@sanity/ailf 2.0.2 → 2.2.0

package/dist/scripts/grader-consistency.js

@@ -1,313 +0,0 @@
-/**
- * grader-consistency.ts
- *
- * CLI script for measuring grader consistency (Phase 1 of grader reliability).
- *
- * Reads existing eval results, re-runs ONLY the grading assertions N additional
- * times with the configured grader model, and analyzes score variance.
- *
- * This does NOT re-run the models under test — it only re-grades the same
- * responses. Cost is low: ~$0.005 per grading call × N replications.
- *
- * Usage:
- *   pnpm grader-consistency                    # 5 replications (default)
- *   pnpm grader-consistency --replications 3   # custom count
- *   pnpm grader-consistency --results <path>   # custom results file
- *
- * Reads: results/latest/eval-results.json (default)
- * Writes: results/latest/grader-consistency.json
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-import { analyzeConsistency, } from "../pipeline/grader-consistency.js";
-import { gradeOnce } from "./grader-api.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// ---------------------------------------------------------------------------
-// CLI argument parsing
-// ---------------------------------------------------------------------------
-const args = process.argv.slice(2);
-function getFlag(name) {
-    return args.includes(`--${name}`);
-}
-function getOption(name) {
-    const idx = args.indexOf(`--${name}`);
-    return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
-}
-const replicationsStr = getOption("replications") ?? "5";
-const replications = parseInt(replicationsStr, 10);
-const resultsPath = getOption("results") ?? join(ROOT, "results", "latest", "eval-results.json");
-const showHelp = getFlag("help") || getFlag("h");
-if (showHelp) {
-    console.log(`
-Usage: pnpm grader-consistency [options]
-
-Measure grader consistency by re-grading existing eval responses N times.
-
-Options:
-  --replications <n>   Number of additional grading replications (default: 5)
-  --results <path>     Path to eval-results.json (default: results/latest/eval-results.json)
-  --help, -h           Show this help
-
-Examples:
-  pnpm grader-consistency                    # 5 replications
-  pnpm grader-consistency --replications 3   # fewer replications (faster)
-  pnpm grader-consistency --results results/latest/eval-results-agentic.json
-`);
-    process.exit(0);
-}
-// RawPromptfooFile and RawTestResult imported from calculate-scores.ts
-// ---------------------------------------------------------------------------
-// Rubric dimension classification (similar to calculate-scores)
-// ---------------------------------------------------------------------------
-function classifyDimension(component) {
-    // Prefer structured metadata
-    const metadata = component.assertion?.metadata;
-    if (metadata?.dimension) {
-        switch (metadata.dimension) {
-            case "code-correctness":
-                return "codeCorrectness";
-            case "doc-coverage":
-                return "docCoverage";
-            case "task-completion":
-                return "taskCompletion";
-            default:
-                return null;
-        }
-    }
-    // Fallback: heuristic name matching
-    const value = (component.assertion?.value ?? "").toLowerCase();
-    if (value.includes("task completion"))
-        return "taskCompletion";
-    if (value.includes("code correctness"))
-        return "codeCorrectness";
-    if (value.includes("documentation coverage") || value.includes("hallucinate"))
-        return "docCoverage";
-    return null;
-}
-// ---------------------------------------------------------------------------
-// Grading judgment extraction
-// ---------------------------------------------------------------------------
-function detectFeatureArea(description) {
-    const desc = description.toLowerCase();
-    if (desc.includes("studio"))
-        return "studio-setup";
-    if (desc.includes("visual") ||
-        desc.includes("presentation") ||
-        desc.includes("live preview"))
-        return "visual-editing";
-    if (desc.includes("function") || desc.includes("webhook"))
-        return "functions";
-    if (desc.startsWith("groq"))
-        return "groq";
-    if (desc.includes("next") || desc.includes("app router"))
-        return "nextjs-live";
-    if (desc.includes("remix") ||
-        desc.includes("nuxt") ||
-        desc.includes("svelte"))
-        return "frameworks";
-    return "other";
-}
-/**
- * Extract all llm-rubric grading judgments from eval results.
- * Only includes gold (with-docs) tests to keep the analysis focused.
- */
-function extractGradingJudgments(file) {
-    const judgments = [];
-    for (const result of file.results.results) {
-        if (!result.gradingResult)
-            continue;
-        const description = result.testCase?.description ?? "unknown";
-        const hasDocs = result.vars?.docs && result.vars.docs.trim().length > 0;
-        // Only grade "gold" (with-docs) tests — baseline tests have abbreviated rubrics
-        if (!hasDocs)
-            continue;
-        const area = detectFeatureArea(description);
-        const providerId = result.provider?.id;
-        for (const comp of result.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const dimension = classifyDimension(comp);
-            if (!dimension)
-                continue;
-            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
-            if (!rubricText)
-                continue;
-            judgments.push({
-                area,
-                description,
-                dimension,
-                originalScore: typeof comp.score === "number" ? comp.score : 0,
-                providerId,
-                responseText: result.response?.output ?? "",
-                rubricText,
-            });
-        }
-    }
-    return judgments;
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log("=== Grader Consistency Analysis ===\n");
-    // Validate inputs
-    if (!existsSync(resultsPath)) {
-        console.error(`❌ Results file not found: ${resultsPath}`);
-        console.error("Run 'pnpm eval' first to generate results.");
-        process.exit(1);
-    }
-    if (replications < 2) {
-        console.error("❌ Need at least 2 replications for meaningful analysis.");
-        process.exit(1);
-    }
-    // Load eval results
-    console.log(`  Results:      ${resultsPath}`);
-    console.log(`  Replications: ${replications}`);
-    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
-    // Extract grader model
-    const graderModel = file.config?.defaultTest?.options?.rubricProvider ??
-        file.config?.defaultTest?.options?.provider;
-    if (!graderModel) {
-        console.error("❌ Could not determine grader model from eval results config.");
-        process.exit(1);
-    }
-    console.log(`  Grader:       ${graderModel}`);
-    // Extract judgments
-    const judgments = extractGradingJudgments(file);
-    console.log(`  Judgments:    ${judgments.length} (gold tests × rubric dimensions)`);
-    if (judgments.length === 0) {
-        console.error("❌ No gradable judgments found in results.");
-        process.exit(1);
-    }
-    const totalCalls = judgments.length * replications;
-    const estimatedCost = totalCalls * 0.005;
-    console.log(`  API calls:    ${totalCalls} (${judgments.length} × ${replications})`);
-    console.log(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
-    console.log();
-    // Re-grade each judgment N times
-    console.log(`  Running ${replications} replications per judgment...`);
-    const gradings = [];
-    let completed = 0;
-    let failed = 0;
-    for (const judgment of judgments) {
-        const scores = [judgment.originalScore]; // Include the original as replication #0
-        for (let i = 0; i < replications; i++) {
-            const score = await gradeOnce(graderModel, judgment.responseText, judgment.rubricText);
-            if (score !== null) {
-                scores.push(score);
-            }
-            else {
-                failed++;
-            }
-        }
-        completed++;
-        if (completed % 10 === 0 || completed === judgments.length) {
-            const pct = Math.round((completed / judgments.length) * 100);
-            process.stdout.write(`\r  Progress: ${completed}/${judgments.length} (${pct}%)`);
-        }
-        gradings.push({
-            area: judgment.area,
-            dimension: judgment.dimension,
-            providerId: judgment.providerId,
-            scores,
-            taskId: judgment.description,
-        });
-    }
-    console.log(); // newline after progress
-    if (failed > 0) {
-        console.log(`  ⚠ ${failed} grading calls failed (excluded from analysis)`);
-    }
-    console.log();
-    // Analyze consistency
-    const result = analyzeConsistency(gradings);
-    // Print report
-    printReport(result, graderModel);
-    // Write output
-    const outDir = join(ROOT, "results", "latest");
-    mkdirSync(outDir, { recursive: true });
-    const outPath = join(outDir, "grader-consistency.json");
-    writeFileSync(outPath, JSON.stringify(result, null, 2));
-    console.log(`\n  📄 Results written to ${outPath}`);
-}
-// ---------------------------------------------------------------------------
-// Report formatting
-// ---------------------------------------------------------------------------
-function printReport(result, graderModel) {
-    console.log("=".repeat(80));
-    console.log("                    GRADER CONSISTENCY REPORT");
-    console.log("=".repeat(80));
-    console.log();
-    console.log(`  Grader model:  ${graderModel}`);
-    console.log(`  Replications:  ${result.replications} (incl. original)`);
-    console.log(`  Judgments:     ${result.totalJudgments}`);
-    console.log();
-    // Overall stats
-    console.log("-".repeat(80));
-    console.log("OVERALL");
-    console.log("-".repeat(80));
-    console.log();
-    console.log(`  Avg σ:          ${result.avgStdDev}`);
-    console.log(`  Max σ:          ${result.maxStdDev}`);
-    console.log(`  Avg range:      ${result.avgRange} points`);
-    console.log();
-    // Per-dimension table
-    console.log("-".repeat(80));
-    console.log("PER-DIMENSION CONSISTENCY");
-    console.log("-".repeat(80));
-    console.log();
-    const h = "| Dimension        | Avg σ | Max σ | Avg Range | Judgments |";
-    const sep = "|------------------|-------|-------|-----------|-----------|";
-    console.log(h);
-    console.log(sep);
-    const dims = [
-        { data: result.perDimension.taskCompletion, name: "Task Completion" },
-        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
-        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
-    ];
-    for (const { data, name } of dims) {
-        console.log(`| ${name.padEnd(16)} | ${String(data.avgStdDev).padStart(5)} | ${String(data.maxStdDev).padStart(5)} | ${String(data.avgRange).padStart(9)} | ${String(data.judgmentCount).padStart(9)} |`);
-    }
-    console.log();
-    // Noise threshold recommendation
-    console.log("-".repeat(80));
-    console.log("NOISE THRESHOLD RECOMMENDATION");
-    console.log("-".repeat(80));
-    console.log();
-    console.log(`  Current default:     ±2 (DEFAULT_NOISE_THRESHOLD)`);
-    console.log(`  Recommended:         ±${result.recommendedThreshold} (based on 2× max dimension σ)`);
-    if (result.recommendedThreshold > 2) {
-        console.log(`  ⚠ Current threshold may be too low — comparison deltas within ±${result.recommendedThreshold}`);
-        console.log(`    should be treated as noise, not real changes.`);
-    }
-    else {
-        console.log(`  ✅ Current threshold is adequate for this grader's consistency.`);
-    }
-    console.log();
-    // Top 5 noisiest judgments
-    const topN = Math.min(5, result.judgments.length);
-    if (topN > 0) {
-        console.log("-".repeat(80));
-        console.log(`TOP ${topN} NOISIEST JUDGMENTS`);
-        console.log("-".repeat(80));
-        console.log();
-        for (let i = 0; i < topN; i++) {
-            const j = result.judgments[i];
-            const provider = j.providerId ? ` [${j.providerId}]` : "";
-            console.log(`  ${i + 1}. ${j.taskId}${provider}`);
-            console.log(`     ${j.dimension}: σ=${j.stdDev}, range=${j.range} (${j.min}–${j.max}), mean=${j.mean}`);
-        }
-        console.log();
-    }
-}
-// Only run when invoked directly
-if (process.argv[1]?.endsWith("grader-consistency.ts") ||
-    process.argv[1]?.endsWith("grader-consistency.js")) {
-    main().catch((err) => {
-        console.error("❌ Fatal error:", err);
-        process.exit(1);
-    });
-}

package/dist/scripts/grader-sensitivity.d.ts

@@ -1,22 +0,0 @@
-/**
- * grader-sensitivity.ts
- *
- * CLI for grader sensitivity testing (Phase 4 of grader reliability).
- *
- * Loads reference solutions, programmatically degrades them, grades both
- * the original and degraded versions with the grader model, then measures
- * whether the grader can distinguish quality levels.
- *
- * Usage:
- *   pnpm grader-sensitivity                       # test all reference solutions
- *   pnpm grader-sensitivity --area groq            # test only groq area
- *   pnpm grader-sensitivity --format json          # machine-readable output
- *
- * Reads: canonical/reference-solutions/**\/*.ts
- * Reads: config/models.yaml (grader model)
- * Reads: config/rubrics.yaml (dimension rubric templates)
- * Writes: results/latest/grader-sensitivity.json
- *
- * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
- */
-export {};

package/dist/scripts/grader-sensitivity.js

@@ -1,354 +0,0 @@
-/**
- * grader-sensitivity.ts
- *
- * CLI for grader sensitivity testing (Phase 4 of grader reliability).
- *
- * Loads reference solutions, programmatically degrades them, grades both
- * the original and degraded versions with the grader model, then measures
- * whether the grader can distinguish quality levels.
- *
- * Usage:
- *   pnpm grader-sensitivity                       # test all reference solutions
- *   pnpm grader-sensitivity --area groq            # test only groq area
- *   pnpm grader-sensitivity --format json          # machine-readable output
- *
- * Reads: canonical/reference-solutions/**\/*.ts
- * Reads: config/models.yaml (grader model)
- * Reads: config/rubrics.yaml (dimension rubric templates)
- * Writes: results/latest/grader-sensitivity.json
- *
- * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
- */
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-import { load } from "js-yaml";
-import { DEGRADATION_STRATEGIES, } from "../pipeline/degradations.js";
-import { analyzeSensitivity, } from "../pipeline/grader-sensitivity.js";
-import { gradeOnce, loadGraderModel } from "./grader-api.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// ---------------------------------------------------------------------------
-// CLI argument parsing
-// ---------------------------------------------------------------------------
-const args = process.argv.slice(2);
-function getFlag(name) {
-    return args.includes(`--${name}`);
-}
-function getOption(name) {
-    const idx = args.indexOf(`--${name}`);
-    return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
-}
-const areaFilter = getOption("area");
-const format = getOption("format") ?? "table";
-const outputPath = getOption("output");
-const showHelp = getFlag("help") || getFlag("h");
-if (showHelp) {
-    console.log(`
-Usage: pnpm grader-sensitivity [options]
-
-Test grader discrimination power using programmatic code degradation.
-
-Options:
-  --area <name>     Test only reference solutions in this area (e.g., groq)
-  --format <fmt>    Output format: table (default) or json
-  --output <path>   Write JSON report to file
-  --help, -h        Show this help
-
-Examples:
-  pnpm grader-sensitivity                  # test all reference solutions
-  pnpm grader-sensitivity --area groq      # test only GROQ area
-  pnpm grader-sensitivity --format json    # machine-readable output
-`);
-    process.exit(0);
-}
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-// DimensionName imported from pipeline/types.ts
-const DIMENSION_NAMES = [
-    "taskCompletion",
-    "codeCorrectness",
-    "docCoverage",
-];
-// ---------------------------------------------------------------------------
-// Load config
-// ---------------------------------------------------------------------------
-/** Build a rubric prompt for a specific dimension */
-function buildRubricPrompt(rubrics, dimension) {
-    const templateKey = dimension === "taskCompletion"
-        ? "task-completion"
-        : dimension === "codeCorrectness"
-            ? "code-correctness"
-            : "doc-coverage";
-    const template = rubrics.templates[templateKey];
-    if (!template) {
-        throw new Error(`No rubric template for ${dimension}`);
-    }
-    const lines = [
-        template.header,
-        "",
-        ...template.scale.map((s) => `- ${s}`),
-        "",
-        rubrics.footer,
-    ];
-    return lines.join("\n");
-}
-function discoverReferenceSolutions() {
-    const refDir = join(ROOT, "canonical", "reference-solutions");
-    if (!existsSync(refDir)) {
-        console.error(`❌ Reference solutions directory not found: ${refDir}`);
-        process.exit(1);
-    }
-    const solutions = [];
-    const areas = readdirSync(refDir, { withFileTypes: true })
-        .filter((d) => d.isDirectory())
-        .map((d) => d.name);
-    for (const area of areas) {
-        if (areaFilter && area !== areaFilter)
-            continue;
-        const areaDir = join(refDir, area);
-        const files = readdirSync(areaDir).filter((f) => f.endsWith(".ts") || f.endsWith(".tsx"));
-        for (const file of files) {
-            const content = readFileSync(join(areaDir, file), "utf-8");
-            solutions.push({
-                area,
-                content,
-                relativePath: `${area}/${file}`,
-            });
-        }
-    }
-    return solutions;
-}
-// ---------------------------------------------------------------------------
-// Discover reference solutions
-// ---------------------------------------------------------------------------
-function formatSensitivityReport(result) {
-    console.log("-".repeat(80));
-    console.log("OVERALL SENSITIVITY");
-    console.log("-".repeat(80));
-    console.log();
-    console.log(`  Grader:              ${result.graderModel}`);
-    console.log(`  Total pairs:         ${result.totalPairs}`);
-    console.log(`  Concordance rate:    ${result.concordanceRate}%`);
-    console.log(`  Avg separation:      ${result.avgSeparation} points`);
-    console.log(`  Failed pairs:        ${result.failedPairs.length}`);
-    console.log();
-    // Per-dimension table
-    console.log("-".repeat(80));
-    console.log("PER-DIMENSION SENSITIVITY");
-    console.log("-".repeat(80));
-    console.log();
-    const h = "| Dimension        | Concordance | Avg Sep | Tied  | Pairs |";
-    const sep = "|------------------|-------------|---------|-------|-------|";
-    console.log(h);
-    console.log(sep);
-    const dims = [
-        { data: result.perDimension.taskCompletion, name: "Task Completion" },
-        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
-        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
-    ];
-    for (const { data, name } of dims) {
-        console.log(`| ${name.padEnd(16)} | ${(data.concordanceRate + "%").padStart(11)} | ${String(data.avgSeparation).padStart(7)} | ${(data.tiedRate + "%").padStart(5)} | ${String(data.pairCount).padStart(5)} |`);
-    }
-    console.log();
-    // Cross-dimension analysis
-    console.log("-".repeat(80));
-    console.log("CROSS-DIMENSION ANALYSIS");
-    console.log("-".repeat(80));
-    console.log();
-    console.log(`  On-target:  ${result.crossDimension.onTarget.concordanceRate}% concordance, ` +
-        `${result.crossDimension.onTarget.avgSeparation} avg sep (${result.crossDimension.onTarget.pairCount} pairs)`);
-    console.log(`  Off-target: ${result.crossDimension.offTarget.concordanceRate}% concordance, ` +
-        `${result.crossDimension.offTarget.avgSeparation} avg sep (${result.crossDimension.offTarget.pairCount} pairs)`);
-    console.log();
-    if (result.crossDimension.onTarget.concordanceRate >
-        result.crossDimension.offTarget.concordanceRate) {
-        console.log("  ✅ Grader is more sensitive to targeted degradations (expected)");
-    }
-    else {
-        console.log("  ⚠ Grader is equally or less sensitive to targeted degradations");
-    }
-    console.log();
-    // Per-degradation breakdown
-    if (result.byDegradation.length > 0) {
-        console.log("-".repeat(80));
-        console.log("PER-DEGRADATION SENSITIVITY (worst first)");
-        console.log("-".repeat(80));
-        console.log();
-        const dh = "| Degradation                                    | Target     | Conc  | Sep   | N  |";
-        const ds = "|------------------------------------------------|------------|-------|-------|----|";
-        console.log(dh);
-        console.log(ds);
-        for (const d of result.byDegradation) {
-            const dimLabel = d.targetDimension === "taskCompletion"
-                ? "Task"
-                : d.targetDimension === "codeCorrectness"
-                    ? "Code"
-                    : "Docs";
-            console.log(`| ${d.description.slice(0, 46).padEnd(46)} | ${dimLabel.padEnd(10)} | ${(d.concordanceRate + "%").padStart(5)} | ${String(d.avgSeparation).padStart(5)} | ${String(d.pairCount).padStart(2)} |`);
-        }
-        console.log();
-    }
-    // Top failures
-    const topN = Math.min(5, result.failedPairs.length);
-    if (topN > 0) {
-        console.log("-".repeat(80));
-        console.log(`TOP ${topN} DISCRIMINATION FAILURES (grader preferred degraded)`);
-        console.log("-".repeat(80));
-        console.log();
-        for (let i = 0; i < topN; i++) {
-            const f = result.failedPairs[i];
-            const delta = f.degradedScore - f.originalScore;
-            console.log(`  ${i + 1}. ${f.sourcePath} — ${f.dimension}`);
-            console.log(`     Original=${f.originalScore}, Degraded=${f.degradedScore} (+${delta} for degraded)`);
-            console.log(`     Degradation: ${f.degradationDescription}`);
-        }
-        console.log();
-    }
-    // Verdict
-    console.log("-".repeat(80));
-    console.log("VERDICT");
-    console.log("-".repeat(80));
-    console.log();
-    if (result.concordanceRate >= 90) {
-        console.log(`  ✅ EXCELLENT: Grader correctly discriminates ${result.concordanceRate}% of pairs`);
-    }
-    else if (result.concordanceRate >= 75) {
-        console.log(`  ⚠️  ACCEPTABLE: Grader correctly discriminates ${result.concordanceRate}% of pairs`);
-    }
-    else {
-        console.log(`  ❌ POOR: Grader only discriminates ${result.concordanceRate}% of pairs`);
-    }
-    console.log();
-}
-function generateDegradedPairs(solutions) {
-    const pairs = [];
-    for (const solution of solutions) {
-        for (const strategy of DEGRADATION_STRATEGIES) {
-            const degraded = strategy.apply(solution.content);
-            // Only include if the degradation actually changed the code
-            if (degraded !== solution.content) {
-                pairs.push({
-                    degradation: strategy,
-                    degraded,
-                    original: solution.content,
-                    sourcePath: solution.relativePath,
-                });
-            }
-        }
-    }
-    return pairs;
-}
-// ---------------------------------------------------------------------------
-// Main execution
-// ---------------------------------------------------------------------------
-function loadRubrics() {
-    const rubricsPath = join(ROOT, "config", "rubrics.yaml");
-    const raw = readFileSync(rubricsPath, "utf-8");
-    return load(raw);
-}
-// ---------------------------------------------------------------------------
-// Formatted output
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log("=".repeat(80));
-    console.log("                    GRADER SENSITIVITY TESTING");
-    console.log("=".repeat(80));
-    console.log();
-    // Load config
-    const grader = loadGraderModel();
-    const rubrics = loadRubrics();
-    console.log(`  Grader:              ${grader.id} (${grader.label})`);
-    if (areaFilter) {
-        console.log(`  Area filter:         ${areaFilter}`);
-    }
-    console.log();
-    // Discover reference solutions
-    const solutions = discoverReferenceSolutions();
-    console.log(`  Reference solutions: ${solutions.length}`);
-    if (solutions.length === 0) {
-        console.error("❌ No reference solutions found.");
-        process.exit(1);
-    }
-    for (const s of solutions) {
-        console.log(`    ${s.relativePath} (${s.content.split("\n").length} lines)`);
-    }
-    console.log();
-    // Generate degraded pairs
-    const degradedPairs = generateDegradedPairs(solutions);
-    console.log(`  Degraded pairs:      ${degradedPairs.length}`);
-    console.log(`  Total gradings:      ${degradedPairs.length * DIMENSION_NAMES.length * 2} (${degradedPairs.length} pairs × ${DIMENSION_NAMES.length} dimensions × 2 versions)`);
-    console.log();
-    // Grade each pair on each dimension
-    const sensitivityPairs = [];
-    let completed = 0;
-    let failed = 0;
-    const totalGradings = degradedPairs.length * DIMENSION_NAMES.length * 2;
-    for (const pair of degradedPairs) {
-        const area = pair.sourcePath.split("/")[0];
-        for (const dimension of DIMENSION_NAMES) {
-            const rubricText = buildRubricPrompt(rubrics, dimension);
-            // Grade original
-            const originalScore = await gradeOnce(grader.id, pair.original, rubricText);
-            completed++;
-            // Grade degraded
-            const degradedScore = await gradeOnce(grader.id, pair.degraded, rubricText);
-            completed++;
-            if (originalScore !== null && degradedScore !== null) {
-                sensitivityPairs.push({
-                    area,
-                    degradationDescription: pair.degradation.description,
-                    degradedScore,
-                    dimension,
-                    originalScore,
-                    sourcePath: pair.sourcePath,
-                    targetDimension: pair.degradation.targetDimension,
-                });
-            }
-            else {
-                failed++;
-            }
-            if (completed % 20 === 0 || completed === totalGradings) {
-                process.stdout.write(`\r  Progress: ${completed}/${totalGradings} gradings${failed > 0 ? ` (${failed} failed)` : ""}`);
-            }
-        }
-    }
-    console.log();
-    console.log();
-    // Analyze
-    const result = analyzeSensitivity(sensitivityPairs, grader.id);
-    // Output
-    if (format === "json") {
-        const json = JSON.stringify(result, null, 2);
-        if (outputPath) {
-            writeFileSync(outputPath, json);
-            console.log(`  ✅ Report written to ${outputPath}`);
-        }
-        else {
-            console.log(json);
-        }
-    }
-    else {
-        formatSensitivityReport(result);
-    }
-    // Write to results/latest/
-    const resultsDir = join(ROOT, "results", "latest");
-    try {
-        mkdirSync(resultsDir, { recursive: true });
-        const resultFilePath = join(resultsDir, "grader-sensitivity.json");
-        writeFileSync(resultFilePath, JSON.stringify(result, null, 2));
-        console.log(`  📄 Report saved: ${resultFilePath}`);
-    }
-    catch {
-        // results/latest/ may not exist yet
-    }
-}
-// Only run when invoked directly
-if (process.argv[1]?.endsWith("grader-sensitivity.ts") ||
-    process.argv[1]?.endsWith("grader-sensitivity.js")) {
-    main().catch((err) => {
-        console.error("❌ Fatal error:", err);
-        process.exit(1);
-    });
-}