@sanity/ailf 2.0.2 → 2.1.0

package/dist/scripts/grader-compare.d.ts

@@ -1,22 +0,0 @@
-/**
- * grader-compare.ts
- *
- * CLI for inter-grader comparison (Phase 3 of grader reliability).
- *
- * Re-runs grading assertions on existing eval results using candidate grader
- * models, then compares the resulting scores against the baseline grader.
- *
- * Usage:
- *   pnpm grader-compare                           # compare vs configured candidates
- *   pnpm grader-compare --candidate openai:gpt-5.5-preview
- *   pnpm grader-compare --candidate openai:gpt-5.5-preview --candidate anthropic:claude-4-opus
- *   pnpm grader-compare --results eval-results.json
- *   pnpm grader-compare --format json             # machine-readable output
- *
- * Reads: results/latest/eval-results.json (model responses to re-grade)
- * Reads: config/models.yaml (baseline grader + optional candidate list)
- * Writes: results/latest/grader-comparison.json
- *
- * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
- */
-export {};

package/dist/scripts/grader-compare.js

@@ -1,368 +0,0 @@
-/**
- * grader-compare.ts
- *
- * CLI for inter-grader comparison (Phase 3 of grader reliability).
- *
- * Re-runs grading assertions on existing eval results using candidate grader
- * models, then compares the resulting scores against the baseline grader.
- *
- * Usage:
- *   pnpm grader-compare                           # compare vs configured candidates
- *   pnpm grader-compare --candidate openai:gpt-5.5-preview
- *   pnpm grader-compare --candidate openai:gpt-5.5-preview --candidate anthropic:claude-4-opus
- *   pnpm grader-compare --results eval-results.json
- *   pnpm grader-compare --format json             # machine-readable output
- *
- * Reads: results/latest/eval-results.json (model responses to re-grade)
- * Reads: config/models.yaml (baseline grader + optional candidate list)
- * Writes: results/latest/grader-comparison.json
- *
- * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
- */
-import { existsSync, readFileSync, writeFileSync } from "fs";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-import { load } from "js-yaml";
-import { compareGraders, } from "../pipeline/grader-comparison.js";
-import { classifyCorrelation } from "../pipeline/grader-validation.js";
-import { gradeOnce } from "./grader-api.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// ---------------------------------------------------------------------------
-// CLI argument parsing
-// ---------------------------------------------------------------------------
-const args = process.argv.slice(2);
-function getAllOptions(name) {
-    const results = [];
-    const flag = `--${name}`;
-    for (let i = 0; i < args.length; i++) {
-        if (args[i] === flag && i + 1 < args.length) {
-            results.push(args[i + 1]);
-        }
-    }
-    return results;
-}
-function getFlag(name) {
-    return args.includes(`--${name}`);
-}
-function getOption(name) {
-    const idx = args.indexOf(`--${name}`);
-    return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
-}
-const candidateArgs = getAllOptions("candidate");
-const resultsPath = getOption("results") ?? "results/latest/eval-results.json";
-const format = getOption("format") ?? "table";
-const outputPath = getOption("output");
-const showHelp = getFlag("help") || getFlag("h");
-if (showHelp) {
-    console.log(`
-Usage: pnpm grader-compare [options]
-
-Compare multiple grader models on the same evaluation responses.
-
-Options:
-  --candidate <model>   Candidate grader model ID (repeatable)
-                        e.g., --candidate openai:gpt-5.5-preview
-  --results <path>      Path to eval results (default: results/latest/eval-results.json)
-  --format <fmt>        Output format: table (default) or json
-  --output <path>       Write JSON report to file
-  --help, -h            Show this help
-
-If no --candidate flags are provided, reads grader-candidates from config/models.yaml.
-
-Examples:
-  pnpm grader-compare --candidate openai:gpt-5.5-preview
-  pnpm grader-compare --candidate openai:gpt-5.5-preview --candidate anthropic:claude-4-opus
-  pnpm grader-compare --format json
-`);
-    process.exit(0);
-}
-// ---------------------------------------------------------------------------
-// Dimension classification
-// ---------------------------------------------------------------------------
-// DimensionName imported from pipeline/types.ts
-const DIMENSION_PATTERNS = [
-    { dimension: "taskCompletion", pattern: /task[_-]?completion/i },
-    { dimension: "codeCorrectness", pattern: /code[_-]?correct/i },
-    { dimension: "docCoverage", pattern: /doc[_-]?coverage/i },
-];
-/** Classify a component result into a dimension based on rubric content or metric */
-function classifyDimension(comp) {
-    // Check the metric name first (structured dimensions)
-    const metric = comp.assertion?.metric ?? "";
-    for (const { dimension, pattern } of DIMENSION_PATTERNS) {
-        if (pattern.test(metric))
-            return dimension;
-    }
-    // Fall back to rubric text analysis
-    const rubric = typeof comp.assertion?.value === "string" ? comp.assertion.value : "";
-    for (const { dimension, pattern } of DIMENSION_PATTERNS) {
-        if (pattern.test(rubric))
-            return dimension;
-    }
-    return null;
-}
-// ---------------------------------------------------------------------------
-// Judgment extraction (same pattern as grader-consistency.ts)
-// ---------------------------------------------------------------------------
-/** Detect feature area from test description */
-function detectFeatureArea(description) {
-    // Pattern: "[gold] Area Name — Task Description" or "Area Name — Task Description"
-    const cleaned = description.replace(/^\[(?:gold|baseline)\]\s*/i, "");
-    const parts = cleaned.split("—");
-    if (parts.length >= 2) {
-        return parts[0].trim().toLowerCase().replace(/\s+/g, "-");
-    }
-    return "unknown";
-}
-/** Detect task ID from test description */
-function detectTaskId(description) {
-    // Description format: "[gold] Area Name — Task Description"
-    const cleaned = description.replace(/^\[(?:gold|baseline)\]\s*/i, "");
-    return cleaned
-        .toLowerCase()
-        .replace(/\s+/g, "-")
-        .replace(/[^a-z0-9-]/g, "")
-        .slice(0, 60);
-}
-// ---------------------------------------------------------------------------
-// OpenAI grading API call (reuses pattern from grader-consistency.ts)
-// ---------------------------------------------------------------------------
-function extractJudgments(evalResults) {
-    const judgments = [];
-    const results = evalResults.results?.results ?? [];
-    for (const result of results) {
-        const description = result.testCase?.description ?? result.description ?? "";
-        // Only process gold tests (with-docs), skip baseline tests
-        if (!description.toLowerCase().includes("[gold]"))
-            continue;
-        const area = detectFeatureArea(description);
-        const taskId = detectTaskId(description);
-        const providerId = result.provider?.id;
-        const components = result.gradingResult?.componentResults ?? [];
-        for (const comp of components) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const dimension = classifyDimension(comp);
-            if (!dimension)
-                continue;
-            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
-            if (!rubricText)
-                continue;
-            judgments.push({
-                area,
-                dimension,
-                originalScore: typeof comp.score === "number" ? comp.score : 0,
-                providerId,
-                responseText: result.response?.output ?? "",
-                rubricText,
-                taskId,
-            });
-        }
-    }
-    return judgments;
-}
-// ---------------------------------------------------------------------------
-// Config loading
-// ---------------------------------------------------------------------------
-function formatComparisonReport(result) {
-    console.log("-".repeat(80));
-    console.log("COMPARISON RESULTS");
-    console.log("-".repeat(80));
-    console.log();
-    console.log(`  Baseline grader: ${result.baselineGrader}`);
-    console.log(`  Candidates:      ${result.candidateGraders.join(", ")}`);
-    console.log();
-    for (const pair of result.pairwise) {
-        console.log("-".repeat(80));
-        console.log(`  ${pair.graderA}  vs  ${pair.graderB}`);
-        console.log("-".repeat(80));
-        console.log();
-        console.log(`  Overall:`);
-        console.log(`    Correlation:       r=${pair.correlation} (${classifyCorrelation(pair.correlation)})`);
-        console.log(`    Bias:              ${pair.bias > 0 ? "+" : ""}${pair.bias} (${pair.bias > 0 ? "candidate grades higher" : pair.bias < 0 ? "candidate grades lower" : "no systematic bias"})`);
-        console.log(`    Mean Abs Diff:     ${pair.meanAbsDiff} points`);
-        console.log();
-        // Per-dimension table
-        const h = "| Dimension        | Correlation | Quality   | Bias   | MAD   | Count |";
-        const sep = "|------------------|-------------|-----------|--------|-------|-------|";
-        console.log(h);
-        console.log(sep);
-        const dims = [
-            { data: pair.perDimension.taskCompletion, name: "Task Completion" },
-            { data: pair.perDimension.codeCorrectness, name: "Code Correctness" },
-            { data: pair.perDimension.docCoverage, name: "Doc Coverage" },
-        ];
-        for (const { data, name } of dims) {
-            const quality = classifyCorrelation(data.correlation);
-            const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
-            console.log(`| ${name.padEnd(16)} | r=${String(data.correlation).padStart(9)} | ${quality.padEnd(9)} | ${biasStr.padStart(6)} | ${String(data.meanAbsDiff).padStart(5)} | ${String(data.count).padStart(5)} |`);
-        }
-        console.log();
-    }
-    // Recommendations
-    if (result.recommendations.length > 0) {
-        console.log("-".repeat(80));
-        console.log("RECOMMENDATIONS");
-        console.log("-".repeat(80));
-        console.log();
-        for (const rec of result.recommendations) {
-            const icon = rec.recommendation === "comparable"
-                ? "✅"
-                : rec.recommendation === "divergent"
-                    ? "⚠️"
-                    : "❌";
-            console.log(`  ${icon} ${rec.modelId}: ${rec.recommendation}`);
-            console.log(`     ${rec.reason}`);
-        }
-        console.log();
-    }
-}
-// ---------------------------------------------------------------------------
-// Main execution
-// ---------------------------------------------------------------------------
-function loadConfig() {
-    const modelsPath = join(ROOT, "config", "models.yaml");
-    if (!existsSync(modelsPath)) {
-        console.error("❌ config/models.yaml not found");
-        process.exit(1);
-    }
-    const raw = readFileSync(modelsPath, "utf-8");
-    const data = load(raw);
-    const grader = {
-        id: data?.grader?.id ?? "openai:gpt-5",
-        label: data?.grader?.label ?? "GPT-5 (grader)",
-    };
-    // CLI candidates override config candidates
-    let candidates;
-    if (candidateArgs.length > 0) {
-        candidates = candidateArgs.map((id) => ({
-            id,
-            label: id.split(":").pop() ?? id,
-        }));
-    }
-    else {
-        const configCandidates = data?.["grader-candidates"] ?? [];
-        candidates = configCandidates.map((c) => ({
-            id: c.id,
-            label: c.label ?? c.id.split(":").pop() ?? c.id,
-        }));
-    }
-    return { baselineGrader: grader, candidates };
-}
-// ---------------------------------------------------------------------------
-// Formatted output
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log("=".repeat(80));
-    console.log("                   INTER-GRADER COMPARISON");
-    console.log("=".repeat(80));
-    console.log();
-    // Load config
-    const { baselineGrader, candidates } = loadConfig();
-    if (candidates.length === 0) {
-        console.error("❌ No candidate graders specified. Use --candidate <model> or add grader-candidates to config/models.yaml.");
-        process.exit(1);
-    }
-    console.log(`  Baseline grader:  ${baselineGrader.id} (${baselineGrader.label})`);
-    for (const c of candidates) {
-        console.log(`  Candidate:        ${c.id} (${c.label})`);
-    }
-    console.log();
-    // Load eval results
-    const evalResultsPath = resolve(ROOT, resultsPath);
-    if (!existsSync(evalResultsPath)) {
-        console.error(`❌ Eval results not found: ${evalResultsPath}`);
-        console.error("  Run the evaluation pipeline first: pnpm pipeline");
-        process.exit(1);
-    }
-    const evalResultsRaw = readFileSync(evalResultsPath, "utf-8");
-    const evalResults = JSON.parse(evalResultsRaw);
-    const evalData = evalResults;
-    // Extract judgments
-    const judgments = extractJudgments(evalData);
-    console.log(`  Judgments found:  ${judgments.length}`);
-    if (judgments.length === 0) {
-        console.error("❌ No gold-test judgments found in eval results.");
-        process.exit(1);
-    }
-    // Build baseline scores from original eval results
-    const baselineScores = judgments.map((j) => ({
-        area: j.area,
-        dimension: j.dimension,
-        score: Math.round(j.originalScore * 100),
-        taskId: j.taskId,
-    }));
-    const baselineScoreSet = {
-        label: baselineGrader.label,
-        modelId: baselineGrader.id,
-        scores: baselineScores,
-    };
-    // Grade with each candidate
-    const candidateScoreSets = [];
-    for (const candidate of candidates) {
-        console.log();
-        console.log(`  Grading with ${candidate.id}...`);
-        const candidateScores = [];
-        let completed = 0;
-        let failed = 0;
-        for (const j of judgments) {
-            const score = await gradeOnce(candidate.id, j.responseText, j.rubricText);
-            completed++;
-            if (score !== null) {
-                candidateScores.push({
-                    area: j.area,
-                    dimension: j.dimension,
-                    score,
-                    taskId: j.taskId,
-                });
-            }
-            else {
-                failed++;
-            }
-            if (completed % 10 === 0 || completed === judgments.length) {
-                process.stdout.write(`\r    Progress: ${completed}/${judgments.length}${failed > 0 ? ` (${failed} failed)` : ""}`);
-            }
-        }
-        console.log();
-        candidateScoreSets.push({
-            label: candidate.label,
-            modelId: candidate.id,
-            scores: candidateScores,
-        });
-    }
-    console.log();
-    // Run comparison
-    const comparison = compareGraders(baselineScoreSet, candidateScoreSets);
-    // Output
-    if (format === "json") {
-        const json = JSON.stringify(comparison, null, 2);
-        if (outputPath) {
-            writeFileSync(outputPath, json);
-            console.log(`  ✅ Report written to ${outputPath}`);
-        }
-        else {
-            console.log(json);
-        }
-    }
-    else {
-        formatComparisonReport(comparison);
-    }
-    // Write to results/latest/
-    const resultFilePath = join(ROOT, "results", "latest", "grader-comparison.json");
-    try {
-        writeFileSync(resultFilePath, JSON.stringify(comparison, null, 2));
-        console.log(`  📄 Report saved: ${resultFilePath}`);
-    }
-    catch {
-        // results/latest/ may not exist yet
-    }
-}
-// Only run when invoked directly
-if (process.argv[1]?.endsWith("grader-compare.ts") ||
-    process.argv[1]?.endsWith("grader-compare.js")) {
-    main().catch((err) => {
-        console.error("❌ Fatal error:", err);
-        process.exit(1);
-    });
-}

package/dist/scripts/grader-consistency.d.ts

@@ -1,20 +0,0 @@
-/**
- * grader-consistency.ts
- *
- * CLI script for measuring grader consistency (Phase 1 of grader reliability).
- *
- * Reads existing eval results, re-runs ONLY the grading assertions N additional
- * times with the configured grader model, and analyzes score variance.
- *
- * This does NOT re-run the models under test — it only re-grades the same
- * responses. Cost is low: ~$0.005 per grading call × N replications.
- *
- * Usage:
- *   pnpm grader-consistency                    # 5 replications (default)
- *   pnpm grader-consistency --replications 3   # custom count
- *   pnpm grader-consistency --results <path>   # custom results file
- *
- * Reads: results/latest/eval-results.json (default)
- * Writes: results/latest/grader-consistency.json
- */
-import "dotenv/config";