@sanity/ailf 1.0.0 → 2.0.1

package/dist/scripts/grader-compare.d.ts

@@ -0,0 +1,22 @@
+/**
+ * grader-compare.ts
+ *
+ * CLI for inter-grader comparison (Phase 3 of grader reliability).
+ *
+ * Re-runs grading assertions on existing eval results using candidate grader
+ * models, then compares the resulting scores against the baseline grader.
+ *
+ * Usage:
+ *   pnpm grader-compare                           # compare vs configured candidates
+ *   pnpm grader-compare --candidate openai:gpt-5.5-preview
+ *   pnpm grader-compare --candidate openai:gpt-5.5-preview --candidate anthropic:claude-4-opus
+ *   pnpm grader-compare --results eval-results.json
+ *   pnpm grader-compare --format json             # machine-readable output
+ *
+ * Reads: results/latest/eval-results.json (model responses to re-grade)
+ * Reads: config/models.yaml (baseline grader + optional candidate list)
+ * Writes: results/latest/grader-comparison.json
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
+ */
+export {};

package/dist/scripts/grader-compare.js

@@ -0,0 +1,368 @@
+/**
+ * grader-compare.ts
+ *
+ * CLI for inter-grader comparison (Phase 3 of grader reliability).
+ *
+ * Re-runs grading assertions on existing eval results using candidate grader
+ * models, then compares the resulting scores against the baseline grader.
+ *
+ * Usage:
+ *   pnpm grader-compare                           # compare vs configured candidates
+ *   pnpm grader-compare --candidate openai:gpt-5.5-preview
+ *   pnpm grader-compare --candidate openai:gpt-5.5-preview --candidate anthropic:claude-4-opus
+ *   pnpm grader-compare --results eval-results.json
+ *   pnpm grader-compare --format json             # machine-readable output
+ *
+ * Reads: results/latest/eval-results.json (model responses to re-grade)
+ * Reads: config/models.yaml (baseline grader + optional candidate list)
+ * Writes: results/latest/grader-comparison.json
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
+ */
+import { existsSync, readFileSync, writeFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { load } from "js-yaml";
+import { compareGraders, } from "../pipeline/grader-comparison.js";
+import { classifyCorrelation } from "../pipeline/grader-validation.js";
+import { gradeOnce } from "./grader-api.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+// ---------------------------------------------------------------------------
+// CLI argument parsing
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+function getAllOptions(name) {
+    const results = [];
+    const flag = `--${name}`;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === flag && i + 1 < args.length) {
+            results.push(args[i + 1]);
+        }
+    }
+    return results;
+}
+function getFlag(name) {
+    return args.includes(`--${name}`);
+}
+function getOption(name) {
+    const idx = args.indexOf(`--${name}`);
+    return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
+}
+const candidateArgs = getAllOptions("candidate");
+const resultsPath = getOption("results") ?? "results/latest/eval-results.json";
+const format = getOption("format") ?? "table";
+const outputPath = getOption("output");
+const showHelp = getFlag("help") || getFlag("h");
+if (showHelp) {
+    console.log(`
+Usage: pnpm grader-compare [options]
+
+Compare multiple grader models on the same evaluation responses.
+
+Options:
+  --candidate <model>   Candidate grader model ID (repeatable)
+                        e.g., --candidate openai:gpt-5.5-preview
+  --results <path>      Path to eval results (default: results/latest/eval-results.json)
+  --format <fmt>        Output format: table (default) or json
+  --output <path>       Write JSON report to file
+  --help, -h            Show this help
+
+If no --candidate flags are provided, reads grader-candidates from config/models.yaml.
+
+Examples:
+  pnpm grader-compare --candidate openai:gpt-5.5-preview
+  pnpm grader-compare --candidate openai:gpt-5.5-preview --candidate anthropic:claude-4-opus
+  pnpm grader-compare --format json
+`);
+    process.exit(0);
+}
+// ---------------------------------------------------------------------------
+// Dimension classification
+// ---------------------------------------------------------------------------
+// DimensionName imported from pipeline/types.ts
+const DIMENSION_PATTERNS = [
+    { dimension: "taskCompletion", pattern: /task[_-]?completion/i },
+    { dimension: "codeCorrectness", pattern: /code[_-]?correct/i },
+    { dimension: "docCoverage", pattern: /doc[_-]?coverage/i },
+];
+/** Classify a component result into a dimension based on rubric content or metric */
+function classifyDimension(comp) {
+    // Check the metric name first (structured dimensions)
+    const metric = comp.assertion?.metric ?? "";
+    for (const { dimension, pattern } of DIMENSION_PATTERNS) {
+        if (pattern.test(metric))
+            return dimension;
+    }
+    // Fall back to rubric text analysis
+    const rubric = typeof comp.assertion?.value === "string" ? comp.assertion.value : "";
+    for (const { dimension, pattern } of DIMENSION_PATTERNS) {
+        if (pattern.test(rubric))
+            return dimension;
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Judgment extraction (same pattern as grader-consistency.ts)
+// ---------------------------------------------------------------------------
+/** Detect feature area from test description */
+function detectFeatureArea(description) {
+    // Pattern: "[gold] Area Name — Task Description" or "Area Name — Task Description"
+    const cleaned = description.replace(/^\[(?:gold|baseline)\]\s*/i, "");
+    const parts = cleaned.split("—");
+    if (parts.length >= 2) {
+        return parts[0].trim().toLowerCase().replace(/\s+/g, "-");
+    }
+    return "unknown";
+}
+/** Detect task ID from test description */
+function detectTaskId(description) {
+    // Description format: "[gold] Area Name — Task Description"
+    const cleaned = description.replace(/^\[(?:gold|baseline)\]\s*/i, "");
+    return cleaned
+        .toLowerCase()
+        .replace(/\s+/g, "-")
+        .replace(/[^a-z0-9-]/g, "")
+        .slice(0, 60);
+}
+// ---------------------------------------------------------------------------
+// OpenAI grading API call (reuses pattern from grader-consistency.ts)
+// ---------------------------------------------------------------------------
+function extractJudgments(evalResults) {
+    const judgments = [];
+    const results = evalResults.results?.results ?? [];
+    for (const result of results) {
+        const description = result.testCase?.description ?? result.description ?? "";
+        // Only process gold tests (with-docs), skip baseline tests
+        if (!description.toLowerCase().includes("[gold]"))
+            continue;
+        const area = detectFeatureArea(description);
+        const taskId = detectTaskId(description);
+        const providerId = result.provider?.id;
+        const components = result.gradingResult?.componentResults ?? [];
+        for (const comp of components) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const dimension = classifyDimension(comp);
+            if (!dimension)
+                continue;
+            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
+            if (!rubricText)
+                continue;
+            judgments.push({
+                area,
+                dimension,
+                originalScore: typeof comp.score === "number" ? comp.score : 0,
+                providerId,
+                responseText: result.response?.output ?? "",
+                rubricText,
+                taskId,
+            });
+        }
+    }
+    return judgments;
+}
+// ---------------------------------------------------------------------------
+// Config loading
+// ---------------------------------------------------------------------------
+function formatComparisonReport(result) {
+    console.log("-".repeat(80));
+    console.log("COMPARISON RESULTS");
+    console.log("-".repeat(80));
+    console.log();
+    console.log(`  Baseline grader: ${result.baselineGrader}`);
+    console.log(`  Candidates:      ${result.candidateGraders.join(", ")}`);
+    console.log();
+    for (const pair of result.pairwise) {
+        console.log("-".repeat(80));
+        console.log(`  ${pair.graderA}  vs  ${pair.graderB}`);
+        console.log("-".repeat(80));
+        console.log();
+        console.log(`  Overall:`);
+        console.log(`    Correlation:       r=${pair.correlation} (${classifyCorrelation(pair.correlation)})`);
+        console.log(`    Bias:              ${pair.bias > 0 ? "+" : ""}${pair.bias} (${pair.bias > 0 ? "candidate grades higher" : pair.bias < 0 ? "candidate grades lower" : "no systematic bias"})`);
+        console.log(`    Mean Abs Diff:     ${pair.meanAbsDiff} points`);
+        console.log();
+        // Per-dimension table
+        const h = "| Dimension        | Correlation | Quality   | Bias   | MAD   | Count |";
+        const sep = "|------------------|-------------|-----------|--------|-------|-------|";
+        console.log(h);
+        console.log(sep);
+        const dims = [
+            { data: pair.perDimension.taskCompletion, name: "Task Completion" },
+            { data: pair.perDimension.codeCorrectness, name: "Code Correctness" },
+            { data: pair.perDimension.docCoverage, name: "Doc Coverage" },
+        ];
+        for (const { data, name } of dims) {
+            const quality = classifyCorrelation(data.correlation);
+            const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
+            console.log(`| ${name.padEnd(16)} | r=${String(data.correlation).padStart(9)} | ${quality.padEnd(9)} | ${biasStr.padStart(6)} | ${String(data.meanAbsDiff).padStart(5)} | ${String(data.count).padStart(5)} |`);
+        }
+        console.log();
+    }
+    // Recommendations
+    if (result.recommendations.length > 0) {
+        console.log("-".repeat(80));
+        console.log("RECOMMENDATIONS");
+        console.log("-".repeat(80));
+        console.log();
+        for (const rec of result.recommendations) {
+            const icon = rec.recommendation === "comparable"
+                ? "✅"
+                : rec.recommendation === "divergent"
+                    ? "⚠️"
+                    : "❌";
+            console.log(`  ${icon} ${rec.modelId}: ${rec.recommendation}`);
+            console.log(`     ${rec.reason}`);
+        }
+        console.log();
+    }
+}
+// ---------------------------------------------------------------------------
+// Main execution
+// ---------------------------------------------------------------------------
+function loadConfig() {
+    const modelsPath = join(ROOT, "config", "models.yaml");
+    if (!existsSync(modelsPath)) {
+        console.error("❌ config/models.yaml not found");
+        process.exit(1);
+    }
+    const raw = readFileSync(modelsPath, "utf-8");
+    const data = load(raw);
+    const grader = {
+        id: data?.grader?.id ?? "openai:gpt-5",
+        label: data?.grader?.label ?? "GPT-5 (grader)",
+    };
+    // CLI candidates override config candidates
+    let candidates;
+    if (candidateArgs.length > 0) {
+        candidates = candidateArgs.map((id) => ({
+            id,
+            label: id.split(":").pop() ?? id,
+        }));
+    }
+    else {
+        const configCandidates = data?.["grader-candidates"] ?? [];
+        candidates = configCandidates.map((c) => ({
+            id: c.id,
+            label: c.label ?? c.id.split(":").pop() ?? c.id,
+        }));
+    }
+    return { baselineGrader: grader, candidates };
+}
+// ---------------------------------------------------------------------------
+// Formatted output
+// ---------------------------------------------------------------------------
+async function main() {
+    console.log("=".repeat(80));
+    console.log("                   INTER-GRADER COMPARISON");
+    console.log("=".repeat(80));
+    console.log();
+    // Load config
+    const { baselineGrader, candidates } = loadConfig();
+    if (candidates.length === 0) {
+        console.error("❌ No candidate graders specified. Use --candidate <model> or add grader-candidates to config/models.yaml.");
+        process.exit(1);
+    }
+    console.log(`  Baseline grader:  ${baselineGrader.id} (${baselineGrader.label})`);
+    for (const c of candidates) {
+        console.log(`  Candidate:        ${c.id} (${c.label})`);
+    }
+    console.log();
+    // Load eval results
+    const evalResultsPath = resolve(ROOT, resultsPath);
+    if (!existsSync(evalResultsPath)) {
+        console.error(`❌ Eval results not found: ${evalResultsPath}`);
+        console.error("  Run the evaluation pipeline first: pnpm pipeline");
+        process.exit(1);
+    }
+    const evalResultsRaw = readFileSync(evalResultsPath, "utf-8");
+    const evalResults = JSON.parse(evalResultsRaw);
+    const evalData = evalResults;
+    // Extract judgments
+    const judgments = extractJudgments(evalData);
+    console.log(`  Judgments found:  ${judgments.length}`);
+    if (judgments.length === 0) {
+        console.error("❌ No gold-test judgments found in eval results.");
+        process.exit(1);
+    }
+    // Build baseline scores from original eval results
+    const baselineScores = judgments.map((j) => ({
+        area: j.area,
+        dimension: j.dimension,
+        score: Math.round(j.originalScore * 100),
+        taskId: j.taskId,
+    }));
+    const baselineScoreSet = {
+        label: baselineGrader.label,
+        modelId: baselineGrader.id,
+        scores: baselineScores,
+    };
+    // Grade with each candidate
+    const candidateScoreSets = [];
+    for (const candidate of candidates) {
+        console.log();
+        console.log(`  Grading with ${candidate.id}...`);
+        const candidateScores = [];
+        let completed = 0;
+        let failed = 0;
+        for (const j of judgments) {
+            const score = await gradeOnce(candidate.id, j.responseText, j.rubricText);
+            completed++;
+            if (score !== null) {
+                candidateScores.push({
+                    area: j.area,
+                    dimension: j.dimension,
+                    score,
+                    taskId: j.taskId,
+                });
+            }
+            else {
+                failed++;
+            }
+            if (completed % 10 === 0 || completed === judgments.length) {
+                process.stdout.write(`\r    Progress: ${completed}/${judgments.length}${failed > 0 ? ` (${failed} failed)` : ""}`);
+            }
+        }
+        console.log();
+        candidateScoreSets.push({
+            label: candidate.label,
+            modelId: candidate.id,
+            scores: candidateScores,
+        });
+    }
+    console.log();
+    // Run comparison
+    const comparison = compareGraders(baselineScoreSet, candidateScoreSets);
+    // Output
+    if (format === "json") {
+        const json = JSON.stringify(comparison, null, 2);
+        if (outputPath) {
+            writeFileSync(outputPath, json);
+            console.log(`  ✅ Report written to ${outputPath}`);
+        }
+        else {
+            console.log(json);
+        }
+    }
+    else {
+        formatComparisonReport(comparison);
+    }
+    // Write to results/latest/
+    const resultFilePath = join(ROOT, "results", "latest", "grader-comparison.json");
+    try {
+        writeFileSync(resultFilePath, JSON.stringify(comparison, null, 2));
+        console.log(`  📄 Report saved: ${resultFilePath}`);
+    }
+    catch {
+        // results/latest/ may not exist yet
+    }
+}
+// Only run when invoked directly
+if (process.argv[1]?.endsWith("grader-compare.ts") ||
+    process.argv[1]?.endsWith("grader-compare.js")) {
+    main().catch((err) => {
+        console.error("❌ Fatal error:", err);
+        process.exit(1);
+    });
+}

package/dist/scripts/grader-consistency.d.ts

@@ -0,0 +1,20 @@
+/**
+ * grader-consistency.ts
+ *
+ * CLI script for measuring grader consistency (Phase 1 of grader reliability).
+ *
+ * Reads existing eval results, re-runs ONLY the grading assertions N additional
+ * times with the configured grader model, and analyzes score variance.
+ *
+ * This does NOT re-run the models under test — it only re-grades the same
+ * responses. Cost is low: ~$0.005 per grading call × N replications.
+ *
+ * Usage:
+ *   pnpm grader-consistency                    # 5 replications (default)
+ *   pnpm grader-consistency --replications 3   # custom count
+ *   pnpm grader-consistency --results <path>   # custom results file
+ *
+ * Reads: results/latest/eval-results.json (default)
+ * Writes: results/latest/grader-consistency.json
+ */
+import "dotenv/config";