@sanity/ailf 0.1.0

package/dist/pipeline/grader-compare-runner.js

@@ -0,0 +1,301 @@
+/**
+ * pipeline/grader-compare-runner.ts
+ *
+ * Orchestration module for inter-grader comparison (Phase 3).
+ *
+ * Reads eval results, extracts grading judgments, re-grades each with
+ * candidate models, and calls `compareGraders()` from the pure
+ * computation module.
+ *
+ * Migrated from lib/grader-compare.ts — no process.argv, no process.exit(),
+ * no module-level constants. Accepts rootDir as parameter.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { load } from "js-yaml";
+import { compareGraders, } from "./grader-comparison.js";
+import { classifyCorrelation } from "./grader-validation.js";
+import { gradeOnce } from "./grader-api.js";
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function classifyDimension(component) {
+    const metadata = component.assertion?.metadata;
+    if (metadata?.dimension) {
+        switch (metadata.dimension) {
+            case "code-correctness":
+                return "codeCorrectness";
+            case "doc-coverage":
+                return "docCoverage";
+            case "task-completion":
+                return "taskCompletion";
+            default:
+                return null;
+        }
+    }
+    const value = (component.assertion?.value ?? "").toLowerCase();
+    if (value.includes("task completion"))
+        return "taskCompletion";
+    if (value.includes("code correctness"))
+        return "codeCorrectness";
+    if (value.includes("documentation coverage") || value.includes("hallucinate"))
+        return "docCoverage";
+    return null;
+}
+function detectFeatureArea(description) {
+    const desc = description.toLowerCase();
+    if (desc.includes("studio"))
+        return "studio-setup";
+    if (desc.includes("visual") ||
+        desc.includes("presentation") ||
+        desc.includes("live preview"))
+        return "visual-editing";
+    if (desc.includes("function") || desc.includes("webhook"))
+        return "functions";
+    if (desc.startsWith("groq"))
+        return "groq";
+    if (desc.includes("next") || desc.includes("app router"))
+        return "nextjs-live";
+    if (desc.includes("remix") ||
+        desc.includes("nuxt") ||
+        desc.includes("svelte"))
+        return "frameworks";
+    return "other";
+}
+/**
+ * Extract llm-rubric judgments from eval results.
+ * Only includes gold (with-docs) tests.
+ */
+function extractJudgments(file) {
+    const judgments = [];
+    for (const result of file.results.results) {
+        if (!result.gradingResult)
+            continue;
+        const description = result.testCase?.description ?? "unknown";
+        const hasDocs = result.vars?.docs && result.vars.docs.trim().length > 0;
+        if (!hasDocs)
+            continue;
+        const area = detectFeatureArea(description);
+        for (const comp of result.gradingResult.componentResults) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const dimension = classifyDimension(comp);
+            if (!dimension)
+                continue;
+            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
+            if (!rubricText)
+                continue;
+            judgments.push({
+                area,
+                description,
+                dimension,
+                responseText: result.response?.output ?? "",
+                rubricText,
+            });
+        }
+    }
+    return judgments;
+}
+/**
+ * Load config: resolve baseline grader and candidate graders.
+ * Candidate overrides take precedence over config/models.yaml.
+ */
+function loadConfig(rootDir, candidateOverrides) {
+    const modelsPath = join(rootDir, "config", "models.yaml");
+    if (!existsSync(modelsPath)) {
+        throw new Error(`config/models.yaml not found at ${modelsPath}`);
+    }
+    const raw = readFileSync(modelsPath, "utf-8");
+    const data = load(raw);
+    const baseline = {
+        id: data?.grader?.id ?? "openai:gpt-5",
+        label: data?.grader?.label ?? "GPT-5 (grader)",
+    };
+    let candidates;
+    if (candidateOverrides && candidateOverrides.length > 0) {
+        candidates = candidateOverrides;
+    }
+    else {
+        const configCandidates = data?.["grader-candidates"] ?? [];
+        candidates = configCandidates.map((c) => ({
+            id: c.id,
+            label: c.label ?? c.id.split(":").pop() ?? c.id,
+        }));
+    }
+    return { baseline, candidates };
+}
+// ---------------------------------------------------------------------------
+// Report formatting (pure)
+// ---------------------------------------------------------------------------
+/**
+ * Format a GraderComparison result as a human-readable table report.
+ * Returns a string — does NOT print to console.
+ */
+export function formatComparisonReport(result) {
+    const lines = [];
+    lines.push("=".repeat(80));
+    lines.push("                    GRADER COMPARISON REPORT");
+    lines.push("=".repeat(80));
+    lines.push("");
+    lines.push(`  Baseline:     ${result.baselineGrader}`);
+    lines.push(`  Candidates:   ${result.candidateGraders.join(", ")}`);
+    lines.push(`  Generated:    ${result.generatedAt}`);
+    lines.push("");
+    for (const pair of result.pairwise) {
+        lines.push("-".repeat(80));
+        lines.push(`${pair.graderA} vs ${pair.graderB}`);
+        lines.push("-".repeat(80));
+        lines.push("");
+        lines.push(`  Correlation:  r=${pair.correlation} (${classifyCorrelation(pair.correlation)})`);
+        lines.push(`  Bias:         ${pair.bias > 0 ? "+" : ""}${pair.bias} (${pair.bias > 0 ? "candidate scores higher" : pair.bias < 0 ? "candidate scores lower" : "no bias"})`);
+        lines.push(`  Mean |Δ|:     ${pair.meanAbsDiff} points`);
+        lines.push("");
+        // Per-dimension table
+        const h = "| Dimension        | Correlation | Bias   | Mean |Δ| | Count |";
+        const sep = "|------------------|-------------|--------|---------|-------|";
+        lines.push(h);
+        lines.push(sep);
+        const dims = [
+            { data: pair.perDimension.taskCompletion, name: "Task Completion" },
+            { data: pair.perDimension.codeCorrectness, name: "Code Correctness" },
+            { data: pair.perDimension.docCoverage, name: "Doc Coverage" },
+        ];
+        for (const { data, name } of dims) {
+            const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
+            lines.push(`| ${name.padEnd(16)} | r=${String(data.correlation).padStart(9)} | ${biasStr.padStart(6)} | ${String(data.meanAbsDiff).padStart(7)} | ${String(data.count).padStart(5)} |`);
+        }
+        lines.push("");
+    }
+    // Recommendations
+    lines.push("-".repeat(80));
+    lines.push("RECOMMENDATIONS");
+    lines.push("-".repeat(80));
+    lines.push("");
+    for (const rec of result.recommendations) {
+        const icon = rec.recommendation === "comparable"
+            ? "✅"
+            : rec.recommendation === "divergent"
+                ? "⚠️"
+                : "❌";
+        lines.push(`  ${icon} ${rec.modelId}: ${rec.recommendation}`);
+        lines.push(`     ${rec.reason}`);
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+/**
+ * Run inter-grader comparison.
+ *
+ * Reads eval results, grades each judgment with the baseline and candidate
+ * graders, and produces a comparison report.
+ *
+ * @throws Error if results file not found, no candidates configured, or no judgments found
+ */
+export async function runGraderCompare(options) {
+    const { rootDir, format = "table" } = options;
+    const resultsPath = options.resultsPath
+        ? join(rootDir, options.resultsPath)
+        : join(rootDir, "results", "latest", "eval-results.json");
+    console.log("=== Grader Comparison ===\n");
+    // Load config
+    const { baseline, candidates } = loadConfig(rootDir, options.candidates);
+    if (candidates.length === 0) {
+        throw new Error("No candidate graders configured. " +
+            "Add grader-candidates to config/models.yaml or pass --candidate.");
+    }
+    // Load eval results
+    if (!existsSync(resultsPath)) {
+        throw new Error(`Results file not found: ${resultsPath}. Run 'pnpm eval' first.`);
+    }
+    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    // Extract judgments
+    const judgments = extractJudgments(file);
+    console.log(`  Baseline:     ${baseline.label} (${baseline.id})`);
+    console.log(`  Candidates:   ${candidates.map((c) => c.label).join(", ")}`);
+    console.log(`  Judgments:    ${judgments.length}`);
+    if (judgments.length === 0) {
+        throw new Error("No gradable judgments found in results.");
+    }
+    const totalCalls = judgments.length * (1 + candidates.length);
+    const estimatedCost = totalCalls * 0.005;
+    console.log(`  API calls:    ${totalCalls} (${judgments.length} × ${1 + candidates.length} models)`);
+    console.log(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
+    console.log();
+    // Grade with baseline
+    console.log(`  Grading with baseline: ${baseline.label}...`);
+    const baselineScores = await gradeJudgments(judgments, baseline.id);
+    // Grade with each candidate
+    const candidateScoreSets = [];
+    for (const candidate of candidates) {
+        console.log(`  Grading with candidate: ${candidate.label}...`);
+        const scores = await gradeJudgments(judgments, candidate.id);
+        candidateScoreSets.push({
+            label: candidate.label,
+            modelId: candidate.id,
+            scores,
+        });
+    }
+    console.log();
+    // Compare
+    const baselineScoreSet = {
+        label: baseline.label,
+        modelId: baseline.id,
+        scores: baselineScores,
+    };
+    const result = compareGraders(baselineScoreSet, candidateScoreSets);
+    // Output
+    if (format === "table") {
+        console.log(formatComparisonReport(result));
+    }
+    else {
+        console.log(JSON.stringify(result, null, 2));
+    }
+    // Write output
+    const outPath = options.outputPath ??
+        join(rootDir, "results", "latest", "grader-comparison.json");
+    const outDir = join(outPath, "..");
+    mkdirSync(outDir, { recursive: true });
+    writeFileSync(outPath, JSON.stringify(result, null, 2));
+    console.log(`\n  📄 Results written to ${outPath}`);
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Grading helper
+// ---------------------------------------------------------------------------
+/**
+ * Grade a set of judgments with a specific grader model.
+ * Returns GraderScore[] with one score per judgment.
+ */
+async function gradeJudgments(judgments, graderModel) {
+    const scores = [];
+    let completed = 0;
+    let failed = 0;
+    for (const judgment of judgments) {
+        const score = await gradeOnce(graderModel, judgment.responseText, judgment.rubricText);
+        completed++;
+        if (completed % 10 === 0 || completed === judgments.length) {
+            const pct = Math.round((completed / judgments.length) * 100);
+            process.stdout.write(`\r    Progress: ${completed}/${judgments.length} (${pct}%)`);
+        }
+        if (score === null) {
+            failed++;
+            continue;
+        }
+        scores.push({
+            area: judgment.area,
+            dimension: judgment.dimension,
+            score,
+            taskId: judgment.description,
+        });
+    }
+    console.log(); // newline after progress
+    if (failed > 0) {
+        console.log(`    ⚠ ${failed} grading calls failed (excluded)`);
+    }
+    return scores;
+}

package/dist/pipeline/grader-comparison.d.ts

@@ -0,0 +1,111 @@
+/**
+ * pipeline/grader-comparison.ts
+ *
+ * Pure computation module for inter-grader comparison.
+ *
+ * Takes score data from multiple grader models (each having graded the same
+ * set of responses) and produces a comparison matrix with correlations,
+ * bias measurements, and per-area deltas between every grader pair.
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ * It operates on pre-collected data only.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
+ */
+import type { ComparisonReport, ScoreSummary } from "./types.js";
+/** Per-dimension comparison between two graders */
+export interface DimensionPairComparison {
+    /** Systematic bias for this dimension */
+    bias: number;
+    /** Pearson correlation for this dimension */
+    correlation: number;
+    /** Number of paired observations */
+    count: number;
+    /** Mean absolute difference for this dimension */
+    meanAbsDiff: number;
+}
+/** Full inter-grader comparison result */
+export interface GraderComparison {
+    /** The baseline grader model ID */
+    baselineGrader: string;
+    /** Candidate grader model IDs */
+    candidateGraders: string[];
+    /** When this comparison was generated */
+    generatedAt: string;
+    /** Pairwise comparisons between the baseline and each candidate */
+    pairwise: GraderPairComparison[];
+    /** Recommendation summary per candidate */
+    recommendations: GraderRecommendation[];
+    /** ScoreSummary-level comparison reports (reuses compare() primitive) */
+    scoreSummaryComparisons?: ComparisonReport[];
+}
+/** Comparison between a pair of grader models */
+export interface GraderPairComparison {
+    /** Systematic bias: mean(B scores - A scores). Positive = B grades higher */
+    bias: number;
+    /** Pearson correlation between the two graders' scores */
+    correlation: number;
+    /** Grader A model ID */
+    graderA: string;
+    /** Grader B model ID */
+    graderB: string;
+    /** Mean absolute difference between scores */
+    meanAbsDiff: number;
+    /** Per-dimension comparisons */
+    perDimension: {
+        taskCompletion: DimensionPairComparison;
+        codeCorrectness: DimensionPairComparison;
+        docCoverage: DimensionPairComparison;
+    };
+}
+/** Recommendation for a candidate grader */
+export interface GraderRecommendation {
+    /** Candidate grader model ID */
+    modelId: string;
+    /** Short explanation */
+    reason: string;
+    /** Human-readable recommendation */
+    recommendation: "comparable" | "divergent" | "strongly-divergent";
+}
+/** Score data from a single grader model on one (task, dimension) judgment */
+export interface GraderScore {
+    /** Feature area (e.g., "groq") */
+    area: string;
+    /** Which scoring dimension */
+    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Score assigned by this grader (0–100) */
+    score: number;
+    /** Task ID (e.g., "groq-blog-queries") */
+    taskId: string;
+}
+/** Score data from a single grader model across all judgments */
+export interface GraderScoreSet {
+    /** Grader model label (human-readable) */
+    label: string;
+    /** Grader model ID (e.g., "openai:gpt-5-2025-08-07") */
+    modelId: string;
+    /** All scores from this grader */
+    scores: GraderScore[];
+}
+/**
+ * Compare graders at the ScoreSummary level using the existing compare() primitive.
+ *
+ * This extends a basic GraderComparison with full ComparisonReport objects,
+ * showing per-area and per-dimension deltas at the aggregate score level.
+ *
+ * @param comparison Basic grader comparison (from compareGraders)
+ * @param baselineSummary ScoreSummary produced using the baseline grader
+ * @param candidateSummaries ScoreSummaries produced using each candidate grader
+ * @returns The comparison object augmented with scoreSummaryComparisons
+ */
+export declare function attachScoreSummaryComparisons(comparison: GraderComparison, baselineSummary: ScoreSummary, candidateSummaries: ScoreSummary[]): GraderComparison;
+/**
+ * Compare multiple grader models against a baseline grader.
+ *
+ * This is the main entry point — a pure function with no side effects.
+ *
+ * @param baseline Score data from the baseline grader
+ * @param candidates Score data from candidate grader(s)
+ * @returns GraderComparison with pairwise metrics and recommendations
+ */
+export declare function compareGraders(baseline: GraderScoreSet, candidates: GraderScoreSet[]): GraderComparison;

package/dist/pipeline/grader-comparison.js

@@ -0,0 +1,161 @@
+/**
+ * pipeline/grader-comparison.ts
+ *
+ * Pure computation module for inter-grader comparison.
+ *
+ * Takes score data from multiple grader models (each having graded the same
+ * set of responses) and produces a comparison matrix with correlations,
+ * bias measurements, and per-area deltas between every grader pair.
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ * It operates on pre-collected data only.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
+ */
+import { compare } from "./compare.js";
+import { pearsonCorrelation } from "./grader-validation.js";
+// ---------------------------------------------------------------------------
+// Pure computation
+// ---------------------------------------------------------------------------
+/**
+ * Compare graders at the ScoreSummary level using the existing compare() primitive.
+ *
+ * This extends a basic GraderComparison with full ComparisonReport objects,
+ * showing per-area and per-dimension deltas at the aggregate score level.
+ *
+ * @param comparison Basic grader comparison (from compareGraders)
+ * @param baselineSummary ScoreSummary produced using the baseline grader
+ * @param candidateSummaries ScoreSummaries produced using each candidate grader
+ * @returns The comparison object augmented with scoreSummaryComparisons
+ */
+export function attachScoreSummaryComparisons(comparison, baselineSummary, candidateSummaries) {
+    const scoreSummaryComparisons = candidateSummaries.map((candidateSummary) => compare(baselineSummary, candidateSummary));
+    return {
+        ...comparison,
+        scoreSummaryComparisons,
+    };
+}
+/**
+ * Compare multiple grader models against a baseline grader.
+ *
+ * This is the main entry point — a pure function with no side effects.
+ *
+ * @param baseline Score data from the baseline grader
+ * @param candidates Score data from candidate grader(s)
+ * @returns GraderComparison with pairwise metrics and recommendations
+ */
+export function compareGraders(baseline, candidates) {
+    const pairwise = candidates.map((candidate) => comparePair(baseline, candidate));
+    const recommendations = pairwise.map((pair) => makeRecommendation(pair));
+    return {
+        baselineGrader: baseline.modelId,
+        candidateGraders: candidates.map((c) => c.modelId),
+        generatedAt: new Date().toISOString(),
+        pairwise,
+        recommendations,
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/** Compare a single pair of graders on their raw judgment scores */
+function comparePair(a, b) {
+    // Build a lookup for grader B scores by (taskId, dimension) key
+    const bScoreMap = new Map();
+    for (const s of b.scores) {
+        bScoreMap.set(`${s.taskId}::${s.dimension}`, s.score);
+    }
+    // Find paired observations (present in both graders)
+    const pairedA = [];
+    const pairedB = [];
+    const dimPairsA = {
+        codeCorrectness: [],
+        docCoverage: [],
+        taskCompletion: [],
+    };
+    const dimPairsB = {
+        codeCorrectness: [],
+        docCoverage: [],
+        taskCompletion: [],
+    };
+    for (const sA of a.scores) {
+        const key = `${sA.taskId}::${sA.dimension}`;
+        const scoreB = bScoreMap.get(key);
+        if (scoreB === undefined)
+            continue;
+        pairedA.push(sA.score);
+        pairedB.push(scoreB);
+        dimPairsA[sA.dimension].push(sA.score);
+        dimPairsB[sA.dimension].push(scoreB);
+    }
+    return {
+        bias: computeBias(pairedA, pairedB),
+        correlation: safeCorrelation(pairedA, pairedB),
+        graderA: a.modelId,
+        graderB: b.modelId,
+        meanAbsDiff: computeMeanAbsDiff(pairedA, pairedB),
+        perDimension: {
+            codeCorrectness: computeDimensionPair(dimPairsA.codeCorrectness, dimPairsB.codeCorrectness),
+            docCoverage: computeDimensionPair(dimPairsA.docCoverage, dimPairsB.docCoverage),
+            taskCompletion: computeDimensionPair(dimPairsA.taskCompletion, dimPairsB.taskCompletion),
+        },
+    };
+}
+/** Mean signed difference (B - A). Positive = B scores higher. */
+function computeBias(a, b) {
+    if (a.length === 0)
+        return 0;
+    const sum = b.reduce((s, v, i) => s + (v - a[i]), 0);
+    return Math.round((sum / a.length) * 10) / 10;
+}
+/** Compute dimension-level pair comparison */
+function computeDimensionPair(scoresA, scoresB) {
+    return {
+        bias: computeBias(scoresA, scoresB),
+        correlation: safeCorrelation(scoresA, scoresB),
+        count: scoresA.length,
+        meanAbsDiff: computeMeanAbsDiff(scoresA, scoresB),
+    };
+}
+/** Mean absolute difference between paired scores */
+function computeMeanAbsDiff(a, b) {
+    if (a.length === 0)
+        return 0;
+    const sum = b.reduce((s, v, i) => s + Math.abs(v - a[i]), 0);
+    return Math.round((sum / a.length) * 10) / 10;
+}
+/** Classify a grader pair as comparable, divergent, or strongly-divergent */
+function makeRecommendation(pair) {
+    const r = pair.correlation;
+    const absBias = Math.abs(pair.bias);
+    const mad = pair.meanAbsDiff;
+    // Thresholds for recommendation
+    // comparable: high correlation, low bias and MAD
+    // divergent: moderate correlation or noticeable bias
+    // strongly-divergent: low correlation or large bias
+    if (r >= 0.9 && absBias <= 5 && mad <= 8) {
+        return {
+            modelId: pair.graderB,
+            reason: `High correlation (r=${r}), low bias (${pair.bias}), low MAD (${mad})`,
+            recommendation: "comparable",
+        };
+    }
+    if (r >= 0.7 && absBias <= 10 && mad <= 15) {
+        return {
+            modelId: pair.graderB,
+            reason: `Moderate correlation (r=${r}), bias=${pair.bias}, MAD=${mad}`,
+            recommendation: "divergent",
+        };
+    }
+    return {
+        modelId: pair.graderB,
+        reason: `Low correlation (r=${r}), bias=${pair.bias}, MAD=${mad}`,
+        recommendation: "strongly-divergent",
+    };
+}
+/** Safe Pearson correlation that handles edge cases */
+function safeCorrelation(a, b) {
+    if (a.length < 2)
+        return 0;
+    return Math.round(pearsonCorrelation(a, b) * 100) / 100;
+}

package/dist/pipeline/grader-consistency-runner.d.ts

@@ -0,0 +1,60 @@
+/**
+ * grader-consistency-runner.ts
+ *
+ * Orchestration logic for grader consistency analysis.
+ *
+ * Reads existing eval results, re-runs ONLY the grading assertions N additional
+ * times with the configured grader model, and analyzes score variance.
+ *
+ * This does NOT re-run the models under test — it only re-grades the same
+ * responses. Cost is low: ~$0.005 per grading call × N replications.
+ *
+ * Migrated from lib/grader-consistency.ts — no process.argv, no process.exit(),
+ * no module-level constants.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 1
+ */
+import type { RawPromptfooFile } from "./calculate-scores.js";
+import { type GraderConsistency } from "./grader-consistency.js";
+import type { DimensionName } from "./types.js";
+/** Options for the grader consistency runner. */
+export interface GraderConsistencyRunnerOptions {
+    /** Number of additional grading replications (default: 5) */
+    replications: number;
+    /** Path to eval-results.json */
+    resultsPath: string;
+    /** Root directory of the eval package (for output paths) */
+    rootDir: string;
+}
+interface GradingJudgment {
+    /** Feature area */
+    area: string;
+    /** Task description */
+    description: string;
+    /** Scoring dimension */
+    dimension: DimensionName;
+    /** The original score from the eval run */
+    originalScore: number;
+    /** Provider (model under test) */
+    providerId?: string;
+    /** The model's response text to be graded */
+    responseText: string;
+    /** The rubric text (assertion value) */
+    rubricText: string;
+}
+/**
+ * Extract all llm-rubric grading judgments from eval results.
+ * Only includes gold (with-docs) tests to keep the analysis focused.
+ */
+export declare function extractGradingJudgments(file: RawPromptfooFile): GradingJudgment[];
+export declare function formatConsistencyReport(result: GraderConsistency, graderModel: string): string;
+/**
+ * Run the grader consistency analysis.
+ *
+ * Reads eval results, re-grades each judgment N times, and writes the
+ * consistency report to results/latest/grader-consistency.json.
+ *
+ * @throws Error if results file not found, replications < 2, or no judgments found
+ */
+export declare function runGraderConsistency(options: GraderConsistencyRunnerOptions): Promise<GraderConsistency>;
+export {};