@sanity/ailf 2.0.1 → 2.1.0

package/dist/scripts/grader-validate.d.ts

@@ -1,19 +0,0 @@
-/**
- * grader-validate.ts
- *
- * CLI script for validating grader accuracy against human reference grades
- * (Phase 2 of grader reliability).
- *
- * Loads human-graded reference samples from canonical/grader-references/,
- * runs the grader model on each sample, and compares against human scores.
- *
- * Usage:
- *   pnpm grader-validate                          # validate with default grader
- *   pnpm grader-validate --grader openai:gpt-5.5  # validate a candidate grader
- *   pnpm grader-validate --threshold 15            # custom MAE threshold
- *
- * Reads: canonical/grader-references/*.yaml
- * Reads: config/models.yaml (for default grader model)
- * Writes: results/latest/grader-validation.json
- */
-import "dotenv/config";

package/dist/scripts/grader-validate.js

@@ -1,267 +0,0 @@
-/**
- * grader-validate.ts
- *
- * CLI script for validating grader accuracy against human reference grades
- * (Phase 2 of grader reliability).
- *
- * Loads human-graded reference samples from canonical/grader-references/,
- * runs the grader model on each sample, and compares against human scores.
- *
- * Usage:
- *   pnpm grader-validate                          # validate with default grader
- *   pnpm grader-validate --grader openai:gpt-5.5  # validate a candidate grader
- *   pnpm grader-validate --threshold 15            # custom MAE threshold
- *
- * Reads: canonical/grader-references/*.yaml
- * Reads: config/models.yaml (for default grader model)
- * Writes: results/latest/grader-validation.json
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-import { load } from "js-yaml";
-import { classifyCorrelation, validateGrader, } from "../pipeline/grader-validation.js";
-import { gradeOnce, loadGraderModel } from "./grader-api.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// ---------------------------------------------------------------------------
-// CLI argument parsing
-// ---------------------------------------------------------------------------
-const args = process.argv.slice(2);
-function getFlag(name) {
-    return args.includes(`--${name}`);
-}
-function getOption(name) {
-    const idx = args.indexOf(`--${name}`);
-    return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
-}
-const graderOverride = getOption("grader");
-const thresholdStr = getOption("threshold");
-const maeThreshold = thresholdStr ? parseFloat(thresholdStr) : 10;
-const showHelp = getFlag("help") || getFlag("h");
-if (showHelp) {
-    console.log(`
-Usage: pnpm grader-validate [options]
-
-Validate grader accuracy against human reference grades.
-
-Options:
-  --grader <model>     Grader model to validate (default: from config/models.yaml)
-  --threshold <n>      MAE threshold for pass/fail (default: 10)
-  --help, -h           Show this help
-
-Examples:
-  pnpm grader-validate                          # validate current grader
-  pnpm grader-validate --grader openai:gpt-5.5  # test a candidate
-  pnpm grader-validate --threshold 15           # lenient threshold
-`);
-    process.exit(0);
-}
-// ---------------------------------------------------------------------------
-// Load reference grades
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-// Dimension mapping
-// ---------------------------------------------------------------------------
-function loadReferenceGrades() {
-    const refsDir = join(ROOT, "canonical", "grader-references");
-    if (!existsSync(refsDir)) {
-        console.error(`❌ Reference grades directory not found: ${refsDir}`);
-        console.error("Create canonical/grader-references/ with YAML reference files.");
-        console.error("See docs/exec-plans/completed/grader-reliability.md — Phase 2.");
-        process.exit(1);
-    }
-    const files = readdirSync(refsDir)
-        .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
-        .sort();
-    if (files.length === 0) {
-        console.error(`❌ No YAML files found in ${refsDir}`);
-        process.exit(1);
-    }
-    const allGrades = [];
-    for (const file of files) {
-        const filePath = join(refsDir, file);
-        const raw = readFileSync(filePath, "utf-8");
-        const parsed = load(raw);
-        if (Array.isArray(parsed)) {
-            allGrades.push(...parsed);
-        }
-        else if (typeof parsed === "object" && parsed !== null) {
-            allGrades.push(parsed);
-        }
-    }
-    return allGrades;
-}
-// ---------------------------------------------------------------------------
-// OpenAI grading API call (reuse from grader-consistency)
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log("=== Grader Validation ===\n");
-    // Resolve grader model
-    const graderModel = graderOverride ?? loadGraderModel().id;
-    console.log(`  Grader:     ${graderModel}`);
-    console.log(`  Threshold:  MAE < ${maeThreshold}`);
-    // Load reference grades
-    const rawGrades = loadReferenceGrades();
-    console.log(`  Samples:    ${rawGrades.length} reference-graded responses`);
-    // Count total rubric judgments
-    let totalJudgments = 0;
-    for (const rg of rawGrades) {
-        totalJudgments += rg.rubrics.length;
-    }
-    console.log(`  Judgments:  ${totalJudgments} (response × rubric pairs)`);
-    const estimatedCost = totalJudgments * 0.005;
-    console.log(`  Est. cost:  ~$${estimatedCost.toFixed(2)}`);
-    console.log();
-    // Grade each reference sample
-    console.log("  Running grader on reference samples...");
-    const grades = [];
-    let completed = 0;
-    let failed = 0;
-    for (const ref of rawGrades) {
-        for (const rubric of ref.rubrics) {
-            const dimension = mapDimension(rubric.dimension);
-            if (!dimension) {
-                console.error(`  ⚠ Unknown dimension '${rubric.dimension}' — skipping`);
-                continue;
-            }
-            const graderScore = await gradeOnce(graderModel, ref.response, rubric.rubricText);
-            completed++;
-            if (completed % 5 === 0 || completed === totalJudgments) {
-                process.stdout.write(`\r  Progress: ${completed}/${totalJudgments}`);
-            }
-            if (graderScore === null) {
-                failed++;
-                continue;
-            }
-            grades.push({
-                area: ref.area,
-                dimension,
-                graderScore,
-                humanScore: rubric.humanScore,
-                taskId: ref.taskId,
-                ...(rubric.notes && { notes: rubric.notes }),
-            });
-        }
-    }
-    console.log(); // newline after progress
-    if (failed > 0) {
-        console.log(`  ⚠ ${failed} grading calls failed (excluded from analysis)`);
-    }
-    console.log();
-    if (grades.length === 0) {
-        console.error("❌ No grades to analyze.");
-        process.exit(1);
-    }
-    // Validate
-    const result = validateGrader(grades, graderModel, { maeThreshold });
-    // Print report
-    printReport(result);
-    // Write output
-    const outDir = join(ROOT, "results", "latest");
-    mkdirSync(outDir, { recursive: true });
-    const outPath = join(outDir, "grader-validation.json");
-    writeFileSync(outPath, JSON.stringify(result, null, 2));
-    console.log(`\n  📄 Results written to ${outPath}`);
-    // Exit with error code if threshold not met
-    if (!result.passesThreshold) {
-        console.error(`\n  ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${maeThreshold}`);
-        process.exit(1);
-    }
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-function mapDimension(dim) {
-    switch (dim) {
-        case "code-correctness":
-            return "codeCorrectness";
-        case "doc-coverage":
-            return "docCoverage";
-        case "task-completion":
-            return "taskCompletion";
-        default:
-            return null;
-    }
-}
-// ---------------------------------------------------------------------------
-// Report formatting
-// ---------------------------------------------------------------------------
-function printReport(result) {
-    console.log("=".repeat(80));
-    console.log("                    GRADER VALIDATION REPORT");
-    console.log("=".repeat(80));
-    console.log();
-    console.log(`  Grader:          ${result.graderModel}`);
-    console.log(`  Observations:    ${result.totalObservations}`);
-    console.log();
-    // Overall metrics
-    console.log("-".repeat(80));
-    console.log("OVERALL METRICS");
-    console.log("-".repeat(80));
-    console.log();
-    console.log(`  MAE:             ${result.overallMae} points`);
-    console.log(`  Correlation:     r=${result.overallCorrelation} (${classifyCorrelation(result.overallCorrelation)})`);
-    console.log(`  Bias:            ${result.overallBias > 0 ? "+" : ""}${result.overallBias} (${result.overallBias > 0 ? "grader scores higher" : result.overallBias < 0 ? "grader scores lower" : "no bias"})`);
-    console.log();
-    // Per-dimension table
-    console.log("-".repeat(80));
-    console.log("PER-DIMENSION VALIDITY");
-    console.log("-".repeat(80));
-    console.log();
-    const h = "| Dimension        |  MAE  | Correlation | Quality   | Bias   | Count |";
-    const sep = "|------------------|-------|-------------|-----------|--------|-------|";
-    console.log(h);
-    console.log(sep);
-    const dims = [
-        { data: result.perDimension.taskCompletion, name: "Task Completion" },
-        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
-        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
-    ];
-    for (const { data, name } of dims) {
-        const quality = classifyCorrelation(data.correlation);
-        const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
-        console.log(`| ${name.padEnd(16)} | ${String(data.mae).padStart(5)} | r=${String(data.correlation).padStart(9)} | ${quality.padEnd(9)} | ${biasStr.padStart(6)} | ${String(data.count).padStart(5)} |`);
-    }
-    console.log();
-    // Pass/fail verdict
-    console.log("-".repeat(80));
-    console.log("VERDICT");
-    console.log("-".repeat(80));
-    console.log();
-    if (result.passesThreshold) {
-        console.log(`  ✅ PASSED: MAE ${result.overallMae} < threshold ${result.maeThreshold}`);
-    }
-    else {
-        console.log(`  ❌ FAILED: MAE ${result.overallMae} >= threshold ${result.maeThreshold}`);
-    }
-    console.log();
-    // Largest disagreements
-    const topN = Math.min(5, result.largestDisagreements.length);
-    if (topN > 0) {
-        console.log("-".repeat(80));
-        console.log(`TOP ${topN} LARGEST DISAGREEMENTS`);
-        console.log("-".repeat(80));
-        console.log();
-        for (let i = 0; i < topN; i++) {
-            const d = result.largestDisagreements[i];
-            const sign = d.signedError > 0 ? "+" : "";
-            console.log(`  ${i + 1}. ${d.taskId} — ${d.dimension}`);
-            console.log(`     Human=${d.humanScore}, Grader=${d.graderScore} (${sign}${d.signedError})`);
-            if (d.notes) {
-                console.log(`     Note: ${d.notes}`);
-            }
-        }
-        console.log();
-    }
-}
-// Only run when invoked directly
-if (process.argv[1]?.endsWith("grader-validate.ts") ||
-    process.argv[1]?.endsWith("grader-validate.js")) {
-    main().catch((err) => {
-        console.error("❌ Fatal error:", err);
-        process.exit(1);
-    });
-}

package/dist/scripts/measure-retrieval.d.ts

@@ -1,10 +0,0 @@
-/**
- * measure-retrieval.ts
- *
- * Evaluates retrieval quality by comparing what Sanity's text search
- * returns against the manually-annotated canonical documents for each
- * evaluation task. Produces Recall@K and NDCG@K metrics.
- *
- * This answers: "Can a retriever find the docs an LLM actually needs?"
- */
-import "dotenv/config";

package/dist/scripts/measure-retrieval.js

@@ -1,145 +0,0 @@
-/**
- * measure-retrieval.ts
- *
- * Evaluates retrieval quality by comparing what Sanity's text search
- * returns against the manually-annotated canonical documents for each
- * evaluation task. Produces Recall@K and NDCG@K metrics.
- *
- * This answers: "Can a retriever find the docs an LLM actually needs?"
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { writeFileSync, mkdirSync } from "fs";
-import { join, dirname } from "path";
-import { resolveMappings, } from "../pipeline/resolve-mappings.js";
-import { getSanityClient } from "../sanity/client.js";
-// ---------------------------------------------------------------------------
-// Retrieval via Sanity text search
-// ---------------------------------------------------------------------------
-function calculateNDCG(canonical, retrieved, k) {
-    const canonicalSet = new Set(canonical);
-    // Discounted Cumulative Gain
-    let dcg = 0;
-    for (let i = 0; i < Math.min(k, retrieved.length); i++) {
-        if (canonicalSet.has(retrieved[i])) {
-            dcg += 1 / Math.log2(i + 2); // +2 because log2(1) = 0
-        }
-    }
-    // Ideal DCG
-    let idcg = 0;
-    for (let i = 0; i < Math.min(k, canonical.length); i++) {
-        idcg += 1 / Math.log2(i + 2);
-    }
-    return idcg === 0 ? 0 : dcg / idcg;
-}
-// ---------------------------------------------------------------------------
-// Metrics
-// ---------------------------------------------------------------------------
-function calculateRecall(canonical, retrieved, k) {
-    const retrievedSet = new Set(retrieved.slice(0, k));
-    const hits = canonical.filter((doc) => retrievedSet.has(doc)).length;
-    return canonical.length === 0 ? 0 : hits / canonical.length;
-}
-async function main() {
-    console.log("=== Sanity AI Literacy — Retrieval Quality Measurement ===\n");
-    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
-    const mappings = resolveMappings(ROOT);
-    const results = [];
-    for (const [area, areaData] of Object.entries(mappings.feature_areas)) {
-        console.log(`Feature area: ${area}`);
-        for (const task of areaData.tasks) {
-            const canonicalSlugs = task.canonical_docs.map((d) => d.slug);
-            // Use the task description as a search query
-            const retrieved = await retrieveDocsForQuery(task.description, 10);
-            const result = {
-                canonical_docs: canonicalSlugs,
-                feature_area: area,
-                ndcg_at_10: calculateNDCG(canonicalSlugs, retrieved, 10),
-                recall_at_5: calculateRecall(canonicalSlugs, retrieved, 5),
-                recall_at_10: calculateRecall(canonicalSlugs, retrieved, 10),
-                retrieved_docs: retrieved,
-                task_id: task.id,
-            };
-            results.push(result);
-            console.log(`  ${task.id}:`);
-            console.log(`    Recall@5:  ${(result.recall_at_5 * 100).toFixed(1)}%`);
-            console.log(`    Recall@10: ${(result.recall_at_10 * 100).toFixed(1)}%`);
-            console.log(`    NDCG@10:   ${(result.ndcg_at_10 * 100).toFixed(1)}%`);
-        }
-        console.log();
-    }
-    // -----------------------------------------------------------------------
-    // Aggregate by feature area
-    // -----------------------------------------------------------------------
-    const byArea = {};
-    for (const area of Object.keys(mappings.feature_areas)) {
-        const areaResults = results.filter((r) => r.feature_area === area);
-        if (areaResults.length === 0)
-            continue;
-        byArea[area] = {
-            avg_ndcg_at_10: areaResults.reduce((s, r) => s + r.ndcg_at_10, 0) / areaResults.length,
-            avg_recall_at_5: areaResults.reduce((s, r) => s + r.recall_at_5, 0) / areaResults.length,
-            avg_recall_at_10: areaResults.reduce((s, r) => s + r.recall_at_10, 0) /
-                areaResults.length,
-            task_count: areaResults.length,
-        };
-    }
-    // -----------------------------------------------------------------------
-    // Overall
-    // -----------------------------------------------------------------------
-    const overall = {
-        avg_ndcg_at_10: results.reduce((s, r) => s + r.ndcg_at_10, 0) / results.length,
-        avg_recall_at_5: results.reduce((s, r) => s + r.recall_at_5, 0) / results.length,
-        avg_recall_at_10: results.reduce((s, r) => s + r.recall_at_10, 0) / results.length,
-    };
-    // -----------------------------------------------------------------------
-    // Print summary
-    // -----------------------------------------------------------------------
-    console.log("=".repeat(70));
-    console.log("RETRIEVAL QUALITY SUMMARY");
-    console.log("=".repeat(70));
-    console.log();
-    console.log("| Feature Area        | Recall@5 | Recall@10 | NDCG@10 | Tasks |");
-    console.log("|---------------------|----------|-----------|---------|-------|");
-    for (const [area, stats] of Object.entries(byArea)) {
-        console.log(`| ${area.padEnd(19)} | ${(stats.avg_recall_at_5 * 100).toFixed(1).padStart(7)}% | ` +
-            `${(stats.avg_recall_at_10 * 100).toFixed(1).padStart(8)}% | ` +
-            `${(stats.avg_ndcg_at_10 * 100).toFixed(1).padStart(6)}% | ` +
-            `${stats.task_count.toString().padStart(5)} |`);
-    }
-    console.log();
-    console.log(`Overall:  Recall@5=${(overall.avg_recall_at_5 * 100).toFixed(1)}%  ` +
-        `Recall@10=${(overall.avg_recall_at_10 * 100).toFixed(1)}%  ` +
-        `NDCG@10=${(overall.avg_ndcg_at_10 * 100).toFixed(1)}%`);
-    // -----------------------------------------------------------------------
-    // Persist results
-    // -----------------------------------------------------------------------
-    const summary = { by_area: byArea, overall, results };
-    const outDir = join(ROOT, "results", "latest");
-    mkdirSync(outDir, { recursive: true });
-    writeFileSync(join(outDir, "retrieval-results.json"), JSON.stringify(summary, null, 2));
-    console.log("\nResults written to results/latest/retrieval-results.json");
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-async function retrieveDocsForQuery(query, k = 10) {
-    const client = getSanityClient();
-    const results = await client.fetch(`
-    *[_type == "article" && !(_id in path("drafts.**"))]
-    | score(
-        boost(title match $query, 3),
-        boost(pt::text(content) match $query, 1)
-      )
-    | order(_score desc)
-    [0...$k] {
-      "slug": slug.current,
-      _score
-    }
-  `, { k, query });
-    return results.map((r) => r.slug);
-}
-main().catch((err) => {
-    console.error("Fatal error:", err);
-    process.exit(1);
-});

package/dist/scripts/migrate-tasks-to-content-lake.d.ts

@@ -1,24 +0,0 @@
-#!/usr/bin/env tsx
-/**
- * Migration script: YAML tasks → Content Lake documents
- *
- * Reads all existing task definitions from tasks/*.yaml via YamlTaskSource,
- * reference solution files from canonical/reference-solutions/, and creates
- * corresponding ailf.featureArea, ailf.referenceSolution, and ailf.task
- * documents in the Sanity Content Lake.
- *
- * Idempotent — uses createOrReplace with deterministic document IDs.
- * Safe to run multiple times during development/testing.
- *
- * Usage:
- *   cd packages/eval
- *   npx tsx src/scripts/migrate-tasks-to-content-lake.ts
- *   npx tsx src/scripts/migrate-tasks-to-content-lake.ts --dry-run
- *
- * Prerequisites:
- *   - SANITY_API_TOKEN (or AILF_REPORT_SANITY_API_TOKEN) with write access
- *   - SANITY_PROJECT_ID and SANITY_DATASET configured
- *
- * @see docs/archive/exec-plans/tasks-as-content/phase-3-migration.md
- */
-export {};