@sanity/ailf 2.0.2 → 2.1.0

package/dist/scripts/migrate-tasks-to-content-lake.js

@@ -1,328 +0,0 @@
-#!/usr/bin/env tsx
-/**
- * Migration script: YAML tasks → Content Lake documents
- *
- * Reads all existing task definitions from tasks/*.yaml via YamlTaskSource,
- * reference solution files from canonical/reference-solutions/, and creates
- * corresponding ailf.featureArea, ailf.referenceSolution, and ailf.task
- * documents in the Sanity Content Lake.
- *
- * Idempotent — uses createOrReplace with deterministic document IDs.
- * Safe to run multiple times during development/testing.
- *
- * Usage:
- *   cd packages/eval
- *   npx tsx src/scripts/migrate-tasks-to-content-lake.ts
- *   npx tsx src/scripts/migrate-tasks-to-content-lake.ts --dry-run
- *
- * Prerequisites:
- *   - SANITY_API_TOKEN (or AILF_REPORT_SANITY_API_TOKEN) with write access
- *   - SANITY_PROJECT_ID and SANITY_DATASET configured
- *
- * @see docs/archive/exec-plans/tasks-as-content/phase-3-migration.md
- */
-import { config as dotenvConfig } from "dotenv";
-import { existsSync, readFileSync } from "fs";
-import { dirname, extname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { isSlugRef, } from "../_vendor/ailf-core/index.js";
-import { getSanityClient } from "../sanity/client.js";
-import { YamlTaskSource } from "../adapters/task-sources/yaml-task-source.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// Load .env from repository root (same as CLI entry point)
-const envPath = resolve(ROOT, "..", "..", ".env");
-if (existsSync(envPath)) {
-    dotenvConfig({ override: true, path: envPath });
-}
-// ---------------------------------------------------------------------------
-// Configuration
-// ---------------------------------------------------------------------------
-const DRY_RUN = process.argv.includes("--dry-run");
-/** Human-readable descriptions for feature areas (from YAML filenames). */
-const AREA_DESCRIPTIONS = {
-    frameworks: "Framework Integrations",
-    functions: "Sanity Functions",
-    groq: "GROQ Query Language",
-    "nextjs-live": "Next.js Integration",
-    "studio-setup": "Studio Setup & Customization",
-    "visual-editing": "Visual Editing",
-};
-// ---------------------------------------------------------------------------
-// Document ID conventions (deterministic for idempotent createOrReplace)
-// ---------------------------------------------------------------------------
-function featureAreaDocId(areaId) {
-    return `ailf.featureArea.${areaId}`;
-}
-function referenceSolutionDocId(taskId) {
-    return `ailf.referenceSolution.${taskId}`;
-}
-function taskDocId(taskId) {
-    return `ailf.task.${taskId}`;
-}
-// ---------------------------------------------------------------------------
-// Portable Text helpers
-// ---------------------------------------------------------------------------
-/** Generate a simple random key for Portable Text array items. */
-function ptKey() {
-    return Math.random().toString(36).slice(2, 14);
-}
-/**
- * Convert a source code file to a Portable Text array with a single code block.
- * This is the most faithful representation — one code block per file.
- */
-function sourceToPortableText(code, language) {
-    return [
-        {
-            _key: ptKey(),
-            _type: "code",
-            code,
-            language,
-        },
-    ];
-}
-/** Detect language from file extension. */
-function languageFromExt(filePath) {
-    const ext = extname(filePath).toLowerCase();
-    if (ext === ".ts" || ext === ".tsx")
-        return "typescript";
-    if (ext === ".js" || ext === ".jsx")
-        return "javascript";
-    return "typescript"; // default
-}
-// ---------------------------------------------------------------------------
-// Slug → article _id resolution
-// ---------------------------------------------------------------------------
-/**
- * Resolve canonical doc slugs to Sanity article document IDs.
- * Returns a map of slug → _id. Unresolved slugs are logged as warnings.
- */
-async function resolveCanonicalDocIds(client, slugs) {
-    if (slugs.length === 0)
-        return new Map();
-    const query = /* groq */ `
-    *[_type == "article" && slug.current in $slugs] {
-      _id,
-      "slug": slug.current
-    }
-  `;
-    const results = await client.fetch(query, {
-        slugs,
-    });
-    const map = new Map();
-    for (const r of results) {
-        map.set(r.slug, r._id);
-    }
-    // Log unresolved slugs
-    for (const slug of slugs) {
-        if (!map.has(slug)) {
-            console.warn(`  ⚠️  Canonical doc slug "${slug}" could not be resolved`);
-        }
-    }
-    return map;
-}
-// ---------------------------------------------------------------------------
-// Document builders
-// ---------------------------------------------------------------------------
-function buildFeatureAreaDoc(areaId) {
-    return {
-        _id: featureAreaDocId(areaId),
-        _type: "ailf.featureArea",
-        areaId: { _type: "slug", current: areaId },
-        description: AREA_DESCRIPTIONS[areaId] ?? areaId,
-    };
-}
-function buildReferenceSolutionDoc(task, code, language) {
-    return {
-        _id: referenceSolutionDocId(task.id),
-        _type: "ailf.referenceSolution",
-        content: sourceToPortableText(code, language),
-        language,
-        title: `${task.title} — Reference Solution`,
-    };
-}
-function buildTaskDoc(task, slugToDocId, hasReferenceSolution) {
-    // Build canonical docs array with resolved references (slug refs only)
-    const canonicalDocs = (task.context?.docs ?? []).map((ref) => {
-        const resolvedId = isSlugRef(ref) ? slugToDocId.get(ref.slug) : undefined;
-        return {
-            _key: ptKey(),
-            ...(resolvedId ? { doc: { _ref: resolvedId, _type: "reference" } } : {}),
-            reason: ref.reason ?? "",
-        };
-    });
-    // Build assertions array
-    const assertArray = (task.assertions ?? []).map((a) => {
-        const entry = {
-            _key: ptKey(),
-            type: a.type,
-        };
-        if (a.type === "llm-rubric" && "template" in a) {
-            entry.template = a.template;
-            if ("criteria" in a && Array.isArray(a.criteria)) {
-                entry.criteria = a.criteria;
-            }
-        }
-        if ("value" in a && a.value !== undefined) {
-            // Store value as a string (matching the Studio schema text field)
-            entry.value =
-                typeof a.value === "string" ? a.value : JSON.stringify(a.value);
-        }
-        if ("threshold" in a && a.threshold !== undefined) {
-            entry.threshold = a.threshold;
-        }
-        if (a.weight !== undefined) {
-            entry.weight = a.weight;
-        }
-        return entry;
-    });
-    const area = task.area ?? "";
-    const doc = {
-        _id: taskDocId(task.id),
-        _type: "ailf.task",
-        assert: assertArray,
-        canonicalDocs,
-        description: task.title,
-        docCoverage: task.docCoverage ?? false,
-        featureArea: {
-            _ref: featureAreaDocId(area),
-            _type: "reference",
-        },
-        id: { _type: "slug", current: task.id },
-        taskPrompt: task.prompt?.text ?? "",
-    };
-    // Optional reference solution
-    if (hasReferenceSolution) {
-        doc.referenceSolution = {
-            _ref: referenceSolutionDocId(task.id),
-            _type: "reference",
-        };
-    }
-    // Optional baseline config
-    if (task.baseline) {
-        doc.baseline = {
-            ...(task.baseline.enabled !== undefined
-                ? { enabled: task.baseline.enabled }
-                : {}),
-            ...(task.baseline.rubric ? { rubric: task.baseline.rubric } : {}),
-        };
-    }
-    return doc;
-}
-// ---------------------------------------------------------------------------
-// Main migration
-// ---------------------------------------------------------------------------
-async function migrate() {
-    console.log("\n📦 Tasks-as-Content Migration");
-    console.log("=".repeat(50));
-    if (DRY_RUN) {
-        console.log("  Mode: DRY RUN (no writes)");
-    }
-    // 1. Load all tasks from YAML
-    console.log("\n1️⃣  Loading tasks from YAML...");
-    const taskSource = new YamlTaskSource(ROOT);
-    const tasks = (await taskSource.loadTasks()).filter((t) => t.mode === "literacy");
-    console.log(`  Loaded ${tasks.length} tasks`);
-    // 2. Extract unique feature areas
-    const areas = [...new Set(tasks.map((t) => t.area ?? ""))].sort();
-    console.log(`  Found ${areas.length} feature areas: ${areas.join(", ")}`);
-    // 3. Collect all canonical doc slugs for batch resolution (slug refs only)
-    const allSlugs = [
-        ...new Set(tasks.flatMap((t) => (t.context?.docs ?? []).filter(isSlugRef).map((d) => d.slug))),
-    ];
-    console.log(`  Found ${allSlugs.length} unique canonical doc slugs`);
-    // 4. Create Sanity client with write token
-    // Use the report token (which has write access) but target the content
-    // dataset (where articles live, so canonicalDocs references resolve).
-    const writeToken = process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN;
-    if (!writeToken && !DRY_RUN) {
-        console.error("  ❌ No write token found. Set AILF_REPORT_SANITY_API_TOKEN or SANITY_API_TOKEN.");
-        process.exit(1);
-    }
-    const client = getSanityClient({
-        ...(writeToken ? { token: writeToken } : {}),
-    });
-    console.log(`  Sanity: project=${client.config().projectId} dataset=${client.config().dataset}`);
-    // 5. Resolve canonical doc slugs → article _ids
-    console.log("\n2️⃣  Resolving canonical doc slugs...");
-    const slugToDocId = await resolveCanonicalDocIds(client, allSlugs);
-    const resolved = slugToDocId.size;
-    const unresolved = allSlugs.length - resolved;
-    console.log(`  Resolved: ${resolved}/${allSlugs.length}${unresolved > 0 ? ` (${unresolved} unresolved)` : ""}`);
-    // 6. Build all documents
-    console.log("\n3️⃣  Building documents...");
-    // Feature areas
-    const areaDocs = areas.map((a) => buildFeatureAreaDoc(a));
-    console.log(`  ${areaDocs.length} ailf.featureArea documents`);
-    // Reference solutions
-    const refSolutionDocs = [];
-    const tasksWithSolutions = new Set();
-    for (const task of tasks) {
-        if (!task.referenceSolution)
-            continue;
-        const filePath = resolve(ROOT, "canonical", task.referenceSolution);
-        try {
-            const code = readFileSync(filePath, "utf-8");
-            const language = languageFromExt(filePath);
-            refSolutionDocs.push(buildReferenceSolutionDoc(task, code, language));
-            tasksWithSolutions.add(task.id);
-        }
-        catch (err) {
-            console.warn(`  ⚠️  Could not read reference solution for ${task.id}: ${filePath}`);
-            if (err instanceof Error) {
-                console.warn(`      ${err.message}`);
-            }
-        }
-    }
-    console.log(`  ${refSolutionDocs.length} ailf.referenceSolution documents`);
-    // Tasks
-    const taskDocs = tasks.map((t) => buildTaskDoc(t, slugToDocId, tasksWithSolutions.has(t.id)));
-    console.log(`  ${taskDocs.length} ailf.task documents`);
-    // 7. Write to Content Lake
-    const allDocs = [...areaDocs, ...refSolutionDocs, ...taskDocs];
-    console.log(`\n4️⃣  Writing ${allDocs.length} documents...`);
-    if (DRY_RUN) {
-        console.log("  DRY RUN — skipping writes");
-        console.log("\n  Documents that would be created:");
-        for (const doc of allDocs) {
-            console.log(`    ${doc._type} → ${doc._id}`);
-        }
-    }
-    else {
-        // Use a transaction for atomicity
-        let tx = client.transaction();
-        for (const doc of allDocs) {
-            tx = tx.createOrReplace(doc);
-        }
-        try {
-            const result = await tx.commit();
-            console.log(`  ✅ Successfully wrote ${result.documentIds.length} documents`);
-        }
-        catch (err) {
-            console.error("  ❌ Transaction failed:");
-            if (err instanceof Error) {
-                console.error(`      ${err.message}`);
-            }
-            process.exit(1);
-        }
-    }
-    // 8. Summary
-    console.log("\n" + "=".repeat(50));
-    console.log("📊 Migration Summary");
-    console.log(`  Feature areas:       ${areaDocs.length}`);
-    console.log(`  Reference solutions: ${refSolutionDocs.length}`);
-    console.log(`  Tasks:               ${taskDocs.length}`);
-    console.log(`  Total documents:     ${allDocs.length}`);
-    console.log(`  Canonical slugs:     ${resolved} resolved, ${unresolved} unresolved`);
-    if (DRY_RUN) {
-        console.log("\n  ℹ️  Run without --dry-run to write to the Content Lake");
-    }
-    console.log("");
-}
-// ---------------------------------------------------------------------------
-// Run
-// ---------------------------------------------------------------------------
-migrate().catch((err) => {
-    console.error("Migration failed:", err);
-    process.exit(1);
-});

package/dist/scripts/pipeline.d.ts

@@ -1,76 +0,0 @@
-/**
- * pipeline.ts
- *
- * CLI orchestrator for the modular evaluation pipeline.
- * Runs steps in sequence with validation between each.
- *
- * This is the single entry point for both local and CI evaluation.
- * The CI workflow (eval.yml) calls this script, then layers on
- * CI-specific post-steps (PR comment posting, artifact upload).
- *
- * Usage:
- *   pnpm pipeline                          # full baseline pipeline
- *   pnpm pipeline --dry-run                # validate only, no execution
- *   pnpm pipeline --skip-fetch             # reuse cached doc contexts
- *   pnpm pipeline --skip-eval              # recalculate from existing results
- *   pnpm pipeline --mode agentic           # run agentic pipeline
- *   pnpm pipeline --mode observed          # run observed pipeline
- *   pnpm pipeline --source staging         # use staging doc source
- *   pnpm pipeline --debug                  # run first 2 tests only (fast)
- *   pnpm pipeline --debug-n 5              # run first 5 tests
- *   pnpm pipeline --debug-pattern "Blog"   # filter by description
- *   pnpm pipeline --debug-sample 3         # random sample of 3 tests
- *   pnpm pipeline --no-cache               # bypass caching, force re-run
- *   pnpm pipeline --concurrency 64         # override max parallel API calls
- *   pnpm pipeline --area groq,frameworks   # only evaluate these areas
- *   pnpm pipeline --task groq-blog-queries # only evaluate this task
- *   pnpm pipeline --changed-docs groq-introduction,how-queries-work
- *                                          # auto-scope to affected tasks
- *   pnpm pipeline --url https://...        # override docs base URL
- *   pnpm pipeline --sanity-dataset staging # override Sanity dataset
- *   pnpm pipeline --sanity-project abc123  # override Sanity project ID
- *   pnpm pipeline --sanity-perspective agent-c7OKTk
- *                                          # evaluate a Sanity release
- *   pnpm pipeline --sanity-document <uuid>
- *                                          # evaluate specific document(s)
- *   pnpm pipeline --sanity-document <uuid> --sanity-documents <uuid>
- *                                          # singular and plural aliases work
- *   pnpm pipeline --header "X-Vercel-Protection-Bypass: <secret>"
- *                                          # custom HTTP header (repeatable)
- *   pnpm pipeline --allowed-origin my-branch.sanity.build
- *                                          # sandbox agent to this origin
- *   pnpm pipeline --before published         # run before/after impact evaluation
- *   pnpm pipeline --before production        # before = production source
- *   pnpm pipeline --before results/baselines/20260310.json  # use existing scores
- *   pnpm pipeline --before latest-baseline   # use most recent baseline
- *   pnpm pipeline --compare                # compare scores against latest baseline
- *   pnpm pipeline --compare --compare-baseline <path>  # compare against specific file
- *   pnpm pipeline --compare --threshold 5  # noise threshold for unchanged (default: 2)
- *   pnpm pipeline --output /tmp/report.md  # write report to specific path
- *   pnpm pipeline --promptfoo-url <url>    # include Promptfoo URL in report
- *   pnpm pipeline --gap-analysis            # run failure mode + impact analysis
- *   pnpm pipeline --publish                # write report to Sanity + fan out to sinks
- *   pnpm pipeline --publish --publish-tag "daily-2026-03-11"  # tag the report
- *   pnpm pipeline --publish --report-dataset ailf-reports    # report store dataset
- *   pnpm pipeline --publish --report-project abc123          # report store project
- *
- * Override precedence (highest wins):
- *   CLI flag (--url, --sanity-dataset, --sanity-project, --allowed-origin)
- *   → Environment variable (DOC_BASE_URL, SANITY_DATASET, SANITY_PROJECT_ID, DOC_ALLOWED_ORIGIN)
- *   → config/sources.yaml default value
- *
- *   --header flags are additive and do not override env vars — they are
- *   always merged with any headers defined in DOC_HEADERS env var.
- *
- * Environment variable fallbacks (for CI):
- *   DEBUG_EVAL=1              → --debug
- *   DEBUG_EVAL_N=2            → --debug-n 2
- *   DEBUG_EVAL_PATTERN        → --debug-pattern
- *   DEBUG_EVAL_SAMPLE         → --debug-sample
- *   EVAL_FILTER_AREAS         → --area
- *   EVAL_FILTER_TASKS         → --task
- *   EVAL_CHANGED_DOCS         → --changed-docs
- *   AILF_REPORT_DATASET       → --report-dataset (report store, not eval)
- *   AILF_REPORT_PROJECT_ID    → --report-project (report store, not eval)
- */
-export {};