@sanity/ailf 0.1.0

package/dist/pipeline/measure-retrieval.js

@@ -0,0 +1,111 @@
+/**
+ * measure-retrieval.ts
+ *
+ * Pure retrieval quality measurement functions.
+ *
+ * Evaluates retrieval quality by comparing what Sanity's text search
+ * returns against the manually-annotated canonical documents for each
+ * evaluation task. Produces Recall@K and NDCG@K metrics.
+ *
+ * This answers: "Can a retriever find the docs an LLM actually needs?"
+ *
+ * Migrated from lib/measure-retrieval.ts — no process.argv/process.env,
+ * accepts rootDir and retriever function as parameters.
+ */
+import { resolveMappings } from "./resolve-mappings.js";
+// ---------------------------------------------------------------------------
+// Metrics (pure, exported for testing)
+// ---------------------------------------------------------------------------
+export function calculateRecall(canonical, retrieved, k) {
+    const retrievedSet = new Set(retrieved.slice(0, k));
+    const hits = canonical.filter((doc) => retrievedSet.has(doc)).length;
+    return canonical.length === 0 ? 0 : hits / canonical.length;
+}
+export function calculateNDCG(canonical, retrieved, k) {
+    const canonicalSet = new Set(canonical);
+    // Discounted Cumulative Gain
+    let dcg = 0;
+    for (let i = 0; i < Math.min(k, retrieved.length); i++) {
+        if (canonicalSet.has(retrieved[i])) {
+            dcg += 1 / Math.log2(i + 2); // +2 because log2(1) = 0
+        }
+    }
+    // Ideal DCG
+    let idcg = 0;
+    for (let i = 0; i < Math.min(k, canonical.length); i++) {
+        idcg += 1 / Math.log2(i + 2);
+    }
+    return idcg === 0 ? 0 : dcg / idcg;
+}
+/**
+ * Run retrieval quality measurement for all tasks.
+ *
+ * @returns A full RetrievalSummary with per-task, per-area, and overall metrics.
+ */
+export async function measureRetrieval(options) {
+    const { rootDir, retriever, onProgress } = options;
+    const mappings = resolveMappings(rootDir);
+    const results = [];
+    for (const [area, areaData] of Object.entries(mappings.feature_areas)) {
+        for (const task of areaData.tasks) {
+            const canonicalSlugs = task.canonical_docs.map((d) => d.slug);
+            // Use the task description as a search query
+            const retrieved = await retriever(task.description, 10);
+            const result = {
+                canonical_docs: canonicalSlugs,
+                feature_area: area,
+                ndcg_at_10: calculateNDCG(canonicalSlugs, retrieved, 10),
+                recall_at_5: calculateRecall(canonicalSlugs, retrieved, 5),
+                recall_at_10: calculateRecall(canonicalSlugs, retrieved, 10),
+                retrieved_docs: retrieved,
+                task_id: task.id,
+            };
+            results.push(result);
+            onProgress?.(area, task.id, result);
+        }
+    }
+    // Aggregate by feature area
+    const byArea = {};
+    for (const area of Object.keys(mappings.feature_areas)) {
+        const areaResults = results.filter((r) => r.feature_area === area);
+        if (areaResults.length === 0)
+            continue;
+        byArea[area] = {
+            avg_ndcg_at_10: areaResults.reduce((s, r) => s + r.ndcg_at_10, 0) / areaResults.length,
+            avg_recall_at_5: areaResults.reduce((s, r) => s + r.recall_at_5, 0) / areaResults.length,
+            avg_recall_at_10: areaResults.reduce((s, r) => s + r.recall_at_10, 0) /
+                areaResults.length,
+            task_count: areaResults.length,
+        };
+    }
+    // Overall
+    const overall = {
+        avg_ndcg_at_10: results.reduce((s, r) => s + r.ndcg_at_10, 0) / (results.length || 1),
+        avg_recall_at_5: results.reduce((s, r) => s + r.recall_at_5, 0) / (results.length || 1),
+        avg_recall_at_10: results.reduce((s, r) => s + r.recall_at_10, 0) / (results.length || 1),
+    };
+    return { by_area: byArea, overall, results };
+}
+/**
+ * Format a retrieval summary for console display.
+ */
+export function formatRetrievalTable(summary) {
+    const lines = [];
+    lines.push("=".repeat(70));
+    lines.push("RETRIEVAL QUALITY SUMMARY");
+    lines.push("=".repeat(70));
+    lines.push("");
+    lines.push("| Feature Area        | Recall@5 | Recall@10 | NDCG@10 | Tasks |");
+    lines.push("|---------------------|----------|-----------|---------|-------|");
+    for (const [area, stats] of Object.entries(summary.by_area)) {
+        lines.push(`| ${area.padEnd(19)} | ${(stats.avg_recall_at_5 * 100).toFixed(1).padStart(7)}% | ` +
+            `${(stats.avg_recall_at_10 * 100).toFixed(1).padStart(8)}% | ` +
+            `${(stats.avg_ndcg_at_10 * 100).toFixed(1).padStart(6)}% | ` +
+            `${stats.task_count.toString().padStart(5)} |`);
+    }
+    lines.push("");
+    lines.push(`Overall:  Recall@5=${(summary.overall.avg_recall_at_5 * 100).toFixed(1)}%  ` +
+        `Recall@10=${(summary.overall.avg_recall_at_10 * 100).toFixed(1)}%  ` +
+        `NDCG@10=${(summary.overall.avg_ndcg_at_10 * 100).toFixed(1)}%`);
+    return lines.join("\n");
+}

package/dist/pipeline/mirror-repo-tasks.d.ts

@@ -0,0 +1,86 @@
+/**
+ * pipeline/mirror-repo-tasks.ts
+ *
+ * Mirrors repo-based TaskDefinitions into the Sanity Content Lake as
+ * ailf.task documents with origin provenance. This makes the Content Lake
+ * the universal registry — every task (native or repo-sourced) is visible
+ * in Studio.
+ *
+ * The mirror is idempotent: deterministic document IDs + content hashing
+ * means unchanged tasks are skipped. Changed tasks are upserted via
+ * createOrReplace.
+ *
+ * @see docs/exec-plans/completed/tasks-as-content/phase-5-content-lake-mirroring.md
+ */
+import type { SanityClient } from "@sanity/client";
+import { type TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
+export interface MirrorOptions {
+    /** Sanity client with write access */
+    client: SanityClient;
+    /** Tasks to mirror (already loaded from repo) */
+    tasks: TaskDefinition[];
+    /** Git context for origin provenance */
+    git: GitContext;
+    /** If true, log what would be done without writing */
+    dryRun?: boolean;
+}
+export interface GitContext {
+    /** Full repo identifier (e.g., "sanity-io/visual-editing") */
+    repo: string;
+    /** Repo owner (e.g., "sanity-io") */
+    owner: string;
+    /** Repo name (e.g., "visual-editing") */
+    name: string;
+    /** Current branch */
+    branch: string;
+    /** HEAD commit SHA */
+    commitSha: string;
+}
+export interface MirrorResult {
+    /** Total tasks processed */
+    total: number;
+    /** Tasks created or updated */
+    upserted: number;
+    /** Tasks skipped (unchanged) */
+    skipped: number;
+    /** Feature areas auto-created */
+    areasCreated: string[];
+    /** Canonical doc slugs that failed to resolve */
+    unresolvedSlugs: string[];
+    /** Errors (non-fatal — mirror continues) */
+    errors: string[];
+}
+/**
+ * Mirror repo tasks to the Content Lake.
+ *
+ * For each task:
+ * 1. Compute deterministic document ID
+ * 2. Compute content hash of the task definition
+ * 3. Check if mirror document exists with same hash → skip if unchanged
+ * 4. Resolve canonical doc slugs → Sanity references
+ * 5. Auto-create feature areas if needed
+ * 6. Upsert the ailf.task document with origin block
+ */
+export declare function mirrorRepoTasks(options: MirrorOptions): Promise<MirrorResult>;
+/**
+ * Detect git context from GitHub Actions environment variables,
+ * falling back to git CLI commands.
+ */
+export declare function detectGitContext(repoTasksPath: string): Promise<GitContext>;
+/**
+ * Deterministic mirror document ID.
+ *
+ * Format: ailf.task.mirror.<owner>.<repo>.<taskId>
+ *
+ * Dots in owner/repo/taskId are replaced with hyphens to avoid
+ * Sanity document ID issues.
+ */
+export declare function mirrorDocId(owner: string, repo: string, taskId: string): string;
+/**
+ * Compute a content hash of a TaskDefinition for change detection.
+ *
+ * Includes all fields that affect the mirror document. Excludes
+ * runtime metadata like referenceSolution (filesystem path) since
+ * that's not mirrored.
+ */
+export declare function computeTaskHash(task: TaskDefinition): string;

package/dist/pipeline/mirror-repo-tasks.js

@@ -0,0 +1,350 @@
+/**
+ * pipeline/mirror-repo-tasks.ts
+ *
+ * Mirrors repo-based TaskDefinitions into the Sanity Content Lake as
+ * ailf.task documents with origin provenance. This makes the Content Lake
+ * the universal registry — every task (native or repo-sourced) is visible
+ * in Studio.
+ *
+ * The mirror is idempotent: deterministic document IDs + content hashing
+ * means unchanged tasks are skipped. Changed tasks are upserted via
+ * createOrReplace.
+ *
+ * @see docs/exec-plans/completed/tasks-as-content/phase-5-content-lake-mirroring.md
+ */
+import { createHash } from "crypto";
+import { isSlugRef, } from "../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Mirror repo tasks to the Content Lake.
+ *
+ * For each task:
+ * 1. Compute deterministic document ID
+ * 2. Compute content hash of the task definition
+ * 3. Check if mirror document exists with same hash → skip if unchanged
+ * 4. Resolve canonical doc slugs → Sanity references
+ * 5. Auto-create feature areas if needed
+ * 6. Upsert the ailf.task document with origin block
+ */
+export async function mirrorRepoTasks(options) {
+    const { client, tasks, git, dryRun = false } = options;
+    const result = {
+        total: tasks.length,
+        upserted: 0,
+        skipped: 0,
+        areasCreated: [],
+        unresolvedSlugs: [],
+        errors: [],
+    };
+    if (tasks.length === 0)
+        return result;
+    // Batch-resolve all canonical doc slugs (slug refs only — other ref types
+    // are stored without a resolved article reference for now)
+    const allSlugs = [
+        ...new Set(tasks.flatMap((t) => t.canonicalDocs.filter(isSlugRef).map((d) => d.slug))),
+    ];
+    const slugToDocId = await batchResolveDocSlugs(client, allSlugs);
+    // Track unresolved slugs
+    for (const slug of allSlugs) {
+        if (!slugToDocId.has(slug)) {
+            result.unresolvedSlugs.push(slug);
+        }
+    }
+    // Ensure all feature areas exist
+    const areas = [...new Set(tasks.map((t) => t.featureArea))];
+    const createdAreas = await ensureFeatureAreas(client, areas, dryRun);
+    result.areasCreated = createdAreas;
+    // Fetch existing mirror document content hashes for change detection
+    const mirrorIds = tasks.map((t) => mirrorDocId(git.owner, git.name, t.id));
+    const existingHashes = await fetchExistingHashes(client, mirrorIds);
+    // Mirror each task
+    for (const task of tasks) {
+        try {
+            const docId = mirrorDocId(git.owner, git.name, task.id);
+            const contentHash = computeTaskHash(task);
+            // Skip unchanged
+            if (existingHashes.get(docId) === contentHash) {
+                result.skipped++;
+                continue;
+            }
+            const doc = buildMirrorDocument(task, {
+                contentHash,
+                docId,
+                git,
+                slugToDocId,
+            });
+            if (dryRun) {
+                console.log(`  [dry-run] Would upsert: ${docId}`);
+                result.upserted++;
+                continue;
+            }
+            await client.createOrReplace(doc);
+            result.upserted++;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            result.errors.push(`Failed to mirror "${task.id}": ${msg}`);
+        }
+    }
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Detect git context from environment or CLI
+// ---------------------------------------------------------------------------
+/**
+ * Detect git context from GitHub Actions environment variables,
+ * falling back to git CLI commands.
+ */
+export async function detectGitContext(repoTasksPath) {
+    // GitHub Actions provides these env vars
+    const ghRepo = process.env.GITHUB_REPOSITORY; // "owner/name"
+    const ghRef = process.env.GITHUB_REF ?? "";
+    const ghSha = process.env.GITHUB_SHA ?? "";
+    const ghHeadRef = process.env.GITHUB_HEAD_REF ?? "";
+    if (ghRepo) {
+        const [owner, name] = ghRepo.split("/");
+        const branch = ghHeadRef || ghRef.replace("refs/heads/", "").replace("refs/tags/", "");
+        return {
+            repo: ghRepo,
+            owner: owner ?? "unknown",
+            name: name ?? "unknown",
+            branch: branch || "unknown",
+            commitSha: ghSha || "unknown",
+        };
+    }
+    // Fallback: try git CLI
+    const { execSync } = await import("child_process");
+    try {
+        const remote = execSync("git remote get-url origin", {
+            encoding: "utf-8",
+            cwd: repoTasksPath,
+        }).trim();
+        const branch = execSync("git rev-parse --abbrev-ref HEAD", {
+            encoding: "utf-8",
+            cwd: repoTasksPath,
+        }).trim();
+        const commitSha = execSync("git rev-parse HEAD", {
+            encoding: "utf-8",
+            cwd: repoTasksPath,
+        }).trim();
+        // Parse remote URL: https://github.com/owner/name.git or git@github.com:owner/name.git
+        const match = remote.match(/github\.com[:/]([^/]+)\/([^/.]+)/) ??
+            remote.match(/([^/]+)\/([^/.]+?)(?:\.git)?$/);
+        const owner = match?.[1] ?? "unknown";
+        const name = match?.[2] ?? "unknown";
+        return {
+            repo: `${owner}/${name}`,
+            owner,
+            name,
+            branch,
+            commitSha,
+        };
+    }
+    catch {
+        return {
+            repo: "unknown/unknown",
+            owner: "unknown",
+            name: "unknown",
+            branch: "unknown",
+            commitSha: "unknown",
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// Document ID scheme
+// ---------------------------------------------------------------------------
+/**
+ * Deterministic mirror document ID.
+ *
+ * Format: ailf.task.mirror.<owner>.<repo>.<taskId>
+ *
+ * Dots in owner/repo/taskId are replaced with hyphens to avoid
+ * Sanity document ID issues.
+ */
+export function mirrorDocId(owner, repo, taskId) {
+    const sanitize = (s) => s.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
+    return `ailf.task.mirror.${sanitize(owner)}.${sanitize(repo)}.${sanitize(taskId)}`;
+}
+// ---------------------------------------------------------------------------
+// Content hashing
+// ---------------------------------------------------------------------------
+/**
+ * Compute a content hash of a TaskDefinition for change detection.
+ *
+ * Includes all fields that affect the mirror document. Excludes
+ * runtime metadata like referenceSolution (filesystem path) since
+ * that's not mirrored.
+ */
+export function computeTaskHash(task) {
+    const payload = JSON.stringify({
+        id: task.id,
+        description: task.description,
+        featureArea: task.featureArea,
+        taskPrompt: task.taskPrompt,
+        canonicalDocs: task.canonicalDocs,
+        docCoverage: task.docCoverage,
+        assertions: task.assertions,
+        baseline: task.baseline,
+    });
+    return createHash("sha256").update(payload).digest("hex").slice(0, 16);
+}
+// ---------------------------------------------------------------------------
+// Batch slug resolution
+// ---------------------------------------------------------------------------
+/**
+ * Resolve an array of document slugs to Sanity document IDs in a single query.
+ */
+async function batchResolveDocSlugs(client, slugs) {
+    if (slugs.length === 0)
+        return new Map();
+    const query = `*[_type == "article" && slug.current in $slugs] {
+    _id,
+    "slug": slug.current
+  }`;
+    const results = await client.fetch(query, {
+        slugs,
+    });
+    const map = new Map();
+    for (const r of results) {
+        map.set(r.slug, r._id);
+    }
+    return map;
+}
+// ---------------------------------------------------------------------------
+// Feature area auto-creation
+// ---------------------------------------------------------------------------
+/**
+ * Ensure ailf.featureArea documents exist for all referenced areas.
+ * Returns the list of newly created area IDs.
+ */
+async function ensureFeatureAreas(client, areas, dryRun) {
+    if (areas.length === 0)
+        return [];
+    // Check which areas already exist
+    const existing = await client.fetch(`*[_type == "ailf.featureArea" && areaId.current in $areas].areaId.current`, { areas });
+    const existingSet = new Set(existing);
+    const missing = areas.filter((a) => !existingSet.has(a));
+    if (missing.length === 0)
+        return [];
+    if (dryRun) {
+        for (const area of missing) {
+            console.log(`  [dry-run] Would create feature area: ${area}`);
+        }
+        return missing;
+    }
+    const transaction = client.transaction();
+    for (const area of missing) {
+        const docId = `ailf.featureArea.${area}`;
+        transaction.createOrReplace({
+            _id: docId,
+            _type: "ailf.featureArea",
+            areaId: { _type: "slug", current: area },
+            description: area.charAt(0).toUpperCase() + area.slice(1).replace(/-/g, " "),
+        });
+    }
+    await transaction.commit();
+    return missing;
+}
+// ---------------------------------------------------------------------------
+// Fetch existing content hashes
+// ---------------------------------------------------------------------------
+/**
+ * Fetch existing mirror documents' content hashes for change detection.
+ * The hash is stored in origin.contentHash on the document.
+ */
+async function fetchExistingHashes(client, docIds) {
+    if (docIds.length === 0)
+        return new Map();
+    const query = `*[_id in $ids] { _id, "hash": origin.contentHash }`;
+    const results = await client.fetch(query, {
+        ids: docIds,
+    });
+    const map = new Map();
+    for (const r of results) {
+        if (r.hash)
+            map.set(r._id, r.hash);
+    }
+    return map;
+}
+// ---------------------------------------------------------------------------
+// Build mirror document
+// ---------------------------------------------------------------------------
+function buildMirrorDocument(task, opts) {
+    const { contentHash, docId, git, slugToDocId } = opts;
+    // Build canonical docs with resolved references.
+    // Only slug refs can be resolved to article references today.
+    // Other ref types (path, id, perspective) are stored with reason only.
+    const canonicalDocs = task.canonicalDocs.map((ref, i) => {
+        const resolvedId = isSlugRef(ref) ? slugToDocId.get(ref.slug) : undefined;
+        return {
+            _key: `cd${i}`,
+            ...(resolvedId ? { doc: { _ref: resolvedId, _type: "reference" } } : {}),
+            reason: ref.reason ?? "",
+        };
+    });
+    // Build assertions
+    const assertArray = task.assertions.map((a, i) => {
+        const entry = {
+            _key: `a${i}`,
+            type: a.type,
+        };
+        if (a.type === "llm-rubric" && "template" in a) {
+            entry.template = a.template;
+            if ("criteria" in a &&
+                Array.isArray(a.criteria)) {
+                entry.criteria = a.criteria;
+            }
+        }
+        if ("value" in a && a.value !== undefined) {
+            entry.value =
+                typeof a.value === "string" ? a.value : JSON.stringify(a.value);
+        }
+        if ("threshold" in a &&
+            a.threshold !== undefined) {
+            entry.threshold = a.threshold;
+        }
+        if (a.weight !== undefined) {
+            entry.weight = a.weight;
+        }
+        return entry;
+    });
+    // Determine the source file path (best-effort from task's featureArea)
+    const filePath = `.ailf/tasks/${task.featureArea}.yaml`;
+    return {
+        _id: docId,
+        _type: "ailf.task",
+        assert: assertArray,
+        canonicalDocs,
+        description: task.description,
+        docCoverage: task.docCoverage,
+        featureArea: {
+            _ref: `ailf.featureArea.${task.featureArea}`,
+            _type: "reference",
+        },
+        id: { _type: "slug", current: task.id },
+        origin: {
+            branch: git.branch,
+            commitSha: git.commitSha,
+            contentHash,
+            lastSyncedAt: new Date().toISOString(),
+            path: filePath,
+            repo: git.repo,
+            repoName: git.name,
+            repoOwner: git.owner,
+            type: "repo",
+        },
+        taskPrompt: task.taskPrompt,
+        ...(task.baseline
+            ? {
+                baseline: {
+                    ...(task.baseline.enabled !== undefined
+                        ? { enabled: task.baseline.enabled }
+                        : {}),
+                    ...(task.baseline.rubric ? { rubric: task.baseline.rubric } : {}),
+                },
+            }
+            : {}),
+    };
+}

package/dist/pipeline/plan-format.d.ts

@@ -0,0 +1,33 @@
+/**
+ * pipeline/plan-format.ts
+ *
+ * Formatters for rendering an ExecutionPlan to console output or JSON.
+ *
+ * The console formatter produces a rich, human-readable preview with
+ * emoji markers, alignment, and color-coding (via unicode markers).
+ * The JSON formatter produces machine-readable output for CI/CD.
+ *
+ * @see docs/exec-plans/active/execution-preview.md
+ */
+import type { ExecutionPlan } from "./plan.js";
+/**
+ * Format an execution plan as a rich console string.
+ *
+ * Produces a multi-section output with emoji markers showing:
+ * - Command summary (mode, source, flags)
+ * - Step plan (which steps will run, cached, or skipped)
+ * - Test/task summary
+ * - Model list
+ * - Cost estimate
+ * - File I/O
+ * - Comparison context
+ * - Warnings/errors
+ */
+export declare function formatPlanConsole(plan: ExecutionPlan): string;
+/**
+ * Format an execution plan as indented JSON.
+ *
+ * Useful for CI/CD integration, approval gates, and programmatic
+ * inspection of the plan.
+ */
+export declare function formatPlanJson(plan: ExecutionPlan): string;