@sanity/ailf 0.1.0

package/dist/commands/interactive.js

@@ -0,0 +1,238 @@
+/**
+ * Interactive mode — guided wizard for the evaluation pipeline.
+ *
+ * When `ailf` is run with no arguments (or `ailf interactive`), this module
+ * prompts the user through mode selection, area scoping, debug options,
+ * and common flags — then builds and executes the equivalent `ailf pipeline`
+ * command.
+ *
+ * Uses @inquirer/prompts for a clean, modern terminal UI.
+ */
+import { Command } from "commander";
+export function createInteractiveCommand() {
+    return new Command("interactive")
+        .description("Guided wizard for common evaluation workflows")
+        .action(async () => {
+        const result = await runInteractiveWizard();
+        if (!result)
+            return;
+        console.log();
+        console.log(`  ▸ ailf ${result.command} ${result.args.join(" ")}`);
+        console.log();
+        // Dynamically import the CLI entry point's program and re-parse
+        // with the synthesized argv. This avoids shelling out and keeps
+        // everything in-process.
+        const { execSync } = await import("child_process");
+        const fullCommand = `tsx src/cli.ts ${result.command} ${result.args.join(" ")}`;
+        try {
+            execSync(fullCommand, { stdio: "inherit" });
+        }
+        catch (err) {
+            const code = err !== null && typeof err === "object" && "status" in err
+                ? err.status
+                : 1;
+            process.exit(code);
+        }
+    });
+}
+// ---------------------------------------------------------------------------
+// Wizard steps
+// ---------------------------------------------------------------------------
+async function runInteractiveWizard() {
+    const { confirm, input, select } = await import("@inquirer/prompts");
+    console.log();
+    console.log("  🧙 AI Literacy Framework — Interactive Mode");
+    console.log("  ────────────────────────────────────────────");
+    console.log();
+    // Step 1: Choose workflow
+    const workflow = await select({
+        choices: [
+            {
+                description: "Full evaluation pipeline (fetch → eval → score → report)",
+                name: "Run pipeline",
+                value: "pipeline",
+            },
+            {
+                description: "Compare current scores against a saved baseline",
+                name: "Compare scores",
+                value: "compare",
+            },
+            {
+                description: "Check YAML files, mappings, and reference solutions",
+                name: "Validate config",
+                value: "validate",
+            },
+            {
+                description: "Save, compare, or list historical score snapshots",
+                name: "Manage baselines",
+                value: "baseline",
+            },
+            {
+                description: "Weekly evaluation trends and area summaries",
+                name: "Weekly digest",
+                value: "weekly-digest",
+            },
+            {
+                description: "Measure grader reliability and discrimination power",
+                name: "Grader tools",
+                value: "grader",
+            },
+        ],
+        message: "What would you like to do?",
+    });
+    if (workflow === "compare") {
+        return { args: [], command: "compare" };
+    }
+    if (workflow === "validate") {
+        return { args: [], command: "validate" };
+    }
+    if (workflow === "weekly-digest") {
+        const dryRun = await confirm({
+            default: true,
+            message: "Dry run? (preview to stdout, don't send to Slack)",
+        });
+        return { args: dryRun ? ["--dry-run"] : [], command: "weekly-digest" };
+    }
+    if (workflow === "baseline") {
+        const subcommand = await select({
+            choices: [
+                { name: "Save current scores", value: "save" },
+                { name: "Compare against latest", value: "compare" },
+                { name: "List saved baselines", value: "history" },
+            ],
+            message: "Baseline operation:",
+        });
+        return { args: [subcommand], command: "baseline" };
+    }
+    if (workflow === "grader") {
+        const subcommand = await select({
+            choices: [
+                {
+                    description: "Measure grading variance across replications",
+                    name: "Consistency analysis",
+                    value: "consistency",
+                },
+                {
+                    description: "Test grader discrimination across quality levels",
+                    name: "Sensitivity test",
+                    value: "sensitivity",
+                },
+                {
+                    description: "Compare two grader models head-to-head",
+                    name: "Compare graders",
+                    value: "compare",
+                },
+                {
+                    description: "Validate against human reference grades",
+                    name: "Validate grader",
+                    value: "validate",
+                },
+            ],
+            message: "Grader tool:",
+        });
+        return { args: [subcommand], command: "grader" };
+    }
+    // --- Pipeline-specific wizard continues below ---
+    const args = [];
+    // Step 2: Evaluation mode
+    const mode = await select({
+        choices: [
+            {
+                description: "Evaluate with pre-fetched documentation context",
+                name: "Baseline (with docs vs without docs)",
+                value: "baseline",
+            },
+            {
+                description: "Baseline + record HTTP request patterns",
+                name: "Observed (instrumented)",
+                value: "observed",
+            },
+            {
+                description: "Agent searches for docs itself via web tools",
+                name: "Agentic (agent-driven retrieval)",
+                value: "agentic",
+            },
+        ],
+        message: "Evaluation mode:",
+    });
+    if (mode !== "baseline") {
+        args.push("--mode", mode);
+    }
+    // Step 3: Area scoping
+    const scopeByArea = await confirm({
+        default: false,
+        message: "Scope to specific feature areas?",
+    });
+    if (scopeByArea) {
+        const areas = await input({
+            message: "Feature areas (comma-separated, e.g. groq,frameworks):",
+        });
+        if (areas.trim()) {
+            args.push("--area", areas.trim());
+        }
+    }
+    // Step 4: Debug mode
+    const debug = await confirm({
+        default: false,
+        message: "Enable debug mode? (run a subset of tests for fast feedback)",
+    });
+    if (debug) {
+        args.push("--debug");
+        const debugStyle = await select({
+            choices: [
+                { name: "First 2 tests (default)", value: "default" },
+                { name: "First N tests", value: "first-n" },
+                { name: "Random sample", value: "sample" },
+                { name: "Filter by pattern", value: "pattern" },
+            ],
+            message: "Debug scope:",
+        });
+        if (debugStyle === "first-n") {
+            const n = await input({ default: "5", message: "Number of tests:" });
+            args.push("--debug-n", n);
+        }
+        else if (debugStyle === "sample") {
+            const n = await input({
+                default: "3",
+                message: "Sample size:",
+            });
+            args.push("--debug-sample", n);
+        }
+        else if (debugStyle === "pattern") {
+            const pattern = await input({
+                message: "Description regex (e.g. Blog, webhook):",
+            });
+            if (pattern.trim()) {
+                args.push("--debug-pattern", pattern.trim());
+            }
+        }
+    }
+    // Step 5: Common pipeline options
+    const dryRun = await confirm({
+        default: false,
+        message: "Dry run? (validate config only, no API calls)",
+    });
+    if (dryRun) {
+        args.push("--dry-run");
+    }
+    if (!dryRun) {
+        const compareAfter = await confirm({
+            default: false,
+            message: "Compare against baseline after evaluation?",
+        });
+        if (compareAfter) {
+            args.push("--compare");
+        }
+    }
+    // Step 6: Preview execution plan before running
+    if (!dryRun) {
+        const preview = await confirm({
+            default: true,
+            message: "Preview execution plan before running? (--explain --yes)",
+        });
+        if (preview) {
+            args.push("--explain", "--yes");
+        }
+    }
+    return { args, command: "pipeline" };
+}

package/dist/commands/lookup-doc.d.ts

@@ -0,0 +1,15 @@
+/**
+ * lookup-doc command — search Sanity for documentation articles by keyword.
+ *
+ * Helps external contributors find the correct `slug` for canonicalDocs
+ * references without needing to browse the CMS or guess from URLs.
+ *
+ * Usage:
+ *   ailf lookup-doc webhooks
+ *   ailf lookup-doc "visual editing"
+ *   ailf lookup-doc groq --limit 20
+ *
+ * @see docs/design-docs/canonical-doc-resolution.md
+ */
+import { Command } from "commander";
+export declare function createLookupDocCommand(): Command;

package/dist/commands/lookup-doc.js

@@ -0,0 +1,84 @@
+/**
+ * lookup-doc command — search Sanity for documentation articles by keyword.
+ *
+ * Helps external contributors find the correct `slug` for canonicalDocs
+ * references without needing to browse the CMS or guess from URLs.
+ *
+ * Usage:
+ *   ailf lookup-doc webhooks
+ *   ailf lookup-doc "visual editing"
+ *   ailf lookup-doc groq --limit 20
+ *
+ * @see docs/design-docs/canonical-doc-resolution.md
+ */
+import { Command } from "commander";
+export function createLookupDocCommand() {
+    return new Command("lookup-doc")
+        .description("Search Sanity docs by keyword — find slugs for canonicalDocs references")
+        .argument("<keyword>", "Search keyword (matches title and slug)")
+        .option("-l, --limit <n>", "Maximum results to show", parseInt, 10)
+        .option("-s, --source <name>", "Documentation source (from sources.yaml)")
+        .action(async (keyword, opts) => {
+        const { getSanityClient } = await import("../sanity/client.js");
+        const { loadSource } = await import("../sources.js");
+        // Resolve source if provided
+        let clientOverrides;
+        if (opts.source) {
+            try {
+                const source = loadSource(opts.source);
+                clientOverrides = {};
+                if (source.dataset)
+                    clientOverrides.dataset = source.dataset;
+                if (source.projectId)
+                    clientOverrides.projectId = source.projectId;
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`❌ Failed to load source "${opts.source}": ${msg}`);
+                process.exit(1);
+            }
+        }
+        const client = getSanityClient(clientOverrides);
+        const escapedKeyword = keyword.replace(/"/g, '\\"');
+        console.log(`\nSearching for "${keyword}"...\n`);
+        const query = `*[_type == "article"
+          && !(_id in path("drafts.**"))
+          && (
+            title match "*${escapedKeyword}*"
+            || slug.current match "*${escapedKeyword}*"
+          )
+        ] | order(title asc) [0...$limit] {
+          title,
+          "slug": slug.current,
+          "section": primarySection->title,
+          "sectionSlug": primarySection->slug.current
+        }`;
+        const results = await client.fetch(query, {
+            limit: opts.limit,
+        });
+        if (results.length === 0) {
+            console.log(`  No articles found matching "${keyword}".\n\n` +
+                "  Tips:\n" +
+                "  - Try a shorter or more general keyword\n" +
+                "  - Use partial matches (e.g., 'webhook' instead of 'webhooks')\n");
+            process.exit(0);
+        }
+        console.log(`  Found ${results.length} article${results.length === 1 ? "" : "s"}:\n`);
+        // Find longest slug for alignment
+        const maxSlugLen = Math.min(40, Math.max(...results.map((r) => r.slug.length)));
+        for (const doc of results) {
+            const section = doc.section ?? "Unknown";
+            const paddedSlug = doc.slug.padEnd(maxSlugLen);
+            console.log(`  slug: ${paddedSlug}  │ ${doc.title}`);
+            console.log(`  ${"".padEnd(maxSlugLen + 6)}  │ Section: ${section}\n`);
+        }
+        console.log("  Usage in .ailf/tasks/*.yaml:\n");
+        console.log("  canonicalDocs:");
+        console.log(`    - slug: ${results[0].slug}`);
+        console.log(`      reason: "${results[0].title}"`);
+        if (results[0].sectionSlug) {
+            console.log(`\n  Or use path: ${results[0].sectionSlug}/${results[0].slug}`);
+        }
+        console.log();
+    });
+}

package/dist/commands/measure-retrieval.d.ts

@@ -0,0 +1,5 @@
+/**
+ * measure-retrieval command — evaluate Sanity text search retrieval quality.
+ */
+import { Command } from "commander";
+export declare function createMeasureRetrievalCommand(): Command;

package/dist/commands/measure-retrieval.js

@@ -0,0 +1,65 @@
+/**
+ * measure-retrieval command — evaluate Sanity text search retrieval quality.
+ */
+import { mkdirSync, writeFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { getSanityClient } from "../sanity/client.js";
+import { formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+// ---------------------------------------------------------------------------
+// Sanity text search retriever
+// ---------------------------------------------------------------------------
+async function retrieveDocsForQuery(query, k = 10) {
+    const client = getSanityClient();
+    const results = await client.fetch(`
+    *[_type == "article" && !(_id in path("drafts.**"))]
+    | score(
+        boost(title match $query, 3),
+        boost(pt::text(content) match $query, 1)
+      )
+    | order(_score desc)
+    [0...$k] {
+      "slug": slug.current,
+      _score
+    }
+  `, { k, query });
+    return results.map((r) => r.slug);
+}
+// ---------------------------------------------------------------------------
+// Command factory
+// ---------------------------------------------------------------------------
+export function createMeasureRetrievalCommand() {
+    return new Command("measure-retrieval")
+        .description("Measure retrieval quality against canonical document annotations")
+        .action(async () => {
+        try {
+            console.log("=== Sanity AI Literacy — Retrieval Quality Measurement ===\n");
+            const summary = await measureRetrieval({
+                onProgress: (_area, taskId, result) => {
+                    console.log(`  ${taskId}:`);
+                    console.log(`    Recall@5:  ${(result.recall_at_5 * 100).toFixed(1)}%`);
+                    console.log(`    Recall@10: ${(result.recall_at_10 * 100).toFixed(1)}%`);
+                    console.log(`    NDCG@10:   ${(result.ndcg_at_10 * 100).toFixed(1)}%`);
+                },
+                retriever: retrieveDocsForQuery,
+                rootDir: ROOT,
+            });
+            // Print summary
+            console.log();
+            console.log(formatRetrievalTable(summary));
+            // Persist results
+            const outDir = join(ROOT, "results", "latest");
+            mkdirSync(outDir, { recursive: true });
+            writeFileSync(join(outDir, "retrieval-results.json"), JSON.stringify(summary, null, 2));
+            console.log("\nResults written to results/latest/retrieval-results.json");
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+}

package/dist/commands/pipeline-action.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Pipeline action — resolves CLI options and orchestrates pipeline steps.
+ *
+ * This file is the thin CLI-to-orchestrator bridge. The bulk of execution
+ * logic lives in packages/eval/src/orchestration/.
+ *
+ * Responsibilities:
+ * - Resolve CLI flags into typed ResolvedOptions
+ * - Delegate to the PipelineOrchestrator for step execution
+ *
+ * @see packages/eval/src/orchestration/ for the step-based pipeline
+ */
+import { type ImpactSummary } from "../pipeline/reverse-mapping.js";
+import type { DebugOptions, EvalMode } from "../pipeline/types.js";
+import type { PipelineCliOptions } from "./pipeline.js";
+export interface ResolvedOptions {
+    allowedOriginArgs: string[];
+    areaOption?: string;
+    beforeOption?: string;
+    changedDocsOption?: string;
+    compareBaseline?: string;
+    compareEnabled: boolean;
+    compareThreshold?: number;
+    concurrency?: number;
+    datasetOverride?: string;
+    debug?: DebugOptions;
+    discoveryReportEnabled: boolean;
+    dryRun: boolean;
+    gapAnalysisEnabled: boolean;
+    graderReplications?: number;
+    headerArgs: string[];
+    impactSummary?: ImpactSummary;
+    mode: EvalMode;
+    noCache: boolean;
+    noRemoteCache: boolean;
+    outputPath?: string;
+    perspectiveOverride?: string;
+    projectIdOverride?: string;
+    promptfooUrl?: string;
+    publishEnabled: boolean;
+    publishTag?: string;
+    readinessEnabled: boolean;
+    reportDataset?: string;
+    reportProjectId?: string;
+    sanityDocumentArgs: string[];
+    searchMode: string;
+    skipEval: boolean;
+    skipFetch: boolean;
+    source?: string;
+    studioOriginOverride?: string;
+    repoTasksPath?: string;
+    taskOption?: string;
+    taskSourceType?: "content-lake" | "yaml";
+    urlArgs: string[];
+}
+/**
+ * Pure option resolution — computes ResolvedOptions from CLI flags without
+ * any side effects. Safe to call from --explain without mutating process.env.
+ *
+ * Exported so the plan builder can call it independently.
+ */
+export declare function computeResolvedOptions(opts: PipelineCliOptions): ResolvedOptions;
+/**
+ * Execute the evaluation pipeline.
+ *
+ * 1. Resolve CLI options into typed ResolvedOptions
+ * 2. Build AppContext (composition root wires adapters)
+ * 3. Build step sequence from context
+ * 4. Delegate to the PipelineOrchestrator
+ */
+export declare function executePipeline(cliOpts: PipelineCliOptions): Promise<void>;