@sanity/ailf 0.1.0

package/dist/commands/fetch-docs.d.ts

@@ -0,0 +1,8 @@
+/**
+ * fetch-docs command — pull documentation from Sanity CMS.
+ *
+ * Uses the composition root to wire adapters, then delegates to
+ * ctx.docFetcher (SanityDocFetcher) — the same code path as the pipeline.
+ */
+import { Command } from "commander";
+export declare function createFetchDocsCommand(): Command;

package/dist/commands/fetch-docs.js

@@ -0,0 +1,128 @@
+/**
+ * fetch-docs command — pull documentation from Sanity CMS.
+ *
+ * Uses the composition root to wire adapters, then delegates to
+ * ctx.docFetcher (SanityDocFetcher) — the same code path as the pipeline.
+ */
+import { mkdirSync, writeFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { createAppContext } from "../composition-root.js";
+import { loadSource } from "../sources.js";
+import { configToSourceOverrides } from "../orchestration/config-to-source-overrides.js";
+import { addSanitySourceOptions } from "./shared/options.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createFetchDocsCommand() {
+    const cmd = new Command("fetch-docs")
+        .description("Fetch documentation contexts from Sanity CMS")
+        .option("-s, --source <name>", "Documentation source name (from sources.yaml)")
+        .option("--include-feature-areas", "Generate feature-area context files", false)
+        .option("--include-corpus", "Generate full corpus context file", false)
+        .action(async (opts) => {
+        try {
+            await executeFetchDocs(opts);
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+    addSanitySourceOptions(cmd);
+    return cmd;
+}
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+async function executeFetchDocs(opts) {
+    console.log("=== ai-literacy-framework — Documentation Fetcher ===\n");
+    // Build a minimal ResolvedConfig for the composition root
+    const ctx = createAppContext({
+        rootDir: ROOT,
+        mode: "baseline",
+        skipFetch: false,
+        skipEval: true,
+        compareEnabled: false,
+        gapAnalysisEnabled: false,
+        readinessEnabled: false,
+        discoveryReportEnabled: false,
+        publishEnabled: false,
+        noCache: true,
+        noRemoteCache: true,
+        searchMode: "open",
+        source: opts.source,
+    });
+    // Resolve source
+    const overrides = configToSourceOverrides(ctx.config);
+    const resolvedSource = loadSource(ctx.config.source, overrides);
+    // Log source info
+    console.log(`  Source: ${resolvedSource.name}`);
+    console.log(`  Base URL: ${resolvedSource.baseUrl}`);
+    if (resolvedSource.dataset)
+        console.log(`  Dataset: ${resolvedSource.dataset}`);
+    if (resolvedSource.perspective)
+        console.log(`  Perspective: ${resolvedSource.perspective}`);
+    if (resolvedSource.documentIds && resolvedSource.documentIds.length > 0) {
+        console.log(`  Documents: ${resolvedSource.documentIds.length} document ID(s)`);
+    }
+    if (resolvedSource.urls.length > 0) {
+        console.log(`  URLs:      ${resolvedSource.urls.length} direct URL(s)`);
+    }
+    console.log();
+    // The composition root wires SanityDocFetcher into ctx.docFetcher.
+    // We cast to access the non-port methods (feature areas, corpus).
+    const fetcher = ctx.docFetcher;
+    // Feature-area contexts (opt-in)
+    if (opts.includeFeatureAreas) {
+        await fetcher.fetchFeatureAreaContexts(resolvedSource);
+    }
+    // Canonical contexts — same code path as the pipeline
+    const tasks = await ctx.taskSource.loadTasks();
+    const tasksWithDocs = tasks.filter((t) => t.canonicalDocs.length > 0);
+    if (tasksWithDocs.length > 0) {
+        console.log("\nGenerating canonical (gold-retrieval) contexts...\n");
+        const result = await fetcher.fetch(tasksWithDocs, resolvedSource);
+        // Write metadata files
+        if (result.metadata) {
+            writeMetadataFiles(ROOT, result.metadata);
+        }
+        console.log(`\n  Canonical contexts: ${result.contexts.length} tasks`);
+        for (const docCtx of result.contexts) {
+            console.log(`    ${docCtx.taskId}: ${docCtx.slugs.length} doc(s), ~${docCtx.tokenCount ?? 0} tokens`);
+        }
+    }
+    // Full corpus (opt-in)
+    if (opts.includeCorpus) {
+        await fetcher.fetchFullCorpus(resolvedSource);
+    }
+    console.log("\nDone!");
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function writeMetadataFiles(rootDir, metadata) {
+    const contextsDir = join(rootDir, "contexts");
+    mkdirSync(contextsDir, { recursive: true });
+    if (metadata.manifest) {
+        const path = join(contextsDir, "document-manifest.json");
+        writeFileSync(path, JSON.stringify(metadata.manifest, null, 2));
+        console.log(`  📋 Document manifest: ${metadata.manifest.length} docs → contexts/document-manifest.json`);
+    }
+    if (metadata.releaseImpact) {
+        const path = join(contextsDir, "release-impact.json");
+        writeFileSync(path, JSON.stringify(metadata.releaseImpact, null, 2));
+        console.log("  📄 Release impact written to contexts/release-impact.json");
+    }
+    if (metadata.documentOverlay) {
+        const path = join(contextsDir, "document-overlay.json");
+        writeFileSync(path, JSON.stringify(metadata.documentOverlay, null, 2));
+        console.log("  📄 Document overlay written to contexts/document-overlay.json");
+    }
+    if (metadata.urlFetch) {
+        const path = join(contextsDir, "url-fetch.json");
+        writeFileSync(path, JSON.stringify(metadata.urlFetch, null, 2));
+        console.log("  📄 URL fetch metadata written to contexts/url-fetch.json");
+    }
+}

package/dist/commands/generate-configs.d.ts

@@ -0,0 +1,8 @@
+/**
+ * generate-configs command — generate promptfoo config files from models.yaml.
+ *
+ * Uses the composition root to wire adapters, then calls generateConfigs()
+ * directly — the same code path as the pipeline.
+ */
+import { Command } from "commander";
+export declare function createGenerateConfigsCommand(): Command;

package/dist/commands/generate-configs.js

@@ -0,0 +1,46 @@
+/**
+ * generate-configs command — generate promptfoo config files from models.yaml.
+ *
+ * Uses the composition root to wire adapters, then calls generateConfigs()
+ * directly — the same code path as the pipeline.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { createAppContext } from "../composition-root.js";
+import { generateConfigs } from "../pipeline/generate-configs.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createGenerateConfigsCommand() {
+    return new Command("generate-configs")
+        .description("Generate promptfoo config files from config/models.yaml")
+        .option("-s, --source <name>", "Documentation source name")
+        .action(async (opts) => {
+        try {
+            const ctx = createAppContext({
+                rootDir: ROOT,
+                mode: "baseline",
+                skipFetch: true,
+                skipEval: true,
+                compareEnabled: false,
+                gapAnalysisEnabled: false,
+                readinessEnabled: false,
+                discoveryReportEnabled: false,
+                publishEnabled: false,
+                noCache: true,
+                noRemoteCache: true,
+                searchMode: "open",
+                source: opts.source,
+            });
+            generateConfigs({
+                rootDir: ctx.config.rootDir,
+                source: opts.source,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+}

package/dist/commands/grader/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * grader subcommand group — tools for measuring grader reliability.
+ *
+ * Exposes four subcommands:
+ *   ailf grader consistency   — measure grading variance (Phase 1)
+ *   ailf grader compare       — inter-grader comparison (Phase 3)
+ *   ailf grader sensitivity   — discrimination power testing (Phase 4)
+ *   ailf grader validate      — accuracy against human references (Phase 2)
+ */
+import { Command } from "commander";
+export declare function createGraderCommand(): Command;

package/dist/commands/grader/index.js

@@ -0,0 +1,118 @@
+/**
+ * grader subcommand group — tools for measuring grader reliability.
+ *
+ * Exposes four subcommands:
+ *   ailf grader consistency   — measure grading variance (Phase 1)
+ *   ailf grader compare       — inter-grader comparison (Phase 3)
+ *   ailf grader sensitivity   — discrimination power testing (Phase 4)
+ *   ailf grader validate      — accuracy against human references (Phase 2)
+ */
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { runGraderCompare } from "../../pipeline/grader-compare-runner.js";
+import { runGraderConsistency } from "../../pipeline/grader-consistency-runner.js";
+import { runGraderSensitivity } from "../../pipeline/grader-sensitivity-runner.js";
+import { runGraderValidate } from "../../pipeline/grader-validate-runner.js";
+import { collect } from "../shared/options.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+export function createGraderCommand() {
+    const cmd = new Command("grader").description("Grader reliability tools");
+    // ── consistency ──────────────────────────────────────────────────────
+    cmd
+        .command("consistency")
+        .description("Measure grader consistency by re-grading existing responses N times")
+        .option("-r, --replications <n>", "Number of additional grading replications", parseInt, 5)
+        .option("--results <path>", "Path to eval-results.json")
+        .action(async (opts) => {
+        try {
+            await runGraderConsistency({
+                replications: opts.replications,
+                resultsPath: opts.results ??
+                    join(ROOT, "results", "latest", "eval-results.json"),
+                rootDir: ROOT,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+    // ── compare ──────────────────────────────────────────────────────────
+    cmd
+        .command("compare")
+        .description("Compare multiple grader models on the same responses")
+        .option("-c, --candidate <model>", "Candidate grader model ID (repeatable)", collect, [])
+        .option("--results <path>", "Path to eval results")
+        .option("-f, --format <fmt>", "Output format: table or json", "table")
+        .option("-o, --output <path>", "Write JSON report to file")
+        .action(async (opts) => {
+        try {
+            const candidates = opts.candidate.map((id) => ({
+                id,
+                label: id.split(":").pop() ?? id,
+            }));
+            await runGraderCompare({
+                candidates,
+                format: opts.format,
+                outputPath: opts.output,
+                resultsPath: opts.results,
+                rootDir: ROOT,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+    // ── sensitivity ──────────────────────────────────────────────────────
+    cmd
+        .command("sensitivity")
+        .description("Test grader discrimination power using programmatic code degradation")
+        .option("-a, --area <name>", "Test only reference solutions in this area")
+        .option("-f, --format <fmt>", "Output format: table or json", "table")
+        .option("-o, --output <path>", "Write JSON report to file")
+        .action(async (opts) => {
+        try {
+            await runGraderSensitivity({
+                areaFilter: opts.area,
+                format: opts.format,
+                outputPath: opts.output,
+                rootDir: ROOT,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+    // ── validate ─────────────────────────────────────────────────────────
+    cmd
+        .command("validate")
+        .description("Validate grader accuracy against human reference grades")
+        .option("-g, --grader <model>", "Grader model to validate")
+        .option("-t, --threshold <n>", "MAE threshold for pass/fail", parseFloat, 10)
+        .action(async (opts) => {
+        try {
+            const result = await runGraderValidate({
+                graderModel: opts.grader,
+                maeThreshold: opts.threshold,
+                rootDir: ROOT,
+            });
+            if (!result.passesThreshold) {
+                console.error(`\n  ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${opts.threshold}`);
+                process.exit(1);
+            }
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+    return cmd;
+}

package/dist/commands/init.d.ts

@@ -0,0 +1,19 @@
+/**
+ * commands/init.ts — Initialize a directory for AI Literacy Framework.
+ *
+ * Creates the .ailf/ directory structure with example configuration and
+ * task files. The generated files are ready-to-edit starting points —
+ * not live evaluation tasks.
+ *
+ * YAML output (default) preserves the inline comments from the source
+ * YAML files in packages/core/examples/. JSON output is a plain
+ * serialization of the parsed data — no comments.
+ *
+ * Usage:
+ *   ailf init                        # YAML output (default)
+ *   ailf init --output-format json   # JSON output
+ *   ailf init --force                # overwrite existing files
+ *   ailf init --path ./my-dir        # target a specific directory
+ */
+import { Command } from "commander";
+export declare function createInitCommand(): Command;

package/dist/commands/init.js

@@ -0,0 +1,150 @@
+/**
+ * commands/init.ts — Initialize a directory for AI Literacy Framework.
+ *
+ * Creates the .ailf/ directory structure with example configuration and
+ * task files. The generated files are ready-to-edit starting points —
+ * not live evaluation tasks.
+ *
+ * YAML output (default) preserves the inline comments from the source
+ * YAML files in packages/core/examples/. JSON output is a plain
+ * serialization of the parsed data — no comments.
+ *
+ * Usage:
+ *   ailf init                        # YAML output (default)
+ *   ailf init --output-format json   # JSON output
+ *   ailf init --force                # overwrite existing files
+ *   ailf init --path ./my-dir        # target a specific directory
+ */
+import { Command } from "commander";
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { resolve, relative } from "path";
+import { ailfConfigData, ailfConfigYaml, taskYamlFiles, TASK_FILE_NAMES, allTaskData, } from "../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Command factory
+// ---------------------------------------------------------------------------
+export function createInitCommand() {
+    return new Command("init")
+        .description("Initialize a directory for AI Literacy Framework evaluation")
+        .option("--output-format <fmt>", 'Output format for generated files: "yaml" (default) or "json"', "yaml")
+        .option("--force", "Overwrite existing files", false)
+        .option("--path <dir>", "Target directory (default: current directory)", ".")
+        .action(async (opts) => {
+        await runInit(opts);
+    });
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Write a file if it doesn't exist (or --force is set).
+ * Returns true if the file was written, false if skipped.
+ */
+function writeIfNew(filePath, content, force) {
+    if (existsSync(filePath) && !force) {
+        return false;
+    }
+    writeFileSync(filePath, content, "utf-8");
+    return true;
+}
+/** Relative path for display, prefixed with ./ */
+function rel(from, to) {
+    const r = relative(from, to);
+    return r.startsWith(".") ? r : `./${r}`;
+}
+// ---------------------------------------------------------------------------
+// Init logic
+// ---------------------------------------------------------------------------
+async function runInit(opts) {
+    const format = opts.outputFormat === "json" ? "json" : "yaml";
+    const ext = format === "json" ? ".json" : ".yaml";
+    const force = opts.force;
+    // Resolve target from the caller's actual working directory
+    const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
+    const targetDir = resolve(callerCwd, opts.path);
+    const ailfDir = resolve(targetDir, ".ailf");
+    const tasksDir = resolve(ailfDir, "tasks");
+    console.log();
+    console.log("  🚀 Initializing AI Literacy Framework");
+    console.log();
+    // 1. Create directories
+    mkdirSync(tasksDir, { recursive: true });
+    console.log(`  ✓ Created ${rel(targetDir, ailfDir)}/`);
+    console.log(`  ✓ Created ${rel(targetDir, tasksDir)}/`);
+    const written = [];
+    const skipped = [];
+    // 2. Write .ailf/config.yaml (or .json)
+    //    YAML: raw string passthrough (preserves comments)
+    //    JSON: serialize the parsed data
+    const configPath = resolve(ailfDir, `config${ext}`);
+    const configContent = format === "yaml"
+        ? ailfConfigYaml
+        : JSON.stringify(ailfConfigData, null, 2) + "\n";
+    if (writeIfNew(configPath, configContent, force)) {
+        written.push(rel(targetDir, configPath));
+    }
+    else {
+        skipped.push(rel(targetDir, configPath));
+    }
+    // 3. Write example tasks to .ailf/tasks/
+    //    YAML: raw string passthrough (preserves comments)
+    //    JSON: serialize individual task data
+    if (format === "yaml") {
+        // Each task is its own commented YAML file — write as-is
+        for (const stem of TASK_FILE_NAMES) {
+            const taskPath = resolve(tasksDir, `${stem}.yaml`);
+            const content = taskYamlFiles[stem];
+            if (writeIfNew(taskPath, content, force)) {
+                written.push(rel(targetDir, taskPath));
+            }
+            else {
+                skipped.push(rel(targetDir, taskPath));
+            }
+        }
+    }
+    else {
+        // JSON: serialize each task's parsed data individually
+        const tasks = Array.isArray(allTaskData)
+            ? allTaskData
+            : [allTaskData];
+        for (const task of tasks) {
+            const taskId = task.id;
+            const taskPath = resolve(tasksDir, `${taskId}.json`);
+            const content = JSON.stringify([task], null, 2) + "\n";
+            if (writeIfNew(taskPath, content, force)) {
+                written.push(rel(targetDir, taskPath));
+            }
+            else {
+                skipped.push(rel(targetDir, taskPath));
+            }
+        }
+    }
+    // 4. Write .gitignore in .ailf/ (keep results out of version control)
+    const gitignorePath = resolve(ailfDir, ".gitignore");
+    const gitignoreContent = `# AILF generated files\nresults/\ncontexts/\n`;
+    if (writeIfNew(gitignorePath, gitignoreContent, force)) {
+        written.push(rel(targetDir, gitignorePath));
+    }
+    else {
+        skipped.push(rel(targetDir, gitignorePath));
+    }
+    // 5. Summary
+    console.log();
+    if (written.length > 0) {
+        for (const f of written) {
+            console.log(`  ✓ Created ${f}`);
+        }
+    }
+    if (skipped.length > 0) {
+        console.log();
+        for (const f of skipped) {
+            console.log(`  ⊘ Skipped ${f} (already exists, use --force to overwrite)`);
+        }
+    }
+    console.log();
+    console.log("  Next steps:");
+    console.log();
+    console.log(`  1. Edit ${rel(targetDir, resolve(ailfDir, `config${ext}`))} with your Sanity project settings`);
+    console.log(`  2. Customize the example tasks in ${rel(targetDir, tasksDir)}/`);
+    console.log("  3. Run: ailf pipeline --repo-tasks-path .ailf/tasks/");
+    console.log();
+}

package/dist/commands/interactive.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Interactive mode — guided wizard for the evaluation pipeline.
+ *
+ * When `ailf` is run with no arguments (or `ailf interactive`), this module
+ * prompts the user through mode selection, area scoping, debug options,
+ * and common flags — then builds and executes the equivalent `ailf pipeline`
+ * command.
+ *
+ * Uses @inquirer/prompts for a clean, modern terminal UI.
+ */
+import { Command } from "commander";
+export declare function createInteractiveCommand(): Command;