@sanity/ailf 0.1.0

package/dist/commands/baseline.js

@@ -0,0 +1,141 @@
+/**
+ * baseline command — manage historical baseline snapshots of evaluation scores.
+ *
+ * Wraps the core baseline functions from pipeline/baseline.ts behind a
+ * Commander subcommand interface: `baseline save`, `baseline compare`,
+ * `baseline history`.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { compareBaseline, listBaselines, saveBaseline, } from "../pipeline/baseline.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "../..");
+export function createBaselineCommand() {
+    const cmd = new Command("baseline").description("Manage historical baseline snapshots of evaluation scores");
+    // -----------------------------------------------------------------------
+    // baseline save
+    // -----------------------------------------------------------------------
+    cmd
+        .command("save")
+        .description("Save current scores as a baseline snapshot")
+        .option("-t, --tag <tag>", "Descriptive tag for the baseline")
+        .action(async (opts) => {
+        console.log("=== Saving baseline snapshot ===\n");
+        const result = saveBaseline(ROOT, opts.tag);
+        if (result.success) {
+            console.log(`  ✅ ${result.message}`);
+        }
+        else {
+            console.error(`  ❌ ${result.message}`);
+            process.exit(1);
+        }
+    });
+    // -----------------------------------------------------------------------
+    // baseline compare
+    // -----------------------------------------------------------------------
+    cmd
+        .command("compare")
+        .description("Compare current scores against a saved baseline")
+        .option("-f, --file <path>", "Specific baseline file to compare against")
+        .action(async (opts) => {
+        console.log("=== Baseline Comparison ===\n");
+        const result = compareBaseline(ROOT, opts.file);
+        if (!result.success) {
+            console.error(`  ❌ ${result.message}`);
+            process.exit(1);
+        }
+        console.log(`  ${result.message}\n`);
+        // Score comparison table
+        console.log("  " +
+            "Feature Area".padEnd(18) +
+            "Current".padEnd(10) +
+            "Baseline".padEnd(10) +
+            "Delta");
+        console.log("  " + "-".repeat(50));
+        for (const c of result.comparisons) {
+            const deltaStr = c.delta > 0 ? `+${c.delta}` : c.delta === 0 ? "=" : String(c.delta);
+            const icon = c.delta > 0 ? "📈" : c.delta < 0 ? "📉" : "➡️";
+            console.log("  " +
+                c.feature.padEnd(18) +
+                String(c.current).padEnd(10) +
+                String(c.baseline).padEnd(10) +
+                `${icon} ${deltaStr}`);
+        }
+        // Cost comparison (only if cost data exists)
+        const hasCostData = result.comparisons.some((c) => c.costCurrent !== undefined || c.costBaseline !== undefined);
+        if (hasCostData) {
+            console.log();
+            console.log("  Cost Comparison:");
+            console.log("  " +
+                "Feature Area".padEnd(18) +
+                "Current".padEnd(10) +
+                "Baseline".padEnd(10) +
+                "Delta");
+            console.log("  " + "-".repeat(50));
+            for (const c of result.comparisons) {
+                if (c.costCurrent === undefined && c.costBaseline === undefined)
+                    continue;
+                const cur = `$${(c.costCurrent ?? 0).toFixed(4)}`;
+                const base = `$${(c.costBaseline ?? 0).toFixed(4)}`;
+                const delta = c.costDelta ?? 0;
+                const deltaStr = delta > 0
+                    ? `+$${delta.toFixed(4)}`
+                    : delta < 0
+                        ? `-$${Math.abs(delta).toFixed(4)}`
+                        : "=";
+                const icon = delta > 0 ? "📈" : delta < 0 ? "📉" : "➡️";
+                console.log("  " +
+                    c.feature.padEnd(18) +
+                    cur.padEnd(10) +
+                    base.padEnd(10) +
+                    `${icon} ${deltaStr}`);
+            }
+        }
+        console.log();
+        const overallIcon = result.overallDelta > 0 ? "📈" : result.overallDelta < 0 ? "📉" : "➡️";
+        const overallStr = result.overallDelta > 0
+            ? `+${result.overallDelta}`
+            : result.overallDelta === 0
+                ? "="
+                : String(result.overallDelta);
+        console.log(`  Overall: ${overallIcon} ${overallStr} points`);
+    });
+    // -----------------------------------------------------------------------
+    // baseline history
+    // -----------------------------------------------------------------------
+    cmd
+        .command("history")
+        .description("List all saved baselines")
+        .action(async () => {
+        console.log("=== Baseline History ===\n");
+        const baselines = listBaselines(ROOT);
+        if (baselines.length === 0) {
+            console.log("  No baselines saved yet.");
+            return;
+        }
+        const hasCosts = baselines.some((b) => b.totalCost !== undefined || b.graderCost !== undefined);
+        const costHeader = hasCosts ? "Cost".padEnd(10) : "";
+        console.log("  " +
+            "Date".padEnd(22) +
+            "Avg".padEnd(6) +
+            "Areas".padEnd(7) +
+            costHeader +
+            "Tag");
+        console.log("  " + "-".repeat(hasCosts ? 60 : 50));
+        for (const b of baselines) {
+            const date = new Date(b.timestamp).toLocaleString();
+            const combinedCost = (b.totalCost ?? 0) + (b.graderCost ?? 0);
+            const costStr = hasCosts
+                ? (combinedCost > 0 ? `$${combinedCost.toFixed(2)}` : "-").padEnd(10)
+                : "";
+            console.log("  " +
+                date.padEnd(22) +
+                String(b.avgScore).padEnd(6) +
+                String(b.areaCount).padEnd(7) +
+                costStr +
+                (b.tag ?? ""));
+        }
+    });
+    return cmd;
+}

package/dist/commands/cache.d.ts

@@ -0,0 +1,13 @@
+/**
+ * cache command — manage the local pipeline cache.
+ *
+ * Subcommands:
+ *   cache clear   Delete all local cache manifests (results/cache/).
+ *   cache status  Show current cache entries and their ages.
+ *
+ * Note: This only affects the local file-system cache used to skip unchanged
+ * pipeline steps. It does NOT touch the remote Content Lake eval cache.
+ * Use --no-remote-cache on pipeline commands to bypass the remote cache.
+ */
+import { Command } from "commander";
+export declare function createCacheCommand(): Command;

package/dist/commands/cache.js

@@ -0,0 +1,135 @@
+/**
+ * cache command — manage the local pipeline cache.
+ *
+ * Subcommands:
+ *   cache clear   Delete all local cache manifests (results/cache/).
+ *   cache status  Show current cache entries and their ages.
+ *
+ * Note: This only affects the local file-system cache used to skip unchanged
+ * pipeline steps. It does NOT touch the remote Content Lake eval cache.
+ * Use --no-remote-cache on pipeline commands to bypass the remote cache.
+ */
+import { Command } from "commander";
+import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+const CACHE_DIR = resolve(ROOT, "results", "cache");
+export function createCacheCommand() {
+    const cmd = new Command("cache").description("Manage the local pipeline cache (does not affect the remote Content Lake cache)");
+    // -----------------------------------------------------------------------
+    // cache clear
+    // -----------------------------------------------------------------------
+    cmd
+        .command("clear")
+        .description("Delete all local cache manifests so every pipeline step re-executes")
+        .action(() => {
+        if (!existsSync(CACHE_DIR)) {
+            console.log("  ℹ️  No local cache directory found — nothing to clear.");
+            return;
+        }
+        const files = readdirSync(CACHE_DIR).filter((f) => f.endsWith(".json"));
+        if (files.length === 0) {
+            console.log("  ℹ️  Local cache directory is empty — nothing to clear.");
+            return;
+        }
+        rmSync(CACHE_DIR, { recursive: true, force: true });
+        console.log(`  🗑️  Cleared ${files.length} local cache manifest(s).`);
+        console.log("  ℹ️  Next pipeline run will re-execute all steps from scratch.");
+        console.log("\n  Note: The remote Content Lake cache is unaffected.");
+        console.log("  Use --no-remote-cache on pipeline commands to bypass it.\n");
+    });
+    // -----------------------------------------------------------------------
+    // cache status
+    // -----------------------------------------------------------------------
+    cmd
+        .command("status")
+        .description("Show current local cache entries and their ages")
+        .action(() => {
+        if (!existsSync(CACHE_DIR)) {
+            console.log("  ℹ️  No local cache directory found.");
+            return;
+        }
+        const files = readdirSync(CACHE_DIR).filter((f) => f.endsWith(".json"));
+        if (files.length === 0) {
+            console.log("  ℹ️  Local cache directory is empty.");
+            return;
+        }
+        console.log("=== Local Pipeline Cache ===\n");
+        console.log("  " +
+            "Step".padEnd(22) +
+            "Age".padEnd(18) +
+            "Duration".padEnd(12) +
+            "Outputs");
+        console.log("  " + "-".repeat(65));
+        for (const file of files.sort()) {
+            const filePath = join(CACHE_DIR, file);
+            try {
+                const raw = readFileSync(filePath, "utf-8");
+                const manifest = JSON.parse(raw);
+                if (!manifest.entry) {
+                    console.log(`  ${manifest.step.padEnd(22)}(empty)`);
+                    continue;
+                }
+                const age = formatAge(manifest.entry.timestamp);
+                const duration = formatDuration(manifest.entry.durationMs);
+                const outputs = manifest.entry.outputPaths.length;
+                console.log("  " +
+                    manifest.step.padEnd(22) +
+                    age.padEnd(18) +
+                    duration.padEnd(12) +
+                    `${outputs} file(s)`);
+            }
+            catch {
+                const step = file.replace(".json", "");
+                console.log(`  ${step.padEnd(22)}(corrupt manifest)`);
+            }
+        }
+        const totalSize = files.reduce((sum, f) => {
+            try {
+                return sum + statSync(join(CACHE_DIR, f)).size;
+            }
+            catch {
+                return sum;
+            }
+        }, 0);
+        console.log();
+        console.log(`  ${files.length} cached step(s), ${formatBytes(totalSize)} on disk`);
+        console.log('  Run "ailf cache clear" to remove all local cache entries.\n');
+    });
+    return cmd;
+}
+// ---------------------------------------------------------------------------
+// Formatting helpers
+// ---------------------------------------------------------------------------
+function formatAge(timestamp) {
+    const then = new Date(timestamp).getTime();
+    const now = Date.now();
+    const diffMs = now - then;
+    if (Number.isNaN(then))
+        return "unknown";
+    const seconds = Math.floor(diffMs / 1_000);
+    const minutes = Math.floor(seconds / 60);
+    const hours = Math.floor(minutes / 60);
+    const days = Math.floor(hours / 24);
+    if (days > 0)
+        return `${days}d ${hours % 24}h ago`;
+    if (hours > 0)
+        return `${hours}h ${minutes % 60}m ago`;
+    if (minutes > 0)
+        return `${minutes}m ago`;
+    return "just now";
+}
+function formatDuration(ms) {
+    if (ms < 1_000)
+        return `${ms}ms`;
+    const seconds = (ms / 1_000).toFixed(1);
+    return `${seconds}s`;
+}
+function formatBytes(bytes) {
+    if (bytes < 1_024)
+        return `${bytes} B`;
+    const kb = (bytes / 1_024).toFixed(1);
+    return `${kb} KB`;
+}

package/dist/commands/calculate-scores.d.ts

@@ -0,0 +1,8 @@
+/**
+ * calculate-scores command — compute AI Literacy Scores from eval results.
+ *
+ * Uses the composition root to wire adapters, then calls
+ * calculateAndWriteScores() directly from pipeline/.
+ */
+import { Command } from "commander";
+export declare function createCalculateScoresCommand(): Command;

package/dist/commands/calculate-scores.js

@@ -0,0 +1,48 @@
+/**
+ * calculate-scores command — compute AI Literacy Scores from eval results.
+ *
+ * Uses the composition root to wire adapters, then calls
+ * calculateAndWriteScores() directly from pipeline/.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { createAppContext } from "../composition-root.js";
+import { calculateAndWriteScores } from "../pipeline/calculate-scores.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createCalculateScoresCommand() {
+    return new Command("calculate-scores")
+        .description("Calculate AI Literacy Scores from Promptfoo evaluation results")
+        .option("--source <name>", "Documentation source name")
+        .argument("[results-path]", "Path to eval-results.json")
+        .action(async (resultsPath, opts) => {
+        try {
+            const ctx = createAppContext({
+                rootDir: ROOT,
+                mode: "baseline",
+                skipFetch: true,
+                skipEval: true,
+                compareEnabled: false,
+                gapAnalysisEnabled: false,
+                readinessEnabled: false,
+                discoveryReportEnabled: false,
+                publishEnabled: false,
+                noCache: true,
+                noRemoteCache: true,
+                searchMode: "open",
+                source: opts.source,
+            });
+            calculateAndWriteScores({
+                resultsPath,
+                rootDir: ctx.config.rootDir,
+                source: opts.source,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+}

package/dist/commands/compare.d.ts

@@ -0,0 +1,8 @@
+/**
+ * compare command — structured comparison between two evaluation runs.
+ *
+ * Wraps the existing compare pipeline logic and formatting utilities
+ * in a Commander.js command for consistent CLI integration.
+ */
+import { Command } from "commander";
+export declare function createCompareCommand(): Command;

package/dist/commands/compare.js

@@ -0,0 +1,120 @@
+/**
+ * compare command — structured comparison between two evaluation runs.
+ *
+ * Wraps the existing compare pipeline logic and formatting utilities
+ * in a Commander.js command for consistent CLI integration.
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { compare } from "../pipeline/compare.js";
+import { DEFAULT_NOISE_THRESHOLD, } from "../pipeline/types.js";
+import { formatComparisonTable } from "../_vendor/ailf-core/index.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+const BASELINES_DIR = join(ROOT, "results", "baselines");
+const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+export function createCompareCommand() {
+    return new Command("compare")
+        .description("Compare two evaluation score summaries")
+        .option("-b, --baseline <path>", "Baseline score-summary.json (default: latest baseline)")
+        .option("-e, --experiment <path>", "Experiment score-summary.json (default: results/latest/score-summary.json)")
+        .option("-t, --threshold <n>", "Noise threshold for unchanged classification", parseFloat)
+        .option("-o, --output <path>", "Write JSON report to file")
+        .option("-f, --format <fmt>", "Output format: table or json", "table")
+        .action(async (opts) => {
+        const threshold = opts.threshold ?? DEFAULT_NOISE_THRESHOLD;
+        // Resolve experiment path
+        const expPath = opts.experiment
+            ? resolve(opts.experiment)
+            : SCORE_SUMMARY_PATH;
+        const experiment = loadSummary(expPath);
+        // Resolve baseline path
+        let basePath;
+        if (opts.baseline) {
+            basePath = resolve(opts.baseline);
+        }
+        else {
+            const latest = findLatestBaseline();
+            if (!latest) {
+                console.error("❌ No baselines found. Run 'pnpm baseline:save' first, or use --baseline <path>.");
+                process.exit(1);
+            }
+            basePath = latest;
+        }
+        const baseline = loadSummary(basePath);
+        // Try to load grader consistency data for empirical thresholds
+        const consistencyPath = join(ROOT, "results", "latest", "grader-consistency.json");
+        let graderConsistency;
+        if (existsSync(consistencyPath) && opts.threshold === undefined) {
+            try {
+                const consistencyRaw = JSON.parse(readFileSync(consistencyPath, "utf-8"));
+                if (consistencyRaw.recommendedThreshold &&
+                    consistencyRaw.perDimension) {
+                    graderConsistency =
+                        consistencyRaw;
+                    console.log(`  📊 Using empirical noise threshold: ±${graderConsistency.recommendedThreshold.toFixed(1)} (from grader consistency data)`);
+                }
+            }
+            catch {
+                // Non-fatal — fall back to default threshold
+            }
+        }
+        console.log(`  Baseline:   ${basePath}`);
+        console.log(`  Experiment: ${expPath}`);
+        if (!graderConsistency) {
+            console.log(`  Threshold:  ±${threshold} (default — run --grader-replications for empirical threshold)`);
+        }
+        console.log("");
+        const report = compare(baseline, experiment, {
+            graderConsistency,
+            noiseThreshold: threshold,
+        });
+        if (opts.format === "json") {
+            const json = JSON.stringify(report, null, 2);
+            if (opts.output) {
+                writeFileSync(opts.output, json);
+                console.log(`  ✅ Comparison report written to ${opts.output}`);
+            }
+            else {
+                console.log(json);
+            }
+        }
+        else {
+            const table = formatComparisonTable(report);
+            console.log(table);
+            if (opts.output) {
+                const json = JSON.stringify(report, null, 2);
+                writeFileSync(opts.output, json);
+                console.log(`  ✅ Comparison report also written to ${opts.output}`);
+            }
+        }
+        // Write comparison report to results/latest for other steps to consume
+        const latestComparisonPath = join(ROOT, "results", "latest", "comparison-report.json");
+        writeFileSync(latestComparisonPath, JSON.stringify(report, null, 2));
+    });
+}
+function findLatestBaseline() {
+    if (!existsSync(BASELINES_DIR))
+        return null;
+    const files = readdirSync(BASELINES_DIR)
+        .filter((f) => f.endsWith(".json"))
+        .sort()
+        .reverse();
+    return files.length > 0 ? join(BASELINES_DIR, files[0]) : null;
+}
+// ---------------------------------------------------------------------------
+// Command factory
+// ---------------------------------------------------------------------------
+function loadSummary(path) {
+    if (!existsSync(path)) {
+        console.error(`❌ File not found: ${path}`);
+        process.exit(1);
+    }
+    const raw = readFileSync(path, "utf-8");
+    return JSON.parse(raw);
+}

package/dist/commands/completion.d.ts

@@ -0,0 +1,18 @@
+/**
+ * completion command — generate shell completion scripts.
+ *
+ * Walks the Commander program tree to produce completions that are
+ * always in sync with the actual CLI surface. Supports bash, zsh, and fish.
+ *
+ * Usage:
+ *   ailf completion bash   # print bash completion script to stdout
+ *   ailf completion zsh    # print zsh completion script to stdout
+ *   ailf completion fish   # print fish completion script to stdout
+ *
+ * Install:
+ *   ailf completion bash >> ~/.bashrc
+ *   ailf completion zsh >> ~/.zshrc
+ *   ailf completion fish > ~/.config/fish/completions/ailf.fish
+ */
+import { Command } from "commander";
+export declare function createCompletionCommand(program: Command): Command;