@sanity/ailf 2.0.0 → 2.0.2

package/dist/commands/capture-compare.js

@@ -0,0 +1,253 @@
+/**
+ * capture compare — compare two pipeline capture directories.
+ *
+ * Reads manifest.json from both captures, runs compareCaptures(),
+ * and prints a human-readable table or JSON diff report.
+ *
+ * Supports both raw directories and .tar.gz archives.
+ *
+ * Exit codes:
+ *   0 — captures are equivalent
+ *   1 — differences found
+ *   2 — error (missing files, invalid manifest, etc.)
+ */
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { Command, Option } from "commander";
+import { compareCaptures } from "../artifact-capture/comparator.js";
+// ---------------------------------------------------------------------------
+// Command factory
+// ---------------------------------------------------------------------------
+export function createCaptureCompareCommand() {
+    return new Command("compare")
+        .description("Compare two pipeline capture directories")
+        .argument("<baseline>", "Path to baseline capture (directory or .tar.gz)")
+        .argument("<experiment>", "Path to experiment capture (directory or .tar.gz)")
+        .addOption(new Option("-m, --mode <mode>", "Comparison mode")
+        .choices(["inventory", "structural", "strict"])
+        .default("inventory"))
+        .option("-f, --format <fmt>", "Output format: table or json", "table")
+        .option("-o, --output <path>", "Write JSON report to file")
+        .option("--score-threshold <n>", "Aggregate score regression threshold (points)", parseFloat, 5)
+        .option("--task-threshold <n>", "Per-task score regression threshold (points)", parseFloat, 10)
+        .option("--timing-threshold <n>", "Step timing multiplier threshold", parseFloat, 2)
+        .option("--json-depth <n>", "JSON structural diff depth", parseInt, 3)
+        .action(async (baselinePath, experimentPath, opts) => {
+        const cleanups = [];
+        try {
+            const baseline = resolveCapturePath(resolve(baselinePath), cleanups);
+            const experiment = resolveCapturePath(resolve(experimentPath), cleanups);
+            console.log("");
+            console.log("  ailf capture compare");
+            console.log("  " + "─".repeat(40));
+            console.log("");
+            console.log(`  Baseline:   ${baselinePath}`);
+            console.log(`  Experiment: ${experimentPath}`);
+            console.log(`  Mode:       ${opts.mode}`);
+            console.log("");
+            const report = compareCaptures(baseline, experiment, {
+                mode: opts.mode,
+                scoreThresholds: {
+                    aggregate: opts.scoreThreshold,
+                    perTask: opts.taskThreshold,
+                },
+                timingThresholds: { multiplier: opts.timingThreshold },
+                jsonDiffDepth: opts.jsonDepth,
+            });
+            if (opts.format === "json") {
+                const json = JSON.stringify(report, null, 2);
+                if (opts.output) {
+                    writeFileSync(opts.output, json, "utf-8");
+                    console.log(`  Report written to ${opts.output}`);
+                }
+                else {
+                    console.log(json);
+                }
+            }
+            else {
+                printTableReport(report);
+                if (opts.output) {
+                    const json = JSON.stringify(report, null, 2);
+                    writeFileSync(opts.output, json, "utf-8");
+                    console.log(`  Report also written to ${opts.output}`);
+                }
+            }
+            process.exitCode = report.equivalent ? 0 : 1;
+        }
+        catch (err) {
+            console.error(`  Error: ${err instanceof Error ? err.message : String(err)}`);
+            process.exitCode = 2;
+        }
+        finally {
+            for (const cleanup of cleanups) {
+                try {
+                    cleanup();
+                }
+                catch {
+                    // Best-effort cleanup
+                }
+            }
+        }
+    });
+}
+// ---------------------------------------------------------------------------
+// Path resolution (handles tar.gz, subdirectories, raw dirs)
+// ---------------------------------------------------------------------------
+function resolveCapturePath(inputPath, cleanups) {
+    if (!existsSync(inputPath)) {
+        throw new Error(`Path does not exist: ${inputPath}`);
+    }
+    if (inputPath.endsWith(".tar.gz")) {
+        const tempDir = mkdtempSync(join(tmpdir(), "ailf-capture-cmp-"));
+        cleanups.push(() => rmSync(tempDir, { recursive: true, force: true }));
+        execFileSync("tar", ["-xzf", inputPath, "-C", tempDir]);
+        return findManifestDir(tempDir);
+    }
+    return findManifestDir(inputPath);
+}
+/**
+ * Find the directory containing manifest.json.
+ *
+ * Handles two cases:
+ * 1. Path IS the capture dir (contains manifest.json directly)
+ * 2. Path is the parent captures/ dir (contains a single timestamped subdir)
+ */
+function findManifestDir(dir) {
+    if (existsSync(join(dir, "manifest.json")))
+        return dir;
+    // Look one level down for a capture subdirectory
+    const entries = readdirSync(dir).filter((e) => !e.startsWith(".") && !e.endsWith(".tar.gz"));
+    for (const entry of entries) {
+        const sub = join(dir, entry);
+        if (existsSync(join(sub, "manifest.json")))
+            return sub;
+    }
+    throw new Error(`No manifest.json found in ${dir} or its subdirectories. ` +
+        `Is this a valid capture directory?`);
+}
+// ---------------------------------------------------------------------------
+// Table formatting
+// ---------------------------------------------------------------------------
+function printTableReport(report) {
+    // Inventory
+    console.log("  Inventory");
+    console.log("  " + "─".repeat(40));
+    console.log(`  Common:    ${report.inventory.common.length} artifact(s)`);
+    if (report.inventory.added.length > 0) {
+        console.log(`  Added:     ${report.inventory.added.length}  (${report.inventory.added.join(", ")})`);
+    }
+    else {
+        console.log("  Added:     0");
+    }
+    if (report.inventory.removed.length > 0) {
+        console.log(`  Removed:   ${report.inventory.removed.length}  (${report.inventory.removed.join(", ")})`);
+    }
+    else {
+        console.log("  Removed:   0");
+    }
+    console.log("");
+    // Content diff preview (structural/strict modes)
+    if (report.content && report.content.length > 0) {
+        console.log("  Content Changes");
+        console.log("  " + "─".repeat(40));
+        for (const diff of report.content.slice(0, 10)) {
+            printContentDiff(diff);
+        }
+        if (report.content.length > 10) {
+            console.log(`  ... and ${report.content.length - 10} more changed artifact(s)`);
+        }
+        console.log("");
+    }
+    // Scores
+    if (report.scores) {
+        console.log("  Scores");
+        console.log("  " + "─".repeat(40));
+        const { baselineMean, currentMean, delta } = report.scores;
+        const icon = delta > 0 ? "+" : delta < 0 ? "" : " ";
+        console.log(`  Aggregate: ${baselineMean.toFixed(1)} -> ${currentMean.toFixed(1)}  (${icon}${delta.toFixed(1)})`);
+        if (report.scores.breaches.length > 0) {
+            console.log(`  Breaches:  ${report.scores.breaches.length}`);
+            for (const b of report.scores.breaches) {
+                console.log(`    - ${b}`);
+            }
+        }
+        else {
+            console.log("  Breaches:  none");
+        }
+        console.log("");
+    }
+    // Timing
+    if (report.timing) {
+        console.log("  Timing");
+        console.log("  " + "─".repeat(40));
+        const { totalDeltaMs } = report.timing;
+        const sign = totalDeltaMs >= 0 ? "+" : "";
+        console.log(`  Total delta: ${sign}${totalDeltaMs}ms`);
+        if (report.timing.breaches.length > 0) {
+            console.log(`  Breaches:    ${report.timing.breaches.length}`);
+            for (const b of report.timing.breaches) {
+                console.log(`    - ${b}`);
+            }
+        }
+        else {
+            console.log("  Breaches:    none");
+        }
+        console.log("");
+    }
+    // Security
+    console.log("  Security");
+    console.log("  " + "─".repeat(40));
+    if (report.security.leaksFound) {
+        console.log(`  Leaks:     ${report.security.violations.length} finding(s)`);
+        for (const v of report.security.violations.slice(0, 5)) {
+            console.log(`    - ${v.file}: ${v.detail}`);
+        }
+        if (report.security.violations.length > 5) {
+            console.log(`    ... and ${report.security.violations.length - 5} more`);
+        }
+    }
+    else {
+        console.log("  Leaks:     none");
+    }
+    console.log("");
+    // Result
+    if (report.equivalent) {
+        console.log("  Result: EQUIVALENT");
+    }
+    else {
+        console.log(`  Result: DIFFERENCES FOUND`);
+        console.log(`  ${report.summary}`);
+    }
+    console.log("");
+}
+/**
+ * Print up to 3 changed key paths for a content diff.
+ */
+function printContentDiff(diff) {
+    console.log(`  ${diff.artifactKey} (${diff.format})`);
+    if (Array.isArray(diff.changes)) {
+        // JSON diff — show up to 3 changed paths
+        const jsonChanges = diff.changes;
+        for (const change of jsonChanges.slice(0, 3)) {
+            if (change.baseline === undefined) {
+                console.log(`    + ${change.path} (added)`);
+            }
+            else if (change.experiment === undefined) {
+                console.log(`    - ${change.path} (removed)`);
+            }
+            else {
+                console.log(`    ~ ${change.path} (changed)`);
+            }
+        }
+        if (jsonChanges.length > 3) {
+            console.log(`    ... ${jsonChanges.length - 3} more change(s)`);
+        }
+    }
+    else {
+        // Text/markdown diff
+        const { addedLines, removedLines } = diff.changes;
+        console.log(`    +${addedLines} / -${removedLines} lines`);
+    }
+}

package/dist/commands/capture-list.d.ts

@@ -0,0 +1,12 @@
+/**
+ * capture list — list pipeline captures in a directory.
+ *
+ * Scans a capture directory for subdirectories containing manifest.json,
+ * reads each manifest, and prints a summary table sorted by date.
+ *
+ * Usage:
+ *   ailf capture list                          # default: results/captures/
+ *   ailf capture list ./my-captures            # custom directory
+ */
+import { Command } from "commander";
+export declare function createCaptureListCommand(): Command;

package/dist/commands/capture-list.js

@@ -0,0 +1,147 @@
+/**
+ * capture list — list pipeline captures in a directory.
+ *
+ * Scans a capture directory for subdirectories containing manifest.json,
+ * reads each manifest, and prints a summary table sorted by date.
+ *
+ * Usage:
+ *   ailf capture list                          # default: results/captures/
+ *   ailf capture list ./my-captures            # custom directory
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { Command } from "commander";
+// ---------------------------------------------------------------------------
+// Command factory
+// ---------------------------------------------------------------------------
+export function createCaptureListCommand() {
+    return new Command("list")
+        .description("List pipeline captures in a directory")
+        .argument("[dir]", "Captures directory (default: results/captures/)")
+        .option("-f, --format <fmt>", "Output format: table or json", "table")
+        .action(async (dir, opts) => {
+        const captureDir = resolve(dir ?? "results/captures");
+        if (!existsSync(captureDir)) {
+            console.error(`  No captures directory found at ${captureDir}`);
+            console.error("  Run 'ailf pipeline --capture' to create captures.");
+            process.exitCode = 1;
+            return;
+        }
+        const captures = discoverCaptures(captureDir);
+        if (captures.length === 0) {
+            console.log(`  No captures found in ${captureDir}`);
+            return;
+        }
+        // Sort by startedAt descending (newest first)
+        captures.sort((a, b) => new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime());
+        if (opts.format === "json") {
+            console.log(JSON.stringify(captures, null, 2));
+            return;
+        }
+        console.log("");
+        console.log("  ailf capture list");
+        console.log("  " + "─".repeat(60));
+        console.log("");
+        console.log("  " +
+            "Date".padEnd(22) +
+            "Mode".padEnd(18) +
+            "Artifacts".padEnd(12) +
+            "Size".padEnd(10) +
+            "Path");
+        console.log("  " + "─".repeat(60));
+        for (const c of captures) {
+            const date = new Date(c.startedAt).toLocaleString("en-US", {
+                month: "short",
+                day: "2-digit",
+                hour: "2-digit",
+                minute: "2-digit",
+            });
+            const mode = c.mode.padEnd(18);
+            const artifacts = String(c.artifactCount).padEnd(12);
+            const size = formatBytes(c.totalBytes).padEnd(10);
+            console.log(`  ${date.padEnd(22)}${mode}${artifacts}${size}${c.name}`);
+        }
+        console.log("");
+        console.log(`  ${captures.length} capture(s) found in ${captureDir}`);
+        console.log("");
+    });
+}
+function discoverCaptures(captureDir) {
+    const entries = [];
+    for (const name of readdirSync(captureDir)) {
+        if (name.startsWith("."))
+            continue;
+        const fullPath = join(captureDir, name);
+        // Raw directory with manifest.json
+        const manifestPath = join(fullPath, "manifest.json");
+        if (existsSync(manifestPath)) {
+            try {
+                const manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
+                const totalBytes = manifest.artifacts.reduce((sum, a) => sum + a.bytes, 0);
+                entries.push({
+                    name,
+                    path: fullPath,
+                    mode: manifest.pipeline.mode,
+                    startedAt: manifest.startedAt,
+                    artifactCount: manifest.artifacts.length,
+                    totalBytes,
+                    compressed: false,
+                });
+            }
+            catch {
+                // Skip unparseable manifests
+            }
+            continue;
+        }
+        // .tar.gz archive — read size but don't extract
+        if (name.endsWith(".tar.gz")) {
+            try {
+                const stat = statSync(fullPath);
+                entries.push({
+                    name,
+                    path: fullPath,
+                    mode: extractModeFromName(name),
+                    startedAt: extractTimestampFromName(name),
+                    artifactCount: -1, // Unknown without extracting
+                    totalBytes: stat.size,
+                    compressed: true,
+                });
+            }
+            catch {
+                // Skip
+            }
+        }
+    }
+    return entries;
+}
+/** Known modes — used to correctly extract mode from hyphenated filenames. */
+const KNOWN_MODES = [
+    "literacy",
+    "mcp-server",
+    "agent-harness",
+    "knowledge-probe",
+];
+function extractModeFromName(name) {
+    for (const mode of KNOWN_MODES) {
+        if (name.startsWith(mode + "-"))
+            return mode;
+    }
+    return name.split("-")[0] ?? "unknown";
+}
+function extractTimestampFromName(name) {
+    // Pattern: {mode}-YYYYMMDD-HHmmss-{id}.tar.gz
+    const match = name.match(/(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})/);
+    if (!match)
+        return "unknown";
+    const [, y, m, d, h, min, s] = match;
+    return `${y}-${m}-${d}T${h}:${min}:${s}Z`;
+}
+function formatBytes(bytes) {
+    if (bytes < 0)
+        return "?";
+    if (bytes < 1024)
+        return `${bytes}B`;
+    if (bytes < 1024 * 1024)
+        return `${(bytes / 1024).toFixed(1)}KB`;
+    return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+}

package/dist/commands/capture.d.ts

@@ -0,0 +1,9 @@
+/**
+ * capture command — manage and compare pipeline artifact captures.
+ *
+ * Parent command for capture-related subcommands:
+ *   ailf capture compare <baseline> <experiment>
+ *   ailf capture list [dir]
+ */
+import { Command } from "commander";
+export declare function createCaptureCommand(): Command;

package/dist/commands/capture.js

@@ -0,0 +1,16 @@
+/**
+ * capture command — manage and compare pipeline artifact captures.
+ *
+ * Parent command for capture-related subcommands:
+ *   ailf capture compare <baseline> <experiment>
+ *   ailf capture list [dir]
+ */
+import { Command } from "commander";
+import { createCaptureCompareCommand } from "./capture-compare.js";
+import { createCaptureListCommand } from "./capture-list.js";
+export function createCaptureCommand() {
+    const cmd = new Command("capture").description("Manage and compare pipeline artifact captures");
+    cmd.addCommand(createCaptureCompareCommand());
+    cmd.addCommand(createCaptureListCommand());
+    return cmd;
+}

package/dist/commands/chronic-failures.d.ts

@@ -0,0 +1,8 @@
+/**
+ * chronic-failures command — query recent reports to find tasks that
+ * consistently error above a configurable threshold.
+ *
+ * @see docs/exec-plans/eval-pipeline-timeout-resilience.md — Phase 5
+ */
+import { Command } from "commander";
+export declare function createChronicFailuresCommand(): Command;

package/dist/commands/chronic-failures.js

@@ -0,0 +1,33 @@
+/**
+ * chronic-failures command — query recent reports to find tasks that
+ * consistently error above a configurable threshold.
+ *
+ * @see docs/exec-plans/eval-pipeline-timeout-resilience.md — Phase 5
+ */
+import { Command } from "commander";
+import { detectChronicFailures, formatChronicFailuresConsole, } from "../pipeline/chronic-failures.js";
+import { ReportStore } from "../report-store.js";
+export function createChronicFailuresCommand() {
+    return new Command("chronic-failures")
+        .description("Identify tasks that error in >50% of recent evaluation runs")
+        .option("--lookback <n>", "Number of recent reports to analyze", (v) => parseInt(v, 10), 10)
+        .option("--threshold <n>", "Error rate threshold (0-1) for chronic classification", (v) => parseFloat(v), 0.5)
+        .option("--json", "Output raw JSON", false)
+        .action(async (opts) => {
+        const reportStore = new ReportStore();
+        const report = await detectChronicFailures(reportStore, {
+            lookback: opts.lookback,
+            threshold: opts.threshold,
+        });
+        if (opts.json) {
+            console.log(JSON.stringify(report, null, 2));
+        }
+        else {
+            console.log(formatChronicFailuresConsole(report));
+        }
+        // Exit with non-zero if chronic failures detected
+        if (report.failures.length > 0) {
+            process.exit(1);
+        }
+    });
+}

package/dist/commands/explain-handler.d.ts

@@ -17,7 +17,7 @@
  * Adding --explain support for a new command = adding one registry entry.
  * Commands not in the registry fall back to a minimal generic plan.
  *
- * @see docs/exec-plans/execution-preview.md
+ * @see docs/archive/exec-plans/execution-preview.md
  */
 import type { Command } from "commander";
 /**

package/dist/commands/explain-handler.js

@@ -17,7 +17,7 @@
  * Adding --explain support for a new command = adding one registry entry.
  * Commands not in the registry fall back to a minimal generic plan.
  *
- * @see docs/exec-plans/execution-preview.md
+ * @see docs/archive/exec-plans/execution-preview.md
  */
 import { TASK_FILE_NAMES } from "../_vendor/ailf-core/index.js";
 import { buildPipelinePlan, buildSimpleCommandPlan, } from "../pipeline/plan.js";
@@ -137,6 +137,24 @@ const EXPLAIN_REGISTRY = {
             },
         ],
     },
+    "chronic-failures": {
+        description: "Identify tasks that error in >50% of recent evaluation runs",
+        filesRead: [],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Query Content Lake",
+                reason: "Fetch testSummary.errors from recent reports",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Aggregate errors",
+                reason: "Compute per-task error rates across runs",
+                willRun: true,
+            },
+        ],
+    },
     "coverage-audit": {
         description: "Cross-reference feature registry against evaluation tasks for coverage gaps",
         filesRead: ["config/features.ts", "tasks/*.{yaml,task.ts,task.js}"],
@@ -295,7 +313,7 @@ const EXPLAIN_REGISTRY = {
     "measure-retrieval": {
         description: "Measure Sanity text search retrieval quality against canonical document annotations",
         filesCreated: ["results/latest/retrieval-metrics.json"],
-        filesRead: ["tasks/*.yaml"],
+        filesRead: ["tasks/literacy/*.task.ts"],
         steps: [
             {
                 cacheStatus: "miss",
@@ -576,16 +594,23 @@ export async function handleExplain(actionCommand, confirmExecution, rootDir) {
  */
 function buildInitExplainPlan(actionCommand, rootDir) {
     const opts = actionCommand.opts();
-    const format = opts.outputFormat === "json" ? "json" : "yaml";
-    const ext = format === "json" ? ".json" : ".yaml";
+    const format = opts.outputFormat === "json"
+        ? "json"
+        : opts.outputFormat === "yaml"
+            ? "yaml"
+            : "ts";
+    const taskExt = format === "ts" ? ".task.ts" : format === "yaml" ? ".yaml" : ".json";
+    const configFile = format === "ts"
+        ? "ailf.config.ts"
+        : `config.${format === "yaml" ? "yaml" : "json"}`;
     const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
     const targetDir = opts.path ?? ".";
     const ailfDir = `${targetDir}/.ailf`;
     const tasksDir = `${ailfDir}/tasks`;
     const taskFileNames = [...TASK_FILE_NAMES];
     const filesCreated = [
-        `${ailfDir}/config${ext}`,
-        ...taskFileNames.map((stem) => `${tasksDir}/${stem}${ext}`),
+        `${ailfDir}/${configFile}`,
+        ...taskFileNames.map((stem) => `${tasksDir}/${stem}${taskExt}`),
         `${ailfDir}/.gitignore`,
     ];
     return buildSimpleCommandPlan({
@@ -603,14 +628,14 @@ function buildInitExplainPlan(actionCommand, rootDir) {
             },
             {
                 cacheStatus: "miss",
-                name: `Write config${ext}`,
+                name: `Write ${configFile}`,
                 reason: `Project configuration template (${format.toUpperCase()} format)`,
                 willRun: true,
             },
             {
                 cacheStatus: "miss",
                 name: `Write example tasks (${taskFileNames.length} files)`,
-                reason: `Commented starter tasks in ${tasksDir}/ (${format.toUpperCase()} format)`,
+                reason: `Starter tasks in ${tasksDir}/ (${format.toUpperCase()} format)`,
                 willRun: true,
             },
             {
@@ -700,6 +725,10 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         taskSource: raw.taskSource,
         remoteCache: raw.remoteCache,
         config: raw.config,
+        capture: raw.capture ?? false,
+        captureCompress: raw.captureCompress ?? true,
+        captureExtras: raw.captureExtras ?? true,
+        captureDir: raw.captureDir,
     };
     const resolved = computeResolvedOptions(withDefaults);
     const planOpts = {

package/dist/commands/fetch-docs.js

@@ -41,6 +41,7 @@ async function executeFetchDocs(opts) {
     // Build a minimal ResolvedConfig for the composition root
     const ctx = createAppContext({
         rootDir: ROOT,
+        outputDir: resolve(ROOT, "results", "latest"),
         mode: "literacy",
         noAutoScope: false,
         skipFetch: false,

package/dist/commands/generate-configs.d.ts

@@ -1,8 +1,8 @@
 /**
- * generate-configs command — generate promptfoo config files from models.yaml.
+ * generate-configs command — generate promptfoo config files via the compiler pipeline.
  *
- * Uses the composition root to wire adapters, then calls generateConfigs()
- * directly — the same code path as the pipeline.
+ * Uses the composition root to wire adapters, then runs GenerateConfigsStep
+ * directly — the same code path as the full pipeline.
  */
 import { Command } from "commander";
 export declare function createGenerateConfigsCommand(): Command;

package/dist/commands/generate-configs.js

@@ -1,14 +1,14 @@
 /**
- * generate-configs command — generate promptfoo config files from models.yaml.
+ * generate-configs command — generate promptfoo config files via the compiler pipeline.
  *
- * Uses the composition root to wire adapters, then calls generateConfigs()
- * directly — the same code path as the pipeline.
+ * Uses the composition root to wire adapters, then runs GenerateConfigsStep
+ * directly — the same code path as the full pipeline.
  */
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
 import { Command } from "commander";
 import { createAppContext } from "../composition-root.js";
-import { generateConfigs } from "../pipeline/generate-configs.js";
+import { GenerateConfigsStep } from "../orchestration/steps/generate-configs-step.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, "..", "..");
 export function createGenerateConfigsCommand() {
@@ -19,6 +19,7 @@ export function createGenerateConfigsCommand() {
         try {
             const ctx = createAppContext({
                 rootDir: ROOT,
+                outputDir: resolve(ROOT, "results", "latest"),
                 mode: "literacy",
                 noAutoScope: false,
                 skipFetch: true,
@@ -35,10 +36,21 @@ export function createGenerateConfigsCommand() {
                 remote: false,
                 apiUrl: "https://ailf-api.sanity.build",
             });
-            generateConfigs({
-                rootDir: ctx.config.rootDir,
-                source: opts.source,
-            });
+            const step = new GenerateConfigsStep();
+            // Run validation checks first
+            const issues = step.check(ctx);
+            if (issues.length > 0) {
+                for (const issue of issues) {
+                    console.error(`  ❌ ${issue.message}`);
+                }
+                process.exitCode = 1;
+                return;
+            }
+            const result = await step.execute(ctx, {});
+            if (result.status === "failed") {
+                console.error(`❌ ${result.error}`);
+                process.exitCode = 1;
+            }
         }
         catch (err) {
             process.exitCode = 1;