@sanity/ailf 0.1.0

package/dist/commands/completion.js

@@ -0,0 +1,260 @@
+/**
+ * completion command — generate shell completion scripts.
+ *
+ * Walks the Commander program tree to produce completions that are
+ * always in sync with the actual CLI surface. Supports bash, zsh, and fish.
+ *
+ * Usage:
+ *   ailf completion bash   # print bash completion script to stdout
+ *   ailf completion zsh    # print zsh completion script to stdout
+ *   ailf completion fish   # print fish completion script to stdout
+ *
+ * Install:
+ *   ailf completion bash >> ~/.bashrc
+ *   ailf completion zsh >> ~/.zshrc
+ *   ailf completion fish > ~/.config/fish/completions/ailf.fish
+ */
+import { Command } from "commander";
+export function createCompletionCommand(program) {
+    return new Command("completion")
+        .description("Generate shell completion scripts")
+        .argument("<shell>", "Shell type: bash, zsh, or fish")
+        .addHelpText("after", `
+Examples:
+  ailf completion bash               # print bash completions to stdout
+  ailf completion zsh                # print zsh completions to stdout
+  ailf completion fish               # print fish completions to stdout
+
+  # Install permanently:
+  ailf completion bash >> ~/.bashrc
+  ailf completion zsh >> ~/.zshrc
+  ailf completion fish > ~/.config/fish/completions/ailf.fish`)
+        .action((shell) => {
+        const commands = extractCommands(program);
+        switch (shell) {
+            case "bash":
+                process.stdout.write(generateBash(commands));
+                break;
+            case "fish":
+                process.stdout.write(generateFish(commands));
+                break;
+            case "zsh":
+                process.stdout.write(generateZsh(commands));
+                break;
+            default:
+                console.error(`Unknown shell: ${shell}. Supported: bash, zsh, fish`);
+                process.exit(1);
+        }
+    });
+}
+// ---------------------------------------------------------------------------
+// Bash completion
+// ---------------------------------------------------------------------------
+function escapeFish(s) {
+    return s.replace(/'/g, "\\'");
+}
+// ---------------------------------------------------------------------------
+// Zsh completion
+// ---------------------------------------------------------------------------
+function escapeZsh(s) {
+    return s.replace(/'/g, "'\\''").replace(/\[/g, "\\[").replace(/]/g, "\\]");
+}
+function extractCommands(cmd) {
+    return cmd.commands.map((sub) => ({
+        description: sub.description(),
+        name: sub.name(),
+        options: sub.options.map((opt) => ({
+            description: opt.description,
+            flags: opt.long
+                ? opt.short
+                    ? [opt.short, opt.long]
+                    : [opt.long]
+                : opt.short
+                    ? [opt.short]
+                    : [],
+        })),
+        subcommands: extractCommands(sub),
+    }));
+}
+// ---------------------------------------------------------------------------
+// Fish completion
+// ---------------------------------------------------------------------------
+function generateBash(commands) {
+    const topLevelNames = commands.map((c) => c.name).join(" ");
+    // Build per-command option lists
+    const cases = [];
+    for (const cmd of commands) {
+        const opts = cmd.options.flatMap((o) => o.flags).join(" ");
+        if (cmd.subcommands.length > 0) {
+            const subNames = cmd.subcommands.map((s) => s.name).join(" ");
+            cases.push(`        ${cmd.name})
+            COMPREPLY=( $(compgen -W "${subNames} ${opts}" -- "\${cur}") )
+            return 0
+            ;;`);
+            for (const sub of cmd.subcommands) {
+                const subOpts = sub.options.flatMap((o) => o.flags).join(" ");
+                cases.push(`        ${sub.name})
+            COMPREPLY=( $(compgen -W "${subOpts}" -- "\${cur}") )
+            return 0
+            ;;`);
+            }
+        }
+        else {
+            cases.push(`        ${cmd.name})
+            COMPREPLY=( $(compgen -W "${opts}" -- "\${cur}") )
+            return 0
+            ;;`);
+        }
+    }
+    return `# bash completion for ailf
+# Generated by: ailf completion bash
+# Install: ailf completion bash >> ~/.bashrc && source ~/.bashrc
+
+_ailf_completions() {
+    local cur prev commands
+    COMPREPLY=()
+    cur="\${COMP_WORDS[COMP_CWORD]}"
+    prev="\${COMP_WORDS[COMP_CWORD-1]}"
+    commands="${topLevelNames}"
+
+    # Determine which command we're in
+    local cmd=""
+    local subcmd=""
+    for ((i=1; i < COMP_CWORD; i++)); do
+        case "\${COMP_WORDS[i]}" in
+            -*)
+                ;;
+            *)
+                if [ -z "$cmd" ]; then
+                    cmd="\${COMP_WORDS[i]}"
+                elif [ -z "$subcmd" ]; then
+                    subcmd="\${COMP_WORDS[i]}"
+                fi
+                ;;
+        esac
+    done
+
+    # Complete subcommand or command-specific options
+    if [ -n "$subcmd" ]; then
+        case "$subcmd" in
+${cases.join("\n")}
+        esac
+    elif [ -n "$cmd" ]; then
+        case "$cmd" in
+${cases.join("\n")}
+        esac
+    else
+        COMPREPLY=( $(compgen -W "\${commands}" -- "\${cur}") )
+    fi
+
+    return 0
+}
+
+complete -F _ailf_completions ailf
+`;
+}
+function generateFish(commands) {
+    const lines = [
+        "# fish completion for ailf",
+        "# Generated by: ailf completion fish",
+        "# Install: ailf completion fish > ~/.config/fish/completions/ailf.fish",
+        "",
+        "# Disable file completions by default",
+        "complete -c ailf -f",
+        "",
+        "# Top-level commands",
+    ];
+    for (const cmd of commands) {
+        lines.push(`complete -c ailf -n '__fish_use_subcommand' -a '${cmd.name}' -d '${escapeFish(cmd.description)}'`);
+    }
+    lines.push("");
+    for (const cmd of commands) {
+        if (cmd.subcommands.length > 0) {
+            lines.push(`# ${cmd.name} subcommands`);
+            for (const sub of cmd.subcommands) {
+                lines.push(`complete -c ailf -n '__fish_seen_subcommand_from ${cmd.name}' -a '${sub.name}' -d '${escapeFish(sub.description)}'`);
+                for (const opt of sub.options) {
+                    const long = opt.flags.find((f) => f.startsWith("--"));
+                    if (long) {
+                        const name = long.replace(/^--/, "");
+                        lines.push(`complete -c ailf -n '__fish_seen_subcommand_from ${sub.name}' -l '${name}' -d '${escapeFish(opt.description)}'`);
+                    }
+                }
+            }
+        }
+        else {
+            lines.push(`# ${cmd.name} options`);
+            for (const opt of cmd.options) {
+                const long = opt.flags.find((f) => f.startsWith("--"));
+                if (long) {
+                    const name = long.replace(/^--/, "");
+                    lines.push(`complete -c ailf -n '__fish_seen_subcommand_from ${cmd.name}' -l '${name}' -d '${escapeFish(opt.description)}'`);
+                }
+            }
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Command factory
+// ---------------------------------------------------------------------------
+function generateZsh(commands) {
+    const cmdEntries = commands
+        .map((c) => `    '${c.name}:${escapeZsh(c.description)}'`)
+        .join("\n");
+    const subcases = [];
+    for (const cmd of commands) {
+        if (cmd.subcommands.length > 0) {
+            const subEntries = cmd.subcommands
+                .map((s) => `        '${s.name}:${escapeZsh(s.description)}'`)
+                .join("\n");
+            subcases.push(`    ${cmd.name})
+      local -a subcmds
+      subcmds=(
+${subEntries}
+      )
+      _describe 'subcommand' subcmds
+      ;;`);
+        }
+        else {
+            const opts = cmd.options
+                .map((o) => {
+                const flag = o.flags[o.flags.length - 1]; // prefer long flag
+                return `'${flag}[${escapeZsh(o.description)}]'`;
+            })
+                .join(" \\\n        ");
+            subcases.push(`    ${cmd.name})
+      _arguments \\
+        ${opts || "'*:file:_files'"}
+      ;;`);
+        }
+    }
+    return `#compdef ailf
+# zsh completion for ailf
+# Generated by: ailf completion zsh
+# Install: ailf completion zsh >> ~/.zshrc && source ~/.zshrc
+
+_ailf() {
+  local -a commands
+  commands=(
+${cmdEntries}
+  )
+
+  _arguments '1:command:->cmd' '*::arg:->args'
+
+  case $state in
+  cmd)
+    _describe 'command' commands
+    ;;
+  args)
+    case $words[1] in
+${subcases.join("\n")}
+    esac
+    ;;
+  esac
+}
+
+compdef _ailf ailf
+`;
+}

package/dist/commands/coverage-audit.d.ts

@@ -0,0 +1,7 @@
+/**
+ * coverage-audit command — cross-references the product feature registry
+ * against task files to produce a documentation coverage audit.
+ *
+ */
+import { Command } from "commander";
+export declare function createCoverageAuditCommand(): Command;

package/dist/commands/coverage-audit.js

@@ -0,0 +1,40 @@
+/**
+ * coverage-audit command — cross-references the product feature registry
+ * against task files to produce a documentation coverage audit.
+ *
+ */
+import { Command } from "commander";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { countReferencedDocs, formatCoverageConsole, formatCoverageMarkdown, runCoverageAudit, } from "../pipeline/coverage-audit.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createCoverageAuditCommand() {
+    return new Command("coverage-audit")
+        .description("Run documentation coverage audit against feature registry")
+        .option("--format <fmt>", "Output format: table, md, markdown")
+        .option("--json", "Output raw JSON", false)
+        .action(async (opts) => {
+        const report = runCoverageAudit(ROOT);
+        if (!report) {
+            console.error("❌ Coverage audit failed. Ensure config/features.yaml exists and is valid.");
+            process.exit(1);
+        }
+        if (opts.json) {
+            console.log(JSON.stringify(report, null, 2));
+        }
+        else if (opts.format === "md" || opts.format === "markdown") {
+            console.log(formatCoverageMarkdown(report));
+        }
+        else {
+            console.log(formatCoverageConsole(report));
+        }
+        // Print document utilization stats for non-JSON console output
+        if (!opts.json && opts.format !== "md" && opts.format !== "markdown") {
+            const docStats = countReferencedDocs(ROOT);
+            console.log("DOCUMENT UTILIZATION:");
+            console.log(`  ${docStats.total} unique document slugs referenced across evaluation tasks`);
+            console.log("");
+        }
+    });
+}

package/dist/commands/discovery-report.d.ts

@@ -0,0 +1,10 @@
+/**
+ * discovery-report command — generates an agent discoverability report
+ * from agentic mode retrieval metrics.
+ *
+ * Reads score-summary.json and produces a markdown report showing
+ * retrieval summary, per-area breakdown, invisible documents, and
+ * recommendations for improving discoverability.
+ */
+import { Command } from "commander";
+export declare function createDiscoveryReportCommand(): Command;

package/dist/commands/discovery-report.js

@@ -0,0 +1,44 @@
+/**
+ * discovery-report command — generates an agent discoverability report
+ * from agentic mode retrieval metrics.
+ *
+ * Reads score-summary.json and produces a markdown report showing
+ * retrieval summary, per-area breakdown, invisible documents, and
+ * recommendations for improving discoverability.
+ */
+import { Command } from "commander";
+import { existsSync, readFileSync, writeFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { formatDiscoveryMarkdown, generateDiscoveryReport, } from "../pipeline/discovery-report.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createDiscoveryReportCommand() {
+    return new Command("discovery-report")
+        .description("Generate agent discoverability report from retrieval metrics")
+        .option("-a, --area <areas>", "Filter by feature areas (comma-separated)")
+        .option("-o, --output <path>", "Write markdown to file instead of stdout")
+        .option("-i, --input <path>", "Path to score-summary.json", join(ROOT, "results", "latest", "score-summary.json"))
+        .action(async (opts) => {
+        const summaryPath = opts.input;
+        if (!existsSync(summaryPath)) {
+            console.error(`❌ Score summary not found: ${summaryPath}`);
+            console.error("Run an agentic evaluation first: pnpm pipeline -- --mode agentic");
+            process.exit(1);
+        }
+        const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+        // Parse area filter
+        const areaFilter = opts.area
+            ? opts.area.split(",").map((a) => a.trim())
+            : undefined;
+        const report = generateDiscoveryReport(summary, areaFilter);
+        const markdown = formatDiscoveryMarkdown(report);
+        if (opts.output) {
+            writeFileSync(opts.output, markdown, "utf-8");
+            console.log(`✅ Discovery report written to ${opts.output}`);
+        }
+        else {
+            console.log(markdown);
+        }
+    });
+}

package/dist/commands/eval.d.ts

@@ -0,0 +1,9 @@
+/**
+ * eval command — run promptfoo evaluation directly (passthrough).
+ *
+ * This is a convenience command that forwards to `promptfoo eval`.
+ * For most use cases, prefer `ailf pipeline` which handles the full
+ * lifecycle (fetch → generate → eval → score → report).
+ */
+import { Command } from "commander";
+export declare function createEvalCommand(): Command;

package/dist/commands/eval.js

@@ -0,0 +1,35 @@
+/**
+ * eval command — run promptfoo evaluation directly (passthrough).
+ *
+ * This is a convenience command that forwards to `promptfoo eval`.
+ * For most use cases, prefer `ailf pipeline` which handles the full
+ * lifecycle (fetch → generate → eval → score → report).
+ */
+import { Command } from "commander";
+export function createEvalCommand() {
+    return new Command("eval")
+        .description("Run Promptfoo evaluation directly (passthrough to promptfoo)")
+        .option("--config <path>", "Promptfoo config file", "promptfooconfig.yaml")
+        .option("--no-cache", "Disable Promptfoo caching")
+        .option("--filter-description <text>", "Filter tests by description")
+        .option("--filter-pattern <regex>", "Filter tests by regex")
+        .option("--filter-first-n <n>", "Run only first N tests")
+        .option("--filter-sample <n>", "Random sample of N tests")
+        .allowUnknownOption(true)
+        .allowExcessArguments(true)
+        .action(async () => {
+        const { execSync } = await import("child_process");
+        // Pass through all raw arguments after "eval" to promptfoo
+        const evalIdx = process.argv.indexOf("eval");
+        const passthroughArgs = evalIdx !== -1 ? process.argv.slice(evalIdx + 1) : [];
+        try {
+            execSync(`dotenv -e ../../.env -o -- promptfoo eval ${passthroughArgs.join(" ")}`, { stdio: "inherit" });
+        }
+        catch (err) {
+            const code = err !== null && typeof err === "object" && "status" in err
+                ? err.status
+                : 1;
+            process.exit(code);
+        }
+    });
+}

package/dist/commands/explain-handler.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Explain-handler.ts
+ *
+ * Handles the global --explain flag for any CLI command.
+ *
+ * Called from the Commander preAction hook in cli.ts. Inspects which
+ * command is about to run, builds the appropriate execution plan, and
+ * prints it. If --yes is also set, prompts the user to confirm execution.
+ *
+ * ## How it works
+ *
+ * Every command is registered in `EXPLAIN_REGISTRY` with either:
+ * - **Static metadata** — description, filesRead, filesCreated, steps
+ * - **A builder function** — for commands that need to inspect CLI options
+ *   or perform async work (e.g., pipeline, init)
+ *
+ * Adding --explain support for a new command = adding one registry entry.
+ * Commands not in the registry fall back to a minimal generic plan.
+ *
+ * @see docs/exec-plans/active/execution-preview.md
+ */
+import type { Command } from "commander";
+/**
+ * Handle the --explain flag for any command.
+ *
+ * Looks up the command in `EXPLAIN_REGISTRY`. Static entries are converted
+ * to a plan via `buildSimpleCommandPlan`. Builder functions are called
+ * directly. Commands not in the registry get a minimal generic plan.
+ *
+ * @param actionCommand - The Commander command about to execute
+ * @param confirmExecution - If true (--yes), prompt to proceed after showing the plan
+ * @param rootDir - Path to the eval package root (packages/eval)
+ */
+export declare function handleExplain(actionCommand: Command, confirmExecution: boolean, rootDir: string): Promise<void>;