@sanity/ailf 0.1.0

package/dist/commands/explain-handler.js

@@ -0,0 +1,719 @@
+/**
+ * Explain-handler.ts
+ *
+ * Handles the global --explain flag for any CLI command.
+ *
+ * Called from the Commander preAction hook in cli.ts. Inspects which
+ * command is about to run, builds the appropriate execution plan, and
+ * prints it. If --yes is also set, prompts the user to confirm execution.
+ *
+ * ## How it works
+ *
+ * Every command is registered in `EXPLAIN_REGISTRY` with either:
+ * - **Static metadata** — description, filesRead, filesCreated, steps
+ * - **A builder function** — for commands that need to inspect CLI options
+ *   or perform async work (e.g., pipeline, init)
+ *
+ * Adding --explain support for a new command = adding one registry entry.
+ * Commands not in the registry fall back to a minimal generic plan.
+ *
+ * @see docs/exec-plans/active/execution-preview.md
+ */
+import { TASK_FILE_NAMES } from "../_vendor/ailf-core/index.js";
+import { buildPipelinePlan, buildSimpleCommandPlan, } from "../pipeline/plan.js";
+import { formatPlanConsole, formatPlanJson } from "../pipeline/plan-format.js";
+import { computeResolvedOptions } from "./pipeline-action.js";
+// ---------------------------------------------------------------------------
+// Registry
+// ---------------------------------------------------------------------------
+/**
+ * Explain metadata for every command.
+ *
+ * To add --explain support for a new command, add an entry here.
+ * Static entries are converted to a plan via `buildSimpleCommandPlan`.
+ * Builder functions receive the Commander command and rootDir.
+ */
+const EXPLAIN_REGISTRY = {
+    // ── Dynamic builders (inspect CLI options) ────────────────────────
+    baseline: buildBaselineExplainPlan,
+    init: buildInitExplainPlan,
+    pipeline: buildPipelineExplainPlan,
+    // ── Static metadata ───────────────────────────────────────────────
+    "agent-report": {
+        description: "Generate an agent behavior observation report from eval results",
+        filesCreated: [
+            "results/latest/agent-report.json",
+            "results/latest/agent-report.md",
+        ],
+        filesRead: ["results/latest/eval-results.json"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load eval results",
+                reason: "Parse eval-results.json for agent behavior data",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Analyze agent behavior",
+                reason: "Extract tool usage, search patterns, and retrieval strategies",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Write report",
+                reason: "Generate JSON + Markdown observation report",
+                willRun: true,
+            },
+        ],
+    },
+    cache: {
+        description: "Manage the local pipeline cache (clear manifests or show status)",
+        filesRead: ["results/cache/*.json"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Inspect cache directory",
+                reason: "Read results/cache/ for manifest files",
+                willRun: true,
+            },
+        ],
+    },
+    "calculate-scores": {
+        description: "Calculate AI Literacy Scores from Promptfoo evaluation results",
+        filesCreated: ["results/latest/score-summary.json"],
+        filesRead: [
+            "results/latest/eval-results.json",
+            "config/rubrics.yaml",
+            "config/models.yaml",
+        ],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load eval results",
+                reason: "Parse eval-results.json for graded responses",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Compute scores",
+                reason: "Calculate per-area, per-model, and overall AI Literacy Scores",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Write summary",
+                reason: "Persist score-summary.json to results/latest/",
+                willRun: true,
+            },
+        ],
+    },
+    compare: {
+        description: "Compare current evaluation scores against a saved baseline",
+        filesCreated: ["results/latest/comparison-report.json"],
+        filesRead: [
+            "results/latest/score-summary.json",
+            "results/baselines/*.json",
+        ],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load current scores",
+                reason: "Read results/latest/score-summary.json",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Load baseline",
+                reason: "Find and load latest baseline snapshot",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Compare",
+                reason: "Compute per-area deltas with noise threshold",
+                willRun: true,
+            },
+        ],
+    },
+    "coverage-audit": {
+        description: "Cross-reference feature registry against evaluation tasks for coverage gaps",
+        filesRead: ["config/features.yaml", "tasks/*.yaml"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load feature registry",
+                reason: "Parse config/features.yaml for product feature list",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Scan tasks",
+                reason: "Read all task YAML files for feature area references",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Compute coverage",
+                reason: "Match features to tasks, identify gaps, count document utilization",
+                willRun: true,
+            },
+        ],
+    },
+    "discovery-report": {
+        description: "Generate agent discoverability report from agentic retrieval metrics",
+        filesCreated: ["results/latest/discovery-report.md"],
+        filesRead: ["results/latest/score-summary.json"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load score summary",
+                reason: "Read retrieval metrics from score-summary.json",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Analyze discoverability",
+                reason: "Per-area retrieval breakdown, invisible docs, recommendations",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Format report",
+                reason: "Generate Markdown discovery report",
+                willRun: true,
+            },
+        ],
+    },
+    eval: {
+        description: "Run Promptfoo evaluation directly (passthrough — all flags forwarded to promptfoo)",
+        filesCreated: ["results/latest/eval-results.json"],
+        filesRead: ["promptfooconfig.yaml"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Exec promptfoo eval",
+                reason: "Spawn `promptfoo eval` subprocess with forwarded arguments",
+                willRun: true,
+            },
+        ],
+    },
+    "fetch-docs": {
+        description: "Fetch documentation from Sanity CMS and generate canonical context files",
+        filesCreated: ["contexts/canonical/*.md"],
+        filesRead: ["config/sources.yaml", "config/models.yaml"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Extract canonical doc slugs",
+                reason: "Read task definitions for canonical_docs fields",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Fetch from Sanity",
+                reason: "Query Sanity Content Lake via GROQ for each document",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Write context files",
+                reason: "Convert Portable Text → Markdown, write per-task context files",
+                willRun: true,
+            },
+        ],
+    },
+    "generate-configs": {
+        description: "Generate Promptfoo config files from models.yaml and task definitions",
+        filesCreated: [
+            "promptfooconfig.yaml",
+            "promptfooconfig.observed.yaml",
+            "promptfooconfig.agentic.yaml",
+            "tasks/.expanded.yaml",
+        ],
+        filesRead: [
+            "config/models.yaml",
+            "config/prompts.yaml",
+            "config/rubrics.yaml",
+            "config/sources.yaml",
+        ],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load models",
+                reason: "Parse config/models.yaml for active model list",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Expand tasks",
+                reason: "Single-definition → gold + baseline test entries",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Generate configs",
+                reason: "Write baseline, observed, and agentic Promptfoo configs",
+                willRun: true,
+            },
+        ],
+    },
+    grader: {
+        description: "Grader reliability tools (consistency, compare, sensitivity, validate)",
+        filesRead: [
+            "results/latest/eval-results.json",
+            "config/rubrics.yaml",
+            "canonical/reference-solutions/",
+        ],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load eval results",
+                reason: "Parse eval-results.json for graded responses to re-evaluate",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Run grader analysis",
+                reason: "Execute selected subcommand (consistency / compare / sensitivity / validate)",
+                willRun: true,
+            },
+        ],
+    },
+    "lookup-doc": {
+        description: "Search Sanity for documentation articles by keyword (find slugs for canonicalDocs)",
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Query Sanity",
+                reason: "Search articles by title and slug keyword match",
+                willRun: true,
+            },
+        ],
+    },
+    "measure-retrieval": {
+        description: "Measure Sanity text search retrieval quality against canonical document annotations",
+        filesCreated: ["results/latest/retrieval-metrics.json"],
+        filesRead: ["tasks/*.yaml"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Extract queries + ground truth",
+                reason: "Build query → expected docs pairs from task canonical_docs",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Run retrieval queries",
+                reason: "Execute Sanity text search for each task query",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Compute metrics",
+                reason: "Calculate precision, recall, F1 per query and overall",
+                willRun: true,
+            },
+        ],
+    },
+    "pr-comment": {
+        description: "Generate a markdown PR comment from evaluation scores for CI posting",
+        filesRead: ["results/latest/score-summary.json"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load scores",
+                reason: "Read score-summary.json",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Generate comment",
+                reason: "Build markdown with score tables, model breakdown, and Promptfoo link",
+                willRun: true,
+            },
+        ],
+    },
+    publish: {
+        description: "Publish a local evaluation report to the Sanity Content Lake (standalone)",
+        filesRead: ["results/latest/score-summary.json"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load score summary",
+                reason: "Read and validate score-summary.json",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Build provenance",
+                reason: "Collect git SHA, branch, CI metadata, source info",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Write to Sanity",
+                reason: "Create immutable ailf.report document in Content Lake",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Fan out to sinks",
+                reason: "Deliver to configured sinks (Slack, BigQuery, webhooks)",
+                willRun: true,
+            },
+        ],
+    },
+    "readiness-report": {
+        description: "Generate launch readiness checklist for a feature area with threshold evaluation",
+        filesRead: [
+            "results/latest/score-summary.json",
+            "results/latest/gap-analysis.json",
+            "config/thresholds.yaml",
+            "results/baselines/",
+        ],
+        filesCreated: ["results/latest/readiness-report.md"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load scores + thresholds",
+                reason: "Read score-summary.json and thresholds.yaml for gate evaluation",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Evaluate readiness",
+                reason: "Check per-dimension scores against thresholds, build go/no-go checklist",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Format report",
+                reason: "Generate Markdown readiness report with historical trends",
+                willRun: true,
+            },
+        ],
+    },
+    validate: {
+        description: "Validate all YAML config files, task definitions, reference solutions, and environment",
+        filesRead: [
+            "config/models.yaml",
+            "config/rubrics.yaml",
+            "config/features.yaml",
+            "config/thresholds.yaml",
+        ],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Validate configuration",
+                reason: "Parse all YAML configs through Zod schemas, cross-reference mappings",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Check environment",
+                reason: "Verify OPENAI_API_KEY, SANITY_API_TOKEN, AILF_REPORT_SANITY_API_TOKEN are set",
+                willRun: true,
+            },
+        ],
+    },
+    "validate-tasks": {
+        description: "Validate repo-based task YAML files (.ailf/tasks/) against the schema",
+        filesRead: [".ailf/tasks/*.yaml"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Scan task files",
+                reason: "Find all .yaml/.yml files in the target directory",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Validate schemas",
+                reason: "Parse each file through RepoTaskSchema (Zod) and cross-reference IDs",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Check invariants",
+                reason: "Verify canonical_docs references, doc_coverage completeness, unique IDs",
+                willRun: true,
+            },
+        ],
+    },
+    "webhook-server": {
+        description: "Start a local webhook development server for testing content-change triggers",
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Start HTTP server",
+                reason: "Listen for Sanity webhook payloads with debounce + budget control",
+                willRun: true,
+            },
+        ],
+    },
+    "weekly-digest": {
+        description: "Generate and deliver a weekly evaluation trend digest via Slack",
+        filesRead: ["config/schedules.yaml", "config/sinks.yaml"],
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Load digest config",
+                reason: "Read schedules.yaml for lookback window and delivery targets",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Query report store",
+                reason: "Fetch reports within lookback window from Sanity Content Lake",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Compute trends",
+                reason: "Calculate score deltas, area regressions, and improvements",
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Deliver digest",
+                reason: "Format and send to Slack (or stdout in --dry-run mode)",
+                willRun: true,
+            },
+        ],
+    },
+};
+// ---------------------------------------------------------------------------
+// Main handler
+// ---------------------------------------------------------------------------
+/**
+ * Handle the --explain flag for any command.
+ *
+ * Looks up the command in `EXPLAIN_REGISTRY`. Static entries are converted
+ * to a plan via `buildSimpleCommandPlan`. Builder functions are called
+ * directly. Commands not in the registry get a minimal generic plan.
+ *
+ * @param actionCommand - The Commander command about to execute
+ * @param confirmExecution - If true (--yes), prompt to proceed after showing the plan
+ * @param rootDir - Path to the eval package root (packages/eval)
+ */
+export async function handleExplain(actionCommand, confirmExecution, rootDir) {
+    const commandName = actionCommand.name();
+    const entry = EXPLAIN_REGISTRY[commandName];
+    let plan;
+    if (typeof entry === "function") {
+        // Dynamic builder
+        plan = await entry(actionCommand, rootDir);
+    }
+    else if (entry) {
+        // Static metadata → simple plan
+        plan = buildSimpleCommandPlan({
+            command: commandName,
+            description: entry.description,
+            filesCreated: entry.filesCreated,
+            filesRead: entry.filesRead,
+            rootDir,
+            steps: entry.steps,
+        });
+    }
+    else {
+        // Unregistered command — minimal fallback
+        plan = buildSimpleCommandPlan({
+            command: commandName,
+            description: `Run the '${commandName}' command`,
+            rootDir,
+        });
+    }
+    // --format is a global option on the root program (actionCommand.parent)
+    const globalParentOpts = actionCommand.parent?.opts();
+    const formatOpt = globalParentOpts?.format ?? "console";
+    if (formatOpt === "json") {
+        console.log(formatPlanJson(plan));
+    }
+    else {
+        console.log();
+        console.log(formatPlanConsole(plan));
+    }
+    // --yes: prompt to confirm, then re-run without --explain
+    if (confirmExecution && plan.errors.length === 0) {
+        try {
+            const { confirm } = await import("@inquirer/prompts");
+            const proceed = await confirm({
+                default: true,
+                message: "Proceed with execution?",
+            });
+            if (proceed) {
+                // Remove --explain and --yes from argv and re-parse
+                const filteredArgv = process.argv.filter((a) => a !== "--explain" && a !== "--yes");
+                // Re-import the program and re-parse with the filtered argv.
+                // Because we're inside a preAction hook that will exit(0), we need
+                // To signal the caller NOT to exit. We do this by throwing a
+                // Sentinel that cli.ts can catch.
+                // oxlint-disable-next-line @typescript-eslint/only-throw-error
+                throw { __proceedArgv: filteredArgv };
+            }
+        }
+        catch (err) {
+            // If it's our sentinel, re-throw for the caller
+            if (err !== null &&
+                typeof err === "object" &&
+                "__proceedArgv" in err) {
+                throw err;
+            }
+            // User cancelled or prompt failed — just exit
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Dynamic plan builders
+// ---------------------------------------------------------------------------
+/**
+ * Build a plan for the `init` command.
+ *
+ * Shows which files and directories will be created, taking into
+ * account the --output-format and --path flags.
+ */
+function buildInitExplainPlan(actionCommand, rootDir) {
+    const opts = actionCommand.opts();
+    const format = opts.outputFormat === "json" ? "json" : "yaml";
+    const ext = format === "json" ? ".json" : ".yaml";
+    const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
+    const targetDir = opts.path ?? ".";
+    const ailfDir = `${targetDir}/.ailf`;
+    const tasksDir = `${ailfDir}/tasks`;
+    const taskFileNames = [...TASK_FILE_NAMES];
+    const filesCreated = [
+        `${ailfDir}/config${ext}`,
+        ...taskFileNames.map((stem) => `${tasksDir}/${stem}${ext}`),
+        `${ailfDir}/.gitignore`,
+    ];
+    return buildSimpleCommandPlan({
+        command: "init",
+        description: `Initialize .ailf/ directory structure in ${callerCwd}/${targetDir === "." ? "" : targetDir}`,
+        filesCreated,
+        filesRead: [],
+        rootDir,
+        steps: [
+            {
+                cacheStatus: "miss",
+                name: "Create directories",
+                reason: `Create ${ailfDir}/ and ${tasksDir}/`,
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: `Write config${ext}`,
+                reason: `Project configuration template (${format.toUpperCase()} format)`,
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: `Write example tasks (${taskFileNames.length} files)`,
+                reason: `Commented starter tasks in ${tasksDir}/ (${format.toUpperCase()} format)`,
+                willRun: true,
+            },
+            {
+                cacheStatus: "miss",
+                name: "Write .gitignore",
+                reason: "Exclude results/ and contexts/ from version control",
+                willRun: true,
+            },
+        ],
+    });
+}
+/**
+ * Build a plan for the `baseline` command.
+ *
+ * Inspects the subcommand argument (save/compare/history) to tailor
+ * the description and file lists.
+ */
+function buildBaselineExplainPlan(actionCommand, rootDir) {
+    const subcommand = actionCommand.args[0] ?? "save";
+    const descriptions = {
+        compare: "Compare current scores against the latest saved baseline",
+        history: "List all saved baseline snapshots with dates and scores",
+        save: "Save current evaluation scores as a new baseline snapshot",
+    };
+    return buildSimpleCommandPlan({
+        command: `baseline ${subcommand}`,
+        description: descriptions[subcommand] ?? `Baseline operation: ${subcommand}`,
+        filesCreated: subcommand === "save" ? ["results/baselines/<timestamp>.json"] : [],
+        filesRead: ["results/latest/score-summary.json", "results/baselines/"],
+        rootDir,
+    });
+}
+/**
+ * Build a plan for the `pipeline` command — the richest plan with steps,
+ * tasks, models, cost estimates, and cache predictions.
+ */
+async function buildPipelineExplainPlan(actionCommand, rootDir) {
+    const raw = actionCommand.opts();
+    // Merge Commander-parsed opts with safe defaults for array/boolean fields
+    const withDefaults = {
+        allowedOrigin: raw.allowedOrigin ?? [],
+        allowedOrigins: raw.allowedOrigins ?? [],
+        area: raw.area,
+        before: raw.before,
+        cache: raw.cache ?? true,
+        changedDocs: raw.changedDocs,
+        compare: raw.compare ?? false,
+        compareBaseline: raw.compareBaseline,
+        concurrency: raw.concurrency,
+        debug: raw.debug ?? false,
+        debugN: raw.debugN,
+        debugPattern: raw.debugPattern,
+        debugSample: raw.debugSample,
+        discoveryReport: raw.discoveryReport ?? false,
+        dryRun: raw.dryRun ?? false,
+        gapAnalysis: raw.gapAnalysis ?? true,
+        graderReplications: raw.graderReplications,
+        header: raw.header ?? [],
+        headers: raw.headers ?? [],
+        mode: raw.mode ?? "full",
+        output: raw.output,
+        promptfooUrl: raw.promptfooUrl,
+        publish: raw.publish,
+        publishTag: raw.publishTag,
+        readiness: raw.readiness ?? false,
+        reportDataset: raw.reportDataset,
+        reportProject: raw.reportProject,
+        sanityDataset: raw.sanityDataset,
+        sanityDocument: raw.sanityDocument ?? [],
+        sanityDocuments: raw.sanityDocuments ?? [],
+        sanityPerspective: raw.sanityPerspective,
+        sanityProject: raw.sanityProject,
+        sanityStudioOrigin: raw.sanityStudioOrigin,
+        search: raw.search,
+        skipEval: raw.skipEval ?? false,
+        skipFetch: raw.skipFetch ?? false,
+        source: raw.source,
+        task: raw.task,
+        threshold: raw.threshold,
+        url: raw.url ?? [],
+        urls: raw.urls ?? [],
+    };
+    const resolved = computeResolvedOptions(withDefaults);
+    const planOpts = {
+        areaOption: resolved.areaOption,
+        beforeOption: resolved.beforeOption,
+        compareBaseline: resolved.compareBaseline,
+        compareEnabled: resolved.compareEnabled,
+        compareThreshold: resolved.compareThreshold,
+        concurrency: resolved.concurrency,
+        debug: resolved.debug,
+        discoveryReportEnabled: resolved.discoveryReportEnabled,
+        dryRun: resolved.dryRun,
+        gapAnalysisEnabled: resolved.gapAnalysisEnabled,
+        graderReplications: resolved.graderReplications,
+        mode: resolved.mode,
+        noCache: resolved.noCache,
+        publishEnabled: resolved.publishEnabled,
+        readinessEnabled: resolved.readinessEnabled,
+        skipEval: resolved.skipEval,
+        skipFetch: resolved.skipFetch,
+        source: resolved.source,
+        repoTasksPath: resolved.repoTasksPath,
+        taskOption: resolved.taskOption,
+    };
+    return await buildPipelinePlan(planOpts, rootDir);
+}