@sanity/ailf 0.1.0

package/dist/commands/pipeline-action.js

@@ -0,0 +1,305 @@
+/**
+ * Pipeline action — resolves CLI options and orchestrates pipeline steps.
+ *
+ * This file is the thin CLI-to-orchestrator bridge. The bulk of execution
+ * logic lives in packages/eval/src/orchestration/.
+ *
+ * Responsibilities:
+ * - Resolve CLI flags into typed ResolvedOptions
+ * - Delegate to the PipelineOrchestrator for step execution
+ *
+ * @see packages/eval/src/orchestration/ for the step-based pipeline
+ */
+import { writeFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { classifyUrls } from "../pipeline/classify-url.js";
+import { assessImpact, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
+import { buildAppContext } from "../orchestration/build-app-context.js";
+import { buildStepSequence } from "../orchestration/build-step-sequence.js";
+import { orchestratePipeline } from "../orchestration/pipeline-orchestrator.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+// ---------------------------------------------------------------------------
+// Valid modes & search modes
+// ---------------------------------------------------------------------------
+const VALID_MODES = ["baseline", "observed", "agentic", "full"];
+const VALID_SEARCH_MODES = ["open", "origin-only", "off"];
+/**
+ * Pure option resolution — computes ResolvedOptions from CLI flags without
+ * any side effects. Safe to call from --explain without mutating process.env.
+ *
+ * Exported so the plan builder can call it independently.
+ */
+export function computeResolvedOptions(opts) {
+    // Validate mode
+    const mode = opts.mode;
+    if (!VALID_MODES.includes(mode)) {
+        console.error(`❌ Invalid mode "${opts.mode}". Must be one of: ${VALID_MODES.join(", ")}`);
+        process.exit(1);
+    }
+    // Debug options — any sub-flag (--debug-n, --debug-pattern, --debug-sample)
+    // implies --debug, so users don't need to pass both.
+    // When DEBUG_EVAL is explicitly "0", ignore the sub-flags from env.
+    // CLI flags (--debug-n, --debug-pattern, --debug-sample) always win.
+    const debugEnvDisabled = process.env.DEBUG_EVAL === "0";
+    const debugN = opts.debugN ??
+        (process.env.DEBUG_EVAL_N && !debugEnvDisabled
+            ? parseInt(process.env.DEBUG_EVAL_N, 10)
+            : undefined);
+    const debugPattern = opts.debugPattern ??
+        (process.env.DEBUG_EVAL_PATTERN && !debugEnvDisabled
+            ? process.env.DEBUG_EVAL_PATTERN
+            : undefined);
+    const debugSample = opts.debugSample ??
+        (process.env.DEBUG_EVAL_SAMPLE && !debugEnvDisabled
+            ? parseInt(process.env.DEBUG_EVAL_SAMPLE, 10)
+            : undefined);
+    const debugEnabled = opts.debug ||
+        process.env.DEBUG_EVAL === "1" ||
+        debugN !== undefined ||
+        debugPattern !== undefined ||
+        debugSample !== undefined;
+    const debug = debugEnabled
+        ? {
+            enabled: true,
+            firstN: debugN,
+            pattern: debugPattern,
+            sample: debugSample,
+        }
+        : undefined;
+    // Search mode validation
+    const searchMode = opts.search ?? process.env.EVAL_SEARCH_MODE ?? "open";
+    if (!VALID_SEARCH_MODES.includes(searchMode)) {
+        console.error(`❌ Invalid --search mode "${searchMode}". Must be one of: ${VALID_SEARCH_MODES.join(", ")}`);
+        process.exit(1);
+    }
+    // Merge repeatable args (singular + plural aliases)
+    const urlArgs = [...opts.url, ...opts.urls];
+    const headerArgs = [...opts.header, ...opts.headers];
+    const allowedOriginArgs = [...opts.allowedOrigin, ...opts.allowedOrigins];
+    const sanityDocumentArgs = [...opts.sanityDocument, ...opts.sanityDocuments];
+    // Source overrides
+    const datasetOverride = opts.sanityDataset;
+    const projectIdOverride = opts.sanityProject;
+    const perspectiveOverride = opts.sanityPerspective;
+    const studioOriginOverride = opts.sanityStudioOrigin;
+    // URL classification (pure computation — results captured, not applied to env)
+    if (urlArgs.length > 0) {
+        const classification = classifyUrls(urlArgs);
+        if (classification.documentIds.length > 0) {
+            const existing = sanityDocumentArgs.length > 0 ? sanityDocumentArgs : [];
+            const merged = [...new Set([...existing, ...classification.documentIds])];
+            sanityDocumentArgs.length = 0;
+            sanityDocumentArgs.push(...merged);
+        }
+    }
+    // Validate custom headers (early error)
+    if (headerArgs.length > 0) {
+        for (const h of headerArgs) {
+            const colonIdx = h.indexOf(":");
+            if (colonIdx === -1) {
+                console.error(`❌ Invalid header format: "${h}". Expected "Key: Value".`);
+                process.exit(1);
+            }
+            const key = h.slice(0, colonIdx).trim();
+            if (!key) {
+                console.error(`❌ Invalid header: empty key in "${h}"`);
+                process.exit(1);
+            }
+        }
+    }
+    // Auto-infer allowed origin from --url
+    if (urlArgs.length > 0 && allowedOriginArgs.length === 0) {
+        try {
+            const hostname = new URL(urlArgs[0]).hostname.replace(/^www\./, "");
+            allowedOriginArgs.push(hostname);
+        }
+        catch {
+            // Invalid URL — will be caught later in validation
+        }
+    }
+    // Scoping
+    const areaOption = opts.area ?? process.env.EVAL_FILTER_AREAS ?? undefined;
+    const taskOption = opts.task ?? process.env.EVAL_FILTER_TASKS ?? undefined;
+    const changedDocsOption = opts.changedDocs ?? process.env.EVAL_CHANGED_DOCS ?? undefined;
+    // Document-driven scoping (pure — computes impactSummary without env writes)
+    let impactSummary;
+    if (changedDocsOption) {
+        const changedSlugs = changedDocsOption
+            .split(",")
+            .map((s) => s.trim())
+            .filter(Boolean);
+        if (changedSlugs.length > 0) {
+            const reverseMapping = buildReverseMapping(ROOT);
+            impactSummary = assessImpact(changedSlugs, reverseMapping);
+            if (impactSummary.areas.length === 0) {
+                console.warn(`\n⚠️  No evaluation tasks reference any of the changed documents:`);
+                for (const slug of changedSlugs) {
+                    console.warn(`    - ${slug}`);
+                }
+                console.warn(`\n    Score impact cannot be measured for these documents.\n`);
+            }
+        }
+    }
+    // Comparison: --before auto-enables --compare
+    const beforeOption = opts.before;
+    const compareEnabled = opts.compare || beforeOption !== undefined;
+    // Publish: smart default — auto-publish full runs when report store is configured
+    const reportStoreToken = process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN;
+    const reportStoreConfigured = Boolean(reportStoreToken);
+    let publishEnabled;
+    if (opts.publish !== undefined) {
+        // Explicit --publish or --no-publish always wins
+        publishEnabled = opts.publish;
+    }
+    else if (process.env.AILF_PUBLISH === "1") {
+        publishEnabled = true;
+    }
+    else if (process.env.AILF_PUBLISH === "0") {
+        publishEnabled = false;
+    }
+    else {
+        // Smart default: full runs auto-publish when store is configured
+        publishEnabled = reportStoreConfigured && !debugEnabled;
+    }
+    // Report store overrides — fall back to the eval dataset so that
+    // perspective evaluations publish reports to the same dataset the
+    // Studio is reading from. AILF_REPORT_DATASET wins when set explicitly.
+    const reportDataset = opts.reportDataset ??
+        process.env.AILF_REPORT_DATASET ??
+        datasetOverride ??
+        undefined;
+    const reportProjectId = opts.reportProject ?? process.env.AILF_REPORT_PROJECT_ID ?? undefined;
+    return {
+        allowedOriginArgs,
+        areaOption,
+        beforeOption,
+        changedDocsOption,
+        compareBaseline: opts.compareBaseline,
+        compareEnabled,
+        compareThreshold: opts.threshold,
+        concurrency: opts.concurrency,
+        datasetOverride,
+        debug,
+        discoveryReportEnabled: opts.discoveryReport,
+        dryRun: opts.dryRun,
+        gapAnalysisEnabled: opts.gapAnalysis,
+        graderReplications: opts.graderReplications,
+        headerArgs,
+        impactSummary,
+        mode,
+        noCache: !opts.cache,
+        noRemoteCache: opts.remoteCache === false,
+        outputPath: opts.output,
+        perspectiveOverride,
+        projectIdOverride,
+        promptfooUrl: opts.promptfooUrl,
+        publishEnabled,
+        publishTag: opts.publishTag,
+        readinessEnabled: opts.readiness,
+        reportDataset,
+        reportProjectId,
+        sanityDocumentArgs,
+        searchMode,
+        skipEval: opts.skipEval,
+        skipFetch: opts.skipFetch,
+        source: opts.source,
+        studioOriginOverride,
+        repoTasksPath: opts.repoTasksPath,
+        taskOption,
+        taskSourceType: resolveTaskSourceType(opts.taskSource),
+        urlArgs,
+    };
+}
+/** Resolve and validate the --task-source flag value. */
+function resolveTaskSourceType(raw) {
+    if (!raw || raw === "content-lake")
+        return undefined; // default — Content Lake
+    if (raw === "yaml")
+        return "yaml";
+    console.error(`❌ Invalid --task-source "${raw}". Must be "yaml" or "content-lake".`);
+    process.exit(1);
+}
+// ---------------------------------------------------------------------------
+// Pipeline entry point
+// ---------------------------------------------------------------------------
+/**
+ * Execute the evaluation pipeline.
+ *
+ * 1. Resolve CLI options into typed ResolvedOptions
+ * 2. Build AppContext (composition root wires adapters)
+ * 3. Build step sequence from context
+ * 4. Delegate to the PipelineOrchestrator
+ */
+export async function executePipeline(cliOpts) {
+    // When --config is provided, resolve config from file instead of CLI flags
+    if (cliOpts.config) {
+        const { existsSync } = await import("fs");
+        if (!existsSync(cliOpts.config)) {
+            console.error(`❌ Config file not found: ${cliOpts.config}`);
+            process.exit(1);
+        }
+        const { FileConfigAdapter } = await import("../adapters/config-sources/file-config-adapter.js");
+        const adapter = new FileConfigAdapter(cliOpts.config, ROOT);
+        const config = await adapter.resolve();
+        const ctx = buildAppContext(
+        // Build a minimal ResolvedOptions to satisfy the bridge.
+        // FileConfigAdapter already resolved to ResolvedConfig, so we
+        // pass it through by constructing AppContext directly.
+        computeResolvedOptions(cliOpts), ROOT);
+        // Override config with the file-based config
+        const fileCtx = { ...ctx, config };
+        const pipelineStart = Date.now();
+        const steps = buildStepSequence(fileCtx, pipelineStart);
+        const result = await orchestratePipeline(fileCtx, steps);
+        writePipelineResult(result);
+        process.exit(result.success ? 0 : 1);
+    }
+    const o = resolveOptions(cliOpts);
+    // Dry-run: validate only, don't execute steps
+    if (o.dryRun) {
+        const { validateConfiguration } = await import("../pipeline/validate.js");
+        const { checkEnvironment } = await import("../pipeline/checks.js");
+        const validation = validateConfiguration(ROOT);
+        const envIssues = checkEnvironment(ROOT);
+        validation.issues.push(...envIssues);
+        const errors = validation.issues.filter((i) => i.severity === "error");
+        if (errors.length > 0) {
+            console.error("❌ Configuration validation failed:\n");
+            for (const e of errors) {
+                console.error(`  ERROR [${e.source}] ${e.message}`);
+            }
+            process.exit(1);
+        }
+        console.log("\n  ✅ Configuration is valid");
+        console.log("  Pipeline configuration is valid. No steps were executed.");
+        console.log("  Remove --dry-run to execute the full pipeline.\n");
+        process.exit(0);
+    }
+    const ctx = buildAppContext(o, ROOT);
+    const pipelineStart = Date.now();
+    const steps = buildStepSequence(ctx, pipelineStart);
+    const result = await orchestratePipeline(ctx, steps);
+    writePipelineResult(result);
+    process.exit(result.success ? 0 : 1);
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Resolve CLI options into typed ResolvedOptions.
+ */
+function resolveOptions(opts) {
+    return computeResolvedOptions(opts);
+}
+function writePipelineResult(result) {
+    const resultFile = resolve(ROOT, "results", "latest", "pipeline-result.json");
+    try {
+        writeFileSync(resultFile, JSON.stringify(result, null, 2));
+        console.log(`  📄 Pipeline result: ${resultFile}\n`);
+    }
+    catch {
+        // results/latest/ may not exist yet — not critical
+    }
+}

package/dist/commands/pipeline.d.ts

@@ -0,0 +1,62 @@
+/**
+ * pipeline command — the main evaluation pipeline orchestrator.
+ *
+ * Defines all 36+ CLI flags via Commander, resolves them into a typed
+ * options object, bridges to process.env for downstream modules, and
+ * delegates to runPipeline().
+ *
+ * @see docs/API.md for the full flag reference.
+ */
+import { Command } from "commander";
+/**
+ * Raw CLI options as parsed by Commander.
+ * Field names follow Commander's camelCase convention for kebab-case flags.
+ */
+export interface PipelineCliOptions {
+    allowedOrigin: string[];
+    allowedOrigins: string[];
+    area?: string;
+    before?: string;
+    cache: boolean;
+    changedDocs?: string;
+    compare: boolean;
+    compareBaseline?: string;
+    concurrency?: number;
+    config?: string;
+    debug: boolean;
+    debugN?: number;
+    debugPattern?: string;
+    debugSample?: number;
+    discoveryReport: boolean;
+    dryRun: boolean;
+    gapAnalysis: boolean;
+    graderReplications?: number;
+    header: string[];
+    headers: string[];
+    mode: string;
+    output?: string;
+    promptfooUrl?: string;
+    publish?: boolean;
+    publishTag?: string;
+    readiness: boolean;
+    remoteCache?: boolean;
+    reportDataset?: string;
+    reportProject?: string;
+    sanityDataset?: string;
+    sanityDocument: string[];
+    sanityDocuments: string[];
+    sanityPerspective?: string;
+    sanityProject?: string;
+    sanityStudioOrigin?: string;
+    search?: string;
+    skipEval: boolean;
+    skipFetch: boolean;
+    source?: string;
+    repoTasksPath?: string;
+    task?: string;
+    taskSource?: string;
+    threshold?: number;
+    url: string[];
+    urls: string[];
+}
+export declare function createPipelineCommand(): Command;

package/dist/commands/pipeline.js

@@ -0,0 +1,53 @@
+/**
+ * pipeline command — the main evaluation pipeline orchestrator.
+ *
+ * Defines all 36+ CLI flags via Commander, resolves them into a typed
+ * options object, bridges to process.env for downstream modules, and
+ * delegates to runPipeline().
+ *
+ * @see docs/API.md for the full flag reference.
+ */
+import { Command } from "commander";
+import { addAgenticOptions, addDebugOptions, addSanitySourceOptions, } from "./shared/options.js";
+export function createPipelineCommand() {
+    const cmd = new Command("pipeline")
+        .description("Run the full evaluation pipeline")
+        .option("-m, --mode <mode>", "Evaluation mode: full (default — floor + ceiling + actual), baseline (floor + ceiling only), agentic (actual only), observed", "full")
+        .option("-s, --source <name>", "Documentation source name (from sources.yaml)")
+        .option("-n, --dry-run", "Validate configuration only, no execution", false)
+        .option("--skip-fetch", "Reuse cached documentation contexts", false)
+        .option("--skip-eval", "Recalculate from existing eval results", false)
+        .option("--no-cache", "Bypass all pipeline-level caching")
+        .option("--no-remote-cache", "Disable Content Lake cache lookup (local cache still active)")
+        .option("-a, --area <areas>", "Scope to feature areas (comma-separated)")
+        .option("-t, --task <id>", "Scope to specific task ID")
+        .option("--changed-docs <slugs>", "Auto-scope to tasks affected by these document slugs")
+        .option("-j, --concurrency <n>", "Max parallel API calls during evaluation", parseInt)
+        .option("--grader-replications <n>", "Grader consistency replications", parseInt)
+        .option("--before <source>", "Before-state for impact evaluation")
+        .option("-c, --compare", "Compare scores against latest baseline", false)
+        .option("--compare-baseline <path>", "Specific baseline file to compare")
+        .option("--threshold <n>", "Noise threshold for comparison (default: 2)", parseFloat)
+        .option("--no-gap-analysis", "Skip failure mode + impact analysis")
+        .option("--readiness", "Generate launch readiness checklist", false)
+        .option("--discovery-report", "Generate agent discoverability report", false)
+        .option("-p, --publish", "Write report to Sanity + fan out to sinks (auto-enabled for full runs when report store is configured)")
+        .option("--no-publish", "Suppress auto-publishing")
+        .option("--publish-tag <tag>", "Label for published report")
+        .option("--report-dataset <name>", "Sanity dataset for report store")
+        .option("--report-project <id>", "Sanity project ID for report store")
+        .option("--config <path>", "Load pipeline config from a JSON/YAML file (overrides most CLI flags)")
+        .option("-o, --output <path>", "Write PR comment markdown to file")
+        .option("--promptfoo-url <url>", "Promptfoo share URL for report")
+        .option("--task-source <type>", "Task definition source: content-lake (default — Sanity Content Lake), yaml (tasks/*.yaml files, legacy)", "content-lake")
+        .option("--repo-tasks-path <path>", "Path to repo-based task definitions (.ailf/tasks/ directory)")
+        .action(async (opts) => {
+        const { executePipeline } = await import("./pipeline-action.js");
+        await executePipeline(opts);
+    });
+    // Add shared option groups
+    addDebugOptions(cmd);
+    addSanitySourceOptions(cmd);
+    addAgenticOptions(cmd);
+    return cmd;
+}

package/dist/commands/pr-comment.d.ts

@@ -0,0 +1,8 @@
+/**
+ * pr-comment command — generate a PR comment from evaluation scores.
+ *
+ * Uses the composition root to wire adapters, then calls
+ * generatePrComment() directly from pipeline/.
+ */
+import { Command } from "commander";
+export declare function createPrCommentCommand(): Command;

package/dist/commands/pr-comment.js

@@ -0,0 +1,47 @@
+/**
+ * pr-comment command — generate a PR comment from evaluation scores.
+ *
+ * Uses the composition root to wire adapters, then calls
+ * generatePrComment() directly from pipeline/.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { createAppContext } from "../composition-root.js";
+import { generatePrComment } from "../pipeline/pr-comment.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createPrCommentCommand() {
+    return new Command("pr-comment")
+        .description("Generate a markdown PR comment from evaluation scores")
+        .option("--output <path>", "Write comment to file (default: stdout)")
+        .option("--promptfoo-url <url>", "Promptfoo share URL to include")
+        .action(async (opts) => {
+        try {
+            const ctx = createAppContext({
+                rootDir: ROOT,
+                mode: "baseline",
+                skipFetch: true,
+                skipEval: true,
+                compareEnabled: false,
+                gapAnalysisEnabled: false,
+                readinessEnabled: false,
+                discoveryReportEnabled: false,
+                publishEnabled: false,
+                noCache: true,
+                noRemoteCache: true,
+                searchMode: "open",
+            });
+            generatePrComment({
+                outputPath: opts.output,
+                promptfooUrl: opts.promptfooUrl,
+                rootDir: ctx.config.rootDir,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+}

package/dist/commands/publish.d.ts

@@ -0,0 +1,26 @@
+/**
+ * publish command — manually publish a local evaluation report to the
+ * Sanity Content Lake without re-running the pipeline.
+ *
+ * Reads a score-summary.json (defaulting to results/latest/score-summary.json),
+ * builds provenance, writes the report to Sanity, and fans out to configured
+ * sinks — exactly the same as the publish step in `ailf pipeline`, but
+ * standalone.
+ *
+ * Uses createAppContext() (composition root) for all infrastructure access.
+ *
+ * @example
+ *   ailf publish                                        # default path
+ *   ailf publish ./my-results/score-summary.json        # custom path
+ *   ailf publish --tag "manual-2026-03-13"              # with a label
+ *   ailf publish --dry-run                              # preview without writing
+ *
+ * @see packages/eval/src/composition-root.ts
+ * @see docs/design-docs/report-store/architecture.md
+ */
+import { Command } from "commander";
+export interface PublishCommandOptions {
+    dryRun: boolean;
+    tag?: string;
+}
+export declare function createPublishCommand(): Command;