@sanity/ailf 3.7.0 → 3.8.0

package/dist/commands/discovery-report.js

@@ -13,17 +13,29 @@ import { fileURLToPath } from "url";
 import { formatDiscoveryMarkdown, generateDiscoveryReport, } from "../pipeline/discovery-report.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, "..", "..");
+const DEFAULT_RESULTS_DIR = join(ROOT, "results", "latest");
+/** Resolve `--from-run` to an absolute path to score-summary.json. */
+function resolveFromRun(value) {
+    if (value === "latest")
+        return join(DEFAULT_RESULTS_DIR, "score-summary.json");
+    // If the value points at a directory, look for score-summary.json inside it.
+    // Otherwise treat it as a direct file path.
+    const resolved = resolve(value);
+    return resolved.endsWith(".json")
+        ? resolved
+        : join(resolved, "score-summary.json");
+}
 export function createDiscoveryReportCommand() {
-    return new Command("discovery-report")
+    return new Command("discovery")
         .description("Generate agent discoverability report from retrieval metrics")
         .option("-a, --area <areas>", "Filter by feature areas (comma-separated)")
         .option("-o, --output <path>", "Write markdown to file instead of stdout")
-        .option("-i, --input <path>", "Path to score-summary.json", join(ROOT, "results", "latest", "score-summary.json"))
+        .option("--from-run <path>", "Results to read from (`latest`, a results directory, or a direct path to score-summary.json)", "latest")
         .action(async (opts) => {
-        const summaryPath = opts.input;
+        const summaryPath = resolveFromRun(opts.fromRun);
         if (!existsSync(summaryPath)) {
             console.error(`❌ Score summary not found: ${summaryPath}`);
-            console.error("Run an agentic evaluation first: pnpm pipeline -- --mode agentic");
+            console.error("Run an agentic evaluation first: ailf run --mode agentic");
             process.exit(1);
         }
         const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));

package/dist/commands/eval.d.ts

@@ -2,7 +2,7 @@
  * eval command — run promptfoo evaluation directly (passthrough).
  *
  * This is a convenience command that forwards to `promptfoo eval`.
- * For most use cases, prefer `ailf pipeline` which handles the full
+ * For most use cases, prefer `ailf run` which handles the full
  * lifecycle (fetch → generate → eval → score → report).
  */
 import { Command } from "commander";

package/dist/commands/eval.js

@@ -2,7 +2,7 @@
  * eval command — run promptfoo evaluation directly (passthrough).
  *
  * This is a convenience command that forwards to `promptfoo eval`.
- * For most use cases, prefer `ailf pipeline` which handles the full
+ * For most use cases, prefer `ailf run` which handles the full
  * lifecycle (fetch → generate → eval → score → report).
  */
 import { Command } from "commander";

package/dist/commands/explain-handler.d.ts

@@ -12,7 +12,7 @@
  * Every command is registered in `EXPLAIN_REGISTRY` with either:
  * - **Static metadata** — description, filesRead, filesCreated, steps
  * - **A builder function** — for commands that need to inspect CLI options
- *   or perform async work (e.g., pipeline, init)
+ *   or perform async work (e.g., run, init)
  *
  * Adding --explain support for a new command = adding one registry entry.
  * Commands not in the registry fall back to a minimal generic plan.

package/dist/commands/explain-handler.js

@@ -12,7 +12,7 @@
  * Every command is registered in `EXPLAIN_REGISTRY` with either:
  * - **Static metadata** — description, filesRead, filesCreated, steps
  * - **A builder function** — for commands that need to inspect CLI options
- *   or perform async work (e.g., pipeline, init)
+ *   or perform async work (e.g., run, init)
  *
  * Adding --explain support for a new command = adding one registry entry.
  * Commands not in the registry fall back to a minimal generic plan.
@@ -39,7 +39,7 @@ const EXPLAIN_REGISTRY = {
     // ── Dynamic builders (inspect CLI options) ────────────────────────
     baseline: buildBaselineExplainPlan,
     init: buildInitExplainPlan,
-    pipeline: buildPipelineExplainPlan,
+    run: buildPipelineExplainPlan,
     // ── Static metadata ───────────────────────────────────────────────
     "agent-report": {
         description: "Generate an agent behavior observation report from eval results",
@@ -588,15 +588,11 @@ export async function handleExplain(actionCommand, confirmExecution, rootDir) {
  * Build a plan for the `init` command.
  *
  * Shows which files and directories will be created, taking into
- * account the --output-format and --path flags.
+ * account the --format and --path flags.
  */
 function buildInitExplainPlan(actionCommand, rootDir) {
     const opts = actionCommand.opts();
-    const format = opts.outputFormat === "json"
-        ? "json"
-        : opts.outputFormat === "yaml"
-            ? "yaml"
-            : "ts";
+    const format = opts.format === "json" ? "json" : opts.format === "yaml" ? "yaml" : "ts";
     const taskExt = format === "ts" ? ".task.ts" : format === "yaml" ? ".yaml" : ".json";
     const configFile = format === "ts"
         ? "ailf.config.ts"
@@ -667,69 +663,44 @@ function buildBaselineExplainPlan(actionCommand, rootDir) {
     });
 }
 /**
- * Build a plan for the `pipeline` command — the richest plan with steps,
+ * Build a plan for the `run` command — the richest plan with steps,
  * tasks, models, cost estimates, and cache predictions.
  */
 async function buildPipelineExplainPlan(actionCommand, rootDir) {
     const raw = actionCommand.opts();
     // Merge Commander-parsed opts with safe defaults for array/boolean fields
     const withDefaults = {
-        allowedOrigin: raw.allowedOrigin ?? [],
-        allowedOrigins: raw.allowedOrigins ?? [],
         area: raw.area,
         autoScope: raw.autoScope ?? true,
-        before: raw.before,
+        beforeSource: raw.beforeSource,
         cache: raw.cache ?? true,
         changedDocs: raw.changedDocs,
-        compare: raw.compare ?? false,
-        compareBaseline: raw.compareBaseline,
-        concurrency: raw.concurrency,
+        compare: raw.compare,
         debug: raw.debug ?? false,
-        debugN: raw.debugN,
-        debugPattern: raw.debugPattern,
-        debugSample: raw.debugSample,
-        discoveryReport: raw.discoveryReport ?? false,
+        filterFirstN: raw.filterFirstN,
+        filterPattern: raw.filterPattern,
+        filterSample: raw.filterSample,
         dryRun: raw.dryRun ?? false,
-        gapAnalysis: raw.gapAnalysis ?? true,
-        graderReplications: raw.graderReplications,
-        header: raw.header ?? [],
-        headers: raw.headers ?? [],
         mode: raw.mode ?? LiteracyVariant.FULL,
         output: raw.output,
         promptfooUrl: raw.promptfooUrl,
         publish: raw.publish,
         publishTag: raw.publishTag,
-        readiness: raw.readiness ?? false,
-        reportDataset: raw.reportDataset,
-        reportProject: raw.reportProject,
-        sanityDataset: raw.sanityDataset,
         sanityDocument: raw.sanityDocument ?? [],
-        sanityDocuments: raw.sanityDocuments ?? [],
         sanityPerspective: raw.sanityPerspective,
-        sanityProject: raw.sanityProject,
-        sanityStudioOrigin: raw.sanityStudioOrigin,
         search: raw.search,
-        skipEval: raw.skipEval ?? false,
-        skipFetch: raw.skipFetch ?? false,
+        eval: raw.eval ?? true,
+        fetch: raw.fetch ?? true,
         source: raw.source,
         tag: raw.tag ?? [],
         task: raw.task,
         threshold: raw.threshold,
         url: raw.url ?? [],
-        urls: raw.urls ?? [],
         remote: raw.remote ?? false,
-        apiUrl: raw.apiUrl,
-        repoTasksPath: raw.repoTasksPath,
-        taskSource: raw.taskSource,
         remoteCache: raw.remoteCache,
         config: raw.config,
-        artifacts: raw.artifacts ?? true,
-        artifactsDir: raw.artifactsDir,
-        artifactsDryRun: raw.artifactsDryRun ?? false,
-        artifactsExclude: raw.artifactsExclude,
+        artifactsWrite: raw.artifactsWrite ?? true,
         classification: raw.classification,
-        ownerTeam: raw.ownerTeam,
-        ownerIndividual: raw.ownerIndividual,
         purpose: raw.purpose,
         label: raw.label ?? [],
     };
@@ -742,7 +713,6 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         compareThreshold: resolved.compareThreshold,
         concurrency: resolved.concurrency,
         debug: resolved.debug,
-        discoveryReportEnabled: resolved.discoveryReportEnabled,
         dryRun: resolved.dryRun,
         gapAnalysisEnabled: resolved.gapAnalysisEnabled,
         graderReplications: resolved.graderReplications,
@@ -750,7 +720,6 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         variant: resolved.variant,
         noCache: resolved.noCache,
         publishEnabled: resolved.publishEnabled,
-        readinessEnabled: resolved.readinessEnabled,
         skipEval: resolved.skipEval,
         skipFetch: resolved.skipFetch,
         source: resolved.source,

package/dist/commands/fetch-docs.js

@@ -50,8 +50,6 @@ async function executeFetchDocs(opts) {
         skipEval: true,
         compareEnabled: false,
         gapAnalysisEnabled: false,
-        readinessEnabled: false,
-        discoveryReportEnabled: false,
         publishEnabled: false,
         noCache: true,
         noRemoteCache: true,

package/dist/commands/generate-configs.js

@@ -28,8 +28,6 @@ export function createGenerateConfigsCommand() {
                 skipEval: true,
                 compareEnabled: false,
                 gapAnalysisEnabled: false,
-                readinessEnabled: false,
-                discoveryReportEnabled: false,
                 publishEnabled: false,
                 noCache: true,
                 noRemoteCache: true,

package/dist/commands/grader/index.js

@@ -95,16 +95,16 @@ export function createGraderCommand() {
         .command("validate")
         .description("Validate grader accuracy against human reference grades")
         .option("-g, --grader <model>", "Grader model to validate")
-        .option("-t, --threshold <n>", "MAE threshold for pass/fail", parseFloat, 10)
+        .option("--mae-threshold <n>", "MAE threshold for pass/fail", parseFloat, 10)
         .action(async (opts) => {
         try {
             const result = await runGraderValidate({
                 graderModel: opts.grader,
-                maeThreshold: opts.threshold,
+                maeThreshold: opts.maeThreshold,
                 rootDir: ROOT,
             });
             if (!result.passesThreshold) {
-                console.error(`\n  ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${opts.threshold}`);
+                console.error(`\n  ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${opts.maeThreshold}`);
                 process.exit(1);
             }
         }

package/dist/commands/init.d.ts

@@ -11,8 +11,8 @@
  *
  * Usage:
  *   ailf init                        # TypeScript output (default)
- *   ailf init --output-format yaml   # YAML output
- *   ailf init --output-format json   # JSON output
+ *   ailf init --format yaml          # YAML output
+ *   ailf init --format json          # JSON output
  *   ailf init --force                # overwrite existing files
  *   ailf init --path ./my-dir        # target a specific directory
  */

package/dist/commands/init.js

@@ -11,8 +11,8 @@
  *
  * Usage:
  *   ailf init                        # TypeScript output (default)
- *   ailf init --output-format yaml   # YAML output
- *   ailf init --output-format json   # JSON output
+ *   ailf init --format yaml          # YAML output
+ *   ailf init --format json          # JSON output
  *   ailf init --force                # overwrite existing files
  *   ailf init --path ./my-dir        # target a specific directory
  */
@@ -27,7 +27,7 @@ import { probeUserLocalAilf } from "../adapters/config-sources/ailf-resolver.js"
 export function createInitCommand() {
     return new Command("init")
         .description("Initialize a directory for AI Literacy Framework evaluation")
-        .option("--output-format <fmt>", 'Output format for generated files: "ts" (default), "yaml", or "json"', "ts")
+        .option("-f, --format <fmt>", 'Output format for generated files: "ts" (default), "yaml", or "json"', "ts")
         .option("--force", "Overwrite existing files", false)
         .option("--path <dir>", "Target directory (default: current directory)", ".")
         .option("--mode <mode>", "Scaffold for a specific mode: literacy, mcp-server, custom (default: all modes)")
@@ -63,15 +63,15 @@ function taskStemsForMode(mode) {
 // ---------------------------------------------------------------------------
 async function runInit(opts) {
     const validFormats = new Set(["ts", "yaml", "json"]);
-    if (!validFormats.has(opts.outputFormat)) {
-        console.error(`  ✗ Invalid output format "${opts.outputFormat}". Valid options: ts, yaml, json`);
+    if (!validFormats.has(opts.format)) {
+        console.error(`  ✗ Invalid output format "${opts.format}". Valid options: ts, yaml, json`);
         process.exitCode = 1;
         return;
     }
-    const format = opts.outputFormat;
+    const format = opts.format;
     const force = opts.force;
     if (format === "yaml") {
-        console.warn("  ⚠ --output-format yaml is deprecated. TypeScript (default) is the\n" +
+        console.warn("  ⚠ --format yaml is deprecated. TypeScript (default) is the\n" +
             "    recommended format — it provides full IDE autocomplete via defineTask().\n" +
             "    YAML output will be removed in a future release.\n");
     }
@@ -285,10 +285,11 @@ async function runInit(opts) {
     console.log("     AILF_API_KEY=... npx @sanity/ailf@latest pipeline --remote --debug");
     console.log();
     console.log("  💡 Or test a remote run against your repo tasks:");
-    console.log("     AILF_API_KEY=... npx @sanity/ailf@latest pipeline --remote --task-source=repo --debug");
+    console.log("     # First, set `taskSource: { type: repo }` in .ailf/config.yaml");
+    console.log("     AILF_API_KEY=... npx @sanity/ailf@latest run --remote --debug");
     console.log();
     console.log("  💡 Or run locally against your repo tasks:");
-    console.log("     AILF_API_KEY=... npx @sanity/ailf@latest pipeline --mode=literacy --variant=full --task-source=repo --debug --explain -y");
+    console.log("     AILF_API_KEY=... npx @sanity/ailf@latest run --mode=literacy --variant=full --debug --explain -y");
     console.log();
 }
 // ---------------------------------------------------------------------------

package/dist/commands/interactive.d.ts

@@ -3,7 +3,7 @@
  *
  * When `ailf` is run with no arguments (or `ailf interactive`), this module
  * prompts the user through mode selection, area scoping, debug options,
- * and common flags — then builds and executes the equivalent `ailf pipeline`
+ * and common flags — then builds and executes the equivalent `ailf run`
  * command.
  *
  * Uses @inquirer/prompts for a clean, modern terminal UI.

package/dist/commands/interactive.js

@@ -3,7 +3,7 @@
  *
  * When `ailf` is run with no arguments (or `ailf interactive`), this module
  * prompts the user through mode selection, area scoping, debug options,
- * and common flags — then builds and executes the equivalent `ailf pipeline`
+ * and common flags — then builds and executes the equivalent `ailf run`
  * command.
  *
  * Uses @inquirer/prompts for a clean, modern terminal UI.
@@ -52,9 +52,9 @@ async function runInteractiveWizard() {
     const workflow = await select({
         choices: [
             {
-                description: "Full evaluation pipeline (fetch → eval → score → report)",
-                name: "Run pipeline",
-                value: "pipeline",
+                description: "Full evaluation run (fetch → eval → score → report)",
+                name: "Run evaluation",
+                value: "run",
             },
             {
                 description: "Compare current scores against a saved baseline",
@@ -193,21 +193,21 @@ async function runInteractiveWizard() {
         });
         if (debugStyle === "first-n") {
             const n = await input({ default: "5", message: "Number of tests:" });
-            args.push("--debug-n", n);
+            args.push("--filter-first-n", n);
         }
         else if (debugStyle === "sample") {
             const n = await input({
                 default: "3",
                 message: "Sample size:",
             });
-            args.push("--debug-sample", n);
+            args.push("--filter-sample", n);
         }
         else if (debugStyle === "pattern") {
             const pattern = await input({
                 message: "Description regex (e.g. Blog, webhook):",
             });
             if (pattern.trim()) {
-                args.push("--debug-pattern", pattern.trim());
+                args.push("--filter-pattern", pattern.trim());
             }
         }
     }
@@ -238,5 +238,5 @@ async function runInteractiveWizard() {
             args.push("--explain", "--yes");
         }
     }
-    return { args, command: "pipeline" };
+    return { args, command: "run" };
 }

package/dist/commands/pipeline-action.d.ts

@@ -12,7 +12,7 @@
  */
 import { type ImpactSummary } from "../pipeline/reverse-mapping.js";
 import type { DebugOptions, EvalMode } from "../pipeline/types.js";
-import type { PipelineCliOptions } from "./pipeline.js";
+import type { PipelineCliOptions } from "./run.js";
 export interface ResolvedOptions {
     allowedOriginArgs: string[];
     areaOption?: string;
@@ -24,7 +24,6 @@ export interface ResolvedOptions {
     concurrency?: number;
     datasetOverride?: string;
     debug?: DebugOptions;
-    discoveryReportEnabled: boolean;
     dryRun: boolean;
     gapAnalysisEnabled: boolean;
     graderReplications?: number;
@@ -46,7 +45,6 @@ export interface ResolvedOptions {
     /** True when --publish or --no-publish was explicitly passed by the user. */
     publishExplicit: boolean;
     publishTag?: string;
-    readinessEnabled: boolean;
     reportDataset?: string;
     reportProjectId?: string;
     sanityDocumentArgs: string[];