@sanity/ailf 3.7.0 → 3.8.1

package/dist/pipeline/plan.d.ts

@@ -61,7 +61,7 @@ export interface DebugPlan {
 export interface ExecutionPlan {
     /** Cache prediction per step */
     cacheStatus: Record<string, "hit" | "miss" | "skipped" | "unknown">;
-    /** Command name (e.g., "pipeline", "compare", "validate") */
+    /** Command name (e.g., "run", "compare", "validate") */
     command: string;
     /** Comparison plan (when --compare is set) */
     comparison?: ComparisonPlan;
@@ -134,7 +134,6 @@ export interface PlanOptions {
     compareThreshold?: number;
     concurrency?: number;
     debug?: DebugOptions;
-    discoveryReportEnabled: boolean;
     dryRun: boolean;
     gapAnalysisEnabled: boolean;
     graderReplications?: number;
@@ -143,7 +142,6 @@ export interface PlanOptions {
     variant?: string;
     noCache: boolean;
     publishEnabled: boolean;
-    readinessEnabled: boolean;
     repoTasksPath?: string;
     skipEval: boolean;
     skipFetch: boolean;
@@ -152,7 +150,7 @@ export interface PlanOptions {
     taskOption?: string;
 }
 /**
- * Build a complete execution plan for the `pipeline` command.
+ * Build a complete execution plan for the `run` command.
  *
  * This is a read-only operation — it computes the plan by calling existing
  * pure functions (task expansion, model loading, cache lookup, pricing)

package/dist/pipeline/plan.js

@@ -99,7 +99,7 @@ const AVG_TOKENS = {
 // Cache prediction
 // ---------------------------------------------------------------------------
 /**
- * Build a complete execution plan for the `pipeline` command.
+ * Build a complete execution plan for the `run` command.
  *
  * This is a read-only operation — it computes the plan by calling existing
  * pure functions (task expansion, model loading, cache lookup, pricing)
@@ -322,13 +322,11 @@ export async function buildPipelinePlan(opts, rootDir) {
     // 7. Build step plan
     const steps = buildStepPlan({
         compareEnabled: opts.compareEnabled,
-        discoveryReportEnabled: opts.discoveryReportEnabled,
         dryRun: opts.dryRun,
         gapAnalysisEnabled: opts.gapAnalysisEnabled,
         graderReplications: opts.graderReplications,
         noCache: opts.noCache,
         publishEnabled: opts.publishEnabled,
-        readinessEnabled: opts.readinessEnabled,
         skipEval: opts.skipEval,
         skipFetch: opts.skipFetch,
     }, cachePrediction);
@@ -338,14 +336,12 @@ export async function buildPipelinePlan(opts, rootDir) {
     const filesRead = collectFilesRead(rootDir, opts.mode);
     const filesCreated = collectFilesCreated({
         compareEnabled: opts.compareEnabled,
-        discoveryReportEnabled: opts.discoveryReportEnabled,
         gapAnalysisEnabled: opts.gapAnalysisEnabled,
         publishEnabled: opts.publishEnabled,
-        readinessEnabled: opts.readinessEnabled,
     });
     return {
         cacheStatus: cachePrediction.predictions,
-        command: "pipeline",
+        command: "run",
         comparison,
         costEstimate,
         debug: debugPlan,
@@ -486,7 +482,7 @@ function buildStepPlan(opts, cachePrediction) {
         estimatedSavedMs: fetchStatus === "hit" ? cachePrediction.estimatedSavedMs : undefined,
         name: "Fetch documentation",
         reason: fetchStatus === "skipped"
-            ? "--skip-fetch: reuse cached contexts"
+            ? "--no-fetch: reuse cached contexts"
             : fetchStatus === "hit"
                 ? "CACHED (inputs unchanged)"
                 : "Fetch from Sanity Content Lake",
@@ -508,7 +504,7 @@ function buildStepPlan(opts, cachePrediction) {
         cacheStatus: evalStatus,
         name: "Run evaluation",
         reason: opts.skipEval
-            ? "--skip-eval: reuse existing results"
+            ? "--no-eval: reuse existing results"
             : evalStatus === "hit"
                 ? "CACHED (inputs unchanged)"
                 : "Execute Promptfoo evaluation against all models",
@@ -567,24 +563,6 @@ function buildStepPlan(opts, cachePrediction) {
             willRun: true,
         });
     }
-    // Step 6: Readiness report (optional)
-    if (opts.readinessEnabled) {
-        steps.push({
-            cacheStatus: "miss",
-            name: "Readiness report",
-            reason: "Generate launch readiness checklist",
-            willRun: true,
-        });
-    }
-    // Step 6c: Discovery report (optional)
-    if (opts.discoveryReportEnabled) {
-        steps.push({
-            cacheStatus: "miss",
-            name: "Discovery report",
-            reason: "Analyze agent discoverability from retrieval metrics",
-            willRun: true,
-        });
-    }
     return steps;
 }
 function collectFilesCreated(opts) {
@@ -601,12 +579,6 @@ function collectFilesCreated(opts) {
         files.push("results/latest/failure-modes.json");
         files.push("results/latest/gap-analysis.json");
     }
-    if (opts.readinessEnabled) {
-        files.push("results/latest/readiness-report.md");
-    }
-    if (opts.discoveryReportEnabled) {
-        files.push("results/latest/discovery-report.md");
-    }
     return files.sort();
 }
 // ---------------------------------------------------------------------------

package/dist/pipeline/run-context.d.ts

@@ -85,7 +85,7 @@ export interface RunContextInput {
 export declare function buildRunContext(input: RunContextInput): RunContext;
 /**
  * Resolve `classification` from `AILF_CLASSIFICATION`, validated against
- * the closed enum. Defaults to `"ad-hoc"` so unannotated runs never leak
+ * the closed enum. Defaults to `"adhoc"` so unannotated runs never leak
  * into the canonical `"official"` series.
  */
 export declare function detectClassification(log: Logger): RunClassification;

package/dist/pipeline/run-context.js

@@ -169,17 +169,17 @@ function detectTrigger() {
 // ---------------------------------------------------------------------------
 /**
  * Resolve `classification` from `AILF_CLASSIFICATION`, validated against
- * the closed enum. Defaults to `"ad-hoc"` so unannotated runs never leak
+ * the closed enum. Defaults to `"adhoc"` so unannotated runs never leak
  * into the canonical `"official"` series.
  */
 export function detectClassification(log) {
     const raw = process.env.AILF_CLASSIFICATION?.trim();
     if (!raw)
-        return "ad-hoc";
+        return "adhoc";
     if (isRunClassification(raw))
         return raw;
-    log.warn(`AILF_CLASSIFICATION="${raw}" is not a recognized value; defaulting to "ad-hoc"`);
-    return "ad-hoc";
+    log.warn(`AILF_CLASSIFICATION="${raw}" is not a recognized value; defaulting to "adhoc"`);
+    return "adhoc";
 }
 /**
  * Resolve `owner` from `AILF_OWNER_TEAM` (+ optional

package/dist/pipeline/validate.d.ts

@@ -63,6 +63,6 @@ export declare function validateTaskFiles(rootDir: string): ValidationIssue[];
  *
  * Returns warnings (not errors) if the file is missing — thresholds are
  * optional and don't block evaluation. They only activate when
- * `--readiness` or severity-aware sink routing is used.
+ * `ailf report readiness` or severity-aware sink routing is used.
  */
 export declare function validateThresholdsYaml(rootDir: string): ValidationIssue[];

package/dist/pipeline/validate.js

@@ -272,7 +272,7 @@ export function validateTaskFiles(rootDir) {
  *
  * Returns warnings (not errors) if the file is missing — thresholds are
  * optional and don't block evaluation. They only activate when
- * `--readiness` or severity-aware sink routing is used.
+ * `ailf report readiness` or severity-aware sink routing is used.
  */
 export function validateThresholdsYaml(rootDir) {
     const source = "validateThresholdsYaml";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "3.7.0",
+  "version": "3.8.1",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -50,6 +50,7 @@
     "@anthropic-ai/claude-agent-sdk": "^0.2.105",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.13.1",
+    "nock": "^14.0.13",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
     "@sanity/ailf-core": "0.1.0",
@@ -67,20 +68,21 @@
     "grader-compare": "tsx src/cli.ts grader compare",
     "grader-sensitivity": "tsx src/cli.ts grader sensitivity",
     "calculate-scores": "tsx src/cli.ts calculate-scores",
-    "agent-report": "tsx src/cli.ts agent-report",
+    "agent-report": "tsx src/cli.ts report agent",
     "share": "dotenv -e ../../.env -o -- promptfoo share",
     "view": "dotenv -e ../../.env -o -- promptfoo view",
     "cli": "tsx src/cli.ts",
     "pipeline": "tsx src/cli.ts pipeline",
-    "validate": "tsx src/cli.ts validate",
-    "test": "tsx --test src/__tests__/*.test.ts",
+    "validate": "tsx src/cli.ts validate config",
+    "test": "tsx --test src/__tests__/*.test.ts src/adapters/**/__tests__/*.adapter.test.ts",
     "test:e2e": "AILF_E2E=1 tsx --test src/__tests__/e2e/*.e2e.test.ts",
-    "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts",
+    "test:e2e:adapters": "AILF_E2E=1 tsx --test src/adapters/**/__tests__/*.adapter.test.ts",
+    "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts src/adapters/**/__tests__/*.adapter.test.ts",
     "pr-comment": "tsx src/cli.ts pr-comment",
-    "coverage-audit": "tsx src/cli.ts coverage-audit",
-    "readiness-report": "tsx src/cli.ts readiness-report",
-    "discovery-report": "tsx src/cli.ts discovery-report",
+    "coverage-audit": "tsx src/cli.ts report coverage",
+    "readiness-report": "tsx src/cli.ts report readiness",
+    "discovery-report": "tsx src/cli.ts report discovery",
     "webhook-server": "tsx src/cli.ts webhook-server",
-    "weekly-digest": "tsx src/cli.ts weekly-digest"
+    "weekly-digest": "tsx src/cli.ts report digest"
   }
 }

package/dist/commands/pipeline.d.ts

@@ -1,77 +0,0 @@
-/**
- * pipeline command — the main evaluation pipeline orchestrator.
- *
- * Defines all 36+ CLI flags via Commander, resolves them into a typed
- * options object, bridges to process.env for downstream modules, and
- * delegates to runPipeline().
- *
- * @see docs/cli.md for the full flag reference.
- */
-import { Command } from "commander";
-/**
- * Raw CLI options as parsed by Commander.
- * Field names follow Commander's camelCase convention for kebab-case flags.
- */
-export interface PipelineCliOptions {
-    allowedOrigin: string[];
-    allowedOrigins: string[];
-    area?: string;
-    autoScope: boolean;
-    before?: string;
-    cache: boolean;
-    changedDocs?: string;
-    compare: boolean;
-    compareBaseline?: string;
-    concurrency?: number;
-    config?: string;
-    debug: boolean;
-    debugN?: number;
-    debugPattern?: string;
-    debugSample?: number;
-    discoveryReport: boolean;
-    dryRun: boolean;
-    gapAnalysis: boolean;
-    graderReplications?: number;
-    header: string[];
-    headers: string[];
-    mode: string;
-    variant?: string;
-    output?: string;
-    outputDir?: string;
-    promptfooUrl?: string;
-    publish?: boolean;
-    publishTag?: string;
-    readiness: boolean;
-    remoteCache?: boolean;
-    reportDataset?: string;
-    reportProject?: string;
-    sanityDataset?: string;
-    sanityDocument: string[];
-    sanityDocuments: string[];
-    sanityPerspective?: string;
-    sanityProject?: string;
-    sanityStudioOrigin?: string;
-    search?: string;
-    skipEval: boolean;
-    skipFetch: boolean;
-    source?: string;
-    remote: boolean;
-    repoTasksPath?: string;
-    task?: string;
-    tag: string[];
-    taskSource?: string;
-    threshold?: number;
-    url: string[];
-    urls: string[];
-    apiUrl?: string;
-    artifacts: boolean;
-    artifactsDir?: string;
-    artifactsDryRun: boolean;
-    artifactsExclude?: string;
-    classification?: string;
-    ownerTeam?: string;
-    ownerIndividual?: string;
-    purpose?: string;
-    label: string[];
-}
-export declare function createPipelineCommand(): Command;

package/dist/orchestration/steps/discovery-report-step.d.ts

@@ -1,13 +0,0 @@
-/**
- * Pipeline step: Discovery report (agent discoverability analysis).
- *
- * Calls pure functions from pipeline/discovery-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
-export declare class DiscoveryReportStep implements PipelineStep {
-    readonly name = "discovery-report";
-    readonly optional = true;
-    check(): ValidationIssue[];
-    execute(ctx: AppContext): Promise<StepResult>;
-}

package/dist/orchestration/steps/discovery-report-step.js

@@ -1,62 +0,0 @@
-/**
- * Pipeline step: Discovery report (agent discoverability analysis).
- *
- * Calls pure functions from pipeline/discovery-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { resolve } from "path";
-import { assoc, } from "../../_vendor/ailf-core/index.js";
-import { emitFileContents } from "../../artifact-capture/emit-file.js";
-import { formatDiscoveryMarkdown, generateDiscoveryReport, } from "../../pipeline/discovery-report.js";
-export class DiscoveryReportStep {
-    name = "discovery-report";
-    optional = true;
-    check() {
-        return [];
-    }
-    async execute(ctx) {
-        const root = ctx.config.rootDir;
-        const start = Date.now();
-        try {
-            const scoreSummaryPath = resolve(root, "results", "latest", "score-summary.json");
-            if (!existsSync(scoreSummaryPath)) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: "score-summary.json not found",
-                    status: "failed",
-                };
-            }
-            const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-            if (!scoreSummary.retrievalMetrics) {
-                return {
-                    status: "skipped",
-                    reason: "No retrieval metrics in score summary — run an agentic evaluation first",
-                };
-            }
-            const report = generateDiscoveryReport(scoreSummary, ctx.config.areas);
-            const md = formatDiscoveryMarkdown(report);
-            // Write to outputDir (respects --output-dir)
-            mkdirSync(ctx.config.outputDir, { recursive: true });
-            const discoveryPath = resolve(ctx.config.outputDir, "discovery-report.md");
-            writeFileSync(discoveryPath, md);
-            // W0050 — discoveryReport is per-entry keyed by mode.
-            await emitFileContents(ctx.artifactWriter, "discoveryReport", assoc(ctx, { mode: ctx.config.mode }), discoveryPath);
-            console.log(md);
-            const invisible = report.invisibleDocs.length;
-            const f1 = report.overall.avgF1.toFixed(2);
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `F1=${f1}, ${invisible} invisible doc${invisible === 1 ? "" : "s"}, ${report.recommendations.length} recommendation${report.recommendations.length === 1 ? "" : "s"}`,
-            };
-        }
-        catch (err) {
-            return {
-                durationMs: Date.now() - start,
-                error: err instanceof Error ? err.message : String(err),
-                status: "failed",
-            };
-        }
-    }
-}

package/dist/orchestration/steps/readiness-step.d.ts

@@ -1,13 +0,0 @@
-/**
- * Pipeline step: Launch readiness report.
- *
- * Calls pure functions from pipeline/readiness-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
-export declare class ReadinessStep implements PipelineStep {
-    readonly name = "readiness";
-    readonly optional = true;
-    check(): ValidationIssue[];
-    execute(ctx: AppContext): Promise<StepResult>;
-}

package/dist/orchestration/steps/readiness-step.js

@@ -1,98 +0,0 @@
-/**
- * Pipeline step: Launch readiness report.
- *
- * Calls pure functions from pipeline/readiness-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { resolve } from "path";
-import { tryLoadConfigFile } from "../../pipeline/compiler/config-loader.js";
-import { assoc, } from "../../_vendor/ailf-core/index.js";
-import { emitFileContents } from "../../artifact-capture/emit-file.js";
-import { formatReadinessMarkdown, generateReadinessReport, } from "../../pipeline/readiness-report.js";
-import { ThresholdConfigSchema } from "../../pipeline/schemas.js";
-export class ReadinessStep {
-    name = "readiness";
-    optional = true;
-    check() {
-        return [];
-    }
-    async execute(ctx) {
-        const root = ctx.config.rootDir;
-        const start = Date.now();
-        try {
-            const scoreSummaryPath = resolve(root, "results", "latest", "score-summary.json");
-            if (!existsSync(scoreSummaryPath)) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: "score-summary.json not found",
-                    status: "failed",
-                };
-            }
-            const thresholdsLoaded = tryLoadConfigFile("thresholds", root);
-            if (!thresholdsLoaded) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: "config/thresholds not found",
-                    status: "failed",
-                };
-            }
-            const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-            const thresholdConfig = ThresholdConfigSchema.parse(thresholdsLoaded.data);
-            // Read gap-analysis.json from outputDir (gap-analysis step writes there)
-            const gapPath = resolve(ctx.config.outputDir, "gap-analysis.json");
-            const gapAnalysis = existsSync(gapPath)
-                ? JSON.parse(readFileSync(gapPath, "utf-8"))
-                : undefined;
-            const readinessAreas = ctx.config.areas ?? scoreSummary.scores.map((s) => s.feature);
-            const readinessLines = [];
-            for (const area of readinessAreas) {
-                const areaScore = scoreSummary.scores.find((s) => s.feature === area);
-                if (!areaScore) {
-                    ctx.logger.warn(`Area "${area}" not found in scores — skipping`);
-                    continue;
-                }
-                const report = generateReadinessReport({
-                    area,
-                    gapAnalysis,
-                    scoreSummary,
-                    thresholdConfig,
-                });
-                const md = formatReadinessMarkdown(report);
-                readinessLines.push(md);
-                console.log(md);
-            }
-            if (readinessLines.length > 0) {
-                // Write to outputDir (respects --output-dir)
-                mkdirSync(ctx.config.outputDir, { recursive: true });
-                const readinessPath = resolve(ctx.config.outputDir, "readiness-report.md");
-                writeFileSync(readinessPath, readinessLines.join("\n---\n\n"));
-                // W0050 — readinessReport is run-scoped bulk markdown.
-                await emitFileContents(ctx.artifactWriter, "readinessReport", assoc(ctx), readinessPath);
-            }
-            const passCount = readinessAreas.filter((area) => {
-                const areaScore = scoreSummary.scores.find((s) => s.feature === area);
-                if (!areaScore)
-                    return false;
-                const report = generateReadinessReport({
-                    area,
-                    scoreSummary,
-                    thresholdConfig,
-                });
-                return report.pass;
-            }).length;
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `${passCount}/${readinessAreas.length} areas ready`,
-            };
-        }
-        catch (err) {
-            return {
-                durationMs: Date.now() - start,
-                error: err instanceof Error ? err.message : String(err),
-                status: "failed",
-            };
-        }
-    }
-}

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts

@@ -1,10 +0,0 @@
-/**
- * agent-harness-handler.test.ts — Tests for agent harness mode compilation.
- *
- * Tests validation, provider assembly, tool permission resolution,
- * assertion mapping, sandbox config, lifecycle extensions, and
- * end-to-end compilation of example tasks.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/agent-harness-handler.test.ts
- */
-export {};