@sanity/ailf 3.7.0 → 3.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "3.7.0",
+  "version": "3.8.0",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -67,20 +67,20 @@
     "grader-compare": "tsx src/cli.ts grader compare",
     "grader-sensitivity": "tsx src/cli.ts grader sensitivity",
     "calculate-scores": "tsx src/cli.ts calculate-scores",
-    "agent-report": "tsx src/cli.ts agent-report",
+    "agent-report": "tsx src/cli.ts report agent",
     "share": "dotenv -e ../../.env -o -- promptfoo share",
     "view": "dotenv -e ../../.env -o -- promptfoo view",
     "cli": "tsx src/cli.ts",
     "pipeline": "tsx src/cli.ts pipeline",
-    "validate": "tsx src/cli.ts validate",
+    "validate": "tsx src/cli.ts validate config",
     "test": "tsx --test src/__tests__/*.test.ts",
     "test:e2e": "AILF_E2E=1 tsx --test src/__tests__/e2e/*.e2e.test.ts",
     "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts",
     "pr-comment": "tsx src/cli.ts pr-comment",
-    "coverage-audit": "tsx src/cli.ts coverage-audit",
-    "readiness-report": "tsx src/cli.ts readiness-report",
-    "discovery-report": "tsx src/cli.ts discovery-report",
+    "coverage-audit": "tsx src/cli.ts report coverage",
+    "readiness-report": "tsx src/cli.ts report readiness",
+    "discovery-report": "tsx src/cli.ts report discovery",
     "webhook-server": "tsx src/cli.ts webhook-server",
-    "weekly-digest": "tsx src/cli.ts weekly-digest"
+    "weekly-digest": "tsx src/cli.ts report digest"
   }
 }

package/dist/commands/pipeline.d.ts

@@ -1,77 +0,0 @@
-/**
- * pipeline command — the main evaluation pipeline orchestrator.
- *
- * Defines all 36+ CLI flags via Commander, resolves them into a typed
- * options object, bridges to process.env for downstream modules, and
- * delegates to runPipeline().
- *
- * @see docs/cli.md for the full flag reference.
- */
-import { Command } from "commander";
-/**
- * Raw CLI options as parsed by Commander.
- * Field names follow Commander's camelCase convention for kebab-case flags.
- */
-export interface PipelineCliOptions {
-    allowedOrigin: string[];
-    allowedOrigins: string[];
-    area?: string;
-    autoScope: boolean;
-    before?: string;
-    cache: boolean;
-    changedDocs?: string;
-    compare: boolean;
-    compareBaseline?: string;
-    concurrency?: number;
-    config?: string;
-    debug: boolean;
-    debugN?: number;
-    debugPattern?: string;
-    debugSample?: number;
-    discoveryReport: boolean;
-    dryRun: boolean;
-    gapAnalysis: boolean;
-    graderReplications?: number;
-    header: string[];
-    headers: string[];
-    mode: string;
-    variant?: string;
-    output?: string;
-    outputDir?: string;
-    promptfooUrl?: string;
-    publish?: boolean;
-    publishTag?: string;
-    readiness: boolean;
-    remoteCache?: boolean;
-    reportDataset?: string;
-    reportProject?: string;
-    sanityDataset?: string;
-    sanityDocument: string[];
-    sanityDocuments: string[];
-    sanityPerspective?: string;
-    sanityProject?: string;
-    sanityStudioOrigin?: string;
-    search?: string;
-    skipEval: boolean;
-    skipFetch: boolean;
-    source?: string;
-    remote: boolean;
-    repoTasksPath?: string;
-    task?: string;
-    tag: string[];
-    taskSource?: string;
-    threshold?: number;
-    url: string[];
-    urls: string[];
-    apiUrl?: string;
-    artifacts: boolean;
-    artifactsDir?: string;
-    artifactsDryRun: boolean;
-    artifactsExclude?: string;
-    classification?: string;
-    ownerTeam?: string;
-    ownerIndividual?: string;
-    purpose?: string;
-    label: string[];
-}
-export declare function createPipelineCommand(): Command;

package/dist/orchestration/steps/discovery-report-step.d.ts

@@ -1,13 +0,0 @@
-/**
- * Pipeline step: Discovery report (agent discoverability analysis).
- *
- * Calls pure functions from pipeline/discovery-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
-export declare class DiscoveryReportStep implements PipelineStep {
-    readonly name = "discovery-report";
-    readonly optional = true;
-    check(): ValidationIssue[];
-    execute(ctx: AppContext): Promise<StepResult>;
-}

package/dist/orchestration/steps/discovery-report-step.js

@@ -1,62 +0,0 @@
-/**
- * Pipeline step: Discovery report (agent discoverability analysis).
- *
- * Calls pure functions from pipeline/discovery-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { resolve } from "path";
-import { assoc, } from "../../_vendor/ailf-core/index.js";
-import { emitFileContents } from "../../artifact-capture/emit-file.js";
-import { formatDiscoveryMarkdown, generateDiscoveryReport, } from "../../pipeline/discovery-report.js";
-export class DiscoveryReportStep {
-    name = "discovery-report";
-    optional = true;
-    check() {
-        return [];
-    }
-    async execute(ctx) {
-        const root = ctx.config.rootDir;
-        const start = Date.now();
-        try {
-            const scoreSummaryPath = resolve(root, "results", "latest", "score-summary.json");
-            if (!existsSync(scoreSummaryPath)) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: "score-summary.json not found",
-                    status: "failed",
-                };
-            }
-            const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-            if (!scoreSummary.retrievalMetrics) {
-                return {
-                    status: "skipped",
-                    reason: "No retrieval metrics in score summary — run an agentic evaluation first",
-                };
-            }
-            const report = generateDiscoveryReport(scoreSummary, ctx.config.areas);
-            const md = formatDiscoveryMarkdown(report);
-            // Write to outputDir (respects --output-dir)
-            mkdirSync(ctx.config.outputDir, { recursive: true });
-            const discoveryPath = resolve(ctx.config.outputDir, "discovery-report.md");
-            writeFileSync(discoveryPath, md);
-            // W0050 — discoveryReport is per-entry keyed by mode.
-            await emitFileContents(ctx.artifactWriter, "discoveryReport", assoc(ctx, { mode: ctx.config.mode }), discoveryPath);
-            console.log(md);
-            const invisible = report.invisibleDocs.length;
-            const f1 = report.overall.avgF1.toFixed(2);
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `F1=${f1}, ${invisible} invisible doc${invisible === 1 ? "" : "s"}, ${report.recommendations.length} recommendation${report.recommendations.length === 1 ? "" : "s"}`,
-            };
-        }
-        catch (err) {
-            return {
-                durationMs: Date.now() - start,
-                error: err instanceof Error ? err.message : String(err),
-                status: "failed",
-            };
-        }
-    }
-}

package/dist/orchestration/steps/readiness-step.d.ts

@@ -1,13 +0,0 @@
-/**
- * Pipeline step: Launch readiness report.
- *
- * Calls pure functions from pipeline/readiness-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
-export declare class ReadinessStep implements PipelineStep {
-    readonly name = "readiness";
-    readonly optional = true;
-    check(): ValidationIssue[];
-    execute(ctx: AppContext): Promise<StepResult>;
-}

package/dist/orchestration/steps/readiness-step.js

@@ -1,98 +0,0 @@
-/**
- * Pipeline step: Launch readiness report.
- *
- * Calls pure functions from pipeline/readiness-report.ts directly.
- * Optional step — failure doesn't stop the pipeline.
- */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { resolve } from "path";
-import { tryLoadConfigFile } from "../../pipeline/compiler/config-loader.js";
-import { assoc, } from "../../_vendor/ailf-core/index.js";
-import { emitFileContents } from "../../artifact-capture/emit-file.js";
-import { formatReadinessMarkdown, generateReadinessReport, } from "../../pipeline/readiness-report.js";
-import { ThresholdConfigSchema } from "../../pipeline/schemas.js";
-export class ReadinessStep {
-    name = "readiness";
-    optional = true;
-    check() {
-        return [];
-    }
-    async execute(ctx) {
-        const root = ctx.config.rootDir;
-        const start = Date.now();
-        try {
-            const scoreSummaryPath = resolve(root, "results", "latest", "score-summary.json");
-            if (!existsSync(scoreSummaryPath)) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: "score-summary.json not found",
-                    status: "failed",
-                };
-            }
-            const thresholdsLoaded = tryLoadConfigFile("thresholds", root);
-            if (!thresholdsLoaded) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: "config/thresholds not found",
-                    status: "failed",
-                };
-            }
-            const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-            const thresholdConfig = ThresholdConfigSchema.parse(thresholdsLoaded.data);
-            // Read gap-analysis.json from outputDir (gap-analysis step writes there)
-            const gapPath = resolve(ctx.config.outputDir, "gap-analysis.json");
-            const gapAnalysis = existsSync(gapPath)
-                ? JSON.parse(readFileSync(gapPath, "utf-8"))
-                : undefined;
-            const readinessAreas = ctx.config.areas ?? scoreSummary.scores.map((s) => s.feature);
-            const readinessLines = [];
-            for (const area of readinessAreas) {
-                const areaScore = scoreSummary.scores.find((s) => s.feature === area);
-                if (!areaScore) {
-                    ctx.logger.warn(`Area "${area}" not found in scores — skipping`);
-                    continue;
-                }
-                const report = generateReadinessReport({
-                    area,
-                    gapAnalysis,
-                    scoreSummary,
-                    thresholdConfig,
-                });
-                const md = formatReadinessMarkdown(report);
-                readinessLines.push(md);
-                console.log(md);
-            }
-            if (readinessLines.length > 0) {
-                // Write to outputDir (respects --output-dir)
-                mkdirSync(ctx.config.outputDir, { recursive: true });
-                const readinessPath = resolve(ctx.config.outputDir, "readiness-report.md");
-                writeFileSync(readinessPath, readinessLines.join("\n---\n\n"));
-                // W0050 — readinessReport is run-scoped bulk markdown.
-                await emitFileContents(ctx.artifactWriter, "readinessReport", assoc(ctx), readinessPath);
-            }
-            const passCount = readinessAreas.filter((area) => {
-                const areaScore = scoreSummary.scores.find((s) => s.feature === area);
-                if (!areaScore)
-                    return false;
-                const report = generateReadinessReport({
-                    area,
-                    scoreSummary,
-                    thresholdConfig,
-                });
-                return report.pass;
-            }).length;
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `${passCount}/${readinessAreas.length} areas ready`,
-            };
-        }
-        catch (err) {
-            return {
-                durationMs: Date.now() - start,
-                error: err instanceof Error ? err.message : String(err),
-                status: "failed",
-            };
-        }
-    }
-}