@sanity/ailf 0.1.0

package/dist/pipeline/steps/grader-consistency-step.js

@@ -0,0 +1,74 @@
+/**
+ * Pipeline step: Measure grader consistency via replicated grading.
+ *
+ * This step is OPTIONAL — it only runs when --grader-replications N is passed.
+ * It re-runs grading assertions N additional times on the same model responses
+ * and measures score variance across replications.
+ *
+ * Preconditions: eval-results.json exists (model responses to re-grade)
+ * Postconditions: grader-consistency.json written to results/latest/
+ *
+ * Not cached: Each run involves fresh API calls to the grader model.
+ * The whole point is to measure variance, so caching would defeat the purpose.
+ */
+import { execSync } from "child_process";
+import { existsSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkResultsExist } from "../checks.js";
+import { RESULTS_FILES } from "./eval-step.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+/**
+ * Run grader consistency analysis.
+ *
+ * @param replications Number of additional grading replications (default: 5)
+ * @param mode Eval mode — determines which results file to read
+ */
+export function runGraderConsistency(replications = 5, mode = "baseline") {
+    const start = Date.now();
+    // For full mode, use baseline results for grader consistency analysis
+    const concreteMode = mode === "full" ? "baseline" : mode;
+    const resultsFile = RESULTS_FILES[concreteMode];
+    const resultsIssues = checkResultsExist(ROOT, resultsFile);
+    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
+    if (resultsErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Results missing: ${resultsErrors.map((e) => e.message).join("; ")}. Run eval first.`,
+            status: "failed",
+        };
+    }
+    // Execute
+    try {
+        execSync(`tsx src/lib/grader-consistency.ts --replications ${replications} --results ${resultsFile}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: "inherit",
+        });
+    }
+    catch (err) {
+        const code = err !== null && typeof err === "object" && "status" in err
+            ? err.status
+            : 1;
+        return {
+            durationMs: Date.now() - start,
+            error: `grader-consistency failed with exit code ${code}`,
+            status: "failed",
+        };
+    }
+    // Postcondition: output file exists
+    const outputPath = resolve(ROOT, "results", "latest", "grader-consistency.json");
+    if (!existsSync(outputPath)) {
+        return {
+            durationMs: Date.now() - start,
+            error: "grader-consistency.json was not created",
+            status: "failed",
+        };
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: `Grader consistency analysis complete (${replications} replications)`,
+    };
+}

package/dist/pipeline/steps/publish-report-step.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Pipeline step: Publish evaluation report to the report store.
+ *
+ * This step wraps ScoreSummary + provenance into a Report, writes it to
+ * the Sanity Content Lake (system of record), optionally auto-compares
+ * against the most recent comparable baseline, and fans out to configured
+ * sinks (BigQuery, Slack, webhooks, etc.).
+ *
+ * Opt-in via `--publish` flag or `AILF_PUBLISH=1` environment variable.
+ * Without this flag, the pipeline writes results locally only (unchanged
+ * from current behavior).
+ *
+ * Design principles:
+ * - P1: Reports are immutable events (write-once to Sanity)
+ * - P5: Local-first (pipeline never fails because of a store write)
+ * - P6: Sinks are fire-and-forget (failures logged, not thrown)
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: Report written to Sanity (best-effort), sinks notified
+ *
+ * @see docs/design-docs/report-store/architecture.md
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { type ProvenanceInput } from "../provenance.js";
+import type { PromptfooUrlEntry, StepResult } from "../types.js";
+export interface PublishOptions {
+    /** Whether this is a debug run (debug runs don't store fingerprints) */
+    debug?: boolean;
+    /** Evaluation fingerprint override (computed externally by the pipeline) */
+    evalFingerprint?: string;
+    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
+    promptfooUrl?: string;
+    /** Per-mode Promptfoo share URLs */
+    promptfooUrls?: PromptfooUrlEntry[];
+    /** Override provenance input (for testing or custom workflows) */
+    provenanceInput?: Partial<ProvenanceInput>;
+    /** Sanity dataset for report storage (independent of eval dataset) */
+    reportDataset?: string;
+    /** Sanity project ID for report storage (independent of eval project) */
+    reportProjectId?: string;
+    /** Sanity API token for writes */
+    sanityToken?: string;
+    /** Optional human-supplied tag */
+    tag?: string;
+}
+/**
+ * Run the publish-report pipeline step.
+ *
+ * 1. Read score-summary.json
+ * 2. Build provenance from pipeline context
+ * 3. Create Report with generated UUID v7 ID
+ * 4. Auto-compare against most recent comparable baseline
+ * 5. Write to Sanity Content Lake (system of record)
+ * 6. Fan out to configured sinks (fire-and-forget)
+ * 7. Return step result with report ID and sink outcomes
+ */
+export declare function runPublishReport(pipelineStart: number, options?: PublishOptions): Promise<StepResult>;

package/dist/pipeline/steps/publish-report-step.js

@@ -0,0 +1,243 @@
+/**
+ * Pipeline step: Publish evaluation report to the report store.
+ *
+ * This step wraps ScoreSummary + provenance into a Report, writes it to
+ * the Sanity Content Lake (system of record), optionally auto-compares
+ * against the most recent comparable baseline, and fans out to configured
+ * sinks (BigQuery, Slack, webhooks, etc.).
+ *
+ * Opt-in via `--publish` flag or `AILF_PUBLISH=1` environment variable.
+ * Without this flag, the pipeline writes results locally only (unchanged
+ * from current behavior).
+ *
+ * Design principles:
+ * - P1: Reports are immutable events (write-once to Sanity)
+ * - P5: Local-first (pipeline never fails because of a store write)
+ * - P6: Sinks are fire-and-forget (failures logged, not thrown)
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: Report written to Sanity (best-effort), sinks notified
+ *
+ * @see docs/design-docs/report-store/architecture.md
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { readFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkScoreSummaryValid } from "../checks.js";
+import { buildProvenance } from "../provenance.js";
+import { generateReportId, ReportStore } from "../../report-store.js";
+import { loadSinks } from "../../sinks/index.js";
+import { withRetry } from "../../sinks/retry.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+/**
+ * Run the publish-report pipeline step.
+ *
+ * 1. Read score-summary.json
+ * 2. Build provenance from pipeline context
+ * 3. Create Report with generated UUID v7 ID
+ * 4. Auto-compare against most recent comparable baseline
+ * 5. Write to Sanity Content Lake (system of record)
+ * 6. Fan out to configured sinks (fire-and-forget)
+ * 7. Return step result with report ID and sink outcomes
+ */
+export async function runPublishReport(pipelineStart, options = {}) {
+    const start = Date.now();
+    // Precondition: score summary exists
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Score summary missing: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Read score summary
+    let summary;
+    try {
+        const summaryPath = resolve(ROOT, "results", "latest", "score-summary.json");
+        summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+    }
+    catch (err) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Failed to read score-summary.json: ${err instanceof Error ? err.message : String(err)}`,
+            status: "failed",
+        };
+    }
+    // Build provenance
+    const provenanceInput = buildProvenanceInput(summary, options);
+    const provenance = buildProvenance(provenanceInput);
+    // Create report
+    const now = new Date().toISOString();
+    const reportId = generateReportId();
+    const durationMs = Date.now() - pipelineStart;
+    // Initialize report store — uses AILF_REPORT_* env vars, independent of
+    // SANITY_DATASET/SANITY_PROJECT_ID which control doc evaluation.
+    const token = options.sanityToken ??
+        process.env.AILF_REPORT_SANITY_API_TOKEN ??
+        process.env.SANITY_API_TOKEN;
+    const dataset = options.reportDataset ?? process.env.AILF_REPORT_DATASET ?? undefined;
+    const projectId = options.reportProjectId ?? process.env.AILF_REPORT_PROJECT_ID ?? undefined;
+    const store = new ReportStore({ dataset, projectId, token });
+    // Auto-compare against most recent comparable baseline
+    const comparison = await store.autoCompare(summary, provenance, now);
+    const report = {
+        comparison: comparison ?? undefined,
+        completedAt: now,
+        durationMs,
+        id: reportId,
+        provenance,
+        summary,
+        tag: options.tag,
+    };
+    // Write to Sanity (system of record — best-effort, P5)
+    const sanityResult = await store.write(report);
+    // Load and run sinks (fire-and-forget, P6)
+    const publishResult = await runSinks(report);
+    // Build result summary
+    const parts = [];
+    if (sanityResult) {
+        parts.push(`report:${sanityResult}`);
+    }
+    else {
+        parts.push("Sanity write skipped (no token or unreachable)");
+    }
+    if (comparison) {
+        const delta = comparison.deltas.overall;
+        const sign = delta >= 0 ? "+" : "";
+        parts.push(`vs baseline: ${sign}${delta.toFixed(1)}`);
+    }
+    if (publishResult.sinkResults.length > 0) {
+        const succeeded = publishResult.sinkResults.filter((r) => r.result.status === "success").length;
+        const total = publishResult.sinkResults.length;
+        parts.push(`sinks: ${succeeded}/${total}`);
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: `Published — ${parts.join(", ")}`,
+    };
+}
+// ---------------------------------------------------------------------------
+// Sink runner
+// ---------------------------------------------------------------------------
+/**
+ * Assemble provenance input from the score summary and pipeline context.
+ */
+function buildProvenanceInput(summary, options) {
+    const areas = summary.scores.map((s) => s.feature);
+    const mode = (process.env.EVAL_MODE ?? "baseline");
+    // Read document IDs from env (set by pipeline.ts from --sanity-document flags)
+    const docIds = process.env.SANITY_DOCUMENT_IDS;
+    const sanityDocumentIds = docIds
+        ? docIds
+            .split(",")
+            .map((id) => id.trim())
+            .filter(Boolean)
+        : undefined;
+    // Read task filter from env
+    const taskFilter = process.env.EVAL_FILTER_TASKS;
+    const taskIds = taskFilter
+        ? taskFilter
+            .split(",")
+            .map((t) => t.trim())
+            .filter(Boolean)
+        : undefined;
+    // Build source from summary metadata or env
+    const source = {
+        baseUrl: summary.source?.baseUrl ?? "https://www.sanity.io/docs",
+        dataset: summary.source?.dataset ?? process.env.SANITY_DATASET ?? "next",
+        documentIds: [],
+        llmsTxt: (summary.source?.baseUrl ?? "https://www.sanity.io/docs") + "/llms.txt",
+        name: summary.source?.name ?? "production",
+        perspective: summary.source?.perspective ??
+            process.env.SANITY_PERSPECTIVE ??
+            undefined,
+        priorityDomain: "sanity.io",
+        projectId: summary.source?.projectId ?? process.env.SANITY_PROJECT_ID ?? "3do82whm",
+        studioOrigin: "https://admin.sanity.io",
+        urls: [],
+    };
+    // Pass through eval fingerprint for cross-environment cache lookup.
+    // Debug runs don't store fingerprints (they evaluate a subset of tests
+    // and would produce misleading cache hits for full runs).
+    const evalFingerprint = !options.debug ? options.evalFingerprint : undefined;
+    return {
+        areas,
+        evalFingerprint,
+        mode,
+        promptfooUrl: options.promptfooUrl,
+        promptfooUrls: options.promptfooUrls,
+        rootDir: ROOT,
+        sanityDocumentIds,
+        source,
+        taskIds,
+        ...options.provenanceInput,
+    };
+}
+// ---------------------------------------------------------------------------
+// Provenance input builder
+// ---------------------------------------------------------------------------
+/**
+ * Fan out a report to all configured sinks.
+ *
+ * Sinks are loaded from config/sinks.yaml. Each sink is run with retry
+ * logic (3 attempts, exponential backoff). Failures are logged but never
+ * block the pipeline.
+ */
+async function runSinks(report) {
+    const sinks = loadSinks();
+    const sinkResults = [];
+    if (sinks.length === 0) {
+        return { report, sinkResults };
+    }
+    // Health check all sinks first (non-blocking)
+    for (const sink of sinks) {
+        if (sink.healthCheck) {
+            try {
+                const health = await sink.healthCheck();
+                if (!health.healthy) {
+                    console.warn(`  ⚠️  Sink ${sink.name} health check failed: ${health.reason}`);
+                }
+            }
+            catch (err) {
+                console.warn(`  ⚠️  Sink ${sink.name} health check error: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    // Publish to all sinks in parallel (fire-and-forget with retries)
+    const settled = await Promise.allSettled(sinks.map(async (sink) => {
+        const result = await withRetry(() => sink.publish(report));
+        return { name: sink.name, result };
+    }));
+    for (const outcome of settled) {
+        if (outcome.status === "fulfilled") {
+            sinkResults.push(outcome.value);
+            const { name, result } = outcome.value;
+            if (result.status === "failed") {
+                console.warn(`  ⚠️  Sink ${name} failed: ${result.error}`);
+            }
+            else if (result.status === "skipped") {
+                console.log(`  ⏭️  Sink ${name} skipped: ${result.reason}`);
+            }
+            else {
+                console.log(`  ✅ Sink ${name} delivered${result.detail ? ` (${result.detail})` : ""}`);
+            }
+        }
+        else {
+            // Promise.allSettled rejection — shouldn't happen with withRetry, but just in case
+            const error = outcome.reason instanceof Error
+                ? outcome.reason.message
+                : String(outcome.reason);
+            sinkResults.push({
+                name: "unknown",
+                result: { error, status: "failed" },
+            });
+            console.warn(`  ⚠️  Sink delivery error: ${error}`);
+        }
+    }
+    return { report, sinkResults };
+}

package/dist/pipeline/steps/report-step.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Pipeline step: Generate PR comment / report from scores.
+ *
+ * Preconditions: score-summary.json exists
+ * Postconditions: report markdown generated
+ *
+ * Cache key: results/latest/score-summary.json
+ * Note: Report is always regenerated (not cached) since it may include
+ * dynamic data like Promptfoo URLs. The cache infrastructure is wired up
+ * for consistency but reports are cheap to generate.
+ */
+import type { StepResult } from "../types.js";
+export declare function runReport(outputPath?: string, promptfooUrl?: string): StepResult;

package/dist/pipeline/steps/report-step.js

@@ -0,0 +1,56 @@
+/**
+ * Pipeline step: Generate PR comment / report from scores.
+ *
+ * Preconditions: score-summary.json exists
+ * Postconditions: report markdown generated
+ *
+ * Cache key: results/latest/score-summary.json
+ * Note: Report is always regenerated (not cached) since it may include
+ * dynamic data like Promptfoo URLs. The cache infrastructure is wired up
+ * for consistency but reports are cheap to generate.
+ */
+import { execSync } from "child_process";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkScoreSummaryValid } from "../checks.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+const DEFAULT_REPORT_PATH = resolve(ROOT, "results/latest/pr-comment.md");
+export function runReport(outputPath, promptfooUrl) {
+    const start = Date.now();
+    // Precondition: score summary exists
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Score summary missing: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Always write to a file — use the caller's path or a default.
+    // This avoids dumping the full PR-comment markdown into the terminal.
+    const resolvedOutput = outputPath ?? DEFAULT_REPORT_PATH;
+    // Execute — reports are always regenerated (cheap, may include dynamic URLs)
+    try {
+        const outputArg = ` --output ${resolvedOutput}`;
+        const urlArg = promptfooUrl ? ` --promptfoo-url ${promptfooUrl}` : "";
+        execSync(`pnpm pr-comment${outputArg}${urlArg}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: ["inherit", "ignore", "inherit"],
+        });
+    }
+    catch (err) {
+        return {
+            durationMs: Date.now() - start,
+            error: `pr-comment failed: ${err instanceof Error ? err.message : String(err)}`,
+            status: "failed",
+        };
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: `Report written to ${resolvedOutput}`,
+    };
+}

package/dist/pipeline/steps/update-scores-step.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pipeline step: Update QUALITY_SCORE.md from score summary.
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: QUALITY_SCORE.md is updated with latest scores
+ *
+ * Note: Not cached — writing to a git-tracked file is intentionally
+ * always re-executed to ensure the file reflects the latest scores.
+ */
+import type { StepResult } from "../types.js";
+export declare function runUpdateQualityScores(): Promise<StepResult>;

package/dist/pipeline/steps/update-scores-step.js

@@ -0,0 +1,42 @@
+/**
+ * Pipeline step: Update QUALITY_SCORE.md from score summary.
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: QUALITY_SCORE.md is updated with latest scores
+ *
+ * Note: Not cached — writing to a git-tracked file is intentionally
+ * always re-executed to ensure the file reflects the latest scores.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkScoreSummaryValid } from "../checks.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+export async function runUpdateQualityScores() {
+    const start = Date.now();
+    // Precondition: score summary exists and is valid
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Score summary missing or invalid: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Dynamic import to avoid loading file-writing code at module parse time
+    const { updateQualityScores } = await import("../../scripts/update-quality-scores.js");
+    const result = updateQualityScores();
+    if (!result.success) {
+        return {
+            durationMs: Date.now() - start,
+            error: result.message,
+            status: "failed",
+        };
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: result.message,
+    };
+}

package/dist/pipeline/targeted-loo.d.ts

@@ -0,0 +1,88 @@
+/**
+ * pipeline/targeted-loo.ts
+ *
+ * Targeted leave-one-out (LOO) attribution for ambiguous cases.
+ *
+ * Phase 4d of the Scenario Matrix implementation.
+ *
+ * When correlation-based attribution (Phase 2c) identifies ambiguous tasks
+ * (2+ changed docs map to the same task), targeted LOO resolves the ambiguity
+ * by running per-document mini-evaluations to measure each document's
+ * marginal contribution.
+ *
+ * This module handles:
+ * - Cost estimation before execution (to enable user confirmation)
+ * - Result analysis after LOO runs complete
+ * - Integration with the existing AttributionReport
+ *
+ * The actual evaluation execution is handled by the pipeline orchestrator —
+ * this module is pure computation on inputs and outputs.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-4-content-release-integration.md
+ * @see docs/design-docs/scenario-matrix/per-document-attribution.md
+ */
+import type { AttributionReport, LOOCostEstimate, LOOResult, TaskAttribution } from "./types.js";
+/**
+ * Analyze LOO evaluation results to compute marginal contributions.
+ *
+ * Given the full-release score and per-document revert scores,
+ * calculates each document's marginal contribution as:
+ *   marginal(doc) = fullReleaseScore - revertedScore(doc)
+ *
+ * @param taskId - The task being analyzed
+ * @param fullReleaseDelta - The total task delta from the full release
+ * @param revertResults - Per-document scores when that document is reverted
+ * @param noiseThreshold - Threshold for marking contributions as noise
+ * @param additionalCost - Actual cost incurred for the LOO evaluations
+ * @returns LOO result with per-document marginal contributions
+ */
+export declare function analyzeLOOResults(taskId: string, fullReleaseDelta: number, revertResults: {
+    /** Score delta with this document reverted */
+    revertedDelta: number;
+    /** The document slug that was reverted */
+    slug: string;
+}[], noiseThreshold: number, additionalCost: number): LOOResult;
+/**
+ * Enrich an attribution report with LOO results for ambiguous tasks.
+ *
+ * Replaces the "ambiguous" classification with resolved per-document
+ * contributions for tasks that have LOO data.
+ *
+ * @param attribution - Original attribution report
+ * @param looResults - LOO results for ambiguous tasks
+ * @returns New attribution report with LOO data integrated
+ */
+export declare function enrichAttributionWithLOO(attribution: AttributionReport, looResults: LOOResult[]): AttributionReport & {
+    looResults: LOOResult[];
+};
+/**
+ * Estimate the cost of running targeted LOO for ambiguous tasks.
+ *
+ * @param ambiguousTasks - Tasks identified as ambiguous
+ * @param testsPerTask - Number of tests per task (from task YAML)
+ * @param costPerTest - Estimated cost per test (default: $0.08)
+ * @returns Cost estimate with per-task breakdown
+ */
+export declare function estimateLOOCost(ambiguousTasks: TaskAttribution[], testsPerTask: Record<string, number>, costPerTest?: number): LOOCostEstimate;
+/**
+ * Identify ambiguous tasks that would benefit from targeted LOO.
+ *
+ * Filters the attribution report to find tasks where 2+ changed documents
+ * are in the canonical set and the delta is outside the noise floor.
+ *
+ * @param attribution - Attribution report from Phase 2c
+ * @returns Ambiguous task attributions suitable for LOO
+ */
+export declare function findAmbiguousTasks(attribution: AttributionReport): TaskAttribution[];
+/**
+ * Format a LOO cost estimate for console output (for user confirmation).
+ */
+export declare function formatLOOCostEstimate(estimate: LOOCostEstimate): string;
+/**
+ * Format LOO results for console output.
+ */
+export declare function formatLOOResultsConsole(results: LOOResult[]): string;
+/**
+ * Format LOO results as markdown.
+ */
+export declare function formatLOOResultsMarkdown(results: LOOResult[]): string;