@sanity/ailf 2.0.0 → 2.0.2

package/dist/pipeline/steps/publish-report-step.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Pipeline step: Publish evaluation report to the report store.
+ *
+ * This step wraps ScoreSummary + provenance into a Report, writes it to
+ * the Sanity Content Lake (system of record), optionally auto-compares
+ * against the most recent comparable baseline, and fans out to configured
+ * sinks (BigQuery, Slack, webhooks, etc.).
+ *
+ * Opt-in via `--publish` flag or `AILF_PUBLISH=1` environment variable.
+ * Without this flag, the pipeline writes results locally only (unchanged
+ * from current behavior).
+ *
+ * Design principles:
+ * - P1: Reports are immutable events (write-once to Sanity)
+ * - P5: Local-first (pipeline never fails because of a store write)
+ * - P6: Sinks are fire-and-forget (failures logged, not thrown)
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: Report written to Sanity (best-effort), sinks notified
+ *
+ * @see docs/design-docs/report-store/architecture.md
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { type ProvenanceInput } from "../provenance.js";
+import type { PromptfooUrlEntry, StepResult } from "../types.js";
+export interface PublishOptions {
+    /** Whether this is a debug run (debug runs don't store fingerprints) */
+    debug?: boolean;
+    /** Evaluation fingerprint override (computed externally by the pipeline) */
+    evalFingerprint?: string;
+    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
+    promptfooUrl?: string;
+    /** Per-mode Promptfoo share URLs */
+    promptfooUrls?: PromptfooUrlEntry[];
+    /** Override provenance input (for testing or custom workflows) */
+    provenanceInput?: Partial<ProvenanceInput>;
+    /** Sanity dataset for report storage (independent of eval dataset) */
+    reportDataset?: string;
+    /** Sanity project ID for report storage (independent of eval project) */
+    reportProjectId?: string;
+    /** Sanity API token for writes */
+    sanityToken?: string;
+    /** Optional human-supplied tag */
+    tag?: string;
+}
+/**
+ * Run the publish-report pipeline step.
+ *
+ * 1. Read score-summary.json
+ * 2. Build provenance from pipeline context
+ * 3. Create Report with generated UUID v7 ID
+ * 4. Auto-compare against most recent comparable baseline
+ * 5. Write to Sanity Content Lake (system of record)
+ * 6. Fan out to configured sinks (fire-and-forget)
+ * 7. Return step result with report ID and sink outcomes
+ */
+export declare function runPublishReport(pipelineStart: number, options?: PublishOptions): Promise<StepResult>;

package/dist/pipeline/steps/publish-report-step.js

@@ -0,0 +1,243 @@
+/**
+ * Pipeline step: Publish evaluation report to the report store.
+ *
+ * This step wraps ScoreSummary + provenance into a Report, writes it to
+ * the Sanity Content Lake (system of record), optionally auto-compares
+ * against the most recent comparable baseline, and fans out to configured
+ * sinks (BigQuery, Slack, webhooks, etc.).
+ *
+ * Opt-in via `--publish` flag or `AILF_PUBLISH=1` environment variable.
+ * Without this flag, the pipeline writes results locally only (unchanged
+ * from current behavior).
+ *
+ * Design principles:
+ * - P1: Reports are immutable events (write-once to Sanity)
+ * - P5: Local-first (pipeline never fails because of a store write)
+ * - P6: Sinks are fire-and-forget (failures logged, not thrown)
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: Report written to Sanity (best-effort), sinks notified
+ *
+ * @see docs/design-docs/report-store/architecture.md
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { readFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkScoreSummaryValid } from "../checks.js";
+import { buildProvenance } from "../provenance.js";
+import { generateReportId, ReportStore } from "../../report-store.js";
+import { loadSinks } from "../../sinks/index.js";
+import { withRetry } from "../../sinks/retry.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+/**
+ * Run the publish-report pipeline step.
+ *
+ * 1. Read score-summary.json
+ * 2. Build provenance from pipeline context
+ * 3. Create Report with generated UUID v7 ID
+ * 4. Auto-compare against most recent comparable baseline
+ * 5. Write to Sanity Content Lake (system of record)
+ * 6. Fan out to configured sinks (fire-and-forget)
+ * 7. Return step result with report ID and sink outcomes
+ */
+export async function runPublishReport(pipelineStart, options = {}) {
+    const start = Date.now();
+    // Precondition: score summary exists
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Score summary missing: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Read score summary
+    let summary;
+    try {
+        const summaryPath = resolve(ROOT, "results", "latest", "score-summary.json");
+        summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+    }
+    catch (err) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Failed to read score-summary.json: ${err instanceof Error ? err.message : String(err)}`,
+            status: "failed",
+        };
+    }
+    // Build provenance
+    const provenanceInput = buildProvenanceInput(summary, options);
+    const provenance = buildProvenance(provenanceInput);
+    // Create report
+    const now = new Date().toISOString();
+    const reportId = generateReportId();
+    const durationMs = Date.now() - pipelineStart;
+    // Initialize report store — uses AILF_REPORT_* env vars, independent of
+    // SANITY_DATASET/SANITY_PROJECT_ID which control doc evaluation.
+    const token = options.sanityToken ??
+        process.env.AILF_REPORT_SANITY_API_TOKEN ??
+        process.env.SANITY_API_TOKEN;
+    const dataset = options.reportDataset ?? process.env.AILF_REPORT_DATASET ?? undefined;
+    const projectId = options.reportProjectId ?? process.env.AILF_REPORT_PROJECT_ID ?? undefined;
+    const store = new ReportStore({ dataset, projectId, token });
+    // Auto-compare against most recent comparable baseline
+    const comparison = await store.autoCompare(summary, provenance, now);
+    const report = {
+        comparison: comparison ?? undefined,
+        completedAt: now,
+        durationMs,
+        id: reportId,
+        provenance,
+        summary,
+        tag: options.tag,
+    };
+    // Write to Sanity (system of record — best-effort, P5)
+    const sanityResult = await store.write(report);
+    // Load and run sinks (fire-and-forget, P6)
+    const publishResult = await runSinks(report);
+    // Build result summary
+    const parts = [];
+    if (sanityResult) {
+        parts.push(`report:${sanityResult}`);
+    }
+    else {
+        parts.push("Sanity write skipped (no token or unreachable)");
+    }
+    if (comparison) {
+        const delta = comparison.deltas.overall;
+        const sign = delta >= 0 ? "+" : "";
+        parts.push(`vs baseline: ${sign}${delta.toFixed(1)}`);
+    }
+    if (publishResult.sinkResults.length > 0) {
+        const succeeded = publishResult.sinkResults.filter((r) => r.result.status === "success").length;
+        const total = publishResult.sinkResults.length;
+        parts.push(`sinks: ${succeeded}/${total}`);
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: `Published — ${parts.join(", ")}`,
+    };
+}
+// ---------------------------------------------------------------------------
+// Sink runner
+// ---------------------------------------------------------------------------
+/**
+ * Assemble provenance input from the score summary and pipeline context.
+ */
+function buildProvenanceInput(summary, options) {
+    const areas = summary.scores.map((s) => s.feature);
+    const mode = (process.env.EVAL_MODE ?? "baseline");
+    // Read document IDs from env (set by pipeline.ts from --sanity-document flags)
+    const docIds = process.env.SANITY_DOCUMENT_IDS;
+    const sanityDocumentIds = docIds
+        ? docIds
+            .split(",")
+            .map((id) => id.trim())
+            .filter(Boolean)
+        : undefined;
+    // Read task filter from env
+    const taskFilter = process.env.EVAL_FILTER_TASKS;
+    const taskIds = taskFilter
+        ? taskFilter
+            .split(",")
+            .map((t) => t.trim())
+            .filter(Boolean)
+        : undefined;
+    // Build source from summary metadata or env
+    const source = {
+        baseUrl: summary.source?.baseUrl ?? "https://www.sanity.io/docs",
+        dataset: summary.source?.dataset ?? process.env.SANITY_DATASET ?? "next",
+        documentIds: [],
+        llmsTxt: (summary.source?.baseUrl ?? "https://www.sanity.io/docs") + "/llms.txt",
+        name: summary.source?.name ?? "production",
+        perspective: summary.source?.perspective ??
+            process.env.SANITY_PERSPECTIVE ??
+            undefined,
+        priorityDomain: "sanity.io",
+        projectId: summary.source?.projectId ?? process.env.SANITY_PROJECT_ID ?? "3do82whm",
+        studioOrigin: "https://admin.sanity.io",
+        urls: [],
+    };
+    // Pass through eval fingerprint for cross-environment cache lookup.
+    // Debug runs don't store fingerprints (they evaluate a subset of tests
+    // and would produce misleading cache hits for full runs).
+    const evalFingerprint = !options.debug ? options.evalFingerprint : undefined;
+    return {
+        areas,
+        evalFingerprint,
+        mode,
+        promptfooUrl: options.promptfooUrl,
+        promptfooUrls: options.promptfooUrls,
+        rootDir: ROOT,
+        sanityDocumentIds,
+        source,
+        taskIds,
+        ...options.provenanceInput,
+    };
+}
+// ---------------------------------------------------------------------------
+// Provenance input builder
+// ---------------------------------------------------------------------------
+/**
+ * Fan out a report to all configured sinks.
+ *
+ * Sinks are loaded from config/sinks.yaml. Each sink is run with retry
+ * logic (3 attempts, exponential backoff). Failures are logged but never
+ * block the pipeline.
+ */
+async function runSinks(report) {
+    const sinks = loadSinks();
+    const sinkResults = [];
+    if (sinks.length === 0) {
+        return { report, sinkResults };
+    }
+    // Health check all sinks first (non-blocking)
+    for (const sink of sinks) {
+        if (sink.healthCheck) {
+            try {
+                const health = await sink.healthCheck();
+                if (!health.healthy) {
+                    console.warn(`  ⚠️  Sink ${sink.name} health check failed: ${health.reason}`);
+                }
+            }
+            catch (err) {
+                console.warn(`  ⚠️  Sink ${sink.name} health check error: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    // Publish to all sinks in parallel (fire-and-forget with retries)
+    const settled = await Promise.allSettled(sinks.map(async (sink) => {
+        const result = await withRetry(() => sink.publish(report));
+        return { name: sink.name, result };
+    }));
+    for (const outcome of settled) {
+        if (outcome.status === "fulfilled") {
+            sinkResults.push(outcome.value);
+            const { name, result } = outcome.value;
+            if (result.status === "failed") {
+                console.warn(`  ⚠️  Sink ${name} failed: ${result.error}`);
+            }
+            else if (result.status === "skipped") {
+                console.log(`  ⏭️  Sink ${name} skipped: ${result.reason}`);
+            }
+            else {
+                console.log(`  ✅ Sink ${name} delivered${result.detail ? ` (${result.detail})` : ""}`);
+            }
+        }
+        else {
+            // Promise.allSettled rejection — shouldn't happen with withRetry, but just in case
+            const error = outcome.reason instanceof Error
+                ? outcome.reason.message
+                : String(outcome.reason);
+            sinkResults.push({
+                name: "unknown",
+                result: { error, status: "failed" },
+            });
+            console.warn(`  ⚠️  Sink delivery error: ${error}`);
+        }
+    }
+    return { report, sinkResults };
+}

package/dist/pipeline/steps/report-step.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Pipeline step: Generate PR comment / report from scores.
+ *
+ * Preconditions: score-summary.json exists
+ * Postconditions: report markdown generated
+ *
+ * Cache key: results/latest/score-summary.json
+ * Note: Report is always regenerated (not cached) since it may include
+ * dynamic data like Promptfoo URLs. The cache infrastructure is wired up
+ * for consistency but reports are cheap to generate.
+ */
+import type { StepResult } from "../types.js";
+export declare function runReport(outputPath?: string, promptfooUrl?: string): StepResult;

package/dist/pipeline/steps/report-step.js

@@ -0,0 +1,56 @@
+/**
+ * Pipeline step: Generate PR comment / report from scores.
+ *
+ * Preconditions: score-summary.json exists
+ * Postconditions: report markdown generated
+ *
+ * Cache key: results/latest/score-summary.json
+ * Note: Report is always regenerated (not cached) since it may include
+ * dynamic data like Promptfoo URLs. The cache infrastructure is wired up
+ * for consistency but reports are cheap to generate.
+ */
+import { execSync } from "child_process";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkScoreSummaryValid } from "../checks.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+const DEFAULT_REPORT_PATH = resolve(ROOT, "results/latest/pr-comment.md");
+export function runReport(outputPath, promptfooUrl) {
+    const start = Date.now();
+    // Precondition: score summary exists
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Score summary missing: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Always write to a file — use the caller's path or a default.
+    // This avoids dumping the full PR-comment markdown into the terminal.
+    const resolvedOutput = outputPath ?? DEFAULT_REPORT_PATH;
+    // Execute — reports are always regenerated (cheap, may include dynamic URLs)
+    try {
+        const outputArg = ` --output ${resolvedOutput}`;
+        const urlArg = promptfooUrl ? ` --promptfoo-url ${promptfooUrl}` : "";
+        execSync(`pnpm pr-comment${outputArg}${urlArg}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: ["inherit", "ignore", "inherit"],
+        });
+    }
+    catch (err) {
+        return {
+            durationMs: Date.now() - start,
+            error: `pr-comment failed: ${err instanceof Error ? err.message : String(err)}`,
+            status: "failed",
+        };
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: `Report written to ${resolvedOutput}`,
+    };
+}

package/dist/pipeline/steps/update-scores-step.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pipeline step: Update QUALITY_SCORE.md from score summary.
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: QUALITY_SCORE.md is updated with latest scores
+ *
+ * Note: Not cached — writing to a git-tracked file is intentionally
+ * always re-executed to ensure the file reflects the latest scores.
+ */
+import type { StepResult } from "../types.js";
+export declare function runUpdateQualityScores(): Promise<StepResult>;

package/dist/pipeline/steps/update-scores-step.js

@@ -0,0 +1,42 @@
+/**
+ * Pipeline step: Update QUALITY_SCORE.md from score summary.
+ *
+ * Preconditions: score-summary.json exists and is valid
+ * Postconditions: QUALITY_SCORE.md is updated with latest scores
+ *
+ * Note: Not cached — writing to a git-tracked file is intentionally
+ * always re-executed to ensure the file reflects the latest scores.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkScoreSummaryValid } from "../checks.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+export async function runUpdateQualityScores() {
+    const start = Date.now();
+    // Precondition: score summary exists and is valid
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Score summary missing or invalid: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Dynamic import to avoid loading file-writing code at module parse time
+    const { updateQualityScores } = await import("../../scripts/update-quality-scores.js");
+    const result = updateQualityScores();
+    if (!result.success) {
+        return {
+            durationMs: Date.now() - start,
+            error: result.message,
+            status: "failed",
+        };
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: result.message,
+    };
+}

package/dist/pipeline/targeted-loo.d.ts

@@ -18,7 +18,7 @@
  * The actual evaluation execution is handled by the pipeline orchestrator —
  * this module is pure computation on inputs and outputs.
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-4-content-release-integration.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-4-content-release-integration.md
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
  */
 import type { AttributionReport, LOOCostEstimate, LOOResult, TaskAttribution } from "./types.js";

package/dist/pipeline/targeted-loo.js

@@ -18,7 +18,7 @@
  * The actual evaluation execution is handled by the pipeline orchestrator —
  * this module is pure computation on inputs and outputs.
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-4-content-release-integration.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-4-content-release-integration.md
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
  */
 // ---------------------------------------------------------------------------

package/dist/pipeline/thresholds.d.ts

@@ -8,7 +8,7 @@
  * publish-report-step.ts) is responsible for loading the threshold config
  * and score summary.
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
  */
 import type { ThresholdConfig } from "./schemas.js";
 import type { ComparisonReport, ScoreSummary, ThresholdEvaluation, ThresholdViolation } from "./types.js";

package/dist/pipeline/thresholds.js

@@ -8,7 +8,7 @@
  * publish-report-step.ts) is responsible for loading the threshold config
  * and score summary.
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
  */
 // ---------------------------------------------------------------------------
 // Severity priority for sorting (higher = more severe)

package/dist/pipeline/validate.js

@@ -160,8 +160,21 @@ export function validateModelsYaml(rootDir) {
             if (!model.label) {
                 issues.push(error(source, `models[${i}] is missing 'label'`, path.join(rootDir, "config")));
             }
+            if (model.timeoutMs !== undefined &&
+                (typeof model.timeoutMs !== "number" ||
+                    !Number.isInteger(model.timeoutMs) ||
+                    model.timeoutMs <= 0)) {
+                issues.push(error(source, `models[${i}] has invalid 'timeoutMs' (must be a positive integer)`, path.join(rootDir, "config")));
+            }
         }
     }
+    // Check evalBudgetMs
+    if (data.evalBudgetMs !== undefined &&
+        (typeof data.evalBudgetMs !== "number" ||
+            !Number.isInteger(data.evalBudgetMs) ||
+            data.evalBudgetMs <= 0)) {
+        issues.push(error(source, "config/models has invalid 'evalBudgetMs' (must be a positive integer)", path.join(rootDir, "config")));
+    }
     // Check grader
     if (!data.grader) {
         issues.push(error(source, "config/models is missing a 'grader' section", path.join(rootDir, "config")));

package/dist/report-store.d.ts

@@ -94,6 +94,23 @@ export declare class ReportStore {
      * @returns The report ID on success, null on failure (logged, not thrown)
      */
     write(report: Report): Promise<null | ReportId>;
+    /**
+     * Query error arrays from the last N reports for chronic failure detection.
+     *
+     * Returns a simplified shape with just the report ID and error details,
+     * suitable for aggregation by the chronic-failures module.
+     *
+     * @param limit - Number of recent reports to query (default: 10)
+     * @returns Array of { reportId, errors } or empty array on failure
+     */
+    queryRecentErrors(limit?: number): Promise<{
+        reportId: string;
+        errors: {
+            task: string;
+            model: string;
+            error: string;
+        }[];
+    }[]>;
 }
 /**
  * Generate a UUID v7 (time-sortable) for report identification.

package/dist/report-store.js

@@ -213,6 +213,30 @@ export class ReportStore {
             return null;
         }
     }
+    /**
+     * Query error arrays from the last N reports for chronic failure detection.
+     *
+     * Returns a simplified shape with just the report ID and error details,
+     * suitable for aggregation by the chronic-failures module.
+     *
+     * @param limit - Number of recent reports to query (default: 10)
+     * @returns Array of { reportId, errors } or empty array on failure
+     */
+    async queryRecentErrors(limit = 10) {
+        try {
+            const groq = `*[_type == $type && defined(summary.testSummary.errors)]
+        | order(completedAt desc) [0...$limit] {
+          "reportId": reportId,
+          "errors": summary.testSummary.errors[] { task, model, error }
+        }`;
+            const results = await this.client.fetch(groq, { limit, type: REPORT_TYPE });
+            return results ?? [];
+        }
+        catch (error) {
+            console.warn(`  ⚠️  Failed to query recent errors: ${error instanceof Error ? error.message : String(error)}`);
+            return [];
+        }
+    }
 }
 /**
  * Generate a UUID v7 (time-sortable) for report identification.

package/dist/scripts/agent-behavior-report.d.ts

@@ -0,0 +1,19 @@
+/**
+ * agent-behavior-report.ts
+ *
+ * Standalone script that reads Promptfoo evaluation results containing
+ * agent behavior observation data and generates a detailed report.
+ *
+ * This provides deeper analysis than the summary included in the main
+ * calculate-scores report, including:
+ *
+ *   - Per-task behavior breakdown (which specific pages each task visited)
+ *   - Canonical doc coverage (did the agent find the "right" docs?)
+ *   - Request timeline and latency analysis
+ *   - Search strategy analysis
+ *   - Cross-task navigation pattern detection
+ *
+ * Usage:
+ *   tsx src/scripts/agent-behavior-report.ts [results-path]
+ */
+import "dotenv/config";