@sanity/ailf 0.1.0 → 0.1.1

package/dist/pipeline/steps/publish-report-step.d.ts

@@ -1,57 +0,0 @@
-/**
- * Pipeline step: Publish evaluation report to the report store.
- *
- * This step wraps ScoreSummary + provenance into a Report, writes it to
- * the Sanity Content Lake (system of record), optionally auto-compares
- * against the most recent comparable baseline, and fans out to configured
- * sinks (BigQuery, Slack, webhooks, etc.).
- *
- * Opt-in via `--publish` flag or `AILF_PUBLISH=1` environment variable.
- * Without this flag, the pipeline writes results locally only (unchanged
- * from current behavior).
- *
- * Design principles:
- * - P1: Reports are immutable events (write-once to Sanity)
- * - P5: Local-first (pipeline never fails because of a store write)
- * - P6: Sinks are fire-and-forget (failures logged, not thrown)
- *
- * Preconditions: score-summary.json exists and is valid
- * Postconditions: Report written to Sanity (best-effort), sinks notified
- *
- * @see docs/design-docs/report-store/architecture.md
- * @see docs/design-docs/report-store/sink-architecture.md
- */
-import { type ProvenanceInput } from "../provenance.js";
-import type { PromptfooUrlEntry, StepResult } from "../types.js";
-export interface PublishOptions {
-    /** Whether this is a debug run (debug runs don't store fingerprints) */
-    debug?: boolean;
-    /** Evaluation fingerprint override (computed externally by the pipeline) */
-    evalFingerprint?: string;
-    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
-    promptfooUrl?: string;
-    /** Per-mode Promptfoo share URLs */
-    promptfooUrls?: PromptfooUrlEntry[];
-    /** Override provenance input (for testing or custom workflows) */
-    provenanceInput?: Partial<ProvenanceInput>;
-    /** Sanity dataset for report storage (independent of eval dataset) */
-    reportDataset?: string;
-    /** Sanity project ID for report storage (independent of eval project) */
-    reportProjectId?: string;
-    /** Sanity API token for writes */
-    sanityToken?: string;
-    /** Optional human-supplied tag */
-    tag?: string;
-}
-/**
- * Run the publish-report pipeline step.
- *
- * 1. Read score-summary.json
- * 2. Build provenance from pipeline context
- * 3. Create Report with generated UUID v7 ID
- * 4. Auto-compare against most recent comparable baseline
- * 5. Write to Sanity Content Lake (system of record)
- * 6. Fan out to configured sinks (fire-and-forget)
- * 7. Return step result with report ID and sink outcomes
- */
-export declare function runPublishReport(pipelineStart: number, options?: PublishOptions): Promise<StepResult>;

package/dist/pipeline/steps/publish-report-step.js

@@ -1,243 +0,0 @@
-/**
- * Pipeline step: Publish evaluation report to the report store.
- *
- * This step wraps ScoreSummary + provenance into a Report, writes it to
- * the Sanity Content Lake (system of record), optionally auto-compares
- * against the most recent comparable baseline, and fans out to configured
- * sinks (BigQuery, Slack, webhooks, etc.).
- *
- * Opt-in via `--publish` flag or `AILF_PUBLISH=1` environment variable.
- * Without this flag, the pipeline writes results locally only (unchanged
- * from current behavior).
- *
- * Design principles:
- * - P1: Reports are immutable events (write-once to Sanity)
- * - P5: Local-first (pipeline never fails because of a store write)
- * - P6: Sinks are fire-and-forget (failures logged, not thrown)
- *
- * Preconditions: score-summary.json exists and is valid
- * Postconditions: Report written to Sanity (best-effort), sinks notified
- *
- * @see docs/design-docs/report-store/architecture.md
- * @see docs/design-docs/report-store/sink-architecture.md
- */
-import { readFileSync } from "fs";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { checkScoreSummaryValid } from "../checks.js";
-import { buildProvenance } from "../provenance.js";
-import { generateReportId, ReportStore } from "../../report-store.js";
-import { loadSinks } from "../../sinks/index.js";
-import { withRetry } from "../../sinks/retry.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-/**
- * Run the publish-report pipeline step.
- *
- * 1. Read score-summary.json
- * 2. Build provenance from pipeline context
- * 3. Create Report with generated UUID v7 ID
- * 4. Auto-compare against most recent comparable baseline
- * 5. Write to Sanity Content Lake (system of record)
- * 6. Fan out to configured sinks (fire-and-forget)
- * 7. Return step result with report ID and sink outcomes
- */
-export async function runPublishReport(pipelineStart, options = {}) {
-    const start = Date.now();
-    // Precondition: score summary exists
-    const summaryIssues = checkScoreSummaryValid(ROOT);
-    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
-    if (summaryErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Score summary missing: ${summaryErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    // Read score summary
-    let summary;
-    try {
-        const summaryPath = resolve(ROOT, "results", "latest", "score-summary.json");
-        summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
-    }
-    catch (err) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Failed to read score-summary.json: ${err instanceof Error ? err.message : String(err)}`,
-            status: "failed",
-        };
-    }
-    // Build provenance
-    const provenanceInput = buildProvenanceInput(summary, options);
-    const provenance = buildProvenance(provenanceInput);
-    // Create report
-    const now = new Date().toISOString();
-    const reportId = generateReportId();
-    const durationMs = Date.now() - pipelineStart;
-    // Initialize report store — uses AILF_REPORT_* env vars, independent of
-    // SANITY_DATASET/SANITY_PROJECT_ID which control doc evaluation.
-    const token = options.sanityToken ??
-        process.env.AILF_REPORT_SANITY_API_TOKEN ??
-        process.env.SANITY_API_TOKEN;
-    const dataset = options.reportDataset ?? process.env.AILF_REPORT_DATASET ?? undefined;
-    const projectId = options.reportProjectId ?? process.env.AILF_REPORT_PROJECT_ID ?? undefined;
-    const store = new ReportStore({ dataset, projectId, token });
-    // Auto-compare against most recent comparable baseline
-    const comparison = await store.autoCompare(summary, provenance, now);
-    const report = {
-        comparison: comparison ?? undefined,
-        completedAt: now,
-        durationMs,
-        id: reportId,
-        provenance,
-        summary,
-        tag: options.tag,
-    };
-    // Write to Sanity (system of record — best-effort, P5)
-    const sanityResult = await store.write(report);
-    // Load and run sinks (fire-and-forget, P6)
-    const publishResult = await runSinks(report);
-    // Build result summary
-    const parts = [];
-    if (sanityResult) {
-        parts.push(`report:${sanityResult}`);
-    }
-    else {
-        parts.push("Sanity write skipped (no token or unreachable)");
-    }
-    if (comparison) {
-        const delta = comparison.deltas.overall;
-        const sign = delta >= 0 ? "+" : "";
-        parts.push(`vs baseline: ${sign}${delta.toFixed(1)}`);
-    }
-    if (publishResult.sinkResults.length > 0) {
-        const succeeded = publishResult.sinkResults.filter((r) => r.result.status === "success").length;
-        const total = publishResult.sinkResults.length;
-        parts.push(`sinks: ${succeeded}/${total}`);
-    }
-    return {
-        durationMs: Date.now() - start,
-        status: "success",
-        summary: `Published — ${parts.join(", ")}`,
-    };
-}
-// ---------------------------------------------------------------------------
-// Sink runner
-// ---------------------------------------------------------------------------
-/**
- * Assemble provenance input from the score summary and pipeline context.
- */
-function buildProvenanceInput(summary, options) {
-    const areas = summary.scores.map((s) => s.feature);
-    const mode = (process.env.EVAL_MODE ?? "baseline");
-    // Read document IDs from env (set by pipeline.ts from --sanity-document flags)
-    const docIds = process.env.SANITY_DOCUMENT_IDS;
-    const sanityDocumentIds = docIds
-        ? docIds
-            .split(",")
-            .map((id) => id.trim())
-            .filter(Boolean)
-        : undefined;
-    // Read task filter from env
-    const taskFilter = process.env.EVAL_FILTER_TASKS;
-    const taskIds = taskFilter
-        ? taskFilter
-            .split(",")
-            .map((t) => t.trim())
-            .filter(Boolean)
-        : undefined;
-    // Build source from summary metadata or env
-    const source = {
-        baseUrl: summary.source?.baseUrl ?? "https://www.sanity.io/docs",
-        dataset: summary.source?.dataset ?? process.env.SANITY_DATASET ?? "next",
-        documentIds: [],
-        llmsTxt: (summary.source?.baseUrl ?? "https://www.sanity.io/docs") + "/llms.txt",
-        name: summary.source?.name ?? "production",
-        perspective: summary.source?.perspective ??
-            process.env.SANITY_PERSPECTIVE ??
-            undefined,
-        priorityDomain: "sanity.io",
-        projectId: summary.source?.projectId ?? process.env.SANITY_PROJECT_ID ?? "3do82whm",
-        studioOrigin: "https://admin.sanity.io",
-        urls: [],
-    };
-    // Pass through eval fingerprint for cross-environment cache lookup.
-    // Debug runs don't store fingerprints (they evaluate a subset of tests
-    // and would produce misleading cache hits for full runs).
-    const evalFingerprint = !options.debug ? options.evalFingerprint : undefined;
-    return {
-        areas,
-        evalFingerprint,
-        mode,
-        promptfooUrl: options.promptfooUrl,
-        promptfooUrls: options.promptfooUrls,
-        rootDir: ROOT,
-        sanityDocumentIds,
-        source,
-        taskIds,
-        ...options.provenanceInput,
-    };
-}
-// ---------------------------------------------------------------------------
-// Provenance input builder
-// ---------------------------------------------------------------------------
-/**
- * Fan out a report to all configured sinks.
- *
- * Sinks are loaded from config/sinks.yaml. Each sink is run with retry
- * logic (3 attempts, exponential backoff). Failures are logged but never
- * block the pipeline.
- */
-async function runSinks(report) {
-    const sinks = loadSinks();
-    const sinkResults = [];
-    if (sinks.length === 0) {
-        return { report, sinkResults };
-    }
-    // Health check all sinks first (non-blocking)
-    for (const sink of sinks) {
-        if (sink.healthCheck) {
-            try {
-                const health = await sink.healthCheck();
-                if (!health.healthy) {
-                    console.warn(`  ⚠️  Sink ${sink.name} health check failed: ${health.reason}`);
-                }
-            }
-            catch (err) {
-                console.warn(`  ⚠️  Sink ${sink.name} health check error: ${err instanceof Error ? err.message : String(err)}`);
-            }
-        }
-    }
-    // Publish to all sinks in parallel (fire-and-forget with retries)
-    const settled = await Promise.allSettled(sinks.map(async (sink) => {
-        const result = await withRetry(() => sink.publish(report));
-        return { name: sink.name, result };
-    }));
-    for (const outcome of settled) {
-        if (outcome.status === "fulfilled") {
-            sinkResults.push(outcome.value);
-            const { name, result } = outcome.value;
-            if (result.status === "failed") {
-                console.warn(`  ⚠️  Sink ${name} failed: ${result.error}`);
-            }
-            else if (result.status === "skipped") {
-                console.log(`  ⏭️  Sink ${name} skipped: ${result.reason}`);
-            }
-            else {
-                console.log(`  ✅ Sink ${name} delivered${result.detail ? ` (${result.detail})` : ""}`);
-            }
-        }
-        else {
-            // Promise.allSettled rejection — shouldn't happen with withRetry, but just in case
-            const error = outcome.reason instanceof Error
-                ? outcome.reason.message
-                : String(outcome.reason);
-            sinkResults.push({
-                name: "unknown",
-                result: { error, status: "failed" },
-            });
-            console.warn(`  ⚠️  Sink delivery error: ${error}`);
-        }
-    }
-    return { report, sinkResults };
-}

package/dist/pipeline/steps/report-step.d.ts

@@ -1,13 +0,0 @@
-/**
- * Pipeline step: Generate PR comment / report from scores.
- *
- * Preconditions: score-summary.json exists
- * Postconditions: report markdown generated
- *
- * Cache key: results/latest/score-summary.json
- * Note: Report is always regenerated (not cached) since it may include
- * dynamic data like Promptfoo URLs. The cache infrastructure is wired up
- * for consistency but reports are cheap to generate.
- */
-import type { StepResult } from "../types.js";
-export declare function runReport(outputPath?: string, promptfooUrl?: string): StepResult;

package/dist/pipeline/steps/report-step.js

@@ -1,56 +0,0 @@
-/**
- * Pipeline step: Generate PR comment / report from scores.
- *
- * Preconditions: score-summary.json exists
- * Postconditions: report markdown generated
- *
- * Cache key: results/latest/score-summary.json
- * Note: Report is always regenerated (not cached) since it may include
- * dynamic data like Promptfoo URLs. The cache infrastructure is wired up
- * for consistency but reports are cheap to generate.
- */
-import { execSync } from "child_process";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { checkScoreSummaryValid } from "../checks.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-const DEFAULT_REPORT_PATH = resolve(ROOT, "results/latest/pr-comment.md");
-export function runReport(outputPath, promptfooUrl) {
-    const start = Date.now();
-    // Precondition: score summary exists
-    const summaryIssues = checkScoreSummaryValid(ROOT);
-    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
-    if (summaryErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Score summary missing: ${summaryErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    // Always write to a file — use the caller's path or a default.
-    // This avoids dumping the full PR-comment markdown into the terminal.
-    const resolvedOutput = outputPath ?? DEFAULT_REPORT_PATH;
-    // Execute — reports are always regenerated (cheap, may include dynamic URLs)
-    try {
-        const outputArg = ` --output ${resolvedOutput}`;
-        const urlArg = promptfooUrl ? ` --promptfoo-url ${promptfooUrl}` : "";
-        execSync(`pnpm pr-comment${outputArg}${urlArg}`, {
-            cwd: ROOT,
-            env: process.env,
-            stdio: ["inherit", "ignore", "inherit"],
-        });
-    }
-    catch (err) {
-        return {
-            durationMs: Date.now() - start,
-            error: `pr-comment failed: ${err instanceof Error ? err.message : String(err)}`,
-            status: "failed",
-        };
-    }
-    return {
-        durationMs: Date.now() - start,
-        status: "success",
-        summary: `Report written to ${resolvedOutput}`,
-    };
-}

package/dist/pipeline/steps/update-scores-step.d.ts

@@ -1,11 +0,0 @@
-/**
- * Pipeline step: Update QUALITY_SCORE.md from score summary.
- *
- * Preconditions: score-summary.json exists and is valid
- * Postconditions: QUALITY_SCORE.md is updated with latest scores
- *
- * Note: Not cached — writing to a git-tracked file is intentionally
- * always re-executed to ensure the file reflects the latest scores.
- */
-import type { StepResult } from "../types.js";
-export declare function runUpdateQualityScores(): Promise<StepResult>;

package/dist/pipeline/steps/update-scores-step.js

@@ -1,42 +0,0 @@
-/**
- * Pipeline step: Update QUALITY_SCORE.md from score summary.
- *
- * Preconditions: score-summary.json exists and is valid
- * Postconditions: QUALITY_SCORE.md is updated with latest scores
- *
- * Note: Not cached — writing to a git-tracked file is intentionally
- * always re-executed to ensure the file reflects the latest scores.
- */
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { checkScoreSummaryValid } from "../checks.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-export async function runUpdateQualityScores() {
-    const start = Date.now();
-    // Precondition: score summary exists and is valid
-    const summaryIssues = checkScoreSummaryValid(ROOT);
-    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
-    if (summaryErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Score summary missing or invalid: ${summaryErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    // Dynamic import to avoid loading file-writing code at module parse time
-    const { updateQualityScores } = await import("../../scripts/update-quality-scores.js");
-    const result = updateQualityScores();
-    if (!result.success) {
-        return {
-            durationMs: Date.now() - start,
-            error: result.message,
-            status: "failed",
-        };
-    }
-    return {
-        durationMs: Date.now() - start,
-        status: "success",
-        summary: result.message,
-    };
-}

package/dist/scripts/agent-behavior-report.d.ts

@@ -1,19 +0,0 @@
-/**
- * agent-behavior-report.ts
- *
- * Standalone script that reads Promptfoo evaluation results containing
- * agent behavior observation data and generates a detailed report.
- *
- * This provides deeper analysis than the summary included in the main
- * calculate-scores report, including:
- *
- *   - Per-task behavior breakdown (which specific pages each task visited)
- *   - Canonical doc coverage (did the agent find the "right" docs?)
- *   - Request timeline and latency analysis
- *   - Search strategy analysis
- *   - Cross-task navigation pattern detection
- *
- * Usage:
- *   tsx src/scripts/agent-behavior-report.ts [results-path]
- */
-import "dotenv/config";