@sanity/ailf 0.1.0

package/dist/sinks/bigquery/index.d.ts

@@ -0,0 +1,131 @@
+/**
+ * sinks/bigquery/index.ts
+ *
+ * BigQuery analytics sink — flattens evaluation reports into rows for
+ * the `ailf.reports` and `ailf.area_scores` tables.
+ *
+ * The BigQuery schema serves as the analytical mirror of the Sanity Content
+ * Lake. While Sanity is the system of record for operational queries (Studio
+ * dashboards, webhooks, real-time listeners), BigQuery handles the analytical
+ * queries that GROQ was never designed for (percentile distributions,
+ * cost-per-quality-point trends, cross-source regression detection).
+ *
+ * Schema evolution: additive-only. New columns with NULLABLE mode don't break
+ * existing queries. The `flattenReportRow` and `flattenAreaScoreRows` functions
+ * are the single place where schema mapping lives.
+ *
+ * @see docs/design-docs/report-store/bigquery.md
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { Report, SinkHealthStatus, SinkResult } from "../../pipeline/types.js";
+import type { ReportSink } from "../types.js";
+/** One row per area per model per report in `ailf.area_scores`. */
+export interface AreaScoreRow {
+    area: string;
+    ceiling_score: number;
+    code_correctness: number;
+    completed_at: string;
+    doc_coverage: number;
+    doc_lift: number;
+    floor_score: number;
+    mode: string;
+    model_id: string;
+    report_id: string;
+    source_name: string;
+    task_completion: number;
+    test_count: number;
+    total_cost: number;
+    total_score: number;
+}
+/** Options for constructing a BigQuerySink instance. */
+export interface BigQuerySinkOptions {
+    /** Path to a service account JSON credentials file */
+    credentials?: string;
+    /** BigQuery dataset name (e.g., "ailf") */
+    dataset: string;
+    /** BigQuery project ID (e.g., "ailf-reports") */
+    project: string;
+}
+/** One row per evaluation run in `ailf.reports`. */
+export interface ReportRow {
+    area_count: number;
+    areas: string[];
+    avg_doc_lift: number;
+    avg_score: number;
+    completed_at: string;
+    duration_ms: number;
+    git_branch: null | string;
+    git_pr_number: null | number;
+    git_repo: null | string;
+    git_sha: null | string;
+    grader_cost: null | number;
+    grader_model: string;
+    mode: string;
+    model_count: number;
+    models: string[];
+    promptfoo_url: null | string;
+    promptfoo_urls: null | string;
+    report_id: string;
+    source_base_url: string;
+    source_dataset: null | string;
+    source_name: string;
+    source_perspective: null | string;
+    tag: null | string;
+    total_cost: null | number;
+    trigger_caller_repo: null | string;
+    trigger_type: string;
+}
+/**
+ * BigQuery sink — inserts flattened report rows into BigQuery tables.
+ *
+ * Transforms the nested `Report` into flat rows suitable for SQL analytics.
+ * Two tables are populated per publish:
+ * - `ailf.reports` — one row per evaluation run
+ * - `ailf.area_scores` — one row per area per model per report
+ *
+ * The sink creates a BigQuery client lazily on first use. If credentials
+ * are not provided, it falls back to Application Default Credentials (ADC).
+ */
+export declare class BigQuerySink implements ReportSink {
+    readonly name = "bigquery";
+    private client;
+    private readonly options;
+    constructor(options: BigQuerySinkOptions);
+    /**
+     * Health check — validates that BigQuery is reachable and the dataset exists.
+     *
+     * This catches common misconfigurations early (wrong project, missing
+     * credentials, non-existent dataset) before the pipeline runs a full eval.
+     */
+    healthCheck(): Promise<SinkHealthStatus>;
+    /**
+     * Publish a report to BigQuery — inserts into both tables.
+     *
+     * The report row goes into `ailf.reports` and the area score rows go
+     * into `ailf.area_scores`. Both inserts use streaming inserts for
+     * near-real-time availability.
+     */
+    publish(report: Report): Promise<SinkResult>;
+    /** Lazily create the BigQuery client (reused across calls). */
+    private getClient;
+}
+/**
+ * Flatten a Report into rows for the `ailf.area_scores` table.
+ *
+ * Produces one row per area per model. When per-model data is available
+ * (multi-model evaluations), each model's area scores become separate rows.
+ * When only aggregate scores exist (single model or no per-model breakdown),
+ * the model_id is set to the first model in provenance.
+ *
+ * Denormalized fields (completed_at, mode, source_name) are included for
+ * BigQuery partitioning and clustering efficiency.
+ */
+export declare function flattenAreaScoreRows(report: Report): AreaScoreRow[];
+/**
+ * Flatten a Report into a single row for the `ailf.reports` table.
+ *
+ * This is the single place where the nested Report structure maps to
+ * the flat BigQuery schema. When the report format evolves, update this
+ * function and add new NULLABLE columns to the BigQuery table.
+ */
+export declare function flattenReportRow(report: Report): ReportRow;

package/dist/sinks/bigquery/index.js

@@ -0,0 +1,222 @@
+/**
+ * sinks/bigquery/index.ts
+ *
+ * BigQuery analytics sink — flattens evaluation reports into rows for
+ * the `ailf.reports` and `ailf.area_scores` tables.
+ *
+ * The BigQuery schema serves as the analytical mirror of the Sanity Content
+ * Lake. While Sanity is the system of record for operational queries (Studio
+ * dashboards, webhooks, real-time listeners), BigQuery handles the analytical
+ * queries that GROQ was never designed for (percentile distributions,
+ * cost-per-quality-point trends, cross-source regression detection).
+ *
+ * Schema evolution: additive-only. New columns with NULLABLE mode don't break
+ * existing queries. The `flattenReportRow` and `flattenAreaScoreRows` functions
+ * are the single place where schema mapping lives.
+ *
+ * @see docs/design-docs/report-store/bigquery.md
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { BigQuery } from "@google-cloud/bigquery";
+// ---------------------------------------------------------------------------
+// BigQuerySink class
+// ---------------------------------------------------------------------------
+/**
+ * BigQuery sink — inserts flattened report rows into BigQuery tables.
+ *
+ * Transforms the nested `Report` into flat rows suitable for SQL analytics.
+ * Two tables are populated per publish:
+ * - `ailf.reports` — one row per evaluation run
+ * - `ailf.area_scores` — one row per area per model per report
+ *
+ * The sink creates a BigQuery client lazily on first use. If credentials
+ * are not provided, it falls back to Application Default Credentials (ADC).
+ */
+export class BigQuerySink {
+    name = "bigquery";
+    client = null;
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    /**
+     * Health check — validates that BigQuery is reachable and the dataset exists.
+     *
+     * This catches common misconfigurations early (wrong project, missing
+     * credentials, non-existent dataset) before the pipeline runs a full eval.
+     */
+    async healthCheck() {
+        try {
+            const client = this.getClient();
+            const dataset = client.dataset(this.options.dataset);
+            const [exists] = await dataset.exists();
+            if (!exists) {
+                return {
+                    healthy: false,
+                    reason: `Dataset "${this.options.dataset}" does not exist in project "${this.options.project}"`,
+                };
+            }
+            return { healthy: true };
+        }
+        catch (error) {
+            return {
+                healthy: false,
+                reason: `BigQuery health check failed: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }
+    }
+    /**
+     * Publish a report to BigQuery — inserts into both tables.
+     *
+     * The report row goes into `ailf.reports` and the area score rows go
+     * into `ailf.area_scores`. Both inserts use streaming inserts for
+     * near-real-time availability.
+     */
+    async publish(report) {
+        try {
+            const client = this.getClient();
+            const dataset = client.dataset(this.options.dataset);
+            const reportRow = flattenReportRow(report);
+            const areaRows = flattenAreaScoreRows(report);
+            // Insert report row
+            const reportsTable = dataset.table("reports");
+            await reportsTable.insert([reportRow]);
+            // Insert area score rows (may be empty if no scores)
+            if (areaRows.length > 0) {
+                const areaScoresTable = dataset.table("area_scores");
+                await areaScoresTable.insert(areaRows);
+            }
+            const totalRows = 1 + areaRows.length;
+            return {
+                detail: `${totalRows} row${totalRows === 1 ? "" : "s"} inserted (1 report + ${areaRows.length} area scores)`,
+                status: "success",
+            };
+        }
+        catch (error) {
+            return {
+                error: `BigQuery insert failed: ${error instanceof Error ? error.message : String(error)}`,
+                status: "failed",
+            };
+        }
+    }
+    // -----------------------------------------------------------------------
+    // Private helpers
+    // -----------------------------------------------------------------------
+    /** Lazily create the BigQuery client (reused across calls). */
+    getClient() {
+        this.client ??= new BigQuery({
+            keyFilename: this.options.credentials ?? undefined,
+            projectId: this.options.project,
+        });
+        return this.client;
+    }
+}
+// ---------------------------------------------------------------------------
+// Flattening functions (the schema mapping layer)
+// ---------------------------------------------------------------------------
+/**
+ * Flatten a Report into rows for the `ailf.area_scores` table.
+ *
+ * Produces one row per area per model. When per-model data is available
+ * (multi-model evaluations), each model's area scores become separate rows.
+ * When only aggregate scores exist (single model or no per-model breakdown),
+ * the model_id is set to the first model in provenance.
+ *
+ * Denormalized fields (completed_at, mode, source_name) are included for
+ * BigQuery partitioning and clustering efficiency.
+ */
+export function flattenAreaScoreRows(report) {
+    const { provenance, summary } = report;
+    const rows = [];
+    // Common denormalized fields for partitioning/clustering
+    const common = {
+        completed_at: report.completedAt,
+        mode: provenance.mode,
+        report_id: report.id,
+        source_name: provenance.source.name,
+    };
+    // When per-model data is available, use it for model-level granularity
+    if (summary.perModel && summary.perModel.length > 0) {
+        for (const modelEntry of summary.perModel) {
+            for (const score of modelEntry.scores) {
+                rows.push({
+                    ...common,
+                    area: score.feature,
+                    ceiling_score: score.ceilingScore,
+                    code_correctness: score.codeCorrectness,
+                    doc_coverage: score.docCoverage,
+                    doc_lift: score.docLift,
+                    floor_score: score.floorScore,
+                    model_id: modelEntry.modelId,
+                    task_completion: score.taskCompletion,
+                    test_count: score.testCount,
+                    total_cost: score.totalCost,
+                    total_score: score.totalScore,
+                });
+            }
+        }
+    }
+    else {
+        // Fallback: use aggregate scores with the first model ID
+        const modelId = provenance.models[0]?.id ?? "unknown";
+        for (const score of summary.scores) {
+            rows.push({
+                ...common,
+                area: score.feature,
+                ceiling_score: score.ceilingScore,
+                code_correctness: score.codeCorrectness,
+                doc_coverage: score.docCoverage,
+                doc_lift: score.docLift,
+                floor_score: score.floorScore,
+                model_id: modelId,
+                task_completion: score.taskCompletion,
+                test_count: score.testCount,
+                total_cost: score.totalCost,
+                total_score: score.totalScore,
+            });
+        }
+    }
+    return rows;
+}
+/**
+ * Flatten a Report into a single row for the `ailf.reports` table.
+ *
+ * This is the single place where the nested Report structure maps to
+ * the flat BigQuery schema. When the report format evolves, update this
+ * function and add new NULLABLE columns to the BigQuery table.
+ */
+export function flattenReportRow(report) {
+    const { provenance, summary } = report;
+    return {
+        area_count: provenance.areas.length,
+        areas: provenance.areas,
+        avg_doc_lift: summary.overall.avgDocLift,
+        avg_score: summary.overall.avgScore,
+        completed_at: report.completedAt,
+        duration_ms: report.durationMs,
+        git_branch: provenance.git?.branch ?? null,
+        git_pr_number: provenance.git?.prNumber ?? null,
+        git_repo: provenance.git?.repo ?? null,
+        git_sha: provenance.git?.sha ?? null,
+        grader_cost: summary.overall.cost?.graderTotal ?? null,
+        grader_model: provenance.graderModel,
+        mode: provenance.mode,
+        model_count: provenance.models.length,
+        models: provenance.models.map((m) => m.id),
+        promptfoo_url: provenance.promptfooUrl ?? null,
+        promptfoo_urls: provenance.promptfooUrls
+            ? JSON.stringify(provenance.promptfooUrls)
+            : null,
+        report_id: report.id,
+        source_base_url: provenance.source.baseUrl,
+        source_dataset: provenance.source.dataset ?? null,
+        source_name: provenance.source.name,
+        source_perspective: provenance.source.perspective ?? null,
+        tag: report.tag ?? null,
+        total_cost: summary.overall.cost?.total ?? null,
+        trigger_caller_repo: provenance.trigger.type === "cross-repo"
+            ? provenance.trigger.callerRepo
+            : null,
+        trigger_type: provenance.trigger.type,
+    };
+}

package/dist/sinks/format-slack.d.ts

@@ -0,0 +1,64 @@
+/**
+ * sinks/format-slack.ts
+ *
+ * Formats evaluation report data into Slack Block Kit structures for the
+ * SlackSink. Provides two message formats:
+ *
+ * - `formatRegressionAlert` — detailed regression notification with
+ *   per-area dimension breakdowns
+ * - `formatScoreSummary` — compact score overview for general reporting
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { Report } from "../pipeline/types.js";
+import type { DigestSummary } from "../schedules/digest.js";
+export interface SlackMessage {
+    blocks: SlackBlock[];
+    text: string;
+}
+interface SlackBlock {
+    elements?: Array<{
+        text: string;
+        type: "mrkdwn" | "plain_text";
+    }>;
+    fields?: Array<{
+        text: string;
+        type: "mrkdwn" | "plain_text";
+    }>;
+    text?: {
+        text: string;
+        type: "mrkdwn" | "plain_text";
+    };
+    type: "context" | "divider" | "header" | "section";
+}
+/**
+ * Format a regression alert for areas that have regressed.
+ *
+ * Produces a rich Slack message with:
+ * - Header with overall score change
+ * - Context metadata (mode, source, timestamp, promptfoo link)
+ * - Per-area regression details with dimension breakdowns
+ * - Brief mentions of improved and unchanged areas
+ */
+export declare function formatRegressionAlert(report: Report): SlackMessage;
+/**
+ * Format a general score summary for Slack reporting.
+ *
+ * Produces a compact overview with:
+ * - Overall score with grade emoji
+ * - Per-area score table
+ * - Cost summary (if available)
+ * - Promptfoo link (if available)
+ */
+export declare function formatScoreSummary(report: Report): SlackMessage;
+/**
+ * Format a weekly digest summary for Slack.
+ *
+ * Produces a summary message covering score trends over a time window:
+ * - Header with overall trend direction and score
+ * - Per-area trend table with arrows
+ * - Lists of improved, regressed, and stable areas
+ * - Report count and time window metadata
+ */
+export declare function formatWeeklyDigest(digest: DigestSummary): SlackMessage;
+export {};