@sanity/ailf 0.1.0

package/dist/sinks/slack/format.js

@@ -0,0 +1,327 @@
+/**
+ * sinks/slack/format.ts
+ *
+ * Formats evaluation report data into Slack Block Kit structures for the
+ * SlackSink. Provides three message formats:
+ *
+ * - `formatRegressionAlert` — detailed regression notification with
+ *   per-area dimension breakdowns
+ * - `formatScoreSummary` — compact score overview for general reporting
+ * - `formatWeeklyDigest` — weekly trend summary for scheduled digests
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Format a regression alert for areas that have regressed.
+ *
+ * Produces a rich Slack message with:
+ * - Header with overall score change
+ * - Context metadata (mode, source, timestamp, promptfoo link)
+ * - Per-area regression details with dimension breakdowns
+ * - Brief mentions of improved and unchanged areas
+ */
+export function formatRegressionAlert(report) {
+    const { comparison, provenance, summary } = report;
+    if (!comparison) {
+        return {
+            blocks: [
+                {
+                    text: { text: "⚠️ No comparison data available", type: "mrkdwn" },
+                    type: "section",
+                },
+            ],
+            text: "No comparison data available",
+        };
+    }
+    const baselineScore = Math.round(comparison.baseline.overall.avgScore);
+    const experimentScore = Math.round(comparison.experiment.overall.avgScore);
+    const delta = Math.round(comparison.deltas.overall);
+    const blocks = [];
+    // Header — emoji + title + overall score change
+    blocks.push({
+        text: {
+            text: `📉 *AI Literacy Score Regression*\n` +
+                `Overall: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,
+            type: "mrkdwn",
+        },
+        type: "section",
+    });
+    // Context — mode, source, timestamp, promptfoo link
+    const contextElements = [
+        { text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
+        { text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
+        { text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
+    ];
+    if (provenance.promptfooUrls && provenance.promptfooUrls.length > 0) {
+        for (const entry of provenance.promptfooUrls) {
+            contextElements.push({
+                text: `<${entry.url}|Promptfoo (${entry.mode})>`,
+                type: "mrkdwn",
+            });
+        }
+    }
+    else if (provenance.promptfooUrl) {
+        contextElements.push({
+            text: `<${provenance.promptfooUrl}|View in Promptfoo>`,
+            type: "mrkdwn",
+        });
+    }
+    blocks.push({ elements: contextElements, type: "context" });
+    // Divider
+    blocks.push({ type: "divider" });
+    // Regressed areas — detailed fields with dimension breakdowns
+    const regressedAreas = comparison.areas.filter((a) => a.change === "regressed");
+    if (regressedAreas.length > 0) {
+        const fields = regressedAreas.map((area) => ({
+            text: `*${area.area}:* ${Math.round(area.baseline)} → ` +
+                `${Math.round(area.experiment)} (${formatDelta(Math.round(area.delta))})\n` +
+                dimensionBreakdown(area.dimensions),
+            type: "mrkdwn",
+        }));
+        blocks.push({
+            fields,
+            text: {
+                text: `*Regressed Areas (${regressedAreas.length})*`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Improved areas — compact mention
+    if (comparison.improved.length > 0) {
+        blocks.push({
+            text: {
+                text: `📈 ${comparison.improved.length} area${comparison.improved.length === 1 ? "" : "s"} improved: ${comparison.improved.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Unchanged areas — brief mention
+    if (comparison.unchanged.length > 0) {
+        blocks.push({
+            text: {
+                text: `➡️ ${comparison.unchanged.length} area${comparison.unchanged.length === 1 ? "" : "s"} unchanged`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    return {
+        blocks,
+        text: `📉 AI Literacy Score Regression: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,
+    };
+}
+/**
+ * Format a general score summary for Slack reporting.
+ *
+ * Produces a compact overview with:
+ * - Overall score with grade emoji
+ * - Per-area score table
+ * - Cost summary (if available)
+ * - Promptfoo link (if available)
+ */
+export function formatScoreSummary(report) {
+    const { provenance, summary } = report;
+    const overall = Math.round(summary.overall.avgScore);
+    const blocks = [];
+    // Header — overall score with emoji
+    blocks.push({
+        text: {
+            text: `${gradeEmoji(overall)} *AI Literacy Score: ${overall}*`,
+            type: "mrkdwn",
+        },
+        type: "section",
+    });
+    // Context — mode, source
+    const contextElements = [
+        { text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
+        { text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
+        { text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
+    ];
+    blocks.push({ elements: contextElements, type: "context" });
+    // Divider
+    blocks.push({ type: "divider" });
+    // Per-area score table as markdown
+    const rows = summary.scores
+        .map((s) => {
+        const emoji = gradeEmoji(s.totalScore);
+        return `${emoji} *${s.feature}*: ${Math.round(s.totalScore)}  _(T:${Math.round(s.taskCompletion)} · C:${Math.round(s.codeCorrectness)} · D:${Math.round(s.docCoverage)})_`;
+    })
+        .join("\n");
+    blocks.push({
+        text: { text: rows, type: "mrkdwn" },
+        type: "section",
+    });
+    // Cost summary
+    if (summary.overall.cost) {
+        blocks.push({
+            text: {
+                text: `💰 Total cost: $${summary.overall.cost.total.toFixed(2)} ($${summary.overall.cost.perTest.toFixed(3)}/test)`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Promptfoo links
+    if (provenance.promptfooUrls && provenance.promptfooUrls.length > 0) {
+        const links = provenance.promptfooUrls
+            .map((e) => `<${e.url}|${e.mode}>`)
+            .join(" · ");
+        blocks.push({
+            text: {
+                text: `🔗 View results in Promptfoo: ${links}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    else if (provenance.promptfooUrl) {
+        blocks.push({
+            text: {
+                text: `🔗 <${provenance.promptfooUrl}|View full results in Promptfoo>`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    return {
+        blocks,
+        text: `${gradeEmoji(overall)} AI Literacy Score: ${overall}`,
+    };
+}
+/**
+ * Format a weekly digest summary for Slack.
+ *
+ * Produces a summary message covering score trends over a time window:
+ * - Header with overall trend direction and score
+ * - Per-area trend table with arrows
+ * - Lists of improved, regressed, and stable areas
+ * - Report count and time window metadata
+ */
+export function formatWeeklyDigest(digest) {
+    const trendEmoji = digest.overallTrend === "improving"
+        ? "📈"
+        : digest.overallTrend === "regressing"
+            ? "📉"
+            : "➡️";
+    const blocks = [];
+    // Header — overall trend
+    blocks.push({
+        text: {
+            text: `${trendEmoji} *Weekly AI Literacy Digest*\n` +
+                `Overall: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))} this week)`,
+            type: "mrkdwn",
+        },
+        type: "section",
+    });
+    // Context — time window and report count
+    blocks.push({
+        elements: [
+            {
+                text: `*Period:* ${readableDate(digest.lookbackStart)} – ${readableDate(digest.lookbackEnd)}`,
+                type: "mrkdwn",
+            },
+            {
+                text: `*Reports:* ${digest.reportCount}`,
+                type: "mrkdwn",
+            },
+        ],
+        type: "context",
+    });
+    blocks.push({ type: "divider" });
+    // Per-area trend table
+    if (digest.areaTrends.length > 0) {
+        const rows = digest.areaTrends
+            .map((t) => {
+            const arrow = t.trend === "improving" ? "↑" : t.trend === "regressing" ? "↓" : "→";
+            const emoji = gradeEmoji(t.lastScore);
+            return `${emoji} *${t.area}*: ${Math.round(t.lastScore)} ${arrow} (${formatDelta(Math.round(t.scoreDelta))})`;
+        })
+            .join("\n");
+        blocks.push({
+            text: { text: rows, type: "mrkdwn" },
+            type: "section",
+        });
+    }
+    // Summary badges
+    if (digest.improved.length > 0) {
+        blocks.push({
+            text: {
+                text: `📈 *Improved:* ${digest.improved.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    if (digest.regressed.length > 0) {
+        blocks.push({
+            text: {
+                text: `📉 *Regressed:* ${digest.regressed.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    if (digest.stable.length > 0) {
+        blocks.push({
+            text: {
+                text: `➡️ *Stable:* ${digest.stable.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Cost summary if available
+    if (digest.totalCost !== undefined) {
+        blocks.push({
+            text: {
+                text: `💰 Total evaluation cost this week: ${digest.totalCost.toFixed(2)}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    return {
+        blocks,
+        text: `${trendEmoji} Weekly AI Literacy Digest: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))})`,
+    };
+}
+/** Build a dimension breakdown string for an area delta */
+function dimensionBreakdown(dimensions) {
+    return [
+        `Task: ${formatDelta(dimensions.taskCompletion.delta)}`,
+        `Code: ${formatDelta(dimensions.codeCorrectness.delta)}`,
+        `Docs: ${formatDelta(dimensions.docCoverage.delta)}`,
+    ].join(" · ");
+}
+/** Format a numeric delta with explicit sign: "+4", "-2", or "0" */
+function formatDelta(n) {
+    if (n > 0)
+        return `+${n}`;
+    if (n < 0)
+        return `${n}`;
+    return "0";
+}
+/** Score-tier emoji: ✅ (≥80), 🟡 (≥70), 🟠 (≥50), 🔴 (<50) */
+function gradeEmoji(score) {
+    if (score >= 80)
+        return "✅";
+    if (score >= 70)
+        return "🟡";
+    if (score >= 50)
+        return "🟠";
+    return "🔴";
+}
+/** Format an ISO timestamp into a readable date string */
+function readableDate(iso) {
+    const d = new Date(iso);
+    return d.toLocaleDateString("en-US", {
+        day: "numeric",
+        month: "short",
+        year: "numeric",
+    });
+}

package/dist/sinks/slack/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * sinks/slack/index.ts
+ *
+ * Slack notification sink — posts formatted score change messages to
+ * configured Slack channels via incoming webhooks.
+ *
+ * By default, only posts when regressions are detected (avoids notification
+ * fatigue). Set `alwaysPost: true` to receive all reports.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ * @see docs/design-docs/report-store/notifications.md
+ */
+import type { Report, SinkHealthStatus, SinkResult } from "../../pipeline/types.js";
+import type { ReportSink } from "../types.js";
+export interface SlackSinkOptions {
+    /** Post all reports, not just regressions (default: false — only regressions) */
+    alwaysPost?: boolean;
+}
+export declare class SlackSink implements ReportSink {
+    private readonly webhookUrl;
+    private readonly channel?;
+    private readonly options;
+    readonly name = "slack";
+    constructor(webhookUrl: string, channel?: string | undefined, options?: SlackSinkOptions);
+    healthCheck(): Promise<SinkHealthStatus>;
+    publish(report: Report): Promise<SinkResult>;
+}

package/dist/sinks/slack/index.js

@@ -0,0 +1,78 @@
+/**
+ * sinks/slack/index.ts
+ *
+ * Slack notification sink — posts formatted score change messages to
+ * configured Slack channels via incoming webhooks.
+ *
+ * By default, only posts when regressions are detected (avoids notification
+ * fatigue). Set `alwaysPost: true` to receive all reports.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ * @see docs/design-docs/report-store/notifications.md
+ */
+import { formatRegressionAlert, formatScoreSummary, } from "./format.js";
+export class SlackSink {
+    webhookUrl;
+    channel;
+    options;
+    name = "slack";
+    constructor(webhookUrl, channel, options = {}) {
+        this.webhookUrl = webhookUrl;
+        this.channel = channel;
+        this.options = options;
+    }
+    healthCheck() {
+        try {
+            const url = new URL(this.webhookUrl);
+            if (url.protocol !== "https:") {
+                return Promise.resolve({
+                    healthy: false,
+                    reason: `Webhook URL must use HTTPS, got ${url.protocol}`,
+                });
+            }
+            if (!this.webhookUrl.startsWith("https://hooks.slack.com/")) {
+                // Non-standard URL — warn but don't fail (could be a proxy)
+                return Promise.resolve({ healthy: true });
+            }
+            return Promise.resolve({ healthy: true });
+        }
+        catch {
+            return Promise.resolve({
+                healthy: false,
+                reason: `Invalid webhook URL: ${this.webhookUrl}`,
+            });
+        }
+    }
+    async publish(report) {
+        const hasRegressions = report.comparison !== undefined && report.comparison.regressed.length > 0;
+        let message;
+        if (hasRegressions) {
+            message = formatRegressionAlert(report);
+        }
+        else if (this.options.alwaysPost) {
+            message = formatScoreSummary(report);
+        }
+        else {
+            return { reason: "No regressions detected", status: "skipped" };
+        }
+        const body = { ...message };
+        if (this.channel) {
+            body.channel = this.channel;
+        }
+        const response = await fetch(this.webhookUrl, {
+            body: JSON.stringify(body),
+            headers: { "Content-Type": "application/json" },
+            method: "POST",
+        });
+        if (!response.ok) {
+            return {
+                error: `Slack webhook returned HTTP ${response.status}`,
+                status: "failed",
+            };
+        }
+        return {
+            detail: hasRegressions ? "regression alert" : "score summary",
+            status: "success",
+        };
+    }
+}

package/dist/sinks/slack-sink.d.ts

@@ -0,0 +1,27 @@
+/**
+ * sinks/slack-sink.ts
+ *
+ * Slack notification sink — posts formatted score change messages to
+ * configured Slack channels via incoming webhooks.
+ *
+ * By default, only posts when regressions are detected (avoids notification
+ * fatigue). Set `alwaysPost: true` to receive all reports.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ * @see docs/design-docs/report-store/notifications.md
+ */
+import type { Report, SinkHealthStatus, SinkResult } from "../pipeline/types.js";
+import type { ReportSink } from "./types.js";
+export interface SlackSinkOptions {
+    /** Post all reports, not just regressions (default: false — only regressions) */
+    alwaysPost?: boolean;
+}
+export declare class SlackSink implements ReportSink {
+    private readonly webhookUrl;
+    private readonly channel?;
+    private readonly options;
+    readonly name = "slack";
+    constructor(webhookUrl: string, channel?: string | undefined, options?: SlackSinkOptions);
+    healthCheck(): Promise<SinkHealthStatus>;
+    publish(report: Report): Promise<SinkResult>;
+}

package/dist/sinks/slack-sink.js

@@ -0,0 +1,78 @@
+/**
+ * sinks/slack-sink.ts
+ *
+ * Slack notification sink — posts formatted score change messages to
+ * configured Slack channels via incoming webhooks.
+ *
+ * By default, only posts when regressions are detected (avoids notification
+ * fatigue). Set `alwaysPost: true` to receive all reports.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ * @see docs/design-docs/report-store/notifications.md
+ */
+import { formatRegressionAlert, formatScoreSummary, } from "./format-slack.js";
+export class SlackSink {
+    webhookUrl;
+    channel;
+    options;
+    name = "slack";
+    constructor(webhookUrl, channel, options = {}) {
+        this.webhookUrl = webhookUrl;
+        this.channel = channel;
+        this.options = options;
+    }
+    healthCheck() {
+        try {
+            const url = new URL(this.webhookUrl);
+            if (url.protocol !== "https:") {
+                return Promise.resolve({
+                    healthy: false,
+                    reason: `Webhook URL must use HTTPS, got ${url.protocol}`,
+                });
+            }
+            if (!this.webhookUrl.startsWith("https://hooks.slack.com/")) {
+                // Non-standard URL — warn but don't fail (could be a proxy)
+                return Promise.resolve({ healthy: true });
+            }
+            return Promise.resolve({ healthy: true });
+        }
+        catch {
+            return Promise.resolve({
+                healthy: false,
+                reason: `Invalid webhook URL: ${this.webhookUrl}`,
+            });
+        }
+    }
+    async publish(report) {
+        const hasRegressions = report.comparison !== undefined && report.comparison.regressed.length > 0;
+        let message;
+        if (hasRegressions) {
+            message = formatRegressionAlert(report);
+        }
+        else if (this.options.alwaysPost) {
+            message = formatScoreSummary(report);
+        }
+        else {
+            return { reason: "No regressions detected", status: "skipped" };
+        }
+        const body = { ...message };
+        if (this.channel) {
+            body.channel = this.channel;
+        }
+        const response = await fetch(this.webhookUrl, {
+            body: JSON.stringify(body),
+            headers: { "Content-Type": "application/json" },
+            method: "POST",
+        });
+        if (!response.ok) {
+            return {
+                error: `Slack webhook returned HTTP ${response.status}`,
+                status: "failed",
+            };
+        }
+        return {
+            detail: hasRegressions ? "regression alert" : "score summary",
+            status: "success",
+        };
+    }
+}

package/dist/sinks/types.d.ts

@@ -0,0 +1,59 @@
+/**
+ * sinks/types.ts
+ *
+ * The ReportSink interface — the contract for all report delivery destinations.
+ *
+ * Sinks receive published reports and deliver them to external systems
+ * (BigQuery, Slack, GitHub, webhooks, etc.). Each sink is self-contained:
+ * it handles its own connection, schema mapping, and error handling.
+ *
+ * Design principle P6: Sinks are fire-and-forget. A sink failure is logged
+ * but never blocks the pipeline, the Sanity write, or other sinks.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { Report, SinkHealthStatus, SinkPayload, SinkResult } from "../pipeline/types.js";
+import type { SinkRouting } from "./schema.js";
+/**
+ * A ReportSink receives published reports and delivers them to an
+ * external system.
+ *
+ * Retries are handled by the sink runner (3 retries, exponential backoff)
+ * — implementations should not retry internally.
+ */
+export interface ReportSink {
+    /**
+     * Optional health check — called at pipeline start to surface
+     * configuration errors early (missing credentials, unreachable endpoint).
+     */
+    healthCheck?(): Promise<SinkHealthStatus>;
+    /** Human-readable name for logging and diagnostics */
+    readonly name: string;
+    /**
+     * Deliver a report to this sink's destination.
+     *
+     * Called after the report is written to the Sanity Content Lake
+     * (system of record). The Sanity document is the source of truth;
+     * sinks receive a copy.
+     *
+     * Implementations may accept either a plain Report (backward compatible)
+     * or a SinkPayload (includes threshold violations and severity context).
+     *
+     * @returns Result indicating success, skip, or failure.
+     */
+    publish(report: Report | SinkPayload): Promise<SinkResult>;
+    /** Optional routing rules from config/sinks.yaml */
+    readonly routing?: SinkRouting;
+}
+/**
+ * Check whether a sink should receive a delivery based on its routing
+ * rules and the payload's severity/comparison data.
+ *
+ * Returns true (deliver) when:
+ * - The sink has no routing config (backward compatible — receives everything)
+ * - The payload's maxSeverity matches an enabled routing rule
+ * - The payload has regressions and the sink's `regression` rule is enabled
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ */
+export declare function shouldDeliver(routing: SinkRouting | undefined, payload: SinkPayload): boolean;

package/dist/sinks/types.js

@@ -0,0 +1,44 @@
+/**
+ * sinks/types.ts
+ *
+ * The ReportSink interface — the contract for all report delivery destinations.
+ *
+ * Sinks receive published reports and deliver them to external systems
+ * (BigQuery, Slack, GitHub, webhooks, etc.). Each sink is self-contained:
+ * it handles its own connection, schema mapping, and error handling.
+ *
+ * Design principle P6: Sinks are fire-and-forget. A sink failure is logged
+ * but never blocks the pipeline, the Sanity write, or other sinks.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+/**
+ * Check whether a sink should receive a delivery based on its routing
+ * rules and the payload's severity/comparison data.
+ *
+ * Returns true (deliver) when:
+ * - The sink has no routing config (backward compatible — receives everything)
+ * - The payload's maxSeverity matches an enabled routing rule
+ * - The payload has regressions and the sink's `regression` rule is enabled
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ */
+export function shouldDeliver(routing, payload) {
+    // No routing config = deliver everything (backward compatible)
+    if (!routing)
+        return true;
+    // Check severity-based routing
+    if (payload.maxSeverity !== "none") {
+        const severityRule = routing[payload.maxSeverity];
+        if (severityRule)
+            return true;
+    }
+    // Check regression routing
+    if (routing.regression &&
+        payload.comparison &&
+        payload.comparison.regressed.length > 0) {
+        return true;
+    }
+    // No matching rule — skip
+    return false;
+}

package/dist/sinks/webhook/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * sinks/webhook/index.ts
+ *
+ * Generic HTTP webhook sink — POSTs the full report JSON to any endpoint.
+ * This is the universal adapter for integrations that don't have a
+ * dedicated sink (Airbyte, Zapier, custom services, etc.).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { Report, SinkHealthStatus, SinkResult } from "../../pipeline/types.js";
+import type { ReportSink } from "../types.js";
+export declare class WebhookSink implements ReportSink {
+    private readonly url;
+    private readonly headers;
+    readonly name: string;
+    constructor(url: string, headers?: Record<string, string>);
+    healthCheck(): Promise<SinkHealthStatus>;
+    publish(report: Report): Promise<SinkResult>;
+}