@sanity/ailf 2.0.1 → 2.1.0

package/dist/sinks/format-slack.d.ts

@@ -1,64 +0,0 @@
-/**
- * sinks/format-slack.ts
- *
- * Formats evaluation report data into Slack Block Kit structures for the
- * SlackSink. Provides two message formats:
- *
- * - `formatRegressionAlert` — detailed regression notification with
- *   per-area dimension breakdowns
- * - `formatScoreSummary` — compact score overview for general reporting
- *
- * @see docs/design-docs/report-store/sink-architecture.md
- */
-import type { Report } from "../pipeline/types.js";
-import type { DigestSummary } from "../schedules/digest.js";
-export interface SlackMessage {
-    blocks: SlackBlock[];
-    text: string;
-}
-interface SlackBlock {
-    elements?: Array<{
-        text: string;
-        type: "mrkdwn" | "plain_text";
-    }>;
-    fields?: Array<{
-        text: string;
-        type: "mrkdwn" | "plain_text";
-    }>;
-    text?: {
-        text: string;
-        type: "mrkdwn" | "plain_text";
-    };
-    type: "context" | "divider" | "header" | "section";
-}
-/**
- * Format a regression alert for areas that have regressed.
- *
- * Produces a rich Slack message with:
- * - Header with overall score change
- * - Context metadata (mode, source, timestamp, promptfoo link)
- * - Per-area regression details with dimension breakdowns
- * - Brief mentions of improved and unchanged areas
- */
-export declare function formatRegressionAlert(report: Report): SlackMessage;
-/**
- * Format a general score summary for Slack reporting.
- *
- * Produces a compact overview with:
- * - Overall score with grade emoji
- * - Per-area score table
- * - Cost summary (if available)
- * - Promptfoo link (if available)
- */
-export declare function formatScoreSummary(report: Report): SlackMessage;
-/**
- * Format a weekly digest summary for Slack.
- *
- * Produces a summary message covering score trends over a time window:
- * - Header with overall trend direction and score
- * - Per-area trend table with arrows
- * - Lists of improved, regressed, and stable areas
- * - Report count and time window metadata
- */
-export declare function formatWeeklyDigest(digest: DigestSummary): SlackMessage;
-export {};

package/dist/sinks/format-slack.js

@@ -1,306 +0,0 @@
-/**
- * sinks/format-slack.ts
- *
- * Formats evaluation report data into Slack Block Kit structures for the
- * SlackSink. Provides two message formats:
- *
- * - `formatRegressionAlert` — detailed regression notification with
- *   per-area dimension breakdowns
- * - `formatScoreSummary` — compact score overview for general reporting
- *
- * @see docs/design-docs/report-store/sink-architecture.md
- */
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-/**
- * Format a regression alert for areas that have regressed.
- *
- * Produces a rich Slack message with:
- * - Header with overall score change
- * - Context metadata (mode, source, timestamp, promptfoo link)
- * - Per-area regression details with dimension breakdowns
- * - Brief mentions of improved and unchanged areas
- */
-export function formatRegressionAlert(report) {
-    const { comparison, provenance, summary } = report;
-    if (!comparison) {
-        return {
-            blocks: [
-                {
-                    text: { text: "⚠️ No comparison data available", type: "mrkdwn" },
-                    type: "section",
-                },
-            ],
-            text: "No comparison data available",
-        };
-    }
-    const baselineScore = Math.round(comparison.baseline.overall.avgScore);
-    const experimentScore = Math.round(comparison.experiment.overall.avgScore);
-    const delta = Math.round(comparison.deltas.overall);
-    const blocks = [];
-    // Header — emoji + title + overall score change
-    blocks.push({
-        text: {
-            text: `📉 *AI Literacy Score Regression*\n` +
-                `Overall: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,
-            type: "mrkdwn",
-        },
-        type: "section",
-    });
-    // Context — mode, source, timestamp, promptfoo link
-    const contextElements = [
-        { text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
-        { text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
-        { text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
-    ];
-    if (provenance.promptfooUrl) {
-        contextElements.push({
-            text: `<${provenance.promptfooUrl}|View in Promptfoo>`,
-            type: "mrkdwn",
-        });
-    }
-    blocks.push({ elements: contextElements, type: "context" });
-    // Divider
-    blocks.push({ type: "divider" });
-    // Regressed areas — detailed fields with dimension breakdowns
-    const regressedAreas = comparison.areas.filter((a) => a.change === "regressed");
-    if (regressedAreas.length > 0) {
-        const fields = regressedAreas.map((area) => ({
-            text: `*${area.area}:* ${Math.round(area.baseline)} → ` +
-                `${Math.round(area.experiment)} (${formatDelta(Math.round(area.delta))})\n` +
-                dimensionBreakdown(area.dimensions),
-            type: "mrkdwn",
-        }));
-        blocks.push({
-            fields,
-            text: {
-                text: `*Regressed Areas (${regressedAreas.length})*`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    // Improved areas — compact mention
-    if (comparison.improved.length > 0) {
-        blocks.push({
-            text: {
-                text: `📈 ${comparison.improved.length} area${comparison.improved.length === 1 ? "" : "s"} improved: ${comparison.improved.join(", ")}`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    // Unchanged areas — brief mention
-    if (comparison.unchanged.length > 0) {
-        blocks.push({
-            text: {
-                text: `➡️ ${comparison.unchanged.length} area${comparison.unchanged.length === 1 ? "" : "s"} unchanged`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    return {
-        blocks,
-        text: `📉 AI Literacy Score Regression: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,
-    };
-}
-/**
- * Format a general score summary for Slack reporting.
- *
- * Produces a compact overview with:
- * - Overall score with grade emoji
- * - Per-area score table
- * - Cost summary (if available)
- * - Promptfoo link (if available)
- */
-export function formatScoreSummary(report) {
-    const { provenance, summary } = report;
-    const overall = Math.round(summary.overall.avgScore);
-    const blocks = [];
-    // Header — overall score with emoji
-    blocks.push({
-        text: {
-            text: `${gradeEmoji(overall)} *AI Literacy Score: ${overall}*`,
-            type: "mrkdwn",
-        },
-        type: "section",
-    });
-    // Context — mode, source
-    const contextElements = [
-        { text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
-        { text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
-        { text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
-    ];
-    blocks.push({ elements: contextElements, type: "context" });
-    // Divider
-    blocks.push({ type: "divider" });
-    // Per-area score table as markdown
-    const rows = summary.scores
-        .map((s) => {
-        const emoji = gradeEmoji(s.totalScore);
-        return `${emoji} *${s.feature}*: ${Math.round(s.totalScore)}  _(T:${Math.round(s.taskCompletion)} · C:${Math.round(s.codeCorrectness)} · D:${Math.round(s.docCoverage)})_`;
-    })
-        .join("\n");
-    blocks.push({
-        text: { text: rows, type: "mrkdwn" },
-        type: "section",
-    });
-    // Cost summary
-    if (summary.overall.cost) {
-        blocks.push({
-            text: {
-                text: `💰 Total cost: $${summary.overall.cost.total.toFixed(2)} ($${summary.overall.cost.perTest.toFixed(3)}/test)`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    // Promptfoo link
-    if (provenance.promptfooUrl) {
-        blocks.push({
-            text: {
-                text: `🔗 <${provenance.promptfooUrl}|View full results in Promptfoo>`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    return {
-        blocks,
-        text: `${gradeEmoji(overall)} AI Literacy Score: ${overall}`,
-    };
-}
-/**
- * Format a weekly digest summary for Slack.
- *
- * Produces a summary message covering score trends over a time window:
- * - Header with overall trend direction and score
- * - Per-area trend table with arrows
- * - Lists of improved, regressed, and stable areas
- * - Report count and time window metadata
- */
-export function formatWeeklyDigest(digest) {
-    const trendEmoji = digest.overallTrend === "improving"
-        ? "📈"
-        : digest.overallTrend === "regressing"
-            ? "📉"
-            : "➡️";
-    const blocks = [];
-    // Header — overall trend
-    blocks.push({
-        text: {
-            text: `${trendEmoji} *Weekly AI Literacy Digest*\n` +
-                `Overall: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))} this week)`,
-            type: "mrkdwn",
-        },
-        type: "section",
-    });
-    // Context — time window and report count
-    blocks.push({
-        elements: [
-            {
-                text: `*Period:* ${readableDate(digest.lookbackStart)} – ${readableDate(digest.lookbackEnd)}`,
-                type: "mrkdwn",
-            },
-            {
-                text: `*Reports:* ${digest.reportCount}`,
-                type: "mrkdwn",
-            },
-        ],
-        type: "context",
-    });
-    blocks.push({ type: "divider" });
-    // Per-area trend table
-    if (digest.areaTrends.length > 0) {
-        const rows = digest.areaTrends
-            .map((t) => {
-            const arrow = t.trend === "improving" ? "↑" : t.trend === "regressing" ? "↓" : "→";
-            const emoji = gradeEmoji(t.lastScore);
-            return `${emoji} *${t.area}*: ${Math.round(t.lastScore)} ${arrow} (${formatDelta(Math.round(t.scoreDelta))})`;
-        })
-            .join("\n");
-        blocks.push({
-            text: { text: rows, type: "mrkdwn" },
-            type: "section",
-        });
-    }
-    // Summary badges
-    if (digest.improved.length > 0) {
-        blocks.push({
-            text: {
-                text: `📈 *Improved:* ${digest.improved.join(", ")}`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    if (digest.regressed.length > 0) {
-        blocks.push({
-            text: {
-                text: `📉 *Regressed:* ${digest.regressed.join(", ")}`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    if (digest.stable.length > 0) {
-        blocks.push({
-            text: {
-                text: `➡️ *Stable:* ${digest.stable.join(", ")}`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    // Cost summary if available
-    if (digest.totalCost !== undefined) {
-        blocks.push({
-            text: {
-                text: `💰 Total evaluation cost this week: ${digest.totalCost.toFixed(2)}`,
-                type: "mrkdwn",
-            },
-            type: "section",
-        });
-    }
-    return {
-        blocks,
-        text: `${trendEmoji} Weekly AI Literacy Digest: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))})`,
-    };
-}
-/** Build a dimension breakdown string for an area delta */
-function dimensionBreakdown(dimensions) {
-    return [
-        `Task: ${formatDelta(dimensions.taskCompletion.delta)}`,
-        `Code: ${formatDelta(dimensions.codeCorrectness.delta)}`,
-        `Docs: ${formatDelta(dimensions.docCoverage.delta)}`,
-    ].join(" · ");
-}
-/** Format a numeric delta with explicit sign: "+4", "-2", or "0" */
-function formatDelta(n) {
-    if (n > 0)
-        return `+${n}`;
-    if (n < 0)
-        return `${n}`;
-    return "0";
-}
-/** Score-tier emoji: ✅ (≥80), 🟡 (≥70), 🟠 (≥50), 🔴 (<50) */
-function gradeEmoji(score) {
-    if (score >= 80)
-        return "✅";
-    if (score >= 70)
-        return "🟡";
-    if (score >= 50)
-        return "🟠";
-    return "🔴";
-}
-/** Format an ISO timestamp into a readable date string */
-function readableDate(iso) {
-    const d = new Date(iso);
-    return d.toLocaleDateString("en-US", {
-        day: "numeric",
-        month: "short",
-        year: "numeric",
-    });
-}

package/dist/sinks/slack-sink.d.ts

@@ -1,27 +0,0 @@
-/**
- * sinks/slack-sink.ts
- *
- * Slack notification sink — posts formatted score change messages to
- * configured Slack channels via incoming webhooks.
- *
- * By default, only posts when regressions are detected (avoids notification
- * fatigue). Set `alwaysPost: true` to receive all reports.
- *
- * @see docs/design-docs/report-store/sink-architecture.md
- * @see docs/design-docs/report-store/notifications.md
- */
-import type { Report, SinkHealthStatus, SinkResult } from "../pipeline/types.js";
-import type { ReportSink } from "./types.js";
-export interface SlackSinkOptions {
-    /** Post all reports, not just regressions (default: false — only regressions) */
-    alwaysPost?: boolean;
-}
-export declare class SlackSink implements ReportSink {
-    private readonly webhookUrl;
-    private readonly channel?;
-    private readonly options;
-    readonly name = "slack";
-    constructor(webhookUrl: string, channel?: string | undefined, options?: SlackSinkOptions);
-    healthCheck(): Promise<SinkHealthStatus>;
-    publish(report: Report): Promise<SinkResult>;
-}

package/dist/sinks/slack-sink.js

@@ -1,78 +0,0 @@
-/**
- * sinks/slack-sink.ts
- *
- * Slack notification sink — posts formatted score change messages to
- * configured Slack channels via incoming webhooks.
- *
- * By default, only posts when regressions are detected (avoids notification
- * fatigue). Set `alwaysPost: true` to receive all reports.
- *
- * @see docs/design-docs/report-store/sink-architecture.md
- * @see docs/design-docs/report-store/notifications.md
- */
-import { formatRegressionAlert, formatScoreSummary, } from "./format-slack.js";
-export class SlackSink {
-    webhookUrl;
-    channel;
-    options;
-    name = "slack";
-    constructor(webhookUrl, channel, options = {}) {
-        this.webhookUrl = webhookUrl;
-        this.channel = channel;
-        this.options = options;
-    }
-    healthCheck() {
-        try {
-            const url = new URL(this.webhookUrl);
-            if (url.protocol !== "https:") {
-                return Promise.resolve({
-                    healthy: false,
-                    reason: `Webhook URL must use HTTPS, got ${url.protocol}`,
-                });
-            }
-            if (!this.webhookUrl.startsWith("https://hooks.slack.com/")) {
-                // Non-standard URL — warn but don't fail (could be a proxy)
-                return Promise.resolve({ healthy: true });
-            }
-            return Promise.resolve({ healthy: true });
-        }
-        catch {
-            return Promise.resolve({
-                healthy: false,
-                reason: `Invalid webhook URL: ${this.webhookUrl}`,
-            });
-        }
-    }
-    async publish(report) {
-        const hasRegressions = report.comparison !== undefined && report.comparison.regressed.length > 0;
-        let message;
-        if (hasRegressions) {
-            message = formatRegressionAlert(report);
-        }
-        else if (this.options.alwaysPost) {
-            message = formatScoreSummary(report);
-        }
-        else {
-            return { reason: "No regressions detected", status: "skipped" };
-        }
-        const body = { ...message };
-        if (this.channel) {
-            body.channel = this.channel;
-        }
-        const response = await fetch(this.webhookUrl, {
-            body: JSON.stringify(body),
-            headers: { "Content-Type": "application/json" },
-            method: "POST",
-        });
-        if (!response.ok) {
-            return {
-                error: `Slack webhook returned HTTP ${response.status}`,
-                status: "failed",
-            };
-        }
-        return {
-            detail: hasRegressions ? "regression alert" : "score summary",
-            status: "success",
-        };
-    }
-}

package/dist/sinks/webhook-sink.d.ts

@@ -1,19 +0,0 @@
-/**
- * sinks/webhook-sink.ts
- *
- * Generic HTTP webhook sink — POSTs the full report JSON to any endpoint.
- * This is the universal adapter for integrations that don't have a
- * dedicated sink (Airbyte, Zapier, custom services, etc.).
- *
- * @see docs/design-docs/report-store/sink-architecture.md
- */
-import type { Report, SinkHealthStatus, SinkResult } from "../pipeline/types.js";
-import type { ReportSink } from "./types.js";
-export declare class WebhookSink implements ReportSink {
-    private readonly url;
-    private readonly headers;
-    readonly name: string;
-    constructor(url: string, headers?: Record<string, string>);
-    healthCheck(): Promise<SinkHealthStatus>;
-    publish(report: Report): Promise<SinkResult>;
-}

package/dist/sinks/webhook-sink.js

@@ -1,50 +0,0 @@
-/**
- * sinks/webhook-sink.ts
- *
- * Generic HTTP webhook sink — POSTs the full report JSON to any endpoint.
- * This is the universal adapter for integrations that don't have a
- * dedicated sink (Airbyte, Zapier, custom services, etc.).
- *
- * @see docs/design-docs/report-store/sink-architecture.md
- */
-export class WebhookSink {
-    url;
-    headers;
-    name;
-    constructor(url, headers = {}) {
-        this.url = url;
-        this.headers = headers;
-        try {
-            this.name = `webhook:${new URL(url).hostname}`;
-        }
-        catch {
-            this.name = "webhook:invalid-url";
-        }
-    }
-    healthCheck() {
-        try {
-            new URL(this.url);
-            return Promise.resolve({ healthy: true });
-        }
-        catch {
-            return Promise.resolve({
-                healthy: false,
-                reason: `Invalid URL: ${this.url}`,
-            });
-        }
-    }
-    async publish(report) {
-        const response = await fetch(this.url, {
-            body: JSON.stringify(report),
-            headers: { "Content-Type": "application/json", ...this.headers },
-            method: "POST",
-        });
-        if (!response.ok) {
-            return {
-                error: `HTTP ${response.status}: ${response.statusText}`,
-                status: "failed",
-            };
-        }
-        return { detail: `HTTP ${response.status}`, status: "success" };
-    }
-}