@sanity/ailf 0.1.0

package/dist/sinks/format-slack.js

@@ -0,0 +1,306 @@
+/**
+ * sinks/format-slack.ts
+ *
+ * Formats evaluation report data into Slack Block Kit structures for the
+ * SlackSink. Provides two message formats:
+ *
+ * - `formatRegressionAlert` — detailed regression notification with
+ *   per-area dimension breakdowns
+ * - `formatScoreSummary` — compact score overview for general reporting
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Format a regression alert for areas that have regressed.
+ *
+ * Produces a rich Slack message with:
+ * - Header with overall score change
+ * - Context metadata (mode, source, timestamp, promptfoo link)
+ * - Per-area regression details with dimension breakdowns
+ * - Brief mentions of improved and unchanged areas
+ */
+export function formatRegressionAlert(report) {
+    const { comparison, provenance, summary } = report;
+    if (!comparison) {
+        return {
+            blocks: [
+                {
+                    text: { text: "⚠️ No comparison data available", type: "mrkdwn" },
+                    type: "section",
+                },
+            ],
+            text: "No comparison data available",
+        };
+    }
+    const baselineScore = Math.round(comparison.baseline.overall.avgScore);
+    const experimentScore = Math.round(comparison.experiment.overall.avgScore);
+    const delta = Math.round(comparison.deltas.overall);
+    const blocks = [];
+    // Header — emoji + title + overall score change
+    blocks.push({
+        text: {
+            text: `📉 *AI Literacy Score Regression*\n` +
+                `Overall: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,
+            type: "mrkdwn",
+        },
+        type: "section",
+    });
+    // Context — mode, source, timestamp, promptfoo link
+    const contextElements = [
+        { text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
+        { text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
+        { text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
+    ];
+    if (provenance.promptfooUrl) {
+        contextElements.push({
+            text: `<${provenance.promptfooUrl}|View in Promptfoo>`,
+            type: "mrkdwn",
+        });
+    }
+    blocks.push({ elements: contextElements, type: "context" });
+    // Divider
+    blocks.push({ type: "divider" });
+    // Regressed areas — detailed fields with dimension breakdowns
+    const regressedAreas = comparison.areas.filter((a) => a.change === "regressed");
+    if (regressedAreas.length > 0) {
+        const fields = regressedAreas.map((area) => ({
+            text: `*${area.area}:* ${Math.round(area.baseline)} → ` +
+                `${Math.round(area.experiment)} (${formatDelta(Math.round(area.delta))})\n` +
+                dimensionBreakdown(area.dimensions),
+            type: "mrkdwn",
+        }));
+        blocks.push({
+            fields,
+            text: {
+                text: `*Regressed Areas (${regressedAreas.length})*`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Improved areas — compact mention
+    if (comparison.improved.length > 0) {
+        blocks.push({
+            text: {
+                text: `📈 ${comparison.improved.length} area${comparison.improved.length === 1 ? "" : "s"} improved: ${comparison.improved.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Unchanged areas — brief mention
+    if (comparison.unchanged.length > 0) {
+        blocks.push({
+            text: {
+                text: `➡️ ${comparison.unchanged.length} area${comparison.unchanged.length === 1 ? "" : "s"} unchanged`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    return {
+        blocks,
+        text: `📉 AI Literacy Score Regression: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,
+    };
+}
+/**
+ * Format a general score summary for Slack reporting.
+ *
+ * Produces a compact overview with:
+ * - Overall score with grade emoji
+ * - Per-area score table
+ * - Cost summary (if available)
+ * - Promptfoo link (if available)
+ */
+export function formatScoreSummary(report) {
+    const { provenance, summary } = report;
+    const overall = Math.round(summary.overall.avgScore);
+    const blocks = [];
+    // Header — overall score with emoji
+    blocks.push({
+        text: {
+            text: `${gradeEmoji(overall)} *AI Literacy Score: ${overall}*`,
+            type: "mrkdwn",
+        },
+        type: "section",
+    });
+    // Context — mode, source
+    const contextElements = [
+        { text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
+        { text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
+        { text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
+    ];
+    blocks.push({ elements: contextElements, type: "context" });
+    // Divider
+    blocks.push({ type: "divider" });
+    // Per-area score table as markdown
+    const rows = summary.scores
+        .map((s) => {
+        const emoji = gradeEmoji(s.totalScore);
+        return `${emoji} *${s.feature}*: ${Math.round(s.totalScore)}  _(T:${Math.round(s.taskCompletion)} · C:${Math.round(s.codeCorrectness)} · D:${Math.round(s.docCoverage)})_`;
+    })
+        .join("\n");
+    blocks.push({
+        text: { text: rows, type: "mrkdwn" },
+        type: "section",
+    });
+    // Cost summary
+    if (summary.overall.cost) {
+        blocks.push({
+            text: {
+                text: `💰 Total cost: $${summary.overall.cost.total.toFixed(2)} ($${summary.overall.cost.perTest.toFixed(3)}/test)`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Promptfoo link
+    if (provenance.promptfooUrl) {
+        blocks.push({
+            text: {
+                text: `🔗 <${provenance.promptfooUrl}|View full results in Promptfoo>`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    return {
+        blocks,
+        text: `${gradeEmoji(overall)} AI Literacy Score: ${overall}`,
+    };
+}
+/**
+ * Format a weekly digest summary for Slack.
+ *
+ * Produces a summary message covering score trends over a time window:
+ * - Header with overall trend direction and score
+ * - Per-area trend table with arrows
+ * - Lists of improved, regressed, and stable areas
+ * - Report count and time window metadata
+ */
+export function formatWeeklyDigest(digest) {
+    const trendEmoji = digest.overallTrend === "improving"
+        ? "📈"
+        : digest.overallTrend === "regressing"
+            ? "📉"
+            : "➡️";
+    const blocks = [];
+    // Header — overall trend
+    blocks.push({
+        text: {
+            text: `${trendEmoji} *Weekly AI Literacy Digest*\n` +
+                `Overall: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))} this week)`,
+            type: "mrkdwn",
+        },
+        type: "section",
+    });
+    // Context — time window and report count
+    blocks.push({
+        elements: [
+            {
+                text: `*Period:* ${readableDate(digest.lookbackStart)} – ${readableDate(digest.lookbackEnd)}`,
+                type: "mrkdwn",
+            },
+            {
+                text: `*Reports:* ${digest.reportCount}`,
+                type: "mrkdwn",
+            },
+        ],
+        type: "context",
+    });
+    blocks.push({ type: "divider" });
+    // Per-area trend table
+    if (digest.areaTrends.length > 0) {
+        const rows = digest.areaTrends
+            .map((t) => {
+            const arrow = t.trend === "improving" ? "↑" : t.trend === "regressing" ? "↓" : "→";
+            const emoji = gradeEmoji(t.lastScore);
+            return `${emoji} *${t.area}*: ${Math.round(t.lastScore)} ${arrow} (${formatDelta(Math.round(t.scoreDelta))})`;
+        })
+            .join("\n");
+        blocks.push({
+            text: { text: rows, type: "mrkdwn" },
+            type: "section",
+        });
+    }
+    // Summary badges
+    if (digest.improved.length > 0) {
+        blocks.push({
+            text: {
+                text: `📈 *Improved:* ${digest.improved.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    if (digest.regressed.length > 0) {
+        blocks.push({
+            text: {
+                text: `📉 *Regressed:* ${digest.regressed.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    if (digest.stable.length > 0) {
+        blocks.push({
+            text: {
+                text: `➡️ *Stable:* ${digest.stable.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    // Cost summary if available
+    if (digest.totalCost !== undefined) {
+        blocks.push({
+            text: {
+                text: `💰 Total evaluation cost this week: ${digest.totalCost.toFixed(2)}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
+    return {
+        blocks,
+        text: `${trendEmoji} Weekly AI Literacy Digest: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))})`,
+    };
+}
+/** Build a dimension breakdown string for an area delta */
+function dimensionBreakdown(dimensions) {
+    return [
+        `Task: ${formatDelta(dimensions.taskCompletion.delta)}`,
+        `Code: ${formatDelta(dimensions.codeCorrectness.delta)}`,
+        `Docs: ${formatDelta(dimensions.docCoverage.delta)}`,
+    ].join(" · ");
+}
+/** Format a numeric delta with explicit sign: "+4", "-2", or "0" */
+function formatDelta(n) {
+    if (n > 0)
+        return `+${n}`;
+    if (n < 0)
+        return `${n}`;
+    return "0";
+}
+/** Score-tier emoji: ✅ (≥80), 🟡 (≥70), 🟠 (≥50), 🔴 (<50) */
+function gradeEmoji(score) {
+    if (score >= 80)
+        return "✅";
+    if (score >= 70)
+        return "🟡";
+    if (score >= 50)
+        return "🟠";
+    return "🔴";
+}
+/** Format an ISO timestamp into a readable date string */
+function readableDate(iso) {
+    const d = new Date(iso);
+    return d.toLocaleDateString("en-US", {
+        day: "numeric",
+        month: "short",
+        year: "numeric",
+    });
+}

package/dist/sinks/index.d.ts

@@ -0,0 +1,23 @@
+/**
+ * sinks/index.ts
+ *
+ * Barrel exports for the report sink module.
+ *
+ * Framework types and utilities live at this level. Concrete sink
+ * implementations live in subdirectories (bigquery/, slack/, webhook/).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+export { BigQuerySink } from "./bigquery/index.js";
+export type { AreaScoreRow, BigQuerySinkOptions, ReportRow, } from "./bigquery/index.js";
+export { flattenAreaScoreRows, flattenReportRow } from "./bigquery/index.js";
+export { loadSinks } from "./loader.js";
+export { withRetry } from "./retry.js";
+export type { SinkRouting } from "./schema.js";
+export { formatRegressionAlert, formatScoreSummary, formatWeeklyDigest, } from "./slack/format.js";
+export type { SlackMessage } from "./slack/format.js";
+export { SlackSink } from "./slack/index.js";
+export type { SlackSinkOptions } from "./slack/index.js";
+export type { ReportSink } from "./types.js";
+export { shouldDeliver } from "./types.js";
+export { WebhookSink } from "./webhook/index.js";

package/dist/sinks/index.js

@@ -0,0 +1,18 @@
+/**
+ * sinks/index.ts
+ *
+ * Barrel exports for the report sink module.
+ *
+ * Framework types and utilities live at this level. Concrete sink
+ * implementations live in subdirectories (bigquery/, slack/, webhook/).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+export { BigQuerySink } from "./bigquery/index.js";
+export { flattenAreaScoreRows, flattenReportRow } from "./bigquery/index.js";
+export { loadSinks } from "./loader.js";
+export { withRetry } from "./retry.js";
+export { formatRegressionAlert, formatScoreSummary, formatWeeklyDigest, } from "./slack/format.js";
+export { SlackSink } from "./slack/index.js";
+export { shouldDeliver } from "./types.js";
+export { WebhookSink } from "./webhook/index.js";

package/dist/sinks/loader.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Sink loader — reads config/sinks.yaml, resolves env vars, validates,
+ * and instantiates sink objects.
+ *
+ * Fire-and-forget: parse failures log a warning and return an empty array
+ * so the pipeline never crashes due to sink misconfiguration.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { ReportSink } from "./types.js";
+/**
+ * Load and instantiate all enabled sinks from `config/sinks.yaml`.
+ *
+ * Returns an empty array when:
+ * - The config file doesn't exist (no sinks configured)
+ * - The config file fails to parse or validate
+ */
+export declare function loadSinks(): ReportSink[];

package/dist/sinks/loader.js

@@ -0,0 +1,82 @@
+/**
+ * Sink loader — reads config/sinks.yaml, resolves env vars, validates,
+ * and instantiates sink objects.
+ *
+ * Fire-and-forget: parse failures log a warning and return an empty array
+ * so the pipeline never crashes due to sink misconfiguration.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { existsSync, readFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { load } from "js-yaml";
+import { interpolate } from "../interpolate.js";
+import { BigQuerySink } from "./bigquery/index.js";
+import { SinksFileSchema } from "./schema.js";
+import { SlackSink } from "./slack/index.js";
+import { WebhookSink } from "./webhook/index.js";
+// ---------------------------------------------------------------------------
+// Paths
+// ---------------------------------------------------------------------------
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SINKS_CONFIG_PATH = resolve(__dirname, "..", "..", "config", "sinks.yaml");
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load and instantiate all enabled sinks from `config/sinks.yaml`.
+ *
+ * Returns an empty array when:
+ * - The config file doesn't exist (no sinks configured)
+ * - The config file fails to parse or validate
+ */
+export function loadSinks() {
+    if (!existsSync(SINKS_CONFIG_PATH)) {
+        return [];
+    }
+    try {
+        const raw = readFileSync(SINKS_CONFIG_PATH, "utf-8");
+        const parsed = load(raw);
+        const interpolated = interpolate(parsed);
+        // When all sinks are commented out, js-yaml returns { sinks: null }.
+        // Normalize null → undefined so Zod's .default([]) can apply.
+        const normalized = interpolated && typeof interpolated === "object"
+            ? Object.fromEntries(Object.entries(interpolated).map(([k, v]) => [k, v === null ? undefined : v]))
+            : interpolated;
+        const validated = SinksFileSchema.parse(normalized);
+        return validated.sinks
+            .filter((s) => s.enabled)
+            .map((s) => createSink(s))
+            .filter((sink) => sink !== null);
+    }
+    catch (error) {
+        console.warn("Failed to load sinks config — continuing without sinks:", error instanceof Error ? error.message : error);
+        return [];
+    }
+}
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+function createSink(config) {
+    switch (config.type) {
+        case "bigquery":
+            return new BigQuerySink({
+                credentials: config.credentials,
+                dataset: config.dataset,
+                project: config.project,
+            });
+        case "github-comment":
+            console.log("  ℹ️  GitHubCommentSink not yet implemented — skipping");
+            return null;
+        case "slack":
+            return new SlackSink(config.webhookUrl, config.channel);
+        case "webhook":
+            return new WebhookSink(config.url, config.headers ?? {});
+        default: {
+            const _exhaustive = config;
+            console.warn(`  ⚠️  Unknown sink type: ${_exhaustive.type}`);
+            return null;
+        }
+    }
+}

package/dist/sinks/retry.d.ts

@@ -0,0 +1,24 @@
+/**
+ * sinks/retry.ts
+ *
+ * Simple retry utility with exponential backoff for sink operations.
+ * Used by the sink runner to retry transient failures (network timeouts,
+ * HTTP 429/5xx) without requiring sinks to implement retry logic internally.
+ *
+ * Policy: 3 retries, exponential backoff (1s → 2s → 4s).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { SinkResult } from "../pipeline/types.js";
+/**
+ * Retry a sink publish operation with exponential backoff.
+ *
+ * Only retries on "failed" results — "success" and "skipped" are returned
+ * immediately. Thrown exceptions are caught and converted to "failed" results.
+ *
+ * @param fn - The publish function to retry
+ * @param maxRetries - Maximum number of retry attempts (default: 3)
+ * @param baseDelayMs - Initial delay in milliseconds (default: 1000)
+ * @returns The final SinkResult after all attempts
+ */
+export declare function withRetry(fn: () => Promise<SinkResult>, maxRetries?: number, baseDelayMs?: number): Promise<SinkResult>;

package/dist/sinks/retry.js

@@ -0,0 +1,52 @@
+/**
+ * sinks/retry.ts
+ *
+ * Simple retry utility with exponential backoff for sink operations.
+ * Used by the sink runner to retry transient failures (network timeouts,
+ * HTTP 429/5xx) without requiring sinks to implement retry logic internally.
+ *
+ * Policy: 3 retries, exponential backoff (1s → 2s → 4s).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+const DEFAULT_MAX_RETRIES = 3;
+const DEFAULT_BASE_DELAY_MS = 1000;
+/**
+ * Retry a sink publish operation with exponential backoff.
+ *
+ * Only retries on "failed" results — "success" and "skipped" are returned
+ * immediately. Thrown exceptions are caught and converted to "failed" results.
+ *
+ * @param fn - The publish function to retry
+ * @param maxRetries - Maximum number of retry attempts (default: 3)
+ * @param baseDelayMs - Initial delay in milliseconds (default: 1000)
+ * @returns The final SinkResult after all attempts
+ */
+export async function withRetry(fn, maxRetries = DEFAULT_MAX_RETRIES, baseDelayMs = DEFAULT_BASE_DELAY_MS) {
+    let lastResult = { error: "No attempts made", status: "failed" };
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            const result = await fn();
+            // Success and skipped are terminal — no retry
+            if (result.status !== "failed") {
+                return result;
+            }
+            lastResult = result;
+        }
+        catch (error) {
+            lastResult = {
+                error: error instanceof Error ? error.message : String(error),
+                status: "failed",
+            };
+        }
+        // Don't delay after the last attempt
+        if (attempt < maxRetries) {
+            const delayMs = baseDelayMs * Math.pow(2, attempt);
+            await sleep(delayMs);
+        }
+    }
+    return lastResult;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/sinks/schema.d.ts

@@ -0,0 +1,9 @@
+/**
+ * sinks/schema.ts — Re-export barrel
+ *
+ * All sink Zod schemas now live in @sanity/ailf-core. This file
+ * re-exports them for backward compatibility.
+ *
+ * @see packages/core/src/schemas/sinks.ts (canonical source)
+ */
+export * from "../_vendor/ailf-core/index.d.ts";

package/dist/sinks/schema.js

@@ -0,0 +1,9 @@
+/**
+ * sinks/schema.ts — Re-export barrel
+ *
+ * All sink Zod schemas now live in @sanity/ailf-core. This file
+ * re-exports them for backward compatibility.
+ *
+ * @see packages/core/src/schemas/sinks.ts (canonical source)
+ */
+export * from "../_vendor/ailf-core/index.js";

package/dist/sinks/slack/format.d.ts

@@ -0,0 +1,65 @@
+/**
+ * sinks/slack/format.ts
+ *
+ * Formats evaluation report data into Slack Block Kit structures for the
+ * SlackSink. Provides three message formats:
+ *
+ * - `formatRegressionAlert` — detailed regression notification with
+ *   per-area dimension breakdowns
+ * - `formatScoreSummary` — compact score overview for general reporting
+ * - `formatWeeklyDigest` — weekly trend summary for scheduled digests
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { Report } from "../../pipeline/types.js";
+import type { DigestSummary } from "../../schedules/digest.js";
+export interface SlackMessage {
+    blocks: SlackBlock[];
+    text: string;
+}
+interface SlackBlock {
+    elements?: {
+        text: string;
+        type: "mrkdwn" | "plain_text";
+    }[];
+    fields?: {
+        text: string;
+        type: "mrkdwn" | "plain_text";
+    }[];
+    text?: {
+        text: string;
+        type: "mrkdwn" | "plain_text";
+    };
+    type: "context" | "divider" | "header" | "section";
+}
+/**
+ * Format a regression alert for areas that have regressed.
+ *
+ * Produces a rich Slack message with:
+ * - Header with overall score change
+ * - Context metadata (mode, source, timestamp, promptfoo link)
+ * - Per-area regression details with dimension breakdowns
+ * - Brief mentions of improved and unchanged areas
+ */
+export declare function formatRegressionAlert(report: Report): SlackMessage;
+/**
+ * Format a general score summary for Slack reporting.
+ *
+ * Produces a compact overview with:
+ * - Overall score with grade emoji
+ * - Per-area score table
+ * - Cost summary (if available)
+ * - Promptfoo link (if available)
+ */
+export declare function formatScoreSummary(report: Report): SlackMessage;
+/**
+ * Format a weekly digest summary for Slack.
+ *
+ * Produces a summary message covering score trends over a time window:
+ * - Header with overall trend direction and score
+ * - Per-area trend table with arrows
+ * - Lists of improved, regressed, and stable areas
+ * - Report count and time window metadata
+ */
+export declare function formatWeeklyDigest(digest: DigestSummary): SlackMessage;
+export {};