@sanity/ailf 0.1.0

package/dist/_vendor/ailf-core/schemas/sinks.js

@@ -0,0 +1,108 @@
+/**
+ * @sanity/ailf-core — Sink schemas
+ *
+ * Zod schemas for runtime validation of config/sinks.yaml — the configuration
+ * file for report delivery sinks.
+ *
+ * Each sink type has its own config schema, and the top-level SinksFileSchema
+ * validates the complete YAML structure. The discriminated union on `type`
+ * ensures type-safe parsing with clear error messages for unknown sink types.
+ *
+ * Usage:
+ *   import { SinksFileSchema } from "./schema.js"
+ *   const parsed = SinksFileSchema.parse(rawYaml)
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import { z } from "zod";
+// ---------------------------------------------------------------------------
+// Sink type enum
+// ---------------------------------------------------------------------------
+/** All supported sink types as a Zod union. */
+export const SinkTypeSchema = z.enum([
+    "bigquery",
+    "github-comment",
+    "slack",
+    "webhook",
+]);
+// ---------------------------------------------------------------------------
+// Routing rules — severity-aware delivery targeting (Phase 5c)
+// ---------------------------------------------------------------------------
+/**
+ * Schema for sink routing rules.
+ * Controls which messages are delivered based on severity and type.
+ * When omitted, the sink receives all reports (backward compatible).
+ */
+export const SinkRoutingSchema = z.object({
+    /** Deliver on critical threshold violations */
+    critical: z.union([z.boolean(), z.string()]).optional(),
+    /** Deliver weekly digest messages */
+    digest: z.union([z.boolean(), z.string()]).optional(),
+    /** Deliver on info-level threshold violations */
+    info: z.union([z.boolean(), z.string()]).optional(),
+    /** Deliver on regression detection */
+    regression: z.union([z.boolean(), z.string()]).optional(),
+    /** Deliver on warning-level threshold violations */
+    warning: z.union([z.boolean(), z.string()]).optional(),
+});
+/** Inferred TypeScript type for sink routing rules. */
+// ---------------------------------------------------------------------------
+// Individual sink config schemas
+// ---------------------------------------------------------------------------
+/** Config schema for a generic webhook sink. */
+export const WebhookSinkConfigSchema = z.object({
+    enabled: z.boolean().default(false),
+    headers: z.record(z.string(), z.string()).optional(),
+    routing: SinkRoutingSchema.optional(),
+    type: z.literal("webhook"),
+    url: z.string().url(),
+});
+// ---------------------------------------------------------------------------
+// Individual sink config schemas
+// ---------------------------------------------------------------------------
+/** Config schema for a Slack notification sink. */
+export const SlackSinkConfigSchema = z.object({
+    channel: z.string().optional(),
+    enabled: z.boolean().default(false),
+    routing: SinkRoutingSchema.optional(),
+    type: z.literal("slack"),
+    webhookUrl: z.string().url(),
+});
+/** Config schema for a GitHub PR comment sink. */
+export const GitHubCommentSinkConfigSchema = z.object({
+    enabled: z.boolean().default(false),
+    routing: SinkRoutingSchema.optional(),
+    token: z.string().optional(),
+    type: z.literal("github-comment"),
+});
+/** Config schema for a BigQuery sink. */
+export const BigQuerySinkConfigSchema = z.object({
+    credentials: z.string().optional(),
+    dataset: z.string(),
+    enabled: z.boolean().default(false),
+    project: z.string(),
+    routing: SinkRoutingSchema.optional(),
+    type: z.literal("bigquery"),
+});
+// ---------------------------------------------------------------------------
+// Discriminated union across all sink types
+// ---------------------------------------------------------------------------
+/**
+ * A single sink configuration — discriminated on the `type` field.
+ *
+ * Zod v4 discriminatedUnion provides clear errors when the `type` value
+ * doesn't match any known sink.
+ */
+export const SinkConfigSchema = z.discriminatedUnion("type", [
+    BigQuerySinkConfigSchema,
+    GitHubCommentSinkConfigSchema,
+    SlackSinkConfigSchema,
+    WebhookSinkConfigSchema,
+]);
+// ---------------------------------------------------------------------------
+// Top-level file schema (config/sinks.yaml)
+// ---------------------------------------------------------------------------
+/** Schema for the complete config/sinks.yaml file. */
+export const SinksFileSchema = z.object({
+    sinks: z.array(SinkConfigSchema).default([]),
+});

package/dist/_vendor/ailf-core/services/comparison-formatters.d.ts

@@ -0,0 +1,18 @@
+/**
+ * core/services/comparison-formatters.ts — Pure formatting functions for
+ * comparison reports.
+ *
+ * No I/O — takes a ComparisonReport, returns a string.
+ *
+ * Extracted from packages/eval/src/lib/compare.ts during the
+ * Ports & Adapters migration (Phase 4e).
+ */
+import type { ComparisonReport } from "../types/index.js";
+/**
+ * Generate a markdown comparison section suitable for PR comments.
+ */
+export declare function formatComparisonMarkdown(report: ComparisonReport): string;
+/**
+ * Generate a human-readable console comparison table.
+ */
+export declare function formatComparisonTable(report: ComparisonReport): string;

package/dist/_vendor/ailf-core/services/comparison-formatters.js

@@ -0,0 +1,189 @@
+/**
+ * core/services/comparison-formatters.ts — Pure formatting functions for
+ * comparison reports.
+ *
+ * No I/O — takes a ComparisonReport, returns a string.
+ *
+ * Extracted from packages/eval/src/lib/compare.ts during the
+ * Ports & Adapters migration (Phase 4e).
+ */
+// ---------------------------------------------------------------------------
+// Markdown format (for PR comments)
+// ---------------------------------------------------------------------------
+/**
+ * Generate a markdown comparison section suitable for PR comments.
+ */
+export function formatComparisonMarkdown(report) {
+    const lines = [];
+    const overall = report.deltas.overall;
+    const overallIcon = changeIcon(overall > report.noiseThreshold
+        ? "improved"
+        : overall < -report.noiseThreshold
+            ? "regressed"
+            : "unchanged");
+    lines.push("### 📊 Score Comparison");
+    lines.push("");
+    lines.push(`**Overall: ${Math.round(report.baseline.overall.avgScore)} → ${Math.round(report.experiment.overall.avgScore)}** (${overallIcon} ${deltaStr(overall)})`);
+    lines.push("");
+    // Per-area table
+    lines.push("| Feature | Baseline | Current | Delta | Task | Code | Docs |");
+    lines.push("|---------|----------|---------|-------|------|------|------|");
+    for (const a of report.areas) {
+        const icon = changeIcon(a.change);
+        lines.push(`| ${a.area} | ${a.baseline} | ${a.experiment} | ${icon} ${deltaStr(a.delta)} | ${deltaStr(a.dimensions.taskCompletion.delta)} | ${deltaStr(a.dimensions.codeCorrectness.delta)} | ${deltaStr(a.dimensions.docCoverage.delta)} |`);
+    }
+    lines.push("");
+    // Summary
+    const parts = [];
+    if (report.improved.length > 0) {
+        parts.push(`📈 ${report.improved.length} improved`);
+    }
+    if (report.regressed.length > 0) {
+        parts.push(`📉 ${report.regressed.length} regressed`);
+    }
+    if (report.unchanged.length > 0) {
+        parts.push(`➡️ ${report.unchanged.length} unchanged`);
+    }
+    if (parts.length > 0) {
+        lines.push(parts.join(" · "));
+        lines.push("");
+    }
+    // Dimension averages in collapsible
+    lines.push("<details>");
+    lines.push("<summary>Dimension averages</summary>");
+    lines.push("");
+    const dim = report.deltas.perDimension;
+    lines.push("| Dimension | Delta |");
+    lines.push("|-----------|-------|");
+    lines.push(`| Task Completion | ${deltaStr(dim.taskCompletion)} |`);
+    lines.push(`| Code Correctness | ${deltaStr(dim.codeCorrectness)} |`);
+    lines.push(`| Doc Coverage | ${deltaStr(dim.docCoverage)} |`);
+    lines.push(`| Doc Lift | ${deltaStr(report.deltas.docLift)} |`);
+    if (report.deltas.cost !== undefined) {
+        const costStr = report.deltas.cost > 0
+            ? `+$${report.deltas.cost.toFixed(4)}`
+            : `-$${Math.abs(report.deltas.cost).toFixed(4)}`;
+        lines.push(`| Cost | ${costStr} |`);
+    }
+    lines.push("");
+    lines.push("</details>");
+    lines.push("");
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Console table format
+// ---------------------------------------------------------------------------
+/**
+ * Generate a human-readable console comparison table.
+ */
+export function formatComparisonTable(report) {
+    const lines = [];
+    lines.push("=".repeat(80));
+    lines.push("                    COMPARISON REPORT");
+    lines.push("=".repeat(80));
+    lines.push("");
+    // Overall summary
+    const overall = report.deltas.overall;
+    const overallIcon = changeIcon(overall > report.noiseThreshold
+        ? "improved"
+        : overall < -report.noiseThreshold
+            ? "regressed"
+            : "unchanged");
+    lines.push(`  Overall: ${Math.round(report.baseline.overall.avgScore)} → ${Math.round(report.experiment.overall.avgScore)} (${overallIcon} ${deltaStr(overall)})`);
+    lines.push("");
+    // Per-dimension averages
+    const dim = report.deltas.perDimension;
+    lines.push("  Dimension averages:");
+    lines.push(`    Task Completion:  ${deltaStr(dim.taskCompletion)}`);
+    lines.push(`    Code Correctness: ${deltaStr(dim.codeCorrectness)}`);
+    lines.push(`    Doc Coverage:     ${deltaStr(dim.docCoverage)}`);
+    lines.push(`    Doc Lift:         ${deltaStr(report.deltas.docLift)}`);
+    if (report.deltas.cost !== undefined) {
+        lines.push(`    Cost:             ${report.deltas.cost > 0 ? "+" : ""}$${report.deltas.cost.toFixed(4)}`);
+    }
+    lines.push("");
+    // Per-area table
+    lines.push("-".repeat(80));
+    lines.push("PER-AREA BREAKDOWN");
+    lines.push("-".repeat(80));
+    lines.push("");
+    const h = "| Feature Area        | Baseline | Experiment | Delta | Task | Code | Docs |";
+    const sep = "|---------------------|----------|------------|-------|------|------|------|";
+    lines.push(h);
+    lines.push(sep);
+    for (const a of report.areas) {
+        const icon = changeIcon(a.change);
+        lines.push(`| ${icon} ${a.area.padEnd(17)} | ${String(a.baseline).padStart(8)} | ${String(a.experiment).padStart(10)} | ${deltaStr(a.delta).padStart(5)} | ${deltaStr(a.dimensions.taskCompletion.delta).padStart(4)} | ${deltaStr(a.dimensions.codeCorrectness.delta).padStart(4)} | ${deltaStr(a.dimensions.docCoverage.delta).padStart(4)} |`);
+    }
+    lines.push("");
+    // Classification summary
+    if (report.improved.length > 0) {
+        lines.push(`  📈 Improved:   ${report.improved.join(", ")}`);
+    }
+    if (report.regressed.length > 0) {
+        lines.push(`  📉 Regressed:  ${report.regressed.join(", ")}`);
+    }
+    if (report.unchanged.length > 0) {
+        lines.push(`  ➡️  Unchanged:  ${report.unchanged.join(", ")}`);
+    }
+    lines.push("");
+    // Mismatched areas
+    if (report.mismatched.onlyInBaseline.length > 0 ||
+        report.mismatched.onlyInExperiment.length > 0) {
+        lines.push("  ⚠️  Area mismatches:");
+        if (report.mismatched.onlyInBaseline.length > 0) {
+            lines.push(`    Only in baseline:   ${report.mismatched.onlyInBaseline.join(", ")}`);
+        }
+        if (report.mismatched.onlyInExperiment.length > 0) {
+            lines.push(`    Only in experiment: ${report.mismatched.onlyInExperiment.join(", ")}`);
+        }
+        lines.push("");
+    }
+    const isEmpirical = "noiseThresholdEmpirical" in report &&
+        report.noiseThresholdEmpirical === true;
+    const thresholdSource = isEmpirical
+        ? "empirical, from grader consistency data"
+        : "default";
+    lines.push(`  Noise threshold: ±${report.noiseThreshold}${Number.isInteger(report.noiseThreshold) ? "" : ` (${report.noiseThreshold.toFixed(1)})`} (${thresholdSource})`);
+    lines.push("");
+    // Ceiling decomposition deltas (when areas have ceiling data)
+    const hasCeilingData = report.areas.some((a) => a.ceilingDelta !== undefined);
+    if (hasCeilingData) {
+        lines.push("-".repeat(80));
+        lines.push("CEILING DECOMPOSITION DELTAS");
+        lines.push("-".repeat(80));
+        lines.push("");
+        const cH = "| Feature Area        | Ceiling Δ | Floor Δ | Doc Lift Δ |";
+        const cSep = "|---------------------|-----------|---------|------------|";
+        lines.push(cH);
+        lines.push(cSep);
+        for (const a of report.areas) {
+            lines.push(`| ${a.area.padEnd(19)} | ` +
+                `${deltaStr(a.ceilingDelta).padStart(9)} | ` +
+                `${deltaStr(a.floorDelta).padStart(7)} | ` +
+                `${deltaStr(a.docLiftDelta).padStart(10)} |`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Helpers (private)
+// ---------------------------------------------------------------------------
+function changeIcon(change) {
+    switch (change) {
+        case "improved":
+            return "📈";
+        case "regressed":
+            return "📉";
+        default:
+            return "➡️";
+    }
+}
+function deltaStr(d) {
+    if (d > 0)
+        return `+${Math.round(d)}`;
+    if (d < 0)
+        return `${Math.round(d)}`;
+    return "0";
+}

package/dist/_vendor/ailf-core/services/config-helpers.d.ts

@@ -0,0 +1,41 @@
+/**
+ * core/services/config-helpers.ts — Pure config generation helpers.
+ *
+ * No I/O — operates on typed inputs and returns typed outputs.
+ *
+ * Extracted from packages/eval/src/lib/generate-configs.ts during
+ * the Ports & Adapters migration (Phase 4e).
+ */
+import type { ModelEntry } from "../types/index.js";
+/**
+ * Extract the raw API model name from a Promptfoo provider ID.
+ *
+ * Promptfoo IDs encode the provider + sub-protocol + model, e.g.:
+ * - "openai:chat:gpt-5.2" → "gpt-5.2"
+ * - "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6"
+ * - "openrouter:deepseek/deepseek-r1" → "deepseek/deepseek-r1"
+ * - "gpt-4o" → "gpt-4o"
+ */
+export declare function extractModelName(id: string): string;
+/**
+ * Extract the LLM provider family from a Promptfoo provider ID.
+ *
+ * - "openai:chat:gpt-5.2" → "openai"
+ * - "anthropic:messages:claude-opus-4-6" → "anthropic"
+ * - "google:gemini-2.5-pro" → "google"
+ * - "gpt-4o" → "openai" (default)
+ */
+export declare function extractProvider(id: string): string;
+/**
+ * Merge default, model-specific, and mode-specific config into a single object.
+ *
+ * Only picks scalar defaults (temperature, max_tokens). Model and mode
+ * overrides are applied on top.
+ */
+export declare function mergeConfig(defaults: Record<string, unknown>, modelConfig?: Record<string, unknown>, overrides?: Record<string, unknown>): Record<string, unknown>;
+/**
+ * Check whether a model entry matches a given evaluation mode.
+ *
+ * Models without a `modes` field match all modes.
+ */
+export declare function modelMatchesMode(model: ModelEntry, mode: string): boolean;

package/dist/_vendor/ailf-core/services/config-helpers.js

@@ -0,0 +1,86 @@
+/**
+ * core/services/config-helpers.ts — Pure config generation helpers.
+ *
+ * No I/O — operates on typed inputs and returns typed outputs.
+ *
+ * Extracted from packages/eval/src/lib/generate-configs.ts during
+ * the Ports & Adapters migration (Phase 4e).
+ */
+/**
+ * Known Promptfoo provider prefixes, ordered longest-first so
+ * "openai:chat:" is tried before "openai:".
+ */
+const PROVIDER_PREFIXES = [
+    "anthropic:messages:",
+    "openai:chat:",
+    "openai:responses:",
+    "openai:",
+    "anthropic:",
+    "google:",
+];
+/**
+ * Extract the raw API model name from a Promptfoo provider ID.
+ *
+ * Promptfoo IDs encode the provider + sub-protocol + model, e.g.:
+ * - "openai:chat:gpt-5.2" → "gpt-5.2"
+ * - "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6"
+ * - "openrouter:deepseek/deepseek-r1" → "deepseek/deepseek-r1"
+ * - "gpt-4o" → "gpt-4o"
+ */
+export function extractModelName(id) {
+    for (const prefix of PROVIDER_PREFIXES) {
+        if (id.startsWith(prefix)) {
+            return id.slice(prefix.length);
+        }
+    }
+    // Fallback: strip first colon-separated segment
+    const parts = id.split(":");
+    return parts.length > 1 ? parts.slice(1).join(":") : id;
+}
+/**
+ * Extract the LLM provider family from a Promptfoo provider ID.
+ *
+ * - "openai:chat:gpt-5.2" → "openai"
+ * - "anthropic:messages:claude-opus-4-6" → "anthropic"
+ * - "google:gemini-2.5-pro" → "google"
+ * - "gpt-4o" → "openai" (default)
+ */
+export function extractProvider(id) {
+    const colon = id.indexOf(":");
+    return colon > 0 ? id.slice(0, colon) : "openai";
+}
+/**
+ * Merge default, model-specific, and mode-specific config into a single object.
+ *
+ * Only picks scalar defaults (temperature, max_tokens). Model and mode
+ * overrides are applied on top.
+ */
+export function mergeConfig(defaults, modelConfig, overrides) {
+    const result = {};
+    // Only pick scalar defaults (temperature, max_tokens)
+    for (const [key, value] of Object.entries(defaults)) {
+        if (typeof value !== "object" || value === null) {
+            result[key] = value;
+        }
+    }
+    // Model-specific overrides
+    if (modelConfig) {
+        Object.assign(result, modelConfig);
+    }
+    // Mode-specific overrides
+    if (overrides) {
+        Object.assign(result, overrides);
+    }
+    return result;
+}
+/**
+ * Check whether a model entry matches a given evaluation mode.
+ *
+ * Models without a `modes` field match all modes.
+ */
+export function modelMatchesMode(model, mode) {
+    if (!model.modes || model.modes.length === 0) {
+        return true;
+    }
+    return model.modes.includes(mode);
+}

package/dist/_vendor/ailf-core/services/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @sanity/ailf-core — Domain services
+ *
+ * Pure computation functions with zero I/O. These operate on typed
+ * inputs and return typed outputs.
+ *
+ * Extracted from packages/eval/src/lib/ during the Ports & Adapters
+ * migration (Phase 4e).
+ */
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "./scoring.js";
+export { formatComparisonMarkdown, formatComparisonTable, } from "./comparison-formatters.js";
+export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "./config-helpers.js";

package/dist/_vendor/ailf-core/services/index.js

@@ -0,0 +1,12 @@
+/**
+ * @sanity/ailf-core — Domain services
+ *
+ * Pure computation functions with zero I/O. These operate on typed
+ * inputs and return typed outputs.
+ *
+ * Extracted from packages/eval/src/lib/ during the Ports & Adapters
+ * migration (Phase 4e).
+ */
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "./scoring.js";
+export { formatComparisonMarkdown, formatComparisonTable, } from "./comparison-formatters.js";
+export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "./config-helpers.js";

package/dist/_vendor/ailf-core/services/scoring.d.ts

@@ -0,0 +1,49 @@
+/**
+ * core/services/scoring.ts — Pure scoring functions.
+ *
+ * These functions operate on typed inputs and produce typed outputs
+ * with zero I/O (no filesystem, network, or console access).
+ *
+ * Extracted from packages/eval/src/lib/calculate-scores.ts during
+ * the Ports & Adapters migration (Phase 4e).
+ */
+import type { FeatureScore } from "../types/index.js";
+import type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata } from "../types/scoring-input.js";
+/**
+ * Classify a grading component into a scoring dimension.
+ *
+ * Prefers structured metadata (Approach 5) over heuristic string matching.
+ * Returns null if the component doesn't map to a known dimension.
+ */
+export declare function classifyRubric(component: ComponentResult): "codeCorrectness" | "docCoverage" | "taskCompletion" | null;
+/**
+ * Detect the feature area from a test description string.
+ *
+ * Uses keyword matching on the lowercased description.
+ */
+export declare function detectFeatureArea(description: string): string;
+/**
+ * Extract a numeric score (0–100) from a grading component result.
+ *
+ * Tries: direct score field → JSON-parsed reason → bare number in reason.
+ */
+export declare function parseRubricScore(component: ComponentResult): number;
+/**
+ * Extract URL metadata from a test result's JavaScript assertion output.
+ *
+ * Returns null if the test doesn't contain URL extraction data.
+ */
+export declare function extractUrlMetadata(test: TestResult): null | UrlMetadata;
+/**
+ * Merge baseline FeatureScore[] with agentic actual scores to produce
+ * the full three-layer decomposition.
+ *
+ * The merge is per feature area. For each area:
+ * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
+ * - If agentic data exists: actualScore is populated
+ * - If both exist: retrievalGap and infrastructureEfficiency are computed
+ *
+ * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
+ * @param agenticScores   Actual scores from agentic evaluation (may be empty)
+ */
+export declare function mergeScores(baselineScores: FeatureScore[], agenticScores: Record<string, ActualScoreEntry>): FeatureScore[];