@sanity/ailf 0.1.0

package/dist/pipeline/eval-fingerprint.d.ts

@@ -0,0 +1,66 @@
+/**
+ * pipeline/eval-fingerprint.ts
+ *
+ * Computes a deterministic fingerprint of all inputs that affect evaluation
+ * output. Used for cross-environment cache lookup: when running in CI, the
+ * pipeline can query the Sanity Content Lake for a previous report with an
+ * identical fingerprint and skip the expensive eval step.
+ *
+ * The fingerprint captures everything that would change evaluation results:
+ * - Evaluation mode (baseline, observed, agentic)
+ * - Model configuration (which models, their settings)
+ * - Grader model identity (different graders score differently)
+ * - Prompt templates (different instructions → different outputs)
+ * - Rubric templates (different criteria → different scores)
+ * - Task definitions (what's being evaluated)
+ * - Reference solutions (used by grader assertions)
+ * - Documentation content (the docs being evaluated — the primary variable)
+ * - Filter flags (which subset of tasks is included)
+ *
+ * The fingerprint intentionally EXCLUDES:
+ * - Source name/URL (content matters, not origin)
+ * - Git metadata (informational, not eval-affecting)
+ * - Trigger type (manual vs CI → same inputs → same results)
+ * - Report tags (human labels)
+ *
+ * @see docs/design-docs/content-lake-eval-caching.md
+ */
+import type { EvalMode, FilterOptions } from "./types.js";
+/** Inputs needed to compute an evaluation fingerprint. */
+export interface FingerprintInput {
+    /** Filter options (areas, taskIds) — determines which tasks are included */
+    filter?: FilterOptions;
+    /** Grader model identifier (e.g., "anthropic:messages:claude-opus-4-5-20251101") */
+    graderModel: string;
+    /** Evaluation mode */
+    mode: EvalMode;
+    /** Path to the packages/eval root directory */
+    rootDir: string;
+}
+/**
+ * Collect all file paths that contribute to the evaluation fingerprint.
+ *
+ * This is similar to `getStepInputPaths()` in `cache.ts` but is more
+ * comprehensive and explicitly designed for cross-environment cache keys:
+ *
+ * - Includes `config/prompts.yaml` and `config/rubrics.yaml` directly
+ *   (the local cache only includes them indirectly via generated configs)
+ * - Includes `config/models.yaml` (model configuration)
+ * - Includes task definitions and reference solutions
+ * - Includes the actual documentation content (contexts/canonical/*.md)
+ * - Respects filter flags to only include relevant files
+ */
+export declare function collectFingerprintInputPaths(rootDir: string, filter?: FilterOptions): string[];
+/**
+ * Compute a deterministic SHA-256 fingerprint of all evaluation inputs.
+ *
+ * The fingerprint is content-addressed: identical inputs always produce
+ * the same fingerprint, regardless of the environment (local, CI, etc.).
+ *
+ * Reuses the existing `hashFiles()` from `cache.ts` to hash file content,
+ * and adds non-file context (mode, grader model, filter flags) as
+ * additional context strings.
+ *
+ * @returns SHA-256 hex string (64 characters)
+ */
+export declare function computeEvalFingerprint(input: FingerprintInput): string;

package/dist/pipeline/eval-fingerprint.js

@@ -0,0 +1,175 @@
+/**
+ * pipeline/eval-fingerprint.ts
+ *
+ * Computes a deterministic fingerprint of all inputs that affect evaluation
+ * output. Used for cross-environment cache lookup: when running in CI, the
+ * pipeline can query the Sanity Content Lake for a previous report with an
+ * identical fingerprint and skip the expensive eval step.
+ *
+ * The fingerprint captures everything that would change evaluation results:
+ * - Evaluation mode (baseline, observed, agentic)
+ * - Model configuration (which models, their settings)
+ * - Grader model identity (different graders score differently)
+ * - Prompt templates (different instructions → different outputs)
+ * - Rubric templates (different criteria → different scores)
+ * - Task definitions (what's being evaluated)
+ * - Reference solutions (used by grader assertions)
+ * - Documentation content (the docs being evaluated — the primary variable)
+ * - Filter flags (which subset of tasks is included)
+ *
+ * The fingerprint intentionally EXCLUDES:
+ * - Source name/URL (content matters, not origin)
+ * - Git metadata (informational, not eval-affecting)
+ * - Trigger type (manual vs CI → same inputs → same results)
+ * - Report tags (human labels)
+ *
+ * @see docs/design-docs/content-lake-eval-caching.md
+ */
+import { existsSync, readdirSync, statSync } from "fs";
+import { join, resolve } from "path";
+import { hashFiles } from "./cache.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/**
+ * Version prefix for the fingerprint hash. Bumping this invalidates all
+ * existing fingerprints in the Content Lake without needing to clear the
+ * store. Change this when adding new inputs to the hash.
+ */
+const FINGERPRINT_VERSION = "eval-fingerprint-v1";
+/**
+ * Collect all file paths that contribute to the evaluation fingerprint.
+ *
+ * This is similar to `getStepInputPaths()` in `cache.ts` but is more
+ * comprehensive and explicitly designed for cross-environment cache keys:
+ *
+ * - Includes `config/prompts.yaml` and `config/rubrics.yaml` directly
+ *   (the local cache only includes them indirectly via generated configs)
+ * - Includes `config/models.yaml` (model configuration)
+ * - Includes task definitions and reference solutions
+ * - Includes the actual documentation content (contexts/canonical/*.md)
+ * - Respects filter flags to only include relevant files
+ */
+export function collectFingerprintInputPaths(rootDir, filter) {
+    const r = (rel) => resolve(rootDir, rel);
+    const paths = [];
+    // -----------------------------------------------------------------------
+    // Config files — always included
+    // -----------------------------------------------------------------------
+    const configFiles = [
+        "config/models.yaml",
+        "config/prompts.yaml",
+        "config/rubrics.yaml",
+    ];
+    for (const f of configFiles) {
+        const p = r(f);
+        if (existsSync(p))
+            paths.push(p);
+    }
+    // -----------------------------------------------------------------------
+    // Task files — filtered if --area is set
+    // -----------------------------------------------------------------------
+    const tasksDir = r("tasks");
+    if (existsSync(tasksDir)) {
+        const taskFiles = readdirSync(tasksDir)
+            .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+            .filter((f) => !f.startsWith(".")); // exclude .expanded.yaml
+        for (const f of taskFiles) {
+            // If area filter is set, only include matching task files
+            if (filter?.areas && filter.areas.length > 0) {
+                const stem = f.replace(/\.ya?ml$/, "");
+                if (!filter.areas.includes(stem))
+                    continue;
+            }
+            paths.push(join(tasksDir, f));
+        }
+    }
+    // -----------------------------------------------------------------------
+    // Reference solutions — all included (they're referenced by tasks)
+    // -----------------------------------------------------------------------
+    const refDir = r("canonical/reference-solutions");
+    if (existsSync(refDir)) {
+        collectFilesRecursive(refDir, paths);
+    }
+    // -----------------------------------------------------------------------
+    // Canonical context files — the documentation content being evaluated
+    // This is the KEY differentiator from the local cache (which doesn't
+    // include Sanity document content in the fetch-docs cache key).
+    // -----------------------------------------------------------------------
+    const canonicalDir = r("contexts/canonical");
+    if (existsSync(canonicalDir)) {
+        const contextFiles = readdirSync(canonicalDir)
+            .filter((f) => f.endsWith(".md"))
+            .sort();
+        for (const f of contextFiles) {
+            // If area or task filter is set, we include all context files anyway
+            // because context filenames map to task IDs, and task-to-area mapping
+            // requires reading the YAML. It's safer to include all — a superset
+            // doesn't cause false cache hits, only potential false misses when
+            // a non-matching context changes. This is acceptable: the filter
+            // flags in the context strings differentiate the fingerprints.
+            paths.push(join(canonicalDir, f));
+        }
+    }
+    return paths;
+}
+/**
+ * Compute a deterministic SHA-256 fingerprint of all evaluation inputs.
+ *
+ * The fingerprint is content-addressed: identical inputs always produce
+ * the same fingerprint, regardless of the environment (local, CI, etc.).
+ *
+ * Reuses the existing `hashFiles()` from `cache.ts` to hash file content,
+ * and adds non-file context (mode, grader model, filter flags) as
+ * additional context strings.
+ *
+ * @returns SHA-256 hex string (64 characters)
+ */
+export function computeEvalFingerprint(input) {
+    const { filter, graderModel, mode, rootDir } = input;
+    // -----------------------------------------------------------------------
+    // 1. Collect context strings (non-file inputs)
+    // -----------------------------------------------------------------------
+    const context = [
+        FINGERPRINT_VERSION,
+        `mode:${mode}`,
+        `grader:${graderModel}`,
+    ];
+    // Include filter flags so that scoped runs produce different fingerprints
+    if (filter?.areas && filter.areas.length > 0) {
+        context.push(`areas:${[...filter.areas].sort().join(",")}`);
+    }
+    if (filter?.taskIds && filter.taskIds.length > 0) {
+        context.push(`tasks:${[...filter.taskIds].sort().join(",")}`);
+    }
+    // -----------------------------------------------------------------------
+    // 2. Collect input file paths (all files that affect eval output)
+    // -----------------------------------------------------------------------
+    const paths = collectFingerprintInputPaths(rootDir, filter);
+    // -----------------------------------------------------------------------
+    // 3. Hash everything together
+    // -----------------------------------------------------------------------
+    return hashFiles(paths, context);
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Recursively collect all file paths under a directory.
+ * Skips hidden files and directories (starting with '.').
+ */
+function collectFilesRecursive(dir, paths) {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+        if (entry.startsWith("."))
+            continue;
+        const fullPath = join(dir, entry);
+        const stat = statSync(fullPath);
+        if (stat.isDirectory()) {
+            collectFilesRecursive(fullPath, paths);
+        }
+        else if (stat.isFile()) {
+            paths.push(fullPath);
+        }
+    }
+}

package/dist/pipeline/expand-tasks.d.ts

@@ -0,0 +1,220 @@
+/**
+ * pipeline/expand-tasks.ts
+ *
+ * Reads task YAML files in the single-definition format and expands each
+ * task into gold + baseline Promptfoo test entries. This eliminates the
+ * manual duplication where every task had to be written twice.
+ *
+ * Rubric templates from config/rubrics.yaml are resolved at expansion time:
+ * tasks specify `template` + `criteria`, and the expander assembles
+ * the full rubric text by injecting criteria into the template.
+ *
+ * Structured dimension metadata (Approach 5):
+ * When a rubric template has a `dimension` field, the resolved assertion
+ * includes `metadata.dimension` and `metadata.maxScore`. This flows through
+ * Promptfoo into component results, allowing the scoring engine to classify
+ * rubrics structurally instead of via heuristic string matching.
+ * See docs/design-docs/structured-dimensions.md.
+ *
+ * Single-definition format:
+ *   - id: groq-blog-queries
+ *     description: "GROQ - Blog queries with filtering and pagination"
+ *     doc_coverage: true
+ *     vars:
+ *       task: |
+ *         Write GROQ queries for a Sanity blog application: ...
+ *       docs: file://contexts/canonical/groq-blog-queries.md
+ *     assert:
+ *       - type: llm-rubric
+ *         template: task-completion
+ *         criteria:
+ *           - GROQ filter with _type == "post"
+ *           - Projection with aliased slug field
+ *       - type: contains-any
+ *         value: ["client.fetch", "createClient"]
+ *     baseline:
+ *       enabled: true
+ *       rubric: abbreviated
+ *
+ * Expands to:
+ *   1. Gold entry — uses vars.docs as-is, resolves templates, appends doc-coverage
+ *   2. Baseline entry — sets docs: "", adds transform, uses abbreviated rubric
+ */
+import type { TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
+import { type RubricConfig } from "./schemas.js";
+import type { FilterOptions } from "./types.js";
+/** Any assertion entry (templated or value-based). */
+export type AssertEntry = TemplatedAssert | ValueAssert;
+/** The output format — a Promptfoo-compatible test entry. */
+export interface ExpandedTestEntry {
+    assert?: ValueAssert[];
+    description: string;
+    /**
+     * Promptfoo prompt filter — restricts which prompts this test runs against.
+     * Matches on prompt `id` or `label`. When absent, all prompts are allowed
+     * (Promptfoo's default cartesian product behavior).
+     *
+     * Gold entries use `['with-docs']` (ceiling measurement).
+     * Baseline entries use `['without-docs']` (floor measurement).
+     *
+     * See: evaluation-ceiling.md for the floor/ceiling/actual decomposition.
+     */
+    prompts?: string[];
+    vars: Record<string, unknown>;
+}
+/** A legacy task entry (the old paired format without an `id` field). */
+export interface LegacyTaskEntry {
+    assert?: AssertEntry[];
+    description: string;
+    transform?: string;
+    vars?: Record<string, unknown>;
+}
+/** A single task definition in the new format (input). */
+export interface SingleTaskDefinition {
+    /** Grading assertions (applied to gold; optionally abbreviated for baseline). */
+    assert: AssertEntry[];
+    /** Baseline generation options. */
+    baseline?: {
+        /** Whether to generate a baseline variant. Default: true. */
+        enabled?: boolean;
+        /** Rubric mode: 'full' copies all asserts, 'abbreviated' generates a
+         *  summary rubric, 'none' omits rubric asserts. Default: 'abbreviated'. */
+        rubric?: "abbreviated" | "full" | "none";
+    };
+    /** Human-readable description of what this task tests. */
+    description: string;
+    /** Opt-in: auto-generate a documentation coverage rubric for gold. */
+    doc_coverage?: boolean;
+    /** Explicit task ID — determines the canonical context filename. */
+    id: string;
+    /** Template variables: task prompt and docs path. */
+    vars: {
+        task: string;
+        docs: string;
+        [key: string]: unknown;
+    };
+}
+/** A templated assertion — references a rubric template. */
+export interface TemplatedAssert {
+    criteria: string[];
+    template: string;
+    type: "llm-rubric";
+    weight?: number;
+}
+/** A standard assertion with a value. */
+export interface ValueAssert {
+    [key: string]: unknown;
+    type: string;
+    value?: unknown;
+    weight?: number;
+}
+/**
+ * Assemble a full rubric text string from a template and criteria.
+ *
+ * Output format:
+ *   {header}
+ *   - {scale[0]}
+ *   - {scale[1]}
+ *   ...
+ *
+ *   {criteria_label}
+ *   - {criteria[0]}
+ *   - {criteria[1]}
+ *   ...
+ *
+ *   {footer}
+ */
+export declare function assembleRubric(templateKey: string, criteria: string[], rubricConfig: RubricConfig): string;
+/**
+ * Build baseline assertions based on the rubric mode.
+ *
+ * - 'full': Copy all assertions as-is
+ * - 'abbreviated': Keep only the first llm-rubric (task completion) with
+ *   a shortened prompt, plus all non-rubric assertions
+ * - 'none': No assertions at all
+ */
+export declare function buildBaselineAsserts(goldAsserts: ValueAssert[], mode: "abbreviated" | "full" | "none"): ValueAssert[];
+/**
+ * Clear the cached rubric config. Used in tests.
+ */
+export declare function clearRubricCache(): void;
+/**
+ * Expand a single task definition into gold + baseline Promptfoo test entries.
+ * Returns 1 entry (gold only) if baseline is disabled, or 2 entries otherwise.
+ *
+ * Resolves templated assertions and appends doc-coverage if opted in.
+ *
+ * @param mode - Controls which entries are generated and how:
+ *   - `'baseline'` (default): Gold + baseline entries with `prompts` filter
+ *     to prevent cartesian product with multiple prompts. Gold entries get
+ *     `prompts: ['with-docs']`, baseline entries get `prompts: ['without-docs']`.
+ *   - `'agentic'`: Gold entries only, no `prompts` filter (agentic mode has
+ *     a single prompt that doesn't use `{{docs}}`; baseline entries would be
+ *     pure waste — identical prompts, wasted API calls).
+ */
+export declare function expandTask(task: SingleTaskDefinition, rubricConfig: RubricConfig, mode?: "agentic" | "baseline"): ExpandedTestEntry[];
+/**
+ * Expand an array of TaskDefinition[] (from any TaskSource adapter) into
+ * Promptfoo-compatible test entries. This is the TaskSource-aware counterpart
+ * of loadAndExpandTasks() — it skips YAML file I/O and works directly with
+ * the canonical domain type.
+ *
+ * @param tasks - Task definitions from any TaskSource adapter
+ * @param rootDir - Eval package root (needed to load rubric templates)
+ * @param mode - Expansion mode: 'baseline' (gold + baseline) or 'agentic' (gold only)
+ * @returns Expanded test entries and statistics
+ */
+export declare function expandTaskDefinitions(tasks: TaskDefinition[], rootDir: string, mode?: "agentic" | "baseline"): {
+    entries: ExpandedTestEntry[];
+    stats: {
+        totalTasks: number;
+        expandedTotal: number;
+    };
+};
+/**
+ * Extract all task IDs from task files. Only works with the new
+ * single-definition format entries (those that have an `id` field).
+ */
+export declare function extractTaskIds(rootDir: string): string[];
+/**
+ * Type guard: checks if an entry is in the new single-definition format.
+ * The distinguishing feature is the presence of an `id` field.
+ */
+export declare function isSingleTaskDefinition(entry: unknown): entry is SingleTaskDefinition;
+/**
+ * Type guard: checks if an assertion uses the templated format.
+ */
+export declare function isTemplatedAssert(entry: AssertEntry): entry is TemplatedAssert;
+/**
+ * Load and expand all task files from the tasks/ directory.
+ * Supports both the new single-definition format (has `id`) and the legacy
+ * paired format (no `id`). Legacy entries pass through unchanged.
+ *
+ * @param mode - Controls expansion behavior:
+ *   - `'baseline'` (default): Gold + baseline entries with prompt filters.
+ *   - `'agentic'`: Gold entries only, no prompt filters.
+ *
+ * Returns the expanded entries grouped by source file.
+ */
+export declare function loadAndExpandTasks(rootDir: string, filter?: FilterOptions, mode?: "agentic" | "baseline"): {
+    /** All expanded test entries, in order. */
+    entries: ExpandedTestEntry[];
+    /** Statistics about what was processed. */
+    stats: {
+        totalFiles: number;
+        singleDefinitions: number;
+        legacyEntries: number;
+        expandedTotal: number;
+    };
+};
+/**
+ * Load and validate config/rubrics.yaml from the given root directory.
+ * Caches the result for subsequent calls with the same rootDir.
+ */
+export declare function loadRubricTemplates(rootDir: string): RubricConfig;
+/**
+ * Resolve a single assertion: if it's templated, assemble the rubric text
+ * and attach structured dimension metadata when the template has a
+ * `dimension` field. Otherwise, pass through unchanged.
+ */
+export declare function resolveAssert(entry: AssertEntry, rubricConfig: RubricConfig): ValueAssert;