@sanity/ailf 0.1.0

package/dist/pipeline/expand-tasks.js

@@ -0,0 +1,421 @@
+/**
+ * pipeline/expand-tasks.ts
+ *
+ * Reads task YAML files in the single-definition format and expands each
+ * task into gold + baseline Promptfoo test entries. This eliminates the
+ * manual duplication where every task had to be written twice.
+ *
+ * Rubric templates from config/rubrics.yaml are resolved at expansion time:
+ * tasks specify `template` + `criteria`, and the expander assembles
+ * the full rubric text by injecting criteria into the template.
+ *
+ * Structured dimension metadata (Approach 5):
+ * When a rubric template has a `dimension` field, the resolved assertion
+ * includes `metadata.dimension` and `metadata.maxScore`. This flows through
+ * Promptfoo into component results, allowing the scoring engine to classify
+ * rubrics structurally instead of via heuristic string matching.
+ * See docs/design-docs/structured-dimensions.md.
+ *
+ * Single-definition format:
+ *   - id: groq-blog-queries
+ *     description: "GROQ - Blog queries with filtering and pagination"
+ *     doc_coverage: true
+ *     vars:
+ *       task: |
+ *         Write GROQ queries for a Sanity blog application: ...
+ *       docs: file://contexts/canonical/groq-blog-queries.md
+ *     assert:
+ *       - type: llm-rubric
+ *         template: task-completion
+ *         criteria:
+ *           - GROQ filter with _type == "post"
+ *           - Projection with aliased slug field
+ *       - type: contains-any
+ *         value: ["client.fetch", "createClient"]
+ *     baseline:
+ *       enabled: true
+ *       rubric: abbreviated
+ *
+ * Expands to:
+ *   1. Gold entry — uses vars.docs as-is, resolves templates, appends doc-coverage
+ *   2. Baseline entry — sets docs: "", adds transform, uses abbreviated rubric
+ */
+import { existsSync, readFileSync, readdirSync } from "fs";
+import { resolve } from "path";
+import { load } from "js-yaml";
+import { RubricConfigSchema } from "./schemas.js";
+// ---------------------------------------------------------------------------
+// Rubric template loading and assembly
+// ---------------------------------------------------------------------------
+/** Cached rubric config — loaded once per process. */
+let cachedRubricConfig = null;
+/**
+ * Assemble a full rubric text string from a template and criteria.
+ *
+ * Output format:
+ *   {header}
+ *   - {scale[0]}
+ *   - {scale[1]}
+ *   ...
+ *
+ *   {criteria_label}
+ *   - {criteria[0]}
+ *   - {criteria[1]}
+ *   ...
+ *
+ *   {footer}
+ */
+export function assembleRubric(templateKey, criteria, rubricConfig) {
+    const template = rubricConfig.templates[templateKey];
+    if (!template) {
+        throw new Error(`Unknown rubric template '${templateKey}'. Available: ${Object.keys(rubricConfig.templates).join(", ")}`);
+    }
+    const parts = [];
+    // Header
+    parts.push(template.header);
+    // Scale
+    for (const item of template.scale) {
+        parts.push(`- ${item}`);
+    }
+    // Criteria (if template has a label and criteria are provided)
+    if (template.criteria_label && criteria.length > 0) {
+        parts.push(""); // blank line
+        parts.push(template.criteria_label);
+        for (const c of criteria) {
+            parts.push(`- ${c}`);
+        }
+    }
+    // Footer
+    parts.push(""); // blank line
+    parts.push(rubricConfig.footer);
+    return parts.join("\n");
+}
+/**
+ * Build baseline assertions based on the rubric mode.
+ *
+ * - 'full': Copy all assertions as-is
+ * - 'abbreviated': Keep only the first llm-rubric (task completion) with
+ *   a shortened prompt, plus all non-rubric assertions
+ * - 'none': No assertions at all
+ */
+export function buildBaselineAsserts(goldAsserts, mode) {
+    if (mode === "none")
+        return [];
+    if (mode === "full")
+        return [...goldAsserts];
+    // Abbreviated: keep the first llm-rubric with a summary, drop the rest
+    const abbreviated = [];
+    let foundFirstRubric = false;
+    for (const a of goldAsserts) {
+        if (a.type === "llm-rubric") {
+            if (!foundFirstRubric) {
+                foundFirstRubric = true;
+                abbreviated.push({
+                    type: "llm-rubric",
+                    value: 'Score task completion from 0 to 100 (same criteria as above).\nReturn ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+                });
+            }
+            // Skip subsequent llm-rubrics (code correctness, doc coverage)
+        }
+        // Non-rubric assertions are excluded from baseline to match current behavior
+    }
+    return abbreviated;
+}
+/**
+ * Clear the cached rubric config. Used in tests.
+ */
+export function clearRubricCache() {
+    cachedRubricConfig = null;
+}
+/**
+ * Expand a single task definition into gold + baseline Promptfoo test entries.
+ * Returns 1 entry (gold only) if baseline is disabled, or 2 entries otherwise.
+ *
+ * Resolves templated assertions and appends doc-coverage if opted in.
+ *
+ * @param mode - Controls which entries are generated and how:
+ *   - `'baseline'` (default): Gold + baseline entries with `prompts` filter
+ *     to prevent cartesian product with multiple prompts. Gold entries get
+ *     `prompts: ['with-docs']`, baseline entries get `prompts: ['without-docs']`.
+ *   - `'agentic'`: Gold entries only, no `prompts` filter (agentic mode has
+ *     a single prompt that doesn't use `{{docs}}`; baseline entries would be
+ *     pure waste — identical prompts, wasted API calls).
+ */
+export function expandTask(task, rubricConfig, mode = "baseline") {
+    const entries = [];
+    // Resolve all templated assertions
+    const resolvedAsserts = task.assert.map((a) => resolveAssert(a, rubricConfig));
+    // Append doc-coverage rubric if opted in
+    if (task.doc_coverage) {
+        const dcTemplate = rubricConfig.templates["doc-coverage"];
+        resolvedAsserts.push({
+            type: "llm-rubric",
+            value: assembleRubric("doc-coverage", [], rubricConfig),
+            ...(dcTemplate?.dimension && {
+                metadata: {
+                    dimension: dcTemplate.dimension,
+                    maxScore: 100,
+                },
+            }),
+        });
+    }
+    // Gold entry — ceiling measurement (canonical docs injected directly).
+    // In baseline/observed mode: restricted to the 'with-docs' prompt to
+    // prevent cartesian product with the 'without-docs' prompt.
+    // In agentic mode: no filter needed (single prompt, id = 'agentic').
+    // See evaluation-ceiling.md for the floor/ceiling/actual decomposition.
+    entries.push({
+        assert: [...resolvedAsserts],
+        description: `${task.description} (gold)`,
+        ...(mode === "baseline" ? { prompts: ["with-docs"] } : {}),
+        vars: { ...task.vars },
+    });
+    // Baseline entry — floor measurement (no docs, parametric knowledge only).
+    // Skipped entirely in agentic mode: the agentic prompt doesn't reference
+    // {{docs}}, so gold and baseline would produce identical prompts — pure
+    // waste of API calls and cost.
+    if (mode === "agentic") {
+        return entries;
+    }
+    // Restricted to the 'without-docs' prompt. Unless explicitly disabled.
+    const baselineEnabled = task.baseline?.enabled !== false;
+    if (baselineEnabled) {
+        const rubricMode = task.baseline?.rubric ?? "abbreviated";
+        const baselineAsserts = buildBaselineAsserts(resolvedAsserts, rubricMode);
+        entries.push({
+            description: `${task.description} (baseline)`,
+            prompts: ["without-docs"],
+            vars: {
+                ...task.vars,
+                docs: "",
+            },
+            ...(baselineAsserts.length > 0 ? { assert: baselineAsserts } : {}),
+        });
+    }
+    return entries;
+}
+/**
+ * Convert a TaskDefinition (from @sanity/ailf-core) to the local
+ * SingleTaskDefinition format used by expandTask().
+ */
+function taskDefinitionToSingle(task) {
+    return {
+        assert: task.assertions.map((a) => ({ ...a })),
+        baseline: task.baseline,
+        description: task.description,
+        doc_coverage: task.docCoverage,
+        id: task.id,
+        vars: {
+            docs: `file://contexts/canonical/${task.id}.md`,
+            task: task.taskPrompt,
+            ...task.extraVars,
+        },
+    };
+}
+/**
+ * Expand an array of TaskDefinition[] (from any TaskSource adapter) into
+ * Promptfoo-compatible test entries. This is the TaskSource-aware counterpart
+ * of loadAndExpandTasks() — it skips YAML file I/O and works directly with
+ * the canonical domain type.
+ *
+ * @param tasks - Task definitions from any TaskSource adapter
+ * @param rootDir - Eval package root (needed to load rubric templates)
+ * @param mode - Expansion mode: 'baseline' (gold + baseline) or 'agentic' (gold only)
+ * @returns Expanded test entries and statistics
+ */
+export function expandTaskDefinitions(tasks, rootDir, mode = "baseline") {
+    const rubricConfig = loadRubricTemplates(rootDir);
+    const entries = [];
+    for (const task of tasks) {
+        const single = taskDefinitionToSingle(task);
+        entries.push(...expandTask(single, rubricConfig, mode));
+    }
+    return {
+        entries,
+        stats: {
+            expandedTotal: entries.length,
+            totalTasks: tasks.length,
+        },
+    };
+}
+/**
+ * Extract all task IDs from task files. Only works with the new
+ * single-definition format entries (those that have an `id` field).
+ */
+export function extractTaskIds(rootDir) {
+    const tasksDir = resolve(rootDir, "tasks");
+    if (!existsSync(tasksDir))
+        return [];
+    const yamlFiles = readdirSync(tasksDir)
+        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
+        .sort();
+    const ids = [];
+    for (const file of yamlFiles) {
+        const filePath = resolve(tasksDir, file);
+        const raw = readFileSync(filePath, "utf-8");
+        const parsed = load(raw);
+        if (!Array.isArray(parsed))
+            continue;
+        for (const entry of parsed) {
+            if (isSingleTaskDefinition(entry)) {
+                ids.push(entry.id);
+            }
+        }
+    }
+    return ids;
+}
+// ---------------------------------------------------------------------------
+// Core expansion logic
+// ---------------------------------------------------------------------------
+/**
+ * Type guard: checks if an entry is in the new single-definition format.
+ * The distinguishing feature is the presence of an `id` field.
+ */
+export function isSingleTaskDefinition(entry) {
+    return (typeof entry === "object" &&
+        entry !== null &&
+        "id" in entry &&
+        typeof entry.id === "string" &&
+        "description" in entry &&
+        "vars" in entry &&
+        "assert" in entry);
+}
+/**
+ * Type guard: checks if an assertion uses the templated format.
+ */
+export function isTemplatedAssert(entry) {
+    return (entry.type === "llm-rubric" &&
+        "template" in entry &&
+        typeof entry.template === "string" &&
+        "criteria" in entry &&
+        Array.isArray(entry.criteria));
+}
+// ---------------------------------------------------------------------------
+// Task file loading
+// ---------------------------------------------------------------------------
+/**
+ * Load and expand all task files from the tasks/ directory.
+ * Supports both the new single-definition format (has `id`) and the legacy
+ * paired format (no `id`). Legacy entries pass through unchanged.
+ *
+ * @param mode - Controls expansion behavior:
+ *   - `'baseline'` (default): Gold + baseline entries with prompt filters.
+ *   - `'agentic'`: Gold entries only, no prompt filters.
+ *
+ * Returns the expanded entries grouped by source file.
+ */
+export function loadAndExpandTasks(rootDir, filter, mode = "baseline") {
+    const tasksDir = resolve(rootDir, "tasks");
+    if (!existsSync(tasksDir)) {
+        // tasks/ may not exist when task definitions come from Content Lake
+        return {
+            entries: [],
+            stats: {
+                expandedTotal: 0,
+                legacyEntries: 0,
+                singleDefinitions: 0,
+                totalFiles: 0,
+            },
+        };
+    }
+    // Load rubric templates
+    const rubricConfig = loadRubricTemplates(rootDir);
+    let yamlFiles = readdirSync(tasksDir)
+        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
+        .sort();
+    // Apply area filter — area name = filename stem (e.g., "groq" matches "groq.yaml")
+    if (filter?.areas && filter.areas.length > 0) {
+        const allowedAreas = new Set(filter.areas.map((a) => a.toLowerCase()));
+        yamlFiles = yamlFiles.filter((f) => {
+            const stem = f.replace(/\.ya?ml$/, "").toLowerCase();
+            return allowedAreas.has(stem);
+        });
+    }
+    const entries = [];
+    let singleDefinitions = 0;
+    let legacyEntries = 0;
+    for (const file of yamlFiles) {
+        const filePath = resolve(tasksDir, file);
+        const raw = readFileSync(filePath, "utf-8");
+        let parsed = load(raw);
+        if (!Array.isArray(parsed)) {
+            throw new Error(`${file} did not parse to an array of tasks`);
+        }
+        // Apply task ID filter
+        if (filter?.taskIds && filter.taskIds.length > 0) {
+            const allowedIds = new Set(filter.taskIds);
+            parsed = parsed.filter((entry) => typeof entry === "object" &&
+                entry !== null &&
+                "id" in entry &&
+                allowedIds.has(entry.id));
+        }
+        for (const entry of parsed) {
+            if (isSingleTaskDefinition(entry)) {
+                singleDefinitions++;
+                entries.push(...expandTask(entry, rubricConfig, mode));
+            }
+            else {
+                legacyEntries++;
+                entries.push(entry);
+            }
+        }
+    }
+    if (filter?.areas || filter?.taskIds) {
+        console.log(`  Filter: ${filter.areas ? `areas=[${filter.areas.join(", ")}]` : ""}${filter.areas && filter.taskIds ? ", " : ""}${filter.taskIds ? `tasks=[${filter.taskIds.join(", ")}]` : ""}`);
+    }
+    return {
+        entries,
+        stats: {
+            expandedTotal: entries.length,
+            legacyEntries,
+            singleDefinitions,
+            totalFiles: yamlFiles.length,
+        },
+    };
+}
+/**
+ * Load and validate config/rubrics.yaml from the given root directory.
+ * Caches the result for subsequent calls with the same rootDir.
+ */
+export function loadRubricTemplates(rootDir) {
+    if (cachedRubricConfig)
+        return cachedRubricConfig;
+    const filePath = resolve(rootDir, "config", "rubrics.yaml");
+    if (!existsSync(filePath)) {
+        throw new Error(`config/rubrics.yaml not found at ${filePath}`);
+    }
+    const raw = readFileSync(filePath, "utf-8");
+    const parsed = load(raw);
+    const result = RubricConfigSchema.safeParse(parsed);
+    if (!result.success) {
+        const messages = result.error.issues
+            .map((i) => `  [${i.path.join(".")}]: ${i.message}`)
+            .join("\n");
+        throw new Error(`Invalid config/rubrics.yaml:\n${messages}`);
+    }
+    cachedRubricConfig = result.data;
+    return result.data;
+}
+/**
+ * Resolve a single assertion: if it's templated, assemble the rubric text
+ * and attach structured dimension metadata when the template has a
+ * `dimension` field. Otherwise, pass through unchanged.
+ */
+export function resolveAssert(entry, rubricConfig) {
+    if (isTemplatedAssert(entry)) {
+        const template = rubricConfig.templates[entry.template];
+        return {
+            type: "llm-rubric",
+            value: assembleRubric(entry.template, entry.criteria, rubricConfig),
+            ...(entry.weight !== undefined ? { weight: entry.weight } : {}),
+            // Structured dimension metadata (Approach 5)
+            ...(template?.dimension && {
+                metadata: {
+                    dimension: template.dimension,
+                    maxScore: 100,
+                },
+            }),
+        };
+    }
+    return entry;
+}

package/dist/pipeline/failure-modes.d.ts

@@ -0,0 +1,46 @@
+/**
+ * pipeline/failure-modes.ts
+ *
+ * Keyword-based failure mode classifier for grader reasoning text,
+ * cross-referenced with ceiling decomposition data.
+ *
+ * Phase 3a of the Scenario Matrix implementation.
+ *
+ * The classifier uses two signal sources:
+ * 1. Keyword matching on grader reason text (primary)
+ * 2. Ceiling decomposition structural signals (supplementary)
+ *
+ * When both sources agree, confidence is boosted. When only ceiling
+ * signals are available, they serve as a fallback for unclassified cases.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+import type { FailureMode, FailureModeReport, FeatureScore, GraderJudgment } from "./types.js";
+/**
+ * Build a complete failure mode report from grader judgments and scores.
+ *
+ * @param judgments - All grader judgments from the evaluation
+ * @param scores - Per-area feature scores (for ceiling decomposition)
+ * @returns Failure mode report with per-area breakdowns
+ */
+export declare function buildFailureModeReport(judgments: GraderJudgment[], scores: FeatureScore[]): FailureModeReport;
+/**
+ * Classify the failure mode of a low-scoring grader judgment.
+ *
+ * Uses keyword matching on the reason text, then cross-references with
+ * ceiling decomposition data for structural confirmation.
+ *
+ * @param judgment - The grader judgment to classify
+ * @param ceilingScore - The area's ceiling score (with-docs best case)
+ * @param floorScore - The area's floor score (no-docs baseline)
+ * @returns Classified failure mode with confidence level
+ */
+export declare function classifyFailureMode(judgment: GraderJudgment, ceilingScore: number, floorScore: number): FailureMode;
+/**
+ * Format a failure mode report for console output.
+ */
+export declare function formatFailureModesConsole(report: FailureModeReport): string;
+/**
+ * Format a failure mode report as markdown for PR comments.
+ */
+export declare function formatFailureModesMarkdown(report: FailureModeReport): string;