@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/generate-configs.js

@@ -1,6 +1,21 @@
 /**
  * pipeline/generate-configs.ts
  *
+ * @deprecated This is the LEGACY compilation path. New code should use the
+ * config compiler pipeline instead:
+ *
+ *   import { compileLiteracyTasks } from "./compiler/literacy-bridge.js"
+ *   import { buildTaskGraph, compileToPromptfoo } from "./compiler/index.js"
+ *
+ * This file is retained behind the `--legacy-compiler` CLI flag as an
+ * emergency fallback during the migration period. It will be removed once
+ * the new compiler has been validated in production.
+ *
+ * @see packages/eval/src/pipeline/compiler/ — the new compiler pipeline
+ * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ *
+ * ---
+ *
  * Reads config/models.yaml (the central model registry) and generates all
  * promptfoo config files with the correct provider entries.
  *
@@ -19,12 +34,15 @@
  * @see docs/exec-plans/eliminate-lib-layer.md
  */
 import { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.js";
-import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { existsSync, readdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
-import { dump, load } from "js-yaml";
+import { dump } from "js-yaml";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
+import { loadConfigFile } from "./compiler/config-loader.js";
+import { LITERACY_PROMPT_TEMPLATES } from "./compiler/mode-handlers/literacy-handler.js";
 import { expandTaskDefinitions, loadAndExpandTasks } from "./expand-tasks.js";
 import { validateModelsYaml } from "./validate.js";
+import { LiteracyVariant } from "./normalize-mode.js";
 import { loadSource } from "../sources.js";
 // Re-export pure functions from core for backward compatibility.
 // Tests and other modules that previously imported from lib/generate-configs
@@ -44,32 +62,43 @@ export function discoverTaskFiles(rootDir) {
         .sort()
         .map((f) => `file://tasks/${f}`);
 }
-/** Load prompt templates from config/prompts.yaml. Throws if missing or malformed. */
+/**
+ * Load prompt templates. Uses handler-owned literacy templates as defaults,
+ * with config/prompts.ts as an override layer for user customization.
+ */
 export function loadPrompts(rootDir) {
-    const promptsPath = resolve(rootDir, "config", "prompts.yaml");
-    if (!existsSync(promptsPath)) {
-        throw new Error(`config/prompts.yaml not found at ${promptsPath}. This file is required — it defines the prompt templates for all evaluation modes.`);
-    }
-    const raw = readFileSync(promptsPath, "utf-8");
-    const data = load(raw);
     const toPrompt = (entry) => ({
         id: entry.id,
         label: entry.label,
         raw: entry.template,
     });
-    if (!data["with-docs"] || !data["without-docs"] || !data["agentic"]) {
-        const missing = ["with-docs", "without-docs", "agentic"].filter((k) => !data[k]);
-        throw new Error(`config/prompts.yaml is missing required keys: ${missing.join(", ")}. Each prompt must have id, label, and template fields.`);
+    // Load user overrides from config/prompts (may be empty after Wave 4)
+    let overrides = {};
+    try {
+        const loaded = loadConfigFile("prompts", rootDir).data;
+        // config/prompts.ts may export a Record (legacy) or an empty array (post-Wave 4)
+        if (loaded && !Array.isArray(loaded)) {
+            overrides = loaded;
+        }
+    }
+    catch {
+        // No config/prompts file — use handler defaults only
     }
+    // Handler-owned templates are the canonical source; overrides take precedence
     return {
-        agentic: toPrompt(data["agentic"]),
-        withDocs: toPrompt(data["with-docs"]),
-        withoutDocs: toPrompt(data["without-docs"]),
+        agentic: overrides[LiteracyVariant.AGENTIC]
+            ? toPrompt(overrides[LiteracyVariant.AGENTIC])
+            : toPrompt(LITERACY_PROMPT_TEMPLATES[LiteracyVariant.AGENTIC]),
+        withDocs: overrides["with-docs"]
+            ? toPrompt(overrides["with-docs"])
+            : toPrompt(LITERACY_PROMPT_TEMPLATES["with-docs"]),
+        withoutDocs: overrides["without-docs"]
+            ? toPrompt(overrides["without-docs"])
+            : toPrompt(LITERACY_PROMPT_TEMPLATES["without-docs"]),
     };
 }
 function loadModels(rootDir) {
-    const raw = readFileSync(resolve(rootDir, "config", "models.yaml"), "utf-8");
-    return load(raw);
+    return loadConfigFile("models", rootDir).data;
 }
 // ---------------------------------------------------------------------------
 // Shared components
@@ -191,7 +220,7 @@ function generateAgenticConfig(models, tests, prompts, source, searchMode, allow
     };
 }
 function generateBaselineConfig(models, tests, prompts) {
-    const baselineModels = models.models.filter((m) => modelMatchesMode(m, "baseline"));
+    const baselineModels = models.models.filter((m) => modelMatchesMode(m, LiteracyVariant.STANDARD));
     const providers = baselineModels.map((model) => ({
         config: mergeConfig(models.defaults, model.config),
         id: model.id,
@@ -217,7 +246,7 @@ function generateBaselineConfig(models, tests, prompts) {
     };
 }
 function generateObservedConfig(models, tests, prompts) {
-    const observedModels = models.models.filter((m) => modelMatchesMode(m, "observed"));
+    const observedModels = models.models.filter((m) => modelMatchesMode(m, LiteracyVariant.OBSERVED));
     const providers = observedModels.map((model) => {
         const modelName = extractModelName(model.id);
         return {
@@ -293,7 +322,7 @@ export function generateConfigs(options) {
     const filter = options.filter?.areas || options.filter?.taskIds
         ? options.filter
         : undefined;
-    // Expand tasks — use TaskDefinition[] from TaskSource when provided,
+    // Expand tasks — use GeneralizedTaskDefinition[] from TaskSource when provided,
     // otherwise fall back to loading from tasks/*.yaml files.
     let entries;
     let agenticEntries;
@@ -303,16 +332,16 @@ export function generateConfigs(options) {
             taskCount: options.tasks.length,
             taskIds: options.tasks.map((t) => t.id),
         });
-        const baselineResult = expandTaskDefinitions(options.tasks, rootDir, "baseline");
+        const baselineResult = expandTaskDefinitions(options.tasks, rootDir, LiteracyVariant.STANDARD);
         entries = baselineResult.entries;
         log.info(`  Expanded ${baselineResult.stats.totalTasks} task(s) → ${baselineResult.stats.expandedTotal} test entries (from TaskSource)`);
-        const agenticResult = expandTaskDefinitions(options.tasks, rootDir, "agentic");
+        const agenticResult = expandTaskDefinitions(options.tasks, rootDir, LiteracyVariant.AGENTIC);
         agenticEntries = agenticResult.entries;
         log.info(`  Agentic: ${agenticResult.stats.expandedTotal} entries (gold only, no baseline)`);
     }
     else {
         // Legacy path — read from tasks/*.yaml files
-        const { entries: baselineEntries, stats } = loadAndExpandTasks(rootDir, filter, "baseline", log);
+        const { entries: baselineEntries, stats } = loadAndExpandTasks(rootDir, filter, LiteracyVariant.STANDARD, log);
         entries = baselineEntries;
         log.info(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
         if (stats.legacyEntries > 0) {
@@ -328,7 +357,7 @@ export function generateConfigs(options) {
             }
             log.info(`  Scoped to: ${parts.join("; ")}`);
         }
-        const { entries: agenticFromYaml, stats: agenticStats } = loadAndExpandTasks(rootDir, filter, "agentic", log);
+        const { entries: agenticFromYaml, stats: agenticStats } = loadAndExpandTasks(rootDir, filter, LiteracyVariant.AGENTIC, log);
         agenticEntries = agenticFromYaml;
         log.info(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
     }

package/dist/pipeline/grader-api.d.ts

@@ -7,7 +7,7 @@
  * grader model prefix. Reads the appropriate API key from environment.
  *
  * Also exports `loadGraderModel()` to resolve the grader from
- * `config/models.yaml`.
+ * `config/models`.
  *
  * Migrated from lib/grader-api.ts — no module-level side effects, no
  * process.exit(), accepts rootDir as parameter for file-based operations.
@@ -26,11 +26,11 @@ interface ProviderConfig {
  */
 export declare function gradeOnce(graderModel: string, responseText: string, rubricText: string, logger?: Logger): Promise<null | number>;
 /**
- * Load the grader model from `config/models.yaml`.
+ * Load the grader model from `config/models`.
  * Returns both the model ID and human-readable label.
  * Falls back to `openai:gpt-5` if not configured.
  *
- * @throws Error if config/models.yaml is not found
+ * @throws Error if config/models is not found
  */
 export declare function loadGraderModel(rootDir: string): {
     id: string;

package/dist/pipeline/grader-api.js

@@ -7,15 +7,13 @@
  * grader model prefix. Reads the appropriate API key from environment.
  *
  * Also exports `loadGraderModel()` to resolve the grader from
- * `config/models.yaml`.
+ * `config/models`.
  *
  * Migrated from lib/grader-api.ts — no module-level side effects, no
  * process.exit(), accepts rootDir as parameter for file-based operations.
  */
-import { existsSync, readFileSync } from "fs";
-import { join } from "path";
-import { load } from "js-yaml";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
+import { loadConfigFile } from "./compiler/config-loader.js";
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -63,19 +61,14 @@ ${rubricText}
     }
 }
 /**
- * Load the grader model from `config/models.yaml`.
+ * Load the grader model from `config/models`.
  * Returns both the model ID and human-readable label.
  * Falls back to `openai:gpt-5` if not configured.
  *
- * @throws Error if config/models.yaml is not found
+ * @throws Error if config/models is not found
  */
 export function loadGraderModel(rootDir) {
-    const modelsPath = join(rootDir, "config", "models.yaml");
-    if (!existsSync(modelsPath)) {
-        throw new Error(`config/models.yaml not found at ${modelsPath}`);
-    }
-    const raw = readFileSync(modelsPath, "utf-8");
-    const data = load(raw);
+    const data = loadConfigFile("models", rootDir).data;
     return {
         id: data?.grader?.id ?? "openai:gpt-5",
         label: data?.grader?.label ?? "GPT-5 (grader)",

package/dist/pipeline/grader-compare-runner.js

@@ -14,8 +14,8 @@
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
-import { load } from "js-yaml";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
+import { loadConfigFile } from "./compiler/config-loader.js";
 import { compareGraders, } from "./grader-comparison.js";
 import { classifyCorrelation } from "./grader-validation.js";
 import { gradeOnce } from "./grader-api.js";
@@ -23,26 +23,20 @@ import { gradeOnce } from "./grader-api.js";
 // Internal helpers
 // ---------------------------------------------------------------------------
 function classifyDimension(component) {
+    // Prefer structured metadata — pass through any dimension name directly,
+    // enabling non-literacy profiles (MCP, agent, knowledge-probe)
     const metadata = component.assertion?.metadata;
     if (metadata?.dimension) {
-        switch (metadata.dimension) {
-            case "code-correctness":
-                return "codeCorrectness";
-            case "doc-coverage":
-                return "docCoverage";
-            case "task-completion":
-                return "taskCompletion";
-            default:
-                return null;
-        }
+        return metadata.dimension;
     }
+    // Fallback: heuristic name matching (returns kebab-case)
     const value = (component.assertion?.value ?? "").toLowerCase();
     if (value.includes("task completion"))
-        return "taskCompletion";
+        return "task-completion";
     if (value.includes("code correctness"))
-        return "codeCorrectness";
+        return "code-correctness";
     if (value.includes("documentation coverage") || value.includes("hallucinate"))
-        return "docCoverage";
+        return "doc-coverage";
     return null;
 }
 function detectFeatureArea(description) {
@@ -101,15 +95,10 @@ function extractJudgments(file) {
 }
 /**
  * Load config: resolve baseline grader and candidate graders.
- * Candidate overrides take precedence over config/models.yaml.
+ * Candidate overrides take precedence over config/models.
  */
 function loadConfig(rootDir, candidateOverrides) {
-    const modelsPath = join(rootDir, "config", "models.yaml");
-    if (!existsSync(modelsPath)) {
-        throw new Error(`config/models.yaml not found at ${modelsPath}`);
-    }
-    const raw = readFileSync(modelsPath, "utf-8");
-    const data = load(raw);
+    const data = loadConfigFile("models", rootDir).data;
     const baseline = {
         id: data?.grader?.id ?? "openai:gpt-5",
         label: data?.grader?.label ?? "GPT-5 (grader)",
@@ -158,11 +147,15 @@ export function formatComparisonReport(result) {
         const sep = "|------------------|-------------|--------|---------|-------|";
         lines.push(h);
         lines.push(sep);
-        const dims = [
-            { data: pair.perDimension.taskCompletion, name: "Task Completion" },
-            { data: pair.perDimension.codeCorrectness, name: "Code Correctness" },
-            { data: pair.perDimension.docCoverage, name: "Doc Coverage" },
-        ];
+        // Derive display rows dynamically from whatever dimensions are present
+        const dims = Object.entries(pair.perDimension).map(([key, data]) => ({
+            data,
+            // kebab-case → Title Case (e.g. 'task-completion' → 'Task Completion')
+            name: key
+                .split("-")
+                .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+                .join(" "),
+        }));
         for (const { data, name } of dims) {
             const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
             lines.push(`| ${name.padEnd(16)} | r=${String(data.correlation).padStart(9)} | ${biasStr.padStart(6)} | ${String(data.meanAbsDiff).padStart(7)} | ${String(data.count).padStart(5)} |`);
@@ -208,7 +201,7 @@ export async function runGraderCompare(options) {
     const { baseline, candidates } = loadConfig(rootDir, options.candidates);
     if (candidates.length === 0) {
         throw new Error("No candidate graders configured. " +
-            "Add grader-candidates to config/models.yaml or pass --candidate.");
+            "Add grader-candidates to config/models or pass --candidate.");
     }
     // Load eval results
     if (!existsSync(resultsPath)) {

package/dist/pipeline/grader-comparison.d.ts

@@ -51,12 +51,8 @@ export interface GraderPairComparison {
     graderB: string;
     /** Mean absolute difference between scores */
     meanAbsDiff: number;
-    /** Per-dimension comparisons */
-    perDimension: {
-        taskCompletion: DimensionPairComparison;
-        codeCorrectness: DimensionPairComparison;
-        docCoverage: DimensionPairComparison;
-    };
+    /** Per-dimension comparisons (keyed by kebab-case dimension name) */
+    perDimension: Record<string, DimensionPairComparison>;
 }
 /** Recommendation for a candidate grader */
 export interface GraderRecommendation {
@@ -71,8 +67,8 @@ export interface GraderRecommendation {
 export interface GraderScore {
     /** Feature area (e.g., "groq") */
     area: string;
-    /** Which scoring dimension */
-    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Which scoring dimension (kebab-case, e.g. 'task-completion') */
+    dimension: string;
     /** Score assigned by this grader (0–100) */
     score: number;
     /** Task ID (e.g., "groq-blog-queries") */

package/dist/pipeline/grader-comparison.js

@@ -68,16 +68,9 @@ function comparePair(a, b) {
     // Find paired observations (present in both graders)
     const pairedA = [];
     const pairedB = [];
-    const dimPairsA = {
-        codeCorrectness: [],
-        docCoverage: [],
-        taskCompletion: [],
-    };
-    const dimPairsB = {
-        codeCorrectness: [],
-        docCoverage: [],
-        taskCompletion: [],
-    };
+    // Group by dimension dynamically — works with any dimension names
+    const dimPairsA = {};
+    const dimPairsB = {};
     for (const sA of a.scores) {
         const key = `${sA.taskId}::${sA.dimension}`;
         const scoreB = bScoreMap.get(key);
@@ -85,8 +78,13 @@ function comparePair(a, b) {
             continue;
         pairedA.push(sA.score);
         pairedB.push(scoreB);
-        dimPairsA[sA.dimension].push(sA.score);
-        dimPairsB[sA.dimension].push(scoreB);
+        (dimPairsA[sA.dimension] ??= []).push(sA.score);
+        (dimPairsB[sA.dimension] ??= []).push(scoreB);
+    }
+    // Build perDimension from all dimensions observed in paired data
+    const perDimension = {};
+    for (const dim of Object.keys(dimPairsA)) {
+        perDimension[dim] = computeDimensionPair(dimPairsA[dim], dimPairsB[dim]);
     }
     return {
         bias: computeBias(pairedA, pairedB),
@@ -94,11 +92,7 @@ function comparePair(a, b) {
         graderA: a.modelId,
         graderB: b.modelId,
         meanAbsDiff: computeMeanAbsDiff(pairedA, pairedB),
-        perDimension: {
-            codeCorrectness: computeDimensionPair(dimPairsA.codeCorrectness, dimPairsB.codeCorrectness),
-            docCoverage: computeDimensionPair(dimPairsA.docCoverage, dimPairsB.docCoverage),
-            taskCompletion: computeDimensionPair(dimPairsA.taskCompletion, dimPairsB.taskCompletion),
-        },
+        perDimension,
     };
 }
 /** Mean signed difference (B - A). Positive = B scores higher. */

package/dist/pipeline/grader-consistency-runner.d.ts

@@ -17,7 +17,6 @@
 import { type Logger } from "../_vendor/ailf-core/index.d.ts";
 import type { RawPromptfooFile } from "./calculate-scores.js";
 import { type GraderConsistency } from "./grader-consistency.js";
-import type { DimensionName } from "./types.js";
 /** Options for the grader consistency runner. */
 export interface GraderConsistencyRunnerOptions {
     /** Logger for structured output. Falls back to ConsoleLogger if omitted. */
@@ -34,8 +33,8 @@ interface GradingJudgment {
     area: string;
     /** Task description */
     description: string;
-    /** Scoring dimension */
-    dimension: DimensionName;
+    /** Scoring dimension (kebab-case, e.g. 'task-completion') */
+    dimension: string;
     /** The original score from the eval run */
     originalScore: number;
     /** Provider (model under test) */

package/dist/pipeline/grader-consistency-runner.js

@@ -23,28 +23,20 @@ import { analyzeConsistency, } from "./grader-consistency.js";
 // Rubric dimension classification (similar to calculate-scores)
 // ---------------------------------------------------------------------------
 function classifyDimension(component) {
-    // Prefer structured metadata
+    // Prefer structured metadata — pass through any dimension name directly,
+    // enabling non-literacy profiles (MCP, agent, knowledge-probe)
     const metadata = component.assertion?.metadata;
     if (metadata?.dimension) {
-        switch (metadata.dimension) {
-            case "code-correctness":
-                return "codeCorrectness";
-            case "doc-coverage":
-                return "docCoverage";
-            case "task-completion":
-                return "taskCompletion";
-            default:
-                return null;
-        }
+        return metadata.dimension;
     }
-    // Fallback: heuristic name matching
+    // Fallback: heuristic name matching (returns kebab-case)
     const value = (component.assertion?.value ?? "").toLowerCase();
     if (value.includes("task completion"))
-        return "taskCompletion";
+        return "task-completion";
     if (value.includes("code correctness"))
-        return "codeCorrectness";
+        return "code-correctness";
     if (value.includes("documentation coverage") || value.includes("hallucinate"))
-        return "docCoverage";
+        return "doc-coverage";
     return null;
 }
 // ---------------------------------------------------------------------------
@@ -140,11 +132,15 @@ export function formatConsistencyReport(result, graderModel) {
     const sep = "|------------------|-------|-------|-----------|-----------| ";
     lines.push(h);
     lines.push(sep);
-    const dims = [
-        { data: result.perDimension.taskCompletion, name: "Task Completion" },
-        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
-        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
-    ];
+    // Derive display rows dynamically from whatever dimensions are present
+    const dims = Object.entries(result.perDimension).map(([key, data]) => ({
+        data,
+        // kebab-case → Title Case (e.g. 'task-completion' → 'Task Completion')
+        name: key
+            .split("-")
+            .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+            .join(" "),
+    }));
     for (const { data, name } of dims) {
         lines.push(`| ${name.padEnd(16)} | ${String(data.avgStdDev).padStart(5)} | ${String(data.maxStdDev).padStart(5)} | ${String(data.avgRange).padStart(9)} | ${String(data.judgmentCount).padStart(9)} |`);
     }

package/dist/pipeline/grader-consistency.d.ts

@@ -35,12 +35,8 @@ export interface GraderConsistency {
     judgments: JudgmentConsistency[];
     /** Maximum standard deviation observed (worst-case noise) */
     maxStdDev: number;
-    /** Per-dimension consistency */
-    perDimension: {
-        taskCompletion: DimensionConsistency;
-        codeCorrectness: DimensionConsistency;
-        docCoverage: DimensionConsistency;
-    };
+    /** Per-dimension consistency (keyed by kebab-case dimension name) */
+    perDimension: Record<string, DimensionConsistency>;
     /** Recommended noise threshold for comparisons (2× max dimension avgStdDev) */
     recommendedThreshold: number;
     /** Number of replications per judgment */
@@ -52,8 +48,8 @@ export interface GraderConsistency {
 export interface JudgmentConsistency {
     /** Feature area */
     area: string;
-    /** Scoring dimension */
-    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Scoring dimension (kebab-case, e.g. 'task-completion') */
+    dimension: string;
     /** Max score observed */
     max: number;
     /** Mean score across replications */
@@ -75,8 +71,8 @@ export interface JudgmentConsistency {
 export interface ReplicatedGrading {
     /** Feature area (derived from task description) */
     area: string;
-    /** Which scoring dimension this rubric measures */
-    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Which scoring dimension this rubric measures (kebab-case, e.g. 'task-completion') */
+    dimension: string;
     /** Provider (model under test) that produced the original response */
     providerId?: string;
     /** The scores from each replication (length = N replications) */

package/dist/pipeline/grader-consistency.js

@@ -31,26 +31,7 @@ export function analyzeConsistency(gradings) {
             generatedAt: new Date().toISOString(),
             judgments: [],
             maxStdDev: 0,
-            perDimension: {
-                codeCorrectness: {
-                    avgRange: 0,
-                    avgStdDev: 0,
-                    judgmentCount: 0,
-                    maxStdDev: 0,
-                },
-                docCoverage: {
-                    avgRange: 0,
-                    avgStdDev: 0,
-                    judgmentCount: 0,
-                    maxStdDev: 0,
-                },
-                taskCompletion: {
-                    avgRange: 0,
-                    avgStdDev: 0,
-                    judgmentCount: 0,
-                    maxStdDev: 0,
-                },
-            },
+            perDimension: {},
             recommendedThreshold: 0,
             replications: 0,
             totalJudgments: 0,
@@ -58,17 +39,16 @@ export function analyzeConsistency(gradings) {
     }
     // Analyze each judgment
     const judgments = gradings.map(analyzeJudgment);
-    // Group by dimension
-    const byDimension = {
-        codeCorrectness: judgments.filter((j) => j.dimension === "codeCorrectness"),
-        docCoverage: judgments.filter((j) => j.dimension === "docCoverage"),
-        taskCompletion: judgments.filter((j) => j.dimension === "taskCompletion"),
-    };
-    const perDimension = {
-        codeCorrectness: aggregateDimension(byDimension.codeCorrectness),
-        docCoverage: aggregateDimension(byDimension.docCoverage),
-        taskCompletion: aggregateDimension(byDimension.taskCompletion),
-    };
+    // Group by dimension dynamically — works with any dimension names
+    const byDimension = {};
+    for (const j of judgments) {
+        ;
+        (byDimension[j.dimension] ??= []).push(j);
+    }
+    const perDimension = {};
+    for (const [dim, dimJudgments] of Object.entries(byDimension)) {
+        perDimension[dim] = aggregateDimension(dimJudgments);
+    }
     // Overall stats
     const allStdDevs = judgments.map((j) => j.stdDev);
     const allRanges = judgments.map((j) => j.range);
@@ -76,7 +56,8 @@ export function analyzeConsistency(gradings) {
     // Recommended threshold: 2× the worst (highest) per-dimension avgStdDev.
     // This means a comparison delta must exceed 2σ of the noisiest dimension
     // to be classified as a real change rather than grader variance.
-    const maxDimensionAvgStdDev = Math.max(perDimension.taskCompletion.avgStdDev, perDimension.codeCorrectness.avgStdDev, perDimension.docCoverage.avgStdDev);
+    const dimAvgStdDevs = Object.values(perDimension).map((d) => d.avgStdDev);
+    const maxDimensionAvgStdDev = dimAvgStdDevs.length > 0 ? Math.max(...dimAvgStdDevs) : 0;
     const recommendedThreshold = Math.ceil(maxDimensionAvgStdDev * 2);
     // Sort judgments by stdDev descending (noisiest first)
     const sortedJudgments = [...judgments].sort((a, b) => b.stdDev - a.stdDev);

package/dist/pipeline/grader-sensitivity-runner.js

@@ -119,11 +119,13 @@ export function formatSensitivityReport(result) {
     const sep = "|------------------|-------------|---------|-------|-------|";
     lines.push(h);
     lines.push(sep);
-    const dims = [
-        { data: result.perDimension.taskCompletion, name: "Task Completion" },
-        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
-        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
-    ];
+    const dims = Object.entries(result.perDimension).map(([key, data]) => ({
+        data,
+        name: key
+            .split(/[-_]/)
+            .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+            .join(" "),
+    }));
     for (const { data, name } of dims) {
         lines.push(`| ${name.padEnd(16)} | ${String(data.concordanceRate + "%").padStart(11)} | ${String(data.avgSeparation).padStart(7)} | ${String(data.tiedRate + "%").padStart(5)} | ${String(data.pairCount).padStart(5)} |`);
     }

package/dist/pipeline/grader-sensitivity.d.ts

@@ -58,12 +58,8 @@ export interface GraderSensitivityResult {
     generatedAt: string;
     /** Grader model used */
     graderModel: string;
-    /** Per-dimension sensitivity metrics */
-    perDimension: {
-        taskCompletion: DimensionSensitivity;
-        codeCorrectness: DimensionSensitivity;
-        docCoverage: DimensionSensitivity;
-    };
+    /** Per-dimension sensitivity metrics (keyed by dimension name) */
+    perDimension: Record<string, DimensionSensitivity>;
     /** Total paired comparisons analyzed */
     totalPairs: number;
 }

package/dist/pipeline/grader-sensitivity.js

@@ -30,11 +30,15 @@ export function analyzeSensitivity(pairs, graderModel) {
     // Overall concordance and separation
     const { avgSeparation, concordanceRate, tiedRate: _tiedRate, } = computeMetrics(pairs);
     // Per-dimension (based on the grading dimension, not the target dimension)
-    const perDimension = {
-        codeCorrectness: computeMetrics(pairs.filter((p) => p.dimension === "codeCorrectness")),
-        docCoverage: computeMetrics(pairs.filter((p) => p.dimension === "docCoverage")),
-        taskCompletion: computeMetrics(pairs.filter((p) => p.dimension === "taskCompletion")),
-    };
+    const dimGroups = {};
+    for (const p of pairs) {
+        ;
+        (dimGroups[p.dimension] ??= []).push(p);
+    }
+    const perDimension = {};
+    for (const [dim, dimPairs] of Object.entries(dimGroups)) {
+        perDimension[dim] = computeMetrics(dimPairs);
+    }
     // Cross-dimension: on-target (dimension matches targetDimension) vs off-target
     const onTargetPairs = pairs.filter((p) => p.dimension === p.targetDimension);
     const offTargetPairs = pairs.filter((p) => p.dimension !== p.targetDimension);
@@ -130,11 +134,7 @@ function emptyResult(graderModel) {
         failedPairs: [],
         generatedAt: new Date().toISOString(),
         graderModel,
-        perDimension: {
-            codeCorrectness: emptyDim,
-            docCoverage: emptyDim,
-            taskCompletion: emptyDim,
-        },
+        perDimension: {},
         totalPairs: 0,
     };
 }