@sanity/ailf 0.1.0

package/dist/pipeline/generate-configs.js

@@ -0,0 +1,395 @@
+/**
+ * pipeline/generate-configs.ts
+ *
+ * Reads config/models.yaml (the central model registry) and generates all
+ * promptfoo config files with the correct provider entries.
+ *
+ * This keeps model definitions in one place — add a model to config/models.yaml
+ * and run `pnpm generate-configs` to propagate it to all eval modes.
+ *
+ * Generated configs:
+ *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
+ *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
+ *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
+ *
+ * All functions accept rootDir as a parameter — no module-level constants.
+ * No process.argv parsing. No env var fallbacks. Callers provide typed options.
+ *
+ * @see config/models.yaml — the central model registry
+ * @see docs/exec-plans/active/eliminate-lib-layer.md
+ */
+import { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.js";
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { resolve } from "path";
+import { dump, load } from "js-yaml";
+import { expandTaskDefinitions, loadAndExpandTasks } from "./expand-tasks.js";
+import { validateModelsYaml } from "./validate.js";
+import { loadSource } from "../sources.js";
+// Re-export pure functions from core for backward compatibility.
+// Tests and other modules that previously imported from lib/generate-configs
+// can import from pipeline/generate-configs instead.
+export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Auto-discover all task YAML files in the tasks/ directory. */
+export function discoverTaskFiles(rootDir) {
+    const tasksDir = resolve(rootDir, "tasks");
+    if (!existsSync(tasksDir)) {
+        return []; // tasks may come from Content Lake instead
+    }
+    return readdirSync(tasksDir)
+        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
+        .sort()
+        .map((f) => `file://tasks/${f}`);
+}
+/** Load prompt templates from config/prompts.yaml. Throws if missing or malformed. */
+export function loadPrompts(rootDir) {
+    const promptsPath = resolve(rootDir, "config", "prompts.yaml");
+    if (!existsSync(promptsPath)) {
+        throw new Error(`config/prompts.yaml not found at ${promptsPath}. This file is required — it defines the prompt templates for all evaluation modes.`);
+    }
+    const raw = readFileSync(promptsPath, "utf-8");
+    const data = load(raw);
+    const toPrompt = (entry) => ({
+        id: entry.id,
+        label: entry.label,
+        raw: entry.template,
+    });
+    if (!data["with-docs"] || !data["without-docs"] || !data["agentic"]) {
+        const missing = ["with-docs", "without-docs", "agentic"].filter((k) => !data[k]);
+        throw new Error(`config/prompts.yaml is missing required keys: ${missing.join(", ")}. Each prompt must have id, label, and template fields.`);
+    }
+    return {
+        agentic: toPrompt(data["agentic"]),
+        withDocs: toPrompt(data["with-docs"]),
+        withoutDocs: toPrompt(data["without-docs"]),
+    };
+}
+function loadModels(rootDir) {
+    const raw = readFileSync(resolve(rootDir, "config", "models.yaml"), "utf-8");
+    return load(raw);
+}
+// ---------------------------------------------------------------------------
+// Shared components
+// ---------------------------------------------------------------------------
+const URL_EXTRACTION_ASSERT = {
+    type: "javascript",
+    value: `const urlPattern = /https?:\\/\\/[^\\s\\)\\"\\'\\\`>]+/g;
+const urls = [...new Set((output.match(urlPattern) || []))];
+const sanityUrls = urls.filter(u => u.includes('sanity.io'));
+return {
+  pass: true,
+  score: 1,
+  reason: JSON.stringify({
+    sanityUrls,
+    otherUrls: urls.filter(u => !u.includes('sanity.io')),
+    totalUrlCount: urls.length,
+    sanityUrlCount: sanityUrls.length
+  })
+};`,
+    weight: 0,
+};
+/**
+ * Source isolation assertion — advisory (weight: 0).
+ * Verifies that the agentic provider only fetched docs from allowed origins.
+ * Only injected when origin sandboxing is active.
+ */
+const SOURCE_ISOLATION_ASSERT = {
+    metadata: { dimension: "source-isolation" },
+    type: "javascript",
+    value: "file://dist/assertions/source-isolation.js",
+    weight: 0,
+};
+// ---------------------------------------------------------------------------
+// Config generators
+// ---------------------------------------------------------------------------
+function generateAgenticConfig(models, tests, prompts, source, searchMode, allowedOrigins) {
+    const naiveModels = models.models.filter((m) => modelMatchesMode(m, "agentic-naive"));
+    const optimizedModels = models.models.filter((m) => modelMatchesMode(m, "agentic-optimized"));
+    const providers = [];
+    // Build doc source config to inject into providers
+    const resolvedSearchMode = searchMode ?? "open";
+    const sourceConfig = source
+        ? {
+            ...(source.allowedOrigins?.length
+                ? { allowedOrigins: source.allowedOrigins }
+                : {}),
+            docBaseUrl: source.baseUrl,
+            ...(source.headers && Object.keys(source.headers).length > 0
+                ? { customHeaders: source.headers }
+                : {}),
+            llmsTxtUrl: source.llmsTxt,
+            ...(source.priorityDomain
+                ? { priorityDomain: source.priorityDomain }
+                : {}),
+            // Tool access control: search mode for web_search behavior
+            ...(resolvedSearchMode !== "open"
+                ? { searchMode: resolvedSearchMode }
+                : {}),
+        }
+        : {};
+    for (const model of naiveModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "naive",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Naive Agent)`,
+        });
+    }
+    for (const model of optimizedModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "optimized",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Optimized Agent)`,
+        });
+    }
+    // Inject source isolation assertion when origin sandboxing is active
+    const hasOriginSandbox = Boolean(allowedOrigins?.length);
+    const agenticAssertions = hasOriginSandbox ? [SOURCE_ISOLATION_ASSERT] : [];
+    return {
+        commandLineOptions: { table: false },
+        defaultTest: {
+            ...(agenticAssertions.length > 0 ? { assert: agenticAssertions } : {}),
+            options: {
+                provider: models.grader.id,
+                rubricProvider: models.grader.id,
+            },
+        },
+        description: "Sanity AI Literacy Evaluation — Agentic (naive vs optimized)",
+        ...(models.maxConcurrency
+            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
+            : {}),
+        outputPath: "results/latest/eval-results-agentic.json",
+        prompts: [prompts.agentic],
+        providers,
+        tests,
+    };
+}
+function generateBaselineConfig(models, tests, prompts) {
+    const baselineModels = models.models.filter((m) => modelMatchesMode(m, "baseline"));
+    const providers = baselineModels.map((model) => ({
+        config: mergeConfig(models.defaults, model.config),
+        id: model.id,
+        label: model.label,
+    }));
+    return {
+        commandLineOptions: { table: false },
+        defaultTest: {
+            assert: [URL_EXTRACTION_ASSERT],
+            options: {
+                provider: models.grader.id,
+                rubricProvider: models.grader.id,
+            },
+        },
+        description: "Sanity AI Literacy Evaluation — Baseline",
+        ...(models.maxConcurrency
+            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
+            : {}),
+        outputPath: "results/latest/eval-results.json",
+        prompts: [prompts.withDocs, prompts.withoutDocs],
+        providers,
+        tests,
+    };
+}
+function generateObservedConfig(models, tests, prompts) {
+    const observedModels = models.models.filter((m) => modelMatchesMode(m, "observed"));
+    const providers = observedModels.map((model) => {
+        const modelName = extractModelName(model.id);
+        return {
+            config: {
+                ...mergeConfig(models.defaults, model.config),
+                modelName,
+                observe: true,
+                recordOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/provider.js",
+            label: `${model.label} (Observed)`,
+        };
+    });
+    return {
+        commandLineOptions: { table: false },
+        defaultTest: {
+            options: {
+                provider: models.grader.id,
+                rubricProvider: models.grader.id,
+            },
+        },
+        description: "Sanity AI Literacy Evaluation — Observed",
+        ...(models.maxConcurrency
+            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
+            : {}),
+        outputPath: "results/latest/eval-results-observed.json",
+        prompts: [prompts.withDocs, prompts.withoutDocs],
+        providers,
+        tests,
+    };
+}
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+/**
+ * Generate Promptfoo config files from models.yaml + task definitions.
+ *
+ * All parameters are passed via the typed options object — no process.argv
+ * parsing or env var fallbacks. Callers (command handlers, orchestration
+ * steps) are responsible for resolving options from their own context.
+ */
+export function generateConfigs(options) {
+    const { rootDir } = options;
+    // Validate config/models.yaml before generating configs
+    const modelIssues = validateModelsYaml(rootDir);
+    const modelErrors = modelIssues.filter((i) => i.severity === "error");
+    if (modelErrors.length > 0) {
+        console.error("❌ config/models.yaml validation failed:");
+        for (const e of modelErrors) {
+            console.error(`  ERROR: ${e.message}`);
+            if (e.path) {
+                console.error(`         at ${e.path}`);
+            }
+        }
+        console.error("\nFix config/models.yaml before generating configs. Run 'pnpm validate' for details.");
+        process.exit(1);
+    }
+    console.log("Loading config/models.yaml...");
+    const models = loadModels(rootDir);
+    const activeModels = models.models.filter((m) => m.id && m.label);
+    console.log(`  Found ${activeModels.length} active model(s):`);
+    for (const m of activeModels) {
+        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty array join → "all"
+        const modes = m.modes?.join(", ") || "all";
+        console.log(`    - ${m.label} (${m.id}) → [${modes}]`);
+    }
+    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty label falls through to id
+    console.log(`  Grader: ${models.grader.label || models.grader.id}`);
+    // Build filter from options
+    const filter = options.filter?.areas || options.filter?.taskIds
+        ? options.filter
+        : undefined;
+    // Expand tasks — use TaskDefinition[] from TaskSource when provided,
+    // otherwise fall back to loading from tasks/*.yaml files.
+    let entries;
+    let agenticEntries;
+    if (options.tasks) {
+        // TaskSource path — tasks already loaded and filtered by the adapter
+        const baselineResult = expandTaskDefinitions(options.tasks, rootDir, "baseline");
+        entries = baselineResult.entries;
+        console.log(`  Expanded ${baselineResult.stats.totalTasks} task(s) → ${baselineResult.stats.expandedTotal} test entries (from TaskSource)`);
+        const agenticResult = expandTaskDefinitions(options.tasks, rootDir, "agentic");
+        agenticEntries = agenticResult.entries;
+        console.log(`  Agentic: ${agenticResult.stats.expandedTotal} entries (gold only, no baseline)`);
+    }
+    else {
+        // Legacy path — read from tasks/*.yaml files
+        const { entries: baselineEntries, stats } = loadAndExpandTasks(rootDir, filter, "baseline");
+        entries = baselineEntries;
+        console.log(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
+        if (stats.legacyEntries > 0) {
+            console.log(`  ⚠ ${stats.legacyEntries} legacy (paired) entries passed through unchanged`);
+        }
+        if (filter) {
+            const parts = [];
+            if (filter.areas) {
+                parts.push(`areas: ${filter.areas.join(", ")}`);
+            }
+            if (filter.taskIds) {
+                parts.push(`tasks: ${filter.taskIds.join(", ")}`);
+            }
+            console.log(`  Scoped to: ${parts.join("; ")}`);
+        }
+        const { entries: agenticFromYaml, stats: agenticStats } = loadAndExpandTasks(rootDir, filter, "agentic");
+        agenticEntries = agenticFromYaml;
+        console.log(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
+    }
+    // Write expanded tasks to generated files for Promptfoo to consume
+    const expandedPath = resolve(rootDir, "tasks", ".expanded.yaml");
+    const expandedYaml = dump(entries, {
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        quotingType: "'",
+    });
+    writeFileSync(expandedPath, `# .expanded.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${expandedYaml}`, "utf-8");
+    console.log(`  ✓ tasks/.expanded.yaml (${entries.length} entries)`);
+    const agenticExpandedPath = resolve(rootDir, "tasks", ".expanded.agentic.yaml");
+    const agenticExpandedYaml = dump(agenticEntries, {
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        quotingType: "'",
+    });
+    writeFileSync(agenticExpandedPath, `# .expanded.agentic.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Gold entries only (no baseline) for agentic evaluation mode.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${agenticExpandedYaml}`, "utf-8");
+    console.log(`  ✓ tasks/.expanded.agentic.yaml (${agenticEntries.length} entries)`);
+    const taskFiles = ["file://tasks/.expanded.yaml"];
+    const agenticTaskFiles = ["file://tasks/.expanded.agentic.yaml"];
+    // Load prompt templates
+    const prompts = loadPrompts(rootDir);
+    console.log(`  Loaded prompts: ${Object.keys(prompts).join(", ")}`);
+    // Load optional documentation source configuration
+    // Pre-resolved source wins over name-based lookup
+    let source = options.resolvedSource;
+    const sourceName = options.source;
+    if (!source && sourceName) {
+        console.log(`\nLoading source: ${sourceName}`);
+        try {
+            source = loadSource(sourceName);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`\n⚠ Failed to load source "${sourceName}": ${msg}`);
+        }
+    }
+    if (source) {
+        console.log(`  Base URL: ${source.baseUrl}`);
+        console.log(`  Dataset: ${source.dataset}`);
+        if (source.allowedOrigins?.length) {
+            console.log(`  Allowed origins: ${source.allowedOrigins.join(", ")}`);
+        }
+    }
+    console.log("\nGenerating configs...");
+    writeConfig(rootDir, "promptfooconfig.yaml", generateBaselineConfig(models, taskFiles, prompts), `# promptfooconfig.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
+    writeConfig(rootDir, "promptfooconfig.observed.yaml", generateObservedConfig(models, taskFiles, prompts), `# promptfooconfig.observed.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
+    writeConfig(rootDir, "promptfooconfig.agentic.yaml", generateAgenticConfig(models, agenticTaskFiles, prompts, source, options.searchMode, options.allowedOrigins), `# promptfooconfig.agentic.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
+    console.log("\nDone! Configs are ready.");
+    if (source) {
+        console.log(`  (using doc source: ${sourceName})`);
+    }
+}
+// ---------------------------------------------------------------------------
+// File writing
+// ---------------------------------------------------------------------------
+function writeConfig(rootDir, filename, config, header) {
+    const yamlStr = dump(config, {
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        quotingType: "'",
+    });
+    const content = `${header}\n${yamlStr}`;
+    const outPath = resolve(rootDir, filename);
+    writeFileSync(outPath, content, "utf-8");
+    console.log(`  ✓ ${filename}`);
+}

package/dist/pipeline/grader-api.d.ts

@@ -0,0 +1,49 @@
+/**
+ * grader-api.ts
+ *
+ * Shared utility for calling LLM grading APIs from grader scripts.
+ *
+ * Dispatches to the correct provider API (OpenAI, Anthropic) based on the
+ * grader model prefix. Reads the appropriate API key from environment.
+ *
+ * Also exports `loadGraderModel()` to resolve the grader from
+ * `config/models.yaml`.
+ *
+ * Migrated from lib/grader-api.ts — no module-level side effects, no
+ * process.exit(), accepts rootDir as parameter for file-based operations.
+ */
+interface ProviderConfig {
+    apiKey: string;
+    baseUrl: string;
+    modelName: string;
+}
+/**
+ * Call the grader model once to score a response against a rubric.
+ *
+ * Dispatches to the correct provider API based on the model prefix.
+ * Returns a numeric score (0–100) or null if the call or parse fails.
+ */
+export declare function gradeOnce(graderModel: string, responseText: string, rubricText: string): Promise<null | number>;
+/**
+ * Load the grader model from `config/models.yaml`.
+ * Returns both the model ID and human-readable label.
+ * Falls back to `openai:gpt-5` if not configured.
+ *
+ * @throws Error if config/models.yaml is not found
+ */
+export declare function loadGraderModel(rootDir: string): {
+    id: string;
+    label: string;
+};
+export declare function extractScore(content: string): null | number;
+/**
+ * Parse a Promptfoo-style model ID and resolve the provider config.
+ *
+ * Supported formats:
+ *   - `openai:chat:gpt-5.2`       → OpenAI, model = `gpt-5.2`
+ *   - `openai:gpt-5`              → OpenAI, model = `gpt-5`
+ *   - `anthropic:messages:claude-opus-4-5-20251101` → Anthropic, model = `claude-opus-4-5-20251101`
+ *   - `anthropic:claude-sonnet-4`  → Anthropic, model = `claude-sonnet-4`
+ */
+export declare function resolveProvider(graderModel: string): ProviderConfig;
+export {};

package/dist/pipeline/grader-api.js

@@ -0,0 +1,200 @@
+/**
+ * grader-api.ts
+ *
+ * Shared utility for calling LLM grading APIs from grader scripts.
+ *
+ * Dispatches to the correct provider API (OpenAI, Anthropic) based on the
+ * grader model prefix. Reads the appropriate API key from environment.
+ *
+ * Also exports `loadGraderModel()` to resolve the grader from
+ * `config/models.yaml`.
+ *
+ * Migrated from lib/grader-api.ts — no module-level side effects, no
+ * process.exit(), accepts rootDir as parameter for file-based operations.
+ */
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+import { load } from "js-yaml";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Call the grader model once to score a response against a rubric.
+ *
+ * Dispatches to the correct provider API based on the model prefix.
+ * Returns a numeric score (0–100) or null if the call or parse fails.
+ */
+export async function gradeOnce(graderModel, responseText, rubricText) {
+    const config = resolveProvider(graderModel);
+    const prompt = `You are evaluating an AI assistant's response. Grade the response according to the following rubric.
+
+## Response to evaluate:
+${responseText.slice(0, 8000)}
+
+## Rubric:
+${rubricText}
+`;
+    try {
+        const provider = graderModel.split(":")[0];
+        let content;
+        if (provider === "anthropic") {
+            content = await callAnthropic(config, prompt);
+        }
+        else if (provider === "openai") {
+            content = await callOpenAI(config, prompt);
+        }
+        else {
+            // resolveProvider already throws for unknown providers, but just in case
+            return null;
+        }
+        if (content === null)
+            return null;
+        const score = extractScore(content);
+        if (score === null) {
+            console.error(`  ⚠ Could not parse grader response: ${content.slice(0, 100)}`);
+        }
+        return score;
+    }
+    catch (err) {
+        console.error(`  ⚠ Grader call failed: ${err instanceof Error ? err.message : String(err)}`);
+        return null;
+    }
+}
+/**
+ * Load the grader model from `config/models.yaml`.
+ * Returns both the model ID and human-readable label.
+ * Falls back to `openai:gpt-5` if not configured.
+ *
+ * @throws Error if config/models.yaml is not found
+ */
+export function loadGraderModel(rootDir) {
+    const modelsPath = join(rootDir, "config", "models.yaml");
+    if (!existsSync(modelsPath)) {
+        throw new Error(`config/models.yaml not found at ${modelsPath}`);
+    }
+    const raw = readFileSync(modelsPath, "utf-8");
+    const data = load(raw);
+    return {
+        id: data?.grader?.id ?? "openai:gpt-5",
+        label: data?.grader?.label ?? "GPT-5 (grader)",
+    };
+}
+// ---------------------------------------------------------------------------
+// Score extraction (exported for testing)
+// ---------------------------------------------------------------------------
+export function extractScore(content) {
+    // Try JSON parse first: {"score": 85, "reason": "..."}
+    try {
+        const parsed = JSON.parse(content);
+        if (typeof parsed === "object" &&
+            parsed !== null &&
+            "score" in parsed &&
+            typeof parsed.score === "number") {
+            return parsed.score;
+        }
+    }
+    catch {
+        // Not JSON — fall through
+    }
+    // Fallback: extract first bare number
+    const match = content.match(/(\d+)/);
+    if (match)
+        return parseInt(match[1], 10);
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Provider resolution (exported for testing)
+// ---------------------------------------------------------------------------
+/**
+ * Parse a Promptfoo-style model ID and resolve the provider config.
+ *
+ * Supported formats:
+ *   - `openai:chat:gpt-5.2`       → OpenAI, model = `gpt-5.2`
+ *   - `openai:gpt-5`              → OpenAI, model = `gpt-5`
+ *   - `anthropic:messages:claude-opus-4-5-20251101` → Anthropic, model = `claude-opus-4-5-20251101`
+ *   - `anthropic:claude-sonnet-4`  → Anthropic, model = `claude-sonnet-4`
+ */
+export function resolveProvider(graderModel) {
+    const parts = graderModel.split(":");
+    const provider = parts[0];
+    if (provider === "anthropic") {
+        // "anthropic:messages:claude-opus-4-5" → "claude-opus-4-5"
+        // "anthropic:claude-sonnet-4" → "claude-sonnet-4"
+        const modelName = parts.length >= 3 && parts[1] === "messages"
+            ? parts.slice(2).join(":")
+            : parts.slice(1).join(":");
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            throw new Error("ANTHROPIC_API_KEY not set. Required for grader model: " + graderModel);
+        }
+        return {
+            apiKey,
+            baseUrl: "https://api.anthropic.com/v1/messages",
+            modelName,
+        };
+    }
+    if (provider === "openai") {
+        // "openai:chat:gpt-5.2" → "gpt-5.2", "openai:gpt-5" → "gpt-5"
+        const modelName = parts.length >= 3 ? parts.slice(2).join(":") : parts[1];
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            throw new Error("OPENAI_API_KEY not set. Required for grader model: " + graderModel);
+        }
+        return {
+            apiKey,
+            baseUrl: "https://api.openai.com/v1/chat/completions",
+            modelName,
+        };
+    }
+    throw new Error(`Unsupported grader provider "${provider}" in model "${graderModel}". ` +
+        "Supported: openai, anthropic.");
+}
+// ---------------------------------------------------------------------------
+// Provider-specific API calls
+// ---------------------------------------------------------------------------
+async function callAnthropic(config, prompt) {
+    const response = await fetch(config.baseUrl, {
+        body: JSON.stringify({
+            max_tokens: 256,
+            messages: [{ content: prompt, role: "user" }],
+            model: config.modelName,
+            temperature: 0.2,
+        }),
+        headers: {
+            "anthropic-version": "2023-06-01",
+            "Content-Type": "application/json",
+            "x-api-key": config.apiKey,
+        },
+        method: "POST",
+    });
+    if (!response.ok) {
+        const text = await response.text();
+        console.error(`  ⚠ Grader API error (Anthropic): ${response.status} ${text.slice(0, 200)}`);
+        return null;
+    }
+    const data = (await response.json());
+    const textBlock = data.content?.find((c) => c.type === "text");
+    return textBlock?.text ?? "";
+}
+async function callOpenAI(config, prompt) {
+    const response = await fetch(config.baseUrl, {
+        body: JSON.stringify({
+            max_tokens: 256,
+            messages: [{ content: prompt, role: "user" }],
+            model: config.modelName,
+            temperature: 0.2,
+        }),
+        headers: {
+            Authorization: `Bearer ${config.apiKey}`,
+            "Content-Type": "application/json",
+        },
+        method: "POST",
+    });
+    if (!response.ok) {
+        const text = await response.text();
+        console.error(`  ⚠ Grader API error (OpenAI): ${response.status} ${text.slice(0, 200)}`);
+        return null;
+    }
+    const data = (await response.json());
+    return data.choices?.[0]?.message?.content ?? "";
+}

package/dist/pipeline/grader-compare-runner.d.ts

@@ -0,0 +1,44 @@
+/**
+ * pipeline/grader-compare-runner.ts
+ *
+ * Orchestration module for inter-grader comparison (Phase 3).
+ *
+ * Reads eval results, extracts grading judgments, re-grades each with
+ * candidate models, and calls `compareGraders()` from the pure
+ * computation module.
+ *
+ * Migrated from lib/grader-compare.ts — no process.argv, no process.exit(),
+ * no module-level constants. Accepts rootDir as parameter.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 3
+ */
+import { type GraderComparison } from "./grader-comparison.js";
+export interface GraderCompareRunnerOptions {
+    /** Candidate grader models to compare against the baseline */
+    candidates: {
+        id: string;
+        label: string;
+    }[];
+    /** Output format */
+    format?: "json" | "table";
+    /** Custom output path (default: results/latest/grader-comparison.json) */
+    outputPath?: string;
+    /** Path to eval results (default: results/latest/eval-results.json) */
+    resultsPath?: string;
+    /** Root directory of the eval package */
+    rootDir: string;
+}
+/**
+ * Format a GraderComparison result as a human-readable table report.
+ * Returns a string — does NOT print to console.
+ */
+export declare function formatComparisonReport(result: GraderComparison): string;
+/**
+ * Run inter-grader comparison.
+ *
+ * Reads eval results, grades each judgment with the baseline and candidate
+ * graders, and produces a comparison report.
+ *
+ * @throws Error if results file not found, no candidates configured, or no judgments found
+ */
+export declare function runGraderCompare(options: GraderCompareRunnerOptions): Promise<GraderComparison>;