@sanity/ailf 2.0.0 → 2.0.2

package/dist/scripts/generate-configs.js

@@ -0,0 +1,459 @@
+/**
+ * Generate-configs.ts
+ *
+ * Reads config/models.yaml (the central model registry) and generates all
+ * promptfoo config files with the correct provider entries.
+ *
+ * This keeps model definitions in one place — add a model to config/models.yaml
+ * and run `pnpm generate-configs` to propagate it to all eval modes.
+ *
+ * Generated configs:
+ *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
+ *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
+ *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
+ *
+ * Usage:
+ *   pnpm generate-configs
+ *   # or
+ *   tsx src/scripts/generate-configs.ts
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { dump, load } from "js-yaml";
+import { loadAndExpandTasks } from "../pipeline/expand-tasks.js";
+import { validateModelsYaml } from "../pipeline/validate.js";
+import { loadSource } from "../sources.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "../..");
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Auto-discover all task YAML files in the tasks/ directory. */
+export function discoverTaskFiles(rootDir) {
+    const tasksDir = resolve(rootDir, "tasks");
+    if (!existsSync(tasksDir)) {
+        throw new Error(`tasks/ directory not found at ${tasksDir}`);
+    }
+    return readdirSync(tasksDir)
+        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
+        .sort()
+        .map((f) => `file://tasks/${f}`);
+}
+/**
+ * Known promptfoo provider prefixes that are NOT part of the model name.
+ * Order matters: longer prefixes must come first to avoid partial matches.
+ */
+const PROVIDER_PREFIXES = [
+    "anthropic:messages:",
+    "openai:chat:",
+    "openai:responses:",
+    "openai:",
+    "anthropic:",
+    "google:",
+];
+/**
+ * Extract the raw API model name from a promptfoo provider ID.
+ *
+ * Promptfoo IDs encode the provider + sub-protocol + model, e.g.:
+ *   "openai:chat:gpt-5.2"       → "gpt-5.2"
+ *   "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6"
+ *   "openai:gpt-4o"             → "gpt-4o"
+ *   "google:gemini-2.5-pro"     → "gemini-2.5-pro"
+ *
+ * Falls back to stripping everything before the first colon for unknown
+ * providers (e.g., "openrouter:deepseek/deepseek-r1" → "deepseek/deepseek-r1").
+ */
+export function extractModelName(id) {
+    for (const prefix of PROVIDER_PREFIXES) {
+        if (id.startsWith(prefix)) {
+            return id.slice(prefix.length);
+        }
+    }
+    // Fallback: strip first colon-separated segment
+    const parts = id.split(":");
+    return parts.length > 1 ? parts.slice(1).join(":") : id;
+}
+/**
+ * Extract the LLM provider family from a promptfoo provider ID.
+ *
+ *   "openai:chat:gpt-5.2"                → "openai"
+ *   "anthropic:messages:claude-opus-4-6"  → "anthropic"
+ *   "google:gemini-2.5-pro"              → "google"
+ */
+export function extractProvider(id) {
+    const colon = id.indexOf(":");
+    return colon > 0 ? id.slice(0, colon) : "openai";
+}
+/** Load prompt templates from config/prompts.yaml. Throws if missing or malformed. */
+export function loadPrompts(rootDir) {
+    const promptsPath = resolve(rootDir, "config", "prompts.yaml");
+    if (!existsSync(promptsPath)) {
+        throw new Error(`config/prompts.yaml not found at ${promptsPath}. This file is required — it defines the prompt templates for all evaluation modes.`);
+    }
+    const raw = readFileSync(promptsPath, "utf-8");
+    const data = load(raw);
+    const toPrompt = (entry) => ({
+        id: entry.id,
+        label: entry.label,
+        raw: entry.template,
+    });
+    if (!data["with-docs"] || !data["without-docs"] || !data["agentic"]) {
+        const missing = ["with-docs", "without-docs", "agentic"].filter((k) => !data[k]);
+        throw new Error(`config/prompts.yaml is missing required keys: ${missing.join(", ")}. Each prompt must have id, label, and template fields.`);
+    }
+    return {
+        agentic: toPrompt(data["agentic"]),
+        withDocs: toPrompt(data["with-docs"]),
+        withoutDocs: toPrompt(data["without-docs"]),
+    };
+}
+// ---------------------------------------------------------------------------
+// Task discovery
+// ---------------------------------------------------------------------------
+/** Merge default config with model-specific config */
+export function mergeConfig(defaults, modelConfig, overrides) {
+    const result = {};
+    // Only pick scalar defaults (temperature, max_tokens)
+    for (const [key, value] of Object.entries(defaults)) {
+        if (typeof value !== "object" || value === null) {
+            result[key] = value;
+        }
+    }
+    // Model-specific overrides
+    if (modelConfig) {
+        Object.assign(result, modelConfig);
+    }
+    // Mode-specific overrides
+    if (overrides) {
+        Object.assign(result, overrides);
+    }
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Prompt loading
+// ---------------------------------------------------------------------------
+export function modelMatchesMode(model, mode) {
+    if (!model.modes || model.modes.length === 0) {
+        return true;
+    }
+    return model.modes.includes(mode);
+}
+function loadModels() {
+    const raw = readFileSync(resolve(ROOT, "config", "models.yaml"), "utf-8");
+    return load(raw);
+}
+// ---------------------------------------------------------------------------
+// Shared components
+// ---------------------------------------------------------------------------
+const URL_EXTRACTION_ASSERT = {
+    type: "javascript",
+    value: `const urlPattern = /https?:\\/\\/[^\\s\\)\\"\\'\\\`>]+/g;
+const urls = [...new Set((output.match(urlPattern) || []))];
+const sanityUrls = urls.filter(u => u.includes('sanity.io'));
+return {
+  pass: true,
+  score: 1,
+  reason: JSON.stringify({
+    sanityUrls,
+    otherUrls: urls.filter(u => !u.includes('sanity.io')),
+    totalUrlCount: urls.length,
+    sanityUrlCount: sanityUrls.length
+  })
+};`,
+    weight: 0,
+};
+/**
+ * Source isolation assertion — advisory (weight: 0).
+ * Verifies that the agentic provider only fetched docs from allowed origins.
+ * Only injected when origin sandboxing is active (DOC_ALLOWED_ORIGINS is set).
+ */
+const SOURCE_ISOLATION_ASSERT = {
+    metadata: { dimension: "source-isolation" },
+    type: "javascript",
+    value: "file://dist/assertions/source-isolation.js",
+    weight: 0,
+};
+// ---------------------------------------------------------------------------
+// Config generators
+// ---------------------------------------------------------------------------
+function generateAgenticConfig(models, tests, prompts, source) {
+    const naiveModels = models.models.filter((m) => modelMatchesMode(m, "agentic-naive"));
+    const optimizedModels = models.models.filter((m) => modelMatchesMode(m, "agentic-optimized"));
+    const providers = [];
+    // Build doc source config to inject into providers
+    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means unset
+    const searchMode = process.env.EVAL_SEARCH_MODE || "open";
+    const sourceConfig = source
+        ? {
+            ...(source.allowedOrigins?.length
+                ? { allowedOrigins: source.allowedOrigins }
+                : {}),
+            docBaseUrl: source.baseUrl,
+            ...(source.headers && Object.keys(source.headers).length > 0
+                ? { customHeaders: source.headers }
+                : {}),
+            llmsTxtUrl: source.llmsTxt,
+            ...(source.priorityDomain
+                ? { priorityDomain: source.priorityDomain }
+                : {}),
+            // Tool access control: search mode for web_search behavior
+            ...(searchMode !== "open" ? { searchMode } : {}),
+        }
+        : {};
+    for (const model of naiveModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "naive",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Naive Agent)`,
+        });
+    }
+    for (const model of optimizedModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "optimized",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Optimized Agent)`,
+        });
+    }
+    // Inject source isolation assertion when origin sandboxing is active
+    const hasOriginSandbox = Boolean(process.env.DOC_ALLOWED_ORIGINS);
+    const agenticAssertions = hasOriginSandbox ? [SOURCE_ISOLATION_ASSERT] : [];
+    return {
+        commandLineOptions: { table: false },
+        defaultTest: {
+            ...(agenticAssertions.length > 0 ? { assert: agenticAssertions } : {}),
+            options: {
+                provider: models.grader.id,
+                rubricProvider: models.grader.id,
+            },
+        },
+        description: "Sanity AI Literacy Evaluation — Agentic (naive vs optimized)",
+        ...(models.maxConcurrency
+            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
+            : {}),
+        outputPath: "results/latest/eval-results-agentic.json",
+        prompts: [prompts.agentic],
+        providers,
+        tests,
+    };
+}
+function generateBaselineConfig(models, tests, prompts) {
+    const baselineModels = models.models.filter((m) => modelMatchesMode(m, "baseline"));
+    const providers = baselineModels.map((model) => ({
+        config: mergeConfig(models.defaults, model.config),
+        id: model.id,
+        label: model.label,
+    }));
+    return {
+        commandLineOptions: { table: false },
+        defaultTest: {
+            assert: [URL_EXTRACTION_ASSERT],
+            options: {
+                provider: models.grader.id,
+                rubricProvider: models.grader.id,
+            },
+        },
+        description: "Sanity AI Literacy Evaluation — Baseline",
+        ...(models.maxConcurrency
+            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
+            : {}),
+        outputPath: "results/latest/eval-results.json",
+        prompts: [prompts.withDocs, prompts.withoutDocs],
+        providers,
+        tests,
+    };
+}
+function generateObservedConfig(models, tests, prompts) {
+    const observedModels = models.models.filter((m) => modelMatchesMode(m, "observed"));
+    const providers = observedModels.map((model) => {
+        const modelName = extractModelName(model.id);
+        return {
+            config: {
+                ...mergeConfig(models.defaults, model.config),
+                modelName,
+                observe: true,
+                recordOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/provider.js",
+            label: `${model.label} (Observed)`,
+        };
+    });
+    return {
+        commandLineOptions: { table: false },
+        defaultTest: {
+            options: {
+                provider: models.grader.id,
+                rubricProvider: models.grader.id,
+            },
+        },
+        description: "Sanity AI Literacy Evaluation — Observed",
+        ...(models.maxConcurrency
+            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
+            : {}),
+        outputPath: "results/latest/eval-results-observed.json",
+        prompts: [prompts.withDocs, prompts.withoutDocs],
+        providers,
+        tests,
+    };
+}
+// ---------------------------------------------------------------------------
+// File writing
+// ---------------------------------------------------------------------------
+function main() {
+    // Validate config/models.yaml before generating configs
+    const modelIssues = validateModelsYaml(ROOT);
+    const modelErrors = modelIssues.filter((i) => i.severity === "error");
+    if (modelErrors.length > 0) {
+        console.error("❌ config/models.yaml validation failed:");
+        for (const e of modelErrors) {
+            console.error(`  ERROR: ${e.message}`);
+            if (e.path) {
+                console.error(`         at ${e.path}`);
+            }
+        }
+        console.error("\nFix config/models.yaml before generating configs. Run 'pnpm validate' for details.");
+        process.exit(1);
+    }
+    console.log("Loading config/models.yaml...");
+    const models = loadModels();
+    const activeModels = models.models.filter((m) => m.id && m.label);
+    console.log(`  Found ${activeModels.length} active model(s):`);
+    for (const m of activeModels) {
+        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty array join → "all"
+        const modes = m.modes?.join(", ") || "all";
+        console.log(`    - ${m.label} (${m.id}) → [${modes}]`);
+    }
+    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty label falls through to id
+    console.log(`  Grader: ${models.grader.label || models.grader.id}`);
+    // Expand tasks: read single-definition tasks and generate gold+baseline pairs
+    // Build filter options from environment variables (set by pipeline.ts)
+    const filterAreas = process.env.EVAL_FILTER_AREAS;
+    const filterTaskIds = process.env.EVAL_FILTER_TASKS;
+    const filter = filterAreas || filterTaskIds
+        ? {
+            areas: filterAreas
+                ? filterAreas.split(",").map((a) => a.trim())
+                : undefined,
+            taskIds: filterTaskIds
+                ? filterTaskIds.split(",").map((t) => t.trim())
+                : undefined,
+        }
+        : undefined;
+    // Expand tasks for baseline/observed mode (gold + baseline with prompt filters)
+    const { entries, stats } = loadAndExpandTasks(ROOT, filter, "baseline");
+    console.log(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
+    if (stats.legacyEntries > 0) {
+        console.log(`  ⚠ ${stats.legacyEntries} legacy (paired) entries passed through unchanged`);
+    }
+    if (filter) {
+        const parts = [];
+        if (filter.areas) {
+            parts.push(`areas: ${filter.areas.join(", ")}`);
+        }
+        if (filter.taskIds) {
+            parts.push(`tasks: ${filter.taskIds.join(", ")}`);
+        }
+        console.log(`  Scoped to: ${parts.join("; ")}`);
+    }
+    // Expand tasks for agentic mode (gold entries only, no prompt filters).
+    // Agentic mode has a single prompt that doesn't use {{docs}}, so baseline
+    // entries would produce identical prompts — pure waste of API calls.
+    const { entries: agenticEntries, stats: agenticStats } = loadAndExpandTasks(ROOT, filter, "agentic");
+    console.log(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
+    // Write expanded tasks to generated files for Promptfoo to consume
+    const expandedPath = resolve(ROOT, "tasks", ".expanded.yaml");
+    const expandedYaml = dump(entries, {
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        quotingType: "'",
+    });
+    writeFileSync(expandedPath, `# .expanded.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${expandedYaml}`, "utf-8");
+    console.log(`  ✓ tasks/.expanded.yaml (${entries.length} entries)`);
+    const agenticExpandedPath = resolve(ROOT, "tasks", ".expanded.agentic.yaml");
+    const agenticExpandedYaml = dump(agenticEntries, {
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        quotingType: "'",
+    });
+    writeFileSync(agenticExpandedPath, `# .expanded.agentic.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Gold entries only (no baseline) for agentic evaluation mode.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${agenticExpandedYaml}`, "utf-8");
+    console.log(`  ✓ tasks/.expanded.agentic.yaml (${agenticEntries.length} entries)`);
+    const taskFiles = ["file://tasks/.expanded.yaml"];
+    const agenticTaskFiles = ["file://tasks/.expanded.agentic.yaml"];
+    // Load prompt templates
+    const prompts = loadPrompts(ROOT);
+    console.log(`  Loaded prompts: ${Object.keys(prompts).join(", ")}`);
+    // Load optional documentation source configuration
+    // Usage: pnpm generate-configs [source-name]
+    //   Or:  DOC_SOURCE=branch pnpm generate-configs
+    const sourceName = process.argv[2] || process.env.DOC_SOURCE;
+    let source;
+    if (sourceName) {
+        console.log(`\nLoading source: ${sourceName}`);
+        try {
+            source = loadSource(sourceName);
+            console.log(`  Base URL: ${source.baseUrl}`);
+            console.log(`  Dataset: ${source.dataset}`);
+            if (source.allowedOrigins?.length) {
+                console.log(`  Allowed origins: ${source.allowedOrigins.join(", ")}`);
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`\n⚠ Failed to load source "${sourceName}": ${msg}`);
+        }
+    }
+    console.log("\nGenerating configs...");
+    writeConfig("promptfooconfig.yaml", generateBaselineConfig(models, taskFiles, prompts), `# promptfooconfig.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
+    writeConfig("promptfooconfig.observed.yaml", generateObservedConfig(models, taskFiles, prompts), `# promptfooconfig.observed.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
+    writeConfig("promptfooconfig.agentic.yaml", generateAgenticConfig(models, agenticTaskFiles, prompts, source), `# promptfooconfig.agentic.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
+    console.log("\nDone! Configs are ready.");
+    if (source) {
+        console.log(`  (using doc source: ${sourceName})`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+function writeConfig(filename, config, header) {
+    const yamlStr = dump(config, {
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        quotingType: "'",
+    });
+    const content = `${header}\n${yamlStr}`;
+    const outPath = resolve(ROOT, filename);
+    writeFileSync(outPath, content, "utf-8");
+    console.log(`  ✓ ${filename}`);
+}
+// Only run when invoked directly (not when imported for testing)
+if (process.argv[1]?.endsWith("generate-configs.ts") ||
+    process.argv[1]?.endsWith("generate-configs.js")) {
+    main();
+}

package/dist/scripts/grader-api.d.ts

@@ -0,0 +1,27 @@
+/**
+ * grader-api.ts
+ *
+ * Shared utility for calling LLM grading APIs from grader scripts.
+ *
+ * Dispatches to the correct provider API (OpenAI, Anthropic) based on the
+ * grader model prefix. Reads the appropriate API key from environment.
+ *
+ * Also exports `loadGraderModel()` to resolve the grader from
+ * `config/models.yaml`.
+ */
+/**
+ * Call the grader model once to score a response against a rubric.
+ *
+ * Dispatches to the correct provider API based on the model prefix.
+ * Returns a numeric score (0–100) or null if the call or parse fails.
+ */
+export declare function gradeOnce(graderModel: string, responseText: string, rubricText: string): Promise<null | number>;
+/**
+ * Load the grader model from `config/models.yaml`.
+ * Returns both the model ID and human-readable label.
+ * Falls back to `openai:gpt-5` if not configured.
+ */
+export declare function loadGraderModel(): {
+    id: string;
+    label: string;
+};

package/dist/scripts/grader-api.js

@@ -0,0 +1,206 @@
+/**
+ * grader-api.ts
+ *
+ * Shared utility for calling LLM grading APIs from grader scripts.
+ *
+ * Dispatches to the correct provider API (OpenAI, Anthropic) based on the
+ * grader model prefix. Reads the appropriate API key from environment.
+ *
+ * Also exports `loadGraderModel()` to resolve the grader from
+ * `config/models.yaml`.
+ */
+import { config as dotenvConfig } from "dotenv";
+import { existsSync, readFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { load } from "js-yaml";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EVAL_ROOT = resolve(__dirname, "..", "..");
+// Load root .env (two levels above packages/eval/) so API keys are available
+// even when this module is invoked via `tsx` from packages/eval/.
+const rootEnvPath = resolve(EVAL_ROOT, "..", "..", ".env");
+if (existsSync(rootEnvPath)) {
+    dotenvConfig({ override: true, path: rootEnvPath });
+}
+// ---------------------------------------------------------------------------
+// Provider detection
+// ---------------------------------------------------------------------------
+/**
+ * Call the grader model once to score a response against a rubric.
+ *
+ * Dispatches to the correct provider API based on the model prefix.
+ * Returns a numeric score (0–100) or null if the call or parse fails.
+ */
+export async function gradeOnce(graderModel, responseText, rubricText) {
+    const config = resolveProvider(graderModel);
+    const prompt = `You are evaluating an AI assistant's response. Grade the response according to the following rubric.
+
+## Response to evaluate:
+${responseText.slice(0, 8000)}
+
+## Rubric:
+${rubricText}
+`;
+    try {
+        const provider = graderModel.split(":")[0];
+        let content;
+        if (provider === "anthropic") {
+            content = await callAnthropic(config, prompt);
+        }
+        else if (provider === "openai") {
+            content = await callOpenAI(config, prompt);
+        }
+        else {
+            // resolveProvider already throws for unknown providers, but just in case
+            return null;
+        }
+        if (content === null)
+            return null;
+        const score = extractScore(content);
+        if (score === null) {
+            console.error(`  ⚠ Could not parse grader response: ${content.slice(0, 100)}`);
+        }
+        return score;
+    }
+    catch (err) {
+        console.error(`  ⚠ Grader call failed: ${err instanceof Error ? err.message : String(err)}`);
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Provider-specific API calls
+// ---------------------------------------------------------------------------
+/**
+ * Load the grader model from `config/models.yaml`.
+ * Returns both the model ID and human-readable label.
+ * Falls back to `openai:gpt-5` if not configured.
+ */
+export function loadGraderModel() {
+    const modelsPath = join(EVAL_ROOT, "config", "models.yaml");
+    if (!existsSync(modelsPath)) {
+        console.error("❌ config/models.yaml not found");
+        process.exit(1);
+    }
+    const raw = readFileSync(modelsPath, "utf-8");
+    const data = load(raw);
+    return {
+        id: data?.grader?.id ?? "openai:gpt-5",
+        label: data?.grader?.label ?? "GPT-5 (grader)",
+    };
+}
+async function callAnthropic(config, prompt) {
+    const response = await fetch(config.baseUrl, {
+        body: JSON.stringify({
+            max_tokens: 256,
+            messages: [{ content: prompt, role: "user" }],
+            model: config.modelName,
+            temperature: 0.2,
+        }),
+        headers: {
+            "anthropic-version": "2023-06-01",
+            "Content-Type": "application/json",
+            "x-api-key": config.apiKey,
+        },
+        method: "POST",
+    });
+    if (!response.ok) {
+        const text = await response.text();
+        console.error(`  ⚠ Grader API error (Anthropic): ${response.status} ${text.slice(0, 200)}`);
+        return null;
+    }
+    const data = (await response.json());
+    const textBlock = data.content?.find((c) => c.type === "text");
+    return textBlock?.text ?? "";
+}
+// ---------------------------------------------------------------------------
+// Score extraction
+// ---------------------------------------------------------------------------
+async function callOpenAI(config, prompt) {
+    const response = await fetch(config.baseUrl, {
+        body: JSON.stringify({
+            max_tokens: 256,
+            messages: [{ content: prompt, role: "user" }],
+            model: config.modelName,
+            temperature: 0.2,
+        }),
+        headers: {
+            Authorization: `Bearer ${config.apiKey}`,
+            "Content-Type": "application/json",
+        },
+        method: "POST",
+    });
+    if (!response.ok) {
+        const text = await response.text();
+        console.error(`  ⚠ Grader API error (OpenAI): ${response.status} ${text.slice(0, 200)}`);
+        return null;
+    }
+    const data = (await response.json());
+    return data.choices?.[0]?.message?.content ?? "";
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+function extractScore(content) {
+    // Try JSON parse first: {"score": 85, "reason": "..."}
+    try {
+        const parsed = JSON.parse(content);
+        if (typeof parsed === "object" &&
+            parsed !== null &&
+            "score" in parsed &&
+            typeof parsed.score === "number") {
+            return parsed.score;
+        }
+    }
+    catch {
+        // Not JSON — fall through
+    }
+    // Fallback: extract first bare number
+    const match = content.match(/(\d+)/);
+    if (match)
+        return parseInt(match[1], 10);
+    return null;
+}
+/**
+ * Parse a Promptfoo-style model ID and resolve the provider config.
+ *
+ * Supported formats:
+ *   - `openai:chat:gpt-5.2`       → OpenAI, model = `gpt-5.2`
+ *   - `openai:gpt-5`              → OpenAI, model = `gpt-5`
+ *   - `anthropic:messages:claude-opus-4-5-20251101` → Anthropic, model = `claude-opus-4-5-20251101`
+ *   - `anthropic:claude-sonnet-4`  → Anthropic, model = `claude-sonnet-4`
+ */
+function resolveProvider(graderModel) {
+    const parts = graderModel.split(":");
+    const provider = parts[0];
+    if (provider === "anthropic") {
+        // "anthropic:messages:claude-opus-4-5" → "claude-opus-4-5"
+        // "anthropic:claude-sonnet-4" → "claude-sonnet-4"
+        const modelName = parts.length >= 3 && parts[1] === "messages"
+            ? parts.slice(2).join(":")
+            : parts.slice(1).join(":");
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            throw new Error("ANTHROPIC_API_KEY not set. Required for grader model: " + graderModel);
+        }
+        return {
+            apiKey,
+            baseUrl: "https://api.anthropic.com/v1/messages",
+            modelName,
+        };
+    }
+    if (provider === "openai") {
+        // "openai:chat:gpt-5.2" → "gpt-5.2", "openai:gpt-5" → "gpt-5"
+        const modelName = parts.length >= 3 ? parts.slice(2).join(":") : parts[1];
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            throw new Error("OPENAI_API_KEY not set. Required for grader model: " + graderModel);
+        }
+        return {
+            apiKey,
+            baseUrl: "https://api.openai.com/v1/chat/completions",
+            modelName,
+        };
+    }
+    throw new Error(`Unsupported grader provider "${provider}" in model "${graderModel}". ` +
+        "Supported: openai, anthropic.");
+}