@sanity/ailf 2.0.2 → 2.2.0

package/dist/scripts/generate-configs.js

@@ -1,459 +0,0 @@
-/**
- * Generate-configs.ts
- *
- * Reads config/models.yaml (the central model registry) and generates all
- * promptfoo config files with the correct provider entries.
- *
- * This keeps model definitions in one place — add a model to config/models.yaml
- * and run `pnpm generate-configs` to propagate it to all eval modes.
- *
- * Generated configs:
- *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
- *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
- *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
- *
- * Usage:
- *   pnpm generate-configs
- *   # or
- *   tsx src/scripts/generate-configs.ts
- */
-import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { dump, load } from "js-yaml";
-import { loadAndExpandTasks } from "../pipeline/expand-tasks.js";
-import { validateModelsYaml } from "../pipeline/validate.js";
-import { loadSource } from "../sources.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "../..");
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-/** Auto-discover all task YAML files in the tasks/ directory. */
-export function discoverTaskFiles(rootDir) {
-    const tasksDir = resolve(rootDir, "tasks");
-    if (!existsSync(tasksDir)) {
-        throw new Error(`tasks/ directory not found at ${tasksDir}`);
-    }
-    return readdirSync(tasksDir)
-        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
-        .sort()
-        .map((f) => `file://tasks/${f}`);
-}
-/**
- * Known promptfoo provider prefixes that are NOT part of the model name.
- * Order matters: longer prefixes must come first to avoid partial matches.
- */
-const PROVIDER_PREFIXES = [
-    "anthropic:messages:",
-    "openai:chat:",
-    "openai:responses:",
-    "openai:",
-    "anthropic:",
-    "google:",
-];
-/**
- * Extract the raw API model name from a promptfoo provider ID.
- *
- * Promptfoo IDs encode the provider + sub-protocol + model, e.g.:
- *   "openai:chat:gpt-5.2"       → "gpt-5.2"
- *   "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6"
- *   "openai:gpt-4o"             → "gpt-4o"
- *   "google:gemini-2.5-pro"     → "gemini-2.5-pro"
- *
- * Falls back to stripping everything before the first colon for unknown
- * providers (e.g., "openrouter:deepseek/deepseek-r1" → "deepseek/deepseek-r1").
- */
-export function extractModelName(id) {
-    for (const prefix of PROVIDER_PREFIXES) {
-        if (id.startsWith(prefix)) {
-            return id.slice(prefix.length);
-        }
-    }
-    // Fallback: strip first colon-separated segment
-    const parts = id.split(":");
-    return parts.length > 1 ? parts.slice(1).join(":") : id;
-}
-/**
- * Extract the LLM provider family from a promptfoo provider ID.
- *
- *   "openai:chat:gpt-5.2"                → "openai"
- *   "anthropic:messages:claude-opus-4-6"  → "anthropic"
- *   "google:gemini-2.5-pro"              → "google"
- */
-export function extractProvider(id) {
-    const colon = id.indexOf(":");
-    return colon > 0 ? id.slice(0, colon) : "openai";
-}
-/** Load prompt templates from config/prompts.yaml. Throws if missing or malformed. */
-export function loadPrompts(rootDir) {
-    const promptsPath = resolve(rootDir, "config", "prompts.yaml");
-    if (!existsSync(promptsPath)) {
-        throw new Error(`config/prompts.yaml not found at ${promptsPath}. This file is required — it defines the prompt templates for all evaluation modes.`);
-    }
-    const raw = readFileSync(promptsPath, "utf-8");
-    const data = load(raw);
-    const toPrompt = (entry) => ({
-        id: entry.id,
-        label: entry.label,
-        raw: entry.template,
-    });
-    if (!data["with-docs"] || !data["without-docs"] || !data["agentic"]) {
-        const missing = ["with-docs", "without-docs", "agentic"].filter((k) => !data[k]);
-        throw new Error(`config/prompts.yaml is missing required keys: ${missing.join(", ")}. Each prompt must have id, label, and template fields.`);
-    }
-    return {
-        agentic: toPrompt(data["agentic"]),
-        withDocs: toPrompt(data["with-docs"]),
-        withoutDocs: toPrompt(data["without-docs"]),
-    };
-}
-// ---------------------------------------------------------------------------
-// Task discovery
-// ---------------------------------------------------------------------------
-/** Merge default config with model-specific config */
-export function mergeConfig(defaults, modelConfig, overrides) {
-    const result = {};
-    // Only pick scalar defaults (temperature, max_tokens)
-    for (const [key, value] of Object.entries(defaults)) {
-        if (typeof value !== "object" || value === null) {
-            result[key] = value;
-        }
-    }
-    // Model-specific overrides
-    if (modelConfig) {
-        Object.assign(result, modelConfig);
-    }
-    // Mode-specific overrides
-    if (overrides) {
-        Object.assign(result, overrides);
-    }
-    return result;
-}
-// ---------------------------------------------------------------------------
-// Prompt loading
-// ---------------------------------------------------------------------------
-export function modelMatchesMode(model, mode) {
-    if (!model.modes || model.modes.length === 0) {
-        return true;
-    }
-    return model.modes.includes(mode);
-}
-function loadModels() {
-    const raw = readFileSync(resolve(ROOT, "config", "models.yaml"), "utf-8");
-    return load(raw);
-}
-// ---------------------------------------------------------------------------
-// Shared components
-// ---------------------------------------------------------------------------
-const URL_EXTRACTION_ASSERT = {
-    type: "javascript",
-    value: `const urlPattern = /https?:\\/\\/[^\\s\\)\\"\\'\\\`>]+/g;
-const urls = [...new Set((output.match(urlPattern) || []))];
-const sanityUrls = urls.filter(u => u.includes('sanity.io'));
-return {
-  pass: true,
-  score: 1,
-  reason: JSON.stringify({
-    sanityUrls,
-    otherUrls: urls.filter(u => !u.includes('sanity.io')),
-    totalUrlCount: urls.length,
-    sanityUrlCount: sanityUrls.length
-  })
-};`,
-    weight: 0,
-};
-/**
- * Source isolation assertion — advisory (weight: 0).
- * Verifies that the agentic provider only fetched docs from allowed origins.
- * Only injected when origin sandboxing is active (DOC_ALLOWED_ORIGINS is set).
- */
-const SOURCE_ISOLATION_ASSERT = {
-    metadata: { dimension: "source-isolation" },
-    type: "javascript",
-    value: "file://dist/assertions/source-isolation.js",
-    weight: 0,
-};
-// ---------------------------------------------------------------------------
-// Config generators
-// ---------------------------------------------------------------------------
-function generateAgenticConfig(models, tests, prompts, source) {
-    const naiveModels = models.models.filter((m) => modelMatchesMode(m, "agentic-naive"));
-    const optimizedModels = models.models.filter((m) => modelMatchesMode(m, "agentic-optimized"));
-    const providers = [];
-    // Build doc source config to inject into providers
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means unset
-    const searchMode = process.env.EVAL_SEARCH_MODE || "open";
-    const sourceConfig = source
-        ? {
-            ...(source.allowedOrigins?.length
-                ? { allowedOrigins: source.allowedOrigins }
-                : {}),
-            docBaseUrl: source.baseUrl,
-            ...(source.headers && Object.keys(source.headers).length > 0
-                ? { customHeaders: source.headers }
-                : {}),
-            llmsTxtUrl: source.llmsTxt,
-            ...(source.priorityDomain
-                ? { priorityDomain: source.priorityDomain }
-                : {}),
-            // Tool access control: search mode for web_search behavior
-            ...(searchMode !== "open" ? { searchMode } : {}),
-        }
-        : {};
-    for (const model of naiveModels) {
-        const modelName = extractModelName(model.id);
-        const provider = extractProvider(model.id);
-        providers.push({
-            config: {
-                ...mergeConfig(models.defaults, model.config, {
-                    agentMode: "naive",
-                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
-                    model: modelName,
-                    provider,
-                }),
-                ...sourceConfig,
-                observe: true,
-                observerOptions: models.defaults.observerOptions ?? {},
-            },
-            id: "file://dist/agent-observer/agentic-provider.js",
-            label: `${model.label} (Naive Agent)`,
-        });
-    }
-    for (const model of optimizedModels) {
-        const modelName = extractModelName(model.id);
-        const provider = extractProvider(model.id);
-        providers.push({
-            config: {
-                ...mergeConfig(models.defaults, model.config, {
-                    agentMode: "optimized",
-                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
-                    model: modelName,
-                    provider,
-                }),
-                ...sourceConfig,
-                observe: true,
-                observerOptions: models.defaults.observerOptions ?? {},
-            },
-            id: "file://dist/agent-observer/agentic-provider.js",
-            label: `${model.label} (Optimized Agent)`,
-        });
-    }
-    // Inject source isolation assertion when origin sandboxing is active
-    const hasOriginSandbox = Boolean(process.env.DOC_ALLOWED_ORIGINS);
-    const agenticAssertions = hasOriginSandbox ? [SOURCE_ISOLATION_ASSERT] : [];
-    return {
-        commandLineOptions: { table: false },
-        defaultTest: {
-            ...(agenticAssertions.length > 0 ? { assert: agenticAssertions } : {}),
-            options: {
-                provider: models.grader.id,
-                rubricProvider: models.grader.id,
-            },
-        },
-        description: "Sanity AI Literacy Evaluation — Agentic (naive vs optimized)",
-        ...(models.maxConcurrency
-            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
-            : {}),
-        outputPath: "results/latest/eval-results-agentic.json",
-        prompts: [prompts.agentic],
-        providers,
-        tests,
-    };
-}
-function generateBaselineConfig(models, tests, prompts) {
-    const baselineModels = models.models.filter((m) => modelMatchesMode(m, "baseline"));
-    const providers = baselineModels.map((model) => ({
-        config: mergeConfig(models.defaults, model.config),
-        id: model.id,
-        label: model.label,
-    }));
-    return {
-        commandLineOptions: { table: false },
-        defaultTest: {
-            assert: [URL_EXTRACTION_ASSERT],
-            options: {
-                provider: models.grader.id,
-                rubricProvider: models.grader.id,
-            },
-        },
-        description: "Sanity AI Literacy Evaluation — Baseline",
-        ...(models.maxConcurrency
-            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
-            : {}),
-        outputPath: "results/latest/eval-results.json",
-        prompts: [prompts.withDocs, prompts.withoutDocs],
-        providers,
-        tests,
-    };
-}
-function generateObservedConfig(models, tests, prompts) {
-    const observedModels = models.models.filter((m) => modelMatchesMode(m, "observed"));
-    const providers = observedModels.map((model) => {
-        const modelName = extractModelName(model.id);
-        return {
-            config: {
-                ...mergeConfig(models.defaults, model.config),
-                modelName,
-                observe: true,
-                recordOptions: models.defaults.observerOptions ?? {},
-            },
-            id: "file://dist/agent-observer/provider.js",
-            label: `${model.label} (Observed)`,
-        };
-    });
-    return {
-        commandLineOptions: { table: false },
-        defaultTest: {
-            options: {
-                provider: models.grader.id,
-                rubricProvider: models.grader.id,
-            },
-        },
-        description: "Sanity AI Literacy Evaluation — Observed",
-        ...(models.maxConcurrency
-            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
-            : {}),
-        outputPath: "results/latest/eval-results-observed.json",
-        prompts: [prompts.withDocs, prompts.withoutDocs],
-        providers,
-        tests,
-    };
-}
-// ---------------------------------------------------------------------------
-// File writing
-// ---------------------------------------------------------------------------
-function main() {
-    // Validate config/models.yaml before generating configs
-    const modelIssues = validateModelsYaml(ROOT);
-    const modelErrors = modelIssues.filter((i) => i.severity === "error");
-    if (modelErrors.length > 0) {
-        console.error("❌ config/models.yaml validation failed:");
-        for (const e of modelErrors) {
-            console.error(`  ERROR: ${e.message}`);
-            if (e.path) {
-                console.error(`         at ${e.path}`);
-            }
-        }
-        console.error("\nFix config/models.yaml before generating configs. Run 'pnpm validate' for details.");
-        process.exit(1);
-    }
-    console.log("Loading config/models.yaml...");
-    const models = loadModels();
-    const activeModels = models.models.filter((m) => m.id && m.label);
-    console.log(`  Found ${activeModels.length} active model(s):`);
-    for (const m of activeModels) {
-        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty array join → "all"
-        const modes = m.modes?.join(", ") || "all";
-        console.log(`    - ${m.label} (${m.id}) → [${modes}]`);
-    }
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty label falls through to id
-    console.log(`  Grader: ${models.grader.label || models.grader.id}`);
-    // Expand tasks: read single-definition tasks and generate gold+baseline pairs
-    // Build filter options from environment variables (set by pipeline.ts)
-    const filterAreas = process.env.EVAL_FILTER_AREAS;
-    const filterTaskIds = process.env.EVAL_FILTER_TASKS;
-    const filter = filterAreas || filterTaskIds
-        ? {
-            areas: filterAreas
-                ? filterAreas.split(",").map((a) => a.trim())
-                : undefined,
-            taskIds: filterTaskIds
-                ? filterTaskIds.split(",").map((t) => t.trim())
-                : undefined,
-        }
-        : undefined;
-    // Expand tasks for baseline/observed mode (gold + baseline with prompt filters)
-    const { entries, stats } = loadAndExpandTasks(ROOT, filter, "baseline");
-    console.log(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
-    if (stats.legacyEntries > 0) {
-        console.log(`  ⚠ ${stats.legacyEntries} legacy (paired) entries passed through unchanged`);
-    }
-    if (filter) {
-        const parts = [];
-        if (filter.areas) {
-            parts.push(`areas: ${filter.areas.join(", ")}`);
-        }
-        if (filter.taskIds) {
-            parts.push(`tasks: ${filter.taskIds.join(", ")}`);
-        }
-        console.log(`  Scoped to: ${parts.join("; ")}`);
-    }
-    // Expand tasks for agentic mode (gold entries only, no prompt filters).
-    // Agentic mode has a single prompt that doesn't use {{docs}}, so baseline
-    // entries would produce identical prompts — pure waste of API calls.
-    const { entries: agenticEntries, stats: agenticStats } = loadAndExpandTasks(ROOT, filter, "agentic");
-    console.log(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
-    // Write expanded tasks to generated files for Promptfoo to consume
-    const expandedPath = resolve(ROOT, "tasks", ".expanded.yaml");
-    const expandedYaml = dump(entries, {
-        forceQuotes: false,
-        lineWidth: 120,
-        noRefs: true,
-        quotingType: "'",
-    });
-    writeFileSync(expandedPath, `# .expanded.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${expandedYaml}`, "utf-8");
-    console.log(`  ✓ tasks/.expanded.yaml (${entries.length} entries)`);
-    const agenticExpandedPath = resolve(ROOT, "tasks", ".expanded.agentic.yaml");
-    const agenticExpandedYaml = dump(agenticEntries, {
-        forceQuotes: false,
-        lineWidth: 120,
-        noRefs: true,
-        quotingType: "'",
-    });
-    writeFileSync(agenticExpandedPath, `# .expanded.agentic.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Gold entries only (no baseline) for agentic evaluation mode.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${agenticExpandedYaml}`, "utf-8");
-    console.log(`  ✓ tasks/.expanded.agentic.yaml (${agenticEntries.length} entries)`);
-    const taskFiles = ["file://tasks/.expanded.yaml"];
-    const agenticTaskFiles = ["file://tasks/.expanded.agentic.yaml"];
-    // Load prompt templates
-    const prompts = loadPrompts(ROOT);
-    console.log(`  Loaded prompts: ${Object.keys(prompts).join(", ")}`);
-    // Load optional documentation source configuration
-    // Usage: pnpm generate-configs [source-name]
-    //   Or:  DOC_SOURCE=branch pnpm generate-configs
-    const sourceName = process.argv[2] || process.env.DOC_SOURCE;
-    let source;
-    if (sourceName) {
-        console.log(`\nLoading source: ${sourceName}`);
-        try {
-            source = loadSource(sourceName);
-            console.log(`  Base URL: ${source.baseUrl}`);
-            console.log(`  Dataset: ${source.dataset}`);
-            if (source.allowedOrigins?.length) {
-                console.log(`  Allowed origins: ${source.allowedOrigins.join(", ")}`);
-            }
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            console.warn(`\n⚠ Failed to load source "${sourceName}": ${msg}`);
-        }
-    }
-    console.log("\nGenerating configs...");
-    writeConfig("promptfooconfig.yaml", generateBaselineConfig(models, taskFiles, prompts), `# promptfooconfig.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
-    writeConfig("promptfooconfig.observed.yaml", generateObservedConfig(models, taskFiles, prompts), `# promptfooconfig.observed.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
-    writeConfig("promptfooconfig.agentic.yaml", generateAgenticConfig(models, agenticTaskFiles, prompts, source), `# promptfooconfig.agentic.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
-    console.log("\nDone! Configs are ready.");
-    if (source) {
-        console.log(`  (using doc source: ${sourceName})`);
-    }
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-function writeConfig(filename, config, header) {
-    const yamlStr = dump(config, {
-        forceQuotes: false,
-        lineWidth: 120,
-        noRefs: true,
-        quotingType: "'",
-    });
-    const content = `${header}\n${yamlStr}`;
-    const outPath = resolve(ROOT, filename);
-    writeFileSync(outPath, content, "utf-8");
-    console.log(`  ✓ ${filename}`);
-}
-// Only run when invoked directly (not when imported for testing)
-if (process.argv[1]?.endsWith("generate-configs.ts") ||
-    process.argv[1]?.endsWith("generate-configs.js")) {
-    main();
-}

package/dist/scripts/grader-api.d.ts

@@ -1,27 +0,0 @@
-/**
- * grader-api.ts
- *
- * Shared utility for calling LLM grading APIs from grader scripts.
- *
- * Dispatches to the correct provider API (OpenAI, Anthropic) based on the
- * grader model prefix. Reads the appropriate API key from environment.
- *
- * Also exports `loadGraderModel()` to resolve the grader from
- * `config/models.yaml`.
- */
-/**
- * Call the grader model once to score a response against a rubric.
- *
- * Dispatches to the correct provider API based on the model prefix.
- * Returns a numeric score (0–100) or null if the call or parse fails.
- */
-export declare function gradeOnce(graderModel: string, responseText: string, rubricText: string): Promise<null | number>;
-/**
- * Load the grader model from `config/models.yaml`.
- * Returns both the model ID and human-readable label.
- * Falls back to `openai:gpt-5` if not configured.
- */
-export declare function loadGraderModel(): {
-    id: string;
-    label: string;
-};

package/dist/scripts/grader-api.js

@@ -1,206 +0,0 @@
-/**
- * grader-api.ts
- *
- * Shared utility for calling LLM grading APIs from grader scripts.
- *
- * Dispatches to the correct provider API (OpenAI, Anthropic) based on the
- * grader model prefix. Reads the appropriate API key from environment.
- *
- * Also exports `loadGraderModel()` to resolve the grader from
- * `config/models.yaml`.
- */
-import { config as dotenvConfig } from "dotenv";
-import { existsSync, readFileSync } from "fs";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-import { load } from "js-yaml";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const EVAL_ROOT = resolve(__dirname, "..", "..");
-// Load root .env (two levels above packages/eval/) so API keys are available
-// even when this module is invoked via `tsx` from packages/eval/.
-const rootEnvPath = resolve(EVAL_ROOT, "..", "..", ".env");
-if (existsSync(rootEnvPath)) {
-    dotenvConfig({ override: true, path: rootEnvPath });
-}
-// ---------------------------------------------------------------------------
-// Provider detection
-// ---------------------------------------------------------------------------
-/**
- * Call the grader model once to score a response against a rubric.
- *
- * Dispatches to the correct provider API based on the model prefix.
- * Returns a numeric score (0–100) or null if the call or parse fails.
- */
-export async function gradeOnce(graderModel, responseText, rubricText) {
-    const config = resolveProvider(graderModel);
-    const prompt = `You are evaluating an AI assistant's response. Grade the response according to the following rubric.
-
-## Response to evaluate:
-${responseText.slice(0, 8000)}
-
-## Rubric:
-${rubricText}
-`;
-    try {
-        const provider = graderModel.split(":")[0];
-        let content;
-        if (provider === "anthropic") {
-            content = await callAnthropic(config, prompt);
-        }
-        else if (provider === "openai") {
-            content = await callOpenAI(config, prompt);
-        }
-        else {
-            // resolveProvider already throws for unknown providers, but just in case
-            return null;
-        }
-        if (content === null)
-            return null;
-        const score = extractScore(content);
-        if (score === null) {
-            console.error(`  ⚠ Could not parse grader response: ${content.slice(0, 100)}`);
-        }
-        return score;
-    }
-    catch (err) {
-        console.error(`  ⚠ Grader call failed: ${err instanceof Error ? err.message : String(err)}`);
-        return null;
-    }
-}
-// ---------------------------------------------------------------------------
-// Provider-specific API calls
-// ---------------------------------------------------------------------------
-/**
- * Load the grader model from `config/models.yaml`.
- * Returns both the model ID and human-readable label.
- * Falls back to `openai:gpt-5` if not configured.
- */
-export function loadGraderModel() {
-    const modelsPath = join(EVAL_ROOT, "config", "models.yaml");
-    if (!existsSync(modelsPath)) {
-        console.error("❌ config/models.yaml not found");
-        process.exit(1);
-    }
-    const raw = readFileSync(modelsPath, "utf-8");
-    const data = load(raw);
-    return {
-        id: data?.grader?.id ?? "openai:gpt-5",
-        label: data?.grader?.label ?? "GPT-5 (grader)",
-    };
-}
-async function callAnthropic(config, prompt) {
-    const response = await fetch(config.baseUrl, {
-        body: JSON.stringify({
-            max_tokens: 256,
-            messages: [{ content: prompt, role: "user" }],
-            model: config.modelName,
-            temperature: 0.2,
-        }),
-        headers: {
-            "anthropic-version": "2023-06-01",
-            "Content-Type": "application/json",
-            "x-api-key": config.apiKey,
-        },
-        method: "POST",
-    });
-    if (!response.ok) {
-        const text = await response.text();
-        console.error(`  ⚠ Grader API error (Anthropic): ${response.status} ${text.slice(0, 200)}`);
-        return null;
-    }
-    const data = (await response.json());
-    const textBlock = data.content?.find((c) => c.type === "text");
-    return textBlock?.text ?? "";
-}
-// ---------------------------------------------------------------------------
-// Score extraction
-// ---------------------------------------------------------------------------
-async function callOpenAI(config, prompt) {
-    const response = await fetch(config.baseUrl, {
-        body: JSON.stringify({
-            max_tokens: 256,
-            messages: [{ content: prompt, role: "user" }],
-            model: config.modelName,
-            temperature: 0.2,
-        }),
-        headers: {
-            Authorization: `Bearer ${config.apiKey}`,
-            "Content-Type": "application/json",
-        },
-        method: "POST",
-    });
-    if (!response.ok) {
-        const text = await response.text();
-        console.error(`  ⚠ Grader API error (OpenAI): ${response.status} ${text.slice(0, 200)}`);
-        return null;
-    }
-    const data = (await response.json());
-    return data.choices?.[0]?.message?.content ?? "";
-}
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-function extractScore(content) {
-    // Try JSON parse first: {"score": 85, "reason": "..."}
-    try {
-        const parsed = JSON.parse(content);
-        if (typeof parsed === "object" &&
-            parsed !== null &&
-            "score" in parsed &&
-            typeof parsed.score === "number") {
-            return parsed.score;
-        }
-    }
-    catch {
-        // Not JSON — fall through
-    }
-    // Fallback: extract first bare number
-    const match = content.match(/(\d+)/);
-    if (match)
-        return parseInt(match[1], 10);
-    return null;
-}
-/**
- * Parse a Promptfoo-style model ID and resolve the provider config.
- *
- * Supported formats:
- *   - `openai:chat:gpt-5.2`       → OpenAI, model = `gpt-5.2`
- *   - `openai:gpt-5`              → OpenAI, model = `gpt-5`
- *   - `anthropic:messages:claude-opus-4-5-20251101` → Anthropic, model = `claude-opus-4-5-20251101`
- *   - `anthropic:claude-sonnet-4`  → Anthropic, model = `claude-sonnet-4`
- */
-function resolveProvider(graderModel) {
-    const parts = graderModel.split(":");
-    const provider = parts[0];
-    if (provider === "anthropic") {
-        // "anthropic:messages:claude-opus-4-5" → "claude-opus-4-5"
-        // "anthropic:claude-sonnet-4" → "claude-sonnet-4"
-        const modelName = parts.length >= 3 && parts[1] === "messages"
-            ? parts.slice(2).join(":")
-            : parts.slice(1).join(":");
-        const apiKey = process.env.ANTHROPIC_API_KEY;
-        if (!apiKey) {
-            throw new Error("ANTHROPIC_API_KEY not set. Required for grader model: " + graderModel);
-        }
-        return {
-            apiKey,
-            baseUrl: "https://api.anthropic.com/v1/messages",
-            modelName,
-        };
-    }
-    if (provider === "openai") {
-        // "openai:chat:gpt-5.2" → "gpt-5.2", "openai:gpt-5" → "gpt-5"
-        const modelName = parts.length >= 3 ? parts.slice(2).join(":") : parts[1];
-        const apiKey = process.env.OPENAI_API_KEY;
-        if (!apiKey) {
-            throw new Error("OPENAI_API_KEY not set. Required for grader model: " + graderModel);
-        }
-        return {
-            apiKey,
-            baseUrl: "https://api.openai.com/v1/chat/completions",
-            modelName,
-        };
-    }
-    throw new Error(`Unsupported grader provider "${provider}" in model "${graderModel}". ` +
-        "Supported: openai, anthropic.");
-}