@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/compiler/mode-handlers/literacy-handler.js

@@ -1,379 +0,0 @@
-/**
- * LiteracyModeHandler — compilation rules for `literacy` mode.
- *
- * This handler replaces the existing `generate-configs.ts` + `expand-tasks.ts`
- * code path for literacy (documentation) evaluation. It compiles
- * LiteracyTaskDefinition objects into Promptfoo structure:
- *
- * - Gold entry (with-docs prompt, canonical docs injected)
- * - Baseline entry (without-docs prompt, empty docs)
- * - Rubric template resolution from config/rubrics
- * - Doc-coverage auto-generation when opted in
- * - Structured dimension metadata on rubric assertions
- *
- * The handler accepts GeneralizedTaskDefinition, narrows to
- * LiteracyTaskDefinition, and produces Promptfoo output.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
- * @see packages/eval/src/pipeline/expand-tasks.ts — the legacy code path
- */
-import { LiteracyVariant, } from "../../normalize-mode.js";
-// ---------------------------------------------------------------------------
-// Canonical literacy prompt templates
-// ---------------------------------------------------------------------------
-// These are the source-of-truth templates for literacy-mode evaluations.
-// Previously lived in config/prompts.ts as global templates; now handler-owned
-// so non-literacy modes can define their own prompts without collision.
-export const LITERACY_PROMPT_TEMPLATES = {
-    "with-docs": {
-        id: "with-docs",
-        label: "With Documentation",
-        template: `You are an expert Sanity.io developer. Use the following documentation to help implement the task.
-
-## Sanity Documentation
-{{docs}}
-
-## Task
-{{task}}
-
-## Requirements
-
-1. Use ONLY the APIs and patterns shown in the documentation
-2. Provide a complete, working implementation
-3. Include all necessary imports
-4. Follow Sanity best practices as documented
-
-Provide your implementation:
-`,
-        variables: ["docs", "task"],
-    },
-    "without-docs": {
-        id: "without-docs",
-        label: "Baseline (No Docs)",
-        template: `You are an expert Sanity.io developer.
-
-## Task
-{{task}}
-
-## Requirements
-
-1. Provide a complete, working implementation
-2. Include all necessary imports
-3. Follow Sanity best practices
-
-Provide your implementation:
-`,
-        variables: ["task"],
-    },
-    agentic: {
-        id: "agentic",
-        label: "Agentic (self-retrieval)",
-        template: `You are an expert developer helping implement a Sanity.io feature.
-You have access to web search and page fetching tools.
-
-IMPORTANT: Before writing any code, search for and read the relevant
-Sanity.io documentation to ensure you are using the latest APIs and
-best practices. Do not rely on memory alone.
-
-## Task
-{{task}}
-
-## Requirements
-
-1. Search for relevant Sanity documentation before implementing
-2. Use ONLY the APIs and patterns from the current official docs
-3. Provide a complete, working implementation
-4. Include all necessary imports
-5. Follow Sanity best practices as documented
-
-Provide your implementation:
-`,
-        variables: ["task"],
-    },
-};
-/**
- * Validate a literacy task definition.
- */
-export function validateLiteracyTask(task) {
-    const errors = [];
-    if (!task.id)
-        errors.push({ field: "id", message: "Task ID is required" });
-    if (!task.title) {
-        errors.push({
-            field: "title",
-            message: "Task title is required",
-        });
-    }
-    const promptText = task.prompt?.text ??
-        task.prompt?.template ??
-        task.prompt?.vars?.task ??
-        "";
-    if (!promptText) {
-        errors.push({
-            field: "prompt",
-            message: "Task prompt text is required",
-        });
-    }
-    return errors;
-}
-// ---------------------------------------------------------------------------
-// Compilation
-// ---------------------------------------------------------------------------
-/**
- * Compile a literacy task into Promptfoo configuration.
- *
- * Produces the same structure as the legacy expand-tasks.ts path:
- * - Gold entry with with-docs prompt and canonical doc context
- * - Baseline entry with without-docs prompt and empty docs
- * - Rubric assertions with structured dimension metadata
- */
-export function compileLiteracyTask(task, options) {
-    const warnings = [];
-    const evalMode = options?.evalMode ?? LiteracyVariant.STANDARD;
-    // Validation
-    for (const err of validateLiteracyTask(task)) {
-        warnings.push(`Literacy task "${task.id}": ${err.field} — ${err.message}`);
-    }
-    // Build providers from model list
-    const providers = buildProviders(options);
-    // Build prompts
-    const prompts = buildPrompts(evalMode);
-    // Build test cases (gold + baseline)
-    const tests = buildTestCases(task, evalMode, options, warnings);
-    return { providers, tests, prompts, warnings };
-}
-// ---------------------------------------------------------------------------
-// Provider assembly
-// ---------------------------------------------------------------------------
-function buildProviders(options) {
-    if (options?.models && options.models.length > 0) {
-        return options.models.map((m) => ({
-            id: m.id,
-            label: m.label,
-            config: m.config,
-        }));
-    }
-    return [];
-}
-// ---------------------------------------------------------------------------
-// Prompt assembly
-// ---------------------------------------------------------------------------
-/** Convert a PromptTemplate to the PromptfooPrompt shape used by compile results */
-function templateToPromptfoo(pt) {
-    return { id: pt.id, label: pt.label, raw: pt.template };
-}
-function buildPrompts(evalMode) {
-    if (evalMode === "agentic") {
-        return [templateToPromptfoo(LITERACY_PROMPT_TEMPLATES["agentic"])];
-    }
-    // Baseline mode: with-docs + without-docs prompts
-    return [
-        templateToPromptfoo(LITERACY_PROMPT_TEMPLATES["with-docs"]),
-        templateToPromptfoo(LITERACY_PROMPT_TEMPLATES["without-docs"]),
-    ];
-}
-// ---------------------------------------------------------------------------
-// Test case assembly
-// ---------------------------------------------------------------------------
-function buildTestCases(task, evalMode, options, warnings) {
-    const tests = [];
-    // Extract fields from the LiteracyTaskDefinition shape
-    const promptText = task.prompt?.text ?? task.prompt?.template ?? "";
-    const contextDocs = task.context?.docs ?? [];
-    const taskArea = task.area ?? "";
-    const taskTitle = task.title;
-    const promptVars = task.prompt?.vars ?? {};
-    // Resolve doc path
-    const hasDocs = contextDocs.length > 0;
-    const docsVar = hasDocs ? `file://contexts/canonical/${task.id}.md` : "";
-    // Resolve assertions
-    const assertions = resolveAssertions(task, options, warnings);
-    // Gold entry — canonical docs injected
-    const goldVars = {
-        task: promptText,
-        docs: docsVar,
-        __featureArea: taskArea,
-        ...promptVars,
-    };
-    tests.push({
-        description: `${taskTitle} (gold)`,
-        vars: goldVars,
-        ...(evalMode === LiteracyVariant.STANDARD
-            ? { prompts: ["with-docs"] }
-            : {}),
-        ...(assertions.length > 0 ? { assert: assertions } : {}),
-    });
-    // Baseline entry — no docs (floor measurement)
-    // Skipped in agentic mode (the prompt doesn't use {{docs}})
-    if (evalMode !== "agentic") {
-        const baselineEnabled = task.baseline?.enabled !== false;
-        if (baselineEnabled) {
-            const baselineAssertions = buildBaselineAssertions(assertions, task.baseline?.rubric);
-            tests.push({
-                description: `${taskTitle} (baseline)`,
-                vars: {
-                    task: promptText,
-                    docs: "",
-                    __featureArea: taskArea,
-                    ...promptVars,
-                },
-                prompts: ["without-docs"],
-                ...(baselineAssertions.length > 0
-                    ? { assert: baselineAssertions }
-                    : {}),
-            });
-        }
-    }
-    return tests;
-}
-// ---------------------------------------------------------------------------
-// Assertion resolution
-// ---------------------------------------------------------------------------
-function resolveAssertions(task, options, warnings) {
-    const assertions = [];
-    for (const a of task.assertions ?? []) {
-        if (a.type === "llm-rubric" && "template" in a) {
-            // Templated assertion — resolve from rubric config
-            const resolved = resolveTemplatedAssertion(a, options?.rubricConfig, options?.graderProvider, warnings);
-            if (resolved)
-                assertions.push(resolved);
-        }
-        else {
-            // Value assertion — pass through with optional grader
-            assertions.push({
-                type: a.type,
-                ...("value" in a ? { value: a.value } : {}),
-                ...(typeof a.weight === "number"
-                    ? { weight: a.weight }
-                    : {}),
-                ...(a.type === "llm-rubric" && options?.graderProvider
-                    ? { provider: options.graderProvider }
-                    : {}),
-            });
-        }
-    }
-    // Doc-coverage auto-generation
-    if (task.docCoverage) {
-        const docCoverageAssertion = buildDocCoverageAssertion(options?.rubricConfig, options?.graderProvider);
-        if (docCoverageAssertion)
-            assertions.push(docCoverageAssertion);
-    }
-    return assertions;
-}
-function resolveTemplatedAssertion(a, rubricConfig, graderProvider, warnings) {
-    if (!rubricConfig) {
-        warnings.push(`No rubric config — template "${a.template}" cannot be resolved`);
-        return null;
-    }
-    const template = rubricConfig.templates[a.template];
-    if (!template) {
-        warnings.push(`Unknown rubric template: "${a.template}"`);
-        return null;
-    }
-    // Assemble the rubric text
-    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
-    const criteriaText = a.criteria.map((c) => `- ${c}`).join("\n");
-    // Match legacy rubric assembly format:
-    //   header\n- scale...\n\ncriteria_label\n- criteria...\n\nfooter
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
-        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
-        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
-    return {
-        type: "llm-rubric",
-        value: rubricValue,
-        ...(graderProvider ? { provider: graderProvider } : {}),
-        ...(template.dimension
-            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
-            : {}),
-    };
-}
-function buildDocCoverageAssertion(rubricConfig, graderProvider) {
-    if (!rubricConfig?.templates["doc-coverage"])
-        return null;
-    const template = rubricConfig.templates["doc-coverage"];
-    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
-        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
-    return {
-        type: "llm-rubric",
-        value: rubricValue,
-        ...(graderProvider ? { provider: graderProvider } : {}),
-        ...(template.dimension
-            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
-            : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// Baseline assertion filtering
-// ---------------------------------------------------------------------------
-/**
- * Build baseline assertions matching the legacy expand-tasks behavior.
- *
- * - "full": all assertions carried over
- * - "abbreviated": only first llm-rubric with shortened prompt
- * - "none": no assertions
- */
-function buildBaselineAssertions(goldAssertions, rubricMode) {
-    const mode = rubricMode ?? "full";
-    if (mode === "none")
-        return [];
-    if (mode === "full")
-        return [...goldAssertions];
-    // Abbreviated: keep first llm-rubric as summary, skip rest
-    const abbreviated = [];
-    let foundFirst = false;
-    for (const a of goldAssertions) {
-        if (a.type === "llm-rubric") {
-            if (!foundFirst) {
-                foundFirst = true;
-                abbreviated.push({
-                    type: "llm-rubric",
-                    value: "Score task completion from 0 to 100 (same criteria as above).\n" +
-                        'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
-                    ...(a.provider ? { provider: a.provider } : {}),
-                });
-            }
-        }
-        // Non-rubric assertions are excluded (matching legacy behavior)
-    }
-    return abbreviated;
-}
-// ---------------------------------------------------------------------------
-// ModeHandler adapter — wraps compileLiteracyTask for registry dispatch
-// ---------------------------------------------------------------------------
-/**
- * ModeHandler-conformant export for the literacy evaluation mode.
- *
- * The pipeline looks up this handler via `registry.getMode("literacy")`
- * and calls `handler.compileTask()`. The handler narrows the union to
- * LiteracyTaskDefinition and delegates to `compileLiteracyTask()`.
- *
- * Note: The literacy handler's `evalMode` variant ("baseline" vs "agentic")
- * is passed via `ctx.evalMode` — a literacy-specific extension of
- * CompilationContext. The pipeline sets this when compiling literacy tasks.
- */
-export const handler = {
-    getPrompts() {
-        return LITERACY_PROMPT_TEMPLATES;
-    },
-    compileTask(task, ctx) {
-        // Type-narrow the union — literacy handler only accepts literacy tasks
-        if (task.mode !== "literacy") {
-            throw new Error(`Literacy handler received task with mode "${task.mode}" — expected "literacy"`);
-        }
-        const result = compileLiteracyTask(task, {
-            graderProvider: ctx.graderProvider,
-            rootDir: ctx.rootDir,
-            models: ctx.models,
-            rubricConfig: ctx.rubricConfig,
-            evalMode: ctx
-                .evalMode,
-        });
-        return {
-            providers: result.providers,
-            tests: result.tests,
-            prompts: result.prompts,
-            warnings: result.warnings,
-        };
-    },
-};

package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts

@@ -1,50 +0,0 @@
-/**
- * MCP-specific assertion types — ergonomic assertions for MCP server testing.
- *
- * Each assertion type compiles down to a Promptfoo `javascript` assertion
- * with the appropriate validation logic. The developer writes:
- *
- * ```typescript
- * assertions: [
- *   { type: "tool-called", value: "getDocument" },
- *   { type: "tool-input-matches", value: { documentId: "doc-123" } },
- *   { type: "tool-output-matches", value: { title: "Hello" } },
- *   { type: "error-returned", value: { code: -32602 } },
- * ]
- * ```
- *
- * The compiler transforms these into Promptfoo-compatible `javascript`
- * assertions that inspect the tool call trace in the evaluation output.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
- */
-import type { PromptfooAssertion } from "../assertion-mapper.js";
-/** Context for building MCP assertions */
-export interface MCPAssertionContext {
-    /** Task ID (for error messages) */
-    taskId: string;
-    /** Expected server capabilities */
-    capabilities: string[];
-    /** Grader provider for LLM-graded assertions */
-    graderProvider?: string;
-}
-/** An AILF assertion definition — accepts both core and generalized types */
-interface AssertionInput {
-    type: string;
-    value?: unknown;
-    weight?: number;
-    /** Allow additional properties from generalized assertions */
-    [key: string]: unknown;
-}
-/**
- * Build MCP-specific assertions from task assertion definitions.
- *
- * Handles both MCP-specific types (tool-called, tool-input-matches, etc.)
- * and standard assertion types (contains, llm-rubric, etc.) which are
- * passed through unchanged.
- */
-export declare function buildMCPAssertions(assertions: AssertionInput[], context: MCPAssertionContext): {
-    assertions: PromptfooAssertion[];
-    warnings: string[];
-};
-export {};