@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.js

@@ -0,0 +1,334 @@
+/**
+ * MCP-specific assertion types — ergonomic assertions for MCP server testing.
+ *
+ * Each assertion type compiles down to a Promptfoo `javascript` assertion
+ * with the appropriate validation logic. The developer writes:
+ *
+ * ```typescript
+ * assertions: [
+ *   { type: "tool-called", value: "getDocument" },
+ *   { type: "tool-input-matches", value: { documentId: "doc-123" } },
+ *   { type: "tool-output-matches", value: { title: "Hello" } },
+ *   { type: "error-returned", value: { code: -32602 } },
+ * ]
+ * ```
+ *
+ * The compiler transforms these into Promptfoo-compatible `javascript`
+ * assertions that inspect the tool call trace in the evaluation output.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ */
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Build MCP-specific assertions from task assertion definitions.
+ *
+ * Handles both MCP-specific types (tool-called, tool-input-matches, etc.)
+ * and standard assertion types (contains, llm-rubric, etc.) which are
+ * passed through unchanged.
+ */
+export function buildMCPAssertions(assertions, context) {
+    const result = [];
+    const warnings = [];
+    for (const assertion of assertions) {
+        const mapped = mapMCPAssertion(assertion, context, warnings);
+        if (mapped) {
+            result.push(mapped);
+        }
+    }
+    return { assertions: result, warnings };
+}
+// ---------------------------------------------------------------------------
+// Assertion mapping
+// ---------------------------------------------------------------------------
+function mapMCPAssertion(assertion, context, warnings) {
+    switch (assertion.type) {
+        case "tool-called":
+            return buildToolCalledAssertion(assertion, context);
+        case "tool-input-matches":
+            return buildToolInputMatchesAssertion(assertion, context);
+        case "tool-output-matches":
+            return buildToolOutputMatchesAssertion(assertion, context);
+        case "error-returned":
+            return buildErrorReturnedAssertion(assertion, context);
+        case "capability-available":
+            return buildCapabilityAssertion(assertion, context);
+        // Standard assertions — pass through
+        case "contains":
+        case "equals":
+        case "regex":
+        case "is-json":
+        case "llm-rubric":
+        case "javascript":
+        case "python":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
+                ...(assertion.type === "llm-rubric" && context.graderProvider
+                    ? { provider: context.graderProvider }
+                    : {}),
+            };
+        default:
+            warnings.push(`MCP task "${context.taskId}": unknown assertion type "${assertion.type}" — passed through`);
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+            };
+    }
+}
+// ---------------------------------------------------------------------------
+// tool-called — asserts the model called a specific tool by name
+// ---------------------------------------------------------------------------
+function buildToolCalledAssertion(assertion, _context) {
+    const toolName = String(assertion.value ?? "");
+    // Strategy: check multiple sources for tool call evidence.
+    // 1. context.vars.__toolCalls (structured, if Promptfoo populates it)
+    // 2. Response metadata toolCallLog (from custom mcp-tool-provider)
+    // 3. Response output text (LLM+MCP providers embed tool_use JSON blocks)
+    return {
+        type: "javascript",
+        value: buildJsAssertion(`tool-called: ${toolName}`, `
+      var toolName = ${JSON.stringify(toolName)};
+
+      // Strategy 1: structured tool calls from Promptfoo
+      var toolCalls = context.vars.__toolCalls || [];
+      if (Array.isArray(toolCalls) && toolCalls.length > 0) {
+        var called = toolCalls.some(function(tc) { return tc.name === toolName; });
+        return {
+          pass: called,
+          score: called ? 1 : 0,
+          reason: called
+            ? 'Tool "' + toolName + '" was called (via __toolCalls)'
+            : 'Expected tool "' + toolName + '" but found: ' + toolCalls.map(function(tc) { return tc.name; }).join(', '),
+        };
+      }
+
+      // Strategy 2: MCP_TOOLS_CALLED summary appended by custom mcp-tool-provider
+      var outputStr = typeof output === 'string' ? output : JSON.stringify(output || '');
+      var summaryMatch = outputStr.match(/<!-- MCP_TOOLS_CALLED: (\\[.*?\\]) -->/);
+      if (summaryMatch) {
+        try {
+          var calledTools = JSON.parse(summaryMatch[1]);
+          var called = calledTools.includes(toolName);
+          var count = calledTools.filter(function(n) { return n === toolName; }).length;
+          return {
+            pass: called,
+            score: called ? 1 : 0,
+            reason: called
+              ? 'Tool "' + toolName + '" was called ' + count + ' time(s)'
+              : 'Expected tool "' + toolName + '" but found: ' + calledTools.join(', '),
+          };
+        } catch (e) { /* fall through to Strategy 3 */ }
+      }
+
+      // Strategy 3: parse output for tool_use blocks (built-in provider fallback)
+      var outputStr = typeof output === 'string' ? output : JSON.stringify(output || '');
+      var toolUsePattern = /"type"\\s*:\\s*"tool_use"[^}]*"name"\\s*:\\s*"([^"]+)"/g;
+      var foundTools = [];
+      var match;
+      while ((match = toolUsePattern.exec(outputStr)) !== null) {
+        foundTools.push(match[1]);
+      }
+      var fnCallPattern = /"function"\\s*:\\s*\\{[^}]*"name"\\s*:\\s*"([^"]+)"/g;
+      while ((match = fnCallPattern.exec(outputStr)) !== null) {
+        foundTools.push(match[1]);
+      }
+      if (foundTools.length === 0 && outputStr.includes(toolName) && outputStr.includes('tool_use')) {
+        foundTools.push(toolName);
+      }
+
+      var called = foundTools.includes(toolName);
+      return {
+        pass: called,
+        score: called ? 1 : 0,
+        reason: called
+          ? 'Tool "' + toolName + '" was called (detected in output)'
+          : 'Expected tool "' + toolName + '" to be called. ' +
+            (foundTools.length > 0
+              ? 'Tools found in output: ' + foundTools.join(', ')
+              : 'No tool calls detected in output'),
+      };`),
+        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// tool-input-matches — asserts tool call inputs match a schema/value
+// ---------------------------------------------------------------------------
+function buildToolInputMatchesAssertion(assertion, _context) {
+    const expected = assertion.value;
+    const toolName = assertion.toolName ?? assertion.tool;
+    return {
+        type: "javascript",
+        value: buildJsAssertion(`tool-input-matches${toolName ? `: ${toolName}` : ""}`, `
+      const toolCalls = context.vars.__toolCalls || [];
+      const expected = ${JSON.stringify(expected)};
+      const toolFilter = ${JSON.stringify(toolName ?? null)};
+
+      const targetCalls = toolFilter
+        ? toolCalls.filter(tc => tc.name === toolFilter)
+        : toolCalls;
+
+      if (targetCalls.length === 0) {
+        return {
+          pass: false,
+          score: 0,
+          reason: toolFilter
+            ? 'No calls to tool "' + toolFilter + '" found'
+            : 'No tool calls found',
+        };
+      }
+
+      // Check if any call's input matches the expected value
+      const match = targetCalls.some(tc => {
+        const input = tc.input || tc.arguments || {};
+        return Object.entries(expected).every(([k, v]) =>
+          JSON.stringify(input[k]) === JSON.stringify(v)
+        );
+      });
+
+      return {
+        pass: match,
+        score: match ? 1 : 0,
+        reason: match
+          ? 'Tool input matches expected values'
+          : 'Tool input does not match. Expected: ' + JSON.stringify(expected) +
+            ', Got: ' + JSON.stringify(targetCalls.map(tc => tc.input || tc.arguments)),
+      };`),
+        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// tool-output-matches — asserts tool outputs match expected shape/values
+// ---------------------------------------------------------------------------
+function buildToolOutputMatchesAssertion(assertion, _context) {
+    const expected = assertion.value;
+    const toolName = assertion.toolName ?? assertion.tool;
+    return {
+        type: "javascript",
+        value: buildJsAssertion(`tool-output-matches${toolName ? `: ${toolName}` : ""}`, `
+      const toolCalls = context.vars.__toolCalls || [];
+      const expected = ${JSON.stringify(expected)};
+      const toolFilter = ${JSON.stringify(toolName ?? null)};
+
+      const targetCalls = toolFilter
+        ? toolCalls.filter(tc => tc.name === toolFilter)
+        : toolCalls;
+
+      if (targetCalls.length === 0) {
+        return {
+          pass: false,
+          score: 0,
+          reason: toolFilter
+            ? 'No calls to tool "' + toolFilter + '" found'
+            : 'No tool calls found',
+        };
+      }
+
+      const match = targetCalls.some(tc => {
+        const output = tc.output || tc.result || {};
+        return Object.entries(expected).every(([k, v]) =>
+          JSON.stringify(output[k]) === JSON.stringify(v)
+        );
+      });
+
+      return {
+        pass: match,
+        score: match ? 1 : 0,
+        reason: match
+          ? 'Tool output matches expected values'
+          : 'Tool output does not match. Expected: ' + JSON.stringify(expected),
+      };`),
+        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// error-returned — asserts the server returned a specific error
+// ---------------------------------------------------------------------------
+function buildErrorReturnedAssertion(assertion, _context) {
+    const expected = assertion.value;
+    return {
+        type: "javascript",
+        value: buildJsAssertion("error-returned", `
+      const toolCalls = context.vars.__toolCalls || [];
+      const expected = ${JSON.stringify(expected ?? {})};
+
+      const errorCall = toolCalls.find(tc => tc.error);
+      if (!errorCall) {
+        return {
+          pass: false,
+          score: 0,
+          reason: 'Expected an error response but no errors were returned',
+        };
+      }
+
+      const error = errorCall.error;
+      let pass = true;
+      const reasons = [];
+
+      if (expected.code !== undefined && error.code !== expected.code) {
+        pass = false;
+        reasons.push('Expected error code ' + expected.code + ', got ' + error.code);
+      }
+
+      if (expected.message !== undefined) {
+        const msgMatch = typeof error.message === 'string' &&
+          error.message.includes(expected.message);
+        if (!msgMatch) {
+          pass = false;
+          reasons.push('Expected error message containing "' + expected.message +
+            '", got "' + (error.message || '') + '"');
+        }
+      }
+
+      if (pass) {
+        reasons.push('Error matches expected pattern');
+      }
+
+      return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: reasons.join('; '),
+      };`),
+        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// capability-available — asserts the server advertises a capability
+// ---------------------------------------------------------------------------
+function buildCapabilityAssertion(assertion, _context) {
+    const capability = String(assertion.value ?? "");
+    return {
+        type: "javascript",
+        value: buildJsAssertion(`capability-available: ${capability}`, `
+      const capabilities = context.vars.__serverCapabilities || [];
+      const expected = ${JSON.stringify(capability)};
+      const available = capabilities.includes(expected);
+
+      return {
+        pass: available,
+        score: available ? 1 : 0,
+        reason: available
+          ? 'Server advertises capability "' + expected + '"'
+          : 'Server does not advertise capability "' + expected + '". ' +
+            'Available: ' + (capabilities.join(', ') || 'none'),
+      };`),
+        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Build a Promptfoo-compatible JavaScript assertion string.
+ *
+ * Wraps the assertion body in a function that receives `output` and `context`
+ * from Promptfoo's assertion runner.
+ */
+function buildJsAssertion(label, body) {
+    // No IIFE wrapper — Promptfoo wraps the assertion in its own function via
+    // new Function('output', 'context', ...). The body must use `return` at
+    // the top level for the result to reach Promptfoo's validator.
+    return `// MCP assertion: ${label}\n${body.trim()}`;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/compiler.d.ts

@@ -0,0 +1,19 @@
+/**
+ * MCP server task compilation — core compiler logic.
+ *
+ * Produces Promptfoo configuration from MCP server task definitions:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+import type { MCPServerTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { MCPCompileOptions, MCPCompileResult } from "./types.js";
+/**
+ * Compile an MCP server task definition into Promptfoo configuration.
+ *
+ * This is the core of the MCP mode handler. It produces:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+export declare function compileMCPTask(task: MCPServerTaskDefinition, options?: MCPCompileOptions): MCPCompileResult;

package/dist/pipeline/compiler/mode-handlers/mcp-server/compiler.js

@@ -0,0 +1,100 @@
+/**
+ * MCP server task compilation — core compiler logic.
+ *
+ * Produces Promptfoo configuration from MCP server task definitions:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+import { buildMCPAssertions } from "./assertions.js";
+import { buildMCPProvider } from "./provider-config.js";
+import { validateMCPTask } from "./validation.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compile an MCP server task definition into Promptfoo configuration.
+ *
+ * This is the core of the MCP mode handler. It produces:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+export function compileMCPTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateMCPTask(task);
+    if (validationErrors.length > 0) {
+        for (const err of validationErrors) {
+            warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
+        }
+    }
+    // Build providers (one LLM provider per model, each with MCP config)
+    const providers = buildMCPProvider(task, options?.models ?? [], warnings);
+    // Build prompts
+    const prompts = buildMCPPrompts(task);
+    // Build test cases
+    const tests = buildMCPTestCases(task, options, warnings);
+    return { providers, tests, prompts, warnings };
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function buildMCPPrompts(task) {
+    // MCP mode uses a single prompt — the task description
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Test MCP server: ${task.title}`;
+    return [
+        {
+            id: "mcp-test",
+            label: `MCP: ${task.title}`,
+            raw: String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildMCPTestCases(task, options, warnings) {
+    const tests = [];
+    // Build assertion context
+    const assertionContext = {
+        capabilities: task.capabilities ?? [],
+        graderProvider: options?.graderProvider,
+        taskId: task.id,
+    };
+    // Compile assertions
+    // Cast GeneralizedAssertionDefinition[] → AssertionInput[] (structurally compatible)
+    const assertions = [];
+    if (task.assertions) {
+        const rawAssertions = task.assertions;
+        const { assertions: mapped, warnings: assertionWarnings } = buildMCPAssertions(rawAssertions, assertionContext);
+        assertions.push(...mapped);
+        warnings.push(...assertionWarnings);
+    }
+    // Build test case vars
+    const vars = {
+        task: task.prompt?.vars?.task ?? task.description ?? `Test: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+    };
+    // Primary test case
+    tests.push({
+        description: `${task.id} — ${task.title}`,
+        vars,
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    });
+    // Multi-turn test cases
+    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
+        tests.push({
+            description: `${task.id} — ${task.title} [multi-turn]`,
+            vars: {
+                ...vars,
+                __multiTurn: task.multiTurn.turns,
+            },
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        });
+    }
+    return tests;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * MCP Server mode handler — directory barrel.
+ *
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export declare const handler: ModeHandler;
+export type { MCPAssertionContext, MCPCompileOptions, MCPCompileResult, MCPValidationError, } from "./types.js";
+export { buildMCPAssertions } from "./assertions.js";
+export { compileMCPTask } from "./compiler.js";
+export { validateMCPTask } from "./validation.js";
+export { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+export { DEFAULT_MAX_TOOL_ROUNDS, MCP_PROVIDER_PATH, } from "./provider-config.js";

package/dist/pipeline/compiler/mode-handlers/mcp-server/index.js

@@ -0,0 +1,54 @@
+/**
+ * MCP Server mode handler — directory barrel.
+ *
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import { compileMCPTask } from "./compiler.js";
+import { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return MCP_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "mcp-server") {
+            throw new Error(`MCP server handler received task with mode "${task.mode ?? "undefined"}" — expected "mcp-server"`);
+        }
+        const result = compileMCPTask(task, {
+            graderProvider: ctx.graderProvider,
+            models: ctx.models,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+        };
+    },
+};
+// Assertions
+export { buildMCPAssertions } from "./assertions.js";
+// Compilation
+export { compileMCPTask } from "./compiler.js";
+// Validation
+export { validateMCPTask } from "./validation.js";
+// Prompts
+export { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+// Provider config
+export { DEFAULT_MAX_TOOL_ROUNDS, MCP_PROVIDER_PATH, } from "./provider-config.js";

package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Canonical MCP server prompt templates.
+ *
+ * Handler-owned prompts for MCP server evaluations. Instructs the model to
+ * interact with MCP tools rather than writing standalone code.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const MCP_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.js

@@ -0,0 +1,28 @@
+/**
+ * Canonical MCP server prompt templates.
+ *
+ * Handler-owned prompts for MCP server evaluations. Instructs the model to
+ * interact with MCP tools rather than writing standalone code.
+ */
+export const MCP_PROMPT_TEMPLATES = {
+    "mcp-server": {
+        id: "mcp-server",
+        label: "MCP Server Tool Use",
+        template: `You are an AI assistant with access to an MCP (Model Context Protocol) server that provides tools for interacting with a Sanity content backend.
+
+## Task
+{{task}}
+
+## Instructions
+
+1. Use the available MCP tools to complete the task
+2. Call tools with the correct parameters as described in their schemas
+3. Interpret tool responses and use the results to accomplish the goal
+4. If a tool returns an error, explain the issue clearly
+5. Prefer using specific tools over broad queries when possible
+
+Complete the task using the MCP tools provided:
+`,
+        variables: ["task"],
+    },
+};

package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.d.ts

@@ -0,0 +1,28 @@
+/**
+ * MCP server provider assembly — builds Promptfoo provider configs.
+ */
+import type { MCPServerTaskDefinition, ModeProviderEntry } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooProvider } from "../../promptfoo-compiler.js";
+/** Default max tool rounds for MCP multi-turn execution */
+export declare const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+export declare const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+export declare function buildMCPProvider(task: MCPServerTaskDefinition, models: ModeProviderEntry[], warnings: string[]): PromptfooProvider[];
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+export declare function buildMCPServerConfig(task: MCPServerTaskDefinition, warnings: string[]): Record<string, unknown>;