@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js

@@ -1,334 +0,0 @@
-/**
- * MCP-specific assertion types — ergonomic assertions for MCP server testing.
- *
- * Each assertion type compiles down to a Promptfoo `javascript` assertion
- * with the appropriate validation logic. The developer writes:
- *
- * ```typescript
- * assertions: [
- *   { type: "tool-called", value: "getDocument" },
- *   { type: "tool-input-matches", value: { documentId: "doc-123" } },
- *   { type: "tool-output-matches", value: { title: "Hello" } },
- *   { type: "error-returned", value: { code: -32602 } },
- * ]
- * ```
- *
- * The compiler transforms these into Promptfoo-compatible `javascript`
- * assertions that inspect the tool call trace in the evaluation output.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
- */
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-/**
- * Build MCP-specific assertions from task assertion definitions.
- *
- * Handles both MCP-specific types (tool-called, tool-input-matches, etc.)
- * and standard assertion types (contains, llm-rubric, etc.) which are
- * passed through unchanged.
- */
-export function buildMCPAssertions(assertions, context) {
-    const result = [];
-    const warnings = [];
-    for (const assertion of assertions) {
-        const mapped = mapMCPAssertion(assertion, context, warnings);
-        if (mapped) {
-            result.push(mapped);
-        }
-    }
-    return { assertions: result, warnings };
-}
-// ---------------------------------------------------------------------------
-// Assertion mapping
-// ---------------------------------------------------------------------------
-function mapMCPAssertion(assertion, context, warnings) {
-    switch (assertion.type) {
-        case "tool-called":
-            return buildToolCalledAssertion(assertion, context);
-        case "tool-input-matches":
-            return buildToolInputMatchesAssertion(assertion, context);
-        case "tool-output-matches":
-            return buildToolOutputMatchesAssertion(assertion, context);
-        case "error-returned":
-            return buildErrorReturnedAssertion(assertion, context);
-        case "capability-available":
-            return buildCapabilityAssertion(assertion, context);
-        // Standard assertions — pass through
-        case "contains":
-        case "equals":
-        case "regex":
-        case "is-json":
-        case "llm-rubric":
-        case "javascript":
-        case "python":
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-                ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
-                ...(assertion.type === "llm-rubric" && context.graderProvider
-                    ? { provider: context.graderProvider }
-                    : {}),
-            };
-        default:
-            warnings.push(`MCP task "${context.taskId}": unknown assertion type "${assertion.type}" — passed through`);
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-            };
-    }
-}
-// ---------------------------------------------------------------------------
-// tool-called — asserts the model called a specific tool by name
-// ---------------------------------------------------------------------------
-function buildToolCalledAssertion(assertion, _context) {
-    const toolName = String(assertion.value ?? "");
-    // Strategy: check multiple sources for tool call evidence.
-    // 1. context.vars.__toolCalls (structured, if Promptfoo populates it)
-    // 2. Response metadata toolCallLog (from custom mcp-tool-provider)
-    // 3. Response output text (LLM+MCP providers embed tool_use JSON blocks)
-    return {
-        type: "javascript",
-        value: buildJsAssertion(`tool-called: ${toolName}`, `
-      var toolName = ${JSON.stringify(toolName)};
-
-      // Strategy 1: structured tool calls from Promptfoo
-      var toolCalls = context.vars.__toolCalls || [];
-      if (Array.isArray(toolCalls) && toolCalls.length > 0) {
-        var called = toolCalls.some(function(tc) { return tc.name === toolName; });
-        return {
-          pass: called,
-          score: called ? 1 : 0,
-          reason: called
-            ? 'Tool "' + toolName + '" was called (via __toolCalls)'
-            : 'Expected tool "' + toolName + '" but found: ' + toolCalls.map(function(tc) { return tc.name; }).join(', '),
-        };
-      }
-
-      // Strategy 2: MCP_TOOLS_CALLED summary appended by custom mcp-tool-provider
-      var outputStr = typeof output === 'string' ? output : JSON.stringify(output || '');
-      var summaryMatch = outputStr.match(/<!-- MCP_TOOLS_CALLED: (\\[.*?\\]) -->/);
-      if (summaryMatch) {
-        try {
-          var calledTools = JSON.parse(summaryMatch[1]);
-          var called = calledTools.includes(toolName);
-          var count = calledTools.filter(function(n) { return n === toolName; }).length;
-          return {
-            pass: called,
-            score: called ? 1 : 0,
-            reason: called
-              ? 'Tool "' + toolName + '" was called ' + count + ' time(s)'
-              : 'Expected tool "' + toolName + '" but found: ' + calledTools.join(', '),
-          };
-        } catch (e) { /* fall through to Strategy 3 */ }
-      }
-
-      // Strategy 3: parse output for tool_use blocks (built-in provider fallback)
-      var outputStr = typeof output === 'string' ? output : JSON.stringify(output || '');
-      var toolUsePattern = /"type"\\s*:\\s*"tool_use"[^}]*"name"\\s*:\\s*"([^"]+)"/g;
-      var foundTools = [];
-      var match;
-      while ((match = toolUsePattern.exec(outputStr)) !== null) {
-        foundTools.push(match[1]);
-      }
-      var fnCallPattern = /"function"\\s*:\\s*\\{[^}]*"name"\\s*:\\s*"([^"]+)"/g;
-      while ((match = fnCallPattern.exec(outputStr)) !== null) {
-        foundTools.push(match[1]);
-      }
-      if (foundTools.length === 0 && outputStr.includes(toolName) && outputStr.includes('tool_use')) {
-        foundTools.push(toolName);
-      }
-
-      var called = foundTools.includes(toolName);
-      return {
-        pass: called,
-        score: called ? 1 : 0,
-        reason: called
-          ? 'Tool "' + toolName + '" was called (detected in output)'
-          : 'Expected tool "' + toolName + '" to be called. ' +
-            (foundTools.length > 0
-              ? 'Tools found in output: ' + foundTools.join(', ')
-              : 'No tool calls detected in output'),
-      };`),
-        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// tool-input-matches — asserts tool call inputs match a schema/value
-// ---------------------------------------------------------------------------
-function buildToolInputMatchesAssertion(assertion, _context) {
-    const expected = assertion.value;
-    const toolName = assertion.toolName ?? assertion.tool;
-    return {
-        type: "javascript",
-        value: buildJsAssertion(`tool-input-matches${toolName ? `: ${toolName}` : ""}`, `
-      const toolCalls = context.vars.__toolCalls || [];
-      const expected = ${JSON.stringify(expected)};
-      const toolFilter = ${JSON.stringify(toolName ?? null)};
-
-      const targetCalls = toolFilter
-        ? toolCalls.filter(tc => tc.name === toolFilter)
-        : toolCalls;
-
-      if (targetCalls.length === 0) {
-        return {
-          pass: false,
-          score: 0,
-          reason: toolFilter
-            ? 'No calls to tool "' + toolFilter + '" found'
-            : 'No tool calls found',
-        };
-      }
-
-      // Check if any call's input matches the expected value
-      const match = targetCalls.some(tc => {
-        const input = tc.input || tc.arguments || {};
-        return Object.entries(expected).every(([k, v]) =>
-          JSON.stringify(input[k]) === JSON.stringify(v)
-        );
-      });
-
-      return {
-        pass: match,
-        score: match ? 1 : 0,
-        reason: match
-          ? 'Tool input matches expected values'
-          : 'Tool input does not match. Expected: ' + JSON.stringify(expected) +
-            ', Got: ' + JSON.stringify(targetCalls.map(tc => tc.input || tc.arguments)),
-      };`),
-        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// tool-output-matches — asserts tool outputs match expected shape/values
-// ---------------------------------------------------------------------------
-function buildToolOutputMatchesAssertion(assertion, _context) {
-    const expected = assertion.value;
-    const toolName = assertion.toolName ?? assertion.tool;
-    return {
-        type: "javascript",
-        value: buildJsAssertion(`tool-output-matches${toolName ? `: ${toolName}` : ""}`, `
-      const toolCalls = context.vars.__toolCalls || [];
-      const expected = ${JSON.stringify(expected)};
-      const toolFilter = ${JSON.stringify(toolName ?? null)};
-
-      const targetCalls = toolFilter
-        ? toolCalls.filter(tc => tc.name === toolFilter)
-        : toolCalls;
-
-      if (targetCalls.length === 0) {
-        return {
-          pass: false,
-          score: 0,
-          reason: toolFilter
-            ? 'No calls to tool "' + toolFilter + '" found'
-            : 'No tool calls found',
-        };
-      }
-
-      const match = targetCalls.some(tc => {
-        const output = tc.output || tc.result || {};
-        return Object.entries(expected).every(([k, v]) =>
-          JSON.stringify(output[k]) === JSON.stringify(v)
-        );
-      });
-
-      return {
-        pass: match,
-        score: match ? 1 : 0,
-        reason: match
-          ? 'Tool output matches expected values'
-          : 'Tool output does not match. Expected: ' + JSON.stringify(expected),
-      };`),
-        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// error-returned — asserts the server returned a specific error
-// ---------------------------------------------------------------------------
-function buildErrorReturnedAssertion(assertion, _context) {
-    const expected = assertion.value;
-    return {
-        type: "javascript",
-        value: buildJsAssertion("error-returned", `
-      const toolCalls = context.vars.__toolCalls || [];
-      const expected = ${JSON.stringify(expected ?? {})};
-
-      const errorCall = toolCalls.find(tc => tc.error);
-      if (!errorCall) {
-        return {
-          pass: false,
-          score: 0,
-          reason: 'Expected an error response but no errors were returned',
-        };
-      }
-
-      const error = errorCall.error;
-      let pass = true;
-      const reasons = [];
-
-      if (expected.code !== undefined && error.code !== expected.code) {
-        pass = false;
-        reasons.push('Expected error code ' + expected.code + ', got ' + error.code);
-      }
-
-      if (expected.message !== undefined) {
-        const msgMatch = typeof error.message === 'string' &&
-          error.message.includes(expected.message);
-        if (!msgMatch) {
-          pass = false;
-          reasons.push('Expected error message containing "' + expected.message +
-            '", got "' + (error.message || '') + '"');
-        }
-      }
-
-      if (pass) {
-        reasons.push('Error matches expected pattern');
-      }
-
-      return {
-        pass,
-        score: pass ? 1 : 0,
-        reason: reasons.join('; '),
-      };`),
-        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// capability-available — asserts the server advertises a capability
-// ---------------------------------------------------------------------------
-function buildCapabilityAssertion(assertion, _context) {
-    const capability = String(assertion.value ?? "");
-    return {
-        type: "javascript",
-        value: buildJsAssertion(`capability-available: ${capability}`, `
-      const capabilities = context.vars.__serverCapabilities || [];
-      const expected = ${JSON.stringify(capability)};
-      const available = capabilities.includes(expected);
-
-      return {
-        pass: available,
-        score: available ? 1 : 0,
-        reason: available
-          ? 'Server advertises capability "' + expected + '"'
-          : 'Server does not advertise capability "' + expected + '". ' +
-            'Available: ' + (capabilities.join(', ') || 'none'),
-      };`),
-        ...(assertion.weight !== undefined ? { weight: assertion.weight } : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-/**
- * Build a Promptfoo-compatible JavaScript assertion string.
- *
- * Wraps the assertion body in a function that receives `output` and `context`
- * from Promptfoo's assertion runner.
- */
-function buildJsAssertion(label, body) {
-    // No IIFE wrapper — Promptfoo wraps the assertion in its own function via
-    // new Function('output', 'context', ...). The body must use `return` at
-    // the top level for the result to reach Promptfoo's validator.
-    return `// MCP assertion: ${label}\n${body.trim()}`;
-}

package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts

@@ -1,69 +0,0 @@
-/**
- * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
- *
- * This is the first non-literacy mode handler, proving the compiler
- * architecture works end-to-end. It translates MCP server task definitions
- * into Promptfoo configuration with:
- *
- * - An MCP provider that wraps the server under test
- * - Tool-call assertions compiled to Promptfoo `javascript` assertions
- * - Server lifecycle management via Promptfoo provider hooks
- * - Multi-turn conversation support via Promptfoo's `steps` syntax
- *
- * Promptfoo supports MCP servers as providers natively:
- * ```yaml
- * providers:
- *   - id: mcp:./my-server
- *     config:
- *       command: node
- *       args: [./dist/server.js]
- *       env: { API_KEY: "..." }
- * ```
- *
- * This handler assembles that config from AILF's `MCPServerTaskDefinition`.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
- * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
- * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
- */
-import type { MCPServerTaskDefinition, ModeHandler, ModeProviderEntry, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
-import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
-export declare const MCP_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
-/** Options for compiling an MCP server task */
-export interface MCPCompileOptions {
-    /** Grader provider for LLM-graded assertions */
-    graderProvider?: string;
-    /** Model providers to evaluate with (from registry, filtered by mcp-server mode) */
-    models?: ModeProviderEntry[];
-}
-/** Result of compiling a single MCP task */
-export interface MCPCompileResult {
-    /** Promptfoo provider config for the MCP server */
-    providers: PromptfooProvider[];
-    /** Compiled test cases */
-    tests: PromptfooTestCase[];
-    /** Prompts for MCP evaluation */
-    prompts: PromptfooPrompt[];
-    /** Warnings generated during compilation */
-    warnings: string[];
-}
-/** Validation errors for MCP task definitions */
-export interface MCPValidationError {
-    field: string;
-    message: string;
-}
-/**
- * Validate that an MCP task definition has all required fields.
- */
-export declare function validateMCPTask(task: MCPServerTaskDefinition): MCPValidationError[];
-/**
- * Compile an MCP server task definition into Promptfoo configuration.
- *
- * This is the core of the MCP mode handler. It produces:
- * 1. A provider config pointing to the MCP server
- * 2. Test cases with tool-call assertions
- * 3. Appropriate prompts for the evaluation
- */
-export declare function compileMCPTask(task: MCPServerTaskDefinition, options?: MCPCompileOptions): MCPCompileResult;
-/** ModeHandler-conformant export for the mcp-server evaluation mode. */
-export declare const handler: ModeHandler;