@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js

@@ -0,0 +1,392 @@
+/**
+ * mcp-server-handler.test.ts — Tests for MCP server mode compilation.
+ *
+ * Tests the full MCP compilation pipeline: task validation, provider
+ * assembly, assertion mapping, test case generation, and end-to-end
+ * compilation of example tasks.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/mcp-server-handler.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { LiteracyVariant } from "../../normalize-mode.js";
+import { buildMCPAssertions, compileMCPTask, handler as mcpHandler, MCP_PROMPT_TEMPLATES, validateMCPTask, } from "../mode-handlers/mcp-server/index.js";
+import { allMCPExampleTasks, createAndPublishTask, inspectSchemaTask, queryDocumentsTask, semanticSearchTask, stdioServerTask, } from "../mode-handlers/__fixtures__/mcp-example-tasks.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function makeMinimalMCPTask(overrides) {
+    return {
+        mode: "mcp-server",
+        id: "test-mcp-task",
+        title: "Test MCP Task",
+        description: "A test MCP server evaluation task",
+        area: "mcp-server",
+        ...overrides,
+    };
+}
+/** Test models for compilation — simulates models from the registry */
+const TEST_MODELS = [
+    {
+        id: "anthropic:messages:claude-opus-4-6",
+        label: "Claude Opus 4.6",
+        config: { temperature: 0.2 },
+    },
+    {
+        id: "openai:responses:gpt-5.4",
+        label: "GPT 5.4",
+        config: { reasoning_effort: "medium" },
+    },
+];
+/** The custom MCP provider file:// path */
+const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+/** Helper to get provider config */
+function cfg(provider) {
+    return provider.config;
+}
+/** Helper to get mcpServer sub-config from provider */
+function serverCfg(provider) {
+    return cfg(provider)?.mcpServer;
+}
+// ---------------------------------------------------------------------------
+// handler.getPrompts() — prompt template ownership
+// ---------------------------------------------------------------------------
+describe("MCPServerHandler.getPrompts", () => {
+    it("returns prompt templates", () => {
+        const prompts = mcpHandler.getPrompts();
+        assert.ok(prompts, "getPrompts() should return a record");
+        assert.ok(Object.keys(prompts).length > 0, "should return at least one template");
+    });
+    it("returns templates keyed by MCP-specific IDs (not literacy names)", () => {
+        const prompts = mcpHandler.getPrompts();
+        const keys = Object.keys(prompts);
+        assert.ok(!keys.includes("with-docs"), "should not use literacy key 'with-docs'");
+        assert.ok(!keys.includes("without-docs"), "should not use literacy key 'without-docs'");
+        assert.ok(!keys.includes(LiteracyVariant.AGENTIC), "should not use literacy key 'agentic'");
+        assert.ok(keys.includes("mcp-server"), "should include 'mcp-server' template");
+    });
+    it("mcp-server template instructs model to use MCP tools", () => {
+        const prompts = mcpHandler.getPrompts();
+        const template = prompts["mcp-server"];
+        assert.ok(template, "mcp-server template should exist");
+        assert.ok(template.template.includes("{{task}}"), "should include {{task}} placeholder");
+        assert.ok(/tool/i.test(template.template), "template should mention tools (MCP-appropriate content)");
+    });
+    it("template has correct PromptTemplate shape", () => {
+        const prompts = mcpHandler.getPrompts();
+        const template = prompts["mcp-server"];
+        assert.equal(template.id, "mcp-server");
+        assert.ok(template.label, "should have a human-readable label");
+        assert.ok(template.template, "should have a template string");
+        assert.ok(Array.isArray(template.variables), "should declare variables");
+        assert.ok(template.variables.includes("task"), "variables should include 'task'");
+    });
+    it("exported MCP_PROMPT_TEMPLATES matches handler.getPrompts()", () => {
+        const fromHandler = mcpHandler.getPrompts();
+        assert.deepEqual(fromHandler, MCP_PROMPT_TEMPLATES);
+    });
+});
+// ---------------------------------------------------------------------------
+// validateMCPTask
+// ---------------------------------------------------------------------------
+describe("validateMCPTask", () => {
+    it("passes for a valid minimal task", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask());
+        assert.equal(errors.length, 0);
+    });
+    it("errors on missing ID", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({ id: "" }));
+        assert.ok(errors.some((e) => e.field === "id"));
+    });
+    it("errors on missing title", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({ title: "" }));
+        assert.ok(errors.some((e) => e.field === "title"));
+    });
+    it("errors on stdio transport without command", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "stdio" },
+        }));
+        assert.ok(errors.some((e) => e.field === "serverConfig.command"));
+    });
+    it("errors on sse transport without url", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "sse" },
+        }));
+        assert.ok(errors.some((e) => e.field === "serverConfig.url"));
+    });
+    it("passes for valid stdio config", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "stdio",
+                command: "node dist/server.js",
+            },
+        }));
+        assert.equal(errors.length, 0);
+    });
+    it("passes for valid sse config", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "sse",
+                url: "http://localhost:3000/sse",
+            },
+        }));
+        assert.equal(errors.length, 0);
+    });
+    it("errors on tool-called without value", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            assertions: [{ type: "tool-called" }],
+        }));
+        assert.ok(errors.some((e) => e.field === "assertions"));
+    });
+});
+// ---------------------------------------------------------------------------
+// compileMCPTask — provider assembly
+// ---------------------------------------------------------------------------
+describe("compileMCPTask", () => {
+    it("produces provider, tests, and prompts", () => {
+        const result = compileMCPTask(makeMinimalMCPTask(), { models: TEST_MODELS });
+        assert.ok(result.providers.length > 0, "Should produce providers");
+        assert.ok(result.tests.length > 0, "Should produce test cases");
+        assert.ok(result.prompts.length > 0, "Should produce prompts");
+    });
+    it("emits file:// providers using the custom MCP tool provider", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "stdio",
+                command: "node dist/server.js --flag",
+            },
+        }), { models: TEST_MODELS });
+        assert.equal(result.providers.length, 2, "One provider per model");
+        // All providers use the custom MCP tool provider path
+        assert.equal(result.providers[0].id, MCP_PROVIDER_PATH);
+        assert.equal(result.providers[1].id, MCP_PROVIDER_PATH);
+        // Model ID is passed in config
+        assert.equal(cfg(result.providers[0]).model, "anthropic:messages:claude-opus-4-6");
+        assert.equal(cfg(result.providers[1]).model, "openai:responses:gpt-5.4");
+        // MCP server config is in config.mcpServer
+        const server = serverCfg(result.providers[0]);
+        assert.equal(server.command, "node dist/server.js --flag");
+    });
+    it("preserves model config in provider config", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "sse", url: "http://localhost:3000/sse" },
+        }), { models: TEST_MODELS });
+        const c = cfg(result.providers[0]);
+        assert.equal(c.temperature, 0.2, "Model config preserved");
+        assert.ok(c.mcpServer, "MCP server config present");
+        assert.equal(c.maxToolRounds, 5, "Default maxToolRounds");
+    });
+    it("builds MCP server config for URL-based transport", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "sse", url: "http://localhost:3000/sse" },
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.equal(server.url, "http://localhost:3000/sse");
+    });
+    it("maps auth config to mcpServer config", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+                auth: { type: "bearer", token: "{{env.MY_TOKEN}}" },
+            },
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.auth, { type: "bearer", token: "{{env.MY_TOKEN}}" });
+    });
+    it("maps capabilities to mcpTools config", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            capabilities: ["query_documents", "get_schema"],
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+            },
+        }), { models: TEST_MODELS });
+        assert.deepEqual(cfg(result.providers[0]).mcpTools, [
+            "query_documents",
+            "get_schema",
+        ]);
+    });
+    it("uses task-level models override when specified", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            models: ["anthropic:messages:claude-sonnet-4-20250514"],
+            serverConfig: { transport: "sse", url: "http://localhost:3000" },
+        }), { models: TEST_MODELS });
+        assert.equal(result.providers.length, 1);
+        assert.equal(cfg(result.providers[0]).model, "anthropic:messages:claude-sonnet-4-20250514");
+    });
+    it("respects task-level maxToolRounds", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({ maxToolRounds: 10 }), {
+            models: TEST_MODELS,
+        });
+        assert.equal(cfg(result.providers[0]).maxToolRounds, 10);
+    });
+    it("falls back to default model when no models provided", () => {
+        const result = compileMCPTask(makeMinimalMCPTask());
+        assert.ok(result.providers.length > 0, "Should have a fallback provider");
+        assert.ok(result.warnings.some((w) => w.includes("no models")));
+    });
+    it("uses task description as prompt text", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            description: "Test the getDocument tool",
+        }));
+        assert.equal(result.prompts[0].raw, "Test the getDocument tool");
+    });
+    it("prefers prompt.text over description", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            description: "Description",
+            prompt: { text: "Custom prompt text" },
+        }));
+        assert.equal(result.prompts[0].raw, "Custom prompt text");
+    });
+    it("includes task vars in test case", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            prompt: {
+                vars: { task: "Do the thing", extra: "value" },
+            },
+        }));
+        assert.equal(result.tests[0].vars.task, "Do the thing");
+        assert.equal(result.tests[0].vars.extra, "value");
+    });
+    it("creates multi-turn test case when multiTurn is defined", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            multiTurn: {
+                turns: [
+                    { role: "user", content: "Hello" },
+                    { role: "assistant", content: "Hi" },
+                ],
+            },
+        }));
+        assert.equal(result.tests.length, 2);
+        assert.ok(result.tests[1].description.includes("[multi-turn]"));
+    });
+    it("warns when serverConfig is missing", () => {
+        const result = compileMCPTask(makeMinimalMCPTask());
+        assert.ok(result.warnings.some((w) => w.includes("no serverConfig")));
+    });
+    it("sets grader provider on LLM assertions", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    value: "Check quality",
+                },
+            ],
+        }), { graderProvider: "openai:chat:gpt-5" });
+        const llmAssert = result.tests[0].assert?.find((a) => a.type === "llm-rubric");
+        assert.ok(llmAssert);
+        assert.equal(llmAssert.provider, "openai:chat:gpt-5");
+    });
+});
+// ---------------------------------------------------------------------------
+// buildMCPAssertions
+// ---------------------------------------------------------------------------
+describe("buildMCPAssertions", () => {
+    const ctx = { taskId: "test", capabilities: [], graderProvider: undefined };
+    it("maps tool-called to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-called", value: "getDocument" }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("getDocument"), "Should reference tool name");
+    });
+    it("maps tool-input-matches to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-input-matches", value: { id: "doc-123" } }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("doc-123"));
+    });
+    it("maps tool-output-matches to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-output-matches", value: { title: "Hello" } }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("Hello"));
+    });
+    it("maps error-returned to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "error-returned", value: { code: -32602 } }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("-32602"));
+    });
+    it("maps capability-available to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "capability-available", value: "tools/list" }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("tools/list"));
+    });
+    it("passes through standard assertion types", () => {
+        const { assertions } = buildMCPAssertions([{ type: "contains", value: "result" }, { type: "is-json" }], ctx);
+        assert.equal(assertions.length, 2);
+        assert.equal(assertions[0].type, "contains");
+        assert.equal(assertions[1].type, "is-json");
+    });
+    it("preserves assertion weights", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-called", value: "test", weight: 0.5 }], ctx);
+        assert.equal(assertions[0].weight, 0.5);
+    });
+    it("warns on unknown assertion type", () => {
+        const { warnings } = buildMCPAssertions([{ type: "unknown-type", value: "x" }], ctx);
+        assert.ok(warnings.some((w) => w.includes("unknown")));
+    });
+});
+// ---------------------------------------------------------------------------
+// Example task compilation (end-to-end)
+// ---------------------------------------------------------------------------
+describe("example MCP tasks — end-to-end compilation", () => {
+    const opts = { models: TEST_MODELS };
+    it("compiles all example tasks without errors", () => {
+        for (const task of allMCPExampleTasks) {
+            const result = compileMCPTask(task, opts);
+            assert.ok(result.providers.length > 0, `${task.id}: should produce providers`);
+            assert.ok(result.tests.length > 0, `${task.id}: should produce test cases`);
+            assert.ok(result.prompts.length > 0, `${task.id}: should produce prompts`);
+        }
+    });
+    it("query task has tool-called + contains + llm-rubric assertions", () => {
+        const result = compileMCPTask(queryDocumentsTask, opts);
+        const asserts = result.tests[0].assert;
+        assert.equal(asserts.length, 4);
+        assert.equal(asserts[0].type, "javascript");
+        assert.equal(asserts[1].type, "contains");
+        assert.equal(asserts[2].type, "contains");
+        assert.equal(asserts[3].type, "llm-rubric");
+    });
+    it("schema task uses get_schema tool", () => {
+        const result = compileMCPTask(inspectSchemaTask, opts);
+        const asserts = result.tests[0].assert;
+        assert.ok(asserts.some((a) => a.type === "javascript" && a.value.includes("get_schema")), "Should have tool-called assertion for get_schema");
+    });
+    it("create-publish task produces multi-turn test case", () => {
+        const result = compileMCPTask(createAndPublishTask, opts);
+        assert.equal(result.tests.length, 2);
+        assert.ok(result.tests[1].description?.includes("[multi-turn]"));
+    });
+    it("stdio task uses custom provider with command config", () => {
+        const result = compileMCPTask(stdioServerTask, opts);
+        assert.equal(result.providers[0].id, MCP_PROVIDER_PATH);
+        assert.equal(cfg(result.providers[0]).model, "anthropic:messages:claude-opus-4-6");
+        const server = serverCfg(result.providers[0]);
+        assert.equal(server.command, "node dist/sanity-mcp-server.js");
+    });
+    it("semantic search task has two tool-called + one llm-rubric assertion", () => {
+        const result = compileMCPTask(semanticSearchTask, opts);
+        const asserts = result.tests[0].assert;
+        assert.equal(asserts.length, 3);
+        assert.equal(asserts[0].type, "javascript");
+        assert.ok(asserts[0].value.includes("list_embeddings_indices"));
+        assert.equal(asserts[1].type, "javascript");
+        assert.ok(asserts[1].value.includes("semantic_search"));
+        assert.equal(asserts[2].type, "llm-rubric");
+    });
+    it("remote task has bearer auth and tools filter", () => {
+        const result = compileMCPTask(queryDocumentsTask, opts);
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.auth, {
+            type: "bearer",
+            token: "{{env.SANITY_MCP_AUTH_TOKEN}}",
+        });
+        assert.deepEqual(cfg(result.providers[0]).mcpTools, [
+            "query_documents",
+            "get_schema",
+        ]);
+    });
+});

package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts

@@ -0,0 +1,9 @@
+/**
+ * promptfoo-compiler.test.ts — Unit tests for the PromptfooCompiler.
+ *
+ * Tests compilation of a TaskGraph into Promptfoo configuration,
+ * including provider assembly, prompt generation, and test case creation.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/promptfoo-compiler.test.ts
+ */
+export {};