@sanity/ailf 0.4.1 → 1.0.0

package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js

@@ -0,0 +1,355 @@
+/**
+ * mcp-server-handler.test.ts — Tests for MCP server mode compilation.
+ *
+ * Tests the full MCP compilation pipeline: task validation, provider
+ * assembly, assertion mapping, test case generation, and end-to-end
+ * compilation of example tasks.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/mcp-server-handler.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { LiteracyVariant } from "../../normalize-mode.js";
+import { compileMCPTask, handler as mcpHandler, MCP_PROMPT_TEMPLATES, validateMCPTask, } from "../mode-handlers/mcp-server-handler.js";
+import { buildMCPAssertions } from "../mode-handlers/mcp-assertions.js";
+import { allMCPExampleTasks, createAndPublishTask, inspectSchemaTask, queryDocumentsTask, semanticSearchTask, stdioServerTask, } from "../mode-handlers/__fixtures__/mcp-example-tasks.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function makeMinimalMCPTask(overrides) {
+    return {
+        mode: "mcp-server",
+        id: "test-mcp-task",
+        title: "Test MCP Task",
+        description: "A test MCP server evaluation task",
+        area: "mcp-server",
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// handler.getPrompts() — prompt template ownership
+// ---------------------------------------------------------------------------
+describe("MCPServerHandler.getPrompts", () => {
+    it("returns prompt templates", () => {
+        const prompts = mcpHandler.getPrompts();
+        assert.ok(prompts, "getPrompts() should return a record");
+        assert.ok(Object.keys(prompts).length > 0, "should return at least one template");
+    });
+    it("returns templates keyed by MCP-specific IDs (not literacy names)", () => {
+        const prompts = mcpHandler.getPrompts();
+        const keys = Object.keys(prompts);
+        // Must not use literacy template names
+        assert.ok(!keys.includes("with-docs"), "should not use literacy key 'with-docs'");
+        assert.ok(!keys.includes("without-docs"), "should not use literacy key 'without-docs'");
+        assert.ok(!keys.includes(LiteracyVariant.AGENTIC), "should not use literacy key 'agentic'");
+        // Must have MCP-appropriate key(s)
+        assert.ok(keys.includes("mcp-server"), "should include 'mcp-server' template");
+    });
+    it("mcp-server template instructs model to use MCP tools", () => {
+        const prompts = mcpHandler.getPrompts();
+        const template = prompts["mcp-server"];
+        assert.ok(template, "mcp-server template should exist");
+        assert.ok(template.template.includes("{{task}}"), "should include {{task}} placeholder");
+        // Should reference MCP tools / tool usage
+        assert.ok(/tool/i.test(template.template), "template should mention tools (MCP-appropriate content)");
+    });
+    it("template has correct PromptTemplate shape", () => {
+        const prompts = mcpHandler.getPrompts();
+        const template = prompts["mcp-server"];
+        assert.equal(template.id, "mcp-server");
+        assert.ok(template.label, "should have a human-readable label");
+        assert.ok(template.template, "should have a template string");
+        assert.ok(Array.isArray(template.variables), "should declare variables");
+        assert.ok(template.variables.includes("task"), "variables should include 'task'");
+    });
+    it("exported MCP_PROMPT_TEMPLATES matches handler.getPrompts()", () => {
+        const fromHandler = mcpHandler.getPrompts();
+        assert.deepEqual(fromHandler, MCP_PROMPT_TEMPLATES);
+    });
+});
+// ---------------------------------------------------------------------------
+// validateMCPTask
+// ---------------------------------------------------------------------------
+describe("validateMCPTask", () => {
+    it("passes for a valid minimal task", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask());
+        assert.equal(errors.length, 0);
+    });
+    it("errors on missing ID", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({ id: "" }));
+        assert.ok(errors.some((e) => e.field === "id"));
+    });
+    it("errors on missing title", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({ title: "" }));
+        assert.ok(errors.some((e) => e.field === "title"));
+    });
+    it("errors on stdio transport without command", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "stdio" },
+        }));
+        assert.ok(errors.some((e) => e.field === "serverConfig.command"));
+    });
+    it("errors on sse transport without url", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "sse" },
+        }));
+        assert.ok(errors.some((e) => e.field === "serverConfig.url"));
+    });
+    it("passes for valid stdio config", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "stdio",
+                command: "node dist/server.js",
+            },
+        }));
+        assert.equal(errors.length, 0);
+    });
+    it("passes for valid sse config", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "sse",
+                url: "http://localhost:3000/sse",
+            },
+        }));
+        assert.equal(errors.length, 0);
+    });
+    it("errors on tool-called without value", () => {
+        const errors = validateMCPTask(makeMinimalMCPTask({
+            assertions: [{ type: "tool-called" }],
+        }));
+        assert.ok(errors.some((e) => e.field === "assertions"));
+    });
+});
+// ---------------------------------------------------------------------------
+// compileMCPTask
+// ---------------------------------------------------------------------------
+describe("compileMCPTask", () => {
+    it("produces provider, tests, and prompts", () => {
+        const result = compileMCPTask(makeMinimalMCPTask());
+        assert.ok(result.providers.length > 0, "Should produce providers");
+        assert.ok(result.tests.length > 0, "Should produce test cases");
+        assert.ok(result.prompts.length > 0, "Should produce prompts");
+    });
+    it("builds Promptfoo-native MCP provider for stdio", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "stdio",
+                command: "node dist/server.js --flag",
+            },
+        }));
+        assert.equal(result.providers.length, 1);
+        assert.equal(result.providers[0].id, "mcp");
+        const config = result.providers[0].config;
+        assert.equal(config.enabled, true);
+        const server = config.server;
+        assert.equal(server.command, "node");
+        assert.deepEqual(server.args, ["dist/server.js", "--flag"]);
+    });
+    it("builds Promptfoo-native MCP provider for URL-based transport", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "sse",
+                url: "http://localhost:3000/sse",
+            },
+        }));
+        assert.equal(result.providers[0].id, "mcp");
+        const config = result.providers[0].config;
+        const server = config.server;
+        assert.equal(server.url, "http://localhost:3000/sse");
+    });
+    it("maps auth config to Promptfoo provider", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+                auth: {
+                    type: "bearer",
+                    token: "{{env.MY_TOKEN}}",
+                },
+            },
+        }));
+        const config = result.providers[0].config;
+        const server = config.server;
+        assert.deepEqual(server.auth, {
+            type: "bearer",
+            token: "{{env.MY_TOKEN}}",
+        });
+    });
+    it("maps capabilities to Promptfoo tools filter", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            capabilities: ["query_documents", "get_schema"],
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+            },
+        }));
+        const config = result.providers[0].config;
+        assert.deepEqual(config.tools, ["query_documents", "get_schema"]);
+    });
+    it("uses task description as prompt text", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            description: "Test the getDocument tool",
+        }));
+        assert.equal(result.prompts[0].raw, "Test the getDocument tool");
+    });
+    it("prefers prompt.text over description", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            description: "Description",
+            prompt: { text: "Custom prompt text" },
+        }));
+        assert.equal(result.prompts[0].raw, "Custom prompt text");
+    });
+    it("includes task vars in test case", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            prompt: {
+                vars: { task: "Do the thing", extra: "value" },
+            },
+        }));
+        assert.equal(result.tests[0].vars.task, "Do the thing");
+        assert.equal(result.tests[0].vars.extra, "value");
+    });
+    it("creates multi-turn test case when multiTurn is defined", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            multiTurn: {
+                turns: [
+                    { role: "user", content: "Hello" },
+                    { role: "assistant", content: "Hi" },
+                ],
+            },
+        }));
+        // Primary + multi-turn test cases
+        assert.equal(result.tests.length, 2);
+        assert.ok(result.tests[1].description.includes("[multi-turn]"));
+    });
+    it("warns when serverConfig is missing", () => {
+        const result = compileMCPTask(makeMinimalMCPTask());
+        assert.ok(result.warnings.some((w) => w.includes("no serverConfig")));
+    });
+    it("sets grader provider on LLM assertions", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    value: "Check quality",
+                },
+            ],
+        }), { graderProvider: "openai:chat:gpt-5" });
+        const llmAssert = result.tests[0].assert?.find((a) => a.type === "llm-rubric");
+        assert.ok(llmAssert);
+        assert.equal(llmAssert.provider, "openai:chat:gpt-5");
+    });
+});
+// ---------------------------------------------------------------------------
+// buildMCPAssertions
+// ---------------------------------------------------------------------------
+describe("buildMCPAssertions", () => {
+    const ctx = { taskId: "test", capabilities: [], graderProvider: undefined };
+    it("maps tool-called to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-called", value: "getDocument" }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("getDocument"), "Should reference tool name");
+    });
+    it("maps tool-input-matches to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-input-matches", value: { id: "doc-123" } }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("doc-123"));
+    });
+    it("maps tool-output-matches to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-output-matches", value: { title: "Hello" } }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("Hello"));
+    });
+    it("maps error-returned to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "error-returned", value: { code: -32602 } }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("-32602"));
+    });
+    it("maps capability-available to javascript assertion", () => {
+        const { assertions } = buildMCPAssertions([{ type: "capability-available", value: "tools/list" }], ctx);
+        assert.equal(assertions.length, 1);
+        assert.equal(assertions[0].type, "javascript");
+        assert.ok(assertions[0].value.includes("tools/list"));
+    });
+    it("passes through standard assertion types", () => {
+        const { assertions } = buildMCPAssertions([{ type: "contains", value: "result" }, { type: "is-json" }], ctx);
+        assert.equal(assertions.length, 2);
+        assert.equal(assertions[0].type, "contains");
+        assert.equal(assertions[1].type, "is-json");
+    });
+    it("preserves assertion weights", () => {
+        const { assertions } = buildMCPAssertions([{ type: "tool-called", value: "test", weight: 0.5 }], ctx);
+        assert.equal(assertions[0].weight, 0.5);
+    });
+    it("warns on unknown assertion type", () => {
+        const { warnings } = buildMCPAssertions([{ type: "unknown-type", value: "x" }], ctx);
+        assert.ok(warnings.some((w) => w.includes("unknown")));
+    });
+});
+// ---------------------------------------------------------------------------
+// Example task compilation (end-to-end)
+// ---------------------------------------------------------------------------
+describe("example MCP tasks — end-to-end compilation", () => {
+    it("compiles all example tasks without errors", () => {
+        for (const task of allMCPExampleTasks) {
+            const result = compileMCPTask(task);
+            assert.ok(result.providers.length > 0, `${task.id}: should produce providers`);
+            assert.ok(result.tests.length > 0, `${task.id}: should produce test cases`);
+            assert.ok(result.prompts.length > 0, `${task.id}: should produce prompts`);
+        }
+    });
+    it("query task has tool-called + contains + llm-rubric assertions", () => {
+        const result = compileMCPTask(queryDocumentsTask);
+        const asserts = result.tests[0].assert;
+        // tool-called (→ javascript), contains × 2, llm-rubric
+        assert.equal(asserts.length, 4);
+        assert.equal(asserts[0].type, "javascript"); // tool-called → javascript
+        assert.equal(asserts[1].type, "contains");
+        assert.equal(asserts[2].type, "contains");
+        assert.equal(asserts[3].type, "llm-rubric");
+    });
+    it("schema task uses get_schema tool", () => {
+        const result = compileMCPTask(inspectSchemaTask);
+        const asserts = result.tests[0].assert;
+        assert.ok(asserts.some((a) => a.type === "javascript" && a.value.includes("get_schema")), "Should have tool-called assertion for get_schema");
+    });
+    it("create-publish task produces multi-turn test case", () => {
+        const result = compileMCPTask(createAndPublishTask);
+        // Primary + multi-turn
+        assert.equal(result.tests.length, 2);
+        assert.ok(result.tests[1].description?.includes("[multi-turn]"));
+    });
+    it("stdio task has Promptfoo-native MCP provider with command", () => {
+        const result = compileMCPTask(stdioServerTask);
+        assert.equal(result.providers[0].id, "mcp");
+        const config = result.providers[0].config;
+        assert.equal(config.enabled, true);
+        const server = config.server;
+        assert.equal(server.command, "node");
+        assert.deepEqual(server.args, ["dist/sanity-mcp-server.js"]);
+    });
+    it("semantic search task has two tool-called + one llm-rubric assertion", () => {
+        const result = compileMCPTask(semanticSearchTask);
+        const asserts = result.tests[0].assert;
+        // tool-called × 2 (→ javascript) + llm-rubric
+        assert.equal(asserts.length, 3);
+        assert.equal(asserts[0].type, "javascript"); // tool-called → javascript
+        assert.ok(asserts[0].value.includes("list_embeddings_indices"), "Should have tool-called assertion for list_embeddings_indices");
+        assert.equal(asserts[1].type, "javascript"); // tool-called → javascript
+        assert.ok(asserts[1].value.includes("semantic_search"), "Should have tool-called assertion for semantic_search");
+        assert.equal(asserts[2].type, "llm-rubric");
+    });
+    it("remote task has bearer auth and tools filter", () => {
+        const result = compileMCPTask(queryDocumentsTask);
+        const config = result.providers[0].config;
+        const server = config.server;
+        assert.deepEqual(server.auth, {
+            type: "bearer",
+            token: "{{env.SANITY_MCP_AUTH_TOKEN}}",
+        });
+        assert.deepEqual(config.tools, ["query_documents", "get_schema"]);
+    });
+});

package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts

@@ -0,0 +1,9 @@
+/**
+ * promptfoo-compiler.test.ts — Unit tests for the PromptfooCompiler.
+ *
+ * Tests compilation of a TaskGraph into Promptfoo configuration,
+ * including provider assembly, prompt generation, and test case creation.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/promptfoo-compiler.test.ts
+ */
+export {};