@langwatch/mcp-server 0.4.0 → 0.6.0

package/src/__tests__/model-provider-tools.unit.test.ts

@@ -0,0 +1,190 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api-model-providers.js", () => ({
+  listModelProviders: vi.fn(),
+  setModelProvider: vi.fn(),
+}));
+
+import {
+  listModelProviders,
+  setModelProvider,
+} from "../langwatch-api-model-providers.js";
+
+import { handleListModelProviders } from "../tools/list-model-providers.js";
+import { handleSetModelProvider } from "../tools/set-model-provider.js";
+
+const mockListModelProviders = vi.mocked(listModelProviders);
+const mockSetModelProvider = vi.mocked(setModelProvider);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleListModelProviders()", () => {
+  const sampleProviders = {
+    openai: {
+      provider: "openai",
+      enabled: true,
+      customKeys: { OPENAI_API_KEY: "HAS_KEY" },
+      models: ["gpt-4o", "gpt-4o-mini"],
+      embeddingsModels: ["text-embedding-3-small"],
+      deploymentMapping: null,
+      extraHeaders: [],
+    },
+    anthropic: {
+      provider: "anthropic",
+      enabled: false,
+      customKeys: null,
+      models: ["claude-sonnet-4-5-20250929"],
+      embeddingsModels: null,
+      deploymentMapping: null,
+      extraHeaders: [],
+    },
+  };
+
+  describe("when providers exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListModelProviders.mockResolvedValue(sampleProviders);
+      result = await handleListModelProviders();
+    });
+
+    it("includes the total count header", () => {
+      expect(result).toContain("# Model Providers (2 total)");
+    });
+
+    it("includes provider name", () => {
+      expect(result).toContain("## openai");
+    });
+
+    it("shows enabled status", () => {
+      expect(result).toContain("**Status**: enabled");
+    });
+
+    it("shows disabled status", () => {
+      expect(result).toContain("**Status**: disabled");
+    });
+
+    it("shows key fields that are set", () => {
+      expect(result).toContain("OPENAI_API_KEY: set");
+    });
+
+    it("shows model count", () => {
+      expect(result).toContain("2 available");
+    });
+  });
+
+  describe("when no providers exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListModelProviders.mockResolvedValue({});
+      result = await handleListModelProviders();
+    });
+
+    it("returns a no-providers message", () => {
+      expect(result).toContain("No model providers configured");
+    });
+
+    it("includes a tip to use platform_set_model_provider", () => {
+      expect(result).toContain("platform_set_model_provider");
+    });
+  });
+});
+
+describe("handleSetModelProvider()", () => {
+  describe("when update succeeds", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockSetModelProvider.mockResolvedValue({
+        openai: {
+          provider: "openai",
+          enabled: true,
+          customKeys: { OPENAI_API_KEY: "HAS_KEY" },
+          models: ["gpt-4o"],
+          embeddingsModels: null,
+          deploymentMapping: null,
+          extraHeaders: [],
+        },
+      });
+      result = await handleSetModelProvider({
+        provider: "openai",
+        enabled: true,
+        customKeys: { OPENAI_API_KEY: "sk-test123" },
+      });
+    });
+
+    it("confirms update", () => {
+      expect(result).toContain("Model provider updated successfully!");
+    });
+
+    it("includes provider name", () => {
+      expect(result).toContain("**Provider**: openai");
+    });
+
+    it("shows enabled status", () => {
+      expect(result).toContain("**Status**: enabled");
+    });
+
+    it("shows key fields", () => {
+      expect(result).toContain("OPENAI_API_KEY: set");
+    });
+  });
+
+  describe("when setting default model without provider prefix", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockSetModelProvider.mockResolvedValue({
+        openai: {
+          provider: "openai",
+          enabled: true,
+          customKeys: null,
+          models: ["gpt-4o"],
+          embeddingsModels: null,
+          deploymentMapping: null,
+          extraHeaders: [],
+        },
+      });
+      result = await handleSetModelProvider({
+        provider: "openai",
+        enabled: true,
+        defaultModel: "gpt-4o",
+      });
+    });
+
+    it("prepends provider prefix in response", () => {
+      expect(result).toContain("**Default Model**: openai/gpt-4o");
+    });
+  });
+
+  describe("when setting default model with provider prefix already", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockSetModelProvider.mockResolvedValue({
+        openai: {
+          provider: "openai",
+          enabled: true,
+          customKeys: null,
+          models: ["gpt-4o"],
+          embeddingsModels: null,
+          deploymentMapping: null,
+          extraHeaders: [],
+        },
+      });
+      result = await handleSetModelProvider({
+        provider: "openai",
+        enabled: true,
+        defaultModel: "openai/gpt-4o",
+      });
+    });
+
+    it("keeps the prefix as-is", () => {
+      expect(result).toContain("**Default Model**: openai/gpt-4o");
+      expect(result).not.toContain("openai/openai/");
+    });
+  });
+});

package/src/__tests__/scenario-tools.integration.test.ts

@@ -0,0 +1,286 @@
+import { createServer, type Server } from "http";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { initConfig } from "../config.js";
+
+// --- Canned responses for scenario API endpoints ---
+
+const CANNED_SCENARIOS_LIST = [
+  {
+    id: "scen_abc123",
+    name: "Login Flow Happy Path",
+    situation: "User attempts to log in with valid credentials",
+    criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+    labels: ["auth", "happy-path"],
+  },
+  {
+    id: "scen_def456",
+    name: "Password Reset",
+    situation: "User requests a password reset link",
+    criteria: ["Sends reset email"],
+    labels: ["auth"],
+  },
+];
+
+const CANNED_SCENARIO_DETAIL = {
+  id: "scen_abc123",
+  name: "Login Flow Happy Path",
+  situation: "User attempts to log in with valid credentials",
+  criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+  labels: ["auth", "happy-path"],
+};
+
+const CANNED_SCENARIO_CREATED = {
+  id: "scen_new789",
+  name: "Login Flow Happy Path",
+  situation: "User attempts to log in with valid creds",
+  criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+  labels: ["auth", "happy-path"],
+};
+
+const CANNED_SCENARIO_UPDATED = {
+  id: "scen_abc123",
+  name: "Login Flow - Valid Credentials",
+  situation: "User logs in with correct email and pass",
+  criteria: [
+    "Responds with welcome message",
+    "Sets session cookie",
+    "Redirects to dashboard",
+  ],
+  labels: ["auth", "happy-path"],
+};
+
+const CANNED_SCENARIO_ARCHIVED = {
+  id: "scen_abc123",
+  archived: true,
+};
+
+// --- Mock HTTP Server ---
+
+function createMockServer(): Server {
+  return createServer((req, res) => {
+    const authToken = req.headers["x-auth-token"];
+    if (authToken !== "test-integration-key") {
+      res.writeHead(401, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ message: "Invalid auth token." }));
+      return;
+    }
+
+    let body = "";
+    req.on("data", (chunk: string) => (body += chunk));
+    req.on("end", () => {
+      const url = req.url ?? "";
+      res.setHeader("Content-Type", "application/json");
+
+      // GET /api/scenarios - list all scenarios
+      if (url === "/api/scenarios" && req.method === "GET") {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIOS_LIST));
+      }
+      // GET /api/scenarios/:id - get scenario detail
+      else if (
+        url.match(/^\/api\/scenarios\/scen_abc123(\?|$)/) &&
+        req.method === "GET"
+      ) {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIO_DETAIL));
+      }
+      // GET /api/scenarios/:id - not found
+      else if (
+        url.match(/^\/api\/scenarios\/scen_nonexistent(\?|$)/) &&
+        req.method === "GET"
+      ) {
+        res.writeHead(404);
+        res.end(JSON.stringify({ message: "Scenario not found" }));
+      }
+      // POST /api/scenarios - create scenario
+      else if (url === "/api/scenarios" && req.method === "POST") {
+        const parsed = JSON.parse(body);
+        if (!parsed.name) {
+          res.writeHead(400);
+          res.end(JSON.stringify({ message: "Validation error: name is required" }));
+        } else {
+          res.writeHead(201);
+          res.end(JSON.stringify(CANNED_SCENARIO_CREATED));
+        }
+      }
+      // PUT /api/scenarios/:id - update scenario
+      else if (
+        url.match(/^\/api\/scenarios\/scen_abc123$/) &&
+        req.method === "PUT"
+      ) {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIO_UPDATED));
+      }
+      // PUT /api/scenarios/:id - not found
+      else if (
+        url.match(/^\/api\/scenarios\/scen_nonexistent$/) &&
+        req.method === "PUT"
+      ) {
+        res.writeHead(404);
+        res.end(JSON.stringify({ message: "Scenario not found" }));
+      }
+      // DELETE /api/scenarios/:id - archive scenario
+      else if (
+        url.match(/^\/api\/scenarios\/scen_abc123$/) &&
+        req.method === "DELETE"
+      ) {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIO_ARCHIVED));
+      }
+      else {
+        res.writeHead(404);
+        res.end(
+          JSON.stringify({ message: `Not found: ${req.method} ${url}` })
+        );
+      }
+    });
+  });
+}
+
+// --- Integration Tests ---
+// These verify that MCP tool handlers correctly communicate with the REST API
+// (auth, HTTP methods, status codes, error propagation).
+// Formatting/digest logic is tested in scenario-tools.unit.test.ts.
+
+describe("MCP scenario tools integration", () => {
+  let server: Server;
+  let port: number;
+
+  beforeAll(async () => {
+    server = createMockServer();
+    await new Promise<void>((resolve) => {
+      server.listen(0, () => {
+        const addr = server.address();
+        port = typeof addr === "object" && addr ? addr.port : 0;
+        initConfig({
+          apiKey: "test-integration-key",
+          endpoint: `http://localhost:${port}`,
+        });
+        resolve();
+      });
+    });
+  });
+
+  afterAll(async () => {
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+  });
+
+  describe("platform_list_scenarios", () => {
+    describe("when the API returns scenarios", () => {
+      it("returns a non-empty result", async () => {
+        const { handleListScenarios } = await import(
+          "../tools/list-scenarios.js"
+        );
+        const result = await handleListScenarios({});
+        expect(result.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe("when format is json", () => {
+      it("returns parseable JSON matching the API response", async () => {
+        const { handleListScenarios } = await import(
+          "../tools/list-scenarios.js"
+        );
+        const result = await handleListScenarios({ format: "json" });
+        expect(JSON.parse(result)).toEqual(CANNED_SCENARIOS_LIST);
+      });
+    });
+  });
+
+  describe("platform_get_scenario", () => {
+    describe("when the scenario exists", () => {
+      it("returns a non-empty result", async () => {
+        const { handleGetScenario } = await import(
+          "../tools/get-scenario.js"
+        );
+        const result = await handleGetScenario({ scenarioId: "scen_abc123" });
+        expect(result.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe("when the scenario does not exist", () => {
+      it("propagates the 404 error", async () => {
+        const { handleGetScenario } = await import(
+          "../tools/get-scenario.js"
+        );
+        await expect(
+          handleGetScenario({ scenarioId: "scen_nonexistent" })
+        ).rejects.toThrow("404");
+      });
+    });
+  });
+
+  describe("platform_create_scenario", () => {
+    describe("when valid data is provided", () => {
+      it("returns confirmation with new scenario ID", async () => {
+        const { handleCreateScenario } = await import(
+          "../tools/create-scenario.js"
+        );
+        const result = await handleCreateScenario({
+          name: "Login Flow Happy Path",
+          situation: "User attempts to log in with valid creds",
+          criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+          labels: ["auth", "happy-path"],
+        });
+        expect(result).toContain("scen_new789");
+      });
+    });
+
+    describe("when name is empty", () => {
+      it("propagates the validation error", async () => {
+        const { handleCreateScenario } = await import(
+          "../tools/create-scenario.js"
+        );
+        await expect(
+          handleCreateScenario({
+            name: "",
+            situation: "Some situation",
+          })
+        ).rejects.toThrow();
+      });
+    });
+  });
+
+  describe("platform_update_scenario", () => {
+    describe("when the scenario exists", () => {
+      it("returns a non-empty result", async () => {
+        const { handleUpdateScenario } = await import(
+          "../tools/update-scenario.js"
+        );
+        const result = await handleUpdateScenario({
+          scenarioId: "scen_abc123",
+          name: "Login Flow - Valid Credentials",
+        });
+        expect(result.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe("when the scenario does not exist", () => {
+      it("propagates the 404 error", async () => {
+        const { handleUpdateScenario } = await import(
+          "../tools/update-scenario.js"
+        );
+        await expect(
+          handleUpdateScenario({
+            scenarioId: "scen_nonexistent",
+            name: "Updated Name",
+          })
+        ).rejects.toThrow("404");
+      });
+    });
+  });
+
+  describe("platform_archive_scenario", () => {
+    describe("when the scenario exists", () => {
+      it("returns confirmation that scenario was archived", async () => {
+        const { handleArchiveScenario } = await import(
+          "../tools/archive-scenario.js"
+        );
+        const result = await handleArchiveScenario({
+          scenarioId: "scen_abc123",
+        });
+        expect(result).toContain("archived");
+      });
+    });
+  });
+});

package/src/__tests__/scenario-tools.unit.test.ts

@@ -0,0 +1,185 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api-scenarios.js", () => ({
+  listScenarios: vi.fn(),
+  getScenario: vi.fn(),
+}));
+
+import {
+  listScenarios,
+  getScenario,
+} from "../langwatch-api-scenarios.js";
+
+import { handleListScenarios } from "../tools/list-scenarios.js";
+import { handleGetScenario } from "../tools/get-scenario.js";
+import { formatScenarioSchema } from "../tools/discover-scenario-schema.js";
+
+const mockListScenarios = vi.mocked(listScenarios);
+const mockGetScenario = vi.mocked(getScenario);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleListScenarios()", () => {
+  const sampleScenarios = [
+    {
+      id: "scen_abc123",
+      name: "Login Flow Happy Path",
+      situation:
+        "User attempts to log in with valid credentials and expects a welcome message back from the system",
+      criteria: [
+        "Responds with a welcome message",
+        "Includes user name in greeting",
+        "Sets session cookie",
+      ],
+      labels: ["auth", "happy-path"],
+    },
+    {
+      id: "scen_def456",
+      name: "Error Handling",
+      situation: "User sends malformed input",
+      criteria: ["Returns 400 status"],
+      labels: ["error"],
+    },
+  ];
+
+  describe("when scenarios exist (digest mode)", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListScenarios.mockResolvedValue(sampleScenarios);
+      result = await handleListScenarios({});
+    });
+
+    it("includes scenario id", () => {
+      expect(result).toContain("scen_abc123");
+    });
+
+    it("includes scenario name", () => {
+      expect(result).toContain("Login Flow Happy Path");
+    });
+
+    it("includes truncated situation preview", () => {
+      expect(result).toContain("User attempts to log in");
+      expect(result).not.toContain(
+        "User attempts to log in with valid credentials and expects a welcome message back from the system"
+      );
+    });
+
+    it("shows criteria count per scenario", () => {
+      expect(result).toContain("3 criteria");
+    });
+
+    it("includes labels", () => {
+      expect(result).toContain("auth");
+    });
+
+    it("includes all scenarios in the list", () => {
+      expect(result).toContain("scen_def456");
+    });
+
+    it("includes the total count header", () => {
+      expect(result).toContain("# Scenarios (2 total)");
+    });
+  });
+
+  describe("when no scenarios exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListScenarios.mockResolvedValue([]);
+      result = await handleListScenarios({});
+    });
+
+    it("returns a no-scenarios message", () => {
+      expect(result).toContain("No scenarios found");
+    });
+
+    it("includes a tip to use platform_create_scenario", () => {
+      expect(result).toContain("platform_create_scenario");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the scenario structure", async () => {
+      mockListScenarios.mockResolvedValue(sampleScenarios);
+      const result = await handleListScenarios({ format: "json" });
+      expect(JSON.parse(result)).toEqual(sampleScenarios);
+    });
+  });
+});
+
+describe("handleGetScenario()", () => {
+  const sampleScenario = {
+    id: "scen_abc123",
+    name: "Login Flow Happy Path",
+    situation: "User attempts to log in with valid credentials",
+    criteria: [
+      "Responds with a welcome message",
+      "Includes user name in greeting",
+    ],
+    labels: ["auth", "happy-path"],
+  };
+
+  describe("when format is digest", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockGetScenario.mockResolvedValue(sampleScenario);
+      result = await handleGetScenario({ scenarioId: "scen_abc123" });
+    });
+
+    it("includes the scenario name in the heading", () => {
+      expect(result).toContain("# Scenario: Login Flow Happy Path");
+    });
+
+    it("includes the situation", () => {
+      expect(result).toContain("User attempts to log in with valid credentials");
+    });
+
+    it("includes each criteria item", () => {
+      expect(result).toContain("- Responds with a welcome message");
+      expect(result).toContain("- Includes user name in greeting");
+    });
+
+    it("includes labels", () => {
+      expect(result).toContain("auth, happy-path");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the scenario structure", async () => {
+      mockGetScenario.mockResolvedValue(sampleScenario);
+      const result = await handleGetScenario({
+        scenarioId: "scen_abc123",
+        format: "json",
+      });
+      expect(JSON.parse(result)).toEqual(sampleScenario);
+    });
+  });
+});
+
+describe("formatScenarioSchema()", () => {
+  it("includes field descriptions with required/optional annotations", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("**name** (required)");
+    expect(result).toContain("**situation** (required)");
+    expect(result).toContain("**criteria** (array of strings)");
+    expect(result).toContain("**labels** (array of strings)");
+  });
+
+  it("includes all target types with descriptions", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("**prompt**: Test a prompt template");
+    expect(result).toContain("**http**: Test an HTTP endpoint");
+    expect(result).toContain("**code**: Test a code function");
+  });
+
+  it("includes authoring guidance for situations and criteria", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("## Writing a Good Situation");
+    expect(result).toContain("## Writing Good Criteria");
+    expect(result).toContain("Specific and testable");
+  });
+});