@langwatch/mcp-server 0.3.3 → 0.5.0

package/src/__tests__/scenario-tools.integration.test.ts

@@ -0,0 +1,286 @@
+import { createServer, type Server } from "http";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { initConfig } from "../config.js";
+
+// --- Canned responses for scenario API endpoints ---
+
+const CANNED_SCENARIOS_LIST = [
+  {
+    id: "scen_abc123",
+    name: "Login Flow Happy Path",
+    situation: "User attempts to log in with valid credentials",
+    criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+    labels: ["auth", "happy-path"],
+  },
+  {
+    id: "scen_def456",
+    name: "Password Reset",
+    situation: "User requests a password reset link",
+    criteria: ["Sends reset email"],
+    labels: ["auth"],
+  },
+];
+
+const CANNED_SCENARIO_DETAIL = {
+  id: "scen_abc123",
+  name: "Login Flow Happy Path",
+  situation: "User attempts to log in with valid credentials",
+  criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+  labels: ["auth", "happy-path"],
+};
+
+const CANNED_SCENARIO_CREATED = {
+  id: "scen_new789",
+  name: "Login Flow Happy Path",
+  situation: "User attempts to log in with valid creds",
+  criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+  labels: ["auth", "happy-path"],
+};
+
+const CANNED_SCENARIO_UPDATED = {
+  id: "scen_abc123",
+  name: "Login Flow - Valid Credentials",
+  situation: "User logs in with correct email and pass",
+  criteria: [
+    "Responds with welcome message",
+    "Sets session cookie",
+    "Redirects to dashboard",
+  ],
+  labels: ["auth", "happy-path"],
+};
+
+const CANNED_SCENARIO_ARCHIVED = {
+  id: "scen_abc123",
+  archived: true,
+};
+
+// --- Mock HTTP Server ---
+
+function createMockServer(): Server {
+  return createServer((req, res) => {
+    const authToken = req.headers["x-auth-token"];
+    if (authToken !== "test-integration-key") {
+      res.writeHead(401, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ message: "Invalid auth token." }));
+      return;
+    }
+
+    let body = "";
+    req.on("data", (chunk: string) => (body += chunk));
+    req.on("end", () => {
+      const url = req.url ?? "";
+      res.setHeader("Content-Type", "application/json");
+
+      // GET /api/scenarios - list all scenarios
+      if (url === "/api/scenarios" && req.method === "GET") {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIOS_LIST));
+      }
+      // GET /api/scenarios/:id - get scenario detail
+      else if (
+        url.match(/^\/api\/scenarios\/scen_abc123(\?|$)/) &&
+        req.method === "GET"
+      ) {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIO_DETAIL));
+      }
+      // GET /api/scenarios/:id - not found
+      else if (
+        url.match(/^\/api\/scenarios\/scen_nonexistent(\?|$)/) &&
+        req.method === "GET"
+      ) {
+        res.writeHead(404);
+        res.end(JSON.stringify({ message: "Scenario not found" }));
+      }
+      // POST /api/scenarios - create scenario
+      else if (url === "/api/scenarios" && req.method === "POST") {
+        const parsed = JSON.parse(body);
+        if (!parsed.name) {
+          res.writeHead(400);
+          res.end(JSON.stringify({ message: "Validation error: name is required" }));
+        } else {
+          res.writeHead(201);
+          res.end(JSON.stringify(CANNED_SCENARIO_CREATED));
+        }
+      }
+      // PUT /api/scenarios/:id - update scenario
+      else if (
+        url.match(/^\/api\/scenarios\/scen_abc123$/) &&
+        req.method === "PUT"
+      ) {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIO_UPDATED));
+      }
+      // PUT /api/scenarios/:id - not found
+      else if (
+        url.match(/^\/api\/scenarios\/scen_nonexistent$/) &&
+        req.method === "PUT"
+      ) {
+        res.writeHead(404);
+        res.end(JSON.stringify({ message: "Scenario not found" }));
+      }
+      // DELETE /api/scenarios/:id - archive scenario
+      else if (
+        url.match(/^\/api\/scenarios\/scen_abc123$/) &&
+        req.method === "DELETE"
+      ) {
+        res.writeHead(200);
+        res.end(JSON.stringify(CANNED_SCENARIO_ARCHIVED));
+      }
+      else {
+        res.writeHead(404);
+        res.end(
+          JSON.stringify({ message: `Not found: ${req.method} ${url}` })
+        );
+      }
+    });
+  });
+}
+
+// --- Integration Tests ---
+// These verify that MCP tool handlers correctly communicate with the REST API
+// (auth, HTTP methods, status codes, error propagation).
+// Formatting/digest logic is tested in scenario-tools.unit.test.ts.
+
+describe("MCP scenario tools integration", () => {
+  let server: Server;
+  let port: number;
+
+  beforeAll(async () => {
+    server = createMockServer();
+    await new Promise<void>((resolve) => {
+      server.listen(0, () => {
+        const addr = server.address();
+        port = typeof addr === "object" && addr ? addr.port : 0;
+        initConfig({
+          apiKey: "test-integration-key",
+          endpoint: `http://localhost:${port}`,
+        });
+        resolve();
+      });
+    });
+  });
+
+  afterAll(async () => {
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+  });
+
+  describe("list_scenarios", () => {
+    describe("when the API returns scenarios", () => {
+      it("returns a non-empty result", async () => {
+        const { handleListScenarios } = await import(
+          "../tools/list-scenarios.js"
+        );
+        const result = await handleListScenarios({});
+        expect(result.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe("when format is json", () => {
+      it("returns parseable JSON matching the API response", async () => {
+        const { handleListScenarios } = await import(
+          "../tools/list-scenarios.js"
+        );
+        const result = await handleListScenarios({ format: "json" });
+        expect(JSON.parse(result)).toEqual(CANNED_SCENARIOS_LIST);
+      });
+    });
+  });
+
+  describe("get_scenario", () => {
+    describe("when the scenario exists", () => {
+      it("returns a non-empty result", async () => {
+        const { handleGetScenario } = await import(
+          "../tools/get-scenario.js"
+        );
+        const result = await handleGetScenario({ scenarioId: "scen_abc123" });
+        expect(result.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe("when the scenario does not exist", () => {
+      it("propagates the 404 error", async () => {
+        const { handleGetScenario } = await import(
+          "../tools/get-scenario.js"
+        );
+        await expect(
+          handleGetScenario({ scenarioId: "scen_nonexistent" })
+        ).rejects.toThrow("404");
+      });
+    });
+  });
+
+  describe("create_scenario", () => {
+    describe("when valid data is provided", () => {
+      it("returns confirmation with new scenario ID", async () => {
+        const { handleCreateScenario } = await import(
+          "../tools/create-scenario.js"
+        );
+        const result = await handleCreateScenario({
+          name: "Login Flow Happy Path",
+          situation: "User attempts to log in with valid creds",
+          criteria: ["Responds with a welcome message", "Includes user name in greeting"],
+          labels: ["auth", "happy-path"],
+        });
+        expect(result).toContain("scen_new789");
+      });
+    });
+
+    describe("when name is empty", () => {
+      it("propagates the validation error", async () => {
+        const { handleCreateScenario } = await import(
+          "../tools/create-scenario.js"
+        );
+        await expect(
+          handleCreateScenario({
+            name: "",
+            situation: "Some situation",
+          })
+        ).rejects.toThrow();
+      });
+    });
+  });
+
+  describe("update_scenario", () => {
+    describe("when the scenario exists", () => {
+      it("returns a non-empty result", async () => {
+        const { handleUpdateScenario } = await import(
+          "../tools/update-scenario.js"
+        );
+        const result = await handleUpdateScenario({
+          scenarioId: "scen_abc123",
+          name: "Login Flow - Valid Credentials",
+        });
+        expect(result.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe("when the scenario does not exist", () => {
+      it("propagates the 404 error", async () => {
+        const { handleUpdateScenario } = await import(
+          "../tools/update-scenario.js"
+        );
+        await expect(
+          handleUpdateScenario({
+            scenarioId: "scen_nonexistent",
+            name: "Updated Name",
+          })
+        ).rejects.toThrow("404");
+      });
+    });
+  });
+
+  describe("archive_scenario", () => {
+    describe("when the scenario exists", () => {
+      it("returns confirmation that scenario was archived", async () => {
+        const { handleArchiveScenario } = await import(
+          "../tools/archive-scenario.js"
+        );
+        const result = await handleArchiveScenario({
+          scenarioId: "scen_abc123",
+        });
+        expect(result).toContain("archived");
+      });
+    });
+  });
+});

package/src/__tests__/scenario-tools.unit.test.ts

@@ -0,0 +1,185 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api-scenarios.js", () => ({
+  listScenarios: vi.fn(),
+  getScenario: vi.fn(),
+}));
+
+import {
+  listScenarios,
+  getScenario,
+} from "../langwatch-api-scenarios.js";
+
+import { handleListScenarios } from "../tools/list-scenarios.js";
+import { handleGetScenario } from "../tools/get-scenario.js";
+import { formatScenarioSchema } from "../tools/discover-scenario-schema.js";
+
+const mockListScenarios = vi.mocked(listScenarios);
+const mockGetScenario = vi.mocked(getScenario);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleListScenarios()", () => {
+  const sampleScenarios = [
+    {
+      id: "scen_abc123",
+      name: "Login Flow Happy Path",
+      situation:
+        "User attempts to log in with valid credentials and expects a welcome message back from the system",
+      criteria: [
+        "Responds with a welcome message",
+        "Includes user name in greeting",
+        "Sets session cookie",
+      ],
+      labels: ["auth", "happy-path"],
+    },
+    {
+      id: "scen_def456",
+      name: "Error Handling",
+      situation: "User sends malformed input",
+      criteria: ["Returns 400 status"],
+      labels: ["error"],
+    },
+  ];
+
+  describe("when scenarios exist (digest mode)", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListScenarios.mockResolvedValue(sampleScenarios);
+      result = await handleListScenarios({});
+    });
+
+    it("includes scenario id", () => {
+      expect(result).toContain("scen_abc123");
+    });
+
+    it("includes scenario name", () => {
+      expect(result).toContain("Login Flow Happy Path");
+    });
+
+    it("includes truncated situation preview", () => {
+      expect(result).toContain("User attempts to log in");
+      expect(result).not.toContain(
+        "User attempts to log in with valid credentials and expects a welcome message back from the system"
+      );
+    });
+
+    it("shows criteria count per scenario", () => {
+      expect(result).toContain("3 criteria");
+    });
+
+    it("includes labels", () => {
+      expect(result).toContain("auth");
+    });
+
+    it("includes all scenarios in the list", () => {
+      expect(result).toContain("scen_def456");
+    });
+
+    it("includes the total count header", () => {
+      expect(result).toContain("# Scenarios (2 total)");
+    });
+  });
+
+  describe("when no scenarios exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListScenarios.mockResolvedValue([]);
+      result = await handleListScenarios({});
+    });
+
+    it("returns a no-scenarios message", () => {
+      expect(result).toContain("No scenarios found");
+    });
+
+    it("includes a tip to use create_scenario", () => {
+      expect(result).toContain("create_scenario");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the scenario structure", async () => {
+      mockListScenarios.mockResolvedValue(sampleScenarios);
+      const result = await handleListScenarios({ format: "json" });
+      expect(JSON.parse(result)).toEqual(sampleScenarios);
+    });
+  });
+});
+
+describe("handleGetScenario()", () => {
+  const sampleScenario = {
+    id: "scen_abc123",
+    name: "Login Flow Happy Path",
+    situation: "User attempts to log in with valid credentials",
+    criteria: [
+      "Responds with a welcome message",
+      "Includes user name in greeting",
+    ],
+    labels: ["auth", "happy-path"],
+  };
+
+  describe("when format is digest", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockGetScenario.mockResolvedValue(sampleScenario);
+      result = await handleGetScenario({ scenarioId: "scen_abc123" });
+    });
+
+    it("includes the scenario name in the heading", () => {
+      expect(result).toContain("# Scenario: Login Flow Happy Path");
+    });
+
+    it("includes the situation", () => {
+      expect(result).toContain("User attempts to log in with valid credentials");
+    });
+
+    it("includes each criteria item", () => {
+      expect(result).toContain("- Responds with a welcome message");
+      expect(result).toContain("- Includes user name in greeting");
+    });
+
+    it("includes labels", () => {
+      expect(result).toContain("auth, happy-path");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the scenario structure", async () => {
+      mockGetScenario.mockResolvedValue(sampleScenario);
+      const result = await handleGetScenario({
+        scenarioId: "scen_abc123",
+        format: "json",
+      });
+      expect(JSON.parse(result)).toEqual(sampleScenario);
+    });
+  });
+});
+
+describe("formatScenarioSchema()", () => {
+  it("includes field descriptions with required/optional annotations", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("**name** (required)");
+    expect(result).toContain("**situation** (required)");
+    expect(result).toContain("**criteria** (array of strings)");
+    expect(result).toContain("**labels** (array of strings)");
+  });
+
+  it("includes all target types with descriptions", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("**prompt**: Test a prompt template");
+    expect(result).toContain("**http**: Test an HTTP endpoint");
+    expect(result).toContain("**code**: Test a code function");
+  });
+
+  it("includes authoring guidance for situations and criteria", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("## Writing a Good Situation");
+    expect(result).toContain("## Writing Good Criteria");
+    expect(result).toContain("Specific and testable");
+  });
+});

package/src/__tests__/schemas.unit.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect } from "vitest";
+import { filterFields } from "../schemas/filter-fields.js";
+import { analyticsMetrics } from "../schemas/analytics-metrics.js";
+import { analyticsGroups } from "../schemas/analytics-groups.js";
+
+describe("schemas", () => {
+  describe("filterFields", () => {
+    it("has at least 22 entries", () => {
+      expect(filterFields.length).toBeGreaterThanOrEqual(22);
+    });
+
+    it("has non-empty field and description for every entry", () => {
+      for (const entry of filterFields) {
+        expect(entry.field).toBeTruthy();
+        expect(entry.description).toBeTruthy();
+      }
+    });
+
+    it("contains expected filter fields", () => {
+      const fieldNames = filterFields.map((f) => f.field);
+      expect(fieldNames).toContain("topics.topics");
+      expect(fieldNames).toContain("metadata.user_id");
+      expect(fieldNames).toContain("spans.model");
+      expect(fieldNames).toContain("evaluations.passed");
+      expect(fieldNames).toContain("events.event_type");
+      expect(fieldNames).toContain("annotations.hasAnnotation");
+      expect(fieldNames).toContain("sentiment.input_sentiment");
+    });
+  });
+
+  describe("analyticsMetrics", () => {
+    it("covers expected categories", () => {
+      const categories = new Set(analyticsMetrics.map((m) => m.category));
+      expect(categories).toContain("metadata");
+      expect(categories).toContain("performance");
+      expect(categories).toContain("evaluations");
+      expect(categories).toContain("sentiment");
+      expect(categories).toContain("events");
+    });
+
+    it("has non-empty allowedAggregations for every metric", () => {
+      for (const metric of analyticsMetrics) {
+        expect(metric.allowedAggregations.length).toBeGreaterThan(0);
+      }
+    });
+
+    it("has non-empty name, label, and description for every metric", () => {
+      for (const metric of analyticsMetrics) {
+        expect(metric.name).toBeTruthy();
+        expect(metric.label).toBeTruthy();
+        expect(metric.description).toBeTruthy();
+      }
+    });
+
+    it("contains expected metrics", () => {
+      const names = analyticsMetrics.map((m) => `${m.category}.${m.name}`);
+      expect(names).toContain("metadata.trace_id");
+      expect(names).toContain("performance.completion_time");
+      expect(names).toContain("performance.total_cost");
+      expect(names).toContain("evaluations.evaluation_score");
+    });
+  });
+
+  describe("analyticsGroups", () => {
+    it("has at least 10 entries", () => {
+      expect(analyticsGroups.length).toBeGreaterThanOrEqual(10);
+    });
+
+    it("has non-empty name, label, and description for every group", () => {
+      for (const group of analyticsGroups) {
+        expect(group.name).toBeTruthy();
+        expect(group.label).toBeTruthy();
+        expect(group.description).toBeTruthy();
+      }
+    });
+
+    it("contains expected group-by options", () => {
+      const names = analyticsGroups.map((g) => g.name);
+      expect(names).toContain("topics.topics");
+      expect(names).toContain("metadata.user_id");
+      expect(names).toContain("metadata.model");
+      expect(names).toContain("events.event_type");
+    });
+  });
+});