@langwatch/mcp-server 2.1.0 → 3.0.0

package/src/__tests__/evaluator-tools.unit.test.ts

@@ -1,262 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
-
-vi.mock("../langwatch-api-evaluators.js", async (importOriginal) => {
-  const actual = await importOriginal() as Record<string, unknown>;
-  return {
-    ...actual,
-    listEvaluators: vi.fn(),
-    getEvaluator: vi.fn(),
-    createEvaluator: vi.fn(),
-    updateEvaluator: vi.fn(),
-  };
-});
-
-import {
-  listEvaluators,
-  getEvaluator,
-  createEvaluator,
-  updateEvaluator,
-} from "../langwatch-api-evaluators.js";
-
-import { handleListEvaluators } from "../tools/list-evaluators.js";
-import { handleGetEvaluator } from "../tools/get-evaluator.js";
-import { handleCreateEvaluator } from "../tools/create-evaluator.js";
-import { handleUpdateEvaluator } from "../tools/update-evaluator.js";
-
-const mockListEvaluators = vi.mocked(listEvaluators);
-const mockGetEvaluator = vi.mocked(getEvaluator);
-const mockCreateEvaluator = vi.mocked(createEvaluator);
-const mockUpdateEvaluator = vi.mocked(updateEvaluator);
-
-beforeEach(() => {
-  vi.clearAllMocks();
-});
-
-describe("handleListEvaluators()", () => {
-  const sampleEvaluators = [
-    {
-      id: "evaluator_abc123",
-      projectId: "proj_1",
-      name: "Toxicity Check",
-      slug: "toxicity-check",
-      type: "evaluator",
-      config: { evaluatorType: "openai/moderation" },
-      workflowId: null,
-      copiedFromEvaluatorId: null,
-      createdAt: "2024-01-01T00:00:00.000Z",
-      updatedAt: "2024-01-01T00:00:00.000Z",
-      fields: [{ identifier: "input", type: "str" }],
-      outputFields: [{ identifier: "passed", type: "bool" }],
-    },
-    {
-      id: "evaluator_def456",
-      projectId: "proj_1",
-      name: "Exact Match",
-      slug: "exact-match",
-      type: "evaluator",
-      config: { evaluatorType: "langevals/exact_match" },
-      workflowId: null,
-      copiedFromEvaluatorId: null,
-      createdAt: "2024-01-02T00:00:00.000Z",
-      updatedAt: "2024-01-02T00:00:00.000Z",
-      fields: [
-        { identifier: "output", type: "str" },
-        { identifier: "expected_output", type: "str" },
-      ],
-      outputFields: [{ identifier: "passed", type: "bool" }],
-    },
-  ];
-
-  describe("when evaluators exist", () => {
-    let result: string;
-
-    beforeEach(async () => {
-      mockListEvaluators.mockResolvedValue(sampleEvaluators);
-      result = await handleListEvaluators();
-    });
-
-    it("includes evaluator id", () => {
-      expect(result).toContain("evaluator_abc123");
-    });
-
-    it("includes evaluator name", () => {
-      expect(result).toContain("Toxicity Check");
-    });
-
-    it("includes evaluator type", () => {
-      expect(result).toContain("openai/moderation");
-    });
-
-    it("includes slug", () => {
-      expect(result).toContain("toxicity-check");
-    });
-
-    it("includes all evaluators in the list", () => {
-      expect(result).toContain("evaluator_def456");
-    });
-
-    it("includes the total count header", () => {
-      expect(result).toContain("# Evaluators (2 total)");
-    });
-  });
-
-  describe("when no evaluators exist", () => {
-    let result: string;
-
-    beforeEach(async () => {
-      mockListEvaluators.mockResolvedValue([]);
-      result = await handleListEvaluators();
-    });
-
-    it("returns a no-evaluators message", () => {
-      expect(result).toContain("No evaluators found");
-    });
-
-    it("includes a tip to use platform_create_evaluator", () => {
-      expect(result).toContain("platform_create_evaluator");
-    });
-  });
-});
-
-describe("handleGetEvaluator()", () => {
-  const sampleEvaluator = {
-    id: "evaluator_abc123",
-    projectId: "proj_1",
-    name: "Toxicity Check",
-    slug: "toxicity-check",
-    type: "evaluator",
-    config: {
-      evaluatorType: "openai/moderation",
-      settings: { model: "text-moderation-stable" },
-    },
-    workflowId: null,
-    copiedFromEvaluatorId: null,
-    createdAt: "2024-01-01T00:00:00.000Z",
-    updatedAt: "2024-01-01T00:00:00.000Z",
-    fields: [
-      { identifier: "input", type: "str" },
-      { identifier: "output", type: "str", optional: true },
-    ],
-    outputFields: [
-      { identifier: "passed", type: "bool" },
-      { identifier: "score", type: "float" },
-    ],
-  };
-
-  describe("when evaluator is found", () => {
-    let result: string;
-
-    beforeEach(async () => {
-      mockGetEvaluator.mockResolvedValue(sampleEvaluator);
-      result = await handleGetEvaluator({ idOrSlug: "evaluator_abc123" });
-    });
-
-    it("includes the evaluator name in the heading", () => {
-      expect(result).toContain("# Evaluator: Toxicity Check");
-    });
-
-    it("includes the evaluator type", () => {
-      expect(result).toContain("openai/moderation");
-    });
-
-    it("includes the config as JSON", () => {
-      expect(result).toContain("text-moderation-stable");
-    });
-
-    it("includes input fields", () => {
-      expect(result).toContain("## Input Fields");
-      expect(result).toContain("**input** (str)");
-    });
-
-    it("marks optional fields", () => {
-      expect(result).toContain("(optional)");
-    });
-
-    it("includes output fields", () => {
-      expect(result).toContain("## Output Fields");
-      expect(result).toContain("**passed** (bool)");
-    });
-  });
-});
-
-describe("handleCreateEvaluator()", () => {
-  describe("when creation succeeds", () => {
-    let result: string;
-
-    beforeEach(async () => {
-      mockCreateEvaluator.mockResolvedValue({
-        id: "evaluator_new123",
-        projectId: "proj_1",
-        name: "My LLM Judge",
-        slug: "my-llm-judge",
-        type: "evaluator",
-        config: { evaluatorType: "langevals/llm_boolean" },
-        workflowId: null,
-        copiedFromEvaluatorId: null,
-        createdAt: "2024-01-01T00:00:00.000Z",
-        updatedAt: "2024-01-01T00:00:00.000Z",
-        fields: [{ identifier: "input", type: "str" }],
-        outputFields: [{ identifier: "passed", type: "bool" }],
-      });
-      result = await handleCreateEvaluator({
-        name: "My LLM Judge",
-        config: { evaluatorType: "langevals/llm_boolean" },
-      });
-    });
-
-    it("confirms creation", () => {
-      expect(result).toContain("Evaluator created successfully!");
-    });
-
-    it("includes the generated ID", () => {
-      expect(result).toContain("evaluator_new123");
-    });
-
-    it("includes the slug", () => {
-      expect(result).toContain("my-llm-judge");
-    });
-
-    it("includes the evaluator type", () => {
-      expect(result).toContain("langevals/llm_boolean");
-    });
-  });
-});
-
-describe("handleUpdateEvaluator()", () => {
-  describe("when update succeeds", () => {
-    let result: string;
-
-    beforeEach(async () => {
-      mockUpdateEvaluator.mockResolvedValue({
-        id: "evaluator_abc123",
-        projectId: "proj_1",
-        name: "Updated Name",
-        slug: "toxicity-check",
-        type: "evaluator",
-        config: { evaluatorType: "openai/moderation" },
-        workflowId: null,
-        copiedFromEvaluatorId: null,
-        createdAt: "2024-01-01T00:00:00.000Z",
-        updatedAt: "2024-01-02T00:00:00.000Z",
-        fields: [],
-        outputFields: [],
-      });
-      result = await handleUpdateEvaluator({
-        evaluatorId: "evaluator_abc123",
-        name: "Updated Name",
-      });
-    });
-
-    it("confirms update", () => {
-      expect(result).toContain("Evaluator updated successfully!");
-    });
-
-    it("includes the evaluator ID", () => {
-      expect(result).toContain("evaluator_abc123");
-    });
-
-    it("includes the updated name", () => {
-      expect(result).toContain("Updated Name");
-    });
-  });
-});

package/src/__tests__/experiment-list-tools.unit.test.ts

@@ -1,157 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
-
-vi.mock("../langwatch-api.js", async (importOriginal) => {
-  const actual = (await importOriginal()) as Record<string, unknown>;
-  return {
-    ...actual,
-    makeRequest: vi.fn(),
-  };
-});
-
-import { makeRequest } from "../langwatch-api.js";
-import { handleExperimentList } from "../tools/list-experiments.js";
-import { handleExperimentListRuns } from "../tools/list-experiment-runs.js";
-
-const mockMakeRequest = vi.mocked(makeRequest);
-
-describe("handleExperimentList()", () => {
-  beforeEach(() => {
-    vi.clearAllMocks();
-  });
-
-  describe("given experiments exist", () => {
-    describe("when invoked with default limit", () => {
-      /** @scenario "Lists experiments as markdown" */
-      it("renders a markdown table with each slug", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          experiments: [
-            {
-              id: "exp_1",
-              slug: "checkout-flow",
-              name: "Checkout Flow",
-              type: "EVALUATIONS_V3",
-              workflowId: null,
-              createdAt: "2026-01-01T00:00:00Z",
-              updatedAt: "2026-01-02T00:00:00Z",
-              runsCount: 3,
-              lastRunAt: "2026-01-02T00:00:00Z",
-            },
-          ],
-          pagination: {
-            page: 1,
-            pageSize: 25,
-            totalHits: 1,
-            hasMore: false,
-          },
-        });
-
-        const out = await handleExperimentList({});
-        expect(out).toContain("# Experiments");
-        expect(out).toContain("`checkout-flow`");
-        expect(out).toContain("Checkout Flow");
-      });
-    });
-
-    describe("when invoked with limit above the cap", () => {
-      /** @scenario "Limit is bounded to protect agent context" */
-      it("clamps the effective pageSize to 100", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          experiments: [],
-          pagination: { page: 1, pageSize: 100, totalHits: 0, hasMore: false },
-        });
-
-        await handleExperimentList({ limit: 5000 });
-        const path = mockMakeRequest.mock.calls[0]![1] as string;
-        expect(path).toContain("pageSize=100");
-      });
-    });
-  });
-
-  describe("given no experiments exist", () => {
-    describe("when invoked", () => {
-      it("returns an empty-state message", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          experiments: [],
-          pagination: { page: 1, pageSize: 25, totalHits: 0, hasMore: false },
-        });
-        const out = await handleExperimentList({});
-        expect(out).toContain("No experiments");
-      });
-    });
-  });
-});
-
-describe("handleExperimentListRuns()", () => {
-  beforeEach(() => {
-    vi.clearAllMocks();
-  });
-
-  describe("given a known experiment with runs", () => {
-    describe("when invoked", () => {
-      /** @scenario "Lists runs for a known experiment" */
-      it("renders a markdown table with each runId", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          experimentId: "exp_1",
-          experimentSlug: "checkout-flow",
-          runs: [
-            {
-              experimentId: "exp_1",
-              runId: "run_visible",
-              workflowVersion: null,
-              timestamps: { createdAt: 1000, updatedAt: 2000, finishedAt: 2000 },
-              summary: { evaluations: {} },
-            },
-          ],
-          pagination: {
-            page: 1,
-            pageSize: 25,
-            totalHits: 1,
-            hasMore: false,
-          },
-        });
-
-        const out = await handleExperimentListRuns({
-          experimentSlug: "checkout-flow",
-        });
-        expect(out).toContain("# Evaluation Runs: checkout-flow");
-        expect(out).toContain("`run_visible`");
-      });
-    });
-  });
-
-  describe("given the experiment slug is unknown", () => {
-    describe("when the API throws 404", () => {
-      /** @scenario "Unknown experiment slug returns a graceful not-found message" */
-      it("returns a graceful not-found message suggesting platform_experiment_list", async () => {
-        mockMakeRequest.mockRejectedValueOnce(
-          new Error("LangWatch API error 404: experiment not found"),
-        );
-
-        const out = await handleExperimentListRuns({
-          experimentSlug: "does-not-exist",
-        });
-        expect(out).toContain("not found");
-        expect(out).toContain("platform_experiment_list");
-      });
-    });
-  });
-
-  describe("given the experiment exists but has no runs", () => {
-    describe("when invoked", () => {
-      it("returns an empty-state message that points at platform_run_experiment", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          experimentId: "exp_1",
-          experimentSlug: "support-bot",
-          runs: [],
-          pagination: { page: 1, pageSize: 25, totalHits: 0, hasMore: false },
-        });
-
-        const out = await handleExperimentListRuns({
-          experimentSlug: "support-bot",
-        });
-        expect(out).toContain("no runs yet");
-        expect(out).toContain("platform_run_experiment");
-      });
-    });
-  });
-});

package/src/__tests__/experiment-status.unit.test.ts

@@ -1,194 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
-
-vi.mock("../langwatch-api.js", async (importOriginal) => {
-  const actual = (await importOriginal()) as Record<string, unknown>;
-  return {
-    ...actual,
-    makeRequest: vi.fn(),
-  };
-});
-
-import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
-import { handleExperimentStatus } from "../tools/run-experiment.js";
-import {
-  deriveRunStatus,
-  isTerminalStatus,
-} from "../tools/experiment-run-status.js";
-
-const mockMakeRequest = vi.mocked(makeRequest);
-
-beforeEach(() => {
-  vi.clearAllMocks();
-});
-
-describe("deriveRunStatus()", () => {
-  describe("when stoppedAt is set", () => {
-    it("is stopped", () => {
-      expect(deriveRunStatus({ stoppedAt: 5 })).toBe("stopped");
-    });
-  });
-
-  describe("when finishedAt is set", () => {
-    it("is completed", () => {
-      expect(deriveRunStatus({ finishedAt: 5 })).toBe("completed");
-    });
-  });
-
-  describe("when no terminal marker and updates are recent", () => {
-    it("is running", () => {
-      const now = 1_000_000;
-      expect(deriveRunStatus({ updatedAt: now - 1000 }, now)).toBe("running");
-    });
-  });
-
-  describe("when no terminal marker and no recent updates", () => {
-    it("is interrupted", () => {
-      const now = 1_000_000;
-      expect(deriveRunStatus({ updatedAt: now - 6 * 60 * 1000 }, now)).toBe(
-        "interrupted",
-      );
-    });
-  });
-
-  describe("terminal classification", () => {
-    it("treats completed and stopped as terminal", () => {
-      expect(isTerminalStatus("completed")).toBe(true);
-      expect(isTerminalStatus("stopped")).toBe(true);
-      expect(isTerminalStatus("running")).toBe(false);
-      expect(isTerminalStatus("interrupted")).toBe(false);
-    });
-  });
-});
-
-describe("handleExperimentStatus()", () => {
-  describe("given the run state exists in Redis", () => {
-    it("reports the status straight from the status endpoint", async () => {
-      mockMakeRequest.mockResolvedValueOnce({
-        runId: "run_1",
-        status: "completed",
-        progress: 3,
-        total: 3,
-      });
-
-      const out = await handleExperimentStatus({ runId: "run_1" });
-
-      expect(mockMakeRequest).toHaveBeenCalledWith(
-        "GET",
-        "/api/experiments/runs/run_1",
-      );
-      expect(out).toContain("**Status**: completed");
-      expect(out).toContain("3/3 cells");
-    });
-  });
-
-  describe("given an SDK-logged run with no Redis state", () => {
-    it("falls back to deriving status from the results endpoint", async () => {
-      mockMakeRequest
-        .mockRejectedValueOnce(
-          new LangWatchApiError("missing", 404, "Run not found"),
-        )
-        .mockResolvedValueOnce({
-          progress: 5,
-          total: 5,
-          dataset: [1, 2, 3, 4, 5],
-          timestamps: {
-            createdAt: 1,
-            updatedAt: 2,
-            finishedAt: 3,
-            stoppedAt: null,
-          },
-        });
-
-      const out = await handleExperimentStatus({
-        runId: "sdk_run",
-        experimentSlug: "doc-qa",
-      });
-
-      expect(mockMakeRequest).toHaveBeenNthCalledWith(
-        1,
-        "GET",
-        "/api/experiments/runs/sdk_run",
-      );
-      expect(mockMakeRequest).toHaveBeenNthCalledWith(
-        2,
-        "GET",
-        "/api/experiments/runs/sdk_run/results?experimentSlug=doc-qa",
-      );
-      expect(out).toContain("**Status**: completed");
-      expect(out).toContain("5/5 cells");
-      expect(out).toContain("platform_experiment_results");
-    });
-
-    it("reports interrupted for a stale unfinished SDK run", async () => {
-      mockMakeRequest
-        .mockRejectedValueOnce(
-          new LangWatchApiError("missing", 404, "Run not found"),
-        )
-        .mockResolvedValueOnce({
-          progress: 2,
-          total: 5,
-          dataset: [1, 2],
-          timestamps: {
-            createdAt: Date.now() - 60 * 60 * 1000,
-            updatedAt: Date.now() - 30 * 60 * 1000,
-            finishedAt: null,
-            stoppedAt: null,
-          },
-        });
-
-      const out = await handleExperimentStatus({
-        runId: "sdk_run",
-        experimentSlug: "doc-qa",
-      });
-
-      expect(out).toContain("**Status**: interrupted");
-      expect(out).toContain("2/5 cells");
-    });
-  });
-
-  describe("given the run cannot be found anywhere", () => {
-    it("returns actionable guidance instead of a raw 404", async () => {
-      mockMakeRequest
-        .mockRejectedValueOnce(
-          new LangWatchApiError("missing", 404, "Run not found"),
-        )
-        .mockRejectedValueOnce(
-          new LangWatchApiError("missing", 404, "Run not found"),
-        );
-
-      const out = await handleExperimentStatus({
-        runId: "nope",
-        experimentSlug: "doc-qa",
-      });
-
-      expect(out).toContain("not found");
-      expect(out).toContain("experimentSlug");
-      expect(out).toContain("platform_experiment_results");
-    });
-
-    it("rethrows non-404 errors from the status endpoint", async () => {
-      mockMakeRequest.mockRejectedValueOnce(
-        new LangWatchApiError("boom", 500, "Internal error"),
-      );
-
-      await expect(
-        handleExperimentStatus({ runId: "nope" }),
-      ).rejects.toThrow();
-    });
-
-    it("propagates a real (non-404) error from the results fallback instead of faking not-found", async () => {
-      mockMakeRequest
-        .mockRejectedValueOnce(
-          new LangWatchApiError("missing", 404, "Run not found"),
-        )
-        .mockRejectedValueOnce(
-          new LangWatchApiError("boom", 500, "Internal error"),
-        );
-
-      // The fallback's 500 ("boom") must surface, not the original 404 ("missing").
-      await expect(
-        handleExperimentStatus({ runId: "sdk_run", experimentSlug: "doc-qa" }),
-      ).rejects.toThrow(/boom/);
-    });
-  });
-});

package/src/__tests__/feature-map-tool-drift.unit.test.ts

@@ -1,93 +0,0 @@
-/**
- * Drift guard between the MCP tools this server registers and the names
- * `feature-map.json` tells agents to call.
- *
- * The feature map is the canonical information architecture: it is embedded
- * into the CLI at codegen time and shipped inside the npx server package, so a
- * name in it is a name an agent will try. Two entries pointed at
- * `platform_run_evaluation` and `platform_evaluation_status`, which have never
- * existed — the tools are `platform_run_experiment` and
- * `platform_experiment_status`. An agent following the map for the experiments
- * feature called two tools that were not there.
- *
- * Nothing compared the two lists, which is why it went unnoticed. This does.
- *
- * @see .claude/skills/feature-map/SKILL.md
- */
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import { describe, expect, it } from "vitest";
-
-const REPO_ROOT = join(__dirname, "../../../..");
-
-/**
- * Every tool name the feature map hands to an agent, from any depth: `mcp`
- * arrays hang off surfaces nested several levels into the tree.
- */
-function toolNamesInFeatureMap(): Set<string> {
-  const map: unknown = JSON.parse(
-    readFileSync(join(REPO_ROOT, "feature-map.json"), "utf-8")
-  );
-  const names = new Set<string>();
-
-  const walk = (node: unknown): void => {
-    if (Array.isArray(node)) {
-      for (const item of node) walk(item);
-      return;
-    }
-    if (node === null || typeof node !== "object") return;
-
-    const entry = node as Record<string, unknown>;
-    if (Array.isArray(entry.mcp)) {
-      for (const name of entry.mcp) {
-        if (typeof name === "string") names.add(name);
-      }
-    }
-    for (const value of Object.values(entry)) walk(value);
-  };
-
-  walk(map);
-  return names;
-}
-
-/**
- * Every name passed as the first argument of a `server.tool(` registration.
- *
- * Read from the source rather than by constructing a server: building one
- * needs config and credentials this test has no business holding, and the
- * registration is a string literal in every case.
- */
-function registeredToolNames(): Set<string> {
-  const source = readFileSync(
-    join(__dirname, "../create-mcp-server.ts"),
-    "utf-8"
-  );
-  const names = new Set<string>();
-  for (const match of source.matchAll(/server\.tool\(\s*"([^"]+)"/g)) {
-    names.add(match[1]!);
-  }
-  return names;
-}
-
-describe("feature map MCP tool names", () => {
-  describe("given the tools this server registers", () => {
-    it("names only tools that exist", () => {
-      const registered = registeredToolNames();
-      const missing = [...toolNamesInFeatureMap()]
-        .filter((name) => !registered.has(name))
-        .sort();
-
-      // Listed, not registered: an agent following the map calls a tool that
-      // is not there. Either rename the entry, or register the tool.
-      expect(missing).toEqual([]);
-    });
-
-    it("finds the registrations it is reading", () => {
-      // A regex that silently matched nothing would make the check above pass
-      // for the wrong reason.
-      const registered = registeredToolNames();
-      expect(registered.size).toBeGreaterThan(50);
-      expect(registered).toContain("platform_run_experiment");
-    });
-  });
-});