@langwatch/mcp-server 0.8.0 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langwatch/mcp-server",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "An MCP server for Langwatch.",
   "type": "module",
   "main": "./dist/index.js",
@@ -45,7 +45,7 @@
     "@types/node": "^25.6.0",
     "@types/yargs": "^17.0.33",
     "@typescript/native-preview": "7.0.0-dev.20250911.1",
-    "ai": "^6.0.0",
+    "ai": "^6.0.161",
     "dotenv": "^17.2.2",
     "esbuild": "^0.25.0",
     "eslint": "^10.2.1",

package/pnpm-workspace.yaml

@@ -1,2 +1,4 @@
+minimumReleaseAge: 10080
+
 onlyBuiltDependencies:
   - node-pty

package/pyproject.toml

@@ -13,5 +13,8 @@ dependencies = [
     "tenacity>=9.1.2",
 ]
 
+[tool.uv]
+exclude-newer = "7 days"
+
 [tool.uv.sources]
 langwatch = { path = "../python-sdk", editable = true }

package/src/__tests__/experiment-list-tools.unit.test.ts

@@ -0,0 +1,157 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api.js", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    makeRequest: vi.fn(),
+  };
+});
+
+import { makeRequest } from "../langwatch-api.js";
+import { handleExperimentList } from "../tools/list-experiments.js";
+import { handleExperimentListRuns } from "../tools/list-experiment-runs.js";
+
+const mockMakeRequest = vi.mocked(makeRequest);
+
+describe("handleExperimentList()", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe("given experiments exist", () => {
+    describe("when invoked with default limit", () => {
+      /** @scenario "Lists experiments as markdown" */
+      it("renders a markdown table with each slug", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experiments: [
+            {
+              id: "exp_1",
+              slug: "checkout-flow",
+              name: "Checkout Flow",
+              type: "EVALUATIONS_V3",
+              workflowId: null,
+              createdAt: "2026-01-01T00:00:00Z",
+              updatedAt: "2026-01-02T00:00:00Z",
+              runsCount: 3,
+              lastRunAt: "2026-01-02T00:00:00Z",
+            },
+          ],
+          pagination: {
+            page: 1,
+            pageSize: 25,
+            totalHits: 1,
+            hasMore: false,
+          },
+        });
+
+        const out = await handleExperimentList({});
+        expect(out).toContain("# Experiments");
+        expect(out).toContain("`checkout-flow`");
+        expect(out).toContain("Checkout Flow");
+      });
+    });
+
+    describe("when invoked with limit above the cap", () => {
+      /** @scenario "Limit is bounded to protect agent context" */
+      it("clamps the effective pageSize to 100", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experiments: [],
+          pagination: { page: 1, pageSize: 100, totalHits: 0, hasMore: false },
+        });
+
+        await handleExperimentList({ limit: 5000 });
+        const path = mockMakeRequest.mock.calls[0]![1] as string;
+        expect(path).toContain("pageSize=100");
+      });
+    });
+  });
+
+  describe("given no experiments exist", () => {
+    describe("when invoked", () => {
+      it("returns an empty-state message", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experiments: [],
+          pagination: { page: 1, pageSize: 25, totalHits: 0, hasMore: false },
+        });
+        const out = await handleExperimentList({});
+        expect(out).toContain("No experiments");
+      });
+    });
+  });
+});
+
+describe("handleExperimentListRuns()", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe("given a known experiment with runs", () => {
+    describe("when invoked", () => {
+      /** @scenario "Lists runs for a known experiment" */
+      it("renders a markdown table with each runId", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experimentId: "exp_1",
+          experimentSlug: "checkout-flow",
+          runs: [
+            {
+              experimentId: "exp_1",
+              runId: "run_visible",
+              workflowVersion: null,
+              timestamps: { createdAt: 1000, updatedAt: 2000, finishedAt: 2000 },
+              summary: { evaluations: {} },
+            },
+          ],
+          pagination: {
+            page: 1,
+            pageSize: 25,
+            totalHits: 1,
+            hasMore: false,
+          },
+        });
+
+        const out = await handleExperimentListRuns({
+          experimentSlug: "checkout-flow",
+        });
+        expect(out).toContain("# Evaluation Runs: checkout-flow");
+        expect(out).toContain("`run_visible`");
+      });
+    });
+  });
+
+  describe("given the experiment slug is unknown", () => {
+    describe("when the API throws 404", () => {
+      /** @scenario "Unknown experiment slug returns a graceful not-found message" */
+      it("returns a graceful not-found message suggesting platform_experiment_list", async () => {
+        mockMakeRequest.mockRejectedValueOnce(
+          new Error("LangWatch API error 404: experiment not found"),
+        );
+
+        const out = await handleExperimentListRuns({
+          experimentSlug: "does-not-exist",
+        });
+        expect(out).toContain("not found");
+        expect(out).toContain("platform_experiment_list");
+      });
+    });
+  });
+
+  describe("given the experiment exists but has no runs", () => {
+    describe("when invoked", () => {
+      it("returns an empty-state message that points at platform_run_experiment", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experimentId: "exp_1",
+          experimentSlug: "support-bot",
+          runs: [],
+          pagination: { page: 1, pageSize: 25, totalHits: 0, hasMore: false },
+        });
+
+        const out = await handleExperimentListRuns({
+          experimentSlug: "support-bot",
+        });
+        expect(out).toContain("no runs yet");
+        expect(out).toContain("platform_run_experiment");
+      });
+    });
+  });
+});

package/src/__tests__/get-experiment-results.unit.test.ts

@@ -0,0 +1,222 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api.js", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    makeRequest: vi.fn(),
+  };
+});
+
+import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
+import { handleExperimentResults } from "../tools/get-experiment-results.js";
+
+const mockMakeRequest = vi.mocked(makeRequest);
+
+const sample = {
+  experimentId: "exp_1",
+  runId: "run_1",
+  projectId: "proj_1",
+  progress: 3,
+  total: 3,
+  dataset: [
+    { index: 0, entry: { input: "hello world" } },
+    { index: 1, entry: { input: "broken" }, error: "boom" },
+    { index: 2, entry: { input: "low" } },
+  ],
+  evaluations: [
+    { evaluator: "quality", index: 0, status: "processed", score: 0.9, passed: true },
+    { evaluator: "quality", index: 2, status: "processed", score: 0.2, passed: false, details: "off-topic" },
+    { evaluator: "safety", index: 0, status: "processed", score: 1.0, passed: true },
+  ],
+  timestamps: { createdAt: 0, updatedAt: 0, finishedAt: 1 },
+};
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleExperimentResults()", () => {
+  describe("given a completed run", () => {
+    describe("when called without filters", () => {
+      it("hits the v3 results endpoint with the run id", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        await handleExperimentResults({ runId: "run_1" });
+        expect(mockMakeRequest).toHaveBeenCalledWith(
+          "GET",
+          "/api/experiments/runs/run_1/results",
+        );
+      });
+
+      it("renders evaluator averages and per-row sections in markdown", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        const out = await handleExperimentResults({ runId: "run_1" });
+        expect(out).toContain("# Evaluation Results: run_1");
+        expect(out).toContain("## Evaluator Summary");
+        expect(out).toContain("quality");
+        expect(out).toContain("safety");
+        expect(out).toContain("Row #0");
+        expect(out).toContain("Row #1");
+        expect(out).toContain("Row #2");
+      });
+    });
+
+    describe("when filter is 'failed'", () => {
+      it("includes only failed/errored rows", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          filter: "failed",
+        });
+        expect(out).toContain("Row #1");
+        expect(out).toContain("Row #2");
+        expect(out).not.toContain("Row #0");
+      });
+    });
+
+    describe("when an evaluator name is supplied", () => {
+      it("limits the report to that evaluator", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          evaluator: "quality",
+        });
+        expect(out).toContain("quality");
+        // The per-row sections should not list the safety evaluator bullet
+        expect(out).not.toMatch(/\*\*safety\*\*/);
+      });
+    });
+
+    describe("when there are more rows than the cap", () => {
+      it("truncates and notes the cap", async () => {
+        const big = {
+          ...sample,
+          dataset: Array.from({ length: 80 }, (_, i) => ({
+            index: i,
+            entry: { input: `row ${i}` },
+          })),
+          evaluations: [],
+        };
+        mockMakeRequest.mockResolvedValueOnce(big);
+        const out = await handleExperimentResults({ runId: "run_1" });
+        expect(out).toContain("Output truncated to 50 rows of 80");
+      });
+    });
+
+    describe("when a caller asks for more than the row cap", () => {
+      it("still caps output at 50 rows", async () => {
+        const big = {
+          ...sample,
+          dataset: Array.from({ length: 80 }, (_, i) => ({
+            index: i,
+            entry: { input: `row ${i}` },
+          })),
+          evaluations: [],
+        };
+        mockMakeRequest.mockResolvedValueOnce(big);
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          limit: 5000,
+        });
+        expect(out).toContain("Output truncated to 50 rows of 80");
+      });
+    });
+
+    describe("when only some rows have evaluations", () => {
+      it("explains rows with no evaluation coverage", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [
+            { index: 0, entry: { input: "covered" } },
+            { index: 1, entry: { input: "uncovered" } },
+          ],
+          evaluations: [
+            {
+              evaluator: "quality",
+              index: 0,
+              status: "processed",
+              score: 0.9,
+              passed: true,
+            },
+          ],
+        });
+
+        const out = await handleExperimentResults({ runId: "run_1" });
+
+        expect(out).toContain("uncovered");
+        expect(out).toContain("No evaluations recorded for this row");
+      });
+    });
+
+    describe("when multiple targets share the same dataset index", () => {
+      it("keeps evaluator rows target-scoped", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [
+            { index: 0, targetId: "a", entry: { input: "target a" } },
+            { index: 0, targetId: "b", entry: { input: "target b" } },
+          ],
+          evaluations: [
+            {
+              evaluator: "quality",
+              index: 0,
+              targetId: "a",
+              status: "processed",
+              score: 0.9,
+              passed: true,
+            },
+            {
+              evaluator: "quality",
+              index: 0,
+              targetId: "b",
+              status: "processed",
+              score: 0.1,
+              passed: false,
+              details: "target-b-only",
+            },
+          ],
+        });
+
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          filter: "failed",
+        });
+
+        expect(out).toContain("target b");
+        expect(out).toContain("target-b-only");
+        expect(out).not.toContain("target a");
+      });
+    });
+  });
+
+  describe("given the run is missing", () => {
+    describe("when the API throws a 404-shaped error", () => {
+      it("returns a graceful 'not found' markdown", async () => {
+        mockMakeRequest.mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        );
+        const out = await handleExperimentResults({ runId: "missing" });
+        expect(out).toContain("not found");
+        expect(out).toContain("platform_experiment_status");
+      });
+    });
+  });
+
+  describe("given the run is still in progress", () => {
+    describe("when results are requested", () => {
+      it("returns guidance to check status instead of partial results", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          progress: 2,
+          total: 3,
+          timestamps: { createdAt: 0, updatedAt: 0, finishedAt: null },
+        });
+
+        const out = await handleExperimentResults({ runId: "unfinished" });
+
+        expect(out).toContain("results are not yet available");
+        expect(out).toContain("platform_experiment_status");
+      });
+    });
+  });
+});