@langwatch/mcp-server 0.8.0 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langwatch/mcp-server",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "An MCP server for Langwatch.",
   "type": "module",
   "main": "./dist/index.js",
@@ -45,7 +45,7 @@
     "@types/node": "^25.6.0",
     "@types/yargs": "^17.0.33",
     "@typescript/native-preview": "7.0.0-dev.20250911.1",
-    "ai": "^6.0.0",
+    "ai": "^6.0.161",
     "dotenv": "^17.2.2",
     "esbuild": "^0.25.0",
     "eslint": "^10.2.1",

package/pnpm-workspace.yaml

@@ -1,2 +1,9 @@
+minimumReleaseAge: 10080
+
+overrides:
+  "protobufjs": ">=7.5.6 <8.0.0 || >=8.0.2"
+  "langsmith@<0.6.0": "0.6.0"
+  "fast-uri@<3.1.2": "3.1.2"
+
 onlyBuiltDependencies:
   - node-pty

package/pyproject.toml

@@ -13,5 +13,8 @@ dependencies = [
     "tenacity>=9.1.2",
 ]
 
+[tool.uv]
+exclude-newer = "7 days"
+
 [tool.uv.sources]
 langwatch = { path = "../python-sdk", editable = true }

package/src/__tests__/experiment-list-tools.unit.test.ts

@@ -0,0 +1,157 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api.js", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    makeRequest: vi.fn(),
+  };
+});
+
+import { makeRequest } from "../langwatch-api.js";
+import { handleExperimentList } from "../tools/list-experiments.js";
+import { handleExperimentListRuns } from "../tools/list-experiment-runs.js";
+
+const mockMakeRequest = vi.mocked(makeRequest);
+
+describe("handleExperimentList()", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe("given experiments exist", () => {
+    describe("when invoked with default limit", () => {
+      /** @scenario "Lists experiments as markdown" */
+      it("renders a markdown table with each slug", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experiments: [
+            {
+              id: "exp_1",
+              slug: "checkout-flow",
+              name: "Checkout Flow",
+              type: "EVALUATIONS_V3",
+              workflowId: null,
+              createdAt: "2026-01-01T00:00:00Z",
+              updatedAt: "2026-01-02T00:00:00Z",
+              runsCount: 3,
+              lastRunAt: "2026-01-02T00:00:00Z",
+            },
+          ],
+          pagination: {
+            page: 1,
+            pageSize: 25,
+            totalHits: 1,
+            hasMore: false,
+          },
+        });
+
+        const out = await handleExperimentList({});
+        expect(out).toContain("# Experiments");
+        expect(out).toContain("`checkout-flow`");
+        expect(out).toContain("Checkout Flow");
+      });
+    });
+
+    describe("when invoked with limit above the cap", () => {
+      /** @scenario "Limit is bounded to protect agent context" */
+      it("clamps the effective pageSize to 100", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experiments: [],
+          pagination: { page: 1, pageSize: 100, totalHits: 0, hasMore: false },
+        });
+
+        await handleExperimentList({ limit: 5000 });
+        const path = mockMakeRequest.mock.calls[0]![1] as string;
+        expect(path).toContain("pageSize=100");
+      });
+    });
+  });
+
+  describe("given no experiments exist", () => {
+    describe("when invoked", () => {
+      it("returns an empty-state message", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experiments: [],
+          pagination: { page: 1, pageSize: 25, totalHits: 0, hasMore: false },
+        });
+        const out = await handleExperimentList({});
+        expect(out).toContain("No experiments");
+      });
+    });
+  });
+});
+
+describe("handleExperimentListRuns()", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe("given a known experiment with runs", () => {
+    describe("when invoked", () => {
+      /** @scenario "Lists runs for a known experiment" */
+      it("renders a markdown table with each runId", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experimentId: "exp_1",
+          experimentSlug: "checkout-flow",
+          runs: [
+            {
+              experimentId: "exp_1",
+              runId: "run_visible",
+              workflowVersion: null,
+              timestamps: { createdAt: 1000, updatedAt: 2000, finishedAt: 2000 },
+              summary: { evaluations: {} },
+            },
+          ],
+          pagination: {
+            page: 1,
+            pageSize: 25,
+            totalHits: 1,
+            hasMore: false,
+          },
+        });
+
+        const out = await handleExperimentListRuns({
+          experimentSlug: "checkout-flow",
+        });
+        expect(out).toContain("# Evaluation Runs: checkout-flow");
+        expect(out).toContain("`run_visible`");
+      });
+    });
+  });
+
+  describe("given the experiment slug is unknown", () => {
+    describe("when the API throws 404", () => {
+      /** @scenario "Unknown experiment slug returns a graceful not-found message" */
+      it("returns a graceful not-found message suggesting platform_experiment_list", async () => {
+        mockMakeRequest.mockRejectedValueOnce(
+          new Error("LangWatch API error 404: experiment not found"),
+        );
+
+        const out = await handleExperimentListRuns({
+          experimentSlug: "does-not-exist",
+        });
+        expect(out).toContain("not found");
+        expect(out).toContain("platform_experiment_list");
+      });
+    });
+  });
+
+  describe("given the experiment exists but has no runs", () => {
+    describe("when invoked", () => {
+      it("returns an empty-state message that points at platform_run_experiment", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          experimentId: "exp_1",
+          experimentSlug: "support-bot",
+          runs: [],
+          pagination: { page: 1, pageSize: 25, totalHits: 0, hasMore: false },
+        });
+
+        const out = await handleExperimentListRuns({
+          experimentSlug: "support-bot",
+        });
+        expect(out).toContain("no runs yet");
+        expect(out).toContain("platform_run_experiment");
+      });
+    });
+  });
+});

package/src/__tests__/experiment-status.unit.test.ts

@@ -0,0 +1,194 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api.js", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    makeRequest: vi.fn(),
+  };
+});
+
+import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
+import { handleExperimentStatus } from "../tools/run-experiment.js";
+import {
+  deriveRunStatus,
+  isTerminalStatus,
+} from "../tools/experiment-run-status.js";
+
+const mockMakeRequest = vi.mocked(makeRequest);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("deriveRunStatus()", () => {
+  describe("when stoppedAt is set", () => {
+    it("is stopped", () => {
+      expect(deriveRunStatus({ stoppedAt: 5 })).toBe("stopped");
+    });
+  });
+
+  describe("when finishedAt is set", () => {
+    it("is completed", () => {
+      expect(deriveRunStatus({ finishedAt: 5 })).toBe("completed");
+    });
+  });
+
+  describe("when no terminal marker and updates are recent", () => {
+    it("is running", () => {
+      const now = 1_000_000;
+      expect(deriveRunStatus({ updatedAt: now - 1000 }, now)).toBe("running");
+    });
+  });
+
+  describe("when no terminal marker and no recent updates", () => {
+    it("is interrupted", () => {
+      const now = 1_000_000;
+      expect(deriveRunStatus({ updatedAt: now - 6 * 60 * 1000 }, now)).toBe(
+        "interrupted",
+      );
+    });
+  });
+
+  describe("terminal classification", () => {
+    it("treats completed and stopped as terminal", () => {
+      expect(isTerminalStatus("completed")).toBe(true);
+      expect(isTerminalStatus("stopped")).toBe(true);
+      expect(isTerminalStatus("running")).toBe(false);
+      expect(isTerminalStatus("interrupted")).toBe(false);
+    });
+  });
+});
+
+describe("handleExperimentStatus()", () => {
+  describe("given the run state exists in Redis", () => {
+    it("reports the status straight from the status endpoint", async () => {
+      mockMakeRequest.mockResolvedValueOnce({
+        runId: "run_1",
+        status: "completed",
+        progress: 3,
+        total: 3,
+      });
+
+      const out = await handleExperimentStatus({ runId: "run_1" });
+
+      expect(mockMakeRequest).toHaveBeenCalledWith(
+        "GET",
+        "/api/experiments/runs/run_1",
+      );
+      expect(out).toContain("**Status**: completed");
+      expect(out).toContain("3/3 cells");
+    });
+  });
+
+  describe("given an SDK-logged run with no Redis state", () => {
+    it("falls back to deriving status from the results endpoint", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockResolvedValueOnce({
+          progress: 5,
+          total: 5,
+          dataset: [1, 2, 3, 4, 5],
+          timestamps: {
+            createdAt: 1,
+            updatedAt: 2,
+            finishedAt: 3,
+            stoppedAt: null,
+          },
+        });
+
+      const out = await handleExperimentStatus({
+        runId: "sdk_run",
+        experimentSlug: "doc-qa",
+      });
+
+      expect(mockMakeRequest).toHaveBeenNthCalledWith(
+        1,
+        "GET",
+        "/api/experiments/runs/sdk_run",
+      );
+      expect(mockMakeRequest).toHaveBeenNthCalledWith(
+        2,
+        "GET",
+        "/api/experiments/runs/sdk_run/results?experimentSlug=doc-qa",
+      );
+      expect(out).toContain("**Status**: completed");
+      expect(out).toContain("5/5 cells");
+      expect(out).toContain("platform_experiment_results");
+    });
+
+    it("reports interrupted for a stale unfinished SDK run", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockResolvedValueOnce({
+          progress: 2,
+          total: 5,
+          dataset: [1, 2],
+          timestamps: {
+            createdAt: Date.now() - 60 * 60 * 1000,
+            updatedAt: Date.now() - 30 * 60 * 1000,
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+
+      const out = await handleExperimentStatus({
+        runId: "sdk_run",
+        experimentSlug: "doc-qa",
+      });
+
+      expect(out).toContain("**Status**: interrupted");
+      expect(out).toContain("2/5 cells");
+    });
+  });
+
+  describe("given the run cannot be found anywhere", () => {
+    it("returns actionable guidance instead of a raw 404", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        );
+
+      const out = await handleExperimentStatus({
+        runId: "nope",
+        experimentSlug: "doc-qa",
+      });
+
+      expect(out).toContain("not found");
+      expect(out).toContain("experimentSlug");
+      expect(out).toContain("platform_experiment_results");
+    });
+
+    it("rethrows non-404 errors from the status endpoint", async () => {
+      mockMakeRequest.mockRejectedValueOnce(
+        new LangWatchApiError("boom", 500, "Internal error"),
+      );
+
+      await expect(
+        handleExperimentStatus({ runId: "nope" }),
+      ).rejects.toThrow();
+    });
+
+    it("propagates a real (non-404) error from the results fallback instead of faking not-found", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockRejectedValueOnce(
+          new LangWatchApiError("boom", 500, "Internal error"),
+        );
+
+      // The fallback's 500 ("boom") must surface, not the original 404 ("missing").
+      await expect(
+        handleExperimentStatus({ runId: "sdk_run", experimentSlug: "doc-qa" }),
+      ).rejects.toThrow(/boom/);
+    });
+  });
+});

package/src/__tests__/get-experiment-results.unit.test.ts

@@ -0,0 +1,313 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api.js", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    makeRequest: vi.fn(),
+  };
+});
+
+import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
+import { handleExperimentResults } from "../tools/get-experiment-results.js";
+
+const mockMakeRequest = vi.mocked(makeRequest);
+
+const sample = {
+  experimentId: "exp_1",
+  runId: "run_1",
+  projectId: "proj_1",
+  progress: 3,
+  total: 3,
+  dataset: [
+    { index: 0, entry: { input: "hello world" } },
+    { index: 1, entry: { input: "broken" }, error: "boom" },
+    { index: 2, entry: { input: "low" } },
+  ],
+  evaluations: [
+    { evaluator: "quality", index: 0, status: "processed", score: 0.9, passed: true },
+    { evaluator: "quality", index: 2, status: "processed", score: 0.2, passed: false, details: "off-topic" },
+    { evaluator: "safety", index: 0, status: "processed", score: 1.0, passed: true },
+  ],
+  timestamps: { createdAt: 0, updatedAt: 0, finishedAt: 1 },
+};
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleExperimentResults()", () => {
+  describe("given a completed run", () => {
+    describe("when called without filters", () => {
+      it("hits the v3 results endpoint with the run id", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        await handleExperimentResults({ runId: "run_1" });
+        expect(mockMakeRequest).toHaveBeenCalledWith(
+          "GET",
+          "/api/experiments/runs/run_1/results",
+        );
+      });
+
+      it("renders evaluator averages and per-row sections in markdown", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        const out = await handleExperimentResults({ runId: "run_1" });
+        expect(out).toContain("# Evaluation Results: run_1");
+        expect(out).toContain("## Evaluator Summary");
+        expect(out).toContain("quality");
+        expect(out).toContain("safety");
+        expect(out).toContain("Row #0");
+        expect(out).toContain("Row #1");
+        expect(out).toContain("Row #2");
+      });
+    });
+
+    describe("when filter is 'failed'", () => {
+      it("includes only failed/errored rows", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          filter: "failed",
+        });
+        expect(out).toContain("Row #1");
+        expect(out).toContain("Row #2");
+        expect(out).not.toContain("Row #0");
+      });
+    });
+
+    describe("when an evaluator name is supplied", () => {
+      it("limits the report to that evaluator", async () => {
+        mockMakeRequest.mockResolvedValueOnce(sample);
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          evaluator: "quality",
+        });
+        expect(out).toContain("quality");
+        // The per-row sections should not list the safety evaluator bullet
+        expect(out).not.toMatch(/\*\*safety\*\*/);
+      });
+    });
+
+    describe("when there are more rows than the cap", () => {
+      it("truncates and notes the cap", async () => {
+        const big = {
+          ...sample,
+          dataset: Array.from({ length: 80 }, (_, i) => ({
+            index: i,
+            entry: { input: `row ${i}` },
+          })),
+          evaluations: [],
+        };
+        mockMakeRequest.mockResolvedValueOnce(big);
+        const out = await handleExperimentResults({ runId: "run_1" });
+        expect(out).toContain("Output truncated to 50 rows of 80");
+      });
+    });
+
+    describe("when a caller asks for more than the row cap", () => {
+      it("still caps output at 50 rows", async () => {
+        const big = {
+          ...sample,
+          dataset: Array.from({ length: 80 }, (_, i) => ({
+            index: i,
+            entry: { input: `row ${i}` },
+          })),
+          evaluations: [],
+        };
+        mockMakeRequest.mockResolvedValueOnce(big);
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          limit: 5000,
+        });
+        expect(out).toContain("Output truncated to 50 rows of 80");
+      });
+    });
+
+    describe("when only some rows have evaluations", () => {
+      it("explains rows with no evaluation coverage", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [
+            { index: 0, entry: { input: "covered" } },
+            { index: 1, entry: { input: "uncovered" } },
+          ],
+          evaluations: [
+            {
+              evaluator: "quality",
+              index: 0,
+              status: "processed",
+              score: 0.9,
+              passed: true,
+            },
+          ],
+        });
+
+        const out = await handleExperimentResults({ runId: "run_1" });
+
+        expect(out).toContain("uncovered");
+        expect(out).toContain("No evaluations recorded for this row");
+      });
+    });
+
+    describe("when multiple targets share the same dataset index", () => {
+      it("keeps evaluator rows target-scoped", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [
+            { index: 0, targetId: "a", entry: { input: "target a" } },
+            { index: 0, targetId: "b", entry: { input: "target b" } },
+          ],
+          evaluations: [
+            {
+              evaluator: "quality",
+              index: 0,
+              targetId: "a",
+              status: "processed",
+              score: 0.9,
+              passed: true,
+            },
+            {
+              evaluator: "quality",
+              index: 0,
+              targetId: "b",
+              status: "processed",
+              score: 0.1,
+              passed: false,
+              details: "target-b-only",
+            },
+          ],
+        });
+
+        const out = await handleExperimentResults({
+          runId: "run_1",
+          filter: "failed",
+        });
+
+        expect(out).toContain("target b");
+        expect(out).toContain("target-b-only");
+        expect(out).not.toContain("target a");
+      });
+    });
+  });
+
+  describe("given the run is missing", () => {
+    describe("when the API throws a 404-shaped error", () => {
+      it("returns a graceful 'not found' markdown", async () => {
+        mockMakeRequest.mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        );
+        const out = await handleExperimentResults({ runId: "missing" });
+        expect(out).toContain("not found");
+        expect(out).toContain("experimentSlug");
+      });
+    });
+  });
+
+  describe("given the run is still in progress", () => {
+    describe("when results are requested", () => {
+      it("serves the partial rows logged so far with an in-progress note", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          progress: 2,
+          total: 3,
+          timestamps: {
+            createdAt: Date.now(),
+            updatedAt: Date.now(),
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+
+        const out = await handleExperimentResults({ runId: "unfinished" });
+
+        expect(out).toContain("**Status**: running");
+        expect(out).toContain("still in progress");
+        expect(out).toContain("Row #0");
+        expect(out).toContain("Progress**: 2/3");
+      });
+    });
+  });
+
+  describe("given the run was interrupted before finishing", () => {
+    describe("when results are requested", () => {
+      it("serves the partial rows with an interrupted note", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          timestamps: {
+            createdAt: Date.now() - 60 * 60 * 1000,
+            updatedAt: Date.now() - 30 * 60 * 1000,
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+
+        const out = await handleExperimentResults({ runId: "interrupted" });
+
+        expect(out).toContain("**Status**: interrupted");
+        expect(out).toContain("interrupted");
+        expect(out).toContain("Row #0");
+      });
+    });
+  });
+
+  describe("given a non-terminal run with zero rows", () => {
+    describe("when the run is still running", () => {
+      it("says the run is in progress", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [],
+          evaluations: [],
+          timestamps: {
+            createdAt: Date.now(),
+            updatedAt: Date.now(),
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+        const out = await handleExperimentResults({ runId: "empty-running" });
+        expect(out).toContain("still in progress");
+        expect(out).not.toContain("interrupted");
+      });
+    });
+
+    describe("when the run was interrupted", () => {
+      it("says no rows were recorded before it was interrupted, not to wait", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [],
+          evaluations: [],
+          timestamps: {
+            createdAt: Date.now() - 60 * 60 * 1000,
+            updatedAt: Date.now() - 30 * 60 * 1000,
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+        const out = await handleExperimentResults({ runId: "empty-interrupted" });
+        expect(out).toContain("interrupted");
+        expect(out).not.toContain("still in progress");
+        expect(out).not.toContain("Call again shortly");
+      });
+    });
+  });
+
+  describe("given no run state but rows exist (SDK / >24h run)", () => {
+    describe("when results are requested with a stopped run", () => {
+      it("renders results for a stopped run", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          timestamps: {
+            createdAt: 0,
+            updatedAt: 10,
+            finishedAt: null,
+            stoppedAt: 20,
+          },
+        });
+
+        const out = await handleExperimentResults({ runId: "stopped" });
+
+        expect(out).toContain("**Status**: stopped");
+        expect(out).toContain("Row #0");
+      });
+    });
+  });
+});