@langwatch/mcp-server 0.10.1 → 2.0.0

package/src/__tests__/langwatch-api.unit.test.ts

@@ -452,4 +452,210 @@ describe("langwatch-api", () => {
       ).rejects.toThrow("Forbidden: invalid API key");
     });
   });
+
+  describe("when the API returns a handled-error JSON body", () => {
+    const handledErrorBody = {
+      error: "invalid_filter",
+      message: "Filter 'llm.model' is not supported",
+      tips: ["Use 'metadata.model' instead", "See the filter reference"],
+      docsUrl: "https://docs.langwatch.ai/api/filters",
+      fault: "customer",
+    };
+
+    it("parses code, tips, docsUrl and fault into LangWatchApiError fields", async () => {
+      const { searchTraces, LangWatchApiError } = await import(
+        "../langwatch-api.js"
+      );
+      mockErrorResponse(400, JSON.stringify(handledErrorBody));
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error).toBeInstanceOf(LangWatchApiError);
+      expect(error.status).toBe(400);
+      expect(error.responseBody).toBe(JSON.stringify(handledErrorBody));
+      expect(error.code).toBe("invalid_filter");
+      expect(error.tips).toEqual(handledErrorBody.tips);
+      expect(error.docsUrl).toBe(handledErrorBody.docsUrl);
+      expect(error.fault).toBe("customer");
+    });
+
+    it("formats the message with the server message, Tips section and Docs link", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      mockErrorResponse(400, JSON.stringify(handledErrorBody));
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.message).toContain(
+        "LangWatch API error 400: Filter 'llm.model' is not supported"
+      );
+      expect(error.message).toContain("Tips:");
+      expect(error.message).toContain("- Use 'metadata.model' instead");
+      expect(error.message).toContain("- See the filter reference");
+      expect(error.message).toContain(
+        "Docs: https://docs.langwatch.ai/api/filters"
+      );
+    });
+
+    // A validation failure names the offending field AND the values it would
+    // have taken, in `reasons[].meta`. Dropping that leaves the message's own
+    // advice ("pick one of the types in this error's expected list") naming a
+    // list the caller was never given.
+    describe("when the failure carries the values it would have accepted", () => {
+      const rejectedTypeBody = {
+        error: "validation_error",
+        message:
+          "Unknown evaluator type. Pick one of the types in this error's expected list and retry.",
+        reasons: [
+          {
+            code: "schema_failure",
+            kind: "schema_failure",
+            meta: {
+              field: "config.evaluatorType",
+              expected: ["ragas/response_relevancy", "langevals/llm_boolean"],
+              received: "ragas/answer_relevancy",
+            },
+          },
+        ],
+      };
+
+      /** @scenario A rejection over MCP carries the accepted types */
+      it("keeps the reasons on the error", async () => {
+        const { createEvaluator } = await import(
+          "../langwatch-api-evaluators.js"
+        );
+        mockErrorResponse(422, JSON.stringify(rejectedTypeBody));
+
+        const error = await createEvaluator({
+          name: "Relevancy",
+          config: { evaluatorType: "ragas/answer_relevancy" },
+        }).catch((e) => e);
+
+        expect(error.reasons).toEqual(rejectedTypeBody.reasons);
+      });
+
+      it("names the rejected field and the accepted values in the message", async () => {
+        const { createEvaluator } = await import(
+          "../langwatch-api-evaluators.js"
+        );
+        mockErrorResponse(422, JSON.stringify(rejectedTypeBody));
+
+        const error = await createEvaluator({
+          name: "Relevancy",
+          config: { evaluatorType: "ragas/answer_relevancy" },
+        }).catch((e) => e);
+
+        expect(error.message).toContain("config.evaluatorType");
+        expect(error.message).toContain("ragas/response_relevancy");
+        expect(error.message).toContain("langevals/llm_boolean");
+        expect(error.message).toContain("ragas/answer_relevancy");
+      });
+    });
+
+    it("parses the tRPC shape using `code` instead of `error`", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      const body = {
+        code: "dataset_not_found",
+        message: "Dataset 'abc' not found",
+        meta: { datasetId: "abc" },
+        httpStatus: 404,
+        fault: "customer",
+        tips: ["Check the dataset slug"],
+        docsUrl: "https://docs.langwatch.ai/datasets",
+        reasons: [],
+      };
+      mockErrorResponse(404, JSON.stringify(body));
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.code).toBe("dataset_not_found");
+      expect(error.tips).toEqual(["Check the dataset slug"]);
+      expect(error.docsUrl).toBe("https://docs.langwatch.ai/datasets");
+      expect(error.fault).toBe("customer");
+      expect(error.message).toContain("Dataset 'abc' not found");
+    });
+
+    it("omits the Tips and Docs sections when absent", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      mockErrorResponse(
+        401,
+        JSON.stringify({ error: "unauthorized", message: "Invalid API key" })
+      );
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.code).toBe("unauthorized");
+      expect(error.tips).toBeUndefined();
+      expect(error.docsUrl).toBeUndefined();
+      expect(error.fault).toBeUndefined();
+      expect(error.message).toBe(
+        "LangWatch API error 401: Invalid API key"
+      );
+    });
+
+    it("ignores an invalid fault value", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      mockErrorResponse(
+        500,
+        JSON.stringify({ error: "boom", message: "Boom", fault: "nobody" })
+      );
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.fault).toBeUndefined();
+    });
+  });
+
+  describe("when the error body is not a handled-error envelope", () => {
+    it("keeps the raw text as the message and sets no extra fields", async () => {
+      const { searchTraces, LangWatchApiError } = await import(
+        "../langwatch-api.js"
+      );
+      mockErrorResponse(502, "<html>Bad Gateway</html>");
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error).toBeInstanceOf(LangWatchApiError);
+      expect(error.status).toBe(502);
+      expect(error.responseBody).toBe("<html>Bad Gateway</html>");
+      expect(error.message).toBe(
+        "LangWatch API error 502: <html>Bad Gateway</html>"
+      );
+      expect(error.code).toBeUndefined();
+      expect(error.tips).toBeUndefined();
+      expect(error.docsUrl).toBeUndefined();
+      expect(error.fault).toBeUndefined();
+    });
+
+    it("treats JSON without error/code/message fields as raw text", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      const body = JSON.stringify({ unexpected: true });
+      mockErrorResponse(500, body);
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.code).toBeUndefined();
+      expect(error.message).toBe(`LangWatch API error 500: ${body}`);
+    });
+  });
 });

package/src/__tests__/public-http-request-security.integration.test.ts

@@ -0,0 +1,118 @@
+import { createServer, type Server } from "node:http";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+
+import {
+  headersForRedirect,
+  requestPublicJson,
+  resolvePublicDestination,
+} from "../public-http-request.js";
+
+describe("public HTTP request security", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  /** @scenario Running an HTTP agent rejects unsafe destinations */
+  it.each([
+    "http://127.0.0.1/secrets",
+    "http://2130706433/secrets",
+    "http://169.254.169.254/latest/meta-data/",
+    "http://10.0.0.1/internal",
+    "http://[::1]/secrets",
+    "file:///etc/passwd",
+    "http://user:password@example.com/",
+  ])("rejects unsafe destination %s", async (url) => {
+    await expect(resolvePublicDestination(url)).rejects.toThrow(/globally routable public addresses/);
+  });
+
+  it("fails closed when any resolved address is not public", async () => {
+    await expect(
+      resolvePublicDestination("https://agent.example/run", async () => [
+        { address: "93.184.216.34", family: 4 },
+        { address: "10.0.0.4", family: 4 },
+      ])
+    ).rejects.toThrow(/globally routable public addresses/);
+  });
+
+  it("pins a validated public destination", async () => {
+    await expect(
+      resolvePublicDestination("https://agent.example/run", async () => [{ address: "93.184.216.34", family: 4 }])
+    ).resolves.toMatchObject({
+      address: "93.184.216.34",
+      family: 4,
+    });
+  });
+
+  it("bounds DNS resolution time", async () => {
+    vi.useFakeTimers();
+    const resolution = resolvePublicDestination(
+      "https://agent.example/run",
+      () => new Promise(() => undefined)
+    );
+    const rejection = expect(resolution).rejects.toThrow(
+      /destination could not be resolved/
+    );
+
+    await vi.advanceTimersByTimeAsync(30_000);
+
+    await rejection;
+  });
+
+  /** @scenario HTTP agent redirects are revalidated */
+  it("rejects private redirect targets and strips cross-origin secrets", async () => {
+    await expect(
+      resolvePublicDestination("http://127.0.0.1/redirect-target")
+    ).rejects.toThrow(/globally routable public addresses/);
+
+    expect(
+      headersForRedirect(
+        {
+          Authorization: "Bearer secret",
+          Cookie: "session=secret",
+          "Content-Type": "application/json",
+          "X-Api-Key": "secret",
+          "X-Request-Id": "request-1",
+        },
+        true
+      )
+    ).toEqual({
+      "Content-Type": "application/json",
+      "X-Request-Id": "request-1",
+    });
+  });
+
+  describe("real HTTP request", () => {
+    let server: Server;
+    let port: number;
+    let hits = 0;
+
+    beforeAll(async () => {
+      server = createServer((_request, response) => {
+        hits += 1;
+        response.setHeader("Content-Type", "application/json");
+        response.end(JSON.stringify({ secret: true }));
+      });
+      await new Promise<void>((resolve) => {
+        server.listen(0, "127.0.0.1", resolve);
+      });
+      const address = server.address();
+      port = typeof address === "object" && address ? address.port : 0;
+    });
+
+    afterAll(async () => {
+      await new Promise<void>((resolve, reject) => {
+        server.close((error) => (error ? reject(error) : resolve()));
+      });
+    });
+
+    it("blocks loopback before opening a connection", async () => {
+      await expect(
+        requestPublicJson(`http://127.0.0.1:${port}/secrets`, {
+          method: "POST",
+          body: "{}",
+        })
+      ).rejects.toThrow(/globally routable public addresses/);
+      expect(hits).toBe(0);
+    });
+  });
+});

package/src/__tests__/report-issue.integration.test.ts

@@ -0,0 +1,147 @@
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+} from "vitest";
+import { initConfig } from "../config.js";
+import { createMcpServer } from "../create-mcp-server.js";
+import { handleReportIssue } from "../tools/report-issue.js";
+
+/**
+ * Integration tests for the report_issue tool against a REAL local HTTP
+ * server. Corresponds to specs/mcp-server/report-issue-tool.feature.
+ */
+describe("report_issue", () => {
+  let server: Server;
+  let endpoint: string;
+  let received: { headers: Record<string, unknown>; body: any }[];
+
+  beforeAll(async () => {
+    server = createServer((req, res) => {
+      let data = "";
+      req.on("data", (chunk: Buffer) => (data += chunk.toString()));
+      req.on("end", () => {
+        received.push({ headers: { ...req.headers }, body: JSON.parse(data) });
+        res.writeHead(201, { "content-type": "application/json" });
+        res.end(JSON.stringify({ id: "report-mcp-1" }));
+      });
+    });
+    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+    endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+  });
+
+  afterAll(async () => {
+    await new Promise((resolve) => server.close(resolve));
+  });
+
+  beforeEach(() => {
+    received = [];
+    initConfig({ endpoint, apiKey: undefined });
+  });
+
+  describe("given the registered tool set", () => {
+    /** @scenario "The report tool is listed with an agent-facing description" */
+    it("lists report_issue with the agent-facing consent description", () => {
+      const server = createMcpServer();
+      const tools = (
+        server as unknown as {
+          _registeredTools: Record<string, { description?: string }>;
+        }
+      )._registeredTools;
+      expect(Object.keys(tools)).toContain("report_issue");
+      const description = tools.report_issue?.description ?? "";
+      expect(description).toMatch(/whenever you struggled/i);
+      expect(description).toMatch(/ask the user for permission first/i);
+    });
+  });
+
+  describe("given the user did not approve sending", () => {
+    /** @scenario "Calls without user approval are rejected with instructions" */
+    it("refuses with instructions to ask the user first", async () => {
+      await expect(
+        handleReportIssue({
+          user_approved: false,
+          title: "broken tool",
+          summary: "search_traces 500s",
+        }),
+      ).rejects.toThrow(/ask the user/i);
+      expect(received).toHaveLength(0);
+    });
+  });
+
+  describe("given nothing to report", () => {
+    it("asks for a summary or session content", async () => {
+      await expect(
+        handleReportIssue({ user_approved: true, title: "empty" }),
+      ).rejects.toThrow(/Nothing to report/);
+    });
+  });
+
+  describe("when the user approved a summary report", () => {
+    /** @scenario "Reporting an issue through MCP reaches the LangWatch backend" */
+    it("delivers it marked as coming from MCP and returns a thank-you", async () => {
+      const text = await handleReportIssue({
+        user_approved: true,
+        title: "evaluator setup confusing",
+        summary: "the evaluator wizard 404s on step 2",
+        agent: "claude-code",
+      });
+
+      expect(received).toHaveLength(1);
+      expect(received[0]?.body.source).toBe("mcp");
+      expect(received[0]?.body.kind).toBe("summary");
+      expect(received[0]?.body.title).toBe("evaluator setup confusing");
+      expect(text).toContain("report-mcp-1");
+      expect(text.toLowerCase()).toContain("thank you");
+    });
+  });
+
+  describe("when the title itself includes a secret", () => {
+    it("redacts it before sending", async () => {
+      await handleReportIssue({
+        user_approved: true,
+        title: "X-Api-Key sk-proj-abcdefghijklmnopqrstuvwxyz123456 rejected",
+        summary: "the key was rejected",
+      });
+      expect(received[0]?.body.title).toBe("X-Api-Key [SECRET] rejected");
+    });
+  });
+
+  describe("when session content includes secrets", () => {
+    /** @scenario "Session content passed through MCP is redacted before sending" */
+    it("redacts them locally before sending", async () => {
+      await handleReportIssue({
+        user_approved: true,
+        title: "agent stuck",
+        session_content: JSON.stringify({
+          role: "user",
+          content: "my key sk-proj-abcdefghijklmnopqrstuvwxyz123456 failed",
+        }),
+      });
+
+      const wire = JSON.stringify(received[0]?.body);
+      expect(wire).not.toContain("sk-proj-abcdefghijklmnopqrstuvwxyz123456");
+      expect(received[0]?.body.sessionData).toContain("[SECRET]");
+      expect(received[0]?.body.kind).toBe("full_session");
+    });
+  });
+
+  describe("when an API key is configured", () => {
+    it("attaches it so the report links to the project", async () => {
+      initConfig({ endpoint, apiKey: "sk-lw-mcp-key-abcdef1234567890" });
+      await handleReportIssue({
+        user_approved: true,
+        title: "linked",
+        summary: "linked report",
+      });
+      expect(received[0]?.headers.authorization).toBe(
+        "Bearer sk-lw-mcp-key-abcdef1234567890",
+      );
+    });
+  });
+});

package/src/__tests__/support/http-server-harness.ts

@@ -0,0 +1,138 @@
+/**
+ * Shared fixtures for the standalone MCP HTTP server tests.
+ */
+
+import { expect, vi } from "vitest";
+
+import type { ApiKeyVerifier } from "../../http-security.js";
+import { startHttpServer } from "../../http-server.js";
+
+/** Standard headers the MCP Streamable HTTP protocol requires on POST. */
+export const MCP_POST_HEADERS = {
+  "Content-Type": "application/json",
+  Accept: "application/json, text/event-stream",
+};
+
+export const VALID_KEY = "sk-lw-valid-key";
+export const OTHER_VALID_KEY = "sk-lw-other-valid-key";
+
+/**
+ * Stands in for the LangWatch API. The standalone server has no database, so
+ * it asks the API whether a key is real; here a fixed list is.
+ */
+export function countingVerifier(validKeys: string[]) {
+  const verify = vi.fn(async (apiKey: string) => validKeys.includes(apiKey));
+  const verifier: ApiKeyVerifier = {
+    verify,
+    sweep: () => undefined,
+    clear: () => undefined,
+  };
+  return { verifier, verify };
+}
+
+/** A verifier whose accepted keys can change mid-test, to model revocation. */
+export function revocableVerifier(initialKeys: string[]) {
+  const live = new Set(initialKeys);
+  const verifier: ApiKeyVerifier = {
+    verify: async (apiKey: string) => live.has(apiKey),
+    sweep: () => undefined,
+    clear: () => undefined,
+  };
+  return { verifier, revoke: (apiKey: string) => live.delete(apiKey) };
+}
+
+export interface Harness {
+  baseUrl: string;
+  close: () => Promise<void>;
+}
+
+export async function startHarness(
+  options: Parameters<typeof startHttpServer>[0]
+): Promise<Harness> {
+  const { server, port, host } = await startHttpServer(options);
+  return {
+    baseUrl: `http://${host === "0.0.0.0" ? "127.0.0.1" : host}:${port}`,
+    close: () =>
+      new Promise<void>((resolve) => {
+        server.close(() => resolve());
+      }),
+  };
+}
+
+export function initializeBody(): string {
+  return JSON.stringify({
+    jsonrpc: "2.0",
+    method: "initialize",
+    params: {
+      protocolVersion: "2025-03-26",
+      capabilities: {},
+      clientInfo: { name: "test-client", version: "1.0.0" },
+    },
+    id: 1,
+  });
+}
+
+export function toolsListBody(): string {
+  return JSON.stringify({ jsonrpc: "2.0", method: "tools/list", id: 2 });
+}
+
+/** Opens a Streamable HTTP session and returns its id. */
+export async function openSession({
+  baseUrl,
+  apiKey = VALID_KEY,
+}: {
+  baseUrl: string;
+  apiKey?: string;
+}): Promise<string> {
+  const response = await fetch(`${baseUrl}/mcp`, {
+    method: "POST",
+    headers: { ...MCP_POST_HEADERS, Authorization: `Bearer ${apiKey}` },
+    body: initializeBody(),
+  });
+  expect(response.status).toBe(200);
+  await response.text();
+  return response.headers.get("mcp-session-id")!;
+}
+
+/**
+ * Reads an SSE stream until the endpoint event carrying the session id has
+ * arrived. A single read can return a partial frame, so waiting for the
+ * pattern rather than the first chunk keeps this from failing intermittently.
+ */
+export async function readSessionId(response: Response): Promise<string> {
+  const reader = response.body!.getReader();
+  const decoder = new TextDecoder();
+  let buffered = "";
+
+  for (let reads = 0; reads < 20; reads++) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    buffered += decoder.decode(value, { stream: true });
+    const match = /sessionId=([\w-]+)/.exec(buffered);
+    if (match?.[1]) return match[1];
+  }
+
+  throw new Error(`No session id in SSE stream. Received: ${buffered}`);
+}
+
+/**
+ * Sends `request` repeatedly until it answers 429, and returns that response
+ * along with how many attempts it took. Returns `rejected: undefined` when the
+ * limit never triggered within `maxAttempts`.
+ */
+export async function requestUntilThrottled({
+  request,
+  maxAttempts = 30,
+}: {
+  request: (attempt: number) => Promise<Response>;
+  maxAttempts?: number;
+}): Promise<{ attempts: number; rejected: Response | undefined }> {
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const response = await request(attempt);
+    if (response.status === 429) {
+      return { attempts: attempt + 1, rejected: response };
+    }
+    await response.text();
+  }
+  return { attempts: maxAttempts, rejected: undefined };
+}

package/src/__tests__/tools.unit.test.ts

@@ -168,6 +168,30 @@ describe("handleSearchTraces()", () => {
     expect(result).toContain("timeout");
   });
 
+  describe("when a trace has evaluation results", () => {
+    it("shows pass/fail status per evaluation without requiring get_trace", async () => {
+      mockSearchTraces.mockResolvedValue({
+        traces: [
+          {
+            trace_id: "trace-eval",
+            input: { value: "" },
+            output: { value: "" },
+            evaluations: [
+              { name: "faithfulness", passed: true, score: 0.92 },
+              { evaluator_id: "toxicity-check", passed: false, label: "flagged" },
+            ],
+          },
+        ],
+        pagination: { totalHits: 1 },
+      });
+
+      const result = await handleSearchTraces({});
+
+      expect(result).toContain("**faithfulness**: PASSED (score: 0.92)");
+      expect(result).toContain("**toxicity-check**: FAILED [flagged]");
+    });
+  });
+
   describe("when no traces are found", () => {
     it("returns a no-results message", async () => {
       mockSearchTraces.mockResolvedValue({ traces: [] });

package/src/config.ts

@@ -49,13 +49,28 @@ function getGlobalState(): McpGlobalState {
   };
 }
 
+/**
+ * Trim surrounding whitespace and drop any trailing slashes.
+ *
+ * Request URLs are built as `${endpoint}/api/...`, so an endpoint written as
+ * `https://app.langwatch.ai/` would produce a double slash the router does not
+ * match, and the caller gets an opaque 404 with nothing pointing at the
+ * endpoint as the cause.
+ */
+function normalizeEndpoint(endpoint: string): string {
+  const trimmed = endpoint.trim();
+  let end = trimmed.length;
+  while (end > 0 && trimmed[end - 1] === "/") end--;
+  return trimmed.slice(0, end);
+}
+
 export function initConfig(args: { apiKey?: string; endpoint?: string }): void {
   const state = getGlobalState();
   state.globalConfig = {
     apiKey: args.apiKey || process.env.LANGWATCH_API_KEY,
     endpoint:
-      args.endpoint ||
-      process.env.LANGWATCH_ENDPOINT ||
+      normalizeEndpoint(args.endpoint ?? "") ||
+      normalizeEndpoint(process.env.LANGWATCH_ENDPOINT ?? "") ||
       "https://app.langwatch.ai",
     projectId: process.env.LANGWATCH_PROJECT_ID,
   };