@langwatch/mcp-server 0.10.1 → 1.0.0

package/src/__tests__/langwatch-api.unit.test.ts

@@ -452,4 +452,210 @@ describe("langwatch-api", () => {
       ).rejects.toThrow("Forbidden: invalid API key");
     });
   });
+
+  describe("when the API returns a handled-error JSON body", () => {
+    const handledErrorBody = {
+      error: "invalid_filter",
+      message: "Filter 'llm.model' is not supported",
+      tips: ["Use 'metadata.model' instead", "See the filter reference"],
+      docsUrl: "https://docs.langwatch.ai/api/filters",
+      fault: "customer",
+    };
+
+    it("parses code, tips, docsUrl and fault into LangWatchApiError fields", async () => {
+      const { searchTraces, LangWatchApiError } = await import(
+        "../langwatch-api.js"
+      );
+      mockErrorResponse(400, JSON.stringify(handledErrorBody));
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error).toBeInstanceOf(LangWatchApiError);
+      expect(error.status).toBe(400);
+      expect(error.responseBody).toBe(JSON.stringify(handledErrorBody));
+      expect(error.code).toBe("invalid_filter");
+      expect(error.tips).toEqual(handledErrorBody.tips);
+      expect(error.docsUrl).toBe(handledErrorBody.docsUrl);
+      expect(error.fault).toBe("customer");
+    });
+
+    it("formats the message with the server message, Tips section and Docs link", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      mockErrorResponse(400, JSON.stringify(handledErrorBody));
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.message).toContain(
+        "LangWatch API error 400: Filter 'llm.model' is not supported"
+      );
+      expect(error.message).toContain("Tips:");
+      expect(error.message).toContain("- Use 'metadata.model' instead");
+      expect(error.message).toContain("- See the filter reference");
+      expect(error.message).toContain(
+        "Docs: https://docs.langwatch.ai/api/filters"
+      );
+    });
+
+    // A validation failure names the offending field AND the values it would
+    // have taken, in `reasons[].meta`. Dropping that leaves the message's own
+    // advice ("pick one of the types in this error's expected list") naming a
+    // list the caller was never given.
+    describe("when the failure carries the values it would have accepted", () => {
+      const rejectedTypeBody = {
+        error: "validation_error",
+        message:
+          "Unknown evaluator type. Pick one of the types in this error's expected list and retry.",
+        reasons: [
+          {
+            code: "schema_failure",
+            kind: "schema_failure",
+            meta: {
+              field: "config.evaluatorType",
+              expected: ["ragas/response_relevancy", "langevals/llm_boolean"],
+              received: "ragas/answer_relevancy",
+            },
+          },
+        ],
+      };
+
+      /** @scenario A rejection over MCP carries the accepted types */
+      it("keeps the reasons on the error", async () => {
+        const { createEvaluator } = await import(
+          "../langwatch-api-evaluators.js"
+        );
+        mockErrorResponse(422, JSON.stringify(rejectedTypeBody));
+
+        const error = await createEvaluator({
+          name: "Relevancy",
+          config: { evaluatorType: "ragas/answer_relevancy" },
+        }).catch((e) => e);
+
+        expect(error.reasons).toEqual(rejectedTypeBody.reasons);
+      });
+
+      it("names the rejected field and the accepted values in the message", async () => {
+        const { createEvaluator } = await import(
+          "../langwatch-api-evaluators.js"
+        );
+        mockErrorResponse(422, JSON.stringify(rejectedTypeBody));
+
+        const error = await createEvaluator({
+          name: "Relevancy",
+          config: { evaluatorType: "ragas/answer_relevancy" },
+        }).catch((e) => e);
+
+        expect(error.message).toContain("config.evaluatorType");
+        expect(error.message).toContain("ragas/response_relevancy");
+        expect(error.message).toContain("langevals/llm_boolean");
+        expect(error.message).toContain("ragas/answer_relevancy");
+      });
+    });
+
+    it("parses the tRPC shape using `code` instead of `error`", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      const body = {
+        code: "dataset_not_found",
+        message: "Dataset 'abc' not found",
+        meta: { datasetId: "abc" },
+        httpStatus: 404,
+        fault: "customer",
+        tips: ["Check the dataset slug"],
+        docsUrl: "https://docs.langwatch.ai/datasets",
+        reasons: [],
+      };
+      mockErrorResponse(404, JSON.stringify(body));
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.code).toBe("dataset_not_found");
+      expect(error.tips).toEqual(["Check the dataset slug"]);
+      expect(error.docsUrl).toBe("https://docs.langwatch.ai/datasets");
+      expect(error.fault).toBe("customer");
+      expect(error.message).toContain("Dataset 'abc' not found");
+    });
+
+    it("omits the Tips and Docs sections when absent", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      mockErrorResponse(
+        401,
+        JSON.stringify({ error: "unauthorized", message: "Invalid API key" })
+      );
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.code).toBe("unauthorized");
+      expect(error.tips).toBeUndefined();
+      expect(error.docsUrl).toBeUndefined();
+      expect(error.fault).toBeUndefined();
+      expect(error.message).toBe(
+        "LangWatch API error 401: Invalid API key"
+      );
+    });
+
+    it("ignores an invalid fault value", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      mockErrorResponse(
+        500,
+        JSON.stringify({ error: "boom", message: "Boom", fault: "nobody" })
+      );
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.fault).toBeUndefined();
+    });
+  });
+
+  describe("when the error body is not a handled-error envelope", () => {
+    it("keeps the raw text as the message and sets no extra fields", async () => {
+      const { searchTraces, LangWatchApiError } = await import(
+        "../langwatch-api.js"
+      );
+      mockErrorResponse(502, "<html>Bad Gateway</html>");
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error).toBeInstanceOf(LangWatchApiError);
+      expect(error.status).toBe(502);
+      expect(error.responseBody).toBe("<html>Bad Gateway</html>");
+      expect(error.message).toBe(
+        "LangWatch API error 502: <html>Bad Gateway</html>"
+      );
+      expect(error.code).toBeUndefined();
+      expect(error.tips).toBeUndefined();
+      expect(error.docsUrl).toBeUndefined();
+      expect(error.fault).toBeUndefined();
+    });
+
+    it("treats JSON without error/code/message fields as raw text", async () => {
+      const { searchTraces } = await import("../langwatch-api.js");
+      const body = JSON.stringify({ unexpected: true });
+      mockErrorResponse(500, body);
+
+      const error = await searchTraces({
+        startDate: 1000,
+        endDate: 2000,
+      }).catch((e) => e);
+
+      expect(error.code).toBeUndefined();
+      expect(error.message).toBe(`LangWatch API error 500: ${body}`);
+    });
+  });
 });

package/src/__tests__/public-http-request-security.integration.test.ts

@@ -0,0 +1,118 @@
+import { createServer, type Server } from "node:http";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+
+import {
+  headersForRedirect,
+  requestPublicJson,
+  resolvePublicDestination,
+} from "../public-http-request.js";
+
+describe("public HTTP request security", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  /** @scenario Running an HTTP agent rejects unsafe destinations */
+  it.each([
+    "http://127.0.0.1/secrets",
+    "http://2130706433/secrets",
+    "http://169.254.169.254/latest/meta-data/",
+    "http://10.0.0.1/internal",
+    "http://[::1]/secrets",
+    "file:///etc/passwd",
+    "http://user:password@example.com/",
+  ])("rejects unsafe destination %s", async (url) => {
+    await expect(resolvePublicDestination(url)).rejects.toThrow(/globally routable public addresses/);
+  });
+
+  it("fails closed when any resolved address is not public", async () => {
+    await expect(
+      resolvePublicDestination("https://agent.example/run", async () => [
+        { address: "93.184.216.34", family: 4 },
+        { address: "10.0.0.4", family: 4 },
+      ])
+    ).rejects.toThrow(/globally routable public addresses/);
+  });
+
+  it("pins a validated public destination", async () => {
+    await expect(
+      resolvePublicDestination("https://agent.example/run", async () => [{ address: "93.184.216.34", family: 4 }])
+    ).resolves.toMatchObject({
+      address: "93.184.216.34",
+      family: 4,
+    });
+  });
+
+  it("bounds DNS resolution time", async () => {
+    vi.useFakeTimers();
+    const resolution = resolvePublicDestination(
+      "https://agent.example/run",
+      () => new Promise(() => undefined)
+    );
+    const rejection = expect(resolution).rejects.toThrow(
+      /destination could not be resolved/
+    );
+
+    await vi.advanceTimersByTimeAsync(30_000);
+
+    await rejection;
+  });
+
+  /** @scenario HTTP agent redirects are revalidated */
+  it("rejects private redirect targets and strips cross-origin secrets", async () => {
+    await expect(
+      resolvePublicDestination("http://127.0.0.1/redirect-target")
+    ).rejects.toThrow(/globally routable public addresses/);
+
+    expect(
+      headersForRedirect(
+        {
+          Authorization: "Bearer secret",
+          Cookie: "session=secret",
+          "Content-Type": "application/json",
+          "X-Api-Key": "secret",
+          "X-Request-Id": "request-1",
+        },
+        true
+      )
+    ).toEqual({
+      "Content-Type": "application/json",
+      "X-Request-Id": "request-1",
+    });
+  });
+
+  describe("real HTTP request", () => {
+    let server: Server;
+    let port: number;
+    let hits = 0;
+
+    beforeAll(async () => {
+      server = createServer((_request, response) => {
+        hits += 1;
+        response.setHeader("Content-Type", "application/json");
+        response.end(JSON.stringify({ secret: true }));
+      });
+      await new Promise<void>((resolve) => {
+        server.listen(0, "127.0.0.1", resolve);
+      });
+      const address = server.address();
+      port = typeof address === "object" && address ? address.port : 0;
+    });
+
+    afterAll(async () => {
+      await new Promise<void>((resolve, reject) => {
+        server.close((error) => (error ? reject(error) : resolve()));
+      });
+    });
+
+    it("blocks loopback before opening a connection", async () => {
+      await expect(
+        requestPublicJson(`http://127.0.0.1:${port}/secrets`, {
+          method: "POST",
+          body: "{}",
+        })
+      ).rejects.toThrow(/globally routable public addresses/);
+      expect(hits).toBe(0);
+    });
+  });
+});

package/src/__tests__/report-issue.integration.test.ts

@@ -0,0 +1,147 @@
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+} from "vitest";
+import { initConfig } from "../config.js";
+import { createMcpServer } from "../create-mcp-server.js";
+import { handleReportIssue } from "../tools/report-issue.js";
+
+/**
+ * Integration tests for the report_issue tool against a REAL local HTTP
+ * server. Corresponds to specs/mcp-server/report-issue-tool.feature.
+ */
+describe("report_issue", () => {
+  let server: Server;
+  let endpoint: string;
+  let received: { headers: Record<string, unknown>; body: any }[];
+
+  beforeAll(async () => {
+    server = createServer((req, res) => {
+      let data = "";
+      req.on("data", (chunk: Buffer) => (data += chunk.toString()));
+      req.on("end", () => {
+        received.push({ headers: { ...req.headers }, body: JSON.parse(data) });
+        res.writeHead(201, { "content-type": "application/json" });
+        res.end(JSON.stringify({ id: "report-mcp-1" }));
+      });
+    });
+    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+    endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+  });
+
+  afterAll(async () => {
+    await new Promise((resolve) => server.close(resolve));
+  });
+
+  beforeEach(() => {
+    received = [];
+    initConfig({ endpoint, apiKey: undefined });
+  });
+
+  describe("given the registered tool set", () => {
+    /** @scenario "The report tool is listed with an agent-facing description" */
+    it("lists report_issue with the agent-facing consent description", () => {
+      const server = createMcpServer();
+      const tools = (
+        server as unknown as {
+          _registeredTools: Record<string, { description?: string }>;
+        }
+      )._registeredTools;
+      expect(Object.keys(tools)).toContain("report_issue");
+      const description = tools.report_issue?.description ?? "";
+      expect(description).toMatch(/whenever you struggled/i);
+      expect(description).toMatch(/ask the user for permission first/i);
+    });
+  });
+
+  describe("given the user did not approve sending", () => {
+    /** @scenario "Calls without user approval are rejected with instructions" */
+    it("refuses with instructions to ask the user first", async () => {
+      await expect(
+        handleReportIssue({
+          user_approved: false,
+          title: "broken tool",
+          summary: "search_traces 500s",
+        }),
+      ).rejects.toThrow(/ask the user/i);
+      expect(received).toHaveLength(0);
+    });
+  });
+
+  describe("given nothing to report", () => {
+    it("asks for a summary or session content", async () => {
+      await expect(
+        handleReportIssue({ user_approved: true, title: "empty" }),
+      ).rejects.toThrow(/Nothing to report/);
+    });
+  });
+
+  describe("when the user approved a summary report", () => {
+    /** @scenario "Reporting an issue through MCP reaches the LangWatch backend" */
+    it("delivers it marked as coming from MCP and returns a thank-you", async () => {
+      const text = await handleReportIssue({
+        user_approved: true,
+        title: "evaluator setup confusing",
+        summary: "the evaluator wizard 404s on step 2",
+        agent: "claude-code",
+      });
+
+      expect(received).toHaveLength(1);
+      expect(received[0]?.body.source).toBe("mcp");
+      expect(received[0]?.body.kind).toBe("summary");
+      expect(received[0]?.body.title).toBe("evaluator setup confusing");
+      expect(text).toContain("report-mcp-1");
+      expect(text.toLowerCase()).toContain("thank you");
+    });
+  });
+
+  describe("when the title itself includes a secret", () => {
+    it("redacts it before sending", async () => {
+      await handleReportIssue({
+        user_approved: true,
+        title: "X-Api-Key sk-proj-abcdefghijklmnopqrstuvwxyz123456 rejected",
+        summary: "the key was rejected",
+      });
+      expect(received[0]?.body.title).toBe("X-Api-Key [SECRET] rejected");
+    });
+  });
+
+  describe("when session content includes secrets", () => {
+    /** @scenario "Session content passed through MCP is redacted before sending" */
+    it("redacts them locally before sending", async () => {
+      await handleReportIssue({
+        user_approved: true,
+        title: "agent stuck",
+        session_content: JSON.stringify({
+          role: "user",
+          content: "my key sk-proj-abcdefghijklmnopqrstuvwxyz123456 failed",
+        }),
+      });
+
+      const wire = JSON.stringify(received[0]?.body);
+      expect(wire).not.toContain("sk-proj-abcdefghijklmnopqrstuvwxyz123456");
+      expect(received[0]?.body.sessionData).toContain("[SECRET]");
+      expect(received[0]?.body.kind).toBe("full_session");
+    });
+  });
+
+  describe("when an API key is configured", () => {
+    it("attaches it so the report links to the project", async () => {
+      initConfig({ endpoint, apiKey: "sk-lw-mcp-key-abcdef1234567890" });
+      await handleReportIssue({
+        user_approved: true,
+        title: "linked",
+        summary: "linked report",
+      });
+      expect(received[0]?.headers.authorization).toBe(
+        "Bearer sk-lw-mcp-key-abcdef1234567890",
+      );
+    });
+  });
+});

package/src/__tests__/tools.unit.test.ts

@@ -168,6 +168,30 @@ describe("handleSearchTraces()", () => {
     expect(result).toContain("timeout");
   });
 
+  describe("when a trace has evaluation results", () => {
+    it("shows pass/fail status per evaluation without requiring get_trace", async () => {
+      mockSearchTraces.mockResolvedValue({
+        traces: [
+          {
+            trace_id: "trace-eval",
+            input: { value: "" },
+            output: { value: "" },
+            evaluations: [
+              { name: "faithfulness", passed: true, score: 0.92 },
+              { evaluator_id: "toxicity-check", passed: false, label: "flagged" },
+            ],
+          },
+        ],
+        pagination: { totalHits: 1 },
+      });
+
+      const result = await handleSearchTraces({});
+
+      expect(result).toContain("**faithfulness**: PASSED (score: 0.92)");
+      expect(result).toContain("**toxicity-check**: FAILED [flagged]");
+    });
+  });
+
   describe("when no traces are found", () => {
     it("returns a no-results message", async () => {
       mockSearchTraces.mockResolvedValue({ traces: [] });

package/src/create-mcp-server.ts

@@ -1,8 +1,9 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
 
-import packageJson from "../package.json" assert { type: "json" };
+import packageJson from "../package.json" with { type: "json" };
 import { requireApiKey } from "./config.js";
+import { fetchDocumentation } from "./documentation-fetch.js";
 import {
   createDatasetSchema,
   datasetColumnDefinitionSchema,
@@ -70,20 +71,46 @@ function registerTools(server: McpServer): void {
         ),
     },
     withToolLogging("fetch_langwatch_docs", async ({ url }) => {
-      let urlToFetch = url || "https://langwatch.ai/docs/llms.txt";
-      if (url && !urlToFetch.endsWith(".md") && !urlToFetch.endsWith(".txt")) {
-        urlToFetch += ".md";
-      }
-      if (!urlToFetch.startsWith("http")) {
-        if (!urlToFetch.startsWith("/")) {
-          urlToFetch = "/" + urlToFetch;
-        }
-        urlToFetch = "https://langwatch.ai/docs" + urlToFetch;
-      }
-      const response = await fetch(urlToFetch);
+      const text = await fetchDocumentation("langwatch", url);
+      return { content: [{ type: "text", text }] };
+    })
+  );
 
+  server.tool(
+    "report_issue",
+    "Report an issue to the LangWatch team: anything that did not work, confused you, or took trial and error while using LangWatch (docs, MCP tools, CLI, SDKs, the platform). Use it whenever you struggled, it is how rough edges get fixed for every agent. Ask the user for permission first, then call with user_approved=true. Works without an API key. Put VERBATIM error messages in the summary; optionally attach raw session content. Secrets, API keys, emails and phone numbers are redacted locally before anything is sent.",
+    {
+      user_approved: z
+        .boolean()
+        .describe(
+          "Must be true, and only after the user explicitly agreed to send this report to LangWatch"
+        ),
+      title: z.string().max(300).describe("One-line description of the issue"),
+      summary: z
+        .string()
+        .optional()
+        .describe(
+          "What you were trying to do, what went wrong (verbatim errors), and what you had to figure out the hard way"
+        ),
+      session_content: z
+        .string()
+        .optional()
+        .describe(
+          "Optional raw session transcript or log excerpt (JSONL or plain text); redacted locally before sending"
+        ),
+      contact_email: z
+        .string()
+        .optional()
+        .describe("Optional contact email for follow-up"),
+      agent: z
+        .string()
+        .optional()
+        .describe("Which coding agent this is, e.g. claude-code, codex, cursor"),
+    },
+    withToolLogging("report_issue", async (params) => {
+      const { handleReportIssue } = await import("./tools/report-issue.js");
       return {
-        content: [{ type: "text", text: await response.text() }],
+        content: [{ type: "text", text: await handleReportIssue(params) }],
       };
     })
   );
@@ -100,21 +127,8 @@ function registerTools(server: McpServer): void {
         ),
     },
     withToolLogging("fetch_scenario_docs", async ({ url }) => {
-      let urlToFetch = url || "https://langwatch.ai/scenario/llms.txt";
-      if (url && !urlToFetch.endsWith(".md") && !urlToFetch.endsWith(".txt")) {
-        urlToFetch += ".md";
-      }
-      if (!urlToFetch.startsWith("http")) {
-        if (!urlToFetch.startsWith("/")) {
-          urlToFetch = "/" + urlToFetch;
-        }
-        urlToFetch = "https://langwatch.ai" + urlToFetch;
-      }
-      const response = await fetch(urlToFetch);
-
-      return {
-        content: [{ type: "text", text: await response.text() }],
-      };
+      const text = await fetchDocumentation("scenario", url);
+      return { content: [{ type: "text", text }] };
     })
   );
 
@@ -139,7 +153,7 @@ function registerTools(server: McpServer): void {
         .string()
         .optional()
         .describe(
-          "When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_judge') to get its full schema details"
+          "When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details"
         ),
     },
     async ({ category, evaluatorType }) => {