@langwatch/mcp-server 0.10.1 → 1.0.0

package/src/documentation-fetch.ts

@@ -0,0 +1,111 @@
+export type DocumentationKind = "langwatch" | "scenario";
+
+const TRUSTED_DOCUMENTATION_ORIGIN = "https://langwatch.ai";
+const MAX_DOCUMENTATION_BYTES = 2 * 1024 * 1024;
+const DOCUMENTATION_TIMEOUT_MS = 30_000;
+const DOCUMENTATION_CONTENT_TYPES = new Set(["text/markdown", "text/plain"]);
+
+const DOCUMENTATION_CONFIG: Record<DocumentationKind, { defaultPath: string; namespace: string }> = {
+  langwatch: {
+    defaultPath: "/docs/llms.txt",
+    namespace: "/docs",
+  },
+  scenario: {
+    defaultPath: "/scenario/llms.txt",
+    namespace: "/scenario",
+  },
+};
+
+function documentationUrlError(namespace: string): Error {
+  return new Error(
+    `Only a trusted LangWatch documentation URL under ${TRUSTED_DOCUMENTATION_ORIGIN}${namespace}/ may be fetched`
+  );
+}
+
+export function resolveDocumentationUrl(kind: DocumentationKind, input?: string): URL {
+  const { defaultPath, namespace } = DOCUMENTATION_CONFIG[kind];
+  const value = input?.trim();
+  let url: URL;
+
+  try {
+    if (!value) {
+      url = new URL(defaultPath, TRUSTED_DOCUMENTATION_ORIGIN);
+    } else if (/^[a-zA-Z][a-zA-Z\d+.-]*:/.test(value) || value.startsWith("//")) {
+      url = new URL(value, TRUSTED_DOCUMENTATION_ORIGIN);
+    } else {
+      const relativePath = value.replace(/^\/+/, "");
+      const namespacePath = namespace.slice(1);
+      const path =
+        relativePath === namespacePath || relativePath.startsWith(`${namespacePath}/`)
+          ? `/${relativePath}`
+          : `${namespace}/${relativePath}`;
+      url = new URL(path, TRUSTED_DOCUMENTATION_ORIGIN);
+    }
+  } catch {
+    throw documentationUrlError(namespace);
+  }
+
+  if (
+    url.protocol !== "https:" ||
+    url.hostname !== "langwatch.ai" ||
+    url.port !== "" ||
+    url.username !== "" ||
+    url.password !== "" ||
+    !url.pathname.startsWith(`${namespace}/`)
+  ) {
+    throw documentationUrlError(namespace);
+  }
+
+  const lowerPath = url.pathname.toLowerCase();
+  if (!lowerPath.endsWith(".md") && !lowerPath.endsWith(".txt")) {
+    url.pathname += ".md";
+  }
+
+  return url;
+}
+
+export async function fetchDocumentation(
+  kind: DocumentationKind,
+  input?: string,
+  fetchImplementation: typeof fetch = fetch
+): Promise<string> {
+  const url = resolveDocumentationUrl(kind, input);
+  const response = await fetchImplementation(url, {
+    redirect: "error",
+    signal: AbortSignal.timeout(DOCUMENTATION_TIMEOUT_MS),
+  });
+  if (!response.ok) {
+    throw new Error(`Documentation request failed with status ${response.status}`);
+  }
+
+  const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim() ?? "";
+  if (!DOCUMENTATION_CONTENT_TYPES.has(contentType)) {
+    throw new Error("Documentation response has an unexpected content type");
+  }
+
+  const contentLength = Number(response.headers.get("content-length"));
+  if (Number.isFinite(contentLength) && contentLength > MAX_DOCUMENTATION_BYTES) {
+    throw new Error("Documentation response is too large");
+  }
+
+  if (!response.body) {
+    return "";
+  }
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let receivedBytes = 0;
+  let text = "";
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) {
+      return text + decoder.decode();
+    }
+    receivedBytes += value.byteLength;
+    if (receivedBytes > MAX_DOCUMENTATION_BYTES) {
+      await reader.cancel();
+      throw new Error("Documentation response is too large");
+    }
+    text += decoder.decode(value, { stream: true });
+  }
+}

package/src/langwatch-api-agents.ts

@@ -1,4 +1,5 @@
 import { makeRequest } from "./langwatch-api.js";
+import { requestPublicJson } from "./public-http-request.js";
 
 export interface AgentSummary {
   id: string;
@@ -76,13 +77,12 @@ export async function runAgent(
       throw new Error("HTTP agent has no URL configured");
     }
 
-    const response = await fetch(url, {
+    const result = await requestPublicJson(url, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify(input ?? {}),
     });
 
-    const result = (await response.json()) as Record<string, unknown>;
     return { agentType: "http", result };
   }
 

package/src/langwatch-api.ts

@@ -1,4 +1,9 @@
+import type {
+  HandledErrorFault,
+  SerializedReason,
+} from "@langwatch/handled-error";
 import { getConfig, requireApiKey } from "./config.js";
+import type { EvaluationSummary } from "./utils/format-evaluations.js";
 
 // --- Response types ---
 
@@ -10,6 +15,7 @@ export interface TraceSearchResult {
   timestamps?: { started_at?: string | number };
   metadata?: Record<string, unknown>;
   error?: Record<string, unknown>;
+  evaluations?: EvaluationSummary[];
 }
 
 export interface SearchTracesResponse {
@@ -39,13 +45,7 @@ export interface TraceDetailResponse {
   };
   error?: Record<string, unknown>;
   ascii_tree?: string;
-  evaluations?: Array<{
-    evaluator_id?: string;
-    name?: string;
-    score?: number;
-    passed?: boolean;
-    label?: string;
-  }>;
+  evaluations?: EvaluationSummary[];
   spans?: Array<{
     span_id: string;
     name?: string;
@@ -106,14 +106,121 @@ export interface PromptMutationResponse {
 // --- HTTP client ---
 
 export class LangWatchApiError extends Error {
+  readonly code?: string;
+  readonly tips?: string[];
+  readonly docsUrl?: string;
+  readonly fault?: HandledErrorFault;
+  /**
+   * The per-field failures behind this error, verbatim from the envelope.
+   * A validation failure carries the offending field and the values it would
+   * have accepted here — the difference between an agent correcting its own
+   * request and an agent guessing again.
+   */
+  readonly reasons?: SerializedReason[];
+
   constructor(
     message: string,
     public readonly status: number,
     public readonly responseBody: string,
+    options: {
+      code?: string;
+      tips?: string[];
+      docsUrl?: string;
+      fault?: HandledErrorFault;
+      reasons?: SerializedReason[];
+    } = {},
   ) {
     super(message);
     this.name = "LangWatchApiError";
+    this.code = options.code;
+    this.tips = options.tips;
+    this.docsUrl = options.docsUrl;
+    this.fault = options.fault;
+    this.reasons = options.reasons;
+  }
+}
+
+/** Structured fields extracted from a handled-error JSON response body. */
+interface ParsedErrorBody {
+  code?: string;
+  message?: string;
+  tips?: string[];
+  docsUrl?: string;
+  fault?: HandledErrorFault;
+  reasons?: SerializedReason[];
+}
+
+const VALID_FAULTS: readonly HandledErrorFault[] = ["customer", "platform", "provider"];
+
+/**
+ * Parses an error response body as a handled-error envelope. Accepts both the
+ * REST shape (`{ error: "<code>", message, tips?, docsUrl?, fault? }`) and the
+ * serialized tRPC shape (`{ code, message?, tips?, docsUrl?, fault?, ... }`).
+ * Returns an empty object when the body is not a recognizable error envelope.
+ */
+function parseErrorBody(responseBody: string): ParsedErrorBody {
+  try {
+    const parsed: unknown = JSON.parse(responseBody);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      return {};
+    }
+    const body = parsed as Record<string, unknown>;
+    // Prefer `code` (the domain discriminant) over `error` — the
+    // packages/api unversioned envelope uses `error` for the HTTP status
+    // text ("Not Found") while `code` holds the real code.
+    const code =
+      typeof body.code === "string"
+        ? body.code
+        : typeof body.error === "string"
+          ? body.error
+          : undefined;
+    const message = typeof body.message === "string" ? body.message : undefined;
+    if (code === undefined && message === undefined) {
+      return {};
+    }
+    const tips =
+      Array.isArray(body.tips) && body.tips.every((t) => typeof t === "string")
+        ? (body.tips as string[])
+        : undefined;
+    const docsUrl = typeof body.docsUrl === "string" ? body.docsUrl : undefined;
+    const fault = VALID_FAULTS.includes(body.fault as HandledErrorFault)
+      ? (body.fault as HandledErrorFault)
+      : undefined;
+    const reasons =
+      Array.isArray(body.reasons) &&
+      body.reasons.every((r) => !!r && typeof r === "object")
+        ? (body.reasons as SerializedReason[])
+        : undefined;
+    return { code, message, tips, docsUrl, fault, reasons };
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * The human line for one per-field failure: the field, what it would have
+ * accepted, and what it got. Returns null when a reason names no field, so a
+ * generic nested error adds no noise to the message.
+ *
+ * This exists because the MCP transport's only channel to the caller is the
+ * error MESSAGE — an agent never sees the `reasons` array itself, so a
+ * rejection whose remedy lives only there is unfollowable.
+ */
+function describeReason(reason: SerializedReason): string | null {
+  const meta = reason.meta;
+  if (!meta || typeof meta !== "object") return null;
+
+  const field = typeof meta.field === "string" ? meta.field : null;
+  if (!field) return null;
+
+  const parts = [`- ${field}`];
+  if (meta.received !== undefined) {
+    parts.push(`received ${JSON.stringify(meta.received)}`);
+  }
+  if (Array.isArray(meta.expected) && meta.expected.length > 0) {
+    parts.push(`expected one of: ${meta.expected.join(", ")}`);
   }
+  return parts.join(" — ");
 }
 
 /**
@@ -151,10 +258,33 @@ export async function makeRequest(
 
   if (!response.ok) {
     const responseBody = await response.text();
+    const parsed = parseErrorBody(responseBody);
+    const lines = [
+      `LangWatch API error ${response.status}: ${parsed.message ?? responseBody}`,
+    ];
+    const reasonLines = (parsed.reasons ?? [])
+      .map(describeReason)
+      .filter((line): line is string => line !== null);
+    if (reasonLines.length > 0) {
+      lines.push("Rejected fields:", ...reasonLines);
+    }
+    if (parsed.tips && parsed.tips.length > 0) {
+      lines.push("Tips:", ...parsed.tips.map((tip) => `- ${tip}`));
+    }
+    if (parsed.docsUrl) {
+      lines.push(`Docs: ${parsed.docsUrl}`);
+    }
     throw new LangWatchApiError(
-      `LangWatch API error ${response.status}: ${responseBody}`,
+      lines.join("\n"),
       response.status,
       responseBody,
+      {
+        code: parsed.code,
+        tips: parsed.tips,
+        docsUrl: parsed.docsUrl,
+        fault: parsed.fault,
+        reasons: parsed.reasons,
+      },
     );
   }
 

package/src/public-http-request.ts

@@ -0,0 +1,242 @@
+import { lookup as dnsLookup } from "node:dns/promises";
+import { request as httpRequest, type RequestOptions } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { isIP } from "node:net";
+
+import { classify } from "@langwatch/ssrf";
+
+const MAX_REDIRECTS = 5;
+const MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
+const REQUEST_TIMEOUT_MS = 30_000;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const SENSITIVE_REDIRECT_HEADERS = new Set([
+  "authorization",
+  "cookie",
+  "cookie2",
+  "proxy-authorization",
+  "x-api-key",
+  "api-key",
+  "x-auth-token",
+  "x-access-token",
+]);
+
+type DnsResolver = (hostname: string, options: { all: true; verbatim: true }) => Promise<ResolvedAddress[]>;
+
+interface ResolvedAddress {
+  address: string;
+  family: number;
+}
+
+export interface PublicDestination {
+  address: string;
+  family: number;
+  url: URL;
+}
+
+function unsafeDestinationError(): Error {
+  return new Error("HTTP agent destinations must resolve only to globally routable public addresses");
+}
+
+function hostnameWithoutBrackets(hostname: string): string {
+  return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+}
+
+function assertPublicAddress(address: string): void {
+  if (classify(address) !== "global") {
+    throw unsafeDestinationError();
+  }
+}
+
+async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      promise,
+      new Promise<never>((_resolve, reject) => {
+        timeout = setTimeout(() => reject(new Error("request timed out")), timeoutMs);
+        timeout.unref();
+      }),
+    ]);
+  } finally {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+  }
+}
+
+export function headersForRedirect(
+  headers: Record<string, string> | undefined,
+  crossOrigin: boolean
+): Record<string, string> | undefined {
+  if (!headers || !crossOrigin) {
+    return headers;
+  }
+
+  const safeHeaders = Object.fromEntries(
+    Object.entries(headers).filter(([name]) => !SENSITIVE_REDIRECT_HEADERS.has(name.toLowerCase()))
+  );
+  return Object.keys(safeHeaders).length > 0 ? safeHeaders : undefined;
+}
+
+export async function resolvePublicDestination(
+  input: string,
+  resolver: DnsResolver = dnsLookup
+): Promise<PublicDestination> {
+  let url: URL;
+  try {
+    url = new URL(input);
+  } catch {
+    throw new Error("HTTP agent URL is invalid");
+  }
+
+  if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username !== "" || url.password !== "") {
+    throw unsafeDestinationError();
+  }
+
+  const hostname = hostnameWithoutBrackets(url.hostname);
+  const literalFamily = isIP(hostname);
+  if (literalFamily !== 0) {
+    assertPublicAddress(hostname);
+    return { address: hostname, family: literalFamily, url };
+  }
+
+  let addresses: ResolvedAddress[];
+  try {
+    addresses = await withTimeout(
+      resolver(hostname, { all: true, verbatim: true }),
+      REQUEST_TIMEOUT_MS
+    );
+  } catch {
+    throw new Error("HTTP agent destination could not be resolved");
+  }
+
+  if (addresses.length === 0) {
+    throw new Error("HTTP agent destination could not be resolved");
+  }
+
+  for (const { address } of addresses) {
+    assertPublicAddress(address);
+  }
+
+  const selected = addresses.find(({ family }) => family === 4) ?? addresses[0]!;
+  return {
+    address: selected.address,
+    family: selected.family,
+    url,
+  };
+}
+
+interface PublicJsonRequestOptions {
+  body?: string;
+  headers?: Record<string, string>;
+  method?: "GET" | "POST";
+  signal?: AbortSignal;
+}
+
+function requestBody(
+  destination: PublicDestination,
+  options: PublicJsonRequestOptions
+): Promise<{ body: string; location?: string; statusCode: number }> {
+  return new Promise((resolve, reject) => {
+    const hostname = hostnameWithoutBrackets(destination.url.hostname);
+    const requestOptions: RequestOptions = {
+      protocol: destination.url.protocol,
+      hostname,
+      port: destination.url.port || undefined,
+      path: `${destination.url.pathname}${destination.url.search}`,
+      method: options.method ?? "GET",
+      headers: options.headers,
+      family: destination.family,
+      signal: options.signal,
+      lookup: (_hostname, _options, callback) => {
+        callback(null, destination.address, destination.family);
+      },
+    };
+    const transport = destination.url.protocol === "https:" ? httpsRequest : httpRequest;
+    const request = transport(requestOptions, (response) => {
+      const chunks: Buffer[] = [];
+      let receivedBytes = 0;
+      response.once("aborted", () => {
+        reject(new Error("HTTP agent response was interrupted"));
+      });
+      response.once("error", reject);
+
+      const contentLength = Number(response.headers["content-length"]);
+      if (Number.isFinite(contentLength) && contentLength > MAX_RESPONSE_BYTES) {
+        response.destroy(new Error("HTTP agent response is too large"));
+        return;
+      }
+
+      response.on("data", (chunk: Buffer) => {
+        receivedBytes += chunk.length;
+        if (receivedBytes > MAX_RESPONSE_BYTES) {
+          response.destroy(new Error("HTTP agent response is too large"));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      response.once("end", () => {
+        resolve({
+          body: Buffer.concat(chunks).toString("utf8"),
+          location: response.headers.location,
+          statusCode: response.statusCode ?? 0,
+        });
+      });
+    });
+
+    request.once("error", reject);
+    request.setTimeout(REQUEST_TIMEOUT_MS, () => {
+      request.destroy(new Error("HTTP agent request timed out"));
+    });
+    if (options.body !== undefined) {
+      request.write(options.body);
+    }
+    request.end();
+  });
+}
+
+export async function requestPublicJson(
+  input: string,
+  options: PublicJsonRequestOptions = {},
+  redirectCount = 0
+): Promise<Record<string, unknown>> {
+  const destination = await resolvePublicDestination(input);
+  const response = await requestBody(destination, options);
+
+  if (REDIRECT_STATUSES.has(response.statusCode) && response.location) {
+    if (redirectCount >= MAX_REDIRECTS) {
+      throw new Error(`HTTP agent returned too many redirects`);
+    }
+    const redirectUrl = new URL(response.location, destination.url);
+    const redirectHeaders = headersForRedirect(
+      options.headers,
+      redirectUrl.origin !== destination.url.origin
+    );
+    const preserveBody = response.statusCode === 307 || response.statusCode === 308;
+    return requestPublicJson(
+      redirectUrl.toString(),
+      preserveBody
+        ? { ...options, headers: redirectHeaders }
+        : {
+            ...options,
+            body: undefined,
+            headers: redirectHeaders,
+            method: "GET",
+          },
+      redirectCount + 1
+    );
+  }
+
+  if (response.statusCode < 200 || response.statusCode >= 300) {
+    throw new Error(`HTTP agent returned an unsuccessful status: ${response.statusCode}`);
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(response.body);
+  } catch {
+    throw new Error("HTTP agent returned invalid JSON");
+  }
+
+  return parsed as Record<string, unknown>;
+}

package/src/tools/get-trace.ts

@@ -1,4 +1,5 @@
 import { getTraceById as apiGetTraceById } from "../langwatch-api.js";
+import { formatEvaluationLines } from "../utils/format-evaluations.js";
 
 /**
  * Handles the get_trace MCP tool invocation.
@@ -35,18 +36,7 @@ export async function handleGetTrace(params: {
   }
 
   if (result.evaluations && result.evaluations.length > 0) {
-    lines.push("\n## Evaluations");
-    for (const evaluation of result.evaluations) {
-      const status =
-        evaluation.passed === true
-          ? "PASSED"
-          : evaluation.passed === false
-            ? "FAILED"
-            : "N/A";
-      lines.push(
-        `- **${evaluation.name || evaluation.evaluator_id}**: ${status}${evaluation.score != null ? ` (score: ${evaluation.score})` : ""}${evaluation.label ? ` [${evaluation.label}]` : ""}`
-      );
-    }
+    lines.push("\n## Evaluations", ...formatEvaluationLines(result.evaluations));
   }
 
   if (result.formatted_trace) {