@langwatch/mcp-server 0.7.0 → 0.9.0

package/src/config.ts

@@ -3,6 +3,7 @@ import { AsyncLocalStorage } from "node:async_hooks";
 export interface McpConfig {
   apiKey: string | undefined;
   endpoint: string;
+  projectId?: string;
 }
 
 // ---------------------------------------------------------------------------
@@ -56,6 +57,7 @@ export function initConfig(args: { apiKey?: string; endpoint?: string }): void {
       args.endpoint ||
       process.env.LANGWATCH_ENDPOINT ||
       "https://app.langwatch.ai",
+    projectId: process.env.LANGWATCH_PROJECT_ID,
   };
 }
 

package/src/create-mcp-server.ts

@@ -7,6 +7,10 @@ import {
   createDatasetSchema,
   datasetColumnDefinitionSchema,
 } from "./schemas/create-dataset.js";
+import { handleExperimentResults } from "./tools/get-experiment-results.js";
+import { handleExperimentListRuns } from "./tools/list-experiment-runs.js";
+import { handleExperimentList } from "./tools/list-experiments.js";
+import { handleRunExperiment, handleExperimentStatus } from "./tools/run-experiment.js";
 
 const modelSchema = z
   .string()
@@ -1541,34 +1545,124 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
     })
   );
 
-  // --- Platform Evaluation Execution Tools (require API key) ---
+  // --- Platform Experiment Execution Tools (require API key) ---
 
   server.tool(
-    "platform_run_evaluation",
-    "Start an evaluation run by slug. Returns a run ID for polling status.",
+    "platform_run_experiment",
+    "Start an experiment run by slug. Returns a run ID for polling status.",
     {
-      slug: z.string().describe("The evaluation slug to run"),
+      slug: z.string().describe("The experiment slug to run"),
     },
-    withToolLogging("platform_run_evaluation", async (params) => {
+    withToolLogging("platform_run_experiment", async (params) => {
       requireApiKey();
-      const { handleRunEvaluation } = await import("./tools/run-evaluation.js");
       return {
-        content: [{ type: "text", text: await handleRunEvaluation(params) }],
+        content: [{ type: "text", text: await handleRunExperiment(params) }],
       };
     })
   );
 
   server.tool(
-    "platform_evaluation_status",
-    "Check the status of an evaluation run. Returns progress and summary when completed.",
+    "platform_experiment_status",
+    "Check the status of an experiment run. Returns progress and summary when completed.",
     {
-      runId: z.string().describe("The run ID returned from platform_run_evaluation"),
+      runId: z.string().describe("The run ID returned from platform_run_experiment"),
     },
-    withToolLogging("platform_evaluation_status", async (params) => {
+    withToolLogging("platform_experiment_status", async (params) => {
       requireApiKey();
-      const { handleEvaluationStatus } = await import("./tools/run-evaluation.js");
       return {
-        content: [{ type: "text", text: await handleEvaluationStatus(params) }],
+        content: [{ type: "text", text: await handleExperimentStatus(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_experiment_list",
+    "List experiments configured in the LangWatch project. Returns a markdown table with each experiment's slug, name, last-run timestamp, and run count. Use this to discover experiment slugs before drilling into runs with platform_experiment_list_runs. Output is capped (default 25, max 100) to protect the agent's context window.",
+    {
+      limit: z
+        .number()
+        .int()
+        .positive()
+        .max(100)
+        .optional()
+        .describe(
+          "Maximum number of experiments to include (default 25, hard-capped at 100)",
+        ),
+    },
+    withToolLogging("platform_experiment_list", async (params) => {
+      requireApiKey();
+      return {
+        content: [{ type: "text", text: await handleExperimentList(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_experiment_list_runs",
+    "List experiment runs for a specific experiment slug. Returns a markdown table with each run's id, status, started/finished timestamps, and pass-rate summary. Use this to discover run ids before calling platform_experiment_results. Output is capped (default 25, max 100) to protect the agent's context window.",
+    {
+      experimentSlug: z
+        .string()
+        .describe(
+          "Experiment slug from platform_experiment_list (e.g. 'checkout-flow')",
+        ),
+      limit: z
+        .number()
+        .int()
+        .positive()
+        .max(100)
+        .optional()
+        .describe(
+          "Maximum number of runs to include (default 25, hard-capped at 100)",
+        ),
+    },
+    withToolLogging("platform_experiment_list_runs", async (params) => {
+      requireApiKey();
+      return {
+        content: [{ type: "text", text: await handleExperimentListRuns(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_experiment_results",
+    "Fetch per-row results for a completed experiment run so you can debug evaluator scores and missed rows. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window — narrow with `filter: 'failed'` or `evaluator` to see what matters.",
+    {
+      runId: z
+        .string()
+        .describe("The run ID returned from platform_run_experiment"),
+      experimentSlug: z
+        .string()
+        .optional()
+        .describe(
+          "Experiment slug — required when fetching results for runs older than 24h (Redis state may have expired). Use platform_experiment_list_runs to find the slug.",
+        ),
+      filter: z
+        .enum(["all", "failed"])
+        .optional()
+        .describe(
+          "Filter rows: 'failed' shows only rows that errored or failed an evaluation; 'all' (default) shows everything",
+        ),
+      evaluator: z
+        .string()
+        .optional()
+        .describe("Show only this evaluator's results (matched on `evaluator` field)"),
+      limit: z
+        .number()
+        .int()
+        .positive()
+        .max(50)
+        .optional()
+        .describe(
+          "Maximum number of rows to include in the markdown output (default 50, hard-capped at 50).",
+        ),
+    },
+    withToolLogging("platform_experiment_results", async (params) => {
+      requireApiKey();
+      return {
+        content: [
+          { type: "text", text: await handleExperimentResults(params) },
+        ],
       };
     })
   );
@@ -1796,4 +1890,154 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
       };
     }
   );
+
+  // --- Platform Project Tools ---
+  // These tools manage projects on the LangWatch platform via the REST API.
+  // They require an API key with org-level permissions.
+
+  server.tool(
+    "platform_list_projects",
+    "List all projects in your LangWatch organization. Requires an org-level API key.",
+    {
+      page: z.number().int().positive().optional().describe("Page number (default: 1)"),
+      limit: z.number().int().positive().max(1000).optional().describe("Results per page (default: 100)"),
+    },
+    withToolLogging("platform_list_projects", async (params) => {
+      requireApiKey();
+      const { handleListProjects } = await import("./tools/list-projects.js");
+      return {
+        content: [{ type: "text", text: await handleListProjects(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_get_project",
+    "Get details of a specific project by its ID.",
+    {
+      id: z.string().describe("The project ID"),
+    },
+    withToolLogging("platform_get_project", async (params) => {
+      requireApiKey();
+      const { handleGetProject } = await import("./tools/get-project.js");
+      return {
+        content: [{ type: "text", text: await handleGetProject(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_create_project",
+    `Create a new project in your LangWatch organization. Returns a one-time service API key for the project.
+
+You must provide either teamId (to add the project to an existing team) or newTeamName (to create a new team for the project).`,
+    {
+      name: z.string().describe("Project name"),
+      language: z.string().describe('Programming language (e.g. "python", "typescript")'),
+      framework: z.string().describe('Framework (e.g. "openai", "langchain", "custom")'),
+      teamId: z.string().optional().describe("ID of an existing team to add the project to"),
+      newTeamName: z.string().optional().describe("Name for a new team to create for this project"),
+    },
+    withToolLogging("platform_create_project", async (params) => {
+      requireApiKey();
+      const { handleCreateProject } = await import("./tools/create-project.js");
+      return {
+        content: [{ type: "text", text: await handleCreateProject(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_update_project",
+    "Update an existing project's name, language, framework, or PII redaction level.",
+    {
+      id: z.string().describe("The project ID to update"),
+      name: z.string().optional().describe("New project name"),
+      language: z.string().optional().describe("New programming language"),
+      framework: z.string().optional().describe("New framework"),
+      piiRedactionLevel: z.enum(["STRICT", "ESSENTIAL", "DISABLED"]).optional().describe("PII redaction level"),
+    },
+    withToolLogging("platform_update_project", async (params) => {
+      requireApiKey();
+      const { handleUpdateProject } = await import("./tools/update-project.js");
+      return {
+        content: [{ type: "text", text: await handleUpdateProject(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_archive_project",
+    "Archive (soft-delete) a project. The project can be restored later if needed.",
+    {
+      id: z.string().describe("The project ID to archive"),
+    },
+    withToolLogging("platform_archive_project", async (params) => {
+      requireApiKey();
+      const { handleArchiveProject } = await import("./tools/archive-project.js");
+      return {
+        content: [{ type: "text", text: await handleArchiveProject(params) }],
+      };
+    })
+  );
+
+  // --- Platform API Key Tools (require org-level API key) ---
+  // These tools manage API keys on the LangWatch platform via the REST API.
+
+  server.tool(
+    "platform_list_api_keys",
+    "List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
+    {},
+    withToolLogging("platform_list_api_keys", async () => {
+      requireApiKey();
+      const { handleListApiKeys } = await import("./tools/list-api-keys.js");
+      return {
+        content: [{ type: "text", text: await handleListApiKeys() }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_create_api_key",
+    `Create a new API key. Two types available:
+
+- **personal**: Tied to a user, requires explicit role bindings (e.g. ADMIN on PROJECT:proj_123).
+- **service**: Not tied to a user. Use projectIds for project-scoped service keys, or omit for org-wide access.
+
+The token is returned once and cannot be retrieved again.`,
+    {
+      keyType: z.enum(["personal", "service"]).describe("Key type: 'personal' (user-tied) or 'service' (headless)"),
+      name: z.string().describe("Key name (max 100 chars)"),
+      description: z.string().optional().describe("Key description (max 500 chars)"),
+      expiresAt: z.string().optional().describe("Expiration date (ISO string)"),
+      bindings: z.array(z.object({
+        role: z.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
+        scopeType: z.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
+        scopeId: z.string().describe("Scope ID (org, team, or project ID)"),
+      })).optional().describe("Role bindings (required for personal keys)"),
+      projectIds: z.array(z.string()).optional().describe("Project IDs for service keys (creates ADMIN bindings per project)"),
+    },
+    withToolLogging("platform_create_api_key", async (params) => {
+      requireApiKey();
+      const { handleCreateApiKey } = await import("./tools/create-api-key.js");
+      return {
+        content: [{ type: "text", text: await handleCreateApiKey(params) }],
+      };
+    })
+  );
+
+  server.tool(
+    "platform_revoke_api_key",
+    "Revoke an API key. Once revoked, the key can no longer be used for authentication.",
+    {
+      id: z.string().describe("The API key ID to revoke"),
+    },
+    withToolLogging("platform_revoke_api_key", async (params) => {
+      requireApiKey();
+      const { handleRevokeApiKey } = await import("./tools/revoke-api-key.js");
+      return {
+        content: [{ type: "text", text: await handleRevokeApiKey(params) }],
+      };
+    })
+  );
 }

package/src/langwatch-api-api-keys.ts

@@ -0,0 +1,54 @@
+import { makeRequest } from "./langwatch-api.js";
+
+export interface RoleBinding {
+  id: string;
+  role: string;
+  scopeType: string;
+  scopeId: string;
+}
+
+export interface ApiKeySummary {
+  id: string;
+  name: string;
+  description: string | null;
+  createdAt: string;
+  expiresAt: string | null;
+  lastUsedAt: string | null;
+  revokedAt: string | null;
+  roleBindings: RoleBinding[];
+}
+
+export interface ApiKeyCreateResponse {
+  token: string;
+  apiKey: {
+    id: string;
+    name: string;
+    createdAt: string;
+  };
+}
+
+export async function listApiKeys(): Promise<{ data: ApiKeySummary[] }> {
+  return makeRequest("GET", "/api/api-keys") as Promise<{ data: ApiKeySummary[] }>;
+}
+
+export async function createApiKey(data: {
+  keyType: "personal" | "service";
+  name: string;
+  description?: string;
+  expiresAt?: string;
+  bindings?: Array<{
+    role: "ADMIN" | "MEMBER" | "VIEWER";
+    scopeType: "ORGANIZATION" | "TEAM" | "PROJECT";
+    scopeId: string;
+  }>;
+  projectIds?: string[];
+}): Promise<ApiKeyCreateResponse> {
+  return makeRequest("POST", "/api/api-keys", data) as Promise<ApiKeyCreateResponse>;
+}
+
+export async function revokeApiKey(id: string): Promise<{ success: boolean }> {
+  return makeRequest(
+    "DELETE",
+    `/api/api-keys/${encodeURIComponent(id)}`,
+  ) as Promise<{ success: boolean }>;
+}

package/src/langwatch-api-projects.ts

@@ -0,0 +1,82 @@
+import { makeRequest } from "./langwatch-api.js";
+
+export interface ProjectSummary {
+  id: string;
+  name: string;
+  slug: string;
+  language: string;
+  framework: string;
+  teamId: string;
+  piiRedactionLevel: string;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface ProjectListResponse {
+  data: ProjectSummary[];
+  pagination: {
+    page: number;
+    limit: number;
+    total: number;
+    totalPages: number;
+  };
+}
+
+export interface ProjectCreateResponse extends ProjectSummary {
+  serviceApiKey: string;
+  serviceApiKeyId: string;
+}
+
+export async function listProjects(params?: {
+  page?: number;
+  limit?: number;
+}): Promise<ProjectListResponse> {
+  const query = new URLSearchParams();
+  if (params?.page) query.set("page", String(params.page));
+  if (params?.limit) query.set("limit", String(params.limit));
+  const qs = query.toString() ? `?${query}` : "";
+  return makeRequest("GET", `/api/projects${qs}`) as Promise<ProjectListResponse>;
+}
+
+export async function getProject(id: string): Promise<ProjectSummary> {
+  return makeRequest(
+    "GET",
+    `/api/projects/${encodeURIComponent(id)}`,
+  ) as Promise<ProjectSummary>;
+}
+
+export async function createProject(data: {
+  name: string;
+  language: string;
+  framework: string;
+  teamId?: string;
+  newTeamName?: string;
+}): Promise<ProjectCreateResponse> {
+  return makeRequest("POST", "/api/projects", data) as Promise<ProjectCreateResponse>;
+}
+
+export async function updateProject(params: {
+  id: string;
+  name?: string;
+  language?: string;
+  framework?: string;
+  piiRedactionLevel?: "STRICT" | "ESSENTIAL" | "DISABLED";
+}): Promise<ProjectSummary> {
+  const { id, ...data } = params;
+  return makeRequest(
+    "PATCH",
+    `/api/projects/${encodeURIComponent(id)}`,
+    data,
+  ) as Promise<ProjectSummary>;
+}
+
+export async function archiveProject(id: string): Promise<{
+  id: string;
+  name: string;
+  archivedAt: string;
+}> {
+  return makeRequest(
+    "DELETE",
+    `/api/projects/${encodeURIComponent(id)}`,
+  ) as Promise<{ id: string; name: string; archivedAt: string }>;
+}

package/src/langwatch-api.ts

@@ -105,23 +105,39 @@ export interface PromptMutationResponse {
 
 // --- HTTP client ---
 
+export class LangWatchApiError extends Error {
+  constructor(
+    message: string,
+    public readonly status: number,
+    public readonly responseBody: string,
+  ) {
+    super(message);
+    this.name = "LangWatchApiError";
+  }
+}
+
 /**
  * Sends an HTTP request to the LangWatch API.
  *
  * Builds the full URL from the configured endpoint, adds authentication,
  * and handles JSON serialization/deserialization.
  *
- * @throws Error with status code and response body when the response is not OK
+ * @throws LangWatchApiError with status code and response body when the
+ * response is not OK.
  */
 export async function makeRequest(
   method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
   path: string,
   body?: unknown
 ): Promise<unknown> {
-  const url = getConfig().endpoint + path;
+  const config = getConfig();
+  const url = config.endpoint + path;
   const headers: Record<string, string> = {
     "X-Auth-Token": requireApiKey(),
   };
+  if (config.projectId) {
+    headers["X-Project-Id"] = config.projectId;
+  }
 
   if (body !== undefined) {
     headers["Content-Type"] = "application/json";
@@ -135,8 +151,10 @@ export async function makeRequest(
 
   if (!response.ok) {
     const responseBody = await response.text();
-    throw new Error(
-      `LangWatch API error ${response.status}: ${responseBody}`
+    throw new LangWatchApiError(
+      `LangWatch API error ${response.status}: ${responseBody}`,
+      response.status,
+      responseBody,
     );
   }
 

package/src/tools/archive-project.ts

@@ -0,0 +1,14 @@
+import { archiveProject as apiArchiveProject } from "../langwatch-api-projects.js";
+
+export async function handleArchiveProject(params: {
+  id: string;
+}): Promise<string> {
+  const result = await apiArchiveProject(params.id);
+
+  return (
+    `Project archived successfully.\n\n` +
+    `**Name**: ${result.name}\n` +
+    `**ID**: ${result.id}\n` +
+    `**Archived At**: ${result.archivedAt}`
+  );
+}

package/src/tools/create-api-key.ts

@@ -0,0 +1,29 @@
+import { createApiKey as apiCreateApiKey } from "../langwatch-api-api-keys.js";
+
+export async function handleCreateApiKey(params: {
+  keyType: "personal" | "service";
+  name: string;
+  description?: string;
+  expiresAt?: string;
+  bindings?: Array<{
+    role: "ADMIN" | "MEMBER" | "VIEWER";
+    scopeType: "ORGANIZATION" | "TEAM" | "PROJECT";
+    scopeId: string;
+  }>;
+  projectIds?: string[];
+}): Promise<string> {
+  const result = await apiCreateApiKey(params);
+
+  const lines: string[] = [];
+  lines.push(`API key created successfully!\n`);
+  lines.push(`**Name**: ${result.apiKey.name}`);
+  lines.push(`**ID**: ${result.apiKey.id}`);
+  lines.push(`**Token**: \`${result.token}\``);
+  lines.push(`**Created**: ${result.apiKey.createdAt}`);
+  lines.push("");
+  lines.push(
+    "> ⚠️ Save this token now — it will not be shown again.",
+  );
+
+  return lines.join("\n");
+}

package/src/tools/create-project.ts

@@ -0,0 +1,45 @@
+import { createProject as apiCreateProject } from "../langwatch-api-projects.js";
+
+export async function handleCreateProject(params: {
+  name: string;
+  language: string;
+  framework: string;
+  teamId?: string;
+  newTeamName?: string;
+}): Promise<string> {
+  if (!params.teamId && !params.newTeamName) {
+    return "Error: You must provide either `teamId` (to add to an existing team) or `newTeamName` (to create a new team for this project).";
+  }
+
+  const result = await apiCreateProject({
+    name: params.name,
+    language: params.language,
+    framework: params.framework,
+    teamId: params.teamId,
+    newTeamName: params.newTeamName,
+  });
+
+  if (!result.serviceApiKey) {
+    throw new Error(
+      "API did not return a service API key. The project was created but the key is missing — check the LangWatch UI.",
+    );
+  }
+
+  const lines: string[] = [];
+  lines.push(`Project created successfully!\n`);
+  lines.push(`**Name**: ${result.name}`);
+  lines.push(`**ID**: ${result.id}`);
+  lines.push(`**Slug**: ${result.slug}`);
+  lines.push(`**Language**: ${result.language}`);
+  lines.push(`**Framework**: ${result.framework}`);
+  lines.push("");
+  lines.push(`**Service API Key**: \`${result.serviceApiKey}\``);
+  lines.push(`**Service API Key ID**: ${result.serviceApiKeyId}`);
+  lines.push("");
+  lines.push(
+    "> ⚠️ Save the service API key now — it will not be shown again. " +
+    "Use it as `LANGWATCH_API_KEY` to authenticate project-scoped operations.",
+  );
+
+  return lines.join("\n");
+}