@langwatch/mcp-server 0.5.0 → 0.6.0

package/src/index.ts

@@ -94,22 +94,32 @@ server.tool(
 
 server.tool(
   "discover_schema",
-  "Discover available filter fields, metrics, aggregation types, group-by options, and scenario schema for LangWatch queries. Call this before using search_traces, get_analytics, or scenario tools to understand available options.",
+  "Discover available filter fields, metrics, aggregation types, group-by options, scenario schema, and evaluator types for LangWatch queries. Call this before using search_traces, get_analytics, scenario tools, or evaluator tools to understand available options.",
   {
     category: z
-      .enum(["filters", "metrics", "aggregations", "groups", "scenarios", "all"])
+      .enum(["filters", "metrics", "aggregations", "groups", "scenarios", "evaluators", "all"])
       .describe("Which schema category to discover"),
+    evaluatorType: z
+      .string()
+      .optional()
+      .describe("When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_judge') to get its full schema details"),
   },
-  async ({ category }) => {
+  async ({ category, evaluatorType }) => {
     if (category === "scenarios") {
       const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.js");
       return { content: [{ type: "text", text: formatScenarioSchema() }] };
     }
+    if (category === "evaluators") {
+      const { formatEvaluatorSchema } = await import("./tools/discover-evaluator-schema.js");
+      return { content: [{ type: "text", text: formatEvaluatorSchema(evaluatorType) }] };
+    }
     const { formatSchema } = await import("./tools/discover-schema.js");
     let text = formatSchema(category);
     if (category === "all") {
       const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.js");
       text += "\n\n" + formatScenarioSchema();
+      const { formatEvaluatorSchema } = await import("./tools/discover-evaluator-schema.js");
+      text += "\n\n" + formatEvaluatorSchema();
     }
     return { content: [{ type: "text", text }] };
   }
@@ -225,9 +235,59 @@ server.tool(
   }
 );
 
+// --- Platform Prompt Tools (require API key) ---
+// These tools manage prompts on the LangWatch platform via API.
+// For code-based prompt management, see `fetch_langwatch_docs` for the CLI/SDK approach.
+
+const modelSchema = z
+  .string()
+  .describe(
+    'Model in "provider/model-name" format, e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929"'
+  );
+
+server.tool(
+  "platform_create_prompt",
+  `Create a new prompt on the LangWatch platform.
+
+NOTE: Prompts can be managed two ways. Determine which approach the user needs:
+
+1. Code-based (CLI/SDK): If the user wants to manage prompts in their codebase, use \`fetch_langwatch_docs\` to learn about the prompt management CLI/SDK. This lets them version-control prompts and pull them into code.
+
+2. Platform-based (LangWatch UI): If the user wants to manage prompts directly on the LangWatch platform, use the \`platform_\` MCP tools (\`platform_create_prompt\`, \`platform_update_prompt\`, etc.).
+`,
+  {
+    name: z.string().describe("Prompt display name"),
+    handle: z
+      .string()
+      .optional()
+      .describe(
+        "URL-friendly handle (auto-generated from name if omitted)"
+      ),
+    messages: z
+      .array(
+        z.object({
+          role: z
+            .enum(["system", "user", "assistant"])
+            .describe("Message role"),
+          content: z.string().describe("Message content"),
+        })
+      )
+      .describe("Prompt messages"),
+    model: modelSchema,
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleCreatePrompt } = await import("./tools/create-prompt.js");
+    return {
+      content: [{ type: "text", text: await handleCreatePrompt(params) }],
+    };
+  }
+);
+
 server.tool(
-  "list_prompts",
-  "List all prompts configured in the LangWatch project.",
+  "platform_list_prompts",
+  "List all prompts configured on the LangWatch platform.",
   {},
   async () => {
     const { requireApiKey } = await import("./config.js");
@@ -240,8 +300,8 @@ server.tool(
 );
 
 server.tool(
-  "get_prompt",
-  "Get a specific prompt by ID or handle, including messages, model config, and version history.",
+  "platform_get_prompt",
+  "Get a specific prompt from the LangWatch platform by ID or handle, including messages, model config, and version history.",
   {
     idOrHandle: z.string().describe("Prompt ID or handle"),
     version: z
@@ -260,84 +320,77 @@ server.tool(
 );
 
 server.tool(
-  "create_prompt",
-  "Create a new prompt in the LangWatch project.",
+  "platform_update_prompt",
+  "Update an existing prompt on the LangWatch platform. Every update creates a new version.",
   {
-    name: z.string().describe("Prompt name"),
-    handle: z
-      .string()
-      .optional()
-      .describe("URL-friendly handle (auto-generated if omitted)"),
+    idOrHandle: z.string().describe("Prompt ID or handle to update"),
     messages: z
       .array(
         z.object({
-          role: z
-            .enum(["system", "user", "assistant"])
-            .describe("Message role"),
-          content: z.string().describe("Message content"),
+          role: z.enum(["system", "user", "assistant"]),
+          content: z.string(),
         })
       )
-      .describe("Prompt messages"),
-    model: z
-      .string()
-      .describe('Model name, e.g., "gpt-4o", "claude-sonnet-4-5-20250929"'),
-    modelProvider: z
+      .optional()
+      .describe("Updated messages"),
+    model: modelSchema.optional(),
+    commitMessage: z
       .string()
-      .describe('Provider name, e.g., "openai", "anthropic"'),
-    description: z.string().optional().describe("Prompt description"),
+      .describe("Commit message describing the change"),
   },
   async (params) => {
     const { requireApiKey } = await import("./config.js");
     requireApiKey();
-    const { handleCreatePrompt } = await import("./tools/create-prompt.js");
+    const { handleUpdatePrompt } = await import("./tools/update-prompt.js");
     return {
-      content: [{ type: "text", text: await handleCreatePrompt(params) }],
+      content: [{ type: "text", text: await handleUpdatePrompt(params) }],
     };
   }
 );
 
+// --- Platform Scenario Tools (require API key) ---
+// These tools manage scenarios on the LangWatch platform via API.
+// For code-based scenario testing, see `fetch_scenario_docs` for the SDK approach.
+
 server.tool(
-  "update_prompt",
-  "Update an existing prompt or create a new version.",
+  "platform_create_scenario",
+  `Create a new scenario on the LangWatch platform. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.
+
+NOTE: Scenarios can be created two ways. Determine which approach the user needs:
+
+1. Code-based (local testing): If the user has a codebase with an AI agent they want to test, use \`fetch_scenario_docs\` to learn about the Scenario Python/TypeScript SDK. This lets them run tests locally and iterate in code.
+
+2. Platform-based (LangWatch UI): If the user wants to manage scenarios directly on the LangWatch platform, use the \`platform_\` MCP tools (\`platform_create_scenario\`, \`platform_update_scenario\`, etc.).
+`,
   {
-    idOrHandle: z.string().describe("Prompt ID or handle to update"),
-    messages: z
-      .array(
-        z.object({
-          role: z.enum(["system", "user", "assistant"]),
-          content: z.string(),
-        })
-      )
-      .optional()
-      .describe("Updated messages"),
-    model: z.string().optional().describe("Updated model name"),
-    modelProvider: z.string().optional().describe("Updated provider"),
-    commitMessage: z
+    name: z.string().describe("Scenario name"),
+    situation: z
       .string()
+      .describe("The context or setup describing what the user/agent is doing"),
+    criteria: z
+      .array(z.string())
       .optional()
-      .describe("Commit message for the change"),
-    createVersion: z
-      .boolean()
+      .describe("Pass/fail conditions the agent's response must satisfy"),
+    labels: z
+      .array(z.string())
       .optional()
-      .describe(
-        "If true, creates a new version instead of updating in place"
-      ),
+      .describe("Tags for organizing and filtering scenarios"),
   },
   async (params) => {
     const { requireApiKey } = await import("./config.js");
     requireApiKey();
-    const { handleUpdatePrompt } = await import("./tools/update-prompt.js");
+    const { handleCreateScenario } = await import(
+      "./tools/create-scenario.js"
+    );
     return {
-      content: [{ type: "text", text: await handleUpdatePrompt(params) }],
+      content: [{ type: "text", text: await handleCreateScenario(params) }],
     };
   }
 );
 
-// --- Scenario Tools (require API key) ---
-
 server.tool(
-  "list_scenarios",
-  "List all scenarios in the LangWatch project. Returns AI-readable digest by default.",
+  "platform_list_scenarios",
+  "List all scenarios on the LangWatch platform. Returns AI-readable digest by default.",
   {
     format: z
       .enum(["digest", "json"])
@@ -357,8 +410,8 @@ server.tool(
 );
 
 server.tool(
-  "get_scenario",
-  "Get full details of a scenario by ID, including situation, criteria, and labels.",
+  "platform_get_scenario",
+  "Get full details of a scenario on the LangWatch platform by ID, including situation, criteria, and labels.",
   {
     scenarioId: z.string().describe("The scenario ID to retrieve"),
     format: z
@@ -379,76 +432,184 @@ server.tool(
 );
 
 server.tool(
-  "create_scenario",
-  "Create a new scenario in the LangWatch project. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.",
+  "platform_update_scenario",
+  "Update an existing scenario on the LangWatch platform.",
   {
-    name: z.string().describe("Scenario name"),
-    situation: z
-      .string()
-      .describe("The context or setup describing what the user/agent is doing"),
+    scenarioId: z.string().describe("The scenario ID to update"),
+    name: z.string().optional().describe("Updated scenario name"),
+    situation: z.string().optional().describe("Updated situation"),
     criteria: z
       .array(z.string())
       .optional()
-      .describe("Pass/fail conditions the agent's response must satisfy"),
+      .describe("Updated criteria"),
     labels: z
       .array(z.string())
       .optional()
-      .describe("Tags for organizing and filtering scenarios"),
+      .describe("Updated labels"),
   },
   async (params) => {
     const { requireApiKey } = await import("./config.js");
     requireApiKey();
-    const { handleCreateScenario } = await import(
-      "./tools/create-scenario.js"
+    const { handleUpdateScenario } = await import(
+      "./tools/update-scenario.js"
     );
     return {
-      content: [{ type: "text", text: await handleCreateScenario(params) }],
+      content: [{ type: "text", text: await handleUpdateScenario(params) }],
     };
   }
 );
 
 server.tool(
-  "update_scenario",
-  "Update an existing scenario.",
+  "platform_archive_scenario",
+  "Archive (soft-delete) a scenario on the LangWatch platform.",
   {
-    scenarioId: z.string().describe("The scenario ID to update"),
-    name: z.string().optional().describe("Updated scenario name"),
-    situation: z.string().optional().describe("Updated situation"),
-    criteria: z
-      .array(z.string())
-      .optional()
-      .describe("Updated criteria"),
-    labels: z
-      .array(z.string())
+    scenarioId: z.string().describe("The scenario ID to archive"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleArchiveScenario } = await import(
+      "./tools/archive-scenario.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleArchiveScenario(params) }],
+    };
+  }
+);
+
+// --- Platform Evaluator Tools (require API key) ---
+// These tools manage evaluators on the LangWatch platform via API.
+
+server.tool(
+  "platform_create_evaluator",
+  `Create an evaluator on the LangWatch platform. Useful for setting up LLM-as-judge and other evaluators to use in evaluation notebooks. Call discover_schema({ category: 'evaluators' }) first to see available evaluator types and their settings.`,
+  {
+    name: z.string().describe("Evaluator name"),
+    config: z
+      .record(z.string(), z.unknown())
+      .describe(
+        'Evaluator config object. Must include "evaluatorType" (e.g. "langevals/llm_boolean") and optional "settings" overrides.'
+      ),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleCreateEvaluator } = await import(
+      "./tools/create-evaluator.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleCreateEvaluator(params) }],
+    };
+  }
+);
+
+server.tool(
+  "platform_list_evaluators",
+  "List all evaluators configured on the LangWatch platform.",
+  {},
+  async () => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleListEvaluators } = await import(
+      "./tools/list-evaluators.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleListEvaluators() }],
+    };
+  }
+);
+
+server.tool(
+  "platform_get_evaluator",
+  "Get full details of an evaluator on the LangWatch platform by ID or slug, including config, input fields, and output fields.",
+  {
+    idOrSlug: z.string().describe("The evaluator ID or slug to retrieve"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleGetEvaluator } = await import("./tools/get-evaluator.js");
+    return {
+      content: [{ type: "text", text: await handleGetEvaluator(params) }],
+    };
+  }
+);
+
+server.tool(
+  "platform_update_evaluator",
+  "Update an existing evaluator on the LangWatch platform. The evaluatorType in config cannot be changed after creation.",
+  {
+    evaluatorId: z.string().describe("The evaluator ID to update"),
+    name: z.string().optional().describe("Updated evaluator name"),
+    config: z
+      .record(z.string(), z.unknown())
       .optional()
-      .describe("Updated labels"),
+      .describe(
+        "Updated config settings. Note: evaluatorType cannot be changed after creation."
+      ),
   },
   async (params) => {
     const { requireApiKey } = await import("./config.js");
     requireApiKey();
-    const { handleUpdateScenario } = await import(
-      "./tools/update-scenario.js"
+    const { handleUpdateEvaluator } = await import(
+      "./tools/update-evaluator.js"
     );
     return {
-      content: [{ type: "text", text: await handleUpdateScenario(params) }],
+      content: [{ type: "text", text: await handleUpdateEvaluator(params) }],
     };
   }
 );
 
+// --- Platform Model Provider Tools (require API key) ---
+// These tools manage model provider API keys on the LangWatch platform.
+
 server.tool(
-  "archive_scenario",
-  "Archive (soft-delete) a scenario.",
+  "platform_set_model_provider",
+  `Set or update a model provider on the LangWatch platform. Use this to configure API keys (e.g. OPENAI_API_KEY) needed to run evaluators. The API key is stored securely and never returned in responses. Omit customKeys to update other settings without changing existing keys.`,
   {
-    scenarioId: z.string().describe("The scenario ID to archive"),
+    provider: z
+      .string()
+      .describe(
+        'Provider name, e.g., "openai", "anthropic", "azure", "custom"'
+      ),
+    enabled: z.boolean().describe("Whether the provider is enabled"),
+    customKeys: z
+      .record(z.string(), z.unknown())
+      .optional()
+      .describe(
+        'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.'
+      ),
+    defaultModel: z
+      .string()
+      .optional()
+      .describe("Set as project default model"),
   },
   async (params) => {
     const { requireApiKey } = await import("./config.js");
     requireApiKey();
-    const { handleArchiveScenario } = await import(
-      "./tools/archive-scenario.js"
+    const { handleSetModelProvider } = await import(
+      "./tools/set-model-provider.js"
     );
     return {
-      content: [{ type: "text", text: await handleArchiveScenario(params) }],
+      content: [
+        { type: "text", text: await handleSetModelProvider(params) },
+      ],
+    };
+  }
+);
+
+server.tool(
+  "platform_list_model_providers",
+  "List all model providers configured on the LangWatch platform. API keys are masked in the response.",
+  {},
+  async () => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleListModelProviders } = await import(
+      "./tools/list-model-providers.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleListModelProviders() }],
     };
   }
 );

package/src/langwatch-api-evaluators.ts

@@ -0,0 +1,70 @@
+import { makeRequest } from "./langwatch-api.js";
+
+// --- Evaluator types ---
+
+export interface EvaluatorSummary {
+  id: string;
+  projectId: string;
+  name: string;
+  slug: string | null;
+  type: string;
+  config: Record<string, unknown> | null;
+  workflowId: string | null;
+  copiedFromEvaluatorId: string | null;
+  createdAt: string;
+  updatedAt: string;
+  fields: Array<{ identifier: string; type: string; optional?: boolean }>;
+  outputFields: Array<{ identifier: string; type: string; optional?: boolean }>;
+  workflowName?: string;
+  workflowIcon?: string;
+}
+
+// --- Helpers ---
+
+/**
+ * Extracts the evaluatorType from an evaluator's config.
+ * Centralises the cast so callers don't repeat it.
+ */
+export function getEvaluatorType(
+  evaluator: Pick<EvaluatorSummary, "config">,
+): string | undefined {
+  return (evaluator.config as Record<string, unknown> | null)
+    ?.evaluatorType as string | undefined;
+}
+
+// --- Evaluator API functions ---
+
+/** Lists all evaluators in the project. */
+export async function listEvaluators(): Promise<EvaluatorSummary[]> {
+  return makeRequest("GET", "/api/evaluators") as Promise<EvaluatorSummary[]>;
+}
+
+/** Retrieves a single evaluator by ID or slug. */
+export async function getEvaluator(idOrSlug: string): Promise<EvaluatorSummary> {
+  return makeRequest(
+    "GET",
+    `/api/evaluators/${encodeURIComponent(idOrSlug)}`,
+  ) as Promise<EvaluatorSummary>;
+}
+
+/** Creates a new evaluator. */
+export async function createEvaluator(data: {
+  name: string;
+  config: Record<string, unknown>;
+}): Promise<EvaluatorSummary> {
+  return makeRequest("POST", "/api/evaluators", data) as Promise<EvaluatorSummary>;
+}
+
+/** Updates an existing evaluator. */
+export async function updateEvaluator(params: {
+  id: string;
+  name?: string;
+  config?: Record<string, unknown>;
+}): Promise<EvaluatorSummary> {
+  const { id, ...data } = params;
+  return makeRequest(
+    "PUT",
+    `/api/evaluators/${encodeURIComponent(id)}`,
+    data,
+  ) as Promise<EvaluatorSummary>;
+}

package/src/langwatch-api-model-providers.ts

@@ -0,0 +1,41 @@
+import { makeRequest } from "./langwatch-api.js";
+
+// --- Model Provider types ---
+
+export interface ModelProviderEntry {
+  id?: string;
+  provider: string;
+  enabled: boolean;
+  customKeys: Record<string, unknown> | null;
+  models?: string[] | null;
+  embeddingsModels?: string[] | null;
+  customModels?: unknown[] | null;
+  customEmbeddingsModels?: unknown[] | null;
+  disabledByDefault?: boolean;
+  deploymentMapping?: unknown;
+  extraHeaders?: Array<{ key: string; value: string }> | null;
+}
+
+// --- Model Provider API functions ---
+
+/** Lists all model providers for the project, with masked API keys. */
+export async function listModelProviders(): Promise<Record<string, ModelProviderEntry>> {
+  return makeRequest("GET", "/api/model-providers") as Promise<
+    Record<string, ModelProviderEntry>
+  >;
+}
+
+/** Creates or updates a model provider. */
+export async function setModelProvider(params: {
+  provider: string;
+  enabled: boolean;
+  customKeys?: Record<string, unknown>;
+  defaultModel?: string;
+}): Promise<Record<string, ModelProviderEntry>> {
+  const { provider, ...data } = params;
+  return makeRequest(
+    "PUT",
+    `/api/model-providers/${encodeURIComponent(provider)}`,
+    data,
+  ) as Promise<Record<string, ModelProviderEntry>>;
+}

package/src/langwatch-api.ts

@@ -78,7 +78,6 @@ export interface PromptSummary {
   id?: string;
   handle?: string;
   name?: string;
-  description?: string | null;
   latestVersionNumber?: number;
   version?: number;
 }
@@ -87,14 +86,12 @@ export interface PromptVersion {
   version?: number;
   commitMessage?: string;
   model?: string;
-  modelProvider?: string;
   messages?: Array<{ role: string; content: string }>;
 }
 
 export interface PromptDetailResponse extends PromptSummary {
   versions?: PromptVersion[];
   model?: string;
-  modelProvider?: string;
   messages?: Array<{ role: string; content: string }>;
   prompt?: Array<{ role: string; content: string }>;
 }
@@ -216,12 +213,9 @@ export async function getPrompt(
 
 /** Creates a new prompt. */
 export async function createPrompt(data: {
-  name: string;
-  handle?: string;
+  handle: string;
   messages: Array<{ role: string; content: string }>;
   model: string;
-  modelProvider: string;
-  description?: string;
 }): Promise<PromptMutationResponse> {
   return makeRequest(
     "POST",
@@ -236,31 +230,12 @@ export async function updatePrompt(
   data: {
     messages?: Array<{ role: string; content: string }>;
     model?: string;
-    modelProvider?: string;
-    commitMessage?: string;
+    commitMessage: string;
   }
 ): Promise<PromptMutationResponse> {
   return makeRequest(
-    "POST",
+    "PUT",
     `/api/prompts/${encodeURIComponent(idOrHandle)}`,
     data
   ) as Promise<PromptMutationResponse>;
 }
-
-/** Creates a new version of an existing prompt. */
-export async function createPromptVersion(
-  idOrHandle: string,
-  data: {
-    messages?: Array<{ role: string; content: string }>;
-    model?: string;
-    modelProvider?: string;
-    commitMessage?: string;
-  }
-): Promise<PromptMutationResponse> {
-  return makeRequest(
-    "POST",
-    `/api/prompts/${encodeURIComponent(idOrHandle)}/versions`,
-    data
-  ) as Promise<PromptMutationResponse>;
-}
-

package/src/tools/archive-scenario.ts

@@ -1,7 +1,7 @@
 import { archiveScenario as apiArchiveScenario } from "../langwatch-api-scenarios.js";
 
 /**
- * Handles the archive_scenario MCP tool invocation.
+ * Handles the platform_archive_scenario MCP tool invocation.
  *
  * Archives (soft-deletes) a scenario and returns confirmation.
  */