@langwatch/mcp-server 2.0.0 → 2.2.0

package/CONTRIBUTING.md

@@ -13,18 +13,21 @@ Thank you for your interest in contributing to the LangWatch MCP Server! This gu
 ### Getting Started
 
 1. **Clone the repository and navigate to the MCP server directory:**
+
    ```bash
    git clone https://github.com/langwatch/langwatch.git
    cd langwatch/mcp/typescript
    ```
 
 2. **Install dependencies and build the MCP server:**
+
    ```bash
    pnpm install
    pnpm run build
    ```
 
 3. **Configure environment variables:**
+
    ```bash
    cp .env.example .env
    ```
@@ -43,12 +46,15 @@ Thank you for your interest in contributing to the LangWatch MCP Server! This gu
 This project follows the **[Agent Testing Pyramid](https://scenario.langwatch.ai/best-practices/the-agent-testing-pyramid/)** methodology, which provides a structured approach to testing AI agents across three layers:
 
 ### 1. Unit Tests (Foundation)
+
 Traditional software tests for deterministic components like API connections, data pipelines, and error handling.
 
 ### 2. Evals & Optimization (Middle Layer)
+
 Component-level evaluation and optimization of probabilistic AI components, including prompt effectiveness and retrieval accuracy.
 
 ### 3. Simulations (Peak)
+
 End-to-end testing that validates the complete agent behavior in realistic scenarios.
 
 ## Running Tests
@@ -63,6 +69,7 @@ code tests/evaluations.ipynb
 ```
 
 The notebook contains lightweight tests that directly test the MCP server with a "mocked" coding agent on single files. These are useful for:
+
 - Quick validation of MCP tool functionality
 - Testing individual instrumentation patterns
 - Rapid prototyping of new features
@@ -76,12 +83,14 @@ pnpm test
 ```
 
 This runs full simulation tests using the Scenario framework, which:
+
 - Launches actual Claude Code sessions
 - Uses the MCP server in a real development environment
 - Tests complete workflows on entire codebases
 - Validates that the agent can successfully instrument various AI frameworks (OpenAI, LangChain, DSPy, etc.)
 
 When tests run successfully, you'll see:
+
 - LangWatch Scenario interface opening
 - Terminal output showing Claude Code using MCP tools
 - Validation of code instrumentation at the end of each scenario
@@ -89,6 +98,7 @@ When tests run successfully, you'll see:
 ## Questions?
 
 If you encounter any issues or have questions about the setup, please:
+
 - Check existing GitHub issues
 - Create a new issue with detailed reproduction steps
 - Join our Discord community for real-time support

package/README.md

@@ -22,15 +22,15 @@ In stdio mode the API key is required for observability and prompt tools, and do
 
 ## Configuration
 
-| Env Var | CLI Arg | Description |
-|---------|---------|-------------|
-| `LANGWATCH_API_KEY` | `--apiKey` | API key for authentication |
-| `LANGWATCH_ENDPOINT` | `--endpoint` | API endpoint (default: `https://app.langwatch.ai`) |
-| | `--http` | Serve over HTTP and SSE instead of stdio |
-| | `--port` | HTTP port (default: `3000`) |
-| `LANGWATCH_MCP_HTTP_HOST` | `--host` | HTTP listen address (default: `127.0.0.1`) |
-| `LANGWATCH_MCP_ALLOWED_ORIGINS` | `--allowedOrigin` | Browser origins allowed to call the HTTP server |
-| `LANGWATCH_MCP_TRUST_PROXY` | | Use `X-Forwarded-For` as the rate limit key (default: off) |
+| Env Var                         | CLI Arg           | Description                                                |
+| ------------------------------- | ----------------- | ---------------------------------------------------------- |
+| `LANGWATCH_API_KEY`             | `--apiKey`        | API key for authentication                                 |
+| `LANGWATCH_ENDPOINT`            | `--endpoint`      | API endpoint (default: `https://app.langwatch.ai`)         |
+|                                 | `--http`          | Serve over HTTP and SSE instead of stdio                   |
+|                                 | `--port`          | HTTP port (default: `3000`)                                |
+| `LANGWATCH_MCP_HTTP_HOST`       | `--host`          | HTTP listen address (default: `127.0.0.1`)                 |
+| `LANGWATCH_MCP_ALLOWED_ORIGINS` | `--allowedOrigin` | Browser origins allowed to call the HTTP server            |
+| `LANGWATCH_MCP_TRUST_PROXY`     |                   | Use `X-Forwarded-For` as the rate limit key (default: off) |
 
 ### HTTP mode
 
@@ -85,67 +85,95 @@ gets in. A spoofable rate limit does not let anyone authenticate.
 
 ### Documentation
 
-| Tool | Description |
-|------|-------------|
-| `fetch_langwatch_docs` | Fetch LangWatch integration docs |
-| `fetch_scenario_docs` | Fetch Scenario agent testing docs |
+| Tool                   | Description                       |
+| ---------------------- | --------------------------------- |
+| `fetch_langwatch_docs` | Fetch LangWatch integration docs  |
+| `fetch_scenario_docs`  | Fetch Scenario agent testing docs |
 
 ### Observability (requires API key)
 
-| Tool | Description |
-|------|-------------|
+| Tool              | Description                                                  |
+| ----------------- | ------------------------------------------------------------ |
 | `discover_schema` | Explore available filters, metrics, aggregations, and groups |
-| `search_traces` | Search traces with filters, text query, and date range |
-| `get_trace` | Get full trace details with AI-readable formatting |
-| `get_analytics` | Query timeseries analytics data |
+| `search_traces`   | Search traces with filters, text query, and date range       |
+| `get_trace`       | Get full trace details with AI-readable formatting           |
+| `get_analytics`   | Query timeseries analytics data                              |
 
 ### Prompts (requires API key)
 
-| Tool | Description |
-|------|-------------|
-| `platform_list_prompts` | List all prompts |
-| `platform_get_prompt` | Get prompt with messages and version history |
-| `platform_create_prompt` | Create a new prompt |
-| `platform_update_prompt` | Update prompt or create new version |
+| Tool                     | Description                                  |
+| ------------------------ | -------------------------------------------- |
+| `platform_list_prompts`  | List all prompts                             |
+| `platform_get_prompt`    | Get prompt with messages and version history |
+| `platform_create_prompt` | Create a new prompt                          |
+| `platform_update_prompt` | Update prompt or create new version          |
 
 ### Datasets (requires API key)
 
-| Tool | Description |
-|------|-------------|
-| `platform_list_datasets` | List all datasets with record counts |
-| `platform_get_dataset` | Get dataset metadata, columns, and record preview |
-| `platform_create_dataset` | Create a new dataset with optional column definitions |
-| `platform_update_dataset` | Update dataset name or column types |
-| `platform_delete_dataset` | Archive a dataset |
-| `platform_create_dataset_records` | Add records to a dataset (max 1000 per call) |
-| `platform_update_dataset_record` | Update a single record |
-| `platform_delete_dataset_records` | Delete records by IDs (max 1000 per call) |
+| Tool                              | Description                                           |
+| --------------------------------- | ----------------------------------------------------- |
+| `platform_list_datasets`          | List all datasets with record counts                  |
+| `platform_get_dataset`            | Get dataset metadata, columns, and record preview     |
+| `platform_create_dataset`         | Create a new dataset with optional column definitions |
+| `platform_update_dataset`         | Update dataset name or column types                   |
+| `platform_delete_dataset`         | Archive a dataset                                     |
+| `platform_create_dataset_records` | Add records to a dataset (max 1000 per call)          |
+| `platform_update_dataset_record`  | Update a single record                                |
+| `platform_delete_dataset_records` | Delete records by IDs (max 1000 per call)             |
 
 ### Scenarios (requires API key)
 
-| Tool | Description |
-|------|-------------|
-| `platform_list_scenarios` | List all scenarios |
-| `platform_get_scenario` | Get scenario details |
-| `platform_create_scenario` | Create a new scenario |
-| `platform_update_scenario` | Update a scenario |
-| `platform_archive_scenario` | Archive a scenario |
+| Tool                        | Description                                                |
+| --------------------------- | ---------------------------------------------------------- |
+| `platform_list_scenarios`   | List all scenarios, or only the ones filed in a test suite |
+| `platform_get_scenario`     | Get scenario details                                       |
+| `platform_create_scenario`  | Create a new scenario, optionally filed in a test suite    |
+| `platform_update_scenario`  | Update a scenario, or file it in another test suite        |
+| `platform_archive_scenario` | Archive a scenario                                         |
+
+### Test suites and run plans (requires API key)
+
+A test suite is a folder of scenarios. A run plan is what you run, and its
+name identifies it: running a name that exists replaces that plan's
+configuration, running a new name creates the plan. Running a test suite is
+sugar over a run plan, and creates or joins the plan `<suite name> <target name>`.
+
+| Tool                          | Description                                        |
+| ----------------------------- | -------------------------------------------------- |
+| `platform_run_plan`           | Run scenarios against targets, by plan name        |
+| `platform_list_run_plans`     | List the run plans of the project                  |
+| `platform_get_run_plan`       | Get the full configuration of a run plan           |
+| `platform_rerun_run_plan`     | Run a plan again with the configuration it holds   |
+| `platform_archive_run_plan`   | Archive a run plan                                 |
+| `platform_list_test_suites`   | List the test suites of the project                |
+| `platform_create_test_suite`  | Create a test suite                                |
+| `platform_get_test_suite`     | Get a test suite and the scenarios filed in it     |
+| `platform_rename_test_suite`  | Rename a test suite                                |
+| `platform_archive_test_suite` | Archive a test suite and the scenarios filed in it |
+| `platform_run_test_suite`     | Run every scenario of a test suite against targets |
+
+### Simulation runs (requires API key)
+
+| Tool                            | Description                                   |
+| ------------------------------- | --------------------------------------------- |
+| `platform_list_simulation_runs` | List simulation run results                   |
+| `platform_get_simulation_run`   | Get one run with its conversation and verdict |
 
 ### Evaluators (requires API key)
 
-| Tool | Description |
-|------|-------------|
-| `platform_list_evaluators` | List all evaluators |
-| `platform_get_evaluator` | Get evaluator details |
+| Tool                        | Description            |
+| --------------------------- | ---------------------- |
+| `platform_list_evaluators`  | List all evaluators    |
+| `platform_get_evaluator`    | Get evaluator details  |
 | `platform_create_evaluator` | Create a new evaluator |
-| `platform_update_evaluator` | Update an evaluator |
+| `platform_update_evaluator` | Update an evaluator    |
 
 ### Model Providers (requires API key)
 
-| Tool | Description |
-|------|-------------|
+| Tool                            | Description                     |
+| ------------------------------- | ------------------------------- |
 | `platform_list_model_providers` | List configured model providers |
-| `platform_set_model_provider` | Configure a model provider |
+| `platform_set_model_provider`   | Configure a model provider      |
 
 ## Output Formats
 
@@ -189,7 +217,7 @@ pnpm build
 
 ```bash
 pnpm test        # Run all tests
-pnpm test:unit   # Unit tests only
+pnpm test   # Unit tests only
 ```
 
 ### Local testing
@@ -203,8 +231,10 @@ Build and point your MCP client to the local dist:
       "command": "node",
       "args": [
         "/path/to/mcp-server/dist/index.js",
-        "--apiKey", "your-api-key",
-        "--endpoint", "http://localhost:5560"
+        "--apiKey",
+        "your-api-key",
+        "--endpoint",
+        "http://localhost:5560"
       ]
     }
   }

package/dist/add-dashboard-widget-4ZC5UN73.js

@@ -0,0 +1,50 @@
+import {
+  assertDashboardExists,
+  assignDashboardWidgetToDashboard,
+  createDashboardWidget,
+  deleteDashboardWidget,
+  normalizeDashboardWidgetQueries
+} from "./chunk-5FNCQRCX.js";
+import "./chunk-K2I7Z3LZ.js";
+import {
+  getCurrentProject
+} from "./chunk-B4J3PJLX.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+import "./chunk-SAS5OB7F.js";
+
+// src/tools/add-dashboard-widget.ts
+async function handleAddDashboardWidget(params) {
+  const queries = normalizeDashboardWidgetQueries({ queries: params.queries });
+  const project = await getCurrentProject();
+  await assertDashboardExists({ projectId: project.id, dashboardId: params.dashboardId });
+  const widget = await createDashboardWidget({
+    projectId: project.id,
+    name: params.name,
+    code: params.code,
+    queries
+  });
+  try {
+    await assignDashboardWidgetToDashboard({
+      projectId: project.id,
+      widgetId: widget.id,
+      dashboardId: params.dashboardId
+    });
+  } catch (error) {
+    await deleteDashboardWidget({
+      projectId: project.id,
+      widgetId: widget.id
+    }).catch(() => {
+    });
+    throw error;
+  }
+  return [
+    `Widget "${params.name}" created and added to dashboard "${params.dashboardId}".`,
+    `**Widget ID**: ${widget.id}`,
+    `**Queries**: ${queries.map((q) => q.name).join(", ")}`
+  ].join("\n");
+}
+export {
+  handleAddDashboardWidget
+};
+//# sourceMappingURL=add-dashboard-widget-4ZC5UN73.js.map

package/dist/add-dashboard-widget-ACC6QYBG.js

@@ -0,0 +1,49 @@
+import {
+  assertDashboardExists,
+  assignDashboardWidgetToDashboard,
+  createDashboardWidget,
+  deleteDashboardWidget,
+  normalizeDashboardWidgetQueries
+} from "./chunk-5FNCQRCX.js";
+import "./chunk-K2I7Z3LZ.js";
+import {
+  getCurrentProject
+} from "./chunk-B4J3PJLX.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+
+// src/tools/add-dashboard-widget.ts
+async function handleAddDashboardWidget(params) {
+  const queries = normalizeDashboardWidgetQueries({ queries: params.queries });
+  const project = await getCurrentProject();
+  await assertDashboardExists({ projectId: project.id, dashboardId: params.dashboardId });
+  const widget = await createDashboardWidget({
+    projectId: project.id,
+    name: params.name,
+    code: params.code,
+    queries
+  });
+  try {
+    await assignDashboardWidgetToDashboard({
+      projectId: project.id,
+      widgetId: widget.id,
+      dashboardId: params.dashboardId
+    });
+  } catch (error) {
+    await deleteDashboardWidget({
+      projectId: project.id,
+      widgetId: widget.id
+    }).catch(() => {
+    });
+    throw error;
+  }
+  return [
+    `Widget "${params.name}" created and added to dashboard "${params.dashboardId}".`,
+    `**Widget ID**: ${widget.id}`,
+    `**Queries**: ${queries.map((q) => q.name).join(", ")}`
+  ].join("\n");
+}
+export {
+  handleAddDashboardWidget
+};
+//# sourceMappingURL=add-dashboard-widget-ACC6QYBG.js.map

package/dist/archive-project-KWFM3I5Z.js

@@ -0,0 +1,19 @@
+import {
+  archiveProject
+} from "./chunk-B4J3PJLX.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+
+// src/tools/archive-project.ts
+async function handleArchiveProject(params) {
+  const result = await archiveProject(params.id);
+  return `Project archived successfully.
+
+**Name**: ${result.name}
+**ID**: ${result.id}
+**Archived At**: ${result.archivedAt}`;
+}
+export {
+  handleArchiveProject
+};
+//# sourceMappingURL=archive-project-KWFM3I5Z.js.map

package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js}

@@ -1,8 +1,8 @@
 import {
   archiveProject
-} from "./chunk-7RHG56OH.js";
-import "./chunk-A2LEVYQ5.js";
-import "./chunk-WT4PO4A3.js";
+} from "./chunk-B4J3PJLX.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
 import "./chunk-SAS5OB7F.js";
 
 // src/tools/archive-project.ts
@@ -17,4 +17,4 @@ async function handleArchiveProject(params) {
 export {
   handleArchiveProject
 };
-//# sourceMappingURL=archive-project-4EQGLBJJ.js.map
+//# sourceMappingURL=archive-project-SPVHXK6A.js.map

package/dist/archive-run-plan-65QN3XRX.js

@@ -0,0 +1,16 @@
+import {
+  archiveRunPlan
+} from "./chunk-PTDR56Z3.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+import "./chunk-SAS5OB7F.js";
+
+// src/tools/archive-run-plan.ts
+async function handleArchiveRunPlan(params) {
+  const result = await archiveRunPlan(params.id);
+  return `Run plan ${result.id} is archived. Its past runs stay readable, and the name is free for a new plan.`;
+}
+export {
+  handleArchiveRunPlan
+};
+//# sourceMappingURL=archive-run-plan-65QN3XRX.js.map

package/dist/archive-run-plan-Z4SYE6BP.js

@@ -0,0 +1,15 @@
+import {
+  archiveRunPlan
+} from "./chunk-PTDR56Z3.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+
+// src/tools/archive-run-plan.ts
+async function handleArchiveRunPlan(params) {
+  const result = await archiveRunPlan(params.id);
+  return `Run plan ${result.id} is archived. Its past runs stay readable, and the name is free for a new plan.`;
+}
+export {
+  handleArchiveRunPlan
+};
+//# sourceMappingURL=archive-run-plan-Z4SYE6BP.js.map

package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js}

@@ -1,8 +1,8 @@
 import {
   archiveScenario
-} from "./chunk-J42FWRJ2.js";
-import "./chunk-A2LEVYQ5.js";
-import "./chunk-WT4PO4A3.js";
+} from "./chunk-I7ATIZED.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
 import "./chunk-SAS5OB7F.js";
 
 // src/tools/archive-scenario.ts
@@ -17,4 +17,4 @@ async function handleArchiveScenario(params) {
 export {
   handleArchiveScenario
 };
-//# sourceMappingURL=archive-scenario-W6BXYDZ3.js.map
+//# sourceMappingURL=archive-scenario-A6QV5LOW.js.map

package/dist/archive-scenario-FXEYQNEH.js

@@ -0,0 +1,19 @@
+import {
+  archiveScenario
+} from "./chunk-I7ATIZED.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+
+// src/tools/archive-scenario.ts
+async function handleArchiveScenario(params) {
+  const result = await archiveScenario(params.scenarioId);
+  const lines = [];
+  lines.push("Scenario archived successfully!\n");
+  lines.push(`**ID**: ${result.id}`);
+  lines.push(`**Status**: ${result.archived ? "archived" : "active"}`);
+  return lines.join("\n");
+}
+export {
+  handleArchiveScenario
+};
+//# sourceMappingURL=archive-scenario-FXEYQNEH.js.map

package/dist/archive-test-suite-5ADQ3KPQ.js

@@ -0,0 +1,15 @@
+import {
+  archiveTestSuite
+} from "./chunk-VMWJD732.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+
+// src/tools/archive-test-suite.ts
+async function handleArchiveTestSuite(params) {
+  const result = await archiveTestSuite(params.id);
+  return `Test suite ${result.id} is archived, and the scenarios filed in it are archived with it.`;
+}
+export {
+  handleArchiveTestSuite
+};
+//# sourceMappingURL=archive-test-suite-5ADQ3KPQ.js.map

package/dist/archive-test-suite-NKZDZY3D.js

@@ -0,0 +1,16 @@
+import {
+  archiveTestSuite
+} from "./chunk-VMWJD732.js";
+import "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+import "./chunk-SAS5OB7F.js";
+
+// src/tools/archive-test-suite.ts
+async function handleArchiveTestSuite(params) {
+  const result = await archiveTestSuite(params.id);
+  return `Test suite ${result.id} is archived, and the scenarios filed in it are archived with it.`;
+}
+export {
+  handleArchiveTestSuite
+};
+//# sourceMappingURL=archive-test-suite-NKZDZY3D.js.map

package/dist/assign-prompt-tag-5IUZXJ2T.js

@@ -0,0 +1,23 @@
+import {
+  assignPromptTag
+} from "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
+
+// src/tools/assign-prompt-tag.ts
+async function handleAssignPromptTag(params) {
+  await assignPromptTag({
+    idOrHandle: params.idOrHandle,
+    tag: params.tag,
+    versionId: params.versionId
+  });
+  const lines = [];
+  lines.push("Tag assigned successfully!\n");
+  lines.push(`**Prompt**: ${params.idOrHandle}`);
+  lines.push(`**Tag**: ${params.tag}`);
+  lines.push(`**Version ID**: ${params.versionId}`);
+  return lines.join("\n");
+}
+export {
+  handleAssignPromptTag
+};
+//# sourceMappingURL=assign-prompt-tag-5IUZXJ2T.js.map

package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js}

@@ -1,12 +1,16 @@
 import {
   assignPromptTag
-} from "./chunk-A2LEVYQ5.js";
-import "./chunk-WT4PO4A3.js";
+} from "./chunk-UWLE5T2F.js";
+import "./chunk-H6WGMG2T.js";
 import "./chunk-SAS5OB7F.js";
 
 // src/tools/assign-prompt-tag.ts
 async function handleAssignPromptTag(params) {
-  await assignPromptTag({ idOrHandle: params.idOrHandle, tag: params.tag, versionId: params.versionId });
+  await assignPromptTag({
+    idOrHandle: params.idOrHandle,
+    tag: params.tag,
+    versionId: params.versionId
+  });
   const lines = [];
   lines.push("Tag assigned successfully!\n");
   lines.push(`**Prompt**: ${params.idOrHandle}`);
@@ -17,4 +21,4 @@ async function handleAssignPromptTag(params) {
 export {
   handleAssignPromptTag
 };
-//# sourceMappingURL=assign-prompt-tag-VLXUWEKU.js.map
+//# sourceMappingURL=assign-prompt-tag-HBZLBIFD.js.map

package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js}

@@ -1,36 +1,33 @@
 import {
   makeRequest
-} from "./chunk-A2LEVYQ5.js";
+} from "./chunk-UWLE5T2F.js";
 
 // src/langwatch-api-evaluators.ts
 function getEvaluatorType(evaluator) {
   return evaluator.config?.evaluatorType;
 }
 async function listEvaluators() {
-  return makeRequest("GET", "/api/evaluators");
+  return makeRequest("GET", "/api/v1/evaluators");
 }
 async function getEvaluator(idOrSlug) {
   return makeRequest(
     "GET",
-    `/api/evaluators/${encodeURIComponent(idOrSlug)}`
+    `/api/v1/evaluators/${encodeURIComponent(idOrSlug)}`
   );
 }
 async function createEvaluator(data) {
-  return makeRequest("POST", "/api/evaluators", data);
+  return makeRequest("POST", "/api/v1/evaluators", data);
 }
 async function updateEvaluator(params) {
   const { id, ...data } = params;
   return makeRequest(
     "PUT",
-    `/api/evaluators/${encodeURIComponent(id)}`,
+    `/api/v1/evaluators/${encodeURIComponent(id)}`,
     data
   );
 }
 async function deleteEvaluator(idOrSlug) {
-  return makeRequest(
-    "DELETE",
-    `/api/evaluators/${encodeURIComponent(idOrSlug)}`
-  );
+  return makeRequest("DELETE", `/api/v1/evaluators/${encodeURIComponent(idOrSlug)}`);
 }
 
 export {
@@ -41,4 +38,4 @@ export {
   updateEvaluator,
   deleteEvaluator
 };
-//# sourceMappingURL=chunk-CDYQOY5U.js.map
+//# sourceMappingURL=chunk-3OCG3QPU.js.map