@langwatch/mcp-server 0.4.0 → 0.5.0

package/src/__tests__/scenario-tools.unit.test.ts

@@ -0,0 +1,185 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api-scenarios.js", () => ({
+  listScenarios: vi.fn(),
+  getScenario: vi.fn(),
+}));
+
+import {
+  listScenarios,
+  getScenario,
+} from "../langwatch-api-scenarios.js";
+
+import { handleListScenarios } from "../tools/list-scenarios.js";
+import { handleGetScenario } from "../tools/get-scenario.js";
+import { formatScenarioSchema } from "../tools/discover-scenario-schema.js";
+
+const mockListScenarios = vi.mocked(listScenarios);
+const mockGetScenario = vi.mocked(getScenario);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleListScenarios()", () => {
+  const sampleScenarios = [
+    {
+      id: "scen_abc123",
+      name: "Login Flow Happy Path",
+      situation:
+        "User attempts to log in with valid credentials and expects a welcome message back from the system",
+      criteria: [
+        "Responds with a welcome message",
+        "Includes user name in greeting",
+        "Sets session cookie",
+      ],
+      labels: ["auth", "happy-path"],
+    },
+    {
+      id: "scen_def456",
+      name: "Error Handling",
+      situation: "User sends malformed input",
+      criteria: ["Returns 400 status"],
+      labels: ["error"],
+    },
+  ];
+
+  describe("when scenarios exist (digest mode)", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListScenarios.mockResolvedValue(sampleScenarios);
+      result = await handleListScenarios({});
+    });
+
+    it("includes scenario id", () => {
+      expect(result).toContain("scen_abc123");
+    });
+
+    it("includes scenario name", () => {
+      expect(result).toContain("Login Flow Happy Path");
+    });
+
+    it("includes truncated situation preview", () => {
+      expect(result).toContain("User attempts to log in");
+      expect(result).not.toContain(
+        "User attempts to log in with valid credentials and expects a welcome message back from the system"
+      );
+    });
+
+    it("shows criteria count per scenario", () => {
+      expect(result).toContain("3 criteria");
+    });
+
+    it("includes labels", () => {
+      expect(result).toContain("auth");
+    });
+
+    it("includes all scenarios in the list", () => {
+      expect(result).toContain("scen_def456");
+    });
+
+    it("includes the total count header", () => {
+      expect(result).toContain("# Scenarios (2 total)");
+    });
+  });
+
+  describe("when no scenarios exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListScenarios.mockResolvedValue([]);
+      result = await handleListScenarios({});
+    });
+
+    it("returns a no-scenarios message", () => {
+      expect(result).toContain("No scenarios found");
+    });
+
+    it("includes a tip to use create_scenario", () => {
+      expect(result).toContain("create_scenario");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the scenario structure", async () => {
+      mockListScenarios.mockResolvedValue(sampleScenarios);
+      const result = await handleListScenarios({ format: "json" });
+      expect(JSON.parse(result)).toEqual(sampleScenarios);
+    });
+  });
+});
+
+describe("handleGetScenario()", () => {
+  const sampleScenario = {
+    id: "scen_abc123",
+    name: "Login Flow Happy Path",
+    situation: "User attempts to log in with valid credentials",
+    criteria: [
+      "Responds with a welcome message",
+      "Includes user name in greeting",
+    ],
+    labels: ["auth", "happy-path"],
+  };
+
+  describe("when format is digest", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockGetScenario.mockResolvedValue(sampleScenario);
+      result = await handleGetScenario({ scenarioId: "scen_abc123" });
+    });
+
+    it("includes the scenario name in the heading", () => {
+      expect(result).toContain("# Scenario: Login Flow Happy Path");
+    });
+
+    it("includes the situation", () => {
+      expect(result).toContain("User attempts to log in with valid credentials");
+    });
+
+    it("includes each criteria item", () => {
+      expect(result).toContain("- Responds with a welcome message");
+      expect(result).toContain("- Includes user name in greeting");
+    });
+
+    it("includes labels", () => {
+      expect(result).toContain("auth, happy-path");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the scenario structure", async () => {
+      mockGetScenario.mockResolvedValue(sampleScenario);
+      const result = await handleGetScenario({
+        scenarioId: "scen_abc123",
+        format: "json",
+      });
+      expect(JSON.parse(result)).toEqual(sampleScenario);
+    });
+  });
+});
+
+describe("formatScenarioSchema()", () => {
+  it("includes field descriptions with required/optional annotations", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("**name** (required)");
+    expect(result).toContain("**situation** (required)");
+    expect(result).toContain("**criteria** (array of strings)");
+    expect(result).toContain("**labels** (array of strings)");
+  });
+
+  it("includes all target types with descriptions", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("**prompt**: Test a prompt template");
+    expect(result).toContain("**http**: Test an HTTP endpoint");
+    expect(result).toContain("**code**: Test a code function");
+  });
+
+  it("includes authoring guidance for situations and criteria", () => {
+    const result = formatScenarioSchema();
+    expect(result).toContain("## Writing a Good Situation");
+    expect(result).toContain("## Writing Good Criteria");
+    expect(result).toContain("Specific and testable");
+  });
+});

package/src/index.ts

@@ -94,15 +94,24 @@ server.tool(
 
 server.tool(
   "discover_schema",
-  "Discover available filter fields, metrics, aggregation types, and group-by options for LangWatch queries. Call this before using search_traces or get_analytics to understand available options.",
+  "Discover available filter fields, metrics, aggregation types, group-by options, and scenario schema for LangWatch queries. Call this before using search_traces, get_analytics, or scenario tools to understand available options.",
   {
     category: z
-      .enum(["filters", "metrics", "aggregations", "groups", "all"])
+      .enum(["filters", "metrics", "aggregations", "groups", "scenarios", "all"])
       .describe("Which schema category to discover"),
   },
   async ({ category }) => {
+    if (category === "scenarios") {
+      const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.js");
+      return { content: [{ type: "text", text: formatScenarioSchema() }] };
+    }
     const { formatSchema } = await import("./tools/discover-schema.js");
-    return { content: [{ type: "text", text: formatSchema(category) }] };
+    let text = formatSchema(category);
+    if (category === "all") {
+      const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.js");
+      text += "\n\n" + formatScenarioSchema();
+    }
+    return { content: [{ type: "text", text }] };
   }
 );
 
@@ -324,4 +333,124 @@ server.tool(
   }
 );
 
+// --- Scenario Tools (require API key) ---
+
+server.tool(
+  "list_scenarios",
+  "List all scenarios in the LangWatch project. Returns AI-readable digest by default.",
+  {
+    format: z
+      .enum(["digest", "json"])
+      .optional()
+      .describe(
+        "Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
+      ),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleListScenarios } = await import("./tools/list-scenarios.js");
+    return {
+      content: [{ type: "text", text: await handleListScenarios(params) }],
+    };
+  }
+);
+
+server.tool(
+  "get_scenario",
+  "Get full details of a scenario by ID, including situation, criteria, and labels.",
+  {
+    scenarioId: z.string().describe("The scenario ID to retrieve"),
+    format: z
+      .enum(["digest", "json"])
+      .optional()
+      .describe(
+        "Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
+      ),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleGetScenario } = await import("./tools/get-scenario.js");
+    return {
+      content: [{ type: "text", text: await handleGetScenario(params) }],
+    };
+  }
+);
+
+server.tool(
+  "create_scenario",
+  "Create a new scenario in the LangWatch project. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.",
+  {
+    name: z.string().describe("Scenario name"),
+    situation: z
+      .string()
+      .describe("The context or setup describing what the user/agent is doing"),
+    criteria: z
+      .array(z.string())
+      .optional()
+      .describe("Pass/fail conditions the agent's response must satisfy"),
+    labels: z
+      .array(z.string())
+      .optional()
+      .describe("Tags for organizing and filtering scenarios"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleCreateScenario } = await import(
+      "./tools/create-scenario.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleCreateScenario(params) }],
+    };
+  }
+);
+
+server.tool(
+  "update_scenario",
+  "Update an existing scenario.",
+  {
+    scenarioId: z.string().describe("The scenario ID to update"),
+    name: z.string().optional().describe("Updated scenario name"),
+    situation: z.string().optional().describe("Updated situation"),
+    criteria: z
+      .array(z.string())
+      .optional()
+      .describe("Updated criteria"),
+    labels: z
+      .array(z.string())
+      .optional()
+      .describe("Updated labels"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleUpdateScenario } = await import(
+      "./tools/update-scenario.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleUpdateScenario(params) }],
+    };
+  }
+);
+
+server.tool(
+  "archive_scenario",
+  "Archive (soft-delete) a scenario.",
+  {
+    scenarioId: z.string().describe("The scenario ID to archive"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleArchiveScenario } = await import(
+      "./tools/archive-scenario.js"
+    );
+    return {
+      content: [{ type: "text", text: await handleArchiveScenario(params) }],
+    };
+  }
+);
+
 await server.connect(transport);

package/src/langwatch-api-scenarios.ts

@@ -0,0 +1,67 @@
+import { makeRequest } from "./langwatch-api.js";
+
+// --- Scenario types ---
+
+export interface ScenarioSummary {
+  id: string;
+  name: string;
+  situation: string;
+  criteria: string[];
+  labels: string[];
+}
+
+export interface ScenarioArchiveResponse {
+  id: string;
+  archived: boolean;
+}
+
+// --- Scenario API functions ---
+
+/** Lists all scenarios in the project. */
+export async function listScenarios(): Promise<ScenarioSummary[]> {
+  return makeRequest("GET", "/api/scenarios") as Promise<ScenarioSummary[]>;
+}
+
+/** Retrieves a single scenario by ID. */
+export async function getScenario(id: string): Promise<ScenarioSummary> {
+  return makeRequest(
+    "GET",
+    `/api/scenarios/${encodeURIComponent(id)}`
+  ) as Promise<ScenarioSummary>;
+}
+
+/** Creates a new scenario. */
+export async function createScenario(data: {
+  name: string;
+  situation: string;
+  criteria?: string[];
+  labels?: string[];
+}): Promise<ScenarioSummary> {
+  return makeRequest("POST", "/api/scenarios", data) as Promise<ScenarioSummary>;
+}
+
+/** Updates an existing scenario. */
+export async function updateScenario(params: {
+  id: string;
+  name?: string;
+  situation?: string;
+  criteria?: string[];
+  labels?: string[];
+}): Promise<ScenarioSummary> {
+  const { id, ...data } = params;
+  return makeRequest(
+    "PUT",
+    `/api/scenarios/${encodeURIComponent(id)}`,
+    data
+  ) as Promise<ScenarioSummary>;
+}
+
+/** Archives (soft-deletes) a scenario. */
+export async function archiveScenario(
+  id: string
+): Promise<ScenarioArchiveResponse> {
+  return makeRequest(
+    "DELETE",
+    `/api/scenarios/${encodeURIComponent(id)}`
+  ) as Promise<ScenarioArchiveResponse>;
+}

package/src/langwatch-api.ts

@@ -116,8 +116,8 @@ export interface PromptMutationResponse {
  *
  * @throws Error with status code and response body when the response is not OK
  */
-async function makeRequest(
-  method: "GET" | "POST",
+export async function makeRequest(
+  method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
   path: string,
   body?: unknown
 ): Promise<unknown> {
@@ -126,7 +126,7 @@ async function makeRequest(
     "X-Auth-Token": requireApiKey(),
   };
 
-  if (method === "POST") {
+  if (body !== undefined) {
     headers["Content-Type"] = "application/json";
   }
 
@@ -263,3 +263,4 @@ export async function createPromptVersion(
     data
   ) as Promise<PromptMutationResponse>;
 }
+

package/src/tools/archive-scenario.ts

@@ -0,0 +1,19 @@
+import { archiveScenario as apiArchiveScenario } from "../langwatch-api-scenarios.js";
+
+/**
+ * Handles the archive_scenario MCP tool invocation.
+ *
+ * Archives (soft-deletes) a scenario and returns confirmation.
+ */
+export async function handleArchiveScenario(params: {
+  scenarioId: string;
+}): Promise<string> {
+  const result = await apiArchiveScenario(params.scenarioId);
+
+  const lines: string[] = [];
+  lines.push("Scenario archived successfully!\n");
+  lines.push(`**ID**: ${result.id}`);
+  lines.push(`**Status**: ${result.archived ? "archived" : "active"}`);
+
+  return lines.join("\n");
+}

package/src/tools/create-scenario.ts

@@ -0,0 +1,30 @@
+import { createScenario as apiCreateScenario } from "../langwatch-api-scenarios.js";
+
+/**
+ * Handles the create_scenario MCP tool invocation.
+ *
+ * Creates a new scenario in the LangWatch project and returns a
+ * confirmation with the created scenario's details.
+ */
+export async function handleCreateScenario(params: {
+  name: string;
+  situation: string;
+  criteria?: string[];
+  labels?: string[];
+}): Promise<string> {
+  const result = await apiCreateScenario(params);
+
+  const lines: string[] = [];
+  lines.push("Scenario created successfully!\n");
+  lines.push(`**ID**: ${result.id}`);
+  lines.push(`**Name**: ${result.name}`);
+  lines.push(`**Situation**: ${result.situation}`);
+  if (Array.isArray(result.criteria) && result.criteria.length > 0) {
+    lines.push(`**Criteria**: ${result.criteria.length} criteria`);
+  }
+  if (Array.isArray(result.labels) && result.labels.length > 0) {
+    lines.push(`**Labels**: ${result.labels.join(", ")}`);
+  }
+
+  return lines.join("\n");
+}

package/src/tools/discover-scenario-schema.ts

@@ -0,0 +1,71 @@
+/**
+ * Returns a human-readable description of the scenario schema,
+ * including field descriptions, authoring guidance, and examples.
+ */
+export function formatScenarioSchema(): string {
+  const lines: string[] = [];
+
+  lines.push("# Scenario Schema\n");
+
+  lines.push("## Fields\n");
+  lines.push(
+    '- **name** (required): A short, descriptive name (e.g., "billing dispute resolution", "password reset with 2FA unavailable")',
+  );
+  lines.push(
+    "- **situation** (required): The context that guides the user simulator — who the user is, what they want, and any constraints (see Writing a Good Situation below)",
+  );
+  lines.push(
+    "- **criteria** (array of strings): Pass/fail conditions a judge evaluates the agent against (see Writing Good Criteria below)",
+  );
+  lines.push(
+    '- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")',
+  );
+
+  lines.push("\n## Writing a Good Situation\n");
+  lines.push(
+    "The situation drives the user simulator. Include these elements:",
+  );
+  lines.push("- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)");
+  lines.push("- **Emotional state**: How are they feeling? (e.g., frustrated, anxious, impatient)");
+  lines.push("- **Background/Context**: What happened before this conversation?");
+  lines.push("- **Intent**: What do they want to accomplish?");
+  lines.push("- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)");
+  lines.push("\nExample:");
+  lines.push("```");
+  lines.push("User is a small business owner stressed about tax deadline.");
+  lines.push("They need help categorizing expenses but aren't familiar with");
+  lines.push("accounting terms. They appreciate patient explanations and examples.");
+  lines.push("They have a spreadsheet of transactions but aren't sure which");
+  lines.push("categories apply to their consulting business.");
+  lines.push("```");
+
+  lines.push("\n## Writing Good Criteria\n");
+  lines.push("Criteria are what the judge uses to pass or fail the agent. Each criterion should be:");
+  lines.push("- **Specific and testable** — not vague like \"responds helpfully\"");
+  lines.push("- **Behavioral** — describes what the agent should *do*, not how it works internally");
+  lines.push("- **Independent** — each criterion checks one thing");
+  lines.push("\nGood criteria patterns:");
+  lines.push("- **Information gathering**: \"Agent asks for the user's account number before proceeding\"");
+  lines.push("- **Safety/guardrails**: \"Agent does not reveal internal system details or error stack traces\"");
+  lines.push("- **Clarification**: \"Agent asks clarifying questions before taking irreversible action\"");
+  lines.push("- **Tone**: \"Agent maintains a professional and empathetic tone throughout\"");
+  lines.push("- **Completeness**: \"Agent confirms the user understands the solution before ending\"");
+  lines.push("- **Domain-specific**: \"Agent recommends releasing a wild frog rather than keeping it as a pet\"");
+  lines.push("\nAvoid vague criteria like:");
+  lines.push('- "Responds correctly" — correct how?');
+  lines.push('- "Is helpful" — helpful in what way?');
+  lines.push('- "Works well" — not testable');
+
+  lines.push("\n## Target Types\n");
+  lines.push("Scenarios can target different execution backends:");
+  lines.push("- **prompt**: Test a prompt template with variable substitution");
+  lines.push("- **http**: Test an HTTP endpoint (e.g., a deployed agent API)");
+  lines.push("- **code**: Test a code function directly");
+
+  lines.push("\n## Tips\n");
+  lines.push("- Start simple, then layer complexity (add constraints, edge cases)");
+  lines.push("- Test edge cases: user changes their mind, gives ambiguous input, makes mistakes");
+  lines.push("- Use `fetch_scenario_docs` for the full authoring guide and advanced patterns");
+
+  return lines.join("\n");
+}

package/src/tools/get-scenario.ts

@@ -0,0 +1,36 @@
+import { getScenario as apiGetScenario } from "../langwatch-api-scenarios.js";
+
+/**
+ * Handles the get_scenario MCP tool invocation.
+ *
+ * Retrieves a specific scenario by ID and formats it as
+ * AI-readable markdown or raw JSON.
+ */
+export async function handleGetScenario(params: {
+  scenarioId: string;
+  format?: "digest" | "json";
+}): Promise<string> {
+  const scenario = await apiGetScenario(params.scenarioId);
+
+  if (params.format === "json") {
+    return JSON.stringify(scenario, null, 2);
+  }
+
+  const lines: string[] = [];
+  lines.push(`# Scenario: ${scenario.name}\n`);
+  lines.push(`**ID**: ${scenario.id}`);
+  lines.push(`**Situation**: ${scenario.situation}`);
+
+  if (Array.isArray(scenario.criteria) && scenario.criteria.length > 0) {
+    lines.push("\n## Criteria");
+    for (const criterion of scenario.criteria) {
+      lines.push(`- ${criterion}`);
+    }
+  }
+
+  if (Array.isArray(scenario.labels) && scenario.labels.length > 0) {
+    lines.push(`\n**Labels**: ${scenario.labels.join(", ")}`);
+  }
+
+  return lines.join("\n");
+}

package/src/tools/list-scenarios.ts

@@ -0,0 +1,47 @@
+import { listScenarios as apiListScenarios } from "../langwatch-api-scenarios.js";
+
+/**
+ * Handles the list_scenarios MCP tool invocation.
+ *
+ * Lists all scenarios in the LangWatch project, formatted as an
+ * AI-readable digest or raw JSON.
+ */
+export async function handleListScenarios(params: {
+  format?: "digest" | "json";
+}): Promise<string> {
+  const scenarios = await apiListScenarios();
+
+  if (params.format === "json") {
+    return JSON.stringify(scenarios, null, 2);
+  }
+
+  if (!Array.isArray(scenarios) || scenarios.length === 0) {
+    return "No scenarios found in this project.\n\n> Tip: Use `create_scenario` to create your first scenario.";
+  }
+
+  const lines: string[] = [];
+  lines.push(`# Scenarios (${scenarios.length} total)\n`);
+
+  for (const s of scenarios) {
+    lines.push(`## ${s.name}`);
+    lines.push(`**ID**: ${s.id}`);
+    const preview =
+      s.situation && s.situation.length > 60
+        ? s.situation.slice(0, 60) + "..."
+        : s.situation;
+    lines.push(`**Situation**: ${preview}`);
+    lines.push(
+      `**Criteria**: ${Array.isArray(s.criteria) ? s.criteria.length : 0} criteria`,
+    );
+    if (Array.isArray(s.labels) && s.labels.length > 0) {
+      lines.push(`**Labels**: ${s.labels.join(", ")}`);
+    }
+    lines.push("");
+  }
+
+  lines.push(
+    "> Use `get_scenario` with the ID to see full scenario details.",
+  );
+
+  return lines.join("\n");
+}

package/src/tools/update-scenario.ts

@@ -0,0 +1,32 @@
+import { updateScenario as apiUpdateScenario } from "../langwatch-api-scenarios.js";
+
+/**
+ * Handles the update_scenario MCP tool invocation.
+ *
+ * Updates an existing scenario and returns a confirmation
+ * with the updated details.
+ */
+export async function handleUpdateScenario(params: {
+  scenarioId: string;
+  name?: string;
+  situation?: string;
+  criteria?: string[];
+  labels?: string[];
+}): Promise<string> {
+  const { scenarioId, ...data } = params;
+  const result = await apiUpdateScenario({ id: scenarioId, ...data });
+
+  const lines: string[] = [];
+  lines.push("Scenario updated successfully!\n");
+  lines.push(`**ID**: ${result.id}`);
+  lines.push(`**Name**: ${result.name}`);
+  if (result.situation) lines.push(`**Situation**: ${result.situation}`);
+  if (Array.isArray(result.criteria) && result.criteria.length > 0) {
+    lines.push(`**Criteria**: ${result.criteria.length} criteria`);
+  }
+  if (Array.isArray(result.labels) && result.labels.length > 0) {
+    lines.push(`**Labels**: ${result.labels.join(", ")}`);
+  }
+
+  return lines.join("\n");
+}