@langwatch/mcp-server 2.0.0 → 2.1.0

package/src/__tests__/run-plan-tools.unit.test.ts

@@ -0,0 +1,565 @@
+/**
+ * Digests of the run plan tools.
+ *
+ * @see specs/mcp-server/run-plan-tools.feature
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../langwatch-api-run-plans.js", () => ({
+  listRunPlans: vi.fn(),
+  getRunPlan: vi.fn(),
+  runRunPlan: vi.fn(),
+  rerunRunPlan: vi.fn(),
+  archiveRunPlan: vi.fn(),
+}));
+
+import {
+  archiveRunPlan,
+  getRunPlan,
+  listRunPlans,
+  rerunRunPlan,
+  runRunPlan,
+  type RunPlan,
+  type RunPlanRunResult,
+} from "../langwatch-api-run-plans.js";
+
+import { handleArchiveRunPlan } from "../tools/archive-run-plan.js";
+import { handleGetRunPlan } from "../tools/get-run-plan.js";
+import { handleListRunPlans } from "../tools/list-run-plans.js";
+import { handleRerunRunPlan } from "../tools/rerun-run-plan.js";
+import { handleRunPlan } from "../tools/run-plan.js";
+
+const mockListRunPlans = vi.mocked(listRunPlans);
+const mockGetRunPlan = vi.mocked(getRunPlan);
+const mockRunRunPlan = vi.mocked(runRunPlan);
+const mockRerunRunPlan = vi.mocked(rerunRunPlan);
+const mockArchiveRunPlan = vi.mocked(archiveRunPlan);
+
+const samplePlan: RunPlan = {
+  id: "plan_abc123",
+  name: "Regression Plan",
+  slug: "regression-plan",
+  scope: { mode: "labels", labels: ["auth", "checkout"] },
+  scenarioIds: ["scen_abc123"],
+  targets: [{ type: "http", referenceId: "agent_abc" }],
+  repeatCount: 3,
+  simulatorModel: "openai/gpt-5-mini",
+  judgeModel: "openai/gpt-5-mini",
+  labels: ["nightly"],
+  archivedAt: null,
+  createdAt: "2024-01-01T00:00:00Z",
+  updatedAt: "2024-01-02T00:00:00Z",
+  platformUrl: "https://app.langwatch.ai/proj/agent-testing/results/regression-plan",
+};
+
+const sampleRun: RunPlanRunResult = {
+  scheduled: true,
+  batchRunId: "batch_123",
+  setId: "set_456",
+  jobCount: 6,
+  skippedArchived: { scenarios: [], targets: [] },
+  items: [],
+  runPlanId: "plan_abc123",
+  planName: "Regression Plan",
+  created: true,
+  platformUrl: "https://app.langwatch.ai/proj/agent-testing/results/regression-plan",
+};
+
+const runInput = {
+  scope: { mode: "labels" as const, labels: ["auth"] },
+  targets: [{ type: "http" as const, referenceId: "agent_abc" }],
+};
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("handleRunPlan()", () => {
+  describe("when the name no plan carries yet", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockRunRunPlan.mockResolvedValue({ ...sampleRun, created: true });
+      result = await handleRunPlan({ name: "Regression Plan", ...runInput });
+    });
+
+    /** @scenario "Agent runs a name that no plan carries yet" */
+    it("says the plan was created and started", () => {
+      expect(result).toContain(
+        'Run plan "Regression Plan" created and started.',
+      );
+    });
+
+    /** @scenario "Agent reads the batch a run started" */
+    it("names the plan, the batch and the job count", () => {
+      expect(result).toContain("**Plan**: Regression Plan (plan_abc123)");
+      expect(result).toContain("**Batch Run ID**: batch_123");
+      expect(result).toContain("**Jobs**: 6");
+    });
+
+    it("includes the platform URL of the batch", () => {
+      expect(result).toContain(
+        "**View**: https://app.langwatch.ai/proj/agent-testing/results/regression-plan",
+      );
+    });
+
+    it("sends the configuration under config and the note beside it", async () => {
+      await handleRunPlan({
+        name: "Regression Plan",
+        ...runInput,
+        repeatCount: 2,
+        note: "nightly regression",
+      });
+
+      expect(mockRunRunPlan).toHaveBeenLastCalledWith({
+        name: "Regression Plan",
+        config: {
+          scope: { mode: "labels", labels: ["auth"] },
+          targets: [{ type: "http", referenceId: "agent_abc" }],
+          repeatCount: 2,
+          simulatorModel: undefined,
+          judgeModel: undefined,
+          scenarioIds: undefined,
+        },
+        idempotencyKey: undefined,
+        parameters: undefined,
+        note: "nightly regression",
+      });
+    });
+  });
+
+  describe("when a plan already carries the name", () => {
+    /** @scenario "Agent runs a name an existing plan carries" */
+    it("says the plan ran with the configuration of this run", async () => {
+      mockRunRunPlan.mockResolvedValue({ ...sampleRun, created: false });
+
+      const result = await handleRunPlan({
+        name: "Regression Plan",
+        ...runInput,
+      });
+
+      expect(result).toContain(
+        'Run plan "Regression Plan" started with the configuration of this run.',
+      );
+    });
+  });
+
+  describe("when two targets name the same agent with different parameters", () => {
+    /** @scenario "Agent compares one agent on two models in one run" */
+    it("sends both targets, each carrying its own runParameters", async () => {
+      await handleRunPlan({
+        scope: { mode: "labels", labels: ["auth"] },
+        targets: [
+          {
+            type: "http",
+            referenceId: "agent_abc",
+            parameters: { model: "gpt-5" },
+          },
+          {
+            type: "http",
+            referenceId: "agent_abc",
+            parameters: { model: "gpt-5-mini" },
+          },
+        ],
+      });
+
+      expect(mockRunRunPlan).toHaveBeenLastCalledWith(
+        expect.objectContaining({
+          config: expect.objectContaining({
+            targets: [
+              {
+                type: "http",
+                referenceId: "agent_abc",
+                runParameters: { model: "gpt-5" },
+              },
+              {
+                type: "http",
+                referenceId: "agent_abc",
+                runParameters: { model: "gpt-5-mini" },
+              },
+            ],
+          }),
+        }),
+      );
+    });
+
+    /** @scenario "Agent compares one agent on two models in one run" */
+    it("keeps the run-level parameters, which a target overrides for itself", async () => {
+      await handleRunPlan({
+        ...runInput,
+        targets: [
+          {
+            type: "http",
+            referenceId: "agent_abc",
+            parameters: { model: "gpt-5" },
+          },
+        ],
+        parameters: { model: "gpt-5-mini", account_tier: "gold" },
+      });
+
+      expect(mockRunRunPlan).toHaveBeenLastCalledWith(
+        expect.objectContaining({
+          parameters: { model: "gpt-5-mini", account_tier: "gold" },
+          config: expect.objectContaining({
+            targets: [
+              {
+                type: "http",
+                referenceId: "agent_abc",
+                runParameters: { model: "gpt-5" },
+              },
+            ],
+          }),
+        }),
+      );
+    });
+
+    it("leaves runParameters out of a target that names none", async () => {
+      await handleRunPlan(runInput);
+
+      expect(mockRunRunPlan).toHaveBeenLastCalledWith(
+        expect.objectContaining({
+          config: expect.objectContaining({
+            targets: [{ type: "http", referenceId: "agent_abc" }],
+          }),
+        }),
+      );
+    });
+  });
+
+  describe("when the run skipped archived rows", () => {
+    /** @scenario "A run reports what it skipped as archived" */
+    it("lists the skipped scenarios and the skipped targets", async () => {
+      mockRunRunPlan.mockResolvedValue({
+        ...sampleRun,
+        skippedArchived: {
+          scenarios: ["scen_gone"],
+          targets: ["agent_gone"],
+        },
+      });
+
+      const result = await handleRunPlan(runInput);
+
+      expect(result).toContain("**Skipped archived scenarios**: scen_gone");
+      expect(result).toContain("**Skipped archived targets**: agent_gone");
+    });
+  });
+});
+
+describe("handleListRunPlans()", () => {
+  describe("when run plans exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListRunPlans.mockResolvedValue([samplePlan]);
+      result = await handleListRunPlans({});
+    });
+
+    /** @scenario "Agent lists the run plans of a project" */
+    it("lists each plan with what it covers", () => {
+      expect(result).toContain("# Run Plans (1 total)");
+      expect(result).toContain("## Regression Plan");
+      expect(result).toContain("**Covers**: labels: auth, checkout");
+    });
+
+    it("includes the targets and the repeat count", () => {
+      expect(result).toContain("**Targets**: 1 (http:agent_abc)");
+      expect(result).toContain("**Repeat**: 3x");
+    });
+
+    it("asks the API for non-archived plans by default", () => {
+      expect(mockListRunPlans).toHaveBeenCalledWith({
+        includeArchived: undefined,
+      });
+    });
+  });
+
+  describe("when includeArchived is set", () => {
+    it("marks an archived plan as archived", async () => {
+      mockListRunPlans.mockResolvedValue([
+        { ...samplePlan, archivedAt: "2024-02-01T00:00:00Z" },
+      ]);
+
+      const result = await handleListRunPlans({ includeArchived: true });
+
+      expect(result).toContain("## Regression Plan (archived)");
+      expect(mockListRunPlans).toHaveBeenCalledWith({ includeArchived: true });
+    });
+  });
+
+  describe("when no run plans exist", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockListRunPlans.mockResolvedValue([]);
+      result = await handleListRunPlans({});
+    });
+
+    /** @scenario "Agent lists run plans when none exist" */
+    it("returns a no-plans message", () => {
+      expect(result).toContain("No run plans found");
+    });
+
+    it("includes a tip to use platform_run_plan", () => {
+      expect(result).toContain("platform_run_plan");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the plan structure", async () => {
+      mockListRunPlans.mockResolvedValue([samplePlan]);
+
+      const result = await handleListRunPlans({ format: "json" });
+
+      expect(JSON.parse(result)).toEqual([samplePlan]);
+    });
+  });
+});
+
+describe("handleGetRunPlan()", () => {
+  describe("when the plan names labels, targets and models", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockGetRunPlan.mockResolvedValue(samplePlan);
+      result = await handleGetRunPlan({ id: "plan_abc123" });
+    });
+
+    /** @scenario "Agent reads the full configuration of a run plan" */
+    it("includes the scope, the targets, the repeat count and the models", () => {
+      expect(result).toContain("**Covers**: labels: auth, checkout");
+      expect(result).toContain("- http:agent_abc");
+      expect(result).toContain("**Repeat**: 3x");
+      expect(result).toContain("**Simulator model**: openai/gpt-5-mini");
+      expect(result).toContain("**Judge model**: openai/gpt-5-mini");
+    });
+
+    it("includes the plan name in the heading", () => {
+      expect(result).toContain("# Run Plan: Regression Plan");
+    });
+  });
+
+  describe("when a plan compares one agent on two models", () => {
+    /** @scenario "Agent reads the full configuration of a run plan" */
+    it("reads the two targets apart by the parameters each one runs with", async () => {
+      mockGetRunPlan.mockResolvedValue({
+        ...samplePlan,
+        targets: [
+          {
+            type: "http",
+            referenceId: "agent_abc",
+            runParameters: { model: "gpt-5" },
+          },
+          {
+            type: "http",
+            referenceId: "agent_abc",
+            runParameters: { model: "gpt-5-mini" },
+          },
+        ],
+      });
+
+      const result = await handleGetRunPlan({ id: "plan_abc123" });
+
+      expect(result).toContain("- http:agent_abc (model=gpt-5)");
+      expect(result).toContain("- http:agent_abc (model=gpt-5-mini)");
+    });
+  });
+
+  describe("when the scope names test suites", () => {
+    /** @scenario "Agent reads a plan that runs the scenarios of a test suite" */
+    it("says the plan covers the scenarios of those test suites", async () => {
+      mockGetRunPlan.mockResolvedValue({
+        ...samplePlan,
+        scope: { mode: "test_suites", testSuiteIds: ["suite_a", "suite_b"] },
+      });
+
+      const result = await handleGetRunPlan({ id: "plan_abc123" });
+
+      expect(result).toContain("**Covers**: test suites: suite_a, suite_b");
+    });
+  });
+
+  describe("when the plan names no models", () => {
+    /** @scenario "Agent reads a plan that names its own models" */
+    it("says both models are the project default", async () => {
+      mockGetRunPlan.mockResolvedValue({
+        ...samplePlan,
+        simulatorModel: null,
+        judgeModel: null,
+      });
+
+      const result = await handleGetRunPlan({ id: "plan_abc123" });
+
+      expect(result).toContain("**Simulator model**: project default");
+      expect(result).toContain("**Judge model**: project default");
+    });
+  });
+
+  describe("when the plan carries no scope", () => {
+    it("reads as the hand-picked scenario list it already held", async () => {
+      mockGetRunPlan.mockResolvedValue({ ...samplePlan, scope: null });
+
+      const result = await handleGetRunPlan({ id: "plan_abc123" });
+
+      expect(result).toContain("**Covers**: hand-picked scenarios (1)");
+    });
+  });
+
+  describe("when format is json", () => {
+    it("returns valid parseable JSON matching the plan structure", async () => {
+      mockGetRunPlan.mockResolvedValue(samplePlan);
+
+      const result = await handleGetRunPlan({
+        id: "plan_abc123",
+        format: "json",
+      });
+
+      expect(JSON.parse(result)).toEqual(samplePlan);
+    });
+  });
+});
+
+describe("handleRerunRunPlan()", () => {
+  describe("when the plan is run again", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockRerunRunPlan.mockResolvedValue({ ...sampleRun, created: false });
+      result = await handleRerunRunPlan({
+        id: "plan_abc123",
+        note: "after the retry fix",
+      });
+    });
+
+    /** @scenario "Agent runs a plan again with the configuration it holds" */
+    it("says the plan ran with the configuration of this run", () => {
+      expect(result).toContain(
+        'Run plan "Regression Plan" started with the configuration of this run.',
+      );
+    });
+
+    it("includes the batch run id", () => {
+      expect(result).toContain("**Batch Run ID**: batch_123");
+    });
+
+    it("sends only the parameters and the note", () => {
+      expect(mockRerunRunPlan).toHaveBeenCalledWith("plan_abc123", {
+        parameters: undefined,
+        note: "after the retry fix",
+      });
+    });
+  });
+});
+
+describe("handleArchiveRunPlan()", () => {
+  describe("when the plan is archived", () => {
+    let result: string;
+
+    beforeEach(async () => {
+      mockArchiveRunPlan.mockResolvedValue({
+        id: "plan_abc123",
+        archived: true,
+      });
+      result = await handleArchiveRunPlan({ id: "plan_abc123" });
+    });
+
+    /** @scenario "Agent archives a run plan" */
+    it("confirms the plan is archived", () => {
+      expect(result).toContain("Run plan plan_abc123 is archived");
+    });
+
+    it("says the past runs stay readable", () => {
+      expect(result).toContain("past runs stay readable");
+    });
+  });
+});
+
+describe("handleRunPlan()", () => {
+  describe("when the plan carries its own evaluators", () => {
+    /** @scenario "Agent runs a plan with its own evaluators" */
+    it("sends the attachments inside the configuration, with an id and a gate settled", async () => {
+      mockRunRunPlan.mockResolvedValue({
+        scheduled: true,
+        batchRunId: "batch_1",
+        setId: "set_1",
+        jobCount: 1,
+        skippedArchived: { scenarios: [], targets: [] },
+        items: [],
+        runPlanId: "plan_1",
+        planName: "Nightly",
+        created: true,
+        platformUrl: "https://app.langwatch.ai/proj/agent-testing/results/nightly",
+      });
+
+      await handleRunPlan({
+        scope: { mode: "all" },
+        targets: [{ type: "http", referenceId: "agent_abc" }],
+        evaluators: [
+          {
+            evaluatorId: "evaluator_pii",
+            required: false,
+            mappings: {
+              input: { type: "source", sourceId: "conversation", path: ["transcript"] },
+            },
+          },
+        ],
+      });
+
+      expect(mockRunRunPlan).toHaveBeenLastCalledWith(
+        expect.objectContaining({
+          config: expect.objectContaining({
+            evaluators: [
+              {
+                id: expect.stringMatching(/^att_/),
+                evaluatorId: "evaluator_pii",
+                required: false,
+                mappings: {
+                  input: { type: "source", sourceId: "conversation", path: ["transcript"] },
+                },
+              },
+            ],
+          }),
+        }),
+      );
+    });
+  });
+});
+
+describe("handleGetRunPlan()", () => {
+  describe("when the plan carries evaluators", () => {
+    /** @scenario "Agent reads a run plan that carries evaluators" */
+    it("lists each evaluator with its gate and its mappings", async () => {
+      mockGetRunPlan.mockResolvedValue({
+        id: "plan_1",
+        name: "Nightly",
+        slug: "nightly",
+        scope: { mode: "all" },
+        scenarioIds: [],
+        targets: [{ type: "http", referenceId: "agent_abc" }],
+        repeatCount: 1,
+        simulatorModel: null,
+        judgeModel: null,
+        labels: [],
+        archivedAt: null,
+        createdAt: "2026-01-01T00:00:00Z",
+        updatedAt: "2026-01-02T00:00:00Z",
+        platformUrl: "https://app.langwatch.ai/proj/agent-testing/results/nightly",
+        evaluators: [
+          {
+            id: "att_pii",
+            evaluatorId: "evaluator_pii",
+            required: false,
+            mappings: {
+              input: { type: "source", sourceId: "conversation", path: ["transcript"] },
+            },
+          },
+        ],
+      });
+
+      const result = await handleGetRunPlan({ id: "plan_1" });
+
+      expect(result).toContain("## Evaluators");
+      expect(result).toContain("evaluator_pii (reports only, attachment att_pii)");
+      expect(result).toContain("input: conversation.transcript");
+    });
+  });
+});

package/src/__tests__/scenario-tools.unit.test.ts

@@ -186,3 +186,22 @@ describe("formatScenarioSchema()", () => {
     expect(result).toContain("Specific and testable");
   });
 });
+
+describe("formatScenarioSchema()", () => {
+  describe("when documenting the fields and evaluators sections", () => {
+    /** @scenario "Agent discovers the suite fields and the evaluator attachments" */
+    it("documents the field rules and every mapping path", () => {
+      const result = formatScenarioSchema();
+
+      expect(result).toContain("## Test Suite Fields");
+      expect(result).toContain("text, number or boolean");
+      expect(result).toContain("## Evaluator Attachments");
+      expect(result).toContain("first_user_message");
+      expect(result).toContain("last_agent_message");
+      expect(result).toContain('["fields", "<identifier>"]');
+      expect(result).toContain('["tool_calls", "<toolName>", "input" | "output"]');
+      expect(result).toContain('["contexts"]');
+      expect(result).toContain("platform_get_simulation_run");
+    });
+  });
+});

package/src/__tests__/simulation-run-tools.unit.test.ts

@@ -0,0 +1,116 @@
+/**
+ * The digest of a simulation run carries the result of every evaluator that
+ * ran after the conversation, beside the judge's verdict.
+ *
+ * @see specs/mcp-server/test-suite-tools.feature
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../langwatch-api-simulation-runs.js", () => ({
+  getSimulationRun: vi.fn(),
+  listSimulationRuns: vi.fn(),
+}));
+
+import {
+  getSimulationRun,
+  type SimulationRunSummary,
+} from "../langwatch-api-simulation-runs.js";
+import { handleGetSimulationRun } from "../tools/get-simulation-run.js";
+
+const mockGetSimulationRun = vi.mocked(getSimulationRun);
+
+const run: SimulationRunSummary = {
+  scenarioRunId: "run_1",
+  scenarioId: "scen_1",
+  batchRunId: "batch_1",
+  name: "Chargebacks by quarter",
+  status: "FAILED",
+  durationInMs: 4200,
+  results: {
+    verdict: "failed",
+    reasoning: "SQL Query Equivalence failed",
+    metCriteria: ["Answers with one row per quarter"],
+    unmetCriteria: [],
+    error: null,
+    evaluations: [
+      {
+        evaluatorId: "evaluator_sql",
+        name: "SQL Query Equivalence",
+        status: "failed",
+        required: true,
+        passed: false,
+        details: "The query groups by month, not by quarter",
+      },
+      {
+        evaluatorId: "evaluator_judge",
+        name: "Answer quality",
+        status: "scored",
+        required: false,
+        score: 0.8,
+      },
+      {
+        evaluatorId: "evaluator_ctx",
+        name: "Context recall",
+        status: "skipped",
+        required: false,
+        details: "no table_schema on this scenario",
+      },
+    ],
+  },
+  messages: [],
+  timestamp: 1_700_000_000_000,
+  updatedAt: 1_700_000_004_200,
+};
+
+describe("handleGetSimulationRun()", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockGetSimulationRun.mockResolvedValue(run);
+  });
+
+  describe("when the run carries evaluator results", () => {
+    /** @scenario "Agent reads a simulation run with evaluator results" */
+    it("lists each evaluator with its status, score, gate and reason", async () => {
+      const result = await handleGetSimulationRun({ scenarioRunId: "run_1" });
+
+      expect(result).toContain("## Evaluators");
+      expect(result).toContain("**SQL Query Equivalence**: failed, required");
+      expect(result).toContain("The query groups by month, not by quarter");
+      expect(result).toContain("**Answer quality**: scored, score 0.8");
+      expect(result).toContain("**Context recall**: skipped");
+      expect(result).toContain("no table_schema on this scenario");
+    });
+
+    /** @scenario "Agent reads a simulation run with evaluator results" */
+    it("serves them under results.evaluations in the json format", async () => {
+      const result = await handleGetSimulationRun({
+        scenarioRunId: "run_1",
+        format: "json",
+      });
+
+      const parsed = JSON.parse(result) as SimulationRunSummary;
+      expect(parsed.results?.evaluations).toHaveLength(3);
+    });
+  });
+
+  describe("when the run is still waiting on its evaluators", () => {
+    /** @scenario "A pending run reads as PENDING_EVALUATION" */
+    it("warns that the verdict is not final yet", async () => {
+      mockGetSimulationRun.mockResolvedValue({
+        ...run,
+        status: "PENDING_EVALUATION",
+      });
+
+      const result = await handleGetSimulationRun({ scenarioRunId: "run_1" });
+
+      expect(result).toContain("**Status**: PENDING_EVALUATION");
+      expect(result).toContain("have not been recorded yet");
+    });
+
+    it("says nothing extra once they are recorded", async () => {
+      const result = await handleGetSimulationRun({ scenarioRunId: "run_1" });
+
+      expect(result).not.toContain("have not been recorded yet");
+    });
+  });
+});