@langwatch/mcp-server 1.0.0 → 2.1.0

package/src/tools/rename-test-suite.ts

@@ -0,0 +1,18 @@
+import { renameTestSuite as apiRenameTestSuite } from "../langwatch-api-test-suites.js";
+
+/**
+ * Handles the platform_rename_test_suite MCP tool invocation.
+ */
+export async function handleRenameTestSuite(params: {
+  id: string;
+  name: string;
+}): Promise<string> {
+  const suite = await apiRenameTestSuite(params);
+
+  return [
+    `Test suite ${suite.id} is now named "${suite.name}".`,
+    "",
+    `**Slug**: ${suite.slug}`,
+    `**Scenarios**: ${suite.scenarioCount}`,
+  ].join("\n");
+}

package/src/tools/rerun-run-plan.ts

@@ -0,0 +1,21 @@
+import {
+  rerunRunPlan as apiRerunRunPlan,
+  type RunParameters,
+} from "../langwatch-api-run-plans.js";
+import { formatRunPlanRun } from "./format-run-plan.js";
+
+/**
+ * Handles the platform_rerun_run_plan MCP tool invocation.
+ */
+export async function handleRerunRunPlan(params: {
+  id: string;
+  parameters?: RunParameters;
+  note?: string;
+}): Promise<string> {
+  const result = await apiRerunRunPlan(params.id, {
+    parameters: params.parameters,
+    note: params.note,
+  });
+
+  return formatRunPlanRun(result);
+}

package/src/tools/run-agent.ts

@@ -1,22 +1,56 @@
-import { runAgent as apiRunAgent } from "../langwatch-api-agents.js";
+import { z } from "zod";
+import {
+  runAgent as apiRunAgent,
+  type AgentCallParams,
+} from "../langwatch-api-agents.js";
+
+/**
+ * A scalar and an array both parse as JSON, and either one reaches the agent
+ * as a body it cannot read, so only an object passes.
+ */
+const jsonObjectSchema = z.looseObject({});
+type JsonObject = z.infer<typeof jsonObjectSchema>;
 
 /**
  * Handles the platform_run_agent MCP tool invocation.
+ *
+ * @see specs/mcp-server/agent-tools.feature
  */
-export async function handleRunAgent(params: {
+export async function handleRunAgent({
+  id,
+  input,
+  message,
+  parameters,
+  threadId,
+}: {
   id: string;
   input?: string;
+  message?: string;
+  parameters?: AgentCallParams;
+  threadId?: string;
 }): Promise<string> {
-  let parsedInput: Record<string, unknown> = {};
-  if (params.input) {
+  let parsedInput: JsonObject = {};
+  if (input) {
+    let decoded: unknown;
     try {
-      parsedInput = JSON.parse(params.input) as Record<string, unknown>;
+      decoded = JSON.parse(input);
     } catch {
       return "Error: `input` must be a valid JSON object.";
     }
+    const result = jsonObjectSchema.safeParse(decoded);
+    if (!result.success) {
+      return "Error: `input` must be a valid JSON object.";
+    }
+    parsedInput = result.data;
   }
 
-  const { agentType, result } = await apiRunAgent(params.id, parsedInput);
+  const { agentType, result } = await apiRunAgent({
+    id,
+    input: parsedInput,
+    message,
+    parameters,
+    threadId,
+  });
 
   const lines: string[] = [];
   lines.push(`Agent executed successfully (type: ${agentType}).\n`);
@@ -30,6 +64,20 @@ export async function handleRunAgent(params: {
       lines.push("**Result:**");
       lines.push(JSON.stringify(result, null, 2));
     }
+    if (agentType === "connected") {
+      const instance = (result as { instance?: { hostname?: string; label?: string | null } }).instance;
+      const durationMs = (result as { durationMs?: number }).durationMs;
+      if (instance?.hostname) {
+        const label = instance.label ? ` (${instance.label})` : "";
+        lines.push(`\n**Instance:** ${instance.hostname}${label}`);
+      }
+      if (typeof durationMs === "number") lines.push(`**Duration:** ${durationMs} ms`);
+      const session = (result as { session?: unknown }).session;
+      if (session !== undefined && session !== null) {
+        lines.push("**Session:**");
+        lines.push(JSON.stringify(session, null, 2));
+      }
+    }
   }
 
   return lines.join("\n");

package/src/tools/run-plan.ts

@@ -0,0 +1,49 @@
+import {
+  runRunPlan as apiRunRunPlan,
+  type RunParameters,
+  type RunPlanScope,
+  type RunPlanTarget,
+} from "../langwatch-api-run-plans.js";
+import { toWireTargets } from "../schemas/run-plan.js";
+import {
+  type EvaluatorAttachmentInput,
+  toWireAttachments,
+} from "../schemas/suite-fields.js";
+import { formatRunPlanRun } from "./format-run-plan.js";
+
+/**
+ * Handles the platform_run_plan MCP tool invocation.
+ */
+export async function handleRunPlan(params: {
+  name?: string;
+  scope: RunPlanScope;
+  scenarioIds?: string[];
+  targets: RunPlanTarget[];
+  repeatCount?: number;
+  simulatorModel?: string;
+  judgeModel?: string;
+  evaluators?: EvaluatorAttachmentInput[];
+  parameters?: RunParameters;
+  note?: string;
+  idempotencyKey?: string;
+}): Promise<string> {
+  const result = await apiRunRunPlan({
+    name: params.name,
+    config: {
+      scope: params.scope,
+      targets: toWireTargets(params.targets),
+      repeatCount: params.repeatCount,
+      simulatorModel: params.simulatorModel,
+      judgeModel: params.judgeModel,
+      scenarioIds: params.scenarioIds,
+      ...(params.evaluators !== undefined
+        ? { evaluators: toWireAttachments(params.evaluators) }
+        : {}),
+    },
+    idempotencyKey: params.idempotencyKey,
+    parameters: params.parameters,
+    note: params.note,
+  });
+
+  return formatRunPlanRun(result);
+}

package/src/tools/run-test-suite.ts

@@ -0,0 +1,30 @@
+import type {
+  RunParameters,
+  RunPlanTarget,
+} from "../langwatch-api-run-plans.js";
+import { toWireTargets } from "../schemas/run-plan.js";
+import { runTestSuite as apiRunTestSuite } from "../langwatch-api-test-suites.js";
+import { formatRunPlanRun } from "./format-run-plan.js";
+
+/**
+ * Handles the platform_run_test_suite MCP tool invocation.
+ */
+export async function handleRunTestSuite(params: {
+  id: string;
+  targets: RunPlanTarget[];
+  name?: string;
+  repeatCount?: number;
+  simulatorModel?: string;
+  judgeModel?: string;
+  parameters?: RunParameters;
+  note?: string;
+  idempotencyKey?: string;
+}): Promise<string> {
+  const { targets, ...rest } = params;
+  const result = await apiRunTestSuite({
+    ...rest,
+    targets: toWireTargets(targets),
+  });
+
+  return formatRunPlanRun(result);
+}

package/src/tools/test-agent.ts

@@ -0,0 +1,18 @@
+import { testAgent as apiTestAgent } from "../langwatch-api-agents.js";
+
+/**
+ * Handles the platform_test_agent MCP tool invocation.
+ *
+ * @see specs/agents/agent-test-run.feature
+ */
+export async function handleTestAgent(params: { id: string }): Promise<string> {
+  const run = await apiTestAgent(params.id);
+  return [
+    "Test run scheduled. The user sends \"ping\"; the run succeeds when the agent answers.",
+    "",
+    `**Scenario run ID:** ${run.scenarioRunId}`,
+    `**Batch run ID:** ${run.batchRunId}`,
+    "",
+    `Follow it with platform_get_simulation_run using scenarioRunId "${run.scenarioRunId}".`,
+  ].join("\n");
+}

package/src/tools/update-prompt.ts

@@ -1,10 +1,24 @@
-import { updatePrompt as apiUpdatePrompt } from "../langwatch-api.js";
+import {
+  getPrompt as apiGetPrompt,
+  getPromptVersions as apiGetPromptVersions,
+  updatePrompt as apiUpdatePrompt,
+  type PromptDetailResponse,
+  type PromptTag,
+} from "../langwatch-api.js";
 
 /**
  * Handles the platform_update_prompt MCP tool invocation.
  *
- * Updates an existing prompt via the PUT endpoint.
- * Every update with a commitMessage creates a new version automatically.
+ * Updates an existing prompt via the PUT endpoint. Every update with a
+ * commitMessage creates a new version automatically. The mutation response
+ * does not carry the tags the server actually applied, so this re-fetches
+ * the prompt via getPrompt: the GET response is the prompt's latest version
+ * flattened to the top level (version, versionId, commitMessage) plus a
+ * `tags` array of { name, versionId } naming which version each tag points
+ * to. The new version is identified by matching the request's commitMessage
+ * against that top level (falling back to the versions listing), and
+ * deployment state is derived from the tags that point at it — never from
+ * the request's tags directly.
  */
 export async function handleUpdatePrompt(params: {
   idOrHandle: string;
@@ -15,17 +29,188 @@ export async function handleUpdatePrompt(params: {
 }): Promise<string> {
   const { idOrHandle, ...data } = params;
 
-  const result = await apiUpdatePrompt(idOrHandle, data);
+  let updated;
+  try {
+    updated = await apiUpdatePrompt(idOrHandle, data);
+  } catch (err) {
+    // The platform commits the version before assigning tags, so a
+    // rejection while tags were requested may still have created a version.
+    // Re-fetch to find out rather than reporting a bare failure.
+    if (params.tags && params.tags.length > 0) {
+      return renderTagFailure({ idOrHandle, params });
+    }
+    throw err;
+  }
+
+  return renderUpdateSuccess({ idOrHandle, params, updated });
+}
+
+interface IdentifiedVersion {
+  version?: number;
+  versionId?: string;
+}
+
+/**
+ * Finds the version created by this update. The GET detail response IS the
+ * latest version, so a top-level commitMessage match identifies it directly;
+ * otherwise (e.g. a concurrent update landed after ours) the versions
+ * listing is consulted, best-effort.
+ */
+async function identifyNewVersion({
+  idOrHandle,
+  prompt,
+  commitMessage,
+}: {
+  idOrHandle: string;
+  prompt: PromptDetailResponse;
+  commitMessage: string;
+}): Promise<IdentifiedVersion | undefined> {
+  if (prompt.commitMessage === commitMessage) {
+    return { version: prompt.version, versionId: prompt.versionId };
+  }
+  try {
+    const versions = await apiGetPromptVersions(idOrHandle);
+    const match = (Array.isArray(versions) ? versions : []).find(
+      (v) => v.commitMessage === commitMessage
+    );
+    if (match) return { version: match.version, versionId: match.versionId };
+  } catch {
+    // Identification failure is reported by the caller.
+  }
+  return undefined;
+}
+
+/** The prompt's deployment tags, excluding the built-in "latest" tag. */
+function deploymentTagsOf(prompt: PromptDetailResponse): PromptTag[] {
+  return (prompt.tags ?? []).filter((tag) => tag?.name !== "latest");
+}
+
+async function renderUpdateSuccess({
+  idOrHandle,
+  params,
+  updated,
+}: {
+  idOrHandle: string;
+  params: { commitMessage: string; tags?: string[] };
+  updated?: { id?: string; handle?: string };
+}): Promise<string> {
+  let prompt;
+  try {
+    prompt = await apiGetPrompt(idOrHandle, {
+      version: undefined,
+      tag: undefined,
+    });
+  } catch {
+    // The update itself succeeded; a failed confirmation read must not
+    // surface as a tool failure — that invites a dangerous retry.
+    const lines: string[] = [];
+    lines.push("Prompt updated successfully!\n");
+    if (updated?.id) lines.push(`**ID**: ${updated.id}`);
+    if (updated?.handle) lines.push(`**Handle**: ${updated.handle}`);
+    lines.push(`**Commit**: ${params.commitMessage}`);
+    lines.push(
+      `**Note**: the update succeeded, but version and deployment details are unavailable (confirmation read failed). Run platform_get_prompt to inspect the current state. Do not retry the update.`
+    );
+    return lines.join("\n");
+  }
+
+  const newVersion = await identifyNewVersion({
+    idOrHandle,
+    prompt,
+    commitMessage: params.commitMessage,
+  });
 
   const lines: string[] = [];
   lines.push("Prompt updated successfully!\n");
-  if (result.id) lines.push(`**ID**: ${result.id}`);
-  if (result.handle) lines.push(`**Handle**: ${result.handle}`);
-  if (result.latestVersionNumber != null)
-    lines.push(`**Version**: v${result.latestVersionNumber}`);
+  if (prompt.id) lines.push(`**ID**: ${prompt.id}`);
+  if (prompt.handle) lines.push(`**Handle**: ${prompt.handle}`);
+  if (newVersion?.version != null)
+    lines.push(`**Version**: v${newVersion.version}`);
+  if (newVersion?.versionId)
+    lines.push(`**Version ID**: ${newVersion.versionId}`);
   lines.push(`**Commit**: ${params.commitMessage}`);
-  if (params.tags && params.tags.length > 0)
-    lines.push(`**Tags**: ${params.tags.join(", ")}`);
+
+  if (!newVersion) {
+    // The update succeeded but the re-fetch could not identify the new
+    // version by commit message — say so instead of silently omitting the
+    // version and deployment lines.
+    lines.push(
+      `**Note**: update succeeded, but the new version could not be identified in the re-fetched prompt — version and deployment details are unavailable. Run platform_get_prompt to inspect the current state.`
+    );
+  }
+
+  if (newVersion) {
+    const allDeployments = deploymentTagsOf(prompt);
+    const newTags = allDeployments
+      .filter(
+        (tag) =>
+          newVersion.versionId != null &&
+          tag.versionId === newVersion.versionId
+      )
+      .map((tag) => tag.name);
+    if (newTags.length > 0) {
+      lines.push(`**Deployed to**: ${newTags.join(", ")}`);
+    } else {
+      lines.push(`**Deployment**: not deployed`);
+
+      // Tags pointing at other versions are untouched by this update —
+      // surface them on their own line so no line pairs a version number
+      // with a deployment tag name.
+      const otherTags = Array.from(
+        new Set(
+          allDeployments
+            .filter((tag) => tag.versionId !== newVersion.versionId)
+            .map((tag) => tag.name)
+        )
+      );
+      if (otherTags.length > 0) {
+        lines.push(
+          `**Existing deployments (untouched)**: ${otherTags.join(", ")}`
+        );
+      }
+    }
+  }
 
   return lines.join("\n");
 }
+
+async function renderTagFailure({
+  idOrHandle,
+  params,
+}: {
+  idOrHandle: string;
+  params: { commitMessage: string; tags?: string[] };
+}): Promise<string> {
+  const failedTags = params.tags?.join(", ") ?? "";
+  let prompt;
+  try {
+    prompt = await apiGetPrompt(idOrHandle, {
+      version: undefined,
+      tag: undefined,
+    });
+  } catch {
+    // The confirmation read failed too — report the tag-assignment failure
+    // without claiming whether a version was created.
+    return `Prompt update failed: could not assign tag(s) ${failedTags} to "${idOrHandle}". Whether a new version was created could not be confirmed (confirmation read failed) — run platform_get_prompt before retrying.`;
+  }
+
+  const matched = await identifyNewVersion({
+    idOrHandle,
+    prompt,
+    commitMessage: params.commitMessage,
+  });
+
+  if (matched) {
+    const lines: string[] = [];
+    lines.push("Prompt update partially failed.\n");
+    lines.push(
+      "A new version was created, but assigning the requested tag(s) failed."
+    );
+    if (matched.versionId) lines.push(`**Version ID**: ${matched.versionId}`);
+    lines.push(`**Status**: created, untagged`);
+    lines.push(`**Failed tag(s)**: ${failedTags}`);
+    return lines.join("\n");
+  }
+
+  return `Prompt update failed: could not assign tag(s) ${failedTags} to "${idOrHandle}".`;
+}

package/src/tools/update-scenario.ts

@@ -1,4 +1,6 @@
 import { updateScenario as apiUpdateScenario } from "../langwatch-api-scenarios.js";
+import type { ScenarioFieldValues } from "../schemas/suite-fields.js";
+import { formatScenarioFields } from "./format-scenario.js";
 
 /**
  * Handles the platform_update_scenario MCP tool invocation.
@@ -12,6 +14,8 @@ export async function handleUpdateScenario(params: {
   situation?: string;
   criteria?: string[];
   labels?: string[];
+  testSuiteId?: string | null;
+  fields?: ScenarioFieldValues;
 }): Promise<string> {
   const { scenarioId, ...data } = params;
   const result = await apiUpdateScenario({ id: scenarioId, ...data });
@@ -27,6 +31,10 @@ export async function handleUpdateScenario(params: {
   if (Array.isArray(result.labels) && result.labels.length > 0) {
     lines.push(`**Labels**: ${result.labels.join(", ")}`);
   }
+  if (result.testSuiteId) {
+    lines.push(`**Test suite**: ${result.testSuiteId}`);
+  }
+  lines.push(...formatScenarioFields(result.fields));
 
   return lines.join("\n");
 }

package/src/tools/update-test-suite.ts

@@ -0,0 +1,45 @@
+import { updateTestSuite as apiUpdateTestSuite } from "../langwatch-api-test-suites.js";
+import {
+  type EvaluatorAttachmentInput,
+  type SuiteField,
+  toWireAttachments,
+} from "../schemas/suite-fields.js";
+import {
+  formatEvaluatorAttachments,
+  formatSuiteFields,
+} from "./format-suite-details.js";
+
+/**
+ * Handles the platform_update_test_suite MCP tool invocation.
+ *
+ * Any of the name, the fields and the evaluators. A field list or an
+ * evaluator list replaces the one the suite holds; a key left out keeps
+ * what the suite has.
+ */
+export async function handleUpdateTestSuite(params: {
+  id: string;
+  name?: string;
+  fields?: SuiteField[];
+  evaluators?: EvaluatorAttachmentInput[];
+}): Promise<string> {
+  const suite = await apiUpdateTestSuite({
+    id: params.id,
+    ...(params.name !== undefined ? { name: params.name } : {}),
+    ...(params.fields !== undefined ? { fields: params.fields } : {}),
+    ...(params.evaluators !== undefined
+      ? { evaluators: toWireAttachments(params.evaluators) }
+      : {}),
+  });
+
+  return [
+    `Test suite "${suite.name}" updated.`,
+    "",
+    `**ID**: ${suite.id}`,
+    `**Slug**: ${suite.slug}`,
+    `**Scenarios**: ${suite.scenarioCount}`,
+    ...formatSuiteFields(suite.fields),
+    ...formatEvaluatorAttachments(suite.evaluators),
+    "",
+    `**View**: ${suite.platformUrl}`,
+  ].join("\n");
+}

package/tsconfig.json

@@ -13,13 +13,18 @@
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,
     "moduleResolution": "bundler",
-    "rootDir": "..",
-    // The evaluator catalog we import (langevals/ts-integration/evaluators.generated.ts)
+    "rootDir": "../..",
+    // The evaluator catalog we import (services/langevals/ts-integration/evaluators.generated.ts)
     // is Zod-first and imports `zod`, but lives outside this package's
     // node_modules tree, so plain resolution cannot find it. Point `zod` at this
     // package's own copy (a declared dependency) so the type-checker resolves it.
     // Paths are resolved relative to this tsconfig (no baseUrl — tsgo removed it).
     "paths": { "zod": ["./node_modules/zod"] },
+    // tsup's dts worker sets `baseUrl` itself, and TypeScript 6 makes that a
+    // hard error rather than a warning — this package cannot build without the
+    // transition flag until a d.ts bundler exists that speaks the TypeScript 7
+    // API. See ADR-099.
+    "ignoreDeprecations": "6.0",
     "isolatedModules": true,
     "jsx": "preserve",
     "incremental": true,