@langwatch/mcp-server 0.6.1 → 0.8.0

package/src/tools/list-workflows.ts

@@ -0,0 +1,31 @@
+import { listWorkflows as apiListWorkflows } from "../langwatch-api-workflows.js";
+
+export async function handleListWorkflows(): Promise<string> {
+  const workflows = await apiListWorkflows();
+
+  if (!Array.isArray(workflows) || workflows.length === 0) {
+    return "No workflows found in this project.";
+  }
+
+  const lines: string[] = [];
+  lines.push(`# Workflows (${workflows.length} total)\n`);
+
+  for (const w of workflows) {
+    const tags: string[] = [];
+    if (w.isEvaluator) tags.push("evaluator");
+    if (w.isComponent) tags.push("component");
+    const tagStr = tags.length > 0 ? ` [${tags.join(", ")}]` : "";
+
+    lines.push(`## ${w.name}${tagStr}`);
+    lines.push(`**ID**: ${w.id}`);
+    if (w.description) lines.push(`**Description**: ${w.description}`);
+    lines.push(`**Updated**: ${w.updatedAt}`);
+    lines.push("");
+  }
+
+  lines.push(
+    "> Use `platform_get_workflow` with the ID to see full workflow details.",
+  );
+
+  return lines.join("\n");
+}

package/src/tools/rename-prompt-tag.ts

@@ -0,0 +1,14 @@
+import { renamePromptTag as apiRenamePromptTag } from "../langwatch-api.js";
+
+export async function handleRenamePromptTag(params: {
+  tag: string;
+  name: string;
+}): Promise<string> {
+  await apiRenamePromptTag({ tag: params.tag, name: params.name });
+
+  const lines: string[] = [];
+  lines.push("Tag renamed successfully!\n");
+  lines.push(`**Old name**: ${params.tag}`);
+  lines.push(`**New name**: ${params.name}`);
+  return lines.join("\n");
+}

package/src/tools/run-agent.ts

@@ -0,0 +1,36 @@
+import { runAgent as apiRunAgent } from "../langwatch-api-agents.js";
+
+/**
+ * Handles the platform_run_agent MCP tool invocation.
+ */
+export async function handleRunAgent(params: {
+  id: string;
+  input?: string;
+}): Promise<string> {
+  let parsedInput: Record<string, unknown> = {};
+  if (params.input) {
+    try {
+      parsedInput = JSON.parse(params.input) as Record<string, unknown>;
+    } catch {
+      return "Error: `input` must be a valid JSON object.";
+    }
+  }
+
+  const { agentType, result } = await apiRunAgent(params.id, parsedInput);
+
+  const lines: string[] = [];
+  lines.push(`Agent executed successfully (type: ${agentType}).\n`);
+
+  if (typeof result === "object" && result !== null) {
+    const output = (result as Record<string, unknown>).output;
+    if (output !== undefined) {
+      lines.push("**Output:**");
+      lines.push(typeof output === "string" ? output : JSON.stringify(output, null, 2));
+    } else {
+      lines.push("**Result:**");
+      lines.push(JSON.stringify(result, null, 2));
+    }
+  }
+
+  return lines.join("\n");
+}

package/src/tools/run-evaluation.ts

@@ -0,0 +1,87 @@
+import { makeRequest } from "../langwatch-api.js";
+
+interface EvaluationRunResponse {
+  runId: string;
+  status: string;
+  total: number;
+  runUrl?: string;
+}
+
+interface EvaluationStatusResponse {
+  runId: string;
+  status: string;
+  progress: number;
+  total: number;
+  startedAt?: number;
+  finishedAt?: number;
+  summary?: {
+    totalCells?: number;
+    completedCells?: number;
+    failedCells?: number;
+    duration?: number;
+    runUrl?: string;
+  };
+}
+
+export async function handleRunEvaluation(params: {
+  slug: string;
+}): Promise<string> {
+  const result = (await makeRequest(
+    "POST",
+    `/api/evaluations/v3/${encodeURIComponent(params.slug)}/run`,
+  )) as EvaluationRunResponse;
+
+  const lines: string[] = [];
+  lines.push(`Evaluation started!\n`);
+  lines.push(`**Run ID**: ${result.runId}`);
+  lines.push(`**Status**: ${result.status}`);
+  lines.push(`**Total cells**: ${result.total}`);
+  if (result.runUrl) lines.push(`**View at**: ${result.runUrl}`);
+  lines.push("");
+  lines.push(
+    `> Use \`platform_evaluation_status\` with run ID "${result.runId}" to check progress.`,
+  );
+
+  return lines.join("\n");
+}
+
+export async function handleEvaluationStatus(params: {
+  runId: string;
+}): Promise<string> {
+  const status = (await makeRequest(
+    "GET",
+    `/api/evaluations/v3/runs/${encodeURIComponent(params.runId)}`,
+  )) as EvaluationStatusResponse;
+
+  const lines: string[] = [];
+  lines.push(`# Evaluation Run ${status.runId}\n`);
+  lines.push(`**Status**: ${status.status}`);
+  lines.push(`**Progress**: ${status.progress}/${status.total} cells`);
+
+  if (status.startedAt) {
+    lines.push(`**Started**: ${new Date(status.startedAt).toISOString()}`);
+  }
+  if (status.finishedAt) {
+    lines.push(`**Finished**: ${new Date(status.finishedAt).toISOString()}`);
+  }
+
+  if (status.summary) {
+    lines.push("\n## Summary\n");
+    if (status.summary.completedCells !== undefined) {
+      lines.push(`**Completed**: ${status.summary.completedCells}`);
+    }
+    if (status.summary.failedCells) {
+      lines.push(`**Failed**: ${status.summary.failedCells}`);
+    }
+    if (status.summary.duration) {
+      lines.push(
+        `**Duration**: ${(status.summary.duration / 1000).toFixed(1)}s`,
+      );
+    }
+    if (status.summary.runUrl) {
+      lines.push(`**View results**: ${status.summary.runUrl}`);
+    }
+  }
+
+  return lines.join("\n");
+}

package/src/tools/run-suite.ts

@@ -0,0 +1,28 @@
+import { runSuite as apiRunSuite } from "../langwatch-api-suites.js";
+
+/**
+ * Handles the platform_run_suite MCP tool invocation.
+ */
+export async function handleRunSuite(params: {
+  id: string;
+}): Promise<string> {
+  const result = await apiRunSuite(params.id);
+
+  const lines: string[] = [];
+  lines.push(`Suite run scheduled successfully.`);
+  lines.push("");
+  lines.push(`**Batch Run ID**: ${result.batchRunId}`);
+  lines.push(`**Jobs**: ${result.jobCount}`);
+  lines.push(`**Set ID**: ${result.setId}`);
+
+  if (result.skippedArchived.scenarios.length > 0) {
+    lines.push(`\n⚠️ Skipped archived scenarios: ${result.skippedArchived.scenarios.join(", ")}`);
+  }
+  if (result.skippedArchived.targets.length > 0) {
+    lines.push(`⚠️ Skipped archived targets: ${result.skippedArchived.targets.join(", ")}`);
+  }
+
+  lines.push("\nView results in the LangWatch dashboard under Simulations.");
+
+  return lines.join("\n");
+}

package/src/tools/run-workflow.ts

@@ -0,0 +1,36 @@
+import { runWorkflow as apiRunWorkflow } from "../langwatch-api-workflows.js";
+
+/**
+ * Handles the platform_run_workflow MCP tool invocation.
+ */
+export async function handleRunWorkflow(params: {
+  id: string;
+  input?: string;
+}): Promise<string> {
+  let parsedInput: Record<string, unknown> = {};
+  if (params.input) {
+    try {
+      parsedInput = JSON.parse(params.input) as Record<string, unknown>;
+    } catch {
+      return "Error: `input` must be a valid JSON object.";
+    }
+  }
+
+  const result = await apiRunWorkflow(params.id, parsedInput);
+
+  const lines: string[] = [];
+  lines.push(`Workflow "${params.id}" executed successfully.\n`);
+
+  if (typeof result === "object" && result !== null) {
+    const output = (result as Record<string, unknown>).output;
+    if (output !== undefined) {
+      lines.push("**Output:**");
+      lines.push(typeof output === "string" ? output : JSON.stringify(output, null, 2));
+    } else {
+      lines.push("**Result:**");
+      lines.push(JSON.stringify(result, null, 2));
+    }
+  }
+
+  return lines.join("\n");
+}

package/src/tools/update-agent.ts

@@ -0,0 +1,13 @@
+import { updateAgent as apiUpdateAgent } from "../langwatch-api-agents.js";
+
+export async function handleUpdateAgent(params: {
+  id: string;
+  name?: string;
+  type?: string;
+  config?: Record<string, unknown>;
+}): Promise<string> {
+  const { id, ...data } = params;
+  const agent = await apiUpdateAgent({ id, ...data });
+
+  return `Agent updated successfully!\n\n**Name**: ${agent.name}\n**ID**: ${agent.id}\n**Type**: ${agent.type}`;
+}

package/src/tools/update-dataset-record.ts

@@ -0,0 +1,21 @@
+import { updateDatasetRecord as apiUpdateDatasetRecord } from "../langwatch-api-datasets.js";
+
+/**
+ * Handles the platform_update_dataset_record MCP tool invocation.
+ *
+ * Updates a single record in a dataset and returns confirmation.
+ */
+export async function handleUpdateDatasetRecord(params: {
+  slugOrId: string;
+  recordId: string;
+  entry: Record<string, unknown>;
+}): Promise<string> {
+  const result = await apiUpdateDatasetRecord(params);
+
+  const lines: string[] = [];
+  lines.push("Record updated successfully!\n");
+  lines.push(`**Record ID**: ${result.id}`);
+  lines.push(`**Entry**: ${JSON.stringify(result.entry)}`);
+
+  return lines.join("\n");
+}

package/src/tools/update-dataset.ts

@@ -0,0 +1,23 @@
+import { updateDataset as apiUpdateDataset } from "../langwatch-api-datasets.js";
+import type { DatasetColumnType } from "../langwatch-api-datasets.js";
+import { formatDatasetMutationDetails } from "./format-dataset-mutation.js";
+
+/**
+ * Handles the platform_update_dataset MCP tool invocation.
+ *
+ * Updates an existing dataset and returns a confirmation
+ * with the updated details.
+ */
+export async function handleUpdateDataset(params: {
+  slugOrId: string;
+  name?: string;
+  columnTypes?: DatasetColumnType[];
+}): Promise<string> {
+  const result = await apiUpdateDataset(params);
+
+  const lines: string[] = [];
+  lines.push("Dataset updated successfully!\n");
+  lines.push(...formatDatasetMutationDetails(result));
+
+  return lines.join("\n");
+}

package/src/tools/update-prompt.ts

@@ -11,6 +11,7 @@ export async function handleUpdatePrompt(params: {
   messages?: Array<{ role: string; content: string }>;
   model?: string;
   commitMessage: string;
+  tags?: string[];
 }): Promise<string> {
   const { idOrHandle, ...data } = params;
 
@@ -23,6 +24,8 @@ export async function handleUpdatePrompt(params: {
   if (result.latestVersionNumber != null)
     lines.push(`**Version**: v${result.latestVersionNumber}`);
   lines.push(`**Commit**: ${params.commitMessage}`);
+  if (params.tags && params.tags.length > 0)
+    lines.push(`**Tags**: ${params.tags.join(", ")}`);
 
   return lines.join("\n");
 }

package/src/tools/update-suite.ts

@@ -0,0 +1,44 @@
+import { updateSuite as apiUpdateSuite } from "../langwatch-api-suites.js";
+import type { SuiteTarget } from "../langwatch-api-suites.js";
+
+/**
+ * Handles the platform_update_suite MCP tool invocation.
+ */
+export async function handleUpdateSuite(params: {
+  id: string;
+  name?: string;
+  description?: string;
+  scenarioIds?: string[];
+  targets?: string;
+  repeatCount?: number;
+  labels?: string[];
+}): Promise<string> {
+  let parsedTargets: SuiteTarget[] | undefined;
+  if (params.targets) {
+    try {
+      parsedTargets = JSON.parse(params.targets) as SuiteTarget[];
+    } catch {
+      return "Error: `targets` must be a valid JSON array.";
+    }
+  }
+
+  const suite = await apiUpdateSuite({
+    id: params.id,
+    name: params.name,
+    description: params.description,
+    scenarioIds: params.scenarioIds,
+    targets: parsedTargets,
+    repeatCount: params.repeatCount,
+    labels: params.labels,
+  });
+
+  return [
+    `Suite "${suite.name}" updated successfully.`,
+    "",
+    `**ID**: ${suite.id}`,
+    `**Slug**: ${suite.slug}`,
+    `**Scenarios**: ${suite.scenarioIds.length}`,
+    `**Targets**: ${suite.targets.length}`,
+    `**Repeat**: ${suite.repeatCount}x`,
+  ].join("\n");
+}

package/src/utils/escape-markdown.ts

@@ -0,0 +1,12 @@
+/**
+ * Escapes characters that would break markdown table formatting.
+ *
+ * Replaces pipe characters (`|`) with `\|` and newlines with spaces
+ * so that values can be safely embedded in markdown tables and headers.
+ */
+export function escapeMarkdown(value: string): string {
+  return value
+    .replace(/\\/g, "\\\\")
+    .replace(/\|/g, "\\|")
+    .replace(/\n/g, " ");
+}