@langwatch/mcp-server 0.6.0 → 0.7.0

package/src/langwatch-api-suites.ts

@@ -0,0 +1,118 @@
+import { makeRequest } from "./langwatch-api.js";
+
+// --- Suite types ---
+
+export interface SuiteTarget {
+  type: "prompt" | "http" | "code" | "workflow";
+  referenceId: string;
+}
+
+export interface SuiteSummary {
+  id: string;
+  name: string;
+  slug: string;
+  description: string | null;
+  scenarioIds: string[];
+  targets: SuiteTarget[];
+  repeatCount: number;
+  labels: string[];
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface SuiteRunResult {
+  scheduled: boolean;
+  batchRunId: string;
+  setId: string;
+  jobCount: number;
+  skippedArchived: {
+    scenarios: string[];
+    targets: string[];
+  };
+  items: Array<{
+    scenarioRunId: string;
+    scenarioId: string;
+    target: SuiteTarget;
+    name: string | null;
+  }>;
+}
+
+export interface SuiteArchiveResponse {
+  id: string;
+  archived: boolean;
+}
+
+// --- Suite API functions ---
+
+/** Lists all suites (run plans) in the project. */
+export async function listSuites(): Promise<SuiteSummary[]> {
+  return makeRequest("GET", "/api/suites") as Promise<SuiteSummary[]>;
+}
+
+/** Retrieves a single suite by ID. */
+export async function getSuite(id: string): Promise<SuiteSummary> {
+  return makeRequest(
+    "GET",
+    `/api/suites/${encodeURIComponent(id)}`
+  ) as Promise<SuiteSummary>;
+}
+
+/** Creates a new suite. */
+export async function createSuite(data: {
+  name: string;
+  description?: string;
+  scenarioIds: string[];
+  targets: SuiteTarget[];
+  repeatCount?: number;
+  labels?: string[];
+}): Promise<SuiteSummary> {
+  return makeRequest("POST", "/api/suites", data) as Promise<SuiteSummary>;
+}
+
+/** Updates an existing suite. */
+export async function updateSuite(params: {
+  id: string;
+  name?: string;
+  description?: string | null;
+  scenarioIds?: string[];
+  targets?: SuiteTarget[];
+  repeatCount?: number;
+  labels?: string[];
+}): Promise<SuiteSummary> {
+  const { id, ...data } = params;
+  return makeRequest(
+    "PATCH",
+    `/api/suites/${encodeURIComponent(id)}`,
+    data
+  ) as Promise<SuiteSummary>;
+}
+
+/** Duplicates a suite. */
+export async function duplicateSuite(id: string): Promise<SuiteSummary> {
+  return makeRequest(
+    "POST",
+    `/api/suites/${encodeURIComponent(id)}/duplicate`
+  ) as Promise<SuiteSummary>;
+}
+
+/** Triggers a suite run. */
+export async function runSuite(
+  id: string,
+  idempotencyKey?: string
+): Promise<SuiteRunResult> {
+  return makeRequest(
+    "POST",
+    `/api/suites/${encodeURIComponent(id)}/run`,
+    { idempotencyKey: idempotencyKey ?? `mcp-${Date.now()}-${Math.random().toString(36).slice(2)}` }
+  ) as Promise<SuiteRunResult>;
+}
+
+/** Archives (soft-deletes) a suite. */
+export async function archiveSuite(
+  id: string
+): Promise<SuiteArchiveResponse> {
+  return makeRequest(
+    "DELETE",
+    `/api/suites/${encodeURIComponent(id)}`
+  ) as Promise<SuiteArchiveResponse>;
+}

package/src/langwatch-api-triggers.ts

@@ -0,0 +1,48 @@
+import { makeRequest } from "./langwatch-api.js";
+
+export interface TriggerSummary {
+  id: string;
+  name: string;
+  action: string;
+  actionParams: Record<string, unknown>;
+  filters: Record<string, unknown>;
+  active: boolean;
+  message: string | null;
+  alertType: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export async function listTriggers(): Promise<TriggerSummary[]> {
+  return makeRequest("GET", "/api/triggers") as Promise<TriggerSummary[]>;
+}
+
+export async function getTrigger(id: string): Promise<TriggerSummary> {
+  return makeRequest("GET", `/api/triggers/${encodeURIComponent(id)}`) as Promise<TriggerSummary>;
+}
+
+export async function createTrigger(data: {
+  name: string;
+  action: string;
+  actionParams?: Record<string, unknown>;
+  filters?: Record<string, unknown>;
+  message?: string;
+  alertType?: string;
+}): Promise<TriggerSummary> {
+  return makeRequest("POST", "/api/triggers", data) as Promise<TriggerSummary>;
+}
+
+export async function updateTrigger(params: {
+  id: string;
+  name?: string;
+  active?: boolean;
+  message?: string | null;
+  alertType?: string | null;
+}): Promise<TriggerSummary> {
+  const { id, ...data } = params;
+  return makeRequest("PATCH", `/api/triggers/${encodeURIComponent(id)}`, data) as Promise<TriggerSummary>;
+}
+
+export async function deleteTrigger(id: string): Promise<{ id: string; deleted: boolean }> {
+  return makeRequest("DELETE", `/api/triggers/${encodeURIComponent(id)}`) as Promise<{ id: string; deleted: boolean }>;
+}

package/src/langwatch-api-workflows.ts

@@ -0,0 +1,41 @@
+import { makeRequest } from "./langwatch-api.js";
+
+export interface WorkflowSummary {
+  id: string;
+  name: string;
+  icon: string | null;
+  description: string | null;
+  isEvaluator: boolean;
+  isComponent: boolean;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export async function listWorkflows(): Promise<WorkflowSummary[]> {
+  return makeRequest("GET", "/api/workflows") as Promise<WorkflowSummary[]>;
+}
+
+export async function getWorkflow(id: string): Promise<WorkflowSummary> {
+  return makeRequest(
+    "GET",
+    `/api/workflows/${encodeURIComponent(id)}`,
+  ) as Promise<WorkflowSummary>;
+}
+
+export async function runWorkflow(
+  id: string,
+  input?: Record<string, unknown>
+): Promise<Record<string, unknown>> {
+  return makeRequest(
+    "POST",
+    `/api/workflows/${encodeURIComponent(id)}/run`,
+    input ?? {},
+  ) as Promise<Record<string, unknown>>;
+}
+
+export async function deleteWorkflow(id: string): Promise<{ id: string; archived: boolean }> {
+  return makeRequest(
+    "DELETE",
+    `/api/workflows/${encodeURIComponent(id)}`,
+  ) as Promise<{ id: string; archived: boolean }>;
+}

package/src/langwatch-api.ts

@@ -140,6 +140,9 @@ export async function makeRequest(
     );
   }
 
+  if (response.status === 204 || response.headers?.get("content-length") === "0") {
+    return null;
+  }
   return response.json();
 }
 
@@ -202,9 +205,12 @@ export async function listPrompts(): Promise<PromptSummary[]> {
 /** Retrieves a single prompt by ID or handle. */
 export async function getPrompt(
   idOrHandle: string,
-  version?: number
+  options?: { version?: number; tag?: string }
 ): Promise<PromptDetailResponse> {
-  const query = version != null ? `?version=${version}` : "";
+  const params = new URLSearchParams();
+  if (options?.version != null) params.set("version", String(options.version));
+  if (options?.tag) params.set("tag", options.tag);
+  const query = params.toString() ? `?${params}` : "";
   return makeRequest(
     "GET",
     `/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
@@ -216,6 +222,7 @@ export async function createPrompt(data: {
   handle: string;
   messages: Array<{ role: string; content: string }>;
   model: string;
+  tags?: string[];
 }): Promise<PromptMutationResponse> {
   return makeRequest(
     "POST",
@@ -231,6 +238,7 @@ export async function updatePrompt(
     messages?: Array<{ role: string; content: string }>;
     model?: string;
     commitMessage: string;
+    tags?: string[];
   }
 ): Promise<PromptMutationResponse> {
   return makeRequest(
@@ -239,3 +247,53 @@ export async function updatePrompt(
     data
   ) as Promise<PromptMutationResponse>;
 }
+
+/** Assigns a tag to a specific prompt version. */
+export async function assignPromptTag({
+  idOrHandle,
+  tag,
+  versionId,
+}: {
+  idOrHandle: string;
+  tag: string;
+  versionId: string;
+}): Promise<unknown> {
+  return makeRequest(
+    "PUT",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}/tags/${encodeURIComponent(tag)}`,
+    { versionId }
+  );
+}
+
+/** Lists all prompt tag definitions for the organization. */
+export async function listPromptTags(): Promise<unknown> {
+  return makeRequest("GET", "/api/prompts/tags");
+}
+
+/** Creates a custom prompt tag definition. */
+export async function createPromptTag(name: string): Promise<unknown> {
+  return makeRequest("POST", "/api/prompts/tags", { name });
+}
+
+/** Renames an existing prompt tag. */
+export async function renamePromptTag({
+  tag,
+  name,
+}: {
+  tag: string;
+  name: string;
+}): Promise<unknown> {
+  return makeRequest(
+    "PUT",
+    `/api/prompts/tags/${encodeURIComponent(tag)}`,
+    { name }
+  );
+}
+
+/** Deletes a prompt tag and all its assignments. */
+export async function deletePromptTag(tag: string): Promise<unknown> {
+  return makeRequest(
+    "DELETE",
+    `/api/prompts/tags/${encodeURIComponent(tag)}`
+  );
+}

package/src/schemas/create-dataset.ts

@@ -0,0 +1,48 @@
+import { z } from "zod";
+
+/**
+ * All supported dataset column types on the LangWatch platform.
+ */
+export const DATASET_COLUMN_TYPES = [
+  "string",
+  "boolean",
+  "number",
+  "date",
+  "list",
+  "json",
+  "spans",
+  "rag_contexts",
+  "chat_messages",
+  "annotations",
+  "evaluations",
+  "image",
+] as const;
+
+/**
+ * Zod schema for a single dataset column type value.
+ */
+export const datasetColumnTypeSchema = z.enum(DATASET_COLUMN_TYPES);
+
+/**
+ * Zod schema for a dataset column definition (name + type).
+ *
+ * Shared between create and update tool registrations.
+ */
+export const datasetColumnDefinitionSchema = z.object({
+  name: z.string().describe("Column name"),
+  type: datasetColumnTypeSchema.describe("Column type"),
+});
+
+/**
+ * Zod schema for the platform_create_dataset MCP tool parameters.
+ *
+ * Extracted so it can be shared between the MCP tool registration
+ * (create-mcp-server.ts) and unit tests.
+ */
+export const createDatasetSchema = z.object({
+  name: z.string().min(1).describe("Dataset name"),
+  columnTypes: z
+    .array(datasetColumnDefinitionSchema)
+    .optional()
+    .describe("Column definitions for the dataset"),
+});

package/src/schemas/index.ts

@@ -1,3 +1,4 @@
 export { filterFields, type FilterFieldInfo } from "./filter-fields.js";
 export { analyticsMetrics, type MetricInfo } from "./analytics-metrics.js";
 export { analyticsGroups, type GroupByInfo } from "./analytics-groups.js";
+export { createDatasetSchema } from "./create-dataset.js";

package/src/tools/archive-suite.ts

@@ -0,0 +1,12 @@
+import { archiveSuite as apiArchiveSuite } from "../langwatch-api-suites.js";
+
+/**
+ * Handles the platform_archive_suite MCP tool invocation.
+ */
+export async function handleArchiveSuite(params: {
+  id: string;
+}): Promise<string> {
+  const result = await apiArchiveSuite(params.id);
+
+  return `Suite ${result.id} has been archived (soft-deleted).`;
+}

package/src/tools/assign-prompt-tag.ts

@@ -0,0 +1,16 @@
+import { assignPromptTag as apiAssignPromptTag } from "../langwatch-api.js";
+
+export async function handleAssignPromptTag(params: {
+  idOrHandle: string;
+  tag: string;
+  versionId: string;
+}): Promise<string> {
+  await apiAssignPromptTag({ idOrHandle: params.idOrHandle, tag: params.tag, versionId: params.versionId });
+
+  const lines: string[] = [];
+  lines.push("Tag assigned successfully!\n");
+  lines.push(`**Prompt**: ${params.idOrHandle}`);
+  lines.push(`**Tag**: ${params.tag}`);
+  lines.push(`**Version ID**: ${params.versionId}`);
+  return lines.join("\n");
+}

package/src/tools/create-agent.ts

@@ -0,0 +1,15 @@
+import { createAgent as apiCreateAgent } from "../langwatch-api-agents.js";
+
+export async function handleCreateAgent(params: {
+  name: string;
+  type: string;
+  config?: Record<string, unknown>;
+}): Promise<string> {
+  const agent = await apiCreateAgent({
+    name: params.name,
+    type: params.type,
+    config: params.config ?? {},
+  });
+
+  return `Agent created successfully!\n\n**Name**: ${agent.name}\n**ID**: ${agent.id}\n**Type**: ${agent.type}`;
+}

package/src/tools/create-annotation.ts

@@ -0,0 +1,18 @@
+import { createAnnotation as apiCreateAnnotation } from "../langwatch-api-annotations.js";
+
+export async function handleCreateAnnotation(params: {
+  traceId: string;
+  comment?: string;
+  isThumbsUp?: boolean;
+  email?: string;
+}): Promise<string> {
+  const annotation = await apiCreateAnnotation(params.traceId, {
+    comment: params.comment,
+    isThumbsUp: params.isThumbsUp,
+    email: params.email,
+  });
+
+  const rating = params.isThumbsUp === true ? " 👍" : params.isThumbsUp === false ? " 👎" : "";
+
+  return `Annotation created successfully!${rating}\n\n**ID**: ${annotation.id ?? "—"}\n**Trace ID**: ${params.traceId}${params.comment ? `\n**Comment**: ${params.comment}` : ""}`;
+}

package/src/tools/create-dashboard.ts

@@ -0,0 +1,7 @@
+import { createDashboard as apiCreateDashboard } from "../langwatch-api-dashboards.js";
+
+export async function handleCreateDashboard(params: { name: string }): Promise<string> {
+  const dashboard = await apiCreateDashboard({ name: params.name });
+
+  return `Dashboard created successfully!\n\n**Name**: ${dashboard.name}\n**ID**: ${dashboard.id}`;
+}

package/src/tools/create-dataset-records.ts

@@ -0,0 +1,23 @@
+import { createDatasetRecords as apiCreateDatasetRecords } from "../langwatch-api-datasets.js";
+
+/**
+ * Handles the platform_create_dataset_records MCP tool invocation.
+ *
+ * Creates records in a dataset in batch and returns a confirmation
+ * with the count of records created.
+ */
+export async function handleCreateDatasetRecords(params: {
+  slugOrId: string;
+  entries: Record<string, unknown>[];
+}): Promise<string> {
+  const result = await apiCreateDatasetRecords(params);
+  const count = Array.isArray(result.data) ? result.data.length : 0;
+
+  const lines: string[] = [];
+  lines.push(`${count} record(s) created successfully!\n`);
+  lines.push(
+    "> Use `platform_get_dataset` to see the updated dataset with all records.",
+  );
+
+  return lines.join("\n");
+}

package/src/tools/create-dataset.ts

@@ -0,0 +1,22 @@
+import { createDataset as apiCreateDataset } from "../langwatch-api-datasets.js";
+import type { DatasetColumnType } from "../langwatch-api-datasets.js";
+import { formatDatasetMutationDetails } from "./format-dataset-mutation.js";
+
+/**
+ * Handles the platform_create_dataset MCP tool invocation.
+ *
+ * Creates a new dataset in the LangWatch project and returns a
+ * confirmation with the created dataset's details.
+ */
+export async function handleCreateDataset(params: {
+  name: string;
+  columnTypes?: DatasetColumnType[];
+}): Promise<string> {
+  const result = await apiCreateDataset(params);
+
+  const lines: string[] = [];
+  lines.push("Dataset created successfully!\n");
+  lines.push(...formatDatasetMutationDetails(result));
+
+  return lines.join("\n");
+}

package/src/tools/create-prompt-tag.ts

@@ -0,0 +1,12 @@
+import { createPromptTag as apiCreatePromptTag } from "../langwatch-api.js";
+
+export async function handleCreatePromptTag(params: {
+  name: string;
+}): Promise<string> {
+  await apiCreatePromptTag(params.name);
+
+  const lines: string[] = [];
+  lines.push("Tag created successfully!\n");
+  lines.push(`**Name**: ${params.name}`);
+  return lines.join("\n");
+}

package/src/tools/create-prompt.ts

@@ -27,6 +27,7 @@ export async function handleCreatePrompt(params: {
   handle?: string;
   messages: Array<{ role: string; content: string }>;
   model: string;
+  tags?: string[];
 }): Promise<string> {
   const handle = params.handle?.trim() || toHandle(params.name);
   if (!handle || !HANDLE_PATTERN.test(handle)) {
@@ -39,6 +40,7 @@ export async function handleCreatePrompt(params: {
     handle,
     messages: params.messages,
     model: params.model,
+    ...(params.tags ? { tags: params.tags } : {}),
   });
 
   const lines: string[] = [];
@@ -49,6 +51,8 @@ export async function handleCreatePrompt(params: {
   lines.push(`**Model**: ${params.model}`);
   if (result.latestVersionNumber != null)
     lines.push(`**Version**: v${result.latestVersionNumber}`);
+  if (params.tags && params.tags.length > 0)
+    lines.push(`**Tags**: ${params.tags.join(", ")}`);
 
   return lines.join("\n");
 }

package/src/tools/create-suite.ts

@@ -0,0 +1,42 @@
+import { createSuite as apiCreateSuite } from "../langwatch-api-suites.js";
+import type { SuiteTarget } from "../langwatch-api-suites.js";
+
+/**
+ * Handles the platform_create_suite MCP tool invocation.
+ */
+export async function handleCreateSuite(params: {
+  name: string;
+  description?: string;
+  scenarioIds: string[];
+  targets: string;
+  repeatCount?: number;
+  labels?: string[];
+}): Promise<string> {
+  let parsedTargets: SuiteTarget[];
+  try {
+    parsedTargets = JSON.parse(params.targets) as SuiteTarget[];
+  } catch {
+    return "Error: `targets` must be a valid JSON array of objects with `type` and `referenceId` fields.\n\nExample: [{\"type\": \"http\", \"referenceId\": \"agent_abc123\"}]";
+  }
+
+  const suite = await apiCreateSuite({
+    name: params.name,
+    description: params.description,
+    scenarioIds: params.scenarioIds,
+    targets: parsedTargets,
+    repeatCount: params.repeatCount,
+    labels: params.labels,
+  });
+
+  return [
+    `Suite "${suite.name}" created successfully.`,
+    "",
+    `**ID**: ${suite.id}`,
+    `**Slug**: ${suite.slug}`,
+    `**Scenarios**: ${suite.scenarioIds.length}`,
+    `**Targets**: ${suite.targets.length}`,
+    `**Repeat**: ${suite.repeatCount}x`,
+    "",
+    `> Use \`platform_run_suite\` with ID \`${suite.id}\` to trigger execution.`,
+  ].join("\n");
+}

package/src/tools/delete-agent.ts

@@ -0,0 +1,7 @@
+import { deleteAgent as apiDeleteAgent } from "../langwatch-api-agents.js";
+
+export async function handleDeleteAgent(params: { id: string }): Promise<string> {
+  const result = await apiDeleteAgent(params.id);
+
+  return `Agent archived successfully!\n\n**Name**: ${result.name}\n**ID**: ${result.id}`;
+}

package/src/tools/delete-annotation.ts

@@ -0,0 +1,7 @@
+import { deleteAnnotation as apiDeleteAnnotation } from "../langwatch-api-annotations.js";
+
+export async function handleDeleteAnnotation(params: { id: string }): Promise<string> {
+  await apiDeleteAnnotation(params.id);
+
+  return `Annotation deleted successfully!\n\n**ID**: ${params.id}`;
+}

package/src/tools/delete-dashboard.ts

@@ -0,0 +1,7 @@
+import { deleteDashboard as apiDeleteDashboard } from "../langwatch-api-dashboards.js";
+
+export async function handleDeleteDashboard(params: { id: string }): Promise<string> {
+  const result = await apiDeleteDashboard(params.id);
+
+  return `Dashboard deleted successfully!\n\n**Name**: ${result.name}\n**ID**: ${result.id}`;
+}

package/src/tools/delete-dataset-records.ts

@@ -0,0 +1,22 @@
+import { deleteDatasetRecords as apiDeleteDatasetRecords } from "../langwatch-api-datasets.js";
+
+/**
+ * Handles the platform_delete_dataset_records MCP tool invocation.
+ *
+ * Deletes records from a dataset by IDs and returns confirmation
+ * with the count of records deleted.
+ */
+export async function handleDeleteDatasetRecords(params: {
+  slugOrId: string;
+  recordIds: string[];
+}): Promise<string> {
+  const result = await apiDeleteDatasetRecords(params);
+
+  const lines: string[] = [];
+  lines.push(`${result.deletedCount} record(s) deleted successfully!\n`);
+  lines.push(
+    "> Use `platform_get_dataset` to see the updated dataset.",
+  );
+
+  return lines.join("\n");
+}

package/src/tools/delete-dataset.ts

@@ -0,0 +1,20 @@
+import { deleteDataset as apiDeleteDataset } from "../langwatch-api-datasets.js";
+
+/**
+ * Handles the platform_delete_dataset MCP tool invocation.
+ *
+ * Archives (soft-deletes) a dataset and returns confirmation.
+ */
+export async function handleDeleteDataset(params: {
+  slugOrId: string;
+}): Promise<string> {
+  await apiDeleteDataset(params.slugOrId);
+
+  const lines: string[] = [];
+  lines.push("Dataset deleted successfully!\n");
+  lines.push(
+    "> The dataset has been archived and will no longer appear in listings.",
+  );
+
+  return lines.join("\n");
+}

package/src/tools/delete-evaluator.ts

@@ -0,0 +1,12 @@
+import { deleteEvaluator as apiDeleteEvaluator } from "../langwatch-api-evaluators.js";
+
+/**
+ * Handles the platform_delete_evaluator MCP tool invocation.
+ */
+export async function handleDeleteEvaluator(params: {
+  idOrSlug: string;
+}): Promise<string> {
+  const result = await apiDeleteEvaluator(params.idOrSlug);
+
+  return `Evaluator ${result.id} has been archived (soft-deleted).`;
+}

package/src/tools/delete-prompt-tag.ts

@@ -0,0 +1,12 @@
+import { deletePromptTag as apiDeletePromptTag } from "../langwatch-api.js";
+
+export async function handleDeletePromptTag(params: {
+  tag: string;
+}): Promise<string> {
+  await apiDeletePromptTag(params.tag);
+
+  const lines: string[] = [];
+  lines.push("Tag deleted successfully!\n");
+  lines.push(`**Tag**: ${params.tag}`);
+  return lines.join("\n");
+}

package/src/tools/delete-workflow.ts

@@ -0,0 +1,7 @@
+import { deleteWorkflow as apiDeleteWorkflow } from "../langwatch-api-workflows.js";
+
+export async function handleDeleteWorkflow(params: { id: string }): Promise<string> {
+  const result = await apiDeleteWorkflow(params.id);
+
+  return `Workflow archived successfully!\n\n**ID**: ${result.id}`;
+}