@langwatch/mcp-server 2.0.0 → 2.2.0

package/src/langwatch-api-suites.ts

@@ -1,118 +0,0 @@
-import { makeRequest } from "./langwatch-api.js";
-
-// --- Suite types ---
-
-export interface SuiteTarget {
-  type: "prompt" | "http" | "code" | "workflow";
-  referenceId: string;
-}
-
-export interface SuiteSummary {
-  id: string;
-  name: string;
-  slug: string;
-  description: string | null;
-  scenarioIds: string[];
-  targets: SuiteTarget[];
-  repeatCount: number;
-  labels: string[];
-  createdAt: string;
-  updatedAt: string;
-}
-
-export interface SuiteRunResult {
-  scheduled: boolean;
-  batchRunId: string;
-  setId: string;
-  jobCount: number;
-  skippedArchived: {
-    scenarios: string[];
-    targets: string[];
-  };
-  items: Array<{
-    scenarioRunId: string;
-    scenarioId: string;
-    target: SuiteTarget;
-    name: string | null;
-  }>;
-}
-
-export interface SuiteArchiveResponse {
-  id: string;
-  archived: boolean;
-}
-
-// --- Suite API functions ---
-
-/** Lists all suites (run plans) in the project. */
-export async function listSuites(): Promise<SuiteSummary[]> {
-  return makeRequest("GET", "/api/suites") as Promise<SuiteSummary[]>;
-}
-
-/** Retrieves a single suite by ID. */
-export async function getSuite(id: string): Promise<SuiteSummary> {
-  return makeRequest(
-    "GET",
-    `/api/suites/${encodeURIComponent(id)}`
-  ) as Promise<SuiteSummary>;
-}
-
-/** Creates a new suite. */
-export async function createSuite(data: {
-  name: string;
-  description?: string;
-  scenarioIds: string[];
-  targets: SuiteTarget[];
-  repeatCount?: number;
-  labels?: string[];
-}): Promise<SuiteSummary> {
-  return makeRequest("POST", "/api/suites", data) as Promise<SuiteSummary>;
-}
-
-/** Updates an existing suite. */
-export async function updateSuite(params: {
-  id: string;
-  name?: string;
-  description?: string | null;
-  scenarioIds?: string[];
-  targets?: SuiteTarget[];
-  repeatCount?: number;
-  labels?: string[];
-}): Promise<SuiteSummary> {
-  const { id, ...data } = params;
-  return makeRequest(
-    "PATCH",
-    `/api/suites/${encodeURIComponent(id)}`,
-    data
-  ) as Promise<SuiteSummary>;
-}
-
-/** Duplicates a suite. */
-export async function duplicateSuite(id: string): Promise<SuiteSummary> {
-  return makeRequest(
-    "POST",
-    `/api/suites/${encodeURIComponent(id)}/duplicate`
-  ) as Promise<SuiteSummary>;
-}
-
-/** Triggers a suite run. */
-export async function runSuite(
-  id: string,
-  idempotencyKey?: string
-): Promise<SuiteRunResult> {
-  return makeRequest(
-    "POST",
-    `/api/suites/${encodeURIComponent(id)}/run`,
-    { idempotencyKey: idempotencyKey ?? `mcp-${Date.now()}-${Math.random().toString(36).slice(2)}` }
-  ) as Promise<SuiteRunResult>;
-}
-
-/** Archives (soft-deletes) a suite. */
-export async function archiveSuite(
-  id: string
-): Promise<SuiteArchiveResponse> {
-  return makeRequest(
-    "DELETE",
-    `/api/suites/${encodeURIComponent(id)}`
-  ) as Promise<SuiteArchiveResponse>;
-}

package/src/schemas/filter-fields.ts

@@ -1,119 +0,0 @@
-export interface FilterFieldInfo {
-  field: string;
-  description: string;
-  example?: string;
-}
-
-export const filterFields: FilterFieldInfo[] = [
-  {
-    field: "topics.topics",
-    description: "Main topic classification of the trace",
-    example: "billing",
-  },
-  {
-    field: "topics.subtopics",
-    description: "Subtopic classification",
-    example: "refund-request",
-  },
-  {
-    field: "metadata.user_id",
-    description: "User ID from trace metadata",
-    example: "user-123",
-  },
-  {
-    field: "metadata.thread_id",
-    description: "Conversation thread ID",
-    example: "thread-456",
-  },
-  {
-    field: "metadata.customer_id",
-    description: "Customer/organization ID",
-    example: "customer-789",
-  },
-  {
-    field: "metadata.labels",
-    description: "Custom labels attached to traces",
-    example: "production",
-  },
-  {
-    field: "metadata.key",
-    description: "Custom metadata key",
-    example: "environment",
-  },
-  {
-    field: "metadata.value",
-    description: "Custom metadata value (used with metadata.key)",
-    example: "staging",
-  },
-  {
-    field: "metadata.prompt_ids",
-    description: "Prompt IDs used in the trace",
-  },
-  {
-    field: "traces.error",
-    description: "Whether the trace has errors",
-    example: "true",
-  },
-  {
-    field: "spans.type",
-    description: "Span type (llm, tool, agent, chain, rag)",
-    example: "llm",
-  },
-  {
-    field: "spans.model",
-    description: "LLM model name used in spans",
-    example: "gpt-4o",
-  },
-  {
-    field: "evaluations.evaluator_id",
-    description: "Evaluator that ran on the trace",
-  },
-  {
-    field: "evaluations.evaluator_id.guardrails_only",
-    description: "Evaluator ID filtered to guardrails only",
-  },
-  {
-    field: "evaluations.passed",
-    description: "Whether evaluations passed",
-    example: "true",
-  },
-  {
-    field: "evaluations.score",
-    description: "Evaluation score (numeric)",
-  },
-  {
-    field: "evaluations.state",
-    description: "Evaluation state (processed, error, skipped)",
-  },
-  {
-    field: "evaluations.label",
-    description: "Evaluation label result",
-  },
-  {
-    field: "events.event_type",
-    description: "Type of event (thumbs_up_down, custom)",
-    example: "thumbs_up_down",
-  },
-  {
-    field: "events.metrics.key",
-    description: "Event metric key",
-  },
-  {
-    field: "events.metrics.value",
-    description: "Event metric value (numeric)",
-  },
-  {
-    field: "events.event_details.key",
-    description: "Event detail key",
-  },
-  {
-    field: "annotations.hasAnnotation",
-    description: "Whether trace has human annotations",
-    example: "true",
-  },
-  {
-    field: "sentiment.input_sentiment",
-    description: "Detected sentiment of input",
-    example: "positive",
-  },
-];

package/src/schemas/index.ts

@@ -1,4 +0,0 @@
-export { filterFields, type FilterFieldInfo } from "./filter-fields.js";
-export { analyticsMetrics, type MetricInfo } from "./analytics-metrics.js";
-export { analyticsGroups, type GroupByInfo } from "./analytics-groups.js";
-export { createDatasetSchema } from "./create-dataset.js";

package/src/tools/archive-suite.ts

@@ -1,12 +0,0 @@
-import { archiveSuite as apiArchiveSuite } from "../langwatch-api-suites.js";
-
-/**
- * Handles the platform_archive_suite MCP tool invocation.
- */
-export async function handleArchiveSuite(params: {
-  id: string;
-}): Promise<string> {
-  const result = await apiArchiveSuite(params.id);
-
-  return `Suite ${result.id} has been archived (soft-deleted).`;
-}

package/src/tools/create-suite.ts

@@ -1,42 +0,0 @@
-import { createSuite as apiCreateSuite } from "../langwatch-api-suites.js";
-import type { SuiteTarget } from "../langwatch-api-suites.js";
-
-/**
- * Handles the platform_create_suite MCP tool invocation.
- */
-export async function handleCreateSuite(params: {
-  name: string;
-  description?: string;
-  scenarioIds: string[];
-  targets: string;
-  repeatCount?: number;
-  labels?: string[];
-}): Promise<string> {
-  let parsedTargets: SuiteTarget[];
-  try {
-    parsedTargets = JSON.parse(params.targets) as SuiteTarget[];
-  } catch {
-    return "Error: `targets` must be a valid JSON array of objects with `type` and `referenceId` fields.\n\nExample: [{\"type\": \"http\", \"referenceId\": \"agent_abc123\"}]";
-  }
-
-  const suite = await apiCreateSuite({
-    name: params.name,
-    description: params.description,
-    scenarioIds: params.scenarioIds,
-    targets: parsedTargets,
-    repeatCount: params.repeatCount,
-    labels: params.labels,
-  });
-
-  return [
-    `Suite "${suite.name}" created successfully.`,
-    "",
-    `**ID**: ${suite.id}`,
-    `**Slug**: ${suite.slug}`,
-    `**Scenarios**: ${suite.scenarioIds.length}`,
-    `**Targets**: ${suite.targets.length}`,
-    `**Repeat**: ${suite.repeatCount}x`,
-    "",
-    `> Use \`platform_run_suite\` with ID \`${suite.id}\` to trigger execution.`,
-  ].join("\n");
-}

package/src/tools/get-suite.ts

@@ -1,47 +0,0 @@
-import { getSuite as apiGetSuite } from "../langwatch-api-suites.js";
-
-/**
- * Handles the platform_get_suite MCP tool invocation.
- */
-export async function handleGetSuite(params: {
-  id: string;
-  format?: "digest" | "json";
-}): Promise<string> {
-  const suite = await apiGetSuite(params.id);
-
-  if (params.format === "json") {
-    return JSON.stringify(suite, null, 2);
-  }
-
-  const lines: string[] = [];
-  lines.push(`# Suite: ${suite.name}\n`);
-  lines.push(`**ID**: ${suite.id}`);
-  lines.push(`**Slug**: ${suite.slug}`);
-  lines.push(`**Description**: ${suite.description ?? "—"}`);
-  lines.push(`**Repeat Count**: ${suite.repeatCount}`);
-  lines.push(`**Created**: ${suite.createdAt}`);
-  lines.push(`**Updated**: ${suite.updatedAt}`);
-
-  if (suite.labels.length > 0) {
-    lines.push(`**Labels**: ${suite.labels.join(", ")}`);
-  }
-
-  lines.push("\n## Scenarios");
-  for (const id of suite.scenarioIds) {
-    lines.push(`- ${id}`);
-  }
-
-  lines.push("\n## Targets");
-  for (const t of suite.targets) {
-    lines.push(`- ${t.type}:${t.referenceId}`);
-  }
-
-  const totalJobs = suite.scenarioIds.length * suite.targets.length * suite.repeatCount;
-  lines.push(`\n**Total jobs per run**: ${totalJobs}`);
-
-  lines.push(
-    "\n> Use `platform_run_suite` to trigger a run, or `platform_update_suite` to modify."
-  );
-
-  return lines.join("\n");
-}

package/src/tools/list-suites.ts

@@ -1,43 +0,0 @@
-import { listSuites as apiListSuites } from "../langwatch-api-suites.js";
-
-/**
- * Handles the platform_list_suites MCP tool invocation.
- */
-export async function handleListSuites(params: {
-  format?: "digest" | "json";
-}): Promise<string> {
-  const suites = await apiListSuites();
-
-  if (params.format === "json") {
-    return JSON.stringify(suites, null, 2);
-  }
-
-  if (!Array.isArray(suites) || suites.length === 0) {
-    return "No suites (run plans) found in this project.\n\n> Tip: Use `platform_create_suite` to create your first suite.";
-  }
-
-  const lines: string[] = [];
-  lines.push(`# Suites / Run Plans (${suites.length} total)\n`);
-
-  for (const s of suites) {
-    lines.push(`## ${s.name}`);
-    lines.push(`**ID**: ${s.id}`);
-    lines.push(`**Slug**: ${s.slug}`);
-    lines.push(`**Scenarios**: ${s.scenarioIds.length}`);
-    lines.push(`**Targets**: ${s.targets.length} (${s.targets.map((t) => `${t.type}:${t.referenceId}`).join(", ")})`);
-    lines.push(`**Repeat**: ${s.repeatCount}x`);
-    if (s.description) {
-      lines.push(`**Description**: ${s.description}`);
-    }
-    if (Array.isArray(s.labels) && s.labels.length > 0) {
-      lines.push(`**Labels**: ${s.labels.join(", ")}`);
-    }
-    lines.push("");
-  }
-
-  lines.push(
-    "> Use `platform_get_suite` with the ID to see full details, or `platform_run_suite` to trigger execution."
-  );
-
-  return lines.join("\n");
-}

package/src/tools/run-suite.ts

@@ -1,28 +0,0 @@
-import { runSuite as apiRunSuite } from "../langwatch-api-suites.js";
-
-/**
- * Handles the platform_run_suite MCP tool invocation.
- */
-export async function handleRunSuite(params: {
-  id: string;
-}): Promise<string> {
-  const result = await apiRunSuite(params.id);
-
-  const lines: string[] = [];
-  lines.push(`Suite run scheduled successfully.`);
-  lines.push("");
-  lines.push(`**Batch Run ID**: ${result.batchRunId}`);
-  lines.push(`**Jobs**: ${result.jobCount}`);
-  lines.push(`**Set ID**: ${result.setId}`);
-
-  if (result.skippedArchived.scenarios.length > 0) {
-    lines.push(`\n⚠️ Skipped archived scenarios: ${result.skippedArchived.scenarios.join(", ")}`);
-  }
-  if (result.skippedArchived.targets.length > 0) {
-    lines.push(`⚠️ Skipped archived targets: ${result.skippedArchived.targets.join(", ")}`);
-  }
-
-  lines.push("\nView results in the LangWatch dashboard under Simulations.");
-
-  return lines.join("\n");
-}

package/src/tools/update-suite.ts

@@ -1,44 +0,0 @@
-import { updateSuite as apiUpdateSuite } from "../langwatch-api-suites.js";
-import type { SuiteTarget } from "../langwatch-api-suites.js";
-
-/**
- * Handles the platform_update_suite MCP tool invocation.
- */
-export async function handleUpdateSuite(params: {
-  id: string;
-  name?: string;
-  description?: string;
-  scenarioIds?: string[];
-  targets?: string;
-  repeatCount?: number;
-  labels?: string[];
-}): Promise<string> {
-  let parsedTargets: SuiteTarget[] | undefined;
-  if (params.targets) {
-    try {
-      parsedTargets = JSON.parse(params.targets) as SuiteTarget[];
-    } catch {
-      return "Error: `targets` must be a valid JSON array.";
-    }
-  }
-
-  const suite = await apiUpdateSuite({
-    id: params.id,
-    name: params.name,
-    description: params.description,
-    scenarioIds: params.scenarioIds,
-    targets: parsedTargets,
-    repeatCount: params.repeatCount,
-    labels: params.labels,
-  });
-
-  return [
-    `Suite "${suite.name}" updated successfully.`,
-    "",
-    `**ID**: ${suite.id}`,
-    `**Slug**: ${suite.slug}`,
-    `**Scenarios**: ${suite.scenarioIds.length}`,
-    `**Targets**: ${suite.targets.length}`,
-    `**Repeat**: ${suite.repeatCount}x`,
-  ].join("\n");
-}