@langwatch/mcp-server 0.3.3 → 0.5.0

package/src/langwatch-api.ts

@@ -0,0 +1,266 @@
+import { getConfig, requireApiKey } from "./config.js";
+
+// --- Response types ---
+
+export interface TraceSearchResult {
+  trace_id: string;
+  formatted_trace?: string;
+  input?: { value: string };
+  output?: { value: string };
+  timestamps?: { started_at?: string | number };
+  metadata?: Record<string, unknown>;
+  error?: Record<string, unknown>;
+}
+
+export interface SearchTracesResponse {
+  traces: TraceSearchResult[];
+  pagination?: {
+    totalHits?: number;
+    scrollId?: string;
+  };
+}
+
+export interface TraceDetailResponse {
+  trace_id: string;
+  formatted_trace?: string;
+  input?: { value: string };
+  output?: { value: string };
+  timestamps?: {
+    started_at?: string | number;
+    updated_at?: string | number;
+    inserted_at?: string | number;
+  };
+  metadata?: {
+    user_id?: string;
+    thread_id?: string;
+    customer_id?: string;
+    labels?: string[];
+    [key: string]: unknown;
+  };
+  error?: Record<string, unknown>;
+  ascii_tree?: string;
+  evaluations?: Array<{
+    evaluator_id?: string;
+    name?: string;
+    score?: number;
+    passed?: boolean;
+    label?: string;
+  }>;
+  spans?: Array<{
+    span_id: string;
+    name?: string;
+    type?: string;
+    model?: string;
+    input?: { value: string };
+    output?: { value: string };
+    timestamps?: { started_at?: number; finished_at?: number };
+    metrics?: {
+      completion_time_ms?: number;
+      prompt_tokens?: number;
+      completion_tokens?: number;
+      tokens_estimated?: boolean;
+      cost?: number;
+    };
+  }>;
+}
+
+export interface AnalyticsBucket {
+  date: string;
+  [key: string]: unknown;
+}
+
+export interface AnalyticsTimeseriesResponse {
+  currentPeriod: AnalyticsBucket[];
+  previousPeriod: AnalyticsBucket[];
+}
+
+export interface PromptSummary {
+  id?: string;
+  handle?: string;
+  name?: string;
+  description?: string | null;
+  latestVersionNumber?: number;
+  version?: number;
+}
+
+export interface PromptVersion {
+  version?: number;
+  commitMessage?: string;
+  model?: string;
+  modelProvider?: string;
+  messages?: Array<{ role: string; content: string }>;
+}
+
+export interface PromptDetailResponse extends PromptSummary {
+  versions?: PromptVersion[];
+  model?: string;
+  modelProvider?: string;
+  messages?: Array<{ role: string; content: string }>;
+  prompt?: Array<{ role: string; content: string }>;
+}
+
+export interface PromptMutationResponse {
+  id?: string;
+  handle?: string;
+  name?: string;
+  latestVersionNumber?: number;
+}
+
+// --- HTTP client ---
+
+/**
+ * Sends an HTTP request to the LangWatch API.
+ *
+ * Builds the full URL from the configured endpoint, adds authentication,
+ * and handles JSON serialization/deserialization.
+ *
+ * @throws Error with status code and response body when the response is not OK
+ */
+export async function makeRequest(
+  method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
+  path: string,
+  body?: unknown
+): Promise<unknown> {
+  const url = getConfig().endpoint + path;
+  const headers: Record<string, string> = {
+    "X-Auth-Token": requireApiKey(),
+  };
+
+  if (body !== undefined) {
+    headers["Content-Type"] = "application/json";
+  }
+
+  const response = await fetch(url, {
+    method,
+    headers,
+    ...(body !== undefined ? { body: JSON.stringify(body) } : {}),
+  });
+
+  if (!response.ok) {
+    const responseBody = await response.text();
+    throw new Error(
+      `LangWatch API error ${response.status}: ${responseBody}`
+    );
+  }
+
+  return response.json();
+}
+
+/** Searches traces with optional filters and pagination. */
+export async function searchTraces(params: {
+  query?: string;
+  filters?: Record<string, string[]>;
+  startDate: number;
+  endDate: number;
+  pageSize?: number;
+  pageOffset?: number;
+  scrollId?: string;
+  format?: "digest" | "json";
+}): Promise<SearchTracesResponse> {
+  const { format = "digest", ...rest } = params;
+  return makeRequest("POST", "/api/traces/search", {
+    ...rest,
+    format,
+  }) as Promise<SearchTracesResponse>;
+}
+
+/** Retrieves a single trace by its ID. */
+export async function getTraceById(
+  traceId: string,
+  format: "digest" | "json" = "digest"
+): Promise<TraceDetailResponse> {
+  return makeRequest(
+    "GET",
+    `/api/traces/${encodeURIComponent(traceId)}?format=${format}`
+  ) as Promise<TraceDetailResponse>;
+}
+
+/** Fetches analytics timeseries data for the given metrics and date range. */
+export async function getAnalyticsTimeseries(params: {
+  series: Array<{
+    metric: string;
+    aggregation: string;
+    key?: string;
+    subkey?: string;
+  }>;
+  startDate: number;
+  endDate: number;
+  timeZone?: string;
+  groupBy?: string;
+  groupByKey?: string;
+  filters?: Record<string, string[]>;
+}): Promise<AnalyticsTimeseriesResponse> {
+  return makeRequest(
+    "POST",
+    "/api/analytics/timeseries",
+    params
+  ) as Promise<AnalyticsTimeseriesResponse>;
+}
+
+/** Lists all prompts in the project. */
+export async function listPrompts(): Promise<PromptSummary[]> {
+  return makeRequest("GET", "/api/prompts") as Promise<PromptSummary[]>;
+}
+
+/** Retrieves a single prompt by ID or handle. */
+export async function getPrompt(
+  idOrHandle: string,
+  version?: number
+): Promise<PromptDetailResponse> {
+  const query = version != null ? `?version=${version}` : "";
+  return makeRequest(
+    "GET",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
+  ) as Promise<PromptDetailResponse>;
+}
+
+/** Creates a new prompt. */
+export async function createPrompt(data: {
+  name: string;
+  handle?: string;
+  messages: Array<{ role: string; content: string }>;
+  model: string;
+  modelProvider: string;
+  description?: string;
+}): Promise<PromptMutationResponse> {
+  return makeRequest(
+    "POST",
+    "/api/prompts",
+    data
+  ) as Promise<PromptMutationResponse>;
+}
+
+/** Updates an existing prompt by ID or handle. */
+export async function updatePrompt(
+  idOrHandle: string,
+  data: {
+    messages?: Array<{ role: string; content: string }>;
+    model?: string;
+    modelProvider?: string;
+    commitMessage?: string;
+  }
+): Promise<PromptMutationResponse> {
+  return makeRequest(
+    "POST",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}`,
+    data
+  ) as Promise<PromptMutationResponse>;
+}
+
+/** Creates a new version of an existing prompt. */
+export async function createPromptVersion(
+  idOrHandle: string,
+  data: {
+    messages?: Array<{ role: string; content: string }>;
+    model?: string;
+    modelProvider?: string;
+    commitMessage?: string;
+  }
+): Promise<PromptMutationResponse> {
+  return makeRequest(
+    "POST",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}/versions`,
+    data
+  ) as Promise<PromptMutationResponse>;
+}
+

package/src/schemas/analytics-groups.ts

@@ -0,0 +1,78 @@
+export interface GroupByInfo {
+  name: string;
+  label: string;
+  description: string;
+}
+
+export const analyticsGroups: GroupByInfo[] = [
+  {
+    name: "topics.topics",
+    label: "Topic",
+    description: "Group by topic classification",
+  },
+  {
+    name: "metadata.user_id",
+    label: "User",
+    description: "Group by user ID",
+  },
+  {
+    name: "metadata.thread_id",
+    label: "Thread",
+    description: "Group by conversation thread",
+  },
+  {
+    name: "metadata.customer_id",
+    label: "Customer ID",
+    description: "Group by customer/organization",
+  },
+  {
+    name: "metadata.labels",
+    label: "Label",
+    description: "Group by custom labels",
+  },
+  {
+    name: "metadata.model",
+    label: "Model",
+    description: "Group by LLM model name",
+  },
+  {
+    name: "metadata.span_type",
+    label: "Span Type",
+    description: "Group by span type (llm, tool, agent, etc.)",
+  },
+  {
+    name: "sentiment.input_sentiment",
+    label: "Input Sentiment",
+    description: "Group by detected input sentiment (positive, negative, neutral)",
+  },
+  {
+    name: "sentiment.thumbs_up_down",
+    label: "Thumbs Up/Down",
+    description: "Group by user feedback (positive, negative, neutral)",
+  },
+  {
+    name: "events.event_type",
+    label: "Event Type",
+    description: "Group by event type",
+  },
+  {
+    name: "evaluations.evaluation_passed",
+    label: "Evaluation Passed",
+    description: "Group by evaluation pass/fail status",
+  },
+  {
+    name: "evaluations.evaluation_label",
+    label: "Evaluation Label",
+    description: "Group by evaluation label result",
+  },
+  {
+    name: "evaluations.evaluation_processing_state",
+    label: "Evaluation Processing State",
+    description: "Group by evaluation processing state",
+  },
+  {
+    name: "error.has_error",
+    label: "Contains Error",
+    description: "Group by whether the trace contains an error",
+  },
+];

package/src/schemas/analytics-metrics.ts

@@ -0,0 +1,179 @@
+export interface MetricInfo {
+  category: string;
+  name: string;
+  label: string;
+  allowedAggregations: string[];
+  description: string;
+}
+
+export const analyticsMetrics: MetricInfo[] = [
+  // metadata
+  {
+    category: "metadata",
+    name: "trace_id",
+    label: "Traces",
+    allowedAggregations: ["cardinality"],
+    description: "Count of unique traces",
+  },
+  {
+    category: "metadata",
+    name: "user_id",
+    label: "Users",
+    allowedAggregations: ["cardinality"],
+    description: "Count of unique users",
+  },
+  {
+    category: "metadata",
+    name: "thread_id",
+    label: "Threads",
+    allowedAggregations: ["cardinality"],
+    description: "Count of unique conversation threads",
+  },
+  {
+    category: "metadata",
+    name: "span_type",
+    label: "Span Type",
+    allowedAggregations: ["cardinality"],
+    description: "Count of spans, optionally filtered by span type",
+  },
+  // sentiment
+  {
+    category: "sentiment",
+    name: "input_sentiment",
+    label: "Input Sentiment Score",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Sentiment analysis score of inputs",
+  },
+  {
+    category: "sentiment",
+    name: "thumbs_up_down",
+    label: "Thumbs Up/Down Score",
+    allowedAggregations: [
+      "terms",
+      "cardinality",
+      "avg",
+      "sum",
+      "min",
+      "max",
+      "median",
+      "p99",
+      "p95",
+      "p90",
+    ],
+    description: "User feedback score (-1 to 1)",
+  },
+  // performance
+  {
+    category: "performance",
+    name: "completion_time",
+    label: "Completion Time",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Time to complete the trace (ms)",
+  },
+  {
+    category: "performance",
+    name: "first_token",
+    label: "Time to First Token",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Time to first token (ms)",
+  },
+  {
+    category: "performance",
+    name: "total_cost",
+    label: "Total Cost",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Cost per trace in USD",
+  },
+  {
+    category: "performance",
+    name: "prompt_tokens",
+    label: "Prompt Tokens",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Input token count",
+  },
+  {
+    category: "performance",
+    name: "completion_tokens",
+    label: "Completion Tokens",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Output token count",
+  },
+  {
+    category: "performance",
+    name: "total_tokens",
+    label: "Total Tokens",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Total token count (input + output)",
+  },
+  {
+    category: "performance",
+    name: "tokens_per_second",
+    label: "Tokens per Second",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Token generation speed",
+  },
+  // events
+  {
+    category: "events",
+    name: "event_type",
+    label: "Event Type",
+    allowedAggregations: ["cardinality"],
+    description: "Count of events, optionally filtered by event type",
+  },
+  {
+    category: "events",
+    name: "event_score",
+    label: "Event Score",
+    allowedAggregations: [
+      "terms",
+      "avg",
+      "sum",
+      "min",
+      "max",
+      "median",
+      "p99",
+      "p95",
+      "p90",
+    ],
+    description: "Numeric score from events (requires event_type key and metrics key)",
+  },
+  {
+    category: "events",
+    name: "event_details",
+    label: "Event Details",
+    allowedAggregations: ["cardinality"],
+    description:
+      "Event detail key/value distribution (requires event_type key and details key)",
+  },
+  // evaluations
+  {
+    category: "evaluations",
+    name: "evaluation_score",
+    label: "Evaluation Score",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Numeric evaluation score (requires evaluator_id key)",
+  },
+  {
+    category: "evaluations",
+    name: "evaluation_pass_rate",
+    label: "Evaluation Pass Rate",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description:
+      "Percentage of traces passing evaluation (requires evaluator_id key)",
+  },
+  {
+    category: "evaluations",
+    name: "evaluation_runs",
+    label: "Evaluation Runs",
+    allowedAggregations: ["cardinality"],
+    description: "Count of evaluation executions",
+  },
+  // threads
+  {
+    category: "threads",
+    name: "average_duration_per_thread",
+    label: "Thread Duration",
+    allowedAggregations: ["avg"],
+    description: "Average duration of conversation threads (ms)",
+  },
+];

package/src/schemas/filter-fields.ts

@@ -0,0 +1,119 @@
+export interface FilterFieldInfo {
+  field: string;
+  description: string;
+  example?: string;
+}
+
+export const filterFields: FilterFieldInfo[] = [
+  {
+    field: "topics.topics",
+    description: "Main topic classification of the trace",
+    example: "billing",
+  },
+  {
+    field: "topics.subtopics",
+    description: "Subtopic classification",
+    example: "refund-request",
+  },
+  {
+    field: "metadata.user_id",
+    description: "User ID from trace metadata",
+    example: "user-123",
+  },
+  {
+    field: "metadata.thread_id",
+    description: "Conversation thread ID",
+    example: "thread-456",
+  },
+  {
+    field: "metadata.customer_id",
+    description: "Customer/organization ID",
+    example: "customer-789",
+  },
+  {
+    field: "metadata.labels",
+    description: "Custom labels attached to traces",
+    example: "production",
+  },
+  {
+    field: "metadata.key",
+    description: "Custom metadata key",
+    example: "environment",
+  },
+  {
+    field: "metadata.value",
+    description: "Custom metadata value (used with metadata.key)",
+    example: "staging",
+  },
+  {
+    field: "metadata.prompt_ids",
+    description: "Prompt IDs used in the trace",
+  },
+  {
+    field: "traces.error",
+    description: "Whether the trace has errors",
+    example: "true",
+  },
+  {
+    field: "spans.type",
+    description: "Span type (llm, tool, agent, chain, rag)",
+    example: "llm",
+  },
+  {
+    field: "spans.model",
+    description: "LLM model name used in spans",
+    example: "gpt-4o",
+  },
+  {
+    field: "evaluations.evaluator_id",
+    description: "Evaluator that ran on the trace",
+  },
+  {
+    field: "evaluations.evaluator_id.guardrails_only",
+    description: "Evaluator ID filtered to guardrails only",
+  },
+  {
+    field: "evaluations.passed",
+    description: "Whether evaluations passed",
+    example: "true",
+  },
+  {
+    field: "evaluations.score",
+    description: "Evaluation score (numeric)",
+  },
+  {
+    field: "evaluations.state",
+    description: "Evaluation state (processed, error, skipped)",
+  },
+  {
+    field: "evaluations.label",
+    description: "Evaluation label result",
+  },
+  {
+    field: "events.event_type",
+    description: "Type of event (thumbs_up_down, custom)",
+    example: "thumbs_up_down",
+  },
+  {
+    field: "events.metrics.key",
+    description: "Event metric key",
+  },
+  {
+    field: "events.metrics.value",
+    description: "Event metric value (numeric)",
+  },
+  {
+    field: "events.event_details.key",
+    description: "Event detail key",
+  },
+  {
+    field: "annotations.hasAnnotation",
+    description: "Whether trace has human annotations",
+    example: "true",
+  },
+  {
+    field: "sentiment.input_sentiment",
+    description: "Detected sentiment of input",
+    example: "positive",
+  },
+];

package/src/schemas/index.ts

@@ -0,0 +1,3 @@
+export { filterFields, type FilterFieldInfo } from "./filter-fields.js";
+export { analyticsMetrics, type MetricInfo } from "./analytics-metrics.js";
+export { analyticsGroups, type GroupByInfo } from "./analytics-groups.js";

package/src/tools/archive-scenario.ts

@@ -0,0 +1,19 @@
+import { archiveScenario as apiArchiveScenario } from "../langwatch-api-scenarios.js";
+
+/**
+ * Handles the archive_scenario MCP tool invocation.
+ *
+ * Archives (soft-deletes) a scenario and returns confirmation.
+ */
+export async function handleArchiveScenario(params: {
+  scenarioId: string;
+}): Promise<string> {
+  const result = await apiArchiveScenario(params.scenarioId);
+
+  const lines: string[] = [];
+  lines.push("Scenario archived successfully!\n");
+  lines.push(`**ID**: ${result.id}`);
+  lines.push(`**Status**: ${result.archived ? "archived" : "active"}`);
+
+  return lines.join("\n");
+}

package/src/tools/create-prompt.ts

@@ -0,0 +1,29 @@
+import { createPrompt as apiCreatePrompt } from "../langwatch-api.js";
+
+/**
+ * Handles the create_prompt MCP tool invocation.
+ *
+ * Creates a new prompt in the LangWatch project and returns a
+ * confirmation with the created prompt's details.
+ */
+export async function handleCreatePrompt(params: {
+  name: string;
+  handle?: string;
+  messages: Array<{ role: string; content: string }>;
+  model: string;
+  modelProvider: string;
+  description?: string;
+}): Promise<string> {
+  const result = await apiCreatePrompt(params);
+
+  const lines: string[] = [];
+  lines.push("Prompt created successfully!\n");
+  if (result.id) lines.push(`**ID**: ${result.id}`);
+  if (result.handle) lines.push(`**Handle**: ${result.handle}`);
+  lines.push(`**Name**: ${result.name || params.name}`);
+  lines.push(`**Model**: ${params.model} (${params.modelProvider})`);
+  if (result.latestVersionNumber != null)
+    lines.push(`**Version**: v${result.latestVersionNumber}`);
+
+  return lines.join("\n");
+}