@langwatch/mcp-server 0.3.2 → 0.4.0

package/src/config.ts

@@ -0,0 +1,31 @@
+export interface McpConfig {
+  apiKey: string | undefined;
+  endpoint: string;
+}
+
+let config: McpConfig | undefined;
+
+export function initConfig(args: { apiKey?: string; endpoint?: string }): void {
+  config = {
+    apiKey: args.apiKey || process.env.LANGWATCH_API_KEY,
+    endpoint:
+      args.endpoint ||
+      process.env.LANGWATCH_ENDPOINT ||
+      "https://app.langwatch.ai",
+  };
+}
+
+export function getConfig(): McpConfig {
+  if (!config) throw new Error("Config not initialized");
+  return config;
+}
+
+export function requireApiKey(): string {
+  const { apiKey } = getConfig();
+  if (!apiKey) {
+    throw new Error(
+      "LANGWATCH_API_KEY is required. Set it via --apiKey flag or LANGWATCH_API_KEY environment variable."
+    );
+  }
+  return apiKey;
+}

package/src/index.ts

@@ -1,8 +1,28 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
 import { z } from "zod";
 
 import packageJson from "../package.json" assert { type: "json" };
+import { initConfig } from "./config.js";
+
+const argv = await yargs(hideBin(process.argv))
+  .option("apiKey", {
+    type: "string",
+    description: "LangWatch API key",
+  })
+  .option("endpoint", {
+    type: "string",
+    description: "LangWatch API endpoint",
+  })
+  .help()
+  .parseAsync();
+
+initConfig({
+  apiKey: argv.apiKey,
+  endpoint: argv.endpoint,
+});
 
 const transport = new StdioServerTransport();
 const server = new McpServer({
@@ -70,4 +90,238 @@ server.tool(
   }
 );
 
+// --- Observability Tools (require API key) ---
+
+server.tool(
+  "discover_schema",
+  "Discover available filter fields, metrics, aggregation types, and group-by options for LangWatch queries. Call this before using search_traces or get_analytics to understand available options.",
+  {
+    category: z
+      .enum(["filters", "metrics", "aggregations", "groups", "all"])
+      .describe("Which schema category to discover"),
+  },
+  async ({ category }) => {
+    const { formatSchema } = await import("./tools/discover-schema.js");
+    return { content: [{ type: "text", text: formatSchema(category) }] };
+  }
+);
+
+server.tool(
+  "search_traces",
+  "Search LangWatch traces with filters, text query, and date range. Returns AI-readable trace digests by default. Use format: 'json' for full raw data.",
+  {
+    query: z.string().optional().describe("Text search query"),
+    filters: z
+      .record(z.string(), z.array(z.string()))
+      .optional()
+      .describe(
+        'Filter traces. Format: {"field": ["value"]}. Use discover_schema for field names.'
+      ),
+    startDate: z
+      .string()
+      .optional()
+      .describe(
+        'Start date: ISO string or relative like "24h", "7d", "30d". Default: 24h ago'
+      ),
+    endDate: z
+      .string()
+      .optional()
+      .describe("End date: ISO string or relative. Default: now"),
+    pageSize: z
+      .number()
+      .optional()
+      .describe("Results per page (default: 25, max: 1000)"),
+    scrollId: z
+      .string()
+      .optional()
+      .describe("Pagination token from previous search"),
+    format: z
+      .enum(["digest", "json"])
+      .optional()
+      .describe(
+        "Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
+      ),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleSearchTraces } = await import("./tools/search-traces.js");
+    return {
+      content: [{ type: "text", text: await handleSearchTraces(params) }],
+    };
+  }
+);
+
+server.tool(
+  "get_trace",
+  "Get full details of a single trace by ID. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
+  {
+    traceId: z.string().describe("The trace ID to retrieve"),
+    format: z
+      .enum(["digest", "json"])
+      .optional()
+      .describe(
+        "Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
+      ),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleGetTrace } = await import("./tools/get-trace.js");
+    return {
+      content: [{ type: "text", text: await handleGetTrace(params) }],
+    };
+  }
+);
+
+server.tool(
+  "get_analytics",
+  'Query analytics timeseries from LangWatch. Metrics use "category.name" format (e.g., "performance.completion_time"). Use discover_schema to see available metrics.',
+  {
+    metric: z
+      .string()
+      .describe(
+        'Metric in "category.name" format, e.g., "metadata.trace_id", "performance.total_cost"'
+      ),
+    aggregation: z
+      .string()
+      .optional()
+      .describe(
+        "Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg"
+      ),
+    startDate: z
+      .string()
+      .optional()
+      .describe('Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'),
+    endDate: z.string().optional().describe("End date. Default: now"),
+    timeZone: z.string().optional().describe("Timezone. Default: UTC"),
+    groupBy: z
+      .string()
+      .optional()
+      .describe(
+        "Group results by field. Use discover_schema for options."
+      ),
+    filters: z
+      .record(z.string(), z.array(z.string()))
+      .optional()
+      .describe("Filters to apply"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleGetAnalytics } = await import("./tools/get-analytics.js");
+    return {
+      content: [{ type: "text", text: await handleGetAnalytics(params) }],
+    };
+  }
+);
+
+server.tool(
+  "list_prompts",
+  "List all prompts configured in the LangWatch project.",
+  {},
+  async () => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleListPrompts } = await import("./tools/list-prompts.js");
+    return {
+      content: [{ type: "text", text: await handleListPrompts() }],
+    };
+  }
+);
+
+server.tool(
+  "get_prompt",
+  "Get a specific prompt by ID or handle, including messages, model config, and version history.",
+  {
+    idOrHandle: z.string().describe("Prompt ID or handle"),
+    version: z
+      .number()
+      .optional()
+      .describe("Specific version number (default: latest)"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleGetPrompt } = await import("./tools/get-prompt.js");
+    return {
+      content: [{ type: "text", text: await handleGetPrompt(params) }],
+    };
+  }
+);
+
+server.tool(
+  "create_prompt",
+  "Create a new prompt in the LangWatch project.",
+  {
+    name: z.string().describe("Prompt name"),
+    handle: z
+      .string()
+      .optional()
+      .describe("URL-friendly handle (auto-generated if omitted)"),
+    messages: z
+      .array(
+        z.object({
+          role: z
+            .enum(["system", "user", "assistant"])
+            .describe("Message role"),
+          content: z.string().describe("Message content"),
+        })
+      )
+      .describe("Prompt messages"),
+    model: z
+      .string()
+      .describe('Model name, e.g., "gpt-4o", "claude-sonnet-4-5-20250929"'),
+    modelProvider: z
+      .string()
+      .describe('Provider name, e.g., "openai", "anthropic"'),
+    description: z.string().optional().describe("Prompt description"),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleCreatePrompt } = await import("./tools/create-prompt.js");
+    return {
+      content: [{ type: "text", text: await handleCreatePrompt(params) }],
+    };
+  }
+);
+
+server.tool(
+  "update_prompt",
+  "Update an existing prompt or create a new version.",
+  {
+    idOrHandle: z.string().describe("Prompt ID or handle to update"),
+    messages: z
+      .array(
+        z.object({
+          role: z.enum(["system", "user", "assistant"]),
+          content: z.string(),
+        })
+      )
+      .optional()
+      .describe("Updated messages"),
+    model: z.string().optional().describe("Updated model name"),
+    modelProvider: z.string().optional().describe("Updated provider"),
+    commitMessage: z
+      .string()
+      .optional()
+      .describe("Commit message for the change"),
+    createVersion: z
+      .boolean()
+      .optional()
+      .describe(
+        "If true, creates a new version instead of updating in place"
+      ),
+  },
+  async (params) => {
+    const { requireApiKey } = await import("./config.js");
+    requireApiKey();
+    const { handleUpdatePrompt } = await import("./tools/update-prompt.js");
+    return {
+      content: [{ type: "text", text: await handleUpdatePrompt(params) }],
+    };
+  }
+);
+
 await server.connect(transport);

package/src/langwatch-api.ts

@@ -0,0 +1,265 @@
+import { getConfig, requireApiKey } from "./config.js";
+
+// --- Response types ---
+
+export interface TraceSearchResult {
+  trace_id: string;
+  formatted_trace?: string;
+  input?: { value: string };
+  output?: { value: string };
+  timestamps?: { started_at?: string | number };
+  metadata?: Record<string, unknown>;
+  error?: Record<string, unknown>;
+}
+
+export interface SearchTracesResponse {
+  traces: TraceSearchResult[];
+  pagination?: {
+    totalHits?: number;
+    scrollId?: string;
+  };
+}
+
+export interface TraceDetailResponse {
+  trace_id: string;
+  formatted_trace?: string;
+  input?: { value: string };
+  output?: { value: string };
+  timestamps?: {
+    started_at?: string | number;
+    updated_at?: string | number;
+    inserted_at?: string | number;
+  };
+  metadata?: {
+    user_id?: string;
+    thread_id?: string;
+    customer_id?: string;
+    labels?: string[];
+    [key: string]: unknown;
+  };
+  error?: Record<string, unknown>;
+  ascii_tree?: string;
+  evaluations?: Array<{
+    evaluator_id?: string;
+    name?: string;
+    score?: number;
+    passed?: boolean;
+    label?: string;
+  }>;
+  spans?: Array<{
+    span_id: string;
+    name?: string;
+    type?: string;
+    model?: string;
+    input?: { value: string };
+    output?: { value: string };
+    timestamps?: { started_at?: number; finished_at?: number };
+    metrics?: {
+      completion_time_ms?: number;
+      prompt_tokens?: number;
+      completion_tokens?: number;
+      tokens_estimated?: boolean;
+      cost?: number;
+    };
+  }>;
+}
+
+export interface AnalyticsBucket {
+  date: string;
+  [key: string]: unknown;
+}
+
+export interface AnalyticsTimeseriesResponse {
+  currentPeriod: AnalyticsBucket[];
+  previousPeriod: AnalyticsBucket[];
+}
+
+export interface PromptSummary {
+  id?: string;
+  handle?: string;
+  name?: string;
+  description?: string | null;
+  latestVersionNumber?: number;
+  version?: number;
+}
+
+export interface PromptVersion {
+  version?: number;
+  commitMessage?: string;
+  model?: string;
+  modelProvider?: string;
+  messages?: Array<{ role: string; content: string }>;
+}
+
+export interface PromptDetailResponse extends PromptSummary {
+  versions?: PromptVersion[];
+  model?: string;
+  modelProvider?: string;
+  messages?: Array<{ role: string; content: string }>;
+  prompt?: Array<{ role: string; content: string }>;
+}
+
+export interface PromptMutationResponse {
+  id?: string;
+  handle?: string;
+  name?: string;
+  latestVersionNumber?: number;
+}
+
+// --- HTTP client ---
+
+/**
+ * Sends an HTTP request to the LangWatch API.
+ *
+ * Builds the full URL from the configured endpoint, adds authentication,
+ * and handles JSON serialization/deserialization.
+ *
+ * @throws Error with status code and response body when the response is not OK
+ */
+async function makeRequest(
+  method: "GET" | "POST",
+  path: string,
+  body?: unknown
+): Promise<unknown> {
+  const url = getConfig().endpoint + path;
+  const headers: Record<string, string> = {
+    "X-Auth-Token": requireApiKey(),
+  };
+
+  if (method === "POST") {
+    headers["Content-Type"] = "application/json";
+  }
+
+  const response = await fetch(url, {
+    method,
+    headers,
+    ...(body !== undefined ? { body: JSON.stringify(body) } : {}),
+  });
+
+  if (!response.ok) {
+    const responseBody = await response.text();
+    throw new Error(
+      `LangWatch API error ${response.status}: ${responseBody}`
+    );
+  }
+
+  return response.json();
+}
+
+/** Searches traces with optional filters and pagination. */
+export async function searchTraces(params: {
+  query?: string;
+  filters?: Record<string, string[]>;
+  startDate: number;
+  endDate: number;
+  pageSize?: number;
+  pageOffset?: number;
+  scrollId?: string;
+  format?: "digest" | "json";
+}): Promise<SearchTracesResponse> {
+  const { format = "digest", ...rest } = params;
+  return makeRequest("POST", "/api/traces/search", {
+    ...rest,
+    format,
+  }) as Promise<SearchTracesResponse>;
+}
+
+/** Retrieves a single trace by its ID. */
+export async function getTraceById(
+  traceId: string,
+  format: "digest" | "json" = "digest"
+): Promise<TraceDetailResponse> {
+  return makeRequest(
+    "GET",
+    `/api/traces/${encodeURIComponent(traceId)}?format=${format}`
+  ) as Promise<TraceDetailResponse>;
+}
+
+/** Fetches analytics timeseries data for the given metrics and date range. */
+export async function getAnalyticsTimeseries(params: {
+  series: Array<{
+    metric: string;
+    aggregation: string;
+    key?: string;
+    subkey?: string;
+  }>;
+  startDate: number;
+  endDate: number;
+  timeZone?: string;
+  groupBy?: string;
+  groupByKey?: string;
+  filters?: Record<string, string[]>;
+}): Promise<AnalyticsTimeseriesResponse> {
+  return makeRequest(
+    "POST",
+    "/api/analytics/timeseries",
+    params
+  ) as Promise<AnalyticsTimeseriesResponse>;
+}
+
+/** Lists all prompts in the project. */
+export async function listPrompts(): Promise<PromptSummary[]> {
+  return makeRequest("GET", "/api/prompts") as Promise<PromptSummary[]>;
+}
+
+/** Retrieves a single prompt by ID or handle. */
+export async function getPrompt(
+  idOrHandle: string,
+  version?: number
+): Promise<PromptDetailResponse> {
+  const query = version != null ? `?version=${version}` : "";
+  return makeRequest(
+    "GET",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
+  ) as Promise<PromptDetailResponse>;
+}
+
+/** Creates a new prompt. */
+export async function createPrompt(data: {
+  name: string;
+  handle?: string;
+  messages: Array<{ role: string; content: string }>;
+  model: string;
+  modelProvider: string;
+  description?: string;
+}): Promise<PromptMutationResponse> {
+  return makeRequest(
+    "POST",
+    "/api/prompts",
+    data
+  ) as Promise<PromptMutationResponse>;
+}
+
+/** Updates an existing prompt by ID or handle. */
+export async function updatePrompt(
+  idOrHandle: string,
+  data: {
+    messages?: Array<{ role: string; content: string }>;
+    model?: string;
+    modelProvider?: string;
+    commitMessage?: string;
+  }
+): Promise<PromptMutationResponse> {
+  return makeRequest(
+    "POST",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}`,
+    data
+  ) as Promise<PromptMutationResponse>;
+}
+
+/** Creates a new version of an existing prompt. */
+export async function createPromptVersion(
+  idOrHandle: string,
+  data: {
+    messages?: Array<{ role: string; content: string }>;
+    model?: string;
+    modelProvider?: string;
+    commitMessage?: string;
+  }
+): Promise<PromptMutationResponse> {
+  return makeRequest(
+    "POST",
+    `/api/prompts/${encodeURIComponent(idOrHandle)}/versions`,
+    data
+  ) as Promise<PromptMutationResponse>;
+}

package/src/schemas/analytics-groups.ts

@@ -0,0 +1,78 @@
+export interface GroupByInfo {
+  name: string;
+  label: string;
+  description: string;
+}
+
+export const analyticsGroups: GroupByInfo[] = [
+  {
+    name: "topics.topics",
+    label: "Topic",
+    description: "Group by topic classification",
+  },
+  {
+    name: "metadata.user_id",
+    label: "User",
+    description: "Group by user ID",
+  },
+  {
+    name: "metadata.thread_id",
+    label: "Thread",
+    description: "Group by conversation thread",
+  },
+  {
+    name: "metadata.customer_id",
+    label: "Customer ID",
+    description: "Group by customer/organization",
+  },
+  {
+    name: "metadata.labels",
+    label: "Label",
+    description: "Group by custom labels",
+  },
+  {
+    name: "metadata.model",
+    label: "Model",
+    description: "Group by LLM model name",
+  },
+  {
+    name: "metadata.span_type",
+    label: "Span Type",
+    description: "Group by span type (llm, tool, agent, etc.)",
+  },
+  {
+    name: "sentiment.input_sentiment",
+    label: "Input Sentiment",
+    description: "Group by detected input sentiment (positive, negative, neutral)",
+  },
+  {
+    name: "sentiment.thumbs_up_down",
+    label: "Thumbs Up/Down",
+    description: "Group by user feedback (positive, negative, neutral)",
+  },
+  {
+    name: "events.event_type",
+    label: "Event Type",
+    description: "Group by event type",
+  },
+  {
+    name: "evaluations.evaluation_passed",
+    label: "Evaluation Passed",
+    description: "Group by evaluation pass/fail status",
+  },
+  {
+    name: "evaluations.evaluation_label",
+    label: "Evaluation Label",
+    description: "Group by evaluation label result",
+  },
+  {
+    name: "evaluations.evaluation_processing_state",
+    label: "Evaluation Processing State",
+    description: "Group by evaluation processing state",
+  },
+  {
+    name: "error.has_error",
+    label: "Contains Error",
+    description: "Group by whether the trace contains an error",
+  },
+];