@langwatch/mcp-server 2.0.0 → 2.2.0

package/src/tools/run-query.ts

@@ -0,0 +1,47 @@
+/**
+ * Handles the `run_query` MCP tool: one LangWatchQL statement rendered as a markdown table. Rows
+ * are capped, and the cap is announced, so an unaggregated query cannot flood the conversation.
+ * @see specs/mcp-server/schema-discovery.feature
+ */
+
+import { runQuery } from "../langwatch-api-query.ts";
+import { markdownTable } from "../utils/markdown-table.ts";
+
+/** Rows the table prints. Beyond this, aggregate or export through the CLI. */
+export const RUN_QUERY_ROW_CAP = 50;
+
+export async function handleRunQuery(params: {
+  sql: string;
+  parameters?: Record<string, string | number | boolean | null>;
+}): Promise<string> {
+  const result = await runQuery(params);
+  const headers = result.columns.map((column) => column.name);
+  const shown = result.rows.slice(0, RUN_QUERY_ROW_CAP);
+
+  const lines: string[] = [];
+  lines.push(
+    markdownTable({
+      headers,
+      rows: shown,
+      emptyMessage: "The statement ran and returned no rows.",
+    }),
+  );
+  lines.push("");
+  lines.push(
+    `Returned ${result.rows.length} row${result.rows.length === 1 ? "" : "s"} in ${result.statistics.elapsedMs}ms.`,
+  );
+  if (result.rows.length > shown.length) {
+    lines.push(
+      `Showing the first ${shown.length}. Aggregate further, or export every row with \`langwatch query "<statement>" --format jsonl\`.`,
+    );
+  }
+  if (result.truncated) {
+    lines.push(
+      "The result hit a response ceiling and was cut short, so this is not the whole answer.",
+    );
+  }
+  for (const diagnostic of result.diagnostics) {
+    lines.push(`${diagnostic.code}: ${diagnostic.message}`);
+  }
+  return lines.join("\n");
+}

package/src/tools/run-test-suite.ts

@@ -0,0 +1,27 @@
+import type { RunParameters, RunPlanTarget } from "../langwatch-api-run-plans.ts";
+import { runTestSuite as apiRunTestSuite } from "../langwatch-api-test-suites.ts";
+import { toWireTargets } from "../schemas/run-plan.ts";
+import { formatRunPlanRun } from "./format-run-plan.ts";
+
+/**
+ * Handles the platform_run_test_suite MCP tool invocation.
+ */
+export async function handleRunTestSuite(params: {
+  id: string;
+  targets: RunPlanTarget[];
+  name?: string;
+  repeatCount?: number;
+  simulatorModel?: string;
+  judgeModel?: string;
+  parameters?: RunParameters;
+  note?: string;
+  idempotencyKey?: string;
+}): Promise<string> {
+  const { targets, ...rest } = params;
+  const result = await apiRunTestSuite({
+    ...rest,
+    targets: toWireTargets(targets),
+  });
+
+  return formatRunPlanRun(result);
+}

package/src/tools/run-workflow.ts

@@ -1,12 +1,9 @@
-import { runWorkflow as apiRunWorkflow } from "../langwatch-api-workflows.js";
+import { runWorkflow as apiRunWorkflow } from "../langwatch-api-workflows.ts";
 
 /**
  * Handles the platform_run_workflow MCP tool invocation.
  */
-export async function handleRunWorkflow(params: {
-  id: string;
-  input?: string;
-}): Promise<string> {
+export async function handleRunWorkflow(params: { id: string; input?: string }): Promise<string> {
   let parsedInput: Record<string, unknown> = {};
   if (params.input) {
     try {

package/src/tools/search-traces.ts

@@ -1,17 +1,142 @@
-import { searchTraces as apiSearchTraces } from "../langwatch-api.js";
-import { parseRelativeDate } from "../utils/date-parsing.js";
-import { formatEvaluationLines } from "../utils/format-evaluations.js";
+import { searchTraces as apiSearchTraces, type TraceSearchResult } from "../langwatch-api.ts";
+import { parseRelativeDate } from "../utils/date-parsing.ts";
+import { formatEvaluationLines } from "../utils/format-evaluations.ts";
+import { looksLikeTraceId } from "../utils/trace-id-shape.ts";
+
+const HOUR_MS = 3600000;
+const DAY_MS = 86400000;
+
+/** Default window for a text search. */
+const TEXT_SEARCH_WINDOW_MS = DAY_MS;
+
+/**
+ * Default window when trace ids are named: an exact-match intent must not be
+ * answered against yesterday alone. Mirrors the platform's 90-day id lookup
+ * bound (ADR-164).
+ */
+const ID_LOOKUP_WINDOW_MS = 90 * DAY_MS;
+
+function previewLine({
+  label,
+  text,
+}: {
+  label: string;
+  text: { value: string } | undefined;
+}): string {
+  const value = text?.value ? String(text.value) : "N/A";
+  return `- **${label}**: ${value.slice(0, 100)}${value.length > 100 ? "..." : ""}`;
+}
+
+function traceLines(trace: TraceSearchResult): string[] {
+  const lines = [`### Trace: ${trace.trace_id}`];
+
+  if (trace.formatted_trace) {
+    lines.push(trace.formatted_trace);
+  } else {
+    lines.push(previewLine({ label: "Input", text: trace.input }));
+    lines.push(previewLine({ label: "Output", text: trace.output }));
+  }
+
+  if (trace.timestamps) {
+    lines.push(`- **Time**: ${trace.timestamps.started_at || "N/A"}`);
+  }
+  if (trace.error) {
+    lines.push(`- **Error**: ${JSON.stringify(trace.error)}`);
+  }
+  if (trace.evaluations && trace.evaluations.length > 0) {
+    lines.push(...formatEvaluationLines(trace.evaluations));
+  }
+  lines.push("");
+  return lines;
+}
+
+function describeSpan({ ms }: { ms: number }): string {
+  if (ms % DAY_MS === 0) {
+    const days = ms / DAY_MS;
+    return days === 1 ? "24 hours" : `${days} days`;
+  }
+  const hours = Math.round(ms / HOUR_MS);
+  return hours === 1 ? "1 hour" : `${hours} hours`;
+}
+
+/** The next sensible window to offer, or undefined once the span is already wide. */
+function suggestWiderWindow({ spanMs }: { spanMs: number }): string | undefined {
+  if (spanMs < 7 * DAY_MS) return "7d";
+  if (spanMs < 30 * DAY_MS) return "30d";
+  if (spanMs < 90 * DAY_MS) return "90d";
+  return undefined;
+}
+
+/** An empty result states the searched window and names the tool that looks an id up. */
+function buildEmptyResult({
+  query,
+  traceIds,
+  spanMs,
+  windowWasDefaulted,
+  startDate,
+  endDate,
+}: {
+  query?: string;
+  traceIds?: string[];
+  spanMs: number;
+  windowWasDefaulted: boolean;
+  startDate: number;
+  endDate: number;
+}): string {
+  const hasTraceIds = (traceIds?.length ?? 0) > 0;
+  const lines = [
+    hasTraceIds
+      ? "No traces matched the requested trace ids in the searched window."
+      : "No traces found matching your query.",
+    "",
+  ];
+
+  const range = `${new Date(startDate).toISOString()} to ${new Date(endDate).toISOString()}`;
+  lines.push(
+    windowWasDefaulted
+      ? `Searched the last ${describeSpan({ ms: spanMs })} (the default): ${range}.`
+      : `Searched ${range}.`,
+  );
+  lines.push("", "Tips:");
+
+  if (query && looksLikeTraceId(query)) {
+    lines.push(
+      `- "${query}" looks like a trace id. Free text never matches trace ids — use \`get_trace\` with traceId: "${query}".`,
+    );
+  }
+
+  const wider = suggestWiderWindow({ spanMs });
+  if (wider) {
+    lines.push(
+      `- Widen the window with startDate, e.g. startDate: "${wider}". Accepts h (hours), d (days), w (weeks), m (30-day months), or an ISO date.`,
+    );
+  }
+
+  if (hasTraceIds) {
+    lines.push(
+      "- Retry with an earlier startDate, or relax the query and filters if you supplied them.",
+      "- Looking up one full id? `get_trace` has no time window. A unique 8–31 character hex prefix searches the last 90 days.",
+    );
+  } else {
+    lines.push(
+      "- Looking up a known trace id? `get_trace` takes a traceId. A full id has no time window; a unique 8–31 character hex prefix searches the last 90 days.",
+      "- To fetch several known trace ids at once, pass `traceIds` instead of `query`.",
+    );
+  }
+
+  return lines.join("\n");
+}
 
 /**
- * Handles the search_traces MCP tool invocation.
- *
- * Searches LangWatch traces with optional filters, text query, and date range.
- * In digest mode (default), returns AI-readable formatted digests per trace.
- * In json mode, returns the full raw JSON.
+ * Handles the search_traces MCP tool: searches with optional filters, text
+ * query, trace ids and date range. Digest mode (default) returns AI-readable
+ * digests per trace; json mode returns the full raw JSON.
  */
 export async function handleSearchTraces(params: {
   query?: string;
+  traceIds?: string[];
   filters?: Record<string, string[]>;
+  filter?: string;
   startDate?: string;
   endDate?: string;
   pageSize?: number;
@@ -19,15 +144,27 @@ export async function handleSearchTraces(params: {
   format?: "digest" | "json";
 }): Promise<string> {
   const now = Date.now();
-  const startDate = params.startDate
-    ? parseRelativeDate(params.startDate)
-    : now - 86400000;
-  const endDate = params.endDate ? parseRelativeDate(params.endDate) : now;
+  const hasTraceIds = (params.traceIds?.length ?? 0) > 0;
+  const defaultSpanMs = hasTraceIds ? ID_LOOKUP_WINDOW_MS : TEXT_SEARCH_WINDOW_MS;
+
+  const endDate = params.endDate !== undefined ? parseRelativeDate(params.endDate) : now;
+  const startDate =
+    params.startDate !== undefined ? parseRelativeDate(params.startDate) : endDate - defaultSpanMs;
+
+  if (startDate >= endDate) {
+    throw new Error(
+      `Invalid trace search window: startDate (${new Date(startDate).toISOString()}) must be before endDate (${new Date(endDate).toISOString()}).`,
+    );
+  }
   const format = params.format ?? "digest";
 
   const result = await apiSearchTraces({
     query: params.query,
+    traceIds: params.traceIds,
     filters: params.filters,
+    // Forwarded as itself: the filter language and the free-text query are two
+    // different searches, and the server combines them.
+    ...(params.filter ? { filter: params.filter } : {}),
     startDate,
     endDate,
     pageSize: params.pageSize ?? 25,
@@ -37,7 +174,14 @@ export async function handleSearchTraces(params: {
 
   const traces = result.traces ?? [];
   if (traces.length === 0) {
-    return "No traces found matching your query.";
+    return buildEmptyResult({
+      query: params.query,
+      traceIds: params.traceIds,
+      spanMs: endDate - startDate,
+      windowWasDefaulted: params.startDate === undefined,
+      startDate,
+      endDate,
+    });
   }
 
   if (format === "json") {
@@ -45,50 +189,20 @@ export async function handleSearchTraces(params: {
   }
 
   const lines: string[] = [];
-  lines.push(
-    `Found ${result.pagination?.totalHits ?? traces.length} traces:\n`
-  );
+  lines.push(`Found ${result.pagination?.totalHits ?? traces.length} traces:\n`);
 
   for (const trace of traces) {
-    lines.push(`### Trace: ${trace.trace_id}`);
-
-    if (trace.formatted_trace) {
-      lines.push(trace.formatted_trace);
-    } else {
-      const inputStr = trace.input?.value
-        ? String(trace.input.value)
-        : "N/A";
-      const outputStr = trace.output?.value
-        ? String(trace.output.value)
-        : "N/A";
-      lines.push(
-        `- **Input**: ${inputStr.slice(0, 100)}${inputStr.length > 100 ? "..." : ""}`
-      );
-      lines.push(
-        `- **Output**: ${outputStr.slice(0, 100)}${outputStr.length > 100 ? "..." : ""}`
-      );
-    }
-
-    if (trace.timestamps) {
-      lines.push(`- **Time**: ${trace.timestamps.started_at || "N/A"}`);
-    }
-    if (trace.error) {
-      lines.push(`- **Error**: ${JSON.stringify(trace.error)}`);
-    }
-    if (trace.evaluations && trace.evaluations.length > 0) {
-      lines.push(...formatEvaluationLines(trace.evaluations));
-    }
-    lines.push("");
+    lines.push(...traceLines(trace));
   }
 
   if (result.pagination?.scrollId) {
     lines.push(
-      `\n**More results available.** Use scrollId: "${result.pagination.scrollId}" to get next page.`
+      `\n**More results available.** Use scrollId: "${result.pagination.scrollId}" to get next page.`,
     );
   }
 
   lines.push(
-    '\n> Tip: Use `get_trace` with a trace_id for full details. Use `search_traces` with `format: "json"` for raw data. Use `discover_schema` to see available filter fields.'
+    '\n> Tip: Use `get_trace` with a traceId for full details. Use `search_traces` with `format: "json"` for raw data. Use `discover_schema` to see available filter fields.',
   );
 
   return lines.join("\n");

package/src/tools/set-model-provider.ts

@@ -1,10 +1,8 @@
-import { setModelProvider as apiSetModelProvider } from "../langwatch-api-model-providers.js";
+import { setModelProvider as apiSetModelProvider } from "../langwatch-api-model-providers.ts";
 
 /**
- * Handles the platform_set_model_provider MCP tool invocation.
- *
- * Creates or updates a model provider configuration, including API keys.
- * Returns confirmation with the updated provider status.
+ * Handles the platform_set_model_provider MCP tool: creates or updates
+ * a provider config (including API keys) and returns its status.
  */
 export async function handleSetModelProvider(params: {
   provider: string;
@@ -18,7 +16,7 @@ export async function handleSetModelProvider(params: {
 
   if (!updated) {
     throw new Error(
-      `Model provider "${params.provider}" was not found in the response. The provider name may be invalid.`
+      `Model provider "${params.provider}" was not found in the response. The provider name may be invalid.`,
     );
   }
 

package/src/tools/test-agent.ts

@@ -0,0 +1,18 @@
+import { testAgent as apiTestAgent } from "../langwatch-api-agents.ts";
+
+/**
+ * Handles the platform_test_agent MCP tool invocation.
+ *
+ * @see specs/agents/agent-test-run.feature
+ */
+export async function handleTestAgent(params: { id: string }): Promise<string> {
+  const run = await apiTestAgent(params.id);
+  return [
+    'Test run scheduled. The user sends "ping"; the run succeeds when the agent answers.',
+    "",
+    `**Scenario run ID:** ${run.scenarioRunId}`,
+    `**Batch run ID:** ${run.batchRunId}`,
+    "",
+    `Follow it with platform_get_simulation_run using scenarioRunId "${run.scenarioRunId}".`,
+  ].join("\n");
+}

package/src/tools/update-agent.ts

@@ -1,4 +1,4 @@
-import { updateAgent as apiUpdateAgent } from "../langwatch-api-agents.js";
+import { updateAgent as apiUpdateAgent } from "../langwatch-api-agents.ts";
 
 export async function handleUpdateAgent(params: {
   id: string;

package/src/tools/update-dashboard-widget.ts

@@ -0,0 +1,97 @@
+/**
+ * The `update_dashboard_widget` MCP tool over the widget PATCH: a field left out keeps its
+ * stored value. The widget must sit on the named dashboard, so an agent never edits one it
+ * found somewhere else by mistake. Spec: modules/dashboard/specs/dashboard-widget-mcp.feature
+ */
+
+import {
+  type DashboardWidget,
+  getDashboardWidget,
+  updateDashboardWidget,
+} from "../langwatch-api-dashboard-widgets.js";
+import { getCurrentProject } from "../langwatch-api-projects.js";
+import { LangWatchApiError } from "../langwatch-api.js";
+import { assertDashboardExists } from "./dashboard-exists.js";
+import {
+  type DashboardWidgetQueriesInput,
+  normalizeDashboardWidgetQueries,
+} from "./dashboard-widget-queries.js";
+
+export interface UpdateDashboardWidgetParams {
+  dashboardId: string;
+  widgetId: string;
+  name?: string;
+  code?: string;
+  queries?: DashboardWidgetQueriesInput;
+  description?: string;
+}
+
+export async function handleUpdateDashboardWidget(
+  params: UpdateDashboardWidgetParams,
+): Promise<string> {
+  const { dashboardId, widgetId, name, code, description } = params;
+  if (
+    name === undefined &&
+    code === undefined &&
+    params.queries === undefined &&
+    description === undefined
+  ) {
+    throw new Error("Provide at least one of name, code, queries or description to change.");
+  }
+  const queries =
+    params.queries === undefined
+      ? undefined
+      : normalizeDashboardWidgetQueries({ queries: params.queries });
+
+  const project = await getCurrentProject();
+  await assertDashboardExists({ projectId: project.id, dashboardId });
+  const current = await getWidgetOnDashboard({ projectId: project.id, widgetId, dashboardId });
+
+  const widget = await updateDashboardWidget({
+    projectId: project.id,
+    widgetId: current.id,
+    ...(name === undefined ? {} : { name }),
+    ...(code === undefined ? {} : { code }),
+    ...(queries === undefined ? {} : { queries }),
+    ...(description === undefined ? {} : { description }),
+  });
+
+  return [
+    `Widget "${widget.name}" updated on dashboard "${dashboardId}".`,
+    `**Widget ID**: ${widget.id}`,
+    `**Queries**: ${widget.definition.queries.map((query) => query.name).join(", ") || "(none)"}`,
+    "",
+    "```json",
+    JSON.stringify(widget, null, 2),
+    "```",
+  ].join("\n");
+}
+
+/** The widget, refused by id when it is not in this project or not on this dashboard. */
+async function getWidgetOnDashboard({
+  projectId,
+  widgetId,
+  dashboardId,
+}: {
+  projectId: string;
+  widgetId: string;
+  dashboardId: string;
+}): Promise<DashboardWidget> {
+  let widget: DashboardWidget;
+  try {
+    widget = await getDashboardWidget({ projectId, widgetId });
+  } catch (error) {
+    if (error instanceof LangWatchApiError && error.status === 404) {
+      throw new Error(
+        `No widget with ID "${widgetId}" was found in this project. Use the widget ID add_dashboard_widget returned.`,
+      );
+    }
+    throw error;
+  }
+  if (widget.dashboardId !== dashboardId) {
+    throw new Error(
+      `Widget "${widgetId}" is not on dashboard "${dashboardId}". Pass the dashboard the widget was added to.`,
+    );
+  }
+  return widget;
+}

package/src/tools/update-dataset-record.ts

@@ -1,4 +1,4 @@
-import { updateDatasetRecord as apiUpdateDatasetRecord } from "../langwatch-api-datasets.js";
+import { updateDatasetRecord as apiUpdateDatasetRecord } from "../langwatch-api-datasets.ts";
 
 /**
  * Handles the platform_update_dataset_record MCP tool invocation.

package/src/tools/update-dataset.ts

@@ -1,12 +1,10 @@
-import { updateDataset as apiUpdateDataset } from "../langwatch-api-datasets.js";
-import type { DatasetColumnType } from "../langwatch-api-datasets.js";
-import { formatDatasetMutationDetails } from "./format-dataset-mutation.js";
+import { updateDataset as apiUpdateDataset } from "../langwatch-api-datasets.ts";
+import type { DatasetColumnType } from "../langwatch-api-datasets.ts";
+import { formatDatasetMutationDetails } from "./format-dataset-mutation.ts";
 
 /**
- * Handles the platform_update_dataset MCP tool invocation.
- *
- * Updates an existing dataset and returns a confirmation
- * with the updated details.
+ * Handles the platform_update_dataset MCP tool: updates a dataset and
+ * returns a confirmation with the updated details.
  */
 export async function handleUpdateDataset(params: {
   slugOrId: string;

package/src/tools/update-evaluator.ts

@@ -1,13 +1,11 @@
 import {
   updateEvaluator as apiUpdateEvaluator,
   getEvaluatorType,
-} from "../langwatch-api-evaluators.js";
+} from "../langwatch-api-evaluators.ts";
 
 /**
- * Handles the platform_update_evaluator MCP tool invocation.
- *
- * Updates an existing evaluator and returns a confirmation
- * with the updated details.
+ * Handles the platform_update_evaluator MCP tool: updates an evaluator
+ * and returns a confirmation with the updated details.
  */
 export async function handleUpdateEvaluator(params: {
   evaluatorId: string;

package/src/tools/update-project.ts

@@ -1,4 +1,4 @@
-import { updateProject as apiUpdateProject } from "../langwatch-api-projects.js";
+import { updateProject as apiUpdateProject } from "../langwatch-api-projects.ts";
 
 export async function handleUpdateProject(params: {
   id: string;