@langwatch/mcp-server 2.1.0 → 3.0.0

package/src/tools/discover-scenario-schema.ts

@@ -32,7 +32,9 @@ export function formatScenarioSchema(): string {
     "A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:",
   );
   lines.push("```json");
-  lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
+  lines.push(
+    '{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }',
+  );
   lines.push("```");
   lines.push(
     "- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
@@ -51,8 +53,12 @@ export function formatScenarioSchema(): string {
   lines.push('  "evaluatorId": "<id from platform_list_evaluators>",');
   lines.push('  "required": true,');
   lines.push('  "mappings": {');
-  lines.push('    "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
-  lines.push('    "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
+  lines.push(
+    '    "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },',
+  );
+  lines.push(
+    '    "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }',
+  );
   lines.push("  }");
   lines.push("}] }");
   lines.push("```");
@@ -60,30 +66,30 @@ export function formatScenarioSchema(): string {
     "Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:",
   );
   lines.push(
-    "- **conversation**: `[\"first_user_message\"]`, `[\"last_agent_message\"]`, `[\"transcript\"]` (role: content lines) or `[\"messages\"]` (JSON)",
+    '- **conversation**: `["first_user_message"]`, `["last_agent_message"]`, `["transcript"]` (role: content lines) or `["messages"]` (JSON)',
   );
   lines.push(
-    "- **scenario**: `[\"situation\"]`, `[\"criteria\"]` or `[\"fields\", \"<identifier>\"]` (a field the suite declares; not on a run plan)",
+    '- **scenario**: `["situation"]`, `["criteria"]` or `["fields", "<identifier>"]` (a field the suite declares; not on a run plan)',
   );
   lines.push(
-    "- **trace**: `[\"contexts\"]` (the retrieved contexts of the run's traces) or `[\"tool_calls\", \"<toolName>\", \"input\" | \"output\"]` (the last matching tool call)",
-  );
-  lines.push(
-    '- a literal: `{ "type": "value", "value": "..." }`',
+    '- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)',
   );
+  lines.push('- a literal: `{ "type": "value", "value": "..." }`');
   lines.push(
     "A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`.",
   );
 
   lines.push("\n## Writing a Good Situation\n");
+  lines.push("The situation drives the user simulator. Include these elements:");
   lines.push(
-    "The situation drives the user simulator. Include these elements:",
+    "- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)",
   );
-  lines.push("- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)");
   lines.push("- **Emotional state**: How are they feeling? (e.g., frustrated, anxious, impatient)");
   lines.push("- **Background/Context**: What happened before this conversation?");
   lines.push("- **Intent**: What do they want to accomplish?");
-  lines.push("- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)");
+  lines.push(
+    "- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)",
+  );
   lines.push("\nExample:");
   lines.push("```");
   lines.push("User is a small business owner stressed about tax deadline.");
@@ -94,17 +100,31 @@ export function formatScenarioSchema(): string {
   lines.push("```");
 
   lines.push("\n## Writing Good Criteria\n");
-  lines.push("Criteria are what the judge uses to pass or fail the agent. Each criterion should be:");
-  lines.push("- **Specific and testable** — not vague like \"responds helpfully\"");
-  lines.push("- **Behavioral** — describes what the agent should *do*, not how it works internally");
+  lines.push(
+    "Criteria are what the judge uses to pass or fail the agent. Each criterion should be:",
+  );
+  lines.push('- **Specific and testable** — not vague like "responds helpfully"');
+  lines.push(
+    "- **Behavioral** — describes what the agent should *do*, not how it works internally",
+  );
   lines.push("- **Independent** — each criterion checks one thing");
   lines.push("\nGood criteria patterns:");
-  lines.push("- **Information gathering**: \"Agent asks for the user's account number before proceeding\"");
-  lines.push("- **Safety/guardrails**: \"Agent does not reveal internal system details or error stack traces\"");
-  lines.push("- **Clarification**: \"Agent asks clarifying questions before taking irreversible action\"");
-  lines.push("- **Tone**: \"Agent maintains a professional and empathetic tone throughout\"");
-  lines.push("- **Completeness**: \"Agent confirms the user understands the solution before ending\"");
-  lines.push("- **Domain-specific**: \"Agent recommends releasing a wild frog rather than keeping it as a pet\"");
+  lines.push(
+    '- **Information gathering**: "Agent asks for the user\'s account number before proceeding"',
+  );
+  lines.push(
+    '- **Safety/guardrails**: "Agent does not reveal internal system details or error stack traces"',
+  );
+  lines.push(
+    '- **Clarification**: "Agent asks clarifying questions before taking irreversible action"',
+  );
+  lines.push('- **Tone**: "Agent maintains a professional and empathetic tone throughout"');
+  lines.push(
+    '- **Completeness**: "Agent confirms the user understands the solution before ending"',
+  );
+  lines.push(
+    '- **Domain-specific**: "Agent recommends releasing a wild frog rather than keeping it as a pet"',
+  );
   lines.push("\nAvoid vague criteria like:");
   lines.push('- "Responds correctly" — correct how?');
   lines.push('- "Is helpful" — helpful in what way?');

package/src/tools/discover-schema.ts

@@ -1,25 +1,39 @@
-import { filterFields } from "../schemas/filter-fields.js";
-import { analyticsMetrics } from "../schemas/analytics-metrics.js";
-import { analyticsGroups } from "../schemas/analytics-groups.js";
+import { getQueryReference, type QueryReferenceResponse } from "../langwatch-api-query.ts";
+import { analyticsGroups } from "../schemas/analytics-groups.ts";
+import { analyticsMetrics } from "../schemas/analytics-metrics.ts";
+import { markdownTable } from "../utils/markdown-table.ts";
 
-export type Category =
-  | "filters"
-  | "metrics"
-  | "aggregations"
-  | "groups"
-  | "all";
+export type Category = "filters" | "lwql" | "metrics" | "aggregations" | "groups" | "all";
 
 /**
- * Formats the LangWatch analytics schema into human-readable markdown.
- *
- * Returns documentation for the requested category of schema elements
- * (filter fields, metrics, aggregation types, or group-by options).
+ * Categories the platform answers, not this package — which is why
+ * `formatSchema` is async: they are read from `GET /api/v1/query/reference`
+ * and need the API key.
  */
-export function formatSchema(category: Category): string {
+const PLATFORM_BACKED: ReadonlySet<Category> = new Set(["filters", "lwql", "all"]);
+
+export function needsQueryReference(category: Category): boolean {
+  return PLATFORM_BACKED.has(category);
+}
+
+/**
+ * Formats the LangWatch query schema into markdown an agent can act on.
+ * `reference` is required for the platform-backed categories and ignored by
+ * the others, so a caller that fetched it once can reuse it.
+ */
+export async function formatSchema(
+  category: Category,
+  reference?: QueryReferenceResponse,
+): Promise<string> {
   const sections: string[] = [];
+  const resolved =
+    needsQueryReference(category) && !reference ? await getQueryReference() : reference;
 
   if (category === "filters" || category === "all") {
-    sections.push(formatFilters());
+    sections.push(formatFilters(resolved));
+  }
+  if (category === "lwql" || category === "all") {
+    sections.push(formatLangWatchQL(resolved));
   }
   if (category === "metrics" || category === "all") {
     sections.push(formatMetrics());
@@ -30,30 +44,140 @@ export function formatSchema(category: Category): string {
   if (category === "groups" || category === "all") {
     sections.push(formatGroups());
   }
+  if (category === "all" && resolved) {
+    sections.push(formatDecisionTable(resolved));
+  }
 
   return sections.join("\n\n");
 }
 
-function formatFilters(): string {
-  const lines = ["## Available Filter Fields", ""];
+/** What the field's values column says: the closed set, or where to read an open one. */
+const knownValuesCell = (field: { knownValues: readonly string[]; facetable: boolean }): string => {
+  if (field.knownValues.length > 0) return field.knownValues.join(", ");
+  return field.facetable ? "open set, read them from GET /api/v1/traces/facets" : "";
+};
+
+function formatFilters(reference?: QueryReferenceResponse): string {
+  if (!reference) {
+    return "## Trace Filter Fields\n\nUnavailable: the platform could not be reached.";
+  }
+  const lines = ["## Trace Filter Fields", ""];
+  lines.push(
+    "Send one of these as the `filter` parameter of `search_traces`, in the language the Trace Explorer's search bar speaks: `status:error AND model:gpt-*`.",
+  );
+  lines.push(
+    "The `filters` parameter is the older per-field map and still works; both are combined when you send both.",
+  );
+  lines.push("");
+  lines.push(
+    markdownTable({
+      headers: ["Field", "Type", "Group", "Known values"],
+      rows: reference.traceFilter.fields.map((field) => ({
+        Field: field.name,
+        Type: field.valueType,
+        Group: field.group ?? "",
+        "Known values": knownValuesCell(field),
+      })),
+    }),
+  );
+  lines.push("");
+  lines.push("### Attribute namespaces", "");
   lines.push(
-    "Use these in the `filters` parameter of `search_traces` and `get_analytics`."
+    markdownTable({
+      headers: ["Prefix", "Matches", "Older spellings"],
+      rows: reference.traceFilter.dynamicPrefixes.map((prefix) => ({
+        Prefix: `${prefix.prefix}<key>`,
+        Matches: prefix.description,
+        "Older spellings": prefix.aliases.join(", "),
+      })),
+    }),
   );
-  lines.push('Format: `{ "field_name": ["value1", "value2"] }`');
   lines.push("");
-  for (const f of filterFields) {
+  lines.push("### Syntax", "");
+  lines.push(reference.traceFilter.syntax);
+  lines.push("");
+  lines.push("### Worked filters", "");
+  for (const example of reference.examples.filter(
+    (candidate) => candidate.language === "trace-filter",
+  )) {
+    lines.push(`- \`${example.text}\` — ${example.title}`);
+  }
+  return lines.join("\n");
+}
+
+function formatLangWatchQL(reference?: QueryReferenceResponse): string {
+  if (!reference) {
+    return "## Analytics SQL\n\nUnavailable: the platform could not be reached.";
+  }
+  if (!reference.lwql.enabled) {
+    return [
+      "## Analytics SQL",
+      "",
+      "Not enabled for this project, so `run_query` will refuse. Use `search_traces` and `get_analytics` instead.",
+    ].join("\n");
+  }
+  const lines = ["## Analytics SQL", ""];
+  lines.push(
+    "Run one read-only SELECT with `run_query`. The statement is executed as written: nothing rewrites it.",
+  );
+  lines.push(
+    `Ceilings: ${reference.lwql.limits.maxRowsReturned} rows, ${reference.lwql.limits.maxExecutionTimeSeconds} seconds. ${reference.lwql.limits.pagination}`,
+  );
+  lines.push("");
+  for (const view of reference.lwql.schema.views) {
+    lines.push(`### ${view.name}`);
+    lines.push(`${view.description} One row is: ${view.grain}`);
+    lines.push(
+      `Filter on \`${view.timeColumn}\` to prune partitions. Join keys: ${view.joinKeys.join(", ") || "none"}.`,
+    );
+    lines.push("");
     lines.push(
-      `- **${f.field}**: ${f.description}${f.example ? ` (e.g., \`${f.example}\`)` : ""}`
+      markdownTable({
+        headers: ["Column", "Type", "Available", "Description"],
+        rows: view.columns.map((column) => ({
+          Column: column.name,
+          Type: column.type,
+          Available: column.available ? "yes" : `needs ${column.gates.join(", ")}`,
+          Description: column.description,
+        })),
+      }),
     );
+    lines.push("");
+  }
+  lines.push("### Worked statements", "");
+  for (const example of reference.examples.filter((candidate) => candidate.language === "lwql")) {
+    lines.push(
+      `**${example.title}**${example.available ? "" : ` (needs ${example.requires.gates.join(", ")})`}`,
+    );
+    lines.push("```sql");
+    lines.push(example.text);
+    lines.push("```");
+    for (const parameter of example.parameters) {
+      lines.push(`- \`{${parameter.name}:${parameter.type}}\` — ${parameter.description}`);
+    }
+    lines.push("");
   }
   return lines.join("\n");
 }
 
-function formatMetrics(): string {
-  const lines = ["## Available Metrics", ""];
+function formatDecisionTable(reference: QueryReferenceResponse): string {
+  const lines = ["## Which one to reach for", ""];
   lines.push(
-    "Use these in `get_analytics` as `metric` parameter in `category.name` format."
+    markdownTable({
+      headers: ["When you want", "Use", "Why"],
+      rows: reference.decisionTable.map((row) => ({
+        "When you want": row.when,
+        Use: row.use,
+        Why: row.why,
+      })),
+    }),
   );
+  return lines.join("\n");
+}
+
+function formatMetrics(): string {
+  const lines = ["## Available Metrics", ""];
+  lines.push("Use these in `get_analytics` as `metric` parameter in `category.name` format.");
   lines.push("");
 
   const byCategory = new Map<string, typeof analyticsMetrics>();
@@ -95,9 +219,7 @@ function formatAggregations(): string {
 
 function formatGroups(): string {
   const lines = ["## Available Group-By Options", ""];
-  lines.push(
-    "Use these in the `groupBy` parameter of `get_analytics`."
-  );
+  lines.push("Use these in the `groupBy` parameter of `get_analytics`.");
   lines.push("");
   for (const g of analyticsGroups) {
     lines.push(`- **${g.name}** (${g.label}): ${g.description}`);

package/src/tools/experiment-run-status.ts

@@ -15,16 +15,10 @@ export interface RunTimestamps {
   stoppedAt?: number | null;
 }
 
-export const deriveRunStatus = (
-  timestamps: RunTimestamps,
-  now: number = Date.now(),
-): RunStatus => {
+export const deriveRunStatus = (timestamps: RunTimestamps, now: number = Date.now()): RunStatus => {
   if (timestamps.stoppedAt != null) return "stopped";
   if (timestamps.finishedAt != null) return "completed";
-  if (
-    timestamps.updatedAt != null &&
-    now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS
-  ) {
+  if (timestamps.updatedAt != null && now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS) {
     return "interrupted";
   }
   return "running";

package/src/tools/format-dataset-mutation.ts

@@ -1,16 +1,11 @@
-import type { DatasetMutationResponse } from "../langwatch-api-datasets.js";
-import { escapeMarkdown } from "../utils/escape-markdown.js";
+import type { DatasetMutationResponse } from "../langwatch-api-datasets.ts";
+import { escapeMarkdown } from "../utils/escape-markdown.ts";
 
 /**
- * Formats the common detail lines for a dataset mutation response
- * (name, slug, id, and column types).
- *
- * Used by both create-dataset and update-dataset handlers to avoid
- * duplicating the formatting logic.
+ * Formats the common detail lines (name, slug, id, column types) for a
+ * dataset mutation response, shared by create-dataset and update-dataset.
  */
-export function formatDatasetMutationDetails(
-  result: DatasetMutationResponse,
-): string[] {
+export function formatDatasetMutationDetails(result: DatasetMutationResponse): string[] {
   const lines: string[] = [];
   lines.push(`**Name**: ${escapeMarkdown(result.name)}`);
   lines.push(`**Slug**: ${escapeMarkdown(result.slug)}`);

package/src/tools/format-run-plan.ts

@@ -1,14 +1,9 @@
-import type {
-  RunPlanRunResult,
-  RunPlanScope,
-} from "../langwatch-api-run-plans.js";
+import type { RunPlanRunResult, RunPlanScope } from "../langwatch-api-run-plans.ts";
 
 /**
  * The digest every run of a plan returns, whichever tool started it.
- *
- * It says which plan ran and whether the run created it or joined one that
- * already carried the name, because that is the difference between a new plan
- * and a plan whose configuration this run just replaced.
+ * Says whether the run created the plan or joined one that already
+ * carried the name — a new plan vs. one this run just reconfigured.
  */
 export function formatRunPlanRun(result: RunPlanRunResult): string {
   const lines: string[] = [];
@@ -25,21 +20,15 @@ export function formatRunPlanRun(result: RunPlanRunResult): string {
   lines.push(`**Jobs**: ${result.jobCount}`);
 
   if (result.skippedArchived.scenarios.length > 0) {
-    lines.push(
-      `**Skipped archived scenarios**: ${result.skippedArchived.scenarios.join(", ")}`,
-    );
+    lines.push(`**Skipped archived scenarios**: ${result.skippedArchived.scenarios.join(", ")}`);
   }
   if (result.skippedArchived.targets.length > 0) {
-    lines.push(
-      `**Skipped archived targets**: ${result.skippedArchived.targets.join(", ")}`,
-    );
+    lines.push(`**Skipped archived targets**: ${result.skippedArchived.targets.join(", ")}`);
   }
 
   lines.push(`**View**: ${result.platformUrl}`);
   lines.push("");
-  lines.push(
-    "> Use `platform_list_simulation_runs` with the batch run ID to read the results.",
-  );
+  lines.push("> Use `platform_list_simulation_runs` with the batch run ID to read the results.");
 
   return lines.join("\n");
 }
@@ -48,10 +37,7 @@ export function formatRunPlanRun(result: RunPlanRunResult): string {
  * One line saying what a plan covers. A plan stored before scopes existed
  * carries none, and runs the scenario list it already held.
  */
-export function describeRunPlanScope(
-  scope: RunPlanScope | null,
-  scenarioIds: string[],
-): string {
+export function describeRunPlanScope(scope: RunPlanScope | null, scenarioIds: string[]): string {
   if (scope === null) {
     return `hand-picked scenarios (${scenarioIds.length})`;
   }

package/src/tools/format-scenario.ts

@@ -1,9 +1,7 @@
-import type { ScenarioFieldValues } from "../schemas/suite-fields.js";
+import type { ScenarioFieldValues } from "../schemas/suite-fields.ts";
 
 /** The digest lines for the values a scenario carries per suite field. */
-export function formatScenarioFields(
-  fields: ScenarioFieldValues | undefined,
-): string[] {
+export function formatScenarioFields(fields: ScenarioFieldValues | undefined): string[] {
   if (!fields || Object.keys(fields).length === 0) return [];
   return [
     "\n## Fields",

package/src/tools/format-suite-details.ts

@@ -2,7 +2,7 @@ import type {
   EvaluatorAttachmentWire,
   ScenarioMapping,
   SuiteField,
-} from "../schemas/suite-fields.js";
+} from "../schemas/suite-fields.ts";
 
 /**
  * The digest lines for a suite's fields, the evaluators attached to a suite
@@ -19,10 +19,7 @@ export function describeMapping(mapping: ScenarioMapping): string {
 
 export function formatSuiteFields(fields: SuiteField[] | undefined): string[] {
   if (!fields || fields.length === 0) return [];
-  return [
-    "\n## Fields",
-    ...fields.map((field) => `- ${field.identifier} (${field.type})`),
-  ];
+  return ["\n## Fields", ...fields.map((field) => `- ${field.identifier} (${field.type})`)];
 }
 
 export function formatEvaluatorAttachments(
@@ -41,10 +38,9 @@ export function formatEvaluatorAttachments(
 }
 
 /**
- * The status the platform reports while a run's evaluators have not been
- * recorded yet. The conversation is over and the judge has decided, but a
- * required evaluator can still turn the run red, so a caller must not read
- * the verdict as final.
+ * Reported while a run's evaluators haven't recorded yet: the judge has
+ * decided, but a required evaluator can still turn the run red — a
+ * caller must not read this as final.
  */
 export const PENDING_EVALUATION_STATUS = "PENDING_EVALUATION";
 
@@ -79,9 +75,7 @@ export interface SimulationRunEvaluation {
   details?: string;
 }
 
-export function formatEvaluations(
-  evaluations: SimulationRunEvaluation[] | undefined,
-): string[] {
+export function formatEvaluations(evaluations: SimulationRunEvaluation[] | undefined): string[] {
   if (!evaluations || evaluations.length === 0) return [];
   const lines = ["\n## Evaluators"];
   for (const evaluation of evaluations) {

package/src/tools/get-agent.ts

@@ -1,4 +1,8 @@
-import { getAgent as apiGetAgent, type AgentParameterSpec } from "../langwatch-api-agents.js";
+import {
+  getAgent as apiGetAgent,
+  type AgentInstance,
+  type AgentParameterSpec,
+} from "../langwatch-api-agents.ts";
 
 export const describeParameter = ({
   name,
@@ -16,6 +20,15 @@ export const describeParameter = ({
   return `- **${name}** (${parts.join(", ")})${suffix}`;
 };
 
+function instancesSection(instances: AgentInstance[]): string[] {
+  const lines = [`\n## Instances (${instances.length})\n`];
+  for (const instance of instances) {
+    const label = instance.label ? ` (${instance.label})` : "";
+    lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
+  }
+  return lines;
+}
+
 /**
  * Handles the platform_get_agent MCP tool invocation.
  *
@@ -45,11 +58,7 @@ export async function handleGetAgent({ id }: { id: string }): Promise<string> {
   }
 
   if (agent.instances && agent.instances.length > 0) {
-    lines.push(`\n## Instances (${agent.instances.length})\n`);
-    for (const instance of agent.instances) {
-      const label = instance.label ? ` (${instance.label})` : "";
-      lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
-    }
+    lines.push(...instancesSection(agent.instances));
   }
 
   if (agent.config && Object.keys(agent.config).length > 0) {

package/src/tools/get-analytics-report.ts

@@ -0,0 +1,23 @@
+import { getAnalyticsReport } from "../langwatch-api-analytics-reports.ts";
+
+/** Reads only `/api/v1/analytics/reports`; a Dashboards dashboard's id is not found there. */
+export async function handleGetAnalyticsReport(params: { reportId: string }): Promise<string> {
+  const report = await getAnalyticsReport({ reportId: params.reportId });
+
+  const lines: string[] = [];
+  lines.push(`# ${report.name}\n`);
+  lines.push(`**ID**: ${report.id}`);
+  lines.push(`**Order**: ${report.order}`);
+  lines.push(`**Graphs**: ${Array.isArray(report.graphs) ? report.graphs.length : 0}`);
+  lines.push(`**Created**: ${report.createdAt}`);
+  lines.push(`**Updated**: ${report.updatedAt}`);
+
+  if (Array.isArray(report.graphs) && report.graphs.length > 0) {
+    lines.push("\n## Graphs\n");
+    lines.push("```json");
+    lines.push(JSON.stringify(report.graphs, null, 2));
+    lines.push("```");
+  }
+
+  return lines.join("\n");
+}