@langwatch/mcp-server 2.1.0 → 3.0.0

package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js}

@@ -28,7 +28,9 @@ function formatScenarioSchema() {
     "A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:"
   );
   lines.push("```json");
-  lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
+  lines.push(
+    '{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }'
+  );
   lines.push("```");
   lines.push(
     "- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved."
@@ -46,8 +48,12 @@ function formatScenarioSchema() {
   lines.push('  "evaluatorId": "<id from platform_list_evaluators>",');
   lines.push('  "required": true,');
   lines.push('  "mappings": {');
-  lines.push('    "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
-  lines.push('    "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
+  lines.push(
+    '    "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },'
+  );
+  lines.push(
+    '    "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }'
+  );
   lines.push("  }");
   lines.push("}] }");
   lines.push("```");
@@ -63,21 +69,21 @@ function formatScenarioSchema() {
   lines.push(
     '- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)'
   );
-  lines.push(
-    '- a literal: `{ "type": "value", "value": "..." }`'
-  );
+  lines.push('- a literal: `{ "type": "value", "value": "..." }`');
   lines.push(
     "A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`."
   );
   lines.push("\n## Writing a Good Situation\n");
+  lines.push("The situation drives the user simulator. Include these elements:");
   lines.push(
-    "The situation drives the user simulator. Include these elements:"
+    "- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)"
   );
-  lines.push("- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)");
   lines.push("- **Emotional state**: How are they feeling? (e.g., frustrated, anxious, impatient)");
   lines.push("- **Background/Context**: What happened before this conversation?");
   lines.push("- **Intent**: What do they want to accomplish?");
-  lines.push("- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)");
+  lines.push(
+    "- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)"
+  );
   lines.push("\nExample:");
   lines.push("```");
   lines.push("User is a small business owner stressed about tax deadline.");
@@ -87,17 +93,31 @@ function formatScenarioSchema() {
   lines.push("categories apply to their consulting business.");
   lines.push("```");
   lines.push("\n## Writing Good Criteria\n");
-  lines.push("Criteria are what the judge uses to pass or fail the agent. Each criterion should be:");
+  lines.push(
+    "Criteria are what the judge uses to pass or fail the agent. Each criterion should be:"
+  );
   lines.push('- **Specific and testable** \u2014 not vague like "responds helpfully"');
-  lines.push("- **Behavioral** \u2014 describes what the agent should *do*, not how it works internally");
+  lines.push(
+    "- **Behavioral** \u2014 describes what the agent should *do*, not how it works internally"
+  );
   lines.push("- **Independent** \u2014 each criterion checks one thing");
   lines.push("\nGood criteria patterns:");
-  lines.push(`- **Information gathering**: "Agent asks for the user's account number before proceeding"`);
-  lines.push('- **Safety/guardrails**: "Agent does not reveal internal system details or error stack traces"');
-  lines.push('- **Clarification**: "Agent asks clarifying questions before taking irreversible action"');
+  lines.push(
+    `- **Information gathering**: "Agent asks for the user's account number before proceeding"`
+  );
+  lines.push(
+    '- **Safety/guardrails**: "Agent does not reveal internal system details or error stack traces"'
+  );
+  lines.push(
+    '- **Clarification**: "Agent asks clarifying questions before taking irreversible action"'
+  );
   lines.push('- **Tone**: "Agent maintains a professional and empathetic tone throughout"');
-  lines.push('- **Completeness**: "Agent confirms the user understands the solution before ending"');
-  lines.push('- **Domain-specific**: "Agent recommends releasing a wild frog rather than keeping it as a pet"');
+  lines.push(
+    '- **Completeness**: "Agent confirms the user understands the solution before ending"'
+  );
+  lines.push(
+    '- **Domain-specific**: "Agent recommends releasing a wild frog rather than keeping it as a pet"'
+  );
   lines.push("\nAvoid vague criteria like:");
   lines.push('- "Responds correctly" \u2014 correct how?');
   lines.push('- "Is helpful" \u2014 helpful in what way?');
@@ -116,4 +136,4 @@ function formatScenarioSchema() {
 export {
   formatScenarioSchema
 };
-//# sourceMappingURL=discover-scenario-schema-W7KBXA4J.js.map
+//# sourceMappingURL=discover-scenario-schema-OOFIQKOG.js.map

package/dist/discover-schema-7WHIIATI.js

@@ -0,0 +1,442 @@
+import {
+  getQueryReference,
+  markdownTable
+} from "./chunk-GA5JUSPT.js";
+import "./chunk-FOKVY3BK.js";
+import "./chunk-7CZQDZ2K.js";
+import "./chunk-DULFP3MK.js";
+
+// src/schemas/analytics-groups.ts
+var analyticsGroups = [
+  {
+    name: "topics.topics",
+    label: "Topic",
+    description: "Group by topic classification"
+  },
+  {
+    name: "metadata.user_id",
+    label: "User",
+    description: "Group by user ID"
+  },
+  {
+    name: "metadata.thread_id",
+    label: "Thread",
+    description: "Group by conversation thread"
+  },
+  {
+    name: "metadata.customer_id",
+    label: "Customer ID",
+    description: "Group by customer/organization"
+  },
+  {
+    name: "metadata.labels",
+    label: "Label",
+    description: "Group by custom labels"
+  },
+  {
+    name: "metadata.model",
+    label: "Model",
+    description: "Group by LLM model name"
+  },
+  {
+    name: "metadata.span_type",
+    label: "Span Type",
+    description: "Group by span type (llm, tool, agent, etc.)"
+  },
+  {
+    name: "sentiment.input_sentiment",
+    label: "Input Sentiment",
+    description: "Group by detected input sentiment (positive, negative, neutral)"
+  },
+  {
+    name: "sentiment.thumbs_up_down",
+    label: "Thumbs Up/Down",
+    description: "Group by user feedback (positive, negative, neutral)"
+  },
+  {
+    name: "events.event_type",
+    label: "Event Type",
+    description: "Group by event type"
+  },
+  {
+    name: "evaluations.evaluation_passed",
+    label: "Evaluation Passed",
+    description: "Group by evaluation pass/fail status"
+  },
+  {
+    name: "evaluations.evaluation_label",
+    label: "Evaluation Label",
+    description: "Group by evaluation label result"
+  },
+  {
+    name: "evaluations.evaluation_processing_state",
+    label: "Evaluation Processing State",
+    description: "Group by evaluation processing state"
+  },
+  {
+    name: "error.has_error",
+    label: "Contains Error",
+    description: "Group by whether the trace contains an error"
+  }
+];
+
+// src/schemas/analytics-metrics.ts
+var analyticsMetrics = [
+  // metadata
+  {
+    category: "metadata",
+    name: "trace_id",
+    label: "Traces",
+    allowedAggregations: ["cardinality"],
+    description: "Count of unique traces"
+  },
+  {
+    category: "metadata",
+    name: "user_id",
+    label: "Users",
+    allowedAggregations: ["cardinality"],
+    description: "Count of unique users"
+  },
+  {
+    category: "metadata",
+    name: "thread_id",
+    label: "Threads",
+    allowedAggregations: ["cardinality"],
+    description: "Count of unique conversation threads"
+  },
+  {
+    category: "metadata",
+    name: "span_type",
+    label: "Span Type",
+    allowedAggregations: ["cardinality"],
+    description: "Count of spans, optionally filtered by span type"
+  },
+  // sentiment
+  {
+    category: "sentiment",
+    name: "input_sentiment",
+    label: "Input Sentiment Score",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Sentiment analysis score of inputs"
+  },
+  {
+    category: "sentiment",
+    name: "thumbs_up_down",
+    label: "Thumbs Up/Down Score",
+    allowedAggregations: [
+      "terms",
+      "cardinality",
+      "avg",
+      "sum",
+      "min",
+      "max",
+      "median",
+      "p99",
+      "p95",
+      "p90"
+    ],
+    description: "User feedback score (-1 to 1)"
+  },
+  // performance
+  {
+    category: "performance",
+    name: "completion_time",
+    label: "Completion Time",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Time to complete the trace (ms)"
+  },
+  {
+    category: "performance",
+    name: "first_token",
+    label: "Time to First Token",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Time to first token (ms)"
+  },
+  {
+    category: "performance",
+    name: "total_cost",
+    label: "Total Cost",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Cost per trace in USD"
+  },
+  {
+    category: "performance",
+    name: "prompt_tokens",
+    label: "Prompt Tokens",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Input token count"
+  },
+  {
+    category: "performance",
+    name: "completion_tokens",
+    label: "Completion Tokens",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Output token count"
+  },
+  {
+    category: "performance",
+    name: "total_tokens",
+    label: "Total Tokens",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Total token count (input + output)"
+  },
+  {
+    category: "performance",
+    name: "tokens_per_second",
+    label: "Tokens per Second",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Token generation speed"
+  },
+  // events
+  {
+    category: "events",
+    name: "event_type",
+    label: "Event Type",
+    allowedAggregations: ["cardinality"],
+    description: "Count of events, optionally filtered by event type"
+  },
+  {
+    category: "events",
+    name: "event_score",
+    label: "Event Score",
+    allowedAggregations: ["terms", "avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Numeric score from events (requires event_type key and metrics key)"
+  },
+  {
+    category: "events",
+    name: "event_details",
+    label: "Event Details",
+    allowedAggregations: ["cardinality"],
+    description: "Event detail key/value distribution (requires event_type key and details key)"
+  },
+  // evaluations
+  {
+    category: "evaluations",
+    name: "evaluation_score",
+    label: "Evaluation Score",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Numeric evaluation score (requires evaluator_id key)"
+  },
+  {
+    category: "evaluations",
+    name: "evaluation_pass_rate",
+    label: "Evaluation Pass Rate",
+    allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
+    description: "Percentage of traces passing evaluation (requires evaluator_id key)"
+  },
+  {
+    category: "evaluations",
+    name: "evaluation_runs",
+    label: "Evaluation Runs",
+    allowedAggregations: ["cardinality"],
+    description: "Count of evaluation executions"
+  },
+  // threads
+  {
+    category: "threads",
+    name: "average_duration_per_thread",
+    label: "Thread Duration",
+    allowedAggregations: ["avg"],
+    description: "Average duration of conversation threads (ms)"
+  }
+];
+
+// src/tools/discover-schema.ts
+var PLATFORM_BACKED = /* @__PURE__ */ new Set(["filters", "lwql", "all"]);
+function needsQueryReference(category) {
+  return PLATFORM_BACKED.has(category);
+}
+async function formatSchema(category, reference) {
+  const sections = [];
+  const resolved = needsQueryReference(category) && !reference ? await getQueryReference() : reference;
+  if (category === "filters" || category === "all") {
+    sections.push(formatFilters(resolved));
+  }
+  if (category === "lwql" || category === "all") {
+    sections.push(formatLangWatchQL(resolved));
+  }
+  if (category === "metrics" || category === "all") {
+    sections.push(formatMetrics());
+  }
+  if (category === "aggregations" || category === "all") {
+    sections.push(formatAggregations());
+  }
+  if (category === "groups" || category === "all") {
+    sections.push(formatGroups());
+  }
+  if (category === "all" && resolved) {
+    sections.push(formatDecisionTable(resolved));
+  }
+  return sections.join("\n\n");
+}
+var knownValuesCell = (field) => {
+  if (field.knownValues.length > 0) return field.knownValues.join(", ");
+  return field.facetable ? "open set, read them from GET /api/v1/traces/facets" : "";
+};
+function formatFilters(reference) {
+  if (!reference) {
+    return "## Trace Filter Fields\n\nUnavailable: the platform could not be reached.";
+  }
+  const lines = ["## Trace Filter Fields", ""];
+  lines.push(
+    "Send one of these as the `filter` parameter of `search_traces`, in the language the Trace Explorer's search bar speaks: `status:error AND model:gpt-*`."
+  );
+  lines.push(
+    "The `filters` parameter is the older per-field map and still works; both are combined when you send both."
+  );
+  lines.push("");
+  lines.push(
+    markdownTable({
+      headers: ["Field", "Type", "Group", "Known values"],
+      rows: reference.traceFilter.fields.map((field) => ({
+        Field: field.name,
+        Type: field.valueType,
+        Group: field.group ?? "",
+        "Known values": knownValuesCell(field)
+      }))
+    })
+  );
+  lines.push("");
+  lines.push("### Attribute namespaces", "");
+  lines.push(
+    markdownTable({
+      headers: ["Prefix", "Matches", "Older spellings"],
+      rows: reference.traceFilter.dynamicPrefixes.map((prefix) => ({
+        Prefix: `${prefix.prefix}<key>`,
+        Matches: prefix.description,
+        "Older spellings": prefix.aliases.join(", ")
+      }))
+    })
+  );
+  lines.push("");
+  lines.push("### Syntax", "");
+  lines.push(reference.traceFilter.syntax);
+  lines.push("");
+  lines.push("### Worked filters", "");
+  for (const example of reference.examples.filter(
+    (candidate) => candidate.language === "trace-filter"
+  )) {
+    lines.push(`- \`${example.text}\` \u2014 ${example.title}`);
+  }
+  return lines.join("\n");
+}
+function formatLangWatchQL(reference) {
+  if (!reference) {
+    return "## Analytics SQL\n\nUnavailable: the platform could not be reached.";
+  }
+  if (!reference.lwql.enabled) {
+    return [
+      "## Analytics SQL",
+      "",
+      "Not enabled for this project, so `run_query` will refuse. Use `search_traces` and `get_analytics` instead."
+    ].join("\n");
+  }
+  const lines = ["## Analytics SQL", ""];
+  lines.push(
+    "Run one read-only SELECT with `run_query`. The statement is executed as written: nothing rewrites it."
+  );
+  lines.push(
+    `Ceilings: ${reference.lwql.limits.maxRowsReturned} rows, ${reference.lwql.limits.maxExecutionTimeSeconds} seconds. ${reference.lwql.limits.pagination}`
+  );
+  lines.push("");
+  for (const view of reference.lwql.schema.views) {
+    lines.push(`### ${view.name}`);
+    lines.push(`${view.description} One row is: ${view.grain}`);
+    lines.push(
+      `Filter on \`${view.timeColumn}\` to prune partitions. Join keys: ${view.joinKeys.join(", ") || "none"}.`
+    );
+    lines.push("");
+    lines.push(
+      markdownTable({
+        headers: ["Column", "Type", "Available", "Description"],
+        rows: view.columns.map((column) => ({
+          Column: column.name,
+          Type: column.type,
+          Available: column.available ? "yes" : `needs ${column.gates.join(", ")}`,
+          Description: column.description
+        }))
+      })
+    );
+    lines.push("");
+  }
+  lines.push("### Worked statements", "");
+  for (const example of reference.examples.filter((candidate) => candidate.language === "lwql")) {
+    lines.push(
+      `**${example.title}**${example.available ? "" : ` (needs ${example.requires.gates.join(", ")})`}`
+    );
+    lines.push("```sql");
+    lines.push(example.text);
+    lines.push("```");
+    for (const parameter of example.parameters) {
+      lines.push(`- \`{${parameter.name}:${parameter.type}}\` \u2014 ${parameter.description}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function formatDecisionTable(reference) {
+  const lines = ["## Which one to reach for", ""];
+  lines.push(
+    markdownTable({
+      headers: ["When you want", "Use", "Why"],
+      rows: reference.decisionTable.map((row) => ({
+        "When you want": row.when,
+        Use: row.use,
+        Why: row.why
+      }))
+    })
+  );
+  return lines.join("\n");
+}
+function formatMetrics() {
+  const lines = ["## Available Metrics", ""];
+  lines.push("Use these in `get_analytics` as `metric` parameter in `category.name` format.");
+  lines.push("");
+  const byCategory = /* @__PURE__ */ new Map();
+  for (const m of analyticsMetrics) {
+    const list = byCategory.get(m.category) || [];
+    list.push(m);
+    byCategory.set(m.category, list);
+  }
+  for (const [cat, metrics] of byCategory) {
+    lines.push(`### ${cat}`);
+    for (const m of metrics) {
+      lines.push(`- **${cat}.${m.name}** (${m.label}): ${m.description}`);
+      lines.push(`  Aggregations: ${m.allowedAggregations.join(", ")}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function formatAggregations() {
+  return [
+    "## Available Aggregation Types",
+    "",
+    "- **cardinality**: Count unique values",
+    "- **terms**: Distribution/breakdown of values",
+    "- **avg**: Average",
+    "- **sum**: Sum total",
+    "- **min**: Minimum",
+    "- **max**: Maximum",
+    "- **median**: 50th percentile",
+    "- **p90**: 90th percentile",
+    "- **p95**: 95th percentile",
+    "- **p99**: 99th percentile",
+    "",
+    "Note: Not all aggregations are available for all metrics. Check the metric's allowed aggregations."
+  ].join("\n");
+}
+function formatGroups() {
+  const lines = ["## Available Group-By Options", ""];
+  lines.push("Use these in the `groupBy` parameter of `get_analytics`.");
+  lines.push("");
+  for (const g of analyticsGroups) {
+    lines.push(`- **${g.name}** (${g.label}): ${g.description}`);
+  }
+  return lines.join("\n");
+}
+export {
+  formatSchema,
+  needsQueryReference
+};
+//# sourceMappingURL=discover-schema-7WHIIATI.js.map