@langwatch/mcp-server 1.0.0 → 2.1.0

package/src/index.ts

@@ -24,6 +24,17 @@ const argv = await yargs(hideBin(process.argv))
     description: "HTTP server port (only used with --http)",
     default: 3000,
   })
+  .option("host", {
+    type: "string",
+    description:
+      "HTTP listen address (only used with --http). Defaults to loopback; pass 0.0.0.0 to accept connections from other machines",
+  })
+  .option("allowedOrigin", {
+    type: "array",
+    string: true,
+    description:
+      "Browser origin allowed to call the HTTP server, repeatable. Loopback origins are always allowed",
+  })
   .help()
   .parseAsync();
 
@@ -34,11 +45,47 @@ initConfig({
 
 if (argv.http) {
   const { startHttpServer } = await import("./http-server.js");
-  const { port } = await startHttpServer({ port: argv.port });
-  console.log(`LangWatch MCP server listening on http://0.0.0.0:${port}/mcp`);
+  const { isLoopbackHost, parseAllowedOrigins } = await import(
+    "./http-security.js"
+  );
+
+  const allowedOrigins =
+    argv.allowedOrigin && argv.allowedOrigin.length > 0
+      ? parseAllowedOrigins(argv.allowedOrigin.join(","))
+      : undefined;
+
+  const { port, host } = await startHttpServer({
+    port: argv.port,
+    host: argv.host,
+    allowedOrigins,
+  });
+
+  // An IPv6 literal has to be bracketed inside a URL, and must not be
+  // double-bracketed when the operator already wrote it that way.
+  const displayHost =
+    host.includes(":") && !host.startsWith("[") ? `[${host}]` : host;
+  console.log(
+    `LangWatch MCP server listening on http://${displayHost}:${port}/mcp`
+  );
   console.log(
     "Clients must provide their API key via Authorization: Bearer <key> header"
   );
+  if (!isLoopbackHost(host)) {
+    console.warn(
+      `Warning: bound to ${host}, so the server is reachable from other machines. ` +
+        "Make sure it sits behind a trusted network boundary and that --allowedOrigin " +
+        "lists only origins you control."
+    );
+  }
+  if (process.env.LANGWATCH_MCP_TRUST_PROXY === "true") {
+    console.warn(
+      "Warning: LANGWATCH_MCP_TRUST_PROXY=true, so the rate limit on failed " +
+        "authentication uses the client address from X-Forwarded-For. Keep this " +
+        "on only when a trusted proxy terminates in front of the port and " +
+        "overwrites that header. Otherwise a client can supply any address there " +
+        "and the limit stops counting."
+    );
+  }
 } else {
   const transport = new StdioServerTransport();
   const server = createMcpServer();

package/src/langwatch-api-agents.ts

@@ -1,6 +1,42 @@
+import { z } from "zod";
+
 import { makeRequest } from "./langwatch-api.js";
 import { requestPublicJson } from "./public-http-request.js";
 
+/**
+ * The run parameters a call carries: a flat record of scalars. The platform
+ * declares run parameters as `string`, `number` or `boolean`, so a nested
+ * object or an array is outside the contract and is refused here.
+ */
+export const agentCallParamsSchema = z.record(
+  z.string(),
+  z.union([z.string(), z.number(), z.boolean()]),
+);
+
+export type AgentCallParams = z.infer<typeof agentCallParamsSchema>;
+
+/**
+ * One run parameter an agent declares, as the platform lists it. An agent
+ * never declares a secret parameter: a secret belongs to the scenario and is
+ * supplied per run.
+ */
+export interface AgentParameterSpec {
+  name: string;
+  type: "string" | "number" | "boolean";
+  options?: string[];
+  default?: string | number | boolean;
+  description?: string;
+  required?: boolean;
+}
+
+/** One process connected as an instance of a connected agent. */
+export interface AgentInstance {
+  id: string;
+  hostname: string;
+  label?: string | null;
+  connectedAt: string;
+}
+
 export interface AgentSummary {
   id: string;
   name: string;
@@ -8,6 +44,19 @@ export interface AgentSummary {
   config: Record<string, unknown> | null;
   createdAt: string;
   updatedAt: string;
+  /** Connected agents: the environment the SDK registered under. */
+  environment?: string | null;
+  /** Connected agents: online while at least one instance is connected. */
+  status?: "online" | "offline" | null;
+  instances?: AgentInstance[];
+  lastSeenAt?: string | null;
+  owner?: { userId: string; name: string } | null;
+  hostLabel?: string | null;
+  /** Whether this key can run the agent. */
+  selectable?: boolean;
+  /** Why the agent cannot be run with this key; null when it can. */
+  notSelectableReason?: "owned_by_another_person" | null;
+  parameters?: AgentParameterSpec[];
 }
 
 export interface AgentListResponse {
@@ -20,6 +69,31 @@ export interface AgentListResponse {
   };
 }
 
+/** One conversation message as the relay carries it, OpenAI style. */
+export interface AgentCallMessage {
+  role: string;
+  content?: unknown;
+  [key: string]: unknown;
+}
+
+/** The body `POST /api/v1/agents/:id/call` takes. */
+export interface AgentCallBody {
+  messages: AgentCallMessage[];
+  newMessages?: AgentCallMessage[];
+  threadId?: string;
+  params?: AgentCallParams;
+  session?: unknown;
+  traceparent?: string;
+}
+
+/** The reply of the relay: the function's output and the instance that ran it. */
+export interface AgentCallResponse {
+  output: unknown;
+  session?: unknown;
+  instance: { hostname: string; label?: string | null };
+  durationMs: number;
+}
+
 export async function listAgents(params?: {
   page?: number;
   limit?: number;
@@ -28,13 +102,13 @@ export async function listAgents(params?: {
   if (params?.page) query.set("page", String(params.page));
   if (params?.limit) query.set("limit", String(params.limit));
   const qs = query.toString() ? `?${query}` : "";
-  return makeRequest("GET", `/api/agents${qs}`) as Promise<AgentListResponse>;
+  return makeRequest("GET", `/api/v1/agents${qs}`) as Promise<AgentListResponse>;
 }
 
 export async function getAgent(id: string): Promise<AgentSummary> {
   return makeRequest(
     "GET",
-    `/api/agents/${encodeURIComponent(id)}`,
+    `/api/v1/agents/${encodeURIComponent(id)}`,
   ) as Promise<AgentSummary>;
 }
 
@@ -43,7 +117,7 @@ export async function createAgent(data: {
   type: string;
   config: Record<string, unknown>;
 }): Promise<AgentSummary> {
-  return makeRequest("POST", "/api/agents", data) as Promise<AgentSummary>;
+  return makeRequest("POST", "/api/v1/agents", data) as Promise<AgentSummary>;
 }
 
 export async function updateAgent(params: {
@@ -55,22 +129,132 @@ export async function updateAgent(params: {
   const { id, ...data } = params;
   return makeRequest(
     "PATCH",
-    `/api/agents/${encodeURIComponent(id)}`,
+    `/api/v1/agents/${encodeURIComponent(id)}`,
     data,
   ) as Promise<AgentSummary>;
 }
 
 /**
- * Run an agent. For HTTP agents, calls the configured URL directly.
- * For workflow-linked agents, calls the workflow run endpoint.
+ * Runs one turn of a connected agent through the relay: the platform
+ * dispatches it to a live instance and answers with the function's output.
+ */
+export async function callAgent({
+  id,
+  body,
+}: {
+  id: string;
+  body: AgentCallBody;
+}): Promise<AgentCallResponse> {
+  return makeRequest(
+    "POST",
+    `/api/v1/agents/${encodeURIComponent(id)}/call`,
+    body,
+  ) as Promise<AgentCallResponse>;
+}
+
+/** The ids `POST /api/v1/agents/:id/test` answers with. */
+export interface AgentTestRunResponse {
+  scenarioRunId: string;
+  batchRunId: string;
+  setId: string;
+}
+
+/**
+ * Runs one scripted scenario against an agent: the user sends "ping", the
+ * agent answers, and the run succeeds when the answer arrives. The project
+ * gains no scenario, run plan or test suite, and the answer carries the run
+ * ids to follow.
+ */
+export async function testAgent(id: string): Promise<AgentTestRunResponse> {
+  return makeRequest(
+    "POST",
+    `/api/v1/agents/${encodeURIComponent(id)}/test`,
+    {},
+  ) as Promise<AgentTestRunResponse>;
+}
+
+const isMessageList = (value: unknown): value is AgentCallMessage[] =>
+  Array.isArray(value) &&
+  value.every(
+    (item) =>
+      typeof item === "object" && item !== null && typeof (item as AgentCallMessage).role === "string",
+  );
+
+/**
+ * The relay body for a connected agent: `message` is one user turn, `input`
+ * is the body itself (it must carry `messages`), `parameters` are the run
+ * parameters and `threadId` continues a conversation. A string is the
+ * refusal, so the tool can answer it.
+ */
+export function buildRelayBody({
+  input,
+  message,
+  parameters,
+  threadId,
+}: {
+  input: Record<string, unknown>;
+  message?: string;
+  parameters?: AgentCallParams;
+  threadId?: string;
+}): AgentCallBody | string {
+  let messages: AgentCallMessage[];
+  if (message !== undefined) {
+    messages = [{ role: "user", content: message }];
+  } else if (isMessageList(input.messages)) {
+    messages = input.messages;
+  } else {
+    return "a connected agent takes a conversation: give `message`, or `input` with a `messages` list.";
+  }
+  const body: AgentCallBody = { messages };
+  const thread = threadId ?? (typeof input.threadId === "string" ? input.threadId : undefined);
+  if (thread) body.threadId = thread;
+  if (isMessageList(input.newMessages)) body.newMessages = input.newMessages;
+  if (input.session !== undefined) body.session = input.session;
+  let inputParams: AgentCallParams | undefined;
+  if (input.params !== undefined) {
+    const parsed = agentCallParamsSchema.safeParse(input.params);
+    if (!parsed.success) {
+      return "`params` takes one flat object of string, number or boolean values. Nested objects and arrays are not run parameters.";
+    }
+    inputParams = parsed.data;
+  }
+  if (inputParams || parameters) body.params = { ...(inputParams ?? {}), ...(parameters ?? {}) };
+  return body;
+}
+
+/**
+ * Run an agent. A connected agent runs one turn through the relay; an HTTP
+ * agent is called at its URL; a workflow-linked agent runs on the workflow
+ * engine.
  */
-export async function runAgent(
-  id: string,
-  input?: Record<string, unknown>,
-): Promise<{ agentType: string; result: Record<string, unknown> }> {
+export async function runAgent({
+  id,
+  input,
+  message,
+  parameters,
+  threadId,
+}: {
+  id: string;
+  input?: Record<string, unknown>;
+  message?: string;
+  parameters?: AgentCallParams;
+  threadId?: string;
+}): Promise<{ agentType: string; result: Record<string, unknown> }> {
   const agent = await getAgent(id);
   const config = agent.config ?? {};
 
+  if (agent.type === "connected") {
+    const body = buildRelayBody({ input: input ?? {}, message, parameters, threadId });
+    if (typeof body === "string") throw new Error(body);
+    if (agent.status === "offline") {
+      throw new Error(
+        `Agent "${agent.name}" is offline. Start the process that calls connectAgent (or connect_agent) and try again.`,
+      );
+    }
+    const result = await callAgent({ id: agent.id, body });
+    return { agentType: "connected", result: result as unknown as Record<string, unknown> };
+  }
+
   if (agent.type === "http") {
     const url = (config as Record<string, unknown>).url as string | undefined;
     if (!url) {
@@ -91,7 +275,7 @@ export async function runAgent(
   if (!workflowId) {
     throw new Error(
       `Agent "${agent.name}" (type: ${agent.type}) cannot be executed directly. ` +
-        `Only HTTP agents and workflow-linked agents can be run.`,
+        `Only connected agents, HTTP agents and workflow-linked agents can be run.`,
     );
   }
 
@@ -103,6 +287,6 @@ export async function runAgent(
 export async function deleteAgent(id: string): Promise<{ id: string; name: string }> {
   return makeRequest(
     "DELETE",
-    `/api/agents/${encodeURIComponent(id)}`,
+    `/api/v1/agents/${encodeURIComponent(id)}`,
   ) as Promise<{ id: string; name: string }>;
 }

package/src/langwatch-api-run-plans.ts

@@ -0,0 +1,157 @@
+import type { z } from "zod";
+import { makeRequest } from "./langwatch-api.js";
+import type {
+  runParametersSchema,
+  runPlanScopeSchema,
+  runPlanTargetSchema,
+  RunPlanTargetWire,
+} from "./schemas/run-plan.js";
+import type { EvaluatorAttachmentWire } from "./schemas/suite-fields.js";
+
+/**
+ * Client for `/api/v1/run-plans`.
+ *
+ * A run plan is what you run, and its NAME identifies it: running against a
+ * name that exists replaces that plan's configuration, running against a new
+ * name creates one. Configuration is the scope, the targets, the repeat count
+ * and the two models. Parameters, the note and the idempotency key belong to
+ * one run, not to the plan.
+ */
+
+/**
+ * The shapes the run plan tools accept, taken from the zod that validates
+ * them. They are inferred rather than written twice, so the schema stays the
+ * one place the shape is stated.
+ */
+
+/** What a run plan covers. Mirrors `suiteScopeSchema` on the platform. */
+export type RunPlanScope = z.infer<typeof runPlanScopeSchema>;
+
+/** One thing a plan runs its scenarios against, as a tool caller writes it. */
+export type RunPlanTarget = z.infer<typeof runPlanTargetSchema>;
+
+/** The values a run supplies for the parameters its scenarios declare. */
+export type RunParameters = z.infer<typeof runParametersSchema>;
+
+export interface RunPlan {
+  id: string;
+  name: string;
+  slug: string;
+  scope: RunPlanScope | null;
+  scenarioIds: string[];
+  targets: RunPlanTargetWire[];
+  repeatCount: number;
+  simulatorModel: string | null;
+  judgeModel: string | null;
+  labels: string[];
+  archivedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+  platformUrl: string;
+  /** The plan's own evaluators. Absent on servers that predate them. */
+  evaluators?: EvaluatorAttachmentWire[];
+}
+
+/** The configuration a run either writes onto a plan or creates it with. */
+export interface RunPlanConfig {
+  scope: RunPlanScope;
+  targets: RunPlanTargetWire[];
+  repeatCount?: number;
+  simulatorModel?: string | null;
+  judgeModel?: string | null;
+  scenarioIds?: string[];
+  /**
+   * The plan's own evaluators, run beside the ones its test suites attach. A
+   * plan evaluator reads the conversation and the trace, never a scenario
+   * field. Left out, the plan keeps what it already holds.
+   */
+  evaluators?: EvaluatorAttachmentWire[];
+}
+
+export interface RunPlanRunResult {
+  scheduled: boolean;
+  batchRunId: string;
+  setId: string;
+  jobCount: number;
+  skippedArchived: {
+    scenarios: string[];
+    targets: string[];
+  };
+  items: Array<{
+    scenarioRunId: string;
+    scenarioId: string;
+    target: RunPlanTargetWire;
+    name: string | null;
+  }>;
+  runPlanId: string;
+  planName: string;
+  /** True when this run created the plan, false when it joined an existing one. */
+  created: boolean;
+  platformUrl: string;
+}
+
+export interface RunPlanArchiveResponse {
+  id: string;
+  archived: boolean;
+}
+
+/** Lists the run plans of the project. */
+export async function listRunPlans(params?: {
+  includeArchived?: boolean;
+}): Promise<RunPlan[]> {
+  const query = params?.includeArchived ? "?includeArchived=true" : "";
+  return makeRequest("GET", `/api/v1/run-plans${query}`) as Promise<RunPlan[]>;
+}
+
+/** Retrieves a single run plan by id. */
+export async function getRunPlan(id: string): Promise<RunPlan> {
+  return makeRequest(
+    "GET",
+    `/api/v1/run-plans/${encodeURIComponent(id)}`,
+  ) as Promise<RunPlan>;
+}
+
+/**
+ * Runs a configuration by plan name. A name that exists is replaced with this
+ * configuration, a name that does not is created. Sending no name lets the
+ * server derive one.
+ */
+export async function runRunPlan(data: {
+  name?: string;
+  config: RunPlanConfig;
+  idempotencyKey?: string;
+  parameters?: RunParameters;
+  note?: string;
+}): Promise<RunPlanRunResult> {
+  return makeRequest(
+    "POST",
+    "/api/v1/run-plans/run",
+    data,
+  ) as Promise<RunPlanRunResult>;
+}
+
+/** Runs a plan again with the configuration it already holds. */
+export async function rerunRunPlan(
+  id: string,
+  data?: {
+    idempotencyKey?: string;
+    parameters?: RunParameters;
+    note?: string;
+  },
+): Promise<RunPlanRunResult> {
+  return makeRequest(
+    "POST",
+    `/api/v1/run-plans/${encodeURIComponent(id)}/run`,
+    data ?? {},
+  ) as Promise<RunPlanRunResult>;
+}
+
+/** Archives a run plan. */
+export async function archiveRunPlan(
+  id: string,
+): Promise<RunPlanArchiveResponse> {
+  return makeRequest(
+    "DELETE",
+    `/api/v1/run-plans/${encodeURIComponent(id)}`,
+  ) as Promise<RunPlanArchiveResponse>;
+}

package/src/langwatch-api-scenarios.ts

@@ -1,4 +1,5 @@
 import { makeRequest } from "./langwatch-api.js";
+import type { ScenarioFieldValues } from "./schemas/suite-fields.js";
 
 // --- Scenario types ---
 
@@ -8,6 +9,10 @@ export interface ScenarioSummary {
   situation: string;
   criteria: string[];
   labels: string[];
+  /** The test suite this scenario is filed in, null when it is unfiled. */
+  testSuiteId?: string | null;
+  /** The scenario's value for each field its test suite declares. */
+  fields?: ScenarioFieldValues;
 }
 
 export interface ScenarioArchiveResponse {
@@ -36,6 +41,8 @@ export async function createScenario(data: {
   situation: string;
   criteria?: string[];
   labels?: string[];
+  testSuiteId?: string | null;
+  fields?: ScenarioFieldValues;
 }): Promise<ScenarioSummary> {
   return makeRequest("POST", "/api/scenarios", data) as Promise<ScenarioSummary>;
 }
@@ -47,6 +54,8 @@ export async function updateScenario(params: {
   situation?: string;
   criteria?: string[];
   labels?: string[];
+  testSuiteId?: string | null;
+  fields?: ScenarioFieldValues;
 }): Promise<ScenarioSummary> {
   const { id, ...data } = params;
   return makeRequest(

package/src/langwatch-api-simulation-runs.ts

@@ -1,4 +1,5 @@
 import { makeRequest } from "./langwatch-api.js";
+import type { SimulationRunEvaluation } from "./tools/format-suite-details.js";
 
 export interface SimulationRunSummary {
   scenarioRunId: string;
@@ -14,8 +15,12 @@ export interface SimulationRunSummary {
     metCriteria?: string[];
     unmetCriteria?: string[];
     error?: string | null;
+    /** One result per evaluator that ran after the conversation. */
+    evaluations?: SimulationRunEvaluation[];
   } | null;
   messages?: Array<{ role: string; content: string }>;
+  /** Why the run was started, as given when it was queued. Null without one. */
+  note?: string | null;
   timestamp: number;
   updatedAt: number;
 }

package/src/langwatch-api-test-suites.ts

@@ -0,0 +1,135 @@
+import { makeRequest } from "./langwatch-api.js";
+import type {
+  RunParameters,
+  RunPlanRunResult,
+} from "./langwatch-api-run-plans.js";
+import type { RunPlanTargetWire } from "./schemas/run-plan.js";
+import type {
+  EvaluatorAttachmentWire,
+  SuiteField,
+} from "./schemas/suite-fields.js";
+
+/**
+ * Client for `/api/v1/test-suites`.
+ *
+ * A test suite groups scenarios: a name and the scenarios filed in it.
+ * Running one is sugar over a run plan, so the run returns the same result a
+ * run plan does.
+ */
+
+export interface TestSuite {
+  id: string;
+  name: string;
+  slug: string;
+  scenarioIds: string[];
+  scenarioCount: number;
+  archivedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+  platformUrl: string;
+  /** The fields the suite declares. Absent on servers that predate them. */
+  fields?: SuiteField[];
+  /** The evaluators attached to the suite. Absent on servers that predate them. */
+  evaluators?: EvaluatorAttachmentWire[];
+}
+
+export interface TestSuiteDetail extends TestSuite {
+  scenarios: Array<{ id: string; name: string }>;
+}
+
+export interface TestSuiteArchiveResponse {
+  id: string;
+  archived: boolean;
+}
+
+/** Lists the test suites of the project. */
+export async function listTestSuites(): Promise<TestSuite[]> {
+  return makeRequest("GET", "/api/v1/test-suites") as Promise<TestSuite[]>;
+}
+
+/** Creates a test suite, with the fields and the evaluators it declares. */
+export async function createTestSuite(data: {
+  name: string;
+  fields?: SuiteField[];
+  evaluators?: EvaluatorAttachmentWire[];
+}): Promise<TestSuite> {
+  return makeRequest(
+    "POST",
+    "/api/v1/test-suites",
+    data,
+  ) as Promise<TestSuite>;
+}
+
+/** Retrieves a test suite with the scenarios filed in it. */
+export async function getTestSuite(id: string): Promise<TestSuiteDetail> {
+  return makeRequest(
+    "GET",
+    `/api/v1/test-suites/${encodeURIComponent(id)}`,
+  ) as Promise<TestSuiteDetail>;
+}
+
+/**
+ * Edits a test suite: any of its name, its fields and its evaluators. A list
+ * given here replaces the one the suite holds.
+ */
+export async function updateTestSuite(params: {
+  id: string;
+  name?: string;
+  fields?: SuiteField[];
+  evaluators?: EvaluatorAttachmentWire[];
+}): Promise<TestSuite> {
+  const { id, ...data } = params;
+  return makeRequest(
+    "PATCH",
+    `/api/v1/test-suites/${encodeURIComponent(id)}`,
+    data,
+  ) as Promise<TestSuite>;
+}
+
+/** Renames a test suite. */
+export async function renameTestSuite(params: {
+  id: string;
+  name: string;
+}): Promise<TestSuite> {
+  return makeRequest(
+    "PATCH",
+    `/api/v1/test-suites/${encodeURIComponent(params.id)}`,
+    { name: params.name },
+  ) as Promise<TestSuite>;
+}
+
+/** Archives a test suite and the scenarios filed in it. */
+export async function archiveTestSuite(
+  id: string,
+): Promise<TestSuiteArchiveResponse> {
+  return makeRequest(
+    "DELETE",
+    `/api/v1/test-suites/${encodeURIComponent(id)}`,
+  ) as Promise<TestSuiteArchiveResponse>;
+}
+
+/**
+ * Runs every scenario of a test suite against the given targets. The server
+ * creates or joins the run plan named "<suite name> <target name>" when no
+ * name is sent.
+ */
+export async function runTestSuite({
+  id,
+  ...data
+}: {
+  id: string;
+  targets: RunPlanTargetWire[];
+  name?: string;
+  repeatCount?: number;
+  simulatorModel?: string | null;
+  judgeModel?: string | null;
+  parameters?: RunParameters;
+  note?: string;
+  idempotencyKey?: string;
+}): Promise<RunPlanRunResult> {
+  return makeRequest(
+    "POST",
+    `/api/v1/test-suites/${encodeURIComponent(id)}/run`,
+    data,
+  ) as Promise<RunPlanRunResult>;
+}