@langwatch/mcp-server 2.0.0 → 2.2.0

package/src/langwatch-api-run-plans.ts

@@ -0,0 +1,140 @@
+import type { z } from "zod";
+
+import { makeRequest } from "./langwatch-api.ts";
+import type {
+  runParametersSchema,
+  runPlanScopeSchema,
+  runPlanTargetSchema,
+  RunPlanTargetWire,
+} from "./schemas/run-plan.ts";
+import type { EvaluatorAttachmentWire } from "./schemas/suite-fields.ts";
+
+// Client for /api/v1/run-plans. A run plan is identified by name; re-using the
+// name replaces its config while parameters/notes belong to one run only.
+
+/**
+ * The shapes the run plan tools accept, taken from the zod that validates
+ * them. They are inferred rather than written twice, so the schema stays the
+ * one place the shape is stated.
+ */
+
+/** What a run plan covers. Mirrors `suiteScopeSchema` on the platform. */
+export type RunPlanScope = z.infer<typeof runPlanScopeSchema>;
+
+/** One thing a plan runs its scenarios against, as a tool caller writes it. */
+export type RunPlanTarget = z.infer<typeof runPlanTargetSchema>;
+
+/** The values a run supplies for the parameters its scenarios declare. */
+export type RunParameters = z.infer<typeof runParametersSchema>;
+
+export interface RunPlan {
+  id: string;
+  name: string;
+  slug: string;
+  scope: RunPlanScope | null;
+  scenarioIds: string[];
+  targets: RunPlanTargetWire[];
+  repeatCount: number;
+  simulatorModel: string | null;
+  judgeModel: string | null;
+  labels: string[];
+  archivedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+  platformUrl: string;
+  /** The plan's own evaluators. Absent on servers that predate them. */
+  evaluators?: EvaluatorAttachmentWire[];
+}
+
+/** The configuration a run either writes onto a plan or creates it with. */
+export interface RunPlanConfig {
+  scope: RunPlanScope;
+  targets: RunPlanTargetWire[];
+  repeatCount?: number;
+  simulatorModel?: string | null;
+  judgeModel?: string | null;
+  scenarioIds?: string[];
+  /**
+   * The plan's own evaluators, run beside the ones its test suites attach. A
+   * plan evaluator reads the conversation and the trace, never a scenario
+   * field. Left out, the plan keeps what it already holds.
+   */
+  evaluators?: EvaluatorAttachmentWire[];
+}
+
+export interface RunPlanRunResult {
+  scheduled: boolean;
+  batchRunId: string;
+  setId: string;
+  jobCount: number;
+  skippedArchived: {
+    scenarios: string[];
+    targets: string[];
+  };
+  items: {
+    scenarioRunId: string;
+    scenarioId: string;
+    target: RunPlanTargetWire;
+    name: string | null;
+  }[];
+  runPlanId: string;
+  planName: string;
+  /** True when this run created the plan, false when it joined an existing one. */
+  created: boolean;
+  platformUrl: string;
+}
+
+export interface RunPlanArchiveResponse {
+  id: string;
+  archived: boolean;
+}
+
+/** Lists the run plans of the project. */
+export async function listRunPlans(params?: { includeArchived?: boolean }): Promise<RunPlan[]> {
+  const query = params?.includeArchived ? "?includeArchived=true" : "";
+  return makeRequest("GET", `/api/v1/run-plans${query}`) as Promise<RunPlan[]>;
+}
+
+/** Retrieves a single run plan by id. */
+export async function getRunPlan(id: string): Promise<RunPlan> {
+  return makeRequest("GET", `/api/v1/run-plans/${encodeURIComponent(id)}`) as Promise<RunPlan>;
+}
+
+/**
+ * Runs a configuration by plan name. A name that exists is replaced with this
+ * configuration, a name that does not is created. Sending no name lets the
+ * server derive one.
+ */
+export async function runRunPlan(data: {
+  name?: string;
+  config: RunPlanConfig;
+  idempotencyKey?: string;
+  parameters?: RunParameters;
+  note?: string;
+}): Promise<RunPlanRunResult> {
+  return makeRequest("POST", "/api/v1/run-plans/run", data) as Promise<RunPlanRunResult>;
+}
+
+/** Runs a plan again with the configuration it already holds. */
+export async function rerunRunPlan(
+  id: string,
+  data?: {
+    idempotencyKey?: string;
+    parameters?: RunParameters;
+    note?: string;
+  },
+): Promise<RunPlanRunResult> {
+  return makeRequest(
+    "POST",
+    `/api/v1/run-plans/${encodeURIComponent(id)}/run`,
+    data ?? {},
+  ) as Promise<RunPlanRunResult>;
+}
+
+/** Archives a run plan. */
+export async function archiveRunPlan(id: string): Promise<RunPlanArchiveResponse> {
+  return makeRequest(
+    "DELETE",
+    `/api/v1/run-plans/${encodeURIComponent(id)}`,
+  ) as Promise<RunPlanArchiveResponse>;
+}

package/src/langwatch-api-scenarios.ts

@@ -1,4 +1,5 @@
-import { makeRequest } from "./langwatch-api.js";
+import { makeRequest } from "./langwatch-api.ts";
+import type { ScenarioFieldValues } from "./schemas/suite-fields.ts";
 
 // --- Scenario types ---
 
@@ -8,6 +9,10 @@ export interface ScenarioSummary {
   situation: string;
   criteria: string[];
   labels: string[];
+  /** The test suite this scenario is filed in, null when it is unfiled. */
+  testSuiteId?: string | null;
+  /** The scenario's value for each field its test suite declares. */
+  fields?: ScenarioFieldValues;
 }
 
 export interface ScenarioArchiveResponse {
@@ -19,14 +24,14 @@ export interface ScenarioArchiveResponse {
 
 /** Lists all scenarios in the project. */
 export async function listScenarios(): Promise<ScenarioSummary[]> {
-  return makeRequest("GET", "/api/scenarios") as Promise<ScenarioSummary[]>;
+  return makeRequest("GET", "/api/v1/scenarios") as Promise<ScenarioSummary[]>;
 }
 
 /** Retrieves a single scenario by ID. */
 export async function getScenario(id: string): Promise<ScenarioSummary> {
   return makeRequest(
     "GET",
-    `/api/scenarios/${encodeURIComponent(id)}`
+    `/api/v1/scenarios/${encodeURIComponent(id)}`,
   ) as Promise<ScenarioSummary>;
 }
 
@@ -36,8 +41,10 @@ export async function createScenario(data: {
   situation: string;
   criteria?: string[];
   labels?: string[];
+  testSuiteId?: string | null;
+  fields?: ScenarioFieldValues;
 }): Promise<ScenarioSummary> {
-  return makeRequest("POST", "/api/scenarios", data) as Promise<ScenarioSummary>;
+  return makeRequest("POST", "/api/v1/scenarios", data) as Promise<ScenarioSummary>;
 }
 
 /** Updates an existing scenario. */
@@ -47,21 +54,21 @@ export async function updateScenario(params: {
   situation?: string;
   criteria?: string[];
   labels?: string[];
+  testSuiteId?: string | null;
+  fields?: ScenarioFieldValues;
 }): Promise<ScenarioSummary> {
   const { id, ...data } = params;
   return makeRequest(
     "PUT",
-    `/api/scenarios/${encodeURIComponent(id)}`,
-    data
+    `/api/v1/scenarios/${encodeURIComponent(id)}`,
+    data,
   ) as Promise<ScenarioSummary>;
 }
 
 /** Archives (soft-deletes) a scenario. */
-export async function archiveScenario(
-  id: string
-): Promise<ScenarioArchiveResponse> {
+export async function archiveScenario(id: string): Promise<ScenarioArchiveResponse> {
   return makeRequest(
     "DELETE",
-    `/api/scenarios/${encodeURIComponent(id)}`
+    `/api/v1/scenarios/${encodeURIComponent(id)}`,
   ) as Promise<ScenarioArchiveResponse>;
 }

package/src/langwatch-api-secrets.ts

@@ -1,4 +1,5 @@
-import { makeRequest } from "./langwatch-api.js";
+import { getConfig } from "./config.ts";
+import { makeRequest } from "./langwatch-api.ts";
 
 export interface SecretSummary {
   id: string;
@@ -8,41 +9,47 @@ export interface SecretSummary {
   updatedAt: string;
 }
 
+function projectId(): string {
+  const projectId = getConfig().projectId;
+  if (!projectId) {
+    throw new Error("LANGWATCH_PROJECT_ID is required for secret operations");
+  }
+  return projectId;
+}
+
 export async function listSecrets(): Promise<SecretSummary[]> {
-  return makeRequest("GET", "/api/secrets") as Promise<SecretSummary[]>;
+  const targetProjectId = projectId();
+  return makeRequest(
+    "GET",
+    `/api/v1/secrets?projectId=${encodeURIComponent(targetProjectId)}`,
+  ) as Promise<SecretSummary[]>;
 }
 
 export async function getSecret(id: string): Promise<SecretSummary> {
+  const targetProjectId = projectId();
   return makeRequest(
     "GET",
-    `/api/secrets/${encodeURIComponent(id)}`
+    `/api/v1/secrets/${encodeURIComponent(id)}?projectId=${encodeURIComponent(targetProjectId)}`,
   ) as Promise<SecretSummary>;
 }
 
-export async function createSecret(data: {
-  name: string;
-  value: string;
-}): Promise<SecretSummary> {
-  return makeRequest("POST", "/api/secrets", data) as Promise<SecretSummary>;
+export async function createSecret(data: { name: string; value: string }): Promise<SecretSummary> {
+  return makeRequest("POST", "/api/v1/secrets", {
+    projectId: projectId(),
+    ...data,
+  }) as Promise<SecretSummary>;
 }
 
-export async function updateSecret(params: {
-  id: string;
-  value: string;
-}): Promise<SecretSummary> {
+export async function updateSecret(params: { id: string; value: string }): Promise<SecretSummary> {
   const { id, value } = params;
-  return makeRequest(
-    "PUT",
-    `/api/secrets/${encodeURIComponent(id)}`,
-    { value }
-  ) as Promise<SecretSummary>;
+  return makeRequest("PUT", `/api/v1/secrets/${encodeURIComponent(id)}`, {
+    projectId: projectId(),
+    value,
+  }) as Promise<SecretSummary>;
 }
 
-export async function deleteSecret(
-  id: string
-): Promise<{ id: string; deleted: boolean }> {
-  return makeRequest(
-    "DELETE",
-    `/api/secrets/${encodeURIComponent(id)}`
-  ) as Promise<{ id: string; deleted: boolean }>;
+export async function deleteSecret(id: string): Promise<{ id: string; deleted: boolean }> {
+  return makeRequest("DELETE", `/api/v1/secrets/${encodeURIComponent(id)}`, {
+    projectId: projectId(),
+  }) as Promise<{ id: string; deleted: boolean }>;
 }

package/src/langwatch-api-simulation-runs.ts

@@ -1,4 +1,5 @@
-import { makeRequest } from "./langwatch-api.js";
+import { makeRequest } from "./langwatch-api.ts";
+import type { SimulationRunEvaluation } from "./tools/format-suite-details.ts";
 
 export interface SimulationRunSummary {
   scenarioRunId: string;
@@ -14,8 +15,12 @@ export interface SimulationRunSummary {
     metCriteria?: string[];
     unmetCriteria?: string[];
     error?: string | null;
+    /** One result per evaluator that ran after the conversation. */
+    evaluations?: SimulationRunEvaluation[];
   } | null;
-  messages?: Array<{ role: string; content: string }>;
+  messages?: { role: string; content: string }[];
+  /** Why the run was started, as given when it was queued. Null without one. */
+  note?: string | null;
   timestamp: number;
   updatedAt: number;
 }
@@ -37,15 +42,13 @@ export async function listSimulationRuns(params?: {
   if (params?.batchRunId) query.set("batchRunId", params.batchRunId);
   if (params?.limit) query.set("limit", String(params.limit));
   const qs = query.toString() ? `?${query}` : "";
-  return makeRequest("GET", `/api/simulation-runs${qs}`) as Promise<SimulationRunListResponse>;
+  return makeRequest("GET", `/api/v1/simulation-runs${qs}`) as Promise<SimulationRunListResponse>;
 }
 
 /** Gets a single simulation run by its ID. */
-export async function getSimulationRun(
-  scenarioRunId: string,
-): Promise<SimulationRunSummary> {
+export async function getSimulationRun(scenarioRunId: string): Promise<SimulationRunSummary> {
   return makeRequest(
     "GET",
-    `/api/simulation-runs/${encodeURIComponent(scenarioRunId)}`,
+    `/api/v1/simulation-runs/${encodeURIComponent(scenarioRunId)}`,
   ) as Promise<SimulationRunSummary>;
 }

package/src/langwatch-api-test-suites.ts

@@ -0,0 +1,116 @@
+import type { RunParameters, RunPlanRunResult } from "./langwatch-api-run-plans.ts";
+import { makeRequest } from "./langwatch-api.ts";
+import type { RunPlanTargetWire } from "./schemas/run-plan.ts";
+import type { EvaluatorAttachmentWire, SuiteField } from "./schemas/suite-fields.ts";
+
+/**
+ * Client for `/api/v1/test-suites`. A test suite groups scenarios by
+ * name; running one is sugar over a run plan, returning the same
+ * result shape.
+ */
+
+export interface TestSuite {
+  id: string;
+  name: string;
+  slug: string;
+  scenarioIds: string[];
+  scenarioCount: number;
+  archivedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+  platformUrl: string;
+  /** The fields the suite declares. Absent on servers that predate them. */
+  fields?: SuiteField[];
+  /** The evaluators attached to the suite. Absent on servers that predate them. */
+  evaluators?: EvaluatorAttachmentWire[];
+}
+
+export interface TestSuiteDetail extends TestSuite {
+  scenarios: { id: string; name: string }[];
+}
+
+export interface TestSuiteArchiveResponse {
+  id: string;
+  archived: boolean;
+}
+
+/** Lists the test suites of the project. */
+export async function listTestSuites(): Promise<TestSuite[]> {
+  return makeRequest("GET", "/api/v1/test-suites") as Promise<TestSuite[]>;
+}
+
+/** Creates a test suite, with the fields and the evaluators it declares. */
+export async function createTestSuite(data: {
+  name: string;
+  fields?: SuiteField[];
+  evaluators?: EvaluatorAttachmentWire[];
+}): Promise<TestSuite> {
+  return makeRequest("POST", "/api/v1/test-suites", data) as Promise<TestSuite>;
+}
+
+/** Retrieves a test suite with the scenarios filed in it. */
+export async function getTestSuite(id: string): Promise<TestSuiteDetail> {
+  return makeRequest(
+    "GET",
+    `/api/v1/test-suites/${encodeURIComponent(id)}`,
+  ) as Promise<TestSuiteDetail>;
+}
+
+/**
+ * Edits a test suite: any of its name, its fields and its evaluators. A list
+ * given here replaces the one the suite holds.
+ */
+export async function updateTestSuite(params: {
+  id: string;
+  name?: string;
+  fields?: SuiteField[];
+  evaluators?: EvaluatorAttachmentWire[];
+}): Promise<TestSuite> {
+  const { id, ...data } = params;
+  return makeRequest(
+    "PATCH",
+    `/api/v1/test-suites/${encodeURIComponent(id)}`,
+    data,
+  ) as Promise<TestSuite>;
+}
+
+/** Renames a test suite. */
+export async function renameTestSuite(params: { id: string; name: string }): Promise<TestSuite> {
+  return makeRequest("PATCH", `/api/v1/test-suites/${encodeURIComponent(params.id)}`, {
+    name: params.name,
+  }) as Promise<TestSuite>;
+}
+
+/** Archives a test suite and the scenarios filed in it. */
+export async function archiveTestSuite(id: string): Promise<TestSuiteArchiveResponse> {
+  return makeRequest(
+    "DELETE",
+    `/api/v1/test-suites/${encodeURIComponent(id)}`,
+  ) as Promise<TestSuiteArchiveResponse>;
+}
+
+/**
+ * Runs every scenario of a test suite against the given targets. The server
+ * creates or joins the run plan named "<suite name> <target name>" when no
+ * name is sent.
+ */
+export async function runTestSuite({
+  id,
+  ...data
+}: {
+  id: string;
+  targets: RunPlanTargetWire[];
+  name?: string;
+  repeatCount?: number;
+  simulatorModel?: string | null;
+  judgeModel?: string | null;
+  parameters?: RunParameters;
+  note?: string;
+  idempotencyKey?: string;
+}): Promise<RunPlanRunResult> {
+  return makeRequest(
+    "POST",
+    `/api/v1/test-suites/${encodeURIComponent(id)}/run`,
+    data,
+  ) as Promise<RunPlanRunResult>;
+}

package/src/langwatch-api-triggers.ts

@@ -1,48 +1,110 @@
-import { makeRequest } from "./langwatch-api.js";
+import { z } from "zod";
 
-export interface TriggerSummary {
-  id: string;
-  name: string;
-  action: string;
-  actionParams: Record<string, unknown>;
-  filters: Record<string, unknown>;
-  active: boolean;
-  message: string | null;
-  alertType: string | null;
-  createdAt: string;
-  updatedAt: string;
-}
+import { makeRequest } from "./langwatch-api.ts";
+import {
+  deletedTriggerSchema,
+  type GraphAlertRule,
+  type NotificationCadence,
+  type ReportRule,
+  type TestFireResult,
+  type Trigger,
+  type TriggerAction,
+  type TriggerActionParams,
+  type TriggerAlertType,
+  type TriggerFirePage,
+  type TriggerTemplates,
+  testFireResultSchema,
+  triggerFirePageSchema,
+  triggerSchema,
+} from "./schemas/triggers.ts";
 
-export async function listTriggers(): Promise<TriggerSummary[]> {
-  return makeRequest("GET", "/api/triggers") as Promise<TriggerSummary[]>;
-}
+/**
+ * The `/api/v1/triggers` calls, each answering with the shape `schemas/triggers` declares.
+ * Responses are read through that schema, not cast, so a deployment that stops sending something
+ * a tool renders fails loudly instead of leaving an `undefined` in an agent's message.
+ */
 
-export async function getTrigger(id: string): Promise<TriggerSummary> {
-  return makeRequest("GET", `/api/triggers/${encodeURIComponent(id)}`) as Promise<TriggerSummary>;
-}
-
-export async function createTrigger(data: {
+export interface CreateTriggerInput {
   name: string;
-  action: string;
-  actionParams?: Record<string, unknown>;
+  action: TriggerAction;
+  /** Omitted sends `{}`, as before `actionParams` was stated per channel. */
+  actionParams?: TriggerActionParams;
   filters?: Record<string, unknown>;
+  filterQuery?: string | null;
   message?: string;
-  alertType?: string;
-}): Promise<TriggerSummary> {
-  return makeRequest("POST", "/api/triggers", data) as Promise<TriggerSummary>;
+  alertType?: TriggerAlertType;
+  customGraphId?: string;
+  graphAlert?: GraphAlertRule;
+  report?: ReportRule;
+  templates?: TriggerTemplates;
+  notificationCadence?: NotificationCadence;
 }
 
-export async function updateTrigger(params: {
+/** `actionParams` replaces the delivery configuration as a whole; a credential
+ *  sent back as `[redacted]` keeps the stored one. The channel cannot change. */
+type UpdateTriggerInput = Partial<
+  Omit<CreateTriggerInput, "action" | "customGraphId" | "message" | "alertType">
+> & {
   id: string;
-  name?: string;
   active?: boolean;
   message?: string | null;
-  alertType?: string | null;
-}): Promise<TriggerSummary> {
-  const { id, ...data } = params;
-  return makeRequest("PATCH", `/api/triggers/${encodeURIComponent(id)}`, data) as Promise<TriggerSummary>;
+  alertType?: TriggerAlertType | null;
+};
+
+export async function listTriggers(): Promise<Trigger[]> {
+  return z.array(triggerSchema).parse(await makeRequest("GET", "/api/v1/triggers"));
+}
+
+export async function getTrigger(id: string): Promise<Trigger> {
+  return triggerSchema.parse(
+    await makeRequest("GET", `/api/v1/triggers/${encodeURIComponent(id)}`),
+  );
+}
+
+export async function createTrigger(input: CreateTriggerInput): Promise<Trigger> {
+  return triggerSchema.parse(
+    await makeRequest("POST", "/api/v1/triggers", {
+      ...input,
+      actionParams: input.actionParams ?? {},
+    }),
+  );
+}
+
+export async function updateTrigger({ id, ...data }: UpdateTriggerInput): Promise<Trigger> {
+  return triggerSchema.parse(
+    await makeRequest("PATCH", `/api/v1/triggers/${encodeURIComponent(id)}`, data),
+  );
+}
+
+/** Send the automation's message to the destination it is saved with. */
+export async function testFireTrigger(id: string): Promise<TestFireResult> {
+  return testFireResultSchema.parse(
+    await makeRequest("POST", `/api/v1/triggers/${encodeURIComponent(id)}/test-fire`),
+  );
+}
+
+/** What the automation has done, newest first. Metadata only. `cursor` reads
+ *  the page after the one that answered with it as `nextCursor`. */
+export async function listTriggerFires({
+  id,
+  limit,
+  cursor,
+}: {
+  id: string;
+  limit?: number;
+  cursor?: string;
+}): Promise<TriggerFirePage> {
+  const query = new URLSearchParams();
+  if (limit !== undefined) query.set("limit", String(limit));
+  if (cursor) query.set("cursor", cursor);
+  const search = query.toString() ? `?${query.toString()}` : "";
+  return triggerFirePageSchema.parse(
+    await makeRequest("GET", `/api/v1/triggers/${encodeURIComponent(id)}/fires${search}`),
+  );
 }
 
-export async function deleteTrigger(id: string): Promise<{ id: string; deleted: boolean }> {
-  return makeRequest("DELETE", `/api/triggers/${encodeURIComponent(id)}`) as Promise<{ id: string; deleted: boolean }>;
+export async function deleteTrigger(id: string): Promise<z.infer<typeof deletedTriggerSchema>> {
+  return deletedTriggerSchema.parse(
+    await makeRequest("DELETE", `/api/v1/triggers/${encodeURIComponent(id)}`),
+  );
 }

package/src/langwatch-api-workflows.ts

@@ -1,4 +1,4 @@
-import { makeRequest } from "./langwatch-api.js";
+import { makeRequest } from "./langwatch-api.ts";
 
 export interface WorkflowSummary {
   id: string;
@@ -12,30 +12,30 @@ export interface WorkflowSummary {
 }
 
 export async function listWorkflows(): Promise<WorkflowSummary[]> {
-  return makeRequest("GET", "/api/workflows") as Promise<WorkflowSummary[]>;
+  return makeRequest("GET", "/api/v1/workflows") as Promise<WorkflowSummary[]>;
 }
 
 export async function getWorkflow(id: string): Promise<WorkflowSummary> {
   return makeRequest(
     "GET",
-    `/api/workflows/${encodeURIComponent(id)}`,
+    `/api/v1/workflows/${encodeURIComponent(id)}`,
   ) as Promise<WorkflowSummary>;
 }
 
 export async function runWorkflow(
   id: string,
-  input?: Record<string, unknown>
+  input?: Record<string, unknown>,
 ): Promise<Record<string, unknown>> {
   return makeRequest(
     "POST",
-    `/api/workflows/${encodeURIComponent(id)}/run`,
+    `/api/v1/workflows/${encodeURIComponent(id)}/run`,
     input ?? {},
   ) as Promise<Record<string, unknown>>;
 }
 
 export async function deleteWorkflow(id: string): Promise<{ id: string; archived: boolean }> {
-  return makeRequest(
-    "DELETE",
-    `/api/workflows/${encodeURIComponent(id)}`,
-  ) as Promise<{ id: string; archived: boolean }>;
+  return makeRequest("DELETE", `/api/v1/workflows/${encodeURIComponent(id)}`) as Promise<{
+    id: string;
+    archived: boolean;
+  }>;
 }