@langwatch/mcp-server 1.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/.python-version +1 -0
  2. package/CHANGELOG.md +110 -0
  3. package/CONTRIBUTING.md +1 -1
  4. package/README.md +86 -4
  5. package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
  6. package/dist/archive-run-plan-474OKHHU.js +16 -0
  7. package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
  8. package/dist/archive-test-suite-O7FLDMXK.js +16 -0
  9. package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
  10. package/dist/chunk-56OGTS24.js +44 -0
  11. package/dist/chunk-6DJDEXET.js +63 -0
  12. package/dist/chunk-7E4FZIYJ.js +15 -0
  13. package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
  14. package/dist/chunk-CKNAH5TG.js +74 -0
  15. package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
  16. package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
  17. package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
  18. package/dist/chunk-HV4DWKBI.js +46 -0
  19. package/dist/chunk-JR6DJZOU.js +49 -0
  20. package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
  21. package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
  22. package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
  23. package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
  24. package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
  25. package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
  26. package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
  27. package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
  28. package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
  29. package/dist/chunk-UF6XIGYR.js +58 -0
  30. package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
  31. package/dist/chunk-UJPXFA26.js +267 -0
  32. package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
  33. package/dist/config.js +2 -2
  34. package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
  35. package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
  36. package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
  37. package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
  38. package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
  39. package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
  40. package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
  41. package/dist/create-mcp-server.js +9 -4
  42. package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
  43. package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
  44. package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
  45. package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
  46. package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
  47. package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
  48. package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
  49. package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
  50. package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
  51. package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
  52. package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
  53. package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
  54. package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
  55. package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
  56. package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
  57. package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
  58. package/dist/get-agent-7THF7HCU.js +66 -0
  59. package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
  60. package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
  61. package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
  62. package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
  63. package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
  64. package/dist/get-prompt-GVKXF5NL.js +100 -0
  65. package/dist/get-run-plan-NY7J5GRE.js +61 -0
  66. package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
  67. package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
  68. package/dist/get-test-suite-4HKA7OKS.js +49 -0
  69. package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
  70. package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
  71. package/dist/http-security-EU3CHTY7.js +26 -0
  72. package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
  73. package/dist/index.js +6437 -4020
  74. package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
  75. package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
  76. package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
  77. package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
  78. package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
  79. package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
  80. package/dist/list-agents-AMVDQPQL.js +43 -0
  81. package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
  82. package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
  83. package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
  84. package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
  85. package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
  86. package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
  87. package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
  88. package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
  89. package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
  90. package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
  91. package/dist/list-run-plans-UBAU2DGH.js +46 -0
  92. package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
  93. package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
  94. package/dist/list-test-suites-L7R2O6DR.js +35 -0
  95. package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
  96. package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
  97. package/dist/rename-test-suite-NTYHAO5X.js +21 -0
  98. package/dist/report-issue-RHZZ33IS.js +885 -0
  99. package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
  100. package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
  101. package/dist/run-agent-CAGWGTYT.js +72 -0
  102. package/dist/run-plan-T4ZYOZGA.js +39 -0
  103. package/dist/run-test-suite-L4MZ7G4D.js +26 -0
  104. package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
  105. package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
  106. package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
  107. package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
  108. package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
  109. package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
  110. package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
  111. package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
  112. package/dist/update-prompt-ZQQ2C3KU.js +143 -0
  113. package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
  114. package/package.json +8 -9
  115. package/pyproject.toml +5 -5
  116. package/src/__tests__/agent-tools.unit.test.ts +257 -0
  117. package/src/__tests__/all-tools.integration.test.ts +321 -75
  118. package/src/__tests__/config.unit.test.ts +33 -0
  119. package/src/__tests__/docker.integration.test.ts +91 -7
  120. package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
  121. package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
  122. package/src/__tests__/http-security-origin.unit.test.ts +97 -0
  123. package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
  124. package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
  125. package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
  126. package/src/__tests__/http-server-origin.integration.test.ts +169 -0
  127. package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
  128. package/src/__tests__/http-transport.integration.test.ts +85 -87
  129. package/src/__tests__/integration.integration.test.ts +7 -11
  130. package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
  131. package/src/__tests__/prompts.integration.test.ts +736 -0
  132. package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
  133. package/src/__tests__/scenario-tools.unit.test.ts +19 -0
  134. package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
  135. package/src/__tests__/support/http-server-harness.ts +138 -0
  136. package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
  137. package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
  138. package/src/config.ts +17 -2
  139. package/src/create-mcp-server.ts +334 -55
  140. package/src/http-security.ts +492 -0
  141. package/src/http-server.ts +536 -152
  142. package/src/index.ts +49 -2
  143. package/src/langwatch-api-agents.ts +196 -12
  144. package/src/langwatch-api-run-plans.ts +157 -0
  145. package/src/langwatch-api-scenarios.ts +9 -0
  146. package/src/langwatch-api-simulation-runs.ts +5 -0
  147. package/src/langwatch-api-test-suites.ts +135 -0
  148. package/src/langwatch-api.ts +62 -12
  149. package/src/schemas/index.ts +5 -0
  150. package/src/schemas/run-plan.ts +92 -0
  151. package/src/schemas/suite-fields.ts +149 -0
  152. package/src/tools/archive-run-plan.ts +12 -0
  153. package/src/tools/archive-test-suite.ts +12 -0
  154. package/src/tools/create-scenario.ts +8 -0
  155. package/src/tools/create-test-suite.ts +43 -0
  156. package/src/tools/discover-evaluator-schema.ts +2 -2
  157. package/src/tools/discover-scenario-schema.ts +54 -0
  158. package/src/tools/format-run-plan.ts +68 -0
  159. package/src/tools/format-scenario.ts +14 -0
  160. package/src/tools/format-suite-details.ts +96 -0
  161. package/src/tools/get-agent.ts +45 -3
  162. package/src/tools/get-prompt.ts +108 -24
  163. package/src/tools/get-run-plan.ts +62 -0
  164. package/src/tools/get-scenario.ts +3 -0
  165. package/src/tools/get-simulation-run.ts +9 -0
  166. package/src/tools/get-test-suite.ts +49 -0
  167. package/src/tools/list-agents.ts +20 -3
  168. package/src/tools/list-run-plans.ts +47 -0
  169. package/src/tools/list-scenarios.ts +12 -2
  170. package/src/tools/list-simulation-runs.ts +6 -2
  171. package/src/tools/list-test-suites.ts +35 -0
  172. package/src/tools/rename-test-suite.ts +18 -0
  173. package/src/tools/rerun-run-plan.ts +21 -0
  174. package/src/tools/run-agent.ts +54 -6
  175. package/src/tools/run-plan.ts +49 -0
  176. package/src/tools/run-test-suite.ts +30 -0
  177. package/src/tools/test-agent.ts +18 -0
  178. package/src/tools/update-prompt.ts +195 -10
  179. package/src/tools/update-scenario.ts +8 -0
  180. package/src/tools/update-test-suite.ts +45 -0
  181. package/tsconfig.json +7 -2
  182. package/uv.lock +418 -298
  183. package/dist/archive-suite-C3KX6IIZ.js +0 -16
  184. package/dist/chunk-WDH5P5RF.js +0 -48
  185. package/dist/create-suite-TWF6BBR4.js +0 -39
  186. package/dist/get-agent-7QJVPH72.js +0 -29
  187. package/dist/get-prompt-I753LZKZ.js +0 -46
  188. package/dist/get-suite-GTOCY5PL.js +0 -45
  189. package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
  190. package/dist/list-agents-TIJTP566.js +0 -33
  191. package/dist/list-suites-5VK4HXEK.js +0 -43
  192. package/dist/report-issue-DXJ6TPTC.js +0 -412
  193. package/dist/run-agent-RBSIFKRT.js +0 -37
  194. package/dist/run-suite-K3DNQ3SE.js +0 -30
  195. package/dist/update-prompt-RQFTEO7Y.js +0 -25
  196. package/dist/update-suite-TRGGLNR7.js +0 -40
  197. package/pnpm-workspace.yaml +0 -34
  198. package/src/langwatch-api-suites.ts +0 -118
  199. package/src/tools/archive-suite.ts +0 -12
  200. package/src/tools/create-suite.ts +0 -42
  201. package/src/tools/get-suite.ts +0 -47
  202. package/src/tools/list-suites.ts +0 -43
  203. package/src/tools/run-suite.ts +0 -28
  204. package/src/tools/update-suite.ts +0 -44
@@ -82,25 +82,48 @@ export interface PromptSummary {
82
82
  version?: number;
83
83
  }
84
84
 
85
+ export interface PromptTag {
86
+ name: string;
87
+ versionId: string;
88
+ }
89
+
90
+ export type PromptFieldList = Array<{ identifier: string; type: string }>;
91
+
92
+ /**
93
+ * A single versioned prompt payload. `GET /api/prompts/:id` returns the
94
+ * requested version's data flattened to the top level (merged with the base
95
+ * prompt data), and `GET /api/prompts/:id/versions` returns an array of
96
+ * entries in this same shape — there is no nested `versions` array on the
97
+ * detail response.
98
+ */
85
99
  export interface PromptVersion {
100
+ versionId?: string;
86
101
  version?: number;
87
- commitMessage?: string;
102
+ commitMessage?: string | null;
88
103
  model?: string;
104
+ /** Legacy single-text prompt body. */
105
+ prompt?: string;
89
106
  messages?: Array<{ role: string; content: string }>;
107
+ temperature?: number;
108
+ maxTokens?: number;
109
+ responseFormat?: Record<string, unknown> | null;
110
+ /** Runtime parameters: an object map of name -> JSON value, not an array. */
111
+ parameters?: Record<string, unknown>;
112
+ inputs?: PromptFieldList;
113
+ outputs?: PromptFieldList;
114
+ /** All tags on the prompt; each names the version it currently points to. */
115
+ tags?: PromptTag[];
90
116
  }
91
117
 
92
- export interface PromptDetailResponse extends PromptSummary {
93
- versions?: PromptVersion[];
94
- model?: string;
95
- messages?: Array<{ role: string; content: string }>;
96
- prompt?: Array<{ role: string; content: string }>;
97
- }
118
+ export interface PromptDetailResponse extends PromptSummary, PromptVersion {}
98
119
 
99
120
  export interface PromptMutationResponse {
100
121
  id?: string;
101
122
  handle?: string;
102
123
  name?: string;
103
124
  latestVersionNumber?: number;
125
+ versionId?: string;
126
+ tags?: string[];
104
127
  }
105
128
 
106
129
  // --- HTTP client ---
@@ -153,8 +176,10 @@ interface ParsedErrorBody {
153
176
  const VALID_FAULTS: readonly HandledErrorFault[] = ["customer", "platform", "provider"];
154
177
 
155
178
  /**
156
- * Parses an error response body as a handled-error envelope. Accepts both the
157
- * REST shape (`{ error: "<code>", message, tips?, docsUrl?, fault? }`) and the
179
+ * Parses an error response body as a handled-error envelope. Accepts three
180
+ * shapes: the canonical v1 envelope
181
+ * (`{ error: { type, code, message, meta?, trace_id } }`), the legacy REST
182
+ * shape (`{ error: "<code>", message, tips?, docsUrl?, fault? }`) and the
158
183
  * serialized tRPC shape (`{ code, message?, tips?, docsUrl?, fault?, ... }`).
159
184
  * Returns an empty object when the body is not a recognizable error envelope.
160
185
  */
@@ -164,7 +189,16 @@ function parseErrorBody(responseBody: string): ParsedErrorBody {
164
189
  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
165
190
  return {};
166
191
  }
167
- const body = parsed as Record<string, unknown>;
192
+ const envelope = parsed as Record<string, unknown>;
193
+ // The canonical v1 envelope nests everything under `error`. Read the
194
+ // nested object as the body so one parser serves both generations.
195
+ const nested =
196
+ typeof envelope.error === "object" &&
197
+ envelope.error !== null &&
198
+ !Array.isArray(envelope.error)
199
+ ? (envelope.error as Record<string, unknown>)
200
+ : null;
201
+ const body = nested ?? envelope;
168
202
  // Prefer `code` (the domain discriminant) over `error` — the
169
203
  // packages/api unversioned envelope uses `error` for the HTTP status
170
204
  // text ("Not Found") while `code` holds the real code.
@@ -294,14 +328,20 @@ export async function makeRequest(
294
328
  return response.json();
295
329
  }
296
330
 
297
- /** Searches traces with optional filters and pagination. */
331
+ /**
332
+ * Searches traces with optional filters and pagination.
333
+ *
334
+ * Paging is by `scrollId` only: pass the one the previous response returned,
335
+ * and stop when a response carries none. `pageOffset` used to be accepted here
336
+ * but was never read by the server, so an offset walk silently re-served the
337
+ * first page; it is rejected at the boundary now (#6808).
338
+ */
298
339
  export async function searchTraces(params: {
299
340
  query?: string;
300
341
  filters?: Record<string, string[]>;
301
342
  startDate: number;
302
343
  endDate: number;
303
344
  pageSize?: number;
304
- pageOffset?: number;
305
345
  scrollId?: string;
306
346
  format?: "digest" | "json";
307
347
  }): Promise<SearchTracesResponse> {
@@ -365,6 +405,16 @@ export async function getPrompt(
365
405
  ) as Promise<PromptDetailResponse>;
366
406
  }
367
407
 
408
+ /** Lists all versions of a prompt (versioned data only). */
409
+ export async function getPromptVersions(
410
+ idOrHandle: string
411
+ ): Promise<PromptVersion[]> {
412
+ return makeRequest(
413
+ "GET",
414
+ `/api/prompts/${encodeURIComponent(idOrHandle)}/versions`
415
+ ) as Promise<PromptVersion[]>;
416
+ }
417
+
368
418
  /** Creates a new prompt. */
369
419
  export async function createPrompt(data: {
370
420
  handle: string;
@@ -2,3 +2,8 @@ export { filterFields, type FilterFieldInfo } from "./filter-fields.js";
2
2
  export { analyticsMetrics, type MetricInfo } from "./analytics-metrics.js";
3
3
  export { analyticsGroups, type GroupByInfo } from "./analytics-groups.js";
4
4
  export { createDatasetSchema } from "./create-dataset.js";
5
+ export {
6
+ runParametersSchema,
7
+ runPlanScopeSchema,
8
+ runPlanTargetSchema,
9
+ } from "./run-plan.js";
@@ -0,0 +1,92 @@
1
+ import { z } from "zod";
2
+
3
+ /**
4
+ * Typed inputs for the run plan tools.
5
+ *
6
+ * Each one mirrors the zod the platform validates the REST body with, so an
7
+ * agent is told what is accepted before the request goes out rather than
8
+ * after: `suiteScopeSchema` and `suiteTargetSchema` on the server side.
9
+ */
10
+
11
+ /** What a run plan covers. */
12
+ export const runPlanScopeSchema = z.discriminatedUnion("mode", [
13
+ z
14
+ .object({ mode: z.literal("all") })
15
+ .describe("Every non-archived scenario of the project"),
16
+ z
17
+ .object({
18
+ mode: z.literal("test_suites"),
19
+ testSuiteIds: z
20
+ .array(z.string())
21
+ .describe("The test suite IDs whose scenarios to run"),
22
+ })
23
+ .describe("The non-archived scenarios filed in the given test suites"),
24
+ z
25
+ .object({
26
+ mode: z.literal("labels"),
27
+ labels: z.array(z.string()).describe("The labels to select scenarios by"),
28
+ })
29
+ .describe(
30
+ "The non-archived scenarios carrying at least one of the given labels",
31
+ ),
32
+ z
33
+ .object({ mode: z.literal("scenarios") })
34
+ .describe("The scenarios named in scenarioIds, picked by hand"),
35
+ ]);
36
+
37
+ /** The values a run supplies for the parameters its scenarios declare. */
38
+ export const runParametersSchema = z.record(
39
+ z.string(),
40
+ z.union([z.string(), z.number(), z.boolean()]),
41
+ );
42
+
43
+ /**
44
+ * One thing a plan runs its scenarios against, and the values that target
45
+ * alone runs with.
46
+ *
47
+ * The parameters are what makes a comparison run: two targets may name the
48
+ * same agent with different values, and the platform runs and reports each one
49
+ * on its own.
50
+ */
51
+ export const runPlanTargetSchema = z.object({
52
+ type: z
53
+ .enum(["connected", "prompt", "http", "code", "workflow"])
54
+ .describe(
55
+ "What kind of thing the target is. A connected agent is one that registered itself from code with connectAgent (TypeScript) or connect_agent (Python).",
56
+ ),
57
+ referenceId: z
58
+ .string()
59
+ .describe(
60
+ "The ID of the prompt, agent or workflow to run against. A connected agent may also be named as <name>@<environment>, for example support-agent@production; the platform resolves it.",
61
+ ),
62
+ parameters: runParametersSchema
63
+ .optional()
64
+ .describe(
65
+ "Parameter values this target alone runs with, by name. They override the run-level parameters for this target. Name the same agent twice with different values here to compare it on two models.",
66
+ ),
67
+ });
68
+
69
+ /**
70
+ * One target as `/api/v1` carries it.
71
+ *
72
+ * The tool input spells the per-target values `parameters`, beside the
73
+ * run-level `parameters` an agent already knows. The REST body spells them
74
+ * `runParameters`, because a target holds the values it runs with rather than
75
+ * the values of a run.
76
+ */
77
+ export interface RunPlanTargetWire {
78
+ type: z.infer<typeof runPlanTargetSchema>["type"];
79
+ referenceId: string;
80
+ runParameters?: z.infer<typeof runParametersSchema>;
81
+ }
82
+
83
+ /** The targets of a tool call, as the REST body carries them. */
84
+ export function toWireTargets(
85
+ targets: z.infer<typeof runPlanTargetSchema>[],
86
+ ): RunPlanTargetWire[] {
87
+ return targets.map((target) => ({
88
+ type: target.type,
89
+ referenceId: target.referenceId,
90
+ ...(target.parameters ? { runParameters: target.parameters } : {}),
91
+ }));
92
+ }
@@ -0,0 +1,149 @@
1
+ import { z } from "zod";
2
+
3
+ /**
4
+ * Typed inputs for the fields a test suite declares, the evaluators attached
5
+ * to a suite or a run plan, and the values a scenario carries per field.
6
+ *
7
+ * Each one mirrors the zod the platform validates the REST body with
8
+ * (`suiteFieldDefinitionsSchema`, `evaluatorAttachmentsSchema` and
9
+ * `scenarioFieldValuesSchema` on the server side), so an agent is told what
10
+ * is accepted before the request goes out rather than after.
11
+ */
12
+
13
+ /** The value types a field can hold. */
14
+ export const SUITE_FIELD_TYPES = ["text", "number", "boolean"] as const;
15
+
16
+ /** One field a test suite declares. */
17
+ export const suiteFieldSchema = z.object({
18
+ identifier: z
19
+ .string()
20
+ .min(1)
21
+ .max(64)
22
+ .regex(
23
+ /^[a-z][a-z0-9_]*$/,
24
+ "Lowercase letters, digits and underscores, starting with a letter",
25
+ )
26
+ .describe(
27
+ "The field name, as scenarios and evaluator mappings address it, for example golden_sql. Lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
28
+ ),
29
+ type: z
30
+ .enum(SUITE_FIELD_TYPES)
31
+ .describe("The value type every scenario carries for this field."),
32
+ });
33
+
34
+ /** The fields a test suite declares, in the order the platform shows them. */
35
+ export const suiteFieldsSchema = z
36
+ .array(suiteFieldSchema)
37
+ .max(30)
38
+ .describe(
39
+ "The fields the test suite declares, in order. Up to 30. Every scenario filed in the suite carries one value per field, and an evaluator attached to the suite reads them through its mappings.",
40
+ );
41
+
42
+ /** The sources an evaluator input can read from. */
43
+ export const SCENARIO_MAPPING_SOURCE_IDS = [
44
+ "conversation",
45
+ "scenario",
46
+ "trace",
47
+ ] as const;
48
+
49
+ /** Where one evaluator input reads its value: a source path or a literal. */
50
+ export const scenarioMappingSchema = z
51
+ .discriminatedUnion("type", [
52
+ z.object({
53
+ type: z.literal("source"),
54
+ sourceId: z
55
+ .enum(SCENARIO_MAPPING_SOURCE_IDS)
56
+ .describe(
57
+ "conversation, scenario or trace. See discover_schema({ category: 'scenarios' }) for the paths each one offers.",
58
+ ),
59
+ path: z
60
+ .array(z.string().min(1).max(256))
61
+ .min(1)
62
+ .max(3)
63
+ .describe(
64
+ "The path inside the source. conversation: [first_user_message] | [last_agent_message] | [transcript] | [messages]. scenario: [situation] | [criteria] | [fields, <identifier>]. trace: [contexts] | [tool_calls, <toolName>, input|output].",
65
+ ),
66
+ }),
67
+ z.object({
68
+ type: z.literal("value"),
69
+ value: z.string().max(65_536).describe("A literal the input reads."),
70
+ }),
71
+ ])
72
+ .describe(
73
+ "Where one evaluator input reads its value: a path into the conversation, the scenario or the trace, or a literal value.",
74
+ );
75
+
76
+ /** One evaluator attached to a test suite or a run plan. */
77
+ export const evaluatorAttachmentSchema = z.object({
78
+ id: z
79
+ .string()
80
+ .min(1)
81
+ .max(64)
82
+ .optional()
83
+ .describe(
84
+ "The attachment id, stable across edits. Leave it out on a new attachment and one is generated.",
85
+ ),
86
+ evaluatorId: z
87
+ .string()
88
+ .min(1)
89
+ .describe(
90
+ "The id of the saved evaluator to run. Read it from platform_list_evaluators or platform_get_evaluator.",
91
+ ),
92
+ required: z
93
+ .boolean()
94
+ .optional()
95
+ .describe(
96
+ "Whether a failing result fails the scenario. Defaults to true. A score-only evaluator reports and never gates, whatever this says.",
97
+ ),
98
+ mappings: z
99
+ .record(z.string().min(1).max(128), scenarioMappingSchema)
100
+ .describe(
101
+ "Where each evaluator input reads its value, keyed by input name. Read the inputs from platform_get_evaluator. A required input with no mapping makes the platform refuse the run until it is set.",
102
+ ),
103
+ });
104
+
105
+ /** The evaluators attached to a test suite or a run plan. */
106
+ export const evaluatorAttachmentsSchema = z
107
+ .array(evaluatorAttachmentSchema)
108
+ .max(20)
109
+ .describe(
110
+ "The evaluators that run after every scenario run, with their mappings. Up to 20. A required evaluator that fails fails the scenario; a score-only one reports beside the verdict.",
111
+ );
112
+
113
+ /** The values a scenario carries for the fields its suite declares. */
114
+ export const scenarioFieldValuesSchema = z
115
+ .record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
116
+ .describe(
117
+ "The scenario's value for each field its test suite declares, by identifier, for example { golden_sql: 'SELECT ...' }. A text field takes a string, a number field a number and a boolean field true or false. A field the suite does not declare is refused. On update, an empty object clears the values.",
118
+ );
119
+
120
+ export type SuiteField = z.infer<typeof suiteFieldSchema>;
121
+ export type ScenarioMapping = z.infer<typeof scenarioMappingSchema>;
122
+ export type EvaluatorAttachmentInput = z.infer<typeof evaluatorAttachmentSchema>;
123
+ export type ScenarioFieldValues = z.infer<typeof scenarioFieldValuesSchema>;
124
+
125
+ /** One attachment as the REST surface carries it: id and gate settled. */
126
+ export interface EvaluatorAttachmentWire {
127
+ id: string;
128
+ evaluatorId: string;
129
+ required: boolean;
130
+ mappings: Record<string, ScenarioMapping>;
131
+ }
132
+
133
+ const newAttachmentId = (): string =>
134
+ `att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
135
+
136
+ /**
137
+ * The attachments of a tool call, as the REST body carries them: a missing
138
+ * id is generated and a missing gate defaults to required.
139
+ */
140
+ export function toWireAttachments(
141
+ attachments: EvaluatorAttachmentInput[],
142
+ ): EvaluatorAttachmentWire[] {
143
+ return attachments.map((attachment) => ({
144
+ id: attachment.id ?? newAttachmentId(),
145
+ evaluatorId: attachment.evaluatorId,
146
+ required: attachment.required ?? true,
147
+ mappings: attachment.mappings,
148
+ }));
149
+ }
@@ -0,0 +1,12 @@
1
+ import { archiveRunPlan as apiArchiveRunPlan } from "../langwatch-api-run-plans.js";
2
+
3
+ /**
4
+ * Handles the platform_archive_run_plan MCP tool invocation.
5
+ */
6
+ export async function handleArchiveRunPlan(params: {
7
+ id: string;
8
+ }): Promise<string> {
9
+ const result = await apiArchiveRunPlan(params.id);
10
+
11
+ return `Run plan ${result.id} is archived. Its past runs stay readable, and the name is free for a new plan.`;
12
+ }
@@ -0,0 +1,12 @@
1
+ import { archiveTestSuite as apiArchiveTestSuite } from "../langwatch-api-test-suites.js";
2
+
3
+ /**
4
+ * Handles the platform_archive_test_suite MCP tool invocation.
5
+ */
6
+ export async function handleArchiveTestSuite(params: {
7
+ id: string;
8
+ }): Promise<string> {
9
+ const result = await apiArchiveTestSuite(params.id);
10
+
11
+ return `Test suite ${result.id} is archived, and the scenarios filed in it are archived with it.`;
12
+ }
@@ -1,4 +1,6 @@
1
1
  import { createScenario as apiCreateScenario } from "../langwatch-api-scenarios.js";
2
+ import type { ScenarioFieldValues } from "../schemas/suite-fields.js";
3
+ import { formatScenarioFields } from "./format-scenario.js";
2
4
 
3
5
  /**
4
6
  * Handles the platform_create_scenario MCP tool invocation.
@@ -11,6 +13,8 @@ export async function handleCreateScenario(params: {
11
13
  situation: string;
12
14
  criteria?: string[];
13
15
  labels?: string[];
16
+ testSuiteId?: string | null;
17
+ fields?: ScenarioFieldValues;
14
18
  }): Promise<string> {
15
19
  const result = await apiCreateScenario(params);
16
20
 
@@ -25,6 +29,10 @@ export async function handleCreateScenario(params: {
25
29
  if (Array.isArray(result.labels) && result.labels.length > 0) {
26
30
  lines.push(`**Labels**: ${result.labels.join(", ")}`);
27
31
  }
32
+ if (result.testSuiteId) {
33
+ lines.push(`**Test suite**: ${result.testSuiteId}`);
34
+ }
35
+ lines.push(...formatScenarioFields(result.fields));
28
36
 
29
37
  return lines.join("\n");
30
38
  }
@@ -0,0 +1,43 @@
1
+ import { createTestSuite as apiCreateTestSuite } from "../langwatch-api-test-suites.js";
2
+ import {
3
+ type EvaluatorAttachmentInput,
4
+ type SuiteField,
5
+ toWireAttachments,
6
+ } from "../schemas/suite-fields.js";
7
+ import {
8
+ formatEvaluatorAttachments,
9
+ formatSuiteFields,
10
+ } from "./format-suite-details.js";
11
+
12
+ /**
13
+ * Handles the platform_create_test_suite MCP tool invocation.
14
+ */
15
+ export async function handleCreateTestSuite(params: {
16
+ name: string;
17
+ fields?: SuiteField[];
18
+ evaluators?: EvaluatorAttachmentInput[];
19
+ }): Promise<string> {
20
+ const suite = await apiCreateTestSuite({
21
+ name: params.name,
22
+ ...(params.fields !== undefined ? { fields: params.fields } : {}),
23
+ ...(params.evaluators !== undefined
24
+ ? { evaluators: toWireAttachments(params.evaluators) }
25
+ : {}),
26
+ });
27
+
28
+ return [
29
+ `Test suite "${suite.name}" created.`,
30
+ "",
31
+ `**ID**: ${suite.id}`,
32
+ `**Slug**: ${suite.slug}`,
33
+ `**View**: ${suite.platformUrl}`,
34
+ ...formatSuiteFields(suite.fields),
35
+ ...formatEvaluatorAttachments(suite.evaluators),
36
+ "",
37
+ `> File scenarios in it with \`platform_create_scenario\` or \`platform_update_scenario\`, passing testSuiteId \`${suite.id}\`${
38
+ suite.fields && suite.fields.length > 0
39
+ ? ` and a value per field under \`fields\``
40
+ : ""
41
+ }.`,
42
+ ].join("\n");
43
+ }
@@ -1,5 +1,5 @@
1
- import { AVAILABLE_EVALUATORS } from "../../../langevals/ts-integration/evaluators.generated.js";
2
- import type { EvaluatorDefinition, EvaluatorTypes } from "../../../langevals/ts-integration/evaluators.generated.js";
1
+ import { AVAILABLE_EVALUATORS } from "../../../../services/langevals/ts-integration/evaluators.generated.js";
2
+ import type { EvaluatorDefinition, EvaluatorTypes } from "../../../../services/langevals/ts-integration/evaluators.generated.js";
3
3
 
4
4
  /**
5
5
  * Formats evaluator schema information for the discover_schema tool.
@@ -20,6 +20,60 @@ export function formatScenarioSchema(): string {
20
20
  lines.push(
21
21
  '- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")',
22
22
  );
23
+ lines.push(
24
+ "- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite.",
25
+ );
26
+ lines.push(
27
+ '- **fields** (object): The scenario\'s value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.',
28
+ );
29
+
30
+ lines.push("\n## Test Suite Fields\n");
31
+ lines.push(
32
+ "A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:",
33
+ );
34
+ lines.push("```json");
35
+ lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
36
+ lines.push("```");
37
+ lines.push(
38
+ "- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
39
+ );
40
+ lines.push("- **type**: text, number or boolean.");
41
+ lines.push(
42
+ "- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed.",
43
+ );
44
+
45
+ lines.push("\n## Evaluator Attachments\n");
46
+ lines.push(
47
+ "A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates.",
48
+ );
49
+ lines.push("```json");
50
+ lines.push('{ "evaluators": [{');
51
+ lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
52
+ lines.push(' "required": true,');
53
+ lines.push(' "mappings": {');
54
+ lines.push(' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
55
+ lines.push(' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
56
+ lines.push(" }");
57
+ lines.push("}] }");
58
+ lines.push("```");
59
+ lines.push(
60
+ "Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:",
61
+ );
62
+ lines.push(
63
+ "- **conversation**: `[\"first_user_message\"]`, `[\"last_agent_message\"]`, `[\"transcript\"]` (role: content lines) or `[\"messages\"]` (JSON)",
64
+ );
65
+ lines.push(
66
+ "- **scenario**: `[\"situation\"]`, `[\"criteria\"]` or `[\"fields\", \"<identifier>\"]` (a field the suite declares; not on a run plan)",
67
+ );
68
+ lines.push(
69
+ "- **trace**: `[\"contexts\"]` (the retrieved contexts of the run's traces) or `[\"tool_calls\", \"<toolName>\", \"input\" | \"output\"]` (the last matching tool call)",
70
+ );
71
+ lines.push(
72
+ '- a literal: `{ "type": "value", "value": "..." }`',
73
+ );
74
+ lines.push(
75
+ "A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`.",
76
+ );
23
77
 
24
78
  lines.push("\n## Writing a Good Situation\n");
25
79
  lines.push(
@@ -0,0 +1,68 @@
1
+ import type {
2
+ RunPlanRunResult,
3
+ RunPlanScope,
4
+ } from "../langwatch-api-run-plans.js";
5
+
6
+ /**
7
+ * The digest every run of a plan returns, whichever tool started it.
8
+ *
9
+ * It says which plan ran and whether the run created it or joined one that
10
+ * already carried the name, because that is the difference between a new plan
11
+ * and a plan whose configuration this run just replaced.
12
+ */
13
+ export function formatRunPlanRun(result: RunPlanRunResult): string {
14
+ const lines: string[] = [];
15
+
16
+ lines.push(
17
+ result.created
18
+ ? `Run plan "${result.planName}" created and started.`
19
+ : `Run plan "${result.planName}" started with the configuration of this run.`,
20
+ );
21
+ lines.push("");
22
+ lines.push(`**Plan**: ${result.planName} (${result.runPlanId})`);
23
+ lines.push(`**Batch Run ID**: ${result.batchRunId}`);
24
+ lines.push(`**Set ID**: ${result.setId}`);
25
+ lines.push(`**Jobs**: ${result.jobCount}`);
26
+
27
+ if (result.skippedArchived.scenarios.length > 0) {
28
+ lines.push(
29
+ `**Skipped archived scenarios**: ${result.skippedArchived.scenarios.join(", ")}`,
30
+ );
31
+ }
32
+ if (result.skippedArchived.targets.length > 0) {
33
+ lines.push(
34
+ `**Skipped archived targets**: ${result.skippedArchived.targets.join(", ")}`,
35
+ );
36
+ }
37
+
38
+ lines.push(`**View**: ${result.platformUrl}`);
39
+ lines.push("");
40
+ lines.push(
41
+ "> Use `platform_list_simulation_runs` with the batch run ID to read the results.",
42
+ );
43
+
44
+ return lines.join("\n");
45
+ }
46
+
47
+ /**
48
+ * One line saying what a plan covers. A plan stored before scopes existed
49
+ * carries none, and runs the scenario list it already held.
50
+ */
51
+ export function describeRunPlanScope(
52
+ scope: RunPlanScope | null,
53
+ scenarioIds: string[],
54
+ ): string {
55
+ if (scope === null) {
56
+ return `hand-picked scenarios (${scenarioIds.length})`;
57
+ }
58
+ switch (scope.mode) {
59
+ case "all":
60
+ return "every scenario in the project";
61
+ case "test_suites":
62
+ return `test suites: ${scope.testSuiteIds.join(", ")}`;
63
+ case "labels":
64
+ return `labels: ${scope.labels.join(", ")}`;
65
+ case "scenarios":
66
+ return `hand-picked scenarios (${scenarioIds.length})`;
67
+ }
68
+ }
@@ -0,0 +1,14 @@
1
+ import type { ScenarioFieldValues } from "../schemas/suite-fields.js";
2
+
3
+ /** The digest lines for the values a scenario carries per suite field. */
4
+ export function formatScenarioFields(
5
+ fields: ScenarioFieldValues | undefined,
6
+ ): string[] {
7
+ if (!fields || Object.keys(fields).length === 0) return [];
8
+ return [
9
+ "\n## Fields",
10
+ ...Object.entries(fields).map(
11
+ ([identifier, value]) => `- ${identifier}: ${JSON.stringify(value)}`,
12
+ ),
13
+ ];
14
+ }