@langwatch/mcp-server 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (430) hide show
  1. package/.gitignore +7 -0
  2. package/.npmignore +41 -0
  3. package/CHANGELOG.md +280 -200
  4. package/CONTRIBUTING.md +10 -0
  5. package/README.md +82 -52
  6. package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
  7. package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
  8. package/dist/archive-project-KWFM3I5Z.js +19 -0
  9. package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js} +4 -4
  10. package/dist/archive-run-plan-65QN3XRX.js +16 -0
  11. package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
  12. package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js} +4 -4
  13. package/dist/archive-scenario-FXEYQNEH.js +19 -0
  14. package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
  15. package/dist/archive-test-suite-NKZDZY3D.js +16 -0
  16. package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
  17. package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
  18. package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js} +7 -10
  19. package/dist/chunk-3YLN3ATJ.js +241 -0
  20. package/dist/chunk-5FNCQRCX.js +91 -0
  21. package/dist/chunk-6YZUOD34.js +55 -0
  22. package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
  23. package/dist/chunk-7VNSRCXO.js +129 -0
  24. package/dist/{chunk-7RHG56OH.js → chunk-B4J3PJLX.js} +11 -10
  25. package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
  26. package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
  27. package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
  28. package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
  29. package/dist/{chunk-TOXJQDIZ.js → chunk-HGELNIVY.js} +135 -51
  30. package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
  31. package/dist/{chunk-J42FWRJ2.js → chunk-I7ATIZED.js} +7 -7
  32. package/dist/{chunk-AQJO2EI5.js → chunk-ILZDO25L.js} +7 -10
  33. package/dist/{chunk-4QC7SA2L.js → chunk-J7PQKBLD.js} +884 -559
  34. package/dist/{chunk-YX3A3CFM.js → chunk-JRJPH75H.js} +7 -13
  35. package/dist/chunk-K2I7Z3LZ.js +28 -0
  36. package/dist/{chunk-5CCZFUXS.js → chunk-NKMTIHIJ.js} +6 -9
  37. package/dist/chunk-O7EMWEVJ.js +44 -0
  38. package/dist/chunk-PTDR56Z3.js +37 -0
  39. package/dist/chunk-RABYBFX5.js +43 -0
  40. package/dist/chunk-SUPUUC4W.js +64 -0
  41. package/dist/{chunk-SFB4MU6H.js → chunk-SXFC3FNH.js} +4 -4
  42. package/dist/chunk-U24EACC4.js +67 -0
  43. package/dist/chunk-UNJM3BIL.js +21 -0
  44. package/dist/{chunk-ZWIL4EFL.js → chunk-UTJCWXPZ.js} +4 -4
  45. package/dist/chunk-UWLE5T2F.js +312 -0
  46. package/dist/chunk-VMWJD732.js +57 -0
  47. package/dist/chunk-XUIJYNRJ.js +15 -0
  48. package/dist/chunk-Y34DHR5K.js +67 -0
  49. package/dist/chunk-YCARO42Z.js +516 -0
  50. package/dist/config-EAKW6F26.js +16 -0
  51. package/dist/config.cjs +109 -0
  52. package/dist/config.js +5 -4
  53. package/dist/{create-agent-7XHO6UIS.js → create-agent-AYZ3LB3V.js} +4 -4
  54. package/dist/create-agent-S5ZBCEFT.js +23 -0
  55. package/dist/create-annotation-LHNCLHYP.js +29 -0
  56. package/dist/{create-annotation-YQXRQC6A.js → create-annotation-VUSXULQK.js} +10 -5
  57. package/dist/{create-api-key-YUS36KBU.js → create-api-key-NNYKARMG.js} +5 -7
  58. package/dist/create-api-key-T4KSNPKB.js +24 -0
  59. package/dist/create-dashboard-DHVFMEU4.js +18 -0
  60. package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-WTKKS5Z6.js} +4 -4
  61. package/dist/{create-dataset-E6NMSVX7.js → create-dataset-2MC4SD4O.js} +6 -6
  62. package/dist/create-dataset-LUP5PVDM.js +22 -0
  63. package/dist/create-dataset-records-NSKBZP3G.js +20 -0
  64. package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-W4GLN2FV.js} +5 -7
  65. package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-643G6LXL.js} +4 -4
  66. package/dist/create-evaluator-HMTYO5SS.js +27 -0
  67. package/dist/create-mcp-server.cjs +9513 -0
  68. package/dist/create-mcp-server.js +2647 -5
  69. package/dist/create-project-6N7PIYRY.js +44 -0
  70. package/dist/{create-project-CS2CB57Y.js → create-project-KAIC46UY.js} +4 -4
  71. package/dist/{create-prompt-F5RMX3L3.js → create-prompt-E67LXIYT.js} +5 -7
  72. package/dist/create-prompt-PKJ7EDRR.js +37 -0
  73. package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
  74. package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-OPQUEXWB.js} +3 -3
  75. package/dist/create-scenario-5PJBMAXO.js +33 -0
  76. package/dist/{create-scenario-JDR2VB2H.js → create-scenario-UEM77OE7.js} +11 -4
  77. package/dist/create-test-suite-4IPXVSFQ.js +37 -0
  78. package/dist/create-test-suite-SZEPVXSE.js +36 -0
  79. package/dist/create-trigger-5HZSCRAN.js +49 -0
  80. package/dist/create-trigger-PRELMT6J.js +48 -0
  81. package/dist/delete-agent-JUIZWAXG.js +18 -0
  82. package/dist/{delete-agent-O3EEH4S5.js → delete-agent-QMEK3XHK.js} +4 -4
  83. package/dist/delete-annotation-HY3O2OGR.js +17 -0
  84. package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-VKIQWUYW.js} +4 -4
  85. package/dist/delete-dashboard-76N66XXL.js +18 -0
  86. package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-JVIVHNJM.js} +4 -4
  87. package/dist/delete-dataset-HQCR66MG.js +18 -0
  88. package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-WKRUW4FP.js} +5 -7
  89. package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
  90. package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
  91. package/dist/delete-evaluator-4SNSCLEE.js +16 -0
  92. package/dist/delete-evaluator-QP3QHTLN.js +15 -0
  93. package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
  94. package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
  95. package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-5J46T73T.js} +4 -4
  96. package/dist/delete-workflow-JSWABL5F.js +17 -0
  97. package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
  98. package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
  99. package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
  100. package/dist/discover-scenario-schema-OOFIQKOG.js +139 -0
  101. package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
  102. package/dist/discover-schema-A33SLMXW.js +442 -0
  103. package/dist/get-agent-LJQX7OF4.js +69 -0
  104. package/dist/get-agent-OTIYUFZA.js +70 -0
  105. package/dist/get-analytics-DCS72353.js +80 -0
  106. package/dist/get-analytics-IQZKJHLW.js +79 -0
  107. package/dist/get-dashboard-5NYJEIX2.js +29 -0
  108. package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-QMTYDQAZ.js} +4 -4
  109. package/dist/get-dataset-ALSP7RTX.js +64 -0
  110. package/dist/get-dataset-X4B5JDG2.js +63 -0
  111. package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-ISMC4LLI.js} +4 -4
  112. package/dist/get-evaluator-XEUZGZCH.js +47 -0
  113. package/dist/{get-project-RAK7URQJ.js → get-project-5VKROSDH.js} +4 -4
  114. package/dist/get-project-RM6FRCBK.js +26 -0
  115. package/dist/get-prompt-OWXJZHQQ.js +95 -0
  116. package/dist/get-prompt-W52DY36E.js +94 -0
  117. package/dist/get-run-plan-BE3M2AUX.js +59 -0
  118. package/dist/get-run-plan-RUXQ6PAX.js +58 -0
  119. package/dist/get-scenario-RNM7BAX2.js +37 -0
  120. package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-YM4ODWW6.js} +8 -4
  121. package/dist/get-simulation-run-6GIATQE5.js +69 -0
  122. package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-LCFSMPGQ.js} +33 -20
  123. package/dist/get-test-suite-GC2O7YH6.js +49 -0
  124. package/dist/get-test-suite-P5LJ6VLF.js +48 -0
  125. package/dist/get-trace-KBRVV4NN.js +45 -0
  126. package/dist/{get-trace-K7U234Q4.js → get-trace-SEF3FMR5.js} +11 -13
  127. package/dist/get-workflow-7PVED7JE.js +25 -0
  128. package/dist/{get-workflow-L5L2PSK3.js → get-workflow-YXULW4GC.js} +4 -4
  129. package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
  130. package/dist/{http-server-YXHFPPGA.js → http-server-IR4QOS43.js} +4071 -342
  131. package/dist/index.js +60221 -38077
  132. package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
  133. package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
  134. package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
  135. package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
  136. package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
  137. package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
  138. package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
  139. package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
  140. package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
  141. package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
  142. package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
  143. package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
  144. package/dist/list-agents-HVYWEQMI.js +43 -0
  145. package/dist/list-agents-RBAOCF53.js +42 -0
  146. package/dist/list-annotations-HJH4NBLD.js +38 -0
  147. package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-J4UAHSUN.js} +10 -5
  148. package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-7UTJRZDG.js} +10 -5
  149. package/dist/list-api-keys-A2PMV435.js +42 -0
  150. package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
  151. package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
  152. package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-ROSYFLDQ.js} +4 -4
  153. package/dist/list-dashboards-RQ635ZU7.js +32 -0
  154. package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-M5TKA6LK.js} +14 -10
  155. package/dist/list-dataset-records-WFM7F33C.js +43 -0
  156. package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-B5X4P7BQ.js} +9 -13
  157. package/dist/list-datasets-THUKQHPX.js +56 -0
  158. package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-MYH2OVCI.js} +5 -7
  159. package/dist/list-evaluators-NLVWF6K7.js +32 -0
  160. package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-3JUGP4YI.js} +4 -4
  161. package/dist/list-model-providers-H3XKORFO.js +35 -0
  162. package/dist/{list-projects-K7QAIX2I.js → list-projects-6EJ5YYEW.js} +5 -7
  163. package/dist/list-projects-YNNSRMLC.js +40 -0
  164. package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-NVFQO6WX.js} +3 -3
  165. package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
  166. package/dist/{list-prompts-SC2HAUFG.js → list-prompts-HSIMMHTF.js} +4 -6
  167. package/dist/list-prompts-IRI6AIXO.js +30 -0
  168. package/dist/list-run-plans-PX2XI6EK.js +46 -0
  169. package/dist/list-run-plans-QALLCN2K.js +45 -0
  170. package/dist/list-scenarios-QZRPVHS4.js +46 -0
  171. package/dist/list-scenarios-SSJQEMJP.js +45 -0
  172. package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
  173. package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-OEADLLQR.js} +19 -9
  174. package/dist/list-test-suites-ANAQBLDG.js +34 -0
  175. package/dist/list-test-suites-N4WVLBKK.js +35 -0
  176. package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
  177. package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
  178. package/dist/list-triggers-SQNIG35Q.js +45 -0
  179. package/dist/list-triggers-V4Y356AK.js +46 -0
  180. package/dist/{list-workflows-QSJWURCK.js → list-workflows-IMSX7SXU.js} +5 -7
  181. package/dist/list-workflows-ZHTX5LCL.js +33 -0
  182. package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
  183. package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
  184. package/dist/rename-test-suite-4J7UX5S2.js +21 -0
  185. package/dist/rename-test-suite-NNVLMLKY.js +20 -0
  186. package/dist/report-issue-AS34ZMJF.js +898 -0
  187. package/dist/report-issue-DCZFLGDO.js +899 -0
  188. package/dist/rerun-run-plan-TG2V4WM4.js +22 -0
  189. package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
  190. package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
  191. package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-SFMEFWJC.js} +4 -4
  192. package/dist/run-agent-W4VJP72Z.js +75 -0
  193. package/dist/run-agent-YNV6ZHW4.js +74 -0
  194. package/dist/run-plan-6C77GWG4.js +38 -0
  195. package/dist/run-plan-PAARQSI4.js +39 -0
  196. package/dist/run-query-KJDH544C.js +47 -0
  197. package/dist/run-query-T4QW2Z5O.js +46 -0
  198. package/dist/run-test-suite-SQJELEZX.js +26 -0
  199. package/dist/run-test-suite-STB6NUJM.js +25 -0
  200. package/dist/{run-workflow-QPXC627O.js → run-workflow-D6PQNBI6.js} +4 -4
  201. package/dist/{run-agent-6SM4XNA7.js → run-workflow-SP32SO5M.js} +10 -11
  202. package/dist/search-traces-HCBGOG6R.js +170 -0
  203. package/dist/search-traces-HSTQ6WTQ.js +171 -0
  204. package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-GF7NXUJA.js} +4 -4
  205. package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
  206. package/dist/{update-agent-SMAEHLXH.js → update-agent-DWG43T2P.js} +4 -4
  207. package/dist/update-agent-MXNOBS4H.js +20 -0
  208. package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
  209. package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
  210. package/dist/update-dataset-DQH3OHOU.js +22 -0
  211. package/dist/{update-dataset-BDUK7TUO.js → update-dataset-SQZ6T4WX.js} +6 -6
  212. package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-3AT4IPBC.js} +4 -4
  213. package/dist/update-dataset-record-7PLDGTMR.js +19 -0
  214. package/dist/update-evaluator-4LFTKFPT.js +24 -0
  215. package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-IGTX6PX4.js} +4 -4
  216. package/dist/{update-project-LKVYXENH.js → update-project-PLJIOZS6.js} +4 -4
  217. package/dist/update-project-QVHSJRUP.js +24 -0
  218. package/dist/update-prompt-GDKRYJI3.js +140 -0
  219. package/dist/update-prompt-MY263DYF.js +141 -0
  220. package/dist/update-scenario-7KA3JDXA.js +34 -0
  221. package/dist/{update-scenario-NEXCAQI7.js → update-scenario-MOHRQKAQ.js} +11 -4
  222. package/package.json +39 -44
  223. package/pyproject.toml +8 -8
  224. package/src/config.d.ts +4 -0
  225. package/src/config.ts +20 -29
  226. package/src/create-mcp-server.ts +1021 -596
  227. package/src/documentation-fetch.ts +21 -20
  228. package/src/http-security.ts +166 -135
  229. package/src/http-server.ts +189 -224
  230. package/src/index.ts +14 -16
  231. package/src/langwatch-api-agents.ts +202 -21
  232. package/src/langwatch-api-annotations.ts +11 -12
  233. package/src/langwatch-api-api-keys.ts +8 -9
  234. package/src/langwatch-api-boards.ts +46 -0
  235. package/src/langwatch-api-dashboard-widgets.ts +112 -0
  236. package/src/langwatch-api-dashboards.ts +13 -10
  237. package/src/langwatch-api-datasets.ts +98 -32
  238. package/src/langwatch-api-evaluators.ts +13 -19
  239. package/src/langwatch-api-model-providers.ts +4 -4
  240. package/src/langwatch-api-monitors.ts +14 -18
  241. package/src/langwatch-api-projects.ts +27 -9
  242. package/src/langwatch-api-query.ts +114 -0
  243. package/src/langwatch-api-run-plans.ts +140 -0
  244. package/src/langwatch-api-scenarios.ts +17 -10
  245. package/src/langwatch-api-secrets.ts +31 -24
  246. package/src/langwatch-api-simulation-runs.ts +10 -7
  247. package/src/langwatch-api-test-suites.ts +116 -0
  248. package/src/langwatch-api-triggers.ts +96 -34
  249. package/src/langwatch-api-workflows.ts +9 -9
  250. package/src/langwatch-api.ts +143 -109
  251. package/src/public-http-request.ts +21 -12
  252. package/src/schemas/analytics-metrics.ts +3 -15
  253. package/src/schemas/create-dataset.ts +3 -4
  254. package/src/schemas/run-plan.ts +76 -0
  255. package/src/schemas/suite-fields.ts +132 -0
  256. package/src/schemas/triggers.ts +365 -0
  257. package/src/tools/add-dashboard-widget.ts +63 -0
  258. package/src/tools/archive-project.ts +2 -4
  259. package/src/tools/archive-run-plan.ts +10 -0
  260. package/src/tools/archive-scenario.ts +2 -4
  261. package/src/tools/archive-test-suite.ts +10 -0
  262. package/src/tools/assign-prompt-tag.ts +6 -2
  263. package/src/tools/create-agent.ts +1 -1
  264. package/src/tools/create-annotation.ts +7 -2
  265. package/src/tools/create-api-key.ts +4 -6
  266. package/src/tools/create-dashboard.ts +1 -1
  267. package/src/tools/create-dataset-records.ts +4 -8
  268. package/src/tools/create-dataset.ts +5 -7
  269. package/src/tools/create-evaluator.ts +3 -5
  270. package/src/tools/create-project.ts +2 -2
  271. package/src/tools/create-prompt-tag.ts +2 -4
  272. package/src/tools/create-prompt.ts +9 -18
  273. package/src/tools/create-scenario.ts +11 -5
  274. package/src/tools/create-test-suite.ts +38 -0
  275. package/src/tools/create-trigger.ts +55 -0
  276. package/src/tools/dashboard-exists.ts +25 -0
  277. package/src/tools/dashboard-widget-queries.ts +32 -0
  278. package/src/tools/delete-agent.ts +1 -1
  279. package/src/tools/delete-annotation.ts +1 -1
  280. package/src/tools/delete-dashboard.ts +1 -1
  281. package/src/tools/delete-dataset-records.ts +4 -8
  282. package/src/tools/delete-dataset.ts +3 -7
  283. package/src/tools/delete-evaluator.ts +4 -6
  284. package/src/tools/delete-prompt-tag.ts +2 -4
  285. package/src/tools/delete-workflow.ts +1 -1
  286. package/src/tools/discover-evaluator-schema.ts +34 -25
  287. package/src/tools/discover-scenario-schema.ts +86 -12
  288. package/src/tools/discover-schema.ts +149 -27
  289. package/src/tools/experiment-run-status.ts +2 -8
  290. package/src/tools/format-dataset-mutation.ts +5 -10
  291. package/src/tools/format-run-plan.ts +54 -0
  292. package/src/tools/format-scenario.ts +12 -0
  293. package/src/tools/format-suite-details.ts +90 -0
  294. package/src/tools/get-agent.ts +54 -3
  295. package/src/tools/get-analytics.ts +52 -46
  296. package/src/tools/get-dashboard.ts +1 -1
  297. package/src/tools/get-dataset.ts +34 -16
  298. package/src/tools/get-evaluator.ts +4 -11
  299. package/src/tools/get-experiment-results.ts +219 -186
  300. package/src/tools/get-project.ts +2 -4
  301. package/src/tools/get-prompt.ts +100 -37
  302. package/src/tools/get-run-plan.ts +60 -0
  303. package/src/tools/get-scenario.ts +6 -5
  304. package/src/tools/get-simulation-run.ts +32 -20
  305. package/src/tools/get-test-suite.ts +46 -0
  306. package/src/tools/get-trace.ts +16 -17
  307. package/src/tools/get-workflow.ts +1 -1
  308. package/src/tools/list-agents.ts +20 -3
  309. package/src/tools/list-annotations.ts +8 -5
  310. package/src/tools/list-api-keys.ts +7 -2
  311. package/src/tools/list-dashboard-boards.ts +28 -0
  312. package/src/tools/list-dashboards.ts +1 -1
  313. package/src/tools/list-dataset-records.ts +8 -6
  314. package/src/tools/list-datasets.ts +12 -23
  315. package/src/tools/list-evaluators.ts +4 -8
  316. package/src/tools/list-experiment-runs.ts +5 -9
  317. package/src/tools/list-experiments.ts +5 -11
  318. package/src/tools/list-model-providers.ts +3 -5
  319. package/src/tools/list-projects.ts +2 -4
  320. package/src/tools/list-prompt-tags.ts +3 -3
  321. package/src/tools/list-prompts.ts +4 -8
  322. package/src/tools/list-run-plans.ts +47 -0
  323. package/src/tools/list-scenarios.ts +26 -17
  324. package/src/tools/list-simulation-runs.ts +13 -5
  325. package/src/tools/list-test-suites.ts +35 -0
  326. package/src/tools/list-trigger-fires.ts +17 -0
  327. package/src/tools/list-triggers.ts +39 -0
  328. package/src/tools/list-workflows.ts +2 -4
  329. package/src/tools/rename-prompt-tag.ts +1 -1
  330. package/src/tools/rename-test-suite.ts +15 -0
  331. package/src/tools/report-issue.ts +5 -9
  332. package/src/tools/rerun-run-plan.ts +18 -0
  333. package/src/tools/revoke-api-key.ts +2 -4
  334. package/src/tools/run-agent.ts +65 -14
  335. package/src/tools/run-experiment.ts +36 -37
  336. package/src/tools/run-plan.ts +46 -0
  337. package/src/tools/run-query.ts +47 -0
  338. package/src/tools/run-test-suite.ts +27 -0
  339. package/src/tools/run-workflow.ts +2 -5
  340. package/src/tools/search-traces.ts +161 -47
  341. package/src/tools/set-model-provider.ts +4 -6
  342. package/src/tools/test-agent.ts +18 -0
  343. package/src/tools/update-agent.ts +1 -1
  344. package/src/tools/update-dashboard-widget.ts +97 -0
  345. package/src/tools/update-dataset-record.ts +1 -1
  346. package/src/tools/update-dataset.ts +5 -7
  347. package/src/tools/update-evaluator.ts +3 -5
  348. package/src/tools/update-project.ts +1 -1
  349. package/src/tools/update-prompt.ts +185 -15
  350. package/src/tools/update-scenario.ts +11 -5
  351. package/src/tools/update-test-suite.ts +40 -0
  352. package/src/utils/date-parsing.ts +1 -1
  353. package/src/utils/escape-markdown.ts +3 -8
  354. package/src/utils/format-evaluations.ts +6 -6
  355. package/src/utils/markdown-table.ts +58 -0
  356. package/src/utils/trace-id-shape.ts +20 -0
  357. package/tsconfig.app.json +24 -0
  358. package/tsconfig.json +19 -12
  359. package/uv.lock +416 -296
  360. package/vitest.integration.config.js +10 -0
  361. package/vitest.scenario.config.js +9 -0
  362. package/.eslintrc.cjs +0 -36
  363. package/dist/archive-suite-4T6BSRS7.js +0 -16
  364. package/dist/chunk-77BNWQ3U.js +0 -86
  365. package/dist/chunk-A2LEVYQ5.js +0 -217
  366. package/dist/chunk-TJCXFNDT.js +0 -24
  367. package/dist/chunk-YZLYHLLI.js +0 -48
  368. package/dist/config.d.ts +0 -17
  369. package/dist/create-mcp-server.d.ts +0 -10
  370. package/dist/create-suite-LZS6QIAF.js +0 -39
  371. package/dist/delete-evaluator-33SP2JFX.js +0 -16
  372. package/dist/discover-evaluator-schema-XTAD4X7D.js +0 -2191
  373. package/dist/discover-scenario-schema-FE64QK2D.js +0 -67
  374. package/dist/get-agent-Y7MWZ65F.js +0 -29
  375. package/dist/get-analytics-TETCSOK6.js +0 -73
  376. package/dist/get-dataset-RCMU5VAH.js +0 -49
  377. package/dist/get-prompt-OY5B7TRI.js +0 -46
  378. package/dist/get-suite-J7JVWO6O.js +0 -45
  379. package/dist/index.d.ts +0 -2
  380. package/dist/langwatch-api-secrets-BUBSNCNO.js +0 -41
  381. package/dist/langwatch-api-triggers-EDFUCJN6.js +0 -31
  382. package/dist/list-agents-KFHKGPXV.js +0 -33
  383. package/dist/list-scenarios-F2WYNRIE.js +0 -41
  384. package/dist/list-suites-CRY4VWW6.js +0 -43
  385. package/dist/report-issue-XYVAPAYJ.js +0 -412
  386. package/dist/run-suite-NKZCJ2IN.js +0 -30
  387. package/dist/search-traces-A6HFKPLJ.js +0 -79
  388. package/dist/update-prompt-ODCRQRN6.js +0 -25
  389. package/dist/update-suite-SNO3GS7T.js +0 -40
  390. package/src/__tests__/all-tools.integration.test.ts +0 -1626
  391. package/src/__tests__/config.unit.test.ts +0 -202
  392. package/src/__tests__/datasets.integration.test.ts +0 -620
  393. package/src/__tests__/datasets.unit.test.ts +0 -244
  394. package/src/__tests__/date-parsing.unit.test.ts +0 -78
  395. package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
  396. package/src/__tests__/discover-schema.unit.test.ts +0 -118
  397. package/src/__tests__/docker.integration.test.ts +0 -318
  398. package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
  399. package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
  400. package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
  401. package/src/__tests__/experiment-status.unit.test.ts +0 -194
  402. package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
  403. package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
  404. package/src/__tests__/http-security-origin.unit.test.ts +0 -97
  405. package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
  406. package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
  407. package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
  408. package/src/__tests__/http-server-origin.integration.test.ts +0 -169
  409. package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
  410. package/src/__tests__/http-transport.integration.test.ts +0 -395
  411. package/src/__tests__/integration.integration.test.ts +0 -288
  412. package/src/__tests__/langwatch-api.unit.test.ts +0 -661
  413. package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
  414. package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
  415. package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
  416. package/src/__tests__/report-issue.integration.test.ts +0 -147
  417. package/src/__tests__/scenario-tools.integration.test.ts +0 -286
  418. package/src/__tests__/scenario-tools.unit.test.ts +0 -188
  419. package/src/__tests__/schemas.unit.test.ts +0 -85
  420. package/src/__tests__/support/http-server-harness.ts +0 -138
  421. package/src/__tests__/tools.unit.test.ts +0 -924
  422. package/src/langwatch-api-suites.ts +0 -118
  423. package/src/schemas/filter-fields.ts +0 -119
  424. package/src/schemas/index.ts +0 -4
  425. package/src/tools/archive-suite.ts +0 -12
  426. package/src/tools/create-suite.ts +0 -42
  427. package/src/tools/get-suite.ts +0 -47
  428. package/src/tools/list-suites.ts +0 -43
  429. package/src/tools/run-suite.ts +0 -28
  430. package/src/tools/update-suite.ts +0 -44
@@ -20,16 +20,76 @@ export function formatScenarioSchema(): string {
20
20
  lines.push(
21
21
  '- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")',
22
22
  );
23
+ lines.push(
24
+ "- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite.",
25
+ );
26
+ lines.push(
27
+ '- **fields** (object): The scenario\'s value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.',
28
+ );
29
+
30
+ lines.push("\n## Test Suite Fields\n");
31
+ lines.push(
32
+ "A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:",
33
+ );
34
+ lines.push("```json");
35
+ lines.push(
36
+ '{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }',
37
+ );
38
+ lines.push("```");
39
+ lines.push(
40
+ "- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
41
+ );
42
+ lines.push("- **type**: text, number or boolean.");
43
+ lines.push(
44
+ "- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed.",
45
+ );
46
+
47
+ lines.push("\n## Evaluator Attachments\n");
48
+ lines.push(
49
+ "A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates.",
50
+ );
51
+ lines.push("```json");
52
+ lines.push('{ "evaluators": [{');
53
+ lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
54
+ lines.push(' "required": true,');
55
+ lines.push(' "mappings": {');
56
+ lines.push(
57
+ ' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },',
58
+ );
59
+ lines.push(
60
+ ' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }',
61
+ );
62
+ lines.push(" }");
63
+ lines.push("}] }");
64
+ lines.push("```");
65
+ lines.push(
66
+ "Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:",
67
+ );
68
+ lines.push(
69
+ '- **conversation**: `["first_user_message"]`, `["last_agent_message"]`, `["transcript"]` (role: content lines) or `["messages"]` (JSON)',
70
+ );
71
+ lines.push(
72
+ '- **scenario**: `["situation"]`, `["criteria"]` or `["fields", "<identifier>"]` (a field the suite declares; not on a run plan)',
73
+ );
74
+ lines.push(
75
+ '- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)',
76
+ );
77
+ lines.push('- a literal: `{ "type": "value", "value": "..." }`');
78
+ lines.push(
79
+ "A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`.",
80
+ );
23
81
 
24
82
  lines.push("\n## Writing a Good Situation\n");
83
+ lines.push("The situation drives the user simulator. Include these elements:");
25
84
  lines.push(
26
- "The situation drives the user simulator. Include these elements:",
85
+ "- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)",
27
86
  );
28
- lines.push("- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)");
29
87
  lines.push("- **Emotional state**: How are they feeling? (e.g., frustrated, anxious, impatient)");
30
88
  lines.push("- **Background/Context**: What happened before this conversation?");
31
89
  lines.push("- **Intent**: What do they want to accomplish?");
32
- lines.push("- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)");
90
+ lines.push(
91
+ "- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)",
92
+ );
33
93
  lines.push("\nExample:");
34
94
  lines.push("```");
35
95
  lines.push("User is a small business owner stressed about tax deadline.");
@@ -40,17 +100,31 @@ export function formatScenarioSchema(): string {
40
100
  lines.push("```");
41
101
 
42
102
  lines.push("\n## Writing Good Criteria\n");
43
- lines.push("Criteria are what the judge uses to pass or fail the agent. Each criterion should be:");
44
- lines.push("- **Specific and testable** — not vague like \"responds helpfully\"");
45
- lines.push("- **Behavioral** — describes what the agent should *do*, not how it works internally");
103
+ lines.push(
104
+ "Criteria are what the judge uses to pass or fail the agent. Each criterion should be:",
105
+ );
106
+ lines.push('- **Specific and testable** — not vague like "responds helpfully"');
107
+ lines.push(
108
+ "- **Behavioral** — describes what the agent should *do*, not how it works internally",
109
+ );
46
110
  lines.push("- **Independent** — each criterion checks one thing");
47
111
  lines.push("\nGood criteria patterns:");
48
- lines.push("- **Information gathering**: \"Agent asks for the user's account number before proceeding\"");
49
- lines.push("- **Safety/guardrails**: \"Agent does not reveal internal system details or error stack traces\"");
50
- lines.push("- **Clarification**: \"Agent asks clarifying questions before taking irreversible action\"");
51
- lines.push("- **Tone**: \"Agent maintains a professional and empathetic tone throughout\"");
52
- lines.push("- **Completeness**: \"Agent confirms the user understands the solution before ending\"");
53
- lines.push("- **Domain-specific**: \"Agent recommends releasing a wild frog rather than keeping it as a pet\"");
112
+ lines.push(
113
+ '- **Information gathering**: "Agent asks for the user\'s account number before proceeding"',
114
+ );
115
+ lines.push(
116
+ '- **Safety/guardrails**: "Agent does not reveal internal system details or error stack traces"',
117
+ );
118
+ lines.push(
119
+ '- **Clarification**: "Agent asks clarifying questions before taking irreversible action"',
120
+ );
121
+ lines.push('- **Tone**: "Agent maintains a professional and empathetic tone throughout"');
122
+ lines.push(
123
+ '- **Completeness**: "Agent confirms the user understands the solution before ending"',
124
+ );
125
+ lines.push(
126
+ '- **Domain-specific**: "Agent recommends releasing a wild frog rather than keeping it as a pet"',
127
+ );
54
128
  lines.push("\nAvoid vague criteria like:");
55
129
  lines.push('- "Responds correctly" — correct how?');
56
130
  lines.push('- "Is helpful" — helpful in what way?');
@@ -1,25 +1,39 @@
1
- import { filterFields } from "../schemas/filter-fields.js";
2
- import { analyticsMetrics } from "../schemas/analytics-metrics.js";
3
- import { analyticsGroups } from "../schemas/analytics-groups.js";
1
+ import { getQueryReference, type QueryReferenceResponse } from "../langwatch-api-query.ts";
2
+ import { analyticsGroups } from "../schemas/analytics-groups.ts";
3
+ import { analyticsMetrics } from "../schemas/analytics-metrics.ts";
4
+ import { markdownTable } from "../utils/markdown-table.ts";
4
5
 
5
- export type Category =
6
- | "filters"
7
- | "metrics"
8
- | "aggregations"
9
- | "groups"
10
- | "all";
6
+ export type Category = "filters" | "lwql" | "metrics" | "aggregations" | "groups" | "all";
11
7
 
12
8
  /**
13
- * Formats the LangWatch analytics schema into human-readable markdown.
14
- *
15
- * Returns documentation for the requested category of schema elements
16
- * (filter fields, metrics, aggregation types, or group-by options).
9
+ * Categories the platform answers, not this package — which is why
10
+ * `formatSchema` is async: they are read from `GET /api/v1/query/reference`
11
+ * and need the API key.
17
12
  */
18
- export function formatSchema(category: Category): string {
13
+ const PLATFORM_BACKED: ReadonlySet<Category> = new Set(["filters", "lwql", "all"]);
14
+
15
+ export function needsQueryReference(category: Category): boolean {
16
+ return PLATFORM_BACKED.has(category);
17
+ }
18
+
19
+ /**
20
+ * Formats the LangWatch query schema into markdown an agent can act on.
21
+ * `reference` is required for the platform-backed categories and ignored by
22
+ * the others, so a caller that fetched it once can reuse it.
23
+ */
24
+ export async function formatSchema(
25
+ category: Category,
26
+ reference?: QueryReferenceResponse,
27
+ ): Promise<string> {
19
28
  const sections: string[] = [];
29
+ const resolved =
30
+ needsQueryReference(category) && !reference ? await getQueryReference() : reference;
20
31
 
21
32
  if (category === "filters" || category === "all") {
22
- sections.push(formatFilters());
33
+ sections.push(formatFilters(resolved));
34
+ }
35
+ if (category === "lwql" || category === "all") {
36
+ sections.push(formatLangWatchQL(resolved));
23
37
  }
24
38
  if (category === "metrics" || category === "all") {
25
39
  sections.push(formatMetrics());
@@ -30,30 +44,140 @@ export function formatSchema(category: Category): string {
30
44
  if (category === "groups" || category === "all") {
31
45
  sections.push(formatGroups());
32
46
  }
47
+ if (category === "all" && resolved) {
48
+ sections.push(formatDecisionTable(resolved));
49
+ }
33
50
 
34
51
  return sections.join("\n\n");
35
52
  }
36
53
 
37
- function formatFilters(): string {
38
- const lines = ["## Available Filter Fields", ""];
54
+ /** What the field's values column says: the closed set, or where to read an open one. */
55
+ const knownValuesCell = (field: { knownValues: readonly string[]; facetable: boolean }): string => {
56
+ if (field.knownValues.length > 0) return field.knownValues.join(", ");
57
+ return field.facetable ? "open set, read them from GET /api/v1/traces/facets" : "";
58
+ };
59
+
60
+ function formatFilters(reference?: QueryReferenceResponse): string {
61
+ if (!reference) {
62
+ return "## Trace Filter Fields\n\nUnavailable: the platform could not be reached.";
63
+ }
64
+ const lines = ["## Trace Filter Fields", ""];
65
+ lines.push(
66
+ "Send one of these as the `filter` parameter of `search_traces`, in the language the Trace Explorer's search bar speaks: `status:error AND model:gpt-*`.",
67
+ );
68
+ lines.push(
69
+ "The `filters` parameter is the older per-field map and still works; both are combined when you send both.",
70
+ );
71
+ lines.push("");
72
+ lines.push(
73
+ markdownTable({
74
+ headers: ["Field", "Type", "Group", "Known values"],
75
+ rows: reference.traceFilter.fields.map((field) => ({
76
+ Field: field.name,
77
+ Type: field.valueType,
78
+ Group: field.group ?? "",
79
+ "Known values": knownValuesCell(field),
80
+ })),
81
+ }),
82
+ );
83
+ lines.push("");
84
+ lines.push("### Attribute namespaces", "");
39
85
  lines.push(
40
- "Use these in the `filters` parameter of `search_traces` and `get_analytics`."
86
+ markdownTable({
87
+ headers: ["Prefix", "Matches", "Older spellings"],
88
+ rows: reference.traceFilter.dynamicPrefixes.map((prefix) => ({
89
+ Prefix: `${prefix.prefix}<key>`,
90
+ Matches: prefix.description,
91
+ "Older spellings": prefix.aliases.join(", "),
92
+ })),
93
+ }),
41
94
  );
42
- lines.push('Format: `{ "field_name": ["value1", "value2"] }`');
43
95
  lines.push("");
44
- for (const f of filterFields) {
96
+ lines.push("### Syntax", "");
97
+ lines.push(reference.traceFilter.syntax);
98
+ lines.push("");
99
+ lines.push("### Worked filters", "");
100
+ for (const example of reference.examples.filter(
101
+ (candidate) => candidate.language === "trace-filter",
102
+ )) {
103
+ lines.push(`- \`${example.text}\` — ${example.title}`);
104
+ }
105
+ return lines.join("\n");
106
+ }
107
+
108
+ function formatLangWatchQL(reference?: QueryReferenceResponse): string {
109
+ if (!reference) {
110
+ return "## Analytics SQL\n\nUnavailable: the platform could not be reached.";
111
+ }
112
+ if (!reference.lwql.enabled) {
113
+ return [
114
+ "## Analytics SQL",
115
+ "",
116
+ "Not enabled for this project, so `run_query` will refuse. Use `search_traces` and `get_analytics` instead.",
117
+ ].join("\n");
118
+ }
119
+ const lines = ["## Analytics SQL", ""];
120
+ lines.push(
121
+ "Run one read-only SELECT with `run_query`. The statement is executed as written: nothing rewrites it.",
122
+ );
123
+ lines.push(
124
+ `Ceilings: ${reference.lwql.limits.maxRowsReturned} rows, ${reference.lwql.limits.maxExecutionTimeSeconds} seconds. ${reference.lwql.limits.pagination}`,
125
+ );
126
+ lines.push("");
127
+ for (const view of reference.lwql.schema.views) {
128
+ lines.push(`### ${view.name}`);
129
+ lines.push(`${view.description} One row is: ${view.grain}`);
130
+ lines.push(
131
+ `Filter on \`${view.timeColumn}\` to prune partitions. Join keys: ${view.joinKeys.join(", ") || "none"}.`,
132
+ );
133
+ lines.push("");
45
134
  lines.push(
46
- `- **${f.field}**: ${f.description}${f.example ? ` (e.g., \`${f.example}\`)` : ""}`
135
+ markdownTable({
136
+ headers: ["Column", "Type", "Available", "Description"],
137
+ rows: view.columns.map((column) => ({
138
+ Column: column.name,
139
+ Type: column.type,
140
+ Available: column.available ? "yes" : `needs ${column.gates.join(", ")}`,
141
+ Description: column.description,
142
+ })),
143
+ }),
47
144
  );
145
+ lines.push("");
146
+ }
147
+ lines.push("### Worked statements", "");
148
+ for (const example of reference.examples.filter((candidate) => candidate.language === "lwql")) {
149
+ lines.push(
150
+ `**${example.title}**${example.available ? "" : ` (needs ${example.requires.gates.join(", ")})`}`,
151
+ );
152
+ lines.push("```sql");
153
+ lines.push(example.text);
154
+ lines.push("```");
155
+ for (const parameter of example.parameters) {
156
+ lines.push(`- \`{${parameter.name}:${parameter.type}}\` — ${parameter.description}`);
157
+ }
158
+ lines.push("");
48
159
  }
49
160
  return lines.join("\n");
50
161
  }
51
162
 
52
- function formatMetrics(): string {
53
- const lines = ["## Available Metrics", ""];
163
+ function formatDecisionTable(reference: QueryReferenceResponse): string {
164
+ const lines = ["## Which one to reach for", ""];
54
165
  lines.push(
55
- "Use these in `get_analytics` as `metric` parameter in `category.name` format."
166
+ markdownTable({
167
+ headers: ["When you want", "Use", "Why"],
168
+ rows: reference.decisionTable.map((row) => ({
169
+ "When you want": row.when,
170
+ Use: row.use,
171
+ Why: row.why,
172
+ })),
173
+ }),
56
174
  );
175
+ return lines.join("\n");
176
+ }
177
+
178
+ function formatMetrics(): string {
179
+ const lines = ["## Available Metrics", ""];
180
+ lines.push("Use these in `get_analytics` as `metric` parameter in `category.name` format.");
57
181
  lines.push("");
58
182
 
59
183
  const byCategory = new Map<string, typeof analyticsMetrics>();
@@ -95,9 +219,7 @@ function formatAggregations(): string {
95
219
 
96
220
  function formatGroups(): string {
97
221
  const lines = ["## Available Group-By Options", ""];
98
- lines.push(
99
- "Use these in the `groupBy` parameter of `get_analytics`."
100
- );
222
+ lines.push("Use these in the `groupBy` parameter of `get_analytics`.");
101
223
  lines.push("");
102
224
  for (const g of analyticsGroups) {
103
225
  lines.push(`- **${g.name}** (${g.label}): ${g.description}`);
@@ -15,16 +15,10 @@ export interface RunTimestamps {
15
15
  stoppedAt?: number | null;
16
16
  }
17
17
 
18
- export const deriveRunStatus = (
19
- timestamps: RunTimestamps,
20
- now: number = Date.now(),
21
- ): RunStatus => {
18
+ export const deriveRunStatus = (timestamps: RunTimestamps, now: number = Date.now()): RunStatus => {
22
19
  if (timestamps.stoppedAt != null) return "stopped";
23
20
  if (timestamps.finishedAt != null) return "completed";
24
- if (
25
- timestamps.updatedAt != null &&
26
- now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS
27
- ) {
21
+ if (timestamps.updatedAt != null && now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS) {
28
22
  return "interrupted";
29
23
  }
30
24
  return "running";
@@ -1,16 +1,11 @@
1
- import type { DatasetMutationResponse } from "../langwatch-api-datasets.js";
2
- import { escapeMarkdown } from "../utils/escape-markdown.js";
1
+ import type { DatasetMutationResponse } from "../langwatch-api-datasets.ts";
2
+ import { escapeMarkdown } from "../utils/escape-markdown.ts";
3
3
 
4
4
  /**
5
- * Formats the common detail lines for a dataset mutation response
6
- * (name, slug, id, and column types).
7
- *
8
- * Used by both create-dataset and update-dataset handlers to avoid
9
- * duplicating the formatting logic.
5
+ * Formats the common detail lines (name, slug, id, column types) for a
6
+ * dataset mutation response, shared by create-dataset and update-dataset.
10
7
  */
11
- export function formatDatasetMutationDetails(
12
- result: DatasetMutationResponse,
13
- ): string[] {
8
+ export function formatDatasetMutationDetails(result: DatasetMutationResponse): string[] {
14
9
  const lines: string[] = [];
15
10
  lines.push(`**Name**: ${escapeMarkdown(result.name)}`);
16
11
  lines.push(`**Slug**: ${escapeMarkdown(result.slug)}`);
@@ -0,0 +1,54 @@
1
+ import type { RunPlanRunResult, RunPlanScope } from "../langwatch-api-run-plans.ts";
2
+
3
+ /**
4
+ * The digest every run of a plan returns, whichever tool started it.
5
+ * Says whether the run created the plan or joined one that already
6
+ * carried the name — a new plan vs. one this run just reconfigured.
7
+ */
8
+ export function formatRunPlanRun(result: RunPlanRunResult): string {
9
+ const lines: string[] = [];
10
+
11
+ lines.push(
12
+ result.created
13
+ ? `Run plan "${result.planName}" created and started.`
14
+ : `Run plan "${result.planName}" started with the configuration of this run.`,
15
+ );
16
+ lines.push("");
17
+ lines.push(`**Plan**: ${result.planName} (${result.runPlanId})`);
18
+ lines.push(`**Batch Run ID**: ${result.batchRunId}`);
19
+ lines.push(`**Set ID**: ${result.setId}`);
20
+ lines.push(`**Jobs**: ${result.jobCount}`);
21
+
22
+ if (result.skippedArchived.scenarios.length > 0) {
23
+ lines.push(`**Skipped archived scenarios**: ${result.skippedArchived.scenarios.join(", ")}`);
24
+ }
25
+ if (result.skippedArchived.targets.length > 0) {
26
+ lines.push(`**Skipped archived targets**: ${result.skippedArchived.targets.join(", ")}`);
27
+ }
28
+
29
+ lines.push(`**View**: ${result.platformUrl}`);
30
+ lines.push("");
31
+ lines.push("> Use `platform_list_simulation_runs` with the batch run ID to read the results.");
32
+
33
+ return lines.join("\n");
34
+ }
35
+
36
+ /**
37
+ * One line saying what a plan covers. A plan stored before scopes existed
38
+ * carries none, and runs the scenario list it already held.
39
+ */
40
+ export function describeRunPlanScope(scope: RunPlanScope | null, scenarioIds: string[]): string {
41
+ if (scope === null) {
42
+ return `hand-picked scenarios (${scenarioIds.length})`;
43
+ }
44
+ switch (scope.mode) {
45
+ case "all":
46
+ return "every scenario in the project";
47
+ case "test_suites":
48
+ return `test suites: ${scope.testSuiteIds.join(", ")}`;
49
+ case "labels":
50
+ return `labels: ${scope.labels.join(", ")}`;
51
+ case "scenarios":
52
+ return `hand-picked scenarios (${scenarioIds.length})`;
53
+ }
54
+ }
@@ -0,0 +1,12 @@
1
+ import type { ScenarioFieldValues } from "../schemas/suite-fields.ts";
2
+
3
+ /** The digest lines for the values a scenario carries per suite field. */
4
+ export function formatScenarioFields(fields: ScenarioFieldValues | undefined): string[] {
5
+ if (!fields || Object.keys(fields).length === 0) return [];
6
+ return [
7
+ "\n## Fields",
8
+ ...Object.entries(fields).map(
9
+ ([identifier, value]) => `- ${identifier}: ${JSON.stringify(value)}`,
10
+ ),
11
+ ];
12
+ }
@@ -0,0 +1,90 @@
1
+ import type {
2
+ EvaluatorAttachmentWire,
3
+ ScenarioMapping,
4
+ SuiteField,
5
+ } from "../schemas/suite-fields.ts";
6
+
7
+ /**
8
+ * The digest lines for a suite's fields, the evaluators attached to a suite
9
+ * or a plan, and the evaluator results on a finished run. Shared by every
10
+ * tool that prints one of them.
11
+ */
12
+
13
+ /** A mapping as one short path: `scenario.fields.golden_sql` or `"literal"`. */
14
+ export function describeMapping(mapping: ScenarioMapping): string {
15
+ return mapping.type === "value"
16
+ ? JSON.stringify(mapping.value)
17
+ : `${mapping.sourceId}.${mapping.path.join(".")}`;
18
+ }
19
+
20
+ export function formatSuiteFields(fields: SuiteField[] | undefined): string[] {
21
+ if (!fields || fields.length === 0) return [];
22
+ return ["\n## Fields", ...fields.map((field) => `- ${field.identifier} (${field.type})`)];
23
+ }
24
+
25
+ export function formatEvaluatorAttachments(
26
+ evaluators: EvaluatorAttachmentWire[] | undefined,
27
+ ): string[] {
28
+ if (!evaluators || evaluators.length === 0) return [];
29
+ const lines = ["\n## Evaluators"];
30
+ for (const attachment of evaluators) {
31
+ const gate = attachment.required ? "required" : "reports only";
32
+ lines.push(`- ${attachment.evaluatorId} (${gate}, attachment ${attachment.id})`);
33
+ for (const [input, mapping] of Object.entries(attachment.mappings)) {
34
+ lines.push(` - ${input}: ${describeMapping(mapping)}`);
35
+ }
36
+ }
37
+ return lines;
38
+ }
39
+
40
+ /**
41
+ * Reported while a run's evaluators haven't recorded yet: the judge has
42
+ * decided, but a required evaluator can still turn the run red — a
43
+ * caller must not read this as final.
44
+ */
45
+ export const PENDING_EVALUATION_STATUS = "PENDING_EVALUATION";
46
+
47
+ /** How a run's status reads in a digest. */
48
+ export function formatRunStatus(status: string): string {
49
+ if (status === PENDING_EVALUATION_STATUS) return "evaluating";
50
+ if (status === "SUCCESS") return "pass";
51
+ if (status === "FAILED") return "FAIL";
52
+ return status;
53
+ }
54
+
55
+ /**
56
+ * The line that warns a caller off a run's verdict while its evaluators are
57
+ * still to be recorded, or nothing when they already are.
58
+ */
59
+ export function pendingEvaluationNote(status: string): string[] {
60
+ if (status !== PENDING_EVALUATION_STATUS) return [];
61
+ return [
62
+ "> The evaluators attached to this run have not been recorded yet. A required one may still fail it, so read the verdict again once the status changes.",
63
+ ];
64
+ }
65
+
66
+ /** One evaluator's result on a finished run. */
67
+ export interface SimulationRunEvaluation {
68
+ evaluatorId: string;
69
+ name: string;
70
+ status: "passed" | "failed" | "scored" | "skipped" | "error";
71
+ required: boolean;
72
+ passed?: boolean;
73
+ score?: number;
74
+ label?: string;
75
+ details?: string;
76
+ }
77
+
78
+ export function formatEvaluations(evaluations: SimulationRunEvaluation[] | undefined): string[] {
79
+ if (!evaluations || evaluations.length === 0) return [];
80
+ const lines = ["\n## Evaluators"];
81
+ for (const evaluation of evaluations) {
82
+ const parts: string[] = [evaluation.status];
83
+ if (evaluation.score !== undefined) parts.push(`score ${evaluation.score}`);
84
+ if (evaluation.label !== undefined) parts.push(evaluation.label);
85
+ if (evaluation.required) parts.push("required");
86
+ lines.push(`- **${evaluation.name}**: ${parts.join(", ")}`);
87
+ if (evaluation.details) lines.push(` ${evaluation.details}`);
88
+ }
89
+ return lines;
90
+ }
@@ -1,15 +1,66 @@
1
- import { getAgent as apiGetAgent } from "../langwatch-api-agents.js";
1
+ import {
2
+ getAgent as apiGetAgent,
3
+ type AgentInstance,
4
+ type AgentParameterSpec,
5
+ } from "../langwatch-api-agents.ts";
2
6
 
3
- export async function handleGetAgent(params: { id: string }): Promise<string> {
4
- const agent = await apiGetAgent(params.id);
7
+ export const describeParameter = ({
8
+ name,
9
+ type,
10
+ options,
11
+ default: defaultValue,
12
+ description,
13
+ required,
14
+ }: AgentParameterSpec): string => {
15
+ const parts: string[] = [type];
16
+ if (options?.length) parts.push(`one of ${options.join(", ")}`);
17
+ if (defaultValue !== undefined) parts.push(`default ${JSON.stringify(defaultValue)}`);
18
+ if (required) parts.push("required");
19
+ const suffix = description ? `: ${description}` : "";
20
+ return `- **${name}** (${parts.join(", ")})${suffix}`;
21
+ };
22
+
23
+ function instancesSection(instances: AgentInstance[]): string[] {
24
+ const lines = [`\n## Instances (${instances.length})\n`];
25
+ for (const instance of instances) {
26
+ const label = instance.label ? ` (${instance.label})` : "";
27
+ lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
28
+ }
29
+ return lines;
30
+ }
31
+
32
+ /**
33
+ * Handles the platform_get_agent MCP tool invocation.
34
+ *
35
+ * @see specs/mcp-server/agent-tools.feature
36
+ */
37
+ export async function handleGetAgent({ id }: { id: string }): Promise<string> {
38
+ const agent = await apiGetAgent(id);
5
39
 
6
40
  const lines: string[] = [];
7
41
  lines.push(`# ${agent.name}\n`);
8
42
  lines.push(`**ID**: ${agent.id}`);
9
43
  lines.push(`**Type**: ${agent.type}`);
44
+ if (agent.environment) lines.push(`**Environment**: ${agent.environment}`);
45
+ if (agent.status) lines.push(`**Status**: ${agent.status}`);
46
+ if (agent.owner?.name) lines.push(`**Owner**: ${agent.owner.name}`);
47
+ else if (agent.hostLabel) lines.push(`**Host**: ${agent.hostLabel}`);
48
+ if (agent.selectable === false) {
49
+ lines.push("**Run**: only its owner can run this agent");
50
+ }
51
+ if (agent.lastSeenAt) lines.push(`**Last seen**: ${agent.lastSeenAt}`);
10
52
  lines.push(`**Created**: ${agent.createdAt}`);
11
53
  lines.push(`**Updated**: ${agent.updatedAt}`);
12
54
 
55
+ if (agent.parameters && agent.parameters.length > 0) {
56
+ lines.push("\n## Parameters\n");
57
+ for (const parameter of agent.parameters) lines.push(describeParameter(parameter));
58
+ }
59
+
60
+ if (agent.instances && agent.instances.length > 0) {
61
+ lines.push(...instancesSection(agent.instances));
62
+ }
63
+
13
64
  if (agent.config && Object.keys(agent.config).length > 0) {
14
65
  lines.push("\n## Config\n");
15
66
  lines.push("```json");