@langwatch/mcp-server 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (430) hide show
  1. package/.gitignore +7 -0
  2. package/.npmignore +41 -0
  3. package/CHANGELOG.md +280 -200
  4. package/CONTRIBUTING.md +10 -0
  5. package/README.md +82 -52
  6. package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
  7. package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
  8. package/dist/archive-project-KWFM3I5Z.js +19 -0
  9. package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js} +4 -4
  10. package/dist/archive-run-plan-65QN3XRX.js +16 -0
  11. package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
  12. package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js} +4 -4
  13. package/dist/archive-scenario-FXEYQNEH.js +19 -0
  14. package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
  15. package/dist/archive-test-suite-NKZDZY3D.js +16 -0
  16. package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
  17. package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
  18. package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js} +7 -10
  19. package/dist/chunk-3YLN3ATJ.js +241 -0
  20. package/dist/chunk-5FNCQRCX.js +91 -0
  21. package/dist/chunk-6YZUOD34.js +55 -0
  22. package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
  23. package/dist/chunk-7VNSRCXO.js +129 -0
  24. package/dist/{chunk-7RHG56OH.js → chunk-B4J3PJLX.js} +11 -10
  25. package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
  26. package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
  27. package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
  28. package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
  29. package/dist/{chunk-TOXJQDIZ.js → chunk-HGELNIVY.js} +135 -51
  30. package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
  31. package/dist/{chunk-J42FWRJ2.js → chunk-I7ATIZED.js} +7 -7
  32. package/dist/{chunk-AQJO2EI5.js → chunk-ILZDO25L.js} +7 -10
  33. package/dist/{chunk-4QC7SA2L.js → chunk-J7PQKBLD.js} +884 -559
  34. package/dist/{chunk-YX3A3CFM.js → chunk-JRJPH75H.js} +7 -13
  35. package/dist/chunk-K2I7Z3LZ.js +28 -0
  36. package/dist/{chunk-5CCZFUXS.js → chunk-NKMTIHIJ.js} +6 -9
  37. package/dist/chunk-O7EMWEVJ.js +44 -0
  38. package/dist/chunk-PTDR56Z3.js +37 -0
  39. package/dist/chunk-RABYBFX5.js +43 -0
  40. package/dist/chunk-SUPUUC4W.js +64 -0
  41. package/dist/{chunk-SFB4MU6H.js → chunk-SXFC3FNH.js} +4 -4
  42. package/dist/chunk-U24EACC4.js +67 -0
  43. package/dist/chunk-UNJM3BIL.js +21 -0
  44. package/dist/{chunk-ZWIL4EFL.js → chunk-UTJCWXPZ.js} +4 -4
  45. package/dist/chunk-UWLE5T2F.js +312 -0
  46. package/dist/chunk-VMWJD732.js +57 -0
  47. package/dist/chunk-XUIJYNRJ.js +15 -0
  48. package/dist/chunk-Y34DHR5K.js +67 -0
  49. package/dist/chunk-YCARO42Z.js +516 -0
  50. package/dist/config-EAKW6F26.js +16 -0
  51. package/dist/config.cjs +109 -0
  52. package/dist/config.js +5 -4
  53. package/dist/{create-agent-7XHO6UIS.js → create-agent-AYZ3LB3V.js} +4 -4
  54. package/dist/create-agent-S5ZBCEFT.js +23 -0
  55. package/dist/create-annotation-LHNCLHYP.js +29 -0
  56. package/dist/{create-annotation-YQXRQC6A.js → create-annotation-VUSXULQK.js} +10 -5
  57. package/dist/{create-api-key-YUS36KBU.js → create-api-key-NNYKARMG.js} +5 -7
  58. package/dist/create-api-key-T4KSNPKB.js +24 -0
  59. package/dist/create-dashboard-DHVFMEU4.js +18 -0
  60. package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-WTKKS5Z6.js} +4 -4
  61. package/dist/{create-dataset-E6NMSVX7.js → create-dataset-2MC4SD4O.js} +6 -6
  62. package/dist/create-dataset-LUP5PVDM.js +22 -0
  63. package/dist/create-dataset-records-NSKBZP3G.js +20 -0
  64. package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-W4GLN2FV.js} +5 -7
  65. package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-643G6LXL.js} +4 -4
  66. package/dist/create-evaluator-HMTYO5SS.js +27 -0
  67. package/dist/create-mcp-server.cjs +9513 -0
  68. package/dist/create-mcp-server.js +2647 -5
  69. package/dist/create-project-6N7PIYRY.js +44 -0
  70. package/dist/{create-project-CS2CB57Y.js → create-project-KAIC46UY.js} +4 -4
  71. package/dist/{create-prompt-F5RMX3L3.js → create-prompt-E67LXIYT.js} +5 -7
  72. package/dist/create-prompt-PKJ7EDRR.js +37 -0
  73. package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
  74. package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-OPQUEXWB.js} +3 -3
  75. package/dist/create-scenario-5PJBMAXO.js +33 -0
  76. package/dist/{create-scenario-JDR2VB2H.js → create-scenario-UEM77OE7.js} +11 -4
  77. package/dist/create-test-suite-4IPXVSFQ.js +37 -0
  78. package/dist/create-test-suite-SZEPVXSE.js +36 -0
  79. package/dist/create-trigger-5HZSCRAN.js +49 -0
  80. package/dist/create-trigger-PRELMT6J.js +48 -0
  81. package/dist/delete-agent-JUIZWAXG.js +18 -0
  82. package/dist/{delete-agent-O3EEH4S5.js → delete-agent-QMEK3XHK.js} +4 -4
  83. package/dist/delete-annotation-HY3O2OGR.js +17 -0
  84. package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-VKIQWUYW.js} +4 -4
  85. package/dist/delete-dashboard-76N66XXL.js +18 -0
  86. package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-JVIVHNJM.js} +4 -4
  87. package/dist/delete-dataset-HQCR66MG.js +18 -0
  88. package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-WKRUW4FP.js} +5 -7
  89. package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
  90. package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
  91. package/dist/delete-evaluator-4SNSCLEE.js +16 -0
  92. package/dist/delete-evaluator-QP3QHTLN.js +15 -0
  93. package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
  94. package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
  95. package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-5J46T73T.js} +4 -4
  96. package/dist/delete-workflow-JSWABL5F.js +17 -0
  97. package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
  98. package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
  99. package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
  100. package/dist/discover-scenario-schema-OOFIQKOG.js +139 -0
  101. package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
  102. package/dist/discover-schema-A33SLMXW.js +442 -0
  103. package/dist/get-agent-LJQX7OF4.js +69 -0
  104. package/dist/get-agent-OTIYUFZA.js +70 -0
  105. package/dist/get-analytics-DCS72353.js +80 -0
  106. package/dist/get-analytics-IQZKJHLW.js +79 -0
  107. package/dist/get-dashboard-5NYJEIX2.js +29 -0
  108. package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-QMTYDQAZ.js} +4 -4
  109. package/dist/get-dataset-ALSP7RTX.js +64 -0
  110. package/dist/get-dataset-X4B5JDG2.js +63 -0
  111. package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-ISMC4LLI.js} +4 -4
  112. package/dist/get-evaluator-XEUZGZCH.js +47 -0
  113. package/dist/{get-project-RAK7URQJ.js → get-project-5VKROSDH.js} +4 -4
  114. package/dist/get-project-RM6FRCBK.js +26 -0
  115. package/dist/get-prompt-OWXJZHQQ.js +95 -0
  116. package/dist/get-prompt-W52DY36E.js +94 -0
  117. package/dist/get-run-plan-BE3M2AUX.js +59 -0
  118. package/dist/get-run-plan-RUXQ6PAX.js +58 -0
  119. package/dist/get-scenario-RNM7BAX2.js +37 -0
  120. package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-YM4ODWW6.js} +8 -4
  121. package/dist/get-simulation-run-6GIATQE5.js +69 -0
  122. package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-LCFSMPGQ.js} +33 -20
  123. package/dist/get-test-suite-GC2O7YH6.js +49 -0
  124. package/dist/get-test-suite-P5LJ6VLF.js +48 -0
  125. package/dist/get-trace-KBRVV4NN.js +45 -0
  126. package/dist/{get-trace-K7U234Q4.js → get-trace-SEF3FMR5.js} +11 -13
  127. package/dist/get-workflow-7PVED7JE.js +25 -0
  128. package/dist/{get-workflow-L5L2PSK3.js → get-workflow-YXULW4GC.js} +4 -4
  129. package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
  130. package/dist/{http-server-YXHFPPGA.js → http-server-IR4QOS43.js} +4071 -342
  131. package/dist/index.js +60221 -38077
  132. package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
  133. package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
  134. package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
  135. package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
  136. package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
  137. package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
  138. package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
  139. package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
  140. package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
  141. package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
  142. package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
  143. package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
  144. package/dist/list-agents-HVYWEQMI.js +43 -0
  145. package/dist/list-agents-RBAOCF53.js +42 -0
  146. package/dist/list-annotations-HJH4NBLD.js +38 -0
  147. package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-J4UAHSUN.js} +10 -5
  148. package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-7UTJRZDG.js} +10 -5
  149. package/dist/list-api-keys-A2PMV435.js +42 -0
  150. package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
  151. package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
  152. package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-ROSYFLDQ.js} +4 -4
  153. package/dist/list-dashboards-RQ635ZU7.js +32 -0
  154. package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-M5TKA6LK.js} +14 -10
  155. package/dist/list-dataset-records-WFM7F33C.js +43 -0
  156. package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-B5X4P7BQ.js} +9 -13
  157. package/dist/list-datasets-THUKQHPX.js +56 -0
  158. package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-MYH2OVCI.js} +5 -7
  159. package/dist/list-evaluators-NLVWF6K7.js +32 -0
  160. package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-3JUGP4YI.js} +4 -4
  161. package/dist/list-model-providers-H3XKORFO.js +35 -0
  162. package/dist/{list-projects-K7QAIX2I.js → list-projects-6EJ5YYEW.js} +5 -7
  163. package/dist/list-projects-YNNSRMLC.js +40 -0
  164. package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-NVFQO6WX.js} +3 -3
  165. package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
  166. package/dist/{list-prompts-SC2HAUFG.js → list-prompts-HSIMMHTF.js} +4 -6
  167. package/dist/list-prompts-IRI6AIXO.js +30 -0
  168. package/dist/list-run-plans-PX2XI6EK.js +46 -0
  169. package/dist/list-run-plans-QALLCN2K.js +45 -0
  170. package/dist/list-scenarios-QZRPVHS4.js +46 -0
  171. package/dist/list-scenarios-SSJQEMJP.js +45 -0
  172. package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
  173. package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-OEADLLQR.js} +19 -9
  174. package/dist/list-test-suites-ANAQBLDG.js +34 -0
  175. package/dist/list-test-suites-N4WVLBKK.js +35 -0
  176. package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
  177. package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
  178. package/dist/list-triggers-SQNIG35Q.js +45 -0
  179. package/dist/list-triggers-V4Y356AK.js +46 -0
  180. package/dist/{list-workflows-QSJWURCK.js → list-workflows-IMSX7SXU.js} +5 -7
  181. package/dist/list-workflows-ZHTX5LCL.js +33 -0
  182. package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
  183. package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
  184. package/dist/rename-test-suite-4J7UX5S2.js +21 -0
  185. package/dist/rename-test-suite-NNVLMLKY.js +20 -0
  186. package/dist/report-issue-AS34ZMJF.js +898 -0
  187. package/dist/report-issue-DCZFLGDO.js +899 -0
  188. package/dist/rerun-run-plan-TG2V4WM4.js +22 -0
  189. package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
  190. package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
  191. package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-SFMEFWJC.js} +4 -4
  192. package/dist/run-agent-W4VJP72Z.js +75 -0
  193. package/dist/run-agent-YNV6ZHW4.js +74 -0
  194. package/dist/run-plan-6C77GWG4.js +38 -0
  195. package/dist/run-plan-PAARQSI4.js +39 -0
  196. package/dist/run-query-KJDH544C.js +47 -0
  197. package/dist/run-query-T4QW2Z5O.js +46 -0
  198. package/dist/run-test-suite-SQJELEZX.js +26 -0
  199. package/dist/run-test-suite-STB6NUJM.js +25 -0
  200. package/dist/{run-workflow-QPXC627O.js → run-workflow-D6PQNBI6.js} +4 -4
  201. package/dist/{run-agent-6SM4XNA7.js → run-workflow-SP32SO5M.js} +10 -11
  202. package/dist/search-traces-HCBGOG6R.js +170 -0
  203. package/dist/search-traces-HSTQ6WTQ.js +171 -0
  204. package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-GF7NXUJA.js} +4 -4
  205. package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
  206. package/dist/{update-agent-SMAEHLXH.js → update-agent-DWG43T2P.js} +4 -4
  207. package/dist/update-agent-MXNOBS4H.js +20 -0
  208. package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
  209. package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
  210. package/dist/update-dataset-DQH3OHOU.js +22 -0
  211. package/dist/{update-dataset-BDUK7TUO.js → update-dataset-SQZ6T4WX.js} +6 -6
  212. package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-3AT4IPBC.js} +4 -4
  213. package/dist/update-dataset-record-7PLDGTMR.js +19 -0
  214. package/dist/update-evaluator-4LFTKFPT.js +24 -0
  215. package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-IGTX6PX4.js} +4 -4
  216. package/dist/{update-project-LKVYXENH.js → update-project-PLJIOZS6.js} +4 -4
  217. package/dist/update-project-QVHSJRUP.js +24 -0
  218. package/dist/update-prompt-GDKRYJI3.js +140 -0
  219. package/dist/update-prompt-MY263DYF.js +141 -0
  220. package/dist/update-scenario-7KA3JDXA.js +34 -0
  221. package/dist/{update-scenario-NEXCAQI7.js → update-scenario-MOHRQKAQ.js} +11 -4
  222. package/package.json +39 -44
  223. package/pyproject.toml +8 -8
  224. package/src/config.d.ts +4 -0
  225. package/src/config.ts +20 -29
  226. package/src/create-mcp-server.ts +1021 -596
  227. package/src/documentation-fetch.ts +21 -20
  228. package/src/http-security.ts +166 -135
  229. package/src/http-server.ts +189 -224
  230. package/src/index.ts +14 -16
  231. package/src/langwatch-api-agents.ts +202 -21
  232. package/src/langwatch-api-annotations.ts +11 -12
  233. package/src/langwatch-api-api-keys.ts +8 -9
  234. package/src/langwatch-api-boards.ts +46 -0
  235. package/src/langwatch-api-dashboard-widgets.ts +112 -0
  236. package/src/langwatch-api-dashboards.ts +13 -10
  237. package/src/langwatch-api-datasets.ts +98 -32
  238. package/src/langwatch-api-evaluators.ts +13 -19
  239. package/src/langwatch-api-model-providers.ts +4 -4
  240. package/src/langwatch-api-monitors.ts +14 -18
  241. package/src/langwatch-api-projects.ts +27 -9
  242. package/src/langwatch-api-query.ts +114 -0
  243. package/src/langwatch-api-run-plans.ts +140 -0
  244. package/src/langwatch-api-scenarios.ts +17 -10
  245. package/src/langwatch-api-secrets.ts +31 -24
  246. package/src/langwatch-api-simulation-runs.ts +10 -7
  247. package/src/langwatch-api-test-suites.ts +116 -0
  248. package/src/langwatch-api-triggers.ts +96 -34
  249. package/src/langwatch-api-workflows.ts +9 -9
  250. package/src/langwatch-api.ts +143 -109
  251. package/src/public-http-request.ts +21 -12
  252. package/src/schemas/analytics-metrics.ts +3 -15
  253. package/src/schemas/create-dataset.ts +3 -4
  254. package/src/schemas/run-plan.ts +76 -0
  255. package/src/schemas/suite-fields.ts +132 -0
  256. package/src/schemas/triggers.ts +365 -0
  257. package/src/tools/add-dashboard-widget.ts +63 -0
  258. package/src/tools/archive-project.ts +2 -4
  259. package/src/tools/archive-run-plan.ts +10 -0
  260. package/src/tools/archive-scenario.ts +2 -4
  261. package/src/tools/archive-test-suite.ts +10 -0
  262. package/src/tools/assign-prompt-tag.ts +6 -2
  263. package/src/tools/create-agent.ts +1 -1
  264. package/src/tools/create-annotation.ts +7 -2
  265. package/src/tools/create-api-key.ts +4 -6
  266. package/src/tools/create-dashboard.ts +1 -1
  267. package/src/tools/create-dataset-records.ts +4 -8
  268. package/src/tools/create-dataset.ts +5 -7
  269. package/src/tools/create-evaluator.ts +3 -5
  270. package/src/tools/create-project.ts +2 -2
  271. package/src/tools/create-prompt-tag.ts +2 -4
  272. package/src/tools/create-prompt.ts +9 -18
  273. package/src/tools/create-scenario.ts +11 -5
  274. package/src/tools/create-test-suite.ts +38 -0
  275. package/src/tools/create-trigger.ts +55 -0
  276. package/src/tools/dashboard-exists.ts +25 -0
  277. package/src/tools/dashboard-widget-queries.ts +32 -0
  278. package/src/tools/delete-agent.ts +1 -1
  279. package/src/tools/delete-annotation.ts +1 -1
  280. package/src/tools/delete-dashboard.ts +1 -1
  281. package/src/tools/delete-dataset-records.ts +4 -8
  282. package/src/tools/delete-dataset.ts +3 -7
  283. package/src/tools/delete-evaluator.ts +4 -6
  284. package/src/tools/delete-prompt-tag.ts +2 -4
  285. package/src/tools/delete-workflow.ts +1 -1
  286. package/src/tools/discover-evaluator-schema.ts +34 -25
  287. package/src/tools/discover-scenario-schema.ts +86 -12
  288. package/src/tools/discover-schema.ts +149 -27
  289. package/src/tools/experiment-run-status.ts +2 -8
  290. package/src/tools/format-dataset-mutation.ts +5 -10
  291. package/src/tools/format-run-plan.ts +54 -0
  292. package/src/tools/format-scenario.ts +12 -0
  293. package/src/tools/format-suite-details.ts +90 -0
  294. package/src/tools/get-agent.ts +54 -3
  295. package/src/tools/get-analytics.ts +52 -46
  296. package/src/tools/get-dashboard.ts +1 -1
  297. package/src/tools/get-dataset.ts +34 -16
  298. package/src/tools/get-evaluator.ts +4 -11
  299. package/src/tools/get-experiment-results.ts +219 -186
  300. package/src/tools/get-project.ts +2 -4
  301. package/src/tools/get-prompt.ts +100 -37
  302. package/src/tools/get-run-plan.ts +60 -0
  303. package/src/tools/get-scenario.ts +6 -5
  304. package/src/tools/get-simulation-run.ts +32 -20
  305. package/src/tools/get-test-suite.ts +46 -0
  306. package/src/tools/get-trace.ts +16 -17
  307. package/src/tools/get-workflow.ts +1 -1
  308. package/src/tools/list-agents.ts +20 -3
  309. package/src/tools/list-annotations.ts +8 -5
  310. package/src/tools/list-api-keys.ts +7 -2
  311. package/src/tools/list-dashboard-boards.ts +28 -0
  312. package/src/tools/list-dashboards.ts +1 -1
  313. package/src/tools/list-dataset-records.ts +8 -6
  314. package/src/tools/list-datasets.ts +12 -23
  315. package/src/tools/list-evaluators.ts +4 -8
  316. package/src/tools/list-experiment-runs.ts +5 -9
  317. package/src/tools/list-experiments.ts +5 -11
  318. package/src/tools/list-model-providers.ts +3 -5
  319. package/src/tools/list-projects.ts +2 -4
  320. package/src/tools/list-prompt-tags.ts +3 -3
  321. package/src/tools/list-prompts.ts +4 -8
  322. package/src/tools/list-run-plans.ts +47 -0
  323. package/src/tools/list-scenarios.ts +26 -17
  324. package/src/tools/list-simulation-runs.ts +13 -5
  325. package/src/tools/list-test-suites.ts +35 -0
  326. package/src/tools/list-trigger-fires.ts +17 -0
  327. package/src/tools/list-triggers.ts +39 -0
  328. package/src/tools/list-workflows.ts +2 -4
  329. package/src/tools/rename-prompt-tag.ts +1 -1
  330. package/src/tools/rename-test-suite.ts +15 -0
  331. package/src/tools/report-issue.ts +5 -9
  332. package/src/tools/rerun-run-plan.ts +18 -0
  333. package/src/tools/revoke-api-key.ts +2 -4
  334. package/src/tools/run-agent.ts +65 -14
  335. package/src/tools/run-experiment.ts +36 -37
  336. package/src/tools/run-plan.ts +46 -0
  337. package/src/tools/run-query.ts +47 -0
  338. package/src/tools/run-test-suite.ts +27 -0
  339. package/src/tools/run-workflow.ts +2 -5
  340. package/src/tools/search-traces.ts +161 -47
  341. package/src/tools/set-model-provider.ts +4 -6
  342. package/src/tools/test-agent.ts +18 -0
  343. package/src/tools/update-agent.ts +1 -1
  344. package/src/tools/update-dashboard-widget.ts +97 -0
  345. package/src/tools/update-dataset-record.ts +1 -1
  346. package/src/tools/update-dataset.ts +5 -7
  347. package/src/tools/update-evaluator.ts +3 -5
  348. package/src/tools/update-project.ts +1 -1
  349. package/src/tools/update-prompt.ts +185 -15
  350. package/src/tools/update-scenario.ts +11 -5
  351. package/src/tools/update-test-suite.ts +40 -0
  352. package/src/utils/date-parsing.ts +1 -1
  353. package/src/utils/escape-markdown.ts +3 -8
  354. package/src/utils/format-evaluations.ts +6 -6
  355. package/src/utils/markdown-table.ts +58 -0
  356. package/src/utils/trace-id-shape.ts +20 -0
  357. package/tsconfig.app.json +24 -0
  358. package/tsconfig.json +19 -12
  359. package/uv.lock +416 -296
  360. package/vitest.integration.config.js +10 -0
  361. package/vitest.scenario.config.js +9 -0
  362. package/.eslintrc.cjs +0 -36
  363. package/dist/archive-suite-4T6BSRS7.js +0 -16
  364. package/dist/chunk-77BNWQ3U.js +0 -86
  365. package/dist/chunk-A2LEVYQ5.js +0 -217
  366. package/dist/chunk-TJCXFNDT.js +0 -24
  367. package/dist/chunk-YZLYHLLI.js +0 -48
  368. package/dist/config.d.ts +0 -17
  369. package/dist/create-mcp-server.d.ts +0 -10
  370. package/dist/create-suite-LZS6QIAF.js +0 -39
  371. package/dist/delete-evaluator-33SP2JFX.js +0 -16
  372. package/dist/discover-evaluator-schema-XTAD4X7D.js +0 -2191
  373. package/dist/discover-scenario-schema-FE64QK2D.js +0 -67
  374. package/dist/get-agent-Y7MWZ65F.js +0 -29
  375. package/dist/get-analytics-TETCSOK6.js +0 -73
  376. package/dist/get-dataset-RCMU5VAH.js +0 -49
  377. package/dist/get-prompt-OY5B7TRI.js +0 -46
  378. package/dist/get-suite-J7JVWO6O.js +0 -45
  379. package/dist/index.d.ts +0 -2
  380. package/dist/langwatch-api-secrets-BUBSNCNO.js +0 -41
  381. package/dist/langwatch-api-triggers-EDFUCJN6.js +0 -31
  382. package/dist/list-agents-KFHKGPXV.js +0 -33
  383. package/dist/list-scenarios-F2WYNRIE.js +0 -41
  384. package/dist/list-suites-CRY4VWW6.js +0 -43
  385. package/dist/report-issue-XYVAPAYJ.js +0 -412
  386. package/dist/run-suite-NKZCJ2IN.js +0 -30
  387. package/dist/search-traces-A6HFKPLJ.js +0 -79
  388. package/dist/update-prompt-ODCRQRN6.js +0 -25
  389. package/dist/update-suite-SNO3GS7T.js +0 -40
  390. package/src/__tests__/all-tools.integration.test.ts +0 -1626
  391. package/src/__tests__/config.unit.test.ts +0 -202
  392. package/src/__tests__/datasets.integration.test.ts +0 -620
  393. package/src/__tests__/datasets.unit.test.ts +0 -244
  394. package/src/__tests__/date-parsing.unit.test.ts +0 -78
  395. package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
  396. package/src/__tests__/discover-schema.unit.test.ts +0 -118
  397. package/src/__tests__/docker.integration.test.ts +0 -318
  398. package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
  399. package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
  400. package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
  401. package/src/__tests__/experiment-status.unit.test.ts +0 -194
  402. package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
  403. package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
  404. package/src/__tests__/http-security-origin.unit.test.ts +0 -97
  405. package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
  406. package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
  407. package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
  408. package/src/__tests__/http-server-origin.integration.test.ts +0 -169
  409. package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
  410. package/src/__tests__/http-transport.integration.test.ts +0 -395
  411. package/src/__tests__/integration.integration.test.ts +0 -288
  412. package/src/__tests__/langwatch-api.unit.test.ts +0 -661
  413. package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
  414. package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
  415. package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
  416. package/src/__tests__/report-issue.integration.test.ts +0 -147
  417. package/src/__tests__/scenario-tools.integration.test.ts +0 -286
  418. package/src/__tests__/scenario-tools.unit.test.ts +0 -188
  419. package/src/__tests__/schemas.unit.test.ts +0 -85
  420. package/src/__tests__/support/http-server-harness.ts +0 -138
  421. package/src/__tests__/tools.unit.test.ts +0 -924
  422. package/src/langwatch-api-suites.ts +0 -118
  423. package/src/schemas/filter-fields.ts +0 -119
  424. package/src/schemas/index.ts +0 -4
  425. package/src/tools/archive-suite.ts +0 -12
  426. package/src/tools/create-suite.ts +0 -42
  427. package/src/tools/get-suite.ts +0 -47
  428. package/src/tools/list-suites.ts +0 -43
  429. package/src/tools/run-suite.ts +0 -28
  430. package/src/tools/update-suite.ts +0 -44
@@ -1,5 +1,5 @@
1
- import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
2
- import { deriveRunStatus, isTerminalStatus } from "./experiment-run-status.js";
1
+ import { LangWatchApiError, makeRequest } from "../langwatch-api.ts";
2
+ import { deriveRunStatus, isTerminalStatus } from "./experiment-run-status.ts";
3
3
 
4
4
  interface DatasetEntry {
5
5
  index: number;
@@ -39,12 +39,17 @@ interface EvaluationRunResults {
39
39
  finishedAt?: number | null;
40
40
  stoppedAt?: number | null;
41
41
  };
42
+ /** What is stored against what the run reported; absent on servers that predate it. */
43
+ completeness?: {
44
+ complete: boolean;
45
+ dataset: { received: number; expected: number | null };
46
+ evaluations: { received: number; expected: number | null };
47
+ };
42
48
  }
43
49
 
44
50
  const DEFAULT_ROW_CAP = 50;
45
51
 
46
- const rowKey = (index: number, targetId?: string | null): string =>
47
- `${index}:${targetId ?? ""}`;
52
+ const rowKey = (index: number, targetId?: string | null): string => `${index}:${targetId ?? ""}`;
48
53
 
49
54
  const summarizeEntry = (entry: Record<string, unknown>): string => {
50
55
  const candidates = ["input", "question", "query", "prompt", "user"];
@@ -68,128 +73,72 @@ const isFailedRow = ({
68
73
  evaluations: EvaluationItem[];
69
74
  }): boolean => Boolean(entry.error) || evaluations.some(isFailedEvaluation);
70
75
 
71
- export async function handleExperimentResults(params: {
76
+ function hasNumericStatus(error: unknown): error is { status: number } {
77
+ if (typeof error !== "object" || error === null) return false;
78
+ return "status" in error && typeof error.status === "number";
79
+ }
80
+
81
+ type RunStatus = ReturnType<typeof deriveRunStatus>;
82
+ type ResultRow = { entry: DatasetEntry; evaluations: EvaluationItem[] };
83
+ type EvaluatorStats = {
84
+ sum: number;
85
+ count: number;
86
+ passed: number;
87
+ failed: number;
88
+ errored: number;
89
+ };
90
+
91
+ function isNotFoundError(error: unknown): boolean {
92
+ const message = error instanceof Error ? error.message : String(error);
93
+ let status: number | undefined;
94
+ if (error instanceof LangWatchApiError) status = error.status;
95
+ else if (hasNumericStatus(error)) status = error.status;
96
+ return status === 404 || (status === undefined && /404|not found/i.test(message));
97
+ }
98
+
99
+ async function fetchRunResults(params: {
72
100
  runId: string;
73
101
  experimentSlug?: string;
74
- filter?: "all" | "failed";
75
- evaluator?: string;
76
- limit?: number;
77
- }): Promise<string> {
78
- const filter = params.filter ?? "all";
79
- const evaluatorFilter = params.evaluator?.trim();
80
- const limit =
81
- typeof params.limit === "number" && params.limit > 0
82
- ? Math.min(params.limit, DEFAULT_ROW_CAP)
83
- : DEFAULT_ROW_CAP;
84
-
102
+ }): Promise<EvaluationRunResults | null> {
85
103
  const search = new URLSearchParams();
86
104
  if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
87
105
  const qs = search.toString() ? `?${search.toString()}` : "";
106
+ return (await makeRequest(
107
+ "GET",
108
+ `/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`,
109
+ )) as EvaluationRunResults;
110
+ }
88
111
 
89
- let results: EvaluationRunResults | null;
90
- try {
91
- results = (await makeRequest(
92
- "GET",
93
- `/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`,
94
- )) as EvaluationRunResults;
95
- } catch (error) {
96
- const message = error instanceof Error ? error.message : String(error);
97
- const status =
98
- error instanceof LangWatchApiError
99
- ? error.status
100
- : error && typeof error === "object" &&
101
- "status" in error &&
102
- typeof (error as { status?: unknown }).status === "number"
103
- ? (error as { status: number }).status
104
- : undefined;
105
- if (
106
- status === 404 ||
107
- (status === undefined && /404|not found/i.test(message))
108
- ) {
109
- return [
110
- `# Evaluation Results: ${params.runId}`,
111
- "",
112
- "**Status**: not found",
113
- "",
114
- `Could not load results for run \`${params.runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
115
- "",
116
- "> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids.",
117
- ].join("\n");
118
- }
119
- throw error;
120
- }
121
-
122
- if (!results) {
123
- return [
124
- `# Evaluation Results: ${params.runId}`,
125
- "",
126
- "**Status**: not found",
127
- "",
128
- `Could not load results for run \`${params.runId}\`.`,
129
- "",
130
- "> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id.",
131
- ].join("\n");
132
- }
133
-
134
- // Partial results are served even while the run is still in progress.
135
- // Rows land in ClickHouse incrementally, so a "running" or "interrupted"
136
- // run can still expose every row recorded so far. A run only fails to set
137
- // finished_at/stopped_at when the SDK process dies before flushing, but the
138
- // rows it did log are still useful, so we never gate on a terminal status.
139
- const runStatus = deriveRunStatus(results.timestamps);
140
- const partial = !isTerminalStatus(runStatus);
141
-
142
- // Group evaluations by target-scoped row key, applying evaluator filter.
143
- const evaluationsByRow = new Map<string, EvaluationItem[]>();
144
- for (const evaluation of results.evaluations) {
112
+ /** Evaluations grouped by target-scoped row key, keeping only the filtered evaluator. */
113
+ function groupEvaluationsByRow({
114
+ evaluations,
115
+ evaluatorFilter,
116
+ }: {
117
+ evaluations: EvaluationItem[];
118
+ evaluatorFilter: string | undefined;
119
+ }): Map<string, EvaluationItem[]> {
120
+ const byRow = new Map<string, EvaluationItem[]>();
121
+ for (const evaluation of evaluations) {
145
122
  if (evaluatorFilter && evaluation.evaluator !== evaluatorFilter) continue;
146
123
  const key = rowKey(evaluation.index, evaluation.targetId);
147
- const list = evaluationsByRow.get(key) ?? [];
124
+ const list = byRow.get(key) ?? [];
148
125
  list.push(evaluation);
149
- evaluationsByRow.set(key, list);
150
- }
151
-
152
- const evaluatorNames = Array.from(
153
- new Set(
154
- (evaluatorFilter
155
- ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter)
156
- : results.evaluations
157
- ).map((e) => e.evaluator),
158
- ),
159
- );
160
-
161
- let rows = results.dataset.map((entry) => ({
162
- entry,
163
- evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? [],
164
- }));
165
-
166
- if (filter === "failed") {
167
- rows = rows.filter((r) =>
168
- isFailedRow({ entry: r.entry, evaluations: r.evaluations }),
169
- );
126
+ byRow.set(key, list);
170
127
  }
128
+ return byRow;
129
+ }
171
130
 
172
- const totalMatching = rows.length;
173
- const rowsForSummary = rows;
174
- const truncated = rows.length > limit;
175
- rows = rows.slice(0, limit);
176
-
177
- // Per-evaluator stats across the filtered rows (before truncation),
178
- // so the summary matches the displayed subset when filter="failed".
179
- const evaluatorAverages = new Map<
180
- string,
181
- { sum: number; count: number; passed: number; failed: number; errored: number }
182
- >();
183
- for (const r of rowsForSummary) {
131
+ function tallyEvaluatorStats(rows: ResultRow[]): Map<string, EvaluatorStats> {
132
+ const averages = new Map<string, EvaluatorStats>();
133
+ for (const r of rows) {
184
134
  for (const e of r.evaluations) {
185
- const stats =
186
- evaluatorAverages.get(e.evaluator) ?? {
187
- sum: 0,
188
- count: 0,
189
- passed: 0,
190
- failed: 0,
191
- errored: 0,
192
- };
135
+ const stats = averages.get(e.evaluator) ?? {
136
+ sum: 0,
137
+ count: 0,
138
+ passed: 0,
139
+ failed: 0,
140
+ errored: 0,
141
+ };
193
142
  if (typeof e.score === "number") {
194
143
  stats.sum += e.score;
195
144
  stats.count += 1;
@@ -197,29 +146,36 @@ export async function handleExperimentResults(params: {
197
146
  if (e.status === "error") stats.errored += 1;
198
147
  else if (e.passed === true) stats.passed += 1;
199
148
  else if (e.passed === false) stats.failed += 1;
200
- evaluatorAverages.set(e.evaluator, stats);
149
+ averages.set(e.evaluator, stats);
201
150
  }
202
151
  }
152
+ return averages;
153
+ }
203
154
 
155
+ function headerLines({
156
+ results,
157
+ runStatus,
158
+ filter,
159
+ evaluatorFilter,
160
+ }: {
161
+ results: EvaluationRunResults;
162
+ runStatus: RunStatus;
163
+ filter: "all" | "failed";
164
+ evaluatorFilter: string | undefined;
165
+ }): string[] {
204
166
  const lines: string[] = [];
205
167
  lines.push(`# Evaluation Results: ${results.runId}`);
206
168
  lines.push("");
207
169
  lines.push(`**Experiment**: ${results.experimentId}`);
208
170
  lines.push(`**Status**: ${runStatus}`);
209
171
  if (typeof results.total === "number" && results.total > 0) {
210
- lines.push(
211
- `**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`,
212
- );
172
+ lines.push(`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`);
213
173
  }
214
174
  lines.push(`**Total rows**: ${results.dataset.length}`);
215
175
  lines.push(`**Total evaluations**: ${results.evaluations.length}`);
216
- if (filter === "failed") {
217
- lines.push(`**Filter**: failed only`);
218
- }
219
- if (evaluatorFilter) {
220
- lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
221
- }
222
- if (partial) {
176
+ if (filter === "failed") lines.push(`**Filter**: failed only`);
177
+ if (evaluatorFilter) lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
178
+ if (!isTerminalStatus(runStatus)) {
223
179
  lines.push("");
224
180
  lines.push(
225
181
  runStatus === "interrupted"
@@ -227,80 +183,157 @@ export async function handleExperimentResults(params: {
227
183
  : "> These are partial results. The run is still in progress, so more rows may appear on a later call.",
228
184
  );
229
185
  }
186
+ const stored = results.completeness;
187
+ if (isTerminalStatus(runStatus) && stored && !stored.complete) {
188
+ lines.push("");
189
+ lines.push(
190
+ `> These are partial results. The run has ended, but the platform is still storing it: ${storedCount(stored.dataset)} rows and ${storedCount(stored.evaluations)} evaluations so far. Call again shortly for the whole run.`,
191
+ );
192
+ }
230
193
  lines.push("");
194
+ return lines;
195
+ }
231
196
 
232
- if (evaluatorAverages.size > 0) {
233
- lines.push("## Evaluator Summary");
234
- lines.push("");
235
- lines.push("| Evaluator | Avg Score | Passed | Failed | Errored |");
236
- lines.push("| --- | --- | --- | --- | --- |");
237
- for (const [name, stats] of evaluatorAverages) {
238
- const avg =
239
- stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "—";
240
- lines.push(
241
- `| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`,
242
- );
243
- }
244
- lines.push("");
197
+ function evaluatorSummaryLines(averages: Map<string, EvaluatorStats>): string[] {
198
+ if (averages.size === 0) return [];
199
+ const lines = [
200
+ "## Evaluator Summary",
201
+ "",
202
+ "| Evaluator | Avg Score | Passed | Failed | Errored |",
203
+ "| --- | --- | --- | --- | --- |",
204
+ ];
205
+ for (const [name, stats] of averages) {
206
+ const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "—";
207
+ lines.push(`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`);
245
208
  }
209
+ lines.push("");
210
+ return lines;
211
+ }
246
212
 
247
- if (rows.length === 0) {
248
- if (filter === "failed") {
249
- lines.push("_No rows matched the filter._");
250
- } else if (runStatus === "running") {
251
- lines.push(
252
- "_No rows recorded yet. The run is still in progress; call again shortly._",
253
- );
254
- } else if (runStatus === "interrupted") {
255
- lines.push("_No rows were recorded before the run was interrupted._");
256
- } else {
257
- lines.push("_No rows recorded for this run._");
258
- }
259
- return lines.join("\n");
213
+ const storedCount = ({ received, expected }: { received: number; expected: number | null }) =>
214
+ expected === null ? `${received}` : `${received} of ${expected}`;
215
+
216
+ function noRowsLine({ filter, runStatus }: { filter: "all" | "failed"; runStatus: RunStatus }) {
217
+ if (filter === "failed") return "_No rows matched the filter._";
218
+ if (runStatus === "running")
219
+ return "_No rows recorded yet. The run is still in progress; call again shortly._";
220
+ if (runStatus === "interrupted") return "_No rows were recorded before the run was interrupted._";
221
+ return "_No rows recorded for this run._";
222
+ }
223
+
224
+ function evaluationLines(e: EvaluationItem): string[] {
225
+ const parts: string[] = [`**${e.evaluator}**`];
226
+ if (e.status === "error") parts.push("status=error");
227
+ else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
228
+ if (typeof e.passed === "boolean") parts.push(`passed=${e.passed ? "yes" : "no"}`);
229
+ if (e.label) parts.push(`label=${e.label}`);
230
+ const lines = [`- ${parts.join(" · ")}`];
231
+ if (e.details) lines.push(` - details: ${e.details}`);
232
+ return lines;
233
+ }
234
+
235
+ function rowLines({
236
+ row: { entry, evaluations },
237
+ anyEvaluators,
238
+ }: {
239
+ row: ResultRow;
240
+ anyEvaluators: boolean;
241
+ }): string[] {
242
+ const summary = summarizeEntry(entry.entry);
243
+ const lines = [`### Row #${entry.index}${summary ? ` — ${summary}` : ""}`];
244
+ if (entry.error) lines.push(`- **Error**: ${entry.error}`);
245
+ if (entry.traceId) lines.push(`- **Trace ID**: \`${entry.traceId}\``);
246
+ if (evaluations.length === 0) {
247
+ lines.push(
248
+ anyEvaluators ? "- _No evaluations recorded for this row_" : "- _No evaluations recorded_",
249
+ );
250
+ }
251
+ for (const e of evaluations) lines.push(...evaluationLines(e));
252
+ lines.push("");
253
+ return lines;
254
+ }
255
+
256
+ function notFoundReport({ runId, expired }: { runId: string; expired: boolean }): string {
257
+ const lines = [`# Evaluation Results: ${runId}`, "", "**Status**: not found", ""];
258
+ if (expired) {
259
+ lines.push(
260
+ `Could not load results for run \`${runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
261
+ "",
262
+ "> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids.",
263
+ );
264
+ } else {
265
+ lines.push(
266
+ `Could not load results for run \`${runId}\`.`,
267
+ "",
268
+ "> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id.",
269
+ );
260
270
  }
271
+ return lines.join("\n");
272
+ }
261
273
 
262
- lines.push(
263
- `## Rows (${rows.length}${truncated ? ` of ${totalMatching}` : ""})`,
274
+ export async function handleExperimentResults(params: {
275
+ runId: string;
276
+ experimentSlug?: string;
277
+ filter?: "all" | "failed";
278
+ evaluator?: string;
279
+ limit?: number;
280
+ }): Promise<string> {
281
+ const filter = params.filter ?? "all";
282
+ const evaluatorFilter = params.evaluator?.trim();
283
+ const limit =
284
+ typeof params.limit === "number" && params.limit > 0
285
+ ? Math.min(params.limit, DEFAULT_ROW_CAP)
286
+ : DEFAULT_ROW_CAP;
287
+
288
+ let results: EvaluationRunResults | null;
289
+ try {
290
+ results = await fetchRunResults(params);
291
+ } catch (error) {
292
+ if (isNotFoundError(error)) return notFoundReport({ runId: params.runId, expired: true });
293
+ throw error;
294
+ }
295
+ if (!results) return notFoundReport({ runId: params.runId, expired: false });
296
+
297
+ // Partial results are served while the run is in progress: rows land
298
+ // incrementally, and a run whose SDK died before flushing still has useful
299
+ // rows, so we never gate on a terminal status.
300
+ const runStatus = deriveRunStatus(results.timestamps);
301
+ const evaluationsByRow = groupEvaluationsByRow({
302
+ evaluations: results.evaluations,
303
+ evaluatorFilter,
304
+ });
305
+ const evaluatorNames = new Set(
306
+ (evaluatorFilter
307
+ ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter)
308
+ : results.evaluations
309
+ ).map((e) => e.evaluator),
264
310
  );
265
- lines.push("");
266
311
 
267
- for (const { entry, evaluations } of rows) {
268
- const summary = summarizeEntry(entry.entry);
269
- lines.push(`### Row #${entry.index}${summary ? ` — ${summary}` : ""}`);
270
- if (entry.error) {
271
- lines.push(`- **Error**: ${entry.error}`);
272
- }
273
- if (entry.traceId) {
274
- lines.push(`- **Trace ID**: \`${entry.traceId}\``);
275
- }
276
- if (evaluations.length === 0) {
277
- lines.push(
278
- evaluatorNames.length === 0
279
- ? "- _No evaluations recorded_"
280
- : "- _No evaluations recorded for this row_",
281
- );
282
- }
283
- for (const e of evaluations) {
284
- const parts: string[] = [`**${e.evaluator}**`];
285
- if (e.status === "error") parts.push("status=error");
286
- else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
287
- if (typeof e.passed === "boolean") {
288
- parts.push(`passed=${e.passed ? "yes" : "no"}`);
289
- }
290
- if (e.label) parts.push(`label=${e.label}`);
291
- lines.push(`- ${parts.join(" · ")}`);
292
- if (e.details) {
293
- lines.push(` - details: ${e.details}`);
294
- }
295
- }
296
- lines.push("");
312
+ const allRows: ResultRow[] = results.dataset.map((entry) => ({
313
+ entry,
314
+ evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? [],
315
+ }));
316
+ // The summary covers the filtered rows before truncation.
317
+ const matching = filter === "failed" ? allRows.filter((r) => isFailedRow(r)) : allRows;
318
+ const truncated = matching.length > limit;
319
+ const rows = matching.slice(0, limit);
320
+
321
+ const lines = [
322
+ ...headerLines({ results, runStatus, filter, evaluatorFilter }),
323
+ ...evaluatorSummaryLines(tallyEvaluatorStats(matching)),
324
+ ];
325
+ if (rows.length === 0) {
326
+ lines.push(noRowsLine({ filter, runStatus }));
327
+ return lines.join("\n");
297
328
  }
298
329
 
330
+ lines.push(`## Rows (${rows.length}${truncated ? ` of ${matching.length}` : ""})`);
331
+ lines.push("");
332
+ for (const row of rows) lines.push(...rowLines({ row, anyEvaluators: evaluatorNames.size > 0 }));
299
333
  if (truncated) {
300
334
  lines.push(
301
- `> Output truncated to ${limit} rows of ${totalMatching} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`,
335
+ `> Output truncated to ${limit} rows of ${matching.length} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`,
302
336
  );
303
337
  }
304
-
305
338
  return lines.join("\n");
306
339
  }
@@ -1,8 +1,6 @@
1
- import { getProject as apiGetProject } from "../langwatch-api-projects.js";
1
+ import { getProject as apiGetProject } from "../langwatch-api-projects.ts";
2
2
 
3
- export async function handleGetProject(params: {
4
- id: string;
5
- }): Promise<string> {
3
+ export async function handleGetProject(params: { id: string }): Promise<string> {
6
4
  const project = await apiGetProject(params.id);
7
5
 
8
6
  const lines: string[] = [];
@@ -1,54 +1,117 @@
1
- import { getPrompt as apiGetPrompt } from "../langwatch-api.js";
1
+ import { getPrompt as apiGetPrompt, type PromptDetailResponse } from "../langwatch-api.ts";
2
+
3
+ interface DeploymentTag {
4
+ name: string;
5
+ versionId?: string;
6
+ }
2
7
 
3
8
  /**
4
- * Handles the platform_get_prompt MCP tool invocation.
5
- *
6
- * Retrieves a specific prompt by ID or handle and formats it as
7
- * AI-readable markdown, including messages, model config, and version history.
9
+ * Normalizes one tag entry from the API. The API returns tag objects
10
+ * ({ name, versionId }); plain strings are tolerated so a shape drift can
11
+ * never crash the tool.
8
12
  */
13
+ function normalizeTag(tag: unknown): DeploymentTag | undefined {
14
+ if (typeof tag === "string") return { name: tag };
15
+ if (tag && typeof tag === "object" && "name" in tag) {
16
+ const candidate = tag as { name: unknown; versionId?: unknown };
17
+ if (typeof candidate.name === "string") {
18
+ return {
19
+ name: candidate.name,
20
+ versionId: typeof candidate.versionId === "string" ? candidate.versionId : undefined,
21
+ };
22
+ }
23
+ }
24
+ return undefined;
25
+ }
26
+
27
+ function fieldEntry(field: unknown): string {
28
+ if (field && typeof field === "object" && "identifier" in field) {
29
+ const typed = field as { identifier: unknown; type?: unknown };
30
+ return `- **${String(typed.identifier)}**: ${typeof typed.type === "string" ? typed.type : "unknown"}`;
31
+ }
32
+ return `- ${JSON.stringify(field)}`;
33
+ }
34
+
35
+ // Renders a field list of any shape the API may return: an array of
36
+ // { identifier, type } entries (inputs/outputs), an object map of
37
+ // name -> value (parameters), or absent/empty (no heading at all).
38
+ function fieldListLines({ heading, fields }: { heading: string; fields: unknown }): string[] {
39
+ const entries: string[] = [];
40
+ if (Array.isArray(fields)) {
41
+ entries.push(...fields.map(fieldEntry));
42
+ } else if (fields && typeof fields === "object") {
43
+ for (const [key, value] of Object.entries(fields)) {
44
+ entries.push(`- **${key}**: ${JSON.stringify(value)}`);
45
+ }
46
+ }
47
+ if (entries.length === 0) return [];
48
+ return [`\n**${heading}**`, ...entries];
49
+ }
50
+
51
+ function messageLines(prompt: PromptDetailResponse): string[] {
52
+ const messages = prompt.messages;
53
+ if (Array.isArray(messages) && messages.length > 0) {
54
+ return ["\n## Messages", ...messages.flatMap((msg) => [`\n### ${msg.role}`, msg.content])];
55
+ }
56
+ if (typeof prompt.prompt === "string" && prompt.prompt.length > 0) {
57
+ return ["\n## Messages", "\n### system", prompt.prompt];
58
+ }
59
+ return [];
60
+ }
61
+
62
+ // Deployments: every non-"latest" tag on the prompt, each shown with the
63
+ // version it currently points to. Always rendered, even when empty, so an
64
+ // empty section is never mistaken for "not checked" — and a tag pointing
65
+ // at another version is listed truthfully instead of implied undeployed.
66
+ function deploymentLines(prompt: PromptDetailResponse): string[] {
67
+ const lines = ["\n## Deployments"];
68
+ const tags = Array.isArray(prompt.tags) ? prompt.tags : [];
69
+ for (const rawTag of tags) {
70
+ const tag = normalizeTag(rawTag);
71
+ if (!tag || tag.name === "latest") continue;
72
+ const target = tag.versionId ?? "unknown version";
73
+ const marker = tag.versionId && tag.versionId === prompt.versionId ? " (this version)" : "";
74
+ lines.push(`- ${tag.name} → ${target}${marker}`);
75
+ }
76
+ return lines;
77
+ }
78
+
79
+ // Retrieves a prompt by ID/handle; returns a markdown digest by default or
80
+ // raw JSON if format: "json" is specified.
9
81
  export async function handleGetPrompt(params: {
10
82
  idOrHandle: string;
11
83
  version?: number;
12
84
  tag?: string;
85
+ format?: "digest" | "json";
13
86
  }): Promise<string> {
14
- const prompt = await apiGetPrompt(params.idOrHandle, { version: params.version, tag: params.tag });
87
+ const prompt = await apiGetPrompt(params.idOrHandle, {
88
+ version: params.version,
89
+ tag: params.tag,
90
+ });
91
+
92
+ if (params.format === "json") {
93
+ return JSON.stringify(prompt, null, 2);
94
+ }
15
95
 
16
96
  const lines: string[] = [];
17
- lines.push(
18
- `# Prompt: ${prompt.name || prompt.handle || prompt.id}\n`
19
- );
97
+ lines.push(`# Prompt: ${prompt.name || prompt.handle || prompt.id}\n`);
20
98
 
21
99
  if (prompt.handle) lines.push(`**Handle**: ${prompt.handle}`);
22
100
  if (prompt.id) lines.push(`**ID**: ${prompt.id}`);
23
- if (prompt.latestVersionNumber != null)
24
- lines.push(`**Latest Version**: v${prompt.latestVersionNumber}`);
25
-
26
- // Show model config
27
- const version = prompt.versions?.[0] ?? prompt;
28
- if (version.model) lines.push(`**Model**: ${version.model}`);
101
+ if (prompt.version != null) lines.push(`**Version**: v${prompt.version}`);
102
+ if (prompt.versionId) lines.push(`**Version ID**: ${prompt.versionId}`);
103
+ if (prompt.commitMessage) lines.push(`**Commit**: ${prompt.commitMessage}`);
104
+ if (prompt.model) lines.push(`**Model**: ${prompt.model}`);
105
+ if (prompt.temperature != null) lines.push(`**Temperature**: ${prompt.temperature}`);
106
+ if (prompt.maxTokens != null) lines.push(`**Max Tokens**: ${prompt.maxTokens}`);
107
+ if (prompt.responseFormat)
108
+ lines.push(`**Response Format**: ${JSON.stringify(prompt.responseFormat)}`);
29
109
 
30
- // Show messages
31
- const messages = version.messages || prompt.prompt || [];
32
- if (Array.isArray(messages) && messages.length > 0) {
33
- lines.push("\n## Messages");
34
- for (const msg of messages) {
35
- lines.push(`\n### ${msg.role}`);
36
- lines.push(msg.content);
37
- }
38
- }
39
-
40
- // Show version history
41
- if (prompt.versions && prompt.versions.length > 0) {
42
- lines.push("\n## Version History");
43
- for (const v of prompt.versions.slice(0, 10)) {
44
- const versionNum = v.version ?? "?";
45
- const commitMsg = v.commitMessage || "No message";
46
- lines.push(`- **v${versionNum}**: ${commitMsg}`);
47
- }
48
- if (prompt.versions.length > 10) {
49
- lines.push(`... and ${prompt.versions.length - 10} more versions`);
50
- }
51
- }
110
+ lines.push(...fieldListLines({ heading: "Parameters", fields: prompt.parameters }));
111
+ lines.push(...fieldListLines({ heading: "Inputs", fields: prompt.inputs }));
112
+ lines.push(...fieldListLines({ heading: "Outputs", fields: prompt.outputs }));
113
+ lines.push(...messageLines(prompt));
114
+ lines.push(...deploymentLines(prompt));
52
115
 
53
116
  return lines.join("\n");
54
117
  }