@langwatch/mcp-server 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/.release-please-shim +1 -0
  2. package/CHANGELOG.md +89 -0
  3. package/README.md +51 -8
  4. package/dist/{archive-scenario-YFD5THOR.js → archive-scenario-YKGLJ4MZ.js} +4 -4
  5. package/dist/archive-suite-QFNDO3Y3.js +15 -0
  6. package/dist/assign-prompt-tag-Y2B4CBBT.js +19 -0
  7. package/dist/chunk-B57MZ4WI.js +1573 -0
  8. package/dist/{chunk-IX6QJKAD.js → chunk-CPUPGDNA.js} +2 -2
  9. package/dist/chunk-FBRKHHJV.js +48 -0
  10. package/dist/{chunk-5UOPNRXW.js → chunk-HDJDFKZR.js} +10 -3
  11. package/dist/chunk-KOO5733S.js +72 -0
  12. package/dist/chunk-KZNBCKWG.js +39 -0
  13. package/dist/chunk-L6QENXYA.js +42 -0
  14. package/dist/{chunk-6U4TCGFC.js → chunk-NACSR6B7.js} +2 -2
  15. package/dist/chunk-NUFPTHKU.js +9 -0
  16. package/dist/chunk-OVQF3D5R.js +86 -0
  17. package/dist/chunk-PO7FM7IX.js +35 -0
  18. package/dist/chunk-PZUWCWOS.js +25 -0
  19. package/dist/chunk-WVI5CJ6S.js +21 -0
  20. package/dist/{chunk-LLRQIF52.js → chunk-YH2IBTEL.js} +54 -6
  21. package/dist/chunk-YI3OG4PI.js +71 -0
  22. package/dist/config.d.ts +17 -0
  23. package/dist/config.js +13 -0
  24. package/dist/create-agent-YRSDH7P7.js +23 -0
  25. package/dist/create-annotation-FYQNKMVJ.js +24 -0
  26. package/dist/create-dashboard-CORBR4YQ.js +18 -0
  27. package/dist/create-dataset-ZC63T6BB.js +22 -0
  28. package/dist/create-dataset-records-ILVNVKVR.js +22 -0
  29. package/dist/{create-evaluator-E5X5ZP3B.js → create-evaluator-2AU3PXFB.js} +4 -4
  30. package/dist/create-mcp-server.d.ts +8 -0
  31. package/dist/create-mcp-server.js +8 -0
  32. package/dist/{create-prompt-7Z35MIL6.js → create-prompt-NKRJ47YY.js} +7 -4
  33. package/dist/create-prompt-tag-S4JX3ULL.js +17 -0
  34. package/dist/{create-scenario-DIMPJRPY.js → create-scenario-BPRF75KF.js} +4 -4
  35. package/dist/create-suite-4YN7D6K2.js +38 -0
  36. package/dist/delete-agent-Y72R2YQA.js +18 -0
  37. package/dist/delete-annotation-UHWDMHSH.js +17 -0
  38. package/dist/delete-dashboard-LPWT36KN.js +18 -0
  39. package/dist/delete-dataset-GGPQRHKX.js +20 -0
  40. package/dist/delete-dataset-records-LTG5LT26.js +21 -0
  41. package/dist/delete-evaluator-F7QLXDHU.js +15 -0
  42. package/dist/delete-prompt-tag-B5UKZL45.js +17 -0
  43. package/dist/delete-workflow-4DGUDIDE.js +17 -0
  44. package/dist/get-agent-YEVICUX4.js +28 -0
  45. package/dist/{get-analytics-4YJW4S5L.js → get-analytics-3W5CMU3F.js} +22 -5
  46. package/dist/get-dashboard-NCQHN6MW.js +29 -0
  47. package/dist/get-dataset-KBK6PH36.js +48 -0
  48. package/dist/{get-evaluator-WDEH2F7M.js → get-evaluator-QDUL5LXR.js} +4 -4
  49. package/dist/{get-prompt-F6PDVC76.js → get-prompt-XCSSNRKR.js} +4 -4
  50. package/dist/{get-scenario-H24ZYNT5.js → get-scenario-FNRKOY7E.js} +4 -4
  51. package/dist/get-simulation-run-WOEKUF7K.js +56 -0
  52. package/dist/get-suite-U5AOD5TC.js +44 -0
  53. package/dist/{get-trace-27USKGO7.js → get-trace-MUWEJM2Z.js} +3 -3
  54. package/dist/get-workflow-SOA7MT7L.js +25 -0
  55. package/dist/http-server-7DMOCB75.js +224 -0
  56. package/dist/index.js +61640 -31868
  57. package/dist/langwatch-api-annotations-Q57NDUVR.js +17 -0
  58. package/dist/langwatch-api-dashboards-VBWWXV3Y.js +17 -0
  59. package/dist/langwatch-api-monitors-L3IY7FDT.js +52 -0
  60. package/dist/langwatch-api-secrets-S5HHF3XR.js +40 -0
  61. package/dist/langwatch-api-triggers-ME33QE26.js +30 -0
  62. package/dist/langwatch-api-workflows-4CJQAYP2.js +15 -0
  63. package/dist/list-agents-3HGIBJFM.js +32 -0
  64. package/dist/list-annotations-KWXYVEPI.js +33 -0
  65. package/dist/list-dashboards-VQ2LH666.js +32 -0
  66. package/dist/list-dataset-records-RNRGROJP.js +39 -0
  67. package/dist/list-datasets-I4A2LT2S.js +60 -0
  68. package/dist/{list-evaluators-KRGI72EH.js → list-evaluators-AKKRJ5MX.js} +4 -4
  69. package/dist/{list-model-providers-A5YCFTPI.js → list-model-providers-DVFEHFHZ.js} +4 -4
  70. package/dist/list-prompt-tags-PL5OISUA.js +23 -0
  71. package/dist/{list-prompts-LKJSE7XN.js → list-prompts-7VTFWE2Z.js} +3 -3
  72. package/dist/{list-scenarios-ZK5CMGC4.js → list-scenarios-5BZWZ6ZN.js} +4 -4
  73. package/dist/list-simulation-runs-WCZ6RLPE.js +43 -0
  74. package/dist/list-suites-RRQ2JX5Y.js +42 -0
  75. package/dist/list-workflows-UQB5FCX5.js +35 -0
  76. package/dist/rename-prompt-tag-TURD6KYU.js +18 -0
  77. package/dist/run-agent-BEJ4K4CM.js +36 -0
  78. package/dist/run-evaluation-AJ7WZNQD.js +64 -0
  79. package/dist/run-suite-U7V432G2.js +29 -0
  80. package/dist/run-workflow-5WX7DK5T.js +36 -0
  81. package/dist/{search-traces-SOKAAMAR.js → search-traces-ZCHNOMTX.js} +3 -3
  82. package/dist/{set-model-provider-7MGULZDH.js → set-model-provider-6CFHMCNE.js} +4 -4
  83. package/dist/update-agent-UE6UWLZP.js +20 -0
  84. package/dist/update-dataset-4XQZZE4S.js +22 -0
  85. package/dist/update-dataset-record-GNNM5TKM.js +19 -0
  86. package/dist/{update-evaluator-A3XINFLJ.js → update-evaluator-LHVDXIZD.js} +4 -4
  87. package/dist/{update-prompt-IW7X2UQM.js → update-prompt-3XEHECJ7.js} +5 -3
  88. package/dist/{update-scenario-ZT7TOBFR.js → update-scenario-DFPREYCD.js} +4 -4
  89. package/dist/update-suite-RATU5THO.js +39 -0
  90. package/package.json +22 -8
  91. package/src/__tests__/all-tools.integration.test.ts +353 -0
  92. package/src/__tests__/config.unit.test.ts +81 -1
  93. package/src/__tests__/datasets.integration.test.ts +620 -0
  94. package/src/__tests__/datasets.unit.test.ts +244 -0
  95. package/src/__tests__/docker.integration.test.ts +234 -0
  96. package/src/__tests__/http-transport.integration.test.ts +397 -0
  97. package/src/__tests__/langwatch-api.unit.test.ts +174 -0
  98. package/src/__tests__/scenario-tools.unit.test.ts +3 -0
  99. package/src/__tests__/tools.unit.test.ts +177 -0
  100. package/src/config.d.ts +15 -0
  101. package/src/config.ts +85 -7
  102. package/src/create-mcp-server.d.ts +6 -0
  103. package/src/create-mcp-server.ts +1799 -0
  104. package/src/http-server.ts +329 -0
  105. package/src/index.ts +22 -593
  106. package/src/langwatch-api-agents.ts +108 -0
  107. package/src/langwatch-api-annotations.ts +49 -0
  108. package/src/langwatch-api-dashboards.ts +48 -0
  109. package/src/langwatch-api-datasets.ts +191 -0
  110. package/src/langwatch-api-evaluators.ts +10 -0
  111. package/src/langwatch-api-monitors.ts +82 -0
  112. package/src/langwatch-api-secrets.ts +48 -0
  113. package/src/langwatch-api-simulation-runs.ts +51 -0
  114. package/src/langwatch-api-suites.ts +118 -0
  115. package/src/langwatch-api-triggers.ts +48 -0
  116. package/src/langwatch-api-workflows.ts +41 -0
  117. package/src/langwatch-api.ts +65 -3
  118. package/src/schemas/create-dataset.ts +48 -0
  119. package/src/schemas/index.ts +1 -0
  120. package/src/tools/archive-suite.ts +12 -0
  121. package/src/tools/assign-prompt-tag.ts +16 -0
  122. package/src/tools/create-agent.ts +15 -0
  123. package/src/tools/create-annotation.ts +18 -0
  124. package/src/tools/create-dashboard.ts +7 -0
  125. package/src/tools/create-dataset-records.ts +23 -0
  126. package/src/tools/create-dataset.ts +22 -0
  127. package/src/tools/create-prompt-tag.ts +12 -0
  128. package/src/tools/create-prompt.ts +4 -0
  129. package/src/tools/create-suite.ts +42 -0
  130. package/src/tools/delete-agent.ts +7 -0
  131. package/src/tools/delete-annotation.ts +7 -0
  132. package/src/tools/delete-dashboard.ts +7 -0
  133. package/src/tools/delete-dataset-records.ts +22 -0
  134. package/src/tools/delete-dataset.ts +20 -0
  135. package/src/tools/delete-evaluator.ts +12 -0
  136. package/src/tools/delete-prompt-tag.ts +12 -0
  137. package/src/tools/delete-workflow.ts +7 -0
  138. package/src/tools/format-dataset-mutation.ts +25 -0
  139. package/src/tools/get-agent.ts +21 -0
  140. package/src/tools/get-analytics.ts +30 -3
  141. package/src/tools/get-dashboard.ts +22 -0
  142. package/src/tools/get-dataset.ts +63 -0
  143. package/src/tools/get-prompt.ts +2 -1
  144. package/src/tools/get-simulation-run.ts +60 -0
  145. package/src/tools/get-suite.ts +47 -0
  146. package/src/tools/get-workflow.ts +17 -0
  147. package/src/tools/list-agents.ts +27 -0
  148. package/src/tools/list-annotations.ts +33 -0
  149. package/src/tools/list-dashboards.ts +27 -0
  150. package/src/tools/list-dataset-records.ts +46 -0
  151. package/src/tools/list-datasets.ts +80 -0
  152. package/src/tools/list-prompt-tags.ts +21 -0
  153. package/src/tools/list-simulation-runs.ts +49 -0
  154. package/src/tools/list-suites.ts +43 -0
  155. package/src/tools/list-workflows.ts +31 -0
  156. package/src/tools/rename-prompt-tag.ts +14 -0
  157. package/src/tools/run-agent.ts +36 -0
  158. package/src/tools/run-evaluation.ts +87 -0
  159. package/src/tools/run-suite.ts +28 -0
  160. package/src/tools/run-workflow.ts +36 -0
  161. package/src/tools/update-agent.ts +13 -0
  162. package/src/tools/update-dataset-record.ts +21 -0
  163. package/src/tools/update-dataset.ts +23 -0
  164. package/src/tools/update-prompt.ts +3 -0
  165. package/src/tools/update-suite.ts +44 -0
  166. package/src/utils/escape-markdown.ts +12 -0
  167. package/uv.lock +188 -530
  168. package/build.js +0 -12
  169. package/dist/archive-scenario-YFD5THOR.js.map +0 -1
  170. package/dist/chunk-5UOPNRXW.js.map +0 -1
  171. package/dist/chunk-6U4TCGFC.js.map +0 -1
  172. package/dist/chunk-AAQNA53E.js +0 -28
  173. package/dist/chunk-AAQNA53E.js.map +0 -1
  174. package/dist/chunk-IX6QJKAD.js.map +0 -1
  175. package/dist/chunk-LLRQIF52.js.map +0 -1
  176. package/dist/chunk-ZXKLPC2E.js.map +0 -1
  177. package/dist/config-FIQWQRUB.js +0 -11
  178. package/dist/config-FIQWQRUB.js.map +0 -1
  179. package/dist/create-evaluator-E5X5ZP3B.js.map +0 -1
  180. package/dist/create-prompt-7Z35MIL6.js.map +0 -1
  181. package/dist/create-scenario-DIMPJRPY.js.map +0 -1
  182. package/dist/discover-evaluator-schema-H23XCLNE.js.map +0 -1
  183. package/dist/discover-scenario-schema-MEEEVND7.js.map +0 -1
  184. package/dist/discover-schema-3T52ORPB.js.map +0 -1
  185. package/dist/get-analytics-4YJW4S5L.js.map +0 -1
  186. package/dist/get-evaluator-WDEH2F7M.js.map +0 -1
  187. package/dist/get-prompt-F6PDVC76.js.map +0 -1
  188. package/dist/get-scenario-H24ZYNT5.js.map +0 -1
  189. package/dist/get-trace-27USKGO7.js.map +0 -1
  190. package/dist/index.js.map +0 -1
  191. package/dist/list-evaluators-KRGI72EH.js.map +0 -1
  192. package/dist/list-model-providers-A5YCFTPI.js.map +0 -1
  193. package/dist/list-prompts-LKJSE7XN.js.map +0 -1
  194. package/dist/list-scenarios-ZK5CMGC4.js.map +0 -1
  195. package/dist/search-traces-SOKAAMAR.js.map +0 -1
  196. package/dist/set-model-provider-7MGULZDH.js.map +0 -1
  197. package/dist/update-evaluator-A3XINFLJ.js.map +0 -1
  198. package/dist/update-prompt-IW7X2UQM.js.map +0 -1
  199. package/dist/update-scenario-ZT7TOBFR.js.map +0 -1
  200. package/tests/evaluations.ipynb +0 -649
  201. package/tests/fixtures/azure/azure_openai_stream_bot_expected.py +0 -102
  202. package/tests/fixtures/azure/azure_openai_stream_bot_input.py +0 -78
  203. package/tests/fixtures/dspy/dspy_bot_expected.py +0 -61
  204. package/tests/fixtures/dspy/dspy_bot_input.py +0 -53
  205. package/tests/fixtures/fastapi/fastapi_app_expected.py +0 -68
  206. package/tests/fixtures/fastapi/fastapi_app_input.py +0 -60
  207. package/tests/fixtures/fastapi/prompt_management_fastapi_expected.py +0 -114
  208. package/tests/fixtures/fastapi/prompt_management_fastapi_input.py +0 -88
  209. package/tests/fixtures/haystack/haystack_bot_expected.py +0 -141
  210. package/tests/fixtures/haystack/haystack_bot_input.py +0 -69
  211. package/tests/fixtures/langchain/langchain_bot_expected.py +0 -53
  212. package/tests/fixtures/langchain/langchain_bot_input.py +0 -45
  213. package/tests/fixtures/langchain/langchain_bot_with_memory_expected.py +0 -69
  214. package/tests/fixtures/langchain/langchain_bot_with_memory_input.py +0 -61
  215. package/tests/fixtures/langchain/langchain_rag_bot_expected.py +0 -97
  216. package/tests/fixtures/langchain/langchain_rag_bot_input.py +0 -77
  217. package/tests/fixtures/langchain/langchain_rag_bot_vertex_ai_expected.py +0 -116
  218. package/tests/fixtures/langchain/langchain_rag_bot_vertex_ai_input.py +0 -81
  219. package/tests/fixtures/langchain/langgraph_rag_bot_with_threads_expected.py +0 -331
  220. package/tests/fixtures/langchain/langgraph_rag_bot_with_threads_input.py +0 -106
  221. package/tests/fixtures/litellm/litellm_bot_expected.py +0 -40
  222. package/tests/fixtures/litellm/litellm_bot_input.py +0 -35
  223. package/tests/fixtures/openai/openai_bot_expected.py +0 -43
  224. package/tests/fixtures/openai/openai_bot_function_call_expected.py +0 -91
  225. package/tests/fixtures/openai/openai_bot_function_call_input.py +0 -82
  226. package/tests/fixtures/openai/openai_bot_input.py +0 -36
  227. package/tests/fixtures/openai/openai_bot_rag_expected.py +0 -73
  228. package/tests/fixtures/openai/openai_bot_rag_input.py +0 -51
  229. package/tests/fixtures/opentelemetry/openinference_dspy_bot_expected.py +0 -63
  230. package/tests/fixtures/opentelemetry/openinference_dspy_bot_input.py +0 -58
  231. package/tests/fixtures/opentelemetry/openinference_langchain_bot_expected.py +0 -53
  232. package/tests/fixtures/opentelemetry/openinference_langchain_bot_input.py +0 -52
  233. package/tests/fixtures/opentelemetry/openinference_openai_bot_expected.py +0 -49
  234. package/tests/fixtures/opentelemetry/openinference_openai_bot_input.py +0 -41
  235. package/tests/fixtures/opentelemetry/openllmetry_openai_bot_expected.py +0 -44
  236. package/tests/fixtures/opentelemetry/openllmetry_openai_bot_input.py +0 -40
  237. package/tests/fixtures/strands/strands_bot_expected.py +0 -84
  238. package/tests/fixtures/strands/strands_bot_input.py +0 -52
  239. package/tests/scenario-openai.test.ts +0 -160
  240. package/tsup.config.ts +0 -10
  241. package/vitest.config.js +0 -7
@@ -278,6 +278,50 @@ const CANNED_MODEL_PROVIDERS_LIST = {
278
278
  },
279
279
  };
280
280
 
281
+ const CANNED_AGENTS_LIST = {
282
+ data: [
283
+ { id: "agent_abc", name: "Test Agent", type: "http", config: { url: "http://example.com" }, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z" },
284
+ ],
285
+ pagination: { page: 1, limit: 50, total: 1, totalPages: 1 },
286
+ };
287
+
288
+ const CANNED_AGENT_DETAIL = {
289
+ id: "agent_abc", name: "Test Agent", type: "http", config: { url: "http://example.com" }, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z",
290
+ };
291
+
292
+ const CANNED_SUITES_LIST = [
293
+ { id: "suite_abc", name: "Regression Suite", slug: "regression-suite", description: null, scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 1, labels: [], createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z" },
294
+ ];
295
+
296
+ const CANNED_SUITE_DETAIL = {
297
+ id: "suite_abc", name: "Regression Suite", slug: "regression-suite", description: "A test suite", scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 1, labels: [], createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z",
298
+ };
299
+
300
+ const CANNED_SUITE_CREATED = {
301
+ id: "suite_new", name: "New Suite", slug: "new-suite", description: null, scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 1, labels: [], createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z",
302
+ };
303
+
304
+ const CANNED_SUITE_RUN = {
305
+ scheduled: true, batchRunId: "batch_123", setId: "set_456", jobCount: 1, skippedArchived: { scenarios: [], targets: [] }, items: [{ scenarioRunId: "run_1", scenarioId: "scen_abc123", target: { type: "http", referenceId: "agent_abc" }, name: "Test" }],
306
+ };
307
+
308
+ const CANNED_SIMULATION_RUNS = {
309
+ runs: [{ scenarioRunId: "run_abc", scenarioId: "scen_abc123", batchRunId: "batch_xyz", name: "Login Flow", status: "SUCCESS", durationInMs: 5200, totalCost: 0.0042, timestamp: 1700000000000, updatedAt: 1700000001000 }],
310
+ hasMore: false,
311
+ };
312
+
313
+ const CANNED_SIMULATION_RUN_DETAIL = {
314
+ scenarioRunId: "run_abc", scenarioId: "scen_abc123", batchRunId: "batch_xyz", name: "Login Flow", status: "SUCCESS", durationInMs: 5200, totalCost: 0.0042, results: { verdict: "passed", reasoning: "All criteria met", metCriteria: ["Greets user"], unmetCriteria: [], error: null }, messages: [{ role: "user", content: "Hello" }, { role: "assistant", content: "Hi there!" }], timestamp: 1700000000000, updatedAt: 1700000001000,
315
+ };
316
+
317
+ const CANNED_DASHBOARDS_LIST = {
318
+ data: [{ id: "dash_abc", name: "Main Dashboard", order: 0, graphCount: 3, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z" }],
319
+ };
320
+
321
+ const CANNED_WORKFLOWS_LIST = [
322
+ { id: "wf_abc", name: "Test Workflow", icon: null, description: "A workflow", isEvaluator: false, isComponent: false, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z" },
323
+ ];
324
+
281
325
  const CANNED_MODEL_PROVIDER_SET = {
282
326
  openai: {
283
327
  provider: "openai",
@@ -436,6 +480,124 @@ function createMockServer(): Server {
436
480
  res.writeHead(200);
437
481
  res.end(JSON.stringify(CANNED_MODEL_PROVIDER_SET));
438
482
  }
483
+ // --- Agent endpoints ---
484
+ else if (url === "/api/agents" && method === "GET") {
485
+ res.writeHead(200);
486
+ res.end(JSON.stringify(CANNED_AGENTS_LIST));
487
+ } else if (url === "/api/agents" && method === "POST") {
488
+ res.writeHead(201);
489
+ res.end(JSON.stringify(CANNED_AGENT_DETAIL));
490
+ } else if (url?.match(/^\/api\/agents\/[^/]+$/) && method === "GET") {
491
+ res.writeHead(200);
492
+ res.end(JSON.stringify(CANNED_AGENT_DETAIL));
493
+ } else if (url?.match(/^\/api\/agents\/[^/]+$/) && method === "PATCH") {
494
+ res.writeHead(200);
495
+ res.end(JSON.stringify(CANNED_AGENT_DETAIL));
496
+ } else if (url?.match(/^\/api\/agents\/[^/]+$/) && method === "DELETE") {
497
+ res.writeHead(200);
498
+ res.end(JSON.stringify({ id: "agent_abc", name: "Test Agent" }));
499
+ }
500
+ // --- Suite endpoints ---
501
+ else if (url === "/api/suites" && method === "GET") {
502
+ res.writeHead(200);
503
+ res.end(JSON.stringify(CANNED_SUITES_LIST));
504
+ } else if (url === "/api/suites" && method === "POST") {
505
+ res.writeHead(201);
506
+ res.end(JSON.stringify(CANNED_SUITE_CREATED));
507
+ } else if (url?.match(/^\/api\/suites\/[^/]+\/run$/) && method === "POST") {
508
+ res.writeHead(200);
509
+ res.end(JSON.stringify(CANNED_SUITE_RUN));
510
+ } else if (url?.match(/^\/api\/suites\/[^/]+\/duplicate$/) && method === "POST") {
511
+ res.writeHead(201);
512
+ res.end(JSON.stringify(CANNED_SUITE_CREATED));
513
+ } else if (url?.match(/^\/api\/suites\/[^/]+$/) && method === "GET") {
514
+ res.writeHead(200);
515
+ res.end(JSON.stringify(CANNED_SUITE_DETAIL));
516
+ } else if (url?.match(/^\/api\/suites\/[^/]+$/) && method === "PATCH") {
517
+ res.writeHead(200);
518
+ res.end(JSON.stringify(CANNED_SUITE_DETAIL));
519
+ } else if (url?.match(/^\/api\/suites\/[^/]+$/) && method === "DELETE") {
520
+ res.writeHead(200);
521
+ res.end(JSON.stringify({ id: "suite_abc", archived: true }));
522
+ }
523
+ // --- Simulation Run endpoints ---
524
+ else if (url?.match(/^\/api\/simulation-runs\/[^/]+$/) && method === "GET") {
525
+ res.writeHead(200);
526
+ res.end(JSON.stringify(CANNED_SIMULATION_RUN_DETAIL));
527
+ } else if (url?.match(/^\/api\/simulation-runs/) && method === "GET") {
528
+ res.writeHead(200);
529
+ res.end(JSON.stringify(CANNED_SIMULATION_RUNS));
530
+ }
531
+ // --- Dashboard endpoints ---
532
+ else if (url === "/api/dashboards" && method === "GET") {
533
+ res.writeHead(200);
534
+ res.end(JSON.stringify(CANNED_DASHBOARDS_LIST));
535
+ } else if (url === "/api/dashboards" && method === "POST") {
536
+ res.writeHead(201);
537
+ res.end(JSON.stringify({ id: "dash_new", name: "New Dashboard" }));
538
+ } else if (url?.match(/^\/api\/dashboards\/[^/]+$/) && method === "GET") {
539
+ res.writeHead(200);
540
+ res.end(JSON.stringify(CANNED_DASHBOARDS_LIST.data[0]));
541
+ } else if (url?.match(/^\/api\/dashboards\/[^/]+$/) && method === "DELETE") {
542
+ res.writeHead(200);
543
+ res.end(JSON.stringify({ id: "dash_abc", name: "Main Dashboard" }));
544
+ }
545
+ // --- Workflow endpoints ---
546
+ else if (url === "/api/workflows" && method === "GET") {
547
+ res.writeHead(200);
548
+ res.end(JSON.stringify(CANNED_WORKFLOWS_LIST));
549
+ } else if (url?.match(/^\/api\/workflows\/[^/]+\/run$/) && method === "POST") {
550
+ res.writeHead(200);
551
+ res.end(JSON.stringify({ output: "workflow result" }));
552
+ } else if (url?.match(/^\/api\/workflows\/[^/]+$/) && method === "GET") {
553
+ res.writeHead(200);
554
+ res.end(JSON.stringify(CANNED_WORKFLOWS_LIST[0]));
555
+ } else if (url?.match(/^\/api\/workflows\/[^/]+$/) && method === "DELETE") {
556
+ res.writeHead(200);
557
+ res.end(JSON.stringify({ id: "wf_abc", archived: true }));
558
+ }
559
+ // --- Monitor endpoints ---
560
+ else if (url === "/api/monitors" && method === "GET") {
561
+ res.writeHead(200);
562
+ res.end(JSON.stringify([
563
+ { id: "mon_abc", name: "Toxicity Check", slug: "toxicity-check-x1y2z", checkType: "ragas/toxicity", enabled: true, executionMode: "ON_MESSAGE", sample: 1.0, level: "trace", evaluatorId: null, preconditions: [], parameters: {}, mappings: {}, threadIdleTimeout: null, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z" },
564
+ ]));
565
+ } else if (url === "/api/monitors" && method === "POST") {
566
+ res.writeHead(201);
567
+ res.end(JSON.stringify({ id: "mon_new", name: "New Monitor", slug: "new-monitor-abc12", checkType: "ragas/toxicity", enabled: true, executionMode: "ON_MESSAGE", sample: 1.0, level: "trace", evaluatorId: null, preconditions: [], parameters: {}, mappings: {}, threadIdleTimeout: null, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z" }));
568
+ } else if (url?.match(/^\/api\/monitors\/[^/]+\/toggle$/) && method === "POST") {
569
+ const monitorId = url?.split("/")[3];
570
+ res.writeHead(200);
571
+ res.end(JSON.stringify({ id: monitorId, enabled: true }));
572
+ } else if (url?.match(/^\/api\/monitors\/[^/]+$/) && method === "GET") {
573
+ res.writeHead(200);
574
+ res.end(JSON.stringify({ id: "mon_abc", name: "Toxicity Check", slug: "toxicity-check-x1y2z", checkType: "ragas/toxicity", enabled: true, executionMode: "ON_MESSAGE", sample: 1.0, level: "trace", evaluatorId: null, preconditions: [], parameters: {}, mappings: {}, threadIdleTimeout: null, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z" }));
575
+ } else if (url?.match(/^\/api\/monitors\/[^/]+$/) && method === "PATCH") {
576
+ res.writeHead(200);
577
+ res.end(JSON.stringify({ id: "mon_abc", name: "Updated Monitor", slug: "toxicity-check-x1y2z", checkType: "ragas/toxicity", enabled: false, executionMode: "ON_MESSAGE", sample: 0.5, level: "trace", evaluatorId: null, preconditions: [], parameters: {}, mappings: {}, threadIdleTimeout: null, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-02T00:00:00Z" }));
578
+ } else if (url?.match(/^\/api\/monitors\/[^/]+$/) && method === "DELETE") {
579
+ const monitorId = url?.split("/").pop();
580
+ res.writeHead(200);
581
+ res.end(JSON.stringify({ id: monitorId, deleted: true }));
582
+ }
583
+ // --- Secret endpoints ---
584
+ else if (url === "/api/secrets" && method === "GET") {
585
+ res.writeHead(200);
586
+ res.end(JSON.stringify([
587
+ { id: "secret_abc", projectId: "proj_123", name: "MY_API_KEY", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z" },
588
+ { id: "secret_def", projectId: "proj_123", name: "DB_PASSWORD", createdAt: "2026-01-02T00:00:00Z", updatedAt: "2026-01-02T00:00:00Z" },
589
+ ]));
590
+ } else if (url === "/api/secrets" && method === "POST") {
591
+ res.writeHead(201);
592
+ res.end(JSON.stringify({ id: "secret_new", projectId: "proj_123", name: "NEW_SECRET", createdAt: "2026-01-03T00:00:00Z", updatedAt: "2026-01-03T00:00:00Z" }));
593
+ } else if (url?.match(/^\/api\/secrets\/[^/]+$/) && method === "PUT") {
594
+ res.writeHead(200);
595
+ res.end(JSON.stringify({ id: "secret_abc", projectId: "proj_123", name: "MY_API_KEY", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-03T00:00:00Z" }));
596
+ } else if (url?.match(/^\/api\/secrets\/[^/]+$/) && method === "DELETE") {
597
+ const secretId = url?.split("/").pop();
598
+ res.writeHead(200);
599
+ res.end(JSON.stringify({ id: secretId, deleted: true }));
600
+ }
439
601
  // --- Fallback ---
440
602
  else {
441
603
  res.writeHead(404);
@@ -619,6 +781,7 @@ describe("All MCP tools integration", () => {
619
781
  // =====================
620
782
  describe("discover_schema", () => {
621
783
  describe("when category is filters", () => {
784
+ /** @scenario Agent discovers available filter fields */
622
785
  it("returns filter field documentation", async () => {
623
786
  const { formatSchema } = await import(
624
787
  "../tools/discover-schema.js"
@@ -631,6 +794,7 @@ describe("All MCP tools integration", () => {
631
794
  });
632
795
 
633
796
  describe("when category is metrics", () => {
797
+ /** @scenario Agent discovers available metrics with allowed aggregations */
634
798
  it("returns metric documentation", async () => {
635
799
  const { formatSchema } = await import(
636
800
  "../tools/discover-schema.js"
@@ -656,6 +820,7 @@ describe("All MCP tools integration", () => {
656
820
  });
657
821
 
658
822
  describe("when category is groups", () => {
823
+ /** @scenario Agent discovers available group-by options */
659
824
  it("returns group-by options", async () => {
660
825
  const { formatSchema } = await import(
661
826
  "../tools/discover-schema.js"
@@ -714,6 +879,7 @@ describe("All MCP tools integration", () => {
714
879
  });
715
880
 
716
881
  describe("when category is all", () => {
882
+ /** @scenario Agent discovers all schema information at once */
717
883
  it("returns all schema categories", async () => {
718
884
  const { formatSchema } = await import(
719
885
  "../tools/discover-schema.js"
@@ -733,6 +899,7 @@ describe("All MCP tools integration", () => {
733
899
  // =====================
734
900
  describe("search_traces", () => {
735
901
  describe("when traces are found", () => {
902
+ /** @scenario Agent searches traces with a text query */
736
903
  it("returns formatted trace digests", async () => {
737
904
  const { handleSearchTraces } = await import(
738
905
  "../tools/search-traces.js"
@@ -762,6 +929,7 @@ describe("All MCP tools integration", () => {
762
929
  });
763
930
 
764
931
  describe("when pagination token is present", () => {
932
+ /** @scenario Agent paginates through trace results */
765
933
  it("includes scroll ID for next page", async () => {
766
934
  const { handleSearchTraces } = await import(
767
935
  "../tools/search-traces.js"
@@ -791,6 +959,7 @@ describe("All MCP tools integration", () => {
791
959
  });
792
960
 
793
961
  describe("when filters are applied", () => {
962
+ /** @scenario Agent searches traces filtered by user_id */
794
963
  it("passes filters to the API", async () => {
795
964
  const { handleSearchTraces } = await import(
796
965
  "../tools/search-traces.js"
@@ -814,6 +983,7 @@ describe("All MCP tools integration", () => {
814
983
  // =====================
815
984
  describe("get_trace", () => {
816
985
  describe("when trace exists", () => {
986
+ /** @scenario Agent gets a single trace by ID in AI-readable format */
817
987
  it("returns formatted trace with metadata and evaluations", async () => {
818
988
  const { handleGetTrace } = await import(
819
989
  "../tools/get-trace.js"
@@ -831,6 +1001,7 @@ describe("All MCP tools integration", () => {
831
1001
  });
832
1002
 
833
1003
  describe("when trace does not exist", () => {
1004
+ /** @scenario Agent gets a trace that does not exist */
834
1005
  it("propagates the 404 error", async () => {
835
1006
  const { handleGetTrace } = await import(
836
1007
  "../tools/get-trace.js"
@@ -1334,4 +1505,186 @@ describe("All MCP tools integration", () => {
1334
1505
  await expect(handleListModelProviders()).rejects.toThrow("401");
1335
1506
  });
1336
1507
  });
1508
+
1509
+ // =====================
1510
+ // Suite Tools
1511
+ // =====================
1512
+ describe("platform_list_suites", () => {
1513
+ it("returns formatted suite list", async () => {
1514
+ const { handleListSuites } = await import("../tools/list-suites.js");
1515
+ const result = await handleListSuites({});
1516
+
1517
+ expect(result).toContain("Suites / Run Plans (1 total)");
1518
+ expect(result).toContain("Regression Suite");
1519
+ });
1520
+ });
1521
+
1522
+ describe("platform_get_suite", () => {
1523
+ it("returns suite details", async () => {
1524
+ const { handleGetSuite } = await import("../tools/get-suite.js");
1525
+ const result = await handleGetSuite({ id: "suite_abc" });
1526
+
1527
+ expect(result).toContain("Regression Suite");
1528
+ expect(result).toContain("suite_abc");
1529
+ });
1530
+ });
1531
+
1532
+ describe("platform_create_suite", () => {
1533
+ it("creates a suite and returns confirmation", async () => {
1534
+ const { handleCreateSuite } = await import("../tools/create-suite.js");
1535
+ const result = await handleCreateSuite({
1536
+ name: "New Suite",
1537
+ scenarioIds: ["scen_abc123"],
1538
+ targets: JSON.stringify([{ type: "http", referenceId: "agent_abc" }]),
1539
+ });
1540
+
1541
+ expect(result).toContain("created successfully");
1542
+ expect(result).toContain("New Suite");
1543
+ });
1544
+ });
1545
+
1546
+ describe("platform_run_suite", () => {
1547
+ it("triggers a suite run and returns batch info", async () => {
1548
+ const { handleRunSuite } = await import("../tools/run-suite.js");
1549
+ const result = await handleRunSuite({ id: "suite_abc" });
1550
+
1551
+ expect(result).toContain("scheduled successfully");
1552
+ expect(result).toContain("batch_123");
1553
+ });
1554
+ });
1555
+
1556
+ describe("platform_archive_suite", () => {
1557
+ it("archives the suite", async () => {
1558
+ const { handleArchiveSuite } = await import("../tools/archive-suite.js");
1559
+ const result = await handleArchiveSuite({ id: "suite_abc" });
1560
+
1561
+ expect(result).toContain("archived");
1562
+ });
1563
+ });
1564
+
1565
+ // =====================
1566
+ // Simulation Run Tools
1567
+ // =====================
1568
+ describe("platform_list_simulation_runs", () => {
1569
+ it("returns formatted run list", async () => {
1570
+ const { handleListSimulationRuns } = await import("../tools/list-simulation-runs.js");
1571
+ const result = await handleListSimulationRuns({});
1572
+
1573
+ expect(result).toContain("Simulation Runs");
1574
+ expect(result).toContain("Login Flow");
1575
+ });
1576
+ });
1577
+
1578
+ describe("platform_get_simulation_run", () => {
1579
+ it("returns run details with conversation", async () => {
1580
+ const { handleGetSimulationRun } = await import("../tools/get-simulation-run.js");
1581
+ const result = await handleGetSimulationRun({ scenarioRunId: "run_abc" });
1582
+
1583
+ expect(result).toContain("Login Flow");
1584
+ expect(result).toContain("passed");
1585
+ expect(result).toContain("Hello");
1586
+ });
1587
+ });
1588
+
1589
+ // =====================
1590
+ // Monitor Tools
1591
+ // =====================
1592
+ describe("platform_list_monitors", () => {
1593
+ it("returns formatted monitor list", async () => {
1594
+ const { listMonitors } = await import("../langwatch-api-monitors.js");
1595
+ const monitors = await listMonitors();
1596
+ expect(monitors).toHaveLength(1);
1597
+ expect(monitors[0]!.name).toBe("Toxicity Check");
1598
+ expect(monitors[0]!.checkType).toBe("ragas/toxicity");
1599
+ });
1600
+ });
1601
+
1602
+ describe("platform_get_monitor", () => {
1603
+ it("returns monitor details", async () => {
1604
+ const { getMonitor } = await import("../langwatch-api-monitors.js");
1605
+ const monitor = await getMonitor("mon_abc");
1606
+ expect(monitor.id).toBe("mon_abc");
1607
+ expect(monitor.name).toBe("Toxicity Check");
1608
+ expect(monitor.enabled).toBe(true);
1609
+ });
1610
+ });
1611
+
1612
+ describe("platform_create_monitor", () => {
1613
+ it("creates a monitor and returns metadata", async () => {
1614
+ const { createMonitor } = await import("../langwatch-api-monitors.js");
1615
+ const monitor = await createMonitor({ name: "New Monitor", checkType: "ragas/toxicity" });
1616
+ expect(monitor.id).toBe("mon_new");
1617
+ expect(monitor.name).toBe("New Monitor");
1618
+ });
1619
+ });
1620
+
1621
+ describe("platform_update_monitor", () => {
1622
+ it("updates a monitor", async () => {
1623
+ const { updateMonitor } = await import("../langwatch-api-monitors.js");
1624
+ const monitor = await updateMonitor({ id: "mon_abc", enabled: false });
1625
+ expect(monitor.id).toBe("mon_abc");
1626
+ expect(monitor.enabled).toBe(false);
1627
+ });
1628
+ });
1629
+
1630
+ describe("platform_delete_monitor", () => {
1631
+ it("deletes a monitor", async () => {
1632
+ const { deleteMonitor } = await import("../langwatch-api-monitors.js");
1633
+ const result = await deleteMonitor("mon_abc");
1634
+ expect(result.id).toBe("mon_abc");
1635
+ expect(result.deleted).toBe(true);
1636
+ });
1637
+ });
1638
+
1639
+ // =====================
1640
+ // Secret Tools
1641
+ // =====================
1642
+ describe("platform_list_secrets", () => {
1643
+ it("returns formatted secret list", async () => {
1644
+ const { listSecrets } = await import("../langwatch-api-secrets.js");
1645
+ const secrets = await listSecrets();
1646
+ expect(secrets).toHaveLength(2);
1647
+ expect(secrets[0]!.name).toBe("MY_API_KEY");
1648
+ });
1649
+ });
1650
+
1651
+ describe("platform_create_secret", () => {
1652
+ it("creates a secret and returns metadata", async () => {
1653
+ const { createSecret } = await import("../langwatch-api-secrets.js");
1654
+ const secret = await createSecret({ name: "NEW_SECRET", value: "sk-123" });
1655
+ expect(secret.id).toBe("secret_new");
1656
+ expect(secret.name).toBe("NEW_SECRET");
1657
+ });
1658
+ });
1659
+
1660
+ describe("platform_update_secret", () => {
1661
+ it("updates a secret value", async () => {
1662
+ const { updateSecret } = await import("../langwatch-api-secrets.js");
1663
+ const secret = await updateSecret({ id: "secret_abc", value: "new-val" });
1664
+ expect(secret.id).toBe("secret_abc");
1665
+ expect(secret.name).toBe("MY_API_KEY");
1666
+ });
1667
+ });
1668
+
1669
+ describe("platform_delete_secret", () => {
1670
+ it("deletes a secret", async () => {
1671
+ const { deleteSecret } = await import("../langwatch-api-secrets.js");
1672
+ const result = await deleteSecret("secret_abc");
1673
+ expect(result.id).toBe("secret_abc");
1674
+ expect(result.deleted).toBe(true);
1675
+ });
1676
+ });
1677
+
1678
+ // =====================
1679
+ // Agent Run Tool
1680
+ // =====================
1681
+ describe("platform_run_workflow", () => {
1682
+ it("executes a workflow and returns result", async () => {
1683
+ const { handleRunWorkflow } = await import("../tools/run-workflow.js");
1684
+ const result = await handleRunWorkflow({ id: "wf_abc" });
1685
+
1686
+ expect(result).toContain("executed successfully");
1687
+ expect(result).toContain("workflow result");
1688
+ });
1689
+ });
1337
1690
  });
@@ -1,5 +1,10 @@
1
1
  import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
2
- import { initConfig, getConfig, requireApiKey } from "../config.js";
2
+ import {
3
+ initConfig,
4
+ getConfig,
5
+ requireApiKey,
6
+ runWithConfig,
7
+ } from "../config.js";
3
8
 
4
9
  describe("config", () => {
5
10
  let originalApiKey: string | undefined;
@@ -81,9 +86,84 @@ describe("config", () => {
81
86
 
82
87
  describe("getConfig", () => {
83
88
  it("throws when config has not been initialized", async () => {
89
+ // globalThis survives vi.resetModules(), so clear it explicitly
90
+ delete (globalThis as Record<string, unknown>).__langwatch_mcp_config;
91
+ delete (globalThis as Record<string, unknown>).__langwatch_mcp_config_storage;
84
92
  vi.resetModules();
85
93
  const freshConfig = await import("../config.js");
86
94
  expect(() => freshConfig.getConfig()).toThrow("Config not initialized");
87
95
  });
88
96
  });
97
+
98
+ describe("runWithConfig()", () => {
99
+ it("overrides global config within the callback", () => {
100
+ initConfig({ apiKey: "global-key" });
101
+
102
+ runWithConfig(
103
+ { apiKey: "session-key", endpoint: "https://session.example.com" },
104
+ () => {
105
+ expect(getConfig().apiKey).toBe("session-key");
106
+ expect(getConfig().endpoint).toBe("https://session.example.com");
107
+ }
108
+ );
109
+ });
110
+
111
+ it("restores global config after the callback completes", () => {
112
+ initConfig({ apiKey: "global-key" });
113
+
114
+ runWithConfig(
115
+ { apiKey: "session-key", endpoint: "https://session.example.com" },
116
+ () => {
117
+ // inside: session config
118
+ }
119
+ );
120
+
121
+ expect(getConfig().apiKey).toBe("global-key");
122
+ });
123
+
124
+ it("isolates concurrent async contexts", async () => {
125
+ initConfig({ apiKey: "global-key" });
126
+
127
+ const results: string[] = [];
128
+
129
+ await Promise.all([
130
+ runWithConfig(
131
+ { apiKey: "key-a", endpoint: "https://a.example.com" },
132
+ async () => {
133
+ await new Promise((r) => setTimeout(r, 10));
134
+ results.push(requireApiKey());
135
+ }
136
+ ),
137
+ runWithConfig(
138
+ { apiKey: "key-b", endpoint: "https://b.example.com" },
139
+ async () => {
140
+ await new Promise((r) => setTimeout(r, 5));
141
+ results.push(requireApiKey());
142
+ }
143
+ ),
144
+ ]);
145
+
146
+ expect(results).toContain("key-a");
147
+ expect(results).toContain("key-b");
148
+ });
149
+
150
+ it("returns the callback result", () => {
151
+ initConfig({});
152
+ const result = runWithConfig(
153
+ { apiKey: "key", endpoint: "https://example.com" },
154
+ () => 42
155
+ );
156
+ expect(result).toBe(42);
157
+ });
158
+
159
+ it("makes requireApiKey() use the scoped key", () => {
160
+ initConfig({});
161
+ runWithConfig(
162
+ { apiKey: "scoped-key", endpoint: "https://example.com" },
163
+ () => {
164
+ expect(requireApiKey()).toBe("scoped-key");
165
+ }
166
+ );
167
+ });
168
+ });
89
169
  });