@langwatch/mcp-server 1.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/.python-version +1 -0
  2. package/CHANGELOG.md +110 -0
  3. package/CONTRIBUTING.md +1 -1
  4. package/README.md +86 -4
  5. package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
  6. package/dist/archive-run-plan-474OKHHU.js +16 -0
  7. package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
  8. package/dist/archive-test-suite-O7FLDMXK.js +16 -0
  9. package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
  10. package/dist/chunk-56OGTS24.js +44 -0
  11. package/dist/chunk-6DJDEXET.js +63 -0
  12. package/dist/chunk-7E4FZIYJ.js +15 -0
  13. package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
  14. package/dist/chunk-CKNAH5TG.js +74 -0
  15. package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
  16. package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
  17. package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
  18. package/dist/chunk-HV4DWKBI.js +46 -0
  19. package/dist/chunk-JR6DJZOU.js +49 -0
  20. package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
  21. package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
  22. package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
  23. package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
  24. package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
  25. package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
  26. package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
  27. package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
  28. package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
  29. package/dist/chunk-UF6XIGYR.js +58 -0
  30. package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
  31. package/dist/chunk-UJPXFA26.js +267 -0
  32. package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
  33. package/dist/config.js +2 -2
  34. package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
  35. package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
  36. package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
  37. package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
  38. package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
  39. package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
  40. package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
  41. package/dist/create-mcp-server.js +9 -4
  42. package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
  43. package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
  44. package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
  45. package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
  46. package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
  47. package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
  48. package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
  49. package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
  50. package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
  51. package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
  52. package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
  53. package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
  54. package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
  55. package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
  56. package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
  57. package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
  58. package/dist/get-agent-7THF7HCU.js +66 -0
  59. package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
  60. package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
  61. package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
  62. package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
  63. package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
  64. package/dist/get-prompt-GVKXF5NL.js +100 -0
  65. package/dist/get-run-plan-NY7J5GRE.js +61 -0
  66. package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
  67. package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
  68. package/dist/get-test-suite-4HKA7OKS.js +49 -0
  69. package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
  70. package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
  71. package/dist/http-security-EU3CHTY7.js +26 -0
  72. package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
  73. package/dist/index.js +6437 -4020
  74. package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
  75. package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
  76. package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
  77. package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
  78. package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
  79. package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
  80. package/dist/list-agents-AMVDQPQL.js +43 -0
  81. package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
  82. package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
  83. package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
  84. package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
  85. package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
  86. package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
  87. package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
  88. package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
  89. package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
  90. package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
  91. package/dist/list-run-plans-UBAU2DGH.js +46 -0
  92. package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
  93. package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
  94. package/dist/list-test-suites-L7R2O6DR.js +35 -0
  95. package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
  96. package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
  97. package/dist/rename-test-suite-NTYHAO5X.js +21 -0
  98. package/dist/report-issue-RHZZ33IS.js +885 -0
  99. package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
  100. package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
  101. package/dist/run-agent-CAGWGTYT.js +72 -0
  102. package/dist/run-plan-T4ZYOZGA.js +39 -0
  103. package/dist/run-test-suite-L4MZ7G4D.js +26 -0
  104. package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
  105. package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
  106. package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
  107. package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
  108. package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
  109. package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
  110. package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
  111. package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
  112. package/dist/update-prompt-ZQQ2C3KU.js +143 -0
  113. package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
  114. package/package.json +8 -9
  115. package/pyproject.toml +5 -5
  116. package/src/__tests__/agent-tools.unit.test.ts +257 -0
  117. package/src/__tests__/all-tools.integration.test.ts +321 -75
  118. package/src/__tests__/config.unit.test.ts +33 -0
  119. package/src/__tests__/docker.integration.test.ts +91 -7
  120. package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
  121. package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
  122. package/src/__tests__/http-security-origin.unit.test.ts +97 -0
  123. package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
  124. package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
  125. package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
  126. package/src/__tests__/http-server-origin.integration.test.ts +169 -0
  127. package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
  128. package/src/__tests__/http-transport.integration.test.ts +85 -87
  129. package/src/__tests__/integration.integration.test.ts +7 -11
  130. package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
  131. package/src/__tests__/prompts.integration.test.ts +736 -0
  132. package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
  133. package/src/__tests__/scenario-tools.unit.test.ts +19 -0
  134. package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
  135. package/src/__tests__/support/http-server-harness.ts +138 -0
  136. package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
  137. package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
  138. package/src/config.ts +17 -2
  139. package/src/create-mcp-server.ts +334 -55
  140. package/src/http-security.ts +492 -0
  141. package/src/http-server.ts +536 -152
  142. package/src/index.ts +49 -2
  143. package/src/langwatch-api-agents.ts +196 -12
  144. package/src/langwatch-api-run-plans.ts +157 -0
  145. package/src/langwatch-api-scenarios.ts +9 -0
  146. package/src/langwatch-api-simulation-runs.ts +5 -0
  147. package/src/langwatch-api-test-suites.ts +135 -0
  148. package/src/langwatch-api.ts +62 -12
  149. package/src/schemas/index.ts +5 -0
  150. package/src/schemas/run-plan.ts +92 -0
  151. package/src/schemas/suite-fields.ts +149 -0
  152. package/src/tools/archive-run-plan.ts +12 -0
  153. package/src/tools/archive-test-suite.ts +12 -0
  154. package/src/tools/create-scenario.ts +8 -0
  155. package/src/tools/create-test-suite.ts +43 -0
  156. package/src/tools/discover-evaluator-schema.ts +2 -2
  157. package/src/tools/discover-scenario-schema.ts +54 -0
  158. package/src/tools/format-run-plan.ts +68 -0
  159. package/src/tools/format-scenario.ts +14 -0
  160. package/src/tools/format-suite-details.ts +96 -0
  161. package/src/tools/get-agent.ts +45 -3
  162. package/src/tools/get-prompt.ts +108 -24
  163. package/src/tools/get-run-plan.ts +62 -0
  164. package/src/tools/get-scenario.ts +3 -0
  165. package/src/tools/get-simulation-run.ts +9 -0
  166. package/src/tools/get-test-suite.ts +49 -0
  167. package/src/tools/list-agents.ts +20 -3
  168. package/src/tools/list-run-plans.ts +47 -0
  169. package/src/tools/list-scenarios.ts +12 -2
  170. package/src/tools/list-simulation-runs.ts +6 -2
  171. package/src/tools/list-test-suites.ts +35 -0
  172. package/src/tools/rename-test-suite.ts +18 -0
  173. package/src/tools/rerun-run-plan.ts +21 -0
  174. package/src/tools/run-agent.ts +54 -6
  175. package/src/tools/run-plan.ts +49 -0
  176. package/src/tools/run-test-suite.ts +30 -0
  177. package/src/tools/test-agent.ts +18 -0
  178. package/src/tools/update-prompt.ts +195 -10
  179. package/src/tools/update-scenario.ts +8 -0
  180. package/src/tools/update-test-suite.ts +45 -0
  181. package/tsconfig.json +7 -2
  182. package/uv.lock +418 -298
  183. package/dist/archive-suite-C3KX6IIZ.js +0 -16
  184. package/dist/chunk-WDH5P5RF.js +0 -48
  185. package/dist/create-suite-TWF6BBR4.js +0 -39
  186. package/dist/get-agent-7QJVPH72.js +0 -29
  187. package/dist/get-prompt-I753LZKZ.js +0 -46
  188. package/dist/get-suite-GTOCY5PL.js +0 -45
  189. package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
  190. package/dist/list-agents-TIJTP566.js +0 -33
  191. package/dist/list-suites-5VK4HXEK.js +0 -43
  192. package/dist/report-issue-DXJ6TPTC.js +0 -412
  193. package/dist/run-agent-RBSIFKRT.js +0 -37
  194. package/dist/run-suite-K3DNQ3SE.js +0 -30
  195. package/dist/update-prompt-RQFTEO7Y.js +0 -25
  196. package/dist/update-suite-TRGGLNR7.js +0 -40
  197. package/pnpm-workspace.yaml +0 -34
  198. package/src/langwatch-api-suites.ts +0 -118
  199. package/src/tools/archive-suite.ts +0 -12
  200. package/src/tools/create-suite.ts +0 -42
  201. package/src/tools/get-suite.ts +0 -47
  202. package/src/tools/list-suites.ts +0 -43
  203. package/src/tools/run-suite.ts +0 -28
  204. package/src/tools/update-suite.ts +0 -44
@@ -5,6 +5,11 @@ import {
5
5
  fetchDocumentation,
6
6
  resolveDocumentationUrl,
7
7
  } from "../documentation-fetch.js";
8
+ import {
9
+ deleteAgent,
10
+ getAgent,
11
+ updateAgent,
12
+ } from "../langwatch-api-agents.js";
8
13
 
9
14
  // --- Canned responses for every API endpoint ---
10
15
 
@@ -99,17 +104,13 @@ const CANNED_PROMPT_DETAIL = {
99
104
  id: "p1",
100
105
  handle: "greeting-bot",
101
106
  name: "Greeting Bot",
102
- latestVersionNumber: 3,
103
- versions: [
104
- {
105
- version: 3,
106
- commitMessage: "Updated tone",
107
- model: "openai/gpt-4o",
108
- messages: [{ role: "system", content: "You are a friendly bot." }],
109
- },
110
- { version: 2, commitMessage: "Added greeting" },
111
- { version: 1, commitMessage: "Initial version" },
112
- ],
107
+ version: 3,
108
+ versionId: "ver_p1v3",
109
+ commitMessage: "Updated tone",
110
+ model: "openai/gpt-4o",
111
+ messages: [{ role: "system", content: "You are a friendly bot." }],
112
+ parameters: {},
113
+ tags: [{ name: "latest", versionId: "ver_p1v3" }],
113
114
  };
114
115
 
115
116
  const CANNED_PROMPT_CREATED = {
@@ -301,22 +302,35 @@ const CANNED_AGENT_DETAIL = {
301
302
  id: "agent_abc", name: "Test Agent", type: "http", config: { url: "http://example.com" }, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z",
302
303
  };
303
304
 
304
- const CANNED_SUITES_LIST = [
305
- { id: "suite_abc", name: "Regression Suite", slug: "regression-suite", description: null, scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 1, labels: [], createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z" },
306
- ];
305
+ const CANNED_RUN_PLAN = {
306
+ id: "plan_abc", name: "Regression Plan", slug: "regression-plan", scope: { mode: "labels", labels: ["auth"] }, scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 2, simulatorModel: "openai/gpt-5-mini", judgeModel: null, labels: [], archivedAt: null, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z", platformUrl: "https://app.langwatch.ai/proj/agent-testing/results/regression-plan",
307
+ };
308
+
309
+ const CANNED_RUN_PLANS_LIST = [CANNED_RUN_PLAN];
307
310
 
308
- const CANNED_SUITE_DETAIL = {
309
- id: "suite_abc", name: "Regression Suite", slug: "regression-suite", description: "A test suite", scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 1, labels: [], createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z",
311
+ const CANNED_RUN_PLAN_RUN = {
312
+ scheduled: true, batchRunId: "batch_123", setId: "set_456", jobCount: 2, skippedArchived: { scenarios: [], targets: [] }, items: [{ scenarioRunId: "run_1", scenarioId: "scen_abc123", target: { type: "http", referenceId: "agent_abc" }, name: "Test" }], runPlanId: "plan_abc", planName: "Regression Plan", created: true, platformUrl: "https://app.langwatch.ai/proj/agent-testing/results/regression-plan",
310
313
  };
311
314
 
312
- const CANNED_SUITE_CREATED = {
313
- id: "suite_new", name: "New Suite", slug: "new-suite", description: null, scenarioIds: ["scen_abc123"], targets: [{ type: "http", referenceId: "agent_abc" }], repeatCount: 1, labels: [], createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z",
315
+ const CANNED_RUN_PLAN_RERUN = { ...CANNED_RUN_PLAN_RUN, created: false };
316
+
317
+ const CANNED_TEST_SUITE = {
318
+ id: "suite_abc", name: "Checkout", slug: "checkout", scenarioIds: ["scen_abc123"], scenarioCount: 1, archivedAt: null, createdAt: "2024-01-01T00:00:00Z", updatedAt: "2024-01-01T00:00:00Z", platformUrl: "https://app.langwatch.ai/proj/agent-testing/suites/checkout",
319
+ };
320
+
321
+ const CANNED_TEST_SUITES_LIST = [CANNED_TEST_SUITE];
322
+
323
+ const CANNED_TEST_SUITE_DETAIL = {
324
+ ...CANNED_TEST_SUITE,
325
+ scenarios: [{ id: "scen_abc123", name: "Login Flow Happy Path" }],
314
326
  };
315
327
 
316
- const CANNED_SUITE_RUN = {
317
- scheduled: true, batchRunId: "batch_123", setId: "set_456", jobCount: 1, skippedArchived: { scenarios: [], targets: [] }, items: [{ scenarioRunId: "run_1", scenarioId: "scen_abc123", target: { type: "http", referenceId: "agent_abc" }, name: "Test" }],
328
+ const CANNED_TEST_SUITE_CREATED = {
329
+ ...CANNED_TEST_SUITE, id: "suite_new", name: "New Suite", slug: "new-suite", scenarioIds: [], scenarioCount: 0,
318
330
  };
319
331
 
332
+ const CANNED_TEST_SUITE_RENAMED = { ...CANNED_TEST_SUITE, name: "Checkout v2", slug: "checkout-v2" };
333
+
320
334
  const CANNED_SIMULATION_RUNS = {
321
335
  runs: [{ scenarioRunId: "run_abc", scenarioId: "scen_abc123", batchRunId: "batch_xyz", name: "Login Flow", status: "SUCCESS", durationInMs: 5200, totalCost: 0.0042, timestamp: 1700000000000, updatedAt: 1700000001000 }],
322
336
  hasMore: false,
@@ -351,6 +365,15 @@ const CANNED_MODEL_PROVIDER_SET = {
351
365
  /** Track last request for each route so tests can assert on request body/params. */
352
366
  const lastRequests: Record<string, { method: string; url: string; body: string }> = {};
353
367
 
368
+ /**
369
+ * Mutable prompt detail so the mock is stateful: a PUT updates it and the
370
+ * subsequent confirmation GET reflects the new version. Reset per test.
371
+ */
372
+ let promptDetailState: typeof CANNED_PROMPT_DETAIL = { ...CANNED_PROMPT_DETAIL };
373
+ beforeEach(() => {
374
+ promptDetailState = { ...CANNED_PROMPT_DETAIL };
375
+ });
376
+
354
377
  function createMockServer(): Server {
355
378
  return createServer((req, res) => {
356
379
  const authToken = req.headers["x-auth-token"];
@@ -415,11 +438,39 @@ function createMockServer(): Server {
415
438
  method === "GET"
416
439
  ) {
417
440
  res.writeHead(200);
418
- res.end(JSON.stringify(CANNED_PROMPT_DETAIL));
441
+ res.end(JSON.stringify(promptDetailState));
419
442
  } else if (
420
443
  url.match(/^\/api\/prompts\/[^/]+$/) &&
421
444
  method === "PUT"
422
445
  ) {
446
+ const parsed = JSON.parse(body) as {
447
+ commitMessage?: string;
448
+ model?: string;
449
+ messages?: Array<{ role: string; content: string }>;
450
+ tags?: string[];
451
+ };
452
+ const newVersionId = "ver_p1v4";
453
+ promptDetailState = {
454
+ ...promptDetailState,
455
+ version: 4,
456
+ versionId: newVersionId,
457
+ commitMessage: parsed.commitMessage ?? promptDetailState.commitMessage,
458
+ model: parsed.model ?? promptDetailState.model,
459
+ messages: parsed.messages ?? promptDetailState.messages,
460
+ tags:
461
+ parsed.tags === undefined
462
+ ? promptDetailState.tags.map((tag) =>
463
+ tag.name === "latest"
464
+ ? { name: "latest", versionId: newVersionId }
465
+ : tag,
466
+ )
467
+ : [
468
+ { name: "latest", versionId: newVersionId },
469
+ ...parsed.tags
470
+ .filter((name) => name !== "latest")
471
+ .map((name) => ({ name, versionId: newVersionId })),
472
+ ],
473
+ };
423
474
  res.writeHead(200);
424
475
  res.end(JSON.stringify(CANNED_PROMPT_UPDATED));
425
476
  }
@@ -493,42 +544,56 @@ function createMockServer(): Server {
493
544
  res.end(JSON.stringify(CANNED_MODEL_PROVIDER_SET));
494
545
  }
495
546
  // --- Agent endpoints ---
496
- else if (url === "/api/agents" && method === "GET") {
547
+ else if (url === "/api/v1/agents" && method === "GET") {
497
548
  res.writeHead(200);
498
549
  res.end(JSON.stringify(CANNED_AGENTS_LIST));
499
- } else if (url === "/api/agents" && method === "POST") {
550
+ } else if (url === "/api/v1/agents" && method === "POST") {
500
551
  res.writeHead(201);
501
552
  res.end(JSON.stringify(CANNED_AGENT_DETAIL));
502
- } else if (url?.match(/^\/api\/agents\/[^/]+$/) && method === "GET") {
553
+ } else if (url?.match(/^\/api\/v1\/agents\/[^/]+$/) && method === "GET") {
503
554
  res.writeHead(200);
504
555
  res.end(JSON.stringify(CANNED_AGENT_DETAIL));
505
- } else if (url?.match(/^\/api\/agents\/[^/]+$/) && method === "PATCH") {
556
+ } else if (url?.match(/^\/api\/v1\/agents\/[^/]+$/) && method === "PATCH") {
506
557
  res.writeHead(200);
507
558
  res.end(JSON.stringify(CANNED_AGENT_DETAIL));
508
- } else if (url?.match(/^\/api\/agents\/[^/]+$/) && method === "DELETE") {
559
+ } else if (url?.match(/^\/api\/v1\/agents\/[^/]+$/) && method === "DELETE") {
509
560
  res.writeHead(200);
510
561
  res.end(JSON.stringify({ id: "agent_abc", name: "Test Agent" }));
511
562
  }
512
- // --- Suite endpoints ---
513
- else if (url === "/api/suites" && method === "GET") {
563
+ // --- Run plan endpoints ---
564
+ else if (url === "/api/v1/run-plans/run" && method === "POST") {
514
565
  res.writeHead(200);
515
- res.end(JSON.stringify(CANNED_SUITES_LIST));
516
- } else if (url === "/api/suites" && method === "POST") {
517
- res.writeHead(201);
518
- res.end(JSON.stringify(CANNED_SUITE_CREATED));
519
- } else if (url?.match(/^\/api\/suites\/[^/]+\/run$/) && method === "POST") {
566
+ res.end(JSON.stringify(CANNED_RUN_PLAN_RUN));
567
+ } else if (url?.match(/^\/api\/v1\/run-plans(\?.*)?$/) && method === "GET") {
568
+ res.writeHead(200);
569
+ res.end(JSON.stringify(CANNED_RUN_PLANS_LIST));
570
+ } else if (url?.match(/^\/api\/v1\/run-plans\/[^/]+\/run$/) && method === "POST") {
571
+ res.writeHead(200);
572
+ res.end(JSON.stringify(CANNED_RUN_PLAN_RERUN));
573
+ } else if (url?.match(/^\/api\/v1\/run-plans\/[^/]+$/) && method === "GET") {
574
+ res.writeHead(200);
575
+ res.end(JSON.stringify(CANNED_RUN_PLAN));
576
+ } else if (url?.match(/^\/api\/v1\/run-plans\/[^/]+$/) && method === "DELETE") {
520
577
  res.writeHead(200);
521
- res.end(JSON.stringify(CANNED_SUITE_RUN));
522
- } else if (url?.match(/^\/api\/suites\/[^/]+\/duplicate$/) && method === "POST") {
578
+ res.end(JSON.stringify({ id: "plan_abc", archived: true }));
579
+ }
580
+ // --- Test suite endpoints ---
581
+ else if (url === "/api/v1/test-suites" && method === "GET") {
582
+ res.writeHead(200);
583
+ res.end(JSON.stringify(CANNED_TEST_SUITES_LIST));
584
+ } else if (url === "/api/v1/test-suites" && method === "POST") {
523
585
  res.writeHead(201);
524
- res.end(JSON.stringify(CANNED_SUITE_CREATED));
525
- } else if (url?.match(/^\/api\/suites\/[^/]+$/) && method === "GET") {
586
+ res.end(JSON.stringify(CANNED_TEST_SUITE_CREATED));
587
+ } else if (url?.match(/^\/api\/v1\/test-suites\/[^/]+\/run$/) && method === "POST") {
588
+ res.writeHead(200);
589
+ res.end(JSON.stringify(CANNED_RUN_PLAN_RUN));
590
+ } else if (url?.match(/^\/api\/v1\/test-suites\/[^/]+$/) && method === "GET") {
526
591
  res.writeHead(200);
527
- res.end(JSON.stringify(CANNED_SUITE_DETAIL));
528
- } else if (url?.match(/^\/api\/suites\/[^/]+$/) && method === "PATCH") {
592
+ res.end(JSON.stringify(CANNED_TEST_SUITE_DETAIL));
593
+ } else if (url?.match(/^\/api\/v1\/test-suites\/[^/]+$/) && method === "PATCH") {
529
594
  res.writeHead(200);
530
- res.end(JSON.stringify(CANNED_SUITE_DETAIL));
531
- } else if (url?.match(/^\/api\/suites\/[^/]+$/) && method === "DELETE") {
595
+ res.end(JSON.stringify(CANNED_TEST_SUITE_RENAMED));
596
+ } else if (url?.match(/^\/api\/v1\/test-suites\/[^/]+$/) && method === "DELETE") {
532
597
  res.writeHead(200);
533
598
  res.end(JSON.stringify({ id: "suite_abc", archived: true }));
534
599
  }
@@ -1055,7 +1120,7 @@ describe("All MCP tools integration", () => {
1055
1120
  // =====================
1056
1121
  describe("platform_get_prompt", () => {
1057
1122
  describe("when prompt exists", () => {
1058
- it("returns formatted prompt details with messages and versions", async () => {
1123
+ it("returns formatted prompt details with messages and deployments", async () => {
1059
1124
  const { handleGetPrompt } = await import(
1060
1125
  "../tools/get-prompt.js"
1061
1126
  );
@@ -1067,7 +1132,7 @@ describe("All MCP tools integration", () => {
1067
1132
  expect(result).toContain("gpt-4o");
1068
1133
  expect(result).toContain("You are a friendly bot.");
1069
1134
  expect(result).toContain("v3");
1070
- expect(result).toContain("## Version History");
1135
+ expect(result).toContain("## Deployments");
1071
1136
  expect(result).toContain("Updated tone");
1072
1137
  });
1073
1138
  });
@@ -1090,6 +1155,70 @@ describe("All MCP tools integration", () => {
1090
1155
 
1091
1156
  expect(result).toContain("updated successfully");
1092
1157
  expect(result).toContain("Switch to mini");
1158
+ expect(result).toContain("**Version**: v4");
1159
+ expect(result).toContain("**Version ID**: ver_p1v4");
1160
+ });
1161
+ });
1162
+
1163
+ describe("when an update supplies only messages and a commit message", () => {
1164
+ /** @scenario Carrying forward prior fields when an update supplies only messages and a commit message */
1165
+ it("does not send fields that would wipe prior prompt configuration", async () => {
1166
+ const { handleUpdatePrompt } = await import(
1167
+ "../tools/update-prompt.js"
1168
+ );
1169
+ await handleUpdatePrompt({
1170
+ idOrHandle: "greeting-bot",
1171
+ messages: [{ role: "system", content: "You are a helpful bot." }],
1172
+ commitMessage: "Only messages changed",
1173
+ });
1174
+
1175
+ const req = lastRequests["PUT /api/prompts/greeting-bot"];
1176
+ expect(req).toBeDefined();
1177
+ const parsed = JSON.parse(req!.body);
1178
+ expect(parsed).not.toHaveProperty("parameters");
1179
+ expect(parsed).not.toHaveProperty("inputs");
1180
+ expect(parsed).not.toHaveProperty("outputs");
1181
+ expect(parsed).not.toHaveProperty("temperature");
1182
+ expect(parsed).not.toHaveProperty("responseFormat");
1183
+ });
1184
+ });
1185
+
1186
+ describe("when an update omits tags", () => {
1187
+ /** @scenario Tag-to-version mapping stays unchanged when an update omits tags */
1188
+ it("does not send a tags field in the update request", async () => {
1189
+ const { handleUpdatePrompt } = await import(
1190
+ "../tools/update-prompt.js"
1191
+ );
1192
+ await handleUpdatePrompt({
1193
+ idOrHandle: "greeting-bot",
1194
+ model: "openai/gpt-4o",
1195
+ commitMessage: "No tag change requested",
1196
+ });
1197
+
1198
+ const req = lastRequests["PUT /api/prompts/greeting-bot"];
1199
+ expect(req).toBeDefined();
1200
+ const parsed = JSON.parse(req!.body);
1201
+ expect(parsed).not.toHaveProperty("tags");
1202
+ });
1203
+ });
1204
+
1205
+ describe("when an update passes tags explicitly", () => {
1206
+ /** @scenario Passing tags explicitly moves the tag to the new version */
1207
+ it("sends the requested tags in the update request", async () => {
1208
+ const { handleUpdatePrompt } = await import(
1209
+ "../tools/update-prompt.js"
1210
+ );
1211
+ await handleUpdatePrompt({
1212
+ idOrHandle: "greeting-bot",
1213
+ model: "openai/gpt-4o",
1214
+ commitMessage: "Deploy new version to production",
1215
+ tags: ["production"],
1216
+ });
1217
+
1218
+ const req = lastRequests["PUT /api/prompts/greeting-bot"];
1219
+ expect(req).toBeDefined();
1220
+ const parsed = JSON.parse(req!.body);
1221
+ expect(parsed.tags).toEqual(["production"]);
1093
1222
  });
1094
1223
  });
1095
1224
  });
@@ -1443,58 +1572,150 @@ describe("All MCP tools integration", () => {
1443
1572
  });
1444
1573
 
1445
1574
  // =====================
1446
- // Suite Tools
1575
+ // Run Plan Tools
1447
1576
  // =====================
1448
- describe("platform_list_suites", () => {
1449
- it("returns formatted suite list", async () => {
1450
- const { handleListSuites } = await import("../tools/list-suites.js");
1451
- const result = await handleListSuites({});
1577
+ describe("platform_run_plan", () => {
1578
+ it("runs a configuration and reports the plan it created", async () => {
1579
+ const { handleRunPlan } = await import("../tools/run-plan.js");
1580
+ const result = await handleRunPlan({
1581
+ name: "Regression Plan",
1582
+ scope: { mode: "labels", labels: ["auth"] },
1583
+ targets: [{ type: "http", referenceId: "agent_abc" }],
1584
+ repeatCount: 2,
1585
+ note: "nightly regression",
1586
+ });
1452
1587
 
1453
- expect(result).toContain("Suites / Run Plans (1 total)");
1454
- expect(result).toContain("Regression Suite");
1588
+ expect(result).toContain("Regression Plan");
1589
+ expect(result).toContain("created and started");
1590
+ expect(result).toContain("batch_123");
1591
+ expect(result).toContain("**Jobs**: 2");
1455
1592
  });
1456
1593
  });
1457
1594
 
1458
- describe("platform_get_suite", () => {
1459
- it("returns suite details", async () => {
1460
- const { handleGetSuite } = await import("../tools/get-suite.js");
1461
- const result = await handleGetSuite({ id: "suite_abc" });
1595
+ describe("platform_list_run_plans", () => {
1596
+ it("returns formatted run plan list", async () => {
1597
+ const { handleListRunPlans } = await import("../tools/list-run-plans.js");
1598
+ const result = await handleListRunPlans({});
1462
1599
 
1463
- expect(result).toContain("Regression Suite");
1464
- expect(result).toContain("suite_abc");
1600
+ expect(result).toContain("Run Plans (1 total)");
1601
+ expect(result).toContain("Regression Plan");
1465
1602
  });
1466
1603
  });
1467
1604
 
1468
- describe("platform_create_suite", () => {
1469
- it("creates a suite and returns confirmation", async () => {
1470
- const { handleCreateSuite } = await import("../tools/create-suite.js");
1471
- const result = await handleCreateSuite({
1472
- name: "New Suite",
1473
- scenarioIds: ["scen_abc123"],
1474
- targets: JSON.stringify([{ type: "http", referenceId: "agent_abc" }]),
1475
- });
1605
+ describe("platform_get_run_plan", () => {
1606
+ it("returns the plan configuration", async () => {
1607
+ const { handleGetRunPlan } = await import("../tools/get-run-plan.js");
1608
+ const result = await handleGetRunPlan({ id: "plan_abc" });
1476
1609
 
1477
- expect(result).toContain("created successfully");
1478
- expect(result).toContain("New Suite");
1610
+ expect(result).toContain("Regression Plan");
1611
+ expect(result).toContain("plan_abc");
1612
+ expect(result).toContain("labels: auth");
1479
1613
  });
1480
1614
  });
1481
1615
 
1482
- describe("platform_run_suite", () => {
1483
- it("triggers a suite run and returns batch info", async () => {
1484
- const { handleRunSuite } = await import("../tools/run-suite.js");
1485
- const result = await handleRunSuite({ id: "suite_abc" });
1616
+ describe("platform_rerun_run_plan", () => {
1617
+ it("runs the stored configuration again and reports it joined the plan", async () => {
1618
+ const { handleRerunRunPlan } = await import(
1619
+ "../tools/rerun-run-plan.js"
1620
+ );
1621
+ const result = await handleRerunRunPlan({ id: "plan_abc" });
1486
1622
 
1487
- expect(result).toContain("scheduled successfully");
1623
+ expect(result).toContain("Regression Plan");
1624
+ expect(result).toContain("configuration of this run");
1488
1625
  expect(result).toContain("batch_123");
1489
1626
  });
1490
1627
  });
1491
1628
 
1492
- describe("platform_archive_suite", () => {
1493
- it("archives the suite", async () => {
1494
- const { handleArchiveSuite } = await import("../tools/archive-suite.js");
1495
- const result = await handleArchiveSuite({ id: "suite_abc" });
1629
+ describe("platform_archive_run_plan", () => {
1630
+ it("archives the run plan", async () => {
1631
+ const { handleArchiveRunPlan } = await import(
1632
+ "../tools/archive-run-plan.js"
1633
+ );
1634
+ const result = await handleArchiveRunPlan({ id: "plan_abc" });
1496
1635
 
1497
1636
  expect(result).toContain("archived");
1637
+ expect(result).toContain("plan_abc");
1638
+ });
1639
+ });
1640
+
1641
+ // =====================
1642
+ // Test Suite Tools
1643
+ // =====================
1644
+ describe("platform_list_test_suites", () => {
1645
+ it("returns formatted test suite list", async () => {
1646
+ const { handleListTestSuites } = await import(
1647
+ "../tools/list-test-suites.js"
1648
+ );
1649
+ const result = await handleListTestSuites({});
1650
+
1651
+ expect(result).toContain("Test Suites (1 total)");
1652
+ expect(result).toContain("Checkout");
1653
+ });
1654
+ });
1655
+
1656
+ describe("platform_create_test_suite", () => {
1657
+ it("creates a test suite and returns confirmation", async () => {
1658
+ const { handleCreateTestSuite } = await import(
1659
+ "../tools/create-test-suite.js"
1660
+ );
1661
+ const result = await handleCreateTestSuite({ name: "New Suite" });
1662
+
1663
+ expect(result).toContain("created");
1664
+ expect(result).toContain("New Suite");
1665
+ expect(result).toContain("suite_new");
1666
+ });
1667
+ });
1668
+
1669
+ describe("platform_get_test_suite", () => {
1670
+ it("returns the suite with the scenarios filed in it", async () => {
1671
+ const { handleGetTestSuite } = await import(
1672
+ "../tools/get-test-suite.js"
1673
+ );
1674
+ const result = await handleGetTestSuite({ id: "suite_abc" });
1675
+
1676
+ expect(result).toContain("Checkout");
1677
+ expect(result).toContain("Login Flow Happy Path");
1678
+ });
1679
+ });
1680
+
1681
+ describe("platform_rename_test_suite", () => {
1682
+ it("renames the test suite", async () => {
1683
+ const { handleRenameTestSuite } = await import(
1684
+ "../tools/rename-test-suite.js"
1685
+ );
1686
+ const result = await handleRenameTestSuite({
1687
+ id: "suite_abc",
1688
+ name: "Checkout v2",
1689
+ });
1690
+
1691
+ expect(result).toContain("Checkout v2");
1692
+ });
1693
+ });
1694
+
1695
+ describe("platform_archive_test_suite", () => {
1696
+ it("archives the suite and says the filed scenarios went with it", async () => {
1697
+ const { handleArchiveTestSuite } = await import(
1698
+ "../tools/archive-test-suite.js"
1699
+ );
1700
+ const result = await handleArchiveTestSuite({ id: "suite_abc" });
1701
+
1702
+ expect(result).toContain("archived");
1703
+ expect(result).toContain("scenarios filed in it");
1704
+ });
1705
+ });
1706
+
1707
+ describe("platform_run_test_suite", () => {
1708
+ it("runs the suite against a target and reports the derived plan", async () => {
1709
+ const { handleRunTestSuite } = await import(
1710
+ "../tools/run-test-suite.js"
1711
+ );
1712
+ const result = await handleRunTestSuite({
1713
+ id: "suite_abc",
1714
+ targets: [{ type: "http", referenceId: "agent_abc" }],
1715
+ });
1716
+
1717
+ expect(result).toContain("Regression Plan");
1718
+ expect(result).toContain("batch_123");
1498
1719
  });
1499
1720
  });
1500
1721
 
@@ -1611,6 +1832,31 @@ describe("All MCP tools integration", () => {
1611
1832
  });
1612
1833
  });
1613
1834
 
1835
+ // =====================
1836
+ // Agent Tools
1837
+ // =====================
1838
+ describe("platform_get_agent", () => {
1839
+ it("returns one agent by id", async () => {
1840
+ const agent = await getAgent("agent_abc");
1841
+ expect(agent.id).toBe("agent_abc");
1842
+ expect(agent.name).toBe("Test Agent");
1843
+ });
1844
+ });
1845
+
1846
+ describe("platform_update_agent", () => {
1847
+ it("updates an agent and returns it", async () => {
1848
+ const agent = await updateAgent({ id: "agent_abc", name: "Test Agent" });
1849
+ expect(agent.id).toBe("agent_abc");
1850
+ });
1851
+ });
1852
+
1853
+ describe("platform_delete_agent", () => {
1854
+ it("deletes an agent", async () => {
1855
+ const result = await deleteAgent("agent_abc");
1856
+ expect(result.id).toBe("agent_abc");
1857
+ });
1858
+ });
1859
+
1614
1860
  // =====================
1615
1861
  // Agent Run Tool
1616
1862
  // =====================
@@ -58,6 +58,39 @@ describe("config", () => {
58
58
  });
59
59
  });
60
60
 
61
+ describe("when the endpoint carries a trailing slash", () => {
62
+ it("strips it from a CLI endpoint", () => {
63
+ initConfig({ endpoint: "https://cli.example.com/" });
64
+ expect(getConfig().endpoint).toBe("https://cli.example.com");
65
+ });
66
+
67
+ it("trims whitespace around a CLI endpoint", () => {
68
+ initConfig({ endpoint: " https://cli.example.com/// " });
69
+ expect(getConfig().endpoint).toBe("https://cli.example.com");
70
+ });
71
+
72
+ it("strips repeated slashes from the env var", () => {
73
+ process.env.LANGWATCH_ENDPOINT = "http://localhost:5560///";
74
+ initConfig({});
75
+ expect(getConfig().endpoint).toBe("http://localhost:5560");
76
+ });
77
+
78
+ it("keeps the built request path free of a double slash", () => {
79
+ initConfig({ endpoint: "https://app.langwatch.ai/" });
80
+ expect(`${getConfig().endpoint}/api/bug-reports`).toBe(
81
+ "https://app.langwatch.ai/api/bug-reports",
82
+ );
83
+ });
84
+ });
85
+
86
+ describe("when the endpoint is blank", () => {
87
+ it("falls back to the default rather than an empty base url", () => {
88
+ process.env.LANGWATCH_ENDPOINT = " ";
89
+ initConfig({});
90
+ expect(getConfig().endpoint).toBe("https://app.langwatch.ai");
91
+ });
92
+ });
93
+
61
94
  describe("when no args or env vars are provided", () => {
62
95
  it("defaults endpoint to https://app.langwatch.ai", () => {
63
96
  initConfig({});