@langwatch/mcp-server 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.python-version +1 -0
- package/CHANGELOG.md +110 -0
- package/CONTRIBUTING.md +1 -1
- package/README.md +86 -4
- package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
- package/dist/archive-run-plan-474OKHHU.js +16 -0
- package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
- package/dist/archive-test-suite-O7FLDMXK.js +16 -0
- package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
- package/dist/chunk-56OGTS24.js +44 -0
- package/dist/chunk-6DJDEXET.js +63 -0
- package/dist/chunk-7E4FZIYJ.js +15 -0
- package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
- package/dist/chunk-CKNAH5TG.js +74 -0
- package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
- package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
- package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
- package/dist/chunk-HV4DWKBI.js +46 -0
- package/dist/chunk-JR6DJZOU.js +49 -0
- package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
- package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
- package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
- package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
- package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
- package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
- package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
- package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
- package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
- package/dist/chunk-UF6XIGYR.js +58 -0
- package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
- package/dist/chunk-UJPXFA26.js +267 -0
- package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
- package/dist/config.js +2 -2
- package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
- package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
- package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
- package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
- package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
- package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
- package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
- package/dist/create-mcp-server.js +9 -4
- package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
- package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
- package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
- package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
- package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
- package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
- package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
- package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
- package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
- package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
- package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
- package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
- package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
- package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
- package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
- package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
- package/dist/get-agent-7THF7HCU.js +66 -0
- package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
- package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
- package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
- package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
- package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
- package/dist/get-prompt-GVKXF5NL.js +100 -0
- package/dist/get-run-plan-NY7J5GRE.js +61 -0
- package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
- package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
- package/dist/get-test-suite-4HKA7OKS.js +49 -0
- package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
- package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
- package/dist/http-security-EU3CHTY7.js +26 -0
- package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
- package/dist/index.js +6437 -4020
- package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
- package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
- package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
- package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
- package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
- package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
- package/dist/list-agents-AMVDQPQL.js +43 -0
- package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
- package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
- package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
- package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
- package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
- package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
- package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
- package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
- package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
- package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
- package/dist/list-run-plans-UBAU2DGH.js +46 -0
- package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
- package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
- package/dist/list-test-suites-L7R2O6DR.js +35 -0
- package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
- package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
- package/dist/rename-test-suite-NTYHAO5X.js +21 -0
- package/dist/report-issue-RHZZ33IS.js +885 -0
- package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
- package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
- package/dist/run-agent-CAGWGTYT.js +72 -0
- package/dist/run-plan-T4ZYOZGA.js +39 -0
- package/dist/run-test-suite-L4MZ7G4D.js +26 -0
- package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
- package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
- package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
- package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
- package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
- package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
- package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
- package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
- package/dist/update-prompt-ZQQ2C3KU.js +143 -0
- package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
- package/package.json +8 -9
- package/pyproject.toml +5 -5
- package/src/__tests__/agent-tools.unit.test.ts +257 -0
- package/src/__tests__/all-tools.integration.test.ts +321 -75
- package/src/__tests__/config.unit.test.ts +33 -0
- package/src/__tests__/docker.integration.test.ts +91 -7
- package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
- package/src/__tests__/http-security-origin.unit.test.ts +97 -0
- package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
- package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
- package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
- package/src/__tests__/http-server-origin.integration.test.ts +169 -0
- package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
- package/src/__tests__/http-transport.integration.test.ts +85 -87
- package/src/__tests__/integration.integration.test.ts +7 -11
- package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
- package/src/__tests__/prompts.integration.test.ts +736 -0
- package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
- package/src/__tests__/scenario-tools.unit.test.ts +19 -0
- package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
- package/src/__tests__/support/http-server-harness.ts +138 -0
- package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
- package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
- package/src/config.ts +17 -2
- package/src/create-mcp-server.ts +334 -55
- package/src/http-security.ts +492 -0
- package/src/http-server.ts +536 -152
- package/src/index.ts +49 -2
- package/src/langwatch-api-agents.ts +196 -12
- package/src/langwatch-api-run-plans.ts +157 -0
- package/src/langwatch-api-scenarios.ts +9 -0
- package/src/langwatch-api-simulation-runs.ts +5 -0
- package/src/langwatch-api-test-suites.ts +135 -0
- package/src/langwatch-api.ts +62 -12
- package/src/schemas/index.ts +5 -0
- package/src/schemas/run-plan.ts +92 -0
- package/src/schemas/suite-fields.ts +149 -0
- package/src/tools/archive-run-plan.ts +12 -0
- package/src/tools/archive-test-suite.ts +12 -0
- package/src/tools/create-scenario.ts +8 -0
- package/src/tools/create-test-suite.ts +43 -0
- package/src/tools/discover-evaluator-schema.ts +2 -2
- package/src/tools/discover-scenario-schema.ts +54 -0
- package/src/tools/format-run-plan.ts +68 -0
- package/src/tools/format-scenario.ts +14 -0
- package/src/tools/format-suite-details.ts +96 -0
- package/src/tools/get-agent.ts +45 -3
- package/src/tools/get-prompt.ts +108 -24
- package/src/tools/get-run-plan.ts +62 -0
- package/src/tools/get-scenario.ts +3 -0
- package/src/tools/get-simulation-run.ts +9 -0
- package/src/tools/get-test-suite.ts +49 -0
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +12 -2
- package/src/tools/list-simulation-runs.ts +6 -2
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/rename-test-suite.ts +18 -0
- package/src/tools/rerun-run-plan.ts +21 -0
- package/src/tools/run-agent.ts +54 -6
- package/src/tools/run-plan.ts +49 -0
- package/src/tools/run-test-suite.ts +30 -0
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-prompt.ts +195 -10
- package/src/tools/update-scenario.ts +8 -0
- package/src/tools/update-test-suite.ts +45 -0
- package/tsconfig.json +7 -2
- package/uv.lock +418 -298
- package/dist/archive-suite-C3KX6IIZ.js +0 -16
- package/dist/chunk-WDH5P5RF.js +0 -48
- package/dist/create-suite-TWF6BBR4.js +0 -39
- package/dist/get-agent-7QJVPH72.js +0 -29
- package/dist/get-prompt-I753LZKZ.js +0 -46
- package/dist/get-suite-GTOCY5PL.js +0 -45
- package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
- package/dist/list-agents-TIJTP566.js +0 -33
- package/dist/list-suites-5VK4HXEK.js +0 -43
- package/dist/report-issue-DXJ6TPTC.js +0 -412
- package/dist/run-agent-RBSIFKRT.js +0 -37
- package/dist/run-suite-K3DNQ3SE.js +0 -30
- package/dist/update-prompt-RQFTEO7Y.js +0 -25
- package/dist/update-suite-TRGGLNR7.js +0 -40
- package/pnpm-workspace.yaml +0 -34
- package/src/langwatch-api-suites.ts +0 -118
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
package/.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,115 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [2.1.0](https://github.com/langwatch/langwatch/compare/mcp-server@v2.0.0...mcp-server@v2.1.0) (2026-09-12)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* **agent-testing:** compare agents in one run ([#7654](https://github.com/langwatch/langwatch/issues/7654)) ([3e73916](https://github.com/langwatch/langwatch/commit/3e7391698f5e9b1cac2f8c5f0d0cacc9dec4d02a))
|
|
9
|
+
* **agent-testing:** suite fields and evaluators on scenario runs ([#7867](https://github.com/langwatch/langwatch/issues/7867)) ([0bcf01e](https://github.com/langwatch/langwatch/commit/0bcf01edb3d8cc36aa62cd48e6ee147a7b01b45b))
|
|
10
|
+
* **agent-testing:** v2 polish round 6 ([#7590](https://github.com/langwatch/langwatch/issues/7590)) ([1f9efad](https://github.com/langwatch/langwatch/commit/1f9efad30ab94e86d00f57ea478bf0685b437f23))
|
|
11
|
+
* **agents:** connected agents, a decorated function is a simulation target ([#7655](https://github.com/langwatch/langwatch/issues/7655)) ([56922c0](https://github.com/langwatch/langwatch/commit/56922c0ee429bd5a38717960c09ede0d1905c0c3))
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### Bug Fixes
|
|
15
|
+
|
|
16
|
+
* **agents:** a finished connected-agent run reaches its verdict, and the transcript stays whole ([#7696](https://github.com/langwatch/langwatch/issues/7696)) ([4f7b665](https://github.com/langwatch/langwatch/commit/4f7b665623e1e453e8f505c259f2351bd074ad72))
|
|
17
|
+
* **agents:** a key waits for its grants, and a listing shows who holds each agent ([#7907](https://github.com/langwatch/langwatch/issues/7907)) ([654e82a](https://github.com/langwatch/langwatch/commit/654e82ae9f53c98c0cc710dc9c0c520a87e2c6cb))
|
|
18
|
+
* **mcp:** stop prompt tools dropping fields in both directions ([#7712](https://github.com/langwatch/langwatch/issues/7712)) ([a97c378](https://github.com/langwatch/langwatch/commit/a97c378d8ee9c3613c2235738e0ea4085b45684e))
|
|
19
|
+
* optional legacy answer fields, one vocabulary for test suites and run plans, and named targets for runs from code ([#7638](https://github.com/langwatch/langwatch/issues/7638)) ([6fd37f3](https://github.com/langwatch/langwatch/commit/6fd37f30428c9fc468230fa9e4a79e3dee8cc58f))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### Miscellaneous
|
|
23
|
+
|
|
24
|
+
* **deps-dev:** bump @anthropic-ai/claude-code ([aac07f5](https://github.com/langwatch/langwatch/commit/aac07f53aa0c7fa4cc988cb365865ff521a0922a))
|
|
25
|
+
* **deps-dev:** bump @anthropic-ai/claude-code ([fbc7dcf](https://github.com/langwatch/langwatch/commit/fbc7dcf5f18f9be74ed7d1a390f922d3c9db7ea8))
|
|
26
|
+
* **deps-dev:** bump @anthropic-ai/claude-code from 2.1.202 to 2.1.226 in the anthropic group across 1 directory ([#6546](https://github.com/langwatch/langwatch/issues/6546)) ([fbc7dcf](https://github.com/langwatch/langwatch/commit/fbc7dcf5f18f9be74ed7d1a390f922d3c9db7ea8))
|
|
27
|
+
* **deps-dev:** bump @anthropic-ai/claude-code from 2.1.226 to 2.1.228 in the anthropic group across 1 directory ([#7264](https://github.com/langwatch/langwatch/issues/7264)) ([aac07f5](https://github.com/langwatch/langwatch/commit/aac07f53aa0c7fa4cc988cb365865ff521a0922a))
|
|
28
|
+
* **deps-dev:** bump the types group across 1 directory with 2 updates ([#6789](https://github.com/langwatch/langwatch/issues/6789)) ([7be2d04](https://github.com/langwatch/langwatch/commit/7be2d04c798b46a29b9b5b046dbb28a1e6c369e7))
|
|
29
|
+
* **deps-dev:** update black requirement from >=26.3.1 to >=26.5.1 in /mcp/typescript ([#7232](https://github.com/langwatch/langwatch/issues/7232)) ([1444629](https://github.com/langwatch/langwatch/commit/14446291f6eafddd99463e7ee83288653ba62346))
|
|
30
|
+
* **deps-dev:** update black requirement in /mcp/typescript ([1444629](https://github.com/langwatch/langwatch/commit/14446291f6eafddd99463e7ee83288653ba62346))
|
|
31
|
+
* **deps-dev:** update cloudpickle requirement from >=3.1.1 to >=3.1.2 in /mcp/typescript ([#7209](https://github.com/langwatch/langwatch/issues/7209)) ([bc02a83](https://github.com/langwatch/langwatch/commit/bc02a8309ad0f69ce350557b35c1bc7d9ea1777b))
|
|
32
|
+
* **deps-dev:** update cloudpickle requirement in /mcp/typescript ([bc02a83](https://github.com/langwatch/langwatch/commit/bc02a8309ad0f69ce350557b35c1bc7d9ea1777b))
|
|
33
|
+
* **deps-dev:** update dspy-ai requirement from <4,>=3.0.3 to >=3.3.0,<4 in /mcp/typescript ([#7225](https://github.com/langwatch/langwatch/issues/7225)) ([81f1590](https://github.com/langwatch/langwatch/commit/81f1590cc08af237d6b6b4fbb81e814ebddbb893))
|
|
34
|
+
* **deps-dev:** update dspy-ai requirement in /mcp/typescript ([81f1590](https://github.com/langwatch/langwatch/commit/81f1590cc08af237d6b6b4fbb81e814ebddbb893))
|
|
35
|
+
* **deps-dev:** update faiss-cpu requirement from <2.0.0,>=1.8.0 to >=1.15.0,<2.0.0 in /mcp/typescript ([#7227](https://github.com/langwatch/langwatch/issues/7227)) ([80b6ae4](https://github.com/langwatch/langwatch/commit/80b6ae45742964508f0a5b44015b433c4887ee19))
|
|
36
|
+
* **deps-dev:** update faiss-cpu requirement in /mcp/typescript ([80b6ae4](https://github.com/langwatch/langwatch/commit/80b6ae45742964508f0a5b44015b433c4887ee19))
|
|
37
|
+
* **deps-dev:** update fastapi requirement from >=0.115.12 to >=0.141.1 in /mcp/typescript ([#7226](https://github.com/langwatch/langwatch/issues/7226)) ([62b3bbf](https://github.com/langwatch/langwatch/commit/62b3bbf4bee00c51316c1e06b3c67788013d8bdb))
|
|
38
|
+
* **deps-dev:** update fastapi requirement in /mcp/typescript ([62b3bbf](https://github.com/langwatch/langwatch/commit/62b3bbf4bee00c51316c1e06b3c67788013d8bdb))
|
|
39
|
+
* **deps-dev:** update fonttools requirement from >=4.62.0 to >=4.63.0 in /mcp/typescript ([#7235](https://github.com/langwatch/langwatch/issues/7235)) ([573fc99](https://github.com/langwatch/langwatch/commit/573fc997c07fbbfbe1dc71acb78635c2c329497c))
|
|
40
|
+
* **deps-dev:** update fonttools requirement in /mcp/typescript ([573fc99](https://github.com/langwatch/langwatch/commit/573fc997c07fbbfbe1dc71acb78635c2c329497c))
|
|
41
|
+
* **deps-dev:** update haystack-ai requirement from <4.0.0,>=2.13.2 to >=3.0.0,<4.0.0 in /mcp/typescript ([#7248](https://github.com/langwatch/langwatch/issues/7248)) ([72df892](https://github.com/langwatch/langwatch/commit/72df892f20134b0a335f3783ef9caa26141888f7))
|
|
42
|
+
* **deps-dev:** update haystack-ai requirement in /mcp/typescript ([72df892](https://github.com/langwatch/langwatch/commit/72df892f20134b0a335f3783ef9caa26141888f7))
|
|
43
|
+
* **deps-dev:** update json-repair requirement from >=0.42.0 to >=0.63.0 in /mcp/typescript ([#7247](https://github.com/langwatch/langwatch/issues/7247)) ([6376930](https://github.com/langwatch/langwatch/commit/6376930151c7d995264c6e1480c08091ecd5830d))
|
|
44
|
+
* **deps-dev:** update json-repair requirement in /mcp/typescript ([6376930](https://github.com/langwatch/langwatch/commit/6376930151c7d995264c6e1480c08091ecd5830d))
|
|
45
|
+
* **deps-dev:** update jupyter requirement from <2.0.0,>=1.0.0 to >=1.1.1,<2.0.0 in /mcp/typescript ([#7243](https://github.com/langwatch/langwatch/issues/7243)) ([7f39887](https://github.com/langwatch/langwatch/commit/7f3988779cc72824bbe3dc5b30c4f0ffbb5bab1c))
|
|
46
|
+
* **deps-dev:** update jupyter requirement in /mcp/typescript ([7f39887](https://github.com/langwatch/langwatch/commit/7f3988779cc72824bbe3dc5b30c4f0ffbb5bab1c))
|
|
47
|
+
* **deps-dev:** update jupyterlab requirement from >=4.5.6 to >=4.6.3 in /mcp/typescript ([#7242](https://github.com/langwatch/langwatch/issues/7242)) ([d714c43](https://github.com/langwatch/langwatch/commit/d714c436982dbda46ce714a4664fa6eb10e47122))
|
|
48
|
+
* **deps-dev:** update jupyterlab requirement in /mcp/typescript ([d714c43](https://github.com/langwatch/langwatch/commit/d714c436982dbda46ce714a4664fa6eb10e47122))
|
|
49
|
+
* **deps-dev:** update langchain-community requirement ([d63a5a8](https://github.com/langwatch/langwatch/commit/d63a5a85e6124030729111f98a4de516f10162bf))
|
|
50
|
+
* **deps-dev:** update langchain-community requirement from <1.0.0,>=0.3.0 to >=0.4.2,<1.0.0 in /mcp/typescript ([#7222](https://github.com/langwatch/langwatch/issues/7222)) ([d63a5a8](https://github.com/langwatch/langwatch/commit/d63a5a85e6124030729111f98a4de516f10162bf))
|
|
51
|
+
* **deps-dev:** update langchain-openai requirement from <2.0.0,>=1.1.14 to >=1.4.3,<2.0.0 in /mcp/typescript ([#7218](https://github.com/langwatch/langwatch/issues/7218)) ([d62efb1](https://github.com/langwatch/langwatch/commit/d62efb14d5cdaf9345a213f86f21804d00890833))
|
|
52
|
+
* **deps-dev:** update langchain-openai requirement in /mcp/typescript ([d62efb1](https://github.com/langwatch/langwatch/commit/d62efb14d5cdaf9345a213f86f21804d00890833))
|
|
53
|
+
* **deps-dev:** update langsmith requirement from >=0.8.0 to >=0.10.18 in /mcp/typescript ([#7230](https://github.com/langwatch/langwatch/issues/7230)) ([f0e1d71](https://github.com/langwatch/langwatch/commit/f0e1d717a416e45f916bd2f777abb5c26598a4d4))
|
|
54
|
+
* **deps-dev:** update langsmith requirement in /mcp/typescript ([f0e1d71](https://github.com/langwatch/langwatch/commit/f0e1d717a416e45f916bd2f777abb5c26598a4d4))
|
|
55
|
+
* **deps-dev:** update nltk requirement from >=3.9.4 to >=3.10.2 in /mcp/typescript ([#7215](https://github.com/langwatch/langwatch/issues/7215)) ([61e9f51](https://github.com/langwatch/langwatch/commit/61e9f515d026cc71f48bffe56f0debff5621f741))
|
|
56
|
+
* **deps-dev:** update nltk requirement in /mcp/typescript ([61e9f51](https://github.com/langwatch/langwatch/commit/61e9f515d026cc71f48bffe56f0debff5621f741))
|
|
57
|
+
* **deps-dev:** update openinference-instrumentation-dspy requirement ([2926e6e](https://github.com/langwatch/langwatch/commit/2926e6efc2aa44faf231c8ed87d26f9f4e588523))
|
|
58
|
+
* **deps-dev:** update openinference-instrumentation-dspy requirement from <0.2.0,>=0.1.23 to >=0.1.39,<0.2.0 in /mcp/typescript ([#7224](https://github.com/langwatch/langwatch/issues/7224)) ([2926e6e](https://github.com/langwatch/langwatch/commit/2926e6efc2aa44faf231c8ed87d26f9f4e588523))
|
|
59
|
+
* **deps-dev:** update openinference-instrumentation-litellm requirement ([8e55d60](https://github.com/langwatch/langwatch/commit/8e55d60f32982070202c61ef53886fb22a74f931))
|
|
60
|
+
* **deps-dev:** update openinference-instrumentation-litellm requirement from >=0.1.19 to >=0.1.36 in /mcp/typescript ([#7250](https://github.com/langwatch/langwatch/issues/7250)) ([8e55d60](https://github.com/langwatch/langwatch/commit/8e55d60f32982070202c61ef53886fb22a74f931))
|
|
61
|
+
* **deps-dev:** update opentelemetry-instrumentation-fastapi requirement ([7a17c8e](https://github.com/langwatch/langwatch/commit/7a17c8edc72629b1500f2f5ee02e398d874d0cb0))
|
|
62
|
+
* **deps-dev:** update opentelemetry-instrumentation-fastapi requirement from >=0.53b1 to >=0.65b0 in /mcp/typescript ([#7228](https://github.com/langwatch/langwatch/issues/7228)) ([7a17c8e](https://github.com/langwatch/langwatch/commit/7a17c8edc72629b1500f2f5ee02e398d874d0cb0))
|
|
63
|
+
* **deps-dev:** update opentelemetry-instrumentation-langchain requirement ([d779795](https://github.com/langwatch/langwatch/commit/d7797954d2099355c3f342cc7a2d580e81926ab0))
|
|
64
|
+
* **deps-dev:** update opentelemetry-instrumentation-langchain requirement from >=0.40.4 to >=0.62.3 in /mcp/typescript ([#7241](https://github.com/langwatch/langwatch/issues/7241)) ([d779795](https://github.com/langwatch/langwatch/commit/d7797954d2099355c3f342cc7a2d580e81926ab0))
|
|
65
|
+
* **deps-dev:** update pypdf requirement from >=6.8.0 to >=6.15.0 in /mcp/typescript ([#7213](https://github.com/langwatch/langwatch/issues/7213)) ([f5160d6](https://github.com/langwatch/langwatch/commit/f5160d686e0d488afc4e6da02e58fc7e973272c2))
|
|
66
|
+
* **deps-dev:** update pypdf requirement in /mcp/typescript ([f5160d6](https://github.com/langwatch/langwatch/commit/f5160d686e0d488afc4e6da02e58fc7e973272c2))
|
|
67
|
+
* **deps-dev:** update pytest requirement from <10.0.0,>=7.4.2 to >=9.1.1,<10.0.0 in /mcp/typescript ([#7211](https://github.com/langwatch/langwatch/issues/7211)) ([d30d016](https://github.com/langwatch/langwatch/commit/d30d0165f77dbd035e25c3fe2ae93ac23d438378))
|
|
68
|
+
* **deps-dev:** update pytest requirement in /mcp/typescript ([d30d016](https://github.com/langwatch/langwatch/commit/d30d0165f77dbd035e25c3fe2ae93ac23d438378))
|
|
69
|
+
* **deps-dev:** update pytest-asyncio requirement from <1.5.0,>=0.21.1 to >=1.4.0,<1.5.0 in /mcp/typescript ([#7231](https://github.com/langwatch/langwatch/issues/7231)) ([752f2f9](https://github.com/langwatch/langwatch/commit/752f2f9e638b5ea98bf6d2d7a4cfc58a67dde7d1))
|
|
70
|
+
* **deps-dev:** update pytest-asyncio requirement in /mcp/typescript ([752f2f9](https://github.com/langwatch/langwatch/commit/752f2f9e638b5ea98bf6d2d7a4cfc58a67dde7d1))
|
|
71
|
+
* **deps-dev:** update strands-agents[otel] requirement ([fd972d9](https://github.com/langwatch/langwatch/commit/fd972d99251385b4dcd16e72a8a444cd4483944b))
|
|
72
|
+
* **deps-dev:** update strands-agents[otel] requirement from <2.0.0,>=1.14.0 to >=1.51.0,<2.0.0 in /mcp/typescript ([#7246](https://github.com/langwatch/langwatch/issues/7246)) ([fd972d9](https://github.com/langwatch/langwatch/commit/fd972d99251385b4dcd16e72a8a444cd4483944b))
|
|
73
|
+
* **deps-dev:** update streamlit requirement from <2.0.0,>=1.37.1 to >=1.61.1,<2.0.0 in /mcp/typescript ([#7233](https://github.com/langwatch/langwatch/issues/7233)) ([042aa3a](https://github.com/langwatch/langwatch/commit/042aa3aa875389929d85cd0896d04ae57490ca43))
|
|
74
|
+
* **deps-dev:** update streamlit requirement in /mcp/typescript ([042aa3a](https://github.com/langwatch/langwatch/commit/042aa3aa875389929d85cd0896d04ae57490ca43))
|
|
75
|
+
* **deps:** bump aiohttp from 3.14.1 to 3.14.3 in /mcp/typescript ([#7216](https://github.com/langwatch/langwatch/issues/7216)) ([590fa78](https://github.com/langwatch/langwatch/commit/590fa78069af737cddfe527f9665fd392355b07f))
|
|
76
|
+
* **deps:** bump attrs from 25.3.0 to 26.1.0 in /mcp/typescript ([#7234](https://github.com/langwatch/langwatch/issues/7234)) ([040afbd](https://github.com/langwatch/langwatch/commit/040afbd9d4292d6ae0ef1da958137e7a32873fc3))
|
|
77
|
+
* **deps:** bump chalk from 5.6.2 to 6.0.0 ([#6803](https://github.com/langwatch/langwatch/issues/6803)) ([82c62ea](https://github.com/langwatch/langwatch/commit/82c62eacb15d9a9ae33e5f4a14da82feaf1de033))
|
|
78
|
+
* **deps:** bump coolname from 2.2.0 to 5.0.0 in /mcp/typescript ([#7240](https://github.com/langwatch/langwatch/issues/7240)) ([0b7d6fa](https://github.com/langwatch/langwatch/commit/0b7d6fa30305ee1ca4d02b0a483593f0792a16dd))
|
|
79
|
+
* **deps:** bump cryptography from 48.0.1 to 50.0.0 in /mcp/typescript ([#7217](https://github.com/langwatch/langwatch/issues/7217)) ([dda8dad](https://github.com/langwatch/langwatch/commit/dda8dad79090bffbed3ed01870e24ee94990dde8))
|
|
80
|
+
* **deps:** bump ipykernel from 6.30.1 to 7.3.0 in /mcp/typescript ([#7221](https://github.com/langwatch/langwatch/issues/7221)) ([8d8442b](https://github.com/langwatch/langwatch/commit/8d8442bb5072ef8b684c17403440efc99bf00470))
|
|
81
|
+
* **deps:** bump mcp from 1.23.0 to 1.28.1 in /mcp/typescript ([#7219](https://github.com/langwatch/langwatch/issues/7219)) ([c243b25](https://github.com/langwatch/langwatch/commit/c243b259905caa263adc530a7a804c7eb9268321))
|
|
82
|
+
* **deps:** bump the minor-and-patch group ([8976b2f](https://github.com/langwatch/langwatch/commit/8976b2f420dc9b9eefd5584465b611b9064bf79e))
|
|
83
|
+
* **deps:** bump the minor-and-patch group in /mcp/typescript with 2 updates ([#7208](https://github.com/langwatch/langwatch/issues/7208)) ([8976b2f](https://github.com/langwatch/langwatch/commit/8976b2f420dc9b9eefd5584465b611b9064bf79e))
|
|
84
|
+
* **deps:** bump the opentelemetry group across 1 directory with 25 updates ([#7296](https://github.com/langwatch/langwatch/issues/7296)) ([12615f1](https://github.com/langwatch/langwatch/commit/12615f1fc27dfc26063d4135f3f322f7c71c7f01))
|
|
85
|
+
* **deps:** update langchain-core requirement from <2.0.0,>=0.3.0 to >=1.5.4,<2.0.0 in /mcp/typescript ([#7239](https://github.com/langwatch/langwatch/issues/7239)) ([ed9ba32](https://github.com/langwatch/langwatch/commit/ed9ba32c4851339a45c3914eb28da8a2ffaf040a))
|
|
86
|
+
* **deps:** update langchain-core requirement in /mcp/typescript ([ed9ba32](https://github.com/langwatch/langwatch/commit/ed9ba32c4851339a45c3914eb28da8a2ffaf040a))
|
|
87
|
+
* **deps:** update mcp[cli] requirement from >=1.13.1 to >=2.0.0 in /mcp/typescript ([#7212](https://github.com/langwatch/langwatch/issues/7212)) ([32859df](https://github.com/langwatch/langwatch/commit/32859dfc457260f098274d0f7df070b2cd9b489e))
|
|
88
|
+
* **deps:** update mcp[cli] requirement in /mcp/typescript ([32859df](https://github.com/langwatch/langwatch/commit/32859dfc457260f098274d0f7df070b2cd9b489e))
|
|
89
|
+
* **deps:** update ruff requirement from >=0.11.1 to >=0.16.2 in /mcp/typescript ([#7244](https://github.com/langwatch/langwatch/issues/7244)) ([5e33035](https://github.com/langwatch/langwatch/commit/5e33035b20bceb14c7c541d784c79d3817f35540))
|
|
90
|
+
* **deps:** update ruff requirement in /mcp/typescript ([5e33035](https://github.com/langwatch/langwatch/commit/5e33035b20bceb14c7c541d784c79d3817f35540))
|
|
91
|
+
|
|
92
|
+
## [2.0.0](https://github.com/langwatch/langwatch/compare/mcp-server@v1.0.0...mcp-server@v2.0.0) (2026-08-07)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
### ⚠ BREAKING CHANGES
|
|
96
|
+
|
|
97
|
+
* **mcp-server:** three behaviors change for callers of the HTTP server. A client that authenticated once at initialize and then sent only the session id must now send Authorization: Bearer <key> on every request. GET /sse?apiKey=<key> no longer authenticates; use the Authorization header. A deployment relying on the implicit 0.0.0.0 bind must pass --host 0.0.0.0, which the published container image already does.
|
|
98
|
+
|
|
99
|
+
### Features
|
|
100
|
+
|
|
101
|
+
* **api-docs:** publish the REST API the SDKs actually call, and gate route coverage ([#6605](https://github.com/langwatch/langwatch/issues/6605)) ([25884bf](https://github.com/langwatch/langwatch/commit/25884bf3992eb99c00c3c7bafb09b979b02f0013))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
### Bug Fixes
|
|
105
|
+
|
|
106
|
+
* **mcp-server:** authorize every request to the standalone HTTP server ([#6641](https://github.com/langwatch/langwatch/issues/6641)) ([af324de](https://github.com/langwatch/langwatch/commit/af324deeb3700f2745783a46e6fd5e8625ef868e))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
### Code Refactoring
|
|
110
|
+
|
|
111
|
+
* **build:** single pnpm workspace + restructure into sdks/ platform/ services/ mcp/ infra/ (ADR-076) ([#6405](https://github.com/langwatch/langwatch/issues/6405)) ([2bbeb7d](https://github.com/langwatch/langwatch/commit/2bbeb7dd2e0747e81478f22c3165adb9163c9f99))
|
|
112
|
+
|
|
3
113
|
## [1.0.0](https://github.com/langwatch/langwatch/compare/mcp-server@v0.10.1...mcp-server@v1.0.0) (2026-07-24)
|
|
4
114
|
|
|
5
115
|
|
package/CONTRIBUTING.md
CHANGED
|
@@ -15,7 +15,7 @@ Thank you for your interest in contributing to the LangWatch MCP Server! This gu
|
|
|
15
15
|
1. **Clone the repository and navigate to the MCP server directory:**
|
|
16
16
|
```bash
|
|
17
17
|
git clone https://github.com/langwatch/langwatch.git
|
|
18
|
-
cd langwatch/mcp
|
|
18
|
+
cd langwatch/mcp/typescript
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
2. **Install dependencies and build the MCP server:**
|
package/README.md
CHANGED
|
@@ -16,7 +16,7 @@ For Claude Code (manual) or other MCP clients (Cursor, Copilot, etc.), add an en
|
|
|
16
16
|
- `args`: `-y`, then the package name `@langwatch/mcp-server` on a separate token
|
|
17
17
|
- `env.LANGWATCH_API_KEY`: your LangWatch API key
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
In stdio mode the API key is required for observability and prompt tools, and documentation tools work without it. In HTTP mode every MCP request needs a valid key, including documentation-tool requests, because the server authenticates the request before it reaches any tool.
|
|
20
20
|
|
|
21
21
|
> The config is described in prose rather than as a JSON snippet because some CLI agents (notably Gemini CLI 0.36.0 and earlier) parse `@`-prefixed runs as file paths and can crash with `ENAMETOOLONG` when a multi-line JSON snippet wraps the scoped package name in double quotes. See [#3104](https://github.com/langwatch/langwatch/issues/3104). The bash form above is safe — the package name is followed by a whitespace terminator.
|
|
22
22
|
|
|
@@ -26,6 +26,60 @@ The API key is required for observability and prompt tools. Documentation tools
|
|
|
26
26
|
|---------|---------|-------------|
|
|
27
27
|
| `LANGWATCH_API_KEY` | `--apiKey` | API key for authentication |
|
|
28
28
|
| `LANGWATCH_ENDPOINT` | `--endpoint` | API endpoint (default: `https://app.langwatch.ai`) |
|
|
29
|
+
| | `--http` | Serve over HTTP and SSE instead of stdio |
|
|
30
|
+
| | `--port` | HTTP port (default: `3000`) |
|
|
31
|
+
| `LANGWATCH_MCP_HTTP_HOST` | `--host` | HTTP listen address (default: `127.0.0.1`) |
|
|
32
|
+
| `LANGWATCH_MCP_ALLOWED_ORIGINS` | `--allowedOrigin` | Browser origins allowed to call the HTTP server |
|
|
33
|
+
| `LANGWATCH_MCP_TRUST_PROXY` | | Use `X-Forwarded-For` as the rate limit key (default: off) |
|
|
34
|
+
|
|
35
|
+
### HTTP mode
|
|
36
|
+
|
|
37
|
+
In HTTP mode each client brings its own API key in an `Authorization: Bearer <key>`
|
|
38
|
+
header, on every MCP request rather than only at session start. That covers
|
|
39
|
+
`/mcp`, `/sse`, and `/messages`. The key is checked against the LangWatch API
|
|
40
|
+
before a session is created and re-checked on each request, so a session id by
|
|
41
|
+
itself grants no access and revoking a key stops its sessions being served
|
|
42
|
+
within a minute. The API key is never read from a query parameter.
|
|
43
|
+
|
|
44
|
+
Because authentication happens before routing, this applies to the documentation
|
|
45
|
+
tools too: unlike stdio mode, they are not reachable without a valid key.
|
|
46
|
+
|
|
47
|
+
Three routes do not take the bearer. `/health` reports liveness only.
|
|
48
|
+
`/.well-known/oauth-authorization-server` is the discovery document clients read
|
|
49
|
+
before they hold a token. `/oauth/token` exchanges an API key, sent as
|
|
50
|
+
`client_secret` in the form body, for a short-lived access token; it verifies
|
|
51
|
+
that key against the LangWatch API before issuing anything, and the token it
|
|
52
|
+
returns is what later MCP requests carry.
|
|
53
|
+
|
|
54
|
+
The server listens on `127.0.0.1` by default, per the MCP transport guidance for
|
|
55
|
+
local servers. Pass `--host 0.0.0.0` to accept connections from other machines,
|
|
56
|
+
and only behind a network boundary you trust.
|
|
57
|
+
|
|
58
|
+
Requests carrying a browser `Origin` header are checked against an allowlist.
|
|
59
|
+
Loopback origins are always allowed; anything else has to be listed:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
npx @langwatch/mcp-server --http --port 3000 \
|
|
63
|
+
--allowedOrigin https://your-app.example.com
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
#### Running behind a proxy
|
|
67
|
+
|
|
68
|
+
Forwarded proxy headers resolve the external scheme that the OAuth metadata
|
|
69
|
+
document advertises, so they are read by default.
|
|
70
|
+
|
|
71
|
+
They do not decide the rate limit. Failed authentication is limited per client
|
|
72
|
+
address, and that address comes from the socket rather than from
|
|
73
|
+
`X-Forwarded-For`, so a client reaching the port directly cannot rotate a header
|
|
74
|
+
to reset its own counter. Behind a real proxy every request shares the proxy's
|
|
75
|
+
socket address, which means the whole proxy is limited as one client. Set
|
|
76
|
+
`LANGWATCH_MCP_TRUST_PROXY=true` there to limit per real client instead, and
|
|
77
|
+
only when the proxy overwrites `X-Forwarded-For` on the way in.
|
|
78
|
+
|
|
79
|
+
This is defense in depth, not the security boundary. API keys are verified
|
|
80
|
+
against the LangWatch API before a session is created and re-checked on every
|
|
81
|
+
request; the rate limit exists to make guessing expensive, not to decide who
|
|
82
|
+
gets in. A spoofable rate limit does not let anyone authenticate.
|
|
29
83
|
|
|
30
84
|
## Tools
|
|
31
85
|
|
|
@@ -71,12 +125,40 @@ The API key is required for observability and prompt tools. Documentation tools
|
|
|
71
125
|
|
|
72
126
|
| Tool | Description |
|
|
73
127
|
|------|-------------|
|
|
74
|
-
| `platform_list_scenarios` | List all scenarios |
|
|
128
|
+
| `platform_list_scenarios` | List all scenarios, or only the ones filed in a test suite |
|
|
75
129
|
| `platform_get_scenario` | Get scenario details |
|
|
76
|
-
| `platform_create_scenario` | Create a new scenario |
|
|
77
|
-
| `platform_update_scenario` | Update a scenario |
|
|
130
|
+
| `platform_create_scenario` | Create a new scenario, optionally filed in a test suite |
|
|
131
|
+
| `platform_update_scenario` | Update a scenario, or file it in another test suite |
|
|
78
132
|
| `platform_archive_scenario` | Archive a scenario |
|
|
79
133
|
|
|
134
|
+
### Test suites and run plans (requires API key)
|
|
135
|
+
|
|
136
|
+
A test suite is a folder of scenarios. A run plan is what you run, and its
|
|
137
|
+
name identifies it: running a name that exists replaces that plan's
|
|
138
|
+
configuration, running a new name creates the plan. Running a test suite is
|
|
139
|
+
sugar over a run plan, and creates or joins the plan `<suite name> <target name>`.
|
|
140
|
+
|
|
141
|
+
| Tool | Description |
|
|
142
|
+
|------|-------------|
|
|
143
|
+
| `platform_run_plan` | Run scenarios against targets, by plan name |
|
|
144
|
+
| `platform_list_run_plans` | List the run plans of the project |
|
|
145
|
+
| `platform_get_run_plan` | Get the full configuration of a run plan |
|
|
146
|
+
| `platform_rerun_run_plan` | Run a plan again with the configuration it holds |
|
|
147
|
+
| `platform_archive_run_plan` | Archive a run plan |
|
|
148
|
+
| `platform_list_test_suites` | List the test suites of the project |
|
|
149
|
+
| `platform_create_test_suite` | Create a test suite |
|
|
150
|
+
| `platform_get_test_suite` | Get a test suite and the scenarios filed in it |
|
|
151
|
+
| `platform_rename_test_suite` | Rename a test suite |
|
|
152
|
+
| `platform_archive_test_suite` | Archive a test suite and the scenarios filed in it |
|
|
153
|
+
| `platform_run_test_suite` | Run every scenario of a test suite against targets |
|
|
154
|
+
|
|
155
|
+
### Simulation runs (requires API key)
|
|
156
|
+
|
|
157
|
+
| Tool | Description |
|
|
158
|
+
|------|-------------|
|
|
159
|
+
| `platform_list_simulation_runs` | List simulation run results |
|
|
160
|
+
| `platform_get_simulation_run` | Get one run with its conversation and verdict |
|
|
161
|
+
|
|
80
162
|
### Evaluators (requires API key)
|
|
81
163
|
|
|
82
164
|
| Tool | Description |
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
archiveProject
|
|
3
|
-
} from "./chunk-
|
|
4
|
-
import "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
3
|
+
} from "./chunk-G4ZAZTZW.js";
|
|
4
|
+
import "./chunk-GELJ2MQR.js";
|
|
5
|
+
import "./chunk-WT4PO4A3.js";
|
|
6
|
+
import "./chunk-SAS5OB7F.js";
|
|
7
7
|
|
|
8
8
|
// src/tools/archive-project.ts
|
|
9
9
|
async function handleArchiveProject(params) {
|
|
@@ -17,4 +17,4 @@ async function handleArchiveProject(params) {
|
|
|
17
17
|
export {
|
|
18
18
|
handleArchiveProject
|
|
19
19
|
};
|
|
20
|
-
//# sourceMappingURL=archive-project-
|
|
20
|
+
//# sourceMappingURL=archive-project-W3IKI7UL.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import {
|
|
2
|
+
archiveRunPlan
|
|
3
|
+
} from "./chunk-56OGTS24.js";
|
|
4
|
+
import "./chunk-GELJ2MQR.js";
|
|
5
|
+
import "./chunk-WT4PO4A3.js";
|
|
6
|
+
import "./chunk-SAS5OB7F.js";
|
|
7
|
+
|
|
8
|
+
// src/tools/archive-run-plan.ts
|
|
9
|
+
async function handleArchiveRunPlan(params) {
|
|
10
|
+
const result = await archiveRunPlan(params.id);
|
|
11
|
+
return `Run plan ${result.id} is archived. Its past runs stay readable, and the name is free for a new plan.`;
|
|
12
|
+
}
|
|
13
|
+
export {
|
|
14
|
+
handleArchiveRunPlan
|
|
15
|
+
};
|
|
16
|
+
//# sourceMappingURL=archive-run-plan-474OKHHU.js.map
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
archiveScenario
|
|
3
|
-
} from "./chunk-
|
|
4
|
-
import "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
3
|
+
} from "./chunk-RVAE2MEA.js";
|
|
4
|
+
import "./chunk-GELJ2MQR.js";
|
|
5
|
+
import "./chunk-WT4PO4A3.js";
|
|
6
|
+
import "./chunk-SAS5OB7F.js";
|
|
7
7
|
|
|
8
8
|
// src/tools/archive-scenario.ts
|
|
9
9
|
async function handleArchiveScenario(params) {
|
|
@@ -17,4 +17,4 @@ async function handleArchiveScenario(params) {
|
|
|
17
17
|
export {
|
|
18
18
|
handleArchiveScenario
|
|
19
19
|
};
|
|
20
|
-
//# sourceMappingURL=archive-scenario-
|
|
20
|
+
//# sourceMappingURL=archive-scenario-V5UNYK26.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import {
|
|
2
|
+
archiveTestSuite
|
|
3
|
+
} from "./chunk-6DJDEXET.js";
|
|
4
|
+
import "./chunk-GELJ2MQR.js";
|
|
5
|
+
import "./chunk-WT4PO4A3.js";
|
|
6
|
+
import "./chunk-SAS5OB7F.js";
|
|
7
|
+
|
|
8
|
+
// src/tools/archive-test-suite.ts
|
|
9
|
+
async function handleArchiveTestSuite(params) {
|
|
10
|
+
const result = await archiveTestSuite(params.id);
|
|
11
|
+
return `Test suite ${result.id} is archived, and the scenarios filed in it are archived with it.`;
|
|
12
|
+
}
|
|
13
|
+
export {
|
|
14
|
+
handleArchiveTestSuite
|
|
15
|
+
};
|
|
16
|
+
//# sourceMappingURL=archive-test-suite-O7FLDMXK.js.map
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
assignPromptTag
|
|
3
|
-
} from "./chunk-
|
|
4
|
-
import "./chunk-
|
|
5
|
-
import "./chunk-
|
|
3
|
+
} from "./chunk-GELJ2MQR.js";
|
|
4
|
+
import "./chunk-WT4PO4A3.js";
|
|
5
|
+
import "./chunk-SAS5OB7F.js";
|
|
6
6
|
|
|
7
7
|
// src/tools/assign-prompt-tag.ts
|
|
8
8
|
async function handleAssignPromptTag(params) {
|
|
@@ -17,4 +17,4 @@ async function handleAssignPromptTag(params) {
|
|
|
17
17
|
export {
|
|
18
18
|
handleAssignPromptTag
|
|
19
19
|
};
|
|
20
|
-
//# sourceMappingURL=assign-prompt-tag-
|
|
20
|
+
//# sourceMappingURL=assign-prompt-tag-Q46WRB6P.js.map
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import {
|
|
2
|
+
makeRequest
|
|
3
|
+
} from "./chunk-GELJ2MQR.js";
|
|
4
|
+
|
|
5
|
+
// src/langwatch-api-run-plans.ts
|
|
6
|
+
async function listRunPlans(params) {
|
|
7
|
+
const query = params?.includeArchived ? "?includeArchived=true" : "";
|
|
8
|
+
return makeRequest("GET", `/api/v1/run-plans${query}`);
|
|
9
|
+
}
|
|
10
|
+
async function getRunPlan(id) {
|
|
11
|
+
return makeRequest(
|
|
12
|
+
"GET",
|
|
13
|
+
`/api/v1/run-plans/${encodeURIComponent(id)}`
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
async function runRunPlan(data) {
|
|
17
|
+
return makeRequest(
|
|
18
|
+
"POST",
|
|
19
|
+
"/api/v1/run-plans/run",
|
|
20
|
+
data
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
async function rerunRunPlan(id, data) {
|
|
24
|
+
return makeRequest(
|
|
25
|
+
"POST",
|
|
26
|
+
`/api/v1/run-plans/${encodeURIComponent(id)}/run`,
|
|
27
|
+
data ?? {}
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
async function archiveRunPlan(id) {
|
|
31
|
+
return makeRequest(
|
|
32
|
+
"DELETE",
|
|
33
|
+
`/api/v1/run-plans/${encodeURIComponent(id)}`
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export {
|
|
38
|
+
listRunPlans,
|
|
39
|
+
getRunPlan,
|
|
40
|
+
runRunPlan,
|
|
41
|
+
rerunRunPlan,
|
|
42
|
+
archiveRunPlan
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=chunk-56OGTS24.js.map
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import {
|
|
2
|
+
makeRequest
|
|
3
|
+
} from "./chunk-GELJ2MQR.js";
|
|
4
|
+
|
|
5
|
+
// src/langwatch-api-test-suites.ts
|
|
6
|
+
async function listTestSuites() {
|
|
7
|
+
return makeRequest("GET", "/api/v1/test-suites");
|
|
8
|
+
}
|
|
9
|
+
async function createTestSuite(data) {
|
|
10
|
+
return makeRequest(
|
|
11
|
+
"POST",
|
|
12
|
+
"/api/v1/test-suites",
|
|
13
|
+
data
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
async function getTestSuite(id) {
|
|
17
|
+
return makeRequest(
|
|
18
|
+
"GET",
|
|
19
|
+
`/api/v1/test-suites/${encodeURIComponent(id)}`
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
async function updateTestSuite(params) {
|
|
23
|
+
const { id, ...data } = params;
|
|
24
|
+
return makeRequest(
|
|
25
|
+
"PATCH",
|
|
26
|
+
`/api/v1/test-suites/${encodeURIComponent(id)}`,
|
|
27
|
+
data
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
async function renameTestSuite(params) {
|
|
31
|
+
return makeRequest(
|
|
32
|
+
"PATCH",
|
|
33
|
+
`/api/v1/test-suites/${encodeURIComponent(params.id)}`,
|
|
34
|
+
{ name: params.name }
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
async function archiveTestSuite(id) {
|
|
38
|
+
return makeRequest(
|
|
39
|
+
"DELETE",
|
|
40
|
+
`/api/v1/test-suites/${encodeURIComponent(id)}`
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
async function runTestSuite({
|
|
44
|
+
id,
|
|
45
|
+
...data
|
|
46
|
+
}) {
|
|
47
|
+
return makeRequest(
|
|
48
|
+
"POST",
|
|
49
|
+
`/api/v1/test-suites/${encodeURIComponent(id)}/run`,
|
|
50
|
+
data
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export {
|
|
55
|
+
listTestSuites,
|
|
56
|
+
createTestSuite,
|
|
57
|
+
getTestSuite,
|
|
58
|
+
updateTestSuite,
|
|
59
|
+
renameTestSuite,
|
|
60
|
+
archiveTestSuite,
|
|
61
|
+
runTestSuite
|
|
62
|
+
};
|
|
63
|
+
//# sourceMappingURL=chunk-6DJDEXET.js.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// src/tools/format-scenario.ts
|
|
2
|
+
function formatScenarioFields(fields) {
|
|
3
|
+
if (!fields || Object.keys(fields).length === 0) return [];
|
|
4
|
+
return [
|
|
5
|
+
"\n## Fields",
|
|
6
|
+
...Object.entries(fields).map(
|
|
7
|
+
([identifier, value]) => `- ${identifier}: ${JSON.stringify(value)}`
|
|
8
|
+
)
|
|
9
|
+
];
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export {
|
|
13
|
+
formatScenarioFields
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=chunk-7E4FZIYJ.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
makeRequest
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-GELJ2MQR.js";
|
|
4
4
|
|
|
5
5
|
// src/langwatch-api-datasets.ts
|
|
6
6
|
async function listDatasets(params) {
|
|
@@ -83,4 +83,4 @@ export {
|
|
|
83
83
|
deleteDatasetRecords,
|
|
84
84
|
listDatasetRecords
|
|
85
85
|
};
|
|
86
|
-
//# sourceMappingURL=chunk-
|
|
86
|
+
//# sourceMappingURL=chunk-AN4OUA4D.js.map
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// src/schemas/suite-fields.ts
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
var SUITE_FIELD_TYPES = ["text", "number", "boolean"];
|
|
4
|
+
var suiteFieldSchema = z.object({
|
|
5
|
+
identifier: z.string().min(1).max(64).regex(
|
|
6
|
+
/^[a-z][a-z0-9_]*$/,
|
|
7
|
+
"Lowercase letters, digits and underscores, starting with a letter"
|
|
8
|
+
).describe(
|
|
9
|
+
"The field name, as scenarios and evaluator mappings address it, for example golden_sql. Lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved."
|
|
10
|
+
),
|
|
11
|
+
type: z.enum(SUITE_FIELD_TYPES).describe("The value type every scenario carries for this field.")
|
|
12
|
+
});
|
|
13
|
+
var suiteFieldsSchema = z.array(suiteFieldSchema).max(30).describe(
|
|
14
|
+
"The fields the test suite declares, in order. Up to 30. Every scenario filed in the suite carries one value per field, and an evaluator attached to the suite reads them through its mappings."
|
|
15
|
+
);
|
|
16
|
+
var SCENARIO_MAPPING_SOURCE_IDS = [
|
|
17
|
+
"conversation",
|
|
18
|
+
"scenario",
|
|
19
|
+
"trace"
|
|
20
|
+
];
|
|
21
|
+
var scenarioMappingSchema = z.discriminatedUnion("type", [
|
|
22
|
+
z.object({
|
|
23
|
+
type: z.literal("source"),
|
|
24
|
+
sourceId: z.enum(SCENARIO_MAPPING_SOURCE_IDS).describe(
|
|
25
|
+
"conversation, scenario or trace. See discover_schema({ category: 'scenarios' }) for the paths each one offers."
|
|
26
|
+
),
|
|
27
|
+
path: z.array(z.string().min(1).max(256)).min(1).max(3).describe(
|
|
28
|
+
"The path inside the source. conversation: [first_user_message] | [last_agent_message] | [transcript] | [messages]. scenario: [situation] | [criteria] | [fields, <identifier>]. trace: [contexts] | [tool_calls, <toolName>, input|output]."
|
|
29
|
+
)
|
|
30
|
+
}),
|
|
31
|
+
z.object({
|
|
32
|
+
type: z.literal("value"),
|
|
33
|
+
value: z.string().max(65536).describe("A literal the input reads.")
|
|
34
|
+
})
|
|
35
|
+
]).describe(
|
|
36
|
+
"Where one evaluator input reads its value: a path into the conversation, the scenario or the trace, or a literal value."
|
|
37
|
+
);
|
|
38
|
+
var evaluatorAttachmentSchema = z.object({
|
|
39
|
+
id: z.string().min(1).max(64).optional().describe(
|
|
40
|
+
"The attachment id, stable across edits. Leave it out on a new attachment and one is generated."
|
|
41
|
+
),
|
|
42
|
+
evaluatorId: z.string().min(1).describe(
|
|
43
|
+
"The id of the saved evaluator to run. Read it from platform_list_evaluators or platform_get_evaluator."
|
|
44
|
+
),
|
|
45
|
+
required: z.boolean().optional().describe(
|
|
46
|
+
"Whether a failing result fails the scenario. Defaults to true. A score-only evaluator reports and never gates, whatever this says."
|
|
47
|
+
),
|
|
48
|
+
mappings: z.record(z.string().min(1).max(128), scenarioMappingSchema).describe(
|
|
49
|
+
"Where each evaluator input reads its value, keyed by input name. Read the inputs from platform_get_evaluator. A required input with no mapping makes the platform refuse the run until it is set."
|
|
50
|
+
)
|
|
51
|
+
});
|
|
52
|
+
var evaluatorAttachmentsSchema = z.array(evaluatorAttachmentSchema).max(20).describe(
|
|
53
|
+
"The evaluators that run after every scenario run, with their mappings. Up to 20. A required evaluator that fails fails the scenario; a score-only one reports beside the verdict."
|
|
54
|
+
);
|
|
55
|
+
var scenarioFieldValuesSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).describe(
|
|
56
|
+
"The scenario's value for each field its test suite declares, by identifier, for example { golden_sql: 'SELECT ...' }. A text field takes a string, a number field a number and a boolean field true or false. A field the suite does not declare is refused. On update, an empty object clears the values."
|
|
57
|
+
);
|
|
58
|
+
var newAttachmentId = () => `att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
|
|
59
|
+
function toWireAttachments(attachments) {
|
|
60
|
+
return attachments.map((attachment) => ({
|
|
61
|
+
id: attachment.id ?? newAttachmentId(),
|
|
62
|
+
evaluatorId: attachment.evaluatorId,
|
|
63
|
+
required: attachment.required ?? true,
|
|
64
|
+
mappings: attachment.mappings
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export {
|
|
69
|
+
suiteFieldsSchema,
|
|
70
|
+
evaluatorAttachmentsSchema,
|
|
71
|
+
scenarioFieldValuesSchema,
|
|
72
|
+
toWireAttachments
|
|
73
|
+
};
|
|
74
|
+
//# sourceMappingURL=chunk-CKNAH5TG.js.map
|