@langwatch/mcp-server 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +192 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-65QN3XRX.js} +4 -4
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-NKZDZY3D.js} +4 -4
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-OJLYYJWI.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-G4ZAZTZW.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-EYXMLE5X.js → chunk-HGELNIVY.js} +56 -47
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-RVAE2MEA.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-S4GVX36W.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-QRO35SOM.js → chunk-J7PQKBLD.js} +652 -528
- package/dist/{chunk-SR6QPRIE.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-UFOYFYI7.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-56OGTS24.js → chunk-PTDR56Z3.js} +4 -11
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-QAMP3CGR.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/{chunk-6DJDEXET.js → chunk-VMWJD732.js} +6 -12
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-Z76AIPLS.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -10
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-ZQ7TFIO2.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-UEM77OE7.js} +5 -5
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-4IPXVSFQ.js} +6 -6
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-LYMRPF27.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-OTIYUFZA.js} +15 -11
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-P4WWVHFU.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-QSQEAB33.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-OWXJZHQQ.js} +53 -58
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-BE3M2AUX.js} +7 -9
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-YM4ODWW6.js} +5 -5
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-LCFSMPGQ.js} +26 -22
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GC2O7YH6.js} +5 -5
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-IR4QOS43.js} +4071 -347
- package/dist/index.js +58671 -37983
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-2KJL2M42.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/{list-agents-AMVDQPQL.js → list-agents-HVYWEQMI.js} +4 -4
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-Z3BZKDGF.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-PX2XI6EK.js} +5 -5
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-QZRPVHS4.js} +13 -11
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-OEADLLQR.js} +12 -8
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-N4WVLBKK.js} +4 -4
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-4J7UX5S2.js} +4 -4
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-DCZFLGDO.js} +81 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-TG2V4WM4.js} +5 -5
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-PAARQSI4.js} +7 -7
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-SQJELEZX.js} +6 -6
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/run-workflow-SP32SO5M.js +36 -0
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-DUWNYBNA.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-Q26FBSYH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-MY263DYF.js} +27 -29
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-MOHRQKAQ.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +755 -609
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +108 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
package/src/create-mcp-server.ts
CHANGED
|
@@ -2,33 +2,43 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
|
|
4
4
|
import packageJson from "../package.json" with { type: "json" };
|
|
5
|
-
import { requireApiKey } from "./config.
|
|
6
|
-
import { fetchDocumentation } from "./documentation-fetch.
|
|
7
|
-
import {
|
|
8
|
-
createDatasetSchema,
|
|
9
|
-
datasetColumnDefinitionSchema,
|
|
10
|
-
} from "./schemas/create-dataset.js";
|
|
5
|
+
import { requireApiKey } from "./config.ts";
|
|
6
|
+
import { fetchDocumentation } from "./documentation-fetch.ts";
|
|
7
|
+
import { createDatasetSchema, datasetColumnDefinitionSchema } from "./schemas/create-dataset.ts";
|
|
11
8
|
import {
|
|
12
9
|
runParametersSchema,
|
|
13
10
|
runPlanScopeSchema,
|
|
14
11
|
runPlanTargetSchema,
|
|
15
|
-
} from "./schemas/run-plan.
|
|
12
|
+
} from "./schemas/run-plan.ts";
|
|
16
13
|
import {
|
|
17
14
|
evaluatorAttachmentsSchema,
|
|
18
15
|
scenarioFieldValuesSchema,
|
|
19
16
|
suiteFieldsSchema,
|
|
20
|
-
} from "./schemas/suite-fields.
|
|
21
|
-
import {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
17
|
+
} from "./schemas/suite-fields.ts";
|
|
18
|
+
import {
|
|
19
|
+
actionParamsSchema,
|
|
20
|
+
alertTypeSchema,
|
|
21
|
+
graphAlertSchema,
|
|
22
|
+
notificationCadenceSchema,
|
|
23
|
+
reportSchema,
|
|
24
|
+
SLACK_DELIVERY_NOTE,
|
|
25
|
+
templatesSchema,
|
|
26
|
+
TRIGGER_FILTER_QUERY_DESCRIPTION,
|
|
27
|
+
TRIGGER_FILTERS_DESCRIPTION,
|
|
28
|
+
triggerActionSchema,
|
|
29
|
+
validateActionParamsForAction,
|
|
30
|
+
} from "./schemas/triggers.ts";
|
|
31
|
+
import { handleExperimentResults } from "./tools/get-experiment-results.ts";
|
|
32
|
+
import { handleExperimentListRuns } from "./tools/list-experiment-runs.ts";
|
|
33
|
+
import { handleExperimentList } from "./tools/list-experiments.ts";
|
|
34
|
+
import { handleRunExperiment, handleExperimentStatus } from "./tools/run-experiment.ts";
|
|
35
|
+
import { handleTestAgent } from "./tools/test-agent.ts";
|
|
36
|
+
import { handleUpdateTestSuite } from "./tools/update-test-suite.ts";
|
|
27
37
|
|
|
28
38
|
const modelSchema = z
|
|
29
39
|
.string()
|
|
30
40
|
.describe(
|
|
31
|
-
'Model in "provider/model-name" format, e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929"'
|
|
41
|
+
'Model in "provider/model-name" format, e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929"',
|
|
32
42
|
);
|
|
33
43
|
|
|
34
44
|
/**
|
|
@@ -71,6 +81,29 @@ function withToolLogging<T extends unknown[], R>(
|
|
|
71
81
|
}
|
|
72
82
|
|
|
73
83
|
function registerTools(server: McpServer): void {
|
|
84
|
+
registerDocsTools(server);
|
|
85
|
+
registerObservabilityTools(server);
|
|
86
|
+
registerPromptTools(server);
|
|
87
|
+
registerScenarioTools(server);
|
|
88
|
+
registerRunPlanTools(server);
|
|
89
|
+
registerTestSuiteTools(server);
|
|
90
|
+
registerSimulationRunTools(server);
|
|
91
|
+
registerEvaluatorTools(server);
|
|
92
|
+
registerModelProviderTools(server);
|
|
93
|
+
registerAgentTools(server);
|
|
94
|
+
registerDashboardTools(server);
|
|
95
|
+
registerWorkflowTools(server);
|
|
96
|
+
registerAnnotationTools(server);
|
|
97
|
+
registerTriggerTools(server);
|
|
98
|
+
registerMonitorTools(server);
|
|
99
|
+
registerSecretTools(server);
|
|
100
|
+
registerExperimentTools(server);
|
|
101
|
+
registerDatasetTools(server);
|
|
102
|
+
registerProjectTools(server);
|
|
103
|
+
registerApiKeyTools(server);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function registerDocsTools(server: McpServer): void {
|
|
74
107
|
server.tool(
|
|
75
108
|
"fetch_langwatch_docs",
|
|
76
109
|
"Fetches the LangWatch docs for understanding how to implement LangWatch in your codebase. Always use this tool when the user asks for help with LangWatch. Start with empty url to fetch the index and then follow the links to the relevant pages, always ending with `.md` extension",
|
|
@@ -79,13 +112,13 @@ function registerTools(server: McpServer): void {
|
|
|
79
112
|
.string()
|
|
80
113
|
.optional()
|
|
81
114
|
.describe(
|
|
82
|
-
"The full url of the specific doc page. If not provided, the docs index will be fetched."
|
|
115
|
+
"The full url of the specific doc page. If not provided, the docs index will be fetched.",
|
|
83
116
|
),
|
|
84
117
|
},
|
|
85
118
|
withToolLogging("fetch_langwatch_docs", async ({ url }) => {
|
|
86
119
|
const text = await fetchDocumentation("langwatch", url);
|
|
87
120
|
return { content: [{ type: "text", text }] };
|
|
88
|
-
})
|
|
121
|
+
}),
|
|
89
122
|
);
|
|
90
123
|
|
|
91
124
|
server.tool(
|
|
@@ -95,36 +128,33 @@ function registerTools(server: McpServer): void {
|
|
|
95
128
|
user_approved: z
|
|
96
129
|
.boolean()
|
|
97
130
|
.describe(
|
|
98
|
-
"Must be true, and only after the user explicitly agreed to send this report to LangWatch"
|
|
131
|
+
"Must be true, and only after the user explicitly agreed to send this report to LangWatch",
|
|
99
132
|
),
|
|
100
133
|
title: z.string().max(300).describe("One-line description of the issue"),
|
|
101
134
|
summary: z
|
|
102
135
|
.string()
|
|
103
136
|
.optional()
|
|
104
137
|
.describe(
|
|
105
|
-
"What you were trying to do, what went wrong (verbatim errors), and what you had to figure out the hard way"
|
|
138
|
+
"What you were trying to do, what went wrong (verbatim errors), and what you had to figure out the hard way",
|
|
106
139
|
),
|
|
107
140
|
session_content: z
|
|
108
141
|
.string()
|
|
109
142
|
.optional()
|
|
110
143
|
.describe(
|
|
111
|
-
"Optional raw session transcript or log excerpt (JSONL or plain text); redacted locally before sending"
|
|
144
|
+
"Optional raw session transcript or log excerpt (JSONL or plain text); redacted locally before sending",
|
|
112
145
|
),
|
|
113
|
-
contact_email: z
|
|
114
|
-
.string()
|
|
115
|
-
.optional()
|
|
116
|
-
.describe("Optional contact email for follow-up"),
|
|
146
|
+
contact_email: z.string().optional().describe("Optional contact email for follow-up"),
|
|
117
147
|
agent: z
|
|
118
148
|
.string()
|
|
119
149
|
.optional()
|
|
120
150
|
.describe("Which coding agent this is, e.g. claude-code, codex, cursor"),
|
|
121
151
|
},
|
|
122
152
|
withToolLogging("report_issue", async (params) => {
|
|
123
|
-
const { handleReportIssue } = await import("./tools/report-issue.
|
|
153
|
+
const { handleReportIssue } = await import("./tools/report-issue.ts");
|
|
124
154
|
return {
|
|
125
155
|
content: [{ type: "text", text: await handleReportIssue(params) }],
|
|
126
156
|
};
|
|
127
|
-
})
|
|
157
|
+
}),
|
|
128
158
|
);
|
|
129
159
|
|
|
130
160
|
server.tool(
|
|
@@ -135,24 +165,26 @@ function registerTools(server: McpServer): void {
|
|
|
135
165
|
.string()
|
|
136
166
|
.optional()
|
|
137
167
|
.describe(
|
|
138
|
-
"The full url of the specific doc page. If not provided, the docs index will be fetched."
|
|
168
|
+
"The full url of the specific doc page. If not provided, the docs index will be fetched.",
|
|
139
169
|
),
|
|
140
170
|
},
|
|
141
171
|
withToolLogging("fetch_scenario_docs", async ({ url }) => {
|
|
142
172
|
const text = await fetchDocumentation("scenario", url);
|
|
143
173
|
return { content: [{ type: "text", text }] };
|
|
144
|
-
})
|
|
174
|
+
}),
|
|
145
175
|
);
|
|
176
|
+
}
|
|
146
177
|
|
|
147
|
-
|
|
148
|
-
|
|
178
|
+
// --- Observability Tools (require API key) ---
|
|
179
|
+
function registerObservabilityTools(server: McpServer): void {
|
|
149
180
|
server.tool(
|
|
150
181
|
"discover_schema",
|
|
151
|
-
"Discover
|
|
182
|
+
"Discover what LangWatch can be queried with: the trace filter fields and syntax, the analytics SQL views and columns, the analytics metrics, aggregation types, group-by options, scenario schema and evaluator types. Call this before using search_traces, run_query, get_analytics, scenario tools or evaluator tools, so you never guess a field, a column or a value.",
|
|
152
183
|
{
|
|
153
184
|
category: z
|
|
154
185
|
.enum([
|
|
155
186
|
"filters",
|
|
187
|
+
"lwql",
|
|
156
188
|
"metrics",
|
|
157
189
|
"aggregations",
|
|
158
190
|
"groups",
|
|
@@ -160,113 +192,149 @@ function registerTools(server: McpServer): void {
|
|
|
160
192
|
"evaluators",
|
|
161
193
|
"all",
|
|
162
194
|
])
|
|
163
|
-
.describe(
|
|
195
|
+
.describe(
|
|
196
|
+
"Which schema category to discover. 'filters' is the trace filter language, 'lwql' the analytics SQL views; both are read from the platform and need the API key",
|
|
197
|
+
),
|
|
164
198
|
evaluatorType: z
|
|
165
199
|
.string()
|
|
166
200
|
.optional()
|
|
167
201
|
.describe(
|
|
168
|
-
"When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details"
|
|
202
|
+
"When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details",
|
|
169
203
|
),
|
|
170
204
|
},
|
|
171
|
-
async ({ category, evaluatorType }) => {
|
|
205
|
+
withToolLogging("discover_schema", async ({ category, evaluatorType }) => {
|
|
172
206
|
if (category === "scenarios") {
|
|
173
|
-
const { formatScenarioSchema } = await import(
|
|
174
|
-
"./tools/discover-scenario-schema.js"
|
|
175
|
-
);
|
|
207
|
+
const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.ts");
|
|
176
208
|
return {
|
|
177
209
|
content: [{ type: "text", text: formatScenarioSchema() }],
|
|
178
210
|
};
|
|
179
211
|
}
|
|
180
212
|
if (category === "evaluators") {
|
|
181
|
-
const { formatEvaluatorSchema } = await import(
|
|
182
|
-
"./tools/discover-evaluator-schema.js"
|
|
183
|
-
);
|
|
213
|
+
const { formatEvaluatorSchema } = await import("./tools/discover-evaluator-schema.ts");
|
|
184
214
|
return {
|
|
185
|
-
content: [
|
|
186
|
-
{ type: "text", text: formatEvaluatorSchema(evaluatorType) },
|
|
187
|
-
],
|
|
215
|
+
content: [{ type: "text", text: formatEvaluatorSchema(evaluatorType) }],
|
|
188
216
|
};
|
|
189
217
|
}
|
|
190
|
-
const { formatSchema } = await import("./tools/discover-schema.
|
|
191
|
-
|
|
218
|
+
const { formatSchema, needsQueryReference } = await import("./tools/discover-schema.ts");
|
|
219
|
+
// The filter and analytics-SQL halves are the PLATFORM's registries, not
|
|
220
|
+
// copies of them, so those categories need the credential. The static
|
|
221
|
+
// ones still answer without it.
|
|
222
|
+
if (needsQueryReference(category)) requireApiKey();
|
|
223
|
+
let text = await formatSchema(category);
|
|
192
224
|
if (category === "all") {
|
|
193
|
-
const { formatScenarioSchema } = await import(
|
|
194
|
-
"./tools/discover-scenario-schema.js"
|
|
195
|
-
);
|
|
225
|
+
const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.ts");
|
|
196
226
|
text += "\n\n" + formatScenarioSchema();
|
|
197
|
-
const { formatEvaluatorSchema } = await import(
|
|
198
|
-
"./tools/discover-evaluator-schema.js"
|
|
199
|
-
);
|
|
227
|
+
const { formatEvaluatorSchema } = await import("./tools/discover-evaluator-schema.ts");
|
|
200
228
|
text += "\n\n" + formatEvaluatorSchema();
|
|
201
229
|
}
|
|
202
230
|
return { content: [{ type: "text", text }] };
|
|
203
|
-
}
|
|
231
|
+
}),
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
server.tool(
|
|
235
|
+
"run_query",
|
|
236
|
+
"Run one read-only analytics SQL statement (LangWatchQL) over this project's traces, spans, evaluations and metrics, and get the rows back as a table. This is the tool for a count, a rate, a sum, a percentile or any of them grouped or over time — questions search_traces would need many calls to answer. The statement runs exactly as written. Call discover_schema with category 'lwql' first for the views, columns and worked statements.",
|
|
237
|
+
{
|
|
238
|
+
sql: z
|
|
239
|
+
.string()
|
|
240
|
+
.describe(
|
|
241
|
+
"The SELECT to run. Filter on the dataset's time column so the read is bounded, and never select a whole attribute map",
|
|
242
|
+
),
|
|
243
|
+
parameters: z
|
|
244
|
+
.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()]))
|
|
245
|
+
.optional()
|
|
246
|
+
.describe(
|
|
247
|
+
'Values for the parameters the statement declares, e.g. {"days": 7} for {days:UInt32}',
|
|
248
|
+
),
|
|
249
|
+
},
|
|
250
|
+
withToolLogging("run_query", async (params) => {
|
|
251
|
+
requireApiKey();
|
|
252
|
+
const { handleRunQuery } = await import("./tools/run-query.ts");
|
|
253
|
+
return {
|
|
254
|
+
content: [{ type: "text", text: await handleRunQuery(params) }],
|
|
255
|
+
};
|
|
256
|
+
}),
|
|
204
257
|
);
|
|
205
258
|
|
|
206
259
|
server.tool(
|
|
207
260
|
"search_traces",
|
|
208
|
-
"Search LangWatch traces
|
|
261
|
+
"Search LangWatch traces by content, with filters and a date range. Returns AI-readable trace digests by default. Use format: 'json' for full raw data. To fetch a trace you already have the ID for, use get_trace instead — free text does not match trace IDs.",
|
|
209
262
|
{
|
|
210
|
-
query: z
|
|
263
|
+
query: z
|
|
264
|
+
.string()
|
|
265
|
+
.optional()
|
|
266
|
+
.describe(
|
|
267
|
+
"Text search query. Matches captured input/output, the trace name and span names ONLY. It does not match trace IDs — to fetch a known trace ID use get_trace, or pass traceIds here for several at once.",
|
|
268
|
+
),
|
|
269
|
+
traceIds: z
|
|
270
|
+
.array(z.string().min(1))
|
|
271
|
+
.min(1)
|
|
272
|
+
.max(1000)
|
|
273
|
+
.optional()
|
|
274
|
+
.describe(
|
|
275
|
+
"Exact trace IDs to fetch (1–1000). Use instead of query when you already know the IDs. When set and no startDate is given, the window defaults to the last 90 days rather than 24 hours.",
|
|
276
|
+
),
|
|
211
277
|
filters: z
|
|
212
278
|
.record(z.string(), z.array(z.string()))
|
|
213
279
|
.optional()
|
|
214
280
|
.describe(
|
|
215
|
-
'Filter traces. Format: {"field": ["value"]}. Use discover_schema for field names.'
|
|
281
|
+
'Filter traces by the older per-field map. Format: {"field": ["value"]}. Use discover_schema for field names.',
|
|
216
282
|
),
|
|
217
|
-
|
|
283
|
+
filter: z
|
|
218
284
|
.string()
|
|
219
285
|
.optional()
|
|
220
286
|
.describe(
|
|
221
|
-
'
|
|
287
|
+
'Filter traces with the Trace Explorer\'s own query language: "status:error AND model:gpt-*", "trace.attribute.langwatch.user_id:alice", "evaluatorVerdict:fail", a quoted phrase for free text. Reaches attribute keys, span events and evaluator verdicts that the `filters` map cannot. Combined with `filters` and `query` when you send more than one. Call discover_schema with category \'filters\' for every field and the syntax.',
|
|
222
288
|
),
|
|
223
|
-
|
|
289
|
+
startDate: z
|
|
224
290
|
.string()
|
|
291
|
+
.min(1)
|
|
225
292
|
.optional()
|
|
226
|
-
.describe(
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
.describe("Results per page (default: 25, max: 1000)"),
|
|
231
|
-
scrollId: z
|
|
293
|
+
.describe(
|
|
294
|
+
'Start of the window: ISO date, or relative like "24h", "7d", "4w", "3m" — units are h (hours), d (days), w (weeks), m (30-day months). Defaults to 24h ago for a text search, or 90d when traceIds is set. Widen this when a search comes back empty.',
|
|
295
|
+
),
|
|
296
|
+
endDate: z
|
|
232
297
|
.string()
|
|
298
|
+
.min(1)
|
|
233
299
|
.optional()
|
|
234
|
-
.describe("
|
|
300
|
+
.describe("End of the window: ISO date or relative. Default: now"),
|
|
301
|
+
pageSize: z.number().optional().describe("Results per page (default: 25, max: 1000)"),
|
|
302
|
+
scrollId: z.string().optional().describe("Pagination token from previous search"),
|
|
235
303
|
format: z
|
|
236
304
|
.enum(["digest", "json"])
|
|
237
305
|
.optional()
|
|
238
|
-
.describe(
|
|
239
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
240
|
-
),
|
|
306
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
241
307
|
},
|
|
242
308
|
withToolLogging("search_traces", async (params) => {
|
|
243
309
|
requireApiKey();
|
|
244
|
-
const { handleSearchTraces } = await import("./tools/search-traces.
|
|
310
|
+
const { handleSearchTraces } = await import("./tools/search-traces.ts");
|
|
245
311
|
return {
|
|
246
312
|
content: [{ type: "text", text: await handleSearchTraces(params) }],
|
|
247
313
|
};
|
|
248
|
-
})
|
|
314
|
+
}),
|
|
249
315
|
);
|
|
250
316
|
|
|
251
317
|
server.tool(
|
|
252
318
|
"get_trace",
|
|
253
|
-
"Get full details of a single trace by ID. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
319
|
+
"Get full details of a single trace by ID. This is the right tool whenever you have a trace ID — search_traces does not match IDs. A full ID resolves at any age; a unique 8–31 character hex prefix resolves within the last 90 days. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
254
320
|
{
|
|
255
|
-
traceId: z
|
|
321
|
+
traceId: z
|
|
322
|
+
.string()
|
|
323
|
+
.describe(
|
|
324
|
+
"The trace ID to retrieve. A full ID resolves at any age; a unique 8–31 character hex prefix also resolves, git-style, within the last 90 days.",
|
|
325
|
+
),
|
|
256
326
|
format: z
|
|
257
327
|
.enum(["digest", "json"])
|
|
258
328
|
.optional()
|
|
259
|
-
.describe(
|
|
260
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
261
|
-
),
|
|
329
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
262
330
|
},
|
|
263
331
|
withToolLogging("get_trace", async (params) => {
|
|
264
332
|
requireApiKey();
|
|
265
|
-
const { handleGetTrace } = await import("./tools/get-trace.
|
|
333
|
+
const { handleGetTrace } = await import("./tools/get-trace.ts");
|
|
266
334
|
return {
|
|
267
335
|
content: [{ type: "text", text: await handleGetTrace(params) }],
|
|
268
336
|
};
|
|
269
|
-
})
|
|
337
|
+
}),
|
|
270
338
|
);
|
|
271
339
|
|
|
272
340
|
server.tool(
|
|
@@ -276,44 +344,40 @@ function registerTools(server: McpServer): void {
|
|
|
276
344
|
metric: z
|
|
277
345
|
.string()
|
|
278
346
|
.describe(
|
|
279
|
-
'Metric in "category.name" format, e.g., "metadata.trace_id", "performance.total_cost"'
|
|
347
|
+
'Metric in "category.name" format, e.g., "metadata.trace_id", "performance.total_cost"',
|
|
280
348
|
),
|
|
281
349
|
aggregation: z
|
|
282
350
|
.string()
|
|
283
351
|
.optional()
|
|
284
352
|
.describe(
|
|
285
|
-
"Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg"
|
|
353
|
+
"Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg",
|
|
286
354
|
),
|
|
287
355
|
startDate: z
|
|
288
356
|
.string()
|
|
289
357
|
.optional()
|
|
290
|
-
.describe(
|
|
291
|
-
'Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'
|
|
292
|
-
),
|
|
358
|
+
.describe('Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'),
|
|
293
359
|
endDate: z.string().optional().describe("End date. Default: now"),
|
|
294
360
|
timeZone: z.string().optional().describe("Timezone. Default: UTC"),
|
|
295
361
|
groupBy: z
|
|
296
362
|
.string()
|
|
297
363
|
.optional()
|
|
298
364
|
.describe("Group results by field. Use discover_schema for options."),
|
|
299
|
-
filters: z
|
|
300
|
-
.record(z.string(), z.array(z.string()))
|
|
301
|
-
.optional()
|
|
302
|
-
.describe("Filters to apply"),
|
|
365
|
+
filters: z.record(z.string(), z.array(z.string())).optional().describe("Filters to apply"),
|
|
303
366
|
},
|
|
304
367
|
withToolLogging("get_analytics", async (params) => {
|
|
305
368
|
requireApiKey();
|
|
306
|
-
const { handleGetAnalytics } = await import("./tools/get-analytics.
|
|
369
|
+
const { handleGetAnalytics } = await import("./tools/get-analytics.ts");
|
|
307
370
|
return {
|
|
308
371
|
content: [{ type: "text", text: await handleGetAnalytics(params) }],
|
|
309
372
|
};
|
|
310
|
-
})
|
|
373
|
+
}),
|
|
311
374
|
);
|
|
375
|
+
}
|
|
312
376
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
377
|
+
// --- Platform Prompt Tools (require API key) ---
|
|
378
|
+
// These tools manage prompts on the LangWatch platform via API.
|
|
379
|
+
// For code-based prompt management, see `fetch_langwatch_docs` for the CLI/SDK approach.
|
|
380
|
+
function registerPromptTools(server: McpServer): void {
|
|
317
381
|
server.tool(
|
|
318
382
|
"platform_create_prompt",
|
|
319
383
|
`Create a new prompt on the LangWatch platform.
|
|
@@ -329,32 +393,31 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
329
393
|
handle: z
|
|
330
394
|
.string()
|
|
331
395
|
.optional()
|
|
332
|
-
.describe(
|
|
333
|
-
"URL-friendly handle (auto-generated from name if omitted)"
|
|
334
|
-
),
|
|
396
|
+
.describe("URL-friendly handle (auto-generated from name if omitted)"),
|
|
335
397
|
messages: z
|
|
336
398
|
.array(
|
|
337
399
|
z.object({
|
|
338
|
-
role: z
|
|
339
|
-
.enum(["system", "user", "assistant"])
|
|
340
|
-
.describe("Message role"),
|
|
400
|
+
role: z.enum(["system", "user", "assistant"]).describe("Message role"),
|
|
341
401
|
content: z.string().describe("Message content"),
|
|
342
|
-
})
|
|
402
|
+
}),
|
|
343
403
|
)
|
|
344
404
|
.describe("Prompt messages"),
|
|
345
405
|
model: modelSchema,
|
|
346
|
-
tags: z
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
406
|
+
tags: z
|
|
407
|
+
.array(z.string())
|
|
408
|
+
.optional()
|
|
409
|
+
.describe(
|
|
410
|
+
'Tags to assign to the initial version (e.g., ["production", "staging"]). ' +
|
|
411
|
+
'Built-in tags: "latest" (auto-assigned), "production", "staging". Custom tags must be created first.',
|
|
412
|
+
),
|
|
350
413
|
},
|
|
351
414
|
withToolLogging("platform_create_prompt", async (params) => {
|
|
352
415
|
requireApiKey();
|
|
353
|
-
const { handleCreatePrompt } = await import("./tools/create-prompt.
|
|
416
|
+
const { handleCreatePrompt } = await import("./tools/create-prompt.ts");
|
|
354
417
|
return {
|
|
355
418
|
content: [{ type: "text", text: await handleCreatePrompt(params) }],
|
|
356
419
|
};
|
|
357
|
-
})
|
|
420
|
+
}),
|
|
358
421
|
);
|
|
359
422
|
|
|
360
423
|
server.tool(
|
|
@@ -363,11 +426,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
363
426
|
{},
|
|
364
427
|
withToolLogging("platform_list_prompts", async () => {
|
|
365
428
|
requireApiKey();
|
|
366
|
-
const { handleListPrompts } = await import("./tools/list-prompts.
|
|
429
|
+
const { handleListPrompts } = await import("./tools/list-prompts.ts");
|
|
367
430
|
return {
|
|
368
431
|
content: [{ type: "text", text: await handleListPrompts() }],
|
|
369
432
|
};
|
|
370
|
-
})
|
|
433
|
+
}),
|
|
371
434
|
);
|
|
372
435
|
|
|
373
436
|
server.tool(
|
|
@@ -375,20 +438,18 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
375
438
|
"Get a specific prompt from the LangWatch platform by ID or handle, including messages, model config, and version history. Use format: 'json' for the full raw API payload, or 'digest' (default) for a formatted summary.",
|
|
376
439
|
{
|
|
377
440
|
idOrHandle: z.string().describe("Prompt ID or handle"),
|
|
378
|
-
version: z
|
|
379
|
-
|
|
441
|
+
version: z.number().optional().describe("Specific version number (default: latest)"),
|
|
442
|
+
tag: z
|
|
443
|
+
.string()
|
|
380
444
|
.optional()
|
|
381
|
-
.describe(
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
),
|
|
445
|
+
.describe(
|
|
446
|
+
'Fetch the version pointed to by this tag (e.g., "production", "staging"). ' +
|
|
447
|
+
'Alternatively, use shorthand in idOrHandle: "pizza-prompt:production"',
|
|
448
|
+
),
|
|
386
449
|
format: z
|
|
387
450
|
.enum(["digest", "json"])
|
|
388
451
|
.optional()
|
|
389
|
-
.describe(
|
|
390
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
391
|
-
),
|
|
452
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
392
453
|
},
|
|
393
454
|
withToolLogging("platform_get_prompt", async (params) => {
|
|
394
455
|
if (params.version != null && params.tag) {
|
|
@@ -398,11 +459,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
398
459
|
};
|
|
399
460
|
}
|
|
400
461
|
requireApiKey();
|
|
401
|
-
const { handleGetPrompt } = await import("./tools/get-prompt.
|
|
462
|
+
const { handleGetPrompt } = await import("./tools/get-prompt.ts");
|
|
402
463
|
return {
|
|
403
464
|
content: [{ type: "text", text: await handleGetPrompt(params) }],
|
|
404
465
|
};
|
|
405
|
-
})
|
|
466
|
+
}),
|
|
406
467
|
);
|
|
407
468
|
|
|
408
469
|
server.tool(
|
|
@@ -415,31 +476,30 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
415
476
|
z.object({
|
|
416
477
|
role: z.enum(["system", "user", "assistant"]),
|
|
417
478
|
content: z.string(),
|
|
418
|
-
})
|
|
479
|
+
}),
|
|
419
480
|
)
|
|
420
481
|
.optional()
|
|
421
482
|
.describe("Updated messages"),
|
|
422
483
|
model: modelSchema.optional(),
|
|
423
|
-
commitMessage: z
|
|
424
|
-
|
|
425
|
-
.
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
),
|
|
484
|
+
commitMessage: z.string().describe("Commit message describing the change"),
|
|
485
|
+
tags: z
|
|
486
|
+
.array(z.string())
|
|
487
|
+
.optional()
|
|
488
|
+
.describe("Tags to assign to the new version created by this update."),
|
|
429
489
|
},
|
|
430
490
|
withToolLogging("platform_update_prompt", async (params) => {
|
|
431
491
|
requireApiKey();
|
|
432
|
-
const { handleUpdatePrompt } = await import("./tools/update-prompt.
|
|
492
|
+
const { handleUpdatePrompt } = await import("./tools/update-prompt.ts");
|
|
433
493
|
return {
|
|
434
494
|
content: [{ type: "text", text: await handleUpdatePrompt(params) }],
|
|
435
495
|
};
|
|
436
|
-
})
|
|
496
|
+
}),
|
|
437
497
|
);
|
|
438
498
|
|
|
439
499
|
server.tool(
|
|
440
500
|
"platform_assign_prompt_tag",
|
|
441
501
|
'Assign a tag (e.g. "production") to a specific version of a prompt. ' +
|
|
442
|
-
|
|
502
|
+
'Use this to "deploy" a version by promoting it to the production tag.',
|
|
443
503
|
{
|
|
444
504
|
idOrHandle: z.string().describe("Prompt ID or handle"),
|
|
445
505
|
tag: z.string().describe('Tag name (e.g., "production", "staging")'),
|
|
@@ -447,41 +507,41 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
447
507
|
},
|
|
448
508
|
withToolLogging("platform_assign_prompt_tag", async (params) => {
|
|
449
509
|
requireApiKey();
|
|
450
|
-
const { handleAssignPromptTag } = await import("./tools/assign-prompt-tag.
|
|
510
|
+
const { handleAssignPromptTag } = await import("./tools/assign-prompt-tag.ts");
|
|
451
511
|
return {
|
|
452
512
|
content: [{ type: "text", text: await handleAssignPromptTag(params) }],
|
|
453
513
|
};
|
|
454
|
-
})
|
|
514
|
+
}),
|
|
455
515
|
);
|
|
456
516
|
|
|
457
517
|
server.tool(
|
|
458
518
|
"platform_list_prompt_tags",
|
|
459
519
|
"List all prompt tag definitions for the organization. " +
|
|
460
|
-
|
|
520
|
+
"Shows built-in tags (latest, production, staging) and any custom tags.",
|
|
461
521
|
{},
|
|
462
522
|
withToolLogging("platform_list_prompt_tags", async () => {
|
|
463
523
|
requireApiKey();
|
|
464
|
-
const { handleListPromptTags } = await import("./tools/list-prompt-tags.
|
|
524
|
+
const { handleListPromptTags } = await import("./tools/list-prompt-tags.ts");
|
|
465
525
|
return {
|
|
466
526
|
content: [{ type: "text", text: await handleListPromptTags() }],
|
|
467
527
|
};
|
|
468
|
-
})
|
|
528
|
+
}),
|
|
469
529
|
);
|
|
470
530
|
|
|
471
531
|
server.tool(
|
|
472
532
|
"platform_create_prompt_tag",
|
|
473
533
|
"Create a custom prompt tag definition for the organization. " +
|
|
474
|
-
|
|
534
|
+
'Tag names must be non-numeric and not "latest".',
|
|
475
535
|
{
|
|
476
536
|
name: z.string().describe("Tag name to create"),
|
|
477
537
|
},
|
|
478
538
|
withToolLogging("platform_create_prompt_tag", async (params) => {
|
|
479
539
|
requireApiKey();
|
|
480
|
-
const { handleCreatePromptTag } = await import("./tools/create-prompt-tag.
|
|
540
|
+
const { handleCreatePromptTag } = await import("./tools/create-prompt-tag.ts");
|
|
481
541
|
return {
|
|
482
542
|
content: [{ type: "text", text: await handleCreatePromptTag(params) }],
|
|
483
543
|
};
|
|
484
|
-
})
|
|
544
|
+
}),
|
|
485
545
|
);
|
|
486
546
|
|
|
487
547
|
server.tool(
|
|
@@ -493,11 +553,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
493
553
|
},
|
|
494
554
|
withToolLogging("platform_rename_prompt_tag", async (params) => {
|
|
495
555
|
requireApiKey();
|
|
496
|
-
const { handleRenamePromptTag } = await import("./tools/rename-prompt-tag.
|
|
556
|
+
const { handleRenamePromptTag } = await import("./tools/rename-prompt-tag.ts");
|
|
497
557
|
return {
|
|
498
558
|
content: [{ type: "text", text: await handleRenamePromptTag(params) }],
|
|
499
559
|
};
|
|
500
|
-
})
|
|
560
|
+
}),
|
|
501
561
|
);
|
|
502
562
|
|
|
503
563
|
server.tool(
|
|
@@ -508,17 +568,18 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
508
568
|
},
|
|
509
569
|
withToolLogging("platform_delete_prompt_tag", async (params) => {
|
|
510
570
|
requireApiKey();
|
|
511
|
-
const { handleDeletePromptTag } = await import("./tools/delete-prompt-tag.
|
|
571
|
+
const { handleDeletePromptTag } = await import("./tools/delete-prompt-tag.ts");
|
|
512
572
|
return {
|
|
513
573
|
content: [{ type: "text", text: await handleDeletePromptTag(params) }],
|
|
514
574
|
};
|
|
515
|
-
})
|
|
575
|
+
}),
|
|
516
576
|
);
|
|
577
|
+
}
|
|
517
578
|
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
579
|
+
// --- Platform Scenario Tools (require API key) ---
|
|
580
|
+
// These tools manage scenarios on the LangWatch platform via API.
|
|
581
|
+
// For code-based scenario testing, see `fetch_scenario_docs` for the SDK approach.
|
|
582
|
+
function registerScenarioTools(server: McpServer): void {
|
|
522
583
|
server.tool(
|
|
523
584
|
"platform_create_scenario",
|
|
524
585
|
`Create a new scenario on the LangWatch platform. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.
|
|
@@ -533,15 +594,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
533
594
|
name: z.string().describe("Scenario name"),
|
|
534
595
|
situation: z
|
|
535
596
|
.string()
|
|
536
|
-
.describe(
|
|
537
|
-
"The context or setup describing what the user/agent is doing"
|
|
538
|
-
),
|
|
597
|
+
.describe("The context or setup describing what the user/agent is doing"),
|
|
539
598
|
criteria: z
|
|
540
599
|
.array(z.string())
|
|
541
600
|
.optional()
|
|
542
|
-
.describe(
|
|
543
|
-
"Pass/fail conditions the agent's response must satisfy"
|
|
544
|
-
),
|
|
601
|
+
.describe("Pass/fail conditions the agent's response must satisfy"),
|
|
545
602
|
labels: z
|
|
546
603
|
.array(z.string())
|
|
547
604
|
.optional()
|
|
@@ -550,49 +607,36 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
550
607
|
.string()
|
|
551
608
|
.nullish()
|
|
552
609
|
.describe(
|
|
553
|
-
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
610
|
+
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it.",
|
|
554
611
|
),
|
|
555
612
|
fields: scenarioFieldValuesSchema.optional(),
|
|
556
613
|
},
|
|
557
614
|
withToolLogging("platform_create_scenario", async (params) => {
|
|
558
615
|
requireApiKey();
|
|
559
|
-
const { handleCreateScenario } = await import(
|
|
560
|
-
"./tools/create-scenario.js"
|
|
561
|
-
);
|
|
616
|
+
const { handleCreateScenario } = await import("./tools/create-scenario.ts");
|
|
562
617
|
return {
|
|
563
|
-
content: [
|
|
564
|
-
{ type: "text", text: await handleCreateScenario(params) },
|
|
565
|
-
],
|
|
618
|
+
content: [{ type: "text", text: await handleCreateScenario(params) }],
|
|
566
619
|
};
|
|
567
|
-
})
|
|
620
|
+
}),
|
|
568
621
|
);
|
|
569
622
|
|
|
570
623
|
server.tool(
|
|
571
624
|
"platform_list_scenarios",
|
|
572
625
|
"List all scenarios on the LangWatch platform. Returns AI-readable digest by default.",
|
|
573
626
|
{
|
|
574
|
-
testSuiteId: z
|
|
575
|
-
.string()
|
|
576
|
-
.optional()
|
|
577
|
-
.describe("Only the scenarios filed in this test suite"),
|
|
627
|
+
testSuiteId: z.string().optional().describe("Only the scenarios filed in this test suite"),
|
|
578
628
|
format: z
|
|
579
629
|
.enum(["digest", "json"])
|
|
580
630
|
.optional()
|
|
581
|
-
.describe(
|
|
582
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
583
|
-
),
|
|
631
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
584
632
|
},
|
|
585
633
|
withToolLogging("platform_list_scenarios", async (params) => {
|
|
586
634
|
requireApiKey();
|
|
587
|
-
const { handleListScenarios } = await import(
|
|
588
|
-
"./tools/list-scenarios.js"
|
|
589
|
-
);
|
|
635
|
+
const { handleListScenarios } = await import("./tools/list-scenarios.ts");
|
|
590
636
|
return {
|
|
591
|
-
content: [
|
|
592
|
-
{ type: "text", text: await handleListScenarios(params) },
|
|
593
|
-
],
|
|
637
|
+
content: [{ type: "text", text: await handleListScenarios(params) }],
|
|
594
638
|
};
|
|
595
|
-
})
|
|
639
|
+
}),
|
|
596
640
|
);
|
|
597
641
|
|
|
598
642
|
server.tool(
|
|
@@ -603,17 +647,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
603
647
|
format: z
|
|
604
648
|
.enum(["digest", "json"])
|
|
605
649
|
.optional()
|
|
606
|
-
.describe(
|
|
607
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
608
|
-
),
|
|
650
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
609
651
|
},
|
|
610
652
|
withToolLogging("platform_get_scenario", async (params) => {
|
|
611
653
|
requireApiKey();
|
|
612
|
-
const { handleGetScenario } = await import("./tools/get-scenario.
|
|
654
|
+
const { handleGetScenario } = await import("./tools/get-scenario.ts");
|
|
613
655
|
return {
|
|
614
656
|
content: [{ type: "text", text: await handleGetScenario(params) }],
|
|
615
657
|
};
|
|
616
|
-
})
|
|
658
|
+
}),
|
|
617
659
|
);
|
|
618
660
|
|
|
619
661
|
server.tool(
|
|
@@ -623,33 +665,23 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
623
665
|
scenarioId: z.string().describe("The scenario ID to update"),
|
|
624
666
|
name: z.string().optional().describe("Updated scenario name"),
|
|
625
667
|
situation: z.string().optional().describe("Updated situation"),
|
|
626
|
-
criteria: z
|
|
627
|
-
|
|
628
|
-
.optional()
|
|
629
|
-
.describe("Updated criteria"),
|
|
630
|
-
labels: z
|
|
631
|
-
.array(z.string())
|
|
632
|
-
.optional()
|
|
633
|
-
.describe("Updated labels"),
|
|
668
|
+
criteria: z.array(z.string()).optional().describe("Updated criteria"),
|
|
669
|
+
labels: z.array(z.string()).optional().describe("Updated labels"),
|
|
634
670
|
testSuiteId: z
|
|
635
671
|
.string()
|
|
636
672
|
.nullish()
|
|
637
673
|
.describe(
|
|
638
|
-
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
674
|
+
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it.",
|
|
639
675
|
),
|
|
640
676
|
fields: scenarioFieldValuesSchema.optional(),
|
|
641
677
|
},
|
|
642
678
|
withToolLogging("platform_update_scenario", async (params) => {
|
|
643
679
|
requireApiKey();
|
|
644
|
-
const { handleUpdateScenario } = await import(
|
|
645
|
-
"./tools/update-scenario.js"
|
|
646
|
-
);
|
|
680
|
+
const { handleUpdateScenario } = await import("./tools/update-scenario.ts");
|
|
647
681
|
return {
|
|
648
|
-
content: [
|
|
649
|
-
{ type: "text", text: await handleUpdateScenario(params) },
|
|
650
|
-
],
|
|
682
|
+
content: [{ type: "text", text: await handleUpdateScenario(params) }],
|
|
651
683
|
};
|
|
652
|
-
})
|
|
684
|
+
}),
|
|
653
685
|
);
|
|
654
686
|
|
|
655
687
|
server.tool(
|
|
@@ -660,23 +692,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
660
692
|
},
|
|
661
693
|
withToolLogging("platform_archive_scenario", async (params) => {
|
|
662
694
|
requireApiKey();
|
|
663
|
-
const { handleArchiveScenario } = await import(
|
|
664
|
-
"./tools/archive-scenario.js"
|
|
665
|
-
);
|
|
695
|
+
const { handleArchiveScenario } = await import("./tools/archive-scenario.ts");
|
|
666
696
|
return {
|
|
667
|
-
content: [
|
|
668
|
-
{ type: "text", text: await handleArchiveScenario(params) },
|
|
669
|
-
],
|
|
697
|
+
content: [{ type: "text", text: await handleArchiveScenario(params) }],
|
|
670
698
|
};
|
|
671
|
-
})
|
|
699
|
+
}),
|
|
672
700
|
);
|
|
701
|
+
}
|
|
673
702
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
703
|
+
// --- Platform Run Plan Tools (require API key) ---
|
|
704
|
+
// A run plan is what you run, and its NAME identifies it: running a name
|
|
705
|
+
// that exists replaces that plan's configuration, running a new name
|
|
706
|
+
// creates the plan. A test suite groups scenarios; running one is
|
|
707
|
+
// sugar that creates or joins the plan "<suite name> <target name>".
|
|
708
|
+
function registerRunPlanTools(server: McpServer): void {
|
|
680
709
|
server.tool(
|
|
681
710
|
"platform_run_plan",
|
|
682
711
|
"Run scenarios against targets. The plan name identifies the run plan: an existing name is re-run with the configuration you send here, a new name creates the plan. Configuration is the scope, the targets, the repeat count and the models; parameters, the note and the idempotency key belong to this run alone. Send more than one target to compare them in the same run, and give a target its own parameters to compare one agent on two models.",
|
|
@@ -693,9 +722,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
693
722
|
scenarioIds: z
|
|
694
723
|
.array(z.string())
|
|
695
724
|
.optional()
|
|
696
|
-
.describe(
|
|
697
|
-
"The scenarios to run. Read only when scope.mode is 'scenarios'.",
|
|
698
|
-
),
|
|
725
|
+
.describe("The scenarios to run. Read only when scope.mode is 'scenarios'."),
|
|
699
726
|
targets: z
|
|
700
727
|
.array(runPlanTargetSchema)
|
|
701
728
|
.describe(
|
|
@@ -734,25 +761,24 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
734
761
|
idempotencyKey: z
|
|
735
762
|
.string()
|
|
736
763
|
.optional()
|
|
737
|
-
.describe(
|
|
764
|
+
.describe(
|
|
765
|
+
"Send the same key to make a retry join the run it already started instead of starting a second one.",
|
|
766
|
+
),
|
|
738
767
|
},
|
|
739
768
|
withToolLogging("platform_run_plan", async (params) => {
|
|
740
769
|
requireApiKey();
|
|
741
|
-
const { handleRunPlan } = await import("./tools/run-plan.
|
|
770
|
+
const { handleRunPlan } = await import("./tools/run-plan.ts");
|
|
742
771
|
return {
|
|
743
772
|
content: [{ type: "text", text: await handleRunPlan(params) }],
|
|
744
773
|
};
|
|
745
|
-
})
|
|
774
|
+
}),
|
|
746
775
|
);
|
|
747
776
|
|
|
748
777
|
server.tool(
|
|
749
778
|
"platform_list_run_plans",
|
|
750
779
|
"List the run plans of the project. A run plan holds the configuration a run used, keyed by its name.",
|
|
751
780
|
{
|
|
752
|
-
includeArchived: z
|
|
753
|
-
.boolean()
|
|
754
|
-
.optional()
|
|
755
|
-
.describe("Include archived plans (default false)"),
|
|
781
|
+
includeArchived: z.boolean().optional().describe("Include archived plans (default false)"),
|
|
756
782
|
format: z
|
|
757
783
|
.enum(["digest", "json"])
|
|
758
784
|
.optional()
|
|
@@ -760,11 +786,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
760
786
|
},
|
|
761
787
|
withToolLogging("platform_list_run_plans", async (params) => {
|
|
762
788
|
requireApiKey();
|
|
763
|
-
const { handleListRunPlans } = await import("./tools/list-run-plans.
|
|
789
|
+
const { handleListRunPlans } = await import("./tools/list-run-plans.ts");
|
|
764
790
|
return {
|
|
765
791
|
content: [{ type: "text", text: await handleListRunPlans(params) }],
|
|
766
792
|
};
|
|
767
|
-
})
|
|
793
|
+
}),
|
|
768
794
|
);
|
|
769
795
|
|
|
770
796
|
server.tool(
|
|
@@ -779,11 +805,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
779
805
|
},
|
|
780
806
|
withToolLogging("platform_get_run_plan", async (params) => {
|
|
781
807
|
requireApiKey();
|
|
782
|
-
const { handleGetRunPlan } = await import("./tools/get-run-plan.
|
|
808
|
+
const { handleGetRunPlan } = await import("./tools/get-run-plan.ts");
|
|
783
809
|
return {
|
|
784
810
|
content: [{ type: "text", text: await handleGetRunPlan(params) }],
|
|
785
811
|
};
|
|
786
|
-
})
|
|
812
|
+
}),
|
|
787
813
|
);
|
|
788
814
|
|
|
789
815
|
server.tool(
|
|
@@ -802,11 +828,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
802
828
|
},
|
|
803
829
|
withToolLogging("platform_rerun_run_plan", async (params) => {
|
|
804
830
|
requireApiKey();
|
|
805
|
-
const { handleRerunRunPlan } = await import("./tools/rerun-run-plan.
|
|
831
|
+
const { handleRerunRunPlan } = await import("./tools/rerun-run-plan.ts");
|
|
806
832
|
return {
|
|
807
833
|
content: [{ type: "text", text: await handleRerunRunPlan(params) }],
|
|
808
834
|
};
|
|
809
|
-
})
|
|
835
|
+
}),
|
|
810
836
|
);
|
|
811
837
|
|
|
812
838
|
server.tool(
|
|
@@ -817,18 +843,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
817
843
|
},
|
|
818
844
|
withToolLogging("platform_archive_run_plan", async (params) => {
|
|
819
845
|
requireApiKey();
|
|
820
|
-
const { handleArchiveRunPlan } = await import(
|
|
821
|
-
"./tools/archive-run-plan.js"
|
|
822
|
-
);
|
|
846
|
+
const { handleArchiveRunPlan } = await import("./tools/archive-run-plan.ts");
|
|
823
847
|
return {
|
|
824
848
|
content: [{ type: "text", text: await handleArchiveRunPlan(params) }],
|
|
825
849
|
};
|
|
826
|
-
})
|
|
850
|
+
}),
|
|
827
851
|
);
|
|
852
|
+
}
|
|
828
853
|
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
854
|
+
// --- Platform Test Suite Tools (require API key) ---
|
|
855
|
+
// A test suite groups scenarios: a name and the scenarios filed in it.
|
|
856
|
+
function registerTestSuiteTools(server: McpServer): void {
|
|
832
857
|
server.tool(
|
|
833
858
|
"platform_list_test_suites",
|
|
834
859
|
"List the test suites of the project. A test suite groups scenarios.",
|
|
@@ -840,13 +865,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
840
865
|
},
|
|
841
866
|
withToolLogging("platform_list_test_suites", async (params) => {
|
|
842
867
|
requireApiKey();
|
|
843
|
-
const { handleListTestSuites } = await import(
|
|
844
|
-
"./tools/list-test-suites.js"
|
|
845
|
-
);
|
|
868
|
+
const { handleListTestSuites } = await import("./tools/list-test-suites.ts");
|
|
846
869
|
return {
|
|
847
870
|
content: [{ type: "text", text: await handleListTestSuites(params) }],
|
|
848
871
|
};
|
|
849
|
-
})
|
|
872
|
+
}),
|
|
850
873
|
);
|
|
851
874
|
|
|
852
875
|
server.tool(
|
|
@@ -859,13 +882,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
859
882
|
},
|
|
860
883
|
withToolLogging("platform_create_test_suite", async (params) => {
|
|
861
884
|
requireApiKey();
|
|
862
|
-
const { handleCreateTestSuite } = await import(
|
|
863
|
-
"./tools/create-test-suite.js"
|
|
864
|
-
);
|
|
885
|
+
const { handleCreateTestSuite } = await import("./tools/create-test-suite.ts");
|
|
865
886
|
return {
|
|
866
887
|
content: [{ type: "text", text: await handleCreateTestSuite(params) }],
|
|
867
888
|
};
|
|
868
|
-
})
|
|
889
|
+
}),
|
|
869
890
|
);
|
|
870
891
|
|
|
871
892
|
server.tool(
|
|
@@ -880,11 +901,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
880
901
|
},
|
|
881
902
|
withToolLogging("platform_get_test_suite", async (params) => {
|
|
882
903
|
requireApiKey();
|
|
883
|
-
const { handleGetTestSuite } = await import("./tools/get-test-suite.
|
|
904
|
+
const { handleGetTestSuite } = await import("./tools/get-test-suite.ts");
|
|
884
905
|
return {
|
|
885
906
|
content: [{ type: "text", text: await handleGetTestSuite(params) }],
|
|
886
907
|
};
|
|
887
|
-
})
|
|
908
|
+
}),
|
|
888
909
|
);
|
|
889
910
|
|
|
890
911
|
server.tool(
|
|
@@ -895,17 +916,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
895
916
|
name: z.string().optional().describe("The new name. The slug is kept."),
|
|
896
917
|
fields: suiteFieldsSchema
|
|
897
918
|
.optional()
|
|
898
|
-
.describe(
|
|
919
|
+
.describe(
|
|
920
|
+
"The full list of fields the suite declares. It replaces the list the suite holds.",
|
|
921
|
+
),
|
|
899
922
|
evaluators: evaluatorAttachmentsSchema
|
|
900
923
|
.optional()
|
|
901
|
-
.describe(
|
|
924
|
+
.describe(
|
|
925
|
+
"The full list of evaluators attached to the suite. It replaces the list the suite holds.",
|
|
926
|
+
),
|
|
902
927
|
},
|
|
903
928
|
withToolLogging("platform_update_test_suite", async (params) => {
|
|
904
929
|
requireApiKey();
|
|
905
930
|
return {
|
|
906
931
|
content: [{ type: "text", text: await handleUpdateTestSuite(params) }],
|
|
907
932
|
};
|
|
908
|
-
})
|
|
933
|
+
}),
|
|
909
934
|
);
|
|
910
935
|
|
|
911
936
|
server.tool(
|
|
@@ -917,13 +942,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
917
942
|
},
|
|
918
943
|
withToolLogging("platform_rename_test_suite", async (params) => {
|
|
919
944
|
requireApiKey();
|
|
920
|
-
const { handleRenameTestSuite } = await import(
|
|
921
|
-
"./tools/rename-test-suite.js"
|
|
922
|
-
);
|
|
945
|
+
const { handleRenameTestSuite } = await import("./tools/rename-test-suite.ts");
|
|
923
946
|
return {
|
|
924
947
|
content: [{ type: "text", text: await handleRenameTestSuite(params) }],
|
|
925
948
|
};
|
|
926
|
-
})
|
|
949
|
+
}),
|
|
927
950
|
);
|
|
928
951
|
|
|
929
952
|
server.tool(
|
|
@@ -934,13 +957,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
934
957
|
},
|
|
935
958
|
withToolLogging("platform_archive_test_suite", async (params) => {
|
|
936
959
|
requireApiKey();
|
|
937
|
-
const { handleArchiveTestSuite } = await import(
|
|
938
|
-
"./tools/archive-test-suite.js"
|
|
939
|
-
);
|
|
960
|
+
const { handleArchiveTestSuite } = await import("./tools/archive-test-suite.ts");
|
|
940
961
|
return {
|
|
941
962
|
content: [{ type: "text", text: await handleArchiveTestSuite(params) }],
|
|
942
963
|
};
|
|
943
|
-
})
|
|
964
|
+
}),
|
|
944
965
|
);
|
|
945
966
|
|
|
946
967
|
server.tool(
|
|
@@ -987,16 +1008,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
987
1008
|
},
|
|
988
1009
|
withToolLogging("platform_run_test_suite", async (params) => {
|
|
989
1010
|
requireApiKey();
|
|
990
|
-
const { handleRunTestSuite } = await import("./tools/run-test-suite.
|
|
1011
|
+
const { handleRunTestSuite } = await import("./tools/run-test-suite.ts");
|
|
991
1012
|
return {
|
|
992
1013
|
content: [{ type: "text", text: await handleRunTestSuite(params) }],
|
|
993
1014
|
};
|
|
994
|
-
})
|
|
1015
|
+
}),
|
|
995
1016
|
);
|
|
1017
|
+
}
|
|
996
1018
|
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1019
|
+
// --- Platform Simulation Run Tools (require API key) ---
|
|
1020
|
+
// These tools query simulation run results from the LangWatch platform.
|
|
1021
|
+
function registerSimulationRunTools(server: McpServer): void {
|
|
1000
1022
|
server.tool(
|
|
1001
1023
|
"platform_list_simulation_runs",
|
|
1002
1024
|
"List simulation run results. Filter by scenario set or batch run ID to see specific results.",
|
|
@@ -1008,11 +1030,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1008
1030
|
},
|
|
1009
1031
|
withToolLogging("platform_list_simulation_runs", async (params) => {
|
|
1010
1032
|
requireApiKey();
|
|
1011
|
-
const { handleListSimulationRuns } = await import("./tools/list-simulation-runs.
|
|
1033
|
+
const { handleListSimulationRuns } = await import("./tools/list-simulation-runs.ts");
|
|
1012
1034
|
return {
|
|
1013
1035
|
content: [{ type: "text", text: await handleListSimulationRuns(params) }],
|
|
1014
1036
|
};
|
|
1015
|
-
})
|
|
1037
|
+
}),
|
|
1016
1038
|
);
|
|
1017
1039
|
|
|
1018
1040
|
server.tool(
|
|
@@ -1024,16 +1046,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1024
1046
|
},
|
|
1025
1047
|
withToolLogging("platform_get_simulation_run", async (params) => {
|
|
1026
1048
|
requireApiKey();
|
|
1027
|
-
const { handleGetSimulationRun } = await import("./tools/get-simulation-run.
|
|
1049
|
+
const { handleGetSimulationRun } = await import("./tools/get-simulation-run.ts");
|
|
1028
1050
|
return {
|
|
1029
1051
|
content: [{ type: "text", text: await handleGetSimulationRun(params) }],
|
|
1030
1052
|
};
|
|
1031
|
-
})
|
|
1053
|
+
}),
|
|
1032
1054
|
);
|
|
1055
|
+
}
|
|
1033
1056
|
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1057
|
+
// --- Platform Evaluator Tools (require API key) ---
|
|
1058
|
+
// These tools manage evaluators on the LangWatch platform via API.
|
|
1059
|
+
function registerEvaluatorTools(server: McpServer): void {
|
|
1037
1060
|
server.tool(
|
|
1038
1061
|
"platform_create_evaluator",
|
|
1039
1062
|
`Create an evaluator on the LangWatch platform. Useful for setting up LLM-as-judge and other evaluators to use in evaluation notebooks. Call discover_schema({ category: 'evaluators' }) first to see available evaluator types and their settings.`,
|
|
@@ -1042,20 +1065,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1042
1065
|
config: z
|
|
1043
1066
|
.record(z.string(), z.unknown())
|
|
1044
1067
|
.describe(
|
|
1045
|
-
'Evaluator config object. Must include "evaluatorType" (e.g. "langevals/llm_boolean") and optional "settings" overrides.'
|
|
1068
|
+
'Evaluator config object. Must include "evaluatorType" (e.g. "langevals/llm_boolean") and optional "settings" overrides.',
|
|
1046
1069
|
),
|
|
1047
1070
|
},
|
|
1048
1071
|
withToolLogging("platform_create_evaluator", async (params) => {
|
|
1049
1072
|
requireApiKey();
|
|
1050
|
-
const { handleCreateEvaluator } = await import(
|
|
1051
|
-
"./tools/create-evaluator.js"
|
|
1052
|
-
);
|
|
1073
|
+
const { handleCreateEvaluator } = await import("./tools/create-evaluator.ts");
|
|
1053
1074
|
return {
|
|
1054
|
-
content: [
|
|
1055
|
-
{ type: "text", text: await handleCreateEvaluator(params) },
|
|
1056
|
-
],
|
|
1075
|
+
content: [{ type: "text", text: await handleCreateEvaluator(params) }],
|
|
1057
1076
|
};
|
|
1058
|
-
})
|
|
1077
|
+
}),
|
|
1059
1078
|
);
|
|
1060
1079
|
|
|
1061
1080
|
server.tool(
|
|
@@ -1064,32 +1083,26 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1064
1083
|
{},
|
|
1065
1084
|
withToolLogging("platform_list_evaluators", async () => {
|
|
1066
1085
|
requireApiKey();
|
|
1067
|
-
const { handleListEvaluators } = await import(
|
|
1068
|
-
"./tools/list-evaluators.js"
|
|
1069
|
-
);
|
|
1086
|
+
const { handleListEvaluators } = await import("./tools/list-evaluators.ts");
|
|
1070
1087
|
return {
|
|
1071
1088
|
content: [{ type: "text", text: await handleListEvaluators() }],
|
|
1072
1089
|
};
|
|
1073
|
-
})
|
|
1090
|
+
}),
|
|
1074
1091
|
);
|
|
1075
1092
|
|
|
1076
1093
|
server.tool(
|
|
1077
1094
|
"platform_get_evaluator",
|
|
1078
1095
|
"Get full details of an evaluator on the LangWatch platform by ID or slug, including config, input fields, and output fields.",
|
|
1079
1096
|
{
|
|
1080
|
-
idOrSlug: z
|
|
1081
|
-
.string()
|
|
1082
|
-
.describe("The evaluator ID or slug to retrieve"),
|
|
1097
|
+
idOrSlug: z.string().describe("The evaluator ID or slug to retrieve"),
|
|
1083
1098
|
},
|
|
1084
1099
|
withToolLogging("platform_get_evaluator", async (params) => {
|
|
1085
1100
|
requireApiKey();
|
|
1086
|
-
const { handleGetEvaluator } = await import(
|
|
1087
|
-
"./tools/get-evaluator.js"
|
|
1088
|
-
);
|
|
1101
|
+
const { handleGetEvaluator } = await import("./tools/get-evaluator.ts");
|
|
1089
1102
|
return {
|
|
1090
1103
|
content: [{ type: "text", text: await handleGetEvaluator(params) }],
|
|
1091
1104
|
};
|
|
1092
|
-
})
|
|
1105
|
+
}),
|
|
1093
1106
|
);
|
|
1094
1107
|
|
|
1095
1108
|
server.tool(
|
|
@@ -1101,21 +1114,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1101
1114
|
config: z
|
|
1102
1115
|
.record(z.string(), z.unknown())
|
|
1103
1116
|
.optional()
|
|
1104
|
-
.describe(
|
|
1105
|
-
"Updated config settings. Note: evaluatorType cannot be changed after creation."
|
|
1106
|
-
),
|
|
1117
|
+
.describe("Updated config settings. Note: evaluatorType cannot be changed after creation."),
|
|
1107
1118
|
},
|
|
1108
1119
|
withToolLogging("platform_update_evaluator", async (params) => {
|
|
1109
1120
|
requireApiKey();
|
|
1110
|
-
const { handleUpdateEvaluator } = await import(
|
|
1111
|
-
"./tools/update-evaluator.js"
|
|
1112
|
-
);
|
|
1121
|
+
const { handleUpdateEvaluator } = await import("./tools/update-evaluator.ts");
|
|
1113
1122
|
return {
|
|
1114
|
-
content: [
|
|
1115
|
-
{ type: "text", text: await handleUpdateEvaluator(params) },
|
|
1116
|
-
],
|
|
1123
|
+
content: [{ type: "text", text: await handleUpdateEvaluator(params) }],
|
|
1117
1124
|
};
|
|
1118
|
-
})
|
|
1125
|
+
}),
|
|
1119
1126
|
);
|
|
1120
1127
|
|
|
1121
1128
|
server.tool(
|
|
@@ -1126,48 +1133,40 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1126
1133
|
},
|
|
1127
1134
|
withToolLogging("platform_delete_evaluator", async (params) => {
|
|
1128
1135
|
requireApiKey();
|
|
1129
|
-
const { handleDeleteEvaluator } = await import("./tools/delete-evaluator.
|
|
1136
|
+
const { handleDeleteEvaluator } = await import("./tools/delete-evaluator.ts");
|
|
1130
1137
|
return {
|
|
1131
1138
|
content: [{ type: "text", text: await handleDeleteEvaluator(params) }],
|
|
1132
1139
|
};
|
|
1133
|
-
})
|
|
1140
|
+
}),
|
|
1134
1141
|
);
|
|
1142
|
+
}
|
|
1135
1143
|
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1144
|
+
// --- Platform Model Provider Tools (require API key) ---
|
|
1145
|
+
// These tools manage model provider API keys on the LangWatch platform.
|
|
1146
|
+
function registerModelProviderTools(server: McpServer): void {
|
|
1139
1147
|
server.tool(
|
|
1140
1148
|
"platform_set_model_provider",
|
|
1141
1149
|
`Set or update a model provider on the LangWatch platform. Use this to configure API keys (e.g. OPENAI_API_KEY) needed to run evaluators. The API key is stored securely and never returned in responses. Omit customKeys to update other settings without changing existing keys.`,
|
|
1142
1150
|
{
|
|
1143
1151
|
provider: z
|
|
1144
1152
|
.string()
|
|
1145
|
-
.describe(
|
|
1146
|
-
'Provider name, e.g., "openai", "anthropic", "azure", "custom"'
|
|
1147
|
-
),
|
|
1153
|
+
.describe('Provider name, e.g., "openai", "anthropic", "azure", "custom"'),
|
|
1148
1154
|
enabled: z.boolean().describe("Whether the provider is enabled"),
|
|
1149
1155
|
customKeys: z
|
|
1150
1156
|
.record(z.string(), z.unknown())
|
|
1151
1157
|
.optional()
|
|
1152
1158
|
.describe(
|
|
1153
|
-
'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.'
|
|
1159
|
+
'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.',
|
|
1154
1160
|
),
|
|
1155
|
-
defaultModel: z
|
|
1156
|
-
.string()
|
|
1157
|
-
.optional()
|
|
1158
|
-
.describe("Set as project default model"),
|
|
1161
|
+
defaultModel: z.string().optional().describe("Set as project default model"),
|
|
1159
1162
|
},
|
|
1160
1163
|
withToolLogging("platform_set_model_provider", async (params) => {
|
|
1161
1164
|
requireApiKey();
|
|
1162
|
-
const { handleSetModelProvider } = await import(
|
|
1163
|
-
"./tools/set-model-provider.js"
|
|
1164
|
-
);
|
|
1165
|
+
const { handleSetModelProvider } = await import("./tools/set-model-provider.ts");
|
|
1165
1166
|
return {
|
|
1166
|
-
content: [
|
|
1167
|
-
{ type: "text", text: await handleSetModelProvider(params) },
|
|
1168
|
-
],
|
|
1167
|
+
content: [{ type: "text", text: await handleSetModelProvider(params) }],
|
|
1169
1168
|
};
|
|
1170
|
-
})
|
|
1169
|
+
}),
|
|
1171
1170
|
);
|
|
1172
1171
|
|
|
1173
1172
|
server.tool(
|
|
@@ -1176,28 +1175,27 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1176
1175
|
{},
|
|
1177
1176
|
withToolLogging("platform_list_model_providers", async () => {
|
|
1178
1177
|
requireApiKey();
|
|
1179
|
-
const { handleListModelProviders } = await import(
|
|
1180
|
-
"./tools/list-model-providers.js"
|
|
1181
|
-
);
|
|
1178
|
+
const { handleListModelProviders } = await import("./tools/list-model-providers.ts");
|
|
1182
1179
|
return {
|
|
1183
1180
|
content: [{ type: "text", text: await handleListModelProviders() }],
|
|
1184
1181
|
};
|
|
1185
|
-
})
|
|
1182
|
+
}),
|
|
1186
1183
|
);
|
|
1184
|
+
}
|
|
1187
1185
|
|
|
1188
|
-
|
|
1189
|
-
|
|
1186
|
+
// --- Platform Agent Tools (require API key) ---
|
|
1187
|
+
function registerAgentTools(server: McpServer): void {
|
|
1190
1188
|
server.tool(
|
|
1191
1189
|
"platform_list_agents",
|
|
1192
1190
|
"List all agents in the LangWatch project with their names, types and IDs. A connected agent (one that registered itself from code with connectAgent or connect_agent) also shows its environment, whether it is online, how many instances are connected, and its owner.",
|
|
1193
1191
|
{},
|
|
1194
1192
|
withToolLogging("platform_list_agents", async () => {
|
|
1195
1193
|
requireApiKey();
|
|
1196
|
-
const { handleListAgents } = await import("./tools/list-agents.
|
|
1194
|
+
const { handleListAgents } = await import("./tools/list-agents.ts");
|
|
1197
1195
|
return {
|
|
1198
1196
|
content: [{ type: "text", text: await handleListAgents() }],
|
|
1199
1197
|
};
|
|
1200
|
-
})
|
|
1198
|
+
}),
|
|
1201
1199
|
);
|
|
1202
1200
|
|
|
1203
1201
|
server.tool(
|
|
@@ -1208,11 +1206,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1208
1206
|
},
|
|
1209
1207
|
withToolLogging("platform_get_agent", async (params) => {
|
|
1210
1208
|
requireApiKey();
|
|
1211
|
-
const { handleGetAgent } = await import("./tools/get-agent.
|
|
1209
|
+
const { handleGetAgent } = await import("./tools/get-agent.ts");
|
|
1212
1210
|
return {
|
|
1213
1211
|
content: [{ type: "text", text: await handleGetAgent(params) }],
|
|
1214
1212
|
};
|
|
1215
|
-
})
|
|
1213
|
+
}),
|
|
1216
1214
|
);
|
|
1217
1215
|
|
|
1218
1216
|
server.tool(
|
|
@@ -1225,12 +1223,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1225
1223
|
},
|
|
1226
1224
|
withToolLogging("platform_create_agent", async (params) => {
|
|
1227
1225
|
requireApiKey();
|
|
1228
|
-
const { handleCreateAgent } = await import("./tools/create-agent.
|
|
1229
|
-
const parsedConfig = params.config
|
|
1226
|
+
const { handleCreateAgent } = await import("./tools/create-agent.ts");
|
|
1227
|
+
const parsedConfig = params.config
|
|
1228
|
+
? (JSON.parse(params.config) as Record<string, unknown>)
|
|
1229
|
+
: undefined;
|
|
1230
1230
|
return {
|
|
1231
|
-
content: [
|
|
1231
|
+
content: [
|
|
1232
|
+
{ type: "text", text: await handleCreateAgent({ ...params, config: parsedConfig }) },
|
|
1233
|
+
],
|
|
1232
1234
|
};
|
|
1233
|
-
})
|
|
1235
|
+
}),
|
|
1234
1236
|
);
|
|
1235
1237
|
|
|
1236
1238
|
server.tool(
|
|
@@ -1240,16 +1242,23 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1240
1242
|
id: z.string().describe("The agent ID"),
|
|
1241
1243
|
name: z.string().optional().describe("New agent name"),
|
|
1242
1244
|
type: z.string().optional().describe("New agent type: signature, code, workflow, or http"),
|
|
1243
|
-
config: z
|
|
1245
|
+
config: z
|
|
1246
|
+
.string()
|
|
1247
|
+
.optional()
|
|
1248
|
+
.describe("Updated configuration as JSON string (will be parsed)"),
|
|
1244
1249
|
},
|
|
1245
1250
|
withToolLogging("platform_update_agent", async (params) => {
|
|
1246
1251
|
requireApiKey();
|
|
1247
|
-
const { handleUpdateAgent } = await import("./tools/update-agent.
|
|
1248
|
-
const parsedConfig = params.config
|
|
1252
|
+
const { handleUpdateAgent } = await import("./tools/update-agent.ts");
|
|
1253
|
+
const parsedConfig = params.config
|
|
1254
|
+
? (JSON.parse(params.config) as Record<string, unknown>)
|
|
1255
|
+
: undefined;
|
|
1249
1256
|
return {
|
|
1250
|
-
content: [
|
|
1257
|
+
content: [
|
|
1258
|
+
{ type: "text", text: await handleUpdateAgent({ ...params, config: parsedConfig }) },
|
|
1259
|
+
],
|
|
1251
1260
|
};
|
|
1252
|
-
})
|
|
1261
|
+
}),
|
|
1253
1262
|
);
|
|
1254
1263
|
|
|
1255
1264
|
server.tool(
|
|
@@ -1260,11 +1269,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1260
1269
|
},
|
|
1261
1270
|
withToolLogging("platform_delete_agent", async (params) => {
|
|
1262
1271
|
requireApiKey();
|
|
1263
|
-
const { handleDeleteAgent } = await import("./tools/delete-agent.
|
|
1272
|
+
const { handleDeleteAgent } = await import("./tools/delete-agent.ts");
|
|
1264
1273
|
return {
|
|
1265
1274
|
content: [{ type: "text", text: await handleDeleteAgent(params) }],
|
|
1266
1275
|
};
|
|
1267
|
-
})
|
|
1276
|
+
}),
|
|
1268
1277
|
);
|
|
1269
1278
|
|
|
1270
1279
|
server.tool(
|
|
@@ -1286,23 +1295,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1286
1295
|
.record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
|
|
1287
1296
|
.optional()
|
|
1288
1297
|
.describe("Run parameter values for a connected agent, by name"),
|
|
1289
|
-
threadId: z
|
|
1290
|
-
.string()
|
|
1291
|
-
.optional()
|
|
1292
|
-
.describe("Continue a conversation on a connected agent"),
|
|
1298
|
+
threadId: z.string().optional().describe("Continue a conversation on a connected agent"),
|
|
1293
1299
|
},
|
|
1294
1300
|
withToolLogging("platform_run_agent", async (params) => {
|
|
1295
1301
|
requireApiKey();
|
|
1296
|
-
const { handleRunAgent } = await import("./tools/run-agent.
|
|
1302
|
+
const { handleRunAgent } = await import("./tools/run-agent.ts");
|
|
1297
1303
|
return {
|
|
1298
1304
|
content: [{ type: "text", text: await handleRunAgent(params) }],
|
|
1299
1305
|
};
|
|
1300
|
-
})
|
|
1306
|
+
}),
|
|
1301
1307
|
);
|
|
1302
1308
|
|
|
1303
1309
|
server.tool(
|
|
1304
1310
|
"platform_test_agent",
|
|
1305
|
-
|
|
1311
|
+
'Test an agent with one scripted scenario run: the user sends "ping", the agent answers, and the run succeeds when the answer arrives. No model is used, and no scenario, run plan or test suite is added to the project. Answers at once with the scenario run id to follow with platform_get_simulation_run; the run itself is asynchronous.',
|
|
1306
1312
|
{
|
|
1307
1313
|
id: z.string().describe("The agent ID to test"),
|
|
1308
1314
|
},
|
|
@@ -1311,22 +1317,23 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1311
1317
|
return {
|
|
1312
1318
|
content: [{ type: "text", text: await handleTestAgent(params) }],
|
|
1313
1319
|
};
|
|
1314
|
-
})
|
|
1320
|
+
}),
|
|
1315
1321
|
);
|
|
1322
|
+
}
|
|
1316
1323
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1324
|
+
// --- Platform Dashboard Tools (require API key) ---
|
|
1325
|
+
function registerDashboardTools(server: McpServer): void {
|
|
1319
1326
|
server.tool(
|
|
1320
1327
|
"platform_list_dashboards",
|
|
1321
1328
|
"List all analytics dashboards in the LangWatch project.",
|
|
1322
1329
|
{},
|
|
1323
1330
|
withToolLogging("platform_list_dashboards", async () => {
|
|
1324
1331
|
requireApiKey();
|
|
1325
|
-
const { handleListDashboards } = await import("./tools/list-dashboards.
|
|
1332
|
+
const { handleListDashboards } = await import("./tools/list-dashboards.ts");
|
|
1326
1333
|
return {
|
|
1327
1334
|
content: [{ type: "text", text: await handleListDashboards() }],
|
|
1328
1335
|
};
|
|
1329
|
-
})
|
|
1336
|
+
}),
|
|
1330
1337
|
);
|
|
1331
1338
|
|
|
1332
1339
|
server.tool(
|
|
@@ -1337,11 +1344,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1337
1344
|
},
|
|
1338
1345
|
withToolLogging("platform_get_dashboard", async (params) => {
|
|
1339
1346
|
requireApiKey();
|
|
1340
|
-
const { handleGetDashboard } = await import("./tools/get-dashboard.
|
|
1347
|
+
const { handleGetDashboard } = await import("./tools/get-dashboard.ts");
|
|
1341
1348
|
return {
|
|
1342
1349
|
content: [{ type: "text", text: await handleGetDashboard(params) }],
|
|
1343
1350
|
};
|
|
1344
|
-
})
|
|
1351
|
+
}),
|
|
1345
1352
|
);
|
|
1346
1353
|
|
|
1347
1354
|
server.tool(
|
|
@@ -1352,11 +1359,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1352
1359
|
},
|
|
1353
1360
|
withToolLogging("platform_create_dashboard", async (params) => {
|
|
1354
1361
|
requireApiKey();
|
|
1355
|
-
const { handleCreateDashboard } = await import("./tools/create-dashboard.
|
|
1362
|
+
const { handleCreateDashboard } = await import("./tools/create-dashboard.ts");
|
|
1356
1363
|
return {
|
|
1357
1364
|
content: [{ type: "text", text: await handleCreateDashboard(params) }],
|
|
1358
1365
|
};
|
|
1359
|
-
})
|
|
1366
|
+
}),
|
|
1360
1367
|
);
|
|
1361
1368
|
|
|
1362
1369
|
server.tool(
|
|
@@ -1367,11 +1374,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1367
1374
|
},
|
|
1368
1375
|
withToolLogging("platform_delete_dashboard", async (params) => {
|
|
1369
1376
|
requireApiKey();
|
|
1370
|
-
const { handleDeleteDashboard } = await import("./tools/delete-dashboard.
|
|
1377
|
+
const { handleDeleteDashboard } = await import("./tools/delete-dashboard.ts");
|
|
1371
1378
|
return {
|
|
1372
1379
|
content: [{ type: "text", text: await handleDeleteDashboard(params) }],
|
|
1373
1380
|
};
|
|
1374
|
-
})
|
|
1381
|
+
}),
|
|
1375
1382
|
);
|
|
1376
1383
|
|
|
1377
1384
|
server.tool(
|
|
@@ -1383,27 +1390,127 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1383
1390
|
},
|
|
1384
1391
|
withToolLogging("platform_rename_dashboard", async (params) => {
|
|
1385
1392
|
requireApiKey();
|
|
1386
|
-
const { renameDashboard } = await import("./langwatch-api-dashboards.
|
|
1393
|
+
const { renameDashboard } = await import("./langwatch-api-dashboards.ts");
|
|
1387
1394
|
const result = await renameDashboard(params.id, { name: params.name });
|
|
1388
1395
|
return {
|
|
1389
|
-
content: [
|
|
1396
|
+
content: [
|
|
1397
|
+
{
|
|
1398
|
+
type: "text",
|
|
1399
|
+
text: `Dashboard "${result.name}" renamed successfully (ID: ${result.id}).`,
|
|
1400
|
+
},
|
|
1401
|
+
],
|
|
1402
|
+
};
|
|
1403
|
+
}),
|
|
1404
|
+
);
|
|
1405
|
+
|
|
1406
|
+
server.tool(
|
|
1407
|
+
"list_dashboard_boards",
|
|
1408
|
+
"Lists the project's boards in the Dashboards area, with each one's ID and widget count. Use an ID from here as the dashboardId of add_dashboard_widget and update_dashboard_widget; the analytics dashboards platform_list_dashboards lists hold no widgets.",
|
|
1409
|
+
{},
|
|
1410
|
+
withToolLogging("list_dashboard_boards", async () => {
|
|
1411
|
+
requireApiKey();
|
|
1412
|
+
const { handleListDashboardBoards } = await import("./tools/list-dashboard-boards.js");
|
|
1413
|
+
return {
|
|
1414
|
+
content: [{ type: "text", text: await handleListDashboardBoards() }],
|
|
1390
1415
|
};
|
|
1391
|
-
})
|
|
1416
|
+
}),
|
|
1392
1417
|
);
|
|
1393
1418
|
|
|
1394
|
-
|
|
1419
|
+
server.tool(
|
|
1420
|
+
"add_dashboard_widget",
|
|
1421
|
+
"Creates a custom dashboard widget and places it on the given dashboard. A widget's `code` is a TSX file run in a sandbox: it receives a global `LW` with `LW.useChartQuery(name)` (reads one of the widget's own named queries), `LW.navigate` and `LW.theme`. `queries` are named LangWatchQL SQL statements the code can read by name — validate each statement's SQL first with run_query before calling this tool, since a query only runs inside the sandbox at render time and a mistake surfaces there, not here. Up to 8 queries per widget.",
|
|
1422
|
+
{
|
|
1423
|
+
dashboardId: z
|
|
1424
|
+
.string()
|
|
1425
|
+
.describe("The Dashboards board to add the widget to, as list_dashboard_boards lists it"),
|
|
1426
|
+
name: z.string().describe("The widget's display name"),
|
|
1427
|
+
code: z
|
|
1428
|
+
.string()
|
|
1429
|
+
.describe(
|
|
1430
|
+
"The widget's TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope.",
|
|
1431
|
+
),
|
|
1432
|
+
queries: z
|
|
1433
|
+
.union([
|
|
1434
|
+
z.record(z.string(), z.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1435
|
+
z
|
|
1436
|
+
.array(
|
|
1437
|
+
z.object({
|
|
1438
|
+
name: z.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1439
|
+
sql: z.string().describe("The LangWatchQL statement"),
|
|
1440
|
+
}),
|
|
1441
|
+
)
|
|
1442
|
+
.describe("Named LWQL queries, array form"),
|
|
1443
|
+
])
|
|
1444
|
+
.describe(
|
|
1445
|
+
"The widget's named LangWatchQL queries: either {name: sql} or an array of {name, sql}. Max 8. Validate each statement with run_query first.",
|
|
1446
|
+
),
|
|
1447
|
+
},
|
|
1448
|
+
withToolLogging("add_dashboard_widget", async (params) => {
|
|
1449
|
+
requireApiKey();
|
|
1450
|
+
const { handleAddDashboardWidget } = await import("./tools/add-dashboard-widget.js");
|
|
1451
|
+
return {
|
|
1452
|
+
content: [{ type: "text", text: await handleAddDashboardWidget(params) }],
|
|
1453
|
+
};
|
|
1454
|
+
}),
|
|
1455
|
+
);
|
|
1395
1456
|
|
|
1457
|
+
server.tool(
|
|
1458
|
+
"update_dashboard_widget",
|
|
1459
|
+
"Changes one custom widget on a dashboard: its name, its TSX `code`, its named LangWatchQL `queries` or its `description`. Pass only the fields to change; every field left out keeps its stored value. Validate any new query's SQL with run_query first. Returns the updated widget.",
|
|
1460
|
+
{
|
|
1461
|
+
dashboardId: z.string().describe("The dashboard the widget is on"),
|
|
1462
|
+
widgetId: z.string().describe("The widget to change, as add_dashboard_widget returned it"),
|
|
1463
|
+
name: z.string().optional().describe("The widget's new display name"),
|
|
1464
|
+
code: z
|
|
1465
|
+
.string()
|
|
1466
|
+
.optional()
|
|
1467
|
+
.describe(
|
|
1468
|
+
"The widget's new TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope.",
|
|
1469
|
+
),
|
|
1470
|
+
queries: z
|
|
1471
|
+
.union([
|
|
1472
|
+
z.record(z.string(), z.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1473
|
+
z
|
|
1474
|
+
.array(
|
|
1475
|
+
z.object({
|
|
1476
|
+
name: z.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1477
|
+
sql: z.string().describe("The LangWatchQL statement"),
|
|
1478
|
+
}),
|
|
1479
|
+
)
|
|
1480
|
+
.describe("Named LWQL queries, array form"),
|
|
1481
|
+
])
|
|
1482
|
+
.optional()
|
|
1483
|
+
.describe(
|
|
1484
|
+
"The widget's new named LangWatchQL queries, replacing the stored set: either {name: sql} or an array of {name, sql}. Max 8.",
|
|
1485
|
+
),
|
|
1486
|
+
description: z
|
|
1487
|
+
.string()
|
|
1488
|
+
.optional()
|
|
1489
|
+
.describe("What the widget's info tip says: a sentence or two"),
|
|
1490
|
+
},
|
|
1491
|
+
withToolLogging("update_dashboard_widget", async (params) => {
|
|
1492
|
+
requireApiKey();
|
|
1493
|
+
const { handleUpdateDashboardWidget } = await import("./tools/update-dashboard-widget.js");
|
|
1494
|
+
return {
|
|
1495
|
+
content: [{ type: "text", text: await handleUpdateDashboardWidget(params) }],
|
|
1496
|
+
};
|
|
1497
|
+
}),
|
|
1498
|
+
);
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
// --- Platform Workflow Tools (require API key) ---
|
|
1502
|
+
function registerWorkflowTools(server: McpServer): void {
|
|
1396
1503
|
server.tool(
|
|
1397
1504
|
"platform_list_workflows",
|
|
1398
1505
|
"List all workflows in the LangWatch project.",
|
|
1399
1506
|
{},
|
|
1400
1507
|
withToolLogging("platform_list_workflows", async () => {
|
|
1401
1508
|
requireApiKey();
|
|
1402
|
-
const { handleListWorkflows } = await import("./tools/list-workflows.
|
|
1509
|
+
const { handleListWorkflows } = await import("./tools/list-workflows.ts");
|
|
1403
1510
|
return {
|
|
1404
1511
|
content: [{ type: "text", text: await handleListWorkflows() }],
|
|
1405
1512
|
};
|
|
1406
|
-
})
|
|
1513
|
+
}),
|
|
1407
1514
|
);
|
|
1408
1515
|
|
|
1409
1516
|
server.tool(
|
|
@@ -1414,11 +1521,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1414
1521
|
},
|
|
1415
1522
|
withToolLogging("platform_get_workflow", async (params) => {
|
|
1416
1523
|
requireApiKey();
|
|
1417
|
-
const { handleGetWorkflow } = await import("./tools/get-workflow.
|
|
1524
|
+
const { handleGetWorkflow } = await import("./tools/get-workflow.ts");
|
|
1418
1525
|
return {
|
|
1419
1526
|
content: [{ type: "text", text: await handleGetWorkflow(params) }],
|
|
1420
1527
|
};
|
|
1421
|
-
})
|
|
1528
|
+
}),
|
|
1422
1529
|
);
|
|
1423
1530
|
|
|
1424
1531
|
server.tool(
|
|
@@ -1429,11 +1536,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1429
1536
|
},
|
|
1430
1537
|
withToolLogging("platform_delete_workflow", async (params) => {
|
|
1431
1538
|
requireApiKey();
|
|
1432
|
-
const { handleDeleteWorkflow } = await import("./tools/delete-workflow.
|
|
1539
|
+
const { handleDeleteWorkflow } = await import("./tools/delete-workflow.ts");
|
|
1433
1540
|
return {
|
|
1434
1541
|
content: [{ type: "text", text: await handleDeleteWorkflow(params) }],
|
|
1435
1542
|
};
|
|
1436
|
-
})
|
|
1543
|
+
}),
|
|
1437
1544
|
);
|
|
1438
1545
|
|
|
1439
1546
|
server.tool(
|
|
@@ -1445,15 +1552,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1445
1552
|
},
|
|
1446
1553
|
withToolLogging("platform_run_workflow", async (params) => {
|
|
1447
1554
|
requireApiKey();
|
|
1448
|
-
const { handleRunWorkflow } = await import("./tools/run-workflow.
|
|
1555
|
+
const { handleRunWorkflow } = await import("./tools/run-workflow.ts");
|
|
1449
1556
|
return {
|
|
1450
1557
|
content: [{ type: "text", text: await handleRunWorkflow(params) }],
|
|
1451
1558
|
};
|
|
1452
|
-
})
|
|
1559
|
+
}),
|
|
1453
1560
|
);
|
|
1561
|
+
}
|
|
1454
1562
|
|
|
1455
|
-
|
|
1456
|
-
|
|
1563
|
+
// --- Platform Annotation Tools (require API key) ---
|
|
1564
|
+
function registerAnnotationTools(server: McpServer): void {
|
|
1457
1565
|
server.tool(
|
|
1458
1566
|
"platform_list_annotations",
|
|
1459
1567
|
"List all annotations for the project, optionally filtered by trace ID.",
|
|
@@ -1462,11 +1570,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1462
1570
|
},
|
|
1463
1571
|
withToolLogging("platform_list_annotations", async (params) => {
|
|
1464
1572
|
requireApiKey();
|
|
1465
|
-
const { handleListAnnotations } = await import("./tools/list-annotations.
|
|
1573
|
+
const { handleListAnnotations } = await import("./tools/list-annotations.ts");
|
|
1466
1574
|
return {
|
|
1467
1575
|
content: [{ type: "text", text: await handleListAnnotations(params) }],
|
|
1468
1576
|
};
|
|
1469
|
-
})
|
|
1577
|
+
}),
|
|
1470
1578
|
);
|
|
1471
1579
|
|
|
1472
1580
|
server.tool(
|
|
@@ -1480,11 +1588,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1480
1588
|
},
|
|
1481
1589
|
withToolLogging("platform_create_annotation", async (params) => {
|
|
1482
1590
|
requireApiKey();
|
|
1483
|
-
const { handleCreateAnnotation } = await import("./tools/create-annotation.
|
|
1591
|
+
const { handleCreateAnnotation } = await import("./tools/create-annotation.ts");
|
|
1484
1592
|
return {
|
|
1485
1593
|
content: [{ type: "text", text: await handleCreateAnnotation(params) }],
|
|
1486
1594
|
};
|
|
1487
|
-
})
|
|
1595
|
+
}),
|
|
1488
1596
|
);
|
|
1489
1597
|
|
|
1490
1598
|
server.tool(
|
|
@@ -1495,11 +1603,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1495
1603
|
},
|
|
1496
1604
|
withToolLogging("platform_delete_annotation", async (params) => {
|
|
1497
1605
|
requireApiKey();
|
|
1498
|
-
const { handleDeleteAnnotation } = await import("./tools/delete-annotation.
|
|
1606
|
+
const { handleDeleteAnnotation } = await import("./tools/delete-annotation.ts");
|
|
1499
1607
|
return {
|
|
1500
1608
|
content: [{ type: "text", text: await handleDeleteAnnotation(params) }],
|
|
1501
1609
|
};
|
|
1502
|
-
})
|
|
1610
|
+
}),
|
|
1503
1611
|
);
|
|
1504
1612
|
|
|
1505
1613
|
server.tool(
|
|
@@ -1510,7 +1618,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1510
1618
|
},
|
|
1511
1619
|
withToolLogging("platform_get_annotation", async (params) => {
|
|
1512
1620
|
requireApiKey();
|
|
1513
|
-
const { getAnnotation } = await import("./langwatch-api-annotations.
|
|
1621
|
+
const { getAnnotation } = await import("./langwatch-api-annotations.ts");
|
|
1514
1622
|
const annotation = await getAnnotation(params.id);
|
|
1515
1623
|
const lines = [
|
|
1516
1624
|
`**ID**: ${annotation.id}`,
|
|
@@ -1522,11 +1630,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1522
1630
|
return {
|
|
1523
1631
|
content: [{ type: "text", text: lines.join("\n") }],
|
|
1524
1632
|
};
|
|
1525
|
-
})
|
|
1633
|
+
}),
|
|
1526
1634
|
);
|
|
1635
|
+
}
|
|
1527
1636
|
|
|
1528
|
-
|
|
1529
|
-
|
|
1637
|
+
// --- Platform Trigger/Automation Tools (require API key) ---
|
|
1638
|
+
function registerTriggerTools(server: McpServer): void {
|
|
1530
1639
|
server.tool(
|
|
1531
1640
|
"platform_list_triggers",
|
|
1532
1641
|
"List all triggers (automations) in the project. Triggers automate actions like sending emails, Slack messages, or adding to datasets when conditions are met.",
|
|
@@ -1535,72 +1644,155 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1535
1644
|
},
|
|
1536
1645
|
withToolLogging("platform_list_triggers", async (params) => {
|
|
1537
1646
|
requireApiKey();
|
|
1538
|
-
const {
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
1556
|
-
})
|
|
1647
|
+
const { handleListTriggers } = await import("./tools/list-triggers.ts");
|
|
1648
|
+
return { content: [{ type: "text", text: await handleListTriggers(params) }] };
|
|
1649
|
+
}),
|
|
1650
|
+
);
|
|
1651
|
+
|
|
1652
|
+
server.tool(
|
|
1653
|
+
"platform_get_trigger",
|
|
1654
|
+
"Read one trigger (automation) by its ID, including its delivery configuration. Credential values come back as [redacted]; sending them back unchanged on an update keeps the stored values.",
|
|
1655
|
+
{
|
|
1656
|
+
id: z.string().describe("The trigger ID"),
|
|
1657
|
+
},
|
|
1658
|
+
withToolLogging("platform_get_trigger", async (params) => {
|
|
1659
|
+
requireApiKey();
|
|
1660
|
+
const { getTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1661
|
+
const trigger = await getTrigger(params.id);
|
|
1662
|
+
return { content: [{ type: "text", text: JSON.stringify(trigger, null, 2) }] };
|
|
1663
|
+
}),
|
|
1557
1664
|
);
|
|
1558
1665
|
|
|
1559
1666
|
server.tool(
|
|
1560
1667
|
"platform_create_trigger",
|
|
1561
|
-
|
|
1668
|
+
[
|
|
1669
|
+
"Create a trigger (automation).",
|
|
1670
|
+
"",
|
|
1671
|
+
"It is about one of three things: matching traces (send `filters` or `filterQuery`), a metric crossing a threshold on a graph (an Alert: send `customGraphId`, `graphAlert` and `alertType`), or a schedule (a Report: send `report`, on an email or Slack channel).",
|
|
1672
|
+
"`actionParams` is the delivery configuration for the channel named in `action` — recipients for SEND_EMAIL, a destination for SEND_SLACK_MESSAGE, a URL for SEND_WEBHOOK, a dataset and mapping for ADD_TO_DATASET, annotators for ADD_TO_ANNOTATION_QUEUE. Omitted, it is sent as {} and the server refuses a channel that cannot deliver without one.",
|
|
1673
|
+
SLACK_DELIVERY_NOTE,
|
|
1674
|
+
`filters: ${TRIGGER_FILTERS_DESCRIPTION}`,
|
|
1675
|
+
"The delivery channel cannot be changed afterwards.",
|
|
1676
|
+
].join("\n"),
|
|
1562
1677
|
{
|
|
1563
1678
|
name: z.string().describe("Trigger name"),
|
|
1564
|
-
action:
|
|
1565
|
-
|
|
1679
|
+
action: triggerActionSchema.describe("Which channel it delivers on"),
|
|
1680
|
+
actionParams: actionParamsSchema
|
|
1681
|
+
.optional()
|
|
1682
|
+
.describe("The delivery configuration the channel named in `action` reads. Defaults to {}"),
|
|
1683
|
+
filters: z.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
1684
|
+
filterQuery: z.string().optional().describe(TRIGGER_FILTER_QUERY_DESCRIPTION),
|
|
1685
|
+
customGraphId: z.string().optional().describe("Set to make this an alert on that graph"),
|
|
1686
|
+
graphAlert: graphAlertSchema.optional(),
|
|
1687
|
+
report: reportSchema.optional().describe("Set to make this a scheduled report"),
|
|
1688
|
+
templates: templatesSchema.optional(),
|
|
1689
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
1566
1690
|
message: z.string().optional().describe("Custom alert message"),
|
|
1567
|
-
alertType:
|
|
1691
|
+
alertType: alertTypeSchema.optional().describe("Alert severity. Required for a graph alert."),
|
|
1568
1692
|
},
|
|
1569
1693
|
withToolLogging("platform_create_trigger", async (params) => {
|
|
1570
1694
|
requireApiKey();
|
|
1571
|
-
const {
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
try { filters = JSON.parse(params.filters) as Record<string, unknown>; }
|
|
1575
|
-
catch { return { content: [{ type: "text", text: "Error: filters must be valid JSON" }] }; }
|
|
1576
|
-
}
|
|
1577
|
-
const trigger = await createTrigger({
|
|
1578
|
-
name: params.name,
|
|
1579
|
-
action: params.action,
|
|
1580
|
-
filters,
|
|
1581
|
-
message: params.message,
|
|
1582
|
-
alertType: params.alertType,
|
|
1583
|
-
});
|
|
1584
|
-
return { content: [{ type: "text", text: `Trigger "${trigger.name}" created (ID: ${trigger.id}, Action: ${trigger.action}).` }] };
|
|
1585
|
-
})
|
|
1695
|
+
const { handleCreateTrigger } = await import("./tools/create-trigger.ts");
|
|
1696
|
+
return handleCreateTrigger(params);
|
|
1697
|
+
}),
|
|
1586
1698
|
);
|
|
1587
1699
|
|
|
1588
1700
|
server.tool(
|
|
1589
1701
|
"platform_update_trigger",
|
|
1590
|
-
|
|
1702
|
+
[
|
|
1703
|
+
"Update a trigger (automation). Anything left out is left alone.",
|
|
1704
|
+
"",
|
|
1705
|
+
"`actionParams` replaces the delivery configuration as a whole, so send every field it should have from now on. A credential the read hid comes back as [redacted]; send that back to keep the stored value. Changing a webhook's `url` means sending its header values in the same call.",
|
|
1706
|
+
"The delivery channel, the kind of automation and an alert's graph cannot be changed.",
|
|
1707
|
+
].join("\n"),
|
|
1591
1708
|
{
|
|
1592
1709
|
id: z.string().describe("The trigger ID"),
|
|
1593
1710
|
name: z.string().optional().describe("New name"),
|
|
1594
|
-
active: z.boolean().optional().describe("
|
|
1595
|
-
|
|
1596
|
-
|
|
1711
|
+
active: z.boolean().optional().describe("Resume or pause it"),
|
|
1712
|
+
actionParams: actionParamsSchema
|
|
1713
|
+
.optional()
|
|
1714
|
+
.describe("The delivery configuration this automation should have from now on"),
|
|
1715
|
+
filters: z.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
1716
|
+
filterQuery: z
|
|
1717
|
+
.string()
|
|
1718
|
+
.nullable()
|
|
1719
|
+
.optional()
|
|
1720
|
+
.describe(`${TRIGGER_FILTER_QUERY_DESCRIPTION} null clears the saved query.`),
|
|
1721
|
+
graphAlert: graphAlertSchema
|
|
1722
|
+
.optional()
|
|
1723
|
+
.describe("Only for an automation that is already a graph alert"),
|
|
1724
|
+
report: reportSchema
|
|
1725
|
+
.optional()
|
|
1726
|
+
.describe("Only for an automation that is already a scheduled report"),
|
|
1727
|
+
templates: templatesSchema.optional(),
|
|
1728
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
1729
|
+
message: z
|
|
1730
|
+
.string()
|
|
1731
|
+
.nullable()
|
|
1732
|
+
.optional()
|
|
1733
|
+
.describe("New alert message. null removes the custom message"),
|
|
1734
|
+
alertType: alertTypeSchema
|
|
1735
|
+
.nullable()
|
|
1736
|
+
.optional()
|
|
1737
|
+
.describe("New alert severity. null clears it"),
|
|
1597
1738
|
},
|
|
1598
1739
|
withToolLogging("platform_update_trigger", async (params) => {
|
|
1599
1740
|
requireApiKey();
|
|
1600
|
-
const { updateTrigger } = await import("./langwatch-api-triggers.
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1741
|
+
const { getTrigger, updateTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1742
|
+
if (params.actionParams !== undefined) {
|
|
1743
|
+
// The channel cannot change on update, so the SAVED action decides
|
|
1744
|
+
// which shape the replacement configuration must fit.
|
|
1745
|
+
const saved = await getTrigger(params.id);
|
|
1746
|
+
const boundParams = validateActionParamsForAction({
|
|
1747
|
+
action: saved.action,
|
|
1748
|
+
actionParams: params.actionParams,
|
|
1749
|
+
});
|
|
1750
|
+
if (!boundParams.ok) {
|
|
1751
|
+
const { toolError } = await import("./tools/create-trigger.ts");
|
|
1752
|
+
return toolError(boundParams.message);
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
const { parseJsonObject, toolError } = await import("./tools/create-trigger.ts");
|
|
1756
|
+
const filters = params.filters ? parseJsonObject(params.filters) : undefined;
|
|
1757
|
+
if (params.filters && !filters) {
|
|
1758
|
+
return toolError("filters must be a JSON object");
|
|
1759
|
+
}
|
|
1760
|
+
const trigger = await updateTrigger({ ...params, filters });
|
|
1761
|
+
return {
|
|
1762
|
+
content: [
|
|
1763
|
+
{ type: "text", text: `Trigger "${trigger.name}" updated (active: ${trigger.active}).` },
|
|
1764
|
+
],
|
|
1765
|
+
};
|
|
1766
|
+
}),
|
|
1767
|
+
);
|
|
1768
|
+
|
|
1769
|
+
server.tool(
|
|
1770
|
+
"platform_test_fire_trigger",
|
|
1771
|
+
"Send a trigger's message to the destination it is configured with, to confirm it arrives. Nothing is recorded as a fire.",
|
|
1772
|
+
{
|
|
1773
|
+
id: z.string().describe("The trigger ID"),
|
|
1774
|
+
},
|
|
1775
|
+
withToolLogging("platform_test_fire_trigger", async (params) => {
|
|
1776
|
+
requireApiKey();
|
|
1777
|
+
const { testFireTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1778
|
+
const result = await testFireTrigger(params.id);
|
|
1779
|
+
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
1780
|
+
}),
|
|
1781
|
+
);
|
|
1782
|
+
|
|
1783
|
+
server.tool(
|
|
1784
|
+
"platform_list_trigger_fires",
|
|
1785
|
+
"What an automation has done, newest first. Metadata only — no message content. Answers with `nextCursor`; pass it back as `cursor` for the next page (null means there is none).",
|
|
1786
|
+
{
|
|
1787
|
+
id: z.string().describe("The trigger ID"),
|
|
1788
|
+
limit: z.number().int().positive().optional().describe("How many fires to return"),
|
|
1789
|
+
cursor: z.string().optional().describe("The `nextCursor` a previous call answered with"),
|
|
1790
|
+
},
|
|
1791
|
+
withToolLogging("platform_list_trigger_fires", async (params) => {
|
|
1792
|
+
requireApiKey();
|
|
1793
|
+
const { handleListTriggerFires } = await import("./tools/list-trigger-fires.ts");
|
|
1794
|
+
return { content: [{ type: "text", text: await handleListTriggerFires(params) }] };
|
|
1795
|
+
}),
|
|
1604
1796
|
);
|
|
1605
1797
|
|
|
1606
1798
|
server.tool(
|
|
@@ -1611,28 +1803,29 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1611
1803
|
},
|
|
1612
1804
|
withToolLogging("platform_delete_trigger", async (params) => {
|
|
1613
1805
|
requireApiKey();
|
|
1614
|
-
const { deleteTrigger } = await import("./langwatch-api-triggers.
|
|
1806
|
+
const { deleteTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1615
1807
|
const result = await deleteTrigger(params.id);
|
|
1616
1808
|
return { content: [{ type: "text", text: `Trigger ${result.id} deleted.` }] };
|
|
1617
|
-
})
|
|
1809
|
+
}),
|
|
1618
1810
|
);
|
|
1811
|
+
}
|
|
1619
1812
|
|
|
1620
|
-
|
|
1621
|
-
|
|
1813
|
+
// --- Platform Monitor Tools (require API key) ---
|
|
1814
|
+
function registerMonitorTools(server: McpServer): void {
|
|
1622
1815
|
server.tool(
|
|
1623
1816
|
"platform_list_monitors",
|
|
1624
1817
|
"List all online evaluation monitors for the project.",
|
|
1625
1818
|
{},
|
|
1626
1819
|
withToolLogging("platform_list_monitors", async () => {
|
|
1627
1820
|
requireApiKey();
|
|
1628
|
-
const { listMonitors } = await import("./langwatch-api-monitors.
|
|
1821
|
+
const { listMonitors } = await import("./langwatch-api-monitors.ts");
|
|
1629
1822
|
const monitors = await listMonitors();
|
|
1630
1823
|
if (monitors.length === 0) {
|
|
1631
1824
|
return { content: [{ type: "text", text: "No monitors found." }] };
|
|
1632
1825
|
}
|
|
1633
1826
|
const lines = monitors.map(
|
|
1634
1827
|
(m) =>
|
|
1635
|
-
`• ${m.name} (id: ${m.id}, type: ${m.checkType}, ${m.enabled ? "enabled" : "disabled"}, mode: ${m.executionMode}, sample: ${Math.round(m.sample * 100)}%)
|
|
1828
|
+
`• ${m.name} (id: ${m.id}, type: ${m.checkType}, ${m.enabled ? "enabled" : "disabled"}, mode: ${m.executionMode}, sample: ${Math.round(m.sample * 100)}%)`,
|
|
1636
1829
|
);
|
|
1637
1830
|
return {
|
|
1638
1831
|
content: [
|
|
@@ -1642,7 +1835,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1642
1835
|
},
|
|
1643
1836
|
],
|
|
1644
1837
|
};
|
|
1645
|
-
})
|
|
1838
|
+
}),
|
|
1646
1839
|
);
|
|
1647
1840
|
|
|
1648
1841
|
server.tool(
|
|
@@ -1653,7 +1846,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1653
1846
|
},
|
|
1654
1847
|
withToolLogging("platform_get_monitor", async (params) => {
|
|
1655
1848
|
requireApiKey();
|
|
1656
|
-
const { getMonitor } = await import("./langwatch-api-monitors.
|
|
1849
|
+
const { getMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1657
1850
|
const m = await getMonitor(params.id);
|
|
1658
1851
|
const lines = [
|
|
1659
1852
|
`**${m.name}** (${m.id})`,
|
|
@@ -1666,7 +1859,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1666
1859
|
`- Created: ${m.createdAt}`,
|
|
1667
1860
|
].filter(Boolean);
|
|
1668
1861
|
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
1669
|
-
})
|
|
1862
|
+
}),
|
|
1670
1863
|
);
|
|
1671
1864
|
|
|
1672
1865
|
server.tool(
|
|
@@ -1674,27 +1867,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1674
1867
|
"Create a new online evaluation monitor that runs an evaluator on incoming traces.",
|
|
1675
1868
|
{
|
|
1676
1869
|
name: z.string().describe("Monitor name"),
|
|
1677
|
-
checkType: z
|
|
1678
|
-
.string()
|
|
1679
|
-
.describe("Evaluator check type (e.g. ragas/toxicity, custom/my-eval)"),
|
|
1870
|
+
checkType: z.string().describe("Evaluator check type (e.g. ragas/toxicity, custom/my-eval)"),
|
|
1680
1871
|
executionMode: z
|
|
1681
1872
|
.enum(["ON_MESSAGE", "AS_GUARDRAIL", "MANUALLY"])
|
|
1682
1873
|
.optional()
|
|
1683
1874
|
.describe("When to run: ON_MESSAGE (default), AS_GUARDRAIL, or MANUALLY"),
|
|
1684
|
-
sample: z
|
|
1685
|
-
|
|
1686
|
-
.min(0)
|
|
1687
|
-
.max(1)
|
|
1688
|
-
.optional()
|
|
1689
|
-
.describe("Sampling rate 0.0-1.0 (default: 1.0)"),
|
|
1690
|
-
evaluatorId: z
|
|
1691
|
-
.string()
|
|
1692
|
-
.optional()
|
|
1693
|
-
.describe("Link to a saved evaluator by ID"),
|
|
1875
|
+
sample: z.number().min(0).max(1).optional().describe("Sampling rate 0.0-1.0 (default: 1.0)"),
|
|
1876
|
+
evaluatorId: z.string().optional().describe("Link to a saved evaluator by ID"),
|
|
1694
1877
|
},
|
|
1695
1878
|
withToolLogging("platform_create_monitor", async (params) => {
|
|
1696
1879
|
requireApiKey();
|
|
1697
|
-
const { createMonitor } = await import("./langwatch-api-monitors.
|
|
1880
|
+
const { createMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1698
1881
|
const monitor = await createMonitor(params);
|
|
1699
1882
|
return {
|
|
1700
1883
|
content: [
|
|
@@ -1704,7 +1887,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1704
1887
|
},
|
|
1705
1888
|
],
|
|
1706
1889
|
};
|
|
1707
|
-
})
|
|
1890
|
+
}),
|
|
1708
1891
|
);
|
|
1709
1892
|
|
|
1710
1893
|
server.tool(
|
|
@@ -1722,7 +1905,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1722
1905
|
},
|
|
1723
1906
|
withToolLogging("platform_update_monitor", async (params) => {
|
|
1724
1907
|
requireApiKey();
|
|
1725
|
-
const { updateMonitor } = await import("./langwatch-api-monitors.
|
|
1908
|
+
const { updateMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1726
1909
|
const monitor = await updateMonitor(params);
|
|
1727
1910
|
return {
|
|
1728
1911
|
content: [
|
|
@@ -1732,7 +1915,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1732
1915
|
},
|
|
1733
1916
|
],
|
|
1734
1917
|
};
|
|
1735
|
-
})
|
|
1918
|
+
}),
|
|
1736
1919
|
);
|
|
1737
1920
|
|
|
1738
1921
|
server.tool(
|
|
@@ -1743,32 +1926,29 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1743
1926
|
},
|
|
1744
1927
|
withToolLogging("platform_delete_monitor", async (params) => {
|
|
1745
1928
|
requireApiKey();
|
|
1746
|
-
const { deleteMonitor } = await import("./langwatch-api-monitors.
|
|
1929
|
+
const { deleteMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1747
1930
|
const result = await deleteMonitor(params.id);
|
|
1748
1931
|
return {
|
|
1749
|
-
content: [
|
|
1750
|
-
{ type: "text", text: `Monitor ${result.id} deleted.` },
|
|
1751
|
-
],
|
|
1932
|
+
content: [{ type: "text", text: `Monitor ${result.id} deleted.` }],
|
|
1752
1933
|
};
|
|
1753
|
-
})
|
|
1934
|
+
}),
|
|
1754
1935
|
);
|
|
1936
|
+
}
|
|
1755
1937
|
|
|
1756
|
-
|
|
1757
|
-
|
|
1938
|
+
// --- Platform Secret Tools (require API key) ---
|
|
1939
|
+
function registerSecretTools(server: McpServer): void {
|
|
1758
1940
|
server.tool(
|
|
1759
1941
|
"platform_list_secrets",
|
|
1760
1942
|
"List all project secrets (values are never returned, only metadata).",
|
|
1761
1943
|
{},
|
|
1762
1944
|
withToolLogging("platform_list_secrets", async () => {
|
|
1763
1945
|
requireApiKey();
|
|
1764
|
-
const { listSecrets } = await import("./langwatch-api-secrets.
|
|
1946
|
+
const { listSecrets } = await import("./langwatch-api-secrets.ts");
|
|
1765
1947
|
const secrets = await listSecrets();
|
|
1766
1948
|
if (secrets.length === 0) {
|
|
1767
1949
|
return { content: [{ type: "text", text: "No secrets found." }] };
|
|
1768
1950
|
}
|
|
1769
|
-
const lines = secrets.map(
|
|
1770
|
-
(s) => `• ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`
|
|
1771
|
-
);
|
|
1951
|
+
const lines = secrets.map((s) => `• ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`);
|
|
1772
1952
|
return {
|
|
1773
1953
|
content: [
|
|
1774
1954
|
{
|
|
@@ -1777,7 +1957,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1777
1957
|
},
|
|
1778
1958
|
],
|
|
1779
1959
|
};
|
|
1780
|
-
})
|
|
1960
|
+
}),
|
|
1781
1961
|
);
|
|
1782
1962
|
|
|
1783
1963
|
server.tool(
|
|
@@ -1789,7 +1969,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1789
1969
|
},
|
|
1790
1970
|
withToolLogging("platform_create_secret", async (params) => {
|
|
1791
1971
|
requireApiKey();
|
|
1792
|
-
const { createSecret } = await import("./langwatch-api-secrets.
|
|
1972
|
+
const { createSecret } = await import("./langwatch-api-secrets.ts");
|
|
1793
1973
|
const secret = await createSecret({ name: params.name, value: params.value });
|
|
1794
1974
|
return {
|
|
1795
1975
|
content: [
|
|
@@ -1799,7 +1979,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1799
1979
|
},
|
|
1800
1980
|
],
|
|
1801
1981
|
};
|
|
1802
|
-
})
|
|
1982
|
+
}),
|
|
1803
1983
|
);
|
|
1804
1984
|
|
|
1805
1985
|
server.tool(
|
|
@@ -1811,7 +1991,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1811
1991
|
},
|
|
1812
1992
|
withToolLogging("platform_update_secret", async (params) => {
|
|
1813
1993
|
requireApiKey();
|
|
1814
|
-
const { updateSecret } = await import("./langwatch-api-secrets.
|
|
1994
|
+
const { updateSecret } = await import("./langwatch-api-secrets.ts");
|
|
1815
1995
|
const secret = await updateSecret({ id: params.id, value: params.value });
|
|
1816
1996
|
return {
|
|
1817
1997
|
content: [
|
|
@@ -1821,7 +2001,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1821
2001
|
},
|
|
1822
2002
|
],
|
|
1823
2003
|
};
|
|
1824
|
-
})
|
|
2004
|
+
}),
|
|
1825
2005
|
);
|
|
1826
2006
|
|
|
1827
2007
|
server.tool(
|
|
@@ -1832,14 +2012,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1832
2012
|
},
|
|
1833
2013
|
withToolLogging("platform_delete_secret", async (params) => {
|
|
1834
2014
|
requireApiKey();
|
|
1835
|
-
const { deleteSecret } = await import("./langwatch-api-secrets.
|
|
2015
|
+
const { deleteSecret } = await import("./langwatch-api-secrets.ts");
|
|
1836
2016
|
const result = await deleteSecret(params.id);
|
|
1837
2017
|
return { content: [{ type: "text", text: `Secret ${result.id} deleted.` }] };
|
|
1838
|
-
})
|
|
2018
|
+
}),
|
|
1839
2019
|
);
|
|
2020
|
+
}
|
|
1840
2021
|
|
|
1841
|
-
|
|
1842
|
-
|
|
2022
|
+
// --- Platform Experiment Execution Tools (require API key) ---
|
|
2023
|
+
function registerExperimentTools(server: McpServer): void {
|
|
1843
2024
|
server.tool(
|
|
1844
2025
|
"platform_run_experiment",
|
|
1845
2026
|
"Start an experiment run by slug. Returns a run ID for polling status.",
|
|
@@ -1851,7 +2032,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1851
2032
|
return {
|
|
1852
2033
|
content: [{ type: "text", text: await handleRunExperiment(params) }],
|
|
1853
2034
|
};
|
|
1854
|
-
})
|
|
2035
|
+
}),
|
|
1855
2036
|
);
|
|
1856
2037
|
|
|
1857
2038
|
server.tool(
|
|
@@ -1871,7 +2052,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1871
2052
|
return {
|
|
1872
2053
|
content: [{ type: "text", text: await handleExperimentStatus(params) }],
|
|
1873
2054
|
};
|
|
1874
|
-
})
|
|
2055
|
+
}),
|
|
1875
2056
|
);
|
|
1876
2057
|
|
|
1877
2058
|
server.tool(
|
|
@@ -1884,16 +2065,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1884
2065
|
.positive()
|
|
1885
2066
|
.max(100)
|
|
1886
2067
|
.optional()
|
|
1887
|
-
.describe(
|
|
1888
|
-
"Maximum number of experiments to include (default 25, hard-capped at 100)",
|
|
1889
|
-
),
|
|
2068
|
+
.describe("Maximum number of experiments to include (default 25, hard-capped at 100)"),
|
|
1890
2069
|
},
|
|
1891
2070
|
withToolLogging("platform_experiment_list", async (params) => {
|
|
1892
2071
|
requireApiKey();
|
|
1893
2072
|
return {
|
|
1894
2073
|
content: [{ type: "text", text: await handleExperimentList(params) }],
|
|
1895
2074
|
};
|
|
1896
|
-
})
|
|
2075
|
+
}),
|
|
1897
2076
|
);
|
|
1898
2077
|
|
|
1899
2078
|
server.tool(
|
|
@@ -1902,34 +2081,28 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1902
2081
|
{
|
|
1903
2082
|
experimentSlug: z
|
|
1904
2083
|
.string()
|
|
1905
|
-
.describe(
|
|
1906
|
-
"Experiment slug from platform_experiment_list (e.g. 'checkout-flow')",
|
|
1907
|
-
),
|
|
2084
|
+
.describe("Experiment slug from platform_experiment_list (e.g. 'checkout-flow')"),
|
|
1908
2085
|
limit: z
|
|
1909
2086
|
.number()
|
|
1910
2087
|
.int()
|
|
1911
2088
|
.positive()
|
|
1912
2089
|
.max(100)
|
|
1913
2090
|
.optional()
|
|
1914
|
-
.describe(
|
|
1915
|
-
"Maximum number of runs to include (default 25, hard-capped at 100)",
|
|
1916
|
-
),
|
|
2091
|
+
.describe("Maximum number of runs to include (default 25, hard-capped at 100)"),
|
|
1917
2092
|
},
|
|
1918
2093
|
withToolLogging("platform_experiment_list_runs", async (params) => {
|
|
1919
2094
|
requireApiKey();
|
|
1920
2095
|
return {
|
|
1921
2096
|
content: [{ type: "text", text: await handleExperimentListRuns(params) }],
|
|
1922
2097
|
};
|
|
1923
|
-
})
|
|
2098
|
+
}),
|
|
1924
2099
|
);
|
|
1925
2100
|
|
|
1926
2101
|
server.tool(
|
|
1927
2102
|
"platform_experiment_results",
|
|
1928
2103
|
"Fetch per-row results for an experiment run so you can debug evaluator scores and missed rows. Serves partial results for runs that are still running or were interrupted (it does not require a completed run), with the run status noted in the output. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window — narrow with `filter: 'failed'` or `evaluator` to see what matters.",
|
|
1929
2104
|
{
|
|
1930
|
-
runId: z
|
|
1931
|
-
.string()
|
|
1932
|
-
.describe("The run ID returned from platform_run_experiment"),
|
|
2105
|
+
runId: z.string().describe("The run ID returned from platform_run_experiment"),
|
|
1933
2106
|
experimentSlug: z
|
|
1934
2107
|
.string()
|
|
1935
2108
|
.optional()
|
|
@@ -1959,16 +2132,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1959
2132
|
withToolLogging("platform_experiment_results", async (params) => {
|
|
1960
2133
|
requireApiKey();
|
|
1961
2134
|
return {
|
|
1962
|
-
content: [
|
|
1963
|
-
{ type: "text", text: await handleExperimentResults(params) },
|
|
1964
|
-
],
|
|
2135
|
+
content: [{ type: "text", text: await handleExperimentResults(params) }],
|
|
1965
2136
|
};
|
|
1966
|
-
})
|
|
2137
|
+
}),
|
|
1967
2138
|
);
|
|
2139
|
+
}
|
|
1968
2140
|
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
2141
|
+
// --- Platform Dataset Tools (require API key) ---
|
|
2142
|
+
// These tools manage datasets on the LangWatch platform via API.
|
|
2143
|
+
function registerDatasetTools(server: McpServer): void {
|
|
1972
2144
|
server.tool(
|
|
1973
2145
|
"platform_list_datasets",
|
|
1974
2146
|
"List all datasets on the LangWatch platform with their names, slugs, columns, and record counts. Returns AI-readable digest by default.",
|
|
@@ -1976,20 +2148,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1976
2148
|
format: z
|
|
1977
2149
|
.enum(["digest", "json"])
|
|
1978
2150
|
.optional()
|
|
1979
|
-
.describe(
|
|
1980
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
1981
|
-
),
|
|
2151
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
1982
2152
|
},
|
|
1983
2153
|
async (params) => {
|
|
1984
|
-
const { requireApiKey } = await import("./config.
|
|
2154
|
+
const { requireApiKey } = await import("./config.ts");
|
|
1985
2155
|
requireApiKey();
|
|
1986
|
-
const { handleListDatasets } = await import(
|
|
1987
|
-
"./tools/list-datasets.js"
|
|
1988
|
-
);
|
|
2156
|
+
const { handleListDatasets } = await import("./tools/list-datasets.ts");
|
|
1989
2157
|
return {
|
|
1990
2158
|
content: [{ type: "text", text: await handleListDatasets(params) }],
|
|
1991
2159
|
};
|
|
1992
|
-
}
|
|
2160
|
+
},
|
|
1993
2161
|
);
|
|
1994
2162
|
|
|
1995
2163
|
server.tool(
|
|
@@ -2000,18 +2168,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2000
2168
|
format: z
|
|
2001
2169
|
.enum(["digest", "json"])
|
|
2002
2170
|
.optional()
|
|
2003
|
-
.describe(
|
|
2004
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
2005
|
-
),
|
|
2171
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
2006
2172
|
},
|
|
2007
2173
|
async (params) => {
|
|
2008
|
-
const { requireApiKey } = await import("./config.
|
|
2174
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2009
2175
|
requireApiKey();
|
|
2010
|
-
const { handleGetDataset } = await import("./tools/get-dataset.
|
|
2176
|
+
const { handleGetDataset } = await import("./tools/get-dataset.ts");
|
|
2011
2177
|
return {
|
|
2012
2178
|
content: [{ type: "text", text: await handleGetDataset(params) }],
|
|
2013
2179
|
};
|
|
2014
|
-
}
|
|
2180
|
+
},
|
|
2015
2181
|
);
|
|
2016
2182
|
|
|
2017
2183
|
server.tool(
|
|
@@ -2019,15 +2185,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2019
2185
|
"Create a new dataset on the LangWatch platform.",
|
|
2020
2186
|
createDatasetSchema.shape,
|
|
2021
2187
|
async (params) => {
|
|
2022
|
-
const { requireApiKey } = await import("./config.
|
|
2188
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2023
2189
|
requireApiKey();
|
|
2024
|
-
const { handleCreateDataset } = await import(
|
|
2025
|
-
"./tools/create-dataset.js"
|
|
2026
|
-
);
|
|
2190
|
+
const { handleCreateDataset } = await import("./tools/create-dataset.ts");
|
|
2027
2191
|
return {
|
|
2028
2192
|
content: [{ type: "text", text: await handleCreateDataset(params) }],
|
|
2029
2193
|
};
|
|
2030
|
-
}
|
|
2194
|
+
},
|
|
2031
2195
|
);
|
|
2032
2196
|
|
|
2033
2197
|
server.tool(
|
|
@@ -2042,15 +2206,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2042
2206
|
.describe("Updated column definitions"),
|
|
2043
2207
|
},
|
|
2044
2208
|
async (params) => {
|
|
2045
|
-
const { requireApiKey } = await import("./config.
|
|
2209
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2046
2210
|
requireApiKey();
|
|
2047
|
-
const { handleUpdateDataset } = await import(
|
|
2048
|
-
"./tools/update-dataset.js"
|
|
2049
|
-
);
|
|
2211
|
+
const { handleUpdateDataset } = await import("./tools/update-dataset.ts");
|
|
2050
2212
|
return {
|
|
2051
2213
|
content: [{ type: "text", text: await handleUpdateDataset(params) }],
|
|
2052
2214
|
};
|
|
2053
|
-
}
|
|
2215
|
+
},
|
|
2054
2216
|
);
|
|
2055
2217
|
|
|
2056
2218
|
server.tool(
|
|
@@ -2060,30 +2222,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2060
2222
|
slugOrId: z.string().describe("The dataset slug or ID to delete"),
|
|
2061
2223
|
},
|
|
2062
2224
|
async (params) => {
|
|
2063
|
-
const { requireApiKey } = await import("./config.
|
|
2225
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2064
2226
|
requireApiKey();
|
|
2065
|
-
const { handleDeleteDataset } = await import(
|
|
2066
|
-
"./tools/delete-dataset.js"
|
|
2067
|
-
);
|
|
2227
|
+
const { handleDeleteDataset } = await import("./tools/delete-dataset.ts");
|
|
2068
2228
|
return {
|
|
2069
2229
|
content: [{ type: "text", text: await handleDeleteDataset(params) }],
|
|
2070
2230
|
};
|
|
2071
|
-
}
|
|
2231
|
+
},
|
|
2072
2232
|
);
|
|
2073
2233
|
|
|
2074
2234
|
server.tool(
|
|
2075
2235
|
"platform_list_dataset_records",
|
|
2076
2236
|
"List records in a dataset on the LangWatch platform with pagination.",
|
|
2077
2237
|
{
|
|
2078
|
-
slugOrId: z
|
|
2079
|
-
|
|
2080
|
-
.describe("The dataset slug or ID to list records from"),
|
|
2081
|
-
page: z
|
|
2082
|
-
.number()
|
|
2083
|
-
.int()
|
|
2084
|
-
.min(1)
|
|
2085
|
-
.optional()
|
|
2086
|
-
.describe("Page number (default: 1)"),
|
|
2238
|
+
slugOrId: z.string().describe("The dataset slug or ID to list records from"),
|
|
2239
|
+
page: z.number().int().min(1).optional().describe("Page number (default: 1)"),
|
|
2087
2240
|
limit: z
|
|
2088
2241
|
.number()
|
|
2089
2242
|
.int()
|
|
@@ -2097,26 +2250,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2097
2250
|
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
2098
2251
|
},
|
|
2099
2252
|
async (params) => {
|
|
2100
|
-
const { requireApiKey } = await import("./config.
|
|
2253
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2101
2254
|
requireApiKey();
|
|
2102
|
-
const { handleListDatasetRecords } = await import(
|
|
2103
|
-
"./tools/list-dataset-records.js"
|
|
2104
|
-
);
|
|
2255
|
+
const { handleListDatasetRecords } = await import("./tools/list-dataset-records.ts");
|
|
2105
2256
|
return {
|
|
2106
|
-
content: [
|
|
2107
|
-
{ type: "text", text: await handleListDatasetRecords(params) },
|
|
2108
|
-
],
|
|
2257
|
+
content: [{ type: "text", text: await handleListDatasetRecords(params) }],
|
|
2109
2258
|
};
|
|
2110
|
-
}
|
|
2259
|
+
},
|
|
2111
2260
|
);
|
|
2112
2261
|
|
|
2113
2262
|
server.tool(
|
|
2114
2263
|
"platform_create_dataset_records",
|
|
2115
2264
|
"Add records to a dataset on the LangWatch platform in batch (max 1000 per call).",
|
|
2116
2265
|
{
|
|
2117
|
-
slugOrId: z
|
|
2118
|
-
.string()
|
|
2119
|
-
.describe("The dataset slug or ID to add records to"),
|
|
2266
|
+
slugOrId: z.string().describe("The dataset slug or ID to add records to"),
|
|
2120
2267
|
entries: z
|
|
2121
2268
|
.array(z.record(z.string(), z.unknown()))
|
|
2122
2269
|
.min(1)
|
|
@@ -2124,90 +2271,75 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2124
2271
|
.describe("Array of record entries to create (key-value objects matching dataset columns)"),
|
|
2125
2272
|
},
|
|
2126
2273
|
async (params) => {
|
|
2127
|
-
const { requireApiKey } = await import("./config.
|
|
2274
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2128
2275
|
requireApiKey();
|
|
2129
|
-
const { handleCreateDatasetRecords } = await import(
|
|
2130
|
-
"./tools/create-dataset-records.js"
|
|
2131
|
-
);
|
|
2276
|
+
const { handleCreateDatasetRecords } = await import("./tools/create-dataset-records.ts");
|
|
2132
2277
|
return {
|
|
2133
|
-
content: [
|
|
2134
|
-
{ type: "text", text: await handleCreateDatasetRecords(params) },
|
|
2135
|
-
],
|
|
2278
|
+
content: [{ type: "text", text: await handleCreateDatasetRecords(params) }],
|
|
2136
2279
|
};
|
|
2137
|
-
}
|
|
2280
|
+
},
|
|
2138
2281
|
);
|
|
2139
2282
|
|
|
2140
2283
|
server.tool(
|
|
2141
2284
|
"platform_update_dataset_record",
|
|
2142
2285
|
"Update a single record in a dataset on the LangWatch platform.",
|
|
2143
2286
|
{
|
|
2144
|
-
slugOrId: z
|
|
2145
|
-
.string()
|
|
2146
|
-
.describe("The dataset slug or ID containing the record"),
|
|
2287
|
+
slugOrId: z.string().describe("The dataset slug or ID containing the record"),
|
|
2147
2288
|
recordId: z.string().describe("The record ID to update"),
|
|
2148
|
-
entry: z
|
|
2149
|
-
.record(z.string(), z.unknown())
|
|
2150
|
-
.describe("Updated record entry (key-value object)"),
|
|
2289
|
+
entry: z.record(z.string(), z.unknown()).describe("Updated record entry (key-value object)"),
|
|
2151
2290
|
},
|
|
2152
2291
|
async (params) => {
|
|
2153
|
-
const { requireApiKey } = await import("./config.
|
|
2292
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2154
2293
|
requireApiKey();
|
|
2155
|
-
const { handleUpdateDatasetRecord } = await import(
|
|
2156
|
-
"./tools/update-dataset-record.js"
|
|
2157
|
-
);
|
|
2294
|
+
const { handleUpdateDatasetRecord } = await import("./tools/update-dataset-record.ts");
|
|
2158
2295
|
return {
|
|
2159
|
-
content: [
|
|
2160
|
-
{ type: "text", text: await handleUpdateDatasetRecord(params) },
|
|
2161
|
-
],
|
|
2296
|
+
content: [{ type: "text", text: await handleUpdateDatasetRecord(params) }],
|
|
2162
2297
|
};
|
|
2163
|
-
}
|
|
2298
|
+
},
|
|
2164
2299
|
);
|
|
2165
2300
|
|
|
2166
2301
|
server.tool(
|
|
2167
2302
|
"platform_delete_dataset_records",
|
|
2168
2303
|
"Delete records from a dataset on the LangWatch platform by their IDs.",
|
|
2169
2304
|
{
|
|
2170
|
-
slugOrId: z
|
|
2171
|
-
|
|
2172
|
-
.describe("The dataset slug or ID containing the records"),
|
|
2173
|
-
recordIds: z
|
|
2174
|
-
.array(z.string())
|
|
2175
|
-
.min(1)
|
|
2176
|
-
.max(1000)
|
|
2177
|
-
.describe("Array of record IDs to delete"),
|
|
2305
|
+
slugOrId: z.string().describe("The dataset slug or ID containing the records"),
|
|
2306
|
+
recordIds: z.array(z.string()).min(1).max(1000).describe("Array of record IDs to delete"),
|
|
2178
2307
|
},
|
|
2179
2308
|
async (params) => {
|
|
2180
|
-
const { requireApiKey } = await import("./config.
|
|
2309
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2181
2310
|
requireApiKey();
|
|
2182
|
-
const { handleDeleteDatasetRecords } = await import(
|
|
2183
|
-
"./tools/delete-dataset-records.js"
|
|
2184
|
-
);
|
|
2311
|
+
const { handleDeleteDatasetRecords } = await import("./tools/delete-dataset-records.ts");
|
|
2185
2312
|
return {
|
|
2186
|
-
content: [
|
|
2187
|
-
{ type: "text", text: await handleDeleteDatasetRecords(params) },
|
|
2188
|
-
],
|
|
2313
|
+
content: [{ type: "text", text: await handleDeleteDatasetRecords(params) }],
|
|
2189
2314
|
};
|
|
2190
|
-
}
|
|
2315
|
+
},
|
|
2191
2316
|
);
|
|
2317
|
+
}
|
|
2192
2318
|
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2319
|
+
// --- Platform Project Tools ---
|
|
2320
|
+
// These tools manage projects on the LangWatch platform via the REST API.
|
|
2321
|
+
// They require an API key with org-level permissions.
|
|
2322
|
+
function registerProjectTools(server: McpServer): void {
|
|
2197
2323
|
server.tool(
|
|
2198
2324
|
"platform_list_projects",
|
|
2199
2325
|
"List all projects in your LangWatch organization. Requires an org-level API key.",
|
|
2200
2326
|
{
|
|
2201
2327
|
page: z.number().int().positive().optional().describe("Page number (default: 1)"),
|
|
2202
|
-
limit: z
|
|
2328
|
+
limit: z
|
|
2329
|
+
.number()
|
|
2330
|
+
.int()
|
|
2331
|
+
.positive()
|
|
2332
|
+
.max(1000)
|
|
2333
|
+
.optional()
|
|
2334
|
+
.describe("Results per page (default: 100)"),
|
|
2203
2335
|
},
|
|
2204
2336
|
withToolLogging("platform_list_projects", async (params) => {
|
|
2205
2337
|
requireApiKey();
|
|
2206
|
-
const { handleListProjects } = await import("./tools/list-projects.
|
|
2338
|
+
const { handleListProjects } = await import("./tools/list-projects.ts");
|
|
2207
2339
|
return {
|
|
2208
2340
|
content: [{ type: "text", text: await handleListProjects(params) }],
|
|
2209
2341
|
};
|
|
2210
|
-
})
|
|
2342
|
+
}),
|
|
2211
2343
|
);
|
|
2212
2344
|
|
|
2213
2345
|
server.tool(
|
|
@@ -2218,11 +2350,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2218
2350
|
},
|
|
2219
2351
|
withToolLogging("platform_get_project", async (params) => {
|
|
2220
2352
|
requireApiKey();
|
|
2221
|
-
const { handleGetProject } = await import("./tools/get-project.
|
|
2353
|
+
const { handleGetProject } = await import("./tools/get-project.ts");
|
|
2222
2354
|
return {
|
|
2223
2355
|
content: [{ type: "text", text: await handleGetProject(params) }],
|
|
2224
2356
|
};
|
|
2225
|
-
})
|
|
2357
|
+
}),
|
|
2226
2358
|
);
|
|
2227
2359
|
|
|
2228
2360
|
server.tool(
|
|
@@ -2239,11 +2371,11 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2239
2371
|
},
|
|
2240
2372
|
withToolLogging("platform_create_project", async (params) => {
|
|
2241
2373
|
requireApiKey();
|
|
2242
|
-
const { handleCreateProject } = await import("./tools/create-project.
|
|
2374
|
+
const { handleCreateProject } = await import("./tools/create-project.ts");
|
|
2243
2375
|
return {
|
|
2244
2376
|
content: [{ type: "text", text: await handleCreateProject(params) }],
|
|
2245
2377
|
};
|
|
2246
|
-
})
|
|
2378
|
+
}),
|
|
2247
2379
|
);
|
|
2248
2380
|
|
|
2249
2381
|
server.tool(
|
|
@@ -2254,15 +2386,18 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2254
2386
|
name: z.string().optional().describe("New project name"),
|
|
2255
2387
|
language: z.string().optional().describe("New programming language"),
|
|
2256
2388
|
framework: z.string().optional().describe("New framework"),
|
|
2257
|
-
piiRedactionLevel: z
|
|
2389
|
+
piiRedactionLevel: z
|
|
2390
|
+
.enum(["STRICT", "ESSENTIAL", "DISABLED"])
|
|
2391
|
+
.optional()
|
|
2392
|
+
.describe("PII redaction level"),
|
|
2258
2393
|
},
|
|
2259
2394
|
withToolLogging("platform_update_project", async (params) => {
|
|
2260
2395
|
requireApiKey();
|
|
2261
|
-
const { handleUpdateProject } = await import("./tools/update-project.
|
|
2396
|
+
const { handleUpdateProject } = await import("./tools/update-project.ts");
|
|
2262
2397
|
return {
|
|
2263
2398
|
content: [{ type: "text", text: await handleUpdateProject(params) }],
|
|
2264
2399
|
};
|
|
2265
|
-
})
|
|
2400
|
+
}),
|
|
2266
2401
|
);
|
|
2267
2402
|
|
|
2268
2403
|
server.tool(
|
|
@@ -2273,27 +2408,28 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2273
2408
|
},
|
|
2274
2409
|
withToolLogging("platform_archive_project", async (params) => {
|
|
2275
2410
|
requireApiKey();
|
|
2276
|
-
const { handleArchiveProject } = await import("./tools/archive-project.
|
|
2411
|
+
const { handleArchiveProject } = await import("./tools/archive-project.ts");
|
|
2277
2412
|
return {
|
|
2278
2413
|
content: [{ type: "text", text: await handleArchiveProject(params) }],
|
|
2279
2414
|
};
|
|
2280
|
-
})
|
|
2415
|
+
}),
|
|
2281
2416
|
);
|
|
2417
|
+
}
|
|
2282
2418
|
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2419
|
+
// --- Platform API Key Tools (require org-level API key) ---
|
|
2420
|
+
// These tools manage API keys on the LangWatch platform via the REST API.
|
|
2421
|
+
function registerApiKeyTools(server: McpServer): void {
|
|
2286
2422
|
server.tool(
|
|
2287
2423
|
"platform_list_api_keys",
|
|
2288
2424
|
"List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
|
|
2289
2425
|
{},
|
|
2290
2426
|
withToolLogging("platform_list_api_keys", async () => {
|
|
2291
2427
|
requireApiKey();
|
|
2292
|
-
const { handleListApiKeys } = await import("./tools/list-api-keys.
|
|
2428
|
+
const { handleListApiKeys } = await import("./tools/list-api-keys.ts");
|
|
2293
2429
|
return {
|
|
2294
2430
|
content: [{ type: "text", text: await handleListApiKeys() }],
|
|
2295
2431
|
};
|
|
2296
|
-
})
|
|
2432
|
+
}),
|
|
2297
2433
|
);
|
|
2298
2434
|
|
|
2299
2435
|
server.tool(
|
|
@@ -2305,24 +2441,34 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2305
2441
|
|
|
2306
2442
|
The token is returned once and cannot be retrieved again.`,
|
|
2307
2443
|
{
|
|
2308
|
-
keyType: z
|
|
2444
|
+
keyType: z
|
|
2445
|
+
.enum(["personal", "service"])
|
|
2446
|
+
.describe("Key type: 'personal' (user-tied) or 'service' (headless)"),
|
|
2309
2447
|
name: z.string().describe("Key name (max 100 chars)"),
|
|
2310
2448
|
description: z.string().optional().describe("Key description (max 500 chars)"),
|
|
2311
2449
|
expiresAt: z.string().optional().describe("Expiration date (ISO string)"),
|
|
2312
|
-
bindings: z
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2450
|
+
bindings: z
|
|
2451
|
+
.array(
|
|
2452
|
+
z.object({
|
|
2453
|
+
role: z.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
|
|
2454
|
+
scopeType: z.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
|
|
2455
|
+
scopeId: z.string().describe("Scope ID (org, team, or project ID)"),
|
|
2456
|
+
}),
|
|
2457
|
+
)
|
|
2458
|
+
.optional()
|
|
2459
|
+
.describe("Role bindings (required for personal keys)"),
|
|
2460
|
+
projectIds: z
|
|
2461
|
+
.array(z.string())
|
|
2462
|
+
.optional()
|
|
2463
|
+
.describe("Project IDs for service keys (creates ADMIN bindings per project)"),
|
|
2318
2464
|
},
|
|
2319
2465
|
withToolLogging("platform_create_api_key", async (params) => {
|
|
2320
2466
|
requireApiKey();
|
|
2321
|
-
const { handleCreateApiKey } = await import("./tools/create-api-key.
|
|
2467
|
+
const { handleCreateApiKey } = await import("./tools/create-api-key.ts");
|
|
2322
2468
|
return {
|
|
2323
2469
|
content: [{ type: "text", text: await handleCreateApiKey(params) }],
|
|
2324
2470
|
};
|
|
2325
|
-
})
|
|
2471
|
+
}),
|
|
2326
2472
|
);
|
|
2327
2473
|
|
|
2328
2474
|
server.tool(
|
|
@@ -2333,10 +2479,10 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2333
2479
|
},
|
|
2334
2480
|
withToolLogging("platform_revoke_api_key", async (params) => {
|
|
2335
2481
|
requireApiKey();
|
|
2336
|
-
const { handleRevokeApiKey } = await import("./tools/revoke-api-key.
|
|
2482
|
+
const { handleRevokeApiKey } = await import("./tools/revoke-api-key.ts");
|
|
2337
2483
|
return {
|
|
2338
2484
|
content: [{ type: "text", text: await handleRevokeApiKey(params) }],
|
|
2339
2485
|
};
|
|
2340
|
-
})
|
|
2486
|
+
}),
|
|
2341
2487
|
);
|
|
2342
2488
|
}
|