@langwatch/mcp-server 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +218 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-55UI6H22.js +44 -0
- package/dist/add-dashboard-widget-MJHIC3CC.js +45 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-K5UD7A6X.js} +4 -4
- package/dist/archive-project-L4PAKT34.js +19 -0
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-5HSQG2HX.js} +4 -4
- package/dist/archive-run-plan-AOKNYQIO.js +15 -0
- package/dist/archive-scenario-2CWJW2VD.js +19 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-5IJQESHH.js} +4 -4
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-OPB7NFI7.js} +4 -4
- package/dist/archive-test-suite-ZC3TN6VH.js +15 -0
- package/dist/assign-prompt-tag-6DDE6ZZ5.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-D7O23MJM.js} +8 -4
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/{chunk-EYXMLE5X.js → chunk-4RX3G62A.js} +56 -47
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7CZQDZ2K.js +319 -0
- package/dist/{chunk-RVAE2MEA.js → chunk-7GETGE6B.js} +7 -7
- package/dist/{chunk-OJLYYJWI.js → chunk-AASW2TO2.js} +7 -10
- package/dist/{chunk-QAMP3CGR.js → chunk-B4RFWIUP.js} +4 -4
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-G4ZAZTZW.js → chunk-D2HS73VI.js} +7 -10
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-WT4PO4A3.js → chunk-DULFP3MK.js} +10 -7
- package/dist/chunk-EYPJBL7P.js +516 -0
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/chunk-GA5JUSPT.js +64 -0
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/chunk-HLQJXUR6.js +67 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-KILVK3FS.js} +4 -4
- package/dist/chunk-MOCU5XWF.js +79 -0
- package/dist/chunk-O2WGVCED.js +41 -0
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-UFOYFYI7.js → chunk-QXLMHLRC.js} +6 -9
- package/dist/{chunk-QRO35SOM.js → chunk-R5YJIYPF.js} +981 -747
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/{chunk-SR6QPRIE.js → chunk-RFC2SBEX.js} +7 -13
- package/dist/chunk-SHSVN3SE.js +21 -0
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/{chunk-56OGTS24.js → chunk-VOUGDY2O.js} +4 -11
- package/dist/chunk-WCKZDOGF.js +31 -0
- package/dist/chunk-XQYH52SI.js +129 -0
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/{chunk-6DJDEXET.js → chunk-YCIUEZVK.js} +6 -12
- package/dist/config-2WQGMAEN.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-6JNLMYQD.js} +4 -4
- package/dist/create-agent-IRZOU4R5.js +23 -0
- package/dist/create-analytics-report-N4L4V5QL.js +19 -0
- package/dist/create-analytics-report-QEHXG7FU.js +18 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-CCEGIZ67.js} +10 -5
- package/dist/create-annotation-DTHCMWHB.js +29 -0
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-JKM2DSZO.js} +5 -7
- package/dist/create-api-key-JPEUIXGM.js +24 -0
- package/dist/create-dataset-DJNXYJ5B.js +22 -0
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-WON4SOTC.js} +6 -6
- package/dist/create-dataset-records-TMEOTLTO.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-UJBW3C7C.js} +5 -7
- package/dist/create-evaluator-N5AEHI4K.js +27 -0
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-XYNY2YFY.js} +4 -4
- package/dist/create-mcp-server.cjs +9712 -0
- package/dist/create-mcp-server.js +2757 -10
- package/dist/{create-project-ZQ7TFIO2.js → create-project-EY4XZ7TH.js} +4 -4
- package/dist/create-project-YL345T5L.js +44 -0
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-JZQNQ7I3.js} +5 -7
- package/dist/create-prompt-ZUMQHHX7.js +37 -0
- package/dist/create-prompt-tag-CJ5FZ436.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-JFP2Q3PO.js} +3 -3
- package/dist/create-scenario-CQU2SSFB.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-S6OWWKJO.js} +5 -5
- package/dist/create-test-suite-OLGCAGJA.js +36 -0
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-Y7P2RI4L.js} +6 -6
- package/dist/create-trigger-4CNBTISJ.js +48 -0
- package/dist/create-trigger-RQ653WYF.js +49 -0
- package/dist/dashboards-BA7M6CMO.js +90 -0
- package/dist/dashboards-U663GUIB.js +91 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-J2Q4DRZ2.js} +4 -4
- package/dist/delete-agent-ZAU4LTXB.js +18 -0
- package/dist/delete-analytics-report-QK6X7P4S.js +19 -0
- package/dist/delete-analytics-report-YUYLDHLL.js +18 -0
- package/dist/delete-annotation-7X4NYXXM.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-KJOCDLV3.js} +4 -4
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-NXP7ZPPV.js} +5 -7
- package/dist/delete-dataset-RUTZBBVD.js +18 -0
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-4Z7DZCDW.js} +5 -7
- package/dist/delete-dataset-records-75UAR5IS.js +19 -0
- package/dist/delete-evaluator-E3CUNGTQ.js +16 -0
- package/dist/delete-evaluator-RI4GALGX.js +15 -0
- package/dist/delete-prompt-tag-LE3WJPKS.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-S4KBZTNG.js} +3 -3
- package/dist/delete-workflow-4CZFIQCQ.js +17 -0
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-PWQPDW6H.js} +4 -4
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/discover-schema-7WHIIATI.js +442 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-J2ANOBKO.js} +181 -186
- package/dist/get-agent-5T3CXQOR.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-ZKYQJUQ7.js} +15 -11
- package/dist/get-analytics-4MPEK4KV.js +79 -0
- package/dist/get-analytics-ZLI3A4F5.js +80 -0
- package/dist/get-analytics-report-JCKYMA2S.js +29 -0
- package/dist/get-analytics-report-SKIP2JWH.js +30 -0
- package/dist/get-dataset-APGZBYMH.js +64 -0
- package/dist/get-dataset-LWXDVMVE.js +63 -0
- package/dist/get-evaluator-3A7BJBAB.js +47 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-IWNJINRY.js} +4 -4
- package/dist/get-project-EQVB4LEL.js +26 -0
- package/dist/{get-project-QSQEAB33.js → get-project-RK6AJTEP.js} +4 -4
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-TLG2T2DS.js} +53 -58
- package/dist/get-prompt-ZER557X3.js +94 -0
- package/dist/get-run-plan-5X7DDJRN.js +58 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-PEHRO5M3.js} +7 -9
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-375W5D6I.js} +5 -5
- package/dist/get-scenario-UQX45ZZO.js +37 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-2NBXJADG.js} +26 -22
- package/dist/get-simulation-run-QZTLGL7W.js +69 -0
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GVIBDG7K.js} +5 -5
- package/dist/get-test-suite-OODIR3RI.js +48 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-AGCHVPJK.js} +11 -13
- package/dist/get-trace-KJLYAZRC.js +45 -0
- package/dist/get-workflow-3TV2YYC5.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-7LZSVPP2.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-7CCN3EHJ.js} +4249 -325
- package/dist/index.js +59595 -38027
- package/dist/langwatch-api-analytics-reports-6CF6KUET.js +17 -0
- package/dist/langwatch-api-analytics-reports-RPFB5AO5.js +18 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-H4AI4TKG.js} +4 -4
- package/dist/langwatch-api-annotations-QSJNUQQS.js +17 -0
- package/dist/langwatch-api-monitors-ENHLOL3S.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-OHSGADTH.js} +11 -16
- package/dist/langwatch-api-secrets-AGW6UUMD.js +55 -0
- package/dist/langwatch-api-secrets-CLTS766Q.js +56 -0
- package/dist/langwatch-api-triggers-G5UWFNN2.js +22 -0
- package/dist/langwatch-api-triggers-QMR7KEXW.js +23 -0
- package/dist/langwatch-api-workflows-HKTBIKKY.js +15 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OMROZH2O.js} +4 -4
- package/dist/{list-agents-AMVDQPQL.js → list-agents-BYWDNAHT.js} +4 -4
- package/dist/list-agents-MG24MHZE.js +42 -0
- package/dist/list-analytics-reports-G3EPWGSQ.js +32 -0
- package/dist/list-analytics-reports-PPWIFRES.js +33 -0
- package/dist/list-annotations-QCLGEK5A.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-Y4EUBJHG.js} +10 -5
- package/dist/list-api-keys-DAVOBPFK.js +42 -0
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-JMW5WRXX.js} +10 -5
- package/dist/list-dashboards-2P3VJFRS.js +29 -0
- package/dist/list-dashboards-MW5O3MKH.js +30 -0
- package/dist/list-dataset-records-KEHT7G6Q.js +43 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-KMLLMZAZ.js} +14 -10
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-P3JK2ZHS.js} +9 -13
- package/dist/list-datasets-Z7OOE4KI.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-7RRESBOU.js} +5 -7
- package/dist/list-evaluators-VATJVOHR.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-OXDRPUCJ.js} +4 -4
- package/dist/list-model-providers-VXQJWDXU.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-HUVK7L3W.js} +5 -7
- package/dist/list-projects-X72ZXVY5.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-3I3LWFIR.js} +3 -3
- package/dist/list-prompt-tags-DRHUH2ND.js +23 -0
- package/dist/list-prompts-6DX3QECJ.js +30 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-PS423RU3.js} +4 -6
- package/dist/list-run-plans-5SXPOV3X.js +45 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-6VNHINPY.js} +5 -5
- package/dist/list-scenarios-EQWCW7Q4.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-GKNH3YND.js} +13 -11
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-KHY7KALI.js} +12 -8
- package/dist/list-simulation-runs-PUDK32C2.js +53 -0
- package/dist/list-test-suites-7H2DRILK.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-GY56LGLY.js} +4 -4
- package/dist/list-trigger-fires-7RZUP54Y.js +19 -0
- package/dist/list-trigger-fires-OZUF65LI.js +20 -0
- package/dist/list-triggers-BGUCUXTJ.js +45 -0
- package/dist/list-triggers-XC5KMX4C.js +46 -0
- package/dist/list-workflows-55NYC3PC.js +33 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-HV7FUWZA.js} +5 -7
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-2I7Z5FCY.js} +3 -3
- package/dist/rename-prompt-tag-SZYTOWDX.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-27AYY33K.js} +4 -4
- package/dist/rename-test-suite-CLIEPZDO.js +20 -0
- package/dist/report-issue-5HSL7CNB.js +924 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-N7ND5TM6.js} +107 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-O3VUZ6CG.js} +5 -5
- package/dist/rerun-run-plan-OJTVVGJI.js +21 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-BU6BKGCG.js} +4 -4
- package/dist/revoke-api-key-NZNCFOIR.js +15 -0
- package/dist/run-agent-244O4Y5M.js +75 -0
- package/dist/run-agent-HPCGZGYW.js +74 -0
- package/dist/run-plan-2ZPGGZ4H.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-KSXUYQT3.js} +7 -7
- package/dist/run-query-4LMOQ5UW.js +47 -0
- package/dist/run-query-BQUHIUC2.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-P3WLVPXC.js} +6 -6
- package/dist/run-test-suite-UEOMCON6.js +25 -0
- package/dist/run-workflow-EFZ6HG5B.js +36 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-I4IHEN54.js} +4 -4
- package/dist/search-traces-6IR7PR4L.js +170 -0
- package/dist/search-traces-BYML2NQ4.js +171 -0
- package/dist/set-model-provider-RBNGWFVI.js +33 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-SHEVKWRC.js} +4 -4
- package/dist/{update-agent-DUWNYBNA.js → update-agent-PTKTM2B5.js} +4 -4
- package/dist/update-agent-ZPXWRMS2.js +20 -0
- package/dist/update-dashboard-widget-HKCBXH7X.js +62 -0
- package/dist/update-dashboard-widget-IP2TTQ5G.js +63 -0
- package/dist/update-dataset-GHIN6JR3.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-IRVMY5PF.js} +6 -6
- package/dist/update-dataset-record-6VI7K2CL.js +19 -0
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-Z3L7ACNF.js} +4 -4
- package/dist/update-evaluator-5NLNE6VY.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-FEH7H6NS.js} +4 -4
- package/dist/update-project-75H3LDQT.js +24 -0
- package/dist/{update-project-Q26FBSYH.js → update-project-QPR3QX55.js} +4 -4
- package/dist/update-prompt-BCDFTNLK.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-NPRXYSWA.js} +27 -29
- package/dist/update-scenario-BNOMDCMV.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-EM5XGMB7.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +23 -8
- package/src/config.d.ts +6 -0
- package/src/config.ts +22 -29
- package/src/create-mcp-server.ts +887 -634
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-analytics-reports.ts +62 -0
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-dashboard-widgets.ts +47 -0
- package/src/langwatch-api-dashboards.ts +109 -24
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +10 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +118 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/schemas/widget-sources.ts +33 -0
- package/src/tools/add-dashboard-widget.ts +51 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-analytics-report.ts +7 -0
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-widget-queries.ts +45 -0
- package/src/tools/dashboards.ts +104 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-analytics-report.ts +7 -0
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics-report.ts +23 -0
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-analytics-reports.ts +27 -0
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboards.ts +20 -21
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +7 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +37 -34
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +86 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/uv.lock +667 -341
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/chunk-S4GVX36W.js +0 -39
- package/dist/config.d.ts +0 -17
- package/dist/create-dashboard-Z76AIPLS.js +0 -19
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-dashboard-LYMRPF27.js +0 -19
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dashboard-P4WWVHFU.js +0 -30
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-dashboards-2KJL2M42.js +0 -18
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/list-dashboards-Z3BZKDGF.js +0 -33
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
- package/src/tools/create-dashboard.ts +0 -7
- package/src/tools/delete-dashboard.ts +0 -7
- package/src/tools/get-dashboard.ts +0 -22
package/src/create-mcp-server.ts
CHANGED
|
@@ -2,33 +2,44 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
|
|
4
4
|
import packageJson from "../package.json" with { type: "json" };
|
|
5
|
-
import { requireApiKey } from "./config.
|
|
6
|
-
import { fetchDocumentation } from "./documentation-fetch.
|
|
7
|
-
import {
|
|
8
|
-
createDatasetSchema,
|
|
9
|
-
datasetColumnDefinitionSchema,
|
|
10
|
-
} from "./schemas/create-dataset.js";
|
|
5
|
+
import { requireApiKey } from "./config.ts";
|
|
6
|
+
import { fetchDocumentation } from "./documentation-fetch.ts";
|
|
7
|
+
import { createDatasetSchema, datasetColumnDefinitionSchema } from "./schemas/create-dataset.ts";
|
|
11
8
|
import {
|
|
12
9
|
runParametersSchema,
|
|
13
10
|
runPlanScopeSchema,
|
|
14
11
|
runPlanTargetSchema,
|
|
15
|
-
} from "./schemas/run-plan.
|
|
12
|
+
} from "./schemas/run-plan.ts";
|
|
16
13
|
import {
|
|
17
14
|
evaluatorAttachmentsSchema,
|
|
18
15
|
scenarioFieldValuesSchema,
|
|
19
16
|
suiteFieldsSchema,
|
|
20
|
-
} from "./schemas/suite-fields.
|
|
21
|
-
import {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
17
|
+
} from "./schemas/suite-fields.ts";
|
|
18
|
+
import {
|
|
19
|
+
actionParamsSchema,
|
|
20
|
+
alertTypeSchema,
|
|
21
|
+
graphAlertSchema,
|
|
22
|
+
notificationCadenceSchema,
|
|
23
|
+
reportSchema,
|
|
24
|
+
SLACK_DELIVERY_NOTE,
|
|
25
|
+
templatesSchema,
|
|
26
|
+
TRIGGER_FILTER_QUERY_DESCRIPTION,
|
|
27
|
+
TRIGGER_FILTERS_DESCRIPTION,
|
|
28
|
+
triggerActionSchema,
|
|
29
|
+
validateActionParamsForAction,
|
|
30
|
+
} from "./schemas/triggers.ts";
|
|
31
|
+
import { widgetSourcesInput } from "./schemas/widget-sources.ts";
|
|
32
|
+
import { handleExperimentResults } from "./tools/get-experiment-results.ts";
|
|
33
|
+
import { handleExperimentListRuns } from "./tools/list-experiment-runs.ts";
|
|
34
|
+
import { handleExperimentList } from "./tools/list-experiments.ts";
|
|
35
|
+
import { handleRunExperiment, handleExperimentStatus } from "./tools/run-experiment.ts";
|
|
36
|
+
import { handleTestAgent } from "./tools/test-agent.ts";
|
|
37
|
+
import { handleUpdateTestSuite } from "./tools/update-test-suite.ts";
|
|
27
38
|
|
|
28
39
|
const modelSchema = z
|
|
29
40
|
.string()
|
|
30
41
|
.describe(
|
|
31
|
-
'Model in "provider/model-name" format, e.g., "openai/gpt-
|
|
42
|
+
'Model in "provider/model-name" format, e.g., "openai/gpt-6-luna", "anthropic/claude-sonnet-4-5-20250929"',
|
|
32
43
|
);
|
|
33
44
|
|
|
34
45
|
/**
|
|
@@ -71,6 +82,30 @@ function withToolLogging<T extends unknown[], R>(
|
|
|
71
82
|
}
|
|
72
83
|
|
|
73
84
|
function registerTools(server: McpServer): void {
|
|
85
|
+
registerDocsTools(server);
|
|
86
|
+
registerObservabilityTools(server);
|
|
87
|
+
registerPromptTools(server);
|
|
88
|
+
registerScenarioTools(server);
|
|
89
|
+
registerRunPlanTools(server);
|
|
90
|
+
registerTestSuiteTools(server);
|
|
91
|
+
registerSimulationRunTools(server);
|
|
92
|
+
registerEvaluatorTools(server);
|
|
93
|
+
registerModelProviderTools(server);
|
|
94
|
+
registerAgentTools(server);
|
|
95
|
+
registerAnalyticsReportTools(server);
|
|
96
|
+
registerDashboardTools(server);
|
|
97
|
+
registerWorkflowTools(server);
|
|
98
|
+
registerAnnotationTools(server);
|
|
99
|
+
registerTriggerTools(server);
|
|
100
|
+
registerMonitorTools(server);
|
|
101
|
+
registerSecretTools(server);
|
|
102
|
+
registerExperimentTools(server);
|
|
103
|
+
registerDatasetTools(server);
|
|
104
|
+
registerProjectTools(server);
|
|
105
|
+
registerApiKeyTools(server);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function registerDocsTools(server: McpServer): void {
|
|
74
109
|
server.tool(
|
|
75
110
|
"fetch_langwatch_docs",
|
|
76
111
|
"Fetches the LangWatch docs for understanding how to implement LangWatch in your codebase. Always use this tool when the user asks for help with LangWatch. Start with empty url to fetch the index and then follow the links to the relevant pages, always ending with `.md` extension",
|
|
@@ -79,13 +114,13 @@ function registerTools(server: McpServer): void {
|
|
|
79
114
|
.string()
|
|
80
115
|
.optional()
|
|
81
116
|
.describe(
|
|
82
|
-
"The full url of the specific doc page. If not provided, the docs index will be fetched."
|
|
117
|
+
"The full url of the specific doc page. If not provided, the docs index will be fetched.",
|
|
83
118
|
),
|
|
84
119
|
},
|
|
85
120
|
withToolLogging("fetch_langwatch_docs", async ({ url }) => {
|
|
86
121
|
const text = await fetchDocumentation("langwatch", url);
|
|
87
122
|
return { content: [{ type: "text", text }] };
|
|
88
|
-
})
|
|
123
|
+
}),
|
|
89
124
|
);
|
|
90
125
|
|
|
91
126
|
server.tool(
|
|
@@ -95,36 +130,33 @@ function registerTools(server: McpServer): void {
|
|
|
95
130
|
user_approved: z
|
|
96
131
|
.boolean()
|
|
97
132
|
.describe(
|
|
98
|
-
"Must be true, and only after the user explicitly agreed to send this report to LangWatch"
|
|
133
|
+
"Must be true, and only after the user explicitly agreed to send this report to LangWatch",
|
|
99
134
|
),
|
|
100
135
|
title: z.string().max(300).describe("One-line description of the issue"),
|
|
101
136
|
summary: z
|
|
102
137
|
.string()
|
|
103
138
|
.optional()
|
|
104
139
|
.describe(
|
|
105
|
-
"What you were trying to do, what went wrong (verbatim errors), and what you had to figure out the hard way"
|
|
140
|
+
"What you were trying to do, what went wrong (verbatim errors), and what you had to figure out the hard way",
|
|
106
141
|
),
|
|
107
142
|
session_content: z
|
|
108
143
|
.string()
|
|
109
144
|
.optional()
|
|
110
145
|
.describe(
|
|
111
|
-
"Optional raw session transcript or log excerpt (JSONL or plain text); redacted locally before sending"
|
|
146
|
+
"Optional raw session transcript or log excerpt (JSONL or plain text); redacted locally before sending",
|
|
112
147
|
),
|
|
113
|
-
contact_email: z
|
|
114
|
-
.string()
|
|
115
|
-
.optional()
|
|
116
|
-
.describe("Optional contact email for follow-up"),
|
|
148
|
+
contact_email: z.string().optional().describe("Optional contact email for follow-up"),
|
|
117
149
|
agent: z
|
|
118
150
|
.string()
|
|
119
151
|
.optional()
|
|
120
152
|
.describe("Which coding agent this is, e.g. claude-code, codex, cursor"),
|
|
121
153
|
},
|
|
122
154
|
withToolLogging("report_issue", async (params) => {
|
|
123
|
-
const { handleReportIssue } = await import("./tools/report-issue.
|
|
155
|
+
const { handleReportIssue } = await import("./tools/report-issue.ts");
|
|
124
156
|
return {
|
|
125
157
|
content: [{ type: "text", text: await handleReportIssue(params) }],
|
|
126
158
|
};
|
|
127
|
-
})
|
|
159
|
+
}),
|
|
128
160
|
);
|
|
129
161
|
|
|
130
162
|
server.tool(
|
|
@@ -135,24 +167,26 @@ function registerTools(server: McpServer): void {
|
|
|
135
167
|
.string()
|
|
136
168
|
.optional()
|
|
137
169
|
.describe(
|
|
138
|
-
"The full url of the specific doc page. If not provided, the docs index will be fetched."
|
|
170
|
+
"The full url of the specific doc page. If not provided, the docs index will be fetched.",
|
|
139
171
|
),
|
|
140
172
|
},
|
|
141
173
|
withToolLogging("fetch_scenario_docs", async ({ url }) => {
|
|
142
174
|
const text = await fetchDocumentation("scenario", url);
|
|
143
175
|
return { content: [{ type: "text", text }] };
|
|
144
|
-
})
|
|
176
|
+
}),
|
|
145
177
|
);
|
|
178
|
+
}
|
|
146
179
|
|
|
147
|
-
|
|
148
|
-
|
|
180
|
+
// --- Observability Tools (require API key) ---
|
|
181
|
+
function registerObservabilityTools(server: McpServer): void {
|
|
149
182
|
server.tool(
|
|
150
183
|
"discover_schema",
|
|
151
|
-
"Discover
|
|
184
|
+
"Discover what LangWatch can be queried with: the trace filter fields and syntax, the analytics SQL views and columns, the analytics metrics, aggregation types, group-by options, scenario schema and evaluator types. Call this before using search_traces, run_query, get_analytics, scenario tools or evaluator tools, so you never guess a field, a column or a value.",
|
|
152
185
|
{
|
|
153
186
|
category: z
|
|
154
187
|
.enum([
|
|
155
188
|
"filters",
|
|
189
|
+
"lwql",
|
|
156
190
|
"metrics",
|
|
157
191
|
"aggregations",
|
|
158
192
|
"groups",
|
|
@@ -160,113 +194,149 @@ function registerTools(server: McpServer): void {
|
|
|
160
194
|
"evaluators",
|
|
161
195
|
"all",
|
|
162
196
|
])
|
|
163
|
-
.describe(
|
|
197
|
+
.describe(
|
|
198
|
+
"Which schema category to discover. 'filters' is the trace filter language, 'lwql' the analytics SQL views; both are read from the platform and need the API key",
|
|
199
|
+
),
|
|
164
200
|
evaluatorType: z
|
|
165
201
|
.string()
|
|
166
202
|
.optional()
|
|
167
203
|
.describe(
|
|
168
|
-
"When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details"
|
|
204
|
+
"When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details",
|
|
169
205
|
),
|
|
170
206
|
},
|
|
171
|
-
async ({ category, evaluatorType }) => {
|
|
207
|
+
withToolLogging("discover_schema", async ({ category, evaluatorType }) => {
|
|
172
208
|
if (category === "scenarios") {
|
|
173
|
-
const { formatScenarioSchema } = await import(
|
|
174
|
-
"./tools/discover-scenario-schema.js"
|
|
175
|
-
);
|
|
209
|
+
const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.ts");
|
|
176
210
|
return {
|
|
177
211
|
content: [{ type: "text", text: formatScenarioSchema() }],
|
|
178
212
|
};
|
|
179
213
|
}
|
|
180
214
|
if (category === "evaluators") {
|
|
181
|
-
const { formatEvaluatorSchema } = await import(
|
|
182
|
-
"./tools/discover-evaluator-schema.js"
|
|
183
|
-
);
|
|
215
|
+
const { formatEvaluatorSchema } = await import("./tools/discover-evaluator-schema.ts");
|
|
184
216
|
return {
|
|
185
|
-
content: [
|
|
186
|
-
{ type: "text", text: formatEvaluatorSchema(evaluatorType) },
|
|
187
|
-
],
|
|
217
|
+
content: [{ type: "text", text: formatEvaluatorSchema(evaluatorType) }],
|
|
188
218
|
};
|
|
189
219
|
}
|
|
190
|
-
const { formatSchema } = await import("./tools/discover-schema.
|
|
191
|
-
|
|
220
|
+
const { formatSchema, needsQueryReference } = await import("./tools/discover-schema.ts");
|
|
221
|
+
// The filter and analytics-SQL halves are the PLATFORM's registries, not
|
|
222
|
+
// copies of them, so those categories need the credential. The static
|
|
223
|
+
// ones still answer without it.
|
|
224
|
+
if (needsQueryReference(category)) requireApiKey();
|
|
225
|
+
let text = await formatSchema(category);
|
|
192
226
|
if (category === "all") {
|
|
193
|
-
const { formatScenarioSchema } = await import(
|
|
194
|
-
"./tools/discover-scenario-schema.js"
|
|
195
|
-
);
|
|
227
|
+
const { formatScenarioSchema } = await import("./tools/discover-scenario-schema.ts");
|
|
196
228
|
text += "\n\n" + formatScenarioSchema();
|
|
197
|
-
const { formatEvaluatorSchema } = await import(
|
|
198
|
-
"./tools/discover-evaluator-schema.js"
|
|
199
|
-
);
|
|
229
|
+
const { formatEvaluatorSchema } = await import("./tools/discover-evaluator-schema.ts");
|
|
200
230
|
text += "\n\n" + formatEvaluatorSchema();
|
|
201
231
|
}
|
|
202
232
|
return { content: [{ type: "text", text }] };
|
|
203
|
-
}
|
|
233
|
+
}),
|
|
234
|
+
);
|
|
235
|
+
|
|
236
|
+
server.tool(
|
|
237
|
+
"run_query",
|
|
238
|
+
"Run one read-only analytics SQL statement (LangWatchQL) over this project's traces, spans, evaluations and metrics, and get the rows back as a table. This is the tool for a count, a rate, a sum, a percentile or any of them grouped or over time — questions search_traces would need many calls to answer. The statement runs exactly as written. Call discover_schema with category 'lwql' first for the views, columns and worked statements.",
|
|
239
|
+
{
|
|
240
|
+
sql: z
|
|
241
|
+
.string()
|
|
242
|
+
.describe(
|
|
243
|
+
"The SELECT to run. Filter on the dataset's time column so the read is bounded, and never select a whole attribute map",
|
|
244
|
+
),
|
|
245
|
+
parameters: z
|
|
246
|
+
.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()]))
|
|
247
|
+
.optional()
|
|
248
|
+
.describe(
|
|
249
|
+
'Values for the parameters the statement declares, e.g. {"days": 7} for {days:UInt32}',
|
|
250
|
+
),
|
|
251
|
+
},
|
|
252
|
+
withToolLogging("run_query", async (params) => {
|
|
253
|
+
requireApiKey();
|
|
254
|
+
const { handleRunQuery } = await import("./tools/run-query.ts");
|
|
255
|
+
return {
|
|
256
|
+
content: [{ type: "text", text: await handleRunQuery(params) }],
|
|
257
|
+
};
|
|
258
|
+
}),
|
|
204
259
|
);
|
|
205
260
|
|
|
206
261
|
server.tool(
|
|
207
262
|
"search_traces",
|
|
208
|
-
"Search LangWatch traces
|
|
263
|
+
"Search LangWatch traces by content, with filters and a date range. Returns AI-readable trace digests by default. Use format: 'json' for full raw data. To fetch a trace you already have the ID for, use get_trace instead — free text does not match trace IDs.",
|
|
209
264
|
{
|
|
210
|
-
query: z
|
|
265
|
+
query: z
|
|
266
|
+
.string()
|
|
267
|
+
.optional()
|
|
268
|
+
.describe(
|
|
269
|
+
"Text search query. Matches captured input/output, the trace name and span names ONLY. It does not match trace IDs — to fetch a known trace ID use get_trace, or pass traceIds here for several at once.",
|
|
270
|
+
),
|
|
271
|
+
traceIds: z
|
|
272
|
+
.array(z.string().min(1))
|
|
273
|
+
.min(1)
|
|
274
|
+
.max(1000)
|
|
275
|
+
.optional()
|
|
276
|
+
.describe(
|
|
277
|
+
"Exact trace IDs to fetch (1–1000). Use instead of query when you already know the IDs. When set and no startDate is given, the window defaults to the last 90 days rather than 24 hours.",
|
|
278
|
+
),
|
|
211
279
|
filters: z
|
|
212
280
|
.record(z.string(), z.array(z.string()))
|
|
213
281
|
.optional()
|
|
214
282
|
.describe(
|
|
215
|
-
'Filter traces. Format: {"field": ["value"]}. Use discover_schema for field names.'
|
|
283
|
+
'Filter traces by the older per-field map. Format: {"field": ["value"]}. Use discover_schema for field names.',
|
|
216
284
|
),
|
|
217
|
-
|
|
285
|
+
filter: z
|
|
218
286
|
.string()
|
|
219
287
|
.optional()
|
|
220
288
|
.describe(
|
|
221
|
-
'
|
|
289
|
+
'Filter traces with the Trace Explorer\'s own query language: "status:error AND model:gpt-*", "trace.attribute.langwatch.user_id:alice", "evaluatorVerdict:fail", a quoted phrase for free text. Reaches attribute keys, span events and evaluator verdicts that the `filters` map cannot. Combined with `filters` and `query` when you send more than one. Call discover_schema with category \'filters\' for every field and the syntax.',
|
|
222
290
|
),
|
|
223
|
-
|
|
291
|
+
startDate: z
|
|
224
292
|
.string()
|
|
293
|
+
.min(1)
|
|
225
294
|
.optional()
|
|
226
|
-
.describe(
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
.describe("Results per page (default: 25, max: 1000)"),
|
|
231
|
-
scrollId: z
|
|
295
|
+
.describe(
|
|
296
|
+
'Start of the window: ISO date, or relative like "24h", "7d", "4w", "3m" — units are h (hours), d (days), w (weeks), m (30-day months). Defaults to 24h ago for a text search, or 90d when traceIds is set. Widen this when a search comes back empty.',
|
|
297
|
+
),
|
|
298
|
+
endDate: z
|
|
232
299
|
.string()
|
|
300
|
+
.min(1)
|
|
233
301
|
.optional()
|
|
234
|
-
.describe("
|
|
302
|
+
.describe("End of the window: ISO date or relative. Default: now"),
|
|
303
|
+
pageSize: z.number().optional().describe("Results per page (default: 25, max: 1000)"),
|
|
304
|
+
scrollId: z.string().optional().describe("Pagination token from previous search"),
|
|
235
305
|
format: z
|
|
236
306
|
.enum(["digest", "json"])
|
|
237
307
|
.optional()
|
|
238
|
-
.describe(
|
|
239
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
240
|
-
),
|
|
308
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
241
309
|
},
|
|
242
310
|
withToolLogging("search_traces", async (params) => {
|
|
243
311
|
requireApiKey();
|
|
244
|
-
const { handleSearchTraces } = await import("./tools/search-traces.
|
|
312
|
+
const { handleSearchTraces } = await import("./tools/search-traces.ts");
|
|
245
313
|
return {
|
|
246
314
|
content: [{ type: "text", text: await handleSearchTraces(params) }],
|
|
247
315
|
};
|
|
248
|
-
})
|
|
316
|
+
}),
|
|
249
317
|
);
|
|
250
318
|
|
|
251
319
|
server.tool(
|
|
252
320
|
"get_trace",
|
|
253
|
-
"Get full details of a single trace by ID. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
321
|
+
"Get full details of a single trace by ID. This is the right tool whenever you have a trace ID — search_traces does not match IDs. A full ID resolves at any age; a unique 8–31 character hex prefix resolves within the last 90 days. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
254
322
|
{
|
|
255
|
-
traceId: z
|
|
323
|
+
traceId: z
|
|
324
|
+
.string()
|
|
325
|
+
.describe(
|
|
326
|
+
"The trace ID to retrieve. A full ID resolves at any age; a unique 8–31 character hex prefix also resolves, git-style, within the last 90 days.",
|
|
327
|
+
),
|
|
256
328
|
format: z
|
|
257
329
|
.enum(["digest", "json"])
|
|
258
330
|
.optional()
|
|
259
|
-
.describe(
|
|
260
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
261
|
-
),
|
|
331
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
262
332
|
},
|
|
263
333
|
withToolLogging("get_trace", async (params) => {
|
|
264
334
|
requireApiKey();
|
|
265
|
-
const { handleGetTrace } = await import("./tools/get-trace.
|
|
335
|
+
const { handleGetTrace } = await import("./tools/get-trace.ts");
|
|
266
336
|
return {
|
|
267
337
|
content: [{ type: "text", text: await handleGetTrace(params) }],
|
|
268
338
|
};
|
|
269
|
-
})
|
|
339
|
+
}),
|
|
270
340
|
);
|
|
271
341
|
|
|
272
342
|
server.tool(
|
|
@@ -276,44 +346,40 @@ function registerTools(server: McpServer): void {
|
|
|
276
346
|
metric: z
|
|
277
347
|
.string()
|
|
278
348
|
.describe(
|
|
279
|
-
'Metric in "category.name" format, e.g., "metadata.trace_id", "performance.total_cost"'
|
|
349
|
+
'Metric in "category.name" format, e.g., "metadata.trace_id", "performance.total_cost"',
|
|
280
350
|
),
|
|
281
351
|
aggregation: z
|
|
282
352
|
.string()
|
|
283
353
|
.optional()
|
|
284
354
|
.describe(
|
|
285
|
-
"Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg"
|
|
355
|
+
"Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg",
|
|
286
356
|
),
|
|
287
357
|
startDate: z
|
|
288
358
|
.string()
|
|
289
359
|
.optional()
|
|
290
|
-
.describe(
|
|
291
|
-
'Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'
|
|
292
|
-
),
|
|
360
|
+
.describe('Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'),
|
|
293
361
|
endDate: z.string().optional().describe("End date. Default: now"),
|
|
294
362
|
timeZone: z.string().optional().describe("Timezone. Default: UTC"),
|
|
295
363
|
groupBy: z
|
|
296
364
|
.string()
|
|
297
365
|
.optional()
|
|
298
366
|
.describe("Group results by field. Use discover_schema for options."),
|
|
299
|
-
filters: z
|
|
300
|
-
.record(z.string(), z.array(z.string()))
|
|
301
|
-
.optional()
|
|
302
|
-
.describe("Filters to apply"),
|
|
367
|
+
filters: z.record(z.string(), z.array(z.string())).optional().describe("Filters to apply"),
|
|
303
368
|
},
|
|
304
369
|
withToolLogging("get_analytics", async (params) => {
|
|
305
370
|
requireApiKey();
|
|
306
|
-
const { handleGetAnalytics } = await import("./tools/get-analytics.
|
|
371
|
+
const { handleGetAnalytics } = await import("./tools/get-analytics.ts");
|
|
307
372
|
return {
|
|
308
373
|
content: [{ type: "text", text: await handleGetAnalytics(params) }],
|
|
309
374
|
};
|
|
310
|
-
})
|
|
375
|
+
}),
|
|
311
376
|
);
|
|
377
|
+
}
|
|
312
378
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
379
|
+
// --- Platform Prompt Tools (require API key) ---
|
|
380
|
+
// These tools manage prompts on the LangWatch platform via API.
|
|
381
|
+
// For code-based prompt management, see `fetch_langwatch_docs` for the CLI/SDK approach.
|
|
382
|
+
function registerPromptTools(server: McpServer): void {
|
|
317
383
|
server.tool(
|
|
318
384
|
"platform_create_prompt",
|
|
319
385
|
`Create a new prompt on the LangWatch platform.
|
|
@@ -329,32 +395,31 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
329
395
|
handle: z
|
|
330
396
|
.string()
|
|
331
397
|
.optional()
|
|
332
|
-
.describe(
|
|
333
|
-
"URL-friendly handle (auto-generated from name if omitted)"
|
|
334
|
-
),
|
|
398
|
+
.describe("URL-friendly handle (auto-generated from name if omitted)"),
|
|
335
399
|
messages: z
|
|
336
400
|
.array(
|
|
337
401
|
z.object({
|
|
338
|
-
role: z
|
|
339
|
-
.enum(["system", "user", "assistant"])
|
|
340
|
-
.describe("Message role"),
|
|
402
|
+
role: z.enum(["system", "user", "assistant"]).describe("Message role"),
|
|
341
403
|
content: z.string().describe("Message content"),
|
|
342
|
-
})
|
|
404
|
+
}),
|
|
343
405
|
)
|
|
344
406
|
.describe("Prompt messages"),
|
|
345
407
|
model: modelSchema,
|
|
346
|
-
tags: z
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
408
|
+
tags: z
|
|
409
|
+
.array(z.string())
|
|
410
|
+
.optional()
|
|
411
|
+
.describe(
|
|
412
|
+
'Tags to assign to the initial version (e.g., ["production", "staging"]). ' +
|
|
413
|
+
'Built-in tags: "latest" (auto-assigned), "production", "staging". Custom tags must be created first.',
|
|
414
|
+
),
|
|
350
415
|
},
|
|
351
416
|
withToolLogging("platform_create_prompt", async (params) => {
|
|
352
417
|
requireApiKey();
|
|
353
|
-
const { handleCreatePrompt } = await import("./tools/create-prompt.
|
|
418
|
+
const { handleCreatePrompt } = await import("./tools/create-prompt.ts");
|
|
354
419
|
return {
|
|
355
420
|
content: [{ type: "text", text: await handleCreatePrompt(params) }],
|
|
356
421
|
};
|
|
357
|
-
})
|
|
422
|
+
}),
|
|
358
423
|
);
|
|
359
424
|
|
|
360
425
|
server.tool(
|
|
@@ -363,11 +428,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
363
428
|
{},
|
|
364
429
|
withToolLogging("platform_list_prompts", async () => {
|
|
365
430
|
requireApiKey();
|
|
366
|
-
const { handleListPrompts } = await import("./tools/list-prompts.
|
|
431
|
+
const { handleListPrompts } = await import("./tools/list-prompts.ts");
|
|
367
432
|
return {
|
|
368
433
|
content: [{ type: "text", text: await handleListPrompts() }],
|
|
369
434
|
};
|
|
370
|
-
})
|
|
435
|
+
}),
|
|
371
436
|
);
|
|
372
437
|
|
|
373
438
|
server.tool(
|
|
@@ -375,20 +440,18 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
375
440
|
"Get a specific prompt from the LangWatch platform by ID or handle, including messages, model config, and version history. Use format: 'json' for the full raw API payload, or 'digest' (default) for a formatted summary.",
|
|
376
441
|
{
|
|
377
442
|
idOrHandle: z.string().describe("Prompt ID or handle"),
|
|
378
|
-
version: z
|
|
379
|
-
|
|
443
|
+
version: z.number().optional().describe("Specific version number (default: latest)"),
|
|
444
|
+
tag: z
|
|
445
|
+
.string()
|
|
380
446
|
.optional()
|
|
381
|
-
.describe(
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
),
|
|
447
|
+
.describe(
|
|
448
|
+
'Fetch the version pointed to by this tag (e.g., "production", "staging"). ' +
|
|
449
|
+
'Alternatively, use shorthand in idOrHandle: "pizza-prompt:production"',
|
|
450
|
+
),
|
|
386
451
|
format: z
|
|
387
452
|
.enum(["digest", "json"])
|
|
388
453
|
.optional()
|
|
389
|
-
.describe(
|
|
390
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
391
|
-
),
|
|
454
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
392
455
|
},
|
|
393
456
|
withToolLogging("platform_get_prompt", async (params) => {
|
|
394
457
|
if (params.version != null && params.tag) {
|
|
@@ -398,11 +461,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
398
461
|
};
|
|
399
462
|
}
|
|
400
463
|
requireApiKey();
|
|
401
|
-
const { handleGetPrompt } = await import("./tools/get-prompt.
|
|
464
|
+
const { handleGetPrompt } = await import("./tools/get-prompt.ts");
|
|
402
465
|
return {
|
|
403
466
|
content: [{ type: "text", text: await handleGetPrompt(params) }],
|
|
404
467
|
};
|
|
405
|
-
})
|
|
468
|
+
}),
|
|
406
469
|
);
|
|
407
470
|
|
|
408
471
|
server.tool(
|
|
@@ -415,31 +478,30 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
415
478
|
z.object({
|
|
416
479
|
role: z.enum(["system", "user", "assistant"]),
|
|
417
480
|
content: z.string(),
|
|
418
|
-
})
|
|
481
|
+
}),
|
|
419
482
|
)
|
|
420
483
|
.optional()
|
|
421
484
|
.describe("Updated messages"),
|
|
422
485
|
model: modelSchema.optional(),
|
|
423
|
-
commitMessage: z
|
|
424
|
-
|
|
425
|
-
.
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
),
|
|
486
|
+
commitMessage: z.string().describe("Commit message describing the change"),
|
|
487
|
+
tags: z
|
|
488
|
+
.array(z.string())
|
|
489
|
+
.optional()
|
|
490
|
+
.describe("Tags to assign to the new version created by this update."),
|
|
429
491
|
},
|
|
430
492
|
withToolLogging("platform_update_prompt", async (params) => {
|
|
431
493
|
requireApiKey();
|
|
432
|
-
const { handleUpdatePrompt } = await import("./tools/update-prompt.
|
|
494
|
+
const { handleUpdatePrompt } = await import("./tools/update-prompt.ts");
|
|
433
495
|
return {
|
|
434
496
|
content: [{ type: "text", text: await handleUpdatePrompt(params) }],
|
|
435
497
|
};
|
|
436
|
-
})
|
|
498
|
+
}),
|
|
437
499
|
);
|
|
438
500
|
|
|
439
501
|
server.tool(
|
|
440
502
|
"platform_assign_prompt_tag",
|
|
441
503
|
'Assign a tag (e.g. "production") to a specific version of a prompt. ' +
|
|
442
|
-
|
|
504
|
+
'Use this to "deploy" a version by promoting it to the production tag.',
|
|
443
505
|
{
|
|
444
506
|
idOrHandle: z.string().describe("Prompt ID or handle"),
|
|
445
507
|
tag: z.string().describe('Tag name (e.g., "production", "staging")'),
|
|
@@ -447,41 +509,41 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
447
509
|
},
|
|
448
510
|
withToolLogging("platform_assign_prompt_tag", async (params) => {
|
|
449
511
|
requireApiKey();
|
|
450
|
-
const { handleAssignPromptTag } = await import("./tools/assign-prompt-tag.
|
|
512
|
+
const { handleAssignPromptTag } = await import("./tools/assign-prompt-tag.ts");
|
|
451
513
|
return {
|
|
452
514
|
content: [{ type: "text", text: await handleAssignPromptTag(params) }],
|
|
453
515
|
};
|
|
454
|
-
})
|
|
516
|
+
}),
|
|
455
517
|
);
|
|
456
518
|
|
|
457
519
|
server.tool(
|
|
458
520
|
"platform_list_prompt_tags",
|
|
459
521
|
"List all prompt tag definitions for the organization. " +
|
|
460
|
-
|
|
522
|
+
"Shows built-in tags (latest, production, staging) and any custom tags.",
|
|
461
523
|
{},
|
|
462
524
|
withToolLogging("platform_list_prompt_tags", async () => {
|
|
463
525
|
requireApiKey();
|
|
464
|
-
const { handleListPromptTags } = await import("./tools/list-prompt-tags.
|
|
526
|
+
const { handleListPromptTags } = await import("./tools/list-prompt-tags.ts");
|
|
465
527
|
return {
|
|
466
528
|
content: [{ type: "text", text: await handleListPromptTags() }],
|
|
467
529
|
};
|
|
468
|
-
})
|
|
530
|
+
}),
|
|
469
531
|
);
|
|
470
532
|
|
|
471
533
|
server.tool(
|
|
472
534
|
"platform_create_prompt_tag",
|
|
473
535
|
"Create a custom prompt tag definition for the organization. " +
|
|
474
|
-
|
|
536
|
+
'Tag names must be non-numeric and not "latest".',
|
|
475
537
|
{
|
|
476
538
|
name: z.string().describe("Tag name to create"),
|
|
477
539
|
},
|
|
478
540
|
withToolLogging("platform_create_prompt_tag", async (params) => {
|
|
479
541
|
requireApiKey();
|
|
480
|
-
const { handleCreatePromptTag } = await import("./tools/create-prompt-tag.
|
|
542
|
+
const { handleCreatePromptTag } = await import("./tools/create-prompt-tag.ts");
|
|
481
543
|
return {
|
|
482
544
|
content: [{ type: "text", text: await handleCreatePromptTag(params) }],
|
|
483
545
|
};
|
|
484
|
-
})
|
|
546
|
+
}),
|
|
485
547
|
);
|
|
486
548
|
|
|
487
549
|
server.tool(
|
|
@@ -493,11 +555,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
493
555
|
},
|
|
494
556
|
withToolLogging("platform_rename_prompt_tag", async (params) => {
|
|
495
557
|
requireApiKey();
|
|
496
|
-
const { handleRenamePromptTag } = await import("./tools/rename-prompt-tag.
|
|
558
|
+
const { handleRenamePromptTag } = await import("./tools/rename-prompt-tag.ts");
|
|
497
559
|
return {
|
|
498
560
|
content: [{ type: "text", text: await handleRenamePromptTag(params) }],
|
|
499
561
|
};
|
|
500
|
-
})
|
|
562
|
+
}),
|
|
501
563
|
);
|
|
502
564
|
|
|
503
565
|
server.tool(
|
|
@@ -508,17 +570,18 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
508
570
|
},
|
|
509
571
|
withToolLogging("platform_delete_prompt_tag", async (params) => {
|
|
510
572
|
requireApiKey();
|
|
511
|
-
const { handleDeletePromptTag } = await import("./tools/delete-prompt-tag.
|
|
573
|
+
const { handleDeletePromptTag } = await import("./tools/delete-prompt-tag.ts");
|
|
512
574
|
return {
|
|
513
575
|
content: [{ type: "text", text: await handleDeletePromptTag(params) }],
|
|
514
576
|
};
|
|
515
|
-
})
|
|
577
|
+
}),
|
|
516
578
|
);
|
|
579
|
+
}
|
|
517
580
|
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
581
|
+
// --- Platform Scenario Tools (require API key) ---
|
|
582
|
+
// These tools manage scenarios on the LangWatch platform via API.
|
|
583
|
+
// For code-based scenario testing, see `fetch_scenario_docs` for the SDK approach.
|
|
584
|
+
function registerScenarioTools(server: McpServer): void {
|
|
522
585
|
server.tool(
|
|
523
586
|
"platform_create_scenario",
|
|
524
587
|
`Create a new scenario on the LangWatch platform. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.
|
|
@@ -533,15 +596,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
533
596
|
name: z.string().describe("Scenario name"),
|
|
534
597
|
situation: z
|
|
535
598
|
.string()
|
|
536
|
-
.describe(
|
|
537
|
-
"The context or setup describing what the user/agent is doing"
|
|
538
|
-
),
|
|
599
|
+
.describe("The context or setup describing what the user/agent is doing"),
|
|
539
600
|
criteria: z
|
|
540
601
|
.array(z.string())
|
|
541
602
|
.optional()
|
|
542
|
-
.describe(
|
|
543
|
-
"Pass/fail conditions the agent's response must satisfy"
|
|
544
|
-
),
|
|
603
|
+
.describe("Pass/fail conditions the agent's response must satisfy"),
|
|
545
604
|
labels: z
|
|
546
605
|
.array(z.string())
|
|
547
606
|
.optional()
|
|
@@ -550,49 +609,36 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
550
609
|
.string()
|
|
551
610
|
.nullish()
|
|
552
611
|
.describe(
|
|
553
|
-
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
612
|
+
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it.",
|
|
554
613
|
),
|
|
555
614
|
fields: scenarioFieldValuesSchema.optional(),
|
|
556
615
|
},
|
|
557
616
|
withToolLogging("platform_create_scenario", async (params) => {
|
|
558
617
|
requireApiKey();
|
|
559
|
-
const { handleCreateScenario } = await import(
|
|
560
|
-
"./tools/create-scenario.js"
|
|
561
|
-
);
|
|
618
|
+
const { handleCreateScenario } = await import("./tools/create-scenario.ts");
|
|
562
619
|
return {
|
|
563
|
-
content: [
|
|
564
|
-
{ type: "text", text: await handleCreateScenario(params) },
|
|
565
|
-
],
|
|
620
|
+
content: [{ type: "text", text: await handleCreateScenario(params) }],
|
|
566
621
|
};
|
|
567
|
-
})
|
|
622
|
+
}),
|
|
568
623
|
);
|
|
569
624
|
|
|
570
625
|
server.tool(
|
|
571
626
|
"platform_list_scenarios",
|
|
572
627
|
"List all scenarios on the LangWatch platform. Returns AI-readable digest by default.",
|
|
573
628
|
{
|
|
574
|
-
testSuiteId: z
|
|
575
|
-
.string()
|
|
576
|
-
.optional()
|
|
577
|
-
.describe("Only the scenarios filed in this test suite"),
|
|
629
|
+
testSuiteId: z.string().optional().describe("Only the scenarios filed in this test suite"),
|
|
578
630
|
format: z
|
|
579
631
|
.enum(["digest", "json"])
|
|
580
632
|
.optional()
|
|
581
|
-
.describe(
|
|
582
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
583
|
-
),
|
|
633
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
584
634
|
},
|
|
585
635
|
withToolLogging("platform_list_scenarios", async (params) => {
|
|
586
636
|
requireApiKey();
|
|
587
|
-
const { handleListScenarios } = await import(
|
|
588
|
-
"./tools/list-scenarios.js"
|
|
589
|
-
);
|
|
637
|
+
const { handleListScenarios } = await import("./tools/list-scenarios.ts");
|
|
590
638
|
return {
|
|
591
|
-
content: [
|
|
592
|
-
{ type: "text", text: await handleListScenarios(params) },
|
|
593
|
-
],
|
|
639
|
+
content: [{ type: "text", text: await handleListScenarios(params) }],
|
|
594
640
|
};
|
|
595
|
-
})
|
|
641
|
+
}),
|
|
596
642
|
);
|
|
597
643
|
|
|
598
644
|
server.tool(
|
|
@@ -603,17 +649,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
603
649
|
format: z
|
|
604
650
|
.enum(["digest", "json"])
|
|
605
651
|
.optional()
|
|
606
|
-
.describe(
|
|
607
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
608
|
-
),
|
|
652
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
609
653
|
},
|
|
610
654
|
withToolLogging("platform_get_scenario", async (params) => {
|
|
611
655
|
requireApiKey();
|
|
612
|
-
const { handleGetScenario } = await import("./tools/get-scenario.
|
|
656
|
+
const { handleGetScenario } = await import("./tools/get-scenario.ts");
|
|
613
657
|
return {
|
|
614
658
|
content: [{ type: "text", text: await handleGetScenario(params) }],
|
|
615
659
|
};
|
|
616
|
-
})
|
|
660
|
+
}),
|
|
617
661
|
);
|
|
618
662
|
|
|
619
663
|
server.tool(
|
|
@@ -623,33 +667,23 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
623
667
|
scenarioId: z.string().describe("The scenario ID to update"),
|
|
624
668
|
name: z.string().optional().describe("Updated scenario name"),
|
|
625
669
|
situation: z.string().optional().describe("Updated situation"),
|
|
626
|
-
criteria: z
|
|
627
|
-
|
|
628
|
-
.optional()
|
|
629
|
-
.describe("Updated criteria"),
|
|
630
|
-
labels: z
|
|
631
|
-
.array(z.string())
|
|
632
|
-
.optional()
|
|
633
|
-
.describe("Updated labels"),
|
|
670
|
+
criteria: z.array(z.string()).optional().describe("Updated criteria"),
|
|
671
|
+
labels: z.array(z.string()).optional().describe("Updated labels"),
|
|
634
672
|
testSuiteId: z
|
|
635
673
|
.string()
|
|
636
674
|
.nullish()
|
|
637
675
|
.describe(
|
|
638
|
-
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
676
|
+
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it.",
|
|
639
677
|
),
|
|
640
678
|
fields: scenarioFieldValuesSchema.optional(),
|
|
641
679
|
},
|
|
642
680
|
withToolLogging("platform_update_scenario", async (params) => {
|
|
643
681
|
requireApiKey();
|
|
644
|
-
const { handleUpdateScenario } = await import(
|
|
645
|
-
"./tools/update-scenario.js"
|
|
646
|
-
);
|
|
682
|
+
const { handleUpdateScenario } = await import("./tools/update-scenario.ts");
|
|
647
683
|
return {
|
|
648
|
-
content: [
|
|
649
|
-
{ type: "text", text: await handleUpdateScenario(params) },
|
|
650
|
-
],
|
|
684
|
+
content: [{ type: "text", text: await handleUpdateScenario(params) }],
|
|
651
685
|
};
|
|
652
|
-
})
|
|
686
|
+
}),
|
|
653
687
|
);
|
|
654
688
|
|
|
655
689
|
server.tool(
|
|
@@ -660,23 +694,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
660
694
|
},
|
|
661
695
|
withToolLogging("platform_archive_scenario", async (params) => {
|
|
662
696
|
requireApiKey();
|
|
663
|
-
const { handleArchiveScenario } = await import(
|
|
664
|
-
"./tools/archive-scenario.js"
|
|
665
|
-
);
|
|
697
|
+
const { handleArchiveScenario } = await import("./tools/archive-scenario.ts");
|
|
666
698
|
return {
|
|
667
|
-
content: [
|
|
668
|
-
{ type: "text", text: await handleArchiveScenario(params) },
|
|
669
|
-
],
|
|
699
|
+
content: [{ type: "text", text: await handleArchiveScenario(params) }],
|
|
670
700
|
};
|
|
671
|
-
})
|
|
701
|
+
}),
|
|
672
702
|
);
|
|
703
|
+
}
|
|
673
704
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
705
|
+
// --- Platform Run Plan Tools (require API key) ---
|
|
706
|
+
// A run plan is what you run, and its NAME identifies it: running a name
|
|
707
|
+
// that exists replaces that plan's configuration, running a new name
|
|
708
|
+
// creates the plan. A test suite groups scenarios; running one is
|
|
709
|
+
// sugar that creates or joins the plan "<suite name> <target name>".
|
|
710
|
+
function registerRunPlanTools(server: McpServer): void {
|
|
680
711
|
server.tool(
|
|
681
712
|
"platform_run_plan",
|
|
682
713
|
"Run scenarios against targets. The plan name identifies the run plan: an existing name is re-run with the configuration you send here, a new name creates the plan. Configuration is the scope, the targets, the repeat count and the models; parameters, the note and the idempotency key belong to this run alone. Send more than one target to compare them in the same run, and give a target its own parameters to compare one agent on two models.",
|
|
@@ -693,9 +724,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
693
724
|
scenarioIds: z
|
|
694
725
|
.array(z.string())
|
|
695
726
|
.optional()
|
|
696
|
-
.describe(
|
|
697
|
-
"The scenarios to run. Read only when scope.mode is 'scenarios'.",
|
|
698
|
-
),
|
|
727
|
+
.describe("The scenarios to run. Read only when scope.mode is 'scenarios'."),
|
|
699
728
|
targets: z
|
|
700
729
|
.array(runPlanTargetSchema)
|
|
701
730
|
.describe(
|
|
@@ -734,25 +763,24 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
734
763
|
idempotencyKey: z
|
|
735
764
|
.string()
|
|
736
765
|
.optional()
|
|
737
|
-
.describe(
|
|
766
|
+
.describe(
|
|
767
|
+
"Send the same key to make a retry join the run it already started instead of starting a second one.",
|
|
768
|
+
),
|
|
738
769
|
},
|
|
739
770
|
withToolLogging("platform_run_plan", async (params) => {
|
|
740
771
|
requireApiKey();
|
|
741
|
-
const { handleRunPlan } = await import("./tools/run-plan.
|
|
772
|
+
const { handleRunPlan } = await import("./tools/run-plan.ts");
|
|
742
773
|
return {
|
|
743
774
|
content: [{ type: "text", text: await handleRunPlan(params) }],
|
|
744
775
|
};
|
|
745
|
-
})
|
|
776
|
+
}),
|
|
746
777
|
);
|
|
747
778
|
|
|
748
779
|
server.tool(
|
|
749
780
|
"platform_list_run_plans",
|
|
750
781
|
"List the run plans of the project. A run plan holds the configuration a run used, keyed by its name.",
|
|
751
782
|
{
|
|
752
|
-
includeArchived: z
|
|
753
|
-
.boolean()
|
|
754
|
-
.optional()
|
|
755
|
-
.describe("Include archived plans (default false)"),
|
|
783
|
+
includeArchived: z.boolean().optional().describe("Include archived plans (default false)"),
|
|
756
784
|
format: z
|
|
757
785
|
.enum(["digest", "json"])
|
|
758
786
|
.optional()
|
|
@@ -760,11 +788,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
760
788
|
},
|
|
761
789
|
withToolLogging("platform_list_run_plans", async (params) => {
|
|
762
790
|
requireApiKey();
|
|
763
|
-
const { handleListRunPlans } = await import("./tools/list-run-plans.
|
|
791
|
+
const { handleListRunPlans } = await import("./tools/list-run-plans.ts");
|
|
764
792
|
return {
|
|
765
793
|
content: [{ type: "text", text: await handleListRunPlans(params) }],
|
|
766
794
|
};
|
|
767
|
-
})
|
|
795
|
+
}),
|
|
768
796
|
);
|
|
769
797
|
|
|
770
798
|
server.tool(
|
|
@@ -779,11 +807,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
779
807
|
},
|
|
780
808
|
withToolLogging("platform_get_run_plan", async (params) => {
|
|
781
809
|
requireApiKey();
|
|
782
|
-
const { handleGetRunPlan } = await import("./tools/get-run-plan.
|
|
810
|
+
const { handleGetRunPlan } = await import("./tools/get-run-plan.ts");
|
|
783
811
|
return {
|
|
784
812
|
content: [{ type: "text", text: await handleGetRunPlan(params) }],
|
|
785
813
|
};
|
|
786
|
-
})
|
|
814
|
+
}),
|
|
787
815
|
);
|
|
788
816
|
|
|
789
817
|
server.tool(
|
|
@@ -802,11 +830,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
802
830
|
},
|
|
803
831
|
withToolLogging("platform_rerun_run_plan", async (params) => {
|
|
804
832
|
requireApiKey();
|
|
805
|
-
const { handleRerunRunPlan } = await import("./tools/rerun-run-plan.
|
|
833
|
+
const { handleRerunRunPlan } = await import("./tools/rerun-run-plan.ts");
|
|
806
834
|
return {
|
|
807
835
|
content: [{ type: "text", text: await handleRerunRunPlan(params) }],
|
|
808
836
|
};
|
|
809
|
-
})
|
|
837
|
+
}),
|
|
810
838
|
);
|
|
811
839
|
|
|
812
840
|
server.tool(
|
|
@@ -817,18 +845,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
817
845
|
},
|
|
818
846
|
withToolLogging("platform_archive_run_plan", async (params) => {
|
|
819
847
|
requireApiKey();
|
|
820
|
-
const { handleArchiveRunPlan } = await import(
|
|
821
|
-
"./tools/archive-run-plan.js"
|
|
822
|
-
);
|
|
848
|
+
const { handleArchiveRunPlan } = await import("./tools/archive-run-plan.ts");
|
|
823
849
|
return {
|
|
824
850
|
content: [{ type: "text", text: await handleArchiveRunPlan(params) }],
|
|
825
851
|
};
|
|
826
|
-
})
|
|
852
|
+
}),
|
|
827
853
|
);
|
|
854
|
+
}
|
|
828
855
|
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
856
|
+
// --- Platform Test Suite Tools (require API key) ---
|
|
857
|
+
// A test suite groups scenarios: a name and the scenarios filed in it.
|
|
858
|
+
function registerTestSuiteTools(server: McpServer): void {
|
|
832
859
|
server.tool(
|
|
833
860
|
"platform_list_test_suites",
|
|
834
861
|
"List the test suites of the project. A test suite groups scenarios.",
|
|
@@ -840,13 +867,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
840
867
|
},
|
|
841
868
|
withToolLogging("platform_list_test_suites", async (params) => {
|
|
842
869
|
requireApiKey();
|
|
843
|
-
const { handleListTestSuites } = await import(
|
|
844
|
-
"./tools/list-test-suites.js"
|
|
845
|
-
);
|
|
870
|
+
const { handleListTestSuites } = await import("./tools/list-test-suites.ts");
|
|
846
871
|
return {
|
|
847
872
|
content: [{ type: "text", text: await handleListTestSuites(params) }],
|
|
848
873
|
};
|
|
849
|
-
})
|
|
874
|
+
}),
|
|
850
875
|
);
|
|
851
876
|
|
|
852
877
|
server.tool(
|
|
@@ -859,13 +884,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
859
884
|
},
|
|
860
885
|
withToolLogging("platform_create_test_suite", async (params) => {
|
|
861
886
|
requireApiKey();
|
|
862
|
-
const { handleCreateTestSuite } = await import(
|
|
863
|
-
"./tools/create-test-suite.js"
|
|
864
|
-
);
|
|
887
|
+
const { handleCreateTestSuite } = await import("./tools/create-test-suite.ts");
|
|
865
888
|
return {
|
|
866
889
|
content: [{ type: "text", text: await handleCreateTestSuite(params) }],
|
|
867
890
|
};
|
|
868
|
-
})
|
|
891
|
+
}),
|
|
869
892
|
);
|
|
870
893
|
|
|
871
894
|
server.tool(
|
|
@@ -880,11 +903,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
880
903
|
},
|
|
881
904
|
withToolLogging("platform_get_test_suite", async (params) => {
|
|
882
905
|
requireApiKey();
|
|
883
|
-
const { handleGetTestSuite } = await import("./tools/get-test-suite.
|
|
906
|
+
const { handleGetTestSuite } = await import("./tools/get-test-suite.ts");
|
|
884
907
|
return {
|
|
885
908
|
content: [{ type: "text", text: await handleGetTestSuite(params) }],
|
|
886
909
|
};
|
|
887
|
-
})
|
|
910
|
+
}),
|
|
888
911
|
);
|
|
889
912
|
|
|
890
913
|
server.tool(
|
|
@@ -895,17 +918,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
895
918
|
name: z.string().optional().describe("The new name. The slug is kept."),
|
|
896
919
|
fields: suiteFieldsSchema
|
|
897
920
|
.optional()
|
|
898
|
-
.describe(
|
|
921
|
+
.describe(
|
|
922
|
+
"The full list of fields the suite declares. It replaces the list the suite holds.",
|
|
923
|
+
),
|
|
899
924
|
evaluators: evaluatorAttachmentsSchema
|
|
900
925
|
.optional()
|
|
901
|
-
.describe(
|
|
926
|
+
.describe(
|
|
927
|
+
"The full list of evaluators attached to the suite. It replaces the list the suite holds.",
|
|
928
|
+
),
|
|
902
929
|
},
|
|
903
930
|
withToolLogging("platform_update_test_suite", async (params) => {
|
|
904
931
|
requireApiKey();
|
|
905
932
|
return {
|
|
906
933
|
content: [{ type: "text", text: await handleUpdateTestSuite(params) }],
|
|
907
934
|
};
|
|
908
|
-
})
|
|
935
|
+
}),
|
|
909
936
|
);
|
|
910
937
|
|
|
911
938
|
server.tool(
|
|
@@ -917,13 +944,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
917
944
|
},
|
|
918
945
|
withToolLogging("platform_rename_test_suite", async (params) => {
|
|
919
946
|
requireApiKey();
|
|
920
|
-
const { handleRenameTestSuite } = await import(
|
|
921
|
-
"./tools/rename-test-suite.js"
|
|
922
|
-
);
|
|
947
|
+
const { handleRenameTestSuite } = await import("./tools/rename-test-suite.ts");
|
|
923
948
|
return {
|
|
924
949
|
content: [{ type: "text", text: await handleRenameTestSuite(params) }],
|
|
925
950
|
};
|
|
926
|
-
})
|
|
951
|
+
}),
|
|
927
952
|
);
|
|
928
953
|
|
|
929
954
|
server.tool(
|
|
@@ -934,13 +959,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
934
959
|
},
|
|
935
960
|
withToolLogging("platform_archive_test_suite", async (params) => {
|
|
936
961
|
requireApiKey();
|
|
937
|
-
const { handleArchiveTestSuite } = await import(
|
|
938
|
-
"./tools/archive-test-suite.js"
|
|
939
|
-
);
|
|
962
|
+
const { handleArchiveTestSuite } = await import("./tools/archive-test-suite.ts");
|
|
940
963
|
return {
|
|
941
964
|
content: [{ type: "text", text: await handleArchiveTestSuite(params) }],
|
|
942
965
|
};
|
|
943
|
-
})
|
|
966
|
+
}),
|
|
944
967
|
);
|
|
945
968
|
|
|
946
969
|
server.tool(
|
|
@@ -987,16 +1010,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
987
1010
|
},
|
|
988
1011
|
withToolLogging("platform_run_test_suite", async (params) => {
|
|
989
1012
|
requireApiKey();
|
|
990
|
-
const { handleRunTestSuite } = await import("./tools/run-test-suite.
|
|
1013
|
+
const { handleRunTestSuite } = await import("./tools/run-test-suite.ts");
|
|
991
1014
|
return {
|
|
992
1015
|
content: [{ type: "text", text: await handleRunTestSuite(params) }],
|
|
993
1016
|
};
|
|
994
|
-
})
|
|
1017
|
+
}),
|
|
995
1018
|
);
|
|
1019
|
+
}
|
|
996
1020
|
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1021
|
+
// --- Platform Simulation Run Tools (require API key) ---
|
|
1022
|
+
// These tools query simulation run results from the LangWatch platform.
|
|
1023
|
+
function registerSimulationRunTools(server: McpServer): void {
|
|
1000
1024
|
server.tool(
|
|
1001
1025
|
"platform_list_simulation_runs",
|
|
1002
1026
|
"List simulation run results. Filter by scenario set or batch run ID to see specific results.",
|
|
@@ -1008,11 +1032,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1008
1032
|
},
|
|
1009
1033
|
withToolLogging("platform_list_simulation_runs", async (params) => {
|
|
1010
1034
|
requireApiKey();
|
|
1011
|
-
const { handleListSimulationRuns } = await import("./tools/list-simulation-runs.
|
|
1035
|
+
const { handleListSimulationRuns } = await import("./tools/list-simulation-runs.ts");
|
|
1012
1036
|
return {
|
|
1013
1037
|
content: [{ type: "text", text: await handleListSimulationRuns(params) }],
|
|
1014
1038
|
};
|
|
1015
|
-
})
|
|
1039
|
+
}),
|
|
1016
1040
|
);
|
|
1017
1041
|
|
|
1018
1042
|
server.tool(
|
|
@@ -1024,16 +1048,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1024
1048
|
},
|
|
1025
1049
|
withToolLogging("platform_get_simulation_run", async (params) => {
|
|
1026
1050
|
requireApiKey();
|
|
1027
|
-
const { handleGetSimulationRun } = await import("./tools/get-simulation-run.
|
|
1051
|
+
const { handleGetSimulationRun } = await import("./tools/get-simulation-run.ts");
|
|
1028
1052
|
return {
|
|
1029
1053
|
content: [{ type: "text", text: await handleGetSimulationRun(params) }],
|
|
1030
1054
|
};
|
|
1031
|
-
})
|
|
1055
|
+
}),
|
|
1032
1056
|
);
|
|
1057
|
+
}
|
|
1033
1058
|
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1059
|
+
// --- Platform Evaluator Tools (require API key) ---
|
|
1060
|
+
// These tools manage evaluators on the LangWatch platform via API.
|
|
1061
|
+
function registerEvaluatorTools(server: McpServer): void {
|
|
1037
1062
|
server.tool(
|
|
1038
1063
|
"platform_create_evaluator",
|
|
1039
1064
|
`Create an evaluator on the LangWatch platform. Useful for setting up LLM-as-judge and other evaluators to use in evaluation notebooks. Call discover_schema({ category: 'evaluators' }) first to see available evaluator types and their settings.`,
|
|
@@ -1042,20 +1067,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1042
1067
|
config: z
|
|
1043
1068
|
.record(z.string(), z.unknown())
|
|
1044
1069
|
.describe(
|
|
1045
|
-
'Evaluator config object. Must include "evaluatorType" (e.g. "langevals/llm_boolean") and optional "settings" overrides.'
|
|
1070
|
+
'Evaluator config object. Must include "evaluatorType" (e.g. "langevals/llm_boolean") and optional "settings" overrides.',
|
|
1046
1071
|
),
|
|
1047
1072
|
},
|
|
1048
1073
|
withToolLogging("platform_create_evaluator", async (params) => {
|
|
1049
1074
|
requireApiKey();
|
|
1050
|
-
const { handleCreateEvaluator } = await import(
|
|
1051
|
-
"./tools/create-evaluator.js"
|
|
1052
|
-
);
|
|
1075
|
+
const { handleCreateEvaluator } = await import("./tools/create-evaluator.ts");
|
|
1053
1076
|
return {
|
|
1054
|
-
content: [
|
|
1055
|
-
{ type: "text", text: await handleCreateEvaluator(params) },
|
|
1056
|
-
],
|
|
1077
|
+
content: [{ type: "text", text: await handleCreateEvaluator(params) }],
|
|
1057
1078
|
};
|
|
1058
|
-
})
|
|
1079
|
+
}),
|
|
1059
1080
|
);
|
|
1060
1081
|
|
|
1061
1082
|
server.tool(
|
|
@@ -1064,32 +1085,26 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1064
1085
|
{},
|
|
1065
1086
|
withToolLogging("platform_list_evaluators", async () => {
|
|
1066
1087
|
requireApiKey();
|
|
1067
|
-
const { handleListEvaluators } = await import(
|
|
1068
|
-
"./tools/list-evaluators.js"
|
|
1069
|
-
);
|
|
1088
|
+
const { handleListEvaluators } = await import("./tools/list-evaluators.ts");
|
|
1070
1089
|
return {
|
|
1071
1090
|
content: [{ type: "text", text: await handleListEvaluators() }],
|
|
1072
1091
|
};
|
|
1073
|
-
})
|
|
1092
|
+
}),
|
|
1074
1093
|
);
|
|
1075
1094
|
|
|
1076
1095
|
server.tool(
|
|
1077
1096
|
"platform_get_evaluator",
|
|
1078
1097
|
"Get full details of an evaluator on the LangWatch platform by ID or slug, including config, input fields, and output fields.",
|
|
1079
1098
|
{
|
|
1080
|
-
idOrSlug: z
|
|
1081
|
-
.string()
|
|
1082
|
-
.describe("The evaluator ID or slug to retrieve"),
|
|
1099
|
+
idOrSlug: z.string().describe("The evaluator ID or slug to retrieve"),
|
|
1083
1100
|
},
|
|
1084
1101
|
withToolLogging("platform_get_evaluator", async (params) => {
|
|
1085
1102
|
requireApiKey();
|
|
1086
|
-
const { handleGetEvaluator } = await import(
|
|
1087
|
-
"./tools/get-evaluator.js"
|
|
1088
|
-
);
|
|
1103
|
+
const { handleGetEvaluator } = await import("./tools/get-evaluator.ts");
|
|
1089
1104
|
return {
|
|
1090
1105
|
content: [{ type: "text", text: await handleGetEvaluator(params) }],
|
|
1091
1106
|
};
|
|
1092
|
-
})
|
|
1107
|
+
}),
|
|
1093
1108
|
);
|
|
1094
1109
|
|
|
1095
1110
|
server.tool(
|
|
@@ -1101,21 +1116,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1101
1116
|
config: z
|
|
1102
1117
|
.record(z.string(), z.unknown())
|
|
1103
1118
|
.optional()
|
|
1104
|
-
.describe(
|
|
1105
|
-
"Updated config settings. Note: evaluatorType cannot be changed after creation."
|
|
1106
|
-
),
|
|
1119
|
+
.describe("Updated config settings. Note: evaluatorType cannot be changed after creation."),
|
|
1107
1120
|
},
|
|
1108
1121
|
withToolLogging("platform_update_evaluator", async (params) => {
|
|
1109
1122
|
requireApiKey();
|
|
1110
|
-
const { handleUpdateEvaluator } = await import(
|
|
1111
|
-
"./tools/update-evaluator.js"
|
|
1112
|
-
);
|
|
1123
|
+
const { handleUpdateEvaluator } = await import("./tools/update-evaluator.ts");
|
|
1113
1124
|
return {
|
|
1114
|
-
content: [
|
|
1115
|
-
{ type: "text", text: await handleUpdateEvaluator(params) },
|
|
1116
|
-
],
|
|
1125
|
+
content: [{ type: "text", text: await handleUpdateEvaluator(params) }],
|
|
1117
1126
|
};
|
|
1118
|
-
})
|
|
1127
|
+
}),
|
|
1119
1128
|
);
|
|
1120
1129
|
|
|
1121
1130
|
server.tool(
|
|
@@ -1126,48 +1135,40 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1126
1135
|
},
|
|
1127
1136
|
withToolLogging("platform_delete_evaluator", async (params) => {
|
|
1128
1137
|
requireApiKey();
|
|
1129
|
-
const { handleDeleteEvaluator } = await import("./tools/delete-evaluator.
|
|
1138
|
+
const { handleDeleteEvaluator } = await import("./tools/delete-evaluator.ts");
|
|
1130
1139
|
return {
|
|
1131
1140
|
content: [{ type: "text", text: await handleDeleteEvaluator(params) }],
|
|
1132
1141
|
};
|
|
1133
|
-
})
|
|
1142
|
+
}),
|
|
1134
1143
|
);
|
|
1144
|
+
}
|
|
1135
1145
|
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1146
|
+
// --- Platform Model Provider Tools (require API key) ---
|
|
1147
|
+
// These tools manage model provider API keys on the LangWatch platform.
|
|
1148
|
+
function registerModelProviderTools(server: McpServer): void {
|
|
1139
1149
|
server.tool(
|
|
1140
1150
|
"platform_set_model_provider",
|
|
1141
1151
|
`Set or update a model provider on the LangWatch platform. Use this to configure API keys (e.g. OPENAI_API_KEY) needed to run evaluators. The API key is stored securely and never returned in responses. Omit customKeys to update other settings without changing existing keys.`,
|
|
1142
1152
|
{
|
|
1143
1153
|
provider: z
|
|
1144
1154
|
.string()
|
|
1145
|
-
.describe(
|
|
1146
|
-
'Provider name, e.g., "openai", "anthropic", "azure", "custom"'
|
|
1147
|
-
),
|
|
1155
|
+
.describe('Provider name, e.g., "openai", "anthropic", "azure", "custom"'),
|
|
1148
1156
|
enabled: z.boolean().describe("Whether the provider is enabled"),
|
|
1149
1157
|
customKeys: z
|
|
1150
1158
|
.record(z.string(), z.unknown())
|
|
1151
1159
|
.optional()
|
|
1152
1160
|
.describe(
|
|
1153
|
-
'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.'
|
|
1161
|
+
'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.',
|
|
1154
1162
|
),
|
|
1155
|
-
defaultModel: z
|
|
1156
|
-
.string()
|
|
1157
|
-
.optional()
|
|
1158
|
-
.describe("Set as project default model"),
|
|
1163
|
+
defaultModel: z.string().optional().describe("Set as project default model"),
|
|
1159
1164
|
},
|
|
1160
1165
|
withToolLogging("platform_set_model_provider", async (params) => {
|
|
1161
1166
|
requireApiKey();
|
|
1162
|
-
const { handleSetModelProvider } = await import(
|
|
1163
|
-
"./tools/set-model-provider.js"
|
|
1164
|
-
);
|
|
1167
|
+
const { handleSetModelProvider } = await import("./tools/set-model-provider.ts");
|
|
1165
1168
|
return {
|
|
1166
|
-
content: [
|
|
1167
|
-
{ type: "text", text: await handleSetModelProvider(params) },
|
|
1168
|
-
],
|
|
1169
|
+
content: [{ type: "text", text: await handleSetModelProvider(params) }],
|
|
1169
1170
|
};
|
|
1170
|
-
})
|
|
1171
|
+
}),
|
|
1171
1172
|
);
|
|
1172
1173
|
|
|
1173
1174
|
server.tool(
|
|
@@ -1176,28 +1177,27 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1176
1177
|
{},
|
|
1177
1178
|
withToolLogging("platform_list_model_providers", async () => {
|
|
1178
1179
|
requireApiKey();
|
|
1179
|
-
const { handleListModelProviders } = await import(
|
|
1180
|
-
"./tools/list-model-providers.js"
|
|
1181
|
-
);
|
|
1180
|
+
const { handleListModelProviders } = await import("./tools/list-model-providers.ts");
|
|
1182
1181
|
return {
|
|
1183
1182
|
content: [{ type: "text", text: await handleListModelProviders() }],
|
|
1184
1183
|
};
|
|
1185
|
-
})
|
|
1184
|
+
}),
|
|
1186
1185
|
);
|
|
1186
|
+
}
|
|
1187
1187
|
|
|
1188
|
-
|
|
1189
|
-
|
|
1188
|
+
// --- Platform Agent Tools (require API key) ---
|
|
1189
|
+
function registerAgentTools(server: McpServer): void {
|
|
1190
1190
|
server.tool(
|
|
1191
1191
|
"platform_list_agents",
|
|
1192
1192
|
"List all agents in the LangWatch project with their names, types and IDs. A connected agent (one that registered itself from code with connectAgent or connect_agent) also shows its environment, whether it is online, how many instances are connected, and its owner.",
|
|
1193
1193
|
{},
|
|
1194
1194
|
withToolLogging("platform_list_agents", async () => {
|
|
1195
1195
|
requireApiKey();
|
|
1196
|
-
const { handleListAgents } = await import("./tools/list-agents.
|
|
1196
|
+
const { handleListAgents } = await import("./tools/list-agents.ts");
|
|
1197
1197
|
return {
|
|
1198
1198
|
content: [{ type: "text", text: await handleListAgents() }],
|
|
1199
1199
|
};
|
|
1200
|
-
})
|
|
1200
|
+
}),
|
|
1201
1201
|
);
|
|
1202
1202
|
|
|
1203
1203
|
server.tool(
|
|
@@ -1208,11 +1208,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1208
1208
|
},
|
|
1209
1209
|
withToolLogging("platform_get_agent", async (params) => {
|
|
1210
1210
|
requireApiKey();
|
|
1211
|
-
const { handleGetAgent } = await import("./tools/get-agent.
|
|
1211
|
+
const { handleGetAgent } = await import("./tools/get-agent.ts");
|
|
1212
1212
|
return {
|
|
1213
1213
|
content: [{ type: "text", text: await handleGetAgent(params) }],
|
|
1214
1214
|
};
|
|
1215
|
-
})
|
|
1215
|
+
}),
|
|
1216
1216
|
);
|
|
1217
1217
|
|
|
1218
1218
|
server.tool(
|
|
@@ -1225,12 +1225,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1225
1225
|
},
|
|
1226
1226
|
withToolLogging("platform_create_agent", async (params) => {
|
|
1227
1227
|
requireApiKey();
|
|
1228
|
-
const { handleCreateAgent } = await import("./tools/create-agent.
|
|
1229
|
-
const parsedConfig = params.config
|
|
1228
|
+
const { handleCreateAgent } = await import("./tools/create-agent.ts");
|
|
1229
|
+
const parsedConfig = params.config
|
|
1230
|
+
? (JSON.parse(params.config) as Record<string, unknown>)
|
|
1231
|
+
: undefined;
|
|
1230
1232
|
return {
|
|
1231
|
-
content: [
|
|
1233
|
+
content: [
|
|
1234
|
+
{ type: "text", text: await handleCreateAgent({ ...params, config: parsedConfig }) },
|
|
1235
|
+
],
|
|
1232
1236
|
};
|
|
1233
|
-
})
|
|
1237
|
+
}),
|
|
1234
1238
|
);
|
|
1235
1239
|
|
|
1236
1240
|
server.tool(
|
|
@@ -1240,16 +1244,23 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1240
1244
|
id: z.string().describe("The agent ID"),
|
|
1241
1245
|
name: z.string().optional().describe("New agent name"),
|
|
1242
1246
|
type: z.string().optional().describe("New agent type: signature, code, workflow, or http"),
|
|
1243
|
-
config: z
|
|
1247
|
+
config: z
|
|
1248
|
+
.string()
|
|
1249
|
+
.optional()
|
|
1250
|
+
.describe("Updated configuration as JSON string (will be parsed)"),
|
|
1244
1251
|
},
|
|
1245
1252
|
withToolLogging("platform_update_agent", async (params) => {
|
|
1246
1253
|
requireApiKey();
|
|
1247
|
-
const { handleUpdateAgent } = await import("./tools/update-agent.
|
|
1248
|
-
const parsedConfig = params.config
|
|
1254
|
+
const { handleUpdateAgent } = await import("./tools/update-agent.ts");
|
|
1255
|
+
const parsedConfig = params.config
|
|
1256
|
+
? (JSON.parse(params.config) as Record<string, unknown>)
|
|
1257
|
+
: undefined;
|
|
1249
1258
|
return {
|
|
1250
|
-
content: [
|
|
1259
|
+
content: [
|
|
1260
|
+
{ type: "text", text: await handleUpdateAgent({ ...params, config: parsedConfig }) },
|
|
1261
|
+
],
|
|
1251
1262
|
};
|
|
1252
|
-
})
|
|
1263
|
+
}),
|
|
1253
1264
|
);
|
|
1254
1265
|
|
|
1255
1266
|
server.tool(
|
|
@@ -1260,11 +1271,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1260
1271
|
},
|
|
1261
1272
|
withToolLogging("platform_delete_agent", async (params) => {
|
|
1262
1273
|
requireApiKey();
|
|
1263
|
-
const { handleDeleteAgent } = await import("./tools/delete-agent.
|
|
1274
|
+
const { handleDeleteAgent } = await import("./tools/delete-agent.ts");
|
|
1264
1275
|
return {
|
|
1265
1276
|
content: [{ type: "text", text: await handleDeleteAgent(params) }],
|
|
1266
1277
|
};
|
|
1267
|
-
})
|
|
1278
|
+
}),
|
|
1268
1279
|
);
|
|
1269
1280
|
|
|
1270
1281
|
server.tool(
|
|
@@ -1286,23 +1297,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1286
1297
|
.record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
|
|
1287
1298
|
.optional()
|
|
1288
1299
|
.describe("Run parameter values for a connected agent, by name"),
|
|
1289
|
-
threadId: z
|
|
1290
|
-
.string()
|
|
1291
|
-
.optional()
|
|
1292
|
-
.describe("Continue a conversation on a connected agent"),
|
|
1300
|
+
threadId: z.string().optional().describe("Continue a conversation on a connected agent"),
|
|
1293
1301
|
},
|
|
1294
1302
|
withToolLogging("platform_run_agent", async (params) => {
|
|
1295
1303
|
requireApiKey();
|
|
1296
|
-
const { handleRunAgent } = await import("./tools/run-agent.
|
|
1304
|
+
const { handleRunAgent } = await import("./tools/run-agent.ts");
|
|
1297
1305
|
return {
|
|
1298
1306
|
content: [{ type: "text", text: await handleRunAgent(params) }],
|
|
1299
1307
|
};
|
|
1300
|
-
})
|
|
1308
|
+
}),
|
|
1301
1309
|
);
|
|
1302
1310
|
|
|
1303
1311
|
server.tool(
|
|
1304
1312
|
"platform_test_agent",
|
|
1305
|
-
|
|
1313
|
+
'Test an agent with one scripted scenario run: the user sends "ping", the agent answers, and the run succeeds when the answer arrives. No model is used, and no scenario, run plan or test suite is added to the project. Answers at once with the scenario run id to follow with platform_get_simulation_run; the run itself is asynchronous.',
|
|
1306
1314
|
{
|
|
1307
1315
|
id: z.string().describe("The agent ID to test"),
|
|
1308
1316
|
},
|
|
@@ -1311,99 +1319,296 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1311
1319
|
return {
|
|
1312
1320
|
content: [{ type: "text", text: await handleTestAgent(params) }],
|
|
1313
1321
|
};
|
|
1314
|
-
})
|
|
1322
|
+
}),
|
|
1315
1323
|
);
|
|
1324
|
+
}
|
|
1316
1325
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1326
|
+
// --- Platform Analytics Report Tools (require API key) ---
|
|
1327
|
+
function registerAnalyticsReportTools(server: McpServer): void {
|
|
1319
1328
|
server.tool(
|
|
1320
|
-
"
|
|
1321
|
-
"List
|
|
1329
|
+
"platform_list_analytics_reports",
|
|
1330
|
+
"List the Analytics Reports in the LangWatch project (/api/v1/analytics/reports). Dashboards, which hold widgets, are listed by list_dashboards.",
|
|
1322
1331
|
{},
|
|
1323
|
-
withToolLogging("
|
|
1332
|
+
withToolLogging("platform_list_analytics_reports", async () => {
|
|
1324
1333
|
requireApiKey();
|
|
1325
|
-
const {
|
|
1334
|
+
const { handleListAnalyticsReports } = await import("./tools/list-analytics-reports.ts");
|
|
1326
1335
|
return {
|
|
1327
|
-
content: [{ type: "text", text: await
|
|
1336
|
+
content: [{ type: "text", text: await handleListAnalyticsReports() }],
|
|
1328
1337
|
};
|
|
1329
|
-
})
|
|
1338
|
+
}),
|
|
1330
1339
|
);
|
|
1331
1340
|
|
|
1332
1341
|
server.tool(
|
|
1333
|
-
"
|
|
1334
|
-
"Get
|
|
1342
|
+
"platform_get_analytics_report",
|
|
1343
|
+
"Get an Analytics Report by its ID, including its graphs. A Dashboards dashboard's ID is not found here; read it with get_dashboard.",
|
|
1335
1344
|
{
|
|
1336
|
-
|
|
1345
|
+
reportId: z.string().describe("The Analytics Report ID"),
|
|
1337
1346
|
},
|
|
1338
|
-
withToolLogging("
|
|
1347
|
+
withToolLogging("platform_get_analytics_report", async (params) => {
|
|
1339
1348
|
requireApiKey();
|
|
1340
|
-
const {
|
|
1349
|
+
const { handleGetAnalyticsReport } = await import("./tools/get-analytics-report.ts");
|
|
1341
1350
|
return {
|
|
1342
|
-
content: [{ type: "text", text: await
|
|
1351
|
+
content: [{ type: "text", text: await handleGetAnalyticsReport(params) }],
|
|
1343
1352
|
};
|
|
1344
|
-
})
|
|
1353
|
+
}),
|
|
1345
1354
|
);
|
|
1346
1355
|
|
|
1347
1356
|
server.tool(
|
|
1348
|
-
"
|
|
1349
|
-
"Create a new
|
|
1357
|
+
"platform_create_analytics_report",
|
|
1358
|
+
"Create a new Analytics Report.",
|
|
1350
1359
|
{
|
|
1351
|
-
name: z.string().describe("
|
|
1360
|
+
name: z.string().describe("Analytics Report name"),
|
|
1352
1361
|
},
|
|
1353
|
-
withToolLogging("
|
|
1362
|
+
withToolLogging("platform_create_analytics_report", async (params) => {
|
|
1363
|
+
requireApiKey();
|
|
1364
|
+
const { handleCreateAnalyticsReport } = await import("./tools/create-analytics-report.ts");
|
|
1365
|
+
return {
|
|
1366
|
+
content: [
|
|
1367
|
+
{
|
|
1368
|
+
type: "text",
|
|
1369
|
+
text: await handleCreateAnalyticsReport(params),
|
|
1370
|
+
},
|
|
1371
|
+
],
|
|
1372
|
+
};
|
|
1373
|
+
}),
|
|
1374
|
+
);
|
|
1375
|
+
|
|
1376
|
+
server.tool(
|
|
1377
|
+
"platform_delete_analytics_report",
|
|
1378
|
+
"Delete an Analytics Report and all its graphs.",
|
|
1379
|
+
{
|
|
1380
|
+
reportId: z.string().describe("The Analytics Report ID to delete"),
|
|
1381
|
+
},
|
|
1382
|
+
withToolLogging("platform_delete_analytics_report", async (params) => {
|
|
1383
|
+
requireApiKey();
|
|
1384
|
+
const { handleDeleteAnalyticsReport } = await import("./tools/delete-analytics-report.ts");
|
|
1385
|
+
return {
|
|
1386
|
+
content: [
|
|
1387
|
+
{
|
|
1388
|
+
type: "text",
|
|
1389
|
+
text: await handleDeleteAnalyticsReport(params),
|
|
1390
|
+
},
|
|
1391
|
+
],
|
|
1392
|
+
};
|
|
1393
|
+
}),
|
|
1394
|
+
);
|
|
1395
|
+
|
|
1396
|
+
server.tool(
|
|
1397
|
+
"platform_rename_analytics_report",
|
|
1398
|
+
"Rename an Analytics Report.",
|
|
1399
|
+
{
|
|
1400
|
+
reportId: z.string().describe("The Analytics Report ID to rename"),
|
|
1401
|
+
name: z.string().describe("The new name for the Analytics Report"),
|
|
1402
|
+
},
|
|
1403
|
+
withToolLogging("platform_rename_analytics_report", async (params) => {
|
|
1404
|
+
requireApiKey();
|
|
1405
|
+
const { renameAnalyticsReport } = await import("./langwatch-api-analytics-reports.ts");
|
|
1406
|
+
const result = await renameAnalyticsReport({ reportId: params.reportId, name: params.name });
|
|
1407
|
+
return {
|
|
1408
|
+
content: [
|
|
1409
|
+
{
|
|
1410
|
+
type: "text",
|
|
1411
|
+
text: `Analytics Report "${result.name}" renamed successfully (ID: ${result.id}).`,
|
|
1412
|
+
},
|
|
1413
|
+
],
|
|
1414
|
+
};
|
|
1415
|
+
}),
|
|
1416
|
+
);
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
// --- Dashboard Tools (require API key) ---
|
|
1420
|
+
function registerDashboardTools(server: McpServer): void {
|
|
1421
|
+
server.tool(
|
|
1422
|
+
"list_dashboards",
|
|
1423
|
+
"Lists the project's dashboards in the Dashboards area, with each one's ID and widget count. Use an ID from here as the dashboardId of add_dashboard_widget and update_dashboard_widget; the Analytics Reports platform_list_analytics_reports lists hold no widgets.",
|
|
1424
|
+
{},
|
|
1425
|
+
withToolLogging("list_dashboards", async () => {
|
|
1354
1426
|
requireApiKey();
|
|
1355
|
-
const {
|
|
1427
|
+
const { handleListDashboards } = await import("./tools/list-dashboards.js");
|
|
1356
1428
|
return {
|
|
1357
|
-
content: [{ type: "text", text: await
|
|
1429
|
+
content: [{ type: "text", text: await handleListDashboards() }],
|
|
1358
1430
|
};
|
|
1359
|
-
})
|
|
1431
|
+
}),
|
|
1360
1432
|
);
|
|
1361
1433
|
|
|
1362
1434
|
server.tool(
|
|
1363
|
-
"
|
|
1364
|
-
"
|
|
1435
|
+
"add_dashboard_widget",
|
|
1436
|
+
"Creates a custom dashboard widget and places it on the given dashboard. A widget's `code` is a TSX file run in a sandbox: it receives a global `LW` with `LW.useChartQuery(name)` (reads one of the widget's own named queries), `LW.navigate` and `LW.theme`. `queries` are named LangWatchQL SQL statements the code can read by name — validate each statement's SQL first with run_query before calling this tool, since a query only runs inside the sandbox at render time and a mistake surfaces there, not here. Up to 8 queries per widget.",
|
|
1365
1437
|
{
|
|
1366
|
-
|
|
1438
|
+
dashboardId: z
|
|
1439
|
+
.string()
|
|
1440
|
+
.describe("The dashboard to add the widget to, as list_dashboards lists it"),
|
|
1441
|
+
name: z.string().describe("The widget's display name"),
|
|
1442
|
+
code: z
|
|
1443
|
+
.string()
|
|
1444
|
+
.describe(
|
|
1445
|
+
"The widget's TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope.",
|
|
1446
|
+
),
|
|
1447
|
+
queries: z
|
|
1448
|
+
.union([
|
|
1449
|
+
z.record(z.string(), z.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1450
|
+
z
|
|
1451
|
+
.array(
|
|
1452
|
+
z.object({
|
|
1453
|
+
name: z.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1454
|
+
sql: z.string().describe("The LangWatchQL statement"),
|
|
1455
|
+
}),
|
|
1456
|
+
)
|
|
1457
|
+
.describe("Named LWQL queries, array form"),
|
|
1458
|
+
])
|
|
1459
|
+
.describe(
|
|
1460
|
+
"The widget's named LangWatchQL queries: either {name: sql} or an array of {name, sql}. Max 8. Validate each statement with run_query first.",
|
|
1461
|
+
),
|
|
1462
|
+
sources: widgetSourcesInput,
|
|
1367
1463
|
},
|
|
1368
|
-
withToolLogging("
|
|
1464
|
+
withToolLogging("add_dashboard_widget", async (params) => {
|
|
1369
1465
|
requireApiKey();
|
|
1370
|
-
const {
|
|
1466
|
+
const { handleAddDashboardWidget } = await import("./tools/add-dashboard-widget.js");
|
|
1371
1467
|
return {
|
|
1372
|
-
content: [{ type: "text", text: await
|
|
1468
|
+
content: [{ type: "text", text: await handleAddDashboardWidget(params) }],
|
|
1373
1469
|
};
|
|
1374
|
-
})
|
|
1470
|
+
}),
|
|
1375
1471
|
);
|
|
1376
1472
|
|
|
1377
1473
|
server.tool(
|
|
1378
|
-
"
|
|
1379
|
-
"
|
|
1474
|
+
"update_dashboard_widget",
|
|
1475
|
+
"Changes one custom widget on a dashboard: its name, its TSX `code`, its named LangWatchQL `queries`, its `description` or the `sources` its queries count. Pass only the fields to change; every field left out keeps its stored value. Validate any new query's SQL with run_query first. Returns the updated widget.",
|
|
1380
1476
|
{
|
|
1381
|
-
|
|
1382
|
-
|
|
1477
|
+
dashboardId: z.string().describe("The dashboard the widget is on"),
|
|
1478
|
+
widgetId: z.string().describe("The widget to change, as add_dashboard_widget returned it"),
|
|
1479
|
+
name: z.string().optional().describe("The widget's new display name"),
|
|
1480
|
+
code: z
|
|
1481
|
+
.string()
|
|
1482
|
+
.optional()
|
|
1483
|
+
.describe(
|
|
1484
|
+
"The widget's new TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope.",
|
|
1485
|
+
),
|
|
1486
|
+
queries: z
|
|
1487
|
+
.union([
|
|
1488
|
+
z.record(z.string(), z.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1489
|
+
z
|
|
1490
|
+
.array(
|
|
1491
|
+
z.object({
|
|
1492
|
+
name: z.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1493
|
+
sql: z.string().describe("The LangWatchQL statement"),
|
|
1494
|
+
}),
|
|
1495
|
+
)
|
|
1496
|
+
.describe("Named LWQL queries, array form"),
|
|
1497
|
+
])
|
|
1498
|
+
.optional()
|
|
1499
|
+
.describe(
|
|
1500
|
+
"The widget's new named LangWatchQL queries, replacing the stored set: either {name: sql} or an array of {name, sql}. Max 8.",
|
|
1501
|
+
),
|
|
1502
|
+
description: z
|
|
1503
|
+
.string()
|
|
1504
|
+
.optional()
|
|
1505
|
+
.describe("What the widget's info tip says: a sentence or two"),
|
|
1506
|
+
sources: widgetSourcesInput,
|
|
1383
1507
|
},
|
|
1384
|
-
withToolLogging("
|
|
1508
|
+
withToolLogging("update_dashboard_widget", async (params) => {
|
|
1385
1509
|
requireApiKey();
|
|
1386
|
-
const {
|
|
1387
|
-
const result = await renameDashboard(params.id, { name: params.name });
|
|
1510
|
+
const { handleUpdateDashboardWidget } = await import("./tools/update-dashboard-widget.js");
|
|
1388
1511
|
return {
|
|
1389
|
-
content: [{ type: "text", text:
|
|
1512
|
+
content: [{ type: "text", text: await handleUpdateDashboardWidget(params) }],
|
|
1390
1513
|
};
|
|
1391
|
-
})
|
|
1514
|
+
}),
|
|
1515
|
+
);
|
|
1516
|
+
|
|
1517
|
+
registerDashboardCrudTools(server);
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
/** Dashboard CRUD and a dashboard's widgets, over `/api/v1/dashboards`. */
|
|
1521
|
+
function registerDashboardCrudTools(server: McpServer): void {
|
|
1522
|
+
const dashboardId = z.string().describe("The dashboard's ID, as list_dashboards lists it");
|
|
1523
|
+
const text = (value: string) => ({ content: [{ type: "text" as const, text: value }] });
|
|
1524
|
+
|
|
1525
|
+
server.tool(
|
|
1526
|
+
"get_dashboard",
|
|
1527
|
+
"Reads one dashboard in the Dashboards area: its name, scope, description and address. An Analytics Report's ID is not found here; read it with platform_get_analytics_report.",
|
|
1528
|
+
{ dashboardId },
|
|
1529
|
+
withToolLogging("get_dashboard", async (params) => {
|
|
1530
|
+
requireApiKey();
|
|
1531
|
+
const { handleGetDashboard } = await import("./tools/dashboards.js");
|
|
1532
|
+
return text(await handleGetDashboard(params));
|
|
1533
|
+
}),
|
|
1534
|
+
);
|
|
1535
|
+
|
|
1536
|
+
server.tool(
|
|
1537
|
+
"create_dashboard",
|
|
1538
|
+
"Creates an empty dashboard in the Dashboards area. Add widgets to it with add_dashboard_widget.",
|
|
1539
|
+
{ name: z.string().describe("The dashboard's name") },
|
|
1540
|
+
withToolLogging("create_dashboard", async (params) => {
|
|
1541
|
+
requireApiKey();
|
|
1542
|
+
const { handleCreateDashboard } = await import("./tools/dashboards.js");
|
|
1543
|
+
return text(await handleCreateDashboard(params));
|
|
1544
|
+
}),
|
|
1545
|
+
);
|
|
1546
|
+
|
|
1547
|
+
server.tool(
|
|
1548
|
+
"rename_dashboard",
|
|
1549
|
+
"Changes a dashboard's name and/or description; a field left out keeps its value.",
|
|
1550
|
+
{
|
|
1551
|
+
dashboardId,
|
|
1552
|
+
name: z.string().optional().describe("The dashboard's new name"),
|
|
1553
|
+
description: z.string().optional().describe("The dashboard's new description"),
|
|
1554
|
+
},
|
|
1555
|
+
withToolLogging("rename_dashboard", async (params) => {
|
|
1556
|
+
requireApiKey();
|
|
1557
|
+
const { handleRenameDashboard } = await import("./tools/dashboards.js");
|
|
1558
|
+
return text(await handleRenameDashboard(params));
|
|
1559
|
+
}),
|
|
1560
|
+
);
|
|
1561
|
+
|
|
1562
|
+
server.tool(
|
|
1563
|
+
"delete_dashboard",
|
|
1564
|
+
"Deletes a dashboard together with the widgets on it. This cannot be undone.",
|
|
1565
|
+
{ dashboardId },
|
|
1566
|
+
withToolLogging("delete_dashboard", async (params) => {
|
|
1567
|
+
requireApiKey();
|
|
1568
|
+
const { handleDeleteDashboard } = await import("./tools/dashboards.js");
|
|
1569
|
+
return text(await handleDeleteDashboard(params));
|
|
1570
|
+
}),
|
|
1571
|
+
);
|
|
1572
|
+
|
|
1573
|
+
server.tool(
|
|
1574
|
+
"list_dashboard_widgets",
|
|
1575
|
+
"Lists the custom widgets on one dashboard, with each widget's ID and query names.",
|
|
1576
|
+
{ dashboardId },
|
|
1577
|
+
withToolLogging("list_dashboard_widgets", async (params) => {
|
|
1578
|
+
requireApiKey();
|
|
1579
|
+
const { handleListDashboardWidgets } = await import("./tools/dashboards.js");
|
|
1580
|
+
return text(await handleListDashboardWidgets(params));
|
|
1581
|
+
}),
|
|
1392
1582
|
);
|
|
1393
1583
|
|
|
1394
|
-
|
|
1584
|
+
server.tool(
|
|
1585
|
+
"delete_dashboard_widget",
|
|
1586
|
+
"Deletes one custom widget from a dashboard. A widget on another dashboard is not found.",
|
|
1587
|
+
{
|
|
1588
|
+
dashboardId,
|
|
1589
|
+
widgetId: z.string().describe("The widget's ID, as list_dashboard_widgets lists it"),
|
|
1590
|
+
},
|
|
1591
|
+
withToolLogging("delete_dashboard_widget", async (params) => {
|
|
1592
|
+
requireApiKey();
|
|
1593
|
+
const { handleDeleteDashboardWidget } = await import("./tools/dashboards.js");
|
|
1594
|
+
return text(await handleDeleteDashboardWidget(params));
|
|
1595
|
+
}),
|
|
1596
|
+
);
|
|
1597
|
+
}
|
|
1395
1598
|
|
|
1599
|
+
// --- Platform Workflow Tools (require API key) ---
|
|
1600
|
+
function registerWorkflowTools(server: McpServer): void {
|
|
1396
1601
|
server.tool(
|
|
1397
1602
|
"platform_list_workflows",
|
|
1398
1603
|
"List all workflows in the LangWatch project.",
|
|
1399
1604
|
{},
|
|
1400
1605
|
withToolLogging("platform_list_workflows", async () => {
|
|
1401
1606
|
requireApiKey();
|
|
1402
|
-
const { handleListWorkflows } = await import("./tools/list-workflows.
|
|
1607
|
+
const { handleListWorkflows } = await import("./tools/list-workflows.ts");
|
|
1403
1608
|
return {
|
|
1404
1609
|
content: [{ type: "text", text: await handleListWorkflows() }],
|
|
1405
1610
|
};
|
|
1406
|
-
})
|
|
1611
|
+
}),
|
|
1407
1612
|
);
|
|
1408
1613
|
|
|
1409
1614
|
server.tool(
|
|
@@ -1414,11 +1619,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1414
1619
|
},
|
|
1415
1620
|
withToolLogging("platform_get_workflow", async (params) => {
|
|
1416
1621
|
requireApiKey();
|
|
1417
|
-
const { handleGetWorkflow } = await import("./tools/get-workflow.
|
|
1622
|
+
const { handleGetWorkflow } = await import("./tools/get-workflow.ts");
|
|
1418
1623
|
return {
|
|
1419
1624
|
content: [{ type: "text", text: await handleGetWorkflow(params) }],
|
|
1420
1625
|
};
|
|
1421
|
-
})
|
|
1626
|
+
}),
|
|
1422
1627
|
);
|
|
1423
1628
|
|
|
1424
1629
|
server.tool(
|
|
@@ -1429,11 +1634,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1429
1634
|
},
|
|
1430
1635
|
withToolLogging("platform_delete_workflow", async (params) => {
|
|
1431
1636
|
requireApiKey();
|
|
1432
|
-
const { handleDeleteWorkflow } = await import("./tools/delete-workflow.
|
|
1637
|
+
const { handleDeleteWorkflow } = await import("./tools/delete-workflow.ts");
|
|
1433
1638
|
return {
|
|
1434
1639
|
content: [{ type: "text", text: await handleDeleteWorkflow(params) }],
|
|
1435
1640
|
};
|
|
1436
|
-
})
|
|
1641
|
+
}),
|
|
1437
1642
|
);
|
|
1438
1643
|
|
|
1439
1644
|
server.tool(
|
|
@@ -1445,15 +1650,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1445
1650
|
},
|
|
1446
1651
|
withToolLogging("platform_run_workflow", async (params) => {
|
|
1447
1652
|
requireApiKey();
|
|
1448
|
-
const { handleRunWorkflow } = await import("./tools/run-workflow.
|
|
1653
|
+
const { handleRunWorkflow } = await import("./tools/run-workflow.ts");
|
|
1449
1654
|
return {
|
|
1450
1655
|
content: [{ type: "text", text: await handleRunWorkflow(params) }],
|
|
1451
1656
|
};
|
|
1452
|
-
})
|
|
1657
|
+
}),
|
|
1453
1658
|
);
|
|
1659
|
+
}
|
|
1454
1660
|
|
|
1455
|
-
|
|
1456
|
-
|
|
1661
|
+
// --- Platform Annotation Tools (require API key) ---
|
|
1662
|
+
function registerAnnotationTools(server: McpServer): void {
|
|
1457
1663
|
server.tool(
|
|
1458
1664
|
"platform_list_annotations",
|
|
1459
1665
|
"List all annotations for the project, optionally filtered by trace ID.",
|
|
@@ -1462,11 +1668,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1462
1668
|
},
|
|
1463
1669
|
withToolLogging("platform_list_annotations", async (params) => {
|
|
1464
1670
|
requireApiKey();
|
|
1465
|
-
const { handleListAnnotations } = await import("./tools/list-annotations.
|
|
1671
|
+
const { handleListAnnotations } = await import("./tools/list-annotations.ts");
|
|
1466
1672
|
return {
|
|
1467
1673
|
content: [{ type: "text", text: await handleListAnnotations(params) }],
|
|
1468
1674
|
};
|
|
1469
|
-
})
|
|
1675
|
+
}),
|
|
1470
1676
|
);
|
|
1471
1677
|
|
|
1472
1678
|
server.tool(
|
|
@@ -1480,11 +1686,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1480
1686
|
},
|
|
1481
1687
|
withToolLogging("platform_create_annotation", async (params) => {
|
|
1482
1688
|
requireApiKey();
|
|
1483
|
-
const { handleCreateAnnotation } = await import("./tools/create-annotation.
|
|
1689
|
+
const { handleCreateAnnotation } = await import("./tools/create-annotation.ts");
|
|
1484
1690
|
return {
|
|
1485
1691
|
content: [{ type: "text", text: await handleCreateAnnotation(params) }],
|
|
1486
1692
|
};
|
|
1487
|
-
})
|
|
1693
|
+
}),
|
|
1488
1694
|
);
|
|
1489
1695
|
|
|
1490
1696
|
server.tool(
|
|
@@ -1495,11 +1701,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1495
1701
|
},
|
|
1496
1702
|
withToolLogging("platform_delete_annotation", async (params) => {
|
|
1497
1703
|
requireApiKey();
|
|
1498
|
-
const { handleDeleteAnnotation } = await import("./tools/delete-annotation.
|
|
1704
|
+
const { handleDeleteAnnotation } = await import("./tools/delete-annotation.ts");
|
|
1499
1705
|
return {
|
|
1500
1706
|
content: [{ type: "text", text: await handleDeleteAnnotation(params) }],
|
|
1501
1707
|
};
|
|
1502
|
-
})
|
|
1708
|
+
}),
|
|
1503
1709
|
);
|
|
1504
1710
|
|
|
1505
1711
|
server.tool(
|
|
@@ -1510,7 +1716,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1510
1716
|
},
|
|
1511
1717
|
withToolLogging("platform_get_annotation", async (params) => {
|
|
1512
1718
|
requireApiKey();
|
|
1513
|
-
const { getAnnotation } = await import("./langwatch-api-annotations.
|
|
1719
|
+
const { getAnnotation } = await import("./langwatch-api-annotations.ts");
|
|
1514
1720
|
const annotation = await getAnnotation(params.id);
|
|
1515
1721
|
const lines = [
|
|
1516
1722
|
`**ID**: ${annotation.id}`,
|
|
@@ -1522,11 +1728,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1522
1728
|
return {
|
|
1523
1729
|
content: [{ type: "text", text: lines.join("\n") }],
|
|
1524
1730
|
};
|
|
1525
|
-
})
|
|
1731
|
+
}),
|
|
1526
1732
|
);
|
|
1733
|
+
}
|
|
1527
1734
|
|
|
1528
|
-
|
|
1529
|
-
|
|
1735
|
+
// --- Platform Trigger/Automation Tools (require API key) ---
|
|
1736
|
+
function registerTriggerTools(server: McpServer): void {
|
|
1530
1737
|
server.tool(
|
|
1531
1738
|
"platform_list_triggers",
|
|
1532
1739
|
"List all triggers (automations) in the project. Triggers automate actions like sending emails, Slack messages, or adding to datasets when conditions are met.",
|
|
@@ -1535,72 +1742,155 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1535
1742
|
},
|
|
1536
1743
|
withToolLogging("platform_list_triggers", async (params) => {
|
|
1537
1744
|
requireApiKey();
|
|
1538
|
-
const {
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
1556
|
-
})
|
|
1745
|
+
const { handleListTriggers } = await import("./tools/list-triggers.ts");
|
|
1746
|
+
return { content: [{ type: "text", text: await handleListTriggers(params) }] };
|
|
1747
|
+
}),
|
|
1748
|
+
);
|
|
1749
|
+
|
|
1750
|
+
server.tool(
|
|
1751
|
+
"platform_get_trigger",
|
|
1752
|
+
"Read one trigger (automation) by its ID, including its delivery configuration. Credential values come back as [redacted]; sending them back unchanged on an update keeps the stored values.",
|
|
1753
|
+
{
|
|
1754
|
+
id: z.string().describe("The trigger ID"),
|
|
1755
|
+
},
|
|
1756
|
+
withToolLogging("platform_get_trigger", async (params) => {
|
|
1757
|
+
requireApiKey();
|
|
1758
|
+
const { getTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1759
|
+
const trigger = await getTrigger(params.id);
|
|
1760
|
+
return { content: [{ type: "text", text: JSON.stringify(trigger, null, 2) }] };
|
|
1761
|
+
}),
|
|
1557
1762
|
);
|
|
1558
1763
|
|
|
1559
1764
|
server.tool(
|
|
1560
1765
|
"platform_create_trigger",
|
|
1561
|
-
|
|
1766
|
+
[
|
|
1767
|
+
"Create a trigger (automation).",
|
|
1768
|
+
"",
|
|
1769
|
+
"It is about one of three things: matching traces (send `filters` or `filterQuery`), a metric crossing a threshold on a graph (an Alert: send `customGraphId`, `graphAlert` and `alertType`), or a schedule (a Report: send `report`, on an email or Slack channel).",
|
|
1770
|
+
"`actionParams` is the delivery configuration for the channel named in `action` — recipients for SEND_EMAIL, a destination for SEND_SLACK_MESSAGE, a URL for SEND_WEBHOOK, a dataset and mapping for ADD_TO_DATASET, annotators for ADD_TO_ANNOTATION_QUEUE. Omitted, it is sent as {} and the server refuses a channel that cannot deliver without one.",
|
|
1771
|
+
SLACK_DELIVERY_NOTE,
|
|
1772
|
+
`filters: ${TRIGGER_FILTERS_DESCRIPTION}`,
|
|
1773
|
+
"The delivery channel cannot be changed afterwards.",
|
|
1774
|
+
].join("\n"),
|
|
1562
1775
|
{
|
|
1563
1776
|
name: z.string().describe("Trigger name"),
|
|
1564
|
-
action:
|
|
1565
|
-
|
|
1777
|
+
action: triggerActionSchema.describe("Which channel it delivers on"),
|
|
1778
|
+
actionParams: actionParamsSchema
|
|
1779
|
+
.optional()
|
|
1780
|
+
.describe("The delivery configuration the channel named in `action` reads. Defaults to {}"),
|
|
1781
|
+
filters: z.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
1782
|
+
filterQuery: z.string().optional().describe(TRIGGER_FILTER_QUERY_DESCRIPTION),
|
|
1783
|
+
customGraphId: z.string().optional().describe("Set to make this an alert on that graph"),
|
|
1784
|
+
graphAlert: graphAlertSchema.optional(),
|
|
1785
|
+
report: reportSchema.optional().describe("Set to make this a scheduled report"),
|
|
1786
|
+
templates: templatesSchema.optional(),
|
|
1787
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
1566
1788
|
message: z.string().optional().describe("Custom alert message"),
|
|
1567
|
-
alertType:
|
|
1789
|
+
alertType: alertTypeSchema.optional().describe("Alert severity. Required for a graph alert."),
|
|
1568
1790
|
},
|
|
1569
1791
|
withToolLogging("platform_create_trigger", async (params) => {
|
|
1570
1792
|
requireApiKey();
|
|
1571
|
-
const {
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
try { filters = JSON.parse(params.filters) as Record<string, unknown>; }
|
|
1575
|
-
catch { return { content: [{ type: "text", text: "Error: filters must be valid JSON" }] }; }
|
|
1576
|
-
}
|
|
1577
|
-
const trigger = await createTrigger({
|
|
1578
|
-
name: params.name,
|
|
1579
|
-
action: params.action,
|
|
1580
|
-
filters,
|
|
1581
|
-
message: params.message,
|
|
1582
|
-
alertType: params.alertType,
|
|
1583
|
-
});
|
|
1584
|
-
return { content: [{ type: "text", text: `Trigger "${trigger.name}" created (ID: ${trigger.id}, Action: ${trigger.action}).` }] };
|
|
1585
|
-
})
|
|
1793
|
+
const { handleCreateTrigger } = await import("./tools/create-trigger.ts");
|
|
1794
|
+
return handleCreateTrigger(params);
|
|
1795
|
+
}),
|
|
1586
1796
|
);
|
|
1587
1797
|
|
|
1588
1798
|
server.tool(
|
|
1589
1799
|
"platform_update_trigger",
|
|
1590
|
-
|
|
1800
|
+
[
|
|
1801
|
+
"Update a trigger (automation). Anything left out is left alone.",
|
|
1802
|
+
"",
|
|
1803
|
+
"`actionParams` replaces the delivery configuration as a whole, so send every field it should have from now on. A credential the read hid comes back as [redacted]; send that back to keep the stored value. Changing a webhook's `url` means sending its header values in the same call.",
|
|
1804
|
+
"The delivery channel, the kind of automation and an alert's graph cannot be changed.",
|
|
1805
|
+
].join("\n"),
|
|
1591
1806
|
{
|
|
1592
1807
|
id: z.string().describe("The trigger ID"),
|
|
1593
1808
|
name: z.string().optional().describe("New name"),
|
|
1594
|
-
active: z.boolean().optional().describe("
|
|
1595
|
-
|
|
1596
|
-
|
|
1809
|
+
active: z.boolean().optional().describe("Resume or pause it"),
|
|
1810
|
+
actionParams: actionParamsSchema
|
|
1811
|
+
.optional()
|
|
1812
|
+
.describe("The delivery configuration this automation should have from now on"),
|
|
1813
|
+
filters: z.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
1814
|
+
filterQuery: z
|
|
1815
|
+
.string()
|
|
1816
|
+
.nullable()
|
|
1817
|
+
.optional()
|
|
1818
|
+
.describe(`${TRIGGER_FILTER_QUERY_DESCRIPTION} null clears the saved query.`),
|
|
1819
|
+
graphAlert: graphAlertSchema
|
|
1820
|
+
.optional()
|
|
1821
|
+
.describe("Only for an automation that is already a graph alert"),
|
|
1822
|
+
report: reportSchema
|
|
1823
|
+
.optional()
|
|
1824
|
+
.describe("Only for an automation that is already a scheduled report"),
|
|
1825
|
+
templates: templatesSchema.optional(),
|
|
1826
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
1827
|
+
message: z
|
|
1828
|
+
.string()
|
|
1829
|
+
.nullable()
|
|
1830
|
+
.optional()
|
|
1831
|
+
.describe("New alert message. null removes the custom message"),
|
|
1832
|
+
alertType: alertTypeSchema
|
|
1833
|
+
.nullable()
|
|
1834
|
+
.optional()
|
|
1835
|
+
.describe("New alert severity. null clears it"),
|
|
1597
1836
|
},
|
|
1598
1837
|
withToolLogging("platform_update_trigger", async (params) => {
|
|
1599
1838
|
requireApiKey();
|
|
1600
|
-
const { updateTrigger } = await import("./langwatch-api-triggers.
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1839
|
+
const { getTrigger, updateTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1840
|
+
if (params.actionParams !== undefined) {
|
|
1841
|
+
// The channel cannot change on update, so the SAVED action decides
|
|
1842
|
+
// which shape the replacement configuration must fit.
|
|
1843
|
+
const saved = await getTrigger(params.id);
|
|
1844
|
+
const boundParams = validateActionParamsForAction({
|
|
1845
|
+
action: saved.action,
|
|
1846
|
+
actionParams: params.actionParams,
|
|
1847
|
+
});
|
|
1848
|
+
if (!boundParams.ok) {
|
|
1849
|
+
const { toolError } = await import("./tools/create-trigger.ts");
|
|
1850
|
+
return toolError(boundParams.message);
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
const { parseJsonObject, toolError } = await import("./tools/create-trigger.ts");
|
|
1854
|
+
const filters = params.filters ? parseJsonObject(params.filters) : undefined;
|
|
1855
|
+
if (params.filters && !filters) {
|
|
1856
|
+
return toolError("filters must be a JSON object");
|
|
1857
|
+
}
|
|
1858
|
+
const trigger = await updateTrigger({ ...params, filters });
|
|
1859
|
+
return {
|
|
1860
|
+
content: [
|
|
1861
|
+
{ type: "text", text: `Trigger "${trigger.name}" updated (active: ${trigger.active}).` },
|
|
1862
|
+
],
|
|
1863
|
+
};
|
|
1864
|
+
}),
|
|
1865
|
+
);
|
|
1866
|
+
|
|
1867
|
+
server.tool(
|
|
1868
|
+
"platform_test_fire_trigger",
|
|
1869
|
+
"Send a trigger's message to the destination it is configured with, to confirm it arrives. Nothing is recorded as a fire.",
|
|
1870
|
+
{
|
|
1871
|
+
id: z.string().describe("The trigger ID"),
|
|
1872
|
+
},
|
|
1873
|
+
withToolLogging("platform_test_fire_trigger", async (params) => {
|
|
1874
|
+
requireApiKey();
|
|
1875
|
+
const { testFireTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1876
|
+
const result = await testFireTrigger(params.id);
|
|
1877
|
+
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
1878
|
+
}),
|
|
1879
|
+
);
|
|
1880
|
+
|
|
1881
|
+
server.tool(
|
|
1882
|
+
"platform_list_trigger_fires",
|
|
1883
|
+
"What an automation has done, newest first. Metadata only — no message content. Answers with `nextCursor`; pass it back as `cursor` for the next page (null means there is none).",
|
|
1884
|
+
{
|
|
1885
|
+
id: z.string().describe("The trigger ID"),
|
|
1886
|
+
limit: z.number().int().positive().optional().describe("How many fires to return"),
|
|
1887
|
+
cursor: z.string().optional().describe("The `nextCursor` a previous call answered with"),
|
|
1888
|
+
},
|
|
1889
|
+
withToolLogging("platform_list_trigger_fires", async (params) => {
|
|
1890
|
+
requireApiKey();
|
|
1891
|
+
const { handleListTriggerFires } = await import("./tools/list-trigger-fires.ts");
|
|
1892
|
+
return { content: [{ type: "text", text: await handleListTriggerFires(params) }] };
|
|
1893
|
+
}),
|
|
1604
1894
|
);
|
|
1605
1895
|
|
|
1606
1896
|
server.tool(
|
|
@@ -1611,28 +1901,29 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1611
1901
|
},
|
|
1612
1902
|
withToolLogging("platform_delete_trigger", async (params) => {
|
|
1613
1903
|
requireApiKey();
|
|
1614
|
-
const { deleteTrigger } = await import("./langwatch-api-triggers.
|
|
1904
|
+
const { deleteTrigger } = await import("./langwatch-api-triggers.ts");
|
|
1615
1905
|
const result = await deleteTrigger(params.id);
|
|
1616
1906
|
return { content: [{ type: "text", text: `Trigger ${result.id} deleted.` }] };
|
|
1617
|
-
})
|
|
1907
|
+
}),
|
|
1618
1908
|
);
|
|
1909
|
+
}
|
|
1619
1910
|
|
|
1620
|
-
|
|
1621
|
-
|
|
1911
|
+
// --- Platform Monitor Tools (require API key) ---
|
|
1912
|
+
function registerMonitorTools(server: McpServer): void {
|
|
1622
1913
|
server.tool(
|
|
1623
1914
|
"platform_list_monitors",
|
|
1624
1915
|
"List all online evaluation monitors for the project.",
|
|
1625
1916
|
{},
|
|
1626
1917
|
withToolLogging("platform_list_monitors", async () => {
|
|
1627
1918
|
requireApiKey();
|
|
1628
|
-
const { listMonitors } = await import("./langwatch-api-monitors.
|
|
1919
|
+
const { listMonitors } = await import("./langwatch-api-monitors.ts");
|
|
1629
1920
|
const monitors = await listMonitors();
|
|
1630
1921
|
if (monitors.length === 0) {
|
|
1631
1922
|
return { content: [{ type: "text", text: "No monitors found." }] };
|
|
1632
1923
|
}
|
|
1633
1924
|
const lines = monitors.map(
|
|
1634
1925
|
(m) =>
|
|
1635
|
-
`• ${m.name} (id: ${m.id}, type: ${m.checkType}, ${m.enabled ? "enabled" : "disabled"}, mode: ${m.executionMode}, sample: ${Math.round(m.sample * 100)}%)
|
|
1926
|
+
`• ${m.name} (id: ${m.id}, type: ${m.checkType}, ${m.enabled ? "enabled" : "disabled"}, mode: ${m.executionMode}, sample: ${Math.round(m.sample * 100)}%)`,
|
|
1636
1927
|
);
|
|
1637
1928
|
return {
|
|
1638
1929
|
content: [
|
|
@@ -1642,7 +1933,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1642
1933
|
},
|
|
1643
1934
|
],
|
|
1644
1935
|
};
|
|
1645
|
-
})
|
|
1936
|
+
}),
|
|
1646
1937
|
);
|
|
1647
1938
|
|
|
1648
1939
|
server.tool(
|
|
@@ -1653,7 +1944,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1653
1944
|
},
|
|
1654
1945
|
withToolLogging("platform_get_monitor", async (params) => {
|
|
1655
1946
|
requireApiKey();
|
|
1656
|
-
const { getMonitor } = await import("./langwatch-api-monitors.
|
|
1947
|
+
const { getMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1657
1948
|
const m = await getMonitor(params.id);
|
|
1658
1949
|
const lines = [
|
|
1659
1950
|
`**${m.name}** (${m.id})`,
|
|
@@ -1666,7 +1957,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1666
1957
|
`- Created: ${m.createdAt}`,
|
|
1667
1958
|
].filter(Boolean);
|
|
1668
1959
|
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
1669
|
-
})
|
|
1960
|
+
}),
|
|
1670
1961
|
);
|
|
1671
1962
|
|
|
1672
1963
|
server.tool(
|
|
@@ -1674,27 +1965,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1674
1965
|
"Create a new online evaluation monitor that runs an evaluator on incoming traces.",
|
|
1675
1966
|
{
|
|
1676
1967
|
name: z.string().describe("Monitor name"),
|
|
1677
|
-
checkType: z
|
|
1678
|
-
.string()
|
|
1679
|
-
.describe("Evaluator check type (e.g. ragas/toxicity, custom/my-eval)"),
|
|
1968
|
+
checkType: z.string().describe("Evaluator check type (e.g. ragas/toxicity, custom/my-eval)"),
|
|
1680
1969
|
executionMode: z
|
|
1681
1970
|
.enum(["ON_MESSAGE", "AS_GUARDRAIL", "MANUALLY"])
|
|
1682
1971
|
.optional()
|
|
1683
1972
|
.describe("When to run: ON_MESSAGE (default), AS_GUARDRAIL, or MANUALLY"),
|
|
1684
|
-
sample: z
|
|
1685
|
-
|
|
1686
|
-
.min(0)
|
|
1687
|
-
.max(1)
|
|
1688
|
-
.optional()
|
|
1689
|
-
.describe("Sampling rate 0.0-1.0 (default: 1.0)"),
|
|
1690
|
-
evaluatorId: z
|
|
1691
|
-
.string()
|
|
1692
|
-
.optional()
|
|
1693
|
-
.describe("Link to a saved evaluator by ID"),
|
|
1973
|
+
sample: z.number().min(0).max(1).optional().describe("Sampling rate 0.0-1.0 (default: 1.0)"),
|
|
1974
|
+
evaluatorId: z.string().optional().describe("Link to a saved evaluator by ID"),
|
|
1694
1975
|
},
|
|
1695
1976
|
withToolLogging("platform_create_monitor", async (params) => {
|
|
1696
1977
|
requireApiKey();
|
|
1697
|
-
const { createMonitor } = await import("./langwatch-api-monitors.
|
|
1978
|
+
const { createMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1698
1979
|
const monitor = await createMonitor(params);
|
|
1699
1980
|
return {
|
|
1700
1981
|
content: [
|
|
@@ -1704,7 +1985,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1704
1985
|
},
|
|
1705
1986
|
],
|
|
1706
1987
|
};
|
|
1707
|
-
})
|
|
1988
|
+
}),
|
|
1708
1989
|
);
|
|
1709
1990
|
|
|
1710
1991
|
server.tool(
|
|
@@ -1722,7 +2003,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1722
2003
|
},
|
|
1723
2004
|
withToolLogging("platform_update_monitor", async (params) => {
|
|
1724
2005
|
requireApiKey();
|
|
1725
|
-
const { updateMonitor } = await import("./langwatch-api-monitors.
|
|
2006
|
+
const { updateMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1726
2007
|
const monitor = await updateMonitor(params);
|
|
1727
2008
|
return {
|
|
1728
2009
|
content: [
|
|
@@ -1732,7 +2013,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1732
2013
|
},
|
|
1733
2014
|
],
|
|
1734
2015
|
};
|
|
1735
|
-
})
|
|
2016
|
+
}),
|
|
1736
2017
|
);
|
|
1737
2018
|
|
|
1738
2019
|
server.tool(
|
|
@@ -1743,32 +2024,29 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1743
2024
|
},
|
|
1744
2025
|
withToolLogging("platform_delete_monitor", async (params) => {
|
|
1745
2026
|
requireApiKey();
|
|
1746
|
-
const { deleteMonitor } = await import("./langwatch-api-monitors.
|
|
2027
|
+
const { deleteMonitor } = await import("./langwatch-api-monitors.ts");
|
|
1747
2028
|
const result = await deleteMonitor(params.id);
|
|
1748
2029
|
return {
|
|
1749
|
-
content: [
|
|
1750
|
-
{ type: "text", text: `Monitor ${result.id} deleted.` },
|
|
1751
|
-
],
|
|
2030
|
+
content: [{ type: "text", text: `Monitor ${result.id} deleted.` }],
|
|
1752
2031
|
};
|
|
1753
|
-
})
|
|
2032
|
+
}),
|
|
1754
2033
|
);
|
|
2034
|
+
}
|
|
1755
2035
|
|
|
1756
|
-
|
|
1757
|
-
|
|
2036
|
+
// --- Platform Secret Tools (require API key) ---
|
|
2037
|
+
function registerSecretTools(server: McpServer): void {
|
|
1758
2038
|
server.tool(
|
|
1759
2039
|
"platform_list_secrets",
|
|
1760
2040
|
"List all project secrets (values are never returned, only metadata).",
|
|
1761
2041
|
{},
|
|
1762
2042
|
withToolLogging("platform_list_secrets", async () => {
|
|
1763
2043
|
requireApiKey();
|
|
1764
|
-
const { listSecrets } = await import("./langwatch-api-secrets.
|
|
2044
|
+
const { listSecrets } = await import("./langwatch-api-secrets.ts");
|
|
1765
2045
|
const secrets = await listSecrets();
|
|
1766
2046
|
if (secrets.length === 0) {
|
|
1767
2047
|
return { content: [{ type: "text", text: "No secrets found." }] };
|
|
1768
2048
|
}
|
|
1769
|
-
const lines = secrets.map(
|
|
1770
|
-
(s) => `• ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`
|
|
1771
|
-
);
|
|
2049
|
+
const lines = secrets.map((s) => `• ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`);
|
|
1772
2050
|
return {
|
|
1773
2051
|
content: [
|
|
1774
2052
|
{
|
|
@@ -1777,7 +2055,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1777
2055
|
},
|
|
1778
2056
|
],
|
|
1779
2057
|
};
|
|
1780
|
-
})
|
|
2058
|
+
}),
|
|
1781
2059
|
);
|
|
1782
2060
|
|
|
1783
2061
|
server.tool(
|
|
@@ -1789,7 +2067,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1789
2067
|
},
|
|
1790
2068
|
withToolLogging("platform_create_secret", async (params) => {
|
|
1791
2069
|
requireApiKey();
|
|
1792
|
-
const { createSecret } = await import("./langwatch-api-secrets.
|
|
2070
|
+
const { createSecret } = await import("./langwatch-api-secrets.ts");
|
|
1793
2071
|
const secret = await createSecret({ name: params.name, value: params.value });
|
|
1794
2072
|
return {
|
|
1795
2073
|
content: [
|
|
@@ -1799,7 +2077,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1799
2077
|
},
|
|
1800
2078
|
],
|
|
1801
2079
|
};
|
|
1802
|
-
})
|
|
2080
|
+
}),
|
|
1803
2081
|
);
|
|
1804
2082
|
|
|
1805
2083
|
server.tool(
|
|
@@ -1811,7 +2089,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1811
2089
|
},
|
|
1812
2090
|
withToolLogging("platform_update_secret", async (params) => {
|
|
1813
2091
|
requireApiKey();
|
|
1814
|
-
const { updateSecret } = await import("./langwatch-api-secrets.
|
|
2092
|
+
const { updateSecret } = await import("./langwatch-api-secrets.ts");
|
|
1815
2093
|
const secret = await updateSecret({ id: params.id, value: params.value });
|
|
1816
2094
|
return {
|
|
1817
2095
|
content: [
|
|
@@ -1821,7 +2099,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1821
2099
|
},
|
|
1822
2100
|
],
|
|
1823
2101
|
};
|
|
1824
|
-
})
|
|
2102
|
+
}),
|
|
1825
2103
|
);
|
|
1826
2104
|
|
|
1827
2105
|
server.tool(
|
|
@@ -1832,26 +2110,36 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1832
2110
|
},
|
|
1833
2111
|
withToolLogging("platform_delete_secret", async (params) => {
|
|
1834
2112
|
requireApiKey();
|
|
1835
|
-
const { deleteSecret } = await import("./langwatch-api-secrets.
|
|
2113
|
+
const { deleteSecret } = await import("./langwatch-api-secrets.ts");
|
|
1836
2114
|
const result = await deleteSecret(params.id);
|
|
1837
2115
|
return { content: [{ type: "text", text: `Secret ${result.id} deleted.` }] };
|
|
1838
|
-
})
|
|
2116
|
+
}),
|
|
1839
2117
|
);
|
|
2118
|
+
}
|
|
1840
2119
|
|
|
1841
|
-
|
|
1842
|
-
|
|
2120
|
+
// --- Platform Experiment Execution Tools (require API key) ---
|
|
2121
|
+
function registerExperimentTools(server: McpServer): void {
|
|
1843
2122
|
server.tool(
|
|
1844
2123
|
"platform_run_experiment",
|
|
1845
2124
|
"Start an experiment run by slug. Returns a run ID for polling status.",
|
|
1846
2125
|
{
|
|
1847
2126
|
slug: z.string().describe("The experiment slug to run"),
|
|
2127
|
+
concurrency: z
|
|
2128
|
+
.number()
|
|
2129
|
+
.int()
|
|
2130
|
+
.min(1)
|
|
2131
|
+
.max(24)
|
|
2132
|
+
.optional()
|
|
2133
|
+
.describe(
|
|
2134
|
+
"Cells (one row against one target) to run at once, for this run only (default: the server's setting). Use 1 to run them one at a time, in order.",
|
|
2135
|
+
),
|
|
1848
2136
|
},
|
|
1849
2137
|
withToolLogging("platform_run_experiment", async (params) => {
|
|
1850
2138
|
requireApiKey();
|
|
1851
2139
|
return {
|
|
1852
2140
|
content: [{ type: "text", text: await handleRunExperiment(params) }],
|
|
1853
2141
|
};
|
|
1854
|
-
})
|
|
2142
|
+
}),
|
|
1855
2143
|
);
|
|
1856
2144
|
|
|
1857
2145
|
server.tool(
|
|
@@ -1871,7 +2159,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1871
2159
|
return {
|
|
1872
2160
|
content: [{ type: "text", text: await handleExperimentStatus(params) }],
|
|
1873
2161
|
};
|
|
1874
|
-
})
|
|
2162
|
+
}),
|
|
1875
2163
|
);
|
|
1876
2164
|
|
|
1877
2165
|
server.tool(
|
|
@@ -1884,16 +2172,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1884
2172
|
.positive()
|
|
1885
2173
|
.max(100)
|
|
1886
2174
|
.optional()
|
|
1887
|
-
.describe(
|
|
1888
|
-
"Maximum number of experiments to include (default 25, hard-capped at 100)",
|
|
1889
|
-
),
|
|
2175
|
+
.describe("Maximum number of experiments to include (default 25, hard-capped at 100)"),
|
|
1890
2176
|
},
|
|
1891
2177
|
withToolLogging("platform_experiment_list", async (params) => {
|
|
1892
2178
|
requireApiKey();
|
|
1893
2179
|
return {
|
|
1894
2180
|
content: [{ type: "text", text: await handleExperimentList(params) }],
|
|
1895
2181
|
};
|
|
1896
|
-
})
|
|
2182
|
+
}),
|
|
1897
2183
|
);
|
|
1898
2184
|
|
|
1899
2185
|
server.tool(
|
|
@@ -1902,34 +2188,28 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1902
2188
|
{
|
|
1903
2189
|
experimentSlug: z
|
|
1904
2190
|
.string()
|
|
1905
|
-
.describe(
|
|
1906
|
-
"Experiment slug from platform_experiment_list (e.g. 'checkout-flow')",
|
|
1907
|
-
),
|
|
2191
|
+
.describe("Experiment slug from platform_experiment_list (e.g. 'checkout-flow')"),
|
|
1908
2192
|
limit: z
|
|
1909
2193
|
.number()
|
|
1910
2194
|
.int()
|
|
1911
2195
|
.positive()
|
|
1912
2196
|
.max(100)
|
|
1913
2197
|
.optional()
|
|
1914
|
-
.describe(
|
|
1915
|
-
"Maximum number of runs to include (default 25, hard-capped at 100)",
|
|
1916
|
-
),
|
|
2198
|
+
.describe("Maximum number of runs to include (default 25, hard-capped at 100)"),
|
|
1917
2199
|
},
|
|
1918
2200
|
withToolLogging("platform_experiment_list_runs", async (params) => {
|
|
1919
2201
|
requireApiKey();
|
|
1920
2202
|
return {
|
|
1921
2203
|
content: [{ type: "text", text: await handleExperimentListRuns(params) }],
|
|
1922
2204
|
};
|
|
1923
|
-
})
|
|
2205
|
+
}),
|
|
1924
2206
|
);
|
|
1925
2207
|
|
|
1926
2208
|
server.tool(
|
|
1927
2209
|
"platform_experiment_results",
|
|
1928
2210
|
"Fetch per-row results for an experiment run so you can debug evaluator scores and missed rows. Serves partial results for runs that are still running or were interrupted (it does not require a completed run), with the run status noted in the output. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window — narrow with `filter: 'failed'` or `evaluator` to see what matters.",
|
|
1929
2211
|
{
|
|
1930
|
-
runId: z
|
|
1931
|
-
.string()
|
|
1932
|
-
.describe("The run ID returned from platform_run_experiment"),
|
|
2212
|
+
runId: z.string().describe("The run ID returned from platform_run_experiment"),
|
|
1933
2213
|
experimentSlug: z
|
|
1934
2214
|
.string()
|
|
1935
2215
|
.optional()
|
|
@@ -1959,16 +2239,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1959
2239
|
withToolLogging("platform_experiment_results", async (params) => {
|
|
1960
2240
|
requireApiKey();
|
|
1961
2241
|
return {
|
|
1962
|
-
content: [
|
|
1963
|
-
{ type: "text", text: await handleExperimentResults(params) },
|
|
1964
|
-
],
|
|
2242
|
+
content: [{ type: "text", text: await handleExperimentResults(params) }],
|
|
1965
2243
|
};
|
|
1966
|
-
})
|
|
2244
|
+
}),
|
|
1967
2245
|
);
|
|
2246
|
+
}
|
|
1968
2247
|
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
2248
|
+
// --- Platform Dataset Tools (require API key) ---
|
|
2249
|
+
// These tools manage datasets on the LangWatch platform via API.
|
|
2250
|
+
function registerDatasetTools(server: McpServer): void {
|
|
1972
2251
|
server.tool(
|
|
1973
2252
|
"platform_list_datasets",
|
|
1974
2253
|
"List all datasets on the LangWatch platform with their names, slugs, columns, and record counts. Returns AI-readable digest by default.",
|
|
@@ -1976,20 +2255,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1976
2255
|
format: z
|
|
1977
2256
|
.enum(["digest", "json"])
|
|
1978
2257
|
.optional()
|
|
1979
|
-
.describe(
|
|
1980
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
1981
|
-
),
|
|
2258
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
1982
2259
|
},
|
|
1983
2260
|
async (params) => {
|
|
1984
|
-
const { requireApiKey } = await import("./config.
|
|
2261
|
+
const { requireApiKey } = await import("./config.ts");
|
|
1985
2262
|
requireApiKey();
|
|
1986
|
-
const { handleListDatasets } = await import(
|
|
1987
|
-
"./tools/list-datasets.js"
|
|
1988
|
-
);
|
|
2263
|
+
const { handleListDatasets } = await import("./tools/list-datasets.ts");
|
|
1989
2264
|
return {
|
|
1990
2265
|
content: [{ type: "text", text: await handleListDatasets(params) }],
|
|
1991
2266
|
};
|
|
1992
|
-
}
|
|
2267
|
+
},
|
|
1993
2268
|
);
|
|
1994
2269
|
|
|
1995
2270
|
server.tool(
|
|
@@ -2000,18 +2275,16 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2000
2275
|
format: z
|
|
2001
2276
|
.enum(["digest", "json"])
|
|
2002
2277
|
.optional()
|
|
2003
|
-
.describe(
|
|
2004
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
2005
|
-
),
|
|
2278
|
+
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
2006
2279
|
},
|
|
2007
2280
|
async (params) => {
|
|
2008
|
-
const { requireApiKey } = await import("./config.
|
|
2281
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2009
2282
|
requireApiKey();
|
|
2010
|
-
const { handleGetDataset } = await import("./tools/get-dataset.
|
|
2283
|
+
const { handleGetDataset } = await import("./tools/get-dataset.ts");
|
|
2011
2284
|
return {
|
|
2012
2285
|
content: [{ type: "text", text: await handleGetDataset(params) }],
|
|
2013
2286
|
};
|
|
2014
|
-
}
|
|
2287
|
+
},
|
|
2015
2288
|
);
|
|
2016
2289
|
|
|
2017
2290
|
server.tool(
|
|
@@ -2019,15 +2292,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2019
2292
|
"Create a new dataset on the LangWatch platform.",
|
|
2020
2293
|
createDatasetSchema.shape,
|
|
2021
2294
|
async (params) => {
|
|
2022
|
-
const { requireApiKey } = await import("./config.
|
|
2295
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2023
2296
|
requireApiKey();
|
|
2024
|
-
const { handleCreateDataset } = await import(
|
|
2025
|
-
"./tools/create-dataset.js"
|
|
2026
|
-
);
|
|
2297
|
+
const { handleCreateDataset } = await import("./tools/create-dataset.ts");
|
|
2027
2298
|
return {
|
|
2028
2299
|
content: [{ type: "text", text: await handleCreateDataset(params) }],
|
|
2029
2300
|
};
|
|
2030
|
-
}
|
|
2301
|
+
},
|
|
2031
2302
|
);
|
|
2032
2303
|
|
|
2033
2304
|
server.tool(
|
|
@@ -2042,15 +2313,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2042
2313
|
.describe("Updated column definitions"),
|
|
2043
2314
|
},
|
|
2044
2315
|
async (params) => {
|
|
2045
|
-
const { requireApiKey } = await import("./config.
|
|
2316
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2046
2317
|
requireApiKey();
|
|
2047
|
-
const { handleUpdateDataset } = await import(
|
|
2048
|
-
"./tools/update-dataset.js"
|
|
2049
|
-
);
|
|
2318
|
+
const { handleUpdateDataset } = await import("./tools/update-dataset.ts");
|
|
2050
2319
|
return {
|
|
2051
2320
|
content: [{ type: "text", text: await handleUpdateDataset(params) }],
|
|
2052
2321
|
};
|
|
2053
|
-
}
|
|
2322
|
+
},
|
|
2054
2323
|
);
|
|
2055
2324
|
|
|
2056
2325
|
server.tool(
|
|
@@ -2060,30 +2329,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2060
2329
|
slugOrId: z.string().describe("The dataset slug or ID to delete"),
|
|
2061
2330
|
},
|
|
2062
2331
|
async (params) => {
|
|
2063
|
-
const { requireApiKey } = await import("./config.
|
|
2332
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2064
2333
|
requireApiKey();
|
|
2065
|
-
const { handleDeleteDataset } = await import(
|
|
2066
|
-
"./tools/delete-dataset.js"
|
|
2067
|
-
);
|
|
2334
|
+
const { handleDeleteDataset } = await import("./tools/delete-dataset.ts");
|
|
2068
2335
|
return {
|
|
2069
2336
|
content: [{ type: "text", text: await handleDeleteDataset(params) }],
|
|
2070
2337
|
};
|
|
2071
|
-
}
|
|
2338
|
+
},
|
|
2072
2339
|
);
|
|
2073
2340
|
|
|
2074
2341
|
server.tool(
|
|
2075
2342
|
"platform_list_dataset_records",
|
|
2076
2343
|
"List records in a dataset on the LangWatch platform with pagination.",
|
|
2077
2344
|
{
|
|
2078
|
-
slugOrId: z
|
|
2079
|
-
|
|
2080
|
-
.describe("The dataset slug or ID to list records from"),
|
|
2081
|
-
page: z
|
|
2082
|
-
.number()
|
|
2083
|
-
.int()
|
|
2084
|
-
.min(1)
|
|
2085
|
-
.optional()
|
|
2086
|
-
.describe("Page number (default: 1)"),
|
|
2345
|
+
slugOrId: z.string().describe("The dataset slug or ID to list records from"),
|
|
2346
|
+
page: z.number().int().min(1).optional().describe("Page number (default: 1)"),
|
|
2087
2347
|
limit: z
|
|
2088
2348
|
.number()
|
|
2089
2349
|
.int()
|
|
@@ -2097,26 +2357,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2097
2357
|
.describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"),
|
|
2098
2358
|
},
|
|
2099
2359
|
async (params) => {
|
|
2100
|
-
const { requireApiKey } = await import("./config.
|
|
2360
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2101
2361
|
requireApiKey();
|
|
2102
|
-
const { handleListDatasetRecords } = await import(
|
|
2103
|
-
"./tools/list-dataset-records.js"
|
|
2104
|
-
);
|
|
2362
|
+
const { handleListDatasetRecords } = await import("./tools/list-dataset-records.ts");
|
|
2105
2363
|
return {
|
|
2106
|
-
content: [
|
|
2107
|
-
{ type: "text", text: await handleListDatasetRecords(params) },
|
|
2108
|
-
],
|
|
2364
|
+
content: [{ type: "text", text: await handleListDatasetRecords(params) }],
|
|
2109
2365
|
};
|
|
2110
|
-
}
|
|
2366
|
+
},
|
|
2111
2367
|
);
|
|
2112
2368
|
|
|
2113
2369
|
server.tool(
|
|
2114
2370
|
"platform_create_dataset_records",
|
|
2115
2371
|
"Add records to a dataset on the LangWatch platform in batch (max 1000 per call).",
|
|
2116
2372
|
{
|
|
2117
|
-
slugOrId: z
|
|
2118
|
-
.string()
|
|
2119
|
-
.describe("The dataset slug or ID to add records to"),
|
|
2373
|
+
slugOrId: z.string().describe("The dataset slug or ID to add records to"),
|
|
2120
2374
|
entries: z
|
|
2121
2375
|
.array(z.record(z.string(), z.unknown()))
|
|
2122
2376
|
.min(1)
|
|
@@ -2124,90 +2378,75 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2124
2378
|
.describe("Array of record entries to create (key-value objects matching dataset columns)"),
|
|
2125
2379
|
},
|
|
2126
2380
|
async (params) => {
|
|
2127
|
-
const { requireApiKey } = await import("./config.
|
|
2381
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2128
2382
|
requireApiKey();
|
|
2129
|
-
const { handleCreateDatasetRecords } = await import(
|
|
2130
|
-
"./tools/create-dataset-records.js"
|
|
2131
|
-
);
|
|
2383
|
+
const { handleCreateDatasetRecords } = await import("./tools/create-dataset-records.ts");
|
|
2132
2384
|
return {
|
|
2133
|
-
content: [
|
|
2134
|
-
{ type: "text", text: await handleCreateDatasetRecords(params) },
|
|
2135
|
-
],
|
|
2385
|
+
content: [{ type: "text", text: await handleCreateDatasetRecords(params) }],
|
|
2136
2386
|
};
|
|
2137
|
-
}
|
|
2387
|
+
},
|
|
2138
2388
|
);
|
|
2139
2389
|
|
|
2140
2390
|
server.tool(
|
|
2141
2391
|
"platform_update_dataset_record",
|
|
2142
2392
|
"Update a single record in a dataset on the LangWatch platform.",
|
|
2143
2393
|
{
|
|
2144
|
-
slugOrId: z
|
|
2145
|
-
.string()
|
|
2146
|
-
.describe("The dataset slug or ID containing the record"),
|
|
2394
|
+
slugOrId: z.string().describe("The dataset slug or ID containing the record"),
|
|
2147
2395
|
recordId: z.string().describe("The record ID to update"),
|
|
2148
|
-
entry: z
|
|
2149
|
-
.record(z.string(), z.unknown())
|
|
2150
|
-
.describe("Updated record entry (key-value object)"),
|
|
2396
|
+
entry: z.record(z.string(), z.unknown()).describe("Updated record entry (key-value object)"),
|
|
2151
2397
|
},
|
|
2152
2398
|
async (params) => {
|
|
2153
|
-
const { requireApiKey } = await import("./config.
|
|
2399
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2154
2400
|
requireApiKey();
|
|
2155
|
-
const { handleUpdateDatasetRecord } = await import(
|
|
2156
|
-
"./tools/update-dataset-record.js"
|
|
2157
|
-
);
|
|
2401
|
+
const { handleUpdateDatasetRecord } = await import("./tools/update-dataset-record.ts");
|
|
2158
2402
|
return {
|
|
2159
|
-
content: [
|
|
2160
|
-
{ type: "text", text: await handleUpdateDatasetRecord(params) },
|
|
2161
|
-
],
|
|
2403
|
+
content: [{ type: "text", text: await handleUpdateDatasetRecord(params) }],
|
|
2162
2404
|
};
|
|
2163
|
-
}
|
|
2405
|
+
},
|
|
2164
2406
|
);
|
|
2165
2407
|
|
|
2166
2408
|
server.tool(
|
|
2167
2409
|
"platform_delete_dataset_records",
|
|
2168
2410
|
"Delete records from a dataset on the LangWatch platform by their IDs.",
|
|
2169
2411
|
{
|
|
2170
|
-
slugOrId: z
|
|
2171
|
-
|
|
2172
|
-
.describe("The dataset slug or ID containing the records"),
|
|
2173
|
-
recordIds: z
|
|
2174
|
-
.array(z.string())
|
|
2175
|
-
.min(1)
|
|
2176
|
-
.max(1000)
|
|
2177
|
-
.describe("Array of record IDs to delete"),
|
|
2412
|
+
slugOrId: z.string().describe("The dataset slug or ID containing the records"),
|
|
2413
|
+
recordIds: z.array(z.string()).min(1).max(1000).describe("Array of record IDs to delete"),
|
|
2178
2414
|
},
|
|
2179
2415
|
async (params) => {
|
|
2180
|
-
const { requireApiKey } = await import("./config.
|
|
2416
|
+
const { requireApiKey } = await import("./config.ts");
|
|
2181
2417
|
requireApiKey();
|
|
2182
|
-
const { handleDeleteDatasetRecords } = await import(
|
|
2183
|
-
"./tools/delete-dataset-records.js"
|
|
2184
|
-
);
|
|
2418
|
+
const { handleDeleteDatasetRecords } = await import("./tools/delete-dataset-records.ts");
|
|
2185
2419
|
return {
|
|
2186
|
-
content: [
|
|
2187
|
-
{ type: "text", text: await handleDeleteDatasetRecords(params) },
|
|
2188
|
-
],
|
|
2420
|
+
content: [{ type: "text", text: await handleDeleteDatasetRecords(params) }],
|
|
2189
2421
|
};
|
|
2190
|
-
}
|
|
2422
|
+
},
|
|
2191
2423
|
);
|
|
2424
|
+
}
|
|
2192
2425
|
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2426
|
+
// --- Platform Project Tools ---
|
|
2427
|
+
// These tools manage projects on the LangWatch platform via the REST API.
|
|
2428
|
+
// They require an API key with org-level permissions.
|
|
2429
|
+
function registerProjectTools(server: McpServer): void {
|
|
2197
2430
|
server.tool(
|
|
2198
2431
|
"platform_list_projects",
|
|
2199
2432
|
"List all projects in your LangWatch organization. Requires an org-level API key.",
|
|
2200
2433
|
{
|
|
2201
2434
|
page: z.number().int().positive().optional().describe("Page number (default: 1)"),
|
|
2202
|
-
limit: z
|
|
2435
|
+
limit: z
|
|
2436
|
+
.number()
|
|
2437
|
+
.int()
|
|
2438
|
+
.positive()
|
|
2439
|
+
.max(1000)
|
|
2440
|
+
.optional()
|
|
2441
|
+
.describe("Results per page (default: 100)"),
|
|
2203
2442
|
},
|
|
2204
2443
|
withToolLogging("platform_list_projects", async (params) => {
|
|
2205
2444
|
requireApiKey();
|
|
2206
|
-
const { handleListProjects } = await import("./tools/list-projects.
|
|
2445
|
+
const { handleListProjects } = await import("./tools/list-projects.ts");
|
|
2207
2446
|
return {
|
|
2208
2447
|
content: [{ type: "text", text: await handleListProjects(params) }],
|
|
2209
2448
|
};
|
|
2210
|
-
})
|
|
2449
|
+
}),
|
|
2211
2450
|
);
|
|
2212
2451
|
|
|
2213
2452
|
server.tool(
|
|
@@ -2218,11 +2457,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
2218
2457
|
},
|
|
2219
2458
|
withToolLogging("platform_get_project", async (params) => {
|
|
2220
2459
|
requireApiKey();
|
|
2221
|
-
const { handleGetProject } = await import("./tools/get-project.
|
|
2460
|
+
const { handleGetProject } = await import("./tools/get-project.ts");
|
|
2222
2461
|
return {
|
|
2223
2462
|
content: [{ type: "text", text: await handleGetProject(params) }],
|
|
2224
2463
|
};
|
|
2225
|
-
})
|
|
2464
|
+
}),
|
|
2226
2465
|
);
|
|
2227
2466
|
|
|
2228
2467
|
server.tool(
|
|
@@ -2239,11 +2478,11 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2239
2478
|
},
|
|
2240
2479
|
withToolLogging("platform_create_project", async (params) => {
|
|
2241
2480
|
requireApiKey();
|
|
2242
|
-
const { handleCreateProject } = await import("./tools/create-project.
|
|
2481
|
+
const { handleCreateProject } = await import("./tools/create-project.ts");
|
|
2243
2482
|
return {
|
|
2244
2483
|
content: [{ type: "text", text: await handleCreateProject(params) }],
|
|
2245
2484
|
};
|
|
2246
|
-
})
|
|
2485
|
+
}),
|
|
2247
2486
|
);
|
|
2248
2487
|
|
|
2249
2488
|
server.tool(
|
|
@@ -2254,15 +2493,18 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2254
2493
|
name: z.string().optional().describe("New project name"),
|
|
2255
2494
|
language: z.string().optional().describe("New programming language"),
|
|
2256
2495
|
framework: z.string().optional().describe("New framework"),
|
|
2257
|
-
piiRedactionLevel: z
|
|
2496
|
+
piiRedactionLevel: z
|
|
2497
|
+
.enum(["STRICT", "ESSENTIAL", "DISABLED"])
|
|
2498
|
+
.optional()
|
|
2499
|
+
.describe("PII redaction level"),
|
|
2258
2500
|
},
|
|
2259
2501
|
withToolLogging("platform_update_project", async (params) => {
|
|
2260
2502
|
requireApiKey();
|
|
2261
|
-
const { handleUpdateProject } = await import("./tools/update-project.
|
|
2503
|
+
const { handleUpdateProject } = await import("./tools/update-project.ts");
|
|
2262
2504
|
return {
|
|
2263
2505
|
content: [{ type: "text", text: await handleUpdateProject(params) }],
|
|
2264
2506
|
};
|
|
2265
|
-
})
|
|
2507
|
+
}),
|
|
2266
2508
|
);
|
|
2267
2509
|
|
|
2268
2510
|
server.tool(
|
|
@@ -2273,27 +2515,28 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2273
2515
|
},
|
|
2274
2516
|
withToolLogging("platform_archive_project", async (params) => {
|
|
2275
2517
|
requireApiKey();
|
|
2276
|
-
const { handleArchiveProject } = await import("./tools/archive-project.
|
|
2518
|
+
const { handleArchiveProject } = await import("./tools/archive-project.ts");
|
|
2277
2519
|
return {
|
|
2278
2520
|
content: [{ type: "text", text: await handleArchiveProject(params) }],
|
|
2279
2521
|
};
|
|
2280
|
-
})
|
|
2522
|
+
}),
|
|
2281
2523
|
);
|
|
2524
|
+
}
|
|
2282
2525
|
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2526
|
+
// --- Platform API Key Tools (require org-level API key) ---
|
|
2527
|
+
// These tools manage API keys on the LangWatch platform via the REST API.
|
|
2528
|
+
function registerApiKeyTools(server: McpServer): void {
|
|
2286
2529
|
server.tool(
|
|
2287
2530
|
"platform_list_api_keys",
|
|
2288
2531
|
"List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
|
|
2289
2532
|
{},
|
|
2290
2533
|
withToolLogging("platform_list_api_keys", async () => {
|
|
2291
2534
|
requireApiKey();
|
|
2292
|
-
const { handleListApiKeys } = await import("./tools/list-api-keys.
|
|
2535
|
+
const { handleListApiKeys } = await import("./tools/list-api-keys.ts");
|
|
2293
2536
|
return {
|
|
2294
2537
|
content: [{ type: "text", text: await handleListApiKeys() }],
|
|
2295
2538
|
};
|
|
2296
|
-
})
|
|
2539
|
+
}),
|
|
2297
2540
|
);
|
|
2298
2541
|
|
|
2299
2542
|
server.tool(
|
|
@@ -2305,24 +2548,34 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2305
2548
|
|
|
2306
2549
|
The token is returned once and cannot be retrieved again.`,
|
|
2307
2550
|
{
|
|
2308
|
-
keyType: z
|
|
2551
|
+
keyType: z
|
|
2552
|
+
.enum(["personal", "service"])
|
|
2553
|
+
.describe("Key type: 'personal' (user-tied) or 'service' (headless)"),
|
|
2309
2554
|
name: z.string().describe("Key name (max 100 chars)"),
|
|
2310
2555
|
description: z.string().optional().describe("Key description (max 500 chars)"),
|
|
2311
2556
|
expiresAt: z.string().optional().describe("Expiration date (ISO string)"),
|
|
2312
|
-
bindings: z
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2557
|
+
bindings: z
|
|
2558
|
+
.array(
|
|
2559
|
+
z.object({
|
|
2560
|
+
role: z.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
|
|
2561
|
+
scopeType: z.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
|
|
2562
|
+
scopeId: z.string().describe("Scope ID (org, team, or project ID)"),
|
|
2563
|
+
}),
|
|
2564
|
+
)
|
|
2565
|
+
.optional()
|
|
2566
|
+
.describe("Role bindings (required for personal keys)"),
|
|
2567
|
+
projectIds: z
|
|
2568
|
+
.array(z.string())
|
|
2569
|
+
.optional()
|
|
2570
|
+
.describe("Project IDs for service keys (creates ADMIN bindings per project)"),
|
|
2318
2571
|
},
|
|
2319
2572
|
withToolLogging("platform_create_api_key", async (params) => {
|
|
2320
2573
|
requireApiKey();
|
|
2321
|
-
const { handleCreateApiKey } = await import("./tools/create-api-key.
|
|
2574
|
+
const { handleCreateApiKey } = await import("./tools/create-api-key.ts");
|
|
2322
2575
|
return {
|
|
2323
2576
|
content: [{ type: "text", text: await handleCreateApiKey(params) }],
|
|
2324
2577
|
};
|
|
2325
|
-
})
|
|
2578
|
+
}),
|
|
2326
2579
|
);
|
|
2327
2580
|
|
|
2328
2581
|
server.tool(
|
|
@@ -2333,10 +2586,10 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2333
2586
|
},
|
|
2334
2587
|
withToolLogging("platform_revoke_api_key", async (params) => {
|
|
2335
2588
|
requireApiKey();
|
|
2336
|
-
const { handleRevokeApiKey } = await import("./tools/revoke-api-key.
|
|
2589
|
+
const { handleRevokeApiKey } = await import("./tools/revoke-api-key.ts");
|
|
2337
2590
|
return {
|
|
2338
2591
|
content: [{ type: "text", text: await handleRevokeApiKey(params) }],
|
|
2339
2592
|
};
|
|
2340
|
-
})
|
|
2593
|
+
}),
|
|
2341
2594
|
);
|
|
2342
2595
|
}
|