@langwatch/mcp-server 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +218 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-55UI6H22.js +44 -0
- package/dist/add-dashboard-widget-MJHIC3CC.js +45 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-K5UD7A6X.js} +4 -4
- package/dist/archive-project-L4PAKT34.js +19 -0
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-5HSQG2HX.js} +4 -4
- package/dist/archive-run-plan-AOKNYQIO.js +15 -0
- package/dist/archive-scenario-2CWJW2VD.js +19 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-5IJQESHH.js} +4 -4
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-OPB7NFI7.js} +4 -4
- package/dist/archive-test-suite-ZC3TN6VH.js +15 -0
- package/dist/assign-prompt-tag-6DDE6ZZ5.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-D7O23MJM.js} +8 -4
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/{chunk-EYXMLE5X.js → chunk-4RX3G62A.js} +56 -47
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7CZQDZ2K.js +319 -0
- package/dist/{chunk-RVAE2MEA.js → chunk-7GETGE6B.js} +7 -7
- package/dist/{chunk-OJLYYJWI.js → chunk-AASW2TO2.js} +7 -10
- package/dist/{chunk-QAMP3CGR.js → chunk-B4RFWIUP.js} +4 -4
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-G4ZAZTZW.js → chunk-D2HS73VI.js} +7 -10
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-WT4PO4A3.js → chunk-DULFP3MK.js} +10 -7
- package/dist/chunk-EYPJBL7P.js +516 -0
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/chunk-GA5JUSPT.js +64 -0
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/chunk-HLQJXUR6.js +67 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-KILVK3FS.js} +4 -4
- package/dist/chunk-MOCU5XWF.js +79 -0
- package/dist/chunk-O2WGVCED.js +41 -0
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-UFOYFYI7.js → chunk-QXLMHLRC.js} +6 -9
- package/dist/{chunk-QRO35SOM.js → chunk-R5YJIYPF.js} +981 -747
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/{chunk-SR6QPRIE.js → chunk-RFC2SBEX.js} +7 -13
- package/dist/chunk-SHSVN3SE.js +21 -0
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/{chunk-56OGTS24.js → chunk-VOUGDY2O.js} +4 -11
- package/dist/chunk-WCKZDOGF.js +31 -0
- package/dist/chunk-XQYH52SI.js +129 -0
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/{chunk-6DJDEXET.js → chunk-YCIUEZVK.js} +6 -12
- package/dist/config-2WQGMAEN.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-6JNLMYQD.js} +4 -4
- package/dist/create-agent-IRZOU4R5.js +23 -0
- package/dist/create-analytics-report-N4L4V5QL.js +19 -0
- package/dist/create-analytics-report-QEHXG7FU.js +18 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-CCEGIZ67.js} +10 -5
- package/dist/create-annotation-DTHCMWHB.js +29 -0
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-JKM2DSZO.js} +5 -7
- package/dist/create-api-key-JPEUIXGM.js +24 -0
- package/dist/create-dataset-DJNXYJ5B.js +22 -0
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-WON4SOTC.js} +6 -6
- package/dist/create-dataset-records-TMEOTLTO.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-UJBW3C7C.js} +5 -7
- package/dist/create-evaluator-N5AEHI4K.js +27 -0
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-XYNY2YFY.js} +4 -4
- package/dist/create-mcp-server.cjs +9712 -0
- package/dist/create-mcp-server.js +2757 -10
- package/dist/{create-project-ZQ7TFIO2.js → create-project-EY4XZ7TH.js} +4 -4
- package/dist/create-project-YL345T5L.js +44 -0
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-JZQNQ7I3.js} +5 -7
- package/dist/create-prompt-ZUMQHHX7.js +37 -0
- package/dist/create-prompt-tag-CJ5FZ436.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-JFP2Q3PO.js} +3 -3
- package/dist/create-scenario-CQU2SSFB.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-S6OWWKJO.js} +5 -5
- package/dist/create-test-suite-OLGCAGJA.js +36 -0
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-Y7P2RI4L.js} +6 -6
- package/dist/create-trigger-4CNBTISJ.js +48 -0
- package/dist/create-trigger-RQ653WYF.js +49 -0
- package/dist/dashboards-BA7M6CMO.js +90 -0
- package/dist/dashboards-U663GUIB.js +91 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-J2Q4DRZ2.js} +4 -4
- package/dist/delete-agent-ZAU4LTXB.js +18 -0
- package/dist/delete-analytics-report-QK6X7P4S.js +19 -0
- package/dist/delete-analytics-report-YUYLDHLL.js +18 -0
- package/dist/delete-annotation-7X4NYXXM.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-KJOCDLV3.js} +4 -4
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-NXP7ZPPV.js} +5 -7
- package/dist/delete-dataset-RUTZBBVD.js +18 -0
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-4Z7DZCDW.js} +5 -7
- package/dist/delete-dataset-records-75UAR5IS.js +19 -0
- package/dist/delete-evaluator-E3CUNGTQ.js +16 -0
- package/dist/delete-evaluator-RI4GALGX.js +15 -0
- package/dist/delete-prompt-tag-LE3WJPKS.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-S4KBZTNG.js} +3 -3
- package/dist/delete-workflow-4CZFIQCQ.js +17 -0
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-PWQPDW6H.js} +4 -4
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/discover-schema-7WHIIATI.js +442 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-J2ANOBKO.js} +181 -186
- package/dist/get-agent-5T3CXQOR.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-ZKYQJUQ7.js} +15 -11
- package/dist/get-analytics-4MPEK4KV.js +79 -0
- package/dist/get-analytics-ZLI3A4F5.js +80 -0
- package/dist/get-analytics-report-JCKYMA2S.js +29 -0
- package/dist/get-analytics-report-SKIP2JWH.js +30 -0
- package/dist/get-dataset-APGZBYMH.js +64 -0
- package/dist/get-dataset-LWXDVMVE.js +63 -0
- package/dist/get-evaluator-3A7BJBAB.js +47 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-IWNJINRY.js} +4 -4
- package/dist/get-project-EQVB4LEL.js +26 -0
- package/dist/{get-project-QSQEAB33.js → get-project-RK6AJTEP.js} +4 -4
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-TLG2T2DS.js} +53 -58
- package/dist/get-prompt-ZER557X3.js +94 -0
- package/dist/get-run-plan-5X7DDJRN.js +58 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-PEHRO5M3.js} +7 -9
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-375W5D6I.js} +5 -5
- package/dist/get-scenario-UQX45ZZO.js +37 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-2NBXJADG.js} +26 -22
- package/dist/get-simulation-run-QZTLGL7W.js +69 -0
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GVIBDG7K.js} +5 -5
- package/dist/get-test-suite-OODIR3RI.js +48 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-AGCHVPJK.js} +11 -13
- package/dist/get-trace-KJLYAZRC.js +45 -0
- package/dist/get-workflow-3TV2YYC5.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-7LZSVPP2.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-7CCN3EHJ.js} +4249 -325
- package/dist/index.js +59595 -38027
- package/dist/langwatch-api-analytics-reports-6CF6KUET.js +17 -0
- package/dist/langwatch-api-analytics-reports-RPFB5AO5.js +18 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-H4AI4TKG.js} +4 -4
- package/dist/langwatch-api-annotations-QSJNUQQS.js +17 -0
- package/dist/langwatch-api-monitors-ENHLOL3S.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-OHSGADTH.js} +11 -16
- package/dist/langwatch-api-secrets-AGW6UUMD.js +55 -0
- package/dist/langwatch-api-secrets-CLTS766Q.js +56 -0
- package/dist/langwatch-api-triggers-G5UWFNN2.js +22 -0
- package/dist/langwatch-api-triggers-QMR7KEXW.js +23 -0
- package/dist/langwatch-api-workflows-HKTBIKKY.js +15 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OMROZH2O.js} +4 -4
- package/dist/{list-agents-AMVDQPQL.js → list-agents-BYWDNAHT.js} +4 -4
- package/dist/list-agents-MG24MHZE.js +42 -0
- package/dist/list-analytics-reports-G3EPWGSQ.js +32 -0
- package/dist/list-analytics-reports-PPWIFRES.js +33 -0
- package/dist/list-annotations-QCLGEK5A.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-Y4EUBJHG.js} +10 -5
- package/dist/list-api-keys-DAVOBPFK.js +42 -0
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-JMW5WRXX.js} +10 -5
- package/dist/list-dashboards-2P3VJFRS.js +29 -0
- package/dist/list-dashboards-MW5O3MKH.js +30 -0
- package/dist/list-dataset-records-KEHT7G6Q.js +43 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-KMLLMZAZ.js} +14 -10
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-P3JK2ZHS.js} +9 -13
- package/dist/list-datasets-Z7OOE4KI.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-7RRESBOU.js} +5 -7
- package/dist/list-evaluators-VATJVOHR.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-OXDRPUCJ.js} +4 -4
- package/dist/list-model-providers-VXQJWDXU.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-HUVK7L3W.js} +5 -7
- package/dist/list-projects-X72ZXVY5.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-3I3LWFIR.js} +3 -3
- package/dist/list-prompt-tags-DRHUH2ND.js +23 -0
- package/dist/list-prompts-6DX3QECJ.js +30 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-PS423RU3.js} +4 -6
- package/dist/list-run-plans-5SXPOV3X.js +45 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-6VNHINPY.js} +5 -5
- package/dist/list-scenarios-EQWCW7Q4.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-GKNH3YND.js} +13 -11
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-KHY7KALI.js} +12 -8
- package/dist/list-simulation-runs-PUDK32C2.js +53 -0
- package/dist/list-test-suites-7H2DRILK.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-GY56LGLY.js} +4 -4
- package/dist/list-trigger-fires-7RZUP54Y.js +19 -0
- package/dist/list-trigger-fires-OZUF65LI.js +20 -0
- package/dist/list-triggers-BGUCUXTJ.js +45 -0
- package/dist/list-triggers-XC5KMX4C.js +46 -0
- package/dist/list-workflows-55NYC3PC.js +33 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-HV7FUWZA.js} +5 -7
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-2I7Z5FCY.js} +3 -3
- package/dist/rename-prompt-tag-SZYTOWDX.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-27AYY33K.js} +4 -4
- package/dist/rename-test-suite-CLIEPZDO.js +20 -0
- package/dist/report-issue-5HSL7CNB.js +924 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-N7ND5TM6.js} +107 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-O3VUZ6CG.js} +5 -5
- package/dist/rerun-run-plan-OJTVVGJI.js +21 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-BU6BKGCG.js} +4 -4
- package/dist/revoke-api-key-NZNCFOIR.js +15 -0
- package/dist/run-agent-244O4Y5M.js +75 -0
- package/dist/run-agent-HPCGZGYW.js +74 -0
- package/dist/run-plan-2ZPGGZ4H.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-KSXUYQT3.js} +7 -7
- package/dist/run-query-4LMOQ5UW.js +47 -0
- package/dist/run-query-BQUHIUC2.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-P3WLVPXC.js} +6 -6
- package/dist/run-test-suite-UEOMCON6.js +25 -0
- package/dist/run-workflow-EFZ6HG5B.js +36 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-I4IHEN54.js} +4 -4
- package/dist/search-traces-6IR7PR4L.js +170 -0
- package/dist/search-traces-BYML2NQ4.js +171 -0
- package/dist/set-model-provider-RBNGWFVI.js +33 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-SHEVKWRC.js} +4 -4
- package/dist/{update-agent-DUWNYBNA.js → update-agent-PTKTM2B5.js} +4 -4
- package/dist/update-agent-ZPXWRMS2.js +20 -0
- package/dist/update-dashboard-widget-HKCBXH7X.js +62 -0
- package/dist/update-dashboard-widget-IP2TTQ5G.js +63 -0
- package/dist/update-dataset-GHIN6JR3.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-IRVMY5PF.js} +6 -6
- package/dist/update-dataset-record-6VI7K2CL.js +19 -0
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-Z3L7ACNF.js} +4 -4
- package/dist/update-evaluator-5NLNE6VY.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-FEH7H6NS.js} +4 -4
- package/dist/update-project-75H3LDQT.js +24 -0
- package/dist/{update-project-Q26FBSYH.js → update-project-QPR3QX55.js} +4 -4
- package/dist/update-prompt-BCDFTNLK.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-NPRXYSWA.js} +27 -29
- package/dist/update-scenario-BNOMDCMV.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-EM5XGMB7.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +23 -8
- package/src/config.d.ts +6 -0
- package/src/config.ts +22 -29
- package/src/create-mcp-server.ts +887 -634
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-analytics-reports.ts +62 -0
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-dashboard-widgets.ts +47 -0
- package/src/langwatch-api-dashboards.ts +109 -24
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +10 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +118 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/schemas/widget-sources.ts +33 -0
- package/src/tools/add-dashboard-widget.ts +51 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-analytics-report.ts +7 -0
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-widget-queries.ts +45 -0
- package/src/tools/dashboards.ts +104 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-analytics-report.ts +7 -0
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics-report.ts +23 -0
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-analytics-reports.ts +27 -0
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboards.ts +20 -21
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +7 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +37 -34
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +86 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/uv.lock +667 -341
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/chunk-S4GVX36W.js +0 -39
- package/dist/config.d.ts +0 -17
- package/dist/create-dashboard-Z76AIPLS.js +0 -19
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-dashboard-LYMRPF27.js +0 -19
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dashboard-P4WWVHFU.js +0 -30
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-dashboards-2KJL2M42.js +0 -18
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/list-dashboards-Z3BZKDGF.js +0 -33
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
- package/src/tools/create-dashboard.ts +0 -7
- package/src/tools/delete-dashboard.ts +0 -7
- package/src/tools/get-dashboard.ts +0 -22
|
@@ -1,117 +1,49 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SLACK_DELIVERY_NOTE,
|
|
3
|
+
TRIGGER_FILTERS_DESCRIPTION,
|
|
4
|
+
TRIGGER_FILTER_QUERY_DESCRIPTION,
|
|
5
|
+
actionParamsSchema,
|
|
6
|
+
alertTypeSchema,
|
|
7
|
+
graphAlertSchema,
|
|
8
|
+
notificationCadenceSchema,
|
|
9
|
+
reportSchema,
|
|
10
|
+
templatesSchema,
|
|
11
|
+
triggerActionSchema,
|
|
12
|
+
validateActionParamsForAction
|
|
13
|
+
} from "./chunk-3YLN3ATJ.js";
|
|
1
14
|
import {
|
|
2
15
|
testAgent
|
|
3
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-4RX3G62A.js";
|
|
4
17
|
import {
|
|
5
18
|
runParametersSchema,
|
|
6
19
|
runPlanScopeSchema,
|
|
7
20
|
runPlanTargetSchema
|
|
8
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-O7EMWEVJ.js";
|
|
9
22
|
import {
|
|
10
23
|
evaluatorAttachmentsSchema,
|
|
11
24
|
scenarioFieldValuesSchema,
|
|
12
25
|
suiteFieldsSchema,
|
|
13
26
|
toWireAttachments
|
|
14
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-U24EACC4.js";
|
|
15
28
|
import {
|
|
16
29
|
formatEvaluatorAttachments,
|
|
17
30
|
formatSuiteFields
|
|
18
|
-
} from "./chunk-
|
|
31
|
+
} from "./chunk-6YZUOD34.js";
|
|
19
32
|
import {
|
|
20
33
|
updateTestSuite
|
|
21
|
-
} from "./chunk-
|
|
34
|
+
} from "./chunk-YCIUEZVK.js";
|
|
22
35
|
import {
|
|
23
36
|
LangWatchApiError,
|
|
24
|
-
makeRequest
|
|
25
|
-
|
|
37
|
+
makeRequest,
|
|
38
|
+
package_default
|
|
39
|
+
} from "./chunk-7CZQDZ2K.js";
|
|
26
40
|
import {
|
|
27
41
|
requireApiKey
|
|
28
|
-
} from "./chunk-
|
|
42
|
+
} from "./chunk-DULFP3MK.js";
|
|
29
43
|
|
|
30
44
|
// src/create-mcp-server.ts
|
|
31
45
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
32
|
-
import { z as
|
|
33
|
-
|
|
34
|
-
// package.json
|
|
35
|
-
var package_default = {
|
|
36
|
-
name: "@langwatch/mcp-server",
|
|
37
|
-
version: "2.1.0",
|
|
38
|
-
description: "An MCP server for Langwatch.",
|
|
39
|
-
type: "module",
|
|
40
|
-
main: "./dist/index.js",
|
|
41
|
-
module: "./dist/index.js",
|
|
42
|
-
types: "./dist/index.d.ts",
|
|
43
|
-
exports: {
|
|
44
|
-
".": {
|
|
45
|
-
import: "./dist/index.js",
|
|
46
|
-
require: "./dist/index.js"
|
|
47
|
-
},
|
|
48
|
-
"./create-mcp-server": {
|
|
49
|
-
types: "./src/create-mcp-server.d.ts",
|
|
50
|
-
import: "./dist/create-mcp-server.js",
|
|
51
|
-
require: "./dist/create-mcp-server.js",
|
|
52
|
-
default: "./dist/create-mcp-server.js"
|
|
53
|
-
},
|
|
54
|
-
"./config": {
|
|
55
|
-
types: "./src/config.d.ts",
|
|
56
|
-
import: "./dist/config.js",
|
|
57
|
-
require: "./dist/config.js",
|
|
58
|
-
default: "./dist/config.js"
|
|
59
|
-
}
|
|
60
|
-
},
|
|
61
|
-
repository: {
|
|
62
|
-
type: "git",
|
|
63
|
-
url: "https://github.com/langwatch/langwatch.git",
|
|
64
|
-
directory: "mcp-server"
|
|
65
|
-
},
|
|
66
|
-
scripts: {
|
|
67
|
-
start: "tsx src/index.ts",
|
|
68
|
-
build: "tsup && node build.js",
|
|
69
|
-
prepublish: "pnpm run build",
|
|
70
|
-
test: "vitest"
|
|
71
|
-
},
|
|
72
|
-
author: "",
|
|
73
|
-
license: "MIT",
|
|
74
|
-
engines: {
|
|
75
|
-
node: ">=18.0.0"
|
|
76
|
-
},
|
|
77
|
-
devDependencies: {
|
|
78
|
-
"@ai-sdk/anthropic": "^3.0.44",
|
|
79
|
-
"@anthropic-ai/claude-code": "2.1.228",
|
|
80
|
-
"@eslint/js": "^10.0.1",
|
|
81
|
-
"@langwatch/handled-error": "workspace:*",
|
|
82
|
-
"@langwatch/redaction": "workspace:*",
|
|
83
|
-
"@langwatch/scenario": "^0.4.6",
|
|
84
|
-
"@langwatch/ssrf": "workspace:*",
|
|
85
|
-
"@opentelemetry/context-async-hooks": "^2.1.0",
|
|
86
|
-
"@opentelemetry/sdk-node": "^0.221.0",
|
|
87
|
-
"@types/debug": "^4.1.13",
|
|
88
|
-
"@types/express": "^5.0.6",
|
|
89
|
-
"@types/node": "^26.2.0",
|
|
90
|
-
"@types/yargs": "^17.0.33",
|
|
91
|
-
ai: "^6.0.161",
|
|
92
|
-
chalk: "^6.0.0",
|
|
93
|
-
"date-fns": "^4.1.0",
|
|
94
|
-
dotenv: "^17.2.2",
|
|
95
|
-
esbuild: "^0.28.1",
|
|
96
|
-
eslint: "^10.2.1",
|
|
97
|
-
express: "^5.2.1",
|
|
98
|
-
"node-pty": "^1.0.0",
|
|
99
|
-
tsup: "^8.1.0",
|
|
100
|
-
tsx: "^4.20.5",
|
|
101
|
-
typescript: "^6.0.3",
|
|
102
|
-
"typescript-eslint": "^8.55.0",
|
|
103
|
-
vitest: "^4.1.8",
|
|
104
|
-
yargs: "^18.0.0",
|
|
105
|
-
"zod-validation-error": "^5.0.0"
|
|
106
|
-
},
|
|
107
|
-
dependencies: {
|
|
108
|
-
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
109
|
-
zod: "^4.3.6"
|
|
110
|
-
},
|
|
111
|
-
bin: {
|
|
112
|
-
"langwatch-mcp-server": "./dist/index.js"
|
|
113
|
-
}
|
|
114
|
-
};
|
|
46
|
+
import { z as z3 } from "zod";
|
|
115
47
|
|
|
116
48
|
// src/documentation-fetch.ts
|
|
117
49
|
var TRUSTED_DOCUMENTATION_ORIGIN = "https://langwatch.ai";
|
|
@@ -151,7 +83,8 @@ function resolveDocumentationUrl(kind, input) {
|
|
|
151
83
|
} catch {
|
|
152
84
|
throw documentationUrlError(namespace);
|
|
153
85
|
}
|
|
154
|
-
|
|
86
|
+
const isTrustedDocumentationUrl = url.protocol === "https:" && url.hostname === "langwatch.ai" && url.port === "" && url.username === "" && url.password === "" && url.pathname.startsWith(`${namespace}/`);
|
|
87
|
+
if (!isTrustedDocumentationUrl) {
|
|
155
88
|
throw documentationUrlError(namespace);
|
|
156
89
|
}
|
|
157
90
|
const lowerPath = url.pathname.toLowerCase();
|
|
@@ -212,7 +145,8 @@ var DATASET_COLUMN_TYPES = [
|
|
|
212
145
|
"chat_messages",
|
|
213
146
|
"annotations",
|
|
214
147
|
"evaluations",
|
|
215
|
-
"image"
|
|
148
|
+
"image",
|
|
149
|
+
"file"
|
|
216
150
|
];
|
|
217
151
|
var datasetColumnTypeSchema = z.enum(DATASET_COLUMN_TYPES);
|
|
218
152
|
var datasetColumnDefinitionSchema = z.object({
|
|
@@ -224,6 +158,25 @@ var createDatasetSchema = z.object({
|
|
|
224
158
|
columnTypes: z.array(datasetColumnDefinitionSchema).optional().describe("Column definitions for the dataset")
|
|
225
159
|
});
|
|
226
160
|
|
|
161
|
+
// src/schemas/widget-sources.ts
|
|
162
|
+
import { z as z2 } from "zod";
|
|
163
|
+
var WIDGET_SOURCES = [
|
|
164
|
+
"application",
|
|
165
|
+
"simulation",
|
|
166
|
+
"evaluation",
|
|
167
|
+
"sample",
|
|
168
|
+
"coding_agent",
|
|
169
|
+
"ai_tool",
|
|
170
|
+
"gateway",
|
|
171
|
+
"workflow",
|
|
172
|
+
"playground",
|
|
173
|
+
"langy",
|
|
174
|
+
"other"
|
|
175
|
+
];
|
|
176
|
+
var widgetSourcesInput = z2.array(z2.enum(WIDGET_SOURCES)).max(WIDGET_SOURCES.length).nullable().optional().describe(
|
|
177
|
+
`The trace origins every query of the widget counts: any of ${WIDGET_SOURCES.join(", ")} (other = any origin not listed). A new widget counts ["application"]; add "langy" to count your own conversations, as a widget about Langy's requests needs. null makes it unset again, counting the default.`
|
|
178
|
+
);
|
|
179
|
+
|
|
227
180
|
// src/tools/experiment-run-status.ts
|
|
228
181
|
var INTERRUPTED_THRESHOLD_MS = 5 * 60 * 1e3;
|
|
229
182
|
var deriveRunStatus = (timestamps, now = Date.now()) => {
|
|
@@ -254,78 +207,45 @@ var isFailedRow = ({
|
|
|
254
207
|
entry,
|
|
255
208
|
evaluations
|
|
256
209
|
}) => Boolean(entry.error) || evaluations.some(isFailedEvaluation);
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
210
|
+
function hasNumericStatus(error) {
|
|
211
|
+
if (typeof error !== "object" || error === null) return false;
|
|
212
|
+
return "status" in error && typeof error.status === "number";
|
|
213
|
+
}
|
|
214
|
+
function isNotFoundError(error) {
|
|
215
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
216
|
+
let status;
|
|
217
|
+
if (error instanceof LangWatchApiError) status = error.status;
|
|
218
|
+
else if (hasNumericStatus(error)) status = error.status;
|
|
219
|
+
return status === 404 || status === void 0 && /404|not found/i.test(message);
|
|
220
|
+
}
|
|
221
|
+
async function fetchRunResults(params) {
|
|
261
222
|
const search = new URLSearchParams();
|
|
262
223
|
if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
|
|
263
224
|
const qs = search.toString() ? `?${search.toString()}` : "";
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
results
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
`# Evaluation Results: ${params.runId}`,
|
|
276
|
-
"",
|
|
277
|
-
"**Status**: not found",
|
|
278
|
-
"",
|
|
279
|
-
`Could not load results for run \`${params.runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
|
|
280
|
-
"",
|
|
281
|
-
"> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids."
|
|
282
|
-
].join("\n");
|
|
283
|
-
}
|
|
284
|
-
throw error;
|
|
285
|
-
}
|
|
286
|
-
if (!results) {
|
|
287
|
-
return [
|
|
288
|
-
`# Evaluation Results: ${params.runId}`,
|
|
289
|
-
"",
|
|
290
|
-
"**Status**: not found",
|
|
291
|
-
"",
|
|
292
|
-
`Could not load results for run \`${params.runId}\`.`,
|
|
293
|
-
"",
|
|
294
|
-
"> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id."
|
|
295
|
-
].join("\n");
|
|
296
|
-
}
|
|
297
|
-
const runStatus2 = deriveRunStatus(results.timestamps);
|
|
298
|
-
const partial = !isTerminalStatus(runStatus2);
|
|
299
|
-
const evaluationsByRow = /* @__PURE__ */ new Map();
|
|
300
|
-
for (const evaluation of results.evaluations) {
|
|
225
|
+
return await makeRequest(
|
|
226
|
+
"GET",
|
|
227
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
function groupEvaluationsByRow({
|
|
231
|
+
evaluations,
|
|
232
|
+
evaluatorFilter
|
|
233
|
+
}) {
|
|
234
|
+
const byRow = /* @__PURE__ */ new Map();
|
|
235
|
+
for (const evaluation of evaluations) {
|
|
301
236
|
if (evaluatorFilter && evaluation.evaluator !== evaluatorFilter) continue;
|
|
302
237
|
const key = rowKey(evaluation.index, evaluation.targetId);
|
|
303
|
-
const list =
|
|
238
|
+
const list = byRow.get(key) ?? [];
|
|
304
239
|
list.push(evaluation);
|
|
305
|
-
|
|
306
|
-
}
|
|
307
|
-
const evaluatorNames = Array.from(
|
|
308
|
-
new Set(
|
|
309
|
-
(evaluatorFilter ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter) : results.evaluations).map((e) => e.evaluator)
|
|
310
|
-
)
|
|
311
|
-
);
|
|
312
|
-
let rows = results.dataset.map((entry) => ({
|
|
313
|
-
entry,
|
|
314
|
-
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? []
|
|
315
|
-
}));
|
|
316
|
-
if (filter === "failed") {
|
|
317
|
-
rows = rows.filter(
|
|
318
|
-
(r) => isFailedRow({ entry: r.entry, evaluations: r.evaluations })
|
|
319
|
-
);
|
|
240
|
+
byRow.set(key, list);
|
|
320
241
|
}
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
const
|
|
326
|
-
for (const r of rowsForSummary) {
|
|
242
|
+
return byRow;
|
|
243
|
+
}
|
|
244
|
+
function tallyEvaluatorStats(rows) {
|
|
245
|
+
const averages = /* @__PURE__ */ new Map();
|
|
246
|
+
for (const r of rows) {
|
|
327
247
|
for (const e of r.evaluations) {
|
|
328
|
-
const stats =
|
|
248
|
+
const stats = averages.get(e.evaluator) ?? {
|
|
329
249
|
sum: 0,
|
|
330
250
|
count: 0,
|
|
331
251
|
passed: 0,
|
|
@@ -339,97 +259,153 @@ async function handleExperimentResults(params) {
|
|
|
339
259
|
if (e.status === "error") stats.errored += 1;
|
|
340
260
|
else if (e.passed === true) stats.passed += 1;
|
|
341
261
|
else if (e.passed === false) stats.failed += 1;
|
|
342
|
-
|
|
262
|
+
averages.set(e.evaluator, stats);
|
|
343
263
|
}
|
|
344
264
|
}
|
|
265
|
+
return averages;
|
|
266
|
+
}
|
|
267
|
+
function headerLines({
|
|
268
|
+
results,
|
|
269
|
+
runStatus: runStatus2,
|
|
270
|
+
filter,
|
|
271
|
+
evaluatorFilter
|
|
272
|
+
}) {
|
|
345
273
|
const lines = [];
|
|
346
274
|
lines.push(`# Evaluation Results: ${results.runId}`);
|
|
347
275
|
lines.push("");
|
|
348
276
|
lines.push(`**Experiment**: ${results.experimentId}`);
|
|
349
277
|
lines.push(`**Status**: ${runStatus2}`);
|
|
350
278
|
if (typeof results.total === "number" && results.total > 0) {
|
|
351
|
-
lines.push(
|
|
352
|
-
`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`
|
|
353
|
-
);
|
|
279
|
+
lines.push(`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`);
|
|
354
280
|
}
|
|
355
281
|
lines.push(`**Total rows**: ${results.dataset.length}`);
|
|
356
282
|
lines.push(`**Total evaluations**: ${results.evaluations.length}`);
|
|
357
|
-
if (filter === "failed")
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
if (evaluatorFilter) {
|
|
361
|
-
lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
362
|
-
}
|
|
363
|
-
if (partial) {
|
|
283
|
+
if (filter === "failed") lines.push(`**Filter**: failed only`);
|
|
284
|
+
if (evaluatorFilter) lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
285
|
+
if (!isTerminalStatus(runStatus2)) {
|
|
364
286
|
lines.push("");
|
|
365
287
|
lines.push(
|
|
366
288
|
runStatus2 === "interrupted" ? "> These are partial results. The run never sent a finished/stopped marker and has had no updates recently, so it likely was interrupted before completing. The rows below are everything recorded so far." : "> These are partial results. The run is still in progress, so more rows may appear on a later call."
|
|
367
289
|
);
|
|
368
290
|
}
|
|
369
|
-
|
|
370
|
-
if (
|
|
371
|
-
lines.push("## Evaluator Summary");
|
|
372
|
-
lines.push("");
|
|
373
|
-
lines.push("| Evaluator | Avg Score | Passed | Failed | Errored |");
|
|
374
|
-
lines.push("| --- | --- | --- | --- | --- |");
|
|
375
|
-
for (const [name, stats] of evaluatorAverages) {
|
|
376
|
-
const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "\u2014";
|
|
377
|
-
lines.push(
|
|
378
|
-
`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`
|
|
379
|
-
);
|
|
380
|
-
}
|
|
291
|
+
const stored = results.completeness;
|
|
292
|
+
if (isTerminalStatus(runStatus2) && stored && !stored.complete) {
|
|
381
293
|
lines.push("");
|
|
294
|
+
lines.push(
|
|
295
|
+
`> These are partial results. The run has ended, but the platform is still storing it: ${storedCount(stored.dataset)} rows and ${storedCount(stored.evaluations)} evaluations so far. Call again shortly for the whole run.`
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
lines.push("");
|
|
299
|
+
return lines;
|
|
300
|
+
}
|
|
301
|
+
function evaluatorSummaryLines(averages) {
|
|
302
|
+
if (averages.size === 0) return [];
|
|
303
|
+
const lines = [
|
|
304
|
+
"## Evaluator Summary",
|
|
305
|
+
"",
|
|
306
|
+
"| Evaluator | Avg Score | Passed | Failed | Errored |",
|
|
307
|
+
"| --- | --- | --- | --- | --- |"
|
|
308
|
+
];
|
|
309
|
+
for (const [name, stats] of averages) {
|
|
310
|
+
const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "\u2014";
|
|
311
|
+
lines.push(`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`);
|
|
382
312
|
}
|
|
313
|
+
lines.push("");
|
|
314
|
+
return lines;
|
|
315
|
+
}
|
|
316
|
+
var storedCount = ({ received, expected }) => expected === null ? `${received}` : `${received} of ${expected}`;
|
|
317
|
+
function noRowsLine({ filter, runStatus: runStatus2 }) {
|
|
318
|
+
if (filter === "failed") return "_No rows matched the filter._";
|
|
319
|
+
if (runStatus2 === "running")
|
|
320
|
+
return "_No rows recorded yet. The run is still in progress; call again shortly._";
|
|
321
|
+
if (runStatus2 === "interrupted") return "_No rows were recorded before the run was interrupted._";
|
|
322
|
+
return "_No rows recorded for this run._";
|
|
323
|
+
}
|
|
324
|
+
function evaluationLines(e) {
|
|
325
|
+
const parts = [`**${e.evaluator}**`];
|
|
326
|
+
if (e.status === "error") parts.push("status=error");
|
|
327
|
+
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
328
|
+
if (typeof e.passed === "boolean") parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
329
|
+
if (e.label) parts.push(`label=${e.label}`);
|
|
330
|
+
const lines = [`- ${parts.join(" \xB7 ")}`];
|
|
331
|
+
if (e.details) lines.push(` - details: ${e.details}`);
|
|
332
|
+
return lines;
|
|
333
|
+
}
|
|
334
|
+
function rowLines({
|
|
335
|
+
row: { entry, evaluations },
|
|
336
|
+
anyEvaluators
|
|
337
|
+
}) {
|
|
338
|
+
const summary = summarizeEntry(entry.entry);
|
|
339
|
+
const lines = [`### Row #${entry.index}${summary ? ` \u2014 ${summary}` : ""}`];
|
|
340
|
+
if (entry.error) lines.push(`- **Error**: ${entry.error}`);
|
|
341
|
+
if (entry.traceId) lines.push(`- **Trace ID**: \`${entry.traceId}\``);
|
|
342
|
+
if (evaluations.length === 0) {
|
|
343
|
+
lines.push(
|
|
344
|
+
anyEvaluators ? "- _No evaluations recorded for this row_" : "- _No evaluations recorded_"
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
for (const e of evaluations) lines.push(...evaluationLines(e));
|
|
348
|
+
lines.push("");
|
|
349
|
+
return lines;
|
|
350
|
+
}
|
|
351
|
+
function notFoundReport({ runId, expired }) {
|
|
352
|
+
const lines = [`# Evaluation Results: ${runId}`, "", "**Status**: not found", ""];
|
|
353
|
+
if (expired) {
|
|
354
|
+
lines.push(
|
|
355
|
+
`Could not load results for run \`${runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
|
|
356
|
+
"",
|
|
357
|
+
"> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids."
|
|
358
|
+
);
|
|
359
|
+
} else {
|
|
360
|
+
lines.push(
|
|
361
|
+
`Could not load results for run \`${runId}\`.`,
|
|
362
|
+
"",
|
|
363
|
+
"> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id."
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
return lines.join("\n");
|
|
367
|
+
}
|
|
368
|
+
async function handleExperimentResults(params) {
|
|
369
|
+
const filter = params.filter ?? "all";
|
|
370
|
+
const evaluatorFilter = params.evaluator?.trim();
|
|
371
|
+
const limit = typeof params.limit === "number" && params.limit > 0 ? Math.min(params.limit, DEFAULT_ROW_CAP) : DEFAULT_ROW_CAP;
|
|
372
|
+
let results;
|
|
373
|
+
try {
|
|
374
|
+
results = await fetchRunResults(params);
|
|
375
|
+
} catch (error) {
|
|
376
|
+
if (isNotFoundError(error)) return notFoundReport({ runId: params.runId, expired: true });
|
|
377
|
+
throw error;
|
|
378
|
+
}
|
|
379
|
+
if (!results) return notFoundReport({ runId: params.runId, expired: false });
|
|
380
|
+
const runStatus2 = deriveRunStatus(results.timestamps);
|
|
381
|
+
const evaluationsByRow = groupEvaluationsByRow({
|
|
382
|
+
evaluations: results.evaluations,
|
|
383
|
+
evaluatorFilter
|
|
384
|
+
});
|
|
385
|
+
const evaluatorNames = new Set(
|
|
386
|
+
(evaluatorFilter ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter) : results.evaluations).map((e) => e.evaluator)
|
|
387
|
+
);
|
|
388
|
+
const allRows = results.dataset.map((entry) => ({
|
|
389
|
+
entry,
|
|
390
|
+
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? []
|
|
391
|
+
}));
|
|
392
|
+
const matching = filter === "failed" ? allRows.filter((r) => isFailedRow(r)) : allRows;
|
|
393
|
+
const truncated = matching.length > limit;
|
|
394
|
+
const rows = matching.slice(0, limit);
|
|
395
|
+
const lines = [
|
|
396
|
+
...headerLines({ results, runStatus: runStatus2, filter, evaluatorFilter }),
|
|
397
|
+
...evaluatorSummaryLines(tallyEvaluatorStats(matching))
|
|
398
|
+
];
|
|
383
399
|
if (rows.length === 0) {
|
|
384
|
-
|
|
385
|
-
lines.push("_No rows matched the filter._");
|
|
386
|
-
} else if (runStatus2 === "running") {
|
|
387
|
-
lines.push(
|
|
388
|
-
"_No rows recorded yet. The run is still in progress; call again shortly._"
|
|
389
|
-
);
|
|
390
|
-
} else if (runStatus2 === "interrupted") {
|
|
391
|
-
lines.push("_No rows were recorded before the run was interrupted._");
|
|
392
|
-
} else {
|
|
393
|
-
lines.push("_No rows recorded for this run._");
|
|
394
|
-
}
|
|
400
|
+
lines.push(noRowsLine({ filter, runStatus: runStatus2 }));
|
|
395
401
|
return lines.join("\n");
|
|
396
402
|
}
|
|
397
|
-
lines.push(
|
|
398
|
-
`## Rows (${rows.length}${truncated ? ` of ${totalMatching}` : ""})`
|
|
399
|
-
);
|
|
403
|
+
lines.push(`## Rows (${rows.length}${truncated ? ` of ${matching.length}` : ""})`);
|
|
400
404
|
lines.push("");
|
|
401
|
-
for (const {
|
|
402
|
-
const summary = summarizeEntry(entry.entry);
|
|
403
|
-
lines.push(`### Row #${entry.index}${summary ? ` \u2014 ${summary}` : ""}`);
|
|
404
|
-
if (entry.error) {
|
|
405
|
-
lines.push(`- **Error**: ${entry.error}`);
|
|
406
|
-
}
|
|
407
|
-
if (entry.traceId) {
|
|
408
|
-
lines.push(`- **Trace ID**: \`${entry.traceId}\``);
|
|
409
|
-
}
|
|
410
|
-
if (evaluations.length === 0) {
|
|
411
|
-
lines.push(
|
|
412
|
-
evaluatorNames.length === 0 ? "- _No evaluations recorded_" : "- _No evaluations recorded for this row_"
|
|
413
|
-
);
|
|
414
|
-
}
|
|
415
|
-
for (const e of evaluations) {
|
|
416
|
-
const parts = [`**${e.evaluator}**`];
|
|
417
|
-
if (e.status === "error") parts.push("status=error");
|
|
418
|
-
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
419
|
-
if (typeof e.passed === "boolean") {
|
|
420
|
-
parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
421
|
-
}
|
|
422
|
-
if (e.label) parts.push(`label=${e.label}`);
|
|
423
|
-
lines.push(`- ${parts.join(" \xB7 ")}`);
|
|
424
|
-
if (e.details) {
|
|
425
|
-
lines.push(` - details: ${e.details}`);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
lines.push("");
|
|
429
|
-
}
|
|
405
|
+
for (const row of rows) lines.push(...rowLines({ row, anyEvaluators: evaluatorNames.size > 0 }));
|
|
430
406
|
if (truncated) {
|
|
431
407
|
lines.push(
|
|
432
|
-
`> Output truncated to ${limit} rows of ${
|
|
408
|
+
`> Output truncated to ${limit} rows of ${matching.length} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`
|
|
433
409
|
);
|
|
434
410
|
}
|
|
435
411
|
return lines.join("\n");
|
|
@@ -470,7 +446,7 @@ async function handleExperimentListRuns(params) {
|
|
|
470
446
|
try {
|
|
471
447
|
result = await makeRequest(
|
|
472
448
|
"GET",
|
|
473
|
-
`/api/experiments/runs?${search.toString()}`
|
|
449
|
+
`/api/v1/experiments/runs?${search.toString()}`
|
|
474
450
|
);
|
|
475
451
|
} catch (error) {
|
|
476
452
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -540,7 +516,7 @@ async function handleExperimentList(params) {
|
|
|
540
516
|
search.set("pageSize", String(effectiveLimit));
|
|
541
517
|
const result = await makeRequest(
|
|
542
518
|
"GET",
|
|
543
|
-
`/api/experiments?${search.toString()}`
|
|
519
|
+
`/api/v1/experiments?${search.toString()}`
|
|
544
520
|
);
|
|
545
521
|
if (result.experiments.length === 0) {
|
|
546
522
|
return [
|
|
@@ -572,9 +548,7 @@ async function handleExperimentList(params) {
|
|
|
572
548
|
);
|
|
573
549
|
}
|
|
574
550
|
lines.push("");
|
|
575
|
-
lines.push(
|
|
576
|
-
"> Use `platform_experiment_list_runs` with one of these slugs to see its runs."
|
|
577
|
-
);
|
|
551
|
+
lines.push("> Use `platform_experiment_list_runs` with one of these slugs to see its runs.");
|
|
578
552
|
return lines.join("\n");
|
|
579
553
|
}
|
|
580
554
|
|
|
@@ -582,7 +556,8 @@ async function handleExperimentList(params) {
|
|
|
582
556
|
async function handleRunExperiment(params) {
|
|
583
557
|
const result = await makeRequest(
|
|
584
558
|
"POST",
|
|
585
|
-
`/api/experiments/${encodeURIComponent(params.slug)}/run
|
|
559
|
+
`/api/v1/experiments/${encodeURIComponent(params.slug)}/run`,
|
|
560
|
+
params.concurrency !== void 0 ? { concurrency: params.concurrency } : void 0
|
|
586
561
|
);
|
|
587
562
|
const lines = [];
|
|
588
563
|
lines.push(`Evaluation started!
|
|
@@ -605,7 +580,7 @@ async function statusFromResults(params) {
|
|
|
605
580
|
try {
|
|
606
581
|
results = await makeRequest(
|
|
607
582
|
"GET",
|
|
608
|
-
`/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
583
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
609
584
|
);
|
|
610
585
|
} catch (error) {
|
|
611
586
|
const code = error instanceof LangWatchApiError ? error.status : void 0;
|
|
@@ -624,19 +599,13 @@ async function statusFromResults(params) {
|
|
|
624
599
|
lines.push(`**Status**: ${status}`);
|
|
625
600
|
lines.push(`**Progress**: ${progress}/${total} cells`);
|
|
626
601
|
if (results.timestamps.createdAt) {
|
|
627
|
-
lines.push(
|
|
628
|
-
`**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`
|
|
629
|
-
);
|
|
602
|
+
lines.push(`**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`);
|
|
630
603
|
}
|
|
631
604
|
if (results.timestamps.finishedAt) {
|
|
632
|
-
lines.push(
|
|
633
|
-
`**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`
|
|
634
|
-
);
|
|
605
|
+
lines.push(`**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`);
|
|
635
606
|
}
|
|
636
607
|
if (results.timestamps.stoppedAt) {
|
|
637
|
-
lines.push(
|
|
638
|
-
`**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`
|
|
639
|
-
);
|
|
608
|
+
lines.push(`**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`);
|
|
640
609
|
}
|
|
641
610
|
lines.push("");
|
|
642
611
|
lines.push(
|
|
@@ -644,30 +613,36 @@ async function statusFromResults(params) {
|
|
|
644
613
|
);
|
|
645
614
|
return lines.join("\n");
|
|
646
615
|
}
|
|
616
|
+
async function statusAfterFailedRead({
|
|
617
|
+
params,
|
|
618
|
+
error
|
|
619
|
+
}) {
|
|
620
|
+
const code = error instanceof LangWatchApiError ? error.status : void 0;
|
|
621
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
622
|
+
if (code === 404 || /404|not found/i.test(message)) {
|
|
623
|
+
const fallback = await statusFromResults(params);
|
|
624
|
+
if (fallback) return fallback;
|
|
625
|
+
return [
|
|
626
|
+
`# Evaluation Run ${params.runId}`,
|
|
627
|
+
"",
|
|
628
|
+
"**Status**: not found",
|
|
629
|
+
"",
|
|
630
|
+
`Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
|
|
631
|
+
"",
|
|
632
|
+
"> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`."
|
|
633
|
+
].join("\n");
|
|
634
|
+
}
|
|
635
|
+
throw error;
|
|
636
|
+
}
|
|
647
637
|
async function handleExperimentStatus(params) {
|
|
648
638
|
let status;
|
|
649
639
|
try {
|
|
650
640
|
status = await makeRequest(
|
|
651
641
|
"GET",
|
|
652
|
-
`/api/experiments/runs/${encodeURIComponent(params.runId)}`
|
|
642
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}`
|
|
653
643
|
);
|
|
654
644
|
} catch (error) {
|
|
655
|
-
|
|
656
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
657
|
-
if (code === 404 || /404|not found/i.test(message)) {
|
|
658
|
-
const fallback = await statusFromResults(params);
|
|
659
|
-
if (fallback) return fallback;
|
|
660
|
-
return [
|
|
661
|
-
`# Evaluation Run ${params.runId}`,
|
|
662
|
-
"",
|
|
663
|
-
"**Status**: not found",
|
|
664
|
-
"",
|
|
665
|
-
`Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
|
|
666
|
-
"",
|
|
667
|
-
"> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`."
|
|
668
|
-
].join("\n");
|
|
669
|
-
}
|
|
670
|
-
throw error;
|
|
645
|
+
return statusAfterFailedRead({ params, error });
|
|
671
646
|
}
|
|
672
647
|
const lines = [];
|
|
673
648
|
lines.push(`# Evaluation Run ${status.runId}
|
|
@@ -689,9 +664,7 @@ async function handleExperimentStatus(params) {
|
|
|
689
664
|
lines.push(`**Failed**: ${status.summary.failedCells}`);
|
|
690
665
|
}
|
|
691
666
|
if (status.summary.duration) {
|
|
692
|
-
lines.push(
|
|
693
|
-
`**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`
|
|
694
|
-
);
|
|
667
|
+
lines.push(`**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`);
|
|
695
668
|
}
|
|
696
669
|
if (status.summary.runUrl) {
|
|
697
670
|
lines.push(`**View results**: ${status.summary.runUrl}`);
|
|
@@ -735,8 +708,8 @@ async function handleUpdateTestSuite(params) {
|
|
|
735
708
|
}
|
|
736
709
|
|
|
737
710
|
// src/create-mcp-server.ts
|
|
738
|
-
var modelSchema =
|
|
739
|
-
'Model in "provider/model-name" format, e.g., "openai/gpt-
|
|
711
|
+
var modelSchema = z3.string().describe(
|
|
712
|
+
'Model in "provider/model-name" format, e.g., "openai/gpt-6-luna", "anthropic/claude-sonnet-4-5-20250929"'
|
|
740
713
|
);
|
|
741
714
|
function createMcpServer() {
|
|
742
715
|
const server = new McpServer({
|
|
@@ -761,11 +734,34 @@ function withToolLogging(toolName, fn) {
|
|
|
761
734
|
};
|
|
762
735
|
}
|
|
763
736
|
function registerTools(server) {
|
|
737
|
+
registerDocsTools(server);
|
|
738
|
+
registerObservabilityTools(server);
|
|
739
|
+
registerPromptTools(server);
|
|
740
|
+
registerScenarioTools(server);
|
|
741
|
+
registerRunPlanTools(server);
|
|
742
|
+
registerTestSuiteTools(server);
|
|
743
|
+
registerSimulationRunTools(server);
|
|
744
|
+
registerEvaluatorTools(server);
|
|
745
|
+
registerModelProviderTools(server);
|
|
746
|
+
registerAgentTools(server);
|
|
747
|
+
registerAnalyticsReportTools(server);
|
|
748
|
+
registerDashboardTools(server);
|
|
749
|
+
registerWorkflowTools(server);
|
|
750
|
+
registerAnnotationTools(server);
|
|
751
|
+
registerTriggerTools(server);
|
|
752
|
+
registerMonitorTools(server);
|
|
753
|
+
registerSecretTools(server);
|
|
754
|
+
registerExperimentTools(server);
|
|
755
|
+
registerDatasetTools(server);
|
|
756
|
+
registerProjectTools(server);
|
|
757
|
+
registerApiKeyTools(server);
|
|
758
|
+
}
|
|
759
|
+
function registerDocsTools(server) {
|
|
764
760
|
server.tool(
|
|
765
761
|
"fetch_langwatch_docs",
|
|
766
762
|
"Fetches the LangWatch docs for understanding how to implement LangWatch in your codebase. Always use this tool when the user asks for help with LangWatch. Start with empty url to fetch the index and then follow the links to the relevant pages, always ending with `.md` extension",
|
|
767
763
|
{
|
|
768
|
-
url:
|
|
764
|
+
url: z3.string().optional().describe(
|
|
769
765
|
"The full url of the specific doc page. If not provided, the docs index will be fetched."
|
|
770
766
|
)
|
|
771
767
|
},
|
|
@@ -778,21 +774,21 @@ function registerTools(server) {
|
|
|
778
774
|
"report_issue",
|
|
779
775
|
"Report an issue to the LangWatch team: anything that did not work, confused you, or took trial and error while using LangWatch (docs, MCP tools, CLI, SDKs, the platform). Use it whenever you struggled, it is how rough edges get fixed for every agent. Ask the user for permission first, then call with user_approved=true. Works without an API key. Put VERBATIM error messages in the summary; optionally attach raw session content. Secrets, API keys, emails and phone numbers are redacted locally before anything is sent.",
|
|
780
776
|
{
|
|
781
|
-
user_approved:
|
|
777
|
+
user_approved: z3.boolean().describe(
|
|
782
778
|
"Must be true, and only after the user explicitly agreed to send this report to LangWatch"
|
|
783
779
|
),
|
|
784
|
-
title:
|
|
785
|
-
summary:
|
|
780
|
+
title: z3.string().max(300).describe("One-line description of the issue"),
|
|
781
|
+
summary: z3.string().optional().describe(
|
|
786
782
|
"What you were trying to do, what went wrong (verbatim errors), and what you had to figure out the hard way"
|
|
787
783
|
),
|
|
788
|
-
session_content:
|
|
784
|
+
session_content: z3.string().optional().describe(
|
|
789
785
|
"Optional raw session transcript or log excerpt (JSONL or plain text); redacted locally before sending"
|
|
790
786
|
),
|
|
791
|
-
contact_email:
|
|
792
|
-
agent:
|
|
787
|
+
contact_email: z3.string().optional().describe("Optional contact email for follow-up"),
|
|
788
|
+
agent: z3.string().optional().describe("Which coding agent this is, e.g. claude-code, codex, cursor")
|
|
793
789
|
},
|
|
794
790
|
withToolLogging("report_issue", async (params) => {
|
|
795
|
-
const { handleReportIssue } = await import("./report-issue-
|
|
791
|
+
const { handleReportIssue } = await import("./report-issue-N7ND5TM6.js");
|
|
796
792
|
return {
|
|
797
793
|
content: [{ type: "text", text: await handleReportIssue(params) }]
|
|
798
794
|
};
|
|
@@ -802,7 +798,7 @@ function registerTools(server) {
|
|
|
802
798
|
"fetch_scenario_docs",
|
|
803
799
|
"Fetches the Scenario docs for understanding how to implement Scenario agent tests in your codebase. Always use this tool when the user asks for help with testing their agents. Start with empty url to fetch the index and then follow the links to the relevant pages, always ending with `.md` extension",
|
|
804
800
|
{
|
|
805
|
-
url:
|
|
801
|
+
url: z3.string().optional().describe(
|
|
806
802
|
"The full url of the specific doc page. If not provided, the docs index will be fetched."
|
|
807
803
|
)
|
|
808
804
|
},
|
|
@@ -811,70 +807,99 @@ function registerTools(server) {
|
|
|
811
807
|
return { content: [{ type: "text", text }] };
|
|
812
808
|
})
|
|
813
809
|
);
|
|
810
|
+
}
|
|
811
|
+
function registerObservabilityTools(server) {
|
|
814
812
|
server.tool(
|
|
815
813
|
"discover_schema",
|
|
816
|
-
"Discover
|
|
814
|
+
"Discover what LangWatch can be queried with: the trace filter fields and syntax, the analytics SQL views and columns, the analytics metrics, aggregation types, group-by options, scenario schema and evaluator types. Call this before using search_traces, run_query, get_analytics, scenario tools or evaluator tools, so you never guess a field, a column or a value.",
|
|
817
815
|
{
|
|
818
|
-
category:
|
|
816
|
+
category: z3.enum([
|
|
819
817
|
"filters",
|
|
818
|
+
"lwql",
|
|
820
819
|
"metrics",
|
|
821
820
|
"aggregations",
|
|
822
821
|
"groups",
|
|
823
822
|
"scenarios",
|
|
824
823
|
"evaluators",
|
|
825
824
|
"all"
|
|
826
|
-
]).describe(
|
|
827
|
-
|
|
825
|
+
]).describe(
|
|
826
|
+
"Which schema category to discover. 'filters' is the trace filter language, 'lwql' the analytics SQL views; both are read from the platform and need the API key"
|
|
827
|
+
),
|
|
828
|
+
evaluatorType: z3.string().optional().describe(
|
|
828
829
|
"When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details"
|
|
829
830
|
)
|
|
830
831
|
},
|
|
831
|
-
async ({ category, evaluatorType }) => {
|
|
832
|
+
withToolLogging("discover_schema", async ({ category, evaluatorType }) => {
|
|
832
833
|
if (category === "scenarios") {
|
|
833
|
-
const { formatScenarioSchema } = await import("./discover-scenario-schema-
|
|
834
|
+
const { formatScenarioSchema } = await import("./discover-scenario-schema-OOFIQKOG.js");
|
|
834
835
|
return {
|
|
835
836
|
content: [{ type: "text", text: formatScenarioSchema() }]
|
|
836
837
|
};
|
|
837
838
|
}
|
|
838
839
|
if (category === "evaluators") {
|
|
839
|
-
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-
|
|
840
|
+
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-UHWA5HVL.js");
|
|
840
841
|
return {
|
|
841
|
-
content: [
|
|
842
|
-
{ type: "text", text: formatEvaluatorSchema(evaluatorType) }
|
|
843
|
-
]
|
|
842
|
+
content: [{ type: "text", text: formatEvaluatorSchema(evaluatorType) }]
|
|
844
843
|
};
|
|
845
844
|
}
|
|
846
|
-
const { formatSchema } = await import("./discover-schema-
|
|
847
|
-
|
|
845
|
+
const { formatSchema, needsQueryReference } = await import("./discover-schema-J2ANOBKO.js");
|
|
846
|
+
if (needsQueryReference(category)) requireApiKey();
|
|
847
|
+
let text = await formatSchema(category);
|
|
848
848
|
if (category === "all") {
|
|
849
|
-
const { formatScenarioSchema } = await import("./discover-scenario-schema-
|
|
849
|
+
const { formatScenarioSchema } = await import("./discover-scenario-schema-OOFIQKOG.js");
|
|
850
850
|
text += "\n\n" + formatScenarioSchema();
|
|
851
|
-
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-
|
|
851
|
+
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-UHWA5HVL.js");
|
|
852
852
|
text += "\n\n" + formatEvaluatorSchema();
|
|
853
853
|
}
|
|
854
854
|
return { content: [{ type: "text", text }] };
|
|
855
|
-
}
|
|
855
|
+
})
|
|
856
|
+
);
|
|
857
|
+
server.tool(
|
|
858
|
+
"run_query",
|
|
859
|
+
"Run one read-only analytics SQL statement (LangWatchQL) over this project's traces, spans, evaluations and metrics, and get the rows back as a table. This is the tool for a count, a rate, a sum, a percentile or any of them grouped or over time \u2014 questions search_traces would need many calls to answer. The statement runs exactly as written. Call discover_schema with category 'lwql' first for the views, columns and worked statements.",
|
|
860
|
+
{
|
|
861
|
+
sql: z3.string().describe(
|
|
862
|
+
"The SELECT to run. Filter on the dataset's time column so the read is bounded, and never select a whole attribute map"
|
|
863
|
+
),
|
|
864
|
+
parameters: z3.record(z3.string(), z3.union([z3.string(), z3.number(), z3.boolean(), z3.null()])).optional().describe(
|
|
865
|
+
'Values for the parameters the statement declares, e.g. {"days": 7} for {days:UInt32}'
|
|
866
|
+
)
|
|
867
|
+
},
|
|
868
|
+
withToolLogging("run_query", async (params) => {
|
|
869
|
+
requireApiKey();
|
|
870
|
+
const { handleRunQuery } = await import("./run-query-4LMOQ5UW.js");
|
|
871
|
+
return {
|
|
872
|
+
content: [{ type: "text", text: await handleRunQuery(params) }]
|
|
873
|
+
};
|
|
874
|
+
})
|
|
856
875
|
);
|
|
857
876
|
server.tool(
|
|
858
877
|
"search_traces",
|
|
859
|
-
"Search LangWatch traces
|
|
878
|
+
"Search LangWatch traces by content, with filters and a date range. Returns AI-readable trace digests by default. Use format: 'json' for full raw data. To fetch a trace you already have the ID for, use get_trace instead \u2014 free text does not match trace IDs.",
|
|
860
879
|
{
|
|
861
|
-
query:
|
|
862
|
-
|
|
863
|
-
'Filter traces. Format: {"field": ["value"]}. Use discover_schema for field names.'
|
|
880
|
+
query: z3.string().optional().describe(
|
|
881
|
+
"Text search query. Matches captured input/output, the trace name and span names ONLY. It does not match trace IDs \u2014 to fetch a known trace ID use get_trace, or pass traceIds here for several at once."
|
|
864
882
|
),
|
|
865
|
-
|
|
866
|
-
|
|
883
|
+
traceIds: z3.array(z3.string().min(1)).min(1).max(1e3).optional().describe(
|
|
884
|
+
"Exact trace IDs to fetch (1\u20131000). Use instead of query when you already know the IDs. When set and no startDate is given, the window defaults to the last 90 days rather than 24 hours."
|
|
867
885
|
),
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
"
|
|
873
|
-
)
|
|
886
|
+
filters: z3.record(z3.string(), z3.array(z3.string())).optional().describe(
|
|
887
|
+
'Filter traces by the older per-field map. Format: {"field": ["value"]}. Use discover_schema for field names.'
|
|
888
|
+
),
|
|
889
|
+
filter: z3.string().optional().describe(
|
|
890
|
+
'Filter traces with the Trace Explorer\'s own query language: "status:error AND model:gpt-*", "trace.attribute.langwatch.user_id:alice", "evaluatorVerdict:fail", a quoted phrase for free text. Reaches attribute keys, span events and evaluator verdicts that the `filters` map cannot. Combined with `filters` and `query` when you send more than one. Call discover_schema with category \'filters\' for every field and the syntax.'
|
|
891
|
+
),
|
|
892
|
+
startDate: z3.string().min(1).optional().describe(
|
|
893
|
+
'Start of the window: ISO date, or relative like "24h", "7d", "4w", "3m" \u2014 units are h (hours), d (days), w (weeks), m (30-day months). Defaults to 24h ago for a text search, or 90d when traceIds is set. Widen this when a search comes back empty.'
|
|
894
|
+
),
|
|
895
|
+
endDate: z3.string().min(1).optional().describe("End of the window: ISO date or relative. Default: now"),
|
|
896
|
+
pageSize: z3.number().optional().describe("Results per page (default: 25, max: 1000)"),
|
|
897
|
+
scrollId: z3.string().optional().describe("Pagination token from previous search"),
|
|
898
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
874
899
|
},
|
|
875
900
|
withToolLogging("search_traces", async (params) => {
|
|
876
901
|
requireApiKey();
|
|
877
|
-
const { handleSearchTraces } = await import("./search-traces-
|
|
902
|
+
const { handleSearchTraces } = await import("./search-traces-BYML2NQ4.js");
|
|
878
903
|
return {
|
|
879
904
|
content: [{ type: "text", text: await handleSearchTraces(params) }]
|
|
880
905
|
};
|
|
@@ -882,16 +907,16 @@ function registerTools(server) {
|
|
|
882
907
|
);
|
|
883
908
|
server.tool(
|
|
884
909
|
"get_trace",
|
|
885
|
-
"Get full details of a single trace by ID. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
910
|
+
"Get full details of a single trace by ID. This is the right tool whenever you have a trace ID \u2014 search_traces does not match IDs. A full ID resolves at any age; a unique 8\u201331 character hex prefix resolves within the last 90 days. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
886
911
|
{
|
|
887
|
-
traceId:
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
)
|
|
912
|
+
traceId: z3.string().describe(
|
|
913
|
+
"The trace ID to retrieve. A full ID resolves at any age; a unique 8\u201331 character hex prefix also resolves, git-style, within the last 90 days."
|
|
914
|
+
),
|
|
915
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
891
916
|
},
|
|
892
917
|
withToolLogging("get_trace", async (params) => {
|
|
893
918
|
requireApiKey();
|
|
894
|
-
const { handleGetTrace } = await import("./get-trace-
|
|
919
|
+
const { handleGetTrace } = await import("./get-trace-AGCHVPJK.js");
|
|
895
920
|
return {
|
|
896
921
|
content: [{ type: "text", text: await handleGetTrace(params) }]
|
|
897
922
|
};
|
|
@@ -901,28 +926,28 @@ function registerTools(server) {
|
|
|
901
926
|
"get_analytics",
|
|
902
927
|
'Query analytics timeseries from LangWatch. Metrics use "category.name" format (e.g., "performance.completion_time"). Use discover_schema to see available metrics.',
|
|
903
928
|
{
|
|
904
|
-
metric:
|
|
929
|
+
metric: z3.string().describe(
|
|
905
930
|
'Metric in "category.name" format, e.g., "metadata.trace_id", "performance.total_cost"'
|
|
906
931
|
),
|
|
907
|
-
aggregation:
|
|
932
|
+
aggregation: z3.string().optional().describe(
|
|
908
933
|
"Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg"
|
|
909
934
|
),
|
|
910
|
-
startDate:
|
|
911
|
-
|
|
912
|
-
),
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
groupBy: z2.string().optional().describe("Group results by field. Use discover_schema for options."),
|
|
916
|
-
filters: z2.record(z2.string(), z2.array(z2.string())).optional().describe("Filters to apply")
|
|
935
|
+
startDate: z3.string().optional().describe('Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'),
|
|
936
|
+
endDate: z3.string().optional().describe("End date. Default: now"),
|
|
937
|
+
timeZone: z3.string().optional().describe("Timezone. Default: UTC"),
|
|
938
|
+
groupBy: z3.string().optional().describe("Group results by field. Use discover_schema for options."),
|
|
939
|
+
filters: z3.record(z3.string(), z3.array(z3.string())).optional().describe("Filters to apply")
|
|
917
940
|
},
|
|
918
941
|
withToolLogging("get_analytics", async (params) => {
|
|
919
942
|
requireApiKey();
|
|
920
|
-
const { handleGetAnalytics } = await import("./get-analytics-
|
|
943
|
+
const { handleGetAnalytics } = await import("./get-analytics-ZLI3A4F5.js");
|
|
921
944
|
return {
|
|
922
945
|
content: [{ type: "text", text: await handleGetAnalytics(params) }]
|
|
923
946
|
};
|
|
924
947
|
})
|
|
925
948
|
);
|
|
949
|
+
}
|
|
950
|
+
function registerPromptTools(server) {
|
|
926
951
|
server.tool(
|
|
927
952
|
"platform_create_prompt",
|
|
928
953
|
`Create a new prompt on the LangWatch platform.
|
|
@@ -934,24 +959,22 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
934
959
|
2. Platform-based (LangWatch UI): If the user wants to manage prompts directly on the LangWatch platform, use the \`platform_\` MCP tools (\`platform_create_prompt\`, \`platform_update_prompt\`, etc.).
|
|
935
960
|
`,
|
|
936
961
|
{
|
|
937
|
-
name:
|
|
938
|
-
handle:
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
role: z2.enum(["system", "user", "assistant"]).describe("Message role"),
|
|
944
|
-
content: z2.string().describe("Message content")
|
|
962
|
+
name: z3.string().describe("Prompt display name"),
|
|
963
|
+
handle: z3.string().optional().describe("URL-friendly handle (auto-generated from name if omitted)"),
|
|
964
|
+
messages: z3.array(
|
|
965
|
+
z3.object({
|
|
966
|
+
role: z3.enum(["system", "user", "assistant"]).describe("Message role"),
|
|
967
|
+
content: z3.string().describe("Message content")
|
|
945
968
|
})
|
|
946
969
|
).describe("Prompt messages"),
|
|
947
970
|
model: modelSchema,
|
|
948
|
-
tags:
|
|
971
|
+
tags: z3.array(z3.string()).optional().describe(
|
|
949
972
|
'Tags to assign to the initial version (e.g., ["production", "staging"]). Built-in tags: "latest" (auto-assigned), "production", "staging". Custom tags must be created first.'
|
|
950
973
|
)
|
|
951
974
|
},
|
|
952
975
|
withToolLogging("platform_create_prompt", async (params) => {
|
|
953
976
|
requireApiKey();
|
|
954
|
-
const { handleCreatePrompt } = await import("./create-prompt-
|
|
977
|
+
const { handleCreatePrompt } = await import("./create-prompt-JZQNQ7I3.js");
|
|
955
978
|
return {
|
|
956
979
|
content: [{ type: "text", text: await handleCreatePrompt(params) }]
|
|
957
980
|
};
|
|
@@ -963,7 +986,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
963
986
|
{},
|
|
964
987
|
withToolLogging("platform_list_prompts", async () => {
|
|
965
988
|
requireApiKey();
|
|
966
|
-
const { handleListPrompts } = await import("./list-prompts-
|
|
989
|
+
const { handleListPrompts } = await import("./list-prompts-PS423RU3.js");
|
|
967
990
|
return {
|
|
968
991
|
content: [{ type: "text", text: await handleListPrompts() }]
|
|
969
992
|
};
|
|
@@ -973,14 +996,12 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
973
996
|
"platform_get_prompt",
|
|
974
997
|
"Get a specific prompt from the LangWatch platform by ID or handle, including messages, model config, and version history. Use format: 'json' for the full raw API payload, or 'digest' (default) for a formatted summary.",
|
|
975
998
|
{
|
|
976
|
-
idOrHandle:
|
|
977
|
-
version:
|
|
978
|
-
tag:
|
|
999
|
+
idOrHandle: z3.string().describe("Prompt ID or handle"),
|
|
1000
|
+
version: z3.number().optional().describe("Specific version number (default: latest)"),
|
|
1001
|
+
tag: z3.string().optional().describe(
|
|
979
1002
|
'Fetch the version pointed to by this tag (e.g., "production", "staging"). Alternatively, use shorthand in idOrHandle: "pizza-prompt:production"'
|
|
980
1003
|
),
|
|
981
|
-
format:
|
|
982
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
983
|
-
)
|
|
1004
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
984
1005
|
},
|
|
985
1006
|
withToolLogging("platform_get_prompt", async (params) => {
|
|
986
1007
|
if (params.version != null && params.tag) {
|
|
@@ -990,7 +1011,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
990
1011
|
};
|
|
991
1012
|
}
|
|
992
1013
|
requireApiKey();
|
|
993
|
-
const { handleGetPrompt } = await import("./get-prompt-
|
|
1014
|
+
const { handleGetPrompt } = await import("./get-prompt-TLG2T2DS.js");
|
|
994
1015
|
return {
|
|
995
1016
|
content: [{ type: "text", text: await handleGetPrompt(params) }]
|
|
996
1017
|
};
|
|
@@ -1000,22 +1021,20 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1000
1021
|
"platform_update_prompt",
|
|
1001
1022
|
"Update an existing prompt on the LangWatch platform. Every update creates a new version.",
|
|
1002
1023
|
{
|
|
1003
|
-
idOrHandle:
|
|
1004
|
-
messages:
|
|
1005
|
-
|
|
1006
|
-
role:
|
|
1007
|
-
content:
|
|
1024
|
+
idOrHandle: z3.string().describe("Prompt ID or handle to update"),
|
|
1025
|
+
messages: z3.array(
|
|
1026
|
+
z3.object({
|
|
1027
|
+
role: z3.enum(["system", "user", "assistant"]),
|
|
1028
|
+
content: z3.string()
|
|
1008
1029
|
})
|
|
1009
1030
|
).optional().describe("Updated messages"),
|
|
1010
1031
|
model: modelSchema.optional(),
|
|
1011
|
-
commitMessage:
|
|
1012
|
-
tags:
|
|
1013
|
-
"Tags to assign to the new version created by this update."
|
|
1014
|
-
)
|
|
1032
|
+
commitMessage: z3.string().describe("Commit message describing the change"),
|
|
1033
|
+
tags: z3.array(z3.string()).optional().describe("Tags to assign to the new version created by this update.")
|
|
1015
1034
|
},
|
|
1016
1035
|
withToolLogging("platform_update_prompt", async (params) => {
|
|
1017
1036
|
requireApiKey();
|
|
1018
|
-
const { handleUpdatePrompt } = await import("./update-prompt-
|
|
1037
|
+
const { handleUpdatePrompt } = await import("./update-prompt-NPRXYSWA.js");
|
|
1019
1038
|
return {
|
|
1020
1039
|
content: [{ type: "text", text: await handleUpdatePrompt(params) }]
|
|
1021
1040
|
};
|
|
@@ -1025,13 +1044,13 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1025
1044
|
"platform_assign_prompt_tag",
|
|
1026
1045
|
'Assign a tag (e.g. "production") to a specific version of a prompt. Use this to "deploy" a version by promoting it to the production tag.',
|
|
1027
1046
|
{
|
|
1028
|
-
idOrHandle:
|
|
1029
|
-
tag:
|
|
1030
|
-
versionId:
|
|
1047
|
+
idOrHandle: z3.string().describe("Prompt ID or handle"),
|
|
1048
|
+
tag: z3.string().describe('Tag name (e.g., "production", "staging")'),
|
|
1049
|
+
versionId: z3.string().describe("The version ID to assign the tag to")
|
|
1031
1050
|
},
|
|
1032
1051
|
withToolLogging("platform_assign_prompt_tag", async (params) => {
|
|
1033
1052
|
requireApiKey();
|
|
1034
|
-
const { handleAssignPromptTag } = await import("./assign-prompt-tag-
|
|
1053
|
+
const { handleAssignPromptTag } = await import("./assign-prompt-tag-D7O23MJM.js");
|
|
1035
1054
|
return {
|
|
1036
1055
|
content: [{ type: "text", text: await handleAssignPromptTag(params) }]
|
|
1037
1056
|
};
|
|
@@ -1043,7 +1062,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1043
1062
|
{},
|
|
1044
1063
|
withToolLogging("platform_list_prompt_tags", async () => {
|
|
1045
1064
|
requireApiKey();
|
|
1046
|
-
const { handleListPromptTags } = await import("./list-prompt-tags-
|
|
1065
|
+
const { handleListPromptTags } = await import("./list-prompt-tags-3I3LWFIR.js");
|
|
1047
1066
|
return {
|
|
1048
1067
|
content: [{ type: "text", text: await handleListPromptTags() }]
|
|
1049
1068
|
};
|
|
@@ -1053,11 +1072,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1053
1072
|
"platform_create_prompt_tag",
|
|
1054
1073
|
'Create a custom prompt tag definition for the organization. Tag names must be non-numeric and not "latest".',
|
|
1055
1074
|
{
|
|
1056
|
-
name:
|
|
1075
|
+
name: z3.string().describe("Tag name to create")
|
|
1057
1076
|
},
|
|
1058
1077
|
withToolLogging("platform_create_prompt_tag", async (params) => {
|
|
1059
1078
|
requireApiKey();
|
|
1060
|
-
const { handleCreatePromptTag } = await import("./create-prompt-tag-
|
|
1079
|
+
const { handleCreatePromptTag } = await import("./create-prompt-tag-JFP2Q3PO.js");
|
|
1061
1080
|
return {
|
|
1062
1081
|
content: [{ type: "text", text: await handleCreatePromptTag(params) }]
|
|
1063
1082
|
};
|
|
@@ -1067,12 +1086,12 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1067
1086
|
"platform_rename_prompt_tag",
|
|
1068
1087
|
'Rename an existing prompt tag. The "latest" tag cannot be renamed.',
|
|
1069
1088
|
{
|
|
1070
|
-
tag:
|
|
1071
|
-
name:
|
|
1089
|
+
tag: z3.string().describe("Current tag name to rename"),
|
|
1090
|
+
name: z3.string().describe("New tag name")
|
|
1072
1091
|
},
|
|
1073
1092
|
withToolLogging("platform_rename_prompt_tag", async (params) => {
|
|
1074
1093
|
requireApiKey();
|
|
1075
|
-
const { handleRenamePromptTag } = await import("./rename-prompt-tag-
|
|
1094
|
+
const { handleRenamePromptTag } = await import("./rename-prompt-tag-2I7Z5FCY.js");
|
|
1076
1095
|
return {
|
|
1077
1096
|
content: [{ type: "text", text: await handleRenamePromptTag(params) }]
|
|
1078
1097
|
};
|
|
@@ -1082,16 +1101,18 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1082
1101
|
"platform_delete_prompt_tag",
|
|
1083
1102
|
'Delete a prompt tag and all its assignments. The "latest" tag cannot be deleted.',
|
|
1084
1103
|
{
|
|
1085
|
-
tag:
|
|
1104
|
+
tag: z3.string().describe("Tag name to delete")
|
|
1086
1105
|
},
|
|
1087
1106
|
withToolLogging("platform_delete_prompt_tag", async (params) => {
|
|
1088
1107
|
requireApiKey();
|
|
1089
|
-
const { handleDeletePromptTag } = await import("./delete-prompt-tag-
|
|
1108
|
+
const { handleDeletePromptTag } = await import("./delete-prompt-tag-S4KBZTNG.js");
|
|
1090
1109
|
return {
|
|
1091
1110
|
content: [{ type: "text", text: await handleDeletePromptTag(params) }]
|
|
1092
1111
|
};
|
|
1093
1112
|
})
|
|
1094
1113
|
);
|
|
1114
|
+
}
|
|
1115
|
+
function registerScenarioTools(server) {
|
|
1095
1116
|
server.tool(
|
|
1096
1117
|
"platform_create_scenario",
|
|
1097
1118
|
`Create a new scenario on the LangWatch platform. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.
|
|
@@ -1103,26 +1124,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1103
1124
|
2. Platform-based (LangWatch UI): If the user wants to manage scenarios directly on the LangWatch platform, use the \`platform_\` MCP tools (\`platform_create_scenario\`, \`platform_update_scenario\`, etc.).
|
|
1104
1125
|
`,
|
|
1105
1126
|
{
|
|
1106
|
-
name:
|
|
1107
|
-
situation:
|
|
1108
|
-
|
|
1109
|
-
),
|
|
1110
|
-
|
|
1111
|
-
"Pass/fail conditions the agent's response must satisfy"
|
|
1112
|
-
),
|
|
1113
|
-
labels: z2.array(z2.string()).optional().describe("Tags for organizing and filtering scenarios"),
|
|
1114
|
-
testSuiteId: z2.string().nullish().describe(
|
|
1127
|
+
name: z3.string().describe("Scenario name"),
|
|
1128
|
+
situation: z3.string().describe("The context or setup describing what the user/agent is doing"),
|
|
1129
|
+
criteria: z3.array(z3.string()).optional().describe("Pass/fail conditions the agent's response must satisfy"),
|
|
1130
|
+
labels: z3.array(z3.string()).optional().describe("Tags for organizing and filtering scenarios"),
|
|
1131
|
+
testSuiteId: z3.string().nullish().describe(
|
|
1115
1132
|
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
1116
1133
|
),
|
|
1117
1134
|
fields: scenarioFieldValuesSchema.optional()
|
|
1118
1135
|
},
|
|
1119
1136
|
withToolLogging("platform_create_scenario", async (params) => {
|
|
1120
1137
|
requireApiKey();
|
|
1121
|
-
const { handleCreateScenario } = await import("./create-scenario-
|
|
1138
|
+
const { handleCreateScenario } = await import("./create-scenario-S6OWWKJO.js");
|
|
1122
1139
|
return {
|
|
1123
|
-
content: [
|
|
1124
|
-
{ type: "text", text: await handleCreateScenario(params) }
|
|
1125
|
-
]
|
|
1140
|
+
content: [{ type: "text", text: await handleCreateScenario(params) }]
|
|
1126
1141
|
};
|
|
1127
1142
|
})
|
|
1128
1143
|
);
|
|
@@ -1130,18 +1145,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1130
1145
|
"platform_list_scenarios",
|
|
1131
1146
|
"List all scenarios on the LangWatch platform. Returns AI-readable digest by default.",
|
|
1132
1147
|
{
|
|
1133
|
-
testSuiteId:
|
|
1134
|
-
format:
|
|
1135
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
1136
|
-
)
|
|
1148
|
+
testSuiteId: z3.string().optional().describe("Only the scenarios filed in this test suite"),
|
|
1149
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
1137
1150
|
},
|
|
1138
1151
|
withToolLogging("platform_list_scenarios", async (params) => {
|
|
1139
1152
|
requireApiKey();
|
|
1140
|
-
const { handleListScenarios } = await import("./list-scenarios-
|
|
1153
|
+
const { handleListScenarios } = await import("./list-scenarios-GKNH3YND.js");
|
|
1141
1154
|
return {
|
|
1142
|
-
content: [
|
|
1143
|
-
{ type: "text", text: await handleListScenarios(params) }
|
|
1144
|
-
]
|
|
1155
|
+
content: [{ type: "text", text: await handleListScenarios(params) }]
|
|
1145
1156
|
};
|
|
1146
1157
|
})
|
|
1147
1158
|
);
|
|
@@ -1149,14 +1160,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1149
1160
|
"platform_get_scenario",
|
|
1150
1161
|
"Get full details of a scenario on the LangWatch platform by ID, including situation, criteria, and labels.",
|
|
1151
1162
|
{
|
|
1152
|
-
scenarioId:
|
|
1153
|
-
format:
|
|
1154
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
1155
|
-
)
|
|
1163
|
+
scenarioId: z3.string().describe("The scenario ID to retrieve"),
|
|
1164
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
1156
1165
|
},
|
|
1157
1166
|
withToolLogging("platform_get_scenario", async (params) => {
|
|
1158
1167
|
requireApiKey();
|
|
1159
|
-
const { handleGetScenario } = await import("./get-scenario-
|
|
1168
|
+
const { handleGetScenario } = await import("./get-scenario-375W5D6I.js");
|
|
1160
1169
|
return {
|
|
1161
1170
|
content: [{ type: "text", text: await handleGetScenario(params) }]
|
|
1162
1171
|
};
|
|
@@ -1166,23 +1175,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1166
1175
|
"platform_update_scenario",
|
|
1167
1176
|
"Update an existing scenario on the LangWatch platform.",
|
|
1168
1177
|
{
|
|
1169
|
-
scenarioId:
|
|
1170
|
-
name:
|
|
1171
|
-
situation:
|
|
1172
|
-
criteria:
|
|
1173
|
-
labels:
|
|
1174
|
-
testSuiteId:
|
|
1178
|
+
scenarioId: z3.string().describe("The scenario ID to update"),
|
|
1179
|
+
name: z3.string().optional().describe("Updated scenario name"),
|
|
1180
|
+
situation: z3.string().optional().describe("Updated situation"),
|
|
1181
|
+
criteria: z3.array(z3.string()).optional().describe("Updated criteria"),
|
|
1182
|
+
labels: z3.array(z3.string()).optional().describe("Updated labels"),
|
|
1183
|
+
testSuiteId: z3.string().nullish().describe(
|
|
1175
1184
|
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
1176
1185
|
),
|
|
1177
1186
|
fields: scenarioFieldValuesSchema.optional()
|
|
1178
1187
|
},
|
|
1179
1188
|
withToolLogging("platform_update_scenario", async (params) => {
|
|
1180
1189
|
requireApiKey();
|
|
1181
|
-
const { handleUpdateScenario } = await import("./update-scenario-
|
|
1190
|
+
const { handleUpdateScenario } = await import("./update-scenario-EM5XGMB7.js");
|
|
1182
1191
|
return {
|
|
1183
|
-
content: [
|
|
1184
|
-
{ type: "text", text: await handleUpdateScenario(params) }
|
|
1185
|
-
]
|
|
1192
|
+
content: [{ type: "text", text: await handleUpdateScenario(params) }]
|
|
1186
1193
|
};
|
|
1187
1194
|
})
|
|
1188
1195
|
);
|
|
@@ -1190,49 +1197,49 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1190
1197
|
"platform_archive_scenario",
|
|
1191
1198
|
"Archive (soft-delete) a scenario on the LangWatch platform.",
|
|
1192
1199
|
{
|
|
1193
|
-
scenarioId:
|
|
1200
|
+
scenarioId: z3.string().describe("The scenario ID to archive")
|
|
1194
1201
|
},
|
|
1195
1202
|
withToolLogging("platform_archive_scenario", async (params) => {
|
|
1196
1203
|
requireApiKey();
|
|
1197
|
-
const { handleArchiveScenario } = await import("./archive-scenario-
|
|
1204
|
+
const { handleArchiveScenario } = await import("./archive-scenario-5IJQESHH.js");
|
|
1198
1205
|
return {
|
|
1199
|
-
content: [
|
|
1200
|
-
{ type: "text", text: await handleArchiveScenario(params) }
|
|
1201
|
-
]
|
|
1206
|
+
content: [{ type: "text", text: await handleArchiveScenario(params) }]
|
|
1202
1207
|
};
|
|
1203
1208
|
})
|
|
1204
1209
|
);
|
|
1210
|
+
}
|
|
1211
|
+
function registerRunPlanTools(server) {
|
|
1205
1212
|
server.tool(
|
|
1206
1213
|
"platform_run_plan",
|
|
1207
1214
|
"Run scenarios against targets. The plan name identifies the run plan: an existing name is re-run with the configuration you send here, a new name creates the plan. Configuration is the scope, the targets, the repeat count and the models; parameters, the note and the idempotency key belong to this run alone. Send more than one target to compare them in the same run, and give a target its own parameters to compare one agent on two models.",
|
|
1208
1215
|
{
|
|
1209
|
-
name:
|
|
1216
|
+
name: z3.string().optional().describe(
|
|
1210
1217
|
"The run plan to run. An existing name is replaced with this configuration, a new name creates the plan. Omit to let the server name it after the suite and the target."
|
|
1211
1218
|
),
|
|
1212
1219
|
scope: runPlanScopeSchema.describe(
|
|
1213
1220
|
"What the plan covers: every scenario in the project, the scenarios filed in given test suites, the scenarios carrying given labels, or the hand-picked list in scenarioIds."
|
|
1214
1221
|
),
|
|
1215
|
-
scenarioIds:
|
|
1216
|
-
|
|
1217
|
-
),
|
|
1218
|
-
targets: z2.array(runPlanTargetSchema).describe(
|
|
1222
|
+
scenarioIds: z3.array(z3.string()).optional().describe("The scenarios to run. Read only when scope.mode is 'scenarios'."),
|
|
1223
|
+
targets: z3.array(runPlanTargetSchema).describe(
|
|
1219
1224
|
"What to run the scenarios against. Every target runs every scenario, and the results show one column for each, so this is how one run compares two agents or one agent on two models."
|
|
1220
1225
|
),
|
|
1221
|
-
repeatCount:
|
|
1222
|
-
simulatorModel:
|
|
1223
|
-
judgeModel:
|
|
1226
|
+
repeatCount: z3.number().int().min(1).max(5).optional().describe("How many times to run each scenario against each target (1 to 5, default 1)"),
|
|
1227
|
+
simulatorModel: z3.string().optional().describe("Model that plays the user. Omit for the project default."),
|
|
1228
|
+
judgeModel: z3.string().optional().describe("Model that judges the criteria. Omit for the project default."),
|
|
1224
1229
|
evaluators: evaluatorAttachmentsSchema.optional().describe(
|
|
1225
1230
|
"The plan's own evaluators, run beside the ones attached to the test suites its scenarios belong to. A plan evaluator reads the conversation and the trace, never a scenario field. Omit to keep what the plan already holds. See discover_schema({ category: 'scenarios' }) for the mapping paths."
|
|
1226
1231
|
),
|
|
1227
1232
|
parameters: runParametersSchema.optional().describe(
|
|
1228
1233
|
"Values for the parameters the scenarios declare, by name. They apply to every target; a target that names the same parameter overrides them for itself."
|
|
1229
1234
|
),
|
|
1230
|
-
note:
|
|
1231
|
-
idempotencyKey:
|
|
1235
|
+
note: z3.string().max(200).optional().describe("One short line saying why this run was started (max 200 characters)"),
|
|
1236
|
+
idempotencyKey: z3.string().optional().describe(
|
|
1237
|
+
"Send the same key to make a retry join the run it already started instead of starting a second one."
|
|
1238
|
+
)
|
|
1232
1239
|
},
|
|
1233
1240
|
withToolLogging("platform_run_plan", async (params) => {
|
|
1234
1241
|
requireApiKey();
|
|
1235
|
-
const { handleRunPlan } = await import("./run-plan-
|
|
1242
|
+
const { handleRunPlan } = await import("./run-plan-KSXUYQT3.js");
|
|
1236
1243
|
return {
|
|
1237
1244
|
content: [{ type: "text", text: await handleRunPlan(params) }]
|
|
1238
1245
|
};
|
|
@@ -1242,12 +1249,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1242
1249
|
"platform_list_run_plans",
|
|
1243
1250
|
"List the run plans of the project. A run plan holds the configuration a run used, keyed by its name.",
|
|
1244
1251
|
{
|
|
1245
|
-
includeArchived:
|
|
1246
|
-
format:
|
|
1252
|
+
includeArchived: z3.boolean().optional().describe("Include archived plans (default false)"),
|
|
1253
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json' (raw data)")
|
|
1247
1254
|
},
|
|
1248
1255
|
withToolLogging("platform_list_run_plans", async (params) => {
|
|
1249
1256
|
requireApiKey();
|
|
1250
|
-
const { handleListRunPlans } = await import("./list-run-plans-
|
|
1257
|
+
const { handleListRunPlans } = await import("./list-run-plans-6VNHINPY.js");
|
|
1251
1258
|
return {
|
|
1252
1259
|
content: [{ type: "text", text: await handleListRunPlans(params) }]
|
|
1253
1260
|
};
|
|
@@ -1257,12 +1264,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1257
1264
|
"platform_get_run_plan",
|
|
1258
1265
|
"Get the full configuration of a run plan: what it covers, its targets, its repeat count and its models.",
|
|
1259
1266
|
{
|
|
1260
|
-
id:
|
|
1261
|
-
format:
|
|
1267
|
+
id: z3.string().describe("The run plan ID"),
|
|
1268
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json'")
|
|
1262
1269
|
},
|
|
1263
1270
|
withToolLogging("platform_get_run_plan", async (params) => {
|
|
1264
1271
|
requireApiKey();
|
|
1265
|
-
const { handleGetRunPlan } = await import("./get-run-plan-
|
|
1272
|
+
const { handleGetRunPlan } = await import("./get-run-plan-PEHRO5M3.js");
|
|
1266
1273
|
return {
|
|
1267
1274
|
content: [{ type: "text", text: await handleGetRunPlan(params) }]
|
|
1268
1275
|
};
|
|
@@ -1272,13 +1279,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1272
1279
|
"platform_rerun_run_plan",
|
|
1273
1280
|
"Run a plan again with the configuration it already holds. Only the parameters and the note belong to the new run.",
|
|
1274
1281
|
{
|
|
1275
|
-
id:
|
|
1282
|
+
id: z3.string().describe("The run plan ID to run again"),
|
|
1276
1283
|
parameters: runParametersSchema.optional().describe("Values for the parameters the scenarios declare, by name."),
|
|
1277
|
-
note:
|
|
1284
|
+
note: z3.string().max(200).optional().describe("One short line saying why this run was started (max 200 characters)")
|
|
1278
1285
|
},
|
|
1279
1286
|
withToolLogging("platform_rerun_run_plan", async (params) => {
|
|
1280
1287
|
requireApiKey();
|
|
1281
|
-
const { handleRerunRunPlan } = await import("./rerun-run-plan-
|
|
1288
|
+
const { handleRerunRunPlan } = await import("./rerun-run-plan-O3VUZ6CG.js");
|
|
1282
1289
|
return {
|
|
1283
1290
|
content: [{ type: "text", text: await handleRerunRunPlan(params) }]
|
|
1284
1291
|
};
|
|
@@ -1288,25 +1295,27 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1288
1295
|
"platform_archive_run_plan",
|
|
1289
1296
|
"Archive a run plan. Its past runs stay readable and the name is free for a new plan.",
|
|
1290
1297
|
{
|
|
1291
|
-
id:
|
|
1298
|
+
id: z3.string().describe("The run plan ID to archive")
|
|
1292
1299
|
},
|
|
1293
1300
|
withToolLogging("platform_archive_run_plan", async (params) => {
|
|
1294
1301
|
requireApiKey();
|
|
1295
|
-
const { handleArchiveRunPlan } = await import("./archive-run-plan-
|
|
1302
|
+
const { handleArchiveRunPlan } = await import("./archive-run-plan-5HSQG2HX.js");
|
|
1296
1303
|
return {
|
|
1297
1304
|
content: [{ type: "text", text: await handleArchiveRunPlan(params) }]
|
|
1298
1305
|
};
|
|
1299
1306
|
})
|
|
1300
1307
|
);
|
|
1308
|
+
}
|
|
1309
|
+
function registerTestSuiteTools(server) {
|
|
1301
1310
|
server.tool(
|
|
1302
1311
|
"platform_list_test_suites",
|
|
1303
1312
|
"List the test suites of the project. A test suite groups scenarios.",
|
|
1304
1313
|
{
|
|
1305
|
-
format:
|
|
1314
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json' (raw data)")
|
|
1306
1315
|
},
|
|
1307
1316
|
withToolLogging("platform_list_test_suites", async (params) => {
|
|
1308
1317
|
requireApiKey();
|
|
1309
|
-
const { handleListTestSuites } = await import("./list-test-suites-
|
|
1318
|
+
const { handleListTestSuites } = await import("./list-test-suites-GY56LGLY.js");
|
|
1310
1319
|
return {
|
|
1311
1320
|
content: [{ type: "text", text: await handleListTestSuites(params) }]
|
|
1312
1321
|
};
|
|
@@ -1316,13 +1325,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1316
1325
|
"platform_create_test_suite",
|
|
1317
1326
|
"Create a test suite. A test suite groups scenarios: file a scenario in it by passing the suite ID as testSuiteId on platform_create_scenario or platform_update_scenario. A suite can declare typed fields every scenario carries a value for, and attach saved evaluators that run after every scenario run with mappings from their inputs to the conversation, the scenario's fields or the trace. Call discover_schema({ category: 'scenarios' }) first for the field rules and the mapping paths.",
|
|
1318
1327
|
{
|
|
1319
|
-
name:
|
|
1328
|
+
name: z3.string().describe("Test suite name"),
|
|
1320
1329
|
fields: suiteFieldsSchema.optional(),
|
|
1321
1330
|
evaluators: evaluatorAttachmentsSchema.optional()
|
|
1322
1331
|
},
|
|
1323
1332
|
withToolLogging("platform_create_test_suite", async (params) => {
|
|
1324
1333
|
requireApiKey();
|
|
1325
|
-
const { handleCreateTestSuite } = await import("./create-test-suite-
|
|
1334
|
+
const { handleCreateTestSuite } = await import("./create-test-suite-Y7P2RI4L.js");
|
|
1326
1335
|
return {
|
|
1327
1336
|
content: [{ type: "text", text: await handleCreateTestSuite(params) }]
|
|
1328
1337
|
};
|
|
@@ -1332,12 +1341,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1332
1341
|
"platform_get_test_suite",
|
|
1333
1342
|
"Get a test suite and the scenarios filed in it.",
|
|
1334
1343
|
{
|
|
1335
|
-
id:
|
|
1336
|
-
format:
|
|
1344
|
+
id: z3.string().describe("The test suite ID"),
|
|
1345
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json'")
|
|
1337
1346
|
},
|
|
1338
1347
|
withToolLogging("platform_get_test_suite", async (params) => {
|
|
1339
1348
|
requireApiKey();
|
|
1340
|
-
const { handleGetTestSuite } = await import("./get-test-suite-
|
|
1349
|
+
const { handleGetTestSuite } = await import("./get-test-suite-GVIBDG7K.js");
|
|
1341
1350
|
return {
|
|
1342
1351
|
content: [{ type: "text", text: await handleGetTestSuite(params) }]
|
|
1343
1352
|
};
|
|
@@ -1347,10 +1356,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1347
1356
|
"platform_update_test_suite",
|
|
1348
1357
|
"Edit a test suite: any of its name, its fields and its evaluators. A field list or an evaluator list replaces the one the suite holds; a key left out keeps what the suite has. A field an attached evaluator still reads cannot be removed. Call discover_schema({ category: 'scenarios' }) first for the field rules and the mapping paths.",
|
|
1349
1358
|
{
|
|
1350
|
-
id:
|
|
1351
|
-
name:
|
|
1352
|
-
fields: suiteFieldsSchema.optional().describe(
|
|
1353
|
-
|
|
1359
|
+
id: z3.string().describe("The test suite ID"),
|
|
1360
|
+
name: z3.string().optional().describe("The new name. The slug is kept."),
|
|
1361
|
+
fields: suiteFieldsSchema.optional().describe(
|
|
1362
|
+
"The full list of fields the suite declares. It replaces the list the suite holds."
|
|
1363
|
+
),
|
|
1364
|
+
evaluators: evaluatorAttachmentsSchema.optional().describe(
|
|
1365
|
+
"The full list of evaluators attached to the suite. It replaces the list the suite holds."
|
|
1366
|
+
)
|
|
1354
1367
|
},
|
|
1355
1368
|
withToolLogging("platform_update_test_suite", async (params) => {
|
|
1356
1369
|
requireApiKey();
|
|
@@ -1363,12 +1376,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1363
1376
|
"platform_rename_test_suite",
|
|
1364
1377
|
"Rename a test suite.",
|
|
1365
1378
|
{
|
|
1366
|
-
id:
|
|
1367
|
-
name:
|
|
1379
|
+
id: z3.string().describe("The test suite ID"),
|
|
1380
|
+
name: z3.string().describe("The new name")
|
|
1368
1381
|
},
|
|
1369
1382
|
withToolLogging("platform_rename_test_suite", async (params) => {
|
|
1370
1383
|
requireApiKey();
|
|
1371
|
-
const { handleRenameTestSuite } = await import("./rename-test-suite-
|
|
1384
|
+
const { handleRenameTestSuite } = await import("./rename-test-suite-27AYY33K.js");
|
|
1372
1385
|
return {
|
|
1373
1386
|
content: [{ type: "text", text: await handleRenameTestSuite(params) }]
|
|
1374
1387
|
};
|
|
@@ -1378,11 +1391,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1378
1391
|
"platform_archive_test_suite",
|
|
1379
1392
|
"Archive a test suite. The scenarios filed in it are archived with it.",
|
|
1380
1393
|
{
|
|
1381
|
-
id:
|
|
1394
|
+
id: z3.string().describe("The test suite ID to archive")
|
|
1382
1395
|
},
|
|
1383
1396
|
withToolLogging("platform_archive_test_suite", async (params) => {
|
|
1384
1397
|
requireApiKey();
|
|
1385
|
-
const { handleArchiveTestSuite } = await import("./archive-test-suite-
|
|
1398
|
+
const { handleArchiveTestSuite } = await import("./archive-test-suite-OPB7NFI7.js");
|
|
1386
1399
|
return {
|
|
1387
1400
|
content: [{ type: "text", text: await handleArchiveTestSuite(params) }]
|
|
1388
1401
|
};
|
|
@@ -1392,41 +1405,43 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1392
1405
|
"platform_run_test_suite",
|
|
1393
1406
|
"Run every scenario of a test suite against targets. This creates or joins the run plan named '<suite name> <target name>' unless you send a name of your own. Send more than one target to compare them in the same run, and give a target its own parameters to compare one agent on two models.",
|
|
1394
1407
|
{
|
|
1395
|
-
id:
|
|
1396
|
-
targets:
|
|
1408
|
+
id: z3.string().describe("The test suite ID to run"),
|
|
1409
|
+
targets: z3.array(runPlanTargetSchema).describe(
|
|
1397
1410
|
"What to run the scenarios against. Every target runs every scenario, and the results show one column for each."
|
|
1398
1411
|
),
|
|
1399
|
-
name:
|
|
1412
|
+
name: z3.string().optional().describe(
|
|
1400
1413
|
"The run plan to run. Omit to let the server name it after the suite and the target."
|
|
1401
1414
|
),
|
|
1402
|
-
repeatCount:
|
|
1403
|
-
simulatorModel:
|
|
1404
|
-
judgeModel:
|
|
1415
|
+
repeatCount: z3.number().int().min(1).max(5).optional().describe("How many times to run each scenario against each target (1 to 5, default 1)"),
|
|
1416
|
+
simulatorModel: z3.string().optional().describe("Model that plays the user. Omit for the project default."),
|
|
1417
|
+
judgeModel: z3.string().optional().describe("Model that judges the criteria. Omit for the project default."),
|
|
1405
1418
|
parameters: runParametersSchema.optional().describe(
|
|
1406
1419
|
"Values for the parameters the scenarios declare, by name. They apply to every target; a target that names the same parameter overrides them for itself."
|
|
1407
1420
|
),
|
|
1408
|
-
note:
|
|
1421
|
+
note: z3.string().max(200).optional().describe("One short line saying why this run was started (max 200 characters)")
|
|
1409
1422
|
},
|
|
1410
1423
|
withToolLogging("platform_run_test_suite", async (params) => {
|
|
1411
1424
|
requireApiKey();
|
|
1412
|
-
const { handleRunTestSuite } = await import("./run-test-suite-
|
|
1425
|
+
const { handleRunTestSuite } = await import("./run-test-suite-P3WLVPXC.js");
|
|
1413
1426
|
return {
|
|
1414
1427
|
content: [{ type: "text", text: await handleRunTestSuite(params) }]
|
|
1415
1428
|
};
|
|
1416
1429
|
})
|
|
1417
1430
|
);
|
|
1431
|
+
}
|
|
1432
|
+
function registerSimulationRunTools(server) {
|
|
1418
1433
|
server.tool(
|
|
1419
1434
|
"platform_list_simulation_runs",
|
|
1420
1435
|
"List simulation run results. Filter by scenario set or batch run ID to see specific results.",
|
|
1421
1436
|
{
|
|
1422
|
-
scenarioSetId:
|
|
1423
|
-
batchRunId:
|
|
1424
|
-
limit:
|
|
1425
|
-
format:
|
|
1437
|
+
scenarioSetId: z3.string().optional().describe("Filter by scenario set ID"),
|
|
1438
|
+
batchRunId: z3.string().optional().describe("Filter by batch run ID"),
|
|
1439
|
+
limit: z3.number().optional().describe("Max results to return (default: 20)"),
|
|
1440
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format")
|
|
1426
1441
|
},
|
|
1427
1442
|
withToolLogging("platform_list_simulation_runs", async (params) => {
|
|
1428
1443
|
requireApiKey();
|
|
1429
|
-
const { handleListSimulationRuns } = await import("./list-simulation-runs-
|
|
1444
|
+
const { handleListSimulationRuns } = await import("./list-simulation-runs-KHY7KALI.js");
|
|
1430
1445
|
return {
|
|
1431
1446
|
content: [{ type: "text", text: await handleListSimulationRuns(params) }]
|
|
1432
1447
|
};
|
|
@@ -1436,33 +1451,33 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1436
1451
|
"platform_get_simulation_run",
|
|
1437
1452
|
"Get full details of a simulation run including conversation messages, results, costs, the verdict and the result of every evaluator attached to its test suite or run plan.",
|
|
1438
1453
|
{
|
|
1439
|
-
scenarioRunId:
|
|
1440
|
-
format:
|
|
1454
|
+
scenarioRunId: z3.string().describe("The simulation run ID"),
|
|
1455
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format")
|
|
1441
1456
|
},
|
|
1442
1457
|
withToolLogging("platform_get_simulation_run", async (params) => {
|
|
1443
1458
|
requireApiKey();
|
|
1444
|
-
const { handleGetSimulationRun } = await import("./get-simulation-run-
|
|
1459
|
+
const { handleGetSimulationRun } = await import("./get-simulation-run-2NBXJADG.js");
|
|
1445
1460
|
return {
|
|
1446
1461
|
content: [{ type: "text", text: await handleGetSimulationRun(params) }]
|
|
1447
1462
|
};
|
|
1448
1463
|
})
|
|
1449
1464
|
);
|
|
1465
|
+
}
|
|
1466
|
+
function registerEvaluatorTools(server) {
|
|
1450
1467
|
server.tool(
|
|
1451
1468
|
"platform_create_evaluator",
|
|
1452
1469
|
`Create an evaluator on the LangWatch platform. Useful for setting up LLM-as-judge and other evaluators to use in evaluation notebooks. Call discover_schema({ category: 'evaluators' }) first to see available evaluator types and their settings.`,
|
|
1453
1470
|
{
|
|
1454
|
-
name:
|
|
1455
|
-
config:
|
|
1471
|
+
name: z3.string().describe("Evaluator name"),
|
|
1472
|
+
config: z3.record(z3.string(), z3.unknown()).describe(
|
|
1456
1473
|
'Evaluator config object. Must include "evaluatorType" (e.g. "langevals/llm_boolean") and optional "settings" overrides.'
|
|
1457
1474
|
)
|
|
1458
1475
|
},
|
|
1459
1476
|
withToolLogging("platform_create_evaluator", async (params) => {
|
|
1460
1477
|
requireApiKey();
|
|
1461
|
-
const { handleCreateEvaluator } = await import("./create-evaluator-
|
|
1478
|
+
const { handleCreateEvaluator } = await import("./create-evaluator-XYNY2YFY.js");
|
|
1462
1479
|
return {
|
|
1463
|
-
content: [
|
|
1464
|
-
{ type: "text", text: await handleCreateEvaluator(params) }
|
|
1465
|
-
]
|
|
1480
|
+
content: [{ type: "text", text: await handleCreateEvaluator(params) }]
|
|
1466
1481
|
};
|
|
1467
1482
|
})
|
|
1468
1483
|
);
|
|
@@ -1472,7 +1487,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1472
1487
|
{},
|
|
1473
1488
|
withToolLogging("platform_list_evaluators", async () => {
|
|
1474
1489
|
requireApiKey();
|
|
1475
|
-
const { handleListEvaluators } = await import("./list-evaluators-
|
|
1490
|
+
const { handleListEvaluators } = await import("./list-evaluators-7RRESBOU.js");
|
|
1476
1491
|
return {
|
|
1477
1492
|
content: [{ type: "text", text: await handleListEvaluators() }]
|
|
1478
1493
|
};
|
|
@@ -1482,11 +1497,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1482
1497
|
"platform_get_evaluator",
|
|
1483
1498
|
"Get full details of an evaluator on the LangWatch platform by ID or slug, including config, input fields, and output fields.",
|
|
1484
1499
|
{
|
|
1485
|
-
idOrSlug:
|
|
1500
|
+
idOrSlug: z3.string().describe("The evaluator ID or slug to retrieve")
|
|
1486
1501
|
},
|
|
1487
1502
|
withToolLogging("platform_get_evaluator", async (params) => {
|
|
1488
1503
|
requireApiKey();
|
|
1489
|
-
const { handleGetEvaluator } = await import("./get-evaluator-
|
|
1504
|
+
const { handleGetEvaluator } = await import("./get-evaluator-IWNJINRY.js");
|
|
1490
1505
|
return {
|
|
1491
1506
|
content: [{ type: "text", text: await handleGetEvaluator(params) }]
|
|
1492
1507
|
};
|
|
@@ -1496,19 +1511,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1496
1511
|
"platform_update_evaluator",
|
|
1497
1512
|
"Update an existing evaluator on the LangWatch platform. The evaluatorType in config cannot be changed after creation.",
|
|
1498
1513
|
{
|
|
1499
|
-
evaluatorId:
|
|
1500
|
-
name:
|
|
1501
|
-
config:
|
|
1502
|
-
"Updated config settings. Note: evaluatorType cannot be changed after creation."
|
|
1503
|
-
)
|
|
1514
|
+
evaluatorId: z3.string().describe("The evaluator ID to update"),
|
|
1515
|
+
name: z3.string().optional().describe("Updated evaluator name"),
|
|
1516
|
+
config: z3.record(z3.string(), z3.unknown()).optional().describe("Updated config settings. Note: evaluatorType cannot be changed after creation.")
|
|
1504
1517
|
},
|
|
1505
1518
|
withToolLogging("platform_update_evaluator", async (params) => {
|
|
1506
1519
|
requireApiKey();
|
|
1507
|
-
const { handleUpdateEvaluator } = await import("./update-evaluator-
|
|
1520
|
+
const { handleUpdateEvaluator } = await import("./update-evaluator-FEH7H6NS.js");
|
|
1508
1521
|
return {
|
|
1509
|
-
content: [
|
|
1510
|
-
{ type: "text", text: await handleUpdateEvaluator(params) }
|
|
1511
|
-
]
|
|
1522
|
+
content: [{ type: "text", text: await handleUpdateEvaluator(params) }]
|
|
1512
1523
|
};
|
|
1513
1524
|
})
|
|
1514
1525
|
);
|
|
@@ -1516,36 +1527,34 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1516
1527
|
"platform_delete_evaluator",
|
|
1517
1528
|
"Archive (soft-delete) an evaluator by ID or slug.",
|
|
1518
1529
|
{
|
|
1519
|
-
idOrSlug:
|
|
1530
|
+
idOrSlug: z3.string().describe("The evaluator ID or slug to archive")
|
|
1520
1531
|
},
|
|
1521
1532
|
withToolLogging("platform_delete_evaluator", async (params) => {
|
|
1522
1533
|
requireApiKey();
|
|
1523
|
-
const { handleDeleteEvaluator } = await import("./delete-evaluator-
|
|
1534
|
+
const { handleDeleteEvaluator } = await import("./delete-evaluator-E3CUNGTQ.js");
|
|
1524
1535
|
return {
|
|
1525
1536
|
content: [{ type: "text", text: await handleDeleteEvaluator(params) }]
|
|
1526
1537
|
};
|
|
1527
1538
|
})
|
|
1528
1539
|
);
|
|
1540
|
+
}
|
|
1541
|
+
function registerModelProviderTools(server) {
|
|
1529
1542
|
server.tool(
|
|
1530
1543
|
"platform_set_model_provider",
|
|
1531
1544
|
`Set or update a model provider on the LangWatch platform. Use this to configure API keys (e.g. OPENAI_API_KEY) needed to run evaluators. The API key is stored securely and never returned in responses. Omit customKeys to update other settings without changing existing keys.`,
|
|
1532
1545
|
{
|
|
1533
|
-
provider:
|
|
1534
|
-
|
|
1535
|
-
),
|
|
1536
|
-
enabled: z2.boolean().describe("Whether the provider is enabled"),
|
|
1537
|
-
customKeys: z2.record(z2.string(), z2.unknown()).optional().describe(
|
|
1546
|
+
provider: z3.string().describe('Provider name, e.g., "openai", "anthropic", "azure", "custom"'),
|
|
1547
|
+
enabled: z3.boolean().describe("Whether the provider is enabled"),
|
|
1548
|
+
customKeys: z3.record(z3.string(), z3.unknown()).optional().describe(
|
|
1538
1549
|
'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.'
|
|
1539
1550
|
),
|
|
1540
|
-
defaultModel:
|
|
1551
|
+
defaultModel: z3.string().optional().describe("Set as project default model")
|
|
1541
1552
|
},
|
|
1542
1553
|
withToolLogging("platform_set_model_provider", async (params) => {
|
|
1543
1554
|
requireApiKey();
|
|
1544
|
-
const { handleSetModelProvider } = await import("./set-model-provider-
|
|
1555
|
+
const { handleSetModelProvider } = await import("./set-model-provider-SHEVKWRC.js");
|
|
1545
1556
|
return {
|
|
1546
|
-
content: [
|
|
1547
|
-
{ type: "text", text: await handleSetModelProvider(params) }
|
|
1548
|
-
]
|
|
1557
|
+
content: [{ type: "text", text: await handleSetModelProvider(params) }]
|
|
1549
1558
|
};
|
|
1550
1559
|
})
|
|
1551
1560
|
);
|
|
@@ -1555,19 +1564,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1555
1564
|
{},
|
|
1556
1565
|
withToolLogging("platform_list_model_providers", async () => {
|
|
1557
1566
|
requireApiKey();
|
|
1558
|
-
const { handleListModelProviders } = await import("./list-model-providers-
|
|
1567
|
+
const { handleListModelProviders } = await import("./list-model-providers-OXDRPUCJ.js");
|
|
1559
1568
|
return {
|
|
1560
1569
|
content: [{ type: "text", text: await handleListModelProviders() }]
|
|
1561
1570
|
};
|
|
1562
1571
|
})
|
|
1563
1572
|
);
|
|
1573
|
+
}
|
|
1574
|
+
function registerAgentTools(server) {
|
|
1564
1575
|
server.tool(
|
|
1565
1576
|
"platform_list_agents",
|
|
1566
1577
|
"List all agents in the LangWatch project with their names, types and IDs. A connected agent (one that registered itself from code with connectAgent or connect_agent) also shows its environment, whether it is online, how many instances are connected, and its owner.",
|
|
1567
1578
|
{},
|
|
1568
1579
|
withToolLogging("platform_list_agents", async () => {
|
|
1569
1580
|
requireApiKey();
|
|
1570
|
-
const { handleListAgents } = await import("./list-agents-
|
|
1581
|
+
const { handleListAgents } = await import("./list-agents-BYWDNAHT.js");
|
|
1571
1582
|
return {
|
|
1572
1583
|
content: [{ type: "text", text: await handleListAgents() }]
|
|
1573
1584
|
};
|
|
@@ -1577,11 +1588,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1577
1588
|
"platform_get_agent",
|
|
1578
1589
|
"Get detailed information about a specific agent by its ID, including its configuration. A connected agent also shows its environment, status, the run parameters it declared (with options and defaults) and the instances connected right now.",
|
|
1579
1590
|
{
|
|
1580
|
-
id:
|
|
1591
|
+
id: z3.string().describe("The agent ID")
|
|
1581
1592
|
},
|
|
1582
1593
|
withToolLogging("platform_get_agent", async (params) => {
|
|
1583
1594
|
requireApiKey();
|
|
1584
|
-
const { handleGetAgent } = await import("./get-agent-
|
|
1595
|
+
const { handleGetAgent } = await import("./get-agent-ZKYQJUQ7.js");
|
|
1585
1596
|
return {
|
|
1586
1597
|
content: [{ type: "text", text: await handleGetAgent(params) }]
|
|
1587
1598
|
};
|
|
@@ -1591,16 +1602,18 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1591
1602
|
"platform_create_agent",
|
|
1592
1603
|
"Create a new agent. Supported types: 'signature' (LLM prompt), 'code' (Python), 'workflow' (sub-workflow), 'http' (external API). A 'connected' agent is not created here: it registers itself from code, with connectAgent from langwatch/agent in TypeScript or langwatch.connect_agent in Python, and appears in the list once that process runs.",
|
|
1593
1604
|
{
|
|
1594
|
-
name:
|
|
1595
|
-
type:
|
|
1596
|
-
config:
|
|
1605
|
+
name: z3.string().describe("Agent name"),
|
|
1606
|
+
type: z3.enum(["signature", "code", "workflow", "http"]).describe("Agent type"),
|
|
1607
|
+
config: z3.string().optional().describe("Agent configuration as JSON string (will be parsed)")
|
|
1597
1608
|
},
|
|
1598
1609
|
withToolLogging("platform_create_agent", async (params) => {
|
|
1599
1610
|
requireApiKey();
|
|
1600
|
-
const { handleCreateAgent } = await import("./create-agent-
|
|
1611
|
+
const { handleCreateAgent } = await import("./create-agent-6JNLMYQD.js");
|
|
1601
1612
|
const parsedConfig = params.config ? JSON.parse(params.config) : void 0;
|
|
1602
1613
|
return {
|
|
1603
|
-
content: [
|
|
1614
|
+
content: [
|
|
1615
|
+
{ type: "text", text: await handleCreateAgent({ ...params, config: parsedConfig }) }
|
|
1616
|
+
]
|
|
1604
1617
|
};
|
|
1605
1618
|
})
|
|
1606
1619
|
);
|
|
@@ -1608,17 +1621,19 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1608
1621
|
"platform_update_agent",
|
|
1609
1622
|
"Update an existing agent's name, type, or configuration.",
|
|
1610
1623
|
{
|
|
1611
|
-
id:
|
|
1612
|
-
name:
|
|
1613
|
-
type:
|
|
1614
|
-
config:
|
|
1624
|
+
id: z3.string().describe("The agent ID"),
|
|
1625
|
+
name: z3.string().optional().describe("New agent name"),
|
|
1626
|
+
type: z3.string().optional().describe("New agent type: signature, code, workflow, or http"),
|
|
1627
|
+
config: z3.string().optional().describe("Updated configuration as JSON string (will be parsed)")
|
|
1615
1628
|
},
|
|
1616
1629
|
withToolLogging("platform_update_agent", async (params) => {
|
|
1617
1630
|
requireApiKey();
|
|
1618
|
-
const { handleUpdateAgent } = await import("./update-agent-
|
|
1631
|
+
const { handleUpdateAgent } = await import("./update-agent-PTKTM2B5.js");
|
|
1619
1632
|
const parsedConfig = params.config ? JSON.parse(params.config) : void 0;
|
|
1620
1633
|
return {
|
|
1621
|
-
content: [
|
|
1634
|
+
content: [
|
|
1635
|
+
{ type: "text", text: await handleUpdateAgent({ ...params, config: parsedConfig }) }
|
|
1636
|
+
]
|
|
1622
1637
|
};
|
|
1623
1638
|
})
|
|
1624
1639
|
);
|
|
@@ -1626,11 +1641,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1626
1641
|
"platform_delete_agent",
|
|
1627
1642
|
"Archive (soft-delete) an agent by its ID.",
|
|
1628
1643
|
{
|
|
1629
|
-
id:
|
|
1644
|
+
id: z3.string().describe("The agent ID to archive")
|
|
1630
1645
|
},
|
|
1631
1646
|
withToolLogging("platform_delete_agent", async (params) => {
|
|
1632
1647
|
requireApiKey();
|
|
1633
|
-
const { handleDeleteAgent } = await import("./delete-agent-
|
|
1648
|
+
const { handleDeleteAgent } = await import("./delete-agent-J2Q4DRZ2.js");
|
|
1634
1649
|
return {
|
|
1635
1650
|
content: [{ type: "text", text: await handleDeleteAgent(params) }]
|
|
1636
1651
|
};
|
|
@@ -1640,17 +1655,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1640
1655
|
"platform_run_agent",
|
|
1641
1656
|
"Run one turn of an agent. A connected agent runs through the platform relay on a live instance: give `message` (one user turn) or `input` with a `messages` list, plus `parameters` and `threadId` to continue a conversation. HTTP agents call their configured URL directly; workflow-linked agents execute via the workflow engine.",
|
|
1642
1657
|
{
|
|
1643
|
-
id:
|
|
1644
|
-
input:
|
|
1658
|
+
id: z3.string().describe("The agent ID to run"),
|
|
1659
|
+
input: z3.string().optional().describe(
|
|
1645
1660
|
"Input data as a JSON object string. For a connected agent it is the relay body: messages (OpenAI style), and optionally threadId, session and params."
|
|
1646
1661
|
),
|
|
1647
|
-
message:
|
|
1648
|
-
parameters:
|
|
1649
|
-
threadId:
|
|
1662
|
+
message: z3.string().optional().describe("One user message to send to a connected agent, instead of input"),
|
|
1663
|
+
parameters: z3.record(z3.string(), z3.union([z3.string(), z3.number(), z3.boolean()])).optional().describe("Run parameter values for a connected agent, by name"),
|
|
1664
|
+
threadId: z3.string().optional().describe("Continue a conversation on a connected agent")
|
|
1650
1665
|
},
|
|
1651
1666
|
withToolLogging("platform_run_agent", async (params) => {
|
|
1652
1667
|
requireApiKey();
|
|
1653
|
-
const { handleRunAgent } = await import("./run-agent-
|
|
1668
|
+
const { handleRunAgent } = await import("./run-agent-244O4Y5M.js");
|
|
1654
1669
|
return {
|
|
1655
1670
|
content: [{ type: "text", text: await handleRunAgent(params) }]
|
|
1656
1671
|
};
|
|
@@ -1660,7 +1675,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1660
1675
|
"platform_test_agent",
|
|
1661
1676
|
'Test an agent with one scripted scenario run: the user sends "ping", the agent answers, and the run succeeds when the answer arrives. No model is used, and no scenario, run plan or test suite is added to the project. Answers at once with the scenario run id to follow with platform_get_simulation_run; the run itself is asynchronous.',
|
|
1662
1677
|
{
|
|
1663
|
-
id:
|
|
1678
|
+
id: z3.string().describe("The agent ID to test")
|
|
1664
1679
|
},
|
|
1665
1680
|
withToolLogging("platform_test_agent", async (params) => {
|
|
1666
1681
|
requireApiKey();
|
|
@@ -1669,83 +1684,250 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1669
1684
|
};
|
|
1670
1685
|
})
|
|
1671
1686
|
);
|
|
1687
|
+
}
|
|
1688
|
+
function registerAnalyticsReportTools(server) {
|
|
1672
1689
|
server.tool(
|
|
1673
|
-
"
|
|
1674
|
-
"List
|
|
1690
|
+
"platform_list_analytics_reports",
|
|
1691
|
+
"List the Analytics Reports in the LangWatch project (/api/v1/analytics/reports). Dashboards, which hold widgets, are listed by list_dashboards.",
|
|
1675
1692
|
{},
|
|
1676
|
-
withToolLogging("
|
|
1693
|
+
withToolLogging("platform_list_analytics_reports", async () => {
|
|
1677
1694
|
requireApiKey();
|
|
1678
|
-
const {
|
|
1695
|
+
const { handleListAnalyticsReports } = await import("./list-analytics-reports-PPWIFRES.js");
|
|
1679
1696
|
return {
|
|
1680
|
-
content: [{ type: "text", text: await
|
|
1697
|
+
content: [{ type: "text", text: await handleListAnalyticsReports() }]
|
|
1698
|
+
};
|
|
1699
|
+
})
|
|
1700
|
+
);
|
|
1701
|
+
server.tool(
|
|
1702
|
+
"platform_get_analytics_report",
|
|
1703
|
+
"Get an Analytics Report by its ID, including its graphs. A Dashboards dashboard's ID is not found here; read it with get_dashboard.",
|
|
1704
|
+
{
|
|
1705
|
+
reportId: z3.string().describe("The Analytics Report ID")
|
|
1706
|
+
},
|
|
1707
|
+
withToolLogging("platform_get_analytics_report", async (params) => {
|
|
1708
|
+
requireApiKey();
|
|
1709
|
+
const { handleGetAnalyticsReport } = await import("./get-analytics-report-SKIP2JWH.js");
|
|
1710
|
+
return {
|
|
1711
|
+
content: [{ type: "text", text: await handleGetAnalyticsReport(params) }]
|
|
1681
1712
|
};
|
|
1682
1713
|
})
|
|
1683
1714
|
);
|
|
1684
1715
|
server.tool(
|
|
1685
|
-
"
|
|
1686
|
-
"
|
|
1716
|
+
"platform_create_analytics_report",
|
|
1717
|
+
"Create a new Analytics Report.",
|
|
1687
1718
|
{
|
|
1688
|
-
|
|
1719
|
+
name: z3.string().describe("Analytics Report name")
|
|
1689
1720
|
},
|
|
1690
|
-
withToolLogging("
|
|
1721
|
+
withToolLogging("platform_create_analytics_report", async (params) => {
|
|
1691
1722
|
requireApiKey();
|
|
1692
|
-
const {
|
|
1723
|
+
const { handleCreateAnalyticsReport } = await import("./create-analytics-report-N4L4V5QL.js");
|
|
1693
1724
|
return {
|
|
1694
|
-
content: [
|
|
1725
|
+
content: [
|
|
1726
|
+
{
|
|
1727
|
+
type: "text",
|
|
1728
|
+
text: await handleCreateAnalyticsReport(params)
|
|
1729
|
+
}
|
|
1730
|
+
]
|
|
1695
1731
|
};
|
|
1696
1732
|
})
|
|
1697
1733
|
);
|
|
1698
1734
|
server.tool(
|
|
1699
|
-
"
|
|
1700
|
-
"
|
|
1735
|
+
"platform_delete_analytics_report",
|
|
1736
|
+
"Delete an Analytics Report and all its graphs.",
|
|
1701
1737
|
{
|
|
1702
|
-
|
|
1738
|
+
reportId: z3.string().describe("The Analytics Report ID to delete")
|
|
1703
1739
|
},
|
|
1704
|
-
withToolLogging("
|
|
1740
|
+
withToolLogging("platform_delete_analytics_report", async (params) => {
|
|
1705
1741
|
requireApiKey();
|
|
1706
|
-
const {
|
|
1742
|
+
const { handleDeleteAnalyticsReport } = await import("./delete-analytics-report-QK6X7P4S.js");
|
|
1707
1743
|
return {
|
|
1708
|
-
content: [
|
|
1744
|
+
content: [
|
|
1745
|
+
{
|
|
1746
|
+
type: "text",
|
|
1747
|
+
text: await handleDeleteAnalyticsReport(params)
|
|
1748
|
+
}
|
|
1749
|
+
]
|
|
1709
1750
|
};
|
|
1710
1751
|
})
|
|
1711
1752
|
);
|
|
1712
1753
|
server.tool(
|
|
1713
|
-
"
|
|
1714
|
-
"
|
|
1754
|
+
"platform_rename_analytics_report",
|
|
1755
|
+
"Rename an Analytics Report.",
|
|
1715
1756
|
{
|
|
1716
|
-
|
|
1757
|
+
reportId: z3.string().describe("The Analytics Report ID to rename"),
|
|
1758
|
+
name: z3.string().describe("The new name for the Analytics Report")
|
|
1717
1759
|
},
|
|
1718
|
-
withToolLogging("
|
|
1760
|
+
withToolLogging("platform_rename_analytics_report", async (params) => {
|
|
1719
1761
|
requireApiKey();
|
|
1720
|
-
const {
|
|
1762
|
+
const { renameAnalyticsReport } = await import("./langwatch-api-analytics-reports-RPFB5AO5.js");
|
|
1763
|
+
const result = await renameAnalyticsReport({ reportId: params.reportId, name: params.name });
|
|
1721
1764
|
return {
|
|
1722
|
-
content: [
|
|
1765
|
+
content: [
|
|
1766
|
+
{
|
|
1767
|
+
type: "text",
|
|
1768
|
+
text: `Analytics Report "${result.name}" renamed successfully (ID: ${result.id}).`
|
|
1769
|
+
}
|
|
1770
|
+
]
|
|
1723
1771
|
};
|
|
1724
1772
|
})
|
|
1725
1773
|
);
|
|
1774
|
+
}
|
|
1775
|
+
function registerDashboardTools(server) {
|
|
1726
1776
|
server.tool(
|
|
1727
|
-
"
|
|
1728
|
-
"
|
|
1777
|
+
"list_dashboards",
|
|
1778
|
+
"Lists the project's dashboards in the Dashboards area, with each one's ID and widget count. Use an ID from here as the dashboardId of add_dashboard_widget and update_dashboard_widget; the Analytics Reports platform_list_analytics_reports lists hold no widgets.",
|
|
1779
|
+
{},
|
|
1780
|
+
withToolLogging("list_dashboards", async () => {
|
|
1781
|
+
requireApiKey();
|
|
1782
|
+
const { handleListDashboards } = await import("./list-dashboards-MW5O3MKH.js");
|
|
1783
|
+
return {
|
|
1784
|
+
content: [{ type: "text", text: await handleListDashboards() }]
|
|
1785
|
+
};
|
|
1786
|
+
})
|
|
1787
|
+
);
|
|
1788
|
+
server.tool(
|
|
1789
|
+
"add_dashboard_widget",
|
|
1790
|
+
"Creates a custom dashboard widget and places it on the given dashboard. A widget's `code` is a TSX file run in a sandbox: it receives a global `LW` with `LW.useChartQuery(name)` (reads one of the widget's own named queries), `LW.navigate` and `LW.theme`. `queries` are named LangWatchQL SQL statements the code can read by name \u2014 validate each statement's SQL first with run_query before calling this tool, since a query only runs inside the sandbox at render time and a mistake surfaces there, not here. Up to 8 queries per widget.",
|
|
1729
1791
|
{
|
|
1730
|
-
|
|
1731
|
-
name:
|
|
1792
|
+
dashboardId: z3.string().describe("The dashboard to add the widget to, as list_dashboards lists it"),
|
|
1793
|
+
name: z3.string().describe("The widget's display name"),
|
|
1794
|
+
code: z3.string().describe(
|
|
1795
|
+
"The widget's TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope."
|
|
1796
|
+
),
|
|
1797
|
+
queries: z3.union([
|
|
1798
|
+
z3.record(z3.string(), z3.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1799
|
+
z3.array(
|
|
1800
|
+
z3.object({
|
|
1801
|
+
name: z3.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1802
|
+
sql: z3.string().describe("The LangWatchQL statement")
|
|
1803
|
+
})
|
|
1804
|
+
).describe("Named LWQL queries, array form")
|
|
1805
|
+
]).describe(
|
|
1806
|
+
"The widget's named LangWatchQL queries: either {name: sql} or an array of {name, sql}. Max 8. Validate each statement with run_query first."
|
|
1807
|
+
),
|
|
1808
|
+
sources: widgetSourcesInput
|
|
1732
1809
|
},
|
|
1733
|
-
withToolLogging("
|
|
1810
|
+
withToolLogging("add_dashboard_widget", async (params) => {
|
|
1734
1811
|
requireApiKey();
|
|
1735
|
-
const {
|
|
1736
|
-
const result = await renameDashboard(params.id, { name: params.name });
|
|
1812
|
+
const { handleAddDashboardWidget } = await import("./add-dashboard-widget-MJHIC3CC.js");
|
|
1737
1813
|
return {
|
|
1738
|
-
content: [{ type: "text", text:
|
|
1814
|
+
content: [{ type: "text", text: await handleAddDashboardWidget(params) }]
|
|
1739
1815
|
};
|
|
1740
1816
|
})
|
|
1741
1817
|
);
|
|
1818
|
+
server.tool(
|
|
1819
|
+
"update_dashboard_widget",
|
|
1820
|
+
"Changes one custom widget on a dashboard: its name, its TSX `code`, its named LangWatchQL `queries`, its `description` or the `sources` its queries count. Pass only the fields to change; every field left out keeps its stored value. Validate any new query's SQL with run_query first. Returns the updated widget.",
|
|
1821
|
+
{
|
|
1822
|
+
dashboardId: z3.string().describe("The dashboard the widget is on"),
|
|
1823
|
+
widgetId: z3.string().describe("The widget to change, as add_dashboard_widget returned it"),
|
|
1824
|
+
name: z3.string().optional().describe("The widget's new display name"),
|
|
1825
|
+
code: z3.string().optional().describe(
|
|
1826
|
+
"The widget's new TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope."
|
|
1827
|
+
),
|
|
1828
|
+
queries: z3.union([
|
|
1829
|
+
z3.record(z3.string(), z3.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1830
|
+
z3.array(
|
|
1831
|
+
z3.object({
|
|
1832
|
+
name: z3.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1833
|
+
sql: z3.string().describe("The LangWatchQL statement")
|
|
1834
|
+
})
|
|
1835
|
+
).describe("Named LWQL queries, array form")
|
|
1836
|
+
]).optional().describe(
|
|
1837
|
+
"The widget's new named LangWatchQL queries, replacing the stored set: either {name: sql} or an array of {name, sql}. Max 8."
|
|
1838
|
+
),
|
|
1839
|
+
description: z3.string().optional().describe("What the widget's info tip says: a sentence or two"),
|
|
1840
|
+
sources: widgetSourcesInput
|
|
1841
|
+
},
|
|
1842
|
+
withToolLogging("update_dashboard_widget", async (params) => {
|
|
1843
|
+
requireApiKey();
|
|
1844
|
+
const { handleUpdateDashboardWidget } = await import("./update-dashboard-widget-IP2TTQ5G.js");
|
|
1845
|
+
return {
|
|
1846
|
+
content: [{ type: "text", text: await handleUpdateDashboardWidget(params) }]
|
|
1847
|
+
};
|
|
1848
|
+
})
|
|
1849
|
+
);
|
|
1850
|
+
registerDashboardCrudTools(server);
|
|
1851
|
+
}
|
|
1852
|
+
function registerDashboardCrudTools(server) {
|
|
1853
|
+
const dashboardId = z3.string().describe("The dashboard's ID, as list_dashboards lists it");
|
|
1854
|
+
const text = (value) => ({ content: [{ type: "text", text: value }] });
|
|
1855
|
+
server.tool(
|
|
1856
|
+
"get_dashboard",
|
|
1857
|
+
"Reads one dashboard in the Dashboards area: its name, scope, description and address. An Analytics Report's ID is not found here; read it with platform_get_analytics_report.",
|
|
1858
|
+
{ dashboardId },
|
|
1859
|
+
withToolLogging("get_dashboard", async (params) => {
|
|
1860
|
+
requireApiKey();
|
|
1861
|
+
const { handleGetDashboard } = await import("./dashboards-U663GUIB.js");
|
|
1862
|
+
return text(await handleGetDashboard(params));
|
|
1863
|
+
})
|
|
1864
|
+
);
|
|
1865
|
+
server.tool(
|
|
1866
|
+
"create_dashboard",
|
|
1867
|
+
"Creates an empty dashboard in the Dashboards area. Add widgets to it with add_dashboard_widget.",
|
|
1868
|
+
{ name: z3.string().describe("The dashboard's name") },
|
|
1869
|
+
withToolLogging("create_dashboard", async (params) => {
|
|
1870
|
+
requireApiKey();
|
|
1871
|
+
const { handleCreateDashboard } = await import("./dashboards-U663GUIB.js");
|
|
1872
|
+
return text(await handleCreateDashboard(params));
|
|
1873
|
+
})
|
|
1874
|
+
);
|
|
1875
|
+
server.tool(
|
|
1876
|
+
"rename_dashboard",
|
|
1877
|
+
"Changes a dashboard's name and/or description; a field left out keeps its value.",
|
|
1878
|
+
{
|
|
1879
|
+
dashboardId,
|
|
1880
|
+
name: z3.string().optional().describe("The dashboard's new name"),
|
|
1881
|
+
description: z3.string().optional().describe("The dashboard's new description")
|
|
1882
|
+
},
|
|
1883
|
+
withToolLogging("rename_dashboard", async (params) => {
|
|
1884
|
+
requireApiKey();
|
|
1885
|
+
const { handleRenameDashboard } = await import("./dashboards-U663GUIB.js");
|
|
1886
|
+
return text(await handleRenameDashboard(params));
|
|
1887
|
+
})
|
|
1888
|
+
);
|
|
1889
|
+
server.tool(
|
|
1890
|
+
"delete_dashboard",
|
|
1891
|
+
"Deletes a dashboard together with the widgets on it. This cannot be undone.",
|
|
1892
|
+
{ dashboardId },
|
|
1893
|
+
withToolLogging("delete_dashboard", async (params) => {
|
|
1894
|
+
requireApiKey();
|
|
1895
|
+
const { handleDeleteDashboard } = await import("./dashboards-U663GUIB.js");
|
|
1896
|
+
return text(await handleDeleteDashboard(params));
|
|
1897
|
+
})
|
|
1898
|
+
);
|
|
1899
|
+
server.tool(
|
|
1900
|
+
"list_dashboard_widgets",
|
|
1901
|
+
"Lists the custom widgets on one dashboard, with each widget's ID and query names.",
|
|
1902
|
+
{ dashboardId },
|
|
1903
|
+
withToolLogging("list_dashboard_widgets", async (params) => {
|
|
1904
|
+
requireApiKey();
|
|
1905
|
+
const { handleListDashboardWidgets } = await import("./dashboards-U663GUIB.js");
|
|
1906
|
+
return text(await handleListDashboardWidgets(params));
|
|
1907
|
+
})
|
|
1908
|
+
);
|
|
1909
|
+
server.tool(
|
|
1910
|
+
"delete_dashboard_widget",
|
|
1911
|
+
"Deletes one custom widget from a dashboard. A widget on another dashboard is not found.",
|
|
1912
|
+
{
|
|
1913
|
+
dashboardId,
|
|
1914
|
+
widgetId: z3.string().describe("The widget's ID, as list_dashboard_widgets lists it")
|
|
1915
|
+
},
|
|
1916
|
+
withToolLogging("delete_dashboard_widget", async (params) => {
|
|
1917
|
+
requireApiKey();
|
|
1918
|
+
const { handleDeleteDashboardWidget } = await import("./dashboards-U663GUIB.js");
|
|
1919
|
+
return text(await handleDeleteDashboardWidget(params));
|
|
1920
|
+
})
|
|
1921
|
+
);
|
|
1922
|
+
}
|
|
1923
|
+
function registerWorkflowTools(server) {
|
|
1742
1924
|
server.tool(
|
|
1743
1925
|
"platform_list_workflows",
|
|
1744
1926
|
"List all workflows in the LangWatch project.",
|
|
1745
1927
|
{},
|
|
1746
1928
|
withToolLogging("platform_list_workflows", async () => {
|
|
1747
1929
|
requireApiKey();
|
|
1748
|
-
const { handleListWorkflows } = await import("./list-workflows-
|
|
1930
|
+
const { handleListWorkflows } = await import("./list-workflows-HV7FUWZA.js");
|
|
1749
1931
|
return {
|
|
1750
1932
|
content: [{ type: "text", text: await handleListWorkflows() }]
|
|
1751
1933
|
};
|
|
@@ -1755,11 +1937,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1755
1937
|
"platform_get_workflow",
|
|
1756
1938
|
"Get a workflow by its ID.",
|
|
1757
1939
|
{
|
|
1758
|
-
id:
|
|
1940
|
+
id: z3.string().describe("The workflow ID")
|
|
1759
1941
|
},
|
|
1760
1942
|
withToolLogging("platform_get_workflow", async (params) => {
|
|
1761
1943
|
requireApiKey();
|
|
1762
|
-
const { handleGetWorkflow } = await import("./get-workflow-
|
|
1944
|
+
const { handleGetWorkflow } = await import("./get-workflow-7LZSVPP2.js");
|
|
1763
1945
|
return {
|
|
1764
1946
|
content: [{ type: "text", text: await handleGetWorkflow(params) }]
|
|
1765
1947
|
};
|
|
@@ -1769,11 +1951,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1769
1951
|
"platform_delete_workflow",
|
|
1770
1952
|
"Archive (soft-delete) a workflow by its ID.",
|
|
1771
1953
|
{
|
|
1772
|
-
id:
|
|
1954
|
+
id: z3.string().describe("The workflow ID to archive")
|
|
1773
1955
|
},
|
|
1774
1956
|
withToolLogging("platform_delete_workflow", async (params) => {
|
|
1775
1957
|
requireApiKey();
|
|
1776
|
-
const { handleDeleteWorkflow } = await import("./delete-workflow-
|
|
1958
|
+
const { handleDeleteWorkflow } = await import("./delete-workflow-PWQPDW6H.js");
|
|
1777
1959
|
return {
|
|
1778
1960
|
content: [{ type: "text", text: await handleDeleteWorkflow(params) }]
|
|
1779
1961
|
};
|
|
@@ -1783,26 +1965,28 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1783
1965
|
"platform_run_workflow",
|
|
1784
1966
|
"Execute a workflow with JSON input. Returns the workflow output.",
|
|
1785
1967
|
{
|
|
1786
|
-
id:
|
|
1787
|
-
input:
|
|
1968
|
+
id: z3.string().describe("The workflow ID to run"),
|
|
1969
|
+
input: z3.string().optional().describe("Input data as a JSON object string")
|
|
1788
1970
|
},
|
|
1789
1971
|
withToolLogging("platform_run_workflow", async (params) => {
|
|
1790
1972
|
requireApiKey();
|
|
1791
|
-
const { handleRunWorkflow } = await import("./run-workflow-
|
|
1973
|
+
const { handleRunWorkflow } = await import("./run-workflow-I4IHEN54.js");
|
|
1792
1974
|
return {
|
|
1793
1975
|
content: [{ type: "text", text: await handleRunWorkflow(params) }]
|
|
1794
1976
|
};
|
|
1795
1977
|
})
|
|
1796
1978
|
);
|
|
1979
|
+
}
|
|
1980
|
+
function registerAnnotationTools(server) {
|
|
1797
1981
|
server.tool(
|
|
1798
1982
|
"platform_list_annotations",
|
|
1799
1983
|
"List all annotations for the project, optionally filtered by trace ID.",
|
|
1800
1984
|
{
|
|
1801
|
-
traceId:
|
|
1985
|
+
traceId: z3.string().optional().describe("Filter annotations by trace ID")
|
|
1802
1986
|
},
|
|
1803
1987
|
withToolLogging("platform_list_annotations", async (params) => {
|
|
1804
1988
|
requireApiKey();
|
|
1805
|
-
const { handleListAnnotations } = await import("./list-annotations-
|
|
1989
|
+
const { handleListAnnotations } = await import("./list-annotations-Y4EUBJHG.js");
|
|
1806
1990
|
return {
|
|
1807
1991
|
content: [{ type: "text", text: await handleListAnnotations(params) }]
|
|
1808
1992
|
};
|
|
@@ -1812,14 +1996,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1812
1996
|
"platform_create_annotation",
|
|
1813
1997
|
"Create an annotation (thumbs up/down, comment) for a trace.",
|
|
1814
1998
|
{
|
|
1815
|
-
traceId:
|
|
1816
|
-
comment:
|
|
1817
|
-
isThumbsUp:
|
|
1818
|
-
email:
|
|
1999
|
+
traceId: z3.string().describe("The trace ID to annotate"),
|
|
2000
|
+
comment: z3.string().optional().describe("Annotation comment"),
|
|
2001
|
+
isThumbsUp: z3.boolean().optional().describe("True for positive feedback, false for negative"),
|
|
2002
|
+
email: z3.string().optional().describe("Email of the annotator")
|
|
1819
2003
|
},
|
|
1820
2004
|
withToolLogging("platform_create_annotation", async (params) => {
|
|
1821
2005
|
requireApiKey();
|
|
1822
|
-
const { handleCreateAnnotation } = await import("./create-annotation-
|
|
2006
|
+
const { handleCreateAnnotation } = await import("./create-annotation-CCEGIZ67.js");
|
|
1823
2007
|
return {
|
|
1824
2008
|
content: [{ type: "text", text: await handleCreateAnnotation(params) }]
|
|
1825
2009
|
};
|
|
@@ -1829,11 +2013,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1829
2013
|
"platform_delete_annotation",
|
|
1830
2014
|
"Delete an annotation by its ID.",
|
|
1831
2015
|
{
|
|
1832
|
-
id:
|
|
2016
|
+
id: z3.string().describe("The annotation ID to delete")
|
|
1833
2017
|
},
|
|
1834
2018
|
withToolLogging("platform_delete_annotation", async (params) => {
|
|
1835
2019
|
requireApiKey();
|
|
1836
|
-
const { handleDeleteAnnotation } = await import("./delete-annotation-
|
|
2020
|
+
const { handleDeleteAnnotation } = await import("./delete-annotation-KJOCDLV3.js");
|
|
1837
2021
|
return {
|
|
1838
2022
|
content: [{ type: "text", text: await handleDeleteAnnotation(params) }]
|
|
1839
2023
|
};
|
|
@@ -1843,11 +2027,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1843
2027
|
"platform_get_annotation",
|
|
1844
2028
|
"Get annotation details by ID.",
|
|
1845
2029
|
{
|
|
1846
|
-
id:
|
|
2030
|
+
id: z3.string().describe("The annotation ID")
|
|
1847
2031
|
},
|
|
1848
2032
|
withToolLogging("platform_get_annotation", async (params) => {
|
|
1849
2033
|
requireApiKey();
|
|
1850
|
-
const { getAnnotation } = await import("./langwatch-api-annotations-
|
|
2034
|
+
const { getAnnotation } = await import("./langwatch-api-annotations-H4AI4TKG.js");
|
|
1851
2035
|
const annotation = await getAnnotation(params.id);
|
|
1852
2036
|
const lines = [
|
|
1853
2037
|
`**ID**: ${annotation.id}`,
|
|
@@ -1861,103 +2045,162 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1861
2045
|
};
|
|
1862
2046
|
})
|
|
1863
2047
|
);
|
|
2048
|
+
}
|
|
2049
|
+
function registerTriggerTools(server) {
|
|
1864
2050
|
server.tool(
|
|
1865
2051
|
"platform_list_triggers",
|
|
1866
2052
|
"List all triggers (automations) in the project. Triggers automate actions like sending emails, Slack messages, or adding to datasets when conditions are met.",
|
|
1867
2053
|
{
|
|
1868
|
-
format:
|
|
2054
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format")
|
|
1869
2055
|
},
|
|
1870
2056
|
withToolLogging("platform_list_triggers", async (params) => {
|
|
1871
2057
|
requireApiKey();
|
|
1872
|
-
const {
|
|
1873
|
-
|
|
1874
|
-
if (params.format === "json") {
|
|
1875
|
-
return { content: [{ type: "text", text: JSON.stringify(triggers, null, 2) }] };
|
|
1876
|
-
}
|
|
1877
|
-
if (triggers.length === 0) {
|
|
1878
|
-
return { content: [{ type: "text", text: "No triggers found. Use `platform_create_trigger` to create one." }] };
|
|
1879
|
-
}
|
|
1880
|
-
const lines = [`# Triggers (${triggers.length} total)
|
|
1881
|
-
`];
|
|
1882
|
-
for (const t of triggers) {
|
|
1883
|
-
lines.push(`## ${t.name}`);
|
|
1884
|
-
lines.push(`**ID**: ${t.id}`);
|
|
1885
|
-
lines.push(`**Action**: ${t.action}`);
|
|
1886
|
-
lines.push(`**Status**: ${t.active ? "active" : "inactive"}`);
|
|
1887
|
-
if (t.alertType) lines.push(`**Alert**: ${t.alertType}`);
|
|
1888
|
-
lines.push("");
|
|
1889
|
-
}
|
|
1890
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
2058
|
+
const { handleListTriggers } = await import("./list-triggers-XC5KMX4C.js");
|
|
2059
|
+
return { content: [{ type: "text", text: await handleListTriggers(params) }] };
|
|
1891
2060
|
})
|
|
1892
2061
|
);
|
|
1893
2062
|
server.tool(
|
|
1894
|
-
"
|
|
1895
|
-
"
|
|
2063
|
+
"platform_get_trigger",
|
|
2064
|
+
"Read one trigger (automation) by its ID, including its delivery configuration. Credential values come back as [redacted]; sending them back unchanged on an update keeps the stored values.",
|
|
1896
2065
|
{
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
2066
|
+
id: z3.string().describe("The trigger ID")
|
|
2067
|
+
},
|
|
2068
|
+
withToolLogging("platform_get_trigger", async (params) => {
|
|
2069
|
+
requireApiKey();
|
|
2070
|
+
const { getTrigger } = await import("./langwatch-api-triggers-QMR7KEXW.js");
|
|
2071
|
+
const trigger = await getTrigger(params.id);
|
|
2072
|
+
return { content: [{ type: "text", text: JSON.stringify(trigger, null, 2) }] };
|
|
2073
|
+
})
|
|
2074
|
+
);
|
|
2075
|
+
server.tool(
|
|
2076
|
+
"platform_create_trigger",
|
|
2077
|
+
[
|
|
2078
|
+
"Create a trigger (automation).",
|
|
2079
|
+
"",
|
|
2080
|
+
"It is about one of three things: matching traces (send `filters` or `filterQuery`), a metric crossing a threshold on a graph (an Alert: send `customGraphId`, `graphAlert` and `alertType`), or a schedule (a Report: send `report`, on an email or Slack channel).",
|
|
2081
|
+
"`actionParams` is the delivery configuration for the channel named in `action` \u2014 recipients for SEND_EMAIL, a destination for SEND_SLACK_MESSAGE, a URL for SEND_WEBHOOK, a dataset and mapping for ADD_TO_DATASET, annotators for ADD_TO_ANNOTATION_QUEUE. Omitted, it is sent as {} and the server refuses a channel that cannot deliver without one.",
|
|
2082
|
+
SLACK_DELIVERY_NOTE,
|
|
2083
|
+
`filters: ${TRIGGER_FILTERS_DESCRIPTION}`,
|
|
2084
|
+
"The delivery channel cannot be changed afterwards."
|
|
2085
|
+
].join("\n"),
|
|
2086
|
+
{
|
|
2087
|
+
name: z3.string().describe("Trigger name"),
|
|
2088
|
+
action: triggerActionSchema.describe("Which channel it delivers on"),
|
|
2089
|
+
actionParams: actionParamsSchema.optional().describe("The delivery configuration the channel named in `action` reads. Defaults to {}"),
|
|
2090
|
+
filters: z3.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
2091
|
+
filterQuery: z3.string().optional().describe(TRIGGER_FILTER_QUERY_DESCRIPTION),
|
|
2092
|
+
customGraphId: z3.string().optional().describe("Set to make this an alert on that graph"),
|
|
2093
|
+
graphAlert: graphAlertSchema.optional(),
|
|
2094
|
+
report: reportSchema.optional().describe("Set to make this a scheduled report"),
|
|
2095
|
+
templates: templatesSchema.optional(),
|
|
2096
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
2097
|
+
message: z3.string().optional().describe("Custom alert message"),
|
|
2098
|
+
alertType: alertTypeSchema.optional().describe("Alert severity. Required for a graph alert.")
|
|
1902
2099
|
},
|
|
1903
2100
|
withToolLogging("platform_create_trigger", async (params) => {
|
|
1904
2101
|
requireApiKey();
|
|
1905
|
-
const {
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
2102
|
+
const { handleCreateTrigger } = await import("./create-trigger-RQ653WYF.js");
|
|
2103
|
+
return handleCreateTrigger(params);
|
|
2104
|
+
})
|
|
2105
|
+
);
|
|
2106
|
+
server.tool(
|
|
2107
|
+
"platform_update_trigger",
|
|
2108
|
+
[
|
|
2109
|
+
"Update a trigger (automation). Anything left out is left alone.",
|
|
2110
|
+
"",
|
|
2111
|
+
"`actionParams` replaces the delivery configuration as a whole, so send every field it should have from now on. A credential the read hid comes back as [redacted]; send that back to keep the stored value. Changing a webhook's `url` means sending its header values in the same call.",
|
|
2112
|
+
"The delivery channel, the kind of automation and an alert's graph cannot be changed."
|
|
2113
|
+
].join("\n"),
|
|
2114
|
+
{
|
|
2115
|
+
id: z3.string().describe("The trigger ID"),
|
|
2116
|
+
name: z3.string().optional().describe("New name"),
|
|
2117
|
+
active: z3.boolean().optional().describe("Resume or pause it"),
|
|
2118
|
+
actionParams: actionParamsSchema.optional().describe("The delivery configuration this automation should have from now on"),
|
|
2119
|
+
filters: z3.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
2120
|
+
filterQuery: z3.string().nullable().optional().describe(`${TRIGGER_FILTER_QUERY_DESCRIPTION} null clears the saved query.`),
|
|
2121
|
+
graphAlert: graphAlertSchema.optional().describe("Only for an automation that is already a graph alert"),
|
|
2122
|
+
report: reportSchema.optional().describe("Only for an automation that is already a scheduled report"),
|
|
2123
|
+
templates: templatesSchema.optional(),
|
|
2124
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
2125
|
+
message: z3.string().nullable().optional().describe("New alert message. null removes the custom message"),
|
|
2126
|
+
alertType: alertTypeSchema.nullable().optional().describe("New alert severity. null clears it")
|
|
2127
|
+
},
|
|
2128
|
+
withToolLogging("platform_update_trigger", async (params) => {
|
|
2129
|
+
requireApiKey();
|
|
2130
|
+
const { getTrigger, updateTrigger } = await import("./langwatch-api-triggers-QMR7KEXW.js");
|
|
2131
|
+
if (params.actionParams !== void 0) {
|
|
2132
|
+
const saved = await getTrigger(params.id);
|
|
2133
|
+
const boundParams = validateActionParamsForAction({
|
|
2134
|
+
action: saved.action,
|
|
2135
|
+
actionParams: params.actionParams
|
|
2136
|
+
});
|
|
2137
|
+
if (!boundParams.ok) {
|
|
2138
|
+
const { toolError: toolError2 } = await import("./create-trigger-RQ653WYF.js");
|
|
2139
|
+
return toolError2(boundParams.message);
|
|
1912
2140
|
}
|
|
1913
2141
|
}
|
|
1914
|
-
const
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
filters
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
2142
|
+
const { parseJsonObject, toolError } = await import("./create-trigger-RQ653WYF.js");
|
|
2143
|
+
const filters = params.filters ? parseJsonObject(params.filters) : void 0;
|
|
2144
|
+
if (params.filters && !filters) {
|
|
2145
|
+
return toolError("filters must be a JSON object");
|
|
2146
|
+
}
|
|
2147
|
+
const trigger = await updateTrigger({ ...params, filters });
|
|
2148
|
+
return {
|
|
2149
|
+
content: [
|
|
2150
|
+
{ type: "text", text: `Trigger "${trigger.name}" updated (active: ${trigger.active}).` }
|
|
2151
|
+
]
|
|
2152
|
+
};
|
|
1922
2153
|
})
|
|
1923
2154
|
);
|
|
1924
2155
|
server.tool(
|
|
1925
|
-
"
|
|
1926
|
-
"
|
|
2156
|
+
"platform_test_fire_trigger",
|
|
2157
|
+
"Send a trigger's message to the destination it is configured with, to confirm it arrives. Nothing is recorded as a fire.",
|
|
1927
2158
|
{
|
|
1928
|
-
id:
|
|
1929
|
-
name: z2.string().optional().describe("New name"),
|
|
1930
|
-
active: z2.boolean().optional().describe("Enable or disable"),
|
|
1931
|
-
message: z2.string().optional().describe("New alert message"),
|
|
1932
|
-
alertType: z2.enum(["CRITICAL", "WARNING", "INFO"]).optional().describe("New alert severity")
|
|
2159
|
+
id: z3.string().describe("The trigger ID")
|
|
1933
2160
|
},
|
|
1934
|
-
withToolLogging("
|
|
2161
|
+
withToolLogging("platform_test_fire_trigger", async (params) => {
|
|
1935
2162
|
requireApiKey();
|
|
1936
|
-
const {
|
|
1937
|
-
const
|
|
1938
|
-
return { content: [{ type: "text", text:
|
|
2163
|
+
const { testFireTrigger } = await import("./langwatch-api-triggers-QMR7KEXW.js");
|
|
2164
|
+
const result = await testFireTrigger(params.id);
|
|
2165
|
+
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
2166
|
+
})
|
|
2167
|
+
);
|
|
2168
|
+
server.tool(
|
|
2169
|
+
"platform_list_trigger_fires",
|
|
2170
|
+
"What an automation has done, newest first. Metadata only \u2014 no message content. Answers with `nextCursor`; pass it back as `cursor` for the next page (null means there is none).",
|
|
2171
|
+
{
|
|
2172
|
+
id: z3.string().describe("The trigger ID"),
|
|
2173
|
+
limit: z3.number().int().positive().optional().describe("How many fires to return"),
|
|
2174
|
+
cursor: z3.string().optional().describe("The `nextCursor` a previous call answered with")
|
|
2175
|
+
},
|
|
2176
|
+
withToolLogging("platform_list_trigger_fires", async (params) => {
|
|
2177
|
+
requireApiKey();
|
|
2178
|
+
const { handleListTriggerFires } = await import("./list-trigger-fires-OZUF65LI.js");
|
|
2179
|
+
return { content: [{ type: "text", text: await handleListTriggerFires(params) }] };
|
|
1939
2180
|
})
|
|
1940
2181
|
);
|
|
1941
2182
|
server.tool(
|
|
1942
2183
|
"platform_delete_trigger",
|
|
1943
2184
|
"Delete a trigger (automation).",
|
|
1944
2185
|
{
|
|
1945
|
-
id:
|
|
2186
|
+
id: z3.string().describe("The trigger ID to delete")
|
|
1946
2187
|
},
|
|
1947
2188
|
withToolLogging("platform_delete_trigger", async (params) => {
|
|
1948
2189
|
requireApiKey();
|
|
1949
|
-
const { deleteTrigger } = await import("./langwatch-api-triggers-
|
|
2190
|
+
const { deleteTrigger } = await import("./langwatch-api-triggers-QMR7KEXW.js");
|
|
1950
2191
|
const result = await deleteTrigger(params.id);
|
|
1951
2192
|
return { content: [{ type: "text", text: `Trigger ${result.id} deleted.` }] };
|
|
1952
2193
|
})
|
|
1953
2194
|
);
|
|
2195
|
+
}
|
|
2196
|
+
function registerMonitorTools(server) {
|
|
1954
2197
|
server.tool(
|
|
1955
2198
|
"platform_list_monitors",
|
|
1956
2199
|
"List all online evaluation monitors for the project.",
|
|
1957
2200
|
{},
|
|
1958
2201
|
withToolLogging("platform_list_monitors", async () => {
|
|
1959
2202
|
requireApiKey();
|
|
1960
|
-
const { listMonitors } = await import("./langwatch-api-monitors-
|
|
2203
|
+
const { listMonitors } = await import("./langwatch-api-monitors-OHSGADTH.js");
|
|
1961
2204
|
const monitors = await listMonitors();
|
|
1962
2205
|
if (monitors.length === 0) {
|
|
1963
2206
|
return { content: [{ type: "text", text: "No monitors found." }] };
|
|
@@ -1980,11 +2223,11 @@ ${lines.join("\n")}`
|
|
|
1980
2223
|
"platform_get_monitor",
|
|
1981
2224
|
"Get details of a specific online evaluation monitor.",
|
|
1982
2225
|
{
|
|
1983
|
-
id:
|
|
2226
|
+
id: z3.string().describe("The monitor ID")
|
|
1984
2227
|
},
|
|
1985
2228
|
withToolLogging("platform_get_monitor", async (params) => {
|
|
1986
2229
|
requireApiKey();
|
|
1987
|
-
const { getMonitor } = await import("./langwatch-api-monitors-
|
|
2230
|
+
const { getMonitor } = await import("./langwatch-api-monitors-OHSGADTH.js");
|
|
1988
2231
|
const m = await getMonitor(params.id);
|
|
1989
2232
|
const lines = [
|
|
1990
2233
|
`**${m.name}** (${m.id})`,
|
|
@@ -2003,15 +2246,15 @@ ${lines.join("\n")}`
|
|
|
2003
2246
|
"platform_create_monitor",
|
|
2004
2247
|
"Create a new online evaluation monitor that runs an evaluator on incoming traces.",
|
|
2005
2248
|
{
|
|
2006
|
-
name:
|
|
2007
|
-
checkType:
|
|
2008
|
-
executionMode:
|
|
2009
|
-
sample:
|
|
2010
|
-
evaluatorId:
|
|
2249
|
+
name: z3.string().describe("Monitor name"),
|
|
2250
|
+
checkType: z3.string().describe("Evaluator check type (e.g. ragas/toxicity, custom/my-eval)"),
|
|
2251
|
+
executionMode: z3.enum(["ON_MESSAGE", "AS_GUARDRAIL", "MANUALLY"]).optional().describe("When to run: ON_MESSAGE (default), AS_GUARDRAIL, or MANUALLY"),
|
|
2252
|
+
sample: z3.number().min(0).max(1).optional().describe("Sampling rate 0.0-1.0 (default: 1.0)"),
|
|
2253
|
+
evaluatorId: z3.string().optional().describe("Link to a saved evaluator by ID")
|
|
2011
2254
|
},
|
|
2012
2255
|
withToolLogging("platform_create_monitor", async (params) => {
|
|
2013
2256
|
requireApiKey();
|
|
2014
|
-
const { createMonitor } = await import("./langwatch-api-monitors-
|
|
2257
|
+
const { createMonitor } = await import("./langwatch-api-monitors-OHSGADTH.js");
|
|
2015
2258
|
const monitor = await createMonitor(params);
|
|
2016
2259
|
return {
|
|
2017
2260
|
content: [
|
|
@@ -2027,15 +2270,15 @@ ${lines.join("\n")}`
|
|
|
2027
2270
|
"platform_update_monitor",
|
|
2028
2271
|
"Update an existing online evaluation monitor.",
|
|
2029
2272
|
{
|
|
2030
|
-
id:
|
|
2031
|
-
name:
|
|
2032
|
-
enabled:
|
|
2033
|
-
executionMode:
|
|
2034
|
-
sample:
|
|
2273
|
+
id: z3.string().describe("The monitor ID to update"),
|
|
2274
|
+
name: z3.string().optional().describe("New monitor name"),
|
|
2275
|
+
enabled: z3.boolean().optional().describe("Enable or disable the monitor"),
|
|
2276
|
+
executionMode: z3.enum(["ON_MESSAGE", "AS_GUARDRAIL", "MANUALLY"]).optional().describe("New execution mode"),
|
|
2277
|
+
sample: z3.number().min(0).max(1).optional().describe("New sampling rate")
|
|
2035
2278
|
},
|
|
2036
2279
|
withToolLogging("platform_update_monitor", async (params) => {
|
|
2037
2280
|
requireApiKey();
|
|
2038
|
-
const { updateMonitor } = await import("./langwatch-api-monitors-
|
|
2281
|
+
const { updateMonitor } = await import("./langwatch-api-monitors-OHSGADTH.js");
|
|
2039
2282
|
const monitor = await updateMonitor(params);
|
|
2040
2283
|
return {
|
|
2041
2284
|
content: [
|
|
@@ -2051,33 +2294,31 @@ ${lines.join("\n")}`
|
|
|
2051
2294
|
"platform_delete_monitor",
|
|
2052
2295
|
"Delete an online evaluation monitor.",
|
|
2053
2296
|
{
|
|
2054
|
-
id:
|
|
2297
|
+
id: z3.string().describe("The monitor ID to delete")
|
|
2055
2298
|
},
|
|
2056
2299
|
withToolLogging("platform_delete_monitor", async (params) => {
|
|
2057
2300
|
requireApiKey();
|
|
2058
|
-
const { deleteMonitor } = await import("./langwatch-api-monitors-
|
|
2301
|
+
const { deleteMonitor } = await import("./langwatch-api-monitors-OHSGADTH.js");
|
|
2059
2302
|
const result = await deleteMonitor(params.id);
|
|
2060
2303
|
return {
|
|
2061
|
-
content: [
|
|
2062
|
-
{ type: "text", text: `Monitor ${result.id} deleted.` }
|
|
2063
|
-
]
|
|
2304
|
+
content: [{ type: "text", text: `Monitor ${result.id} deleted.` }]
|
|
2064
2305
|
};
|
|
2065
2306
|
})
|
|
2066
2307
|
);
|
|
2308
|
+
}
|
|
2309
|
+
function registerSecretTools(server) {
|
|
2067
2310
|
server.tool(
|
|
2068
2311
|
"platform_list_secrets",
|
|
2069
2312
|
"List all project secrets (values are never returned, only metadata).",
|
|
2070
2313
|
{},
|
|
2071
2314
|
withToolLogging("platform_list_secrets", async () => {
|
|
2072
2315
|
requireApiKey();
|
|
2073
|
-
const { listSecrets } = await import("./langwatch-api-secrets-
|
|
2316
|
+
const { listSecrets } = await import("./langwatch-api-secrets-CLTS766Q.js");
|
|
2074
2317
|
const secrets = await listSecrets();
|
|
2075
2318
|
if (secrets.length === 0) {
|
|
2076
2319
|
return { content: [{ type: "text", text: "No secrets found." }] };
|
|
2077
2320
|
}
|
|
2078
|
-
const lines = secrets.map(
|
|
2079
|
-
(s) => `\u2022 ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`
|
|
2080
|
-
);
|
|
2321
|
+
const lines = secrets.map((s) => `\u2022 ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`);
|
|
2081
2322
|
return {
|
|
2082
2323
|
content: [
|
|
2083
2324
|
{
|
|
@@ -2093,12 +2334,12 @@ ${lines.join("\n")}`
|
|
|
2093
2334
|
"platform_create_secret",
|
|
2094
2335
|
"Create a new encrypted project secret. Name must be UPPER_SNAKE_CASE (e.g. MY_API_KEY).",
|
|
2095
2336
|
{
|
|
2096
|
-
name:
|
|
2097
|
-
value:
|
|
2337
|
+
name: z3.string().describe("Secret name in UPPER_SNAKE_CASE (e.g. OPENAI_API_KEY)"),
|
|
2338
|
+
value: z3.string().describe("The secret value (will be encrypted at rest)")
|
|
2098
2339
|
},
|
|
2099
2340
|
withToolLogging("platform_create_secret", async (params) => {
|
|
2100
2341
|
requireApiKey();
|
|
2101
|
-
const { createSecret } = await import("./langwatch-api-secrets-
|
|
2342
|
+
const { createSecret } = await import("./langwatch-api-secrets-CLTS766Q.js");
|
|
2102
2343
|
const secret = await createSecret({ name: params.name, value: params.value });
|
|
2103
2344
|
return {
|
|
2104
2345
|
content: [
|
|
@@ -2114,12 +2355,12 @@ ${lines.join("\n")}`
|
|
|
2114
2355
|
"platform_update_secret",
|
|
2115
2356
|
"Update an existing project secret's value.",
|
|
2116
2357
|
{
|
|
2117
|
-
id:
|
|
2118
|
-
value:
|
|
2358
|
+
id: z3.string().describe("The secret ID to update"),
|
|
2359
|
+
value: z3.string().describe("The new secret value (will be encrypted at rest)")
|
|
2119
2360
|
},
|
|
2120
2361
|
withToolLogging("platform_update_secret", async (params) => {
|
|
2121
2362
|
requireApiKey();
|
|
2122
|
-
const { updateSecret } = await import("./langwatch-api-secrets-
|
|
2363
|
+
const { updateSecret } = await import("./langwatch-api-secrets-CLTS766Q.js");
|
|
2123
2364
|
const secret = await updateSecret({ id: params.id, value: params.value });
|
|
2124
2365
|
return {
|
|
2125
2366
|
content: [
|
|
@@ -2135,20 +2376,25 @@ ${lines.join("\n")}`
|
|
|
2135
2376
|
"platform_delete_secret",
|
|
2136
2377
|
"Delete a project secret.",
|
|
2137
2378
|
{
|
|
2138
|
-
id:
|
|
2379
|
+
id: z3.string().describe("The secret ID to delete")
|
|
2139
2380
|
},
|
|
2140
2381
|
withToolLogging("platform_delete_secret", async (params) => {
|
|
2141
2382
|
requireApiKey();
|
|
2142
|
-
const { deleteSecret } = await import("./langwatch-api-secrets-
|
|
2383
|
+
const { deleteSecret } = await import("./langwatch-api-secrets-CLTS766Q.js");
|
|
2143
2384
|
const result = await deleteSecret(params.id);
|
|
2144
2385
|
return { content: [{ type: "text", text: `Secret ${result.id} deleted.` }] };
|
|
2145
2386
|
})
|
|
2146
2387
|
);
|
|
2388
|
+
}
|
|
2389
|
+
function registerExperimentTools(server) {
|
|
2147
2390
|
server.tool(
|
|
2148
2391
|
"platform_run_experiment",
|
|
2149
2392
|
"Start an experiment run by slug. Returns a run ID for polling status.",
|
|
2150
2393
|
{
|
|
2151
|
-
slug:
|
|
2394
|
+
slug: z3.string().describe("The experiment slug to run"),
|
|
2395
|
+
concurrency: z3.number().int().min(1).max(24).optional().describe(
|
|
2396
|
+
"Cells (one row against one target) to run at once, for this run only (default: the server's setting). Use 1 to run them one at a time, in order."
|
|
2397
|
+
)
|
|
2152
2398
|
},
|
|
2153
2399
|
withToolLogging("platform_run_experiment", async (params) => {
|
|
2154
2400
|
requireApiKey();
|
|
@@ -2161,8 +2407,8 @@ ${lines.join("\n")}`
|
|
|
2161
2407
|
"platform_experiment_status",
|
|
2162
2408
|
"Check the status of an experiment run (running, completed, stopped, or interrupted) plus its progress. Works for both platform-run and SDK-logged experiments; pass experimentSlug for SDK runs or runs older than 24h.",
|
|
2163
2409
|
{
|
|
2164
|
-
runId:
|
|
2165
|
-
experimentSlug:
|
|
2410
|
+
runId: z3.string().describe("The run ID returned from platform_run_experiment"),
|
|
2411
|
+
experimentSlug: z3.string().optional().describe(
|
|
2166
2412
|
"Experiment slug \u2014 required for SDK-logged runs and runs older than 24h (the Redis run-state they would otherwise be read from is absent/expired). Use platform_experiment_list_runs to find the slug."
|
|
2167
2413
|
)
|
|
2168
2414
|
},
|
|
@@ -2177,9 +2423,7 @@ ${lines.join("\n")}`
|
|
|
2177
2423
|
"platform_experiment_list",
|
|
2178
2424
|
"List experiments configured in the LangWatch project. Returns a markdown table with each experiment's slug, name, last-run timestamp, and run count. Use this to discover experiment slugs before drilling into runs with platform_experiment_list_runs. Output is capped (default 25, max 100) to protect the agent's context window.",
|
|
2179
2425
|
{
|
|
2180
|
-
limit:
|
|
2181
|
-
"Maximum number of experiments to include (default 25, hard-capped at 100)"
|
|
2182
|
-
)
|
|
2426
|
+
limit: z3.number().int().positive().max(100).optional().describe("Maximum number of experiments to include (default 25, hard-capped at 100)")
|
|
2183
2427
|
},
|
|
2184
2428
|
withToolLogging("platform_experiment_list", async (params) => {
|
|
2185
2429
|
requireApiKey();
|
|
@@ -2192,12 +2436,8 @@ ${lines.join("\n")}`
|
|
|
2192
2436
|
"platform_experiment_list_runs",
|
|
2193
2437
|
"List experiment runs for a specific experiment slug. Returns a markdown table with each run's id, status, started/finished timestamps, and pass-rate summary. Use this to discover run ids before calling platform_experiment_results. Output is capped (default 25, max 100) to protect the agent's context window.",
|
|
2194
2438
|
{
|
|
2195
|
-
experimentSlug:
|
|
2196
|
-
|
|
2197
|
-
),
|
|
2198
|
-
limit: z2.number().int().positive().max(100).optional().describe(
|
|
2199
|
-
"Maximum number of runs to include (default 25, hard-capped at 100)"
|
|
2200
|
-
)
|
|
2439
|
+
experimentSlug: z3.string().describe("Experiment slug from platform_experiment_list (e.g. 'checkout-flow')"),
|
|
2440
|
+
limit: z3.number().int().positive().max(100).optional().describe("Maximum number of runs to include (default 25, hard-capped at 100)")
|
|
2201
2441
|
},
|
|
2202
2442
|
withToolLogging("platform_experiment_list_runs", async (params) => {
|
|
2203
2443
|
requireApiKey();
|
|
@@ -2210,39 +2450,37 @@ ${lines.join("\n")}`
|
|
|
2210
2450
|
"platform_experiment_results",
|
|
2211
2451
|
"Fetch per-row results for an experiment run so you can debug evaluator scores and missed rows. Serves partial results for runs that are still running or were interrupted (it does not require a completed run), with the run status noted in the output. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window \u2014 narrow with `filter: 'failed'` or `evaluator` to see what matters.",
|
|
2212
2452
|
{
|
|
2213
|
-
runId:
|
|
2214
|
-
experimentSlug:
|
|
2453
|
+
runId: z3.string().describe("The run ID returned from platform_run_experiment"),
|
|
2454
|
+
experimentSlug: z3.string().optional().describe(
|
|
2215
2455
|
"Experiment slug \u2014 required when fetching results for runs older than 24h (Redis state may have expired). Use platform_experiment_list_runs to find the slug."
|
|
2216
2456
|
),
|
|
2217
|
-
filter:
|
|
2457
|
+
filter: z3.enum(["all", "failed"]).optional().describe(
|
|
2218
2458
|
"Filter rows: 'failed' shows only rows that errored or failed an evaluation; 'all' (default) shows everything"
|
|
2219
2459
|
),
|
|
2220
|
-
evaluator:
|
|
2221
|
-
limit:
|
|
2460
|
+
evaluator: z3.string().optional().describe("Show only this evaluator's results (matched on `evaluator` field)"),
|
|
2461
|
+
limit: z3.number().int().positive().max(50).optional().describe(
|
|
2222
2462
|
"Maximum number of rows to include in the markdown output (default 50, hard-capped at 50)."
|
|
2223
2463
|
)
|
|
2224
2464
|
},
|
|
2225
2465
|
withToolLogging("platform_experiment_results", async (params) => {
|
|
2226
2466
|
requireApiKey();
|
|
2227
2467
|
return {
|
|
2228
|
-
content: [
|
|
2229
|
-
{ type: "text", text: await handleExperimentResults(params) }
|
|
2230
|
-
]
|
|
2468
|
+
content: [{ type: "text", text: await handleExperimentResults(params) }]
|
|
2231
2469
|
};
|
|
2232
2470
|
})
|
|
2233
2471
|
);
|
|
2472
|
+
}
|
|
2473
|
+
function registerDatasetTools(server) {
|
|
2234
2474
|
server.tool(
|
|
2235
2475
|
"platform_list_datasets",
|
|
2236
2476
|
"List all datasets on the LangWatch platform with their names, slugs, columns, and record counts. Returns AI-readable digest by default.",
|
|
2237
2477
|
{
|
|
2238
|
-
format:
|
|
2239
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
2240
|
-
)
|
|
2478
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
2241
2479
|
},
|
|
2242
2480
|
async (params) => {
|
|
2243
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2481
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2244
2482
|
requireApiKey2();
|
|
2245
|
-
const { handleListDatasets } = await import("./list-datasets-
|
|
2483
|
+
const { handleListDatasets } = await import("./list-datasets-P3JK2ZHS.js");
|
|
2246
2484
|
return {
|
|
2247
2485
|
content: [{ type: "text", text: await handleListDatasets(params) }]
|
|
2248
2486
|
};
|
|
@@ -2252,15 +2490,13 @@ ${lines.join("\n")}`
|
|
|
2252
2490
|
"platform_get_dataset",
|
|
2253
2491
|
"Get full details of a dataset on the LangWatch platform by slug or ID, including column definitions and a preview of records.",
|
|
2254
2492
|
{
|
|
2255
|
-
slugOrId:
|
|
2256
|
-
format:
|
|
2257
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
2258
|
-
)
|
|
2493
|
+
slugOrId: z3.string().describe("The dataset slug or ID to retrieve"),
|
|
2494
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
2259
2495
|
},
|
|
2260
2496
|
async (params) => {
|
|
2261
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2497
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2262
2498
|
requireApiKey2();
|
|
2263
|
-
const { handleGetDataset } = await import("./get-dataset-
|
|
2499
|
+
const { handleGetDataset } = await import("./get-dataset-APGZBYMH.js");
|
|
2264
2500
|
return {
|
|
2265
2501
|
content: [{ type: "text", text: await handleGetDataset(params) }]
|
|
2266
2502
|
};
|
|
@@ -2271,9 +2507,9 @@ ${lines.join("\n")}`
|
|
|
2271
2507
|
"Create a new dataset on the LangWatch platform.",
|
|
2272
2508
|
createDatasetSchema.shape,
|
|
2273
2509
|
async (params) => {
|
|
2274
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2510
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2275
2511
|
requireApiKey2();
|
|
2276
|
-
const { handleCreateDataset } = await import("./create-dataset-
|
|
2512
|
+
const { handleCreateDataset } = await import("./create-dataset-WON4SOTC.js");
|
|
2277
2513
|
return {
|
|
2278
2514
|
content: [{ type: "text", text: await handleCreateDataset(params) }]
|
|
2279
2515
|
};
|
|
@@ -2283,14 +2519,14 @@ ${lines.join("\n")}`
|
|
|
2283
2519
|
"platform_update_dataset",
|
|
2284
2520
|
"Update an existing dataset on the LangWatch platform (name and/or column types).",
|
|
2285
2521
|
{
|
|
2286
|
-
slugOrId:
|
|
2287
|
-
name:
|
|
2288
|
-
columnTypes:
|
|
2522
|
+
slugOrId: z3.string().describe("The dataset slug or ID to update"),
|
|
2523
|
+
name: z3.string().optional().describe("Updated dataset name"),
|
|
2524
|
+
columnTypes: z3.array(datasetColumnDefinitionSchema).optional().describe("Updated column definitions")
|
|
2289
2525
|
},
|
|
2290
2526
|
async (params) => {
|
|
2291
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2527
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2292
2528
|
requireApiKey2();
|
|
2293
|
-
const { handleUpdateDataset } = await import("./update-dataset-
|
|
2529
|
+
const { handleUpdateDataset } = await import("./update-dataset-IRVMY5PF.js");
|
|
2294
2530
|
return {
|
|
2295
2531
|
content: [{ type: "text", text: await handleUpdateDataset(params) }]
|
|
2296
2532
|
};
|
|
@@ -2300,12 +2536,12 @@ ${lines.join("\n")}`
|
|
|
2300
2536
|
"platform_delete_dataset",
|
|
2301
2537
|
"Delete (archive) a dataset on the LangWatch platform.",
|
|
2302
2538
|
{
|
|
2303
|
-
slugOrId:
|
|
2539
|
+
slugOrId: z3.string().describe("The dataset slug or ID to delete")
|
|
2304
2540
|
},
|
|
2305
2541
|
async (params) => {
|
|
2306
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2542
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2307
2543
|
requireApiKey2();
|
|
2308
|
-
const { handleDeleteDataset } = await import("./delete-dataset-
|
|
2544
|
+
const { handleDeleteDataset } = await import("./delete-dataset-NXP7ZPPV.js");
|
|
2309
2545
|
return {
|
|
2310
2546
|
content: [{ type: "text", text: await handleDeleteDataset(params) }]
|
|
2311
2547
|
};
|
|
@@ -2315,19 +2551,17 @@ ${lines.join("\n")}`
|
|
|
2315
2551
|
"platform_list_dataset_records",
|
|
2316
2552
|
"List records in a dataset on the LangWatch platform with pagination.",
|
|
2317
2553
|
{
|
|
2318
|
-
slugOrId:
|
|
2319
|
-
page:
|
|
2320
|
-
limit:
|
|
2321
|
-
format:
|
|
2554
|
+
slugOrId: z3.string().describe("The dataset slug or ID to list records from"),
|
|
2555
|
+
page: z3.number().int().min(1).optional().describe("Page number (default: 1)"),
|
|
2556
|
+
limit: z3.number().int().min(1).max(1e3).optional().describe("Records per page (default: 50, max: 1000)"),
|
|
2557
|
+
format: z3.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
2322
2558
|
},
|
|
2323
2559
|
async (params) => {
|
|
2324
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2560
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2325
2561
|
requireApiKey2();
|
|
2326
|
-
const { handleListDatasetRecords } = await import("./list-dataset-records-
|
|
2562
|
+
const { handleListDatasetRecords } = await import("./list-dataset-records-KMLLMZAZ.js");
|
|
2327
2563
|
return {
|
|
2328
|
-
content: [
|
|
2329
|
-
{ type: "text", text: await handleListDatasetRecords(params) }
|
|
2330
|
-
]
|
|
2564
|
+
content: [{ type: "text", text: await handleListDatasetRecords(params) }]
|
|
2331
2565
|
};
|
|
2332
2566
|
}
|
|
2333
2567
|
);
|
|
@@ -2335,17 +2569,15 @@ ${lines.join("\n")}`
|
|
|
2335
2569
|
"platform_create_dataset_records",
|
|
2336
2570
|
"Add records to a dataset on the LangWatch platform in batch (max 1000 per call).",
|
|
2337
2571
|
{
|
|
2338
|
-
slugOrId:
|
|
2339
|
-
entries:
|
|
2572
|
+
slugOrId: z3.string().describe("The dataset slug or ID to add records to"),
|
|
2573
|
+
entries: z3.array(z3.record(z3.string(), z3.unknown())).min(1).max(1e3).describe("Array of record entries to create (key-value objects matching dataset columns)")
|
|
2340
2574
|
},
|
|
2341
2575
|
async (params) => {
|
|
2342
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2576
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2343
2577
|
requireApiKey2();
|
|
2344
|
-
const { handleCreateDatasetRecords } = await import("./create-dataset-records-
|
|
2578
|
+
const { handleCreateDatasetRecords } = await import("./create-dataset-records-UJBW3C7C.js");
|
|
2345
2579
|
return {
|
|
2346
|
-
content: [
|
|
2347
|
-
{ type: "text", text: await handleCreateDatasetRecords(params) }
|
|
2348
|
-
]
|
|
2580
|
+
content: [{ type: "text", text: await handleCreateDatasetRecords(params) }]
|
|
2349
2581
|
};
|
|
2350
2582
|
}
|
|
2351
2583
|
);
|
|
@@ -2353,18 +2585,16 @@ ${lines.join("\n")}`
|
|
|
2353
2585
|
"platform_update_dataset_record",
|
|
2354
2586
|
"Update a single record in a dataset on the LangWatch platform.",
|
|
2355
2587
|
{
|
|
2356
|
-
slugOrId:
|
|
2357
|
-
recordId:
|
|
2358
|
-
entry:
|
|
2588
|
+
slugOrId: z3.string().describe("The dataset slug or ID containing the record"),
|
|
2589
|
+
recordId: z3.string().describe("The record ID to update"),
|
|
2590
|
+
entry: z3.record(z3.string(), z3.unknown()).describe("Updated record entry (key-value object)")
|
|
2359
2591
|
},
|
|
2360
2592
|
async (params) => {
|
|
2361
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2593
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2362
2594
|
requireApiKey2();
|
|
2363
|
-
const { handleUpdateDatasetRecord } = await import("./update-dataset-record-
|
|
2595
|
+
const { handleUpdateDatasetRecord } = await import("./update-dataset-record-Z3L7ACNF.js");
|
|
2364
2596
|
return {
|
|
2365
|
-
content: [
|
|
2366
|
-
{ type: "text", text: await handleUpdateDatasetRecord(params) }
|
|
2367
|
-
]
|
|
2597
|
+
content: [{ type: "text", text: await handleUpdateDatasetRecord(params) }]
|
|
2368
2598
|
};
|
|
2369
2599
|
}
|
|
2370
2600
|
);
|
|
@@ -2372,30 +2602,30 @@ ${lines.join("\n")}`
|
|
|
2372
2602
|
"platform_delete_dataset_records",
|
|
2373
2603
|
"Delete records from a dataset on the LangWatch platform by their IDs.",
|
|
2374
2604
|
{
|
|
2375
|
-
slugOrId:
|
|
2376
|
-
recordIds:
|
|
2605
|
+
slugOrId: z3.string().describe("The dataset slug or ID containing the records"),
|
|
2606
|
+
recordIds: z3.array(z3.string()).min(1).max(1e3).describe("Array of record IDs to delete")
|
|
2377
2607
|
},
|
|
2378
2608
|
async (params) => {
|
|
2379
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2609
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-2WQGMAEN.js");
|
|
2380
2610
|
requireApiKey2();
|
|
2381
|
-
const { handleDeleteDatasetRecords } = await import("./delete-dataset-records-
|
|
2611
|
+
const { handleDeleteDatasetRecords } = await import("./delete-dataset-records-4Z7DZCDW.js");
|
|
2382
2612
|
return {
|
|
2383
|
-
content: [
|
|
2384
|
-
{ type: "text", text: await handleDeleteDatasetRecords(params) }
|
|
2385
|
-
]
|
|
2613
|
+
content: [{ type: "text", text: await handleDeleteDatasetRecords(params) }]
|
|
2386
2614
|
};
|
|
2387
2615
|
}
|
|
2388
2616
|
);
|
|
2617
|
+
}
|
|
2618
|
+
function registerProjectTools(server) {
|
|
2389
2619
|
server.tool(
|
|
2390
2620
|
"platform_list_projects",
|
|
2391
2621
|
"List all projects in your LangWatch organization. Requires an org-level API key.",
|
|
2392
2622
|
{
|
|
2393
|
-
page:
|
|
2394
|
-
limit:
|
|
2623
|
+
page: z3.number().int().positive().optional().describe("Page number (default: 1)"),
|
|
2624
|
+
limit: z3.number().int().positive().max(1e3).optional().describe("Results per page (default: 100)")
|
|
2395
2625
|
},
|
|
2396
2626
|
withToolLogging("platform_list_projects", async (params) => {
|
|
2397
2627
|
requireApiKey();
|
|
2398
|
-
const { handleListProjects } = await import("./list-projects-
|
|
2628
|
+
const { handleListProjects } = await import("./list-projects-HUVK7L3W.js");
|
|
2399
2629
|
return {
|
|
2400
2630
|
content: [{ type: "text", text: await handleListProjects(params) }]
|
|
2401
2631
|
};
|
|
@@ -2405,11 +2635,11 @@ ${lines.join("\n")}`
|
|
|
2405
2635
|
"platform_get_project",
|
|
2406
2636
|
"Get details of a specific project by its ID.",
|
|
2407
2637
|
{
|
|
2408
|
-
id:
|
|
2638
|
+
id: z3.string().describe("The project ID")
|
|
2409
2639
|
},
|
|
2410
2640
|
withToolLogging("platform_get_project", async (params) => {
|
|
2411
2641
|
requireApiKey();
|
|
2412
|
-
const { handleGetProject } = await import("./get-project-
|
|
2642
|
+
const { handleGetProject } = await import("./get-project-RK6AJTEP.js");
|
|
2413
2643
|
return {
|
|
2414
2644
|
content: [{ type: "text", text: await handleGetProject(params) }]
|
|
2415
2645
|
};
|
|
@@ -2421,15 +2651,15 @@ ${lines.join("\n")}`
|
|
|
2421
2651
|
|
|
2422
2652
|
You must provide either teamId (to add the project to an existing team) or newTeamName (to create a new team for the project).`,
|
|
2423
2653
|
{
|
|
2424
|
-
name:
|
|
2425
|
-
language:
|
|
2426
|
-
framework:
|
|
2427
|
-
teamId:
|
|
2428
|
-
newTeamName:
|
|
2654
|
+
name: z3.string().describe("Project name"),
|
|
2655
|
+
language: z3.string().describe('Programming language (e.g. "python", "typescript")'),
|
|
2656
|
+
framework: z3.string().describe('Framework (e.g. "openai", "langchain", "custom")'),
|
|
2657
|
+
teamId: z3.string().optional().describe("ID of an existing team to add the project to"),
|
|
2658
|
+
newTeamName: z3.string().optional().describe("Name for a new team to create for this project")
|
|
2429
2659
|
},
|
|
2430
2660
|
withToolLogging("platform_create_project", async (params) => {
|
|
2431
2661
|
requireApiKey();
|
|
2432
|
-
const { handleCreateProject } = await import("./create-project-
|
|
2662
|
+
const { handleCreateProject } = await import("./create-project-EY4XZ7TH.js");
|
|
2433
2663
|
return {
|
|
2434
2664
|
content: [{ type: "text", text: await handleCreateProject(params) }]
|
|
2435
2665
|
};
|
|
@@ -2439,15 +2669,15 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2439
2669
|
"platform_update_project",
|
|
2440
2670
|
"Update an existing project's name, language, framework, or PII redaction level.",
|
|
2441
2671
|
{
|
|
2442
|
-
id:
|
|
2443
|
-
name:
|
|
2444
|
-
language:
|
|
2445
|
-
framework:
|
|
2446
|
-
piiRedactionLevel:
|
|
2672
|
+
id: z3.string().describe("The project ID to update"),
|
|
2673
|
+
name: z3.string().optional().describe("New project name"),
|
|
2674
|
+
language: z3.string().optional().describe("New programming language"),
|
|
2675
|
+
framework: z3.string().optional().describe("New framework"),
|
|
2676
|
+
piiRedactionLevel: z3.enum(["STRICT", "ESSENTIAL", "DISABLED"]).optional().describe("PII redaction level")
|
|
2447
2677
|
},
|
|
2448
2678
|
withToolLogging("platform_update_project", async (params) => {
|
|
2449
2679
|
requireApiKey();
|
|
2450
|
-
const { handleUpdateProject } = await import("./update-project-
|
|
2680
|
+
const { handleUpdateProject } = await import("./update-project-QPR3QX55.js");
|
|
2451
2681
|
return {
|
|
2452
2682
|
content: [{ type: "text", text: await handleUpdateProject(params) }]
|
|
2453
2683
|
};
|
|
@@ -2457,23 +2687,25 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2457
2687
|
"platform_archive_project",
|
|
2458
2688
|
"Archive (soft-delete) a project. The project can be restored later if needed.",
|
|
2459
2689
|
{
|
|
2460
|
-
id:
|
|
2690
|
+
id: z3.string().describe("The project ID to archive")
|
|
2461
2691
|
},
|
|
2462
2692
|
withToolLogging("platform_archive_project", async (params) => {
|
|
2463
2693
|
requireApiKey();
|
|
2464
|
-
const { handleArchiveProject } = await import("./archive-project-
|
|
2694
|
+
const { handleArchiveProject } = await import("./archive-project-K5UD7A6X.js");
|
|
2465
2695
|
return {
|
|
2466
2696
|
content: [{ type: "text", text: await handleArchiveProject(params) }]
|
|
2467
2697
|
};
|
|
2468
2698
|
})
|
|
2469
2699
|
);
|
|
2700
|
+
}
|
|
2701
|
+
function registerApiKeyTools(server) {
|
|
2470
2702
|
server.tool(
|
|
2471
2703
|
"platform_list_api_keys",
|
|
2472
2704
|
"List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
|
|
2473
2705
|
{},
|
|
2474
2706
|
withToolLogging("platform_list_api_keys", async () => {
|
|
2475
2707
|
requireApiKey();
|
|
2476
|
-
const { handleListApiKeys } = await import("./list-api-keys-
|
|
2708
|
+
const { handleListApiKeys } = await import("./list-api-keys-JMW5WRXX.js");
|
|
2477
2709
|
return {
|
|
2478
2710
|
content: [{ type: "text", text: await handleListApiKeys() }]
|
|
2479
2711
|
};
|
|
@@ -2488,20 +2720,22 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2488
2720
|
|
|
2489
2721
|
The token is returned once and cannot be retrieved again.`,
|
|
2490
2722
|
{
|
|
2491
|
-
keyType:
|
|
2492
|
-
name:
|
|
2493
|
-
description:
|
|
2494
|
-
expiresAt:
|
|
2495
|
-
bindings:
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2723
|
+
keyType: z3.enum(["personal", "service"]).describe("Key type: 'personal' (user-tied) or 'service' (headless)"),
|
|
2724
|
+
name: z3.string().describe("Key name (max 100 chars)"),
|
|
2725
|
+
description: z3.string().optional().describe("Key description (max 500 chars)"),
|
|
2726
|
+
expiresAt: z3.string().optional().describe("Expiration date (ISO string)"),
|
|
2727
|
+
bindings: z3.array(
|
|
2728
|
+
z3.object({
|
|
2729
|
+
role: z3.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
|
|
2730
|
+
scopeType: z3.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
|
|
2731
|
+
scopeId: z3.string().describe("Scope ID (org, team, or project ID)")
|
|
2732
|
+
})
|
|
2733
|
+
).optional().describe("Role bindings (required for personal keys)"),
|
|
2734
|
+
projectIds: z3.array(z3.string()).optional().describe("Project IDs for service keys (creates ADMIN bindings per project)")
|
|
2501
2735
|
},
|
|
2502
2736
|
withToolLogging("platform_create_api_key", async (params) => {
|
|
2503
2737
|
requireApiKey();
|
|
2504
|
-
const { handleCreateApiKey } = await import("./create-api-key-
|
|
2738
|
+
const { handleCreateApiKey } = await import("./create-api-key-JKM2DSZO.js");
|
|
2505
2739
|
return {
|
|
2506
2740
|
content: [{ type: "text", text: await handleCreateApiKey(params) }]
|
|
2507
2741
|
};
|
|
@@ -2511,11 +2745,11 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2511
2745
|
"platform_revoke_api_key",
|
|
2512
2746
|
"Revoke an API key. Once revoked, the key can no longer be used for authentication.",
|
|
2513
2747
|
{
|
|
2514
|
-
id:
|
|
2748
|
+
id: z3.string().describe("The API key ID to revoke")
|
|
2515
2749
|
},
|
|
2516
2750
|
withToolLogging("platform_revoke_api_key", async (params) => {
|
|
2517
2751
|
requireApiKey();
|
|
2518
|
-
const { handleRevokeApiKey } = await import("./revoke-api-key-
|
|
2752
|
+
const { handleRevokeApiKey } = await import("./revoke-api-key-BU6BKGCG.js");
|
|
2519
2753
|
return {
|
|
2520
2754
|
content: [{ type: "text", text: await handleRevokeApiKey(params) }]
|
|
2521
2755
|
};
|
|
@@ -2526,4 +2760,4 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2526
2760
|
export {
|
|
2527
2761
|
createMcpServer
|
|
2528
2762
|
};
|
|
2529
|
-
//# sourceMappingURL=chunk-
|
|
2763
|
+
//# sourceMappingURL=chunk-R5YJIYPF.js.map
|