@langwatch/mcp-server 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +218 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-55UI6H22.js +44 -0
- package/dist/add-dashboard-widget-MJHIC3CC.js +45 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-K5UD7A6X.js} +4 -4
- package/dist/archive-project-L4PAKT34.js +19 -0
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-5HSQG2HX.js} +4 -4
- package/dist/archive-run-plan-AOKNYQIO.js +15 -0
- package/dist/archive-scenario-2CWJW2VD.js +19 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-5IJQESHH.js} +4 -4
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-OPB7NFI7.js} +4 -4
- package/dist/archive-test-suite-ZC3TN6VH.js +15 -0
- package/dist/assign-prompt-tag-6DDE6ZZ5.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-D7O23MJM.js} +8 -4
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/{chunk-EYXMLE5X.js → chunk-4RX3G62A.js} +56 -47
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7CZQDZ2K.js +319 -0
- package/dist/{chunk-RVAE2MEA.js → chunk-7GETGE6B.js} +7 -7
- package/dist/{chunk-OJLYYJWI.js → chunk-AASW2TO2.js} +7 -10
- package/dist/{chunk-QAMP3CGR.js → chunk-B4RFWIUP.js} +4 -4
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-G4ZAZTZW.js → chunk-D2HS73VI.js} +7 -10
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-WT4PO4A3.js → chunk-DULFP3MK.js} +10 -7
- package/dist/chunk-EYPJBL7P.js +516 -0
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/chunk-GA5JUSPT.js +64 -0
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/chunk-HLQJXUR6.js +67 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-KILVK3FS.js} +4 -4
- package/dist/chunk-MOCU5XWF.js +79 -0
- package/dist/chunk-O2WGVCED.js +41 -0
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-UFOYFYI7.js → chunk-QXLMHLRC.js} +6 -9
- package/dist/{chunk-QRO35SOM.js → chunk-R5YJIYPF.js} +981 -747
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/{chunk-SR6QPRIE.js → chunk-RFC2SBEX.js} +7 -13
- package/dist/chunk-SHSVN3SE.js +21 -0
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/{chunk-56OGTS24.js → chunk-VOUGDY2O.js} +4 -11
- package/dist/chunk-WCKZDOGF.js +31 -0
- package/dist/chunk-XQYH52SI.js +129 -0
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/{chunk-6DJDEXET.js → chunk-YCIUEZVK.js} +6 -12
- package/dist/config-2WQGMAEN.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-6JNLMYQD.js} +4 -4
- package/dist/create-agent-IRZOU4R5.js +23 -0
- package/dist/create-analytics-report-N4L4V5QL.js +19 -0
- package/dist/create-analytics-report-QEHXG7FU.js +18 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-CCEGIZ67.js} +10 -5
- package/dist/create-annotation-DTHCMWHB.js +29 -0
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-JKM2DSZO.js} +5 -7
- package/dist/create-api-key-JPEUIXGM.js +24 -0
- package/dist/create-dataset-DJNXYJ5B.js +22 -0
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-WON4SOTC.js} +6 -6
- package/dist/create-dataset-records-TMEOTLTO.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-UJBW3C7C.js} +5 -7
- package/dist/create-evaluator-N5AEHI4K.js +27 -0
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-XYNY2YFY.js} +4 -4
- package/dist/create-mcp-server.cjs +9712 -0
- package/dist/create-mcp-server.js +2757 -10
- package/dist/{create-project-ZQ7TFIO2.js → create-project-EY4XZ7TH.js} +4 -4
- package/dist/create-project-YL345T5L.js +44 -0
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-JZQNQ7I3.js} +5 -7
- package/dist/create-prompt-ZUMQHHX7.js +37 -0
- package/dist/create-prompt-tag-CJ5FZ436.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-JFP2Q3PO.js} +3 -3
- package/dist/create-scenario-CQU2SSFB.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-S6OWWKJO.js} +5 -5
- package/dist/create-test-suite-OLGCAGJA.js +36 -0
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-Y7P2RI4L.js} +6 -6
- package/dist/create-trigger-4CNBTISJ.js +48 -0
- package/dist/create-trigger-RQ653WYF.js +49 -0
- package/dist/dashboards-BA7M6CMO.js +90 -0
- package/dist/dashboards-U663GUIB.js +91 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-J2Q4DRZ2.js} +4 -4
- package/dist/delete-agent-ZAU4LTXB.js +18 -0
- package/dist/delete-analytics-report-QK6X7P4S.js +19 -0
- package/dist/delete-analytics-report-YUYLDHLL.js +18 -0
- package/dist/delete-annotation-7X4NYXXM.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-KJOCDLV3.js} +4 -4
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-NXP7ZPPV.js} +5 -7
- package/dist/delete-dataset-RUTZBBVD.js +18 -0
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-4Z7DZCDW.js} +5 -7
- package/dist/delete-dataset-records-75UAR5IS.js +19 -0
- package/dist/delete-evaluator-E3CUNGTQ.js +16 -0
- package/dist/delete-evaluator-RI4GALGX.js +15 -0
- package/dist/delete-prompt-tag-LE3WJPKS.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-S4KBZTNG.js} +3 -3
- package/dist/delete-workflow-4CZFIQCQ.js +17 -0
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-PWQPDW6H.js} +4 -4
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/discover-schema-7WHIIATI.js +442 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-J2ANOBKO.js} +181 -186
- package/dist/get-agent-5T3CXQOR.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-ZKYQJUQ7.js} +15 -11
- package/dist/get-analytics-4MPEK4KV.js +79 -0
- package/dist/get-analytics-ZLI3A4F5.js +80 -0
- package/dist/get-analytics-report-JCKYMA2S.js +29 -0
- package/dist/get-analytics-report-SKIP2JWH.js +30 -0
- package/dist/get-dataset-APGZBYMH.js +64 -0
- package/dist/get-dataset-LWXDVMVE.js +63 -0
- package/dist/get-evaluator-3A7BJBAB.js +47 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-IWNJINRY.js} +4 -4
- package/dist/get-project-EQVB4LEL.js +26 -0
- package/dist/{get-project-QSQEAB33.js → get-project-RK6AJTEP.js} +4 -4
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-TLG2T2DS.js} +53 -58
- package/dist/get-prompt-ZER557X3.js +94 -0
- package/dist/get-run-plan-5X7DDJRN.js +58 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-PEHRO5M3.js} +7 -9
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-375W5D6I.js} +5 -5
- package/dist/get-scenario-UQX45ZZO.js +37 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-2NBXJADG.js} +26 -22
- package/dist/get-simulation-run-QZTLGL7W.js +69 -0
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GVIBDG7K.js} +5 -5
- package/dist/get-test-suite-OODIR3RI.js +48 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-AGCHVPJK.js} +11 -13
- package/dist/get-trace-KJLYAZRC.js +45 -0
- package/dist/get-workflow-3TV2YYC5.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-7LZSVPP2.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-7CCN3EHJ.js} +4249 -325
- package/dist/index.js +59595 -38027
- package/dist/langwatch-api-analytics-reports-6CF6KUET.js +17 -0
- package/dist/langwatch-api-analytics-reports-RPFB5AO5.js +18 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-H4AI4TKG.js} +4 -4
- package/dist/langwatch-api-annotations-QSJNUQQS.js +17 -0
- package/dist/langwatch-api-monitors-ENHLOL3S.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-OHSGADTH.js} +11 -16
- package/dist/langwatch-api-secrets-AGW6UUMD.js +55 -0
- package/dist/langwatch-api-secrets-CLTS766Q.js +56 -0
- package/dist/langwatch-api-triggers-G5UWFNN2.js +22 -0
- package/dist/langwatch-api-triggers-QMR7KEXW.js +23 -0
- package/dist/langwatch-api-workflows-HKTBIKKY.js +15 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OMROZH2O.js} +4 -4
- package/dist/{list-agents-AMVDQPQL.js → list-agents-BYWDNAHT.js} +4 -4
- package/dist/list-agents-MG24MHZE.js +42 -0
- package/dist/list-analytics-reports-G3EPWGSQ.js +32 -0
- package/dist/list-analytics-reports-PPWIFRES.js +33 -0
- package/dist/list-annotations-QCLGEK5A.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-Y4EUBJHG.js} +10 -5
- package/dist/list-api-keys-DAVOBPFK.js +42 -0
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-JMW5WRXX.js} +10 -5
- package/dist/list-dashboards-2P3VJFRS.js +29 -0
- package/dist/list-dashboards-MW5O3MKH.js +30 -0
- package/dist/list-dataset-records-KEHT7G6Q.js +43 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-KMLLMZAZ.js} +14 -10
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-P3JK2ZHS.js} +9 -13
- package/dist/list-datasets-Z7OOE4KI.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-7RRESBOU.js} +5 -7
- package/dist/list-evaluators-VATJVOHR.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-OXDRPUCJ.js} +4 -4
- package/dist/list-model-providers-VXQJWDXU.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-HUVK7L3W.js} +5 -7
- package/dist/list-projects-X72ZXVY5.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-3I3LWFIR.js} +3 -3
- package/dist/list-prompt-tags-DRHUH2ND.js +23 -0
- package/dist/list-prompts-6DX3QECJ.js +30 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-PS423RU3.js} +4 -6
- package/dist/list-run-plans-5SXPOV3X.js +45 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-6VNHINPY.js} +5 -5
- package/dist/list-scenarios-EQWCW7Q4.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-GKNH3YND.js} +13 -11
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-KHY7KALI.js} +12 -8
- package/dist/list-simulation-runs-PUDK32C2.js +53 -0
- package/dist/list-test-suites-7H2DRILK.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-GY56LGLY.js} +4 -4
- package/dist/list-trigger-fires-7RZUP54Y.js +19 -0
- package/dist/list-trigger-fires-OZUF65LI.js +20 -0
- package/dist/list-triggers-BGUCUXTJ.js +45 -0
- package/dist/list-triggers-XC5KMX4C.js +46 -0
- package/dist/list-workflows-55NYC3PC.js +33 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-HV7FUWZA.js} +5 -7
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-2I7Z5FCY.js} +3 -3
- package/dist/rename-prompt-tag-SZYTOWDX.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-27AYY33K.js} +4 -4
- package/dist/rename-test-suite-CLIEPZDO.js +20 -0
- package/dist/report-issue-5HSL7CNB.js +924 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-N7ND5TM6.js} +107 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-O3VUZ6CG.js} +5 -5
- package/dist/rerun-run-plan-OJTVVGJI.js +21 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-BU6BKGCG.js} +4 -4
- package/dist/revoke-api-key-NZNCFOIR.js +15 -0
- package/dist/run-agent-244O4Y5M.js +75 -0
- package/dist/run-agent-HPCGZGYW.js +74 -0
- package/dist/run-plan-2ZPGGZ4H.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-KSXUYQT3.js} +7 -7
- package/dist/run-query-4LMOQ5UW.js +47 -0
- package/dist/run-query-BQUHIUC2.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-P3WLVPXC.js} +6 -6
- package/dist/run-test-suite-UEOMCON6.js +25 -0
- package/dist/run-workflow-EFZ6HG5B.js +36 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-I4IHEN54.js} +4 -4
- package/dist/search-traces-6IR7PR4L.js +170 -0
- package/dist/search-traces-BYML2NQ4.js +171 -0
- package/dist/set-model-provider-RBNGWFVI.js +33 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-SHEVKWRC.js} +4 -4
- package/dist/{update-agent-DUWNYBNA.js → update-agent-PTKTM2B5.js} +4 -4
- package/dist/update-agent-ZPXWRMS2.js +20 -0
- package/dist/update-dashboard-widget-HKCBXH7X.js +62 -0
- package/dist/update-dashboard-widget-IP2TTQ5G.js +63 -0
- package/dist/update-dataset-GHIN6JR3.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-IRVMY5PF.js} +6 -6
- package/dist/update-dataset-record-6VI7K2CL.js +19 -0
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-Z3L7ACNF.js} +4 -4
- package/dist/update-evaluator-5NLNE6VY.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-FEH7H6NS.js} +4 -4
- package/dist/update-project-75H3LDQT.js +24 -0
- package/dist/{update-project-Q26FBSYH.js → update-project-QPR3QX55.js} +4 -4
- package/dist/update-prompt-BCDFTNLK.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-NPRXYSWA.js} +27 -29
- package/dist/update-scenario-BNOMDCMV.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-EM5XGMB7.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +23 -8
- package/src/config.d.ts +6 -0
- package/src/config.ts +22 -29
- package/src/create-mcp-server.ts +887 -634
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-analytics-reports.ts +62 -0
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-dashboard-widgets.ts +47 -0
- package/src/langwatch-api-dashboards.ts +109 -24
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +10 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +118 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/schemas/widget-sources.ts +33 -0
- package/src/tools/add-dashboard-widget.ts +51 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-analytics-report.ts +7 -0
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-widget-queries.ts +45 -0
- package/src/tools/dashboards.ts +104 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-analytics-report.ts +7 -0
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics-report.ts +23 -0
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-analytics-reports.ts +27 -0
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboards.ts +20 -21
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +7 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +37 -34
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +86 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/uv.lock +667 -341
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/chunk-S4GVX36W.js +0 -39
- package/dist/config.d.ts +0 -17
- package/dist/create-dashboard-Z76AIPLS.js +0 -19
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-dashboard-LYMRPF27.js +0 -19
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dashboard-P4WWVHFU.js +0 -30
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-dashboards-2KJL2M42.js +0 -18
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/list-dashboards-Z3BZKDGF.js +0 -33
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
- package/src/tools/create-dashboard.ts +0 -7
- package/src/tools/delete-dashboard.ts +0 -7
- package/src/tools/get-dashboard.ts +0 -22
package/src/schemas/run-plan.ts
CHANGED
|
@@ -1,24 +1,18 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Typed inputs for the run plan tools
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* agent is told what is accepted before the request goes out rather than
|
|
8
|
-
* after: `suiteScopeSchema` and `suiteTargetSchema` on the server side.
|
|
4
|
+
* Typed inputs for the run plan tools, mirroring the platform's REST
|
|
5
|
+
* validation (`suiteScopeSchema` and `suiteTargetSchema`) so an agent
|
|
6
|
+
* learns what's accepted before the request goes out.
|
|
9
7
|
*/
|
|
10
8
|
|
|
11
9
|
/** What a run plan covers. */
|
|
12
10
|
export const runPlanScopeSchema = z.discriminatedUnion("mode", [
|
|
13
|
-
z
|
|
14
|
-
.object({ mode: z.literal("all") })
|
|
15
|
-
.describe("Every non-archived scenario of the project"),
|
|
11
|
+
z.object({ mode: z.literal("all") }).describe("Every non-archived scenario of the project"),
|
|
16
12
|
z
|
|
17
13
|
.object({
|
|
18
14
|
mode: z.literal("test_suites"),
|
|
19
|
-
testSuiteIds: z
|
|
20
|
-
.array(z.string())
|
|
21
|
-
.describe("The test suite IDs whose scenarios to run"),
|
|
15
|
+
testSuiteIds: z.array(z.string()).describe("The test suite IDs whose scenarios to run"),
|
|
22
16
|
})
|
|
23
17
|
.describe("The non-archived scenarios filed in the given test suites"),
|
|
24
18
|
z
|
|
@@ -26,9 +20,7 @@ export const runPlanScopeSchema = z.discriminatedUnion("mode", [
|
|
|
26
20
|
mode: z.literal("labels"),
|
|
27
21
|
labels: z.array(z.string()).describe("The labels to select scenarios by"),
|
|
28
22
|
})
|
|
29
|
-
.describe(
|
|
30
|
-
"The non-archived scenarios carrying at least one of the given labels",
|
|
31
|
-
),
|
|
23
|
+
.describe("The non-archived scenarios carrying at least one of the given labels"),
|
|
32
24
|
z
|
|
33
25
|
.object({ mode: z.literal("scenarios") })
|
|
34
26
|
.describe("The scenarios named in scenarioIds, picked by hand"),
|
|
@@ -41,12 +33,9 @@ export const runParametersSchema = z.record(
|
|
|
41
33
|
);
|
|
42
34
|
|
|
43
35
|
/**
|
|
44
|
-
* One thing a plan runs its scenarios against, and the values
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
* The parameters are what makes a comparison run: two targets may name the
|
|
48
|
-
* same agent with different values, and the platform runs and reports each one
|
|
49
|
-
* on its own.
|
|
36
|
+
* One thing a plan runs its scenarios against, and the values it alone
|
|
37
|
+
* runs with. Parameters make a comparison run: two targets may name the
|
|
38
|
+
* same agent with different values, each run and reported on its own.
|
|
50
39
|
*/
|
|
51
40
|
export const runPlanTargetSchema = z.object({
|
|
52
41
|
type: z
|
|
@@ -67,12 +56,9 @@ export const runPlanTargetSchema = z.object({
|
|
|
67
56
|
});
|
|
68
57
|
|
|
69
58
|
/**
|
|
70
|
-
* One target as `/api/v1` carries it.
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* run-level `parameters` an agent already knows. The REST body spells them
|
|
74
|
-
* `runParameters`, because a target holds the values it runs with rather than
|
|
75
|
-
* the values of a run.
|
|
59
|
+
* One target as `/api/v1` carries it. Tool input spells per-target
|
|
60
|
+
* values `parameters`, beside run-level `parameters`; the REST body
|
|
61
|
+
* spells them `runParameters` since a target holds the values it runs with.
|
|
76
62
|
*/
|
|
77
63
|
export interface RunPlanTargetWire {
|
|
78
64
|
type: z.infer<typeof runPlanTargetSchema>["type"];
|
|
@@ -81,9 +67,7 @@ export interface RunPlanTargetWire {
|
|
|
81
67
|
}
|
|
82
68
|
|
|
83
69
|
/** The targets of a tool call, as the REST body carries them. */
|
|
84
|
-
export function toWireTargets(
|
|
85
|
-
targets: z.infer<typeof runPlanTargetSchema>[],
|
|
86
|
-
): RunPlanTargetWire[] {
|
|
70
|
+
export function toWireTargets(targets: z.infer<typeof runPlanTargetSchema>[]): RunPlanTargetWire[] {
|
|
87
71
|
return targets.map((target) => ({
|
|
88
72
|
type: target.type,
|
|
89
73
|
referenceId: target.referenceId,
|
|
@@ -1,14 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* to a suite or a run plan, and the values a scenario carries per field.
|
|
6
|
-
*
|
|
7
|
-
* Each one mirrors the zod the platform validates the REST body with
|
|
8
|
-
* (`suiteFieldDefinitionsSchema`, `evaluatorAttachmentsSchema` and
|
|
9
|
-
* `scenarioFieldValuesSchema` on the server side), so an agent is told what
|
|
10
|
-
* is accepted before the request goes out rather than after.
|
|
11
|
-
*/
|
|
3
|
+
// Typed inputs for suite fields, evaluators, and scenario values that mirror the
|
|
4
|
+
// server-side validation schemas so agents know what's accepted before sending.
|
|
12
5
|
|
|
13
6
|
/** The value types a field can hold. */
|
|
14
7
|
export const SUITE_FIELD_TYPES = ["text", "number", "boolean"] as const;
|
|
@@ -19,16 +12,11 @@ export const suiteFieldSchema = z.object({
|
|
|
19
12
|
.string()
|
|
20
13
|
.min(1)
|
|
21
14
|
.max(64)
|
|
22
|
-
.regex(
|
|
23
|
-
/^[a-z][a-z0-9_]*$/,
|
|
24
|
-
"Lowercase letters, digits and underscores, starting with a letter",
|
|
25
|
-
)
|
|
15
|
+
.regex(/^[a-z][a-z0-9_]*$/, "Lowercase letters, digits and underscores, starting with a letter")
|
|
26
16
|
.describe(
|
|
27
17
|
"The field name, as scenarios and evaluator mappings address it, for example golden_sql. Lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
|
|
28
18
|
),
|
|
29
|
-
type: z
|
|
30
|
-
.enum(SUITE_FIELD_TYPES)
|
|
31
|
-
.describe("The value type every scenario carries for this field."),
|
|
19
|
+
type: z.enum(SUITE_FIELD_TYPES).describe("The value type every scenario carries for this field."),
|
|
32
20
|
});
|
|
33
21
|
|
|
34
22
|
/** The fields a test suite declares, in the order the platform shows them. */
|
|
@@ -40,11 +28,7 @@ export const suiteFieldsSchema = z
|
|
|
40
28
|
);
|
|
41
29
|
|
|
42
30
|
/** The sources an evaluator input can read from. */
|
|
43
|
-
export const SCENARIO_MAPPING_SOURCE_IDS = [
|
|
44
|
-
"conversation",
|
|
45
|
-
"scenario",
|
|
46
|
-
"trace",
|
|
47
|
-
] as const;
|
|
31
|
+
export const SCENARIO_MAPPING_SOURCE_IDS = ["conversation", "scenario", "trace"] as const;
|
|
48
32
|
|
|
49
33
|
/** Where one evaluator input reads its value: a source path or a literal. */
|
|
50
34
|
export const scenarioMappingSchema = z
|
|
@@ -130,8 +114,7 @@ export interface EvaluatorAttachmentWire {
|
|
|
130
114
|
mappings: Record<string, ScenarioMapping>;
|
|
131
115
|
}
|
|
132
116
|
|
|
133
|
-
const newAttachmentId = (): string =>
|
|
134
|
-
`att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
|
|
117
|
+
const newAttachmentId = (): string => `att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
|
|
135
118
|
|
|
136
119
|
/**
|
|
137
120
|
* The attachments of a tool call, as the REST body carries them: a missing
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/** The shape of an automation on the `/api/v1/triggers` surface, stated once for
|
|
4
|
+
* the tool registrations and the API client. Every type is inferred from its
|
|
5
|
+
* schema, so what the API sends and what a caller reads cannot drift. */
|
|
6
|
+
|
|
7
|
+
export const triggerActionSchema = z.enum([
|
|
8
|
+
"SEND_EMAIL",
|
|
9
|
+
"SEND_SLACK_MESSAGE",
|
|
10
|
+
"SEND_WEBHOOK",
|
|
11
|
+
"ADD_TO_DATASET",
|
|
12
|
+
"ADD_TO_ANNOTATION_QUEUE",
|
|
13
|
+
]);
|
|
14
|
+
export const alertTypeSchema = z.enum(["CRITICAL", "WARNING", "INFO"]);
|
|
15
|
+
|
|
16
|
+
/** The delivery configuration each channel reads, stated per channel so an agent
|
|
17
|
+
* can configure one without a round trip. Each passes unknown fields through:
|
|
18
|
+
* every Slack field is optional, so a webhook destination would otherwise read
|
|
19
|
+
* as an empty Slack one and be sent as nothing. */
|
|
20
|
+
const emailActionParamsSchema = z
|
|
21
|
+
.object({
|
|
22
|
+
members: z
|
|
23
|
+
.array(z.string())
|
|
24
|
+
.min(1)
|
|
25
|
+
.describe("Who receives the email. Any address, not only teammates."),
|
|
26
|
+
})
|
|
27
|
+
.passthrough();
|
|
28
|
+
|
|
29
|
+
const slackActionParamsSchema = z
|
|
30
|
+
.object({
|
|
31
|
+
slackIntegrationId: z
|
|
32
|
+
.string()
|
|
33
|
+
.optional()
|
|
34
|
+
.describe(
|
|
35
|
+
"The Slack connection to post through (an organization connection or one of this project's; they are listed in LangWatch under Settings, Integrations, Slack). A bot connection also needs `slackChannelId`; a webhook connection needs nothing else. Preferred over `slackWebhook` and `slackBotToken`.",
|
|
36
|
+
),
|
|
37
|
+
slackDelivery: z
|
|
38
|
+
.enum(["webhook", "bot"])
|
|
39
|
+
.optional()
|
|
40
|
+
.describe(
|
|
41
|
+
"How the message reaches Slack. `webhook` posts to an incoming webhook URL, `bot` posts as the LangWatch Slack app. With `slackIntegrationId` it follows the connection's kind. Absent without a connection means `webhook`.",
|
|
42
|
+
),
|
|
43
|
+
slackWebhook: z
|
|
44
|
+
.string()
|
|
45
|
+
.optional()
|
|
46
|
+
.describe(
|
|
47
|
+
"Legacy, accepted for one release: an incoming webhook URL (https://hooks.slack.com/...). It is stored as a Slack connection and the automation keeps only the connection id. Send `slackIntegrationId` instead.",
|
|
48
|
+
),
|
|
49
|
+
slackChannelId: z
|
|
50
|
+
.string()
|
|
51
|
+
.optional()
|
|
52
|
+
.describe("The channel the bot posts in, for a bot connection or `bot` delivery."),
|
|
53
|
+
slackBotToken: z
|
|
54
|
+
.string()
|
|
55
|
+
.optional()
|
|
56
|
+
.describe(
|
|
57
|
+
"Legacy, accepted for one release: a bot token, stored as a Slack connection. It never reads back. Send `slackIntegrationId` instead.",
|
|
58
|
+
),
|
|
59
|
+
slackBotTokenSet: z
|
|
60
|
+
.boolean()
|
|
61
|
+
.optional()
|
|
62
|
+
.describe(
|
|
63
|
+
"Legacy. Read: whether an automation not yet moved to a connection stores its own bot token. Write: true keeps it.",
|
|
64
|
+
),
|
|
65
|
+
})
|
|
66
|
+
.passthrough();
|
|
67
|
+
|
|
68
|
+
const webhookActionParamsSchema = z
|
|
69
|
+
.object({
|
|
70
|
+
url: z.string().describe("Where the request goes. https only, and not a private host."),
|
|
71
|
+
method: z.enum(["POST", "PUT", "PATCH"]).optional(),
|
|
72
|
+
headers: z
|
|
73
|
+
.record(z.string(), z.string())
|
|
74
|
+
.optional()
|
|
75
|
+
.describe(
|
|
76
|
+
"Static headers sent with every delivery. The values are credentials: they read back as [redacted], and sending [redacted] back keeps the stored ones. Changing `url` means sending the values again in the same call.",
|
|
77
|
+
),
|
|
78
|
+
bodyTemplate: z
|
|
79
|
+
.string()
|
|
80
|
+
.nullable()
|
|
81
|
+
.optional()
|
|
82
|
+
.describe(
|
|
83
|
+
"A Liquid template for the JSON body. Absent sends the standard LangWatch envelope.",
|
|
84
|
+
),
|
|
85
|
+
signingSecret: z
|
|
86
|
+
.string()
|
|
87
|
+
.nullable()
|
|
88
|
+
.optional()
|
|
89
|
+
.describe("Signs every delivery so the receiver can verify it."),
|
|
90
|
+
})
|
|
91
|
+
.passthrough();
|
|
92
|
+
|
|
93
|
+
const datasetActionParamsSchema = z
|
|
94
|
+
.object({
|
|
95
|
+
datasetId: z.string().describe("The dataset matched traces are appended to."),
|
|
96
|
+
datasetMapping: z
|
|
97
|
+
.object({
|
|
98
|
+
mapping: z.record(z.string(), z.unknown()),
|
|
99
|
+
expansions: z.array(z.string()).optional(),
|
|
100
|
+
})
|
|
101
|
+
.describe("How a trace becomes a row in that dataset."),
|
|
102
|
+
})
|
|
103
|
+
.passthrough();
|
|
104
|
+
|
|
105
|
+
const annotationQueueActionParamsSchema = z
|
|
106
|
+
.object({
|
|
107
|
+
annotators: z
|
|
108
|
+
.array(z.object({ id: z.string(), name: z.string() }))
|
|
109
|
+
.min(1)
|
|
110
|
+
.describe("Who the queued items go to."),
|
|
111
|
+
})
|
|
112
|
+
.passthrough();
|
|
113
|
+
|
|
114
|
+
export const actionParamsSchema = z.union([
|
|
115
|
+
emailActionParamsSchema,
|
|
116
|
+
slackActionParamsSchema,
|
|
117
|
+
webhookActionParamsSchema,
|
|
118
|
+
datasetActionParamsSchema,
|
|
119
|
+
annotationQueueActionParamsSchema,
|
|
120
|
+
]);
|
|
121
|
+
|
|
122
|
+
/** The write-side Slack shape: a redacted read round-trips, but a WRITE with no
|
|
123
|
+
* destination cannot deliver, so it is refused with the field named. A
|
|
124
|
+
* connection decides its own kind; without one, the legacy fields need a
|
|
125
|
+
* webhook URL, or a channel and a token. */
|
|
126
|
+
const slackActionParamsWriteSchema = slackActionParamsSchema.superRefine((params, ctx) => {
|
|
127
|
+
const delivery = params.slackDelivery ?? "webhook";
|
|
128
|
+
if (delivery === "bot" && !params.slackChannelId) {
|
|
129
|
+
ctx.addIssue({
|
|
130
|
+
code: "custom",
|
|
131
|
+
path: ["slackChannelId"],
|
|
132
|
+
message: params.slackIntegrationId
|
|
133
|
+
? "a bot connection needs the channel to post in"
|
|
134
|
+
: "bot delivery needs the channel to post in",
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (params.slackIntegrationId) return;
|
|
138
|
+
if (delivery === "webhook" && !params.slackWebhook) {
|
|
139
|
+
ctx.addIssue({
|
|
140
|
+
code: "custom",
|
|
141
|
+
path: ["slackIntegrationId"],
|
|
142
|
+
message: "name the Slack connection to post through",
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (delivery === "bot" && !params.slackBotToken && params.slackBotTokenSet !== true) {
|
|
146
|
+
ctx.addIssue({
|
|
147
|
+
code: "custom",
|
|
148
|
+
path: ["slackIntegrationId"],
|
|
149
|
+
message: "bot delivery needs the Slack connection to post through",
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const ACTION_PARAMS_SCHEMA_BY_ACTION: Partial<Record<string, z.ZodTypeAny>> = {
|
|
155
|
+
SEND_EMAIL: emailActionParamsSchema,
|
|
156
|
+
SEND_SLACK_MESSAGE: slackActionParamsWriteSchema,
|
|
157
|
+
SEND_WEBHOOK: webhookActionParamsSchema,
|
|
158
|
+
ADD_TO_DATASET: datasetActionParamsSchema,
|
|
159
|
+
ADD_TO_ANNOTATION_QUEUE: annotationQueueActionParamsSchema,
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/** A WRITE's `actionParams` is bound to the channel named in `action`, or a
|
|
163
|
+
* `SEND_WEBHOOK` with `{}` slips through the all-optional Slack shape and is
|
|
164
|
+
* refused server-side with worse words. An action this build does not know
|
|
165
|
+
* validates against the loose union, the tolerance reads keep. */
|
|
166
|
+
export function validateActionParamsForAction({
|
|
167
|
+
action,
|
|
168
|
+
actionParams,
|
|
169
|
+
}: {
|
|
170
|
+
action: string;
|
|
171
|
+
actionParams: unknown;
|
|
172
|
+
}): { ok: true } | { ok: false; message: string } {
|
|
173
|
+
const schema = ACTION_PARAMS_SCHEMA_BY_ACTION[action] ?? actionParamsSchema;
|
|
174
|
+
const parsed = schema.safeParse(actionParams);
|
|
175
|
+
if (parsed.success) return { ok: true };
|
|
176
|
+
const issues = parsed.error.issues
|
|
177
|
+
.map((issue) => `${issue.path.join(".") || "(actionParams)"}: ${issue.message}`)
|
|
178
|
+
.join("; ");
|
|
179
|
+
return {
|
|
180
|
+
ok: false,
|
|
181
|
+
message: `actionParams does not fit ${action} — ${issues}`,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** How `filters` is written. A keyed field sent flat never matches, so each
|
|
186
|
+
* keyed shape is spelled out; see the `/api/v1/triggers` 422 for the refusal. */
|
|
187
|
+
export const TRIGGER_FILTERS_DESCRIPTION = [
|
|
188
|
+
"Trace conditions as a JSON object string.",
|
|
189
|
+
'Unkeyed fields take a list of values: {"traces.error":["true"]}.',
|
|
190
|
+
'Keyed fields nest a key above the list. evaluations.* is keyed by the MONITOR id (the `id` from `platform_list_monitors` / GET /api/v1/monitors, not its evaluatorId): {"evaluations.passed":{"<monitorId>":["false"]}}.',
|
|
191
|
+
'metadata.value is keyed by the metadata key: {"metadata.value":{"<key>":["true"]}}.',
|
|
192
|
+
"A keyed field sent as a bare list, or keyed by anything but a monitor id, is refused.",
|
|
193
|
+
].join(" ");
|
|
194
|
+
|
|
195
|
+
export const TRIGGER_FILTER_QUERY_DESCRIPTION = [
|
|
196
|
+
"Trace query in the syntax the traces view uses; the alternative to `filters`, and it supersedes them.",
|
|
197
|
+
'Examples: status:error · evaluator:"<evaluator name>" AND evaluatorVerdict:fail · trace.attribute.<key>:<value>.',
|
|
198
|
+
].join(" ");
|
|
199
|
+
|
|
200
|
+
/** Stated on the create tool: Slack needs a connection or it never posts. */
|
|
201
|
+
export const SLACK_DELIVERY_NOTE =
|
|
202
|
+
'For SEND_SLACK_MESSAGE, send {"slackIntegrationId":"<connection id>"} for a webhook connection, or {"slackIntegrationId":"<connection id>","slackChannelId":"C..."} for a bot connection. Slack connections are listed and added in LangWatch under Settings, Integrations, Slack (organization-wide or for one project); they are also listed by GET /api/v1/slack-connections and `langwatch slack-connection list`, or ask the user for the id. slackIntegrationId is preferred: a legacy {"slackWebhook":"https://hooks.slack.com/..."} or {"slackDelivery":"bot","slackBotToken":"xoxb-...","slackChannelId":"C..."} is still accepted for one release and is stored as a connection. A Slack automation with no connection will never post.';
|
|
203
|
+
|
|
204
|
+
export const graphAlertSchema = z
|
|
205
|
+
.object({
|
|
206
|
+
seriesName: z.string().describe("The series on the graph to watch."),
|
|
207
|
+
operator: z.enum(["gt", "lt", "gte", "lte", "eq"]),
|
|
208
|
+
threshold: z.number(),
|
|
209
|
+
timePeriod: z
|
|
210
|
+
.union([
|
|
211
|
+
z.literal(1),
|
|
212
|
+
z.literal(5),
|
|
213
|
+
z.literal(15),
|
|
214
|
+
z.literal(30),
|
|
215
|
+
z.literal(60),
|
|
216
|
+
z.literal(1440),
|
|
217
|
+
])
|
|
218
|
+
.describe("The window, in minutes, the series is read over."),
|
|
219
|
+
})
|
|
220
|
+
.describe("The rule an alert fires by. Send it with `customGraphId` and `alertType`.");
|
|
221
|
+
|
|
222
|
+
/** What a scheduled report renders and when. Structured, not a loose record: an
|
|
223
|
+
* agent that cannot see the field names guesses them, and a guess is a 422.
|
|
224
|
+
* The server holds a write to this same shape, so looseness only delays it. */
|
|
225
|
+
export const reportSchema = z
|
|
226
|
+
.object({
|
|
227
|
+
source: z
|
|
228
|
+
.discriminatedUnion("kind", [
|
|
229
|
+
z.object({
|
|
230
|
+
kind: z.literal("dashboard"),
|
|
231
|
+
dashboardId: z.string(),
|
|
232
|
+
}),
|
|
233
|
+
z.object({
|
|
234
|
+
kind: z.literal("customGraph"),
|
|
235
|
+
customGraphId: z.string(),
|
|
236
|
+
}),
|
|
237
|
+
z.object({
|
|
238
|
+
kind: z.literal("traceQuery"),
|
|
239
|
+
filters: z.record(z.string(), z.unknown()).optional(),
|
|
240
|
+
metric: z.string().optional(),
|
|
241
|
+
topN: z.number().int().min(1).max(100).optional(),
|
|
242
|
+
}),
|
|
243
|
+
])
|
|
244
|
+
.describe("What the report renders: a dashboard, one graph, or a table of traces."),
|
|
245
|
+
schedule: z
|
|
246
|
+
.object({
|
|
247
|
+
cron: z
|
|
248
|
+
.string()
|
|
249
|
+
.describe(
|
|
250
|
+
'A 5-field cron expression (minute hour day-of-month month day-of-week), for example "0 9 * * 1". It can send at most every 15 minutes.',
|
|
251
|
+
),
|
|
252
|
+
timezone: z.string().describe('An IANA timezone, for example "Europe/Amsterdam" or "UTC".'),
|
|
253
|
+
})
|
|
254
|
+
.describe("When it sends."),
|
|
255
|
+
compareToPrevious: z
|
|
256
|
+
.boolean()
|
|
257
|
+
.optional()
|
|
258
|
+
.describe("Include a this-period-versus-last comparison."),
|
|
259
|
+
})
|
|
260
|
+
.describe(
|
|
261
|
+
"What a scheduled report renders and when. Send it with a channel that can carry a message: email or Slack.",
|
|
262
|
+
);
|
|
263
|
+
|
|
264
|
+
export const templatesSchema = z
|
|
265
|
+
.object({
|
|
266
|
+
slackTemplateType: z.enum(["string", "block_kit"]).nullable().optional(),
|
|
267
|
+
slackTemplate: z.string().nullable().optional(),
|
|
268
|
+
emailSubjectTemplate: z.string().nullable().optional(),
|
|
269
|
+
emailBodyTemplate: z.string().nullable().optional(),
|
|
270
|
+
})
|
|
271
|
+
.describe(
|
|
272
|
+
"The Liquid templates the message is rendered from. Absent fields render the LangWatch default.",
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
export const notificationCadenceSchema = z
|
|
276
|
+
.enum(["immediate", "5min_digest", "15min_digest", "hourly_digest"])
|
|
277
|
+
.describe(
|
|
278
|
+
"How often a notification automation may send. A new one starts on a five-minute digest.",
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* What a read answers with. Permissive on purpose: an MCP client talks to whichever LangWatch it
|
|
283
|
+
* is pointed at, so an unknown field is carried through and a field a deployment does not send
|
|
284
|
+
* yet is absent rather than fatal.
|
|
285
|
+
*/
|
|
286
|
+
export const triggerSchema = z
|
|
287
|
+
.object({
|
|
288
|
+
id: z.string(),
|
|
289
|
+
name: z.string(),
|
|
290
|
+
action: z.string(),
|
|
291
|
+
/** Where it delivers. The rule it fires by is stated separately, below,
|
|
292
|
+
* the same way a write states it. */
|
|
293
|
+
actionParams: z.record(z.string(), z.unknown()).default({}),
|
|
294
|
+
graphAlert: graphAlertSchema.nullable().optional(),
|
|
295
|
+
// A read stays tolerant of a deployment that words this differently; a
|
|
296
|
+
// write is held to `reportSchema`.
|
|
297
|
+
report: z.record(z.string(), z.unknown()).nullable().optional(),
|
|
298
|
+
filters: z.record(z.string(), z.unknown()).default({}),
|
|
299
|
+
filterQuery: z.string().nullable().optional(),
|
|
300
|
+
kind: z.string().optional(),
|
|
301
|
+
customGraphId: z.string().nullable().optional(),
|
|
302
|
+
notificationCadence: z.string().nullable().optional(),
|
|
303
|
+
traceDebounceMs: z.number().nullable().optional(),
|
|
304
|
+
templates: z.record(z.string(), z.unknown()).optional(),
|
|
305
|
+
active: z.boolean(),
|
|
306
|
+
message: z.string().nullable(),
|
|
307
|
+
alertType: z.string().nullable(),
|
|
308
|
+
createdAt: z.string(),
|
|
309
|
+
updatedAt: z.string(),
|
|
310
|
+
platformUrl: z.string().optional(),
|
|
311
|
+
})
|
|
312
|
+
.passthrough();
|
|
313
|
+
|
|
314
|
+
const triggerFireSchema = z
|
|
315
|
+
.object({
|
|
316
|
+
id: z.string(),
|
|
317
|
+
triggerId: z.string(),
|
|
318
|
+
customGraphId: z.string().nullable(),
|
|
319
|
+
firedAt: z.string(),
|
|
320
|
+
resolvedAt: z.string().nullable(),
|
|
321
|
+
})
|
|
322
|
+
.passthrough();
|
|
323
|
+
|
|
324
|
+
/** A page of fires: `{ fires, nextCursor }`, or the bare array an older
|
|
325
|
+
* deployment answers with, which has no next page. */
|
|
326
|
+
export const triggerFirePageSchema = z.union([
|
|
327
|
+
z.array(triggerFireSchema).transform((fires) => ({ fires, nextCursor: null })),
|
|
328
|
+
z
|
|
329
|
+
.object({
|
|
330
|
+
fires: z.array(triggerFireSchema).optional(),
|
|
331
|
+
data: z.array(triggerFireSchema).optional(),
|
|
332
|
+
nextCursor: z.string().nullable().optional(),
|
|
333
|
+
})
|
|
334
|
+
.transform(({ fires, data, nextCursor }) => ({
|
|
335
|
+
fires: fires ?? data ?? [],
|
|
336
|
+
nextCursor: nextCursor ?? null,
|
|
337
|
+
})),
|
|
338
|
+
]);
|
|
339
|
+
|
|
340
|
+
export const testFireResultSchema = z
|
|
341
|
+
.object({
|
|
342
|
+
channel: z.string(),
|
|
343
|
+
recipientCount: z.number(),
|
|
344
|
+
usedDefault: z.boolean(),
|
|
345
|
+
missingVariables: z.array(z.string()).default([]),
|
|
346
|
+
errors: z.array(z.string()).default([]),
|
|
347
|
+
httpStatus: z.number().optional(),
|
|
348
|
+
})
|
|
349
|
+
.passthrough();
|
|
350
|
+
|
|
351
|
+
export const deletedTriggerSchema = z.object({
|
|
352
|
+
id: z.string(),
|
|
353
|
+
deleted: z.boolean(),
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
export type Trigger = z.infer<typeof triggerSchema>;
|
|
357
|
+
export type TriggerFirePage = z.infer<typeof triggerFirePageSchema>;
|
|
358
|
+
export type TestFireResult = z.infer<typeof testFireResultSchema>;
|
|
359
|
+
export type TriggerAction = z.infer<typeof triggerActionSchema>;
|
|
360
|
+
export type TriggerAlertType = z.infer<typeof alertTypeSchema>;
|
|
361
|
+
export type TriggerActionParams = z.infer<typeof actionParamsSchema>;
|
|
362
|
+
export type GraphAlertRule = z.infer<typeof graphAlertSchema>;
|
|
363
|
+
export type ReportRule = z.infer<typeof reportSchema>;
|
|
364
|
+
export type TriggerTemplates = z.infer<typeof templatesSchema>;
|
|
365
|
+
export type NotificationCadence = z.infer<typeof notificationCadenceSchema>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `sources` input of the widget tools: the trace origins a widget's queries count, checked
|
|
3
|
+
* here so an unknown one never reaches the platform. Spec: dashboards-v2.feature AC200j, AC200n.
|
|
4
|
+
*/
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
|
|
7
|
+
/** The sources the platform takes, "other" being every origin not listed before it. */
|
|
8
|
+
export const WIDGET_SOURCES = [
|
|
9
|
+
"application",
|
|
10
|
+
"simulation",
|
|
11
|
+
"evaluation",
|
|
12
|
+
"sample",
|
|
13
|
+
"coding_agent",
|
|
14
|
+
"ai_tool",
|
|
15
|
+
"gateway",
|
|
16
|
+
"workflow",
|
|
17
|
+
"playground",
|
|
18
|
+
"langy",
|
|
19
|
+
"other",
|
|
20
|
+
] as const;
|
|
21
|
+
|
|
22
|
+
/** A trace origin a widget query can count. */
|
|
23
|
+
export type WidgetSource = (typeof WIDGET_SOURCES)[number];
|
|
24
|
+
|
|
25
|
+
/** The tools' `sources` input: refused at the tool when it names anything else. */
|
|
26
|
+
export const widgetSourcesInput = z
|
|
27
|
+
.array(z.enum(WIDGET_SOURCES))
|
|
28
|
+
.max(WIDGET_SOURCES.length)
|
|
29
|
+
.nullable()
|
|
30
|
+
.optional()
|
|
31
|
+
.describe(
|
|
32
|
+
`The trace origins every query of the widget counts: any of ${WIDGET_SOURCES.join(", ")} (other = any origin not listed). A new widget counts ["application"]; add "langy" to count your own conversations, as a widget about Langy's requests needs. null makes it unset again, counting the default.`,
|
|
33
|
+
);
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `add_dashboard_widget`: one create under the dashboard's path, which also places the widget.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { createWidgetOnDashboard } from "../langwatch-api-dashboards.js";
|
|
6
|
+
import { LangWatchApiError } from "../langwatch-api.js";
|
|
7
|
+
import type { WidgetSource } from "../schemas/widget-sources.js";
|
|
8
|
+
import {
|
|
9
|
+
type DashboardWidgetQueriesInput,
|
|
10
|
+
normalizeDashboardWidgetQueries,
|
|
11
|
+
withSources,
|
|
12
|
+
} from "./dashboard-widget-queries.js";
|
|
13
|
+
|
|
14
|
+
export interface AddDashboardWidgetParams {
|
|
15
|
+
dashboardId: string;
|
|
16
|
+
name: string;
|
|
17
|
+
code: string;
|
|
18
|
+
queries: DashboardWidgetQueriesInput;
|
|
19
|
+
/** Set on every query; absent, the platform stores application on each. */
|
|
20
|
+
sources?: WidgetSource[] | null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function handleAddDashboardWidget(params: AddDashboardWidgetParams): Promise<string> {
|
|
24
|
+
const queries = withSources({
|
|
25
|
+
queries: normalizeDashboardWidgetQueries({ queries: params.queries }),
|
|
26
|
+
sources: params.sources,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
let widget;
|
|
30
|
+
try {
|
|
31
|
+
widget = await createWidgetOnDashboard({
|
|
32
|
+
dashboardId: params.dashboardId,
|
|
33
|
+
name: params.name,
|
|
34
|
+
code: params.code,
|
|
35
|
+
queries,
|
|
36
|
+
});
|
|
37
|
+
} catch (error) {
|
|
38
|
+
if (error instanceof LangWatchApiError && error.status === 404) {
|
|
39
|
+
throw new Error(
|
|
40
|
+
`No dashboard with ID "${params.dashboardId}" was found in this project. Use list_dashboards to find a valid dashboard ID.`,
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
throw error;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return [
|
|
47
|
+
`Widget "${params.name}" created and added to dashboard "${params.dashboardId}".`,
|
|
48
|
+
`**Widget ID**: ${widget.id}`,
|
|
49
|
+
`**Queries**: ${queries.map((q) => q.name).join(", ")}`,
|
|
50
|
+
].join("\n");
|
|
51
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { archiveProject as apiArchiveProject } from "../langwatch-api-projects.
|
|
1
|
+
import { archiveProject as apiArchiveProject } from "../langwatch-api-projects.ts";
|
|
2
2
|
|
|
3
|
-
export async function handleArchiveProject(params: {
|
|
4
|
-
id: string;
|
|
5
|
-
}): Promise<string> {
|
|
3
|
+
export async function handleArchiveProject(params: { id: string }): Promise<string> {
|
|
6
4
|
const result = await apiArchiveProject(params.id);
|
|
7
5
|
|
|
8
6
|
return (
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { archiveRunPlan as apiArchiveRunPlan } from "../langwatch-api-run-plans.
|
|
1
|
+
import { archiveRunPlan as apiArchiveRunPlan } from "../langwatch-api-run-plans.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Handles the platform_archive_run_plan MCP tool invocation.
|
|
5
5
|
*/
|
|
6
|
-
export async function handleArchiveRunPlan(params: {
|
|
7
|
-
id: string;
|
|
8
|
-
}): Promise<string> {
|
|
6
|
+
export async function handleArchiveRunPlan(params: { id: string }): Promise<string> {
|
|
9
7
|
const result = await apiArchiveRunPlan(params.id);
|
|
10
8
|
|
|
11
9
|
return `Run plan ${result.id} is archived. Its past runs stay readable, and the name is free for a new plan.`;
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import { archiveScenario as apiArchiveScenario } from "../langwatch-api-scenarios.
|
|
1
|
+
import { archiveScenario as apiArchiveScenario } from "../langwatch-api-scenarios.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Handles the platform_archive_scenario MCP tool invocation.
|
|
5
5
|
*
|
|
6
6
|
* Archives (soft-deletes) a scenario and returns confirmation.
|
|
7
7
|
*/
|
|
8
|
-
export async function handleArchiveScenario(params: {
|
|
9
|
-
scenarioId: string;
|
|
10
|
-
}): Promise<string> {
|
|
8
|
+
export async function handleArchiveScenario(params: { scenarioId: string }): Promise<string> {
|
|
11
9
|
const result = await apiArchiveScenario(params.scenarioId);
|
|
12
10
|
|
|
13
11
|
const lines: string[] = [];
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { archiveTestSuite as apiArchiveTestSuite } from "../langwatch-api-test-suites.
|
|
1
|
+
import { archiveTestSuite as apiArchiveTestSuite } from "../langwatch-api-test-suites.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Handles the platform_archive_test_suite MCP tool invocation.
|
|
5
5
|
*/
|
|
6
|
-
export async function handleArchiveTestSuite(params: {
|
|
7
|
-
id: string;
|
|
8
|
-
}): Promise<string> {
|
|
6
|
+
export async function handleArchiveTestSuite(params: { id: string }): Promise<string> {
|
|
9
7
|
const result = await apiArchiveTestSuite(params.id);
|
|
10
8
|
|
|
11
9
|
return `Test suite ${result.id} is archived, and the scenarios filed in it are archived with it.`;
|
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
import { assignPromptTag as apiAssignPromptTag } from "../langwatch-api.
|
|
1
|
+
import { assignPromptTag as apiAssignPromptTag } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleAssignPromptTag(params: {
|
|
4
4
|
idOrHandle: string;
|
|
5
5
|
tag: string;
|
|
6
6
|
versionId: string;
|
|
7
7
|
}): Promise<string> {
|
|
8
|
-
await apiAssignPromptTag({
|
|
8
|
+
await apiAssignPromptTag({
|
|
9
|
+
idOrHandle: params.idOrHandle,
|
|
10
|
+
tag: params.tag,
|
|
11
|
+
versionId: params.versionId,
|
|
12
|
+
});
|
|
9
13
|
|
|
10
14
|
const lines: string[] = [];
|
|
11
15
|
lines.push("Tag assigned successfully!\n");
|