@langwatch/mcp-server 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +192 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-65QN3XRX.js} +4 -4
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-NKZDZY3D.js} +4 -4
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-OJLYYJWI.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-G4ZAZTZW.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-EYXMLE5X.js → chunk-HGELNIVY.js} +56 -47
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-RVAE2MEA.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-S4GVX36W.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-QRO35SOM.js → chunk-J7PQKBLD.js} +652 -528
- package/dist/{chunk-SR6QPRIE.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-UFOYFYI7.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-56OGTS24.js → chunk-PTDR56Z3.js} +4 -11
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-QAMP3CGR.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/{chunk-6DJDEXET.js → chunk-VMWJD732.js} +6 -12
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-Z76AIPLS.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -10
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-ZQ7TFIO2.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-UEM77OE7.js} +5 -5
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-4IPXVSFQ.js} +6 -6
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-LYMRPF27.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-OTIYUFZA.js} +15 -11
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-P4WWVHFU.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-QSQEAB33.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-OWXJZHQQ.js} +53 -58
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-BE3M2AUX.js} +7 -9
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-YM4ODWW6.js} +5 -5
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-LCFSMPGQ.js} +26 -22
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GC2O7YH6.js} +5 -5
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-IR4QOS43.js} +4071 -347
- package/dist/index.js +58671 -37983
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-2KJL2M42.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/{list-agents-AMVDQPQL.js → list-agents-HVYWEQMI.js} +4 -4
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-Z3BZKDGF.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-PX2XI6EK.js} +5 -5
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-QZRPVHS4.js} +13 -11
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-OEADLLQR.js} +12 -8
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-N4WVLBKK.js} +4 -4
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-4J7UX5S2.js} +4 -4
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-DCZFLGDO.js} +81 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-TG2V4WM4.js} +5 -5
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-PAARQSI4.js} +7 -7
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-SQJELEZX.js} +6 -6
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/run-workflow-SP32SO5M.js +36 -0
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-DUWNYBNA.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-Q26FBSYH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-MY263DYF.js} +27 -29
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-MOHRQKAQ.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +755 -609
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +108 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
|
@@ -3,7 +3,7 @@ import { request as httpRequest, type RequestOptions } from "node:http";
|
|
|
3
3
|
import { request as httpsRequest } from "node:https";
|
|
4
4
|
import { isIP } from "node:net";
|
|
5
5
|
|
|
6
|
-
import { classify } from "@langwatch/ssrf";
|
|
6
|
+
import { classify } from "@langwatch/egress/ssrf";
|
|
7
7
|
|
|
8
8
|
const MAX_REDIRECTS = 5;
|
|
9
9
|
const MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
|
|
@@ -20,7 +20,10 @@ const SENSITIVE_REDIRECT_HEADERS = new Set([
|
|
|
20
20
|
"x-access-token",
|
|
21
21
|
]);
|
|
22
22
|
|
|
23
|
-
type DnsResolver = (
|
|
23
|
+
type DnsResolver = (
|
|
24
|
+
hostname: string,
|
|
25
|
+
options: { all: true; verbatim: true },
|
|
26
|
+
) => Promise<ResolvedAddress[]>;
|
|
24
27
|
|
|
25
28
|
interface ResolvedAddress {
|
|
26
29
|
address: string;
|
|
@@ -34,7 +37,9 @@ export interface PublicDestination {
|
|
|
34
37
|
}
|
|
35
38
|
|
|
36
39
|
function unsafeDestinationError(): Error {
|
|
37
|
-
return new Error(
|
|
40
|
+
return new Error(
|
|
41
|
+
"HTTP agent destinations must resolve only to globally routable public addresses",
|
|
42
|
+
);
|
|
38
43
|
}
|
|
39
44
|
|
|
40
45
|
function hostnameWithoutBrackets(hostname: string): string {
|
|
@@ -66,21 +71,21 @@ async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T
|
|
|
66
71
|
|
|
67
72
|
export function headersForRedirect(
|
|
68
73
|
headers: Record<string, string> | undefined,
|
|
69
|
-
crossOrigin: boolean
|
|
74
|
+
crossOrigin: boolean,
|
|
70
75
|
): Record<string, string> | undefined {
|
|
71
76
|
if (!headers || !crossOrigin) {
|
|
72
77
|
return headers;
|
|
73
78
|
}
|
|
74
79
|
|
|
75
80
|
const safeHeaders = Object.fromEntries(
|
|
76
|
-
Object.entries(headers).filter(([name]) => !SENSITIVE_REDIRECT_HEADERS.has(name.toLowerCase()))
|
|
81
|
+
Object.entries(headers).filter(([name]) => !SENSITIVE_REDIRECT_HEADERS.has(name.toLowerCase())),
|
|
77
82
|
);
|
|
78
83
|
return Object.keys(safeHeaders).length > 0 ? safeHeaders : undefined;
|
|
79
84
|
}
|
|
80
85
|
|
|
81
86
|
export async function resolvePublicDestination(
|
|
82
87
|
input: string,
|
|
83
|
-
resolver: DnsResolver = dnsLookup
|
|
88
|
+
resolver: DnsResolver = dnsLookup,
|
|
84
89
|
): Promise<PublicDestination> {
|
|
85
90
|
let url: URL;
|
|
86
91
|
try {
|
|
@@ -89,7 +94,11 @@ export async function resolvePublicDestination(
|
|
|
89
94
|
throw new Error("HTTP agent URL is invalid");
|
|
90
95
|
}
|
|
91
96
|
|
|
92
|
-
if (
|
|
97
|
+
if (
|
|
98
|
+
(url.protocol !== "http:" && url.protocol !== "https:") ||
|
|
99
|
+
url.username !== "" ||
|
|
100
|
+
url.password !== ""
|
|
101
|
+
) {
|
|
93
102
|
throw unsafeDestinationError();
|
|
94
103
|
}
|
|
95
104
|
|
|
@@ -104,7 +113,7 @@ export async function resolvePublicDestination(
|
|
|
104
113
|
try {
|
|
105
114
|
addresses = await withTimeout(
|
|
106
115
|
resolver(hostname, { all: true, verbatim: true }),
|
|
107
|
-
REQUEST_TIMEOUT_MS
|
|
116
|
+
REQUEST_TIMEOUT_MS,
|
|
108
117
|
);
|
|
109
118
|
} catch {
|
|
110
119
|
throw new Error("HTTP agent destination could not be resolved");
|
|
@@ -135,7 +144,7 @@ interface PublicJsonRequestOptions {
|
|
|
135
144
|
|
|
136
145
|
function requestBody(
|
|
137
146
|
destination: PublicDestination,
|
|
138
|
-
options: PublicJsonRequestOptions
|
|
147
|
+
options: PublicJsonRequestOptions,
|
|
139
148
|
): Promise<{ body: string; location?: string; statusCode: number }> {
|
|
140
149
|
return new Promise((resolve, reject) => {
|
|
141
150
|
const hostname = hostnameWithoutBrackets(destination.url.hostname);
|
|
@@ -198,7 +207,7 @@ function requestBody(
|
|
|
198
207
|
export async function requestPublicJson(
|
|
199
208
|
input: string,
|
|
200
209
|
options: PublicJsonRequestOptions = {},
|
|
201
|
-
redirectCount = 0
|
|
210
|
+
redirectCount = 0,
|
|
202
211
|
): Promise<Record<string, unknown>> {
|
|
203
212
|
const destination = await resolvePublicDestination(input);
|
|
204
213
|
const response = await requestBody(destination, options);
|
|
@@ -210,7 +219,7 @@ export async function requestPublicJson(
|
|
|
210
219
|
const redirectUrl = new URL(response.location, destination.url);
|
|
211
220
|
const redirectHeaders = headersForRedirect(
|
|
212
221
|
options.headers,
|
|
213
|
-
redirectUrl.origin !== destination.url.origin
|
|
222
|
+
redirectUrl.origin !== destination.url.origin,
|
|
214
223
|
);
|
|
215
224
|
const preserveBody = response.statusCode === 307 || response.statusCode === 308;
|
|
216
225
|
return requestPublicJson(
|
|
@@ -223,7 +232,7 @@ export async function requestPublicJson(
|
|
|
223
232
|
headers: redirectHeaders,
|
|
224
233
|
method: "GET",
|
|
225
234
|
},
|
|
226
|
-
redirectCount + 1
|
|
235
|
+
redirectCount + 1,
|
|
227
236
|
);
|
|
228
237
|
}
|
|
229
238
|
|
|
@@ -124,17 +124,7 @@ export const analyticsMetrics: MetricInfo[] = [
|
|
|
124
124
|
category: "events",
|
|
125
125
|
name: "event_score",
|
|
126
126
|
label: "Event Score",
|
|
127
|
-
allowedAggregations: [
|
|
128
|
-
"terms",
|
|
129
|
-
"avg",
|
|
130
|
-
"sum",
|
|
131
|
-
"min",
|
|
132
|
-
"max",
|
|
133
|
-
"median",
|
|
134
|
-
"p99",
|
|
135
|
-
"p95",
|
|
136
|
-
"p90",
|
|
137
|
-
],
|
|
127
|
+
allowedAggregations: ["terms", "avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
|
|
138
128
|
description: "Numeric score from events (requires event_type key and metrics key)",
|
|
139
129
|
},
|
|
140
130
|
{
|
|
@@ -142,8 +132,7 @@ export const analyticsMetrics: MetricInfo[] = [
|
|
|
142
132
|
name: "event_details",
|
|
143
133
|
label: "Event Details",
|
|
144
134
|
allowedAggregations: ["cardinality"],
|
|
145
|
-
description:
|
|
146
|
-
"Event detail key/value distribution (requires event_type key and details key)",
|
|
135
|
+
description: "Event detail key/value distribution (requires event_type key and details key)",
|
|
147
136
|
},
|
|
148
137
|
// evaluations
|
|
149
138
|
{
|
|
@@ -158,8 +147,7 @@ export const analyticsMetrics: MetricInfo[] = [
|
|
|
158
147
|
name: "evaluation_pass_rate",
|
|
159
148
|
label: "Evaluation Pass Rate",
|
|
160
149
|
allowedAggregations: ["avg", "sum", "min", "max", "median", "p99", "p95", "p90"],
|
|
161
|
-
description:
|
|
162
|
-
"Percentage of traces passing evaluation (requires evaluator_id key)",
|
|
150
|
+
description: "Percentage of traces passing evaluation (requires evaluator_id key)",
|
|
163
151
|
},
|
|
164
152
|
{
|
|
165
153
|
category: "evaluations",
|
|
@@ -16,6 +16,7 @@ export const DATASET_COLUMN_TYPES = [
|
|
|
16
16
|
"annotations",
|
|
17
17
|
"evaluations",
|
|
18
18
|
"image",
|
|
19
|
+
"file",
|
|
19
20
|
] as const;
|
|
20
21
|
|
|
21
22
|
/**
|
|
@@ -34,10 +35,8 @@ export const datasetColumnDefinitionSchema = z.object({
|
|
|
34
35
|
});
|
|
35
36
|
|
|
36
37
|
/**
|
|
37
|
-
* Zod schema for the platform_create_dataset MCP tool parameters
|
|
38
|
-
*
|
|
39
|
-
* Extracted so it can be shared between the MCP tool registration
|
|
40
|
-
* (create-mcp-server.ts) and unit tests.
|
|
38
|
+
* Zod schema for the platform_create_dataset MCP tool parameters,
|
|
39
|
+
* extracted so it's shared between create-mcp-server.ts and unit tests.
|
|
41
40
|
*/
|
|
42
41
|
export const createDatasetSchema = z.object({
|
|
43
42
|
name: z.string().min(1).describe("Dataset name"),
|
package/src/schemas/run-plan.ts
CHANGED
|
@@ -1,24 +1,18 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Typed inputs for the run plan tools
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* agent is told what is accepted before the request goes out rather than
|
|
8
|
-
* after: `suiteScopeSchema` and `suiteTargetSchema` on the server side.
|
|
4
|
+
* Typed inputs for the run plan tools, mirroring the platform's REST
|
|
5
|
+
* validation (`suiteScopeSchema` and `suiteTargetSchema`) so an agent
|
|
6
|
+
* learns what's accepted before the request goes out.
|
|
9
7
|
*/
|
|
10
8
|
|
|
11
9
|
/** What a run plan covers. */
|
|
12
10
|
export const runPlanScopeSchema = z.discriminatedUnion("mode", [
|
|
13
|
-
z
|
|
14
|
-
.object({ mode: z.literal("all") })
|
|
15
|
-
.describe("Every non-archived scenario of the project"),
|
|
11
|
+
z.object({ mode: z.literal("all") }).describe("Every non-archived scenario of the project"),
|
|
16
12
|
z
|
|
17
13
|
.object({
|
|
18
14
|
mode: z.literal("test_suites"),
|
|
19
|
-
testSuiteIds: z
|
|
20
|
-
.array(z.string())
|
|
21
|
-
.describe("The test suite IDs whose scenarios to run"),
|
|
15
|
+
testSuiteIds: z.array(z.string()).describe("The test suite IDs whose scenarios to run"),
|
|
22
16
|
})
|
|
23
17
|
.describe("The non-archived scenarios filed in the given test suites"),
|
|
24
18
|
z
|
|
@@ -26,9 +20,7 @@ export const runPlanScopeSchema = z.discriminatedUnion("mode", [
|
|
|
26
20
|
mode: z.literal("labels"),
|
|
27
21
|
labels: z.array(z.string()).describe("The labels to select scenarios by"),
|
|
28
22
|
})
|
|
29
|
-
.describe(
|
|
30
|
-
"The non-archived scenarios carrying at least one of the given labels",
|
|
31
|
-
),
|
|
23
|
+
.describe("The non-archived scenarios carrying at least one of the given labels"),
|
|
32
24
|
z
|
|
33
25
|
.object({ mode: z.literal("scenarios") })
|
|
34
26
|
.describe("The scenarios named in scenarioIds, picked by hand"),
|
|
@@ -41,12 +33,9 @@ export const runParametersSchema = z.record(
|
|
|
41
33
|
);
|
|
42
34
|
|
|
43
35
|
/**
|
|
44
|
-
* One thing a plan runs its scenarios against, and the values
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
* The parameters are what makes a comparison run: two targets may name the
|
|
48
|
-
* same agent with different values, and the platform runs and reports each one
|
|
49
|
-
* on its own.
|
|
36
|
+
* One thing a plan runs its scenarios against, and the values it alone
|
|
37
|
+
* runs with. Parameters make a comparison run: two targets may name the
|
|
38
|
+
* same agent with different values, each run and reported on its own.
|
|
50
39
|
*/
|
|
51
40
|
export const runPlanTargetSchema = z.object({
|
|
52
41
|
type: z
|
|
@@ -67,12 +56,9 @@ export const runPlanTargetSchema = z.object({
|
|
|
67
56
|
});
|
|
68
57
|
|
|
69
58
|
/**
|
|
70
|
-
* One target as `/api/v1` carries it.
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* run-level `parameters` an agent already knows. The REST body spells them
|
|
74
|
-
* `runParameters`, because a target holds the values it runs with rather than
|
|
75
|
-
* the values of a run.
|
|
59
|
+
* One target as `/api/v1` carries it. Tool input spells per-target
|
|
60
|
+
* values `parameters`, beside run-level `parameters`; the REST body
|
|
61
|
+
* spells them `runParameters` since a target holds the values it runs with.
|
|
76
62
|
*/
|
|
77
63
|
export interface RunPlanTargetWire {
|
|
78
64
|
type: z.infer<typeof runPlanTargetSchema>["type"];
|
|
@@ -81,9 +67,7 @@ export interface RunPlanTargetWire {
|
|
|
81
67
|
}
|
|
82
68
|
|
|
83
69
|
/** The targets of a tool call, as the REST body carries them. */
|
|
84
|
-
export function toWireTargets(
|
|
85
|
-
targets: z.infer<typeof runPlanTargetSchema>[],
|
|
86
|
-
): RunPlanTargetWire[] {
|
|
70
|
+
export function toWireTargets(targets: z.infer<typeof runPlanTargetSchema>[]): RunPlanTargetWire[] {
|
|
87
71
|
return targets.map((target) => ({
|
|
88
72
|
type: target.type,
|
|
89
73
|
referenceId: target.referenceId,
|
|
@@ -1,14 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* to a suite or a run plan, and the values a scenario carries per field.
|
|
6
|
-
*
|
|
7
|
-
* Each one mirrors the zod the platform validates the REST body with
|
|
8
|
-
* (`suiteFieldDefinitionsSchema`, `evaluatorAttachmentsSchema` and
|
|
9
|
-
* `scenarioFieldValuesSchema` on the server side), so an agent is told what
|
|
10
|
-
* is accepted before the request goes out rather than after.
|
|
11
|
-
*/
|
|
3
|
+
// Typed inputs for suite fields, evaluators, and scenario values that mirror the
|
|
4
|
+
// server-side validation schemas so agents know what's accepted before sending.
|
|
12
5
|
|
|
13
6
|
/** The value types a field can hold. */
|
|
14
7
|
export const SUITE_FIELD_TYPES = ["text", "number", "boolean"] as const;
|
|
@@ -19,16 +12,11 @@ export const suiteFieldSchema = z.object({
|
|
|
19
12
|
.string()
|
|
20
13
|
.min(1)
|
|
21
14
|
.max(64)
|
|
22
|
-
.regex(
|
|
23
|
-
/^[a-z][a-z0-9_]*$/,
|
|
24
|
-
"Lowercase letters, digits and underscores, starting with a letter",
|
|
25
|
-
)
|
|
15
|
+
.regex(/^[a-z][a-z0-9_]*$/, "Lowercase letters, digits and underscores, starting with a letter")
|
|
26
16
|
.describe(
|
|
27
17
|
"The field name, as scenarios and evaluator mappings address it, for example golden_sql. Lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
|
|
28
18
|
),
|
|
29
|
-
type: z
|
|
30
|
-
.enum(SUITE_FIELD_TYPES)
|
|
31
|
-
.describe("The value type every scenario carries for this field."),
|
|
19
|
+
type: z.enum(SUITE_FIELD_TYPES).describe("The value type every scenario carries for this field."),
|
|
32
20
|
});
|
|
33
21
|
|
|
34
22
|
/** The fields a test suite declares, in the order the platform shows them. */
|
|
@@ -40,11 +28,7 @@ export const suiteFieldsSchema = z
|
|
|
40
28
|
);
|
|
41
29
|
|
|
42
30
|
/** The sources an evaluator input can read from. */
|
|
43
|
-
export const SCENARIO_MAPPING_SOURCE_IDS = [
|
|
44
|
-
"conversation",
|
|
45
|
-
"scenario",
|
|
46
|
-
"trace",
|
|
47
|
-
] as const;
|
|
31
|
+
export const SCENARIO_MAPPING_SOURCE_IDS = ["conversation", "scenario", "trace"] as const;
|
|
48
32
|
|
|
49
33
|
/** Where one evaluator input reads its value: a source path or a literal. */
|
|
50
34
|
export const scenarioMappingSchema = z
|
|
@@ -130,8 +114,7 @@ export interface EvaluatorAttachmentWire {
|
|
|
130
114
|
mappings: Record<string, ScenarioMapping>;
|
|
131
115
|
}
|
|
132
116
|
|
|
133
|
-
const newAttachmentId = (): string =>
|
|
134
|
-
`att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
|
|
117
|
+
const newAttachmentId = (): string => `att_${crypto.randomUUID().replace(/-/g, "").slice(0, 21)}`;
|
|
135
118
|
|
|
136
119
|
/**
|
|
137
120
|
* The attachments of a tool call, as the REST body carries them: a missing
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/** The shape of an automation on the `/api/v1/triggers` surface, stated once for
|
|
4
|
+
* the tool registrations and the API client. Every type is inferred from its
|
|
5
|
+
* schema, so what the API sends and what a caller reads cannot drift. */
|
|
6
|
+
|
|
7
|
+
export const triggerActionSchema = z.enum([
|
|
8
|
+
"SEND_EMAIL",
|
|
9
|
+
"SEND_SLACK_MESSAGE",
|
|
10
|
+
"SEND_WEBHOOK",
|
|
11
|
+
"ADD_TO_DATASET",
|
|
12
|
+
"ADD_TO_ANNOTATION_QUEUE",
|
|
13
|
+
]);
|
|
14
|
+
export const alertTypeSchema = z.enum(["CRITICAL", "WARNING", "INFO"]);
|
|
15
|
+
|
|
16
|
+
/** The delivery configuration each channel reads, stated per channel so an agent
|
|
17
|
+
* can configure one without a round trip. Each passes unknown fields through:
|
|
18
|
+
* every Slack field is optional, so a webhook destination would otherwise read
|
|
19
|
+
* as an empty Slack one and be sent as nothing. */
|
|
20
|
+
const emailActionParamsSchema = z
|
|
21
|
+
.object({
|
|
22
|
+
members: z
|
|
23
|
+
.array(z.string())
|
|
24
|
+
.min(1)
|
|
25
|
+
.describe("Who receives the email. Any address, not only teammates."),
|
|
26
|
+
})
|
|
27
|
+
.passthrough();
|
|
28
|
+
|
|
29
|
+
const slackActionParamsSchema = z
|
|
30
|
+
.object({
|
|
31
|
+
slackIntegrationId: z
|
|
32
|
+
.string()
|
|
33
|
+
.optional()
|
|
34
|
+
.describe(
|
|
35
|
+
"The Slack connection to post through (an organization connection or one of this project's; they are listed in LangWatch under Settings, Integrations, Slack). A bot connection also needs `slackChannelId`; a webhook connection needs nothing else. Preferred over `slackWebhook` and `slackBotToken`.",
|
|
36
|
+
),
|
|
37
|
+
slackDelivery: z
|
|
38
|
+
.enum(["webhook", "bot"])
|
|
39
|
+
.optional()
|
|
40
|
+
.describe(
|
|
41
|
+
"How the message reaches Slack. `webhook` posts to an incoming webhook URL, `bot` posts as the LangWatch Slack app. With `slackIntegrationId` it follows the connection's kind. Absent without a connection means `webhook`.",
|
|
42
|
+
),
|
|
43
|
+
slackWebhook: z
|
|
44
|
+
.string()
|
|
45
|
+
.optional()
|
|
46
|
+
.describe(
|
|
47
|
+
"Legacy, accepted for one release: an incoming webhook URL (https://hooks.slack.com/...). It is stored as a Slack connection and the automation keeps only the connection id. Send `slackIntegrationId` instead.",
|
|
48
|
+
),
|
|
49
|
+
slackChannelId: z
|
|
50
|
+
.string()
|
|
51
|
+
.optional()
|
|
52
|
+
.describe("The channel the bot posts in, for a bot connection or `bot` delivery."),
|
|
53
|
+
slackBotToken: z
|
|
54
|
+
.string()
|
|
55
|
+
.optional()
|
|
56
|
+
.describe(
|
|
57
|
+
"Legacy, accepted for one release: a bot token, stored as a Slack connection. It never reads back. Send `slackIntegrationId` instead.",
|
|
58
|
+
),
|
|
59
|
+
slackBotTokenSet: z
|
|
60
|
+
.boolean()
|
|
61
|
+
.optional()
|
|
62
|
+
.describe(
|
|
63
|
+
"Legacy. Read: whether an automation not yet moved to a connection stores its own bot token. Write: true keeps it.",
|
|
64
|
+
),
|
|
65
|
+
})
|
|
66
|
+
.passthrough();
|
|
67
|
+
|
|
68
|
+
const webhookActionParamsSchema = z
|
|
69
|
+
.object({
|
|
70
|
+
url: z.string().describe("Where the request goes. https only, and not a private host."),
|
|
71
|
+
method: z.enum(["POST", "PUT", "PATCH"]).optional(),
|
|
72
|
+
headers: z
|
|
73
|
+
.record(z.string(), z.string())
|
|
74
|
+
.optional()
|
|
75
|
+
.describe(
|
|
76
|
+
"Static headers sent with every delivery. The values are credentials: they read back as [redacted], and sending [redacted] back keeps the stored ones. Changing `url` means sending the values again in the same call.",
|
|
77
|
+
),
|
|
78
|
+
bodyTemplate: z
|
|
79
|
+
.string()
|
|
80
|
+
.nullable()
|
|
81
|
+
.optional()
|
|
82
|
+
.describe(
|
|
83
|
+
"A Liquid template for the JSON body. Absent sends the standard LangWatch envelope.",
|
|
84
|
+
),
|
|
85
|
+
signingSecret: z
|
|
86
|
+
.string()
|
|
87
|
+
.nullable()
|
|
88
|
+
.optional()
|
|
89
|
+
.describe("Signs every delivery so the receiver can verify it."),
|
|
90
|
+
})
|
|
91
|
+
.passthrough();
|
|
92
|
+
|
|
93
|
+
const datasetActionParamsSchema = z
|
|
94
|
+
.object({
|
|
95
|
+
datasetId: z.string().describe("The dataset matched traces are appended to."),
|
|
96
|
+
datasetMapping: z
|
|
97
|
+
.object({
|
|
98
|
+
mapping: z.record(z.string(), z.unknown()),
|
|
99
|
+
expansions: z.array(z.string()).optional(),
|
|
100
|
+
})
|
|
101
|
+
.describe("How a trace becomes a row in that dataset."),
|
|
102
|
+
})
|
|
103
|
+
.passthrough();
|
|
104
|
+
|
|
105
|
+
const annotationQueueActionParamsSchema = z
|
|
106
|
+
.object({
|
|
107
|
+
annotators: z
|
|
108
|
+
.array(z.object({ id: z.string(), name: z.string() }))
|
|
109
|
+
.min(1)
|
|
110
|
+
.describe("Who the queued items go to."),
|
|
111
|
+
})
|
|
112
|
+
.passthrough();
|
|
113
|
+
|
|
114
|
+
export const actionParamsSchema = z.union([
|
|
115
|
+
emailActionParamsSchema,
|
|
116
|
+
slackActionParamsSchema,
|
|
117
|
+
webhookActionParamsSchema,
|
|
118
|
+
datasetActionParamsSchema,
|
|
119
|
+
annotationQueueActionParamsSchema,
|
|
120
|
+
]);
|
|
121
|
+
|
|
122
|
+
/** The write-side Slack shape: a redacted read round-trips, but a WRITE with no
|
|
123
|
+
* destination cannot deliver, so it is refused with the field named. A
|
|
124
|
+
* connection decides its own kind; without one, the legacy fields need a
|
|
125
|
+
* webhook URL, or a channel and a token. */
|
|
126
|
+
const slackActionParamsWriteSchema = slackActionParamsSchema.superRefine((params, ctx) => {
|
|
127
|
+
const delivery = params.slackDelivery ?? "webhook";
|
|
128
|
+
if (delivery === "bot" && !params.slackChannelId) {
|
|
129
|
+
ctx.addIssue({
|
|
130
|
+
code: "custom",
|
|
131
|
+
path: ["slackChannelId"],
|
|
132
|
+
message: params.slackIntegrationId
|
|
133
|
+
? "a bot connection needs the channel to post in"
|
|
134
|
+
: "bot delivery needs the channel to post in",
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (params.slackIntegrationId) return;
|
|
138
|
+
if (delivery === "webhook" && !params.slackWebhook) {
|
|
139
|
+
ctx.addIssue({
|
|
140
|
+
code: "custom",
|
|
141
|
+
path: ["slackIntegrationId"],
|
|
142
|
+
message: "name the Slack connection to post through",
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (delivery === "bot" && !params.slackBotToken && params.slackBotTokenSet !== true) {
|
|
146
|
+
ctx.addIssue({
|
|
147
|
+
code: "custom",
|
|
148
|
+
path: ["slackIntegrationId"],
|
|
149
|
+
message: "bot delivery needs the Slack connection to post through",
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const ACTION_PARAMS_SCHEMA_BY_ACTION: Partial<Record<string, z.ZodTypeAny>> = {
|
|
155
|
+
SEND_EMAIL: emailActionParamsSchema,
|
|
156
|
+
SEND_SLACK_MESSAGE: slackActionParamsWriteSchema,
|
|
157
|
+
SEND_WEBHOOK: webhookActionParamsSchema,
|
|
158
|
+
ADD_TO_DATASET: datasetActionParamsSchema,
|
|
159
|
+
ADD_TO_ANNOTATION_QUEUE: annotationQueueActionParamsSchema,
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/** A WRITE's `actionParams` is bound to the channel named in `action`, or a
|
|
163
|
+
* `SEND_WEBHOOK` with `{}` slips through the all-optional Slack shape and is
|
|
164
|
+
* refused server-side with worse words. An action this build does not know
|
|
165
|
+
* validates against the loose union, the tolerance reads keep. */
|
|
166
|
+
export function validateActionParamsForAction({
|
|
167
|
+
action,
|
|
168
|
+
actionParams,
|
|
169
|
+
}: {
|
|
170
|
+
action: string;
|
|
171
|
+
actionParams: unknown;
|
|
172
|
+
}): { ok: true } | { ok: false; message: string } {
|
|
173
|
+
const schema = ACTION_PARAMS_SCHEMA_BY_ACTION[action] ?? actionParamsSchema;
|
|
174
|
+
const parsed = schema.safeParse(actionParams);
|
|
175
|
+
if (parsed.success) return { ok: true };
|
|
176
|
+
const issues = parsed.error.issues
|
|
177
|
+
.map((issue) => `${issue.path.join(".") || "(actionParams)"}: ${issue.message}`)
|
|
178
|
+
.join("; ");
|
|
179
|
+
return {
|
|
180
|
+
ok: false,
|
|
181
|
+
message: `actionParams does not fit ${action} — ${issues}`,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** How `filters` is written. A keyed field sent flat never matches, so each
|
|
186
|
+
* keyed shape is spelled out; see the `/api/v1/triggers` 422 for the refusal. */
|
|
187
|
+
export const TRIGGER_FILTERS_DESCRIPTION = [
|
|
188
|
+
"Trace conditions as a JSON object string.",
|
|
189
|
+
'Unkeyed fields take a list of values: {"traces.error":["true"]}.',
|
|
190
|
+
'Keyed fields nest a key above the list. evaluations.* is keyed by the MONITOR id (the `id` from `platform_list_monitors` / GET /api/v1/monitors, not its evaluatorId): {"evaluations.passed":{"<monitorId>":["false"]}}.',
|
|
191
|
+
'metadata.value is keyed by the metadata key: {"metadata.value":{"<key>":["true"]}}.',
|
|
192
|
+
"A keyed field sent as a bare list, or keyed by anything but a monitor id, is refused.",
|
|
193
|
+
].join(" ");
|
|
194
|
+
|
|
195
|
+
export const TRIGGER_FILTER_QUERY_DESCRIPTION = [
|
|
196
|
+
"Trace query in the syntax the traces view uses; the alternative to `filters`, and it supersedes them.",
|
|
197
|
+
'Examples: status:error · evaluator:"<evaluator name>" AND evaluatorVerdict:fail · trace.attribute.<key>:<value>.',
|
|
198
|
+
].join(" ");
|
|
199
|
+
|
|
200
|
+
/** Stated on the create tool: Slack needs a connection or it never posts. */
|
|
201
|
+
export const SLACK_DELIVERY_NOTE =
|
|
202
|
+
'For SEND_SLACK_MESSAGE, send {"slackIntegrationId":"<connection id>"} for a webhook connection, or {"slackIntegrationId":"<connection id>","slackChannelId":"C..."} for a bot connection. Slack connections are listed and added in LangWatch under Settings, Integrations, Slack (organization-wide or for one project); they are also listed by GET /api/v1/slack-connections and `langwatch slack-connection list`, or ask the user for the id. slackIntegrationId is preferred: a legacy {"slackWebhook":"https://hooks.slack.com/..."} or {"slackDelivery":"bot","slackBotToken":"xoxb-...","slackChannelId":"C..."} is still accepted for one release and is stored as a connection. A Slack automation with no connection will never post.';
|
|
203
|
+
|
|
204
|
+
export const graphAlertSchema = z
|
|
205
|
+
.object({
|
|
206
|
+
seriesName: z.string().describe("The series on the graph to watch."),
|
|
207
|
+
operator: z.enum(["gt", "lt", "gte", "lte", "eq"]),
|
|
208
|
+
threshold: z.number(),
|
|
209
|
+
timePeriod: z
|
|
210
|
+
.union([
|
|
211
|
+
z.literal(1),
|
|
212
|
+
z.literal(5),
|
|
213
|
+
z.literal(15),
|
|
214
|
+
z.literal(30),
|
|
215
|
+
z.literal(60),
|
|
216
|
+
z.literal(1440),
|
|
217
|
+
])
|
|
218
|
+
.describe("The window, in minutes, the series is read over."),
|
|
219
|
+
})
|
|
220
|
+
.describe("The rule an alert fires by. Send it with `customGraphId` and `alertType`.");
|
|
221
|
+
|
|
222
|
+
/** What a scheduled report renders and when. Structured, not a loose record: an
|
|
223
|
+
* agent that cannot see the field names guesses them, and a guess is a 422.
|
|
224
|
+
* The server holds a write to this same shape, so looseness only delays it. */
|
|
225
|
+
export const reportSchema = z
|
|
226
|
+
.object({
|
|
227
|
+
source: z
|
|
228
|
+
.discriminatedUnion("kind", [
|
|
229
|
+
z.object({
|
|
230
|
+
kind: z.literal("dashboard"),
|
|
231
|
+
dashboardId: z.string(),
|
|
232
|
+
}),
|
|
233
|
+
z.object({
|
|
234
|
+
kind: z.literal("customGraph"),
|
|
235
|
+
customGraphId: z.string(),
|
|
236
|
+
}),
|
|
237
|
+
z.object({
|
|
238
|
+
kind: z.literal("traceQuery"),
|
|
239
|
+
filters: z.record(z.string(), z.unknown()).optional(),
|
|
240
|
+
metric: z.string().optional(),
|
|
241
|
+
topN: z.number().int().min(1).max(100).optional(),
|
|
242
|
+
}),
|
|
243
|
+
])
|
|
244
|
+
.describe("What the report renders: a dashboard, one graph, or a table of traces."),
|
|
245
|
+
schedule: z
|
|
246
|
+
.object({
|
|
247
|
+
cron: z
|
|
248
|
+
.string()
|
|
249
|
+
.describe(
|
|
250
|
+
'A 5-field cron expression (minute hour day-of-month month day-of-week), for example "0 9 * * 1". It can send at most every 15 minutes.',
|
|
251
|
+
),
|
|
252
|
+
timezone: z.string().describe('An IANA timezone, for example "Europe/Amsterdam" or "UTC".'),
|
|
253
|
+
})
|
|
254
|
+
.describe("When it sends."),
|
|
255
|
+
compareToPrevious: z
|
|
256
|
+
.boolean()
|
|
257
|
+
.optional()
|
|
258
|
+
.describe("Include a this-period-versus-last comparison."),
|
|
259
|
+
})
|
|
260
|
+
.describe(
|
|
261
|
+
"What a scheduled report renders and when. Send it with a channel that can carry a message: email or Slack.",
|
|
262
|
+
);
|
|
263
|
+
|
|
264
|
+
export const templatesSchema = z
|
|
265
|
+
.object({
|
|
266
|
+
slackTemplateType: z.enum(["string", "block_kit"]).nullable().optional(),
|
|
267
|
+
slackTemplate: z.string().nullable().optional(),
|
|
268
|
+
emailSubjectTemplate: z.string().nullable().optional(),
|
|
269
|
+
emailBodyTemplate: z.string().nullable().optional(),
|
|
270
|
+
})
|
|
271
|
+
.describe(
|
|
272
|
+
"The Liquid templates the message is rendered from. Absent fields render the LangWatch default.",
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
export const notificationCadenceSchema = z
|
|
276
|
+
.enum(["immediate", "5min_digest", "15min_digest", "hourly_digest"])
|
|
277
|
+
.describe(
|
|
278
|
+
"How often a notification automation may send. A new one starts on a five-minute digest.",
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* What a read answers with. Permissive on purpose: an MCP client talks to whichever LangWatch it
|
|
283
|
+
* is pointed at, so an unknown field is carried through and a field a deployment does not send
|
|
284
|
+
* yet is absent rather than fatal.
|
|
285
|
+
*/
|
|
286
|
+
export const triggerSchema = z
|
|
287
|
+
.object({
|
|
288
|
+
id: z.string(),
|
|
289
|
+
name: z.string(),
|
|
290
|
+
action: z.string(),
|
|
291
|
+
/** Where it delivers. The rule it fires by is stated separately, below,
|
|
292
|
+
* the same way a write states it. */
|
|
293
|
+
actionParams: z.record(z.string(), z.unknown()).default({}),
|
|
294
|
+
graphAlert: graphAlertSchema.nullable().optional(),
|
|
295
|
+
// A read stays tolerant of a deployment that words this differently; a
|
|
296
|
+
// write is held to `reportSchema`.
|
|
297
|
+
report: z.record(z.string(), z.unknown()).nullable().optional(),
|
|
298
|
+
filters: z.record(z.string(), z.unknown()).default({}),
|
|
299
|
+
filterQuery: z.string().nullable().optional(),
|
|
300
|
+
kind: z.string().optional(),
|
|
301
|
+
customGraphId: z.string().nullable().optional(),
|
|
302
|
+
notificationCadence: z.string().nullable().optional(),
|
|
303
|
+
traceDebounceMs: z.number().nullable().optional(),
|
|
304
|
+
templates: z.record(z.string(), z.unknown()).optional(),
|
|
305
|
+
active: z.boolean(),
|
|
306
|
+
message: z.string().nullable(),
|
|
307
|
+
alertType: z.string().nullable(),
|
|
308
|
+
createdAt: z.string(),
|
|
309
|
+
updatedAt: z.string(),
|
|
310
|
+
platformUrl: z.string().optional(),
|
|
311
|
+
})
|
|
312
|
+
.passthrough();
|
|
313
|
+
|
|
314
|
+
const triggerFireSchema = z
|
|
315
|
+
.object({
|
|
316
|
+
id: z.string(),
|
|
317
|
+
triggerId: z.string(),
|
|
318
|
+
customGraphId: z.string().nullable(),
|
|
319
|
+
firedAt: z.string(),
|
|
320
|
+
resolvedAt: z.string().nullable(),
|
|
321
|
+
})
|
|
322
|
+
.passthrough();
|
|
323
|
+
|
|
324
|
+
/** A page of fires: `{ fires, nextCursor }`, or the bare array an older
|
|
325
|
+
* deployment answers with, which has no next page. */
|
|
326
|
+
export const triggerFirePageSchema = z.union([
|
|
327
|
+
z.array(triggerFireSchema).transform((fires) => ({ fires, nextCursor: null })),
|
|
328
|
+
z
|
|
329
|
+
.object({
|
|
330
|
+
fires: z.array(triggerFireSchema).optional(),
|
|
331
|
+
data: z.array(triggerFireSchema).optional(),
|
|
332
|
+
nextCursor: z.string().nullable().optional(),
|
|
333
|
+
})
|
|
334
|
+
.transform(({ fires, data, nextCursor }) => ({
|
|
335
|
+
fires: fires ?? data ?? [],
|
|
336
|
+
nextCursor: nextCursor ?? null,
|
|
337
|
+
})),
|
|
338
|
+
]);
|
|
339
|
+
|
|
340
|
+
export const testFireResultSchema = z
|
|
341
|
+
.object({
|
|
342
|
+
channel: z.string(),
|
|
343
|
+
recipientCount: z.number(),
|
|
344
|
+
usedDefault: z.boolean(),
|
|
345
|
+
missingVariables: z.array(z.string()).default([]),
|
|
346
|
+
errors: z.array(z.string()).default([]),
|
|
347
|
+
httpStatus: z.number().optional(),
|
|
348
|
+
})
|
|
349
|
+
.passthrough();
|
|
350
|
+
|
|
351
|
+
export const deletedTriggerSchema = z.object({
|
|
352
|
+
id: z.string(),
|
|
353
|
+
deleted: z.boolean(),
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
export type Trigger = z.infer<typeof triggerSchema>;
|
|
357
|
+
export type TriggerFirePage = z.infer<typeof triggerFirePageSchema>;
|
|
358
|
+
export type TestFireResult = z.infer<typeof testFireResultSchema>;
|
|
359
|
+
export type TriggerAction = z.infer<typeof triggerActionSchema>;
|
|
360
|
+
export type TriggerAlertType = z.infer<typeof alertTypeSchema>;
|
|
361
|
+
export type TriggerActionParams = z.infer<typeof actionParamsSchema>;
|
|
362
|
+
export type GraphAlertRule = z.infer<typeof graphAlertSchema>;
|
|
363
|
+
export type ReportRule = z.infer<typeof reportSchema>;
|
|
364
|
+
export type TriggerTemplates = z.infer<typeof templatesSchema>;
|
|
365
|
+
export type NotificationCadence = z.infer<typeof notificationCadenceSchema>;
|