@langwatch/mcp-server 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +192 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-65QN3XRX.js} +4 -4
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-NKZDZY3D.js} +4 -4
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-OJLYYJWI.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-G4ZAZTZW.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-EYXMLE5X.js → chunk-HGELNIVY.js} +56 -47
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-RVAE2MEA.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-S4GVX36W.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-QRO35SOM.js → chunk-J7PQKBLD.js} +652 -528
- package/dist/{chunk-SR6QPRIE.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-UFOYFYI7.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-56OGTS24.js → chunk-PTDR56Z3.js} +4 -11
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-QAMP3CGR.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/{chunk-6DJDEXET.js → chunk-VMWJD732.js} +6 -12
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-Z76AIPLS.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -10
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-ZQ7TFIO2.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-UEM77OE7.js} +5 -5
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-4IPXVSFQ.js} +6 -6
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-LYMRPF27.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-OTIYUFZA.js} +15 -11
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-P4WWVHFU.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-QSQEAB33.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-OWXJZHQQ.js} +53 -58
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-BE3M2AUX.js} +7 -9
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-YM4ODWW6.js} +5 -5
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-LCFSMPGQ.js} +26 -22
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GC2O7YH6.js} +5 -5
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-IR4QOS43.js} +4071 -347
- package/dist/index.js +58671 -37983
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-2KJL2M42.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/{list-agents-AMVDQPQL.js → list-agents-HVYWEQMI.js} +4 -4
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-Z3BZKDGF.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-PX2XI6EK.js} +5 -5
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-QZRPVHS4.js} +13 -11
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-OEADLLQR.js} +12 -8
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-N4WVLBKK.js} +4 -4
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-4J7UX5S2.js} +4 -4
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-DCZFLGDO.js} +81 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-TG2V4WM4.js} +5 -5
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-PAARQSI4.js} +7 -7
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-SQJELEZX.js} +6 -6
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/run-workflow-SP32SO5M.js +36 -0
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-DUWNYBNA.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-Q26FBSYH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-MY263DYF.js} +27 -29
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-MOHRQKAQ.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +755 -609
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +108 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { listTriggerFires as apiListTriggerFires } from "../langwatch-api-triggers.ts";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Handles the platform_list_trigger_fires MCP tool invocation: one page of
|
|
5
|
+
* fires, newest first, and the cursor for the next page when there is one.
|
|
6
|
+
*/
|
|
7
|
+
export async function handleListTriggerFires(params: {
|
|
8
|
+
id: string;
|
|
9
|
+
limit?: number;
|
|
10
|
+
cursor?: string;
|
|
11
|
+
}): Promise<string> {
|
|
12
|
+
const page = await apiListTriggerFires(params);
|
|
13
|
+
if (page.fires.length === 0) {
|
|
14
|
+
return params.cursor ? "No more fires." : "This automation has not fired yet.";
|
|
15
|
+
}
|
|
16
|
+
return JSON.stringify(page, null, 2);
|
|
17
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { listTriggers as apiListTriggers } from "../langwatch-api-triggers.ts";
|
|
2
|
+
import type { Trigger } from "../schemas/triggers.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Handles the platform_list_triggers MCP tool invocation.
|
|
6
|
+
*/
|
|
7
|
+
export async function handleListTriggers(params: { format?: "digest" | "json" }): Promise<string> {
|
|
8
|
+
const triggers = await apiListTriggers();
|
|
9
|
+
if (params.format === "json") return JSON.stringify(triggers, null, 2);
|
|
10
|
+
if (triggers.length === 0) {
|
|
11
|
+
return "No triggers found. Use `platform_create_trigger` to create one.";
|
|
12
|
+
}
|
|
13
|
+
const lines = [`# Triggers (${triggers.length} total)\n`];
|
|
14
|
+
for (const trigger of triggers) lines.push(...digestTrigger(trigger), "");
|
|
15
|
+
return lines.join("\n");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function digestTrigger(trigger: Trigger): string[] {
|
|
19
|
+
const lines = [
|
|
20
|
+
`## ${trigger.name}`,
|
|
21
|
+
`**ID**: ${trigger.id}`,
|
|
22
|
+
`**Kind**: ${trigger.kind ?? "AUTOMATION"}`,
|
|
23
|
+
`**Action**: ${trigger.action}`,
|
|
24
|
+
`**Status**: ${trigger.active ? "active" : "inactive"}`,
|
|
25
|
+
];
|
|
26
|
+
if (trigger.alertType) lines.push(`**Alert**: ${trigger.alertType}`);
|
|
27
|
+
if (trigger.filterQuery) lines.push(`**Filter query**: ${trigger.filterQuery}`);
|
|
28
|
+
else if (Object.keys(trigger.filters).length > 0) {
|
|
29
|
+
lines.push(`**Filters**: ${JSON.stringify(trigger.filters)}`);
|
|
30
|
+
}
|
|
31
|
+
if (trigger.graphAlert) {
|
|
32
|
+
const rule = trigger.graphAlert;
|
|
33
|
+
lines.push(
|
|
34
|
+
`**Fires when**: ${rule.seriesName} ${rule.operator} ${rule.threshold} over ${rule.timePeriod}m${trigger.customGraphId ? ` on graph ${trigger.customGraphId}` : ""}`,
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
if (trigger.report) lines.push(`**Report**: ${JSON.stringify(trigger.report)}`);
|
|
38
|
+
return lines;
|
|
39
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listWorkflows as apiListWorkflows } from "../langwatch-api-workflows.
|
|
1
|
+
import { listWorkflows as apiListWorkflows } from "../langwatch-api-workflows.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListWorkflows(): Promise<string> {
|
|
4
4
|
const workflows = await apiListWorkflows();
|
|
@@ -23,9 +23,7 @@ export async function handleListWorkflows(): Promise<string> {
|
|
|
23
23
|
lines.push("");
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
lines.push(
|
|
27
|
-
"> Use `platform_get_workflow` with the ID to see full workflow details.",
|
|
28
|
-
);
|
|
26
|
+
lines.push("> Use `platform_get_workflow` with the ID to see full workflow details.");
|
|
29
27
|
|
|
30
28
|
return lines.join("\n");
|
|
31
29
|
}
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
import { renameTestSuite as apiRenameTestSuite } from "../langwatch-api-test-suites.
|
|
1
|
+
import { renameTestSuite as apiRenameTestSuite } from "../langwatch-api-test-suites.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Handles the platform_rename_test_suite MCP tool invocation.
|
|
5
5
|
*/
|
|
6
|
-
export async function handleRenameTestSuite(params: {
|
|
7
|
-
id: string;
|
|
8
|
-
name: string;
|
|
9
|
-
}): Promise<string> {
|
|
6
|
+
export async function handleRenameTestSuite(params: { id: string; name: string }): Promise<string> {
|
|
10
7
|
const suite = await apiRenameTestSuite(params);
|
|
11
8
|
|
|
12
9
|
return [
|
|
@@ -5,7 +5,8 @@ import {
|
|
|
5
5
|
redactSessionJsonl,
|
|
6
6
|
truncateJsonlToByteBudget,
|
|
7
7
|
} from "@langwatch/redaction";
|
|
8
|
-
|
|
8
|
+
|
|
9
|
+
import { getConfig } from "../config.ts";
|
|
9
10
|
|
|
10
11
|
/** Transcripts are capped after redaction; oldest lines are dropped first. */
|
|
11
12
|
const MAX_SESSION_BYTES = 8 * 1024 * 1024;
|
|
@@ -68,14 +69,11 @@ function redactReport(params: ReportIssueParams): RedactedReport {
|
|
|
68
69
|
};
|
|
69
70
|
}
|
|
70
71
|
|
|
71
|
-
async function deliverReport(
|
|
72
|
-
report: RedactedReport,
|
|
73
|
-
params: ReportIssueParams,
|
|
74
|
-
): Promise<string> {
|
|
72
|
+
async function deliverReport(report: RedactedReport, params: ReportIssueParams): Promise<string> {
|
|
75
73
|
const config = getConfig();
|
|
76
74
|
const endpoint = config.endpoint.replace(/\/+$/, "");
|
|
77
75
|
|
|
78
|
-
const response = await fetch(`${endpoint}/api/bug-reports`, {
|
|
76
|
+
const response = await fetch(`${endpoint}/api/v1/bug-reports`, {
|
|
79
77
|
method: "POST",
|
|
80
78
|
headers: {
|
|
81
79
|
"content-type": "application/json",
|
|
@@ -117,9 +115,7 @@ async function deliverReport(
|
|
|
117
115
|
* an API key: the reporter may be struggling precisely because setup failed.
|
|
118
116
|
* When a key is configured it only links the report to the project.
|
|
119
117
|
*/
|
|
120
|
-
export async function handleReportIssue(
|
|
121
|
-
params: ReportIssueParams,
|
|
122
|
-
): Promise<string> {
|
|
118
|
+
export async function handleReportIssue(params: ReportIssueParams): Promise<string> {
|
|
123
119
|
if (!params.user_approved) {
|
|
124
120
|
throw new Error(
|
|
125
121
|
"This sends a report to the LangWatch team, so the user must approve it first. " +
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
type RunParameters,
|
|
4
|
-
} from "../langwatch-api-run-plans.js";
|
|
5
|
-
import { formatRunPlanRun } from "./format-run-plan.js";
|
|
1
|
+
import { rerunRunPlan as apiRerunRunPlan, type RunParameters } from "../langwatch-api-run-plans.ts";
|
|
2
|
+
import { formatRunPlanRun } from "./format-run-plan.ts";
|
|
6
3
|
|
|
7
4
|
/**
|
|
8
5
|
* Handles the platform_rerun_run_plan MCP tool invocation.
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { revokeApiKey as apiRevokeApiKey } from "../langwatch-api-api-keys.
|
|
1
|
+
import { revokeApiKey as apiRevokeApiKey } from "../langwatch-api-api-keys.ts";
|
|
2
2
|
|
|
3
|
-
export async function handleRevokeApiKey(params: {
|
|
4
|
-
id: string;
|
|
5
|
-
}): Promise<string> {
|
|
3
|
+
export async function handleRevokeApiKey(params: { id: string }): Promise<string> {
|
|
6
4
|
await apiRevokeApiKey(params.id);
|
|
7
5
|
|
|
8
6
|
return `API key ${params.id} revoked successfully.`;
|
package/src/tools/run-agent.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
type AgentCallParams,
|
|
5
|
-
} from "../langwatch-api-agents.js";
|
|
2
|
+
|
|
3
|
+
import { runAgent as apiRunAgent, type AgentCallParams } from "../langwatch-api-agents.ts";
|
|
6
4
|
|
|
7
5
|
/**
|
|
8
6
|
* A scalar and an array both parse as JSON, and either one reaches the agent
|
|
@@ -11,6 +9,31 @@ import {
|
|
|
11
9
|
const jsonObjectSchema = z.looseObject({});
|
|
12
10
|
type JsonObject = z.infer<typeof jsonObjectSchema>;
|
|
13
11
|
|
|
12
|
+
function outputLines(result: object): string[] {
|
|
13
|
+
const output = (result as Record<string, unknown>).output;
|
|
14
|
+
if (output !== undefined) {
|
|
15
|
+
return ["**Output:**", typeof output === "string" ? output : JSON.stringify(output, null, 2)];
|
|
16
|
+
}
|
|
17
|
+
return ["**Result:**", JSON.stringify(result, null, 2)];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function connectedLines(result: object): string[] {
|
|
21
|
+
const lines: string[] = [];
|
|
22
|
+
const instance = (result as { instance?: { hostname?: string; label?: string | null } }).instance;
|
|
23
|
+
const durationMs = (result as { durationMs?: number }).durationMs;
|
|
24
|
+
if (instance?.hostname) {
|
|
25
|
+
const label = instance.label ? ` (${instance.label})` : "";
|
|
26
|
+
lines.push(`\n**Instance:** ${instance.hostname}${label}`);
|
|
27
|
+
}
|
|
28
|
+
if (typeof durationMs === "number") lines.push(`**Duration:** ${durationMs} ms`);
|
|
29
|
+
const session = (result as { session?: unknown }).session;
|
|
30
|
+
if (session !== undefined && session !== null) {
|
|
31
|
+
lines.push("**Session:**");
|
|
32
|
+
lines.push(JSON.stringify(session, null, 2));
|
|
33
|
+
}
|
|
34
|
+
return lines;
|
|
35
|
+
}
|
|
36
|
+
|
|
14
37
|
/**
|
|
15
38
|
* Handles the platform_run_agent MCP tool invocation.
|
|
16
39
|
*
|
|
@@ -56,28 +79,8 @@ export async function handleRunAgent({
|
|
|
56
79
|
lines.push(`Agent executed successfully (type: ${agentType}).\n`);
|
|
57
80
|
|
|
58
81
|
if (typeof result === "object" && result !== null) {
|
|
59
|
-
|
|
60
|
-
if (
|
|
61
|
-
lines.push("**Output:**");
|
|
62
|
-
lines.push(typeof output === "string" ? output : JSON.stringify(output, null, 2));
|
|
63
|
-
} else {
|
|
64
|
-
lines.push("**Result:**");
|
|
65
|
-
lines.push(JSON.stringify(result, null, 2));
|
|
66
|
-
}
|
|
67
|
-
if (agentType === "connected") {
|
|
68
|
-
const instance = (result as { instance?: { hostname?: string; label?: string | null } }).instance;
|
|
69
|
-
const durationMs = (result as { durationMs?: number }).durationMs;
|
|
70
|
-
if (instance?.hostname) {
|
|
71
|
-
const label = instance.label ? ` (${instance.label})` : "";
|
|
72
|
-
lines.push(`\n**Instance:** ${instance.hostname}${label}`);
|
|
73
|
-
}
|
|
74
|
-
if (typeof durationMs === "number") lines.push(`**Duration:** ${durationMs} ms`);
|
|
75
|
-
const session = (result as { session?: unknown }).session;
|
|
76
|
-
if (session !== undefined && session !== null) {
|
|
77
|
-
lines.push("**Session:**");
|
|
78
|
-
lines.push(JSON.stringify(session, null, 2));
|
|
79
|
-
}
|
|
80
|
-
}
|
|
82
|
+
lines.push(...outputLines(result));
|
|
83
|
+
if (agentType === "connected") lines.push(...connectedLines(result));
|
|
81
84
|
}
|
|
82
85
|
|
|
83
86
|
return lines.join("\n");
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { LangWatchApiError, makeRequest } from "../langwatch-api.
|
|
2
|
-
import { deriveRunStatus } from "./experiment-run-status.
|
|
1
|
+
import { LangWatchApiError, makeRequest } from "../langwatch-api.ts";
|
|
2
|
+
import { deriveRunStatus } from "./experiment-run-status.ts";
|
|
3
3
|
|
|
4
4
|
interface EvaluationRunResponse {
|
|
5
5
|
runId: string;
|
|
@@ -24,12 +24,10 @@ interface EvaluationStatusResponse {
|
|
|
24
24
|
};
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
export async function handleRunExperiment(params: {
|
|
28
|
-
slug: string;
|
|
29
|
-
}): Promise<string> {
|
|
27
|
+
export async function handleRunExperiment(params: { slug: string }): Promise<string> {
|
|
30
28
|
const result = (await makeRequest(
|
|
31
29
|
"POST",
|
|
32
|
-
`/api/experiments/${encodeURIComponent(params.slug)}/run`,
|
|
30
|
+
`/api/v1/experiments/${encodeURIComponent(params.slug)}/run`,
|
|
33
31
|
)) as EvaluationRunResponse;
|
|
34
32
|
|
|
35
33
|
const lines: string[] = [];
|
|
@@ -75,7 +73,7 @@ async function statusFromResults(params: {
|
|
|
75
73
|
try {
|
|
76
74
|
results = (await makeRequest(
|
|
77
75
|
"GET",
|
|
78
|
-
`/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`,
|
|
76
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`,
|
|
79
77
|
)) as ResultsForStatus;
|
|
80
78
|
} catch (error) {
|
|
81
79
|
// Only a genuine "no such run" is a fallback miss (-> guidance). Real
|
|
@@ -97,19 +95,13 @@ async function statusFromResults(params: {
|
|
|
97
95
|
lines.push(`**Status**: ${status}`);
|
|
98
96
|
lines.push(`**Progress**: ${progress}/${total} cells`);
|
|
99
97
|
if (results.timestamps.createdAt) {
|
|
100
|
-
lines.push(
|
|
101
|
-
`**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`,
|
|
102
|
-
);
|
|
98
|
+
lines.push(`**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`);
|
|
103
99
|
}
|
|
104
100
|
if (results.timestamps.finishedAt) {
|
|
105
|
-
lines.push(
|
|
106
|
-
`**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`,
|
|
107
|
-
);
|
|
101
|
+
lines.push(`**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`);
|
|
108
102
|
}
|
|
109
103
|
if (results.timestamps.stoppedAt) {
|
|
110
|
-
lines.push(
|
|
111
|
-
`**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`,
|
|
112
|
-
);
|
|
104
|
+
lines.push(`**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`);
|
|
113
105
|
}
|
|
114
106
|
lines.push("");
|
|
115
107
|
lines.push(
|
|
@@ -118,6 +110,31 @@ async function statusFromResults(params: {
|
|
|
118
110
|
return lines.join("\n");
|
|
119
111
|
}
|
|
120
112
|
|
|
113
|
+
async function statusAfterFailedRead({
|
|
114
|
+
params,
|
|
115
|
+
error,
|
|
116
|
+
}: {
|
|
117
|
+
params: { runId: string; experimentSlug?: string };
|
|
118
|
+
error: unknown;
|
|
119
|
+
}): Promise<string> {
|
|
120
|
+
const code = error instanceof LangWatchApiError ? error.status : undefined;
|
|
121
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
122
|
+
if (code === 404 || /404|not found/i.test(message)) {
|
|
123
|
+
const fallback = await statusFromResults(params);
|
|
124
|
+
if (fallback) return fallback;
|
|
125
|
+
return [
|
|
126
|
+
`# Evaluation Run ${params.runId}`,
|
|
127
|
+
"",
|
|
128
|
+
"**Status**: not found",
|
|
129
|
+
"",
|
|
130
|
+
`Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
|
|
131
|
+
"",
|
|
132
|
+
"> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`.",
|
|
133
|
+
].join("\n");
|
|
134
|
+
}
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
|
|
121
138
|
export async function handleExperimentStatus(params: {
|
|
122
139
|
runId: string;
|
|
123
140
|
experimentSlug?: string;
|
|
@@ -126,26 +143,10 @@ export async function handleExperimentStatus(params: {
|
|
|
126
143
|
try {
|
|
127
144
|
status = (await makeRequest(
|
|
128
145
|
"GET",
|
|
129
|
-
`/api/experiments/runs/${encodeURIComponent(params.runId)}`,
|
|
146
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}`,
|
|
130
147
|
)) as EvaluationStatusResponse;
|
|
131
148
|
} catch (error) {
|
|
132
|
-
|
|
133
|
-
error instanceof LangWatchApiError ? error.status : undefined;
|
|
134
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
135
|
-
if (code === 404 || /404|not found/i.test(message)) {
|
|
136
|
-
const fallback = await statusFromResults(params);
|
|
137
|
-
if (fallback) return fallback;
|
|
138
|
-
return [
|
|
139
|
-
`# Evaluation Run ${params.runId}`,
|
|
140
|
-
"",
|
|
141
|
-
"**Status**: not found",
|
|
142
|
-
"",
|
|
143
|
-
`Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
|
|
144
|
-
"",
|
|
145
|
-
"> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`.",
|
|
146
|
-
].join("\n");
|
|
147
|
-
}
|
|
148
|
-
throw error;
|
|
149
|
+
return statusAfterFailedRead({ params, error });
|
|
149
150
|
}
|
|
150
151
|
|
|
151
152
|
const lines: string[] = [];
|
|
@@ -169,9 +170,7 @@ export async function handleExperimentStatus(params: {
|
|
|
169
170
|
lines.push(`**Failed**: ${status.summary.failedCells}`);
|
|
170
171
|
}
|
|
171
172
|
if (status.summary.duration) {
|
|
172
|
-
lines.push(
|
|
173
|
-
`**Duration**: ${(status.summary.duration / 1000).toFixed(1)}s`,
|
|
174
|
-
);
|
|
173
|
+
lines.push(`**Duration**: ${(status.summary.duration / 1000).toFixed(1)}s`);
|
|
175
174
|
}
|
|
176
175
|
if (status.summary.runUrl) {
|
|
177
176
|
lines.push(`**View results**: ${status.summary.runUrl}`);
|
package/src/tools/run-plan.ts
CHANGED
|
@@ -3,13 +3,10 @@ import {
|
|
|
3
3
|
type RunParameters,
|
|
4
4
|
type RunPlanScope,
|
|
5
5
|
type RunPlanTarget,
|
|
6
|
-
} from "../langwatch-api-run-plans.
|
|
7
|
-
import { toWireTargets } from "../schemas/run-plan.
|
|
8
|
-
import {
|
|
9
|
-
|
|
10
|
-
toWireAttachments,
|
|
11
|
-
} from "../schemas/suite-fields.js";
|
|
12
|
-
import { formatRunPlanRun } from "./format-run-plan.js";
|
|
6
|
+
} from "../langwatch-api-run-plans.ts";
|
|
7
|
+
import { toWireTargets } from "../schemas/run-plan.ts";
|
|
8
|
+
import { type EvaluatorAttachmentInput, toWireAttachments } from "../schemas/suite-fields.ts";
|
|
9
|
+
import { formatRunPlanRun } from "./format-run-plan.ts";
|
|
13
10
|
|
|
14
11
|
/**
|
|
15
12
|
* Handles the platform_run_plan MCP tool invocation.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Handles the `run_query` MCP tool: one LangWatchQL statement rendered as a markdown table. Rows
|
|
3
|
+
* are capped, and the cap is announced, so an unaggregated query cannot flood the conversation.
|
|
4
|
+
* @see specs/mcp-server/schema-discovery.feature
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { runQuery } from "../langwatch-api-query.ts";
|
|
8
|
+
import { markdownTable } from "../utils/markdown-table.ts";
|
|
9
|
+
|
|
10
|
+
/** Rows the table prints. Beyond this, aggregate or export through the CLI. */
|
|
11
|
+
export const RUN_QUERY_ROW_CAP = 50;
|
|
12
|
+
|
|
13
|
+
export async function handleRunQuery(params: {
|
|
14
|
+
sql: string;
|
|
15
|
+
parameters?: Record<string, string | number | boolean | null>;
|
|
16
|
+
}): Promise<string> {
|
|
17
|
+
const result = await runQuery(params);
|
|
18
|
+
const headers = result.columns.map((column) => column.name);
|
|
19
|
+
const shown = result.rows.slice(0, RUN_QUERY_ROW_CAP);
|
|
20
|
+
|
|
21
|
+
const lines: string[] = [];
|
|
22
|
+
lines.push(
|
|
23
|
+
markdownTable({
|
|
24
|
+
headers,
|
|
25
|
+
rows: shown,
|
|
26
|
+
emptyMessage: "The statement ran and returned no rows.",
|
|
27
|
+
}),
|
|
28
|
+
);
|
|
29
|
+
lines.push("");
|
|
30
|
+
lines.push(
|
|
31
|
+
`Returned ${result.rows.length} row${result.rows.length === 1 ? "" : "s"} in ${result.statistics.elapsedMs}ms.`,
|
|
32
|
+
);
|
|
33
|
+
if (result.rows.length > shown.length) {
|
|
34
|
+
lines.push(
|
|
35
|
+
`Showing the first ${shown.length}. Aggregate further, or export every row with \`langwatch query "<statement>" --format jsonl\`.`,
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
if (result.truncated) {
|
|
39
|
+
lines.push(
|
|
40
|
+
"The result hit a response ceiling and was cut short, so this is not the whole answer.",
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
for (const diagnostic of result.diagnostics) {
|
|
44
|
+
lines.push(`${diagnostic.code}: ${diagnostic.message}`);
|
|
45
|
+
}
|
|
46
|
+
return lines.join("\n");
|
|
47
|
+
}
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
} from "
|
|
5
|
-
import { toWireTargets } from "../schemas/run-plan.js";
|
|
6
|
-
import { runTestSuite as apiRunTestSuite } from "../langwatch-api-test-suites.js";
|
|
7
|
-
import { formatRunPlanRun } from "./format-run-plan.js";
|
|
1
|
+
import type { RunParameters, RunPlanTarget } from "../langwatch-api-run-plans.ts";
|
|
2
|
+
import { runTestSuite as apiRunTestSuite } from "../langwatch-api-test-suites.ts";
|
|
3
|
+
import { toWireTargets } from "../schemas/run-plan.ts";
|
|
4
|
+
import { formatRunPlanRun } from "./format-run-plan.ts";
|
|
8
5
|
|
|
9
6
|
/**
|
|
10
7
|
* Handles the platform_run_test_suite MCP tool invocation.
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
import { runWorkflow as apiRunWorkflow } from "../langwatch-api-workflows.
|
|
1
|
+
import { runWorkflow as apiRunWorkflow } from "../langwatch-api-workflows.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Handles the platform_run_workflow MCP tool invocation.
|
|
5
5
|
*/
|
|
6
|
-
export async function handleRunWorkflow(params: {
|
|
7
|
-
id: string;
|
|
8
|
-
input?: string;
|
|
9
|
-
}): Promise<string> {
|
|
6
|
+
export async function handleRunWorkflow(params: { id: string; input?: string }): Promise<string> {
|
|
10
7
|
let parsedInput: Record<string, unknown> = {};
|
|
11
8
|
if (params.input) {
|
|
12
9
|
try {
|