@langwatch/mcp-server 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +192 -201
- package/CONTRIBUTING.md +10 -0
- package/README.md +70 -68
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-W3IKI7UL.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/{archive-run-plan-474OKHHU.js → archive-run-plan-65QN3XRX.js} +4 -4
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-V5UNYK26.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/{archive-test-suite-O7FLDMXK.js → archive-test-suite-NKZDZY3D.js} +4 -4
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-Q46WRB6P.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-OJLYYJWI.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/{chunk-UF6XIGYR.js → chunk-6YZUOD34.js} +2 -5
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-G4ZAZTZW.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-EYXMLE5X.js → chunk-HGELNIVY.js} +56 -47
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-RVAE2MEA.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-S4GVX36W.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-QRO35SOM.js → chunk-J7PQKBLD.js} +652 -528
- package/dist/{chunk-SR6QPRIE.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-UFOYFYI7.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/{chunk-HV4DWKBI.js → chunk-O7EMWEVJ.js} +2 -4
- package/dist/{chunk-56OGTS24.js → chunk-PTDR56Z3.js} +4 -11
- package/dist/{chunk-JR6DJZOU.js → chunk-RABYBFX5.js} +4 -10
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-JTQJRNFJ.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/{chunk-CKNAH5TG.js → chunk-U24EACC4.js} +3 -10
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-QAMP3CGR.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/{chunk-6DJDEXET.js → chunk-VMWJD732.js} +6 -12
- package/dist/{chunk-7E4FZIYJ.js → chunk-XUIJYNRJ.js} +1 -1
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-JFJ2K6X3.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-EUXXQ6NS.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-BGBB76JX.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-Z76AIPLS.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-FD5VAXU4.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-WQUXXOHI.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-6NV33Z52.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -10
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-ZQ7TFIO2.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-BS6QRCUU.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-FCQ5ELSQ.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-DRPTPXU3.js → create-scenario-UEM77OE7.js} +5 -5
- package/dist/{create-test-suite-ZIZCCFJ4.js → create-test-suite-4IPXVSFQ.js} +6 -6
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-SA4JMUZF.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-X3KKK7QY.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-LYMRPF27.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-W6EZ44HY.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-7CNOTALB.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-QB3OCZUV.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-NG2VXYYM.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/{discover-scenario-schema-W7KBXA4J.js → discover-scenario-schema-OOFIQKOG.js} +37 -17
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/{get-agent-7THF7HCU.js → get-agent-OTIYUFZA.js} +15 -11
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-P4WWVHFU.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-RZ44UIN6.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-QSQEAB33.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/{get-prompt-GVKXF5NL.js → get-prompt-OWXJZHQQ.js} +53 -58
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/{get-run-plan-NY7J5GRE.js → get-run-plan-BE3M2AUX.js} +7 -9
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-Q3LLQE4T.js → get-scenario-YM4ODWW6.js} +5 -5
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-WINEYDMK.js → get-simulation-run-LCFSMPGQ.js} +26 -22
- package/dist/{get-test-suite-4HKA7OKS.js → get-test-suite-GC2O7YH6.js} +5 -5
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-RCUV7N4O.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-EEOC7Y45.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-WCDUARTH.js → http-server-IR4QOS43.js} +4071 -347
- package/dist/index.js +58671 -37983
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-6DHVZWUK.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-2KJL2M42.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-6ISFJ7WX.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-6UJL5U4Z.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/{list-agents-AMVDQPQL.js → list-agents-HVYWEQMI.js} +4 -4
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-6PR5NLIC.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-GVPNBIHQ.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-Z3BZKDGF.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-O67HZ35Y.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-WXZL6UMP.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-PAX72WCN.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-VVLZMLCX.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-AQVI6SUK.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-WZFBJOHG.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-BQOHRHYT.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/{list-run-plans-UBAU2DGH.js → list-run-plans-PX2XI6EK.js} +5 -5
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/{list-scenarios-MS4IUIVM.js → list-scenarios-QZRPVHS4.js} +13 -11
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-D6YF4ZOU.js → list-simulation-runs-OEADLLQR.js} +12 -8
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/{list-test-suites-L7R2O6DR.js → list-test-suites-N4WVLBKK.js} +4 -4
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-6MV4XJHF.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-K5LQ4CY6.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/{rename-test-suite-NTYHAO5X.js → rename-test-suite-4J7UX5S2.js} +4 -4
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/{report-issue-RHZZ33IS.js → report-issue-DCZFLGDO.js} +81 -67
- package/dist/{rerun-run-plan-MUDWJBE4.js → rerun-run-plan-TG2V4WM4.js} +5 -5
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-6LAZ3QZZ.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/{run-plan-T4ZYOZGA.js → run-plan-PAARQSI4.js} +7 -7
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/{run-test-suite-L4MZ7G4D.js → run-test-suite-SQJELEZX.js} +6 -6
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-DFIS23AP.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/run-workflow-SP32SO5M.js +36 -0
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-XRNMFOVR.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-DUWNYBNA.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-YSOMDCOB.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-DEA2ITYR.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-NH7Q5GWV.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-Q26FBSYH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/{update-prompt-ZQQ2C3KU.js → update-prompt-MY263DYF.js} +27 -29
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-WUX34JAR.js → update-scenario-MOHRQKAQ.js} +5 -5
- package/package.json +35 -39
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +755 -609
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +18 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +12 -29
- package/src/langwatch-api-scenarios.ts +9 -11
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +6 -8
- package/src/langwatch-api-test-suites.ts +14 -33
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +108 -124
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +13 -29
- package/src/schemas/suite-fields.ts +6 -23
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +2 -4
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +2 -4
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +5 -7
- package/src/tools/create-test-suite.ts +4 -9
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +41 -21
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +7 -21
- package/src/tools/format-scenario.ts +2 -4
- package/src/tools/format-suite-details.ts +6 -12
- package/src/tools/get-agent.ts +15 -6
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +63 -84
- package/src/tools/get-run-plan.ts +4 -6
- package/src/tools/get-scenario.ts +4 -6
- package/src/tools/get-simulation-run.ts +27 -24
- package/src/tools/get-test-suite.ts +2 -5
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +1 -1
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +2 -2
- package/src/tools/list-scenarios.ts +14 -15
- package/src/tools/list-simulation-runs.ts +8 -4
- package/src/tools/list-test-suites.ts +1 -1
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +2 -5
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +2 -5
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +29 -26
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +4 -7
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +4 -7
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +2 -2
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +44 -59
- package/src/tools/update-scenario.ts +5 -7
- package/src/tools/update-test-suite.ts +6 -11
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +14 -12
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/chunk-AN4OUA4D.js +0 -86
- package/dist/chunk-GELJ2MQR.js +0 -226
- package/dist/chunk-K37ZNPXK.js +0 -24
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/delete-evaluator-RKYC4TNB.js +0 -16
- package/dist/discover-evaluator-schema-ESSHQGPY.js +0 -1912
- package/dist/get-analytics-R6AZD3AW.js +0 -73
- package/dist/get-dataset-AKTYSQDI.js +0 -49
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-OEBD6GBU.js +0 -41
- package/dist/langwatch-api-triggers-5E6DDGX6.js +0 -31
- package/dist/run-agent-CAGWGTYT.js +0 -72
- package/dist/search-traces-4XTJYXIK.js +0 -79
- package/src/__tests__/agent-tools.unit.test.ts +0 -257
- package/src/__tests__/all-tools.integration.test.ts +0 -1872
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -284
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/prompt-response-types.unit.test.ts +0 -51
- package/src/__tests__/prompts.integration.test.ts +0 -736
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/run-plan-tools.unit.test.ts +0 -565
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -207
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/simulation-run-tools.unit.test.ts +0 -116
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/test-suite-tools.unit.test.ts +0 -615
- package/src/__tests__/tools.integration.test.ts +0 -896
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -9
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { getRunPlan as apiGetRunPlan } from "../langwatch-api-run-plans.
|
|
2
|
-
import { describeRunPlanScope } from "./format-run-plan.
|
|
3
|
-
import { formatEvaluatorAttachments } from "./format-suite-details.
|
|
1
|
+
import { getRunPlan as apiGetRunPlan } from "../langwatch-api-run-plans.ts";
|
|
2
|
+
import { describeRunPlanScope } from "./format-run-plan.ts";
|
|
3
|
+
import { formatEvaluatorAttachments } from "./format-suite-details.ts";
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Handles the platform_get_run_plan MCP tool invocation.
|
|
@@ -39,9 +39,7 @@ export async function handleGetRunPlan(params: {
|
|
|
39
39
|
const overrides = Object.entries(target.runParameters ?? {})
|
|
40
40
|
.map(([name, value]) => `${name}=${String(value)}`)
|
|
41
41
|
.join(", ");
|
|
42
|
-
lines.push(
|
|
43
|
-
`- ${target.type}:${target.referenceId}${overrides ? ` (${overrides})` : ""}`,
|
|
44
|
-
);
|
|
42
|
+
lines.push(`- ${target.type}:${target.referenceId}${overrides ? ` (${overrides})` : ""}`);
|
|
45
43
|
}
|
|
46
44
|
|
|
47
45
|
if (plan.scenarioIds.length > 0) {
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { getScenario as apiGetScenario } from "../langwatch-api-scenarios.
|
|
2
|
-
import { formatScenarioFields } from "./format-scenario.
|
|
1
|
+
import { getScenario as apiGetScenario } from "../langwatch-api-scenarios.ts";
|
|
2
|
+
import { formatScenarioFields } from "./format-scenario.ts";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Handles the get_scenario MCP tool
|
|
6
|
-
*
|
|
7
|
-
* Retrieves a specific scenario by ID and formats it as
|
|
8
|
-
* AI-readable markdown or raw JSON.
|
|
5
|
+
* Handles the get_scenario MCP tool: retrieves a scenario by ID and
|
|
6
|
+
* formats it as AI-readable markdown or raw JSON.
|
|
9
7
|
*/
|
|
10
8
|
export async function handleGetScenario(params: {
|
|
11
9
|
scenarioId: string;
|
|
@@ -1,8 +1,29 @@
|
|
|
1
|
-
import { getSimulationRun as apiGetSimulationRun } from "../langwatch-api-simulation-runs.js";
|
|
2
1
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
} from "
|
|
2
|
+
getSimulationRun as apiGetSimulationRun,
|
|
3
|
+
type SimulationRunSummary,
|
|
4
|
+
} from "../langwatch-api-simulation-runs.ts";
|
|
5
|
+
import { formatEvaluations, pendingEvaluationNote } from "./format-suite-details.ts";
|
|
6
|
+
|
|
7
|
+
function resultLines(results: NonNullable<SimulationRunSummary["results"]>): string[] {
|
|
8
|
+
const lines = ["\n## Results"];
|
|
9
|
+
if (results.verdict) {
|
|
10
|
+
lines.push(`**Verdict**: ${results.verdict}`);
|
|
11
|
+
}
|
|
12
|
+
if (results.reasoning) {
|
|
13
|
+
lines.push(`**Reasoning**: ${results.reasoning}`);
|
|
14
|
+
}
|
|
15
|
+
if (results.metCriteria && results.metCriteria.length > 0) {
|
|
16
|
+
lines.push(`**Met Criteria**: ${results.metCriteria.join(", ")}`);
|
|
17
|
+
}
|
|
18
|
+
if (results.unmetCriteria && results.unmetCriteria.length > 0) {
|
|
19
|
+
lines.push(`**Unmet Criteria**: ${results.unmetCriteria.join(", ")}`);
|
|
20
|
+
}
|
|
21
|
+
if (results.error) {
|
|
22
|
+
lines.push(`**Error**: ${results.error}`);
|
|
23
|
+
}
|
|
24
|
+
lines.push(...formatEvaluations(results.evaluations));
|
|
25
|
+
return lines;
|
|
26
|
+
}
|
|
6
27
|
|
|
7
28
|
/**
|
|
8
29
|
* Handles the platform_get_simulation_run MCP tool invocation.
|
|
@@ -36,31 +57,13 @@ export async function handleGetSimulationRun(params: {
|
|
|
36
57
|
}
|
|
37
58
|
|
|
38
59
|
if (run.results) {
|
|
39
|
-
lines.push(
|
|
40
|
-
if (run.results.verdict) {
|
|
41
|
-
lines.push(`**Verdict**: ${run.results.verdict}`);
|
|
42
|
-
}
|
|
43
|
-
if (run.results.reasoning) {
|
|
44
|
-
lines.push(`**Reasoning**: ${run.results.reasoning}`);
|
|
45
|
-
}
|
|
46
|
-
if (run.results.metCriteria && run.results.metCriteria.length > 0) {
|
|
47
|
-
lines.push(`**Met Criteria**: ${run.results.metCriteria.join(", ")}`);
|
|
48
|
-
}
|
|
49
|
-
if (run.results.unmetCriteria && run.results.unmetCriteria.length > 0) {
|
|
50
|
-
lines.push(`**Unmet Criteria**: ${run.results.unmetCriteria.join(", ")}`);
|
|
51
|
-
}
|
|
52
|
-
if (run.results.error) {
|
|
53
|
-
lines.push(`**Error**: ${run.results.error}`);
|
|
54
|
-
}
|
|
55
|
-
lines.push(...formatEvaluations(run.results.evaluations));
|
|
60
|
+
lines.push(...resultLines(run.results));
|
|
56
61
|
}
|
|
57
62
|
|
|
58
63
|
if (run.messages && run.messages.length > 0) {
|
|
59
64
|
lines.push("\n## Conversation");
|
|
60
65
|
for (const msg of run.messages) {
|
|
61
|
-
const content = msg.content.length > 300
|
|
62
|
-
? msg.content.slice(0, 300) + "..."
|
|
63
|
-
: msg.content;
|
|
66
|
+
const content = msg.content.length > 300 ? msg.content.slice(0, 300) + "..." : msg.content;
|
|
64
67
|
lines.push(`**[${msg.role}]**: ${content}`);
|
|
65
68
|
}
|
|
66
69
|
}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import { getTestSuite as apiGetTestSuite } from "../langwatch-api-test-suites.
|
|
2
|
-
import {
|
|
3
|
-
formatEvaluatorAttachments,
|
|
4
|
-
formatSuiteFields,
|
|
5
|
-
} from "./format-suite-details.js";
|
|
1
|
+
import { getTestSuite as apiGetTestSuite } from "../langwatch-api-test-suites.ts";
|
|
2
|
+
import { formatEvaluatorAttachments, formatSuiteFields } from "./format-suite-details.ts";
|
|
6
3
|
|
|
7
4
|
/**
|
|
8
5
|
* Handles the platform_get_test_suite MCP tool invocation.
|
package/src/tools/get-trace.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { getTraceById as apiGetTraceById } from "../langwatch-api.
|
|
2
|
-
import { formatEvaluationLines } from "../utils/format-evaluations.
|
|
1
|
+
import { getTraceById as apiGetTraceById } from "../langwatch-api.ts";
|
|
2
|
+
import { formatEvaluationLines } from "../utils/format-evaluations.ts";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Handles the get_trace MCP tool
|
|
6
|
-
*
|
|
7
|
-
* Retrieves a single trace by ID. In digest mode (default), returns the
|
|
8
|
-
* AI-readable formatted digest. In json mode, returns the full raw JSON.
|
|
5
|
+
* Handles the get_trace MCP tool: retrieves a trace by ID as an
|
|
6
|
+
* AI-readable digest (default) or full raw JSON.
|
|
9
7
|
*/
|
|
10
8
|
export async function handleGetTrace(params: {
|
|
11
9
|
traceId: string;
|
|
@@ -20,11 +18,20 @@ export async function handleGetTrace(params: {
|
|
|
20
18
|
|
|
21
19
|
const lines: string[] = [];
|
|
22
20
|
lines.push(`# Trace: ${params.traceId}\n`);
|
|
21
|
+
addTraceDigest(lines, result);
|
|
23
22
|
|
|
23
|
+
lines.push('\n> Tip: Use `get_trace` with `format: "json"` to get the full raw trace data.');
|
|
24
|
+
|
|
25
|
+
return lines.join("\n");
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function addTraceDigest(
|
|
29
|
+
lines: string[],
|
|
30
|
+
result: Awaited<ReturnType<typeof apiGetTraceById>>,
|
|
31
|
+
): void {
|
|
24
32
|
if (result.timestamps) {
|
|
25
33
|
lines.push(`**Started**: ${result.timestamps.started_at}`);
|
|
26
|
-
if (result.timestamps.updated_at)
|
|
27
|
-
lines.push(`**Updated**: ${result.timestamps.updated_at}`);
|
|
34
|
+
if (result.timestamps.updated_at) lines.push(`**Updated**: ${result.timestamps.updated_at}`);
|
|
28
35
|
}
|
|
29
36
|
|
|
30
37
|
if (result.metadata) {
|
|
@@ -39,13 +46,5 @@ export async function handleGetTrace(params: {
|
|
|
39
46
|
lines.push("\n## Evaluations", ...formatEvaluationLines(result.evaluations));
|
|
40
47
|
}
|
|
41
48
|
|
|
42
|
-
if (result.formatted_trace) {
|
|
43
|
-
lines.push(`\n## Trace Details\n${result.formatted_trace}`);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
lines.push(
|
|
47
|
-
'\n> Tip: Use `get_trace` with `format: "json"` to get the full raw trace data.'
|
|
48
|
-
);
|
|
49
|
-
|
|
50
|
-
return lines.join("\n");
|
|
49
|
+
if (result.formatted_trace) lines.push(`\n## Trace Details\n${result.formatted_trace}`);
|
|
51
50
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getWorkflow as apiGetWorkflow } from "../langwatch-api-workflows.
|
|
1
|
+
import { getWorkflow as apiGetWorkflow } from "../langwatch-api-workflows.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleGetWorkflow(params: { id: string }): Promise<string> {
|
|
4
4
|
const workflow = await apiGetWorkflow(params.id);
|
package/src/tools/list-agents.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listAgents as apiListAgents, type AgentSummary } from "../langwatch-api-agents.
|
|
1
|
+
import { listAgents as apiListAgents, type AgentSummary } from "../langwatch-api-agents.ts";
|
|
2
2
|
|
|
3
3
|
/** Who a personal or host-scoped agent belongs to, empty for a shared one. */
|
|
4
4
|
export const agentOwnerLabel = (agent: AgentSummary): string =>
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
listAnnotations as apiListAnnotations,
|
|
3
3
|
getAnnotationsByTrace as apiGetByTrace,
|
|
4
|
-
} from "../langwatch-api-annotations.
|
|
4
|
+
} from "../langwatch-api-annotations.ts";
|
|
5
5
|
|
|
6
|
-
export async function handleListAnnotations(params: {
|
|
7
|
-
traceId?: string;
|
|
8
|
-
}): Promise<string> {
|
|
6
|
+
export async function handleListAnnotations(params: { traceId?: string }): Promise<string> {
|
|
9
7
|
const annotations = params.traceId
|
|
10
8
|
? await apiGetByTrace(params.traceId)
|
|
11
9
|
: await apiListAnnotations();
|
|
@@ -20,7 +18,12 @@ export async function handleListAnnotations(params: {
|
|
|
20
18
|
lines.push(`# Annotations (${annotations.length} total)\n`);
|
|
21
19
|
|
|
22
20
|
for (const a of annotations) {
|
|
23
|
-
|
|
21
|
+
let rating = "—";
|
|
22
|
+
if (a.isThumbsUp === true) {
|
|
23
|
+
rating = "👍";
|
|
24
|
+
} else if (a.isThumbsUp === false) {
|
|
25
|
+
rating = "👎";
|
|
26
|
+
}
|
|
24
27
|
lines.push(`## Annotation ${a.id ?? "—"}`);
|
|
25
28
|
lines.push(`**Trace ID**: ${a.traceId ?? "—"}`);
|
|
26
29
|
lines.push(`**Rating**: ${rating}`);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listApiKeys as apiListApiKeys } from "../langwatch-api-api-keys.
|
|
1
|
+
import { listApiKeys as apiListApiKeys } from "../langwatch-api-api-keys.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListApiKeys(): Promise<string> {
|
|
4
4
|
const result = await apiListApiKeys();
|
|
@@ -12,7 +12,12 @@ export async function handleListApiKeys(): Promise<string> {
|
|
|
12
12
|
lines.push(`# API Keys (${keys.length} total)\n`);
|
|
13
13
|
|
|
14
14
|
for (const k of keys) {
|
|
15
|
-
|
|
15
|
+
let status = "ACTIVE";
|
|
16
|
+
if (k.revokedAt) {
|
|
17
|
+
status = "REVOKED";
|
|
18
|
+
} else if (k.expiresAt && new Date(k.expiresAt) < new Date()) {
|
|
19
|
+
status = "EXPIRED";
|
|
20
|
+
}
|
|
16
21
|
lines.push(`## ${k.name}`);
|
|
17
22
|
lines.push(`**ID**: ${k.id}`);
|
|
18
23
|
lines.push(`**Status**: ${status}`);
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `list_dashboard_boards` MCP tool: the Dashboards boards a widget can be added to, with
|
|
3
|
+
* the id `add_dashboard_widget` and `update_dashboard_widget` take.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { listBoards } from "../langwatch-api-boards.js";
|
|
7
|
+
import { getCurrentProject } from "../langwatch-api-projects.js";
|
|
8
|
+
|
|
9
|
+
export async function handleListDashboardBoards(): Promise<string> {
|
|
10
|
+
const project = await getCurrentProject();
|
|
11
|
+
const { data: boards } = await listBoards({ projectId: project.id });
|
|
12
|
+
|
|
13
|
+
if (boards.length === 0) {
|
|
14
|
+
return "No dashboards found in this project. Create one in the Dashboards area first.";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
return [
|
|
18
|
+
`# Dashboards (${boards.length} total)\n`,
|
|
19
|
+
...boards.flatMap((board) => [
|
|
20
|
+
`## ${board.name}`,
|
|
21
|
+
`**ID**: ${board.id}`,
|
|
22
|
+
`**Widgets**: ${board.widgetCount}`,
|
|
23
|
+
`**URL**: ${board.platformUrl}`,
|
|
24
|
+
"",
|
|
25
|
+
]),
|
|
26
|
+
"> Pass the ID to `add_dashboard_widget` to add a widget to that dashboard.",
|
|
27
|
+
].join("\n");
|
|
28
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listDashboards as apiListDashboards } from "../langwatch-api-dashboards.
|
|
1
|
+
import { listDashboards as apiListDashboards } from "../langwatch-api-dashboards.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListDashboards(): Promise<string> {
|
|
4
4
|
const result = await apiListDashboards();
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
} from "../langwatch-api-datasets.js";
|
|
4
|
-
import { escapeMarkdown } from "../utils/escape-markdown.js";
|
|
1
|
+
import { listDatasetRecords as apiListDatasetRecords } from "../langwatch-api-datasets.ts";
|
|
2
|
+
import { escapeMarkdown } from "../utils/escape-markdown.ts";
|
|
5
3
|
|
|
6
4
|
/**
|
|
7
5
|
* Handles the platform_list_dataset_records MCP tool invocation.
|
|
@@ -31,7 +29,9 @@ export async function handleListDatasetRecords(params: {
|
|
|
31
29
|
}
|
|
32
30
|
|
|
33
31
|
const lines: string[] = [];
|
|
34
|
-
lines.push(
|
|
32
|
+
lines.push(
|
|
33
|
+
`# Records (page ${pagination.page} of ${pagination.totalPages}, ${pagination.total} total)\n`,
|
|
34
|
+
);
|
|
35
35
|
|
|
36
36
|
for (const record of records) {
|
|
37
37
|
lines.push(`**${escapeMarkdown(record.id)}**: ${escapeMarkdown(JSON.stringify(record.entry))}`);
|
|
@@ -39,7 +39,9 @@ export async function handleListDatasetRecords(params: {
|
|
|
39
39
|
|
|
40
40
|
if (pagination.page < pagination.totalPages) {
|
|
41
41
|
lines.push("");
|
|
42
|
-
lines.push(
|
|
42
|
+
lines.push(
|
|
43
|
+
`> Use \`platform_list_dataset_records\` with \`page: ${pagination.page + 1}\` to see the next page.`,
|
|
44
|
+
);
|
|
43
45
|
}
|
|
44
46
|
|
|
45
47
|
return lines.join("\n");
|
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
type DatasetSummary,
|
|
4
|
-
} from "../langwatch-api-datasets.js";
|
|
5
|
-
import { escapeMarkdown } from "../utils/escape-markdown.js";
|
|
1
|
+
import { listDatasets as apiListDatasets, type DatasetSummary } from "../langwatch-api-datasets.ts";
|
|
2
|
+
import { escapeMarkdown } from "../utils/escape-markdown.ts";
|
|
6
3
|
|
|
7
4
|
/**
|
|
8
|
-
* Fetches all datasets by paginating
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* Returns the accumulated dataset summaries along with the total count
|
|
12
|
-
* reported by the server.
|
|
5
|
+
* Fetches all datasets by paginating until every page is retrieved.
|
|
6
|
+
* Returns the accumulated summaries plus the total count from the server.
|
|
13
7
|
*/
|
|
14
8
|
async function fetchAllDatasets(): Promise<{
|
|
15
9
|
datasets: DatasetSummary[];
|
|
@@ -20,7 +14,6 @@ async function fetchAllDatasets(): Promise<{
|
|
|
20
14
|
let page = 1;
|
|
21
15
|
let total = 0;
|
|
22
16
|
|
|
23
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- loop until break
|
|
24
17
|
while (true) {
|
|
25
18
|
const response = await apiListDatasets({ page, limit: PAGE_SIZE });
|
|
26
19
|
total = response.pagination.total;
|
|
@@ -36,14 +29,14 @@ async function fetchAllDatasets(): Promise<{
|
|
|
36
29
|
}
|
|
37
30
|
|
|
38
31
|
/**
|
|
39
|
-
* Handles the platform_list_datasets MCP tool
|
|
40
|
-
*
|
|
41
|
-
* Lists all datasets in the LangWatch project, formatted as an
|
|
32
|
+
* Handles the platform_list_datasets MCP tool: lists datasets as an
|
|
42
33
|
* AI-readable digest or raw JSON.
|
|
43
34
|
*/
|
|
44
|
-
export async function handleListDatasets(
|
|
45
|
-
|
|
46
|
-
|
|
35
|
+
export async function handleListDatasets(
|
|
36
|
+
params: {
|
|
37
|
+
format?: "digest" | "json";
|
|
38
|
+
} = {},
|
|
39
|
+
): Promise<string> {
|
|
47
40
|
if (params.format === "json") {
|
|
48
41
|
const { datasets, total } = await fetchAllDatasets();
|
|
49
42
|
return JSON.stringify({ data: datasets, total }, null, 2);
|
|
@@ -66,15 +59,11 @@ export async function handleListDatasets(params: {
|
|
|
66
59
|
const colNames = ds.columnTypes.map((c) => c.name).join(", ");
|
|
67
60
|
lines.push(`**Columns**: ${colNames}`);
|
|
68
61
|
}
|
|
69
|
-
lines.push(
|
|
70
|
-
`**Records**: ${ds.recordCount ?? "unknown"}`,
|
|
71
|
-
);
|
|
62
|
+
lines.push(`**Records**: ${ds.recordCount ?? "unknown"}`);
|
|
72
63
|
lines.push("");
|
|
73
64
|
}
|
|
74
65
|
|
|
75
|
-
lines.push(
|
|
76
|
-
"> Use `platform_get_dataset` with the slug to see full dataset details and records.",
|
|
77
|
-
);
|
|
66
|
+
lines.push("> Use `platform_get_dataset` with the slug to see full dataset details and records.");
|
|
78
67
|
|
|
79
68
|
return lines.join("\n");
|
|
80
69
|
}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
listEvaluators as apiListEvaluators,
|
|
3
3
|
getEvaluatorType,
|
|
4
|
-
} from "../langwatch-api-evaluators.
|
|
4
|
+
} from "../langwatch-api-evaluators.ts";
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Handles the platform_list_evaluators MCP tool
|
|
8
|
-
*
|
|
9
|
-
* Lists all evaluators in the LangWatch project, formatted as an
|
|
10
|
-
* AI-readable digest.
|
|
7
|
+
* Handles the platform_list_evaluators MCP tool: lists evaluators in
|
|
8
|
+
* the project as an AI-readable digest.
|
|
11
9
|
*/
|
|
12
10
|
export async function handleListEvaluators(): Promise<string> {
|
|
13
11
|
const evaluators = await apiListEvaluators();
|
|
@@ -29,9 +27,7 @@ export async function handleListEvaluators(): Promise<string> {
|
|
|
29
27
|
lines.push("");
|
|
30
28
|
}
|
|
31
29
|
|
|
32
|
-
lines.push(
|
|
33
|
-
"> Use `platform_get_evaluator` with the ID or slug to see full evaluator details.",
|
|
34
|
-
);
|
|
30
|
+
lines.push("> Use `platform_get_evaluator` with the ID or slug to see full evaluator details.");
|
|
35
31
|
|
|
36
32
|
return lines.join("\n");
|
|
37
33
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { makeRequest } from "../langwatch-api.
|
|
1
|
+
import { makeRequest } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
interface RunSummaryEvaluator {
|
|
4
4
|
name: string;
|
|
@@ -52,12 +52,10 @@ const passRate = (evaluations: RunSummary["evaluations"]): string => {
|
|
|
52
52
|
if (passed.length === 0) {
|
|
53
53
|
const scored = entries.filter((e) => typeof e.averageScore === "number");
|
|
54
54
|
if (scored.length === 0) return "—";
|
|
55
|
-
const avg =
|
|
56
|
-
scored.reduce((sum, e) => sum + (e.averageScore ?? 0), 0) / scored.length;
|
|
55
|
+
const avg = scored.reduce((sum, e) => sum + (e.averageScore ?? 0), 0) / scored.length;
|
|
57
56
|
return `${avg.toFixed(2)} avg`;
|
|
58
57
|
}
|
|
59
|
-
const avg =
|
|
60
|
-
passed.reduce((sum, e) => sum + (e.averagePassed ?? 0), 0) / passed.length;
|
|
58
|
+
const avg = passed.reduce((sum, e) => sum + (e.averagePassed ?? 0), 0) / passed.length;
|
|
61
59
|
return `${(avg * 100).toFixed(0)}% pass`;
|
|
62
60
|
};
|
|
63
61
|
|
|
@@ -72,9 +70,7 @@ export async function handleExperimentListRuns(params: {
|
|
|
72
70
|
limit?: number;
|
|
73
71
|
}): Promise<string> {
|
|
74
72
|
const requested =
|
|
75
|
-
typeof params.limit === "number" && params.limit > 0
|
|
76
|
-
? params.limit
|
|
77
|
-
: DEFAULT_LIMIT;
|
|
73
|
+
typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT;
|
|
78
74
|
const effectiveLimit = Math.min(requested, MAX_LIMIT);
|
|
79
75
|
|
|
80
76
|
const search = new URLSearchParams();
|
|
@@ -85,7 +81,7 @@ export async function handleExperimentListRuns(params: {
|
|
|
85
81
|
try {
|
|
86
82
|
result = (await makeRequest(
|
|
87
83
|
"GET",
|
|
88
|
-
`/api/experiments/runs?${search.toString()}`,
|
|
84
|
+
`/api/v1/experiments/runs?${search.toString()}`,
|
|
89
85
|
)) as ExperimentRunsListResponse;
|
|
90
86
|
} catch (error) {
|
|
91
87
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { makeRequest } from "../langwatch-api.
|
|
1
|
+
import { makeRequest } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
interface ExperimentSummary {
|
|
4
4
|
id: string;
|
|
@@ -38,13 +38,9 @@ const escapeMarkdownTableCell = (value: string): string =>
|
|
|
38
38
|
const formatInlineCode = (value: string): string =>
|
|
39
39
|
`\`${value.replace(/[\\`|]/g, (c) => `\\${c}`).replace(/\n/g, " ")}\``;
|
|
40
40
|
|
|
41
|
-
export async function handleExperimentList(params: {
|
|
42
|
-
limit?: number;
|
|
43
|
-
}): Promise<string> {
|
|
41
|
+
export async function handleExperimentList(params: { limit?: number }): Promise<string> {
|
|
44
42
|
const requested =
|
|
45
|
-
typeof params.limit === "number" && params.limit > 0
|
|
46
|
-
? params.limit
|
|
47
|
-
: DEFAULT_LIMIT;
|
|
43
|
+
typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT;
|
|
48
44
|
const effectiveLimit = Math.min(requested, MAX_LIMIT);
|
|
49
45
|
|
|
50
46
|
const search = new URLSearchParams();
|
|
@@ -52,7 +48,7 @@ export async function handleExperimentList(params: {
|
|
|
52
48
|
|
|
53
49
|
const result = (await makeRequest(
|
|
54
50
|
"GET",
|
|
55
|
-
`/api/experiments?${search.toString()}`,
|
|
51
|
+
`/api/v1/experiments?${search.toString()}`,
|
|
56
52
|
)) as ExperimentListResponse;
|
|
57
53
|
|
|
58
54
|
if (result.experiments.length === 0) {
|
|
@@ -88,8 +84,6 @@ export async function handleExperimentList(params: {
|
|
|
88
84
|
}
|
|
89
85
|
|
|
90
86
|
lines.push("");
|
|
91
|
-
lines.push(
|
|
92
|
-
"> Use `platform_experiment_list_runs` with one of these slugs to see its runs.",
|
|
93
|
-
);
|
|
87
|
+
lines.push("> Use `platform_experiment_list_runs` with one of these slugs to see its runs.");
|
|
94
88
|
return lines.join("\n");
|
|
95
89
|
}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { listModelProviders as apiListModelProviders } from "../langwatch-api-model-providers.
|
|
1
|
+
import { listModelProviders as apiListModelProviders } from "../langwatch-api-model-providers.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Handles the platform_list_model_providers MCP tool
|
|
5
|
-
*
|
|
6
|
-
* Lists all model providers for the project, showing provider name,
|
|
7
|
-
* enabled status, and which key fields are set (masked).
|
|
4
|
+
* Handles the platform_list_model_providers MCP tool: lists providers
|
|
5
|
+
* with name, enabled status, and which key fields are set (masked).
|
|
8
6
|
*/
|
|
9
7
|
export async function handleListModelProviders(): Promise<string> {
|
|
10
8
|
const providers = await apiListModelProviders();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listProjects as apiListProjects } from "../langwatch-api-projects.
|
|
1
|
+
import { listProjects as apiListProjects } from "../langwatch-api-projects.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListProjects(params?: {
|
|
4
4
|
page?: number;
|
|
@@ -33,9 +33,7 @@ export async function handleListProjects(params?: {
|
|
|
33
33
|
);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
lines.push(
|
|
37
|
-
"> Use `platform_get_project` with the ID to see full project details.",
|
|
38
|
-
);
|
|
36
|
+
lines.push("> Use `platform_get_project` with the ID to see full project details.");
|
|
39
37
|
|
|
40
38
|
return lines.join("\n");
|
|
41
39
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { listPromptTags as apiListPromptTags } from "../langwatch-api.
|
|
1
|
+
import { listPromptTags as apiListPromptTags } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListPromptTags(): Promise<string> {
|
|
4
|
-
const tags = (await apiListPromptTags()) as
|
|
4
|
+
const tags = (await apiListPromptTags()) as {
|
|
5
5
|
id: string;
|
|
6
6
|
name: string;
|
|
7
7
|
createdAt?: string;
|
|
8
|
-
}
|
|
8
|
+
}[];
|
|
9
9
|
|
|
10
10
|
if (!tags || tags.length === 0) {
|
|
11
11
|
return "No prompt tags found. The `latest` tag is always available.";
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { listPrompts as apiListPrompts } from "../langwatch-api.
|
|
1
|
+
import { listPrompts as apiListPrompts } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Handles the platform_list_prompts MCP tool
|
|
5
|
-
*
|
|
6
|
-
* Lists all prompts in the LangWatch project, formatted as an
|
|
7
|
-
* AI-readable markdown table.
|
|
4
|
+
* Handles the platform_list_prompts MCP tool: lists prompts in the
|
|
5
|
+
* project as an AI-readable markdown table.
|
|
8
6
|
*/
|
|
9
7
|
export async function handleListPrompts(): Promise<string> {
|
|
10
8
|
const prompts = await apiListPrompts();
|
|
@@ -26,9 +24,7 @@ export async function handleListPrompts(): Promise<string> {
|
|
|
26
24
|
lines.push(`| ${handle} | ${name} | ${version} |`);
|
|
27
25
|
}
|
|
28
26
|
|
|
29
|
-
lines.push(
|
|
30
|
-
"\n> Use `platform_get_prompt` with the handle or ID to see full prompt details."
|
|
31
|
-
);
|
|
27
|
+
lines.push("\n> Use `platform_get_prompt` with the handle or ID to see full prompt details.");
|
|
32
28
|
|
|
33
29
|
return lines.join("\n");
|
|
34
30
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { listRunPlans as apiListRunPlans } from "../langwatch-api-run-plans.
|
|
2
|
-
import { describeRunPlanScope } from "./format-run-plan.
|
|
1
|
+
import { listRunPlans as apiListRunPlans } from "../langwatch-api-run-plans.ts";
|
|
2
|
+
import { describeRunPlanScope } from "./format-run-plan.ts";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handles the platform_list_run_plans MCP tool invocation.
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { listScenarios as apiListScenarios } from "../langwatch-api-scenarios.
|
|
1
|
+
import { listScenarios as apiListScenarios } from "../langwatch-api-scenarios.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Handles the platform_list_scenarios MCP tool
|
|
5
|
-
*
|
|
6
|
-
* Lists all scenarios in the LangWatch project, formatted as an
|
|
7
|
-
* AI-readable digest or raw JSON.
|
|
4
|
+
* Handles the platform_list_scenarios MCP tool: lists scenarios in the
|
|
5
|
+
* project as an AI-readable digest or raw JSON.
|
|
8
6
|
*/
|
|
9
7
|
export async function handleListScenarios(params: {
|
|
10
8
|
testSuiteId?: string;
|
|
@@ -35,23 +33,24 @@ export async function handleListScenarios(params: {
|
|
|
35
33
|
for (const s of scenarios) {
|
|
36
34
|
lines.push(`## ${s.name}`);
|
|
37
35
|
lines.push(`**ID**: ${s.id}`);
|
|
38
|
-
const preview =
|
|
39
|
-
s.situation && s.situation.length > 60
|
|
40
|
-
? s.situation.slice(0, 60) + "..."
|
|
41
|
-
: s.situation;
|
|
36
|
+
const preview = formatSituationPreview(s.situation);
|
|
42
37
|
lines.push(`**Situation**: ${preview}`);
|
|
43
|
-
lines.push(
|
|
44
|
-
`**Criteria**: ${Array.isArray(s.criteria) ? s.criteria.length : 0} criteria`,
|
|
45
|
-
);
|
|
38
|
+
lines.push(`**Criteria**: ${Array.isArray(s.criteria) ? s.criteria.length : 0} criteria`);
|
|
46
39
|
if (Array.isArray(s.labels) && s.labels.length > 0) {
|
|
47
40
|
lines.push(`**Labels**: ${s.labels.join(", ")}`);
|
|
48
41
|
}
|
|
49
42
|
lines.push("");
|
|
50
43
|
}
|
|
51
44
|
|
|
52
|
-
lines.push(
|
|
53
|
-
"> Use `platform_get_scenario` with the ID to see full scenario details.",
|
|
54
|
-
);
|
|
45
|
+
lines.push("> Use `platform_get_scenario` with the ID to see full scenario details.");
|
|
55
46
|
|
|
56
47
|
return lines.join("\n");
|
|
57
48
|
}
|
|
49
|
+
|
|
50
|
+
function formatSituationPreview(situation: string | undefined): string | undefined {
|
|
51
|
+
if (situation && situation.length > 60) {
|
|
52
|
+
return situation.slice(0, 60) + "...";
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return situation;
|
|
56
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { listSimulationRuns as apiListSimulationRuns } from "../langwatch-api-simulation-runs.
|
|
2
|
-
import { formatRunStatus } from "./format-suite-details.
|
|
1
|
+
import { listSimulationRuns as apiListSimulationRuns } from "../langwatch-api-simulation-runs.ts";
|
|
2
|
+
import { formatRunStatus } from "./format-suite-details.ts";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Handles the platform_list_simulation_runs MCP tool invocation.
|
|
@@ -27,7 +27,9 @@ export async function handleListSimulationRuns(params: {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
const lines: string[] = [];
|
|
30
|
-
lines.push(
|
|
30
|
+
lines.push(
|
|
31
|
+
`# Simulation Runs (${runs.length} results${result.hasMore ? ", more available" : ""})\n`,
|
|
32
|
+
);
|
|
31
33
|
|
|
32
34
|
for (const run of runs) {
|
|
33
35
|
const statusIcon = formatRunStatus(run.status);
|
|
@@ -47,7 +49,9 @@ export async function handleListSimulationRuns(params: {
|
|
|
47
49
|
lines.push("");
|
|
48
50
|
}
|
|
49
51
|
|
|
50
|
-
lines.push(
|
|
52
|
+
lines.push(
|
|
53
|
+
"> Use `platform_get_simulation_run` with a run ID to see full details (messages, results).",
|
|
54
|
+
);
|
|
51
55
|
|
|
52
56
|
return lines.join("\n");
|
|
53
57
|
}
|