@langwatch/mcp-server 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +280 -200
- package/CONTRIBUTING.md +10 -0
- package/README.md +82 -52
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/archive-run-plan-65QN3XRX.js +16 -0
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/archive-test-suite-NKZDZY3D.js +16 -0
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/chunk-6YZUOD34.js +55 -0
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-7RHG56OH.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-TOXJQDIZ.js → chunk-HGELNIVY.js} +135 -51
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-J42FWRJ2.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-AQJO2EI5.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-4QC7SA2L.js → chunk-J7PQKBLD.js} +884 -559
- package/dist/{chunk-YX3A3CFM.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-5CCZFUXS.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/chunk-O7EMWEVJ.js +44 -0
- package/dist/chunk-PTDR56Z3.js +37 -0
- package/dist/chunk-RABYBFX5.js +43 -0
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-SFB4MU6H.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/chunk-U24EACC4.js +67 -0
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-ZWIL4EFL.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/chunk-VMWJD732.js +57 -0
- package/dist/chunk-XUIJYNRJ.js +15 -0
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-7XHO6UIS.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-YQXRQC6A.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-YUS36KBU.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-E6NMSVX7.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -5
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-CS2CB57Y.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-F5RMX3L3.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-JDR2VB2H.js → create-scenario-UEM77OE7.js} +11 -4
- package/dist/create-test-suite-4IPXVSFQ.js +37 -0
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-O3EEH4S5.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/discover-scenario-schema-OOFIQKOG.js +139 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/get-agent-OTIYUFZA.js +70 -0
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-RAK7URQJ.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/get-prompt-OWXJZHQQ.js +95 -0
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/get-run-plan-BE3M2AUX.js +59 -0
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-YM4ODWW6.js} +8 -4
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-LCFSMPGQ.js} +33 -20
- package/dist/get-test-suite-GC2O7YH6.js +49 -0
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-K7U234Q4.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-L5L2PSK3.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-YXHFPPGA.js → http-server-IR4QOS43.js} +4071 -342
- package/dist/index.js +60221 -38077
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/list-agents-HVYWEQMI.js +43 -0
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-K7QAIX2I.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-SC2HAUFG.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/list-run-plans-PX2XI6EK.js +46 -0
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/list-scenarios-QZRPVHS4.js +46 -0
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-OEADLLQR.js} +19 -9
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/list-test-suites-N4WVLBKK.js +35 -0
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-QSJWURCK.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/rename-test-suite-4J7UX5S2.js +21 -0
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/report-issue-DCZFLGDO.js +899 -0
- package/dist/rerun-run-plan-TG2V4WM4.js +22 -0
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/run-plan-PAARQSI4.js +39 -0
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/run-test-suite-SQJELEZX.js +26 -0
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-QPXC627O.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/{run-agent-6SM4XNA7.js → run-workflow-SP32SO5M.js} +10 -11
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-SMAEHLXH.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-BDUK7TUO.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-LKVYXENH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/update-prompt-MY263DYF.js +141 -0
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-NEXCAQI7.js → update-scenario-MOHRQKAQ.js} +11 -4
- package/package.json +39 -44
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +1021 -596
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +202 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +140 -0
- package/src/langwatch-api-scenarios.ts +17 -10
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +10 -7
- package/src/langwatch-api-test-suites.ts +116 -0
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +143 -109
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +76 -0
- package/src/schemas/suite-fields.ts +132 -0
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +10 -0
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +10 -0
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +11 -5
- package/src/tools/create-test-suite.ts +38 -0
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +86 -12
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +54 -0
- package/src/tools/format-scenario.ts +12 -0
- package/src/tools/format-suite-details.ts +90 -0
- package/src/tools/get-agent.ts +54 -3
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +100 -37
- package/src/tools/get-run-plan.ts +60 -0
- package/src/tools/get-scenario.ts +6 -5
- package/src/tools/get-simulation-run.ts +32 -20
- package/src/tools/get-test-suite.ts +46 -0
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +26 -17
- package/src/tools/list-simulation-runs.ts +13 -5
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +15 -0
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +18 -0
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +65 -14
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +46 -0
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +27 -0
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +185 -15
- package/src/tools/update-scenario.ts +11 -5
- package/src/tools/update-test-suite.ts +40 -0
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +19 -12
- package/uv.lock +416 -296
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/archive-suite-4T6BSRS7.js +0 -16
- package/dist/chunk-77BNWQ3U.js +0 -86
- package/dist/chunk-A2LEVYQ5.js +0 -217
- package/dist/chunk-TJCXFNDT.js +0 -24
- package/dist/chunk-YZLYHLLI.js +0 -48
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/create-suite-LZS6QIAF.js +0 -39
- package/dist/delete-evaluator-33SP2JFX.js +0 -16
- package/dist/discover-evaluator-schema-XTAD4X7D.js +0 -2191
- package/dist/discover-scenario-schema-FE64QK2D.js +0 -67
- package/dist/get-agent-Y7MWZ65F.js +0 -29
- package/dist/get-analytics-TETCSOK6.js +0 -73
- package/dist/get-dataset-RCMU5VAH.js +0 -49
- package/dist/get-prompt-OY5B7TRI.js +0 -46
- package/dist/get-suite-J7JVWO6O.js +0 -45
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-BUBSNCNO.js +0 -41
- package/dist/langwatch-api-triggers-EDFUCJN6.js +0 -31
- package/dist/list-agents-KFHKGPXV.js +0 -33
- package/dist/list-scenarios-F2WYNRIE.js +0 -41
- package/dist/list-suites-CRY4VWW6.js +0 -43
- package/dist/report-issue-XYVAPAYJ.js +0 -412
- package/dist/run-suite-NKZCJ2IN.js +0 -30
- package/dist/search-traces-A6HFKPLJ.js +0 -79
- package/dist/update-prompt-ODCRQRN6.js +0 -25
- package/dist/update-suite-SNO3GS7T.js +0 -40
- package/src/__tests__/all-tools.integration.test.ts +0 -1626
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -288
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -188
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/tools.unit.test.ts +0 -924
- package/src/langwatch-api-suites.ts +0 -118
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -4
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { getRunPlan as apiGetRunPlan } from "../langwatch-api-run-plans.ts";
|
|
2
|
+
import { describeRunPlanScope } from "./format-run-plan.ts";
|
|
3
|
+
import { formatEvaluatorAttachments } from "./format-suite-details.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Handles the platform_get_run_plan MCP tool invocation.
|
|
7
|
+
*/
|
|
8
|
+
export async function handleGetRunPlan(params: {
|
|
9
|
+
id: string;
|
|
10
|
+
format?: "digest" | "json";
|
|
11
|
+
}): Promise<string> {
|
|
12
|
+
const plan = await apiGetRunPlan(params.id);
|
|
13
|
+
|
|
14
|
+
if (params.format === "json") {
|
|
15
|
+
return JSON.stringify(plan, null, 2);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const lines: string[] = [];
|
|
19
|
+
lines.push(`# Run Plan: ${plan.name}\n`);
|
|
20
|
+
lines.push(`**ID**: ${plan.id}`);
|
|
21
|
+
lines.push(`**Slug**: ${plan.slug}`);
|
|
22
|
+
lines.push(`**Covers**: ${describeRunPlanScope(plan.scope, plan.scenarioIds)}`);
|
|
23
|
+
lines.push(`**Repeat**: ${plan.repeatCount}x`);
|
|
24
|
+
lines.push(`**Simulator model**: ${plan.simulatorModel ?? "project default"}`);
|
|
25
|
+
lines.push(`**Judge model**: ${plan.judgeModel ?? "project default"}`);
|
|
26
|
+
if (plan.labels.length > 0) {
|
|
27
|
+
lines.push(`**Labels**: ${plan.labels.join(", ")}`);
|
|
28
|
+
}
|
|
29
|
+
if (plan.archivedAt) {
|
|
30
|
+
lines.push(`**Archived**: ${plan.archivedAt}`);
|
|
31
|
+
}
|
|
32
|
+
lines.push(`**Created**: ${plan.createdAt}`);
|
|
33
|
+
lines.push(`**Updated**: ${plan.updatedAt}`);
|
|
34
|
+
|
|
35
|
+
lines.push("\n## Targets");
|
|
36
|
+
for (const target of plan.targets) {
|
|
37
|
+
// The parameters are part of the target's identity, so a plan that
|
|
38
|
+
// compares one agent on two models reads as two lines that differ.
|
|
39
|
+
const overrides = Object.entries(target.runParameters ?? {})
|
|
40
|
+
.map(([name, value]) => `${name}=${String(value)}`)
|
|
41
|
+
.join(", ");
|
|
42
|
+
lines.push(`- ${target.type}:${target.referenceId}${overrides ? ` (${overrides})` : ""}`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (plan.scenarioIds.length > 0) {
|
|
46
|
+
lines.push("\n## Scenarios");
|
|
47
|
+
for (const id of plan.scenarioIds) {
|
|
48
|
+
lines.push(`- ${id}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
lines.push(...formatEvaluatorAttachments(plan.evaluators));
|
|
53
|
+
|
|
54
|
+
lines.push(`\n**View**: ${plan.platformUrl}`);
|
|
55
|
+
lines.push(
|
|
56
|
+
"\n> Use `platform_rerun_run_plan` to run this configuration again, or `platform_run_plan` with this name to replace it.",
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
return lines.join("\n");
|
|
60
|
+
}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { getScenario as apiGetScenario } from "../langwatch-api-scenarios.
|
|
1
|
+
import { getScenario as apiGetScenario } from "../langwatch-api-scenarios.ts";
|
|
2
|
+
import { formatScenarioFields } from "./format-scenario.ts";
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
|
-
* Handles the get_scenario MCP tool
|
|
5
|
-
*
|
|
6
|
-
* Retrieves a specific scenario by ID and formats it as
|
|
7
|
-
* AI-readable markdown or raw JSON.
|
|
5
|
+
* Handles the get_scenario MCP tool: retrieves a scenario by ID and
|
|
6
|
+
* formats it as AI-readable markdown or raw JSON.
|
|
8
7
|
*/
|
|
9
8
|
export async function handleGetScenario(params: {
|
|
10
9
|
scenarioId: string;
|
|
@@ -32,5 +31,7 @@ export async function handleGetScenario(params: {
|
|
|
32
31
|
lines.push(`\n**Labels**: ${scenario.labels.join(", ")}`);
|
|
33
32
|
}
|
|
34
33
|
|
|
34
|
+
lines.push(...formatScenarioFields(scenario.fields));
|
|
35
|
+
|
|
35
36
|
return lines.join("\n");
|
|
36
37
|
}
|
|
@@ -1,4 +1,29 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
getSimulationRun as apiGetSimulationRun,
|
|
3
|
+
type SimulationRunSummary,
|
|
4
|
+
} from "../langwatch-api-simulation-runs.ts";
|
|
5
|
+
import { formatEvaluations, pendingEvaluationNote } from "./format-suite-details.ts";
|
|
6
|
+
|
|
7
|
+
function resultLines(results: NonNullable<SimulationRunSummary["results"]>): string[] {
|
|
8
|
+
const lines = ["\n## Results"];
|
|
9
|
+
if (results.verdict) {
|
|
10
|
+
lines.push(`**Verdict**: ${results.verdict}`);
|
|
11
|
+
}
|
|
12
|
+
if (results.reasoning) {
|
|
13
|
+
lines.push(`**Reasoning**: ${results.reasoning}`);
|
|
14
|
+
}
|
|
15
|
+
if (results.metCriteria && results.metCriteria.length > 0) {
|
|
16
|
+
lines.push(`**Met Criteria**: ${results.metCriteria.join(", ")}`);
|
|
17
|
+
}
|
|
18
|
+
if (results.unmetCriteria && results.unmetCriteria.length > 0) {
|
|
19
|
+
lines.push(`**Unmet Criteria**: ${results.unmetCriteria.join(", ")}`);
|
|
20
|
+
}
|
|
21
|
+
if (results.error) {
|
|
22
|
+
lines.push(`**Error**: ${results.error}`);
|
|
23
|
+
}
|
|
24
|
+
lines.push(...formatEvaluations(results.evaluations));
|
|
25
|
+
return lines;
|
|
26
|
+
}
|
|
2
27
|
|
|
3
28
|
/**
|
|
4
29
|
* Handles the platform_get_simulation_run MCP tool invocation.
|
|
@@ -19,6 +44,7 @@ export async function handleGetSimulationRun(params: {
|
|
|
19
44
|
lines.push(`**Scenario ID**: ${run.scenarioId}`);
|
|
20
45
|
lines.push(`**Batch ID**: ${run.batchRunId}`);
|
|
21
46
|
lines.push(`**Status**: ${run.status}`);
|
|
47
|
+
lines.push(...pendingEvaluationNote(run.status));
|
|
22
48
|
|
|
23
49
|
const duration = run.durationInMs > 0 ? `${(run.durationInMs / 1000).toFixed(1)}s` : "—";
|
|
24
50
|
lines.push(`**Duration**: ${duration}`);
|
|
@@ -26,32 +52,18 @@ export async function handleGetSimulationRun(params: {
|
|
|
26
52
|
lines.push(`**Cost**: $${run.totalCost.toFixed(4)}`);
|
|
27
53
|
}
|
|
28
54
|
lines.push(`**Started**: ${new Date(run.timestamp).toISOString()}`);
|
|
55
|
+
if (run.note) {
|
|
56
|
+
lines.push(`**Note**: ${run.note}`);
|
|
57
|
+
}
|
|
29
58
|
|
|
30
59
|
if (run.results) {
|
|
31
|
-
lines.push(
|
|
32
|
-
if (run.results.verdict) {
|
|
33
|
-
lines.push(`**Verdict**: ${run.results.verdict}`);
|
|
34
|
-
}
|
|
35
|
-
if (run.results.reasoning) {
|
|
36
|
-
lines.push(`**Reasoning**: ${run.results.reasoning}`);
|
|
37
|
-
}
|
|
38
|
-
if (run.results.metCriteria && run.results.metCriteria.length > 0) {
|
|
39
|
-
lines.push(`**Met Criteria**: ${run.results.metCriteria.join(", ")}`);
|
|
40
|
-
}
|
|
41
|
-
if (run.results.unmetCriteria && run.results.unmetCriteria.length > 0) {
|
|
42
|
-
lines.push(`**Unmet Criteria**: ${run.results.unmetCriteria.join(", ")}`);
|
|
43
|
-
}
|
|
44
|
-
if (run.results.error) {
|
|
45
|
-
lines.push(`**Error**: ${run.results.error}`);
|
|
46
|
-
}
|
|
60
|
+
lines.push(...resultLines(run.results));
|
|
47
61
|
}
|
|
48
62
|
|
|
49
63
|
if (run.messages && run.messages.length > 0) {
|
|
50
64
|
lines.push("\n## Conversation");
|
|
51
65
|
for (const msg of run.messages) {
|
|
52
|
-
const content = msg.content.length > 300
|
|
53
|
-
? msg.content.slice(0, 300) + "..."
|
|
54
|
-
: msg.content;
|
|
66
|
+
const content = msg.content.length > 300 ? msg.content.slice(0, 300) + "..." : msg.content;
|
|
55
67
|
lines.push(`**[${msg.role}]**: ${content}`);
|
|
56
68
|
}
|
|
57
69
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { getTestSuite as apiGetTestSuite } from "../langwatch-api-test-suites.ts";
|
|
2
|
+
import { formatEvaluatorAttachments, formatSuiteFields } from "./format-suite-details.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Handles the platform_get_test_suite MCP tool invocation.
|
|
6
|
+
*/
|
|
7
|
+
export async function handleGetTestSuite(params: {
|
|
8
|
+
id: string;
|
|
9
|
+
format?: "digest" | "json";
|
|
10
|
+
}): Promise<string> {
|
|
11
|
+
const suite = await apiGetTestSuite(params.id);
|
|
12
|
+
|
|
13
|
+
if (params.format === "json") {
|
|
14
|
+
return JSON.stringify(suite, null, 2);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const lines: string[] = [];
|
|
18
|
+
lines.push(`# Test Suite: ${suite.name}\n`);
|
|
19
|
+
lines.push(`**ID**: ${suite.id}`);
|
|
20
|
+
lines.push(`**Slug**: ${suite.slug}`);
|
|
21
|
+
lines.push(`**Scenarios**: ${suite.scenarioCount}`);
|
|
22
|
+
if (suite.archivedAt) {
|
|
23
|
+
lines.push(`**Archived**: ${suite.archivedAt}`);
|
|
24
|
+
}
|
|
25
|
+
lines.push(`**Created**: ${suite.createdAt}`);
|
|
26
|
+
lines.push(`**Updated**: ${suite.updatedAt}`);
|
|
27
|
+
|
|
28
|
+
lines.push(...formatSuiteFields(suite.fields));
|
|
29
|
+
lines.push(...formatEvaluatorAttachments(suite.evaluators));
|
|
30
|
+
|
|
31
|
+
lines.push("\n## Scenarios");
|
|
32
|
+
if (suite.scenarios.length === 0) {
|
|
33
|
+
lines.push("None filed yet.");
|
|
34
|
+
} else {
|
|
35
|
+
for (const scenario of suite.scenarios) {
|
|
36
|
+
lines.push(`- ${scenario.name} (${scenario.id})`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
lines.push(`\n**View**: ${suite.platformUrl}`);
|
|
41
|
+
lines.push(
|
|
42
|
+
"\n> Use `platform_run_test_suite` to run every scenario of this suite against a target.",
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
return lines.join("\n");
|
|
46
|
+
}
|
package/src/tools/get-trace.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { getTraceById as apiGetTraceById } from "../langwatch-api.
|
|
2
|
-
import { formatEvaluationLines } from "../utils/format-evaluations.
|
|
1
|
+
import { getTraceById as apiGetTraceById } from "../langwatch-api.ts";
|
|
2
|
+
import { formatEvaluationLines } from "../utils/format-evaluations.ts";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Handles the get_trace MCP tool
|
|
6
|
-
*
|
|
7
|
-
* Retrieves a single trace by ID. In digest mode (default), returns the
|
|
8
|
-
* AI-readable formatted digest. In json mode, returns the full raw JSON.
|
|
5
|
+
* Handles the get_trace MCP tool: retrieves a trace by ID as an
|
|
6
|
+
* AI-readable digest (default) or full raw JSON.
|
|
9
7
|
*/
|
|
10
8
|
export async function handleGetTrace(params: {
|
|
11
9
|
traceId: string;
|
|
@@ -20,11 +18,20 @@ export async function handleGetTrace(params: {
|
|
|
20
18
|
|
|
21
19
|
const lines: string[] = [];
|
|
22
20
|
lines.push(`# Trace: ${params.traceId}\n`);
|
|
21
|
+
addTraceDigest(lines, result);
|
|
23
22
|
|
|
23
|
+
lines.push('\n> Tip: Use `get_trace` with `format: "json"` to get the full raw trace data.');
|
|
24
|
+
|
|
25
|
+
return lines.join("\n");
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function addTraceDigest(
|
|
29
|
+
lines: string[],
|
|
30
|
+
result: Awaited<ReturnType<typeof apiGetTraceById>>,
|
|
31
|
+
): void {
|
|
24
32
|
if (result.timestamps) {
|
|
25
33
|
lines.push(`**Started**: ${result.timestamps.started_at}`);
|
|
26
|
-
if (result.timestamps.updated_at)
|
|
27
|
-
lines.push(`**Updated**: ${result.timestamps.updated_at}`);
|
|
34
|
+
if (result.timestamps.updated_at) lines.push(`**Updated**: ${result.timestamps.updated_at}`);
|
|
28
35
|
}
|
|
29
36
|
|
|
30
37
|
if (result.metadata) {
|
|
@@ -39,13 +46,5 @@ export async function handleGetTrace(params: {
|
|
|
39
46
|
lines.push("\n## Evaluations", ...formatEvaluationLines(result.evaluations));
|
|
40
47
|
}
|
|
41
48
|
|
|
42
|
-
if (result.formatted_trace) {
|
|
43
|
-
lines.push(`\n## Trace Details\n${result.formatted_trace}`);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
lines.push(
|
|
47
|
-
'\n> Tip: Use `get_trace` with `format: "json"` to get the full raw trace data.'
|
|
48
|
-
);
|
|
49
|
-
|
|
50
|
-
return lines.join("\n");
|
|
49
|
+
if (result.formatted_trace) lines.push(`\n## Trace Details\n${result.formatted_trace}`);
|
|
51
50
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getWorkflow as apiGetWorkflow } from "../langwatch-api-workflows.
|
|
1
|
+
import { getWorkflow as apiGetWorkflow } from "../langwatch-api-workflows.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleGetWorkflow(params: { id: string }): Promise<string> {
|
|
4
4
|
const workflow = await apiGetWorkflow(params.id);
|
package/src/tools/list-agents.ts
CHANGED
|
@@ -1,11 +1,20 @@
|
|
|
1
|
-
import { listAgents as apiListAgents } from "../langwatch-api-agents.
|
|
1
|
+
import { listAgents as apiListAgents, type AgentSummary } from "../langwatch-api-agents.ts";
|
|
2
2
|
|
|
3
|
+
/** Who a personal or host-scoped agent belongs to, empty for a shared one. */
|
|
4
|
+
export const agentOwnerLabel = (agent: AgentSummary): string =>
|
|
5
|
+
agent.owner?.name ?? agent.hostLabel ?? "";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Handles the platform_list_agents MCP tool invocation.
|
|
9
|
+
*
|
|
10
|
+
* @see specs/mcp-server/agent-tools.feature
|
|
11
|
+
*/
|
|
3
12
|
export async function handleListAgents(): Promise<string> {
|
|
4
13
|
const result = await apiListAgents({ limit: 100 });
|
|
5
14
|
const agents = result.data;
|
|
6
15
|
|
|
7
16
|
if (!Array.isArray(agents) || agents.length === 0) {
|
|
8
|
-
return "No agents found in this project.\n\n> Tip:
|
|
17
|
+
return "No agents found in this project.\n\n> Tip: connect one from code with `connectAgent` (TypeScript, `langwatch/agent`) or `connect_agent` (Python), or use `platform_create_agent` for an HTTP agent.";
|
|
9
18
|
}
|
|
10
19
|
|
|
11
20
|
const lines: string[] = [];
|
|
@@ -15,12 +24,20 @@ export async function handleListAgents(): Promise<string> {
|
|
|
15
24
|
lines.push(`## ${a.name}`);
|
|
16
25
|
lines.push(`**ID**: ${a.id}`);
|
|
17
26
|
lines.push(`**Type**: ${a.type}`);
|
|
27
|
+
if (a.environment) lines.push(`**Environment**: ${a.environment}`);
|
|
28
|
+
if (a.status) lines.push(`**Status**: ${a.status}`);
|
|
29
|
+
if (a.instances) lines.push(`**Instances**: ${a.instances.length}`);
|
|
30
|
+
const owner = agentOwnerLabel(a);
|
|
31
|
+
if (owner) lines.push(`**Owner**: ${owner}`);
|
|
32
|
+
if (a.selectable === false) {
|
|
33
|
+
lines.push("**Run**: only its owner can run this agent");
|
|
34
|
+
}
|
|
18
35
|
lines.push(`**Updated**: ${a.updatedAt}`);
|
|
19
36
|
lines.push("");
|
|
20
37
|
}
|
|
21
38
|
|
|
22
39
|
lines.push(
|
|
23
|
-
"> Use `platform_get_agent` with the ID to see full agent details including config.",
|
|
40
|
+
"> Use `platform_get_agent` with the ID to see full agent details including parameters, instances and config.",
|
|
24
41
|
);
|
|
25
42
|
|
|
26
43
|
return lines.join("\n");
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
listAnnotations as apiListAnnotations,
|
|
3
3
|
getAnnotationsByTrace as apiGetByTrace,
|
|
4
|
-
} from "../langwatch-api-annotations.
|
|
4
|
+
} from "../langwatch-api-annotations.ts";
|
|
5
5
|
|
|
6
|
-
export async function handleListAnnotations(params: {
|
|
7
|
-
traceId?: string;
|
|
8
|
-
}): Promise<string> {
|
|
6
|
+
export async function handleListAnnotations(params: { traceId?: string }): Promise<string> {
|
|
9
7
|
const annotations = params.traceId
|
|
10
8
|
? await apiGetByTrace(params.traceId)
|
|
11
9
|
: await apiListAnnotations();
|
|
@@ -20,7 +18,12 @@ export async function handleListAnnotations(params: {
|
|
|
20
18
|
lines.push(`# Annotations (${annotations.length} total)\n`);
|
|
21
19
|
|
|
22
20
|
for (const a of annotations) {
|
|
23
|
-
|
|
21
|
+
let rating = "—";
|
|
22
|
+
if (a.isThumbsUp === true) {
|
|
23
|
+
rating = "👍";
|
|
24
|
+
} else if (a.isThumbsUp === false) {
|
|
25
|
+
rating = "👎";
|
|
26
|
+
}
|
|
24
27
|
lines.push(`## Annotation ${a.id ?? "—"}`);
|
|
25
28
|
lines.push(`**Trace ID**: ${a.traceId ?? "—"}`);
|
|
26
29
|
lines.push(`**Rating**: ${rating}`);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listApiKeys as apiListApiKeys } from "../langwatch-api-api-keys.
|
|
1
|
+
import { listApiKeys as apiListApiKeys } from "../langwatch-api-api-keys.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListApiKeys(): Promise<string> {
|
|
4
4
|
const result = await apiListApiKeys();
|
|
@@ -12,7 +12,12 @@ export async function handleListApiKeys(): Promise<string> {
|
|
|
12
12
|
lines.push(`# API Keys (${keys.length} total)\n`);
|
|
13
13
|
|
|
14
14
|
for (const k of keys) {
|
|
15
|
-
|
|
15
|
+
let status = "ACTIVE";
|
|
16
|
+
if (k.revokedAt) {
|
|
17
|
+
status = "REVOKED";
|
|
18
|
+
} else if (k.expiresAt && new Date(k.expiresAt) < new Date()) {
|
|
19
|
+
status = "EXPIRED";
|
|
20
|
+
}
|
|
16
21
|
lines.push(`## ${k.name}`);
|
|
17
22
|
lines.push(`**ID**: ${k.id}`);
|
|
18
23
|
lines.push(`**Status**: ${status}`);
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `list_dashboard_boards` MCP tool: the Dashboards boards a widget can be added to, with
|
|
3
|
+
* the id `add_dashboard_widget` and `update_dashboard_widget` take.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { listBoards } from "../langwatch-api-boards.js";
|
|
7
|
+
import { getCurrentProject } from "../langwatch-api-projects.js";
|
|
8
|
+
|
|
9
|
+
export async function handleListDashboardBoards(): Promise<string> {
|
|
10
|
+
const project = await getCurrentProject();
|
|
11
|
+
const { data: boards } = await listBoards({ projectId: project.id });
|
|
12
|
+
|
|
13
|
+
if (boards.length === 0) {
|
|
14
|
+
return "No dashboards found in this project. Create one in the Dashboards area first.";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
return [
|
|
18
|
+
`# Dashboards (${boards.length} total)\n`,
|
|
19
|
+
...boards.flatMap((board) => [
|
|
20
|
+
`## ${board.name}`,
|
|
21
|
+
`**ID**: ${board.id}`,
|
|
22
|
+
`**Widgets**: ${board.widgetCount}`,
|
|
23
|
+
`**URL**: ${board.platformUrl}`,
|
|
24
|
+
"",
|
|
25
|
+
]),
|
|
26
|
+
"> Pass the ID to `add_dashboard_widget` to add a widget to that dashboard.",
|
|
27
|
+
].join("\n");
|
|
28
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listDashboards as apiListDashboards } from "../langwatch-api-dashboards.
|
|
1
|
+
import { listDashboards as apiListDashboards } from "../langwatch-api-dashboards.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListDashboards(): Promise<string> {
|
|
4
4
|
const result = await apiListDashboards();
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
} from "../langwatch-api-datasets.js";
|
|
4
|
-
import { escapeMarkdown } from "../utils/escape-markdown.js";
|
|
1
|
+
import { listDatasetRecords as apiListDatasetRecords } from "../langwatch-api-datasets.ts";
|
|
2
|
+
import { escapeMarkdown } from "../utils/escape-markdown.ts";
|
|
5
3
|
|
|
6
4
|
/**
|
|
7
5
|
* Handles the platform_list_dataset_records MCP tool invocation.
|
|
@@ -31,7 +29,9 @@ export async function handleListDatasetRecords(params: {
|
|
|
31
29
|
}
|
|
32
30
|
|
|
33
31
|
const lines: string[] = [];
|
|
34
|
-
lines.push(
|
|
32
|
+
lines.push(
|
|
33
|
+
`# Records (page ${pagination.page} of ${pagination.totalPages}, ${pagination.total} total)\n`,
|
|
34
|
+
);
|
|
35
35
|
|
|
36
36
|
for (const record of records) {
|
|
37
37
|
lines.push(`**${escapeMarkdown(record.id)}**: ${escapeMarkdown(JSON.stringify(record.entry))}`);
|
|
@@ -39,7 +39,9 @@ export async function handleListDatasetRecords(params: {
|
|
|
39
39
|
|
|
40
40
|
if (pagination.page < pagination.totalPages) {
|
|
41
41
|
lines.push("");
|
|
42
|
-
lines.push(
|
|
42
|
+
lines.push(
|
|
43
|
+
`> Use \`platform_list_dataset_records\` with \`page: ${pagination.page + 1}\` to see the next page.`,
|
|
44
|
+
);
|
|
43
45
|
}
|
|
44
46
|
|
|
45
47
|
return lines.join("\n");
|
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
type DatasetSummary,
|
|
4
|
-
} from "../langwatch-api-datasets.js";
|
|
5
|
-
import { escapeMarkdown } from "../utils/escape-markdown.js";
|
|
1
|
+
import { listDatasets as apiListDatasets, type DatasetSummary } from "../langwatch-api-datasets.ts";
|
|
2
|
+
import { escapeMarkdown } from "../utils/escape-markdown.ts";
|
|
6
3
|
|
|
7
4
|
/**
|
|
8
|
-
* Fetches all datasets by paginating
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* Returns the accumulated dataset summaries along with the total count
|
|
12
|
-
* reported by the server.
|
|
5
|
+
* Fetches all datasets by paginating until every page is retrieved.
|
|
6
|
+
* Returns the accumulated summaries plus the total count from the server.
|
|
13
7
|
*/
|
|
14
8
|
async function fetchAllDatasets(): Promise<{
|
|
15
9
|
datasets: DatasetSummary[];
|
|
@@ -20,7 +14,6 @@ async function fetchAllDatasets(): Promise<{
|
|
|
20
14
|
let page = 1;
|
|
21
15
|
let total = 0;
|
|
22
16
|
|
|
23
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- loop until break
|
|
24
17
|
while (true) {
|
|
25
18
|
const response = await apiListDatasets({ page, limit: PAGE_SIZE });
|
|
26
19
|
total = response.pagination.total;
|
|
@@ -36,14 +29,14 @@ async function fetchAllDatasets(): Promise<{
|
|
|
36
29
|
}
|
|
37
30
|
|
|
38
31
|
/**
|
|
39
|
-
* Handles the platform_list_datasets MCP tool
|
|
40
|
-
*
|
|
41
|
-
* Lists all datasets in the LangWatch project, formatted as an
|
|
32
|
+
* Handles the platform_list_datasets MCP tool: lists datasets as an
|
|
42
33
|
* AI-readable digest or raw JSON.
|
|
43
34
|
*/
|
|
44
|
-
export async function handleListDatasets(
|
|
45
|
-
|
|
46
|
-
|
|
35
|
+
export async function handleListDatasets(
|
|
36
|
+
params: {
|
|
37
|
+
format?: "digest" | "json";
|
|
38
|
+
} = {},
|
|
39
|
+
): Promise<string> {
|
|
47
40
|
if (params.format === "json") {
|
|
48
41
|
const { datasets, total } = await fetchAllDatasets();
|
|
49
42
|
return JSON.stringify({ data: datasets, total }, null, 2);
|
|
@@ -66,15 +59,11 @@ export async function handleListDatasets(params: {
|
|
|
66
59
|
const colNames = ds.columnTypes.map((c) => c.name).join(", ");
|
|
67
60
|
lines.push(`**Columns**: ${colNames}`);
|
|
68
61
|
}
|
|
69
|
-
lines.push(
|
|
70
|
-
`**Records**: ${ds.recordCount ?? "unknown"}`,
|
|
71
|
-
);
|
|
62
|
+
lines.push(`**Records**: ${ds.recordCount ?? "unknown"}`);
|
|
72
63
|
lines.push("");
|
|
73
64
|
}
|
|
74
65
|
|
|
75
|
-
lines.push(
|
|
76
|
-
"> Use `platform_get_dataset` with the slug to see full dataset details and records.",
|
|
77
|
-
);
|
|
66
|
+
lines.push("> Use `platform_get_dataset` with the slug to see full dataset details and records.");
|
|
78
67
|
|
|
79
68
|
return lines.join("\n");
|
|
80
69
|
}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
listEvaluators as apiListEvaluators,
|
|
3
3
|
getEvaluatorType,
|
|
4
|
-
} from "../langwatch-api-evaluators.
|
|
4
|
+
} from "../langwatch-api-evaluators.ts";
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Handles the platform_list_evaluators MCP tool
|
|
8
|
-
*
|
|
9
|
-
* Lists all evaluators in the LangWatch project, formatted as an
|
|
10
|
-
* AI-readable digest.
|
|
7
|
+
* Handles the platform_list_evaluators MCP tool: lists evaluators in
|
|
8
|
+
* the project as an AI-readable digest.
|
|
11
9
|
*/
|
|
12
10
|
export async function handleListEvaluators(): Promise<string> {
|
|
13
11
|
const evaluators = await apiListEvaluators();
|
|
@@ -29,9 +27,7 @@ export async function handleListEvaluators(): Promise<string> {
|
|
|
29
27
|
lines.push("");
|
|
30
28
|
}
|
|
31
29
|
|
|
32
|
-
lines.push(
|
|
33
|
-
"> Use `platform_get_evaluator` with the ID or slug to see full evaluator details.",
|
|
34
|
-
);
|
|
30
|
+
lines.push("> Use `platform_get_evaluator` with the ID or slug to see full evaluator details.");
|
|
35
31
|
|
|
36
32
|
return lines.join("\n");
|
|
37
33
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { makeRequest } from "../langwatch-api.
|
|
1
|
+
import { makeRequest } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
interface RunSummaryEvaluator {
|
|
4
4
|
name: string;
|
|
@@ -52,12 +52,10 @@ const passRate = (evaluations: RunSummary["evaluations"]): string => {
|
|
|
52
52
|
if (passed.length === 0) {
|
|
53
53
|
const scored = entries.filter((e) => typeof e.averageScore === "number");
|
|
54
54
|
if (scored.length === 0) return "—";
|
|
55
|
-
const avg =
|
|
56
|
-
scored.reduce((sum, e) => sum + (e.averageScore ?? 0), 0) / scored.length;
|
|
55
|
+
const avg = scored.reduce((sum, e) => sum + (e.averageScore ?? 0), 0) / scored.length;
|
|
57
56
|
return `${avg.toFixed(2)} avg`;
|
|
58
57
|
}
|
|
59
|
-
const avg =
|
|
60
|
-
passed.reduce((sum, e) => sum + (e.averagePassed ?? 0), 0) / passed.length;
|
|
58
|
+
const avg = passed.reduce((sum, e) => sum + (e.averagePassed ?? 0), 0) / passed.length;
|
|
61
59
|
return `${(avg * 100).toFixed(0)}% pass`;
|
|
62
60
|
};
|
|
63
61
|
|
|
@@ -72,9 +70,7 @@ export async function handleExperimentListRuns(params: {
|
|
|
72
70
|
limit?: number;
|
|
73
71
|
}): Promise<string> {
|
|
74
72
|
const requested =
|
|
75
|
-
typeof params.limit === "number" && params.limit > 0
|
|
76
|
-
? params.limit
|
|
77
|
-
: DEFAULT_LIMIT;
|
|
73
|
+
typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT;
|
|
78
74
|
const effectiveLimit = Math.min(requested, MAX_LIMIT);
|
|
79
75
|
|
|
80
76
|
const search = new URLSearchParams();
|
|
@@ -85,7 +81,7 @@ export async function handleExperimentListRuns(params: {
|
|
|
85
81
|
try {
|
|
86
82
|
result = (await makeRequest(
|
|
87
83
|
"GET",
|
|
88
|
-
`/api/experiments/runs?${search.toString()}`,
|
|
84
|
+
`/api/v1/experiments/runs?${search.toString()}`,
|
|
89
85
|
)) as ExperimentRunsListResponse;
|
|
90
86
|
} catch (error) {
|
|
91
87
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { makeRequest } from "../langwatch-api.
|
|
1
|
+
import { makeRequest } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
interface ExperimentSummary {
|
|
4
4
|
id: string;
|
|
@@ -38,13 +38,9 @@ const escapeMarkdownTableCell = (value: string): string =>
|
|
|
38
38
|
const formatInlineCode = (value: string): string =>
|
|
39
39
|
`\`${value.replace(/[\\`|]/g, (c) => `\\${c}`).replace(/\n/g, " ")}\``;
|
|
40
40
|
|
|
41
|
-
export async function handleExperimentList(params: {
|
|
42
|
-
limit?: number;
|
|
43
|
-
}): Promise<string> {
|
|
41
|
+
export async function handleExperimentList(params: { limit?: number }): Promise<string> {
|
|
44
42
|
const requested =
|
|
45
|
-
typeof params.limit === "number" && params.limit > 0
|
|
46
|
-
? params.limit
|
|
47
|
-
: DEFAULT_LIMIT;
|
|
43
|
+
typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT;
|
|
48
44
|
const effectiveLimit = Math.min(requested, MAX_LIMIT);
|
|
49
45
|
|
|
50
46
|
const search = new URLSearchParams();
|
|
@@ -52,7 +48,7 @@ export async function handleExperimentList(params: {
|
|
|
52
48
|
|
|
53
49
|
const result = (await makeRequest(
|
|
54
50
|
"GET",
|
|
55
|
-
`/api/experiments?${search.toString()}`,
|
|
51
|
+
`/api/v1/experiments?${search.toString()}`,
|
|
56
52
|
)) as ExperimentListResponse;
|
|
57
53
|
|
|
58
54
|
if (result.experiments.length === 0) {
|
|
@@ -88,8 +84,6 @@ export async function handleExperimentList(params: {
|
|
|
88
84
|
}
|
|
89
85
|
|
|
90
86
|
lines.push("");
|
|
91
|
-
lines.push(
|
|
92
|
-
"> Use `platform_experiment_list_runs` with one of these slugs to see its runs.",
|
|
93
|
-
);
|
|
87
|
+
lines.push("> Use `platform_experiment_list_runs` with one of these slugs to see its runs.");
|
|
94
88
|
return lines.join("\n");
|
|
95
89
|
}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { listModelProviders as apiListModelProviders } from "../langwatch-api-model-providers.
|
|
1
|
+
import { listModelProviders as apiListModelProviders } from "../langwatch-api-model-providers.ts";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Handles the platform_list_model_providers MCP tool
|
|
5
|
-
*
|
|
6
|
-
* Lists all model providers for the project, showing provider name,
|
|
7
|
-
* enabled status, and which key fields are set (masked).
|
|
4
|
+
* Handles the platform_list_model_providers MCP tool: lists providers
|
|
5
|
+
* with name, enabled status, and which key fields are set (masked).
|
|
8
6
|
*/
|
|
9
7
|
export async function handleListModelProviders(): Promise<string> {
|
|
10
8
|
const providers = await apiListModelProviders();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listProjects as apiListProjects } from "../langwatch-api-projects.
|
|
1
|
+
import { listProjects as apiListProjects } from "../langwatch-api-projects.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListProjects(params?: {
|
|
4
4
|
page?: number;
|
|
@@ -33,9 +33,7 @@ export async function handleListProjects(params?: {
|
|
|
33
33
|
);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
lines.push(
|
|
37
|
-
"> Use `platform_get_project` with the ID to see full project details.",
|
|
38
|
-
);
|
|
36
|
+
lines.push("> Use `platform_get_project` with the ID to see full project details.");
|
|
39
37
|
|
|
40
38
|
return lines.join("\n");
|
|
41
39
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { listPromptTags as apiListPromptTags } from "../langwatch-api.
|
|
1
|
+
import { listPromptTags as apiListPromptTags } from "../langwatch-api.ts";
|
|
2
2
|
|
|
3
3
|
export async function handleListPromptTags(): Promise<string> {
|
|
4
|
-
const tags = (await apiListPromptTags()) as
|
|
4
|
+
const tags = (await apiListPromptTags()) as {
|
|
5
5
|
id: string;
|
|
6
6
|
name: string;
|
|
7
7
|
createdAt?: string;
|
|
8
|
-
}
|
|
8
|
+
}[];
|
|
9
9
|
|
|
10
10
|
if (!tags || tags.length === 0) {
|
|
11
11
|
return "No prompt tags found. The `latest` tag is always available.";
|