@langwatch/mcp-server 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +280 -200
- package/CONTRIBUTING.md +10 -0
- package/README.md +82 -52
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/archive-run-plan-65QN3XRX.js +16 -0
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/archive-test-suite-NKZDZY3D.js +16 -0
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/chunk-6YZUOD34.js +55 -0
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-7RHG56OH.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-TOXJQDIZ.js → chunk-HGELNIVY.js} +135 -51
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-J42FWRJ2.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-AQJO2EI5.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-4QC7SA2L.js → chunk-J7PQKBLD.js} +884 -559
- package/dist/{chunk-YX3A3CFM.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-5CCZFUXS.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/chunk-O7EMWEVJ.js +44 -0
- package/dist/chunk-PTDR56Z3.js +37 -0
- package/dist/chunk-RABYBFX5.js +43 -0
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-SFB4MU6H.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/chunk-U24EACC4.js +67 -0
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-ZWIL4EFL.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/chunk-VMWJD732.js +57 -0
- package/dist/chunk-XUIJYNRJ.js +15 -0
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-7XHO6UIS.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-YQXRQC6A.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-YUS36KBU.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-E6NMSVX7.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -5
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-CS2CB57Y.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-F5RMX3L3.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-JDR2VB2H.js → create-scenario-UEM77OE7.js} +11 -4
- package/dist/create-test-suite-4IPXVSFQ.js +37 -0
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-O3EEH4S5.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/discover-scenario-schema-OOFIQKOG.js +139 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/get-agent-OTIYUFZA.js +70 -0
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-RAK7URQJ.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/get-prompt-OWXJZHQQ.js +95 -0
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/get-run-plan-BE3M2AUX.js +59 -0
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-YM4ODWW6.js} +8 -4
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-LCFSMPGQ.js} +33 -20
- package/dist/get-test-suite-GC2O7YH6.js +49 -0
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-K7U234Q4.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-L5L2PSK3.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-YXHFPPGA.js → http-server-IR4QOS43.js} +4071 -342
- package/dist/index.js +60221 -38077
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/list-agents-HVYWEQMI.js +43 -0
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-K7QAIX2I.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-SC2HAUFG.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/list-run-plans-PX2XI6EK.js +46 -0
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/list-scenarios-QZRPVHS4.js +46 -0
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-OEADLLQR.js} +19 -9
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/list-test-suites-N4WVLBKK.js +35 -0
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-QSJWURCK.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/rename-test-suite-4J7UX5S2.js +21 -0
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/report-issue-DCZFLGDO.js +899 -0
- package/dist/rerun-run-plan-TG2V4WM4.js +22 -0
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/run-plan-PAARQSI4.js +39 -0
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/run-test-suite-SQJELEZX.js +26 -0
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-QPXC627O.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/{run-agent-6SM4XNA7.js → run-workflow-SP32SO5M.js} +10 -11
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-SMAEHLXH.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-BDUK7TUO.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-LKVYXENH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/update-prompt-MY263DYF.js +141 -0
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-NEXCAQI7.js → update-scenario-MOHRQKAQ.js} +11 -4
- package/package.json +39 -44
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +1021 -596
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +202 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +140 -0
- package/src/langwatch-api-scenarios.ts +17 -10
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +10 -7
- package/src/langwatch-api-test-suites.ts +116 -0
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +143 -109
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +76 -0
- package/src/schemas/suite-fields.ts +132 -0
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +10 -0
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +10 -0
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +11 -5
- package/src/tools/create-test-suite.ts +38 -0
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +86 -12
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +54 -0
- package/src/tools/format-scenario.ts +12 -0
- package/src/tools/format-suite-details.ts +90 -0
- package/src/tools/get-agent.ts +54 -3
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +100 -37
- package/src/tools/get-run-plan.ts +60 -0
- package/src/tools/get-scenario.ts +6 -5
- package/src/tools/get-simulation-run.ts +32 -20
- package/src/tools/get-test-suite.ts +46 -0
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +26 -17
- package/src/tools/list-simulation-runs.ts +13 -5
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +15 -0
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +18 -0
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +65 -14
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +46 -0
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +27 -0
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +185 -15
- package/src/tools/update-scenario.ts +11 -5
- package/src/tools/update-test-suite.ts +40 -0
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +19 -12
- package/uv.lock +416 -296
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/archive-suite-4T6BSRS7.js +0 -16
- package/dist/chunk-77BNWQ3U.js +0 -86
- package/dist/chunk-A2LEVYQ5.js +0 -217
- package/dist/chunk-TJCXFNDT.js +0 -24
- package/dist/chunk-YZLYHLLI.js +0 -48
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/create-suite-LZS6QIAF.js +0 -39
- package/dist/delete-evaluator-33SP2JFX.js +0 -16
- package/dist/discover-evaluator-schema-XTAD4X7D.js +0 -2191
- package/dist/discover-scenario-schema-FE64QK2D.js +0 -67
- package/dist/get-agent-Y7MWZ65F.js +0 -29
- package/dist/get-analytics-TETCSOK6.js +0 -73
- package/dist/get-dataset-RCMU5VAH.js +0 -49
- package/dist/get-prompt-OY5B7TRI.js +0 -46
- package/dist/get-suite-J7JVWO6O.js +0 -45
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-BUBSNCNO.js +0 -41
- package/dist/langwatch-api-triggers-EDFUCJN6.js +0 -31
- package/dist/list-agents-KFHKGPXV.js +0 -33
- package/dist/list-scenarios-F2WYNRIE.js +0 -41
- package/dist/list-suites-CRY4VWW6.js +0 -43
- package/dist/report-issue-XYVAPAYJ.js +0 -412
- package/dist/run-suite-NKZCJ2IN.js +0 -30
- package/dist/search-traces-A6HFKPLJ.js +0 -79
- package/dist/update-prompt-ODCRQRN6.js +0 -25
- package/dist/update-suite-SNO3GS7T.js +0 -40
- package/src/__tests__/all-tools.integration.test.ts +0 -1626
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -288
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -188
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/tools.unit.test.ts +0 -924
- package/src/langwatch-api-suites.ts +0 -118
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -4
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
|
@@ -1,98 +1,50 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SLACK_DELIVERY_NOTE,
|
|
3
|
+
TRIGGER_FILTERS_DESCRIPTION,
|
|
4
|
+
TRIGGER_FILTER_QUERY_DESCRIPTION,
|
|
5
|
+
actionParamsSchema,
|
|
6
|
+
alertTypeSchema,
|
|
7
|
+
graphAlertSchema,
|
|
8
|
+
notificationCadenceSchema,
|
|
9
|
+
reportSchema,
|
|
10
|
+
templatesSchema,
|
|
11
|
+
triggerActionSchema,
|
|
12
|
+
validateActionParamsForAction
|
|
13
|
+
} from "./chunk-3YLN3ATJ.js";
|
|
14
|
+
import {
|
|
15
|
+
testAgent
|
|
16
|
+
} from "./chunk-HGELNIVY.js";
|
|
17
|
+
import {
|
|
18
|
+
runParametersSchema,
|
|
19
|
+
runPlanScopeSchema,
|
|
20
|
+
runPlanTargetSchema
|
|
21
|
+
} from "./chunk-O7EMWEVJ.js";
|
|
22
|
+
import {
|
|
23
|
+
evaluatorAttachmentsSchema,
|
|
24
|
+
scenarioFieldValuesSchema,
|
|
25
|
+
suiteFieldsSchema,
|
|
26
|
+
toWireAttachments
|
|
27
|
+
} from "./chunk-U24EACC4.js";
|
|
28
|
+
import {
|
|
29
|
+
formatEvaluatorAttachments,
|
|
30
|
+
formatSuiteFields
|
|
31
|
+
} from "./chunk-6YZUOD34.js";
|
|
32
|
+
import {
|
|
33
|
+
updateTestSuite
|
|
34
|
+
} from "./chunk-VMWJD732.js";
|
|
1
35
|
import {
|
|
2
36
|
LangWatchApiError,
|
|
3
|
-
makeRequest
|
|
4
|
-
|
|
37
|
+
makeRequest,
|
|
38
|
+
package_default
|
|
39
|
+
} from "./chunk-UWLE5T2F.js";
|
|
5
40
|
import {
|
|
6
41
|
requireApiKey
|
|
7
|
-
} from "./chunk-
|
|
42
|
+
} from "./chunk-H6WGMG2T.js";
|
|
8
43
|
|
|
9
44
|
// src/create-mcp-server.ts
|
|
10
45
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
11
46
|
import { z as z2 } from "zod";
|
|
12
47
|
|
|
13
|
-
// package.json
|
|
14
|
-
var package_default = {
|
|
15
|
-
name: "@langwatch/mcp-server",
|
|
16
|
-
version: "2.0.0",
|
|
17
|
-
description: "An MCP server for Langwatch.",
|
|
18
|
-
type: "module",
|
|
19
|
-
main: "./dist/index.js",
|
|
20
|
-
module: "./dist/index.js",
|
|
21
|
-
types: "./dist/index.d.ts",
|
|
22
|
-
exports: {
|
|
23
|
-
".": {
|
|
24
|
-
import: "./dist/index.js",
|
|
25
|
-
require: "./dist/index.js"
|
|
26
|
-
},
|
|
27
|
-
"./create-mcp-server": {
|
|
28
|
-
types: "./src/create-mcp-server.d.ts",
|
|
29
|
-
import: "./dist/create-mcp-server.js",
|
|
30
|
-
require: "./dist/create-mcp-server.js",
|
|
31
|
-
default: "./dist/create-mcp-server.js"
|
|
32
|
-
},
|
|
33
|
-
"./config": {
|
|
34
|
-
types: "./src/config.d.ts",
|
|
35
|
-
import: "./dist/config.js",
|
|
36
|
-
require: "./dist/config.js",
|
|
37
|
-
default: "./dist/config.js"
|
|
38
|
-
}
|
|
39
|
-
},
|
|
40
|
-
repository: {
|
|
41
|
-
type: "git",
|
|
42
|
-
url: "https://github.com/langwatch/langwatch.git",
|
|
43
|
-
directory: "mcp-server"
|
|
44
|
-
},
|
|
45
|
-
scripts: {
|
|
46
|
-
start: "tsx src/index.ts",
|
|
47
|
-
build: "tsup && node build.js",
|
|
48
|
-
prepublish: "pnpm run build",
|
|
49
|
-
test: "vitest"
|
|
50
|
-
},
|
|
51
|
-
author: "",
|
|
52
|
-
license: "MIT",
|
|
53
|
-
engines: {
|
|
54
|
-
node: ">=18.0.0"
|
|
55
|
-
},
|
|
56
|
-
devDependencies: {
|
|
57
|
-
"@ai-sdk/anthropic": "^3.0.44",
|
|
58
|
-
"@anthropic-ai/claude-code": "2.1.202",
|
|
59
|
-
"@eslint/js": "^10.0.1",
|
|
60
|
-
"@langwatch/handled-error": "workspace:*",
|
|
61
|
-
"@langwatch/redaction": "workspace:*",
|
|
62
|
-
"@langwatch/scenario": "^0.4.6",
|
|
63
|
-
"@langwatch/ssrf": "workspace:*",
|
|
64
|
-
"@opentelemetry/context-async-hooks": "^2.1.0",
|
|
65
|
-
"@opentelemetry/sdk-node": "^0.219.0",
|
|
66
|
-
"@types/debug": "^4.1.13",
|
|
67
|
-
"@types/express": "^5.0.6",
|
|
68
|
-
"@types/node": "^25.6.0",
|
|
69
|
-
"@types/yargs": "^17.0.33",
|
|
70
|
-
"@typescript/native-preview": "7.0.0-dev.20260705.1",
|
|
71
|
-
ai: "^6.0.161",
|
|
72
|
-
chalk: "^5.6.2",
|
|
73
|
-
"date-fns": "^4.1.0",
|
|
74
|
-
dotenv: "^17.2.2",
|
|
75
|
-
esbuild: "^0.28.1",
|
|
76
|
-
eslint: "^10.2.1",
|
|
77
|
-
express: "^5.2.1",
|
|
78
|
-
"node-pty": "^1.0.0",
|
|
79
|
-
tsup: "^8.1.0",
|
|
80
|
-
tsx: "^4.20.5",
|
|
81
|
-
typescript: "^5.0.0",
|
|
82
|
-
"typescript-eslint": "^8.55.0",
|
|
83
|
-
vitest: "^4.1.8",
|
|
84
|
-
yargs: "^18.0.0",
|
|
85
|
-
"zod-validation-error": "^5.0.0"
|
|
86
|
-
},
|
|
87
|
-
dependencies: {
|
|
88
|
-
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
89
|
-
zod: "^4.3.6"
|
|
90
|
-
},
|
|
91
|
-
bin: {
|
|
92
|
-
"langwatch-mcp-server": "./dist/index.js"
|
|
93
|
-
}
|
|
94
|
-
};
|
|
95
|
-
|
|
96
48
|
// src/documentation-fetch.ts
|
|
97
49
|
var TRUSTED_DOCUMENTATION_ORIGIN = "https://langwatch.ai";
|
|
98
50
|
var MAX_DOCUMENTATION_BYTES = 2 * 1024 * 1024;
|
|
@@ -131,7 +83,8 @@ function resolveDocumentationUrl(kind, input) {
|
|
|
131
83
|
} catch {
|
|
132
84
|
throw documentationUrlError(namespace);
|
|
133
85
|
}
|
|
134
|
-
|
|
86
|
+
const isTrustedDocumentationUrl = url.protocol === "https:" && url.hostname === "langwatch.ai" && url.port === "" && url.username === "" && url.password === "" && url.pathname.startsWith(`${namespace}/`);
|
|
87
|
+
if (!isTrustedDocumentationUrl) {
|
|
135
88
|
throw documentationUrlError(namespace);
|
|
136
89
|
}
|
|
137
90
|
const lowerPath = url.pathname.toLowerCase();
|
|
@@ -192,7 +145,8 @@ var DATASET_COLUMN_TYPES = [
|
|
|
192
145
|
"chat_messages",
|
|
193
146
|
"annotations",
|
|
194
147
|
"evaluations",
|
|
195
|
-
"image"
|
|
148
|
+
"image",
|
|
149
|
+
"file"
|
|
196
150
|
];
|
|
197
151
|
var datasetColumnTypeSchema = z.enum(DATASET_COLUMN_TYPES);
|
|
198
152
|
var datasetColumnDefinitionSchema = z.object({
|
|
@@ -234,78 +188,45 @@ var isFailedRow = ({
|
|
|
234
188
|
entry,
|
|
235
189
|
evaluations
|
|
236
190
|
}) => Boolean(entry.error) || evaluations.some(isFailedEvaluation);
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
191
|
+
function hasNumericStatus(error) {
|
|
192
|
+
if (typeof error !== "object" || error === null) return false;
|
|
193
|
+
return "status" in error && typeof error.status === "number";
|
|
194
|
+
}
|
|
195
|
+
function isNotFoundError(error) {
|
|
196
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
197
|
+
let status;
|
|
198
|
+
if (error instanceof LangWatchApiError) status = error.status;
|
|
199
|
+
else if (hasNumericStatus(error)) status = error.status;
|
|
200
|
+
return status === 404 || status === void 0 && /404|not found/i.test(message);
|
|
201
|
+
}
|
|
202
|
+
async function fetchRunResults(params) {
|
|
241
203
|
const search = new URLSearchParams();
|
|
242
204
|
if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
|
|
243
205
|
const qs = search.toString() ? `?${search.toString()}` : "";
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
results
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
`# Evaluation Results: ${params.runId}`,
|
|
256
|
-
"",
|
|
257
|
-
"**Status**: not found",
|
|
258
|
-
"",
|
|
259
|
-
`Could not load results for run \`${params.runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
|
|
260
|
-
"",
|
|
261
|
-
"> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids."
|
|
262
|
-
].join("\n");
|
|
263
|
-
}
|
|
264
|
-
throw error;
|
|
265
|
-
}
|
|
266
|
-
if (!results) {
|
|
267
|
-
return [
|
|
268
|
-
`# Evaluation Results: ${params.runId}`,
|
|
269
|
-
"",
|
|
270
|
-
"**Status**: not found",
|
|
271
|
-
"",
|
|
272
|
-
`Could not load results for run \`${params.runId}\`.`,
|
|
273
|
-
"",
|
|
274
|
-
"> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id."
|
|
275
|
-
].join("\n");
|
|
276
|
-
}
|
|
277
|
-
const runStatus2 = deriveRunStatus(results.timestamps);
|
|
278
|
-
const partial = !isTerminalStatus(runStatus2);
|
|
279
|
-
const evaluationsByRow = /* @__PURE__ */ new Map();
|
|
280
|
-
for (const evaluation of results.evaluations) {
|
|
206
|
+
return await makeRequest(
|
|
207
|
+
"GET",
|
|
208
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
function groupEvaluationsByRow({
|
|
212
|
+
evaluations,
|
|
213
|
+
evaluatorFilter
|
|
214
|
+
}) {
|
|
215
|
+
const byRow = /* @__PURE__ */ new Map();
|
|
216
|
+
for (const evaluation of evaluations) {
|
|
281
217
|
if (evaluatorFilter && evaluation.evaluator !== evaluatorFilter) continue;
|
|
282
218
|
const key = rowKey(evaluation.index, evaluation.targetId);
|
|
283
|
-
const list =
|
|
219
|
+
const list = byRow.get(key) ?? [];
|
|
284
220
|
list.push(evaluation);
|
|
285
|
-
|
|
286
|
-
}
|
|
287
|
-
const evaluatorNames = Array.from(
|
|
288
|
-
new Set(
|
|
289
|
-
(evaluatorFilter ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter) : results.evaluations).map((e) => e.evaluator)
|
|
290
|
-
)
|
|
291
|
-
);
|
|
292
|
-
let rows = results.dataset.map((entry) => ({
|
|
293
|
-
entry,
|
|
294
|
-
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? []
|
|
295
|
-
}));
|
|
296
|
-
if (filter === "failed") {
|
|
297
|
-
rows = rows.filter(
|
|
298
|
-
(r) => isFailedRow({ entry: r.entry, evaluations: r.evaluations })
|
|
299
|
-
);
|
|
221
|
+
byRow.set(key, list);
|
|
300
222
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const
|
|
306
|
-
for (const r of rowsForSummary) {
|
|
223
|
+
return byRow;
|
|
224
|
+
}
|
|
225
|
+
function tallyEvaluatorStats(rows) {
|
|
226
|
+
const averages = /* @__PURE__ */ new Map();
|
|
227
|
+
for (const r of rows) {
|
|
307
228
|
for (const e of r.evaluations) {
|
|
308
|
-
const stats =
|
|
229
|
+
const stats = averages.get(e.evaluator) ?? {
|
|
309
230
|
sum: 0,
|
|
310
231
|
count: 0,
|
|
311
232
|
passed: 0,
|
|
@@ -319,97 +240,153 @@ async function handleExperimentResults(params) {
|
|
|
319
240
|
if (e.status === "error") stats.errored += 1;
|
|
320
241
|
else if (e.passed === true) stats.passed += 1;
|
|
321
242
|
else if (e.passed === false) stats.failed += 1;
|
|
322
|
-
|
|
243
|
+
averages.set(e.evaluator, stats);
|
|
323
244
|
}
|
|
324
245
|
}
|
|
246
|
+
return averages;
|
|
247
|
+
}
|
|
248
|
+
function headerLines({
|
|
249
|
+
results,
|
|
250
|
+
runStatus: runStatus2,
|
|
251
|
+
filter,
|
|
252
|
+
evaluatorFilter
|
|
253
|
+
}) {
|
|
325
254
|
const lines = [];
|
|
326
255
|
lines.push(`# Evaluation Results: ${results.runId}`);
|
|
327
256
|
lines.push("");
|
|
328
257
|
lines.push(`**Experiment**: ${results.experimentId}`);
|
|
329
258
|
lines.push(`**Status**: ${runStatus2}`);
|
|
330
259
|
if (typeof results.total === "number" && results.total > 0) {
|
|
331
|
-
lines.push(
|
|
332
|
-
`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`
|
|
333
|
-
);
|
|
260
|
+
lines.push(`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`);
|
|
334
261
|
}
|
|
335
262
|
lines.push(`**Total rows**: ${results.dataset.length}`);
|
|
336
263
|
lines.push(`**Total evaluations**: ${results.evaluations.length}`);
|
|
337
|
-
if (filter === "failed")
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
if (evaluatorFilter) {
|
|
341
|
-
lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
342
|
-
}
|
|
343
|
-
if (partial) {
|
|
264
|
+
if (filter === "failed") lines.push(`**Filter**: failed only`);
|
|
265
|
+
if (evaluatorFilter) lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
266
|
+
if (!isTerminalStatus(runStatus2)) {
|
|
344
267
|
lines.push("");
|
|
345
268
|
lines.push(
|
|
346
269
|
runStatus2 === "interrupted" ? "> These are partial results. The run never sent a finished/stopped marker and has had no updates recently, so it likely was interrupted before completing. The rows below are everything recorded so far." : "> These are partial results. The run is still in progress, so more rows may appear on a later call."
|
|
347
270
|
);
|
|
348
271
|
}
|
|
349
|
-
|
|
350
|
-
if (
|
|
351
|
-
lines.push("## Evaluator Summary");
|
|
352
|
-
lines.push("");
|
|
353
|
-
lines.push("| Evaluator | Avg Score | Passed | Failed | Errored |");
|
|
354
|
-
lines.push("| --- | --- | --- | --- | --- |");
|
|
355
|
-
for (const [name, stats] of evaluatorAverages) {
|
|
356
|
-
const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "\u2014";
|
|
357
|
-
lines.push(
|
|
358
|
-
`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`
|
|
359
|
-
);
|
|
360
|
-
}
|
|
272
|
+
const stored = results.completeness;
|
|
273
|
+
if (isTerminalStatus(runStatus2) && stored && !stored.complete) {
|
|
361
274
|
lines.push("");
|
|
275
|
+
lines.push(
|
|
276
|
+
`> These are partial results. The run has ended, but the platform is still storing it: ${storedCount(stored.dataset)} rows and ${storedCount(stored.evaluations)} evaluations so far. Call again shortly for the whole run.`
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
lines.push("");
|
|
280
|
+
return lines;
|
|
281
|
+
}
|
|
282
|
+
function evaluatorSummaryLines(averages) {
|
|
283
|
+
if (averages.size === 0) return [];
|
|
284
|
+
const lines = [
|
|
285
|
+
"## Evaluator Summary",
|
|
286
|
+
"",
|
|
287
|
+
"| Evaluator | Avg Score | Passed | Failed | Errored |",
|
|
288
|
+
"| --- | --- | --- | --- | --- |"
|
|
289
|
+
];
|
|
290
|
+
for (const [name, stats] of averages) {
|
|
291
|
+
const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "\u2014";
|
|
292
|
+
lines.push(`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`);
|
|
293
|
+
}
|
|
294
|
+
lines.push("");
|
|
295
|
+
return lines;
|
|
296
|
+
}
|
|
297
|
+
var storedCount = ({ received, expected }) => expected === null ? `${received}` : `${received} of ${expected}`;
|
|
298
|
+
function noRowsLine({ filter, runStatus: runStatus2 }) {
|
|
299
|
+
if (filter === "failed") return "_No rows matched the filter._";
|
|
300
|
+
if (runStatus2 === "running")
|
|
301
|
+
return "_No rows recorded yet. The run is still in progress; call again shortly._";
|
|
302
|
+
if (runStatus2 === "interrupted") return "_No rows were recorded before the run was interrupted._";
|
|
303
|
+
return "_No rows recorded for this run._";
|
|
304
|
+
}
|
|
305
|
+
function evaluationLines(e) {
|
|
306
|
+
const parts = [`**${e.evaluator}**`];
|
|
307
|
+
if (e.status === "error") parts.push("status=error");
|
|
308
|
+
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
309
|
+
if (typeof e.passed === "boolean") parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
310
|
+
if (e.label) parts.push(`label=${e.label}`);
|
|
311
|
+
const lines = [`- ${parts.join(" \xB7 ")}`];
|
|
312
|
+
if (e.details) lines.push(` - details: ${e.details}`);
|
|
313
|
+
return lines;
|
|
314
|
+
}
|
|
315
|
+
function rowLines({
|
|
316
|
+
row: { entry, evaluations },
|
|
317
|
+
anyEvaluators
|
|
318
|
+
}) {
|
|
319
|
+
const summary = summarizeEntry(entry.entry);
|
|
320
|
+
const lines = [`### Row #${entry.index}${summary ? ` \u2014 ${summary}` : ""}`];
|
|
321
|
+
if (entry.error) lines.push(`- **Error**: ${entry.error}`);
|
|
322
|
+
if (entry.traceId) lines.push(`- **Trace ID**: \`${entry.traceId}\``);
|
|
323
|
+
if (evaluations.length === 0) {
|
|
324
|
+
lines.push(
|
|
325
|
+
anyEvaluators ? "- _No evaluations recorded for this row_" : "- _No evaluations recorded_"
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
for (const e of evaluations) lines.push(...evaluationLines(e));
|
|
329
|
+
lines.push("");
|
|
330
|
+
return lines;
|
|
331
|
+
}
|
|
332
|
+
function notFoundReport({ runId, expired }) {
|
|
333
|
+
const lines = [`# Evaluation Results: ${runId}`, "", "**Status**: not found", ""];
|
|
334
|
+
if (expired) {
|
|
335
|
+
lines.push(
|
|
336
|
+
`Could not load results for run \`${runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
|
|
337
|
+
"",
|
|
338
|
+
"> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids."
|
|
339
|
+
);
|
|
340
|
+
} else {
|
|
341
|
+
lines.push(
|
|
342
|
+
`Could not load results for run \`${runId}\`.`,
|
|
343
|
+
"",
|
|
344
|
+
"> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id."
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
return lines.join("\n");
|
|
348
|
+
}
|
|
349
|
+
async function handleExperimentResults(params) {
|
|
350
|
+
const filter = params.filter ?? "all";
|
|
351
|
+
const evaluatorFilter = params.evaluator?.trim();
|
|
352
|
+
const limit = typeof params.limit === "number" && params.limit > 0 ? Math.min(params.limit, DEFAULT_ROW_CAP) : DEFAULT_ROW_CAP;
|
|
353
|
+
let results;
|
|
354
|
+
try {
|
|
355
|
+
results = await fetchRunResults(params);
|
|
356
|
+
} catch (error) {
|
|
357
|
+
if (isNotFoundError(error)) return notFoundReport({ runId: params.runId, expired: true });
|
|
358
|
+
throw error;
|
|
362
359
|
}
|
|
360
|
+
if (!results) return notFoundReport({ runId: params.runId, expired: false });
|
|
361
|
+
const runStatus2 = deriveRunStatus(results.timestamps);
|
|
362
|
+
const evaluationsByRow = groupEvaluationsByRow({
|
|
363
|
+
evaluations: results.evaluations,
|
|
364
|
+
evaluatorFilter
|
|
365
|
+
});
|
|
366
|
+
const evaluatorNames = new Set(
|
|
367
|
+
(evaluatorFilter ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter) : results.evaluations).map((e) => e.evaluator)
|
|
368
|
+
);
|
|
369
|
+
const allRows = results.dataset.map((entry) => ({
|
|
370
|
+
entry,
|
|
371
|
+
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? []
|
|
372
|
+
}));
|
|
373
|
+
const matching = filter === "failed" ? allRows.filter((r) => isFailedRow(r)) : allRows;
|
|
374
|
+
const truncated = matching.length > limit;
|
|
375
|
+
const rows = matching.slice(0, limit);
|
|
376
|
+
const lines = [
|
|
377
|
+
...headerLines({ results, runStatus: runStatus2, filter, evaluatorFilter }),
|
|
378
|
+
...evaluatorSummaryLines(tallyEvaluatorStats(matching))
|
|
379
|
+
];
|
|
363
380
|
if (rows.length === 0) {
|
|
364
|
-
|
|
365
|
-
lines.push("_No rows matched the filter._");
|
|
366
|
-
} else if (runStatus2 === "running") {
|
|
367
|
-
lines.push(
|
|
368
|
-
"_No rows recorded yet. The run is still in progress; call again shortly._"
|
|
369
|
-
);
|
|
370
|
-
} else if (runStatus2 === "interrupted") {
|
|
371
|
-
lines.push("_No rows were recorded before the run was interrupted._");
|
|
372
|
-
} else {
|
|
373
|
-
lines.push("_No rows recorded for this run._");
|
|
374
|
-
}
|
|
381
|
+
lines.push(noRowsLine({ filter, runStatus: runStatus2 }));
|
|
375
382
|
return lines.join("\n");
|
|
376
383
|
}
|
|
377
|
-
lines.push(
|
|
378
|
-
`## Rows (${rows.length}${truncated ? ` of ${totalMatching}` : ""})`
|
|
379
|
-
);
|
|
384
|
+
lines.push(`## Rows (${rows.length}${truncated ? ` of ${matching.length}` : ""})`);
|
|
380
385
|
lines.push("");
|
|
381
|
-
for (const {
|
|
382
|
-
const summary = summarizeEntry(entry.entry);
|
|
383
|
-
lines.push(`### Row #${entry.index}${summary ? ` \u2014 ${summary}` : ""}`);
|
|
384
|
-
if (entry.error) {
|
|
385
|
-
lines.push(`- **Error**: ${entry.error}`);
|
|
386
|
-
}
|
|
387
|
-
if (entry.traceId) {
|
|
388
|
-
lines.push(`- **Trace ID**: \`${entry.traceId}\``);
|
|
389
|
-
}
|
|
390
|
-
if (evaluations.length === 0) {
|
|
391
|
-
lines.push(
|
|
392
|
-
evaluatorNames.length === 0 ? "- _No evaluations recorded_" : "- _No evaluations recorded for this row_"
|
|
393
|
-
);
|
|
394
|
-
}
|
|
395
|
-
for (const e of evaluations) {
|
|
396
|
-
const parts = [`**${e.evaluator}**`];
|
|
397
|
-
if (e.status === "error") parts.push("status=error");
|
|
398
|
-
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
399
|
-
if (typeof e.passed === "boolean") {
|
|
400
|
-
parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
401
|
-
}
|
|
402
|
-
if (e.label) parts.push(`label=${e.label}`);
|
|
403
|
-
lines.push(`- ${parts.join(" \xB7 ")}`);
|
|
404
|
-
if (e.details) {
|
|
405
|
-
lines.push(` - details: ${e.details}`);
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
lines.push("");
|
|
409
|
-
}
|
|
386
|
+
for (const row of rows) lines.push(...rowLines({ row, anyEvaluators: evaluatorNames.size > 0 }));
|
|
410
387
|
if (truncated) {
|
|
411
388
|
lines.push(
|
|
412
|
-
`> Output truncated to ${limit} rows of ${
|
|
389
|
+
`> Output truncated to ${limit} rows of ${matching.length} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`
|
|
413
390
|
);
|
|
414
391
|
}
|
|
415
392
|
return lines.join("\n");
|
|
@@ -450,7 +427,7 @@ async function handleExperimentListRuns(params) {
|
|
|
450
427
|
try {
|
|
451
428
|
result = await makeRequest(
|
|
452
429
|
"GET",
|
|
453
|
-
`/api/experiments/runs?${search.toString()}`
|
|
430
|
+
`/api/v1/experiments/runs?${search.toString()}`
|
|
454
431
|
);
|
|
455
432
|
} catch (error) {
|
|
456
433
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -520,7 +497,7 @@ async function handleExperimentList(params) {
|
|
|
520
497
|
search.set("pageSize", String(effectiveLimit));
|
|
521
498
|
const result = await makeRequest(
|
|
522
499
|
"GET",
|
|
523
|
-
`/api/experiments?${search.toString()}`
|
|
500
|
+
`/api/v1/experiments?${search.toString()}`
|
|
524
501
|
);
|
|
525
502
|
if (result.experiments.length === 0) {
|
|
526
503
|
return [
|
|
@@ -552,9 +529,7 @@ async function handleExperimentList(params) {
|
|
|
552
529
|
);
|
|
553
530
|
}
|
|
554
531
|
lines.push("");
|
|
555
|
-
lines.push(
|
|
556
|
-
"> Use `platform_experiment_list_runs` with one of these slugs to see its runs."
|
|
557
|
-
);
|
|
532
|
+
lines.push("> Use `platform_experiment_list_runs` with one of these slugs to see its runs.");
|
|
558
533
|
return lines.join("\n");
|
|
559
534
|
}
|
|
560
535
|
|
|
@@ -562,7 +537,7 @@ async function handleExperimentList(params) {
|
|
|
562
537
|
async function handleRunExperiment(params) {
|
|
563
538
|
const result = await makeRequest(
|
|
564
539
|
"POST",
|
|
565
|
-
`/api/experiments/${encodeURIComponent(params.slug)}/run`
|
|
540
|
+
`/api/v1/experiments/${encodeURIComponent(params.slug)}/run`
|
|
566
541
|
);
|
|
567
542
|
const lines = [];
|
|
568
543
|
lines.push(`Evaluation started!
|
|
@@ -585,7 +560,7 @@ async function statusFromResults(params) {
|
|
|
585
560
|
try {
|
|
586
561
|
results = await makeRequest(
|
|
587
562
|
"GET",
|
|
588
|
-
`/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
563
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
589
564
|
);
|
|
590
565
|
} catch (error) {
|
|
591
566
|
const code = error instanceof LangWatchApiError ? error.status : void 0;
|
|
@@ -604,19 +579,13 @@ async function statusFromResults(params) {
|
|
|
604
579
|
lines.push(`**Status**: ${status}`);
|
|
605
580
|
lines.push(`**Progress**: ${progress}/${total} cells`);
|
|
606
581
|
if (results.timestamps.createdAt) {
|
|
607
|
-
lines.push(
|
|
608
|
-
`**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`
|
|
609
|
-
);
|
|
582
|
+
lines.push(`**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`);
|
|
610
583
|
}
|
|
611
584
|
if (results.timestamps.finishedAt) {
|
|
612
|
-
lines.push(
|
|
613
|
-
`**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`
|
|
614
|
-
);
|
|
585
|
+
lines.push(`**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`);
|
|
615
586
|
}
|
|
616
587
|
if (results.timestamps.stoppedAt) {
|
|
617
|
-
lines.push(
|
|
618
|
-
`**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`
|
|
619
|
-
);
|
|
588
|
+
lines.push(`**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`);
|
|
620
589
|
}
|
|
621
590
|
lines.push("");
|
|
622
591
|
lines.push(
|
|
@@ -624,30 +593,36 @@ async function statusFromResults(params) {
|
|
|
624
593
|
);
|
|
625
594
|
return lines.join("\n");
|
|
626
595
|
}
|
|
596
|
+
async function statusAfterFailedRead({
|
|
597
|
+
params,
|
|
598
|
+
error
|
|
599
|
+
}) {
|
|
600
|
+
const code = error instanceof LangWatchApiError ? error.status : void 0;
|
|
601
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
602
|
+
if (code === 404 || /404|not found/i.test(message)) {
|
|
603
|
+
const fallback = await statusFromResults(params);
|
|
604
|
+
if (fallback) return fallback;
|
|
605
|
+
return [
|
|
606
|
+
`# Evaluation Run ${params.runId}`,
|
|
607
|
+
"",
|
|
608
|
+
"**Status**: not found",
|
|
609
|
+
"",
|
|
610
|
+
`Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
|
|
611
|
+
"",
|
|
612
|
+
"> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`."
|
|
613
|
+
].join("\n");
|
|
614
|
+
}
|
|
615
|
+
throw error;
|
|
616
|
+
}
|
|
627
617
|
async function handleExperimentStatus(params) {
|
|
628
618
|
let status;
|
|
629
619
|
try {
|
|
630
620
|
status = await makeRequest(
|
|
631
621
|
"GET",
|
|
632
|
-
`/api/experiments/runs/${encodeURIComponent(params.runId)}`
|
|
622
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}`
|
|
633
623
|
);
|
|
634
624
|
} catch (error) {
|
|
635
|
-
|
|
636
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
637
|
-
if (code === 404 || /404|not found/i.test(message)) {
|
|
638
|
-
const fallback = await statusFromResults(params);
|
|
639
|
-
if (fallback) return fallback;
|
|
640
|
-
return [
|
|
641
|
-
`# Evaluation Run ${params.runId}`,
|
|
642
|
-
"",
|
|
643
|
-
"**Status**: not found",
|
|
644
|
-
"",
|
|
645
|
-
`Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
|
|
646
|
-
"",
|
|
647
|
-
"> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`."
|
|
648
|
-
].join("\n");
|
|
649
|
-
}
|
|
650
|
-
throw error;
|
|
625
|
+
return statusAfterFailedRead({ params, error });
|
|
651
626
|
}
|
|
652
627
|
const lines = [];
|
|
653
628
|
lines.push(`# Evaluation Run ${status.runId}
|
|
@@ -669,9 +644,7 @@ async function handleExperimentStatus(params) {
|
|
|
669
644
|
lines.push(`**Failed**: ${status.summary.failedCells}`);
|
|
670
645
|
}
|
|
671
646
|
if (status.summary.duration) {
|
|
672
|
-
lines.push(
|
|
673
|
-
`**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`
|
|
674
|
-
);
|
|
647
|
+
lines.push(`**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`);
|
|
675
648
|
}
|
|
676
649
|
if (status.summary.runUrl) {
|
|
677
650
|
lines.push(`**View results**: ${status.summary.runUrl}`);
|
|
@@ -680,6 +653,40 @@ async function handleExperimentStatus(params) {
|
|
|
680
653
|
return lines.join("\n");
|
|
681
654
|
}
|
|
682
655
|
|
|
656
|
+
// src/tools/test-agent.ts
|
|
657
|
+
async function handleTestAgent(params) {
|
|
658
|
+
const run = await testAgent(params.id);
|
|
659
|
+
return [
|
|
660
|
+
'Test run scheduled. The user sends "ping"; the run succeeds when the agent answers.',
|
|
661
|
+
"",
|
|
662
|
+
`**Scenario run ID:** ${run.scenarioRunId}`,
|
|
663
|
+
`**Batch run ID:** ${run.batchRunId}`,
|
|
664
|
+
"",
|
|
665
|
+
`Follow it with platform_get_simulation_run using scenarioRunId "${run.scenarioRunId}".`
|
|
666
|
+
].join("\n");
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// src/tools/update-test-suite.ts
|
|
670
|
+
async function handleUpdateTestSuite(params) {
|
|
671
|
+
const suite = await updateTestSuite({
|
|
672
|
+
id: params.id,
|
|
673
|
+
...params.name !== void 0 ? { name: params.name } : {},
|
|
674
|
+
...params.fields !== void 0 ? { fields: params.fields } : {},
|
|
675
|
+
...params.evaluators !== void 0 ? { evaluators: toWireAttachments(params.evaluators) } : {}
|
|
676
|
+
});
|
|
677
|
+
return [
|
|
678
|
+
`Test suite "${suite.name}" updated.`,
|
|
679
|
+
"",
|
|
680
|
+
`**ID**: ${suite.id}`,
|
|
681
|
+
`**Slug**: ${suite.slug}`,
|
|
682
|
+
`**Scenarios**: ${suite.scenarioCount}`,
|
|
683
|
+
...formatSuiteFields(suite.fields),
|
|
684
|
+
...formatEvaluatorAttachments(suite.evaluators),
|
|
685
|
+
"",
|
|
686
|
+
`**View**: ${suite.platformUrl}`
|
|
687
|
+
].join("\n");
|
|
688
|
+
}
|
|
689
|
+
|
|
683
690
|
// src/create-mcp-server.ts
|
|
684
691
|
var modelSchema = z2.string().describe(
|
|
685
692
|
'Model in "provider/model-name" format, e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929"'
|
|
@@ -707,6 +714,28 @@ function withToolLogging(toolName, fn) {
|
|
|
707
714
|
};
|
|
708
715
|
}
|
|
709
716
|
function registerTools(server) {
|
|
717
|
+
registerDocsTools(server);
|
|
718
|
+
registerObservabilityTools(server);
|
|
719
|
+
registerPromptTools(server);
|
|
720
|
+
registerScenarioTools(server);
|
|
721
|
+
registerRunPlanTools(server);
|
|
722
|
+
registerTestSuiteTools(server);
|
|
723
|
+
registerSimulationRunTools(server);
|
|
724
|
+
registerEvaluatorTools(server);
|
|
725
|
+
registerModelProviderTools(server);
|
|
726
|
+
registerAgentTools(server);
|
|
727
|
+
registerDashboardTools(server);
|
|
728
|
+
registerWorkflowTools(server);
|
|
729
|
+
registerAnnotationTools(server);
|
|
730
|
+
registerTriggerTools(server);
|
|
731
|
+
registerMonitorTools(server);
|
|
732
|
+
registerSecretTools(server);
|
|
733
|
+
registerExperimentTools(server);
|
|
734
|
+
registerDatasetTools(server);
|
|
735
|
+
registerProjectTools(server);
|
|
736
|
+
registerApiKeyTools(server);
|
|
737
|
+
}
|
|
738
|
+
function registerDocsTools(server) {
|
|
710
739
|
server.tool(
|
|
711
740
|
"fetch_langwatch_docs",
|
|
712
741
|
"Fetches the LangWatch docs for understanding how to implement LangWatch in your codebase. Always use this tool when the user asks for help with LangWatch. Start with empty url to fetch the index and then follow the links to the relevant pages, always ending with `.md` extension",
|
|
@@ -738,7 +767,7 @@ function registerTools(server) {
|
|
|
738
767
|
agent: z2.string().optional().describe("Which coding agent this is, e.g. claude-code, codex, cursor")
|
|
739
768
|
},
|
|
740
769
|
withToolLogging("report_issue", async (params) => {
|
|
741
|
-
const { handleReportIssue } = await import("./report-issue-
|
|
770
|
+
const { handleReportIssue } = await import("./report-issue-DCZFLGDO.js");
|
|
742
771
|
return {
|
|
743
772
|
content: [{ type: "text", text: await handleReportIssue(params) }]
|
|
744
773
|
};
|
|
@@ -757,70 +786,99 @@ function registerTools(server) {
|
|
|
757
786
|
return { content: [{ type: "text", text }] };
|
|
758
787
|
})
|
|
759
788
|
);
|
|
789
|
+
}
|
|
790
|
+
function registerObservabilityTools(server) {
|
|
760
791
|
server.tool(
|
|
761
792
|
"discover_schema",
|
|
762
|
-
"Discover
|
|
793
|
+
"Discover what LangWatch can be queried with: the trace filter fields and syntax, the analytics SQL views and columns, the analytics metrics, aggregation types, group-by options, scenario schema and evaluator types. Call this before using search_traces, run_query, get_analytics, scenario tools or evaluator tools, so you never guess a field, a column or a value.",
|
|
763
794
|
{
|
|
764
795
|
category: z2.enum([
|
|
765
796
|
"filters",
|
|
797
|
+
"lwql",
|
|
766
798
|
"metrics",
|
|
767
799
|
"aggregations",
|
|
768
800
|
"groups",
|
|
769
801
|
"scenarios",
|
|
770
802
|
"evaluators",
|
|
771
803
|
"all"
|
|
772
|
-
]).describe(
|
|
804
|
+
]).describe(
|
|
805
|
+
"Which schema category to discover. 'filters' is the trace filter language, 'lwql' the analytics SQL views; both are read from the platform and need the API key"
|
|
806
|
+
),
|
|
773
807
|
evaluatorType: z2.string().optional().describe(
|
|
774
808
|
"When category is 'evaluators', provide a specific evaluator type (e.g. 'langevals/llm_boolean') to get its full schema details"
|
|
775
809
|
)
|
|
776
810
|
},
|
|
777
|
-
async ({ category, evaluatorType }) => {
|
|
811
|
+
withToolLogging("discover_schema", async ({ category, evaluatorType }) => {
|
|
778
812
|
if (category === "scenarios") {
|
|
779
|
-
const { formatScenarioSchema } = await import("./discover-scenario-schema-
|
|
813
|
+
const { formatScenarioSchema } = await import("./discover-scenario-schema-OOFIQKOG.js");
|
|
780
814
|
return {
|
|
781
815
|
content: [{ type: "text", text: formatScenarioSchema() }]
|
|
782
816
|
};
|
|
783
817
|
}
|
|
784
818
|
if (category === "evaluators") {
|
|
785
|
-
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-
|
|
819
|
+
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-UHWA5HVL.js");
|
|
786
820
|
return {
|
|
787
|
-
content: [
|
|
788
|
-
{ type: "text", text: formatEvaluatorSchema(evaluatorType) }
|
|
789
|
-
]
|
|
821
|
+
content: [{ type: "text", text: formatEvaluatorSchema(evaluatorType) }]
|
|
790
822
|
};
|
|
791
823
|
}
|
|
792
|
-
const { formatSchema } = await import("./discover-schema-
|
|
793
|
-
|
|
824
|
+
const { formatSchema, needsQueryReference } = await import("./discover-schema-55YJBFGQ.js");
|
|
825
|
+
if (needsQueryReference(category)) requireApiKey();
|
|
826
|
+
let text = await formatSchema(category);
|
|
794
827
|
if (category === "all") {
|
|
795
|
-
const { formatScenarioSchema } = await import("./discover-scenario-schema-
|
|
828
|
+
const { formatScenarioSchema } = await import("./discover-scenario-schema-OOFIQKOG.js");
|
|
796
829
|
text += "\n\n" + formatScenarioSchema();
|
|
797
|
-
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-
|
|
830
|
+
const { formatEvaluatorSchema } = await import("./discover-evaluator-schema-UHWA5HVL.js");
|
|
798
831
|
text += "\n\n" + formatEvaluatorSchema();
|
|
799
832
|
}
|
|
800
833
|
return { content: [{ type: "text", text }] };
|
|
801
|
-
}
|
|
834
|
+
})
|
|
835
|
+
);
|
|
836
|
+
server.tool(
|
|
837
|
+
"run_query",
|
|
838
|
+
"Run one read-only analytics SQL statement (LangWatchQL) over this project's traces, spans, evaluations and metrics, and get the rows back as a table. This is the tool for a count, a rate, a sum, a percentile or any of them grouped or over time \u2014 questions search_traces would need many calls to answer. The statement runs exactly as written. Call discover_schema with category 'lwql' first for the views, columns and worked statements.",
|
|
839
|
+
{
|
|
840
|
+
sql: z2.string().describe(
|
|
841
|
+
"The SELECT to run. Filter on the dataset's time column so the read is bounded, and never select a whole attribute map"
|
|
842
|
+
),
|
|
843
|
+
parameters: z2.record(z2.string(), z2.union([z2.string(), z2.number(), z2.boolean(), z2.null()])).optional().describe(
|
|
844
|
+
'Values for the parameters the statement declares, e.g. {"days": 7} for {days:UInt32}'
|
|
845
|
+
)
|
|
846
|
+
},
|
|
847
|
+
withToolLogging("run_query", async (params) => {
|
|
848
|
+
requireApiKey();
|
|
849
|
+
const { handleRunQuery } = await import("./run-query-KJDH544C.js");
|
|
850
|
+
return {
|
|
851
|
+
content: [{ type: "text", text: await handleRunQuery(params) }]
|
|
852
|
+
};
|
|
853
|
+
})
|
|
802
854
|
);
|
|
803
855
|
server.tool(
|
|
804
856
|
"search_traces",
|
|
805
|
-
"Search LangWatch traces
|
|
857
|
+
"Search LangWatch traces by content, with filters and a date range. Returns AI-readable trace digests by default. Use format: 'json' for full raw data. To fetch a trace you already have the ID for, use get_trace instead \u2014 free text does not match trace IDs.",
|
|
806
858
|
{
|
|
807
|
-
query: z2.string().optional().describe(
|
|
859
|
+
query: z2.string().optional().describe(
|
|
860
|
+
"Text search query. Matches captured input/output, the trace name and span names ONLY. It does not match trace IDs \u2014 to fetch a known trace ID use get_trace, or pass traceIds here for several at once."
|
|
861
|
+
),
|
|
862
|
+
traceIds: z2.array(z2.string().min(1)).min(1).max(1e3).optional().describe(
|
|
863
|
+
"Exact trace IDs to fetch (1\u20131000). Use instead of query when you already know the IDs. When set and no startDate is given, the window defaults to the last 90 days rather than 24 hours."
|
|
864
|
+
),
|
|
808
865
|
filters: z2.record(z2.string(), z2.array(z2.string())).optional().describe(
|
|
809
|
-
'Filter traces. Format: {"field": ["value"]}. Use discover_schema for field names.'
|
|
866
|
+
'Filter traces by the older per-field map. Format: {"field": ["value"]}. Use discover_schema for field names.'
|
|
867
|
+
),
|
|
868
|
+
filter: z2.string().optional().describe(
|
|
869
|
+
'Filter traces with the Trace Explorer\'s own query language: "status:error AND model:gpt-*", "trace.attribute.langwatch.user_id:alice", "evaluatorVerdict:fail", a quoted phrase for free text. Reaches attribute keys, span events and evaluator verdicts that the `filters` map cannot. Combined with `filters` and `query` when you send more than one. Call discover_schema with category \'filters\' for every field and the syntax.'
|
|
810
870
|
),
|
|
811
|
-
startDate: z2.string().optional().describe(
|
|
812
|
-
'Start
|
|
871
|
+
startDate: z2.string().min(1).optional().describe(
|
|
872
|
+
'Start of the window: ISO date, or relative like "24h", "7d", "4w", "3m" \u2014 units are h (hours), d (days), w (weeks), m (30-day months). Defaults to 24h ago for a text search, or 90d when traceIds is set. Widen this when a search comes back empty.'
|
|
813
873
|
),
|
|
814
|
-
endDate: z2.string().optional().describe("End
|
|
874
|
+
endDate: z2.string().min(1).optional().describe("End of the window: ISO date or relative. Default: now"),
|
|
815
875
|
pageSize: z2.number().optional().describe("Results per page (default: 25, max: 1000)"),
|
|
816
876
|
scrollId: z2.string().optional().describe("Pagination token from previous search"),
|
|
817
|
-
format: z2.enum(["digest", "json"]).optional().describe(
|
|
818
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
819
|
-
)
|
|
877
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
820
878
|
},
|
|
821
879
|
withToolLogging("search_traces", async (params) => {
|
|
822
880
|
requireApiKey();
|
|
823
|
-
const { handleSearchTraces } = await import("./search-traces-
|
|
881
|
+
const { handleSearchTraces } = await import("./search-traces-HSTQ6WTQ.js");
|
|
824
882
|
return {
|
|
825
883
|
content: [{ type: "text", text: await handleSearchTraces(params) }]
|
|
826
884
|
};
|
|
@@ -828,16 +886,16 @@ function registerTools(server) {
|
|
|
828
886
|
);
|
|
829
887
|
server.tool(
|
|
830
888
|
"get_trace",
|
|
831
|
-
"Get full details of a single trace by ID. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
889
|
+
"Get full details of a single trace by ID. This is the right tool whenever you have a trace ID \u2014 search_traces does not match IDs. A full ID resolves at any age; a unique 8\u201331 character hex prefix resolves within the last 90 days. Returns AI-readable trace digest by default. Use format: 'json' for full raw data including all spans.",
|
|
832
890
|
{
|
|
833
|
-
traceId: z2.string().describe(
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
)
|
|
891
|
+
traceId: z2.string().describe(
|
|
892
|
+
"The trace ID to retrieve. A full ID resolves at any age; a unique 8\u201331 character hex prefix also resolves, git-style, within the last 90 days."
|
|
893
|
+
),
|
|
894
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
837
895
|
},
|
|
838
896
|
withToolLogging("get_trace", async (params) => {
|
|
839
897
|
requireApiKey();
|
|
840
|
-
const { handleGetTrace } = await import("./get-trace-
|
|
898
|
+
const { handleGetTrace } = await import("./get-trace-SEF3FMR5.js");
|
|
841
899
|
return {
|
|
842
900
|
content: [{ type: "text", text: await handleGetTrace(params) }]
|
|
843
901
|
};
|
|
@@ -853,9 +911,7 @@ function registerTools(server) {
|
|
|
853
911
|
aggregation: z2.string().optional().describe(
|
|
854
912
|
"Aggregation type: avg, sum, min, max, median, p90, p95, p99, cardinality, terms. Default: avg"
|
|
855
913
|
),
|
|
856
|
-
startDate: z2.string().optional().describe(
|
|
857
|
-
'Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'
|
|
858
|
-
),
|
|
914
|
+
startDate: z2.string().optional().describe('Start date: ISO or relative ("7d", "30d"). Default: 7 days ago'),
|
|
859
915
|
endDate: z2.string().optional().describe("End date. Default: now"),
|
|
860
916
|
timeZone: z2.string().optional().describe("Timezone. Default: UTC"),
|
|
861
917
|
groupBy: z2.string().optional().describe("Group results by field. Use discover_schema for options."),
|
|
@@ -863,12 +919,14 @@ function registerTools(server) {
|
|
|
863
919
|
},
|
|
864
920
|
withToolLogging("get_analytics", async (params) => {
|
|
865
921
|
requireApiKey();
|
|
866
|
-
const { handleGetAnalytics } = await import("./get-analytics-
|
|
922
|
+
const { handleGetAnalytics } = await import("./get-analytics-DCS72353.js");
|
|
867
923
|
return {
|
|
868
924
|
content: [{ type: "text", text: await handleGetAnalytics(params) }]
|
|
869
925
|
};
|
|
870
926
|
})
|
|
871
927
|
);
|
|
928
|
+
}
|
|
929
|
+
function registerPromptTools(server) {
|
|
872
930
|
server.tool(
|
|
873
931
|
"platform_create_prompt",
|
|
874
932
|
`Create a new prompt on the LangWatch platform.
|
|
@@ -881,9 +939,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
881
939
|
`,
|
|
882
940
|
{
|
|
883
941
|
name: z2.string().describe("Prompt display name"),
|
|
884
|
-
handle: z2.string().optional().describe(
|
|
885
|
-
"URL-friendly handle (auto-generated from name if omitted)"
|
|
886
|
-
),
|
|
942
|
+
handle: z2.string().optional().describe("URL-friendly handle (auto-generated from name if omitted)"),
|
|
887
943
|
messages: z2.array(
|
|
888
944
|
z2.object({
|
|
889
945
|
role: z2.enum(["system", "user", "assistant"]).describe("Message role"),
|
|
@@ -897,7 +953,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
897
953
|
},
|
|
898
954
|
withToolLogging("platform_create_prompt", async (params) => {
|
|
899
955
|
requireApiKey();
|
|
900
|
-
const { handleCreatePrompt } = await import("./create-prompt-
|
|
956
|
+
const { handleCreatePrompt } = await import("./create-prompt-E67LXIYT.js");
|
|
901
957
|
return {
|
|
902
958
|
content: [{ type: "text", text: await handleCreatePrompt(params) }]
|
|
903
959
|
};
|
|
@@ -909,7 +965,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
909
965
|
{},
|
|
910
966
|
withToolLogging("platform_list_prompts", async () => {
|
|
911
967
|
requireApiKey();
|
|
912
|
-
const { handleListPrompts } = await import("./list-prompts-
|
|
968
|
+
const { handleListPrompts } = await import("./list-prompts-HSIMMHTF.js");
|
|
913
969
|
return {
|
|
914
970
|
content: [{ type: "text", text: await handleListPrompts() }]
|
|
915
971
|
};
|
|
@@ -917,13 +973,14 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
917
973
|
);
|
|
918
974
|
server.tool(
|
|
919
975
|
"platform_get_prompt",
|
|
920
|
-
"Get a specific prompt from the LangWatch platform by ID or handle, including messages, model config, and version history.",
|
|
976
|
+
"Get a specific prompt from the LangWatch platform by ID or handle, including messages, model config, and version history. Use format: 'json' for the full raw API payload, or 'digest' (default) for a formatted summary.",
|
|
921
977
|
{
|
|
922
978
|
idOrHandle: z2.string().describe("Prompt ID or handle"),
|
|
923
979
|
version: z2.number().optional().describe("Specific version number (default: latest)"),
|
|
924
980
|
tag: z2.string().optional().describe(
|
|
925
981
|
'Fetch the version pointed to by this tag (e.g., "production", "staging"). Alternatively, use shorthand in idOrHandle: "pizza-prompt:production"'
|
|
926
|
-
)
|
|
982
|
+
),
|
|
983
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
927
984
|
},
|
|
928
985
|
withToolLogging("platform_get_prompt", async (params) => {
|
|
929
986
|
if (params.version != null && params.tag) {
|
|
@@ -933,7 +990,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
933
990
|
};
|
|
934
991
|
}
|
|
935
992
|
requireApiKey();
|
|
936
|
-
const { handleGetPrompt } = await import("./get-prompt-
|
|
993
|
+
const { handleGetPrompt } = await import("./get-prompt-OWXJZHQQ.js");
|
|
937
994
|
return {
|
|
938
995
|
content: [{ type: "text", text: await handleGetPrompt(params) }]
|
|
939
996
|
};
|
|
@@ -952,13 +1009,11 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
952
1009
|
).optional().describe("Updated messages"),
|
|
953
1010
|
model: modelSchema.optional(),
|
|
954
1011
|
commitMessage: z2.string().describe("Commit message describing the change"),
|
|
955
|
-
tags: z2.array(z2.string()).optional().describe(
|
|
956
|
-
"Tags to assign to the new version created by this update."
|
|
957
|
-
)
|
|
1012
|
+
tags: z2.array(z2.string()).optional().describe("Tags to assign to the new version created by this update.")
|
|
958
1013
|
},
|
|
959
1014
|
withToolLogging("platform_update_prompt", async (params) => {
|
|
960
1015
|
requireApiKey();
|
|
961
|
-
const { handleUpdatePrompt } = await import("./update-prompt-
|
|
1016
|
+
const { handleUpdatePrompt } = await import("./update-prompt-MY263DYF.js");
|
|
962
1017
|
return {
|
|
963
1018
|
content: [{ type: "text", text: await handleUpdatePrompt(params) }]
|
|
964
1019
|
};
|
|
@@ -974,7 +1029,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
974
1029
|
},
|
|
975
1030
|
withToolLogging("platform_assign_prompt_tag", async (params) => {
|
|
976
1031
|
requireApiKey();
|
|
977
|
-
const { handleAssignPromptTag } = await import("./assign-prompt-tag-
|
|
1032
|
+
const { handleAssignPromptTag } = await import("./assign-prompt-tag-HBZLBIFD.js");
|
|
978
1033
|
return {
|
|
979
1034
|
content: [{ type: "text", text: await handleAssignPromptTag(params) }]
|
|
980
1035
|
};
|
|
@@ -986,7 +1041,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
986
1041
|
{},
|
|
987
1042
|
withToolLogging("platform_list_prompt_tags", async () => {
|
|
988
1043
|
requireApiKey();
|
|
989
|
-
const { handleListPromptTags } = await import("./list-prompt-tags-
|
|
1044
|
+
const { handleListPromptTags } = await import("./list-prompt-tags-NVFQO6WX.js");
|
|
990
1045
|
return {
|
|
991
1046
|
content: [{ type: "text", text: await handleListPromptTags() }]
|
|
992
1047
|
};
|
|
@@ -1000,7 +1055,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1000
1055
|
},
|
|
1001
1056
|
withToolLogging("platform_create_prompt_tag", async (params) => {
|
|
1002
1057
|
requireApiKey();
|
|
1003
|
-
const { handleCreatePromptTag } = await import("./create-prompt-tag-
|
|
1058
|
+
const { handleCreatePromptTag } = await import("./create-prompt-tag-OPQUEXWB.js");
|
|
1004
1059
|
return {
|
|
1005
1060
|
content: [{ type: "text", text: await handleCreatePromptTag(params) }]
|
|
1006
1061
|
};
|
|
@@ -1015,7 +1070,7 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1015
1070
|
},
|
|
1016
1071
|
withToolLogging("platform_rename_prompt_tag", async (params) => {
|
|
1017
1072
|
requireApiKey();
|
|
1018
|
-
const { handleRenamePromptTag } = await import("./rename-prompt-tag-
|
|
1073
|
+
const { handleRenamePromptTag } = await import("./rename-prompt-tag-IQI7MHOI.js");
|
|
1019
1074
|
return {
|
|
1020
1075
|
content: [{ type: "text", text: await handleRenamePromptTag(params) }]
|
|
1021
1076
|
};
|
|
@@ -1029,12 +1084,14 @@ NOTE: Prompts can be managed two ways. Determine which approach the user needs:
|
|
|
1029
1084
|
},
|
|
1030
1085
|
withToolLogging("platform_delete_prompt_tag", async (params) => {
|
|
1031
1086
|
requireApiKey();
|
|
1032
|
-
const { handleDeletePromptTag } = await import("./delete-prompt-tag-
|
|
1087
|
+
const { handleDeletePromptTag } = await import("./delete-prompt-tag-YYQ4VMAD.js");
|
|
1033
1088
|
return {
|
|
1034
1089
|
content: [{ type: "text", text: await handleDeletePromptTag(params) }]
|
|
1035
1090
|
};
|
|
1036
1091
|
})
|
|
1037
1092
|
);
|
|
1093
|
+
}
|
|
1094
|
+
function registerScenarioTools(server) {
|
|
1038
1095
|
server.tool(
|
|
1039
1096
|
"platform_create_scenario",
|
|
1040
1097
|
`Create a new scenario on the LangWatch platform. Call discover_schema({ category: 'scenarios' }) first to learn how to write effective situations and criteria.
|
|
@@ -1047,21 +1104,19 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1047
1104
|
`,
|
|
1048
1105
|
{
|
|
1049
1106
|
name: z2.string().describe("Scenario name"),
|
|
1050
|
-
situation: z2.string().describe(
|
|
1051
|
-
|
|
1052
|
-
),
|
|
1053
|
-
|
|
1054
|
-
"Pass
|
|
1107
|
+
situation: z2.string().describe("The context or setup describing what the user/agent is doing"),
|
|
1108
|
+
criteria: z2.array(z2.string()).optional().describe("Pass/fail conditions the agent's response must satisfy"),
|
|
1109
|
+
labels: z2.array(z2.string()).optional().describe("Tags for organizing and filtering scenarios"),
|
|
1110
|
+
testSuiteId: z2.string().nullish().describe(
|
|
1111
|
+
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
1055
1112
|
),
|
|
1056
|
-
|
|
1113
|
+
fields: scenarioFieldValuesSchema.optional()
|
|
1057
1114
|
},
|
|
1058
1115
|
withToolLogging("platform_create_scenario", async (params) => {
|
|
1059
1116
|
requireApiKey();
|
|
1060
|
-
const { handleCreateScenario } = await import("./create-scenario-
|
|
1117
|
+
const { handleCreateScenario } = await import("./create-scenario-UEM77OE7.js");
|
|
1061
1118
|
return {
|
|
1062
|
-
content: [
|
|
1063
|
-
{ type: "text", text: await handleCreateScenario(params) }
|
|
1064
|
-
]
|
|
1119
|
+
content: [{ type: "text", text: await handleCreateScenario(params) }]
|
|
1065
1120
|
};
|
|
1066
1121
|
})
|
|
1067
1122
|
);
|
|
@@ -1069,17 +1124,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1069
1124
|
"platform_list_scenarios",
|
|
1070
1125
|
"List all scenarios on the LangWatch platform. Returns AI-readable digest by default.",
|
|
1071
1126
|
{
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
)
|
|
1127
|
+
testSuiteId: z2.string().optional().describe("Only the scenarios filed in this test suite"),
|
|
1128
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
1075
1129
|
},
|
|
1076
1130
|
withToolLogging("platform_list_scenarios", async (params) => {
|
|
1077
1131
|
requireApiKey();
|
|
1078
|
-
const { handleListScenarios } = await import("./list-scenarios-
|
|
1132
|
+
const { handleListScenarios } = await import("./list-scenarios-QZRPVHS4.js");
|
|
1079
1133
|
return {
|
|
1080
|
-
content: [
|
|
1081
|
-
{ type: "text", text: await handleListScenarios(params) }
|
|
1082
|
-
]
|
|
1134
|
+
content: [{ type: "text", text: await handleListScenarios(params) }]
|
|
1083
1135
|
};
|
|
1084
1136
|
})
|
|
1085
1137
|
);
|
|
@@ -1088,13 +1140,11 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1088
1140
|
"Get full details of a scenario on the LangWatch platform by ID, including situation, criteria, and labels.",
|
|
1089
1141
|
{
|
|
1090
1142
|
scenarioId: z2.string().describe("The scenario ID to retrieve"),
|
|
1091
|
-
format: z2.enum(["digest", "json"]).optional().describe(
|
|
1092
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
1093
|
-
)
|
|
1143
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
1094
1144
|
},
|
|
1095
1145
|
withToolLogging("platform_get_scenario", async (params) => {
|
|
1096
1146
|
requireApiKey();
|
|
1097
|
-
const { handleGetScenario } = await import("./get-scenario-
|
|
1147
|
+
const { handleGetScenario } = await import("./get-scenario-YM4ODWW6.js");
|
|
1098
1148
|
return {
|
|
1099
1149
|
content: [{ type: "text", text: await handleGetScenario(params) }]
|
|
1100
1150
|
};
|
|
@@ -1108,15 +1158,17 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1108
1158
|
name: z2.string().optional().describe("Updated scenario name"),
|
|
1109
1159
|
situation: z2.string().optional().describe("Updated situation"),
|
|
1110
1160
|
criteria: z2.array(z2.string()).optional().describe("Updated criteria"),
|
|
1111
|
-
labels: z2.array(z2.string()).optional().describe("Updated labels")
|
|
1161
|
+
labels: z2.array(z2.string()).optional().describe("Updated labels"),
|
|
1162
|
+
testSuiteId: z2.string().nullish().describe(
|
|
1163
|
+
"The test suite to file this scenario in. Pass a test suite ID, or null to unfile it."
|
|
1164
|
+
),
|
|
1165
|
+
fields: scenarioFieldValuesSchema.optional()
|
|
1112
1166
|
},
|
|
1113
1167
|
withToolLogging("platform_update_scenario", async (params) => {
|
|
1114
1168
|
requireApiKey();
|
|
1115
|
-
const { handleUpdateScenario } = await import("./update-scenario-
|
|
1169
|
+
const { handleUpdateScenario } = await import("./update-scenario-MOHRQKAQ.js");
|
|
1116
1170
|
return {
|
|
1117
|
-
content: [
|
|
1118
|
-
{ type: "text", text: await handleUpdateScenario(params) }
|
|
1119
|
-
]
|
|
1171
|
+
content: [{ type: "text", text: await handleUpdateScenario(params) }]
|
|
1120
1172
|
};
|
|
1121
1173
|
})
|
|
1122
1174
|
);
|
|
@@ -1128,110 +1180,235 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1128
1180
|
},
|
|
1129
1181
|
withToolLogging("platform_archive_scenario", async (params) => {
|
|
1130
1182
|
requireApiKey();
|
|
1131
|
-
const { handleArchiveScenario } = await import("./archive-scenario-
|
|
1183
|
+
const { handleArchiveScenario } = await import("./archive-scenario-A6QV5LOW.js");
|
|
1132
1184
|
return {
|
|
1133
|
-
content: [
|
|
1134
|
-
|
|
1135
|
-
|
|
1185
|
+
content: [{ type: "text", text: await handleArchiveScenario(params) }]
|
|
1186
|
+
};
|
|
1187
|
+
})
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
function registerRunPlanTools(server) {
|
|
1191
|
+
server.tool(
|
|
1192
|
+
"platform_run_plan",
|
|
1193
|
+
"Run scenarios against targets. The plan name identifies the run plan: an existing name is re-run with the configuration you send here, a new name creates the plan. Configuration is the scope, the targets, the repeat count and the models; parameters, the note and the idempotency key belong to this run alone. Send more than one target to compare them in the same run, and give a target its own parameters to compare one agent on two models.",
|
|
1194
|
+
{
|
|
1195
|
+
name: z2.string().optional().describe(
|
|
1196
|
+
"The run plan to run. An existing name is replaced with this configuration, a new name creates the plan. Omit to let the server name it after the suite and the target."
|
|
1197
|
+
),
|
|
1198
|
+
scope: runPlanScopeSchema.describe(
|
|
1199
|
+
"What the plan covers: every scenario in the project, the scenarios filed in given test suites, the scenarios carrying given labels, or the hand-picked list in scenarioIds."
|
|
1200
|
+
),
|
|
1201
|
+
scenarioIds: z2.array(z2.string()).optional().describe("The scenarios to run. Read only when scope.mode is 'scenarios'."),
|
|
1202
|
+
targets: z2.array(runPlanTargetSchema).describe(
|
|
1203
|
+
"What to run the scenarios against. Every target runs every scenario, and the results show one column for each, so this is how one run compares two agents or one agent on two models."
|
|
1204
|
+
),
|
|
1205
|
+
repeatCount: z2.number().int().min(1).max(5).optional().describe("How many times to run each scenario against each target (1 to 5, default 1)"),
|
|
1206
|
+
simulatorModel: z2.string().optional().describe("Model that plays the user. Omit for the project default."),
|
|
1207
|
+
judgeModel: z2.string().optional().describe("Model that judges the criteria. Omit for the project default."),
|
|
1208
|
+
evaluators: evaluatorAttachmentsSchema.optional().describe(
|
|
1209
|
+
"The plan's own evaluators, run beside the ones attached to the test suites its scenarios belong to. A plan evaluator reads the conversation and the trace, never a scenario field. Omit to keep what the plan already holds. See discover_schema({ category: 'scenarios' }) for the mapping paths."
|
|
1210
|
+
),
|
|
1211
|
+
parameters: runParametersSchema.optional().describe(
|
|
1212
|
+
"Values for the parameters the scenarios declare, by name. They apply to every target; a target that names the same parameter overrides them for itself."
|
|
1213
|
+
),
|
|
1214
|
+
note: z2.string().max(200).optional().describe("One short line saying why this run was started (max 200 characters)"),
|
|
1215
|
+
idempotencyKey: z2.string().optional().describe(
|
|
1216
|
+
"Send the same key to make a retry join the run it already started instead of starting a second one."
|
|
1217
|
+
)
|
|
1218
|
+
},
|
|
1219
|
+
withToolLogging("platform_run_plan", async (params) => {
|
|
1220
|
+
requireApiKey();
|
|
1221
|
+
const { handleRunPlan } = await import("./run-plan-PAARQSI4.js");
|
|
1222
|
+
return {
|
|
1223
|
+
content: [{ type: "text", text: await handleRunPlan(params) }]
|
|
1224
|
+
};
|
|
1225
|
+
})
|
|
1226
|
+
);
|
|
1227
|
+
server.tool(
|
|
1228
|
+
"platform_list_run_plans",
|
|
1229
|
+
"List the run plans of the project. A run plan holds the configuration a run used, keyed by its name.",
|
|
1230
|
+
{
|
|
1231
|
+
includeArchived: z2.boolean().optional().describe("Include archived plans (default false)"),
|
|
1232
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json' (raw data)")
|
|
1233
|
+
},
|
|
1234
|
+
withToolLogging("platform_list_run_plans", async (params) => {
|
|
1235
|
+
requireApiKey();
|
|
1236
|
+
const { handleListRunPlans } = await import("./list-run-plans-PX2XI6EK.js");
|
|
1237
|
+
return {
|
|
1238
|
+
content: [{ type: "text", text: await handleListRunPlans(params) }]
|
|
1239
|
+
};
|
|
1240
|
+
})
|
|
1241
|
+
);
|
|
1242
|
+
server.tool(
|
|
1243
|
+
"platform_get_run_plan",
|
|
1244
|
+
"Get the full configuration of a run plan: what it covers, its targets, its repeat count and its models.",
|
|
1245
|
+
{
|
|
1246
|
+
id: z2.string().describe("The run plan ID"),
|
|
1247
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json'")
|
|
1248
|
+
},
|
|
1249
|
+
withToolLogging("platform_get_run_plan", async (params) => {
|
|
1250
|
+
requireApiKey();
|
|
1251
|
+
const { handleGetRunPlan } = await import("./get-run-plan-BE3M2AUX.js");
|
|
1252
|
+
return {
|
|
1253
|
+
content: [{ type: "text", text: await handleGetRunPlan(params) }]
|
|
1254
|
+
};
|
|
1255
|
+
})
|
|
1256
|
+
);
|
|
1257
|
+
server.tool(
|
|
1258
|
+
"platform_rerun_run_plan",
|
|
1259
|
+
"Run a plan again with the configuration it already holds. Only the parameters and the note belong to the new run.",
|
|
1260
|
+
{
|
|
1261
|
+
id: z2.string().describe("The run plan ID to run again"),
|
|
1262
|
+
parameters: runParametersSchema.optional().describe("Values for the parameters the scenarios declare, by name."),
|
|
1263
|
+
note: z2.string().max(200).optional().describe("One short line saying why this run was started (max 200 characters)")
|
|
1264
|
+
},
|
|
1265
|
+
withToolLogging("platform_rerun_run_plan", async (params) => {
|
|
1266
|
+
requireApiKey();
|
|
1267
|
+
const { handleRerunRunPlan } = await import("./rerun-run-plan-TG2V4WM4.js");
|
|
1268
|
+
return {
|
|
1269
|
+
content: [{ type: "text", text: await handleRerunRunPlan(params) }]
|
|
1270
|
+
};
|
|
1271
|
+
})
|
|
1272
|
+
);
|
|
1273
|
+
server.tool(
|
|
1274
|
+
"platform_archive_run_plan",
|
|
1275
|
+
"Archive a run plan. Its past runs stay readable and the name is free for a new plan.",
|
|
1276
|
+
{
|
|
1277
|
+
id: z2.string().describe("The run plan ID to archive")
|
|
1278
|
+
},
|
|
1279
|
+
withToolLogging("platform_archive_run_plan", async (params) => {
|
|
1280
|
+
requireApiKey();
|
|
1281
|
+
const { handleArchiveRunPlan } = await import("./archive-run-plan-65QN3XRX.js");
|
|
1282
|
+
return {
|
|
1283
|
+
content: [{ type: "text", text: await handleArchiveRunPlan(params) }]
|
|
1136
1284
|
};
|
|
1137
1285
|
})
|
|
1138
1286
|
);
|
|
1287
|
+
}
|
|
1288
|
+
function registerTestSuiteTools(server) {
|
|
1139
1289
|
server.tool(
|
|
1140
|
-
"
|
|
1141
|
-
"List
|
|
1290
|
+
"platform_list_test_suites",
|
|
1291
|
+
"List the test suites of the project. A test suite groups scenarios.",
|
|
1142
1292
|
{
|
|
1143
1293
|
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json' (raw data)")
|
|
1144
1294
|
},
|
|
1145
|
-
withToolLogging("
|
|
1295
|
+
withToolLogging("platform_list_test_suites", async (params) => {
|
|
1296
|
+
requireApiKey();
|
|
1297
|
+
const { handleListTestSuites } = await import("./list-test-suites-N4WVLBKK.js");
|
|
1298
|
+
return {
|
|
1299
|
+
content: [{ type: "text", text: await handleListTestSuites(params) }]
|
|
1300
|
+
};
|
|
1301
|
+
})
|
|
1302
|
+
);
|
|
1303
|
+
server.tool(
|
|
1304
|
+
"platform_create_test_suite",
|
|
1305
|
+
"Create a test suite. A test suite groups scenarios: file a scenario in it by passing the suite ID as testSuiteId on platform_create_scenario or platform_update_scenario. A suite can declare typed fields every scenario carries a value for, and attach saved evaluators that run after every scenario run with mappings from their inputs to the conversation, the scenario's fields or the trace. Call discover_schema({ category: 'scenarios' }) first for the field rules and the mapping paths.",
|
|
1306
|
+
{
|
|
1307
|
+
name: z2.string().describe("Test suite name"),
|
|
1308
|
+
fields: suiteFieldsSchema.optional(),
|
|
1309
|
+
evaluators: evaluatorAttachmentsSchema.optional()
|
|
1310
|
+
},
|
|
1311
|
+
withToolLogging("platform_create_test_suite", async (params) => {
|
|
1146
1312
|
requireApiKey();
|
|
1147
|
-
const {
|
|
1313
|
+
const { handleCreateTestSuite } = await import("./create-test-suite-4IPXVSFQ.js");
|
|
1148
1314
|
return {
|
|
1149
|
-
content: [{ type: "text", text: await
|
|
1315
|
+
content: [{ type: "text", text: await handleCreateTestSuite(params) }]
|
|
1150
1316
|
};
|
|
1151
1317
|
})
|
|
1152
1318
|
);
|
|
1153
1319
|
server.tool(
|
|
1154
|
-
"
|
|
1155
|
-
"Get
|
|
1320
|
+
"platform_get_test_suite",
|
|
1321
|
+
"Get a test suite and the scenarios filed in it.",
|
|
1156
1322
|
{
|
|
1157
|
-
id: z2.string().describe("The suite ID"),
|
|
1323
|
+
id: z2.string().describe("The test suite ID"),
|
|
1158
1324
|
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default) or 'json'")
|
|
1159
1325
|
},
|
|
1160
|
-
withToolLogging("
|
|
1326
|
+
withToolLogging("platform_get_test_suite", async (params) => {
|
|
1161
1327
|
requireApiKey();
|
|
1162
|
-
const {
|
|
1328
|
+
const { handleGetTestSuite } = await import("./get-test-suite-GC2O7YH6.js");
|
|
1163
1329
|
return {
|
|
1164
|
-
content: [{ type: "text", text: await
|
|
1330
|
+
content: [{ type: "text", text: await handleGetTestSuite(params) }]
|
|
1165
1331
|
};
|
|
1166
1332
|
})
|
|
1167
1333
|
);
|
|
1168
1334
|
server.tool(
|
|
1169
|
-
"
|
|
1170
|
-
"
|
|
1335
|
+
"platform_update_test_suite",
|
|
1336
|
+
"Edit a test suite: any of its name, its fields and its evaluators. A field list or an evaluator list replaces the one the suite holds; a key left out keeps what the suite has. A field an attached evaluator still reads cannot be removed. Call discover_schema({ category: 'scenarios' }) first for the field rules and the mapping paths.",
|
|
1171
1337
|
{
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1338
|
+
id: z2.string().describe("The test suite ID"),
|
|
1339
|
+
name: z2.string().optional().describe("The new name. The slug is kept."),
|
|
1340
|
+
fields: suiteFieldsSchema.optional().describe(
|
|
1341
|
+
"The full list of fields the suite declares. It replaces the list the suite holds."
|
|
1342
|
+
),
|
|
1343
|
+
evaluators: evaluatorAttachmentsSchema.optional().describe(
|
|
1344
|
+
"The full list of evaluators attached to the suite. It replaces the list the suite holds."
|
|
1345
|
+
)
|
|
1178
1346
|
},
|
|
1179
|
-
withToolLogging("
|
|
1347
|
+
withToolLogging("platform_update_test_suite", async (params) => {
|
|
1180
1348
|
requireApiKey();
|
|
1181
|
-
const { handleCreateSuite } = await import("./create-suite-LZS6QIAF.js");
|
|
1182
1349
|
return {
|
|
1183
|
-
content: [{ type: "text", text: await
|
|
1350
|
+
content: [{ type: "text", text: await handleUpdateTestSuite(params) }]
|
|
1184
1351
|
};
|
|
1185
1352
|
})
|
|
1186
1353
|
);
|
|
1187
1354
|
server.tool(
|
|
1188
|
-
"
|
|
1189
|
-
"
|
|
1355
|
+
"platform_rename_test_suite",
|
|
1356
|
+
"Rename a test suite.",
|
|
1190
1357
|
{
|
|
1191
|
-
id: z2.string().describe("The suite ID
|
|
1192
|
-
name: z2.string().
|
|
1193
|
-
description: z2.string().optional().describe("New description"),
|
|
1194
|
-
scenarioIds: z2.array(z2.string()).optional().describe("New array of scenario IDs"),
|
|
1195
|
-
targets: z2.string().optional().describe("New JSON array of targets"),
|
|
1196
|
-
repeatCount: z2.number().optional().describe("New repeat count"),
|
|
1197
|
-
labels: z2.array(z2.string()).optional().describe("New labels")
|
|
1358
|
+
id: z2.string().describe("The test suite ID"),
|
|
1359
|
+
name: z2.string().describe("The new name")
|
|
1198
1360
|
},
|
|
1199
|
-
withToolLogging("
|
|
1361
|
+
withToolLogging("platform_rename_test_suite", async (params) => {
|
|
1200
1362
|
requireApiKey();
|
|
1201
|
-
const {
|
|
1363
|
+
const { handleRenameTestSuite } = await import("./rename-test-suite-4J7UX5S2.js");
|
|
1202
1364
|
return {
|
|
1203
|
-
content: [{ type: "text", text: await
|
|
1365
|
+
content: [{ type: "text", text: await handleRenameTestSuite(params) }]
|
|
1204
1366
|
};
|
|
1205
1367
|
})
|
|
1206
1368
|
);
|
|
1207
1369
|
server.tool(
|
|
1208
|
-
"
|
|
1209
|
-
"
|
|
1370
|
+
"platform_archive_test_suite",
|
|
1371
|
+
"Archive a test suite. The scenarios filed in it are archived with it.",
|
|
1210
1372
|
{
|
|
1211
|
-
id: z2.string().describe("The suite ID to
|
|
1373
|
+
id: z2.string().describe("The test suite ID to archive")
|
|
1212
1374
|
},
|
|
1213
|
-
withToolLogging("
|
|
1375
|
+
withToolLogging("platform_archive_test_suite", async (params) => {
|
|
1214
1376
|
requireApiKey();
|
|
1215
|
-
const {
|
|
1377
|
+
const { handleArchiveTestSuite } = await import("./archive-test-suite-NKZDZY3D.js");
|
|
1216
1378
|
return {
|
|
1217
|
-
content: [{ type: "text", text: await
|
|
1379
|
+
content: [{ type: "text", text: await handleArchiveTestSuite(params) }]
|
|
1218
1380
|
};
|
|
1219
1381
|
})
|
|
1220
1382
|
);
|
|
1221
1383
|
server.tool(
|
|
1222
|
-
"
|
|
1223
|
-
"
|
|
1384
|
+
"platform_run_test_suite",
|
|
1385
|
+
"Run every scenario of a test suite against targets. This creates or joins the run plan named '<suite name> <target name>' unless you send a name of your own. Send more than one target to compare them in the same run, and give a target its own parameters to compare one agent on two models.",
|
|
1224
1386
|
{
|
|
1225
|
-
id: z2.string().describe("The suite ID to
|
|
1387
|
+
id: z2.string().describe("The test suite ID to run"),
|
|
1388
|
+
targets: z2.array(runPlanTargetSchema).describe(
|
|
1389
|
+
"What to run the scenarios against. Every target runs every scenario, and the results show one column for each."
|
|
1390
|
+
),
|
|
1391
|
+
name: z2.string().optional().describe(
|
|
1392
|
+
"The run plan to run. Omit to let the server name it after the suite and the target."
|
|
1393
|
+
),
|
|
1394
|
+
repeatCount: z2.number().int().min(1).max(5).optional().describe("How many times to run each scenario against each target (1 to 5, default 1)"),
|
|
1395
|
+
simulatorModel: z2.string().optional().describe("Model that plays the user. Omit for the project default."),
|
|
1396
|
+
judgeModel: z2.string().optional().describe("Model that judges the criteria. Omit for the project default."),
|
|
1397
|
+
parameters: runParametersSchema.optional().describe(
|
|
1398
|
+
"Values for the parameters the scenarios declare, by name. They apply to every target; a target that names the same parameter overrides them for itself."
|
|
1399
|
+
),
|
|
1400
|
+
note: z2.string().max(200).optional().describe("One short line saying why this run was started (max 200 characters)")
|
|
1226
1401
|
},
|
|
1227
|
-
withToolLogging("
|
|
1402
|
+
withToolLogging("platform_run_test_suite", async (params) => {
|
|
1228
1403
|
requireApiKey();
|
|
1229
|
-
const {
|
|
1404
|
+
const { handleRunTestSuite } = await import("./run-test-suite-SQJELEZX.js");
|
|
1230
1405
|
return {
|
|
1231
|
-
content: [{ type: "text", text: await
|
|
1406
|
+
content: [{ type: "text", text: await handleRunTestSuite(params) }]
|
|
1232
1407
|
};
|
|
1233
1408
|
})
|
|
1234
1409
|
);
|
|
1410
|
+
}
|
|
1411
|
+
function registerSimulationRunTools(server) {
|
|
1235
1412
|
server.tool(
|
|
1236
1413
|
"platform_list_simulation_runs",
|
|
1237
1414
|
"List simulation run results. Filter by scenario set or batch run ID to see specific results.",
|
|
@@ -1243,7 +1420,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1243
1420
|
},
|
|
1244
1421
|
withToolLogging("platform_list_simulation_runs", async (params) => {
|
|
1245
1422
|
requireApiKey();
|
|
1246
|
-
const { handleListSimulationRuns } = await import("./list-simulation-runs-
|
|
1423
|
+
const { handleListSimulationRuns } = await import("./list-simulation-runs-OEADLLQR.js");
|
|
1247
1424
|
return {
|
|
1248
1425
|
content: [{ type: "text", text: await handleListSimulationRuns(params) }]
|
|
1249
1426
|
};
|
|
@@ -1251,19 +1428,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1251
1428
|
);
|
|
1252
1429
|
server.tool(
|
|
1253
1430
|
"platform_get_simulation_run",
|
|
1254
|
-
"Get full details of a simulation run including conversation messages, results, costs, and
|
|
1431
|
+
"Get full details of a simulation run including conversation messages, results, costs, the verdict and the result of every evaluator attached to its test suite or run plan.",
|
|
1255
1432
|
{
|
|
1256
1433
|
scenarioRunId: z2.string().describe("The simulation run ID"),
|
|
1257
1434
|
format: z2.enum(["digest", "json"]).optional().describe("Output format")
|
|
1258
1435
|
},
|
|
1259
1436
|
withToolLogging("platform_get_simulation_run", async (params) => {
|
|
1260
1437
|
requireApiKey();
|
|
1261
|
-
const { handleGetSimulationRun } = await import("./get-simulation-run-
|
|
1438
|
+
const { handleGetSimulationRun } = await import("./get-simulation-run-LCFSMPGQ.js");
|
|
1262
1439
|
return {
|
|
1263
1440
|
content: [{ type: "text", text: await handleGetSimulationRun(params) }]
|
|
1264
1441
|
};
|
|
1265
1442
|
})
|
|
1266
1443
|
);
|
|
1444
|
+
}
|
|
1445
|
+
function registerEvaluatorTools(server) {
|
|
1267
1446
|
server.tool(
|
|
1268
1447
|
"platform_create_evaluator",
|
|
1269
1448
|
`Create an evaluator on the LangWatch platform. Useful for setting up LLM-as-judge and other evaluators to use in evaluation notebooks. Call discover_schema({ category: 'evaluators' }) first to see available evaluator types and their settings.`,
|
|
@@ -1275,11 +1454,9 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1275
1454
|
},
|
|
1276
1455
|
withToolLogging("platform_create_evaluator", async (params) => {
|
|
1277
1456
|
requireApiKey();
|
|
1278
|
-
const { handleCreateEvaluator } = await import("./create-evaluator-
|
|
1457
|
+
const { handleCreateEvaluator } = await import("./create-evaluator-643G6LXL.js");
|
|
1279
1458
|
return {
|
|
1280
|
-
content: [
|
|
1281
|
-
{ type: "text", text: await handleCreateEvaluator(params) }
|
|
1282
|
-
]
|
|
1459
|
+
content: [{ type: "text", text: await handleCreateEvaluator(params) }]
|
|
1283
1460
|
};
|
|
1284
1461
|
})
|
|
1285
1462
|
);
|
|
@@ -1289,7 +1466,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1289
1466
|
{},
|
|
1290
1467
|
withToolLogging("platform_list_evaluators", async () => {
|
|
1291
1468
|
requireApiKey();
|
|
1292
|
-
const { handleListEvaluators } = await import("./list-evaluators-
|
|
1469
|
+
const { handleListEvaluators } = await import("./list-evaluators-MYH2OVCI.js");
|
|
1293
1470
|
return {
|
|
1294
1471
|
content: [{ type: "text", text: await handleListEvaluators() }]
|
|
1295
1472
|
};
|
|
@@ -1303,7 +1480,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1303
1480
|
},
|
|
1304
1481
|
withToolLogging("platform_get_evaluator", async (params) => {
|
|
1305
1482
|
requireApiKey();
|
|
1306
|
-
const { handleGetEvaluator } = await import("./get-evaluator-
|
|
1483
|
+
const { handleGetEvaluator } = await import("./get-evaluator-ISMC4LLI.js");
|
|
1307
1484
|
return {
|
|
1308
1485
|
content: [{ type: "text", text: await handleGetEvaluator(params) }]
|
|
1309
1486
|
};
|
|
@@ -1315,17 +1492,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1315
1492
|
{
|
|
1316
1493
|
evaluatorId: z2.string().describe("The evaluator ID to update"),
|
|
1317
1494
|
name: z2.string().optional().describe("Updated evaluator name"),
|
|
1318
|
-
config: z2.record(z2.string(), z2.unknown()).optional().describe(
|
|
1319
|
-
"Updated config settings. Note: evaluatorType cannot be changed after creation."
|
|
1320
|
-
)
|
|
1495
|
+
config: z2.record(z2.string(), z2.unknown()).optional().describe("Updated config settings. Note: evaluatorType cannot be changed after creation.")
|
|
1321
1496
|
},
|
|
1322
1497
|
withToolLogging("platform_update_evaluator", async (params) => {
|
|
1323
1498
|
requireApiKey();
|
|
1324
|
-
const { handleUpdateEvaluator } = await import("./update-evaluator-
|
|
1499
|
+
const { handleUpdateEvaluator } = await import("./update-evaluator-IGTX6PX4.js");
|
|
1325
1500
|
return {
|
|
1326
|
-
content: [
|
|
1327
|
-
{ type: "text", text: await handleUpdateEvaluator(params) }
|
|
1328
|
-
]
|
|
1501
|
+
content: [{ type: "text", text: await handleUpdateEvaluator(params) }]
|
|
1329
1502
|
};
|
|
1330
1503
|
})
|
|
1331
1504
|
);
|
|
@@ -1337,19 +1510,19 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1337
1510
|
},
|
|
1338
1511
|
withToolLogging("platform_delete_evaluator", async (params) => {
|
|
1339
1512
|
requireApiKey();
|
|
1340
|
-
const { handleDeleteEvaluator } = await import("./delete-evaluator-
|
|
1513
|
+
const { handleDeleteEvaluator } = await import("./delete-evaluator-4SNSCLEE.js");
|
|
1341
1514
|
return {
|
|
1342
1515
|
content: [{ type: "text", text: await handleDeleteEvaluator(params) }]
|
|
1343
1516
|
};
|
|
1344
1517
|
})
|
|
1345
1518
|
);
|
|
1519
|
+
}
|
|
1520
|
+
function registerModelProviderTools(server) {
|
|
1346
1521
|
server.tool(
|
|
1347
1522
|
"platform_set_model_provider",
|
|
1348
1523
|
`Set or update a model provider on the LangWatch platform. Use this to configure API keys (e.g. OPENAI_API_KEY) needed to run evaluators. The API key is stored securely and never returned in responses. Omit customKeys to update other settings without changing existing keys.`,
|
|
1349
1524
|
{
|
|
1350
|
-
provider: z2.string().describe(
|
|
1351
|
-
'Provider name, e.g., "openai", "anthropic", "azure", "custom"'
|
|
1352
|
-
),
|
|
1525
|
+
provider: z2.string().describe('Provider name, e.g., "openai", "anthropic", "azure", "custom"'),
|
|
1353
1526
|
enabled: z2.boolean().describe("Whether the provider is enabled"),
|
|
1354
1527
|
customKeys: z2.record(z2.string(), z2.unknown()).optional().describe(
|
|
1355
1528
|
'API key configuration, e.g. { "OPENAI_API_KEY": "sk-..." }. Omit to keep existing keys.'
|
|
@@ -1358,11 +1531,9 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1358
1531
|
},
|
|
1359
1532
|
withToolLogging("platform_set_model_provider", async (params) => {
|
|
1360
1533
|
requireApiKey();
|
|
1361
|
-
const { handleSetModelProvider } = await import("./set-model-provider-
|
|
1534
|
+
const { handleSetModelProvider } = await import("./set-model-provider-GF7NXUJA.js");
|
|
1362
1535
|
return {
|
|
1363
|
-
content: [
|
|
1364
|
-
{ type: "text", text: await handleSetModelProvider(params) }
|
|
1365
|
-
]
|
|
1536
|
+
content: [{ type: "text", text: await handleSetModelProvider(params) }]
|
|
1366
1537
|
};
|
|
1367
1538
|
})
|
|
1368
1539
|
);
|
|
@@ -1372,19 +1543,21 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1372
1543
|
{},
|
|
1373
1544
|
withToolLogging("platform_list_model_providers", async () => {
|
|
1374
1545
|
requireApiKey();
|
|
1375
|
-
const { handleListModelProviders } = await import("./list-model-providers-
|
|
1546
|
+
const { handleListModelProviders } = await import("./list-model-providers-3JUGP4YI.js");
|
|
1376
1547
|
return {
|
|
1377
1548
|
content: [{ type: "text", text: await handleListModelProviders() }]
|
|
1378
1549
|
};
|
|
1379
1550
|
})
|
|
1380
1551
|
);
|
|
1552
|
+
}
|
|
1553
|
+
function registerAgentTools(server) {
|
|
1381
1554
|
server.tool(
|
|
1382
1555
|
"platform_list_agents",
|
|
1383
|
-
"List all agents in the LangWatch project with their names, types
|
|
1556
|
+
"List all agents in the LangWatch project with their names, types and IDs. A connected agent (one that registered itself from code with connectAgent or connect_agent) also shows its environment, whether it is online, how many instances are connected, and its owner.",
|
|
1384
1557
|
{},
|
|
1385
1558
|
withToolLogging("platform_list_agents", async () => {
|
|
1386
1559
|
requireApiKey();
|
|
1387
|
-
const { handleListAgents } = await import("./list-agents-
|
|
1560
|
+
const { handleListAgents } = await import("./list-agents-HVYWEQMI.js");
|
|
1388
1561
|
return {
|
|
1389
1562
|
content: [{ type: "text", text: await handleListAgents() }]
|
|
1390
1563
|
};
|
|
@@ -1392,13 +1565,13 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1392
1565
|
);
|
|
1393
1566
|
server.tool(
|
|
1394
1567
|
"platform_get_agent",
|
|
1395
|
-
"Get detailed information about a specific agent by its ID, including its configuration.",
|
|
1568
|
+
"Get detailed information about a specific agent by its ID, including its configuration. A connected agent also shows its environment, status, the run parameters it declared (with options and defaults) and the instances connected right now.",
|
|
1396
1569
|
{
|
|
1397
1570
|
id: z2.string().describe("The agent ID")
|
|
1398
1571
|
},
|
|
1399
1572
|
withToolLogging("platform_get_agent", async (params) => {
|
|
1400
1573
|
requireApiKey();
|
|
1401
|
-
const { handleGetAgent } = await import("./get-agent-
|
|
1574
|
+
const { handleGetAgent } = await import("./get-agent-OTIYUFZA.js");
|
|
1402
1575
|
return {
|
|
1403
1576
|
content: [{ type: "text", text: await handleGetAgent(params) }]
|
|
1404
1577
|
};
|
|
@@ -1406,7 +1579,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1406
1579
|
);
|
|
1407
1580
|
server.tool(
|
|
1408
1581
|
"platform_create_agent",
|
|
1409
|
-
"Create a new agent. Supported types: 'signature' (LLM prompt), 'code' (Python), 'workflow' (sub-workflow), 'http' (external API).",
|
|
1582
|
+
"Create a new agent. Supported types: 'signature' (LLM prompt), 'code' (Python), 'workflow' (sub-workflow), 'http' (external API). A 'connected' agent is not created here: it registers itself from code, with connectAgent from langwatch/agent in TypeScript or langwatch.connect_agent in Python, and appears in the list once that process runs.",
|
|
1410
1583
|
{
|
|
1411
1584
|
name: z2.string().describe("Agent name"),
|
|
1412
1585
|
type: z2.enum(["signature", "code", "workflow", "http"]).describe("Agent type"),
|
|
@@ -1414,10 +1587,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1414
1587
|
},
|
|
1415
1588
|
withToolLogging("platform_create_agent", async (params) => {
|
|
1416
1589
|
requireApiKey();
|
|
1417
|
-
const { handleCreateAgent } = await import("./create-agent-
|
|
1590
|
+
const { handleCreateAgent } = await import("./create-agent-AYZ3LB3V.js");
|
|
1418
1591
|
const parsedConfig = params.config ? JSON.parse(params.config) : void 0;
|
|
1419
1592
|
return {
|
|
1420
|
-
content: [
|
|
1593
|
+
content: [
|
|
1594
|
+
{ type: "text", text: await handleCreateAgent({ ...params, config: parsedConfig }) }
|
|
1595
|
+
]
|
|
1421
1596
|
};
|
|
1422
1597
|
})
|
|
1423
1598
|
);
|
|
@@ -1432,10 +1607,12 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1432
1607
|
},
|
|
1433
1608
|
withToolLogging("platform_update_agent", async (params) => {
|
|
1434
1609
|
requireApiKey();
|
|
1435
|
-
const { handleUpdateAgent } = await import("./update-agent-
|
|
1610
|
+
const { handleUpdateAgent } = await import("./update-agent-DWG43T2P.js");
|
|
1436
1611
|
const parsedConfig = params.config ? JSON.parse(params.config) : void 0;
|
|
1437
1612
|
return {
|
|
1438
|
-
content: [
|
|
1613
|
+
content: [
|
|
1614
|
+
{ type: "text", text: await handleUpdateAgent({ ...params, config: parsedConfig }) }
|
|
1615
|
+
]
|
|
1439
1616
|
};
|
|
1440
1617
|
})
|
|
1441
1618
|
);
|
|
@@ -1447,7 +1624,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1447
1624
|
},
|
|
1448
1625
|
withToolLogging("platform_delete_agent", async (params) => {
|
|
1449
1626
|
requireApiKey();
|
|
1450
|
-
const { handleDeleteAgent } = await import("./delete-agent-
|
|
1627
|
+
const { handleDeleteAgent } = await import("./delete-agent-QMEK3XHK.js");
|
|
1451
1628
|
return {
|
|
1452
1629
|
content: [{ type: "text", text: await handleDeleteAgent(params) }]
|
|
1453
1630
|
};
|
|
@@ -1455,26 +1632,46 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1455
1632
|
);
|
|
1456
1633
|
server.tool(
|
|
1457
1634
|
"platform_run_agent",
|
|
1458
|
-
"
|
|
1635
|
+
"Run one turn of an agent. A connected agent runs through the platform relay on a live instance: give `message` (one user turn) or `input` with a `messages` list, plus `parameters` and `threadId` to continue a conversation. HTTP agents call their configured URL directly; workflow-linked agents execute via the workflow engine.",
|
|
1459
1636
|
{
|
|
1460
1637
|
id: z2.string().describe("The agent ID to run"),
|
|
1461
|
-
input: z2.string().optional().describe(
|
|
1638
|
+
input: z2.string().optional().describe(
|
|
1639
|
+
"Input data as a JSON object string. For a connected agent it is the relay body: messages (OpenAI style), and optionally threadId, session and params."
|
|
1640
|
+
),
|
|
1641
|
+
message: z2.string().optional().describe("One user message to send to a connected agent, instead of input"),
|
|
1642
|
+
parameters: z2.record(z2.string(), z2.union([z2.string(), z2.number(), z2.boolean()])).optional().describe("Run parameter values for a connected agent, by name"),
|
|
1643
|
+
threadId: z2.string().optional().describe("Continue a conversation on a connected agent")
|
|
1462
1644
|
},
|
|
1463
1645
|
withToolLogging("platform_run_agent", async (params) => {
|
|
1464
1646
|
requireApiKey();
|
|
1465
|
-
const { handleRunAgent } = await import("./run-agent-
|
|
1647
|
+
const { handleRunAgent } = await import("./run-agent-W4VJP72Z.js");
|
|
1466
1648
|
return {
|
|
1467
1649
|
content: [{ type: "text", text: await handleRunAgent(params) }]
|
|
1468
1650
|
};
|
|
1469
1651
|
})
|
|
1470
1652
|
);
|
|
1653
|
+
server.tool(
|
|
1654
|
+
"platform_test_agent",
|
|
1655
|
+
'Test an agent with one scripted scenario run: the user sends "ping", the agent answers, and the run succeeds when the answer arrives. No model is used, and no scenario, run plan or test suite is added to the project. Answers at once with the scenario run id to follow with platform_get_simulation_run; the run itself is asynchronous.',
|
|
1656
|
+
{
|
|
1657
|
+
id: z2.string().describe("The agent ID to test")
|
|
1658
|
+
},
|
|
1659
|
+
withToolLogging("platform_test_agent", async (params) => {
|
|
1660
|
+
requireApiKey();
|
|
1661
|
+
return {
|
|
1662
|
+
content: [{ type: "text", text: await handleTestAgent(params) }]
|
|
1663
|
+
};
|
|
1664
|
+
})
|
|
1665
|
+
);
|
|
1666
|
+
}
|
|
1667
|
+
function registerDashboardTools(server) {
|
|
1471
1668
|
server.tool(
|
|
1472
1669
|
"platform_list_dashboards",
|
|
1473
1670
|
"List all analytics dashboards in the LangWatch project.",
|
|
1474
1671
|
{},
|
|
1475
1672
|
withToolLogging("platform_list_dashboards", async () => {
|
|
1476
1673
|
requireApiKey();
|
|
1477
|
-
const { handleListDashboards } = await import("./list-dashboards-
|
|
1674
|
+
const { handleListDashboards } = await import("./list-dashboards-ROSYFLDQ.js");
|
|
1478
1675
|
return {
|
|
1479
1676
|
content: [{ type: "text", text: await handleListDashboards() }]
|
|
1480
1677
|
};
|
|
@@ -1488,7 +1685,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1488
1685
|
},
|
|
1489
1686
|
withToolLogging("platform_get_dashboard", async (params) => {
|
|
1490
1687
|
requireApiKey();
|
|
1491
|
-
const { handleGetDashboard } = await import("./get-dashboard-
|
|
1688
|
+
const { handleGetDashboard } = await import("./get-dashboard-QMTYDQAZ.js");
|
|
1492
1689
|
return {
|
|
1493
1690
|
content: [{ type: "text", text: await handleGetDashboard(params) }]
|
|
1494
1691
|
};
|
|
@@ -1502,7 +1699,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1502
1699
|
},
|
|
1503
1700
|
withToolLogging("platform_create_dashboard", async (params) => {
|
|
1504
1701
|
requireApiKey();
|
|
1505
|
-
const { handleCreateDashboard } = await import("./create-dashboard-
|
|
1702
|
+
const { handleCreateDashboard } = await import("./create-dashboard-WTKKS5Z6.js");
|
|
1506
1703
|
return {
|
|
1507
1704
|
content: [{ type: "text", text: await handleCreateDashboard(params) }]
|
|
1508
1705
|
};
|
|
@@ -1516,7 +1713,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1516
1713
|
},
|
|
1517
1714
|
withToolLogging("platform_delete_dashboard", async (params) => {
|
|
1518
1715
|
requireApiKey();
|
|
1519
|
-
const { handleDeleteDashboard } = await import("./delete-dashboard-
|
|
1716
|
+
const { handleDeleteDashboard } = await import("./delete-dashboard-JVIVHNJM.js");
|
|
1520
1717
|
return {
|
|
1521
1718
|
content: [{ type: "text", text: await handleDeleteDashboard(params) }]
|
|
1522
1719
|
};
|
|
@@ -1531,20 +1728,99 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1531
1728
|
},
|
|
1532
1729
|
withToolLogging("platform_rename_dashboard", async (params) => {
|
|
1533
1730
|
requireApiKey();
|
|
1534
|
-
const { renameDashboard } = await import("./langwatch-api-dashboards-
|
|
1731
|
+
const { renameDashboard } = await import("./langwatch-api-dashboards-NHU5BZD7.js");
|
|
1535
1732
|
const result = await renameDashboard(params.id, { name: params.name });
|
|
1536
1733
|
return {
|
|
1537
|
-
content: [
|
|
1734
|
+
content: [
|
|
1735
|
+
{
|
|
1736
|
+
type: "text",
|
|
1737
|
+
text: `Dashboard "${result.name}" renamed successfully (ID: ${result.id}).`
|
|
1738
|
+
}
|
|
1739
|
+
]
|
|
1740
|
+
};
|
|
1741
|
+
})
|
|
1742
|
+
);
|
|
1743
|
+
server.tool(
|
|
1744
|
+
"list_dashboard_boards",
|
|
1745
|
+
"Lists the project's boards in the Dashboards area, with each one's ID and widget count. Use an ID from here as the dashboardId of add_dashboard_widget and update_dashboard_widget; the analytics dashboards platform_list_dashboards lists hold no widgets.",
|
|
1746
|
+
{},
|
|
1747
|
+
withToolLogging("list_dashboard_boards", async () => {
|
|
1748
|
+
requireApiKey();
|
|
1749
|
+
const { handleListDashboardBoards } = await import("./list-dashboard-boards-U2GJ7GC5.js");
|
|
1750
|
+
return {
|
|
1751
|
+
content: [{ type: "text", text: await handleListDashboardBoards() }]
|
|
1752
|
+
};
|
|
1753
|
+
})
|
|
1754
|
+
);
|
|
1755
|
+
server.tool(
|
|
1756
|
+
"add_dashboard_widget",
|
|
1757
|
+
"Creates a custom dashboard widget and places it on the given dashboard. A widget's `code` is a TSX file run in a sandbox: it receives a global `LW` with `LW.useChartQuery(name)` (reads one of the widget's own named queries), `LW.navigate` and `LW.theme`. `queries` are named LangWatchQL SQL statements the code can read by name \u2014 validate each statement's SQL first with run_query before calling this tool, since a query only runs inside the sandbox at render time and a mistake surfaces there, not here. Up to 8 queries per widget.",
|
|
1758
|
+
{
|
|
1759
|
+
dashboardId: z2.string().describe("The Dashboards board to add the widget to, as list_dashboard_boards lists it"),
|
|
1760
|
+
name: z2.string().describe("The widget's display name"),
|
|
1761
|
+
code: z2.string().describe(
|
|
1762
|
+
"The widget's TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope."
|
|
1763
|
+
),
|
|
1764
|
+
queries: z2.union([
|
|
1765
|
+
z2.record(z2.string(), z2.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1766
|
+
z2.array(
|
|
1767
|
+
z2.object({
|
|
1768
|
+
name: z2.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1769
|
+
sql: z2.string().describe("The LangWatchQL statement")
|
|
1770
|
+
})
|
|
1771
|
+
).describe("Named LWQL queries, array form")
|
|
1772
|
+
]).describe(
|
|
1773
|
+
"The widget's named LangWatchQL queries: either {name: sql} or an array of {name, sql}. Max 8. Validate each statement with run_query first."
|
|
1774
|
+
)
|
|
1775
|
+
},
|
|
1776
|
+
withToolLogging("add_dashboard_widget", async (params) => {
|
|
1777
|
+
requireApiKey();
|
|
1778
|
+
const { handleAddDashboardWidget } = await import("./add-dashboard-widget-4ZC5UN73.js");
|
|
1779
|
+
return {
|
|
1780
|
+
content: [{ type: "text", text: await handleAddDashboardWidget(params) }]
|
|
1781
|
+
};
|
|
1782
|
+
})
|
|
1783
|
+
);
|
|
1784
|
+
server.tool(
|
|
1785
|
+
"update_dashboard_widget",
|
|
1786
|
+
"Changes one custom widget on a dashboard: its name, its TSX `code`, its named LangWatchQL `queries` or its `description`. Pass only the fields to change; every field left out keeps its stored value. Validate any new query's SQL with run_query first. Returns the updated widget.",
|
|
1787
|
+
{
|
|
1788
|
+
dashboardId: z2.string().describe("The dashboard the widget is on"),
|
|
1789
|
+
widgetId: z2.string().describe("The widget to change, as add_dashboard_widget returned it"),
|
|
1790
|
+
name: z2.string().optional().describe("The widget's new display name"),
|
|
1791
|
+
code: z2.string().optional().describe(
|
|
1792
|
+
"The widget's new TSX source. Runs sandboxed with a global LW (LW.useChartQuery(name), LW.navigate, LW.theme) in scope."
|
|
1793
|
+
),
|
|
1794
|
+
queries: z2.union([
|
|
1795
|
+
z2.record(z2.string(), z2.string()).describe('{"<name>": "<LWQL SQL>"}'),
|
|
1796
|
+
z2.array(
|
|
1797
|
+
z2.object({
|
|
1798
|
+
name: z2.string().describe('Referenced from code as LW.useChartQuery("name")'),
|
|
1799
|
+
sql: z2.string().describe("The LangWatchQL statement")
|
|
1800
|
+
})
|
|
1801
|
+
).describe("Named LWQL queries, array form")
|
|
1802
|
+
]).optional().describe(
|
|
1803
|
+
"The widget's new named LangWatchQL queries, replacing the stored set: either {name: sql} or an array of {name, sql}. Max 8."
|
|
1804
|
+
),
|
|
1805
|
+
description: z2.string().optional().describe("What the widget's info tip says: a sentence or two")
|
|
1806
|
+
},
|
|
1807
|
+
withToolLogging("update_dashboard_widget", async (params) => {
|
|
1808
|
+
requireApiKey();
|
|
1809
|
+
const { handleUpdateDashboardWidget } = await import("./update-dashboard-widget-CJKJLCDY.js");
|
|
1810
|
+
return {
|
|
1811
|
+
content: [{ type: "text", text: await handleUpdateDashboardWidget(params) }]
|
|
1538
1812
|
};
|
|
1539
1813
|
})
|
|
1540
1814
|
);
|
|
1815
|
+
}
|
|
1816
|
+
function registerWorkflowTools(server) {
|
|
1541
1817
|
server.tool(
|
|
1542
1818
|
"platform_list_workflows",
|
|
1543
1819
|
"List all workflows in the LangWatch project.",
|
|
1544
1820
|
{},
|
|
1545
1821
|
withToolLogging("platform_list_workflows", async () => {
|
|
1546
1822
|
requireApiKey();
|
|
1547
|
-
const { handleListWorkflows } = await import("./list-workflows-
|
|
1823
|
+
const { handleListWorkflows } = await import("./list-workflows-IMSX7SXU.js");
|
|
1548
1824
|
return {
|
|
1549
1825
|
content: [{ type: "text", text: await handleListWorkflows() }]
|
|
1550
1826
|
};
|
|
@@ -1558,7 +1834,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1558
1834
|
},
|
|
1559
1835
|
withToolLogging("platform_get_workflow", async (params) => {
|
|
1560
1836
|
requireApiKey();
|
|
1561
|
-
const { handleGetWorkflow } = await import("./get-workflow-
|
|
1837
|
+
const { handleGetWorkflow } = await import("./get-workflow-YXULW4GC.js");
|
|
1562
1838
|
return {
|
|
1563
1839
|
content: [{ type: "text", text: await handleGetWorkflow(params) }]
|
|
1564
1840
|
};
|
|
@@ -1572,7 +1848,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1572
1848
|
},
|
|
1573
1849
|
withToolLogging("platform_delete_workflow", async (params) => {
|
|
1574
1850
|
requireApiKey();
|
|
1575
|
-
const { handleDeleteWorkflow } = await import("./delete-workflow-
|
|
1851
|
+
const { handleDeleteWorkflow } = await import("./delete-workflow-5J46T73T.js");
|
|
1576
1852
|
return {
|
|
1577
1853
|
content: [{ type: "text", text: await handleDeleteWorkflow(params) }]
|
|
1578
1854
|
};
|
|
@@ -1587,12 +1863,14 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1587
1863
|
},
|
|
1588
1864
|
withToolLogging("platform_run_workflow", async (params) => {
|
|
1589
1865
|
requireApiKey();
|
|
1590
|
-
const { handleRunWorkflow } = await import("./run-workflow-
|
|
1866
|
+
const { handleRunWorkflow } = await import("./run-workflow-D6PQNBI6.js");
|
|
1591
1867
|
return {
|
|
1592
1868
|
content: [{ type: "text", text: await handleRunWorkflow(params) }]
|
|
1593
1869
|
};
|
|
1594
1870
|
})
|
|
1595
1871
|
);
|
|
1872
|
+
}
|
|
1873
|
+
function registerAnnotationTools(server) {
|
|
1596
1874
|
server.tool(
|
|
1597
1875
|
"platform_list_annotations",
|
|
1598
1876
|
"List all annotations for the project, optionally filtered by trace ID.",
|
|
@@ -1601,7 +1879,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1601
1879
|
},
|
|
1602
1880
|
withToolLogging("platform_list_annotations", async (params) => {
|
|
1603
1881
|
requireApiKey();
|
|
1604
|
-
const { handleListAnnotations } = await import("./list-annotations-
|
|
1882
|
+
const { handleListAnnotations } = await import("./list-annotations-J4UAHSUN.js");
|
|
1605
1883
|
return {
|
|
1606
1884
|
content: [{ type: "text", text: await handleListAnnotations(params) }]
|
|
1607
1885
|
};
|
|
@@ -1618,7 +1896,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1618
1896
|
},
|
|
1619
1897
|
withToolLogging("platform_create_annotation", async (params) => {
|
|
1620
1898
|
requireApiKey();
|
|
1621
|
-
const { handleCreateAnnotation } = await import("./create-annotation-
|
|
1899
|
+
const { handleCreateAnnotation } = await import("./create-annotation-VUSXULQK.js");
|
|
1622
1900
|
return {
|
|
1623
1901
|
content: [{ type: "text", text: await handleCreateAnnotation(params) }]
|
|
1624
1902
|
};
|
|
@@ -1632,7 +1910,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1632
1910
|
},
|
|
1633
1911
|
withToolLogging("platform_delete_annotation", async (params) => {
|
|
1634
1912
|
requireApiKey();
|
|
1635
|
-
const { handleDeleteAnnotation } = await import("./delete-annotation-
|
|
1913
|
+
const { handleDeleteAnnotation } = await import("./delete-annotation-VKIQWUYW.js");
|
|
1636
1914
|
return {
|
|
1637
1915
|
content: [{ type: "text", text: await handleDeleteAnnotation(params) }]
|
|
1638
1916
|
};
|
|
@@ -1646,7 +1924,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1646
1924
|
},
|
|
1647
1925
|
withToolLogging("platform_get_annotation", async (params) => {
|
|
1648
1926
|
requireApiKey();
|
|
1649
|
-
const { getAnnotation } = await import("./langwatch-api-annotations-
|
|
1927
|
+
const { getAnnotation } = await import("./langwatch-api-annotations-TBHLCQ5R.js");
|
|
1650
1928
|
const annotation = await getAnnotation(params.id);
|
|
1651
1929
|
const lines = [
|
|
1652
1930
|
`**ID**: ${annotation.id}`,
|
|
@@ -1660,6 +1938,8 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1660
1938
|
};
|
|
1661
1939
|
})
|
|
1662
1940
|
);
|
|
1941
|
+
}
|
|
1942
|
+
function registerTriggerTools(server) {
|
|
1663
1943
|
server.tool(
|
|
1664
1944
|
"platform_list_triggers",
|
|
1665
1945
|
"List all triggers (automations) in the project. Triggers automate actions like sending emails, Slack messages, or adding to datasets when conditions are met.",
|
|
@@ -1668,73 +1948,128 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1668
1948
|
},
|
|
1669
1949
|
withToolLogging("platform_list_triggers", async (params) => {
|
|
1670
1950
|
requireApiKey();
|
|
1671
|
-
const {
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
if (t.alertType) lines.push(`**Alert**: ${t.alertType}`);
|
|
1687
|
-
lines.push("");
|
|
1688
|
-
}
|
|
1689
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
1951
|
+
const { handleListTriggers } = await import("./list-triggers-V4Y356AK.js");
|
|
1952
|
+
return { content: [{ type: "text", text: await handleListTriggers(params) }] };
|
|
1953
|
+
})
|
|
1954
|
+
);
|
|
1955
|
+
server.tool(
|
|
1956
|
+
"platform_get_trigger",
|
|
1957
|
+
"Read one trigger (automation) by its ID, including its delivery configuration. Credential values come back as [redacted]; sending them back unchanged on an update keeps the stored values.",
|
|
1958
|
+
{
|
|
1959
|
+
id: z2.string().describe("The trigger ID")
|
|
1960
|
+
},
|
|
1961
|
+
withToolLogging("platform_get_trigger", async (params) => {
|
|
1962
|
+
requireApiKey();
|
|
1963
|
+
const { getTrigger } = await import("./langwatch-api-triggers-QS7HDIVW.js");
|
|
1964
|
+
const trigger = await getTrigger(params.id);
|
|
1965
|
+
return { content: [{ type: "text", text: JSON.stringify(trigger, null, 2) }] };
|
|
1690
1966
|
})
|
|
1691
1967
|
);
|
|
1692
1968
|
server.tool(
|
|
1693
1969
|
"platform_create_trigger",
|
|
1694
|
-
|
|
1970
|
+
[
|
|
1971
|
+
"Create a trigger (automation).",
|
|
1972
|
+
"",
|
|
1973
|
+
"It is about one of three things: matching traces (send `filters` or `filterQuery`), a metric crossing a threshold on a graph (an Alert: send `customGraphId`, `graphAlert` and `alertType`), or a schedule (a Report: send `report`, on an email or Slack channel).",
|
|
1974
|
+
"`actionParams` is the delivery configuration for the channel named in `action` \u2014 recipients for SEND_EMAIL, a destination for SEND_SLACK_MESSAGE, a URL for SEND_WEBHOOK, a dataset and mapping for ADD_TO_DATASET, annotators for ADD_TO_ANNOTATION_QUEUE. Omitted, it is sent as {} and the server refuses a channel that cannot deliver without one.",
|
|
1975
|
+
SLACK_DELIVERY_NOTE,
|
|
1976
|
+
`filters: ${TRIGGER_FILTERS_DESCRIPTION}`,
|
|
1977
|
+
"The delivery channel cannot be changed afterwards."
|
|
1978
|
+
].join("\n"),
|
|
1695
1979
|
{
|
|
1696
1980
|
name: z2.string().describe("Trigger name"),
|
|
1697
|
-
action:
|
|
1698
|
-
|
|
1981
|
+
action: triggerActionSchema.describe("Which channel it delivers on"),
|
|
1982
|
+
actionParams: actionParamsSchema.optional().describe("The delivery configuration the channel named in `action` reads. Defaults to {}"),
|
|
1983
|
+
filters: z2.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
1984
|
+
filterQuery: z2.string().optional().describe(TRIGGER_FILTER_QUERY_DESCRIPTION),
|
|
1985
|
+
customGraphId: z2.string().optional().describe("Set to make this an alert on that graph"),
|
|
1986
|
+
graphAlert: graphAlertSchema.optional(),
|
|
1987
|
+
report: reportSchema.optional().describe("Set to make this a scheduled report"),
|
|
1988
|
+
templates: templatesSchema.optional(),
|
|
1989
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
1699
1990
|
message: z2.string().optional().describe("Custom alert message"),
|
|
1700
|
-
alertType:
|
|
1991
|
+
alertType: alertTypeSchema.optional().describe("Alert severity. Required for a graph alert.")
|
|
1701
1992
|
},
|
|
1702
1993
|
withToolLogging("platform_create_trigger", async (params) => {
|
|
1703
1994
|
requireApiKey();
|
|
1704
|
-
const {
|
|
1705
|
-
|
|
1706
|
-
if (params.filters) {
|
|
1707
|
-
try {
|
|
1708
|
-
filters = JSON.parse(params.filters);
|
|
1709
|
-
} catch {
|
|
1710
|
-
return { content: [{ type: "text", text: "Error: filters must be valid JSON" }] };
|
|
1711
|
-
}
|
|
1712
|
-
}
|
|
1713
|
-
const trigger = await createTrigger({
|
|
1714
|
-
name: params.name,
|
|
1715
|
-
action: params.action,
|
|
1716
|
-
filters,
|
|
1717
|
-
message: params.message,
|
|
1718
|
-
alertType: params.alertType
|
|
1719
|
-
});
|
|
1720
|
-
return { content: [{ type: "text", text: `Trigger "${trigger.name}" created (ID: ${trigger.id}, Action: ${trigger.action}).` }] };
|
|
1995
|
+
const { handleCreateTrigger } = await import("./create-trigger-5HZSCRAN.js");
|
|
1996
|
+
return handleCreateTrigger(params);
|
|
1721
1997
|
})
|
|
1722
1998
|
);
|
|
1723
1999
|
server.tool(
|
|
1724
2000
|
"platform_update_trigger",
|
|
1725
|
-
|
|
2001
|
+
[
|
|
2002
|
+
"Update a trigger (automation). Anything left out is left alone.",
|
|
2003
|
+
"",
|
|
2004
|
+
"`actionParams` replaces the delivery configuration as a whole, so send every field it should have from now on. A credential the read hid comes back as [redacted]; send that back to keep the stored value. Changing a webhook's `url` means sending its header values in the same call.",
|
|
2005
|
+
"The delivery channel, the kind of automation and an alert's graph cannot be changed."
|
|
2006
|
+
].join("\n"),
|
|
1726
2007
|
{
|
|
1727
2008
|
id: z2.string().describe("The trigger ID"),
|
|
1728
2009
|
name: z2.string().optional().describe("New name"),
|
|
1729
|
-
active: z2.boolean().optional().describe("
|
|
1730
|
-
|
|
1731
|
-
|
|
2010
|
+
active: z2.boolean().optional().describe("Resume or pause it"),
|
|
2011
|
+
actionParams: actionParamsSchema.optional().describe("The delivery configuration this automation should have from now on"),
|
|
2012
|
+
filters: z2.string().optional().describe(TRIGGER_FILTERS_DESCRIPTION),
|
|
2013
|
+
filterQuery: z2.string().nullable().optional().describe(`${TRIGGER_FILTER_QUERY_DESCRIPTION} null clears the saved query.`),
|
|
2014
|
+
graphAlert: graphAlertSchema.optional().describe("Only for an automation that is already a graph alert"),
|
|
2015
|
+
report: reportSchema.optional().describe("Only for an automation that is already a scheduled report"),
|
|
2016
|
+
templates: templatesSchema.optional(),
|
|
2017
|
+
notificationCadence: notificationCadenceSchema.optional(),
|
|
2018
|
+
message: z2.string().nullable().optional().describe("New alert message. null removes the custom message"),
|
|
2019
|
+
alertType: alertTypeSchema.nullable().optional().describe("New alert severity. null clears it")
|
|
1732
2020
|
},
|
|
1733
2021
|
withToolLogging("platform_update_trigger", async (params) => {
|
|
1734
2022
|
requireApiKey();
|
|
1735
|
-
const { updateTrigger } = await import("./langwatch-api-triggers-
|
|
1736
|
-
|
|
1737
|
-
|
|
2023
|
+
const { getTrigger, updateTrigger } = await import("./langwatch-api-triggers-QS7HDIVW.js");
|
|
2024
|
+
if (params.actionParams !== void 0) {
|
|
2025
|
+
const saved = await getTrigger(params.id);
|
|
2026
|
+
const boundParams = validateActionParamsForAction({
|
|
2027
|
+
action: saved.action,
|
|
2028
|
+
actionParams: params.actionParams
|
|
2029
|
+
});
|
|
2030
|
+
if (!boundParams.ok) {
|
|
2031
|
+
const { toolError: toolError2 } = await import("./create-trigger-5HZSCRAN.js");
|
|
2032
|
+
return toolError2(boundParams.message);
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
const { parseJsonObject, toolError } = await import("./create-trigger-5HZSCRAN.js");
|
|
2036
|
+
const filters = params.filters ? parseJsonObject(params.filters) : void 0;
|
|
2037
|
+
if (params.filters && !filters) {
|
|
2038
|
+
return toolError("filters must be a JSON object");
|
|
2039
|
+
}
|
|
2040
|
+
const trigger = await updateTrigger({ ...params, filters });
|
|
2041
|
+
return {
|
|
2042
|
+
content: [
|
|
2043
|
+
{ type: "text", text: `Trigger "${trigger.name}" updated (active: ${trigger.active}).` }
|
|
2044
|
+
]
|
|
2045
|
+
};
|
|
2046
|
+
})
|
|
2047
|
+
);
|
|
2048
|
+
server.tool(
|
|
2049
|
+
"platform_test_fire_trigger",
|
|
2050
|
+
"Send a trigger's message to the destination it is configured with, to confirm it arrives. Nothing is recorded as a fire.",
|
|
2051
|
+
{
|
|
2052
|
+
id: z2.string().describe("The trigger ID")
|
|
2053
|
+
},
|
|
2054
|
+
withToolLogging("platform_test_fire_trigger", async (params) => {
|
|
2055
|
+
requireApiKey();
|
|
2056
|
+
const { testFireTrigger } = await import("./langwatch-api-triggers-QS7HDIVW.js");
|
|
2057
|
+
const result = await testFireTrigger(params.id);
|
|
2058
|
+
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
2059
|
+
})
|
|
2060
|
+
);
|
|
2061
|
+
server.tool(
|
|
2062
|
+
"platform_list_trigger_fires",
|
|
2063
|
+
"What an automation has done, newest first. Metadata only \u2014 no message content. Answers with `nextCursor`; pass it back as `cursor` for the next page (null means there is none).",
|
|
2064
|
+
{
|
|
2065
|
+
id: z2.string().describe("The trigger ID"),
|
|
2066
|
+
limit: z2.number().int().positive().optional().describe("How many fires to return"),
|
|
2067
|
+
cursor: z2.string().optional().describe("The `nextCursor` a previous call answered with")
|
|
2068
|
+
},
|
|
2069
|
+
withToolLogging("platform_list_trigger_fires", async (params) => {
|
|
2070
|
+
requireApiKey();
|
|
2071
|
+
const { handleListTriggerFires } = await import("./list-trigger-fires-WTW4DVIW.js");
|
|
2072
|
+
return { content: [{ type: "text", text: await handleListTriggerFires(params) }] };
|
|
1738
2073
|
})
|
|
1739
2074
|
);
|
|
1740
2075
|
server.tool(
|
|
@@ -1745,18 +2080,20 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
|
|
|
1745
2080
|
},
|
|
1746
2081
|
withToolLogging("platform_delete_trigger", async (params) => {
|
|
1747
2082
|
requireApiKey();
|
|
1748
|
-
const { deleteTrigger } = await import("./langwatch-api-triggers-
|
|
2083
|
+
const { deleteTrigger } = await import("./langwatch-api-triggers-QS7HDIVW.js");
|
|
1749
2084
|
const result = await deleteTrigger(params.id);
|
|
1750
2085
|
return { content: [{ type: "text", text: `Trigger ${result.id} deleted.` }] };
|
|
1751
2086
|
})
|
|
1752
2087
|
);
|
|
2088
|
+
}
|
|
2089
|
+
function registerMonitorTools(server) {
|
|
1753
2090
|
server.tool(
|
|
1754
2091
|
"platform_list_monitors",
|
|
1755
2092
|
"List all online evaluation monitors for the project.",
|
|
1756
2093
|
{},
|
|
1757
2094
|
withToolLogging("platform_list_monitors", async () => {
|
|
1758
2095
|
requireApiKey();
|
|
1759
|
-
const { listMonitors } = await import("./langwatch-api-monitors-
|
|
2096
|
+
const { listMonitors } = await import("./langwatch-api-monitors-YNWA2M5Y.js");
|
|
1760
2097
|
const monitors = await listMonitors();
|
|
1761
2098
|
if (monitors.length === 0) {
|
|
1762
2099
|
return { content: [{ type: "text", text: "No monitors found." }] };
|
|
@@ -1783,7 +2120,7 @@ ${lines.join("\n")}`
|
|
|
1783
2120
|
},
|
|
1784
2121
|
withToolLogging("platform_get_monitor", async (params) => {
|
|
1785
2122
|
requireApiKey();
|
|
1786
|
-
const { getMonitor } = await import("./langwatch-api-monitors-
|
|
2123
|
+
const { getMonitor } = await import("./langwatch-api-monitors-YNWA2M5Y.js");
|
|
1787
2124
|
const m = await getMonitor(params.id);
|
|
1788
2125
|
const lines = [
|
|
1789
2126
|
`**${m.name}** (${m.id})`,
|
|
@@ -1810,7 +2147,7 @@ ${lines.join("\n")}`
|
|
|
1810
2147
|
},
|
|
1811
2148
|
withToolLogging("platform_create_monitor", async (params) => {
|
|
1812
2149
|
requireApiKey();
|
|
1813
|
-
const { createMonitor } = await import("./langwatch-api-monitors-
|
|
2150
|
+
const { createMonitor } = await import("./langwatch-api-monitors-YNWA2M5Y.js");
|
|
1814
2151
|
const monitor = await createMonitor(params);
|
|
1815
2152
|
return {
|
|
1816
2153
|
content: [
|
|
@@ -1834,7 +2171,7 @@ ${lines.join("\n")}`
|
|
|
1834
2171
|
},
|
|
1835
2172
|
withToolLogging("platform_update_monitor", async (params) => {
|
|
1836
2173
|
requireApiKey();
|
|
1837
|
-
const { updateMonitor } = await import("./langwatch-api-monitors-
|
|
2174
|
+
const { updateMonitor } = await import("./langwatch-api-monitors-YNWA2M5Y.js");
|
|
1838
2175
|
const monitor = await updateMonitor(params);
|
|
1839
2176
|
return {
|
|
1840
2177
|
content: [
|
|
@@ -1854,29 +2191,27 @@ ${lines.join("\n")}`
|
|
|
1854
2191
|
},
|
|
1855
2192
|
withToolLogging("platform_delete_monitor", async (params) => {
|
|
1856
2193
|
requireApiKey();
|
|
1857
|
-
const { deleteMonitor } = await import("./langwatch-api-monitors-
|
|
2194
|
+
const { deleteMonitor } = await import("./langwatch-api-monitors-YNWA2M5Y.js");
|
|
1858
2195
|
const result = await deleteMonitor(params.id);
|
|
1859
2196
|
return {
|
|
1860
|
-
content: [
|
|
1861
|
-
{ type: "text", text: `Monitor ${result.id} deleted.` }
|
|
1862
|
-
]
|
|
2197
|
+
content: [{ type: "text", text: `Monitor ${result.id} deleted.` }]
|
|
1863
2198
|
};
|
|
1864
2199
|
})
|
|
1865
2200
|
);
|
|
2201
|
+
}
|
|
2202
|
+
function registerSecretTools(server) {
|
|
1866
2203
|
server.tool(
|
|
1867
2204
|
"platform_list_secrets",
|
|
1868
2205
|
"List all project secrets (values are never returned, only metadata).",
|
|
1869
2206
|
{},
|
|
1870
2207
|
withToolLogging("platform_list_secrets", async () => {
|
|
1871
2208
|
requireApiKey();
|
|
1872
|
-
const { listSecrets } = await import("./langwatch-api-secrets-
|
|
2209
|
+
const { listSecrets } = await import("./langwatch-api-secrets-ADTOJELP.js");
|
|
1873
2210
|
const secrets = await listSecrets();
|
|
1874
2211
|
if (secrets.length === 0) {
|
|
1875
2212
|
return { content: [{ type: "text", text: "No secrets found." }] };
|
|
1876
2213
|
}
|
|
1877
|
-
const lines = secrets.map(
|
|
1878
|
-
(s) => `\u2022 ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`
|
|
1879
|
-
);
|
|
2214
|
+
const lines = secrets.map((s) => `\u2022 ${s.name} (id: ${s.id}, updated: ${s.updatedAt})`);
|
|
1880
2215
|
return {
|
|
1881
2216
|
content: [
|
|
1882
2217
|
{
|
|
@@ -1897,7 +2232,7 @@ ${lines.join("\n")}`
|
|
|
1897
2232
|
},
|
|
1898
2233
|
withToolLogging("platform_create_secret", async (params) => {
|
|
1899
2234
|
requireApiKey();
|
|
1900
|
-
const { createSecret } = await import("./langwatch-api-secrets-
|
|
2235
|
+
const { createSecret } = await import("./langwatch-api-secrets-ADTOJELP.js");
|
|
1901
2236
|
const secret = await createSecret({ name: params.name, value: params.value });
|
|
1902
2237
|
return {
|
|
1903
2238
|
content: [
|
|
@@ -1918,7 +2253,7 @@ ${lines.join("\n")}`
|
|
|
1918
2253
|
},
|
|
1919
2254
|
withToolLogging("platform_update_secret", async (params) => {
|
|
1920
2255
|
requireApiKey();
|
|
1921
|
-
const { updateSecret } = await import("./langwatch-api-secrets-
|
|
2256
|
+
const { updateSecret } = await import("./langwatch-api-secrets-ADTOJELP.js");
|
|
1922
2257
|
const secret = await updateSecret({ id: params.id, value: params.value });
|
|
1923
2258
|
return {
|
|
1924
2259
|
content: [
|
|
@@ -1938,11 +2273,13 @@ ${lines.join("\n")}`
|
|
|
1938
2273
|
},
|
|
1939
2274
|
withToolLogging("platform_delete_secret", async (params) => {
|
|
1940
2275
|
requireApiKey();
|
|
1941
|
-
const { deleteSecret } = await import("./langwatch-api-secrets-
|
|
2276
|
+
const { deleteSecret } = await import("./langwatch-api-secrets-ADTOJELP.js");
|
|
1942
2277
|
const result = await deleteSecret(params.id);
|
|
1943
2278
|
return { content: [{ type: "text", text: `Secret ${result.id} deleted.` }] };
|
|
1944
2279
|
})
|
|
1945
2280
|
);
|
|
2281
|
+
}
|
|
2282
|
+
function registerExperimentTools(server) {
|
|
1946
2283
|
server.tool(
|
|
1947
2284
|
"platform_run_experiment",
|
|
1948
2285
|
"Start an experiment run by slug. Returns a run ID for polling status.",
|
|
@@ -1976,9 +2313,7 @@ ${lines.join("\n")}`
|
|
|
1976
2313
|
"platform_experiment_list",
|
|
1977
2314
|
"List experiments configured in the LangWatch project. Returns a markdown table with each experiment's slug, name, last-run timestamp, and run count. Use this to discover experiment slugs before drilling into runs with platform_experiment_list_runs. Output is capped (default 25, max 100) to protect the agent's context window.",
|
|
1978
2315
|
{
|
|
1979
|
-
limit: z2.number().int().positive().max(100).optional().describe(
|
|
1980
|
-
"Maximum number of experiments to include (default 25, hard-capped at 100)"
|
|
1981
|
-
)
|
|
2316
|
+
limit: z2.number().int().positive().max(100).optional().describe("Maximum number of experiments to include (default 25, hard-capped at 100)")
|
|
1982
2317
|
},
|
|
1983
2318
|
withToolLogging("platform_experiment_list", async (params) => {
|
|
1984
2319
|
requireApiKey();
|
|
@@ -1991,12 +2326,8 @@ ${lines.join("\n")}`
|
|
|
1991
2326
|
"platform_experiment_list_runs",
|
|
1992
2327
|
"List experiment runs for a specific experiment slug. Returns a markdown table with each run's id, status, started/finished timestamps, and pass-rate summary. Use this to discover run ids before calling platform_experiment_results. Output is capped (default 25, max 100) to protect the agent's context window.",
|
|
1993
2328
|
{
|
|
1994
|
-
experimentSlug: z2.string().describe(
|
|
1995
|
-
|
|
1996
|
-
),
|
|
1997
|
-
limit: z2.number().int().positive().max(100).optional().describe(
|
|
1998
|
-
"Maximum number of runs to include (default 25, hard-capped at 100)"
|
|
1999
|
-
)
|
|
2329
|
+
experimentSlug: z2.string().describe("Experiment slug from platform_experiment_list (e.g. 'checkout-flow')"),
|
|
2330
|
+
limit: z2.number().int().positive().max(100).optional().describe("Maximum number of runs to include (default 25, hard-capped at 100)")
|
|
2000
2331
|
},
|
|
2001
2332
|
withToolLogging("platform_experiment_list_runs", async (params) => {
|
|
2002
2333
|
requireApiKey();
|
|
@@ -2024,24 +2355,22 @@ ${lines.join("\n")}`
|
|
|
2024
2355
|
withToolLogging("platform_experiment_results", async (params) => {
|
|
2025
2356
|
requireApiKey();
|
|
2026
2357
|
return {
|
|
2027
|
-
content: [
|
|
2028
|
-
{ type: "text", text: await handleExperimentResults(params) }
|
|
2029
|
-
]
|
|
2358
|
+
content: [{ type: "text", text: await handleExperimentResults(params) }]
|
|
2030
2359
|
};
|
|
2031
2360
|
})
|
|
2032
2361
|
);
|
|
2362
|
+
}
|
|
2363
|
+
function registerDatasetTools(server) {
|
|
2033
2364
|
server.tool(
|
|
2034
2365
|
"platform_list_datasets",
|
|
2035
2366
|
"List all datasets on the LangWatch platform with their names, slugs, columns, and record counts. Returns AI-readable digest by default.",
|
|
2036
2367
|
{
|
|
2037
|
-
format: z2.enum(["digest", "json"]).optional().describe(
|
|
2038
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
2039
|
-
)
|
|
2368
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
2040
2369
|
},
|
|
2041
2370
|
async (params) => {
|
|
2042
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2371
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2043
2372
|
requireApiKey2();
|
|
2044
|
-
const { handleListDatasets } = await import("./list-datasets-
|
|
2373
|
+
const { handleListDatasets } = await import("./list-datasets-B5X4P7BQ.js");
|
|
2045
2374
|
return {
|
|
2046
2375
|
content: [{ type: "text", text: await handleListDatasets(params) }]
|
|
2047
2376
|
};
|
|
@@ -2052,14 +2381,12 @@ ${lines.join("\n")}`
|
|
|
2052
2381
|
"Get full details of a dataset on the LangWatch platform by slug or ID, including column definitions and a preview of records.",
|
|
2053
2382
|
{
|
|
2054
2383
|
slugOrId: z2.string().describe("The dataset slug or ID to retrieve"),
|
|
2055
|
-
format: z2.enum(["digest", "json"]).optional().describe(
|
|
2056
|
-
"Output format: 'digest' (default, AI-readable) or 'json' (full raw data)"
|
|
2057
|
-
)
|
|
2384
|
+
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
2058
2385
|
},
|
|
2059
2386
|
async (params) => {
|
|
2060
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2387
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2061
2388
|
requireApiKey2();
|
|
2062
|
-
const { handleGetDataset } = await import("./get-dataset-
|
|
2389
|
+
const { handleGetDataset } = await import("./get-dataset-ALSP7RTX.js");
|
|
2063
2390
|
return {
|
|
2064
2391
|
content: [{ type: "text", text: await handleGetDataset(params) }]
|
|
2065
2392
|
};
|
|
@@ -2070,9 +2397,9 @@ ${lines.join("\n")}`
|
|
|
2070
2397
|
"Create a new dataset on the LangWatch platform.",
|
|
2071
2398
|
createDatasetSchema.shape,
|
|
2072
2399
|
async (params) => {
|
|
2073
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2400
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2074
2401
|
requireApiKey2();
|
|
2075
|
-
const { handleCreateDataset } = await import("./create-dataset-
|
|
2402
|
+
const { handleCreateDataset } = await import("./create-dataset-2MC4SD4O.js");
|
|
2076
2403
|
return {
|
|
2077
2404
|
content: [{ type: "text", text: await handleCreateDataset(params) }]
|
|
2078
2405
|
};
|
|
@@ -2087,9 +2414,9 @@ ${lines.join("\n")}`
|
|
|
2087
2414
|
columnTypes: z2.array(datasetColumnDefinitionSchema).optional().describe("Updated column definitions")
|
|
2088
2415
|
},
|
|
2089
2416
|
async (params) => {
|
|
2090
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2417
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2091
2418
|
requireApiKey2();
|
|
2092
|
-
const { handleUpdateDataset } = await import("./update-dataset-
|
|
2419
|
+
const { handleUpdateDataset } = await import("./update-dataset-SQZ6T4WX.js");
|
|
2093
2420
|
return {
|
|
2094
2421
|
content: [{ type: "text", text: await handleUpdateDataset(params) }]
|
|
2095
2422
|
};
|
|
@@ -2102,9 +2429,9 @@ ${lines.join("\n")}`
|
|
|
2102
2429
|
slugOrId: z2.string().describe("The dataset slug or ID to delete")
|
|
2103
2430
|
},
|
|
2104
2431
|
async (params) => {
|
|
2105
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2432
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2106
2433
|
requireApiKey2();
|
|
2107
|
-
const { handleDeleteDataset } = await import("./delete-dataset-
|
|
2434
|
+
const { handleDeleteDataset } = await import("./delete-dataset-WKRUW4FP.js");
|
|
2108
2435
|
return {
|
|
2109
2436
|
content: [{ type: "text", text: await handleDeleteDataset(params) }]
|
|
2110
2437
|
};
|
|
@@ -2120,13 +2447,11 @@ ${lines.join("\n")}`
|
|
|
2120
2447
|
format: z2.enum(["digest", "json"]).optional().describe("Output format: 'digest' (default, AI-readable) or 'json' (full raw data)")
|
|
2121
2448
|
},
|
|
2122
2449
|
async (params) => {
|
|
2123
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2450
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2124
2451
|
requireApiKey2();
|
|
2125
|
-
const { handleListDatasetRecords } = await import("./list-dataset-records-
|
|
2452
|
+
const { handleListDatasetRecords } = await import("./list-dataset-records-M5TKA6LK.js");
|
|
2126
2453
|
return {
|
|
2127
|
-
content: [
|
|
2128
|
-
{ type: "text", text: await handleListDatasetRecords(params) }
|
|
2129
|
-
]
|
|
2454
|
+
content: [{ type: "text", text: await handleListDatasetRecords(params) }]
|
|
2130
2455
|
};
|
|
2131
2456
|
}
|
|
2132
2457
|
);
|
|
@@ -2138,13 +2463,11 @@ ${lines.join("\n")}`
|
|
|
2138
2463
|
entries: z2.array(z2.record(z2.string(), z2.unknown())).min(1).max(1e3).describe("Array of record entries to create (key-value objects matching dataset columns)")
|
|
2139
2464
|
},
|
|
2140
2465
|
async (params) => {
|
|
2141
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2466
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2142
2467
|
requireApiKey2();
|
|
2143
|
-
const { handleCreateDatasetRecords } = await import("./create-dataset-records-
|
|
2468
|
+
const { handleCreateDatasetRecords } = await import("./create-dataset-records-W4GLN2FV.js");
|
|
2144
2469
|
return {
|
|
2145
|
-
content: [
|
|
2146
|
-
{ type: "text", text: await handleCreateDatasetRecords(params) }
|
|
2147
|
-
]
|
|
2470
|
+
content: [{ type: "text", text: await handleCreateDatasetRecords(params) }]
|
|
2148
2471
|
};
|
|
2149
2472
|
}
|
|
2150
2473
|
);
|
|
@@ -2157,13 +2480,11 @@ ${lines.join("\n")}`
|
|
|
2157
2480
|
entry: z2.record(z2.string(), z2.unknown()).describe("Updated record entry (key-value object)")
|
|
2158
2481
|
},
|
|
2159
2482
|
async (params) => {
|
|
2160
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2483
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2161
2484
|
requireApiKey2();
|
|
2162
|
-
const { handleUpdateDatasetRecord } = await import("./update-dataset-record-
|
|
2485
|
+
const { handleUpdateDatasetRecord } = await import("./update-dataset-record-3AT4IPBC.js");
|
|
2163
2486
|
return {
|
|
2164
|
-
content: [
|
|
2165
|
-
{ type: "text", text: await handleUpdateDatasetRecord(params) }
|
|
2166
|
-
]
|
|
2487
|
+
content: [{ type: "text", text: await handleUpdateDatasetRecord(params) }]
|
|
2167
2488
|
};
|
|
2168
2489
|
}
|
|
2169
2490
|
);
|
|
@@ -2175,16 +2496,16 @@ ${lines.join("\n")}`
|
|
|
2175
2496
|
recordIds: z2.array(z2.string()).min(1).max(1e3).describe("Array of record IDs to delete")
|
|
2176
2497
|
},
|
|
2177
2498
|
async (params) => {
|
|
2178
|
-
const { requireApiKey: requireApiKey2 } = await import("./config.js");
|
|
2499
|
+
const { requireApiKey: requireApiKey2 } = await import("./config-EAKW6F26.js");
|
|
2179
2500
|
requireApiKey2();
|
|
2180
|
-
const { handleDeleteDatasetRecords } = await import("./delete-dataset-records-
|
|
2501
|
+
const { handleDeleteDatasetRecords } = await import("./delete-dataset-records-C2IQFFVQ.js");
|
|
2181
2502
|
return {
|
|
2182
|
-
content: [
|
|
2183
|
-
{ type: "text", text: await handleDeleteDatasetRecords(params) }
|
|
2184
|
-
]
|
|
2503
|
+
content: [{ type: "text", text: await handleDeleteDatasetRecords(params) }]
|
|
2185
2504
|
};
|
|
2186
2505
|
}
|
|
2187
2506
|
);
|
|
2507
|
+
}
|
|
2508
|
+
function registerProjectTools(server) {
|
|
2188
2509
|
server.tool(
|
|
2189
2510
|
"platform_list_projects",
|
|
2190
2511
|
"List all projects in your LangWatch organization. Requires an org-level API key.",
|
|
@@ -2194,7 +2515,7 @@ ${lines.join("\n")}`
|
|
|
2194
2515
|
},
|
|
2195
2516
|
withToolLogging("platform_list_projects", async (params) => {
|
|
2196
2517
|
requireApiKey();
|
|
2197
|
-
const { handleListProjects } = await import("./list-projects-
|
|
2518
|
+
const { handleListProjects } = await import("./list-projects-6EJ5YYEW.js");
|
|
2198
2519
|
return {
|
|
2199
2520
|
content: [{ type: "text", text: await handleListProjects(params) }]
|
|
2200
2521
|
};
|
|
@@ -2208,7 +2529,7 @@ ${lines.join("\n")}`
|
|
|
2208
2529
|
},
|
|
2209
2530
|
withToolLogging("platform_get_project", async (params) => {
|
|
2210
2531
|
requireApiKey();
|
|
2211
|
-
const { handleGetProject } = await import("./get-project-
|
|
2532
|
+
const { handleGetProject } = await import("./get-project-5VKROSDH.js");
|
|
2212
2533
|
return {
|
|
2213
2534
|
content: [{ type: "text", text: await handleGetProject(params) }]
|
|
2214
2535
|
};
|
|
@@ -2228,7 +2549,7 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2228
2549
|
},
|
|
2229
2550
|
withToolLogging("platform_create_project", async (params) => {
|
|
2230
2551
|
requireApiKey();
|
|
2231
|
-
const { handleCreateProject } = await import("./create-project-
|
|
2552
|
+
const { handleCreateProject } = await import("./create-project-KAIC46UY.js");
|
|
2232
2553
|
return {
|
|
2233
2554
|
content: [{ type: "text", text: await handleCreateProject(params) }]
|
|
2234
2555
|
};
|
|
@@ -2246,7 +2567,7 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2246
2567
|
},
|
|
2247
2568
|
withToolLogging("platform_update_project", async (params) => {
|
|
2248
2569
|
requireApiKey();
|
|
2249
|
-
const { handleUpdateProject } = await import("./update-project-
|
|
2570
|
+
const { handleUpdateProject } = await import("./update-project-PLJIOZS6.js");
|
|
2250
2571
|
return {
|
|
2251
2572
|
content: [{ type: "text", text: await handleUpdateProject(params) }]
|
|
2252
2573
|
};
|
|
@@ -2260,19 +2581,21 @@ You must provide either teamId (to add the project to an existing team) or newTe
|
|
|
2260
2581
|
},
|
|
2261
2582
|
withToolLogging("platform_archive_project", async (params) => {
|
|
2262
2583
|
requireApiKey();
|
|
2263
|
-
const { handleArchiveProject } = await import("./archive-project-
|
|
2584
|
+
const { handleArchiveProject } = await import("./archive-project-SPVHXK6A.js");
|
|
2264
2585
|
return {
|
|
2265
2586
|
content: [{ type: "text", text: await handleArchiveProject(params) }]
|
|
2266
2587
|
};
|
|
2267
2588
|
})
|
|
2268
2589
|
);
|
|
2590
|
+
}
|
|
2591
|
+
function registerApiKeyTools(server) {
|
|
2269
2592
|
server.tool(
|
|
2270
2593
|
"platform_list_api_keys",
|
|
2271
2594
|
"List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
|
|
2272
2595
|
{},
|
|
2273
2596
|
withToolLogging("platform_list_api_keys", async () => {
|
|
2274
2597
|
requireApiKey();
|
|
2275
|
-
const { handleListApiKeys } = await import("./list-api-keys-
|
|
2598
|
+
const { handleListApiKeys } = await import("./list-api-keys-7UTJRZDG.js");
|
|
2276
2599
|
return {
|
|
2277
2600
|
content: [{ type: "text", text: await handleListApiKeys() }]
|
|
2278
2601
|
};
|
|
@@ -2291,16 +2614,18 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2291
2614
|
name: z2.string().describe("Key name (max 100 chars)"),
|
|
2292
2615
|
description: z2.string().optional().describe("Key description (max 500 chars)"),
|
|
2293
2616
|
expiresAt: z2.string().optional().describe("Expiration date (ISO string)"),
|
|
2294
|
-
bindings: z2.array(
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2617
|
+
bindings: z2.array(
|
|
2618
|
+
z2.object({
|
|
2619
|
+
role: z2.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
|
|
2620
|
+
scopeType: z2.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
|
|
2621
|
+
scopeId: z2.string().describe("Scope ID (org, team, or project ID)")
|
|
2622
|
+
})
|
|
2623
|
+
).optional().describe("Role bindings (required for personal keys)"),
|
|
2299
2624
|
projectIds: z2.array(z2.string()).optional().describe("Project IDs for service keys (creates ADMIN bindings per project)")
|
|
2300
2625
|
},
|
|
2301
2626
|
withToolLogging("platform_create_api_key", async (params) => {
|
|
2302
2627
|
requireApiKey();
|
|
2303
|
-
const { handleCreateApiKey } = await import("./create-api-key-
|
|
2628
|
+
const { handleCreateApiKey } = await import("./create-api-key-NNYKARMG.js");
|
|
2304
2629
|
return {
|
|
2305
2630
|
content: [{ type: "text", text: await handleCreateApiKey(params) }]
|
|
2306
2631
|
};
|
|
@@ -2314,7 +2639,7 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2314
2639
|
},
|
|
2315
2640
|
withToolLogging("platform_revoke_api_key", async (params) => {
|
|
2316
2641
|
requireApiKey();
|
|
2317
|
-
const { handleRevokeApiKey } = await import("./revoke-api-key-
|
|
2642
|
+
const { handleRevokeApiKey } = await import("./revoke-api-key-SFMEFWJC.js");
|
|
2318
2643
|
return {
|
|
2319
2644
|
content: [{ type: "text", text: await handleRevokeApiKey(params) }]
|
|
2320
2645
|
};
|
|
@@ -2325,4 +2650,4 @@ The token is returned once and cannot be retrieved again.`,
|
|
|
2325
2650
|
export {
|
|
2326
2651
|
createMcpServer
|
|
2327
2652
|
};
|
|
2328
|
-
//# sourceMappingURL=chunk-
|
|
2653
|
+
//# sourceMappingURL=chunk-J7PQKBLD.js.map
|