@langwatch/mcp-server 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +280 -200
- package/CONTRIBUTING.md +10 -0
- package/README.md +82 -52
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/archive-run-plan-65QN3XRX.js +16 -0
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/archive-test-suite-NKZDZY3D.js +16 -0
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/chunk-6YZUOD34.js +55 -0
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-7RHG56OH.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-TOXJQDIZ.js → chunk-HGELNIVY.js} +135 -51
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-J42FWRJ2.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-AQJO2EI5.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-4QC7SA2L.js → chunk-J7PQKBLD.js} +884 -559
- package/dist/{chunk-YX3A3CFM.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-5CCZFUXS.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/chunk-O7EMWEVJ.js +44 -0
- package/dist/chunk-PTDR56Z3.js +37 -0
- package/dist/chunk-RABYBFX5.js +43 -0
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-SFB4MU6H.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/chunk-U24EACC4.js +67 -0
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-ZWIL4EFL.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/chunk-VMWJD732.js +57 -0
- package/dist/chunk-XUIJYNRJ.js +15 -0
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-7XHO6UIS.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-YQXRQC6A.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-YUS36KBU.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-E6NMSVX7.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -5
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-CS2CB57Y.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-F5RMX3L3.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-JDR2VB2H.js → create-scenario-UEM77OE7.js} +11 -4
- package/dist/create-test-suite-4IPXVSFQ.js +37 -0
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-O3EEH4S5.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/discover-scenario-schema-OOFIQKOG.js +139 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/get-agent-OTIYUFZA.js +70 -0
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-RAK7URQJ.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/get-prompt-OWXJZHQQ.js +95 -0
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/get-run-plan-BE3M2AUX.js +59 -0
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-YM4ODWW6.js} +8 -4
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-LCFSMPGQ.js} +33 -20
- package/dist/get-test-suite-GC2O7YH6.js +49 -0
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-K7U234Q4.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-L5L2PSK3.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-YXHFPPGA.js → http-server-IR4QOS43.js} +4071 -342
- package/dist/index.js +60221 -38077
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/list-agents-HVYWEQMI.js +43 -0
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-K7QAIX2I.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-SC2HAUFG.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/list-run-plans-PX2XI6EK.js +46 -0
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/list-scenarios-QZRPVHS4.js +46 -0
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-OEADLLQR.js} +19 -9
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/list-test-suites-N4WVLBKK.js +35 -0
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-QSJWURCK.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/rename-test-suite-4J7UX5S2.js +21 -0
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/report-issue-DCZFLGDO.js +899 -0
- package/dist/rerun-run-plan-TG2V4WM4.js +22 -0
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/run-plan-PAARQSI4.js +39 -0
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/run-test-suite-SQJELEZX.js +26 -0
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-QPXC627O.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/{run-agent-6SM4XNA7.js → run-workflow-SP32SO5M.js} +10 -11
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-SMAEHLXH.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-BDUK7TUO.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-LKVYXENH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/update-prompt-MY263DYF.js +141 -0
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-NEXCAQI7.js → update-scenario-MOHRQKAQ.js} +11 -4
- package/package.json +39 -44
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +1021 -596
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +202 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +140 -0
- package/src/langwatch-api-scenarios.ts +17 -10
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +10 -7
- package/src/langwatch-api-test-suites.ts +116 -0
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +143 -109
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +76 -0
- package/src/schemas/suite-fields.ts +132 -0
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +10 -0
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +10 -0
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +11 -5
- package/src/tools/create-test-suite.ts +38 -0
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +86 -12
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +54 -0
- package/src/tools/format-scenario.ts +12 -0
- package/src/tools/format-suite-details.ts +90 -0
- package/src/tools/get-agent.ts +54 -3
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +100 -37
- package/src/tools/get-run-plan.ts +60 -0
- package/src/tools/get-scenario.ts +6 -5
- package/src/tools/get-simulation-run.ts +32 -20
- package/src/tools/get-test-suite.ts +46 -0
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +26 -17
- package/src/tools/list-simulation-runs.ts +13 -5
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +15 -0
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +18 -0
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +65 -14
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +46 -0
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +27 -0
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +185 -15
- package/src/tools/update-scenario.ts +11 -5
- package/src/tools/update-test-suite.ts +40 -0
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +19 -12
- package/uv.lock +416 -296
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/archive-suite-4T6BSRS7.js +0 -16
- package/dist/chunk-77BNWQ3U.js +0 -86
- package/dist/chunk-A2LEVYQ5.js +0 -217
- package/dist/chunk-TJCXFNDT.js +0 -24
- package/dist/chunk-YZLYHLLI.js +0 -48
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/create-suite-LZS6QIAF.js +0 -39
- package/dist/delete-evaluator-33SP2JFX.js +0 -16
- package/dist/discover-evaluator-schema-XTAD4X7D.js +0 -2191
- package/dist/discover-scenario-schema-FE64QK2D.js +0 -67
- package/dist/get-agent-Y7MWZ65F.js +0 -29
- package/dist/get-analytics-TETCSOK6.js +0 -73
- package/dist/get-dataset-RCMU5VAH.js +0 -49
- package/dist/get-prompt-OY5B7TRI.js +0 -46
- package/dist/get-suite-J7JVWO6O.js +0 -45
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-BUBSNCNO.js +0 -41
- package/dist/langwatch-api-triggers-EDFUCJN6.js +0 -31
- package/dist/list-agents-KFHKGPXV.js +0 -33
- package/dist/list-scenarios-F2WYNRIE.js +0 -41
- package/dist/list-suites-CRY4VWW6.js +0 -43
- package/dist/report-issue-XYVAPAYJ.js +0 -412
- package/dist/run-suite-NKZCJ2IN.js +0 -30
- package/dist/search-traces-A6HFKPLJ.js +0 -79
- package/dist/update-prompt-ODCRQRN6.js +0 -25
- package/dist/update-suite-SNO3GS7T.js +0 -40
- package/src/__tests__/all-tools.integration.test.ts +0 -1626
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -288
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -188
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/tools.unit.test.ts +0 -924
- package/src/langwatch-api-suites.ts +0 -118
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -4
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
|
@@ -20,16 +20,76 @@ export function formatScenarioSchema(): string {
|
|
|
20
20
|
lines.push(
|
|
21
21
|
'- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")',
|
|
22
22
|
);
|
|
23
|
+
lines.push(
|
|
24
|
+
"- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite.",
|
|
25
|
+
);
|
|
26
|
+
lines.push(
|
|
27
|
+
'- **fields** (object): The scenario\'s value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.',
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
lines.push("\n## Test Suite Fields\n");
|
|
31
|
+
lines.push(
|
|
32
|
+
"A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:",
|
|
33
|
+
);
|
|
34
|
+
lines.push("```json");
|
|
35
|
+
lines.push(
|
|
36
|
+
'{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }',
|
|
37
|
+
);
|
|
38
|
+
lines.push("```");
|
|
39
|
+
lines.push(
|
|
40
|
+
"- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved.",
|
|
41
|
+
);
|
|
42
|
+
lines.push("- **type**: text, number or boolean.");
|
|
43
|
+
lines.push(
|
|
44
|
+
"- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed.",
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
lines.push("\n## Evaluator Attachments\n");
|
|
48
|
+
lines.push(
|
|
49
|
+
"A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates.",
|
|
50
|
+
);
|
|
51
|
+
lines.push("```json");
|
|
52
|
+
lines.push('{ "evaluators": [{');
|
|
53
|
+
lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
|
|
54
|
+
lines.push(' "required": true,');
|
|
55
|
+
lines.push(' "mappings": {');
|
|
56
|
+
lines.push(
|
|
57
|
+
' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },',
|
|
58
|
+
);
|
|
59
|
+
lines.push(
|
|
60
|
+
' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }',
|
|
61
|
+
);
|
|
62
|
+
lines.push(" }");
|
|
63
|
+
lines.push("}] }");
|
|
64
|
+
lines.push("```");
|
|
65
|
+
lines.push(
|
|
66
|
+
"Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:",
|
|
67
|
+
);
|
|
68
|
+
lines.push(
|
|
69
|
+
'- **conversation**: `["first_user_message"]`, `["last_agent_message"]`, `["transcript"]` (role: content lines) or `["messages"]` (JSON)',
|
|
70
|
+
);
|
|
71
|
+
lines.push(
|
|
72
|
+
'- **scenario**: `["situation"]`, `["criteria"]` or `["fields", "<identifier>"]` (a field the suite declares; not on a run plan)',
|
|
73
|
+
);
|
|
74
|
+
lines.push(
|
|
75
|
+
'- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)',
|
|
76
|
+
);
|
|
77
|
+
lines.push('- a literal: `{ "type": "value", "value": "..." }`');
|
|
78
|
+
lines.push(
|
|
79
|
+
"A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`.",
|
|
80
|
+
);
|
|
23
81
|
|
|
24
82
|
lines.push("\n## Writing a Good Situation\n");
|
|
83
|
+
lines.push("The situation drives the user simulator. Include these elements:");
|
|
25
84
|
lines.push(
|
|
26
|
-
"
|
|
85
|
+
"- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)",
|
|
27
86
|
);
|
|
28
|
-
lines.push("- **Persona**: Who is the user? (e.g., a stressed small business owner, a confused teenager)");
|
|
29
87
|
lines.push("- **Emotional state**: How are they feeling? (e.g., frustrated, anxious, impatient)");
|
|
30
88
|
lines.push("- **Background/Context**: What happened before this conversation?");
|
|
31
89
|
lines.push("- **Intent**: What do they want to accomplish?");
|
|
32
|
-
lines.push(
|
|
90
|
+
lines.push(
|
|
91
|
+
"- **Constraints**: What limitations do they have? (e.g., no phone for 2FA, unfamiliar with technical terms)",
|
|
92
|
+
);
|
|
33
93
|
lines.push("\nExample:");
|
|
34
94
|
lines.push("```");
|
|
35
95
|
lines.push("User is a small business owner stressed about tax deadline.");
|
|
@@ -40,17 +100,31 @@ export function formatScenarioSchema(): string {
|
|
|
40
100
|
lines.push("```");
|
|
41
101
|
|
|
42
102
|
lines.push("\n## Writing Good Criteria\n");
|
|
43
|
-
lines.push(
|
|
44
|
-
|
|
45
|
-
|
|
103
|
+
lines.push(
|
|
104
|
+
"Criteria are what the judge uses to pass or fail the agent. Each criterion should be:",
|
|
105
|
+
);
|
|
106
|
+
lines.push('- **Specific and testable** — not vague like "responds helpfully"');
|
|
107
|
+
lines.push(
|
|
108
|
+
"- **Behavioral** — describes what the agent should *do*, not how it works internally",
|
|
109
|
+
);
|
|
46
110
|
lines.push("- **Independent** — each criterion checks one thing");
|
|
47
111
|
lines.push("\nGood criteria patterns:");
|
|
48
|
-
lines.push(
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
lines.push(
|
|
52
|
-
|
|
53
|
-
|
|
112
|
+
lines.push(
|
|
113
|
+
'- **Information gathering**: "Agent asks for the user\'s account number before proceeding"',
|
|
114
|
+
);
|
|
115
|
+
lines.push(
|
|
116
|
+
'- **Safety/guardrails**: "Agent does not reveal internal system details or error stack traces"',
|
|
117
|
+
);
|
|
118
|
+
lines.push(
|
|
119
|
+
'- **Clarification**: "Agent asks clarifying questions before taking irreversible action"',
|
|
120
|
+
);
|
|
121
|
+
lines.push('- **Tone**: "Agent maintains a professional and empathetic tone throughout"');
|
|
122
|
+
lines.push(
|
|
123
|
+
'- **Completeness**: "Agent confirms the user understands the solution before ending"',
|
|
124
|
+
);
|
|
125
|
+
lines.push(
|
|
126
|
+
'- **Domain-specific**: "Agent recommends releasing a wild frog rather than keeping it as a pet"',
|
|
127
|
+
);
|
|
54
128
|
lines.push("\nAvoid vague criteria like:");
|
|
55
129
|
lines.push('- "Responds correctly" — correct how?');
|
|
56
130
|
lines.push('- "Is helpful" — helpful in what way?');
|
|
@@ -1,25 +1,39 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { getQueryReference, type QueryReferenceResponse } from "../langwatch-api-query.ts";
|
|
2
|
+
import { analyticsGroups } from "../schemas/analytics-groups.ts";
|
|
3
|
+
import { analyticsMetrics } from "../schemas/analytics-metrics.ts";
|
|
4
|
+
import { markdownTable } from "../utils/markdown-table.ts";
|
|
4
5
|
|
|
5
|
-
export type Category =
|
|
6
|
-
| "filters"
|
|
7
|
-
| "metrics"
|
|
8
|
-
| "aggregations"
|
|
9
|
-
| "groups"
|
|
10
|
-
| "all";
|
|
6
|
+
export type Category = "filters" | "lwql" | "metrics" | "aggregations" | "groups" | "all";
|
|
11
7
|
|
|
12
8
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* (filter fields, metrics, aggregation types, or group-by options).
|
|
9
|
+
* Categories the platform answers, not this package — which is why
|
|
10
|
+
* `formatSchema` is async: they are read from `GET /api/v1/query/reference`
|
|
11
|
+
* and need the API key.
|
|
17
12
|
*/
|
|
18
|
-
|
|
13
|
+
const PLATFORM_BACKED: ReadonlySet<Category> = new Set(["filters", "lwql", "all"]);
|
|
14
|
+
|
|
15
|
+
export function needsQueryReference(category: Category): boolean {
|
|
16
|
+
return PLATFORM_BACKED.has(category);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Formats the LangWatch query schema into markdown an agent can act on.
|
|
21
|
+
* `reference` is required for the platform-backed categories and ignored by
|
|
22
|
+
* the others, so a caller that fetched it once can reuse it.
|
|
23
|
+
*/
|
|
24
|
+
export async function formatSchema(
|
|
25
|
+
category: Category,
|
|
26
|
+
reference?: QueryReferenceResponse,
|
|
27
|
+
): Promise<string> {
|
|
19
28
|
const sections: string[] = [];
|
|
29
|
+
const resolved =
|
|
30
|
+
needsQueryReference(category) && !reference ? await getQueryReference() : reference;
|
|
20
31
|
|
|
21
32
|
if (category === "filters" || category === "all") {
|
|
22
|
-
sections.push(formatFilters());
|
|
33
|
+
sections.push(formatFilters(resolved));
|
|
34
|
+
}
|
|
35
|
+
if (category === "lwql" || category === "all") {
|
|
36
|
+
sections.push(formatLangWatchQL(resolved));
|
|
23
37
|
}
|
|
24
38
|
if (category === "metrics" || category === "all") {
|
|
25
39
|
sections.push(formatMetrics());
|
|
@@ -30,30 +44,140 @@ export function formatSchema(category: Category): string {
|
|
|
30
44
|
if (category === "groups" || category === "all") {
|
|
31
45
|
sections.push(formatGroups());
|
|
32
46
|
}
|
|
47
|
+
if (category === "all" && resolved) {
|
|
48
|
+
sections.push(formatDecisionTable(resolved));
|
|
49
|
+
}
|
|
33
50
|
|
|
34
51
|
return sections.join("\n\n");
|
|
35
52
|
}
|
|
36
53
|
|
|
37
|
-
|
|
38
|
-
|
|
54
|
+
/** What the field's values column says: the closed set, or where to read an open one. */
|
|
55
|
+
const knownValuesCell = (field: { knownValues: readonly string[]; facetable: boolean }): string => {
|
|
56
|
+
if (field.knownValues.length > 0) return field.knownValues.join(", ");
|
|
57
|
+
return field.facetable ? "open set, read them from GET /api/v1/traces/facets" : "";
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
function formatFilters(reference?: QueryReferenceResponse): string {
|
|
61
|
+
if (!reference) {
|
|
62
|
+
return "## Trace Filter Fields\n\nUnavailable: the platform could not be reached.";
|
|
63
|
+
}
|
|
64
|
+
const lines = ["## Trace Filter Fields", ""];
|
|
65
|
+
lines.push(
|
|
66
|
+
"Send one of these as the `filter` parameter of `search_traces`, in the language the Trace Explorer's search bar speaks: `status:error AND model:gpt-*`.",
|
|
67
|
+
);
|
|
68
|
+
lines.push(
|
|
69
|
+
"The `filters` parameter is the older per-field map and still works; both are combined when you send both.",
|
|
70
|
+
);
|
|
71
|
+
lines.push("");
|
|
72
|
+
lines.push(
|
|
73
|
+
markdownTable({
|
|
74
|
+
headers: ["Field", "Type", "Group", "Known values"],
|
|
75
|
+
rows: reference.traceFilter.fields.map((field) => ({
|
|
76
|
+
Field: field.name,
|
|
77
|
+
Type: field.valueType,
|
|
78
|
+
Group: field.group ?? "",
|
|
79
|
+
"Known values": knownValuesCell(field),
|
|
80
|
+
})),
|
|
81
|
+
}),
|
|
82
|
+
);
|
|
83
|
+
lines.push("");
|
|
84
|
+
lines.push("### Attribute namespaces", "");
|
|
39
85
|
lines.push(
|
|
40
|
-
|
|
86
|
+
markdownTable({
|
|
87
|
+
headers: ["Prefix", "Matches", "Older spellings"],
|
|
88
|
+
rows: reference.traceFilter.dynamicPrefixes.map((prefix) => ({
|
|
89
|
+
Prefix: `${prefix.prefix}<key>`,
|
|
90
|
+
Matches: prefix.description,
|
|
91
|
+
"Older spellings": prefix.aliases.join(", "),
|
|
92
|
+
})),
|
|
93
|
+
}),
|
|
41
94
|
);
|
|
42
|
-
lines.push('Format: `{ "field_name": ["value1", "value2"] }`');
|
|
43
95
|
lines.push("");
|
|
44
|
-
|
|
96
|
+
lines.push("### Syntax", "");
|
|
97
|
+
lines.push(reference.traceFilter.syntax);
|
|
98
|
+
lines.push("");
|
|
99
|
+
lines.push("### Worked filters", "");
|
|
100
|
+
for (const example of reference.examples.filter(
|
|
101
|
+
(candidate) => candidate.language === "trace-filter",
|
|
102
|
+
)) {
|
|
103
|
+
lines.push(`- \`${example.text}\` — ${example.title}`);
|
|
104
|
+
}
|
|
105
|
+
return lines.join("\n");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function formatLangWatchQL(reference?: QueryReferenceResponse): string {
|
|
109
|
+
if (!reference) {
|
|
110
|
+
return "## Analytics SQL\n\nUnavailable: the platform could not be reached.";
|
|
111
|
+
}
|
|
112
|
+
if (!reference.lwql.enabled) {
|
|
113
|
+
return [
|
|
114
|
+
"## Analytics SQL",
|
|
115
|
+
"",
|
|
116
|
+
"Not enabled for this project, so `run_query` will refuse. Use `search_traces` and `get_analytics` instead.",
|
|
117
|
+
].join("\n");
|
|
118
|
+
}
|
|
119
|
+
const lines = ["## Analytics SQL", ""];
|
|
120
|
+
lines.push(
|
|
121
|
+
"Run one read-only SELECT with `run_query`. The statement is executed as written: nothing rewrites it.",
|
|
122
|
+
);
|
|
123
|
+
lines.push(
|
|
124
|
+
`Ceilings: ${reference.lwql.limits.maxRowsReturned} rows, ${reference.lwql.limits.maxExecutionTimeSeconds} seconds. ${reference.lwql.limits.pagination}`,
|
|
125
|
+
);
|
|
126
|
+
lines.push("");
|
|
127
|
+
for (const view of reference.lwql.schema.views) {
|
|
128
|
+
lines.push(`### ${view.name}`);
|
|
129
|
+
lines.push(`${view.description} One row is: ${view.grain}`);
|
|
130
|
+
lines.push(
|
|
131
|
+
`Filter on \`${view.timeColumn}\` to prune partitions. Join keys: ${view.joinKeys.join(", ") || "none"}.`,
|
|
132
|
+
);
|
|
133
|
+
lines.push("");
|
|
45
134
|
lines.push(
|
|
46
|
-
|
|
135
|
+
markdownTable({
|
|
136
|
+
headers: ["Column", "Type", "Available", "Description"],
|
|
137
|
+
rows: view.columns.map((column) => ({
|
|
138
|
+
Column: column.name,
|
|
139
|
+
Type: column.type,
|
|
140
|
+
Available: column.available ? "yes" : `needs ${column.gates.join(", ")}`,
|
|
141
|
+
Description: column.description,
|
|
142
|
+
})),
|
|
143
|
+
}),
|
|
47
144
|
);
|
|
145
|
+
lines.push("");
|
|
146
|
+
}
|
|
147
|
+
lines.push("### Worked statements", "");
|
|
148
|
+
for (const example of reference.examples.filter((candidate) => candidate.language === "lwql")) {
|
|
149
|
+
lines.push(
|
|
150
|
+
`**${example.title}**${example.available ? "" : ` (needs ${example.requires.gates.join(", ")})`}`,
|
|
151
|
+
);
|
|
152
|
+
lines.push("```sql");
|
|
153
|
+
lines.push(example.text);
|
|
154
|
+
lines.push("```");
|
|
155
|
+
for (const parameter of example.parameters) {
|
|
156
|
+
lines.push(`- \`{${parameter.name}:${parameter.type}}\` — ${parameter.description}`);
|
|
157
|
+
}
|
|
158
|
+
lines.push("");
|
|
48
159
|
}
|
|
49
160
|
return lines.join("\n");
|
|
50
161
|
}
|
|
51
162
|
|
|
52
|
-
function
|
|
53
|
-
const lines = ["##
|
|
163
|
+
function formatDecisionTable(reference: QueryReferenceResponse): string {
|
|
164
|
+
const lines = ["## Which one to reach for", ""];
|
|
54
165
|
lines.push(
|
|
55
|
-
|
|
166
|
+
markdownTable({
|
|
167
|
+
headers: ["When you want", "Use", "Why"],
|
|
168
|
+
rows: reference.decisionTable.map((row) => ({
|
|
169
|
+
"When you want": row.when,
|
|
170
|
+
Use: row.use,
|
|
171
|
+
Why: row.why,
|
|
172
|
+
})),
|
|
173
|
+
}),
|
|
56
174
|
);
|
|
175
|
+
return lines.join("\n");
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function formatMetrics(): string {
|
|
179
|
+
const lines = ["## Available Metrics", ""];
|
|
180
|
+
lines.push("Use these in `get_analytics` as `metric` parameter in `category.name` format.");
|
|
57
181
|
lines.push("");
|
|
58
182
|
|
|
59
183
|
const byCategory = new Map<string, typeof analyticsMetrics>();
|
|
@@ -95,9 +219,7 @@ function formatAggregations(): string {
|
|
|
95
219
|
|
|
96
220
|
function formatGroups(): string {
|
|
97
221
|
const lines = ["## Available Group-By Options", ""];
|
|
98
|
-
lines.push(
|
|
99
|
-
"Use these in the `groupBy` parameter of `get_analytics`."
|
|
100
|
-
);
|
|
222
|
+
lines.push("Use these in the `groupBy` parameter of `get_analytics`.");
|
|
101
223
|
lines.push("");
|
|
102
224
|
for (const g of analyticsGroups) {
|
|
103
225
|
lines.push(`- **${g.name}** (${g.label}): ${g.description}`);
|
|
@@ -15,16 +15,10 @@ export interface RunTimestamps {
|
|
|
15
15
|
stoppedAt?: number | null;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
export const deriveRunStatus = (
|
|
19
|
-
timestamps: RunTimestamps,
|
|
20
|
-
now: number = Date.now(),
|
|
21
|
-
): RunStatus => {
|
|
18
|
+
export const deriveRunStatus = (timestamps: RunTimestamps, now: number = Date.now()): RunStatus => {
|
|
22
19
|
if (timestamps.stoppedAt != null) return "stopped";
|
|
23
20
|
if (timestamps.finishedAt != null) return "completed";
|
|
24
|
-
if (
|
|
25
|
-
timestamps.updatedAt != null &&
|
|
26
|
-
now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS
|
|
27
|
-
) {
|
|
21
|
+
if (timestamps.updatedAt != null && now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS) {
|
|
28
22
|
return "interrupted";
|
|
29
23
|
}
|
|
30
24
|
return "running";
|
|
@@ -1,16 +1,11 @@
|
|
|
1
|
-
import type { DatasetMutationResponse } from "../langwatch-api-datasets.
|
|
2
|
-
import { escapeMarkdown } from "../utils/escape-markdown.
|
|
1
|
+
import type { DatasetMutationResponse } from "../langwatch-api-datasets.ts";
|
|
2
|
+
import { escapeMarkdown } from "../utils/escape-markdown.ts";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Formats the common detail lines
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* Used by both create-dataset and update-dataset handlers to avoid
|
|
9
|
-
* duplicating the formatting logic.
|
|
5
|
+
* Formats the common detail lines (name, slug, id, column types) for a
|
|
6
|
+
* dataset mutation response, shared by create-dataset and update-dataset.
|
|
10
7
|
*/
|
|
11
|
-
export function formatDatasetMutationDetails(
|
|
12
|
-
result: DatasetMutationResponse,
|
|
13
|
-
): string[] {
|
|
8
|
+
export function formatDatasetMutationDetails(result: DatasetMutationResponse): string[] {
|
|
14
9
|
const lines: string[] = [];
|
|
15
10
|
lines.push(`**Name**: ${escapeMarkdown(result.name)}`);
|
|
16
11
|
lines.push(`**Slug**: ${escapeMarkdown(result.slug)}`);
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { RunPlanRunResult, RunPlanScope } from "../langwatch-api-run-plans.ts";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The digest every run of a plan returns, whichever tool started it.
|
|
5
|
+
* Says whether the run created the plan or joined one that already
|
|
6
|
+
* carried the name — a new plan vs. one this run just reconfigured.
|
|
7
|
+
*/
|
|
8
|
+
export function formatRunPlanRun(result: RunPlanRunResult): string {
|
|
9
|
+
const lines: string[] = [];
|
|
10
|
+
|
|
11
|
+
lines.push(
|
|
12
|
+
result.created
|
|
13
|
+
? `Run plan "${result.planName}" created and started.`
|
|
14
|
+
: `Run plan "${result.planName}" started with the configuration of this run.`,
|
|
15
|
+
);
|
|
16
|
+
lines.push("");
|
|
17
|
+
lines.push(`**Plan**: ${result.planName} (${result.runPlanId})`);
|
|
18
|
+
lines.push(`**Batch Run ID**: ${result.batchRunId}`);
|
|
19
|
+
lines.push(`**Set ID**: ${result.setId}`);
|
|
20
|
+
lines.push(`**Jobs**: ${result.jobCount}`);
|
|
21
|
+
|
|
22
|
+
if (result.skippedArchived.scenarios.length > 0) {
|
|
23
|
+
lines.push(`**Skipped archived scenarios**: ${result.skippedArchived.scenarios.join(", ")}`);
|
|
24
|
+
}
|
|
25
|
+
if (result.skippedArchived.targets.length > 0) {
|
|
26
|
+
lines.push(`**Skipped archived targets**: ${result.skippedArchived.targets.join(", ")}`);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
lines.push(`**View**: ${result.platformUrl}`);
|
|
30
|
+
lines.push("");
|
|
31
|
+
lines.push("> Use `platform_list_simulation_runs` with the batch run ID to read the results.");
|
|
32
|
+
|
|
33
|
+
return lines.join("\n");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* One line saying what a plan covers. A plan stored before scopes existed
|
|
38
|
+
* carries none, and runs the scenario list it already held.
|
|
39
|
+
*/
|
|
40
|
+
export function describeRunPlanScope(scope: RunPlanScope | null, scenarioIds: string[]): string {
|
|
41
|
+
if (scope === null) {
|
|
42
|
+
return `hand-picked scenarios (${scenarioIds.length})`;
|
|
43
|
+
}
|
|
44
|
+
switch (scope.mode) {
|
|
45
|
+
case "all":
|
|
46
|
+
return "every scenario in the project";
|
|
47
|
+
case "test_suites":
|
|
48
|
+
return `test suites: ${scope.testSuiteIds.join(", ")}`;
|
|
49
|
+
case "labels":
|
|
50
|
+
return `labels: ${scope.labels.join(", ")}`;
|
|
51
|
+
case "scenarios":
|
|
52
|
+
return `hand-picked scenarios (${scenarioIds.length})`;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ScenarioFieldValues } from "../schemas/suite-fields.ts";
|
|
2
|
+
|
|
3
|
+
/** The digest lines for the values a scenario carries per suite field. */
|
|
4
|
+
export function formatScenarioFields(fields: ScenarioFieldValues | undefined): string[] {
|
|
5
|
+
if (!fields || Object.keys(fields).length === 0) return [];
|
|
6
|
+
return [
|
|
7
|
+
"\n## Fields",
|
|
8
|
+
...Object.entries(fields).map(
|
|
9
|
+
([identifier, value]) => `- ${identifier}: ${JSON.stringify(value)}`,
|
|
10
|
+
),
|
|
11
|
+
];
|
|
12
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
EvaluatorAttachmentWire,
|
|
3
|
+
ScenarioMapping,
|
|
4
|
+
SuiteField,
|
|
5
|
+
} from "../schemas/suite-fields.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The digest lines for a suite's fields, the evaluators attached to a suite
|
|
9
|
+
* or a plan, and the evaluator results on a finished run. Shared by every
|
|
10
|
+
* tool that prints one of them.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** A mapping as one short path: `scenario.fields.golden_sql` or `"literal"`. */
|
|
14
|
+
export function describeMapping(mapping: ScenarioMapping): string {
|
|
15
|
+
return mapping.type === "value"
|
|
16
|
+
? JSON.stringify(mapping.value)
|
|
17
|
+
: `${mapping.sourceId}.${mapping.path.join(".")}`;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function formatSuiteFields(fields: SuiteField[] | undefined): string[] {
|
|
21
|
+
if (!fields || fields.length === 0) return [];
|
|
22
|
+
return ["\n## Fields", ...fields.map((field) => `- ${field.identifier} (${field.type})`)];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function formatEvaluatorAttachments(
|
|
26
|
+
evaluators: EvaluatorAttachmentWire[] | undefined,
|
|
27
|
+
): string[] {
|
|
28
|
+
if (!evaluators || evaluators.length === 0) return [];
|
|
29
|
+
const lines = ["\n## Evaluators"];
|
|
30
|
+
for (const attachment of evaluators) {
|
|
31
|
+
const gate = attachment.required ? "required" : "reports only";
|
|
32
|
+
lines.push(`- ${attachment.evaluatorId} (${gate}, attachment ${attachment.id})`);
|
|
33
|
+
for (const [input, mapping] of Object.entries(attachment.mappings)) {
|
|
34
|
+
lines.push(` - ${input}: ${describeMapping(mapping)}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return lines;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Reported while a run's evaluators haven't recorded yet: the judge has
|
|
42
|
+
* decided, but a required evaluator can still turn the run red — a
|
|
43
|
+
* caller must not read this as final.
|
|
44
|
+
*/
|
|
45
|
+
export const PENDING_EVALUATION_STATUS = "PENDING_EVALUATION";
|
|
46
|
+
|
|
47
|
+
/** How a run's status reads in a digest. */
|
|
48
|
+
export function formatRunStatus(status: string): string {
|
|
49
|
+
if (status === PENDING_EVALUATION_STATUS) return "evaluating";
|
|
50
|
+
if (status === "SUCCESS") return "pass";
|
|
51
|
+
if (status === "FAILED") return "FAIL";
|
|
52
|
+
return status;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The line that warns a caller off a run's verdict while its evaluators are
|
|
57
|
+
* still to be recorded, or nothing when they already are.
|
|
58
|
+
*/
|
|
59
|
+
export function pendingEvaluationNote(status: string): string[] {
|
|
60
|
+
if (status !== PENDING_EVALUATION_STATUS) return [];
|
|
61
|
+
return [
|
|
62
|
+
"> The evaluators attached to this run have not been recorded yet. A required one may still fail it, so read the verdict again once the status changes.",
|
|
63
|
+
];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** One evaluator's result on a finished run. */
|
|
67
|
+
export interface SimulationRunEvaluation {
|
|
68
|
+
evaluatorId: string;
|
|
69
|
+
name: string;
|
|
70
|
+
status: "passed" | "failed" | "scored" | "skipped" | "error";
|
|
71
|
+
required: boolean;
|
|
72
|
+
passed?: boolean;
|
|
73
|
+
score?: number;
|
|
74
|
+
label?: string;
|
|
75
|
+
details?: string;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function formatEvaluations(evaluations: SimulationRunEvaluation[] | undefined): string[] {
|
|
79
|
+
if (!evaluations || evaluations.length === 0) return [];
|
|
80
|
+
const lines = ["\n## Evaluators"];
|
|
81
|
+
for (const evaluation of evaluations) {
|
|
82
|
+
const parts: string[] = [evaluation.status];
|
|
83
|
+
if (evaluation.score !== undefined) parts.push(`score ${evaluation.score}`);
|
|
84
|
+
if (evaluation.label !== undefined) parts.push(evaluation.label);
|
|
85
|
+
if (evaluation.required) parts.push("required");
|
|
86
|
+
lines.push(`- **${evaluation.name}**: ${parts.join(", ")}`);
|
|
87
|
+
if (evaluation.details) lines.push(` ${evaluation.details}`);
|
|
88
|
+
}
|
|
89
|
+
return lines;
|
|
90
|
+
}
|
package/src/tools/get-agent.ts
CHANGED
|
@@ -1,15 +1,66 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
getAgent as apiGetAgent,
|
|
3
|
+
type AgentInstance,
|
|
4
|
+
type AgentParameterSpec,
|
|
5
|
+
} from "../langwatch-api-agents.ts";
|
|
2
6
|
|
|
3
|
-
export
|
|
4
|
-
|
|
7
|
+
export const describeParameter = ({
|
|
8
|
+
name,
|
|
9
|
+
type,
|
|
10
|
+
options,
|
|
11
|
+
default: defaultValue,
|
|
12
|
+
description,
|
|
13
|
+
required,
|
|
14
|
+
}: AgentParameterSpec): string => {
|
|
15
|
+
const parts: string[] = [type];
|
|
16
|
+
if (options?.length) parts.push(`one of ${options.join(", ")}`);
|
|
17
|
+
if (defaultValue !== undefined) parts.push(`default ${JSON.stringify(defaultValue)}`);
|
|
18
|
+
if (required) parts.push("required");
|
|
19
|
+
const suffix = description ? `: ${description}` : "";
|
|
20
|
+
return `- **${name}** (${parts.join(", ")})${suffix}`;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
function instancesSection(instances: AgentInstance[]): string[] {
|
|
24
|
+
const lines = [`\n## Instances (${instances.length})\n`];
|
|
25
|
+
for (const instance of instances) {
|
|
26
|
+
const label = instance.label ? ` (${instance.label})` : "";
|
|
27
|
+
lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
|
|
28
|
+
}
|
|
29
|
+
return lines;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Handles the platform_get_agent MCP tool invocation.
|
|
34
|
+
*
|
|
35
|
+
* @see specs/mcp-server/agent-tools.feature
|
|
36
|
+
*/
|
|
37
|
+
export async function handleGetAgent({ id }: { id: string }): Promise<string> {
|
|
38
|
+
const agent = await apiGetAgent(id);
|
|
5
39
|
|
|
6
40
|
const lines: string[] = [];
|
|
7
41
|
lines.push(`# ${agent.name}\n`);
|
|
8
42
|
lines.push(`**ID**: ${agent.id}`);
|
|
9
43
|
lines.push(`**Type**: ${agent.type}`);
|
|
44
|
+
if (agent.environment) lines.push(`**Environment**: ${agent.environment}`);
|
|
45
|
+
if (agent.status) lines.push(`**Status**: ${agent.status}`);
|
|
46
|
+
if (agent.owner?.name) lines.push(`**Owner**: ${agent.owner.name}`);
|
|
47
|
+
else if (agent.hostLabel) lines.push(`**Host**: ${agent.hostLabel}`);
|
|
48
|
+
if (agent.selectable === false) {
|
|
49
|
+
lines.push("**Run**: only its owner can run this agent");
|
|
50
|
+
}
|
|
51
|
+
if (agent.lastSeenAt) lines.push(`**Last seen**: ${agent.lastSeenAt}`);
|
|
10
52
|
lines.push(`**Created**: ${agent.createdAt}`);
|
|
11
53
|
lines.push(`**Updated**: ${agent.updatedAt}`);
|
|
12
54
|
|
|
55
|
+
if (agent.parameters && agent.parameters.length > 0) {
|
|
56
|
+
lines.push("\n## Parameters\n");
|
|
57
|
+
for (const parameter of agent.parameters) lines.push(describeParameter(parameter));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (agent.instances && agent.instances.length > 0) {
|
|
61
|
+
lines.push(...instancesSection(agent.instances));
|
|
62
|
+
}
|
|
63
|
+
|
|
13
64
|
if (agent.config && Object.keys(agent.config).length > 0) {
|
|
14
65
|
lines.push("\n## Config\n");
|
|
15
66
|
lines.push("```json");
|