@langwatch/mcp-server 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +7 -0
- package/.npmignore +41 -0
- package/CHANGELOG.md +280 -200
- package/CONTRIBUTING.md +10 -0
- package/README.md +82 -52
- package/dist/add-dashboard-widget-4ZC5UN73.js +50 -0
- package/dist/add-dashboard-widget-ACC6QYBG.js +49 -0
- package/dist/archive-project-KWFM3I5Z.js +19 -0
- package/dist/{archive-project-4EQGLBJJ.js → archive-project-SPVHXK6A.js} +4 -4
- package/dist/archive-run-plan-65QN3XRX.js +16 -0
- package/dist/archive-run-plan-Z4SYE6BP.js +15 -0
- package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-A6QV5LOW.js} +4 -4
- package/dist/archive-scenario-FXEYQNEH.js +19 -0
- package/dist/archive-test-suite-5ADQ3KPQ.js +15 -0
- package/dist/archive-test-suite-NKZDZY3D.js +16 -0
- package/dist/assign-prompt-tag-5IUZXJ2T.js +23 -0
- package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-HBZLBIFD.js} +8 -4
- package/dist/{chunk-CDYQOY5U.js → chunk-3OCG3QPU.js} +7 -10
- package/dist/chunk-3YLN3ATJ.js +241 -0
- package/dist/chunk-5FNCQRCX.js +91 -0
- package/dist/chunk-6YZUOD34.js +55 -0
- package/dist/{chunk-WVI5CJ6S.js → chunk-7AQLPVFL.js} +2 -2
- package/dist/chunk-7VNSRCXO.js +129 -0
- package/dist/{chunk-7RHG56OH.js → chunk-B4J3PJLX.js} +11 -10
- package/dist/{chunk-ZXKLPC2E.js → chunk-BIIKE3EI.js} +1 -1
- package/dist/{chunk-37SMPMQS.js → chunk-DMM3JQ42.js} +7 -2
- package/dist/{chunk-NUFPTHKU.js → chunk-FOKVY3BK.js} +1 -1
- package/dist/{chunk-WT4PO4A3.js → chunk-H6WGMG2T.js} +10 -7
- package/dist/{chunk-TOXJQDIZ.js → chunk-HGELNIVY.js} +135 -51
- package/dist/{chunk-UJPXFA26.js → chunk-HHMNZAPD.js} +101 -74
- package/dist/{chunk-J42FWRJ2.js → chunk-I7ATIZED.js} +7 -7
- package/dist/{chunk-AQJO2EI5.js → chunk-ILZDO25L.js} +7 -10
- package/dist/{chunk-4QC7SA2L.js → chunk-J7PQKBLD.js} +884 -559
- package/dist/{chunk-YX3A3CFM.js → chunk-JRJPH75H.js} +7 -13
- package/dist/chunk-K2I7Z3LZ.js +28 -0
- package/dist/{chunk-5CCZFUXS.js → chunk-NKMTIHIJ.js} +6 -9
- package/dist/chunk-O7EMWEVJ.js +44 -0
- package/dist/chunk-PTDR56Z3.js +37 -0
- package/dist/chunk-RABYBFX5.js +43 -0
- package/dist/chunk-SUPUUC4W.js +64 -0
- package/dist/{chunk-SFB4MU6H.js → chunk-SXFC3FNH.js} +4 -4
- package/dist/chunk-U24EACC4.js +67 -0
- package/dist/chunk-UNJM3BIL.js +21 -0
- package/dist/{chunk-ZWIL4EFL.js → chunk-UTJCWXPZ.js} +4 -4
- package/dist/chunk-UWLE5T2F.js +312 -0
- package/dist/chunk-VMWJD732.js +57 -0
- package/dist/chunk-XUIJYNRJ.js +15 -0
- package/dist/chunk-Y34DHR5K.js +67 -0
- package/dist/chunk-YCARO42Z.js +516 -0
- package/dist/config-EAKW6F26.js +16 -0
- package/dist/config.cjs +109 -0
- package/dist/config.js +5 -4
- package/dist/{create-agent-7XHO6UIS.js → create-agent-AYZ3LB3V.js} +4 -4
- package/dist/create-agent-S5ZBCEFT.js +23 -0
- package/dist/create-annotation-LHNCLHYP.js +29 -0
- package/dist/{create-annotation-YQXRQC6A.js → create-annotation-VUSXULQK.js} +10 -5
- package/dist/{create-api-key-YUS36KBU.js → create-api-key-NNYKARMG.js} +5 -7
- package/dist/create-api-key-T4KSNPKB.js +24 -0
- package/dist/create-dashboard-DHVFMEU4.js +18 -0
- package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-WTKKS5Z6.js} +4 -4
- package/dist/{create-dataset-E6NMSVX7.js → create-dataset-2MC4SD4O.js} +6 -6
- package/dist/create-dataset-LUP5PVDM.js +22 -0
- package/dist/create-dataset-records-NSKBZP3G.js +20 -0
- package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-W4GLN2FV.js} +5 -7
- package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-643G6LXL.js} +4 -4
- package/dist/create-evaluator-HMTYO5SS.js +27 -0
- package/dist/create-mcp-server.cjs +9513 -0
- package/dist/create-mcp-server.js +2647 -5
- package/dist/create-project-6N7PIYRY.js +44 -0
- package/dist/{create-project-CS2CB57Y.js → create-project-KAIC46UY.js} +4 -4
- package/dist/{create-prompt-F5RMX3L3.js → create-prompt-E67LXIYT.js} +5 -7
- package/dist/create-prompt-PKJ7EDRR.js +37 -0
- package/dist/create-prompt-tag-4YO3FSJ3.js +17 -0
- package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-OPQUEXWB.js} +3 -3
- package/dist/create-scenario-5PJBMAXO.js +33 -0
- package/dist/{create-scenario-JDR2VB2H.js → create-scenario-UEM77OE7.js} +11 -4
- package/dist/create-test-suite-4IPXVSFQ.js +37 -0
- package/dist/create-test-suite-SZEPVXSE.js +36 -0
- package/dist/create-trigger-5HZSCRAN.js +49 -0
- package/dist/create-trigger-PRELMT6J.js +48 -0
- package/dist/delete-agent-JUIZWAXG.js +18 -0
- package/dist/{delete-agent-O3EEH4S5.js → delete-agent-QMEK3XHK.js} +4 -4
- package/dist/delete-annotation-HY3O2OGR.js +17 -0
- package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-VKIQWUYW.js} +4 -4
- package/dist/delete-dashboard-76N66XXL.js +18 -0
- package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-JVIVHNJM.js} +4 -4
- package/dist/delete-dataset-HQCR66MG.js +18 -0
- package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-WKRUW4FP.js} +5 -7
- package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-C2IQFFVQ.js} +5 -7
- package/dist/delete-dataset-records-SDUHQLLM.js +19 -0
- package/dist/delete-evaluator-4SNSCLEE.js +16 -0
- package/dist/delete-evaluator-QP3QHTLN.js +15 -0
- package/dist/delete-prompt-tag-NZ4JQMYG.js +17 -0
- package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-YYQ4VMAD.js} +3 -3
- package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-5J46T73T.js} +4 -4
- package/dist/delete-workflow-JSWABL5F.js +17 -0
- package/dist/discover-evaluator-schema-UHWA5HVL.js +120 -0
- package/dist/discover-evaluator-schema-XKCLLCB3.js +118 -0
- package/dist/discover-scenario-schema-NOJY6S7E.js +137 -0
- package/dist/discover-scenario-schema-OOFIQKOG.js +139 -0
- package/dist/{discover-schema-L6QDPON3.js → discover-schema-55YJBFGQ.js} +181 -186
- package/dist/discover-schema-A33SLMXW.js +442 -0
- package/dist/get-agent-LJQX7OF4.js +69 -0
- package/dist/get-agent-OTIYUFZA.js +70 -0
- package/dist/get-analytics-DCS72353.js +80 -0
- package/dist/get-analytics-IQZKJHLW.js +79 -0
- package/dist/get-dashboard-5NYJEIX2.js +29 -0
- package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-QMTYDQAZ.js} +4 -4
- package/dist/get-dataset-ALSP7RTX.js +64 -0
- package/dist/get-dataset-X4B5JDG2.js +63 -0
- package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-ISMC4LLI.js} +4 -4
- package/dist/get-evaluator-XEUZGZCH.js +47 -0
- package/dist/{get-project-RAK7URQJ.js → get-project-5VKROSDH.js} +4 -4
- package/dist/get-project-RM6FRCBK.js +26 -0
- package/dist/get-prompt-OWXJZHQQ.js +95 -0
- package/dist/get-prompt-W52DY36E.js +94 -0
- package/dist/get-run-plan-BE3M2AUX.js +59 -0
- package/dist/get-run-plan-RUXQ6PAX.js +58 -0
- package/dist/get-scenario-RNM7BAX2.js +37 -0
- package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-YM4ODWW6.js} +8 -4
- package/dist/get-simulation-run-6GIATQE5.js +69 -0
- package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-LCFSMPGQ.js} +33 -20
- package/dist/get-test-suite-GC2O7YH6.js +49 -0
- package/dist/get-test-suite-P5LJ6VLF.js +48 -0
- package/dist/get-trace-KBRVV4NN.js +45 -0
- package/dist/{get-trace-K7U234Q4.js → get-trace-SEF3FMR5.js} +11 -13
- package/dist/get-workflow-7PVED7JE.js +25 -0
- package/dist/{get-workflow-L5L2PSK3.js → get-workflow-YXULW4GC.js} +4 -4
- package/dist/{http-security-EU3CHTY7.js → http-security-DNLBNQZL.js} +2 -2
- package/dist/{http-server-YXHFPPGA.js → http-server-IR4QOS43.js} +4071 -342
- package/dist/index.js +60221 -38077
- package/dist/langwatch-api-annotations-6SA4OB4C.js +17 -0
- package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-TBHLCQ5R.js} +4 -4
- package/dist/langwatch-api-dashboards-6EO2VVZW.js +17 -0
- package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-NHU5BZD7.js} +4 -4
- package/dist/langwatch-api-monitors-CBF3FGIM.js +47 -0
- package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-YNWA2M5Y.js} +11 -16
- package/dist/langwatch-api-secrets-ADTOJELP.js +56 -0
- package/dist/langwatch-api-secrets-RNFO4LF2.js +55 -0
- package/dist/langwatch-api-triggers-KQDQC6L5.js +22 -0
- package/dist/langwatch-api-triggers-QS7HDIVW.js +23 -0
- package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-OKC4FXIF.js} +4 -4
- package/dist/langwatch-api-workflows-XGJE6SR5.js +15 -0
- package/dist/list-agents-HVYWEQMI.js +43 -0
- package/dist/list-agents-RBAOCF53.js +42 -0
- package/dist/list-annotations-HJH4NBLD.js +38 -0
- package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-J4UAHSUN.js} +10 -5
- package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-7UTJRZDG.js} +10 -5
- package/dist/list-api-keys-A2PMV435.js +42 -0
- package/dist/list-dashboard-boards-OPWROHMG.js +33 -0
- package/dist/list-dashboard-boards-U2GJ7GC5.js +34 -0
- package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-ROSYFLDQ.js} +4 -4
- package/dist/list-dashboards-RQ635ZU7.js +32 -0
- package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-M5TKA6LK.js} +14 -10
- package/dist/list-dataset-records-WFM7F33C.js +43 -0
- package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-B5X4P7BQ.js} +9 -13
- package/dist/list-datasets-THUKQHPX.js +56 -0
- package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-MYH2OVCI.js} +5 -7
- package/dist/list-evaluators-NLVWF6K7.js +32 -0
- package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-3JUGP4YI.js} +4 -4
- package/dist/list-model-providers-H3XKORFO.js +35 -0
- package/dist/{list-projects-K7QAIX2I.js → list-projects-6EJ5YYEW.js} +5 -7
- package/dist/list-projects-YNNSRMLC.js +40 -0
- package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-NVFQO6WX.js} +3 -3
- package/dist/list-prompt-tags-XUQTTAH4.js +23 -0
- package/dist/{list-prompts-SC2HAUFG.js → list-prompts-HSIMMHTF.js} +4 -6
- package/dist/list-prompts-IRI6AIXO.js +30 -0
- package/dist/list-run-plans-PX2XI6EK.js +46 -0
- package/dist/list-run-plans-QALLCN2K.js +45 -0
- package/dist/list-scenarios-QZRPVHS4.js +46 -0
- package/dist/list-scenarios-SSJQEMJP.js +45 -0
- package/dist/list-simulation-runs-KNOLGMDV.js +53 -0
- package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-OEADLLQR.js} +19 -9
- package/dist/list-test-suites-ANAQBLDG.js +34 -0
- package/dist/list-test-suites-N4WVLBKK.js +35 -0
- package/dist/list-trigger-fires-JV3WFYYG.js +19 -0
- package/dist/list-trigger-fires-WTW4DVIW.js +20 -0
- package/dist/list-triggers-SQNIG35Q.js +45 -0
- package/dist/list-triggers-V4Y356AK.js +46 -0
- package/dist/{list-workflows-QSJWURCK.js → list-workflows-IMSX7SXU.js} +5 -7
- package/dist/list-workflows-ZHTX5LCL.js +33 -0
- package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-IQI7MHOI.js} +3 -3
- package/dist/rename-prompt-tag-PA2BRIE6.js +18 -0
- package/dist/rename-test-suite-4J7UX5S2.js +21 -0
- package/dist/rename-test-suite-NNVLMLKY.js +20 -0
- package/dist/report-issue-AS34ZMJF.js +898 -0
- package/dist/report-issue-DCZFLGDO.js +899 -0
- package/dist/rerun-run-plan-TG2V4WM4.js +22 -0
- package/dist/rerun-run-plan-ZI5OY3VA.js +21 -0
- package/dist/revoke-api-key-3J3ZA4UB.js +15 -0
- package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-SFMEFWJC.js} +4 -4
- package/dist/run-agent-W4VJP72Z.js +75 -0
- package/dist/run-agent-YNV6ZHW4.js +74 -0
- package/dist/run-plan-6C77GWG4.js +38 -0
- package/dist/run-plan-PAARQSI4.js +39 -0
- package/dist/run-query-KJDH544C.js +47 -0
- package/dist/run-query-T4QW2Z5O.js +46 -0
- package/dist/run-test-suite-SQJELEZX.js +26 -0
- package/dist/run-test-suite-STB6NUJM.js +25 -0
- package/dist/{run-workflow-QPXC627O.js → run-workflow-D6PQNBI6.js} +4 -4
- package/dist/{run-agent-6SM4XNA7.js → run-workflow-SP32SO5M.js} +10 -11
- package/dist/search-traces-HCBGOG6R.js +170 -0
- package/dist/search-traces-HSTQ6WTQ.js +171 -0
- package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-GF7NXUJA.js} +4 -4
- package/dist/set-model-provider-KDQ7C4EZ.js +33 -0
- package/dist/{update-agent-SMAEHLXH.js → update-agent-DWG43T2P.js} +4 -4
- package/dist/update-agent-MXNOBS4H.js +20 -0
- package/dist/update-dashboard-widget-CJKJLCDY.js +71 -0
- package/dist/update-dashboard-widget-CYQDS2NR.js +70 -0
- package/dist/update-dataset-DQH3OHOU.js +22 -0
- package/dist/{update-dataset-BDUK7TUO.js → update-dataset-SQZ6T4WX.js} +6 -6
- package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-3AT4IPBC.js} +4 -4
- package/dist/update-dataset-record-7PLDGTMR.js +19 -0
- package/dist/update-evaluator-4LFTKFPT.js +24 -0
- package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-IGTX6PX4.js} +4 -4
- package/dist/{update-project-LKVYXENH.js → update-project-PLJIOZS6.js} +4 -4
- package/dist/update-project-QVHSJRUP.js +24 -0
- package/dist/update-prompt-GDKRYJI3.js +140 -0
- package/dist/update-prompt-MY263DYF.js +141 -0
- package/dist/update-scenario-7KA3JDXA.js +34 -0
- package/dist/{update-scenario-NEXCAQI7.js → update-scenario-MOHRQKAQ.js} +11 -4
- package/package.json +39 -44
- package/pyproject.toml +8 -8
- package/src/config.d.ts +4 -0
- package/src/config.ts +20 -29
- package/src/create-mcp-server.ts +1021 -596
- package/src/documentation-fetch.ts +21 -20
- package/src/http-security.ts +166 -135
- package/src/http-server.ts +189 -224
- package/src/index.ts +14 -16
- package/src/langwatch-api-agents.ts +202 -21
- package/src/langwatch-api-annotations.ts +11 -12
- package/src/langwatch-api-api-keys.ts +8 -9
- package/src/langwatch-api-boards.ts +46 -0
- package/src/langwatch-api-dashboard-widgets.ts +112 -0
- package/src/langwatch-api-dashboards.ts +13 -10
- package/src/langwatch-api-datasets.ts +98 -32
- package/src/langwatch-api-evaluators.ts +13 -19
- package/src/langwatch-api-model-providers.ts +4 -4
- package/src/langwatch-api-monitors.ts +14 -18
- package/src/langwatch-api-projects.ts +27 -9
- package/src/langwatch-api-query.ts +114 -0
- package/src/langwatch-api-run-plans.ts +140 -0
- package/src/langwatch-api-scenarios.ts +17 -10
- package/src/langwatch-api-secrets.ts +31 -24
- package/src/langwatch-api-simulation-runs.ts +10 -7
- package/src/langwatch-api-test-suites.ts +116 -0
- package/src/langwatch-api-triggers.ts +96 -34
- package/src/langwatch-api-workflows.ts +9 -9
- package/src/langwatch-api.ts +143 -109
- package/src/public-http-request.ts +21 -12
- package/src/schemas/analytics-metrics.ts +3 -15
- package/src/schemas/create-dataset.ts +3 -4
- package/src/schemas/run-plan.ts +76 -0
- package/src/schemas/suite-fields.ts +132 -0
- package/src/schemas/triggers.ts +365 -0
- package/src/tools/add-dashboard-widget.ts +63 -0
- package/src/tools/archive-project.ts +2 -4
- package/src/tools/archive-run-plan.ts +10 -0
- package/src/tools/archive-scenario.ts +2 -4
- package/src/tools/archive-test-suite.ts +10 -0
- package/src/tools/assign-prompt-tag.ts +6 -2
- package/src/tools/create-agent.ts +1 -1
- package/src/tools/create-annotation.ts +7 -2
- package/src/tools/create-api-key.ts +4 -6
- package/src/tools/create-dashboard.ts +1 -1
- package/src/tools/create-dataset-records.ts +4 -8
- package/src/tools/create-dataset.ts +5 -7
- package/src/tools/create-evaluator.ts +3 -5
- package/src/tools/create-project.ts +2 -2
- package/src/tools/create-prompt-tag.ts +2 -4
- package/src/tools/create-prompt.ts +9 -18
- package/src/tools/create-scenario.ts +11 -5
- package/src/tools/create-test-suite.ts +38 -0
- package/src/tools/create-trigger.ts +55 -0
- package/src/tools/dashboard-exists.ts +25 -0
- package/src/tools/dashboard-widget-queries.ts +32 -0
- package/src/tools/delete-agent.ts +1 -1
- package/src/tools/delete-annotation.ts +1 -1
- package/src/tools/delete-dashboard.ts +1 -1
- package/src/tools/delete-dataset-records.ts +4 -8
- package/src/tools/delete-dataset.ts +3 -7
- package/src/tools/delete-evaluator.ts +4 -6
- package/src/tools/delete-prompt-tag.ts +2 -4
- package/src/tools/delete-workflow.ts +1 -1
- package/src/tools/discover-evaluator-schema.ts +34 -25
- package/src/tools/discover-scenario-schema.ts +86 -12
- package/src/tools/discover-schema.ts +149 -27
- package/src/tools/experiment-run-status.ts +2 -8
- package/src/tools/format-dataset-mutation.ts +5 -10
- package/src/tools/format-run-plan.ts +54 -0
- package/src/tools/format-scenario.ts +12 -0
- package/src/tools/format-suite-details.ts +90 -0
- package/src/tools/get-agent.ts +54 -3
- package/src/tools/get-analytics.ts +52 -46
- package/src/tools/get-dashboard.ts +1 -1
- package/src/tools/get-dataset.ts +34 -16
- package/src/tools/get-evaluator.ts +4 -11
- package/src/tools/get-experiment-results.ts +219 -186
- package/src/tools/get-project.ts +2 -4
- package/src/tools/get-prompt.ts +100 -37
- package/src/tools/get-run-plan.ts +60 -0
- package/src/tools/get-scenario.ts +6 -5
- package/src/tools/get-simulation-run.ts +32 -20
- package/src/tools/get-test-suite.ts +46 -0
- package/src/tools/get-trace.ts +16 -17
- package/src/tools/get-workflow.ts +1 -1
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-annotations.ts +8 -5
- package/src/tools/list-api-keys.ts +7 -2
- package/src/tools/list-dashboard-boards.ts +28 -0
- package/src/tools/list-dashboards.ts +1 -1
- package/src/tools/list-dataset-records.ts +8 -6
- package/src/tools/list-datasets.ts +12 -23
- package/src/tools/list-evaluators.ts +4 -8
- package/src/tools/list-experiment-runs.ts +5 -9
- package/src/tools/list-experiments.ts +5 -11
- package/src/tools/list-model-providers.ts +3 -5
- package/src/tools/list-projects.ts +2 -4
- package/src/tools/list-prompt-tags.ts +3 -3
- package/src/tools/list-prompts.ts +4 -8
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +26 -17
- package/src/tools/list-simulation-runs.ts +13 -5
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/list-trigger-fires.ts +17 -0
- package/src/tools/list-triggers.ts +39 -0
- package/src/tools/list-workflows.ts +2 -4
- package/src/tools/rename-prompt-tag.ts +1 -1
- package/src/tools/rename-test-suite.ts +15 -0
- package/src/tools/report-issue.ts +5 -9
- package/src/tools/rerun-run-plan.ts +18 -0
- package/src/tools/revoke-api-key.ts +2 -4
- package/src/tools/run-agent.ts +65 -14
- package/src/tools/run-experiment.ts +36 -37
- package/src/tools/run-plan.ts +46 -0
- package/src/tools/run-query.ts +47 -0
- package/src/tools/run-test-suite.ts +27 -0
- package/src/tools/run-workflow.ts +2 -5
- package/src/tools/search-traces.ts +161 -47
- package/src/tools/set-model-provider.ts +4 -6
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-agent.ts +1 -1
- package/src/tools/update-dashboard-widget.ts +97 -0
- package/src/tools/update-dataset-record.ts +1 -1
- package/src/tools/update-dataset.ts +5 -7
- package/src/tools/update-evaluator.ts +3 -5
- package/src/tools/update-project.ts +1 -1
- package/src/tools/update-prompt.ts +185 -15
- package/src/tools/update-scenario.ts +11 -5
- package/src/tools/update-test-suite.ts +40 -0
- package/src/utils/date-parsing.ts +1 -1
- package/src/utils/escape-markdown.ts +3 -8
- package/src/utils/format-evaluations.ts +6 -6
- package/src/utils/markdown-table.ts +58 -0
- package/src/utils/trace-id-shape.ts +20 -0
- package/tsconfig.app.json +24 -0
- package/tsconfig.json +19 -12
- package/uv.lock +416 -296
- package/vitest.integration.config.js +10 -0
- package/vitest.scenario.config.js +9 -0
- package/.eslintrc.cjs +0 -36
- package/dist/archive-suite-4T6BSRS7.js +0 -16
- package/dist/chunk-77BNWQ3U.js +0 -86
- package/dist/chunk-A2LEVYQ5.js +0 -217
- package/dist/chunk-TJCXFNDT.js +0 -24
- package/dist/chunk-YZLYHLLI.js +0 -48
- package/dist/config.d.ts +0 -17
- package/dist/create-mcp-server.d.ts +0 -10
- package/dist/create-suite-LZS6QIAF.js +0 -39
- package/dist/delete-evaluator-33SP2JFX.js +0 -16
- package/dist/discover-evaluator-schema-XTAD4X7D.js +0 -2191
- package/dist/discover-scenario-schema-FE64QK2D.js +0 -67
- package/dist/get-agent-Y7MWZ65F.js +0 -29
- package/dist/get-analytics-TETCSOK6.js +0 -73
- package/dist/get-dataset-RCMU5VAH.js +0 -49
- package/dist/get-prompt-OY5B7TRI.js +0 -46
- package/dist/get-suite-J7JVWO6O.js +0 -45
- package/dist/index.d.ts +0 -2
- package/dist/langwatch-api-secrets-BUBSNCNO.js +0 -41
- package/dist/langwatch-api-triggers-EDFUCJN6.js +0 -31
- package/dist/list-agents-KFHKGPXV.js +0 -33
- package/dist/list-scenarios-F2WYNRIE.js +0 -41
- package/dist/list-suites-CRY4VWW6.js +0 -43
- package/dist/report-issue-XYVAPAYJ.js +0 -412
- package/dist/run-suite-NKZCJ2IN.js +0 -30
- package/dist/search-traces-A6HFKPLJ.js +0 -79
- package/dist/update-prompt-ODCRQRN6.js +0 -25
- package/dist/update-suite-SNO3GS7T.js +0 -40
- package/src/__tests__/all-tools.integration.test.ts +0 -1626
- package/src/__tests__/config.unit.test.ts +0 -202
- package/src/__tests__/datasets.integration.test.ts +0 -620
- package/src/__tests__/datasets.unit.test.ts +0 -244
- package/src/__tests__/date-parsing.unit.test.ts +0 -78
- package/src/__tests__/discover-evaluator-schema.unit.test.ts +0 -89
- package/src/__tests__/discover-schema.unit.test.ts +0 -118
- package/src/__tests__/docker.integration.test.ts +0 -318
- package/src/__tests__/documentation-fetch-security.integration.test.ts +0 -182
- package/src/__tests__/evaluator-tools.unit.test.ts +0 -262
- package/src/__tests__/experiment-list-tools.unit.test.ts +0 -157
- package/src/__tests__/experiment-status.unit.test.ts +0 -194
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +0 -93
- package/src/__tests__/get-experiment-results.unit.test.ts +0 -313
- package/src/__tests__/http-security-origin.unit.test.ts +0 -97
- package/src/__tests__/http-security-sessions.unit.test.ts +0 -209
- package/src/__tests__/http-security-verifier.unit.test.ts +0 -248
- package/src/__tests__/http-server-allocation.integration.test.ts +0 -279
- package/src/__tests__/http-server-origin.integration.test.ts +0 -169
- package/src/__tests__/http-server-session-auth.integration.test.ts +0 -341
- package/src/__tests__/http-transport.integration.test.ts +0 -395
- package/src/__tests__/integration.integration.test.ts +0 -288
- package/src/__tests__/langwatch-api.unit.test.ts +0 -661
- package/src/__tests__/model-provider-tools.unit.test.ts +0 -190
- package/src/__tests__/project-api-key-tools.unit.test.ts +0 -451
- package/src/__tests__/public-http-request-security.integration.test.ts +0 -118
- package/src/__tests__/report-issue.integration.test.ts +0 -147
- package/src/__tests__/scenario-tools.integration.test.ts +0 -286
- package/src/__tests__/scenario-tools.unit.test.ts +0 -188
- package/src/__tests__/schemas.unit.test.ts +0 -85
- package/src/__tests__/support/http-server-harness.ts +0 -138
- package/src/__tests__/tools.unit.test.ts +0 -924
- package/src/langwatch-api-suites.ts +0 -118
- package/src/schemas/filter-fields.ts +0 -119
- package/src/schemas/index.ts +0 -4
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { LangWatchApiError, makeRequest } from "../langwatch-api.
|
|
2
|
-
import { deriveRunStatus, isTerminalStatus } from "./experiment-run-status.
|
|
1
|
+
import { LangWatchApiError, makeRequest } from "../langwatch-api.ts";
|
|
2
|
+
import { deriveRunStatus, isTerminalStatus } from "./experiment-run-status.ts";
|
|
3
3
|
|
|
4
4
|
interface DatasetEntry {
|
|
5
5
|
index: number;
|
|
@@ -39,12 +39,17 @@ interface EvaluationRunResults {
|
|
|
39
39
|
finishedAt?: number | null;
|
|
40
40
|
stoppedAt?: number | null;
|
|
41
41
|
};
|
|
42
|
+
/** What is stored against what the run reported; absent on servers that predate it. */
|
|
43
|
+
completeness?: {
|
|
44
|
+
complete: boolean;
|
|
45
|
+
dataset: { received: number; expected: number | null };
|
|
46
|
+
evaluations: { received: number; expected: number | null };
|
|
47
|
+
};
|
|
42
48
|
}
|
|
43
49
|
|
|
44
50
|
const DEFAULT_ROW_CAP = 50;
|
|
45
51
|
|
|
46
|
-
const rowKey = (index: number, targetId?: string | null): string =>
|
|
47
|
-
`${index}:${targetId ?? ""}`;
|
|
52
|
+
const rowKey = (index: number, targetId?: string | null): string => `${index}:${targetId ?? ""}`;
|
|
48
53
|
|
|
49
54
|
const summarizeEntry = (entry: Record<string, unknown>): string => {
|
|
50
55
|
const candidates = ["input", "question", "query", "prompt", "user"];
|
|
@@ -68,128 +73,72 @@ const isFailedRow = ({
|
|
|
68
73
|
evaluations: EvaluationItem[];
|
|
69
74
|
}): boolean => Boolean(entry.error) || evaluations.some(isFailedEvaluation);
|
|
70
75
|
|
|
71
|
-
|
|
76
|
+
function hasNumericStatus(error: unknown): error is { status: number } {
|
|
77
|
+
if (typeof error !== "object" || error === null) return false;
|
|
78
|
+
return "status" in error && typeof error.status === "number";
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
type RunStatus = ReturnType<typeof deriveRunStatus>;
|
|
82
|
+
type ResultRow = { entry: DatasetEntry; evaluations: EvaluationItem[] };
|
|
83
|
+
type EvaluatorStats = {
|
|
84
|
+
sum: number;
|
|
85
|
+
count: number;
|
|
86
|
+
passed: number;
|
|
87
|
+
failed: number;
|
|
88
|
+
errored: number;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
function isNotFoundError(error: unknown): boolean {
|
|
92
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
93
|
+
let status: number | undefined;
|
|
94
|
+
if (error instanceof LangWatchApiError) status = error.status;
|
|
95
|
+
else if (hasNumericStatus(error)) status = error.status;
|
|
96
|
+
return status === 404 || (status === undefined && /404|not found/i.test(message));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function fetchRunResults(params: {
|
|
72
100
|
runId: string;
|
|
73
101
|
experimentSlug?: string;
|
|
74
|
-
|
|
75
|
-
evaluator?: string;
|
|
76
|
-
limit?: number;
|
|
77
|
-
}): Promise<string> {
|
|
78
|
-
const filter = params.filter ?? "all";
|
|
79
|
-
const evaluatorFilter = params.evaluator?.trim();
|
|
80
|
-
const limit =
|
|
81
|
-
typeof params.limit === "number" && params.limit > 0
|
|
82
|
-
? Math.min(params.limit, DEFAULT_ROW_CAP)
|
|
83
|
-
: DEFAULT_ROW_CAP;
|
|
84
|
-
|
|
102
|
+
}): Promise<EvaluationRunResults | null> {
|
|
85
103
|
const search = new URLSearchParams();
|
|
86
104
|
if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
|
|
87
105
|
const qs = search.toString() ? `?${search.toString()}` : "";
|
|
106
|
+
return (await makeRequest(
|
|
107
|
+
"GET",
|
|
108
|
+
`/api/v1/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`,
|
|
109
|
+
)) as EvaluationRunResults;
|
|
110
|
+
}
|
|
88
111
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
? error.status
|
|
100
|
-
: error && typeof error === "object" &&
|
|
101
|
-
"status" in error &&
|
|
102
|
-
typeof (error as { status?: unknown }).status === "number"
|
|
103
|
-
? (error as { status: number }).status
|
|
104
|
-
: undefined;
|
|
105
|
-
if (
|
|
106
|
-
status === 404 ||
|
|
107
|
-
(status === undefined && /404|not found/i.test(message))
|
|
108
|
-
) {
|
|
109
|
-
return [
|
|
110
|
-
`# Evaluation Results: ${params.runId}`,
|
|
111
|
-
"",
|
|
112
|
-
"**Status**: not found",
|
|
113
|
-
"",
|
|
114
|
-
`Could not load results for run \`${params.runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
|
|
115
|
-
"",
|
|
116
|
-
"> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids.",
|
|
117
|
-
].join("\n");
|
|
118
|
-
}
|
|
119
|
-
throw error;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
if (!results) {
|
|
123
|
-
return [
|
|
124
|
-
`# Evaluation Results: ${params.runId}`,
|
|
125
|
-
"",
|
|
126
|
-
"**Status**: not found",
|
|
127
|
-
"",
|
|
128
|
-
`Could not load results for run \`${params.runId}\`.`,
|
|
129
|
-
"",
|
|
130
|
-
"> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id.",
|
|
131
|
-
].join("\n");
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
// Partial results are served even while the run is still in progress.
|
|
135
|
-
// Rows land in ClickHouse incrementally, so a "running" or "interrupted"
|
|
136
|
-
// run can still expose every row recorded so far. A run only fails to set
|
|
137
|
-
// finished_at/stopped_at when the SDK process dies before flushing, but the
|
|
138
|
-
// rows it did log are still useful, so we never gate on a terminal status.
|
|
139
|
-
const runStatus = deriveRunStatus(results.timestamps);
|
|
140
|
-
const partial = !isTerminalStatus(runStatus);
|
|
141
|
-
|
|
142
|
-
// Group evaluations by target-scoped row key, applying evaluator filter.
|
|
143
|
-
const evaluationsByRow = new Map<string, EvaluationItem[]>();
|
|
144
|
-
for (const evaluation of results.evaluations) {
|
|
112
|
+
/** Evaluations grouped by target-scoped row key, keeping only the filtered evaluator. */
|
|
113
|
+
function groupEvaluationsByRow({
|
|
114
|
+
evaluations,
|
|
115
|
+
evaluatorFilter,
|
|
116
|
+
}: {
|
|
117
|
+
evaluations: EvaluationItem[];
|
|
118
|
+
evaluatorFilter: string | undefined;
|
|
119
|
+
}): Map<string, EvaluationItem[]> {
|
|
120
|
+
const byRow = new Map<string, EvaluationItem[]>();
|
|
121
|
+
for (const evaluation of evaluations) {
|
|
145
122
|
if (evaluatorFilter && evaluation.evaluator !== evaluatorFilter) continue;
|
|
146
123
|
const key = rowKey(evaluation.index, evaluation.targetId);
|
|
147
|
-
const list =
|
|
124
|
+
const list = byRow.get(key) ?? [];
|
|
148
125
|
list.push(evaluation);
|
|
149
|
-
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
const evaluatorNames = Array.from(
|
|
153
|
-
new Set(
|
|
154
|
-
(evaluatorFilter
|
|
155
|
-
? results.evaluations.filter((e) => e.evaluator === evaluatorFilter)
|
|
156
|
-
: results.evaluations
|
|
157
|
-
).map((e) => e.evaluator),
|
|
158
|
-
),
|
|
159
|
-
);
|
|
160
|
-
|
|
161
|
-
let rows = results.dataset.map((entry) => ({
|
|
162
|
-
entry,
|
|
163
|
-
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? [],
|
|
164
|
-
}));
|
|
165
|
-
|
|
166
|
-
if (filter === "failed") {
|
|
167
|
-
rows = rows.filter((r) =>
|
|
168
|
-
isFailedRow({ entry: r.entry, evaluations: r.evaluations }),
|
|
169
|
-
);
|
|
126
|
+
byRow.set(key, list);
|
|
170
127
|
}
|
|
128
|
+
return byRow;
|
|
129
|
+
}
|
|
171
130
|
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
const
|
|
175
|
-
rows = rows.slice(0, limit);
|
|
176
|
-
|
|
177
|
-
// Per-evaluator stats across the filtered rows (before truncation),
|
|
178
|
-
// so the summary matches the displayed subset when filter="failed".
|
|
179
|
-
const evaluatorAverages = new Map<
|
|
180
|
-
string,
|
|
181
|
-
{ sum: number; count: number; passed: number; failed: number; errored: number }
|
|
182
|
-
>();
|
|
183
|
-
for (const r of rowsForSummary) {
|
|
131
|
+
function tallyEvaluatorStats(rows: ResultRow[]): Map<string, EvaluatorStats> {
|
|
132
|
+
const averages = new Map<string, EvaluatorStats>();
|
|
133
|
+
for (const r of rows) {
|
|
184
134
|
for (const e of r.evaluations) {
|
|
185
|
-
const stats =
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
};
|
|
135
|
+
const stats = averages.get(e.evaluator) ?? {
|
|
136
|
+
sum: 0,
|
|
137
|
+
count: 0,
|
|
138
|
+
passed: 0,
|
|
139
|
+
failed: 0,
|
|
140
|
+
errored: 0,
|
|
141
|
+
};
|
|
193
142
|
if (typeof e.score === "number") {
|
|
194
143
|
stats.sum += e.score;
|
|
195
144
|
stats.count += 1;
|
|
@@ -197,29 +146,36 @@ export async function handleExperimentResults(params: {
|
|
|
197
146
|
if (e.status === "error") stats.errored += 1;
|
|
198
147
|
else if (e.passed === true) stats.passed += 1;
|
|
199
148
|
else if (e.passed === false) stats.failed += 1;
|
|
200
|
-
|
|
149
|
+
averages.set(e.evaluator, stats);
|
|
201
150
|
}
|
|
202
151
|
}
|
|
152
|
+
return averages;
|
|
153
|
+
}
|
|
203
154
|
|
|
155
|
+
function headerLines({
|
|
156
|
+
results,
|
|
157
|
+
runStatus,
|
|
158
|
+
filter,
|
|
159
|
+
evaluatorFilter,
|
|
160
|
+
}: {
|
|
161
|
+
results: EvaluationRunResults;
|
|
162
|
+
runStatus: RunStatus;
|
|
163
|
+
filter: "all" | "failed";
|
|
164
|
+
evaluatorFilter: string | undefined;
|
|
165
|
+
}): string[] {
|
|
204
166
|
const lines: string[] = [];
|
|
205
167
|
lines.push(`# Evaluation Results: ${results.runId}`);
|
|
206
168
|
lines.push("");
|
|
207
169
|
lines.push(`**Experiment**: ${results.experimentId}`);
|
|
208
170
|
lines.push(`**Status**: ${runStatus}`);
|
|
209
171
|
if (typeof results.total === "number" && results.total > 0) {
|
|
210
|
-
lines.push(
|
|
211
|
-
`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`,
|
|
212
|
-
);
|
|
172
|
+
lines.push(`**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`);
|
|
213
173
|
}
|
|
214
174
|
lines.push(`**Total rows**: ${results.dataset.length}`);
|
|
215
175
|
lines.push(`**Total evaluations**: ${results.evaluations.length}`);
|
|
216
|
-
if (filter === "failed")
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
if (evaluatorFilter) {
|
|
220
|
-
lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
221
|
-
}
|
|
222
|
-
if (partial) {
|
|
176
|
+
if (filter === "failed") lines.push(`**Filter**: failed only`);
|
|
177
|
+
if (evaluatorFilter) lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
178
|
+
if (!isTerminalStatus(runStatus)) {
|
|
223
179
|
lines.push("");
|
|
224
180
|
lines.push(
|
|
225
181
|
runStatus === "interrupted"
|
|
@@ -227,80 +183,157 @@ export async function handleExperimentResults(params: {
|
|
|
227
183
|
: "> These are partial results. The run is still in progress, so more rows may appear on a later call.",
|
|
228
184
|
);
|
|
229
185
|
}
|
|
186
|
+
const stored = results.completeness;
|
|
187
|
+
if (isTerminalStatus(runStatus) && stored && !stored.complete) {
|
|
188
|
+
lines.push("");
|
|
189
|
+
lines.push(
|
|
190
|
+
`> These are partial results. The run has ended, but the platform is still storing it: ${storedCount(stored.dataset)} rows and ${storedCount(stored.evaluations)} evaluations so far. Call again shortly for the whole run.`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
230
193
|
lines.push("");
|
|
194
|
+
return lines;
|
|
195
|
+
}
|
|
231
196
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
}
|
|
244
|
-
lines.push("");
|
|
197
|
+
function evaluatorSummaryLines(averages: Map<string, EvaluatorStats>): string[] {
|
|
198
|
+
if (averages.size === 0) return [];
|
|
199
|
+
const lines = [
|
|
200
|
+
"## Evaluator Summary",
|
|
201
|
+
"",
|
|
202
|
+
"| Evaluator | Avg Score | Passed | Failed | Errored |",
|
|
203
|
+
"| --- | --- | --- | --- | --- |",
|
|
204
|
+
];
|
|
205
|
+
for (const [name, stats] of averages) {
|
|
206
|
+
const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "—";
|
|
207
|
+
lines.push(`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`);
|
|
245
208
|
}
|
|
209
|
+
lines.push("");
|
|
210
|
+
return lines;
|
|
211
|
+
}
|
|
246
212
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
213
|
+
const storedCount = ({ received, expected }: { received: number; expected: number | null }) =>
|
|
214
|
+
expected === null ? `${received}` : `${received} of ${expected}`;
|
|
215
|
+
|
|
216
|
+
function noRowsLine({ filter, runStatus }: { filter: "all" | "failed"; runStatus: RunStatus }) {
|
|
217
|
+
if (filter === "failed") return "_No rows matched the filter._";
|
|
218
|
+
if (runStatus === "running")
|
|
219
|
+
return "_No rows recorded yet. The run is still in progress; call again shortly._";
|
|
220
|
+
if (runStatus === "interrupted") return "_No rows were recorded before the run was interrupted._";
|
|
221
|
+
return "_No rows recorded for this run._";
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function evaluationLines(e: EvaluationItem): string[] {
|
|
225
|
+
const parts: string[] = [`**${e.evaluator}**`];
|
|
226
|
+
if (e.status === "error") parts.push("status=error");
|
|
227
|
+
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
228
|
+
if (typeof e.passed === "boolean") parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
229
|
+
if (e.label) parts.push(`label=${e.label}`);
|
|
230
|
+
const lines = [`- ${parts.join(" · ")}`];
|
|
231
|
+
if (e.details) lines.push(` - details: ${e.details}`);
|
|
232
|
+
return lines;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function rowLines({
|
|
236
|
+
row: { entry, evaluations },
|
|
237
|
+
anyEvaluators,
|
|
238
|
+
}: {
|
|
239
|
+
row: ResultRow;
|
|
240
|
+
anyEvaluators: boolean;
|
|
241
|
+
}): string[] {
|
|
242
|
+
const summary = summarizeEntry(entry.entry);
|
|
243
|
+
const lines = [`### Row #${entry.index}${summary ? ` — ${summary}` : ""}`];
|
|
244
|
+
if (entry.error) lines.push(`- **Error**: ${entry.error}`);
|
|
245
|
+
if (entry.traceId) lines.push(`- **Trace ID**: \`${entry.traceId}\``);
|
|
246
|
+
if (evaluations.length === 0) {
|
|
247
|
+
lines.push(
|
|
248
|
+
anyEvaluators ? "- _No evaluations recorded for this row_" : "- _No evaluations recorded_",
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
for (const e of evaluations) lines.push(...evaluationLines(e));
|
|
252
|
+
lines.push("");
|
|
253
|
+
return lines;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function notFoundReport({ runId, expired }: { runId: string; expired: boolean }): string {
|
|
257
|
+
const lines = [`# Evaluation Results: ${runId}`, "", "**Status**: not found", ""];
|
|
258
|
+
if (expired) {
|
|
259
|
+
lines.push(
|
|
260
|
+
`Could not load results for run \`${runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
|
|
261
|
+
"",
|
|
262
|
+
"> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids.",
|
|
263
|
+
);
|
|
264
|
+
} else {
|
|
265
|
+
lines.push(
|
|
266
|
+
`Could not load results for run \`${runId}\`.`,
|
|
267
|
+
"",
|
|
268
|
+
"> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id.",
|
|
269
|
+
);
|
|
260
270
|
}
|
|
271
|
+
return lines.join("\n");
|
|
272
|
+
}
|
|
261
273
|
|
|
262
|
-
|
|
263
|
-
|
|
274
|
+
export async function handleExperimentResults(params: {
|
|
275
|
+
runId: string;
|
|
276
|
+
experimentSlug?: string;
|
|
277
|
+
filter?: "all" | "failed";
|
|
278
|
+
evaluator?: string;
|
|
279
|
+
limit?: number;
|
|
280
|
+
}): Promise<string> {
|
|
281
|
+
const filter = params.filter ?? "all";
|
|
282
|
+
const evaluatorFilter = params.evaluator?.trim();
|
|
283
|
+
const limit =
|
|
284
|
+
typeof params.limit === "number" && params.limit > 0
|
|
285
|
+
? Math.min(params.limit, DEFAULT_ROW_CAP)
|
|
286
|
+
: DEFAULT_ROW_CAP;
|
|
287
|
+
|
|
288
|
+
let results: EvaluationRunResults | null;
|
|
289
|
+
try {
|
|
290
|
+
results = await fetchRunResults(params);
|
|
291
|
+
} catch (error) {
|
|
292
|
+
if (isNotFoundError(error)) return notFoundReport({ runId: params.runId, expired: true });
|
|
293
|
+
throw error;
|
|
294
|
+
}
|
|
295
|
+
if (!results) return notFoundReport({ runId: params.runId, expired: false });
|
|
296
|
+
|
|
297
|
+
// Partial results are served while the run is in progress: rows land
|
|
298
|
+
// incrementally, and a run whose SDK died before flushing still has useful
|
|
299
|
+
// rows, so we never gate on a terminal status.
|
|
300
|
+
const runStatus = deriveRunStatus(results.timestamps);
|
|
301
|
+
const evaluationsByRow = groupEvaluationsByRow({
|
|
302
|
+
evaluations: results.evaluations,
|
|
303
|
+
evaluatorFilter,
|
|
304
|
+
});
|
|
305
|
+
const evaluatorNames = new Set(
|
|
306
|
+
(evaluatorFilter
|
|
307
|
+
? results.evaluations.filter((e) => e.evaluator === evaluatorFilter)
|
|
308
|
+
: results.evaluations
|
|
309
|
+
).map((e) => e.evaluator),
|
|
264
310
|
);
|
|
265
|
-
lines.push("");
|
|
266
311
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
for (const e of evaluations) {
|
|
284
|
-
const parts: string[] = [`**${e.evaluator}**`];
|
|
285
|
-
if (e.status === "error") parts.push("status=error");
|
|
286
|
-
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
287
|
-
if (typeof e.passed === "boolean") {
|
|
288
|
-
parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
289
|
-
}
|
|
290
|
-
if (e.label) parts.push(`label=${e.label}`);
|
|
291
|
-
lines.push(`- ${parts.join(" · ")}`);
|
|
292
|
-
if (e.details) {
|
|
293
|
-
lines.push(` - details: ${e.details}`);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
lines.push("");
|
|
312
|
+
const allRows: ResultRow[] = results.dataset.map((entry) => ({
|
|
313
|
+
entry,
|
|
314
|
+
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? [],
|
|
315
|
+
}));
|
|
316
|
+
// The summary covers the filtered rows before truncation.
|
|
317
|
+
const matching = filter === "failed" ? allRows.filter((r) => isFailedRow(r)) : allRows;
|
|
318
|
+
const truncated = matching.length > limit;
|
|
319
|
+
const rows = matching.slice(0, limit);
|
|
320
|
+
|
|
321
|
+
const lines = [
|
|
322
|
+
...headerLines({ results, runStatus, filter, evaluatorFilter }),
|
|
323
|
+
...evaluatorSummaryLines(tallyEvaluatorStats(matching)),
|
|
324
|
+
];
|
|
325
|
+
if (rows.length === 0) {
|
|
326
|
+
lines.push(noRowsLine({ filter, runStatus }));
|
|
327
|
+
return lines.join("\n");
|
|
297
328
|
}
|
|
298
329
|
|
|
330
|
+
lines.push(`## Rows (${rows.length}${truncated ? ` of ${matching.length}` : ""})`);
|
|
331
|
+
lines.push("");
|
|
332
|
+
for (const row of rows) lines.push(...rowLines({ row, anyEvaluators: evaluatorNames.size > 0 }));
|
|
299
333
|
if (truncated) {
|
|
300
334
|
lines.push(
|
|
301
|
-
`> Output truncated to ${limit} rows of ${
|
|
335
|
+
`> Output truncated to ${limit} rows of ${matching.length} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`,
|
|
302
336
|
);
|
|
303
337
|
}
|
|
304
|
-
|
|
305
338
|
return lines.join("\n");
|
|
306
339
|
}
|
package/src/tools/get-project.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { getProject as apiGetProject } from "../langwatch-api-projects.
|
|
1
|
+
import { getProject as apiGetProject } from "../langwatch-api-projects.ts";
|
|
2
2
|
|
|
3
|
-
export async function handleGetProject(params: {
|
|
4
|
-
id: string;
|
|
5
|
-
}): Promise<string> {
|
|
3
|
+
export async function handleGetProject(params: { id: string }): Promise<string> {
|
|
6
4
|
const project = await apiGetProject(params.id);
|
|
7
5
|
|
|
8
6
|
const lines: string[] = [];
|
package/src/tools/get-prompt.ts
CHANGED
|
@@ -1,54 +1,117 @@
|
|
|
1
|
-
import { getPrompt as apiGetPrompt } from "../langwatch-api.
|
|
1
|
+
import { getPrompt as apiGetPrompt, type PromptDetailResponse } from "../langwatch-api.ts";
|
|
2
|
+
|
|
3
|
+
interface DeploymentTag {
|
|
4
|
+
name: string;
|
|
5
|
+
versionId?: string;
|
|
6
|
+
}
|
|
2
7
|
|
|
3
8
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* AI-readable markdown, including messages, model config, and version history.
|
|
9
|
+
* Normalizes one tag entry from the API. The API returns tag objects
|
|
10
|
+
* ({ name, versionId }); plain strings are tolerated so a shape drift can
|
|
11
|
+
* never crash the tool.
|
|
8
12
|
*/
|
|
13
|
+
function normalizeTag(tag: unknown): DeploymentTag | undefined {
|
|
14
|
+
if (typeof tag === "string") return { name: tag };
|
|
15
|
+
if (tag && typeof tag === "object" && "name" in tag) {
|
|
16
|
+
const candidate = tag as { name: unknown; versionId?: unknown };
|
|
17
|
+
if (typeof candidate.name === "string") {
|
|
18
|
+
return {
|
|
19
|
+
name: candidate.name,
|
|
20
|
+
versionId: typeof candidate.versionId === "string" ? candidate.versionId : undefined,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function fieldEntry(field: unknown): string {
|
|
28
|
+
if (field && typeof field === "object" && "identifier" in field) {
|
|
29
|
+
const typed = field as { identifier: unknown; type?: unknown };
|
|
30
|
+
return `- **${String(typed.identifier)}**: ${typeof typed.type === "string" ? typed.type : "unknown"}`;
|
|
31
|
+
}
|
|
32
|
+
return `- ${JSON.stringify(field)}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Renders a field list of any shape the API may return: an array of
|
|
36
|
+
// { identifier, type } entries (inputs/outputs), an object map of
|
|
37
|
+
// name -> value (parameters), or absent/empty (no heading at all).
|
|
38
|
+
function fieldListLines({ heading, fields }: { heading: string; fields: unknown }): string[] {
|
|
39
|
+
const entries: string[] = [];
|
|
40
|
+
if (Array.isArray(fields)) {
|
|
41
|
+
entries.push(...fields.map(fieldEntry));
|
|
42
|
+
} else if (fields && typeof fields === "object") {
|
|
43
|
+
for (const [key, value] of Object.entries(fields)) {
|
|
44
|
+
entries.push(`- **${key}**: ${JSON.stringify(value)}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (entries.length === 0) return [];
|
|
48
|
+
return [`\n**${heading}**`, ...entries];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function messageLines(prompt: PromptDetailResponse): string[] {
|
|
52
|
+
const messages = prompt.messages;
|
|
53
|
+
if (Array.isArray(messages) && messages.length > 0) {
|
|
54
|
+
return ["\n## Messages", ...messages.flatMap((msg) => [`\n### ${msg.role}`, msg.content])];
|
|
55
|
+
}
|
|
56
|
+
if (typeof prompt.prompt === "string" && prompt.prompt.length > 0) {
|
|
57
|
+
return ["\n## Messages", "\n### system", prompt.prompt];
|
|
58
|
+
}
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Deployments: every non-"latest" tag on the prompt, each shown with the
|
|
63
|
+
// version it currently points to. Always rendered, even when empty, so an
|
|
64
|
+
// empty section is never mistaken for "not checked" — and a tag pointing
|
|
65
|
+
// at another version is listed truthfully instead of implied undeployed.
|
|
66
|
+
function deploymentLines(prompt: PromptDetailResponse): string[] {
|
|
67
|
+
const lines = ["\n## Deployments"];
|
|
68
|
+
const tags = Array.isArray(prompt.tags) ? prompt.tags : [];
|
|
69
|
+
for (const rawTag of tags) {
|
|
70
|
+
const tag = normalizeTag(rawTag);
|
|
71
|
+
if (!tag || tag.name === "latest") continue;
|
|
72
|
+
const target = tag.versionId ?? "unknown version";
|
|
73
|
+
const marker = tag.versionId && tag.versionId === prompt.versionId ? " (this version)" : "";
|
|
74
|
+
lines.push(`- ${tag.name} → ${target}${marker}`);
|
|
75
|
+
}
|
|
76
|
+
return lines;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Retrieves a prompt by ID/handle; returns a markdown digest by default or
|
|
80
|
+
// raw JSON if format: "json" is specified.
|
|
9
81
|
export async function handleGetPrompt(params: {
|
|
10
82
|
idOrHandle: string;
|
|
11
83
|
version?: number;
|
|
12
84
|
tag?: string;
|
|
85
|
+
format?: "digest" | "json";
|
|
13
86
|
}): Promise<string> {
|
|
14
|
-
const prompt = await apiGetPrompt(params.idOrHandle, {
|
|
87
|
+
const prompt = await apiGetPrompt(params.idOrHandle, {
|
|
88
|
+
version: params.version,
|
|
89
|
+
tag: params.tag,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
if (params.format === "json") {
|
|
93
|
+
return JSON.stringify(prompt, null, 2);
|
|
94
|
+
}
|
|
15
95
|
|
|
16
96
|
const lines: string[] = [];
|
|
17
|
-
lines.push(
|
|
18
|
-
`# Prompt: ${prompt.name || prompt.handle || prompt.id}\n`
|
|
19
|
-
);
|
|
97
|
+
lines.push(`# Prompt: ${prompt.name || prompt.handle || prompt.id}\n`);
|
|
20
98
|
|
|
21
99
|
if (prompt.handle) lines.push(`**Handle**: ${prompt.handle}`);
|
|
22
100
|
if (prompt.id) lines.push(`**ID**: ${prompt.id}`);
|
|
23
|
-
if (prompt.
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
if (
|
|
101
|
+
if (prompt.version != null) lines.push(`**Version**: v${prompt.version}`);
|
|
102
|
+
if (prompt.versionId) lines.push(`**Version ID**: ${prompt.versionId}`);
|
|
103
|
+
if (prompt.commitMessage) lines.push(`**Commit**: ${prompt.commitMessage}`);
|
|
104
|
+
if (prompt.model) lines.push(`**Model**: ${prompt.model}`);
|
|
105
|
+
if (prompt.temperature != null) lines.push(`**Temperature**: ${prompt.temperature}`);
|
|
106
|
+
if (prompt.maxTokens != null) lines.push(`**Max Tokens**: ${prompt.maxTokens}`);
|
|
107
|
+
if (prompt.responseFormat)
|
|
108
|
+
lines.push(`**Response Format**: ${JSON.stringify(prompt.responseFormat)}`);
|
|
29
109
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
lines.push(`\n### ${msg.role}`);
|
|
36
|
-
lines.push(msg.content);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Show version history
|
|
41
|
-
if (prompt.versions && prompt.versions.length > 0) {
|
|
42
|
-
lines.push("\n## Version History");
|
|
43
|
-
for (const v of prompt.versions.slice(0, 10)) {
|
|
44
|
-
const versionNum = v.version ?? "?";
|
|
45
|
-
const commitMsg = v.commitMessage || "No message";
|
|
46
|
-
lines.push(`- **v${versionNum}**: ${commitMsg}`);
|
|
47
|
-
}
|
|
48
|
-
if (prompt.versions.length > 10) {
|
|
49
|
-
lines.push(`... and ${prompt.versions.length - 10} more versions`);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
110
|
+
lines.push(...fieldListLines({ heading: "Parameters", fields: prompt.parameters }));
|
|
111
|
+
lines.push(...fieldListLines({ heading: "Inputs", fields: prompt.inputs }));
|
|
112
|
+
lines.push(...fieldListLines({ heading: "Outputs", fields: prompt.outputs }));
|
|
113
|
+
lines.push(...messageLines(prompt));
|
|
114
|
+
lines.push(...deploymentLines(prompt));
|
|
52
115
|
|
|
53
116
|
return lines.join("\n");
|
|
54
117
|
}
|