@langwatch/mcp-server 1.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/.python-version +1 -0
  2. package/CHANGELOG.md +110 -0
  3. package/CONTRIBUTING.md +1 -1
  4. package/README.md +86 -4
  5. package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
  6. package/dist/archive-run-plan-474OKHHU.js +16 -0
  7. package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
  8. package/dist/archive-test-suite-O7FLDMXK.js +16 -0
  9. package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
  10. package/dist/chunk-56OGTS24.js +44 -0
  11. package/dist/chunk-6DJDEXET.js +63 -0
  12. package/dist/chunk-7E4FZIYJ.js +15 -0
  13. package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
  14. package/dist/chunk-CKNAH5TG.js +74 -0
  15. package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
  16. package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
  17. package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
  18. package/dist/chunk-HV4DWKBI.js +46 -0
  19. package/dist/chunk-JR6DJZOU.js +49 -0
  20. package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
  21. package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
  22. package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
  23. package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
  24. package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
  25. package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
  26. package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
  27. package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
  28. package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
  29. package/dist/chunk-UF6XIGYR.js +58 -0
  30. package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
  31. package/dist/chunk-UJPXFA26.js +267 -0
  32. package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
  33. package/dist/config.js +2 -2
  34. package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
  35. package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
  36. package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
  37. package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
  38. package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
  39. package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
  40. package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
  41. package/dist/create-mcp-server.js +9 -4
  42. package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
  43. package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
  44. package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
  45. package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
  46. package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
  47. package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
  48. package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
  49. package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
  50. package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
  51. package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
  52. package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
  53. package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
  54. package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
  55. package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
  56. package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
  57. package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
  58. package/dist/get-agent-7THF7HCU.js +66 -0
  59. package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
  60. package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
  61. package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
  62. package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
  63. package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
  64. package/dist/get-prompt-GVKXF5NL.js +100 -0
  65. package/dist/get-run-plan-NY7J5GRE.js +61 -0
  66. package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
  67. package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
  68. package/dist/get-test-suite-4HKA7OKS.js +49 -0
  69. package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
  70. package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
  71. package/dist/http-security-EU3CHTY7.js +26 -0
  72. package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
  73. package/dist/index.js +6437 -4020
  74. package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
  75. package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
  76. package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
  77. package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
  78. package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
  79. package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
  80. package/dist/list-agents-AMVDQPQL.js +43 -0
  81. package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
  82. package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
  83. package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
  84. package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
  85. package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
  86. package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
  87. package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
  88. package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
  89. package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
  90. package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
  91. package/dist/list-run-plans-UBAU2DGH.js +46 -0
  92. package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
  93. package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
  94. package/dist/list-test-suites-L7R2O6DR.js +35 -0
  95. package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
  96. package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
  97. package/dist/rename-test-suite-NTYHAO5X.js +21 -0
  98. package/dist/report-issue-RHZZ33IS.js +885 -0
  99. package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
  100. package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
  101. package/dist/run-agent-CAGWGTYT.js +72 -0
  102. package/dist/run-plan-T4ZYOZGA.js +39 -0
  103. package/dist/run-test-suite-L4MZ7G4D.js +26 -0
  104. package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
  105. package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
  106. package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
  107. package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
  108. package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
  109. package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
  110. package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
  111. package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
  112. package/dist/update-prompt-ZQQ2C3KU.js +143 -0
  113. package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
  114. package/package.json +8 -9
  115. package/pyproject.toml +5 -5
  116. package/src/__tests__/agent-tools.unit.test.ts +257 -0
  117. package/src/__tests__/all-tools.integration.test.ts +321 -75
  118. package/src/__tests__/config.unit.test.ts +33 -0
  119. package/src/__tests__/docker.integration.test.ts +91 -7
  120. package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
  121. package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
  122. package/src/__tests__/http-security-origin.unit.test.ts +97 -0
  123. package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
  124. package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
  125. package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
  126. package/src/__tests__/http-server-origin.integration.test.ts +169 -0
  127. package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
  128. package/src/__tests__/http-transport.integration.test.ts +85 -87
  129. package/src/__tests__/integration.integration.test.ts +7 -11
  130. package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
  131. package/src/__tests__/prompts.integration.test.ts +736 -0
  132. package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
  133. package/src/__tests__/scenario-tools.unit.test.ts +19 -0
  134. package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
  135. package/src/__tests__/support/http-server-harness.ts +138 -0
  136. package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
  137. package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
  138. package/src/config.ts +17 -2
  139. package/src/create-mcp-server.ts +334 -55
  140. package/src/http-security.ts +492 -0
  141. package/src/http-server.ts +536 -152
  142. package/src/index.ts +49 -2
  143. package/src/langwatch-api-agents.ts +196 -12
  144. package/src/langwatch-api-run-plans.ts +157 -0
  145. package/src/langwatch-api-scenarios.ts +9 -0
  146. package/src/langwatch-api-simulation-runs.ts +5 -0
  147. package/src/langwatch-api-test-suites.ts +135 -0
  148. package/src/langwatch-api.ts +62 -12
  149. package/src/schemas/index.ts +5 -0
  150. package/src/schemas/run-plan.ts +92 -0
  151. package/src/schemas/suite-fields.ts +149 -0
  152. package/src/tools/archive-run-plan.ts +12 -0
  153. package/src/tools/archive-test-suite.ts +12 -0
  154. package/src/tools/create-scenario.ts +8 -0
  155. package/src/tools/create-test-suite.ts +43 -0
  156. package/src/tools/discover-evaluator-schema.ts +2 -2
  157. package/src/tools/discover-scenario-schema.ts +54 -0
  158. package/src/tools/format-run-plan.ts +68 -0
  159. package/src/tools/format-scenario.ts +14 -0
  160. package/src/tools/format-suite-details.ts +96 -0
  161. package/src/tools/get-agent.ts +45 -3
  162. package/src/tools/get-prompt.ts +108 -24
  163. package/src/tools/get-run-plan.ts +62 -0
  164. package/src/tools/get-scenario.ts +3 -0
  165. package/src/tools/get-simulation-run.ts +9 -0
  166. package/src/tools/get-test-suite.ts +49 -0
  167. package/src/tools/list-agents.ts +20 -3
  168. package/src/tools/list-run-plans.ts +47 -0
  169. package/src/tools/list-scenarios.ts +12 -2
  170. package/src/tools/list-simulation-runs.ts +6 -2
  171. package/src/tools/list-test-suites.ts +35 -0
  172. package/src/tools/rename-test-suite.ts +18 -0
  173. package/src/tools/rerun-run-plan.ts +21 -0
  174. package/src/tools/run-agent.ts +54 -6
  175. package/src/tools/run-plan.ts +49 -0
  176. package/src/tools/run-test-suite.ts +30 -0
  177. package/src/tools/test-agent.ts +18 -0
  178. package/src/tools/update-prompt.ts +195 -10
  179. package/src/tools/update-scenario.ts +8 -0
  180. package/src/tools/update-test-suite.ts +45 -0
  181. package/tsconfig.json +7 -2
  182. package/uv.lock +418 -298
  183. package/dist/archive-suite-C3KX6IIZ.js +0 -16
  184. package/dist/chunk-WDH5P5RF.js +0 -48
  185. package/dist/create-suite-TWF6BBR4.js +0 -39
  186. package/dist/get-agent-7QJVPH72.js +0 -29
  187. package/dist/get-prompt-I753LZKZ.js +0 -46
  188. package/dist/get-suite-GTOCY5PL.js +0 -45
  189. package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
  190. package/dist/list-agents-TIJTP566.js +0 -33
  191. package/dist/list-suites-5VK4HXEK.js +0 -43
  192. package/dist/report-issue-DXJ6TPTC.js +0 -412
  193. package/dist/run-agent-RBSIFKRT.js +0 -37
  194. package/dist/run-suite-K3DNQ3SE.js +0 -30
  195. package/dist/update-prompt-RQFTEO7Y.js +0 -25
  196. package/dist/update-suite-TRGGLNR7.js +0 -40
  197. package/pnpm-workspace.yaml +0 -34
  198. package/src/langwatch-api-suites.ts +0 -118
  199. package/src/tools/archive-suite.ts +0 -12
  200. package/src/tools/create-suite.ts +0 -42
  201. package/src/tools/get-suite.ts +0 -47
  202. package/src/tools/list-suites.ts +0 -43
  203. package/src/tools/run-suite.ts +0 -28
  204. package/src/tools/update-suite.ts +0 -44
@@ -1,6 +1,6 @@
1
- import "./chunk-BJ7B4R26.js";
1
+ import "./chunk-SAS5OB7F.js";
2
2
 
3
- // ../langevals/ts-integration/evaluators.generated.ts
3
+ // ../../services/langevals/ts-integration/evaluators.generated.ts
4
4
  import { z } from "zod";
5
5
  var evaluatorTypesSchema = z.string();
6
6
  var moneySchema = z.object({
@@ -18,7 +18,8 @@ var evaluationResultSchema = z.object({
18
18
  });
19
19
  var evaluationResultSkippedSchema = z.object({
20
20
  status: z.literal("skipped"),
21
- details: z.string().optional()
21
+ details: z.string().optional(),
22
+ cost: moneySchema.optional()
22
23
  });
23
24
  var evaluationResultErrorSchema = z.object({
24
25
  status: z.literal("error"),
@@ -379,69 +380,6 @@ var evaluatorsSchema = z.object({
379
380
  ).default(0.25)
380
381
  })
381
382
  }),
382
- "legacy/ragas_answer_correctness": z.object({
383
- settings: z.object({
384
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
385
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
386
- max_tokens: z.number().describe(
387
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
388
- ).default(2048)
389
- })
390
- }),
391
- "legacy/ragas_answer_relevancy": z.object({
392
- settings: z.object({
393
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
394
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
395
- max_tokens: z.number().describe(
396
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
397
- ).default(2048)
398
- })
399
- }),
400
- "legacy/ragas_context_precision": z.object({
401
- settings: z.object({
402
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
403
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
404
- max_tokens: z.number().describe(
405
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
406
- ).default(2048)
407
- })
408
- }),
409
- "legacy/ragas_context_recall": z.object({
410
- settings: z.object({
411
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
412
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
413
- max_tokens: z.number().describe(
414
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
415
- ).default(2048)
416
- })
417
- }),
418
- "legacy/ragas_context_relevancy": z.object({
419
- settings: z.object({
420
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
421
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
422
- max_tokens: z.number().describe(
423
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
424
- ).default(2048)
425
- })
426
- }),
427
- "legacy/ragas_context_utilization": z.object({
428
- settings: z.object({
429
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
430
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
431
- max_tokens: z.number().describe(
432
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
433
- ).default(2048)
434
- })
435
- }),
436
- "legacy/ragas_faithfulness": z.object({
437
- settings: z.object({
438
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
439
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
440
- max_tokens: z.number().describe(
441
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
442
- ).default(2048)
443
- })
444
- }),
445
383
  "langevals/basic": z.object({
446
384
  settings: z.object({
447
385
  rules: z.array(
@@ -580,6 +518,12 @@ var evaluatorsSchema = z.object({
580
518
  include_metrics: z.array(z.union([z.literal("cost"), z.literal("duration")])).describe("Per-candidate metrics to inject into the judge prompt").default([])
581
519
  })
582
520
  }),
521
+ "langevals/query_resolution": z.object({
522
+ settings: z.object({
523
+ model: z.string().describe("The model to use for evaluation").default("openai/gpt-5-mini"),
524
+ max_tokens: z.number().describe("Max tokens allowed for evaluation").default(128e3)
525
+ })
526
+ }),
583
527
  "langevals/select_best_compare": z.object({
584
528
  settings: z.object({
585
529
  model: z.string().describe("The model to use for evaluation").default("openai/gpt-5-mini"),
@@ -609,13 +553,13 @@ slot label. Use "tie" only when no candidate is clearly better.
609
553
  allow_tie: z.boolean().describe(
610
554
  "Allow the judge to return 'tie' when candidates are equivalent"
611
555
  ).default(true),
612
- include_metrics: z.array(z.union([z.literal("cost"), z.literal("duration")])).describe("Per-candidate metrics to inject into the judge prompt").default([])
613
- })
614
- }),
615
- "langevals/query_resolution": z.object({
616
- settings: z.object({
617
- model: z.string().describe("The model to use for evaluation").default("openai/gpt-5-mini"),
618
- max_tokens: z.number().describe("Max tokens allowed for evaluation").default(128e3)
556
+ include_metrics: z.array(z.union([z.literal("cost"), z.literal("duration")])).describe("Per-candidate metrics to inject into the judge prompt").default([]),
557
+ temperature: z.number().describe(
558
+ "Sampling temperature for the judge call. Lower is more deterministic."
559
+ ).default(0),
560
+ swap_and_reconcile: z.boolean().describe(
561
+ "Check each row twice, the second time with the candidate order reversed, and record no result for that row when the two disagree \u2014 rather than a tie, which would claim the candidates are equally good. Doubles judge-call cost per row (literature: swapping candidate order alone flips 10-30% of verdicts on contested rows)."
562
+ ).default(true)
619
563
  })
620
564
  }),
621
565
  "langevals/sentiment": z.object({
@@ -1237,223 +1181,6 @@ or if it's in a specific expected language.
1237
1181
  }
1238
1182
  }
1239
1183
  },
1240
- "legacy/ragas_answer_correctness": {
1241
- name: `Ragas Answer Correctness`,
1242
- description: `
1243
- Computes with an LLM a weighted combination of factual as well as semantic similarity between the generated answer and the expected output.
1244
- `,
1245
- category: "rag",
1246
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/answer_correctness.html",
1247
- isGuardrail: false,
1248
- requiredFields: ["output", "expected_output"],
1249
- optionalFields: ["input"],
1250
- settings: {
1251
- model: {
1252
- description: "The model to use for evaluation.",
1253
- default: "openai/gpt-5-mini"
1254
- },
1255
- embeddings_model: {
1256
- description: "The model to use for embeddings.",
1257
- default: "openai/text-embedding-ada-002"
1258
- },
1259
- max_tokens: {
1260
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1261
- default: 2048
1262
- }
1263
- },
1264
- envVars: [],
1265
- result: {
1266
- score: {
1267
- description: "A score between 0.0 and 1.0 indicating the correctness of the answer."
1268
- }
1269
- }
1270
- },
1271
- "legacy/ragas_answer_relevancy": {
1272
- name: `Ragas Answer Relevancy`,
1273
- description: `
1274
- Evaluates how pertinent the generated answer is to the given prompt. Higher scores indicate better relevancy.
1275
- `,
1276
- category: "rag",
1277
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/answer_relevance.html",
1278
- isGuardrail: false,
1279
- requiredFields: ["input", "output"],
1280
- optionalFields: [],
1281
- settings: {
1282
- model: {
1283
- description: "The model to use for evaluation.",
1284
- default: "openai/gpt-5-mini"
1285
- },
1286
- embeddings_model: {
1287
- description: "The model to use for embeddings.",
1288
- default: "openai/text-embedding-ada-002"
1289
- },
1290
- max_tokens: {
1291
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1292
- default: 2048
1293
- }
1294
- },
1295
- envVars: [],
1296
- result: {
1297
- score: {
1298
- description: "A score between 0.0 and 1.0 indicating the relevance of the answer."
1299
- }
1300
- }
1301
- },
1302
- "legacy/ragas_context_precision": {
1303
- name: `Ragas Context Precision`,
1304
- description: `
1305
- This metric evaluates whether all of the ground-truth relevant items present in the contexts are ranked higher or not. Higher scores indicate better precision.
1306
- `,
1307
- category: "rag",
1308
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_precision.html",
1309
- isGuardrail: false,
1310
- requiredFields: ["input", "contexts", "expected_output"],
1311
- optionalFields: [],
1312
- settings: {
1313
- model: {
1314
- description: "The model to use for evaluation.",
1315
- default: "openai/gpt-5-mini"
1316
- },
1317
- embeddings_model: {
1318
- description: "The model to use for embeddings.",
1319
- default: "openai/text-embedding-ada-002"
1320
- },
1321
- max_tokens: {
1322
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1323
- default: 2048
1324
- }
1325
- },
1326
- envVars: [],
1327
- result: {
1328
- score: {
1329
- description: "A score between 0.0 and 1.0 indicating the precision of the context."
1330
- }
1331
- }
1332
- },
1333
- "legacy/ragas_context_recall": {
1334
- name: `Ragas Context Recall`,
1335
- description: `
1336
- This evaluator measures the extent to which the retrieved context aligns with the annotated answer, treated as the ground truth. Higher values indicate better performance.
1337
- `,
1338
- category: "rag",
1339
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_recall.html",
1340
- isGuardrail: false,
1341
- requiredFields: ["input", "contexts", "expected_output"],
1342
- optionalFields: [],
1343
- settings: {
1344
- model: {
1345
- description: "The model to use for evaluation.",
1346
- default: "openai/gpt-5-mini"
1347
- },
1348
- embeddings_model: {
1349
- description: "The model to use for embeddings.",
1350
- default: "openai/text-embedding-ada-002"
1351
- },
1352
- max_tokens: {
1353
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1354
- default: 2048
1355
- }
1356
- },
1357
- envVars: [],
1358
- result: {
1359
- score: {
1360
- description: "A score between 0.0 and 1.0 indicating the recall of the context."
1361
- }
1362
- }
1363
- },
1364
- "legacy/ragas_context_relevancy": {
1365
- name: `Ragas Context Relevancy`,
1366
- description: `
1367
- This metric gauges the relevancy of the retrieved context, calculated based on both the question and contexts. The values fall within the range of (0, 1), with higher values indicating better relevancy.
1368
- `,
1369
- category: "rag",
1370
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_relevancy.html",
1371
- isGuardrail: false,
1372
- requiredFields: ["output", "contexts"],
1373
- optionalFields: [],
1374
- settings: {
1375
- model: {
1376
- description: "The model to use for evaluation.",
1377
- default: "openai/gpt-5-mini"
1378
- },
1379
- embeddings_model: {
1380
- description: "The model to use for embeddings.",
1381
- default: "openai/text-embedding-ada-002"
1382
- },
1383
- max_tokens: {
1384
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1385
- default: 2048
1386
- }
1387
- },
1388
- envVars: [],
1389
- result: {
1390
- score: {
1391
- description: "A score between 0.0 and 1.0 indicating the relevancy of the context."
1392
- }
1393
- }
1394
- },
1395
- "legacy/ragas_context_utilization": {
1396
- name: `Ragas Context Utilization`,
1397
- description: `
1398
- This metric evaluates whether all of the output relevant items present in the contexts are ranked higher or not. Higher scores indicate better utilization.
1399
- `,
1400
- category: "rag",
1401
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_precision.html",
1402
- isGuardrail: false,
1403
- requiredFields: ["input", "output", "contexts"],
1404
- optionalFields: [],
1405
- settings: {
1406
- model: {
1407
- description: "The model to use for evaluation.",
1408
- default: "openai/gpt-5-mini"
1409
- },
1410
- embeddings_model: {
1411
- description: "The model to use for embeddings.",
1412
- default: "openai/text-embedding-ada-002"
1413
- },
1414
- max_tokens: {
1415
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1416
- default: 2048
1417
- }
1418
- },
1419
- envVars: [],
1420
- result: {
1421
- score: {
1422
- description: "A score between 0.0 and 1.0 indicating the utilization of the context."
1423
- }
1424
- }
1425
- },
1426
- "legacy/ragas_faithfulness": {
1427
- name: `Ragas Faithfulness`,
1428
- description: `
1429
- This evaluator assesses the extent to which the generated answer is consistent with the provided context. Higher scores indicate better faithfulness to the context, useful for detecting hallucinations.
1430
- `,
1431
- category: "rag",
1432
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/faithfulness.html",
1433
- isGuardrail: false,
1434
- requiredFields: ["output", "contexts"],
1435
- optionalFields: [],
1436
- settings: {
1437
- model: {
1438
- description: "The model to use for evaluation.",
1439
- default: "openai/gpt-5-mini"
1440
- },
1441
- embeddings_model: {
1442
- description: "The model to use for embeddings.",
1443
- default: "openai/text-embedding-ada-002"
1444
- },
1445
- max_tokens: {
1446
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1447
- default: 2048
1448
- }
1449
- },
1450
- envVars: [],
1451
- result: {
1452
- score: {
1453
- description: "A score between 0.0 and 1.0 indicating the faithfulness of the answer."
1454
- }
1455
- }
1456
- },
1457
1184
  "langevals/basic": {
1458
1185
  name: `Custom Basic Evaluator`,
1459
1186
  description: `
@@ -1597,8 +1324,8 @@ A simple evaluator that checks if the output matches the expected_output exactly
1597
1324
  category: "quality",
1598
1325
  docsUrl: "",
1599
1326
  isGuardrail: false,
1600
- requiredFields: [],
1601
- optionalFields: ["output", "expected_output"],
1327
+ requiredFields: ["output", "expected_output"],
1328
+ optionalFields: [],
1602
1329
  settings: {
1603
1330
  case_sensitive: {
1604
1331
  description: "True if the comparison should be case-sensitive, False otherwise",
@@ -1628,8 +1355,8 @@ Uses an LLM to check if the generated output answers a question correctly the sa
1628
1355
  category: "quality",
1629
1356
  docsUrl: "",
1630
1357
  isGuardrail: false,
1631
- requiredFields: [],
1632
- optionalFields: ["input", "output", "expected_output"],
1358
+ requiredFields: ["output", "expected_output"],
1359
+ optionalFields: ["input"],
1633
1360
  settings: {
1634
1361
  model: {
1635
1362
  description: "The model to use for evaluation",
@@ -1866,19 +1593,44 @@ better one. Superseded by Comparison, which judges two or more candidates.
1866
1593
  }
1867
1594
  }
1868
1595
  },
1596
+ "langevals/query_resolution": {
1597
+ name: `Query Resolution`,
1598
+ description: `
1599
+ This evaluator checks if all the user queries in the conversation were resolved. Useful to detect when the bot doesn't know how to answer or can't help the user.
1600
+ `,
1601
+ category: "quality",
1602
+ docsUrl: "",
1603
+ isGuardrail: false,
1604
+ requiredFields: ["conversation"],
1605
+ optionalFields: [],
1606
+ settings: {
1607
+ model: {
1608
+ description: "The model to use for evaluation",
1609
+ default: "openai/gpt-5-mini"
1610
+ },
1611
+ max_tokens: {
1612
+ description: "Max tokens allowed for evaluation",
1613
+ default: 128e3
1614
+ }
1615
+ },
1616
+ envVars: [],
1617
+ result: {}
1618
+ },
1869
1619
  "langevals/select_best_compare": {
1870
1620
  name: `Comparison`,
1871
1621
  description: `
1872
1622
  Compare two or more candidate outputs and pick the best one, optionally
1873
1623
  against a reference answer. The judge sees every candidate at once and
1874
1624
  explains why the winner is better. Candidate order is shuffled so that a
1875
- candidate's position never sways the verdict.
1625
+ candidate's position never sways the verdict, and by default each row is
1626
+ checked twice \u2014 once more with the order reversed \u2014 recording no result
1627
+ for that row when the two checks disagree, rather than guessing.
1876
1628
  `,
1877
1629
  category: "quality",
1878
1630
  docsUrl: "",
1879
1631
  isGuardrail: false,
1880
- requiredFields: ["candidates"],
1881
- optionalFields: ["input", "golden", "row_index"],
1632
+ requiredFields: [],
1633
+ optionalFields: ["input", "golden", "candidates", "row_index"],
1882
1634
  settings: {
1883
1635
  model: {
1884
1636
  description: "The model to use for evaluation",
@@ -1918,6 +1670,14 @@ slot label. Use "tie" only when no candidate is clearly better.
1918
1670
  include_metrics: {
1919
1671
  description: "Per-candidate metrics to inject into the judge prompt",
1920
1672
  default: []
1673
+ },
1674
+ temperature: {
1675
+ description: "Sampling temperature for the judge call. Lower is more deterministic.",
1676
+ default: 0
1677
+ },
1678
+ swap_and_reconcile: {
1679
+ description: "Check each row twice, the second time with the candidate order reversed, and record no result for that row when the two disagree \u2014 rather than a tie, which would claim the candidates are equally good. Doubles judge-call cost per row (literature: swapping candidate order alone flips 10-30% of verdicts on contested rows).",
1680
+ default: true
1921
1681
  }
1922
1682
  },
1923
1683
  envVars: [],
@@ -1930,29 +1690,6 @@ slot label. Use "tie" only when no candidate is clearly better.
1930
1690
  }
1931
1691
  }
1932
1692
  },
1933
- "langevals/query_resolution": {
1934
- name: `Query Resolution`,
1935
- description: `
1936
- This evaluator checks if all the user queries in the conversation were resolved. Useful to detect when the bot doesn't know how to answer or can't help the user.
1937
- `,
1938
- category: "quality",
1939
- docsUrl: "",
1940
- isGuardrail: false,
1941
- requiredFields: ["conversation"],
1942
- optionalFields: [],
1943
- settings: {
1944
- model: {
1945
- description: "The model to use for evaluation",
1946
- default: "openai/gpt-5-mini"
1947
- },
1948
- max_tokens: {
1949
- description: "Max tokens allowed for evaluation",
1950
- default: 128e3
1951
- }
1952
- },
1953
- envVars: [],
1954
- result: {}
1955
- },
1956
1693
  "langevals/sentiment": {
1957
1694
  name: `Sentiment Evaluator`,
1958
1695
  description: `
@@ -2172,4 +1909,4 @@ function extractFirstLine(description) {
2172
1909
  export {
2173
1910
  formatEvaluatorSchema
2174
1911
  };
2175
- //# sourceMappingURL=discover-evaluator-schema-MRGHVNTT.js.map
1912
+ //# sourceMappingURL=discover-evaluator-schema-ESSHQGPY.js.map
@@ -1,4 +1,4 @@
1
- import "./chunk-BJ7B4R26.js";
1
+ import "./chunk-SAS5OB7F.js";
2
2
 
3
3
  // src/tools/discover-scenario-schema.ts
4
4
  function formatScenarioSchema() {
@@ -17,6 +17,58 @@ function formatScenarioSchema() {
17
17
  lines.push(
18
18
  '- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")'
19
19
  );
20
+ lines.push(
21
+ "- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite."
22
+ );
23
+ lines.push(
24
+ `- **fields** (object): The scenario's value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.`
25
+ );
26
+ lines.push("\n## Test Suite Fields\n");
27
+ lines.push(
28
+ "A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:"
29
+ );
30
+ lines.push("```json");
31
+ lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
32
+ lines.push("```");
33
+ lines.push(
34
+ "- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved."
35
+ );
36
+ lines.push("- **type**: text, number or boolean.");
37
+ lines.push(
38
+ "- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed."
39
+ );
40
+ lines.push("\n## Evaluator Attachments\n");
41
+ lines.push(
42
+ "A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates."
43
+ );
44
+ lines.push("```json");
45
+ lines.push('{ "evaluators": [{');
46
+ lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
47
+ lines.push(' "required": true,');
48
+ lines.push(' "mappings": {');
49
+ lines.push(' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
50
+ lines.push(' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
51
+ lines.push(" }");
52
+ lines.push("}] }");
53
+ lines.push("```");
54
+ lines.push(
55
+ "Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:"
56
+ );
57
+ lines.push(
58
+ '- **conversation**: `["first_user_message"]`, `["last_agent_message"]`, `["transcript"]` (role: content lines) or `["messages"]` (JSON)'
59
+ );
60
+ lines.push(
61
+ '- **scenario**: `["situation"]`, `["criteria"]` or `["fields", "<identifier>"]` (a field the suite declares; not on a run plan)'
62
+ );
63
+ lines.push(
64
+ '- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)'
65
+ );
66
+ lines.push(
67
+ '- a literal: `{ "type": "value", "value": "..." }`'
68
+ );
69
+ lines.push(
70
+ "A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`."
71
+ );
20
72
  lines.push("\n## Writing a Good Situation\n");
21
73
  lines.push(
22
74
  "The situation drives the user simulator. Include these elements:"
@@ -64,4 +116,4 @@ function formatScenarioSchema() {
64
116
  export {
65
117
  formatScenarioSchema
66
118
  };
67
- //# sourceMappingURL=discover-scenario-schema-UPBTCWPG.js.map
119
+ //# sourceMappingURL=discover-scenario-schema-W7KBXA4J.js.map
@@ -1,4 +1,4 @@
1
- import "./chunk-BJ7B4R26.js";
1
+ import "./chunk-SAS5OB7F.js";
2
2
 
3
3
  // src/schemas/filter-fields.ts
4
4
  var filterFields = [
@@ -445,4 +445,4 @@ function formatGroups() {
445
445
  export {
446
446
  formatSchema
447
447
  };
448
- //# sourceMappingURL=discover-schema-ICGXPHUW.js.map
448
+ //# sourceMappingURL=discover-schema-L6QDPON3.js.map
@@ -0,0 +1,66 @@
1
+ import {
2
+ getAgent
3
+ } from "./chunk-EYXMLE5X.js";
4
+ import "./chunk-GELJ2MQR.js";
5
+ import "./chunk-WT4PO4A3.js";
6
+ import "./chunk-SAS5OB7F.js";
7
+
8
+ // src/tools/get-agent.ts
9
+ var describeParameter = ({
10
+ name,
11
+ type,
12
+ options,
13
+ default: defaultValue,
14
+ description,
15
+ required
16
+ }) => {
17
+ const parts = [type];
18
+ if (options?.length) parts.push(`one of ${options.join(", ")}`);
19
+ if (defaultValue !== void 0) parts.push(`default ${JSON.stringify(defaultValue)}`);
20
+ if (required) parts.push("required");
21
+ const suffix = description ? `: ${description}` : "";
22
+ return `- **${name}** (${parts.join(", ")})${suffix}`;
23
+ };
24
+ async function handleGetAgent({ id }) {
25
+ const agent = await getAgent(id);
26
+ const lines = [];
27
+ lines.push(`# ${agent.name}
28
+ `);
29
+ lines.push(`**ID**: ${agent.id}`);
30
+ lines.push(`**Type**: ${agent.type}`);
31
+ if (agent.environment) lines.push(`**Environment**: ${agent.environment}`);
32
+ if (agent.status) lines.push(`**Status**: ${agent.status}`);
33
+ if (agent.owner?.name) lines.push(`**Owner**: ${agent.owner.name}`);
34
+ else if (agent.hostLabel) lines.push(`**Host**: ${agent.hostLabel}`);
35
+ if (agent.selectable === false) {
36
+ lines.push("**Run**: only its owner can run this agent");
37
+ }
38
+ if (agent.lastSeenAt) lines.push(`**Last seen**: ${agent.lastSeenAt}`);
39
+ lines.push(`**Created**: ${agent.createdAt}`);
40
+ lines.push(`**Updated**: ${agent.updatedAt}`);
41
+ if (agent.parameters && agent.parameters.length > 0) {
42
+ lines.push("\n## Parameters\n");
43
+ for (const parameter of agent.parameters) lines.push(describeParameter(parameter));
44
+ }
45
+ if (agent.instances && agent.instances.length > 0) {
46
+ lines.push(`
47
+ ## Instances (${agent.instances.length})
48
+ `);
49
+ for (const instance of agent.instances) {
50
+ const label = instance.label ? ` (${instance.label})` : "";
51
+ lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
52
+ }
53
+ }
54
+ if (agent.config && Object.keys(agent.config).length > 0) {
55
+ lines.push("\n## Config\n");
56
+ lines.push("```json");
57
+ lines.push(JSON.stringify(agent.config, null, 2));
58
+ lines.push("```");
59
+ }
60
+ return lines.join("\n");
61
+ }
62
+ export {
63
+ describeParameter,
64
+ handleGetAgent
65
+ };
66
+ //# sourceMappingURL=get-agent-7THF7HCU.js.map
@@ -3,9 +3,9 @@ import {
3
3
  } from "./chunk-ZXKLPC2E.js";
4
4
  import {
5
5
  getAnalyticsTimeseries
6
- } from "./chunk-5UWY24RP.js";
7
- import "./chunk-KOO5733S.js";
8
- import "./chunk-BJ7B4R26.js";
6
+ } from "./chunk-GELJ2MQR.js";
7
+ import "./chunk-WT4PO4A3.js";
8
+ import "./chunk-SAS5OB7F.js";
9
9
 
10
10
  // src/tools/get-analytics.ts
11
11
  function getGroupedData(bucket, groupBy) {
@@ -70,4 +70,4 @@ async function handleGetAnalytics(params) {
70
70
  export {
71
71
  handleGetAnalytics
72
72
  };
73
- //# sourceMappingURL=get-analytics-PVERWOA5.js.map
73
+ //# sourceMappingURL=get-analytics-R6AZD3AW.js.map
@@ -1,9 +1,9 @@
1
1
  import {
2
2
  getDashboard
3
- } from "./chunk-UXXA4D6K.js";
4
- import "./chunk-5UWY24RP.js";
5
- import "./chunk-KOO5733S.js";
6
- import "./chunk-BJ7B4R26.js";
3
+ } from "./chunk-S4GVX36W.js";
4
+ import "./chunk-GELJ2MQR.js";
5
+ import "./chunk-WT4PO4A3.js";
6
+ import "./chunk-SAS5OB7F.js";
7
7
 
8
8
  // src/tools/get-dashboard.ts
9
9
  async function handleGetDashboard(params) {
@@ -27,4 +27,4 @@ async function handleGetDashboard(params) {
27
27
  export {
28
28
  handleGetDashboard
29
29
  };
30
- //# sourceMappingURL=get-dashboard-ZZPC3WMD.js.map
30
+ //# sourceMappingURL=get-dashboard-P4WWVHFU.js.map
@@ -3,10 +3,10 @@ import {
3
3
  } from "./chunk-NUFPTHKU.js";
4
4
  import {
5
5
  getDataset
6
- } from "./chunk-J45SVWPR.js";
7
- import "./chunk-5UWY24RP.js";
8
- import "./chunk-KOO5733S.js";
9
- import "./chunk-BJ7B4R26.js";
6
+ } from "./chunk-AN4OUA4D.js";
7
+ import "./chunk-GELJ2MQR.js";
8
+ import "./chunk-WT4PO4A3.js";
9
+ import "./chunk-SAS5OB7F.js";
10
10
 
11
11
  // src/tools/get-dataset.ts
12
12
  function formatDatasetResponse(dataset) {
@@ -46,4 +46,4 @@ export {
46
46
  formatDatasetResponse,
47
47
  handleGetDataset
48
48
  };
49
- //# sourceMappingURL=get-dataset-25TB2LQH.js.map
49
+ //# sourceMappingURL=get-dataset-AKTYSQDI.js.map