@langwatch/mcp-server 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/CHANGELOG.md +89 -0
  2. package/README.md +31 -3
  3. package/dist/{archive-project-4EQGLBJJ.js → archive-project-W3IKI7UL.js} +3 -3
  4. package/dist/archive-run-plan-474OKHHU.js +16 -0
  5. package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-V5UNYK26.js} +3 -3
  6. package/dist/archive-test-suite-O7FLDMXK.js +16 -0
  7. package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-Q46WRB6P.js} +2 -2
  8. package/dist/chunk-56OGTS24.js +44 -0
  9. package/dist/chunk-6DJDEXET.js +63 -0
  10. package/dist/chunk-7E4FZIYJ.js +15 -0
  11. package/dist/{chunk-77BNWQ3U.js → chunk-AN4OUA4D.js} +2 -2
  12. package/dist/chunk-CKNAH5TG.js +74 -0
  13. package/dist/{chunk-TOXJQDIZ.js → chunk-EYXMLE5X.js} +85 -10
  14. package/dist/{chunk-7RHG56OH.js → chunk-G4ZAZTZW.js} +2 -2
  15. package/dist/{chunk-A2LEVYQ5.js → chunk-GELJ2MQR.js} +11 -2
  16. package/dist/chunk-HV4DWKBI.js +46 -0
  17. package/dist/chunk-JR6DJZOU.js +49 -0
  18. package/dist/{chunk-SFB4MU6H.js → chunk-JTQJRNFJ.js} +2 -2
  19. package/dist/{chunk-TJCXFNDT.js → chunk-K37ZNPXK.js} +2 -2
  20. package/dist/{chunk-CDYQOY5U.js → chunk-OJLYYJWI.js} +2 -2
  21. package/dist/{chunk-ZWIL4EFL.js → chunk-QAMP3CGR.js} +2 -2
  22. package/dist/{chunk-4QC7SA2L.js → chunk-QRO35SOM.js} +345 -144
  23. package/dist/{chunk-J42FWRJ2.js → chunk-RVAE2MEA.js} +2 -2
  24. package/dist/{chunk-AQJO2EI5.js → chunk-S4GVX36W.js} +2 -2
  25. package/dist/{chunk-YX3A3CFM.js → chunk-SR6QPRIE.js} +2 -2
  26. package/dist/chunk-UF6XIGYR.js +58 -0
  27. package/dist/{chunk-5CCZFUXS.js → chunk-UFOYFYI7.js} +2 -2
  28. package/dist/{create-agent-7XHO6UIS.js → create-agent-JFJ2K6X3.js} +3 -3
  29. package/dist/{create-annotation-YQXRQC6A.js → create-annotation-EUXXQ6NS.js} +3 -3
  30. package/dist/{create-api-key-YUS36KBU.js → create-api-key-BGBB76JX.js} +3 -3
  31. package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-Z76AIPLS.js} +3 -3
  32. package/dist/{create-dataset-E6NMSVX7.js → create-dataset-FD5VAXU4.js} +3 -3
  33. package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-WQUXXOHI.js} +3 -3
  34. package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-6NV33Z52.js} +3 -3
  35. package/dist/create-mcp-server.js +7 -2
  36. package/dist/{create-project-CS2CB57Y.js → create-project-ZQ7TFIO2.js} +3 -3
  37. package/dist/{create-prompt-F5RMX3L3.js → create-prompt-BS6QRCUU.js} +2 -2
  38. package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-FCQ5ELSQ.js} +2 -2
  39. package/dist/{create-scenario-JDR2VB2H.js → create-scenario-DRPTPXU3.js} +10 -3
  40. package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
  41. package/dist/{delete-agent-O3EEH4S5.js → delete-agent-SA4JMUZF.js} +3 -3
  42. package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-X3KKK7QY.js} +3 -3
  43. package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-LYMRPF27.js} +3 -3
  44. package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-W6EZ44HY.js} +3 -3
  45. package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-7CNOTALB.js} +3 -3
  46. package/dist/{delete-evaluator-33SP2JFX.js → delete-evaluator-RKYC4TNB.js} +3 -3
  47. package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-QB3OCZUV.js} +2 -2
  48. package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-NG2VXYYM.js} +3 -3
  49. package/dist/{discover-evaluator-schema-XTAD4X7D.js → discover-evaluator-schema-ESSHQGPY.js} +7 -286
  50. package/dist/{discover-scenario-schema-FE64QK2D.js → discover-scenario-schema-W7KBXA4J.js} +53 -1
  51. package/dist/get-agent-7THF7HCU.js +66 -0
  52. package/dist/{get-analytics-TETCSOK6.js → get-analytics-R6AZD3AW.js} +2 -2
  53. package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-P4WWVHFU.js} +3 -3
  54. package/dist/{get-dataset-RCMU5VAH.js → get-dataset-AKTYSQDI.js} +3 -3
  55. package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-RZ44UIN6.js} +3 -3
  56. package/dist/{get-project-RAK7URQJ.js → get-project-QSQEAB33.js} +3 -3
  57. package/dist/get-prompt-GVKXF5NL.js +100 -0
  58. package/dist/get-run-plan-NY7J5GRE.js +61 -0
  59. package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-Q3LLQE4T.js} +7 -3
  60. package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-WINEYDMK.js} +12 -3
  61. package/dist/get-test-suite-4HKA7OKS.js +49 -0
  62. package/dist/{get-trace-K7U234Q4.js → get-trace-RCUV7N4O.js} +2 -2
  63. package/dist/{get-workflow-L5L2PSK3.js → get-workflow-EEOC7Y45.js} +3 -3
  64. package/dist/{http-server-YXHFPPGA.js → http-server-WCDUARTH.js} +8 -3
  65. package/dist/index.js +3366 -1910
  66. package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-6DHVZWUK.js} +3 -3
  67. package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-2KJL2M42.js} +3 -3
  68. package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-6ISFJ7WX.js} +2 -2
  69. package/dist/{langwatch-api-secrets-BUBSNCNO.js → langwatch-api-secrets-OEBD6GBU.js} +2 -2
  70. package/dist/{langwatch-api-triggers-EDFUCJN6.js → langwatch-api-triggers-5E6DDGX6.js} +2 -2
  71. package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-6UJL5U4Z.js} +3 -3
  72. package/dist/list-agents-AMVDQPQL.js +43 -0
  73. package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-6PR5NLIC.js} +3 -3
  74. package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-GVPNBIHQ.js} +3 -3
  75. package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-Z3BZKDGF.js} +3 -3
  76. package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-O67HZ35Y.js} +3 -3
  77. package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-WXZL6UMP.js} +3 -3
  78. package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-PAX72WCN.js} +3 -3
  79. package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-VVLZMLCX.js} +3 -3
  80. package/dist/{list-projects-K7QAIX2I.js → list-projects-AQVI6SUK.js} +3 -3
  81. package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-WZFBJOHG.js} +2 -2
  82. package/dist/{list-prompts-SC2HAUFG.js → list-prompts-BQOHRHYT.js} +2 -2
  83. package/dist/list-run-plans-UBAU2DGH.js +46 -0
  84. package/dist/{list-scenarios-F2WYNRIE.js → list-scenarios-MS4IUIVM.js} +8 -5
  85. package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-D6YF4ZOU.js} +11 -5
  86. package/dist/list-test-suites-L7R2O6DR.js +35 -0
  87. package/dist/{list-workflows-QSJWURCK.js → list-workflows-6MV4XJHF.js} +3 -3
  88. package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-K5LQ4CY6.js} +2 -2
  89. package/dist/rename-test-suite-NTYHAO5X.js +21 -0
  90. package/dist/report-issue-RHZZ33IS.js +885 -0
  91. package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
  92. package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-6LAZ3QZZ.js} +3 -3
  93. package/dist/run-agent-CAGWGTYT.js +72 -0
  94. package/dist/run-plan-T4ZYOZGA.js +39 -0
  95. package/dist/run-test-suite-L4MZ7G4D.js +26 -0
  96. package/dist/{run-workflow-QPXC627O.js → run-workflow-DFIS23AP.js} +3 -3
  97. package/dist/{search-traces-A6HFKPLJ.js → search-traces-4XTJYXIK.js} +2 -2
  98. package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-XRNMFOVR.js} +3 -3
  99. package/dist/{update-agent-SMAEHLXH.js → update-agent-DUWNYBNA.js} +3 -3
  100. package/dist/{update-dataset-BDUK7TUO.js → update-dataset-YSOMDCOB.js} +3 -3
  101. package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-DEA2ITYR.js} +3 -3
  102. package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-NH7Q5GWV.js} +3 -3
  103. package/dist/{update-project-LKVYXENH.js → update-project-Q26FBSYH.js} +3 -3
  104. package/dist/update-prompt-ZQQ2C3KU.js +143 -0
  105. package/dist/{update-scenario-NEXCAQI7.js → update-scenario-WUX34JAR.js} +10 -3
  106. package/package.json +8 -9
  107. package/pyproject.toml +4 -4
  108. package/src/__tests__/agent-tools.unit.test.ts +257 -0
  109. package/src/__tests__/all-tools.integration.test.ts +321 -75
  110. package/src/__tests__/integration.integration.test.ts +7 -11
  111. package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
  112. package/src/__tests__/prompts.integration.test.ts +736 -0
  113. package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
  114. package/src/__tests__/scenario-tools.unit.test.ts +19 -0
  115. package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
  116. package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
  117. package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
  118. package/src/create-mcp-server.ts +334 -55
  119. package/src/langwatch-api-agents.ts +196 -12
  120. package/src/langwatch-api-run-plans.ts +157 -0
  121. package/src/langwatch-api-scenarios.ts +9 -0
  122. package/src/langwatch-api-simulation-runs.ts +5 -0
  123. package/src/langwatch-api-test-suites.ts +135 -0
  124. package/src/langwatch-api.ts +62 -12
  125. package/src/schemas/index.ts +5 -0
  126. package/src/schemas/run-plan.ts +92 -0
  127. package/src/schemas/suite-fields.ts +149 -0
  128. package/src/tools/archive-run-plan.ts +12 -0
  129. package/src/tools/archive-test-suite.ts +12 -0
  130. package/src/tools/create-scenario.ts +8 -0
  131. package/src/tools/create-test-suite.ts +43 -0
  132. package/src/tools/discover-scenario-schema.ts +54 -0
  133. package/src/tools/format-run-plan.ts +68 -0
  134. package/src/tools/format-scenario.ts +14 -0
  135. package/src/tools/format-suite-details.ts +96 -0
  136. package/src/tools/get-agent.ts +45 -3
  137. package/src/tools/get-prompt.ts +108 -24
  138. package/src/tools/get-run-plan.ts +62 -0
  139. package/src/tools/get-scenario.ts +3 -0
  140. package/src/tools/get-simulation-run.ts +9 -0
  141. package/src/tools/get-test-suite.ts +49 -0
  142. package/src/tools/list-agents.ts +20 -3
  143. package/src/tools/list-run-plans.ts +47 -0
  144. package/src/tools/list-scenarios.ts +12 -2
  145. package/src/tools/list-simulation-runs.ts +6 -2
  146. package/src/tools/list-test-suites.ts +35 -0
  147. package/src/tools/rename-test-suite.ts +18 -0
  148. package/src/tools/rerun-run-plan.ts +21 -0
  149. package/src/tools/run-agent.ts +54 -6
  150. package/src/tools/run-plan.ts +49 -0
  151. package/src/tools/run-test-suite.ts +30 -0
  152. package/src/tools/test-agent.ts +18 -0
  153. package/src/tools/update-prompt.ts +195 -10
  154. package/src/tools/update-scenario.ts +8 -0
  155. package/src/tools/update-test-suite.ts +45 -0
  156. package/tsconfig.json +5 -0
  157. package/uv.lock +416 -296
  158. package/dist/archive-suite-4T6BSRS7.js +0 -16
  159. package/dist/chunk-YZLYHLLI.js +0 -48
  160. package/dist/create-suite-LZS6QIAF.js +0 -39
  161. package/dist/get-agent-Y7MWZ65F.js +0 -29
  162. package/dist/get-prompt-OY5B7TRI.js +0 -46
  163. package/dist/get-suite-J7JVWO6O.js +0 -45
  164. package/dist/list-agents-KFHKGPXV.js +0 -33
  165. package/dist/list-suites-CRY4VWW6.js +0 -43
  166. package/dist/report-issue-XYVAPAYJ.js +0 -412
  167. package/dist/run-agent-6SM4XNA7.js +0 -37
  168. package/dist/run-suite-NKZCJ2IN.js +0 -30
  169. package/dist/update-prompt-ODCRQRN6.js +0 -25
  170. package/dist/update-suite-SNO3GS7T.js +0 -40
  171. package/src/langwatch-api-suites.ts +0 -118
  172. package/src/tools/archive-suite.ts +0 -12
  173. package/src/tools/create-suite.ts +0 -42
  174. package/src/tools/get-suite.ts +0 -47
  175. package/src/tools/list-suites.ts +0 -43
  176. package/src/tools/run-suite.ts +0 -28
  177. package/src/tools/update-suite.ts +0 -44
@@ -18,7 +18,8 @@ var evaluationResultSchema = z.object({
18
18
  });
19
19
  var evaluationResultSkippedSchema = z.object({
20
20
  status: z.literal("skipped"),
21
- details: z.string().optional()
21
+ details: z.string().optional(),
22
+ cost: moneySchema.optional()
22
23
  });
23
24
  var evaluationResultErrorSchema = z.object({
24
25
  status: z.literal("error"),
@@ -379,69 +380,6 @@ var evaluatorsSchema = z.object({
379
380
  ).default(0.25)
380
381
  })
381
382
  }),
382
- "legacy/ragas_answer_correctness": z.object({
383
- settings: z.object({
384
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
385
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
386
- max_tokens: z.number().describe(
387
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
388
- ).default(2048)
389
- })
390
- }),
391
- "legacy/ragas_answer_relevancy": z.object({
392
- settings: z.object({
393
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
394
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
395
- max_tokens: z.number().describe(
396
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
397
- ).default(2048)
398
- })
399
- }),
400
- "legacy/ragas_context_precision": z.object({
401
- settings: z.object({
402
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
403
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
404
- max_tokens: z.number().describe(
405
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
406
- ).default(2048)
407
- })
408
- }),
409
- "legacy/ragas_context_recall": z.object({
410
- settings: z.object({
411
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
412
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
413
- max_tokens: z.number().describe(
414
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
415
- ).default(2048)
416
- })
417
- }),
418
- "legacy/ragas_context_relevancy": z.object({
419
- settings: z.object({
420
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
421
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
422
- max_tokens: z.number().describe(
423
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
424
- ).default(2048)
425
- })
426
- }),
427
- "legacy/ragas_context_utilization": z.object({
428
- settings: z.object({
429
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
430
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
431
- max_tokens: z.number().describe(
432
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
433
- ).default(2048)
434
- })
435
- }),
436
- "legacy/ragas_faithfulness": z.object({
437
- settings: z.object({
438
- model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
439
- embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
440
- max_tokens: z.number().describe(
441
- "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
442
- ).default(2048)
443
- })
444
- }),
445
383
  "langevals/basic": z.object({
446
384
  settings: z.object({
447
385
  rules: z.array(
@@ -1243,223 +1181,6 @@ or if it's in a specific expected language.
1243
1181
  }
1244
1182
  }
1245
1183
  },
1246
- "legacy/ragas_answer_correctness": {
1247
- name: `Ragas Answer Correctness`,
1248
- description: `
1249
- Computes with an LLM a weighted combination of factual as well as semantic similarity between the generated answer and the expected output.
1250
- `,
1251
- category: "rag",
1252
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/answer_correctness.html",
1253
- isGuardrail: false,
1254
- requiredFields: ["output", "expected_output"],
1255
- optionalFields: ["input"],
1256
- settings: {
1257
- model: {
1258
- description: "The model to use for evaluation.",
1259
- default: "openai/gpt-5-mini"
1260
- },
1261
- embeddings_model: {
1262
- description: "The model to use for embeddings.",
1263
- default: "openai/text-embedding-ada-002"
1264
- },
1265
- max_tokens: {
1266
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1267
- default: 2048
1268
- }
1269
- },
1270
- envVars: [],
1271
- result: {
1272
- score: {
1273
- description: "A score between 0.0 and 1.0 indicating the correctness of the answer."
1274
- }
1275
- }
1276
- },
1277
- "legacy/ragas_answer_relevancy": {
1278
- name: `Ragas Answer Relevancy`,
1279
- description: `
1280
- Evaluates how pertinent the generated answer is to the given prompt. Higher scores indicate better relevancy.
1281
- `,
1282
- category: "rag",
1283
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/answer_relevance.html",
1284
- isGuardrail: false,
1285
- requiredFields: ["input", "output"],
1286
- optionalFields: [],
1287
- settings: {
1288
- model: {
1289
- description: "The model to use for evaluation.",
1290
- default: "openai/gpt-5-mini"
1291
- },
1292
- embeddings_model: {
1293
- description: "The model to use for embeddings.",
1294
- default: "openai/text-embedding-ada-002"
1295
- },
1296
- max_tokens: {
1297
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1298
- default: 2048
1299
- }
1300
- },
1301
- envVars: [],
1302
- result: {
1303
- score: {
1304
- description: "A score between 0.0 and 1.0 indicating the relevance of the answer."
1305
- }
1306
- }
1307
- },
1308
- "legacy/ragas_context_precision": {
1309
- name: `Ragas Context Precision`,
1310
- description: `
1311
- This metric evaluates whether all of the ground-truth relevant items present in the contexts are ranked higher or not. Higher scores indicate better precision.
1312
- `,
1313
- category: "rag",
1314
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_precision.html",
1315
- isGuardrail: false,
1316
- requiredFields: ["input", "contexts", "expected_output"],
1317
- optionalFields: [],
1318
- settings: {
1319
- model: {
1320
- description: "The model to use for evaluation.",
1321
- default: "openai/gpt-5-mini"
1322
- },
1323
- embeddings_model: {
1324
- description: "The model to use for embeddings.",
1325
- default: "openai/text-embedding-ada-002"
1326
- },
1327
- max_tokens: {
1328
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1329
- default: 2048
1330
- }
1331
- },
1332
- envVars: [],
1333
- result: {
1334
- score: {
1335
- description: "A score between 0.0 and 1.0 indicating the precision of the context."
1336
- }
1337
- }
1338
- },
1339
- "legacy/ragas_context_recall": {
1340
- name: `Ragas Context Recall`,
1341
- description: `
1342
- This evaluator measures the extent to which the retrieved context aligns with the annotated answer, treated as the ground truth. Higher values indicate better performance.
1343
- `,
1344
- category: "rag",
1345
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_recall.html",
1346
- isGuardrail: false,
1347
- requiredFields: ["input", "contexts", "expected_output"],
1348
- optionalFields: [],
1349
- settings: {
1350
- model: {
1351
- description: "The model to use for evaluation.",
1352
- default: "openai/gpt-5-mini"
1353
- },
1354
- embeddings_model: {
1355
- description: "The model to use for embeddings.",
1356
- default: "openai/text-embedding-ada-002"
1357
- },
1358
- max_tokens: {
1359
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1360
- default: 2048
1361
- }
1362
- },
1363
- envVars: [],
1364
- result: {
1365
- score: {
1366
- description: "A score between 0.0 and 1.0 indicating the recall of the context."
1367
- }
1368
- }
1369
- },
1370
- "legacy/ragas_context_relevancy": {
1371
- name: `Ragas Context Relevancy`,
1372
- description: `
1373
- This metric gauges the relevancy of the retrieved context, calculated based on both the question and contexts. The values fall within the range of (0, 1), with higher values indicating better relevancy.
1374
- `,
1375
- category: "rag",
1376
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_relevancy.html",
1377
- isGuardrail: false,
1378
- requiredFields: ["output", "contexts"],
1379
- optionalFields: [],
1380
- settings: {
1381
- model: {
1382
- description: "The model to use for evaluation.",
1383
- default: "openai/gpt-5-mini"
1384
- },
1385
- embeddings_model: {
1386
- description: "The model to use for embeddings.",
1387
- default: "openai/text-embedding-ada-002"
1388
- },
1389
- max_tokens: {
1390
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1391
- default: 2048
1392
- }
1393
- },
1394
- envVars: [],
1395
- result: {
1396
- score: {
1397
- description: "A score between 0.0 and 1.0 indicating the relevancy of the context."
1398
- }
1399
- }
1400
- },
1401
- "legacy/ragas_context_utilization": {
1402
- name: `Ragas Context Utilization`,
1403
- description: `
1404
- This metric evaluates whether all of the output relevant items present in the contexts are ranked higher or not. Higher scores indicate better utilization.
1405
- `,
1406
- category: "rag",
1407
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_precision.html",
1408
- isGuardrail: false,
1409
- requiredFields: ["input", "output", "contexts"],
1410
- optionalFields: [],
1411
- settings: {
1412
- model: {
1413
- description: "The model to use for evaluation.",
1414
- default: "openai/gpt-5-mini"
1415
- },
1416
- embeddings_model: {
1417
- description: "The model to use for embeddings.",
1418
- default: "openai/text-embedding-ada-002"
1419
- },
1420
- max_tokens: {
1421
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1422
- default: 2048
1423
- }
1424
- },
1425
- envVars: [],
1426
- result: {
1427
- score: {
1428
- description: "A score between 0.0 and 1.0 indicating the utilization of the context."
1429
- }
1430
- }
1431
- },
1432
- "legacy/ragas_faithfulness": {
1433
- name: `Ragas Faithfulness`,
1434
- description: `
1435
- This evaluator assesses the extent to which the generated answer is consistent with the provided context. Higher scores indicate better faithfulness to the context, useful for detecting hallucinations.
1436
- `,
1437
- category: "rag",
1438
- docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/faithfulness.html",
1439
- isGuardrail: false,
1440
- requiredFields: ["output", "contexts"],
1441
- optionalFields: [],
1442
- settings: {
1443
- model: {
1444
- description: "The model to use for evaluation.",
1445
- default: "openai/gpt-5-mini"
1446
- },
1447
- embeddings_model: {
1448
- description: "The model to use for embeddings.",
1449
- default: "openai/text-embedding-ada-002"
1450
- },
1451
- max_tokens: {
1452
- description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
1453
- default: 2048
1454
- }
1455
- },
1456
- envVars: [],
1457
- result: {
1458
- score: {
1459
- description: "A score between 0.0 and 1.0 indicating the faithfulness of the answer."
1460
- }
1461
- }
1462
- },
1463
1184
  "langevals/basic": {
1464
1185
  name: `Custom Basic Evaluator`,
1465
1186
  description: `
@@ -1603,8 +1324,8 @@ A simple evaluator that checks if the output matches the expected_output exactly
1603
1324
  category: "quality",
1604
1325
  docsUrl: "",
1605
1326
  isGuardrail: false,
1606
- requiredFields: [],
1607
- optionalFields: ["output", "expected_output"],
1327
+ requiredFields: ["output", "expected_output"],
1328
+ optionalFields: [],
1608
1329
  settings: {
1609
1330
  case_sensitive: {
1610
1331
  description: "True if the comparison should be case-sensitive, False otherwise",
@@ -1634,8 +1355,8 @@ Uses an LLM to check if the generated output answers a question correctly the sa
1634
1355
  category: "quality",
1635
1356
  docsUrl: "",
1636
1357
  isGuardrail: false,
1637
- requiredFields: [],
1638
- optionalFields: ["input", "output", "expected_output"],
1358
+ requiredFields: ["output", "expected_output"],
1359
+ optionalFields: ["input"],
1639
1360
  settings: {
1640
1361
  model: {
1641
1362
  description: "The model to use for evaluation",
@@ -2188,4 +1909,4 @@ function extractFirstLine(description) {
2188
1909
  export {
2189
1910
  formatEvaluatorSchema
2190
1911
  };
2191
- //# sourceMappingURL=discover-evaluator-schema-XTAD4X7D.js.map
1912
+ //# sourceMappingURL=discover-evaluator-schema-ESSHQGPY.js.map
@@ -17,6 +17,58 @@ function formatScenarioSchema() {
17
17
  lines.push(
18
18
  '- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")'
19
19
  );
20
+ lines.push(
21
+ "- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite."
22
+ );
23
+ lines.push(
24
+ `- **fields** (object): The scenario's value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.`
25
+ );
26
+ lines.push("\n## Test Suite Fields\n");
27
+ lines.push(
28
+ "A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:"
29
+ );
30
+ lines.push("```json");
31
+ lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
32
+ lines.push("```");
33
+ lines.push(
34
+ "- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved."
35
+ );
36
+ lines.push("- **type**: text, number or boolean.");
37
+ lines.push(
38
+ "- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed."
39
+ );
40
+ lines.push("\n## Evaluator Attachments\n");
41
+ lines.push(
42
+ "A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates."
43
+ );
44
+ lines.push("```json");
45
+ lines.push('{ "evaluators": [{');
46
+ lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
47
+ lines.push(' "required": true,');
48
+ lines.push(' "mappings": {');
49
+ lines.push(' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
50
+ lines.push(' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
51
+ lines.push(" }");
52
+ lines.push("}] }");
53
+ lines.push("```");
54
+ lines.push(
55
+ "Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:"
56
+ );
57
+ lines.push(
58
+ '- **conversation**: `["first_user_message"]`, `["last_agent_message"]`, `["transcript"]` (role: content lines) or `["messages"]` (JSON)'
59
+ );
60
+ lines.push(
61
+ '- **scenario**: `["situation"]`, `["criteria"]` or `["fields", "<identifier>"]` (a field the suite declares; not on a run plan)'
62
+ );
63
+ lines.push(
64
+ '- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)'
65
+ );
66
+ lines.push(
67
+ '- a literal: `{ "type": "value", "value": "..." }`'
68
+ );
69
+ lines.push(
70
+ "A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`."
71
+ );
20
72
  lines.push("\n## Writing a Good Situation\n");
21
73
  lines.push(
22
74
  "The situation drives the user simulator. Include these elements:"
@@ -64,4 +116,4 @@ function formatScenarioSchema() {
64
116
  export {
65
117
  formatScenarioSchema
66
118
  };
67
- //# sourceMappingURL=discover-scenario-schema-FE64QK2D.js.map
119
+ //# sourceMappingURL=discover-scenario-schema-W7KBXA4J.js.map
@@ -0,0 +1,66 @@
1
+ import {
2
+ getAgent
3
+ } from "./chunk-EYXMLE5X.js";
4
+ import "./chunk-GELJ2MQR.js";
5
+ import "./chunk-WT4PO4A3.js";
6
+ import "./chunk-SAS5OB7F.js";
7
+
8
+ // src/tools/get-agent.ts
9
+ var describeParameter = ({
10
+ name,
11
+ type,
12
+ options,
13
+ default: defaultValue,
14
+ description,
15
+ required
16
+ }) => {
17
+ const parts = [type];
18
+ if (options?.length) parts.push(`one of ${options.join(", ")}`);
19
+ if (defaultValue !== void 0) parts.push(`default ${JSON.stringify(defaultValue)}`);
20
+ if (required) parts.push("required");
21
+ const suffix = description ? `: ${description}` : "";
22
+ return `- **${name}** (${parts.join(", ")})${suffix}`;
23
+ };
24
+ async function handleGetAgent({ id }) {
25
+ const agent = await getAgent(id);
26
+ const lines = [];
27
+ lines.push(`# ${agent.name}
28
+ `);
29
+ lines.push(`**ID**: ${agent.id}`);
30
+ lines.push(`**Type**: ${agent.type}`);
31
+ if (agent.environment) lines.push(`**Environment**: ${agent.environment}`);
32
+ if (agent.status) lines.push(`**Status**: ${agent.status}`);
33
+ if (agent.owner?.name) lines.push(`**Owner**: ${agent.owner.name}`);
34
+ else if (agent.hostLabel) lines.push(`**Host**: ${agent.hostLabel}`);
35
+ if (agent.selectable === false) {
36
+ lines.push("**Run**: only its owner can run this agent");
37
+ }
38
+ if (agent.lastSeenAt) lines.push(`**Last seen**: ${agent.lastSeenAt}`);
39
+ lines.push(`**Created**: ${agent.createdAt}`);
40
+ lines.push(`**Updated**: ${agent.updatedAt}`);
41
+ if (agent.parameters && agent.parameters.length > 0) {
42
+ lines.push("\n## Parameters\n");
43
+ for (const parameter of agent.parameters) lines.push(describeParameter(parameter));
44
+ }
45
+ if (agent.instances && agent.instances.length > 0) {
46
+ lines.push(`
47
+ ## Instances (${agent.instances.length})
48
+ `);
49
+ for (const instance of agent.instances) {
50
+ const label = instance.label ? ` (${instance.label})` : "";
51
+ lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
52
+ }
53
+ }
54
+ if (agent.config && Object.keys(agent.config).length > 0) {
55
+ lines.push("\n## Config\n");
56
+ lines.push("```json");
57
+ lines.push(JSON.stringify(agent.config, null, 2));
58
+ lines.push("```");
59
+ }
60
+ return lines.join("\n");
61
+ }
62
+ export {
63
+ describeParameter,
64
+ handleGetAgent
65
+ };
66
+ //# sourceMappingURL=get-agent-7THF7HCU.js.map
@@ -3,7 +3,7 @@ import {
3
3
  } from "./chunk-ZXKLPC2E.js";
4
4
  import {
5
5
  getAnalyticsTimeseries
6
- } from "./chunk-A2LEVYQ5.js";
6
+ } from "./chunk-GELJ2MQR.js";
7
7
  import "./chunk-WT4PO4A3.js";
8
8
  import "./chunk-SAS5OB7F.js";
9
9
 
@@ -70,4 +70,4 @@ async function handleGetAnalytics(params) {
70
70
  export {
71
71
  handleGetAnalytics
72
72
  };
73
- //# sourceMappingURL=get-analytics-TETCSOK6.js.map
73
+ //# sourceMappingURL=get-analytics-R6AZD3AW.js.map
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  getDashboard
3
- } from "./chunk-AQJO2EI5.js";
4
- import "./chunk-A2LEVYQ5.js";
3
+ } from "./chunk-S4GVX36W.js";
4
+ import "./chunk-GELJ2MQR.js";
5
5
  import "./chunk-WT4PO4A3.js";
6
6
  import "./chunk-SAS5OB7F.js";
7
7
 
@@ -27,4 +27,4 @@ async function handleGetDashboard(params) {
27
27
  export {
28
28
  handleGetDashboard
29
29
  };
30
- //# sourceMappingURL=get-dashboard-KHBEGZGL.js.map
30
+ //# sourceMappingURL=get-dashboard-P4WWVHFU.js.map
@@ -3,8 +3,8 @@ import {
3
3
  } from "./chunk-NUFPTHKU.js";
4
4
  import {
5
5
  getDataset
6
- } from "./chunk-77BNWQ3U.js";
7
- import "./chunk-A2LEVYQ5.js";
6
+ } from "./chunk-AN4OUA4D.js";
7
+ import "./chunk-GELJ2MQR.js";
8
8
  import "./chunk-WT4PO4A3.js";
9
9
  import "./chunk-SAS5OB7F.js";
10
10
 
@@ -46,4 +46,4 @@ export {
46
46
  formatDatasetResponse,
47
47
  handleGetDataset
48
48
  };
49
- //# sourceMappingURL=get-dataset-RCMU5VAH.js.map
49
+ //# sourceMappingURL=get-dataset-AKTYSQDI.js.map
@@ -1,8 +1,8 @@
1
1
  import {
2
2
  getEvaluator,
3
3
  getEvaluatorType
4
- } from "./chunk-CDYQOY5U.js";
5
- import "./chunk-A2LEVYQ5.js";
4
+ } from "./chunk-OJLYYJWI.js";
5
+ import "./chunk-GELJ2MQR.js";
6
6
  import "./chunk-WT4PO4A3.js";
7
7
  import "./chunk-SAS5OB7F.js";
8
8
 
@@ -45,4 +45,4 @@ async function handleGetEvaluator(params) {
45
45
  export {
46
46
  handleGetEvaluator
47
47
  };
48
- //# sourceMappingURL=get-evaluator-ADMKVWYZ.js.map
48
+ //# sourceMappingURL=get-evaluator-RZ44UIN6.js.map
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  getProject
3
- } from "./chunk-7RHG56OH.js";
4
- import "./chunk-A2LEVYQ5.js";
3
+ } from "./chunk-G4ZAZTZW.js";
4
+ import "./chunk-GELJ2MQR.js";
5
5
  import "./chunk-WT4PO4A3.js";
6
6
  import "./chunk-SAS5OB7F.js";
7
7
 
@@ -24,4 +24,4 @@ async function handleGetProject(params) {
24
24
  export {
25
25
  handleGetProject
26
26
  };
27
- //# sourceMappingURL=get-project-RAK7URQJ.js.map
27
+ //# sourceMappingURL=get-project-QSQEAB33.js.map
@@ -0,0 +1,100 @@
1
+ import {
2
+ getPrompt
3
+ } from "./chunk-GELJ2MQR.js";
4
+ import "./chunk-WT4PO4A3.js";
5
+ import "./chunk-SAS5OB7F.js";
6
+
7
+ // src/tools/get-prompt.ts
8
+ function normalizeTag(tag) {
9
+ if (typeof tag === "string") return { name: tag };
10
+ if (tag && typeof tag === "object" && "name" in tag) {
11
+ const candidate = tag;
12
+ if (typeof candidate.name === "string") {
13
+ return {
14
+ name: candidate.name,
15
+ versionId: typeof candidate.versionId === "string" ? candidate.versionId : void 0
16
+ };
17
+ }
18
+ }
19
+ return void 0;
20
+ }
21
+ async function handleGetPrompt(params) {
22
+ const prompt = await getPrompt(params.idOrHandle, {
23
+ version: params.version,
24
+ tag: params.tag
25
+ });
26
+ if (params.format === "json") {
27
+ return JSON.stringify(prompt, null, 2);
28
+ }
29
+ const lines = [];
30
+ lines.push(`# Prompt: ${prompt.name || prompt.handle || prompt.id}
31
+ `);
32
+ if (prompt.handle) lines.push(`**Handle**: ${prompt.handle}`);
33
+ if (prompt.id) lines.push(`**ID**: ${prompt.id}`);
34
+ if (prompt.version != null) lines.push(`**Version**: v${prompt.version}`);
35
+ if (prompt.versionId) lines.push(`**Version ID**: ${prompt.versionId}`);
36
+ if (prompt.commitMessage) lines.push(`**Commit**: ${prompt.commitMessage}`);
37
+ if (prompt.model) lines.push(`**Model**: ${prompt.model}`);
38
+ if (prompt.temperature != null)
39
+ lines.push(`**Temperature**: ${prompt.temperature}`);
40
+ if (prompt.maxTokens != null)
41
+ lines.push(`**Max Tokens**: ${prompt.maxTokens}`);
42
+ if (prompt.responseFormat)
43
+ lines.push(`**Response Format**: ${JSON.stringify(prompt.responseFormat)}`);
44
+ const renderFieldList = ({
45
+ heading,
46
+ fields
47
+ }) => {
48
+ const entries = [];
49
+ if (Array.isArray(fields)) {
50
+ for (const field of fields) {
51
+ if (field && typeof field === "object" && "identifier" in field) {
52
+ const typed = field;
53
+ entries.push(
54
+ `- **${String(typed.identifier)}**: ${String(typed.type ?? "unknown")}`
55
+ );
56
+ } else {
57
+ entries.push(`- ${JSON.stringify(field)}`);
58
+ }
59
+ }
60
+ } else if (fields && typeof fields === "object") {
61
+ for (const [key, value] of Object.entries(fields)) {
62
+ entries.push(`- **${key}**: ${JSON.stringify(value)}`);
63
+ }
64
+ }
65
+ if (entries.length === 0) return;
66
+ lines.push(`
67
+ **${heading}**`);
68
+ lines.push(...entries);
69
+ };
70
+ renderFieldList({ heading: "Parameters", fields: prompt.parameters });
71
+ renderFieldList({ heading: "Inputs", fields: prompt.inputs });
72
+ renderFieldList({ heading: "Outputs", fields: prompt.outputs });
73
+ const messages = prompt.messages;
74
+ if (Array.isArray(messages) && messages.length > 0) {
75
+ lines.push("\n## Messages");
76
+ for (const msg of messages) {
77
+ lines.push(`
78
+ ### ${msg.role}`);
79
+ lines.push(msg.content);
80
+ }
81
+ } else if (typeof prompt.prompt === "string" && prompt.prompt.length > 0) {
82
+ lines.push("\n## Messages");
83
+ lines.push("\n### system");
84
+ lines.push(prompt.prompt);
85
+ }
86
+ lines.push("\n## Deployments");
87
+ const tags = Array.isArray(prompt.tags) ? prompt.tags : [];
88
+ for (const rawTag of tags) {
89
+ const tag = normalizeTag(rawTag);
90
+ if (!tag || tag.name === "latest") continue;
91
+ const target = tag.versionId ?? "unknown version";
92
+ const marker = tag.versionId && tag.versionId === prompt.versionId ? " (this version)" : "";
93
+ lines.push(`- ${tag.name} \u2192 ${target}${marker}`);
94
+ }
95
+ return lines.join("\n");
96
+ }
97
+ export {
98
+ handleGetPrompt
99
+ };
100
+ //# sourceMappingURL=get-prompt-GVKXF5NL.js.map