@langwatch/mcp-server 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/CHANGELOG.md +89 -0
  2. package/README.md +31 -3
  3. package/dist/{archive-project-4EQGLBJJ.js → archive-project-W3IKI7UL.js} +3 -3
  4. package/dist/archive-run-plan-474OKHHU.js +16 -0
  5. package/dist/{archive-scenario-W6BXYDZ3.js → archive-scenario-V5UNYK26.js} +3 -3
  6. package/dist/archive-test-suite-O7FLDMXK.js +16 -0
  7. package/dist/{assign-prompt-tag-VLXUWEKU.js → assign-prompt-tag-Q46WRB6P.js} +2 -2
  8. package/dist/chunk-56OGTS24.js +44 -0
  9. package/dist/chunk-6DJDEXET.js +63 -0
  10. package/dist/chunk-7E4FZIYJ.js +15 -0
  11. package/dist/{chunk-77BNWQ3U.js → chunk-AN4OUA4D.js} +2 -2
  12. package/dist/chunk-CKNAH5TG.js +74 -0
  13. package/dist/{chunk-TOXJQDIZ.js → chunk-EYXMLE5X.js} +85 -10
  14. package/dist/{chunk-7RHG56OH.js → chunk-G4ZAZTZW.js} +2 -2
  15. package/dist/{chunk-A2LEVYQ5.js → chunk-GELJ2MQR.js} +11 -2
  16. package/dist/chunk-HV4DWKBI.js +46 -0
  17. package/dist/chunk-JR6DJZOU.js +49 -0
  18. package/dist/{chunk-SFB4MU6H.js → chunk-JTQJRNFJ.js} +2 -2
  19. package/dist/{chunk-TJCXFNDT.js → chunk-K37ZNPXK.js} +2 -2
  20. package/dist/{chunk-CDYQOY5U.js → chunk-OJLYYJWI.js} +2 -2
  21. package/dist/{chunk-ZWIL4EFL.js → chunk-QAMP3CGR.js} +2 -2
  22. package/dist/{chunk-4QC7SA2L.js → chunk-QRO35SOM.js} +345 -144
  23. package/dist/{chunk-J42FWRJ2.js → chunk-RVAE2MEA.js} +2 -2
  24. package/dist/{chunk-AQJO2EI5.js → chunk-S4GVX36W.js} +2 -2
  25. package/dist/{chunk-YX3A3CFM.js → chunk-SR6QPRIE.js} +2 -2
  26. package/dist/chunk-UF6XIGYR.js +58 -0
  27. package/dist/{chunk-5CCZFUXS.js → chunk-UFOYFYI7.js} +2 -2
  28. package/dist/{create-agent-7XHO6UIS.js → create-agent-JFJ2K6X3.js} +3 -3
  29. package/dist/{create-annotation-YQXRQC6A.js → create-annotation-EUXXQ6NS.js} +3 -3
  30. package/dist/{create-api-key-YUS36KBU.js → create-api-key-BGBB76JX.js} +3 -3
  31. package/dist/{create-dashboard-4Z5JO5LB.js → create-dashboard-Z76AIPLS.js} +3 -3
  32. package/dist/{create-dataset-E6NMSVX7.js → create-dataset-FD5VAXU4.js} +3 -3
  33. package/dist/{create-dataset-records-SKHPRD2K.js → create-dataset-records-WQUXXOHI.js} +3 -3
  34. package/dist/{create-evaluator-JXF5PF33.js → create-evaluator-6NV33Z52.js} +3 -3
  35. package/dist/create-mcp-server.js +7 -2
  36. package/dist/{create-project-CS2CB57Y.js → create-project-ZQ7TFIO2.js} +3 -3
  37. package/dist/{create-prompt-F5RMX3L3.js → create-prompt-BS6QRCUU.js} +2 -2
  38. package/dist/{create-prompt-tag-GMRWTEFR.js → create-prompt-tag-FCQ5ELSQ.js} +2 -2
  39. package/dist/{create-scenario-JDR2VB2H.js → create-scenario-DRPTPXU3.js} +10 -3
  40. package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
  41. package/dist/{delete-agent-O3EEH4S5.js → delete-agent-SA4JMUZF.js} +3 -3
  42. package/dist/{delete-annotation-XP6FQM7A.js → delete-annotation-X3KKK7QY.js} +3 -3
  43. package/dist/{delete-dashboard-RFUNYCNP.js → delete-dashboard-LYMRPF27.js} +3 -3
  44. package/dist/{delete-dataset-3MJFRDLP.js → delete-dataset-W6EZ44HY.js} +3 -3
  45. package/dist/{delete-dataset-records-WPMU5GL5.js → delete-dataset-records-7CNOTALB.js} +3 -3
  46. package/dist/{delete-evaluator-33SP2JFX.js → delete-evaluator-RKYC4TNB.js} +3 -3
  47. package/dist/{delete-prompt-tag-KNMZVSPB.js → delete-prompt-tag-QB3OCZUV.js} +2 -2
  48. package/dist/{delete-workflow-3NUXO46U.js → delete-workflow-NG2VXYYM.js} +3 -3
  49. package/dist/{discover-evaluator-schema-XTAD4X7D.js → discover-evaluator-schema-ESSHQGPY.js} +7 -286
  50. package/dist/{discover-scenario-schema-FE64QK2D.js → discover-scenario-schema-W7KBXA4J.js} +53 -1
  51. package/dist/get-agent-7THF7HCU.js +66 -0
  52. package/dist/{get-analytics-TETCSOK6.js → get-analytics-R6AZD3AW.js} +2 -2
  53. package/dist/{get-dashboard-KHBEGZGL.js → get-dashboard-P4WWVHFU.js} +3 -3
  54. package/dist/{get-dataset-RCMU5VAH.js → get-dataset-AKTYSQDI.js} +3 -3
  55. package/dist/{get-evaluator-ADMKVWYZ.js → get-evaluator-RZ44UIN6.js} +3 -3
  56. package/dist/{get-project-RAK7URQJ.js → get-project-QSQEAB33.js} +3 -3
  57. package/dist/get-prompt-GVKXF5NL.js +100 -0
  58. package/dist/get-run-plan-NY7J5GRE.js +61 -0
  59. package/dist/{get-scenario-ZTAJJL5T.js → get-scenario-Q3LLQE4T.js} +7 -3
  60. package/dist/{get-simulation-run-T2ZQWYH2.js → get-simulation-run-WINEYDMK.js} +12 -3
  61. package/dist/get-test-suite-4HKA7OKS.js +49 -0
  62. package/dist/{get-trace-K7U234Q4.js → get-trace-RCUV7N4O.js} +2 -2
  63. package/dist/{get-workflow-L5L2PSK3.js → get-workflow-EEOC7Y45.js} +3 -3
  64. package/dist/{http-server-YXHFPPGA.js → http-server-WCDUARTH.js} +8 -3
  65. package/dist/index.js +3366 -1910
  66. package/dist/{langwatch-api-annotations-FDAS6NQ7.js → langwatch-api-annotations-6DHVZWUK.js} +3 -3
  67. package/dist/{langwatch-api-dashboards-JTVCS7ZJ.js → langwatch-api-dashboards-2KJL2M42.js} +3 -3
  68. package/dist/{langwatch-api-monitors-3J7UIQME.js → langwatch-api-monitors-6ISFJ7WX.js} +2 -2
  69. package/dist/{langwatch-api-secrets-BUBSNCNO.js → langwatch-api-secrets-OEBD6GBU.js} +2 -2
  70. package/dist/{langwatch-api-triggers-EDFUCJN6.js → langwatch-api-triggers-5E6DDGX6.js} +2 -2
  71. package/dist/{langwatch-api-workflows-NBJX5KOG.js → langwatch-api-workflows-6UJL5U4Z.js} +3 -3
  72. package/dist/list-agents-AMVDQPQL.js +43 -0
  73. package/dist/{list-annotations-ZTVP4EEL.js → list-annotations-6PR5NLIC.js} +3 -3
  74. package/dist/{list-api-keys-CWD4UQT5.js → list-api-keys-GVPNBIHQ.js} +3 -3
  75. package/dist/{list-dashboards-PWG6P6KR.js → list-dashboards-Z3BZKDGF.js} +3 -3
  76. package/dist/{list-dataset-records-IOE7LQXS.js → list-dataset-records-O67HZ35Y.js} +3 -3
  77. package/dist/{list-datasets-ZT4JYUDT.js → list-datasets-WXZL6UMP.js} +3 -3
  78. package/dist/{list-evaluators-SJHC7VBH.js → list-evaluators-PAX72WCN.js} +3 -3
  79. package/dist/{list-model-providers-PPD2WPD2.js → list-model-providers-VVLZMLCX.js} +3 -3
  80. package/dist/{list-projects-K7QAIX2I.js → list-projects-AQVI6SUK.js} +3 -3
  81. package/dist/{list-prompt-tags-FOO6TNHB.js → list-prompt-tags-WZFBJOHG.js} +2 -2
  82. package/dist/{list-prompts-SC2HAUFG.js → list-prompts-BQOHRHYT.js} +2 -2
  83. package/dist/list-run-plans-UBAU2DGH.js +46 -0
  84. package/dist/{list-scenarios-F2WYNRIE.js → list-scenarios-MS4IUIVM.js} +8 -5
  85. package/dist/{list-simulation-runs-KF5THG5H.js → list-simulation-runs-D6YF4ZOU.js} +11 -5
  86. package/dist/list-test-suites-L7R2O6DR.js +35 -0
  87. package/dist/{list-workflows-QSJWURCK.js → list-workflows-6MV4XJHF.js} +3 -3
  88. package/dist/{rename-prompt-tag-EKWU5QF3.js → rename-prompt-tag-K5LQ4CY6.js} +2 -2
  89. package/dist/rename-test-suite-NTYHAO5X.js +21 -0
  90. package/dist/report-issue-RHZZ33IS.js +885 -0
  91. package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
  92. package/dist/{revoke-api-key-FTRTZMOF.js → revoke-api-key-6LAZ3QZZ.js} +3 -3
  93. package/dist/run-agent-CAGWGTYT.js +72 -0
  94. package/dist/run-plan-T4ZYOZGA.js +39 -0
  95. package/dist/run-test-suite-L4MZ7G4D.js +26 -0
  96. package/dist/{run-workflow-QPXC627O.js → run-workflow-DFIS23AP.js} +3 -3
  97. package/dist/{search-traces-A6HFKPLJ.js → search-traces-4XTJYXIK.js} +2 -2
  98. package/dist/{set-model-provider-3WKT63ZN.js → set-model-provider-XRNMFOVR.js} +3 -3
  99. package/dist/{update-agent-SMAEHLXH.js → update-agent-DUWNYBNA.js} +3 -3
  100. package/dist/{update-dataset-BDUK7TUO.js → update-dataset-YSOMDCOB.js} +3 -3
  101. package/dist/{update-dataset-record-4ZSDRBSW.js → update-dataset-record-DEA2ITYR.js} +3 -3
  102. package/dist/{update-evaluator-LS6TIBY7.js → update-evaluator-NH7Q5GWV.js} +3 -3
  103. package/dist/{update-project-LKVYXENH.js → update-project-Q26FBSYH.js} +3 -3
  104. package/dist/update-prompt-ZQQ2C3KU.js +143 -0
  105. package/dist/{update-scenario-NEXCAQI7.js → update-scenario-WUX34JAR.js} +10 -3
  106. package/package.json +8 -9
  107. package/pyproject.toml +4 -4
  108. package/src/__tests__/agent-tools.unit.test.ts +257 -0
  109. package/src/__tests__/all-tools.integration.test.ts +321 -75
  110. package/src/__tests__/integration.integration.test.ts +7 -11
  111. package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
  112. package/src/__tests__/prompts.integration.test.ts +736 -0
  113. package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
  114. package/src/__tests__/scenario-tools.unit.test.ts +19 -0
  115. package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
  116. package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
  117. package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
  118. package/src/create-mcp-server.ts +334 -55
  119. package/src/langwatch-api-agents.ts +196 -12
  120. package/src/langwatch-api-run-plans.ts +157 -0
  121. package/src/langwatch-api-scenarios.ts +9 -0
  122. package/src/langwatch-api-simulation-runs.ts +5 -0
  123. package/src/langwatch-api-test-suites.ts +135 -0
  124. package/src/langwatch-api.ts +62 -12
  125. package/src/schemas/index.ts +5 -0
  126. package/src/schemas/run-plan.ts +92 -0
  127. package/src/schemas/suite-fields.ts +149 -0
  128. package/src/tools/archive-run-plan.ts +12 -0
  129. package/src/tools/archive-test-suite.ts +12 -0
  130. package/src/tools/create-scenario.ts +8 -0
  131. package/src/tools/create-test-suite.ts +43 -0
  132. package/src/tools/discover-scenario-schema.ts +54 -0
  133. package/src/tools/format-run-plan.ts +68 -0
  134. package/src/tools/format-scenario.ts +14 -0
  135. package/src/tools/format-suite-details.ts +96 -0
  136. package/src/tools/get-agent.ts +45 -3
  137. package/src/tools/get-prompt.ts +108 -24
  138. package/src/tools/get-run-plan.ts +62 -0
  139. package/src/tools/get-scenario.ts +3 -0
  140. package/src/tools/get-simulation-run.ts +9 -0
  141. package/src/tools/get-test-suite.ts +49 -0
  142. package/src/tools/list-agents.ts +20 -3
  143. package/src/tools/list-run-plans.ts +47 -0
  144. package/src/tools/list-scenarios.ts +12 -2
  145. package/src/tools/list-simulation-runs.ts +6 -2
  146. package/src/tools/list-test-suites.ts +35 -0
  147. package/src/tools/rename-test-suite.ts +18 -0
  148. package/src/tools/rerun-run-plan.ts +21 -0
  149. package/src/tools/run-agent.ts +54 -6
  150. package/src/tools/run-plan.ts +49 -0
  151. package/src/tools/run-test-suite.ts +30 -0
  152. package/src/tools/test-agent.ts +18 -0
  153. package/src/tools/update-prompt.ts +195 -10
  154. package/src/tools/update-scenario.ts +8 -0
  155. package/src/tools/update-test-suite.ts +45 -0
  156. package/tsconfig.json +5 -0
  157. package/uv.lock +416 -296
  158. package/dist/archive-suite-4T6BSRS7.js +0 -16
  159. package/dist/chunk-YZLYHLLI.js +0 -48
  160. package/dist/create-suite-LZS6QIAF.js +0 -39
  161. package/dist/get-agent-Y7MWZ65F.js +0 -29
  162. package/dist/get-prompt-OY5B7TRI.js +0 -46
  163. package/dist/get-suite-J7JVWO6O.js +0 -45
  164. package/dist/list-agents-KFHKGPXV.js +0 -33
  165. package/dist/list-suites-CRY4VWW6.js +0 -43
  166. package/dist/report-issue-XYVAPAYJ.js +0 -412
  167. package/dist/run-agent-6SM4XNA7.js +0 -37
  168. package/dist/run-suite-NKZCJ2IN.js +0 -30
  169. package/dist/update-prompt-ODCRQRN6.js +0 -25
  170. package/dist/update-suite-SNO3GS7T.js +0 -40
  171. package/src/langwatch-api-suites.ts +0 -118
  172. package/src/tools/archive-suite.ts +0 -12
  173. package/src/tools/create-suite.ts +0 -42
  174. package/src/tools/get-suite.ts +0 -47
  175. package/src/tools/list-suites.ts +0 -43
  176. package/src/tools/run-suite.ts +0 -28
  177. package/src/tools/update-suite.ts +0 -44
@@ -1,42 +0,0 @@
1
- import { createSuite as apiCreateSuite } from "../langwatch-api-suites.js";
2
- import type { SuiteTarget } from "../langwatch-api-suites.js";
3
-
4
- /**
5
- * Handles the platform_create_suite MCP tool invocation.
6
- */
7
- export async function handleCreateSuite(params: {
8
- name: string;
9
- description?: string;
10
- scenarioIds: string[];
11
- targets: string;
12
- repeatCount?: number;
13
- labels?: string[];
14
- }): Promise<string> {
15
- let parsedTargets: SuiteTarget[];
16
- try {
17
- parsedTargets = JSON.parse(params.targets) as SuiteTarget[];
18
- } catch {
19
- return "Error: `targets` must be a valid JSON array of objects with `type` and `referenceId` fields.\n\nExample: [{\"type\": \"http\", \"referenceId\": \"agent_abc123\"}]";
20
- }
21
-
22
- const suite = await apiCreateSuite({
23
- name: params.name,
24
- description: params.description,
25
- scenarioIds: params.scenarioIds,
26
- targets: parsedTargets,
27
- repeatCount: params.repeatCount,
28
- labels: params.labels,
29
- });
30
-
31
- return [
32
- `Suite "${suite.name}" created successfully.`,
33
- "",
34
- `**ID**: ${suite.id}`,
35
- `**Slug**: ${suite.slug}`,
36
- `**Scenarios**: ${suite.scenarioIds.length}`,
37
- `**Targets**: ${suite.targets.length}`,
38
- `**Repeat**: ${suite.repeatCount}x`,
39
- "",
40
- `> Use \`platform_run_suite\` with ID \`${suite.id}\` to trigger execution.`,
41
- ].join("\n");
42
- }
@@ -1,47 +0,0 @@
1
- import { getSuite as apiGetSuite } from "../langwatch-api-suites.js";
2
-
3
- /**
4
- * Handles the platform_get_suite MCP tool invocation.
5
- */
6
- export async function handleGetSuite(params: {
7
- id: string;
8
- format?: "digest" | "json";
9
- }): Promise<string> {
10
- const suite = await apiGetSuite(params.id);
11
-
12
- if (params.format === "json") {
13
- return JSON.stringify(suite, null, 2);
14
- }
15
-
16
- const lines: string[] = [];
17
- lines.push(`# Suite: ${suite.name}\n`);
18
- lines.push(`**ID**: ${suite.id}`);
19
- lines.push(`**Slug**: ${suite.slug}`);
20
- lines.push(`**Description**: ${suite.description ?? "—"}`);
21
- lines.push(`**Repeat Count**: ${suite.repeatCount}`);
22
- lines.push(`**Created**: ${suite.createdAt}`);
23
- lines.push(`**Updated**: ${suite.updatedAt}`);
24
-
25
- if (suite.labels.length > 0) {
26
- lines.push(`**Labels**: ${suite.labels.join(", ")}`);
27
- }
28
-
29
- lines.push("\n## Scenarios");
30
- for (const id of suite.scenarioIds) {
31
- lines.push(`- ${id}`);
32
- }
33
-
34
- lines.push("\n## Targets");
35
- for (const t of suite.targets) {
36
- lines.push(`- ${t.type}:${t.referenceId}`);
37
- }
38
-
39
- const totalJobs = suite.scenarioIds.length * suite.targets.length * suite.repeatCount;
40
- lines.push(`\n**Total jobs per run**: ${totalJobs}`);
41
-
42
- lines.push(
43
- "\n> Use `platform_run_suite` to trigger a run, or `platform_update_suite` to modify."
44
- );
45
-
46
- return lines.join("\n");
47
- }
@@ -1,43 +0,0 @@
1
- import { listSuites as apiListSuites } from "../langwatch-api-suites.js";
2
-
3
- /**
4
- * Handles the platform_list_suites MCP tool invocation.
5
- */
6
- export async function handleListSuites(params: {
7
- format?: "digest" | "json";
8
- }): Promise<string> {
9
- const suites = await apiListSuites();
10
-
11
- if (params.format === "json") {
12
- return JSON.stringify(suites, null, 2);
13
- }
14
-
15
- if (!Array.isArray(suites) || suites.length === 0) {
16
- return "No suites (run plans) found in this project.\n\n> Tip: Use `platform_create_suite` to create your first suite.";
17
- }
18
-
19
- const lines: string[] = [];
20
- lines.push(`# Suites / Run Plans (${suites.length} total)\n`);
21
-
22
- for (const s of suites) {
23
- lines.push(`## ${s.name}`);
24
- lines.push(`**ID**: ${s.id}`);
25
- lines.push(`**Slug**: ${s.slug}`);
26
- lines.push(`**Scenarios**: ${s.scenarioIds.length}`);
27
- lines.push(`**Targets**: ${s.targets.length} (${s.targets.map((t) => `${t.type}:${t.referenceId}`).join(", ")})`);
28
- lines.push(`**Repeat**: ${s.repeatCount}x`);
29
- if (s.description) {
30
- lines.push(`**Description**: ${s.description}`);
31
- }
32
- if (Array.isArray(s.labels) && s.labels.length > 0) {
33
- lines.push(`**Labels**: ${s.labels.join(", ")}`);
34
- }
35
- lines.push("");
36
- }
37
-
38
- lines.push(
39
- "> Use `platform_get_suite` with the ID to see full details, or `platform_run_suite` to trigger execution."
40
- );
41
-
42
- return lines.join("\n");
43
- }
@@ -1,28 +0,0 @@
1
- import { runSuite as apiRunSuite } from "../langwatch-api-suites.js";
2
-
3
- /**
4
- * Handles the platform_run_suite MCP tool invocation.
5
- */
6
- export async function handleRunSuite(params: {
7
- id: string;
8
- }): Promise<string> {
9
- const result = await apiRunSuite(params.id);
10
-
11
- const lines: string[] = [];
12
- lines.push(`Suite run scheduled successfully.`);
13
- lines.push("");
14
- lines.push(`**Batch Run ID**: ${result.batchRunId}`);
15
- lines.push(`**Jobs**: ${result.jobCount}`);
16
- lines.push(`**Set ID**: ${result.setId}`);
17
-
18
- if (result.skippedArchived.scenarios.length > 0) {
19
- lines.push(`\n⚠️ Skipped archived scenarios: ${result.skippedArchived.scenarios.join(", ")}`);
20
- }
21
- if (result.skippedArchived.targets.length > 0) {
22
- lines.push(`⚠️ Skipped archived targets: ${result.skippedArchived.targets.join(", ")}`);
23
- }
24
-
25
- lines.push("\nView results in the LangWatch dashboard under Simulations.");
26
-
27
- return lines.join("\n");
28
- }
@@ -1,44 +0,0 @@
1
- import { updateSuite as apiUpdateSuite } from "../langwatch-api-suites.js";
2
- import type { SuiteTarget } from "../langwatch-api-suites.js";
3
-
4
- /**
5
- * Handles the platform_update_suite MCP tool invocation.
6
- */
7
- export async function handleUpdateSuite(params: {
8
- id: string;
9
- name?: string;
10
- description?: string;
11
- scenarioIds?: string[];
12
- targets?: string;
13
- repeatCount?: number;
14
- labels?: string[];
15
- }): Promise<string> {
16
- let parsedTargets: SuiteTarget[] | undefined;
17
- if (params.targets) {
18
- try {
19
- parsedTargets = JSON.parse(params.targets) as SuiteTarget[];
20
- } catch {
21
- return "Error: `targets` must be a valid JSON array.";
22
- }
23
- }
24
-
25
- const suite = await apiUpdateSuite({
26
- id: params.id,
27
- name: params.name,
28
- description: params.description,
29
- scenarioIds: params.scenarioIds,
30
- targets: parsedTargets,
31
- repeatCount: params.repeatCount,
32
- labels: params.labels,
33
- });
34
-
35
- return [
36
- `Suite "${suite.name}" updated successfully.`,
37
- "",
38
- `**ID**: ${suite.id}`,
39
- `**Slug**: ${suite.slug}`,
40
- `**Scenarios**: ${suite.scenarioIds.length}`,
41
- `**Targets**: ${suite.targets.length}`,
42
- `**Repeat**: ${suite.repeatCount}x`,
43
- ].join("\n");
44
- }