@langwatch/mcp-server 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/LICENSE +7 -0
  3. package/dist/archive-project-HZBRPR2S.js +19 -0
  4. package/dist/{archive-scenario-YKGLJ4MZ.js → archive-scenario-P5RRITRM.js} +3 -3
  5. package/dist/{archive-suite-QFNDO3Y3.js → archive-suite-NUUFD2EM.js} +3 -3
  6. package/dist/{assign-prompt-tag-Y2B4CBBT.js → assign-prompt-tag-FKCCZPML.js} +2 -2
  7. package/dist/{chunk-YH2IBTEL.js → chunk-42ZHHHIJ.js} +14 -3
  8. package/dist/chunk-ADSJ2FDA.js +44 -0
  9. package/dist/{chunk-L6QENXYA.js → chunk-BB7EOKHR.js} +2 -2
  10. package/dist/chunk-BBJ5WAJB.js +24 -0
  11. package/dist/{chunk-PZUWCWOS.js → chunk-BYWYM2RP.js} +2 -2
  12. package/dist/{chunk-NACSR6B7.js → chunk-CCM4ZFGF.js} +2 -2
  13. package/dist/{chunk-FBRKHHJV.js → chunk-CHKCEZBP.js} +2 -2
  14. package/dist/{chunk-PO7FM7IX.js → chunk-HKY3GW45.js} +2 -2
  15. package/dist/{chunk-KZNBCKWG.js → chunk-JHOSVOHT.js} +2 -2
  16. package/dist/{chunk-B57MZ4WI.js → chunk-JVLJKQJG.js} +657 -88
  17. package/dist/{chunk-HDJDFKZR.js → chunk-R7C7OUVG.js} +2 -2
  18. package/dist/{chunk-OVQF3D5R.js → chunk-TROIWSEQ.js} +2 -2
  19. package/dist/{chunk-CPUPGDNA.js → chunk-X2SEURT2.js} +2 -2
  20. package/dist/{chunk-YI3OG4PI.js → chunk-XC62KJIT.js} +3 -3
  21. package/dist/{create-agent-YRSDH7P7.js → create-agent-K5Q7BFGZ.js} +3 -3
  22. package/dist/{create-annotation-FYQNKMVJ.js → create-annotation-KN3GE3DK.js} +3 -3
  23. package/dist/create-api-key-OQI5PUNQ.js +26 -0
  24. package/dist/{create-dashboard-CORBR4YQ.js → create-dashboard-G4QA5AXP.js} +3 -3
  25. package/dist/{create-dataset-ZC63T6BB.js → create-dataset-WQSC37N7.js} +3 -3
  26. package/dist/{create-dataset-records-ILVNVKVR.js → create-dataset-records-B4RL3FLC.js} +3 -3
  27. package/dist/{create-evaluator-2AU3PXFB.js → create-evaluator-YFFQIR7E.js} +3 -3
  28. package/dist/create-mcp-server.js +2 -1
  29. package/dist/create-project-CUVQ67VN.js +44 -0
  30. package/dist/{create-prompt-NKRJ47YY.js → create-prompt-NGXKSP4X.js} +2 -2
  31. package/dist/{create-prompt-tag-S4JX3ULL.js → create-prompt-tag-IUKYPVHH.js} +2 -2
  32. package/dist/{create-scenario-BPRF75KF.js → create-scenario-FNBXALH3.js} +3 -3
  33. package/dist/{create-suite-4YN7D6K2.js → create-suite-DJEPD4ZB.js} +3 -3
  34. package/dist/{delete-agent-Y72R2YQA.js → delete-agent-TJI5XBBM.js} +3 -3
  35. package/dist/{delete-annotation-UHWDMHSH.js → delete-annotation-KRP53TUU.js} +3 -3
  36. package/dist/{delete-dashboard-LPWT36KN.js → delete-dashboard-N5SZJIR3.js} +3 -3
  37. package/dist/{delete-dataset-GGPQRHKX.js → delete-dataset-KDUOHL2P.js} +3 -3
  38. package/dist/{delete-dataset-records-LTG5LT26.js → delete-dataset-records-DM4W3PAW.js} +3 -3
  39. package/dist/{delete-evaluator-F7QLXDHU.js → delete-evaluator-7ZATLJOA.js} +3 -3
  40. package/dist/{delete-prompt-tag-B5UKZL45.js → delete-prompt-tag-I6HNMSVJ.js} +2 -2
  41. package/dist/{delete-workflow-4DGUDIDE.js → delete-workflow-QEO74JQR.js} +3 -3
  42. package/dist/{get-agent-YEVICUX4.js → get-agent-3KDWCP5N.js} +3 -3
  43. package/dist/{get-analytics-3W5CMU3F.js → get-analytics-GPVJX7SF.js} +2 -2
  44. package/dist/{get-dashboard-NCQHN6MW.js → get-dashboard-YPTIE5YY.js} +3 -3
  45. package/dist/{get-dataset-KBK6PH36.js → get-dataset-6DQFYITM.js} +3 -3
  46. package/dist/{get-evaluator-QDUL5LXR.js → get-evaluator-PCM6XQAU.js} +3 -3
  47. package/dist/get-project-JCMWXVLU.js +26 -0
  48. package/dist/{get-prompt-XCSSNRKR.js → get-prompt-5XAGWWBG.js} +2 -2
  49. package/dist/{get-scenario-FNRKOY7E.js → get-scenario-7XBCDZWX.js} +3 -3
  50. package/dist/{get-simulation-run-WOEKUF7K.js → get-simulation-run-AV24FJWI.js} +3 -3
  51. package/dist/{get-suite-U5AOD5TC.js → get-suite-4DBVHFEM.js} +3 -3
  52. package/dist/{get-trace-MUWEJM2Z.js → get-trace-MRBDS3HJ.js} +2 -2
  53. package/dist/{get-workflow-SOA7MT7L.js → get-workflow-UR4Y6UFX.js} +3 -3
  54. package/dist/{http-server-7DMOCB75.js → http-server-DEGNV5EP.js} +3 -2
  55. package/dist/index.js +1060 -220
  56. package/dist/{langwatch-api-annotations-Q57NDUVR.js → langwatch-api-annotations-ZJDRB4JH.js} +3 -3
  57. package/dist/{langwatch-api-dashboards-VBWWXV3Y.js → langwatch-api-dashboards-WARLI6NF.js} +3 -3
  58. package/dist/{langwatch-api-monitors-L3IY7FDT.js → langwatch-api-monitors-4XKXHYCG.js} +2 -2
  59. package/dist/{langwatch-api-secrets-S5HHF3XR.js → langwatch-api-secrets-6SHK5G7Q.js} +2 -2
  60. package/dist/{langwatch-api-triggers-ME33QE26.js → langwatch-api-triggers-5SX6R2WU.js} +2 -2
  61. package/dist/{langwatch-api-workflows-4CJQAYP2.js → langwatch-api-workflows-NLJENNXE.js} +3 -3
  62. package/dist/{list-agents-3HGIBJFM.js → list-agents-NECO7MPI.js} +3 -3
  63. package/dist/{list-annotations-KWXYVEPI.js → list-annotations-3OJEO774.js} +3 -3
  64. package/dist/list-api-keys-AWW7ROLB.js +37 -0
  65. package/dist/{list-dashboards-VQ2LH666.js → list-dashboards-DVPVVVKW.js} +3 -3
  66. package/dist/{list-dataset-records-RNRGROJP.js → list-dataset-records-I4BYORF5.js} +3 -3
  67. package/dist/{list-datasets-I4A2LT2S.js → list-datasets-WUAE5HGX.js} +3 -3
  68. package/dist/{list-evaluators-AKKRJ5MX.js → list-evaluators-B6OSVAM2.js} +3 -3
  69. package/dist/{list-model-providers-DVFEHFHZ.js → list-model-providers-NL6IP4UZ.js} +3 -3
  70. package/dist/list-projects-5FHMJ4PQ.js +42 -0
  71. package/dist/{list-prompt-tags-PL5OISUA.js → list-prompt-tags-Z2DD6FLQ.js} +2 -2
  72. package/dist/{list-prompts-7VTFWE2Z.js → list-prompts-VNXKHK7T.js} +2 -2
  73. package/dist/{list-scenarios-5BZWZ6ZN.js → list-scenarios-CFZHXL76.js} +3 -3
  74. package/dist/{list-simulation-runs-WCZ6RLPE.js → list-simulation-runs-AM2BXLBW.js} +3 -3
  75. package/dist/{list-suites-RRQ2JX5Y.js → list-suites-ZDOHAAXF.js} +3 -3
  76. package/dist/{list-workflows-UQB5FCX5.js → list-workflows-WBRBKLIT.js} +3 -3
  77. package/dist/{rename-prompt-tag-TURD6KYU.js → rename-prompt-tag-A4RJUK7Z.js} +2 -2
  78. package/dist/revoke-api-key-ZSPSPF5G.js +15 -0
  79. package/dist/{run-agent-BEJ4K4CM.js → run-agent-JFNDY7MR.js} +3 -3
  80. package/dist/{run-suite-U7V432G2.js → run-suite-TH7LO5TJ.js} +3 -3
  81. package/dist/{run-workflow-5WX7DK5T.js → run-workflow-CK3JP5U2.js} +3 -3
  82. package/dist/{search-traces-ZCHNOMTX.js → search-traces-3BBVYPC3.js} +2 -2
  83. package/dist/{set-model-provider-6CFHMCNE.js → set-model-provider-WOC5FNMV.js} +3 -3
  84. package/dist/{update-agent-UE6UWLZP.js → update-agent-YMEV5MTL.js} +3 -3
  85. package/dist/{update-dataset-4XQZZE4S.js → update-dataset-EYT5POFM.js} +3 -3
  86. package/dist/{update-dataset-record-GNNM5TKM.js → update-dataset-record-VFF5JTX4.js} +3 -3
  87. package/dist/{update-evaluator-LHVDXIZD.js → update-evaluator-6IJHFG2E.js} +3 -3
  88. package/dist/update-project-OIT4WPFI.js +24 -0
  89. package/dist/{update-prompt-3XEHECJ7.js → update-prompt-ZU2DD6KM.js} +2 -2
  90. package/dist/{update-scenario-DFPREYCD.js → update-scenario-3VUTPDRU.js} +3 -3
  91. package/dist/{update-suite-RATU5THO.js → update-suite-C6JFDQHL.js} +3 -3
  92. package/package.json +2 -2
  93. package/pnpm-workspace.yaml +2 -0
  94. package/pyproject.toml +3 -0
  95. package/src/__tests__/experiment-list-tools.unit.test.ts +157 -0
  96. package/src/__tests__/get-experiment-results.unit.test.ts +222 -0
  97. package/src/__tests__/project-api-key-tools.unit.test.ts +451 -0
  98. package/src/create-mcp-server.ts +257 -13
  99. package/src/langwatch-api-api-keys.ts +54 -0
  100. package/src/langwatch-api-projects.ts +82 -0
  101. package/src/langwatch-api.ts +17 -3
  102. package/src/tools/archive-project.ts +14 -0
  103. package/src/tools/create-api-key.ts +29 -0
  104. package/src/tools/create-project.ts +45 -0
  105. package/src/tools/get-experiment-results.ts +273 -0
  106. package/src/tools/get-project.ts +20 -0
  107. package/src/tools/list-api-keys.ts +33 -0
  108. package/src/tools/list-experiment-runs.ts +142 -0
  109. package/src/tools/list-experiments.ts +95 -0
  110. package/src/tools/list-projects.ts +41 -0
  111. package/src/tools/revoke-api-key.ts +9 -0
  112. package/src/tools/{run-evaluation.ts → run-experiment.ts} +5 -5
  113. package/src/tools/update-project.ts +22 -0
  114. package/uv.lock +7 -3
  115. package/dist/run-evaluation-AJ7WZNQD.js +0 -64
package/dist/index.js CHANGED
@@ -31109,7 +31109,7 @@ var init_package = __esm({
31109
31109
  "package.json"() {
31110
31110
  package_default = {
31111
31111
  name: "@langwatch/mcp-server",
31112
- version: "0.8.0",
31112
+ version: "0.9.0",
31113
31113
  description: "An MCP server for Langwatch.",
31114
31114
  type: "module",
31115
31115
  main: "./dist/index.js",
@@ -31160,7 +31160,7 @@ var init_package = __esm({
31160
31160
  "@types/node": "^25.6.0",
31161
31161
  "@types/yargs": "^17.0.33",
31162
31162
  "@typescript/native-preview": "7.0.0-dev.20250911.1",
31163
- ai: "^6.0.0",
31163
+ ai: "^6.0.161",
31164
31164
  dotenv: "^17.2.2",
31165
31165
  esbuild: "^0.25.0",
31166
31166
  eslint: "^10.2.1",
@@ -31188,6 +31188,7 @@ var init_package = __esm({
31188
31188
  pnpm: {
31189
31189
  overrides: {
31190
31190
  protobufjs: ">=7.5.6 <8.0.0 || >=8.0.2",
31191
+ "langsmith@<0.6.0": "0.6.0",
31191
31192
  "fast-uri@<3.1.2": "3.1.2"
31192
31193
  }
31193
31194
  }
@@ -31227,6 +31228,533 @@ var init_create_dataset = __esm({
31227
31228
  }
31228
31229
  });
31229
31230
 
31231
+ // src/langwatch-api.ts
31232
+ async function makeRequest(method, path, body) {
31233
+ const config2 = getConfig();
31234
+ const url2 = config2.endpoint + path;
31235
+ const headers = {
31236
+ "X-Auth-Token": requireApiKey()
31237
+ };
31238
+ if (config2.projectId) {
31239
+ headers["X-Project-Id"] = config2.projectId;
31240
+ }
31241
+ if (body !== void 0) {
31242
+ headers["Content-Type"] = "application/json";
31243
+ }
31244
+ const response = await fetch(url2, {
31245
+ method,
31246
+ headers,
31247
+ ...body !== void 0 ? { body: JSON.stringify(body) } : {}
31248
+ });
31249
+ if (!response.ok) {
31250
+ const responseBody = await response.text();
31251
+ throw new LangWatchApiError(
31252
+ `LangWatch API error ${response.status}: ${responseBody}`,
31253
+ response.status,
31254
+ responseBody
31255
+ );
31256
+ }
31257
+ if (response.status === 204 || response.headers?.get("content-length") === "0") {
31258
+ return null;
31259
+ }
31260
+ return response.json();
31261
+ }
31262
+ async function searchTraces(params) {
31263
+ const { format: format3 = "digest", ...rest } = params;
31264
+ return makeRequest("POST", "/api/traces/search", {
31265
+ ...rest,
31266
+ format: format3
31267
+ });
31268
+ }
31269
+ async function getTraceById(traceId, format3 = "digest") {
31270
+ return makeRequest(
31271
+ "GET",
31272
+ `/api/traces/${encodeURIComponent(traceId)}?format=${format3}`
31273
+ );
31274
+ }
31275
+ async function getAnalyticsTimeseries(params) {
31276
+ return makeRequest(
31277
+ "POST",
31278
+ "/api/analytics/timeseries",
31279
+ params
31280
+ );
31281
+ }
31282
+ async function listPrompts() {
31283
+ return makeRequest("GET", "/api/prompts");
31284
+ }
31285
+ async function getPrompt(idOrHandle, options) {
31286
+ const params = new URLSearchParams();
31287
+ if (options?.version != null) params.set("version", String(options.version));
31288
+ if (options?.tag) params.set("tag", options.tag);
31289
+ const query = params.toString() ? `?${params}` : "";
31290
+ return makeRequest(
31291
+ "GET",
31292
+ `/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
31293
+ );
31294
+ }
31295
+ async function createPrompt(data) {
31296
+ return makeRequest(
31297
+ "POST",
31298
+ "/api/prompts",
31299
+ data
31300
+ );
31301
+ }
31302
+ async function updatePrompt(idOrHandle, data) {
31303
+ return makeRequest(
31304
+ "PUT",
31305
+ `/api/prompts/${encodeURIComponent(idOrHandle)}`,
31306
+ data
31307
+ );
31308
+ }
31309
+ async function assignPromptTag({
31310
+ idOrHandle,
31311
+ tag,
31312
+ versionId
31313
+ }) {
31314
+ return makeRequest(
31315
+ "PUT",
31316
+ `/api/prompts/${encodeURIComponent(idOrHandle)}/tags/${encodeURIComponent(tag)}`,
31317
+ { versionId }
31318
+ );
31319
+ }
31320
+ async function listPromptTags() {
31321
+ return makeRequest("GET", "/api/prompts/tags");
31322
+ }
31323
+ async function createPromptTag(name) {
31324
+ return makeRequest("POST", "/api/prompts/tags", { name });
31325
+ }
31326
+ async function renamePromptTag({
31327
+ tag,
31328
+ name
31329
+ }) {
31330
+ return makeRequest(
31331
+ "PUT",
31332
+ `/api/prompts/tags/${encodeURIComponent(tag)}`,
31333
+ { name }
31334
+ );
31335
+ }
31336
+ async function deletePromptTag(tag) {
31337
+ return makeRequest(
31338
+ "DELETE",
31339
+ `/api/prompts/tags/${encodeURIComponent(tag)}`
31340
+ );
31341
+ }
31342
+ var LangWatchApiError;
31343
+ var init_langwatch_api = __esm({
31344
+ "src/langwatch-api.ts"() {
31345
+ "use strict";
31346
+ init_config();
31347
+ LangWatchApiError = class extends Error {
31348
+ constructor(message, status, responseBody) {
31349
+ super(message);
31350
+ this.status = status;
31351
+ this.responseBody = responseBody;
31352
+ this.name = "LangWatchApiError";
31353
+ }
31354
+ };
31355
+ }
31356
+ });
31357
+
31358
+ // src/tools/get-experiment-results.ts
31359
+ async function handleExperimentResults(params) {
31360
+ const filter = params.filter ?? "all";
31361
+ const evaluatorFilter = params.evaluator?.trim();
31362
+ const limit = typeof params.limit === "number" && params.limit > 0 ? Math.min(params.limit, DEFAULT_ROW_CAP) : DEFAULT_ROW_CAP;
31363
+ const search = new URLSearchParams();
31364
+ if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
31365
+ const qs = search.toString() ? `?${search.toString()}` : "";
31366
+ let results;
31367
+ try {
31368
+ results = await makeRequest(
31369
+ "GET",
31370
+ `/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
31371
+ );
31372
+ } catch (error48) {
31373
+ const message = error48 instanceof Error ? error48.message : String(error48);
31374
+ const status = error48 instanceof LangWatchApiError ? error48.status : error48 && typeof error48 === "object" && "status" in error48 && typeof error48.status === "number" ? error48.status : void 0;
31375
+ if (status === 404 || status === void 0 && /404|not found/i.test(message)) {
31376
+ return [
31377
+ `# Evaluation Results: ${params.runId}`,
31378
+ "",
31379
+ "**Status**: not found",
31380
+ "",
31381
+ `Could not load results for run \`${params.runId}\`. The run may still be in progress, or the id may be incorrect.`,
31382
+ "",
31383
+ "> Try `platform_experiment_status` first to confirm the run id and that it is completed."
31384
+ ].join("\n");
31385
+ }
31386
+ throw error48;
31387
+ }
31388
+ if (!results || results.timestamps.finishedAt == null && results.timestamps.stoppedAt == null) {
31389
+ return [
31390
+ `# Evaluation Results: ${params.runId}`,
31391
+ "",
31392
+ "**Status**: results are not yet available",
31393
+ "",
31394
+ `Results for run \`${params.runId}\` are not ready yet.`,
31395
+ "",
31396
+ "> Try `platform_experiment_status` first to confirm the run id and that it is completed."
31397
+ ].join("\n");
31398
+ }
31399
+ const evaluationsByRow = /* @__PURE__ */ new Map();
31400
+ for (const evaluation of results.evaluations) {
31401
+ if (evaluatorFilter && evaluation.evaluator !== evaluatorFilter) continue;
31402
+ const key = rowKey(evaluation.index, evaluation.targetId);
31403
+ const list = evaluationsByRow.get(key) ?? [];
31404
+ list.push(evaluation);
31405
+ evaluationsByRow.set(key, list);
31406
+ }
31407
+ const evaluatorNames = Array.from(
31408
+ new Set(
31409
+ (evaluatorFilter ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter) : results.evaluations).map((e) => e.evaluator)
31410
+ )
31411
+ );
31412
+ let rows = results.dataset.map((entry) => ({
31413
+ entry,
31414
+ evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? []
31415
+ }));
31416
+ if (filter === "failed") {
31417
+ rows = rows.filter(
31418
+ (r) => isFailedRow({ entry: r.entry, evaluations: r.evaluations })
31419
+ );
31420
+ }
31421
+ const totalMatching = rows.length;
31422
+ const rowsForSummary = rows;
31423
+ const truncated = rows.length > limit;
31424
+ rows = rows.slice(0, limit);
31425
+ const evaluatorAverages = /* @__PURE__ */ new Map();
31426
+ for (const r of rowsForSummary) {
31427
+ for (const e of r.evaluations) {
31428
+ const stats = evaluatorAverages.get(e.evaluator) ?? {
31429
+ sum: 0,
31430
+ count: 0,
31431
+ passed: 0,
31432
+ failed: 0,
31433
+ errored: 0
31434
+ };
31435
+ if (typeof e.score === "number") {
31436
+ stats.sum += e.score;
31437
+ stats.count += 1;
31438
+ }
31439
+ if (e.status === "error") stats.errored += 1;
31440
+ else if (e.passed === true) stats.passed += 1;
31441
+ else if (e.passed === false) stats.failed += 1;
31442
+ evaluatorAverages.set(e.evaluator, stats);
31443
+ }
31444
+ }
31445
+ const lines = [];
31446
+ lines.push(`# Evaluation Results: ${results.runId}`);
31447
+ lines.push("");
31448
+ lines.push(`**Experiment**: ${results.experimentId}`);
31449
+ lines.push(`**Total rows**: ${results.dataset.length}`);
31450
+ lines.push(`**Total evaluations**: ${results.evaluations.length}`);
31451
+ if (filter === "failed") {
31452
+ lines.push(`**Filter**: failed only`);
31453
+ }
31454
+ if (evaluatorFilter) {
31455
+ lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
31456
+ }
31457
+ lines.push("");
31458
+ if (evaluatorAverages.size > 0) {
31459
+ lines.push("## Evaluator Summary");
31460
+ lines.push("");
31461
+ lines.push("| Evaluator | Avg Score | Passed | Failed | Errored |");
31462
+ lines.push("| --- | --- | --- | --- | --- |");
31463
+ for (const [name, stats] of evaluatorAverages) {
31464
+ const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "\u2014";
31465
+ lines.push(
31466
+ `| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`
31467
+ );
31468
+ }
31469
+ lines.push("");
31470
+ }
31471
+ if (rows.length === 0) {
31472
+ lines.push("_No rows matched the filter._");
31473
+ return lines.join("\n");
31474
+ }
31475
+ lines.push(
31476
+ `## Rows (${rows.length}${truncated ? ` of ${totalMatching}` : ""})`
31477
+ );
31478
+ lines.push("");
31479
+ for (const { entry, evaluations } of rows) {
31480
+ const summary = summarizeEntry(entry.entry);
31481
+ lines.push(`### Row #${entry.index}${summary ? ` \u2014 ${summary}` : ""}`);
31482
+ if (entry.error) {
31483
+ lines.push(`- **Error**: ${entry.error}`);
31484
+ }
31485
+ if (entry.traceId) {
31486
+ lines.push(`- **Trace ID**: \`${entry.traceId}\``);
31487
+ }
31488
+ if (evaluations.length === 0) {
31489
+ lines.push(
31490
+ evaluatorNames.length === 0 ? "- _No evaluations recorded_" : "- _No evaluations recorded for this row_"
31491
+ );
31492
+ }
31493
+ for (const e of evaluations) {
31494
+ const parts = [`**${e.evaluator}**`];
31495
+ if (e.status === "error") parts.push("status=error");
31496
+ else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
31497
+ if (typeof e.passed === "boolean") {
31498
+ parts.push(`passed=${e.passed ? "yes" : "no"}`);
31499
+ }
31500
+ if (e.label) parts.push(`label=${e.label}`);
31501
+ lines.push(`- ${parts.join(" \xB7 ")}`);
31502
+ if (e.details) {
31503
+ lines.push(` - details: ${e.details}`);
31504
+ }
31505
+ }
31506
+ lines.push("");
31507
+ }
31508
+ if (truncated) {
31509
+ lines.push(
31510
+ `> Output truncated to ${limit} rows of ${totalMatching} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`
31511
+ );
31512
+ }
31513
+ return lines.join("\n");
31514
+ }
31515
+ var DEFAULT_ROW_CAP, rowKey, summarizeEntry, isFailedEvaluation, isFailedRow;
31516
+ var init_get_experiment_results = __esm({
31517
+ "src/tools/get-experiment-results.ts"() {
31518
+ "use strict";
31519
+ init_langwatch_api();
31520
+ DEFAULT_ROW_CAP = 50;
31521
+ rowKey = (index, targetId) => `${index}:${targetId ?? ""}`;
31522
+ summarizeEntry = (entry) => {
31523
+ const candidates = ["input", "question", "query", "prompt", "user"];
31524
+ for (const key of candidates) {
31525
+ const value = entry[key];
31526
+ if (typeof value === "string" && value.length > 0) {
31527
+ return value.length > 80 ? `${value.slice(0, 77)}...` : value;
31528
+ }
31529
+ }
31530
+ return "";
31531
+ };
31532
+ isFailedEvaluation = (e) => e.status === "error" || e.passed === false;
31533
+ isFailedRow = ({
31534
+ entry,
31535
+ evaluations
31536
+ }) => Boolean(entry.error) || evaluations.some(isFailedEvaluation);
31537
+ }
31538
+ });
31539
+
31540
+ // src/tools/list-experiment-runs.ts
31541
+ async function handleExperimentListRuns(params) {
31542
+ const requested = typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT;
31543
+ const effectiveLimit = Math.min(requested, MAX_LIMIT);
31544
+ const search = new URLSearchParams();
31545
+ search.set("experimentSlug", params.experimentSlug);
31546
+ search.set("pageSize", String(effectiveLimit));
31547
+ let result;
31548
+ try {
31549
+ result = await makeRequest(
31550
+ "GET",
31551
+ `/api/experiments/runs?${search.toString()}`
31552
+ );
31553
+ } catch (error48) {
31554
+ const message = error48 instanceof Error ? error48.message : String(error48);
31555
+ if (/404|not found/i.test(message)) {
31556
+ return [
31557
+ `# Evaluation Runs: ${params.experimentSlug}`,
31558
+ "",
31559
+ "**Status**: experiment not found",
31560
+ "",
31561
+ `No experiment with slug \`${params.experimentSlug}\` exists in this project.`,
31562
+ "",
31563
+ "> Run `platform_experiment_list` to discover available experiment slugs."
31564
+ ].join("\n");
31565
+ }
31566
+ throw error48;
31567
+ }
31568
+ if (result.runs.length === 0) {
31569
+ return [
31570
+ `# Evaluation Runs: ${result.experimentSlug}`,
31571
+ "",
31572
+ "_This experiment has no runs yet._",
31573
+ "",
31574
+ `> Trigger a run with \`platform_run_experiment\` (slug: \`${result.experimentSlug}\`) to populate this list.`
31575
+ ].join("\n");
31576
+ }
31577
+ const lines = [];
31578
+ lines.push(`# Evaluation Runs: ${result.experimentSlug}`);
31579
+ lines.push("");
31580
+ lines.push(
31581
+ `Showing ${result.runs.length} of ${result.pagination.totalHits} run${result.pagination.totalHits === 1 ? "" : "s"}.`
31582
+ );
31583
+ if (result.pagination.hasMore) {
31584
+ lines.push(
31585
+ `> More runs exist beyond this limit. Re-run with a higher \`limit\` (max ${MAX_LIMIT}) if needed.`
31586
+ );
31587
+ }
31588
+ lines.push("");
31589
+ lines.push("| Run ID | Status | Started | Finished | Result |");
31590
+ lines.push("|--------|--------|---------|----------|--------|");
31591
+ for (const run of result.runs) {
31592
+ lines.push(
31593
+ `| \`${run.runId}\` | ${runStatus(run)} | ${formatEpoch(run.timestamps.createdAt)} | ${formatEpoch(run.timestamps.finishedAt ?? null)} | ${passRate(run.summary?.evaluations ?? {})} |`
31594
+ );
31595
+ }
31596
+ lines.push("");
31597
+ lines.push(
31598
+ "> Use `platform_experiment_status <runId>` for live progress, or `platform_experiment_results <runId>` for per-row results."
31599
+ );
31600
+ return lines.join("\n");
31601
+ }
31602
+ var DEFAULT_LIMIT, MAX_LIMIT, formatEpoch, passRate, runStatus;
31603
+ var init_list_experiment_runs = __esm({
31604
+ "src/tools/list-experiment-runs.ts"() {
31605
+ "use strict";
31606
+ init_langwatch_api();
31607
+ DEFAULT_LIMIT = 25;
31608
+ MAX_LIMIT = 100;
31609
+ formatEpoch = (epoch) => {
31610
+ if (!epoch || !Number.isFinite(epoch)) return "\u2014";
31611
+ return new Date(epoch).toISOString().replace("T", " ").slice(0, 19) + " UTC";
31612
+ };
31613
+ passRate = (evaluations) => {
31614
+ const entries = Object.values(evaluations ?? {});
31615
+ if (entries.length === 0) return "\u2014";
31616
+ const passed = entries.filter((e) => typeof e.averagePassed === "number");
31617
+ if (passed.length === 0) {
31618
+ const scored = entries.filter((e) => typeof e.averageScore === "number");
31619
+ if (scored.length === 0) return "\u2014";
31620
+ const avg2 = scored.reduce((sum, e) => sum + (e.averageScore ?? 0), 0) / scored.length;
31621
+ return `${avg2.toFixed(2)} avg`;
31622
+ }
31623
+ const avg = passed.reduce((sum, e) => sum + (e.averagePassed ?? 0), 0) / passed.length;
31624
+ return `${(avg * 100).toFixed(0)}% pass`;
31625
+ };
31626
+ runStatus = (run) => {
31627
+ if (run.timestamps.stoppedAt) return "stopped";
31628
+ if (run.timestamps.finishedAt) return "completed";
31629
+ return "running";
31630
+ };
31631
+ }
31632
+ });
31633
+
31634
+ // src/tools/list-experiments.ts
31635
+ async function handleExperimentList(params) {
31636
+ const requested = typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT2;
31637
+ const effectiveLimit = Math.min(requested, MAX_LIMIT2);
31638
+ const search = new URLSearchParams();
31639
+ search.set("pageSize", String(effectiveLimit));
31640
+ const result = await makeRequest(
31641
+ "GET",
31642
+ `/api/experiments?${search.toString()}`
31643
+ );
31644
+ if (result.experiments.length === 0) {
31645
+ return [
31646
+ "# Experiments",
31647
+ "",
31648
+ "_No experiments found in this project._",
31649
+ "",
31650
+ "> Create an experiment in the LangWatch dashboard, then re-run this tool to discover its slug."
31651
+ ].join("\n");
31652
+ }
31653
+ const lines = [];
31654
+ lines.push("# Experiments");
31655
+ lines.push("");
31656
+ lines.push(
31657
+ `Showing ${result.experiments.length} of ${result.pagination.totalHits} experiment${result.pagination.totalHits === 1 ? "" : "s"}.`
31658
+ );
31659
+ if (result.pagination.hasMore) {
31660
+ lines.push(
31661
+ `> More experiments exist beyond this limit. Re-run with a higher \`limit\` (max ${MAX_LIMIT2}) if needed.`
31662
+ );
31663
+ }
31664
+ lines.push("");
31665
+ lines.push("| Slug | Name | Runs | Last Run |");
31666
+ lines.push("|------|------|------|----------|");
31667
+ for (const exp of result.experiments) {
31668
+ const name = escapeMarkdownTableCell(exp.name ?? exp.slug);
31669
+ lines.push(
31670
+ `| ${formatInlineCode(exp.slug)} | ${name} | ${exp.runsCount} | ${formatTimestamp(exp.lastRunAt)} |`
31671
+ );
31672
+ }
31673
+ lines.push("");
31674
+ lines.push(
31675
+ "> Use `platform_experiment_list_runs` with one of these slugs to see its runs."
31676
+ );
31677
+ return lines.join("\n");
31678
+ }
31679
+ var DEFAULT_LIMIT2, MAX_LIMIT2, formatTimestamp, escapeMarkdownTableCell, formatInlineCode;
31680
+ var init_list_experiments = __esm({
31681
+ "src/tools/list-experiments.ts"() {
31682
+ "use strict";
31683
+ init_langwatch_api();
31684
+ DEFAULT_LIMIT2 = 25;
31685
+ MAX_LIMIT2 = 100;
31686
+ formatTimestamp = (iso) => {
31687
+ if (!iso) return "\u2014";
31688
+ const d = new Date(iso);
31689
+ if (Number.isNaN(d.getTime())) return iso;
31690
+ return d.toISOString().replace("T", " ").slice(0, 19) + " UTC";
31691
+ };
31692
+ escapeMarkdownTableCell = (value) => value.replace(/[\\|]/g, (c) => `\\${c}`).replace(/\n/g, " ");
31693
+ formatInlineCode = (value) => `\`${value.replace(/[\\`|]/g, (c) => `\\${c}`).replace(/\n/g, " ")}\``;
31694
+ }
31695
+ });
31696
+
31697
+ // src/tools/run-experiment.ts
31698
+ async function handleRunExperiment(params) {
31699
+ const result = await makeRequest(
31700
+ "POST",
31701
+ `/api/experiments/${encodeURIComponent(params.slug)}/run`
31702
+ );
31703
+ const lines = [];
31704
+ lines.push(`Evaluation started!
31705
+ `);
31706
+ lines.push(`**Run ID**: ${result.runId}`);
31707
+ lines.push(`**Status**: ${result.status}`);
31708
+ lines.push(`**Total cells**: ${result.total}`);
31709
+ if (result.runUrl) lines.push(`**View at**: ${result.runUrl}`);
31710
+ lines.push("");
31711
+ lines.push(
31712
+ `> Use \`platform_experiment_status\` with run ID "${result.runId}" to check progress.`
31713
+ );
31714
+ return lines.join("\n");
31715
+ }
31716
+ async function handleExperimentStatus(params) {
31717
+ const status = await makeRequest(
31718
+ "GET",
31719
+ `/api/experiments/runs/${encodeURIComponent(params.runId)}`
31720
+ );
31721
+ const lines = [];
31722
+ lines.push(`# Evaluation Run ${status.runId}
31723
+ `);
31724
+ lines.push(`**Status**: ${status.status}`);
31725
+ lines.push(`**Progress**: ${status.progress}/${status.total} cells`);
31726
+ if (status.startedAt) {
31727
+ lines.push(`**Started**: ${new Date(status.startedAt).toISOString()}`);
31728
+ }
31729
+ if (status.finishedAt) {
31730
+ lines.push(`**Finished**: ${new Date(status.finishedAt).toISOString()}`);
31731
+ }
31732
+ if (status.summary) {
31733
+ lines.push("\n## Summary\n");
31734
+ if (status.summary.completedCells !== void 0) {
31735
+ lines.push(`**Completed**: ${status.summary.completedCells}`);
31736
+ }
31737
+ if (status.summary.failedCells) {
31738
+ lines.push(`**Failed**: ${status.summary.failedCells}`);
31739
+ }
31740
+ if (status.summary.duration) {
31741
+ lines.push(
31742
+ `**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`
31743
+ );
31744
+ }
31745
+ if (status.summary.runUrl) {
31746
+ lines.push(`**View results**: ${status.summary.runUrl}`);
31747
+ }
31748
+ }
31749
+ return lines.join("\n");
31750
+ }
31751
+ var init_run_experiment = __esm({
31752
+ "src/tools/run-experiment.ts"() {
31753
+ "use strict";
31754
+ init_langwatch_api();
31755
+ }
31756
+ });
31757
+
31230
31758
  // src/tools/discover-scenario-schema.ts
31231
31759
  var discover_scenario_schema_exports = {};
31232
31760
  __export(discover_scenario_schema_exports, {
@@ -33185,122 +33713,6 @@ var init_discover_schema = __esm({
33185
33713
  }
33186
33714
  });
33187
33715
 
33188
- // src/langwatch-api.ts
33189
- async function makeRequest(method, path, body) {
33190
- const config2 = getConfig();
33191
- const url2 = config2.endpoint + path;
33192
- const headers = {
33193
- "X-Auth-Token": requireApiKey()
33194
- };
33195
- if (config2.projectId) {
33196
- headers["X-Project-Id"] = config2.projectId;
33197
- }
33198
- if (body !== void 0) {
33199
- headers["Content-Type"] = "application/json";
33200
- }
33201
- const response = await fetch(url2, {
33202
- method,
33203
- headers,
33204
- ...body !== void 0 ? { body: JSON.stringify(body) } : {}
33205
- });
33206
- if (!response.ok) {
33207
- const responseBody = await response.text();
33208
- throw new Error(
33209
- `LangWatch API error ${response.status}: ${responseBody}`
33210
- );
33211
- }
33212
- if (response.status === 204 || response.headers?.get("content-length") === "0") {
33213
- return null;
33214
- }
33215
- return response.json();
33216
- }
33217
- async function searchTraces(params) {
33218
- const { format: format3 = "digest", ...rest } = params;
33219
- return makeRequest("POST", "/api/traces/search", {
33220
- ...rest,
33221
- format: format3
33222
- });
33223
- }
33224
- async function getTraceById(traceId, format3 = "digest") {
33225
- return makeRequest(
33226
- "GET",
33227
- `/api/traces/${encodeURIComponent(traceId)}?format=${format3}`
33228
- );
33229
- }
33230
- async function getAnalyticsTimeseries(params) {
33231
- return makeRequest(
33232
- "POST",
33233
- "/api/analytics/timeseries",
33234
- params
33235
- );
33236
- }
33237
- async function listPrompts() {
33238
- return makeRequest("GET", "/api/prompts");
33239
- }
33240
- async function getPrompt(idOrHandle, options) {
33241
- const params = new URLSearchParams();
33242
- if (options?.version != null) params.set("version", String(options.version));
33243
- if (options?.tag) params.set("tag", options.tag);
33244
- const query = params.toString() ? `?${params}` : "";
33245
- return makeRequest(
33246
- "GET",
33247
- `/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
33248
- );
33249
- }
33250
- async function createPrompt(data) {
33251
- return makeRequest(
33252
- "POST",
33253
- "/api/prompts",
33254
- data
33255
- );
33256
- }
33257
- async function updatePrompt(idOrHandle, data) {
33258
- return makeRequest(
33259
- "PUT",
33260
- `/api/prompts/${encodeURIComponent(idOrHandle)}`,
33261
- data
33262
- );
33263
- }
33264
- async function assignPromptTag({
33265
- idOrHandle,
33266
- tag,
33267
- versionId
33268
- }) {
33269
- return makeRequest(
33270
- "PUT",
33271
- `/api/prompts/${encodeURIComponent(idOrHandle)}/tags/${encodeURIComponent(tag)}`,
33272
- { versionId }
33273
- );
33274
- }
33275
- async function listPromptTags() {
33276
- return makeRequest("GET", "/api/prompts/tags");
33277
- }
33278
- async function createPromptTag(name) {
33279
- return makeRequest("POST", "/api/prompts/tags", { name });
33280
- }
33281
- async function renamePromptTag({
33282
- tag,
33283
- name
33284
- }) {
33285
- return makeRequest(
33286
- "PUT",
33287
- `/api/prompts/tags/${encodeURIComponent(tag)}`,
33288
- { name }
33289
- );
33290
- }
33291
- async function deletePromptTag(tag) {
33292
- return makeRequest(
33293
- "DELETE",
33294
- `/api/prompts/tags/${encodeURIComponent(tag)}`
33295
- );
33296
- }
33297
- var init_langwatch_api = __esm({
33298
- "src/langwatch-api.ts"() {
33299
- "use strict";
33300
- init_config();
33301
- }
33302
- });
33303
-
33304
33716
  // src/utils/date-parsing.ts
33305
33717
  function parseRelativeDate(input) {
33306
33718
  if (input === "now") return Date.now();
@@ -35355,97 +35767,31 @@ __export(langwatch_api_secrets_exports, {
35355
35767
  async function listSecrets() {
35356
35768
  return makeRequest("GET", "/api/secrets");
35357
35769
  }
35358
- async function getSecret(id) {
35359
- return makeRequest(
35360
- "GET",
35361
- `/api/secrets/${encodeURIComponent(id)}`
35362
- );
35363
- }
35364
- async function createSecret(data) {
35365
- return makeRequest("POST", "/api/secrets", data);
35366
- }
35367
- async function updateSecret(params) {
35368
- const { id, value } = params;
35369
- return makeRequest(
35370
- "PUT",
35371
- `/api/secrets/${encodeURIComponent(id)}`,
35372
- { value }
35373
- );
35374
- }
35375
- async function deleteSecret(id) {
35376
- return makeRequest(
35377
- "DELETE",
35378
- `/api/secrets/${encodeURIComponent(id)}`
35379
- );
35380
- }
35381
- var init_langwatch_api_secrets = __esm({
35382
- "src/langwatch-api-secrets.ts"() {
35383
- "use strict";
35384
- init_langwatch_api();
35385
- }
35386
- });
35387
-
35388
- // src/tools/run-evaluation.ts
35389
- var run_evaluation_exports = {};
35390
- __export(run_evaluation_exports, {
35391
- handleEvaluationStatus: () => handleEvaluationStatus,
35392
- handleRunEvaluation: () => handleRunEvaluation
35393
- });
35394
- async function handleRunEvaluation(params) {
35395
- const result = await makeRequest(
35396
- "POST",
35397
- `/api/evaluations/v3/${encodeURIComponent(params.slug)}/run`
35398
- );
35399
- const lines = [];
35400
- lines.push(`Evaluation started!
35401
- `);
35402
- lines.push(`**Run ID**: ${result.runId}`);
35403
- lines.push(`**Status**: ${result.status}`);
35404
- lines.push(`**Total cells**: ${result.total}`);
35405
- if (result.runUrl) lines.push(`**View at**: ${result.runUrl}`);
35406
- lines.push("");
35407
- lines.push(
35408
- `> Use \`platform_evaluation_status\` with run ID "${result.runId}" to check progress.`
35409
- );
35410
- return lines.join("\n");
35411
- }
35412
- async function handleEvaluationStatus(params) {
35413
- const status = await makeRequest(
35414
- "GET",
35415
- `/api/evaluations/v3/runs/${encodeURIComponent(params.runId)}`
35416
- );
35417
- const lines = [];
35418
- lines.push(`# Evaluation Run ${status.runId}
35419
- `);
35420
- lines.push(`**Status**: ${status.status}`);
35421
- lines.push(`**Progress**: ${status.progress}/${status.total} cells`);
35422
- if (status.startedAt) {
35423
- lines.push(`**Started**: ${new Date(status.startedAt).toISOString()}`);
35424
- }
35425
- if (status.finishedAt) {
35426
- lines.push(`**Finished**: ${new Date(status.finishedAt).toISOString()}`);
35427
- }
35428
- if (status.summary) {
35429
- lines.push("\n## Summary\n");
35430
- if (status.summary.completedCells !== void 0) {
35431
- lines.push(`**Completed**: ${status.summary.completedCells}`);
35432
- }
35433
- if (status.summary.failedCells) {
35434
- lines.push(`**Failed**: ${status.summary.failedCells}`);
35435
- }
35436
- if (status.summary.duration) {
35437
- lines.push(
35438
- `**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`
35439
- );
35440
- }
35441
- if (status.summary.runUrl) {
35442
- lines.push(`**View results**: ${status.summary.runUrl}`);
35443
- }
35444
- }
35445
- return lines.join("\n");
35770
+ async function getSecret(id) {
35771
+ return makeRequest(
35772
+ "GET",
35773
+ `/api/secrets/${encodeURIComponent(id)}`
35774
+ );
35775
+ }
35776
+ async function createSecret(data) {
35777
+ return makeRequest("POST", "/api/secrets", data);
35778
+ }
35779
+ async function updateSecret(params) {
35780
+ const { id, value } = params;
35781
+ return makeRequest(
35782
+ "PUT",
35783
+ `/api/secrets/${encodeURIComponent(id)}`,
35784
+ { value }
35785
+ );
35446
35786
  }
35447
- var init_run_evaluation = __esm({
35448
- "src/tools/run-evaluation.ts"() {
35787
+ async function deleteSecret(id) {
35788
+ return makeRequest(
35789
+ "DELETE",
35790
+ `/api/secrets/${encodeURIComponent(id)}`
35791
+ );
35792
+ }
35793
+ var init_langwatch_api_secrets = __esm({
35794
+ "src/langwatch-api-secrets.ts"() {
35449
35795
  "use strict";
35450
35796
  init_langwatch_api();
35451
35797
  }
@@ -35826,6 +36172,305 @@ var init_delete_dataset_records = __esm({
35826
36172
  }
35827
36173
  });
35828
36174
 
36175
+ // src/langwatch-api-projects.ts
36176
+ async function listProjects(params) {
36177
+ const query = new URLSearchParams();
36178
+ if (params?.page) query.set("page", String(params.page));
36179
+ if (params?.limit) query.set("limit", String(params.limit));
36180
+ const qs = query.toString() ? `?${query}` : "";
36181
+ return makeRequest("GET", `/api/projects${qs}`);
36182
+ }
36183
+ async function getProject(id) {
36184
+ return makeRequest(
36185
+ "GET",
36186
+ `/api/projects/${encodeURIComponent(id)}`
36187
+ );
36188
+ }
36189
+ async function createProject(data) {
36190
+ return makeRequest("POST", "/api/projects", data);
36191
+ }
36192
+ async function updateProject(params) {
36193
+ const { id, ...data } = params;
36194
+ return makeRequest(
36195
+ "PATCH",
36196
+ `/api/projects/${encodeURIComponent(id)}`,
36197
+ data
36198
+ );
36199
+ }
36200
+ async function archiveProject(id) {
36201
+ return makeRequest(
36202
+ "DELETE",
36203
+ `/api/projects/${encodeURIComponent(id)}`
36204
+ );
36205
+ }
36206
+ var init_langwatch_api_projects = __esm({
36207
+ "src/langwatch-api-projects.ts"() {
36208
+ "use strict";
36209
+ init_langwatch_api();
36210
+ }
36211
+ });
36212
+
36213
+ // src/tools/list-projects.ts
36214
+ var list_projects_exports = {};
36215
+ __export(list_projects_exports, {
36216
+ handleListProjects: () => handleListProjects
36217
+ });
36218
+ async function handleListProjects(params) {
36219
+ const result = await listProjects({
36220
+ page: params?.page,
36221
+ limit: params?.limit ?? 100
36222
+ });
36223
+ const projects = result.data;
36224
+ if (!Array.isArray(projects) || projects.length === 0) {
36225
+ return "No projects found in this organization.\n\n> Tip: Use `platform_create_project` to create your first project.";
36226
+ }
36227
+ const lines = [];
36228
+ lines.push(`# Projects (${result.pagination.total} total)
36229
+ `);
36230
+ for (const p of projects) {
36231
+ lines.push(`## ${p.name}`);
36232
+ lines.push(`**ID**: ${p.id}`);
36233
+ lines.push(`**Slug**: ${p.slug}`);
36234
+ lines.push(`**Language**: ${p.language}`);
36235
+ lines.push(`**Framework**: ${p.framework}`);
36236
+ lines.push(`**Updated**: ${p.updatedAt}`);
36237
+ lines.push("");
36238
+ }
36239
+ if (result.pagination.totalPages > 1) {
36240
+ lines.push(
36241
+ `> Showing page ${result.pagination.page} of ${result.pagination.totalPages}. Use \`page\` parameter to navigate.`
36242
+ );
36243
+ }
36244
+ lines.push(
36245
+ "> Use `platform_get_project` with the ID to see full project details."
36246
+ );
36247
+ return lines.join("\n");
36248
+ }
36249
+ var init_list_projects = __esm({
36250
+ "src/tools/list-projects.ts"() {
36251
+ "use strict";
36252
+ init_langwatch_api_projects();
36253
+ }
36254
+ });
36255
+
36256
+ // src/tools/get-project.ts
36257
+ var get_project_exports = {};
36258
+ __export(get_project_exports, {
36259
+ handleGetProject: () => handleGetProject
36260
+ });
36261
+ async function handleGetProject(params) {
36262
+ const project = await getProject(params.id);
36263
+ const lines = [];
36264
+ lines.push(`# ${project.name}
36265
+ `);
36266
+ lines.push(`**ID**: ${project.id}`);
36267
+ lines.push(`**Slug**: ${project.slug}`);
36268
+ lines.push(`**Language**: ${project.language}`);
36269
+ lines.push(`**Framework**: ${project.framework}`);
36270
+ lines.push(`**Team ID**: ${project.teamId}`);
36271
+ lines.push(`**PII Redaction**: ${project.piiRedactionLevel}`);
36272
+ lines.push(`**Created**: ${project.createdAt}`);
36273
+ lines.push(`**Updated**: ${project.updatedAt}`);
36274
+ return lines.join("\n");
36275
+ }
36276
+ var init_get_project = __esm({
36277
+ "src/tools/get-project.ts"() {
36278
+ "use strict";
36279
+ init_langwatch_api_projects();
36280
+ }
36281
+ });
36282
+
36283
+ // src/tools/create-project.ts
36284
+ var create_project_exports = {};
36285
+ __export(create_project_exports, {
36286
+ handleCreateProject: () => handleCreateProject
36287
+ });
36288
+ async function handleCreateProject(params) {
36289
+ if (!params.teamId && !params.newTeamName) {
36290
+ return "Error: You must provide either `teamId` (to add to an existing team) or `newTeamName` (to create a new team for this project).";
36291
+ }
36292
+ const result = await createProject({
36293
+ name: params.name,
36294
+ language: params.language,
36295
+ framework: params.framework,
36296
+ teamId: params.teamId,
36297
+ newTeamName: params.newTeamName
36298
+ });
36299
+ if (!result.serviceApiKey) {
36300
+ throw new Error(
36301
+ "API did not return a service API key. The project was created but the key is missing \u2014 check the LangWatch UI."
36302
+ );
36303
+ }
36304
+ const lines = [];
36305
+ lines.push(`Project created successfully!
36306
+ `);
36307
+ lines.push(`**Name**: ${result.name}`);
36308
+ lines.push(`**ID**: ${result.id}`);
36309
+ lines.push(`**Slug**: ${result.slug}`);
36310
+ lines.push(`**Language**: ${result.language}`);
36311
+ lines.push(`**Framework**: ${result.framework}`);
36312
+ lines.push("");
36313
+ lines.push(`**Service API Key**: \`${result.serviceApiKey}\``);
36314
+ lines.push(`**Service API Key ID**: ${result.serviceApiKeyId}`);
36315
+ lines.push("");
36316
+ lines.push(
36317
+ "> \u26A0\uFE0F Save the service API key now \u2014 it will not be shown again. Use it as `LANGWATCH_API_KEY` to authenticate project-scoped operations."
36318
+ );
36319
+ return lines.join("\n");
36320
+ }
36321
+ var init_create_project = __esm({
36322
+ "src/tools/create-project.ts"() {
36323
+ "use strict";
36324
+ init_langwatch_api_projects();
36325
+ }
36326
+ });
36327
+
36328
+ // src/tools/update-project.ts
36329
+ var update_project_exports = {};
36330
+ __export(update_project_exports, {
36331
+ handleUpdateProject: () => handleUpdateProject
36332
+ });
36333
+ async function handleUpdateProject(params) {
36334
+ const project = await updateProject(params);
36335
+ const lines = [];
36336
+ lines.push(`Project updated successfully!
36337
+ `);
36338
+ lines.push(`**Name**: ${project.name}`);
36339
+ lines.push(`**ID**: ${project.id}`);
36340
+ lines.push(`**Slug**: ${project.slug}`);
36341
+ lines.push(`**Language**: ${project.language}`);
36342
+ lines.push(`**Framework**: ${project.framework}`);
36343
+ lines.push(`**PII Redaction**: ${project.piiRedactionLevel}`);
36344
+ return lines.join("\n");
36345
+ }
36346
+ var init_update_project = __esm({
36347
+ "src/tools/update-project.ts"() {
36348
+ "use strict";
36349
+ init_langwatch_api_projects();
36350
+ }
36351
+ });
36352
+
36353
+ // src/tools/archive-project.ts
36354
+ var archive_project_exports = {};
36355
+ __export(archive_project_exports, {
36356
+ handleArchiveProject: () => handleArchiveProject
36357
+ });
36358
+ async function handleArchiveProject(params) {
36359
+ const result = await archiveProject(params.id);
36360
+ return `Project archived successfully.
36361
+
36362
+ **Name**: ${result.name}
36363
+ **ID**: ${result.id}
36364
+ **Archived At**: ${result.archivedAt}`;
36365
+ }
36366
+ var init_archive_project = __esm({
36367
+ "src/tools/archive-project.ts"() {
36368
+ "use strict";
36369
+ init_langwatch_api_projects();
36370
+ }
36371
+ });
36372
+
36373
+ // src/langwatch-api-api-keys.ts
36374
+ async function listApiKeys() {
36375
+ return makeRequest("GET", "/api/api-keys");
36376
+ }
36377
+ async function createApiKey(data) {
36378
+ return makeRequest("POST", "/api/api-keys", data);
36379
+ }
36380
+ async function revokeApiKey(id) {
36381
+ return makeRequest(
36382
+ "DELETE",
36383
+ `/api/api-keys/${encodeURIComponent(id)}`
36384
+ );
36385
+ }
36386
+ var init_langwatch_api_api_keys = __esm({
36387
+ "src/langwatch-api-api-keys.ts"() {
36388
+ "use strict";
36389
+ init_langwatch_api();
36390
+ }
36391
+ });
36392
+
36393
+ // src/tools/list-api-keys.ts
36394
+ var list_api_keys_exports = {};
36395
+ __export(list_api_keys_exports, {
36396
+ handleListApiKeys: () => handleListApiKeys
36397
+ });
36398
+ async function handleListApiKeys() {
36399
+ const result = await listApiKeys();
36400
+ const keys = result.data;
36401
+ if (!Array.isArray(keys) || keys.length === 0) {
36402
+ return "No API keys found.\n\n> Tip: Use `platform_create_api_key` to create a new key.";
36403
+ }
36404
+ const lines = [];
36405
+ lines.push(`# API Keys (${keys.length} total)
36406
+ `);
36407
+ for (const k of keys) {
36408
+ const status = k.revokedAt ? "REVOKED" : k.expiresAt && new Date(k.expiresAt) < /* @__PURE__ */ new Date() ? "EXPIRED" : "ACTIVE";
36409
+ lines.push(`## ${k.name}`);
36410
+ lines.push(`**ID**: ${k.id}`);
36411
+ lines.push(`**Status**: ${status}`);
36412
+ if (k.description) lines.push(`**Description**: ${k.description}`);
36413
+ if (k.expiresAt) lines.push(`**Expires**: ${k.expiresAt}`);
36414
+ if (k.lastUsedAt) lines.push(`**Last Used**: ${k.lastUsedAt}`);
36415
+ lines.push(`**Created**: ${k.createdAt}`);
36416
+ if (k.roleBindings.length > 0) {
36417
+ const bindings = k.roleBindings.map((rb) => `${rb.role} on ${rb.scopeType}:${rb.scopeId}`).join(", ");
36418
+ lines.push(`**Bindings**: ${bindings}`);
36419
+ }
36420
+ lines.push("");
36421
+ }
36422
+ return lines.join("\n");
36423
+ }
36424
+ var init_list_api_keys = __esm({
36425
+ "src/tools/list-api-keys.ts"() {
36426
+ "use strict";
36427
+ init_langwatch_api_api_keys();
36428
+ }
36429
+ });
36430
+
36431
+ // src/tools/create-api-key.ts
36432
+ var create_api_key_exports = {};
36433
+ __export(create_api_key_exports, {
36434
+ handleCreateApiKey: () => handleCreateApiKey
36435
+ });
36436
+ async function handleCreateApiKey(params) {
36437
+ const result = await createApiKey(params);
36438
+ const lines = [];
36439
+ lines.push(`API key created successfully!
36440
+ `);
36441
+ lines.push(`**Name**: ${result.apiKey.name}`);
36442
+ lines.push(`**ID**: ${result.apiKey.id}`);
36443
+ lines.push(`**Token**: \`${result.token}\``);
36444
+ lines.push(`**Created**: ${result.apiKey.createdAt}`);
36445
+ lines.push("");
36446
+ lines.push(
36447
+ "> \u26A0\uFE0F Save this token now \u2014 it will not be shown again."
36448
+ );
36449
+ return lines.join("\n");
36450
+ }
36451
+ var init_create_api_key = __esm({
36452
+ "src/tools/create-api-key.ts"() {
36453
+ "use strict";
36454
+ init_langwatch_api_api_keys();
36455
+ }
36456
+ });
36457
+
36458
+ // src/tools/revoke-api-key.ts
36459
+ var revoke_api_key_exports = {};
36460
+ __export(revoke_api_key_exports, {
36461
+ handleRevokeApiKey: () => handleRevokeApiKey
36462
+ });
36463
+ async function handleRevokeApiKey(params) {
36464
+ await revokeApiKey(params.id);
36465
+ return `API key ${params.id} revoked successfully.`;
36466
+ }
36467
+ var init_revoke_api_key = __esm({
36468
+ "src/tools/revoke-api-key.ts"() {
36469
+ "use strict";
36470
+ init_langwatch_api_api_keys();
36471
+ }
36472
+ });
36473
+
35829
36474
  // src/create-mcp-server.ts
35830
36475
  function createMcpServer() {
35831
36476
  const server = new McpServer({
@@ -37087,30 +37732,86 @@ ${lines.join("\n")}`
37087
37732
  })
37088
37733
  );
37089
37734
  server.tool(
37090
- "platform_run_evaluation",
37091
- "Start an evaluation run by slug. Returns a run ID for polling status.",
37735
+ "platform_run_experiment",
37736
+ "Start an experiment run by slug. Returns a run ID for polling status.",
37737
+ {
37738
+ slug: external_exports.string().describe("The experiment slug to run")
37739
+ },
37740
+ withToolLogging("platform_run_experiment", async (params) => {
37741
+ requireApiKey();
37742
+ return {
37743
+ content: [{ type: "text", text: await handleRunExperiment(params) }]
37744
+ };
37745
+ })
37746
+ );
37747
+ server.tool(
37748
+ "platform_experiment_status",
37749
+ "Check the status of an experiment run. Returns progress and summary when completed.",
37750
+ {
37751
+ runId: external_exports.string().describe("The run ID returned from platform_run_experiment")
37752
+ },
37753
+ withToolLogging("platform_experiment_status", async (params) => {
37754
+ requireApiKey();
37755
+ return {
37756
+ content: [{ type: "text", text: await handleExperimentStatus(params) }]
37757
+ };
37758
+ })
37759
+ );
37760
+ server.tool(
37761
+ "platform_experiment_list",
37762
+ "List experiments configured in the LangWatch project. Returns a markdown table with each experiment's slug, name, last-run timestamp, and run count. Use this to discover experiment slugs before drilling into runs with platform_experiment_list_runs. Output is capped (default 25, max 100) to protect the agent's context window.",
37763
+ {
37764
+ limit: external_exports.number().int().positive().max(100).optional().describe(
37765
+ "Maximum number of experiments to include (default 25, hard-capped at 100)"
37766
+ )
37767
+ },
37768
+ withToolLogging("platform_experiment_list", async (params) => {
37769
+ requireApiKey();
37770
+ return {
37771
+ content: [{ type: "text", text: await handleExperimentList(params) }]
37772
+ };
37773
+ })
37774
+ );
37775
+ server.tool(
37776
+ "platform_experiment_list_runs",
37777
+ "List experiment runs for a specific experiment slug. Returns a markdown table with each run's id, status, started/finished timestamps, and pass-rate summary. Use this to discover run ids before calling platform_experiment_results. Output is capped (default 25, max 100) to protect the agent's context window.",
37092
37778
  {
37093
- slug: external_exports.string().describe("The evaluation slug to run")
37779
+ experimentSlug: external_exports.string().describe(
37780
+ "Experiment slug from platform_experiment_list (e.g. 'checkout-flow')"
37781
+ ),
37782
+ limit: external_exports.number().int().positive().max(100).optional().describe(
37783
+ "Maximum number of runs to include (default 25, hard-capped at 100)"
37784
+ )
37094
37785
  },
37095
- withToolLogging("platform_run_evaluation", async (params) => {
37786
+ withToolLogging("platform_experiment_list_runs", async (params) => {
37096
37787
  requireApiKey();
37097
- const { handleRunEvaluation: handleRunEvaluation2 } = await Promise.resolve().then(() => (init_run_evaluation(), run_evaluation_exports));
37098
37788
  return {
37099
- content: [{ type: "text", text: await handleRunEvaluation2(params) }]
37789
+ content: [{ type: "text", text: await handleExperimentListRuns(params) }]
37100
37790
  };
37101
37791
  })
37102
37792
  );
37103
37793
  server.tool(
37104
- "platform_evaluation_status",
37105
- "Check the status of an evaluation run. Returns progress and summary when completed.",
37794
+ "platform_experiment_results",
37795
+ "Fetch per-row results for a completed experiment run so you can debug evaluator scores and missed rows. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window \u2014 narrow with `filter: 'failed'` or `evaluator` to see what matters.",
37106
37796
  {
37107
- runId: external_exports.string().describe("The run ID returned from platform_run_evaluation")
37797
+ runId: external_exports.string().describe("The run ID returned from platform_run_experiment"),
37798
+ experimentSlug: external_exports.string().optional().describe(
37799
+ "Experiment slug \u2014 required when fetching results for runs older than 24h (Redis state may have expired). Use platform_experiment_list_runs to find the slug."
37800
+ ),
37801
+ filter: external_exports.enum(["all", "failed"]).optional().describe(
37802
+ "Filter rows: 'failed' shows only rows that errored or failed an evaluation; 'all' (default) shows everything"
37803
+ ),
37804
+ evaluator: external_exports.string().optional().describe("Show only this evaluator's results (matched on `evaluator` field)"),
37805
+ limit: external_exports.number().int().positive().max(50).optional().describe(
37806
+ "Maximum number of rows to include in the markdown output (default 50, hard-capped at 50)."
37807
+ )
37108
37808
  },
37109
- withToolLogging("platform_evaluation_status", async (params) => {
37809
+ withToolLogging("platform_experiment_results", async (params) => {
37110
37810
  requireApiKey();
37111
- const { handleEvaluationStatus: handleEvaluationStatus2 } = await Promise.resolve().then(() => (init_run_evaluation(), run_evaluation_exports));
37112
37811
  return {
37113
- content: [{ type: "text", text: await handleEvaluationStatus2(params) }]
37812
+ content: [
37813
+ { type: "text", text: await handleExperimentResults(params) }
37814
+ ]
37114
37815
  };
37115
37816
  })
37116
37817
  );
@@ -37269,6 +37970,141 @@ ${lines.join("\n")}`
37269
37970
  };
37270
37971
  }
37271
37972
  );
37973
+ server.tool(
37974
+ "platform_list_projects",
37975
+ "List all projects in your LangWatch organization. Requires an org-level API key.",
37976
+ {
37977
+ page: external_exports.number().int().positive().optional().describe("Page number (default: 1)"),
37978
+ limit: external_exports.number().int().positive().max(1e3).optional().describe("Results per page (default: 100)")
37979
+ },
37980
+ withToolLogging("platform_list_projects", async (params) => {
37981
+ requireApiKey();
37982
+ const { handleListProjects: handleListProjects2 } = await Promise.resolve().then(() => (init_list_projects(), list_projects_exports));
37983
+ return {
37984
+ content: [{ type: "text", text: await handleListProjects2(params) }]
37985
+ };
37986
+ })
37987
+ );
37988
+ server.tool(
37989
+ "platform_get_project",
37990
+ "Get details of a specific project by its ID.",
37991
+ {
37992
+ id: external_exports.string().describe("The project ID")
37993
+ },
37994
+ withToolLogging("platform_get_project", async (params) => {
37995
+ requireApiKey();
37996
+ const { handleGetProject: handleGetProject2 } = await Promise.resolve().then(() => (init_get_project(), get_project_exports));
37997
+ return {
37998
+ content: [{ type: "text", text: await handleGetProject2(params) }]
37999
+ };
38000
+ })
38001
+ );
38002
+ server.tool(
38003
+ "platform_create_project",
38004
+ `Create a new project in your LangWatch organization. Returns a one-time service API key for the project.
38005
+
38006
+ You must provide either teamId (to add the project to an existing team) or newTeamName (to create a new team for the project).`,
38007
+ {
38008
+ name: external_exports.string().describe("Project name"),
38009
+ language: external_exports.string().describe('Programming language (e.g. "python", "typescript")'),
38010
+ framework: external_exports.string().describe('Framework (e.g. "openai", "langchain", "custom")'),
38011
+ teamId: external_exports.string().optional().describe("ID of an existing team to add the project to"),
38012
+ newTeamName: external_exports.string().optional().describe("Name for a new team to create for this project")
38013
+ },
38014
+ withToolLogging("platform_create_project", async (params) => {
38015
+ requireApiKey();
38016
+ const { handleCreateProject: handleCreateProject2 } = await Promise.resolve().then(() => (init_create_project(), create_project_exports));
38017
+ return {
38018
+ content: [{ type: "text", text: await handleCreateProject2(params) }]
38019
+ };
38020
+ })
38021
+ );
38022
+ server.tool(
38023
+ "platform_update_project",
38024
+ "Update an existing project's name, language, framework, or PII redaction level.",
38025
+ {
38026
+ id: external_exports.string().describe("The project ID to update"),
38027
+ name: external_exports.string().optional().describe("New project name"),
38028
+ language: external_exports.string().optional().describe("New programming language"),
38029
+ framework: external_exports.string().optional().describe("New framework"),
38030
+ piiRedactionLevel: external_exports.enum(["STRICT", "ESSENTIAL", "DISABLED"]).optional().describe("PII redaction level")
38031
+ },
38032
+ withToolLogging("platform_update_project", async (params) => {
38033
+ requireApiKey();
38034
+ const { handleUpdateProject: handleUpdateProject2 } = await Promise.resolve().then(() => (init_update_project(), update_project_exports));
38035
+ return {
38036
+ content: [{ type: "text", text: await handleUpdateProject2(params) }]
38037
+ };
38038
+ })
38039
+ );
38040
+ server.tool(
38041
+ "platform_archive_project",
38042
+ "Archive (soft-delete) a project. The project can be restored later if needed.",
38043
+ {
38044
+ id: external_exports.string().describe("The project ID to archive")
38045
+ },
38046
+ withToolLogging("platform_archive_project", async (params) => {
38047
+ requireApiKey();
38048
+ const { handleArchiveProject: handleArchiveProject2 } = await Promise.resolve().then(() => (init_archive_project(), archive_project_exports));
38049
+ return {
38050
+ content: [{ type: "text", text: await handleArchiveProject2(params) }]
38051
+ };
38052
+ })
38053
+ );
38054
+ server.tool(
38055
+ "platform_list_api_keys",
38056
+ "List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
38057
+ {},
38058
+ withToolLogging("platform_list_api_keys", async () => {
38059
+ requireApiKey();
38060
+ const { handleListApiKeys: handleListApiKeys2 } = await Promise.resolve().then(() => (init_list_api_keys(), list_api_keys_exports));
38061
+ return {
38062
+ content: [{ type: "text", text: await handleListApiKeys2() }]
38063
+ };
38064
+ })
38065
+ );
38066
+ server.tool(
38067
+ "platform_create_api_key",
38068
+ `Create a new API key. Two types available:
38069
+
38070
+ - **personal**: Tied to a user, requires explicit role bindings (e.g. ADMIN on PROJECT:proj_123).
38071
+ - **service**: Not tied to a user. Use projectIds for project-scoped service keys, or omit for org-wide access.
38072
+
38073
+ The token is returned once and cannot be retrieved again.`,
38074
+ {
38075
+ keyType: external_exports.enum(["personal", "service"]).describe("Key type: 'personal' (user-tied) or 'service' (headless)"),
38076
+ name: external_exports.string().describe("Key name (max 100 chars)"),
38077
+ description: external_exports.string().optional().describe("Key description (max 500 chars)"),
38078
+ expiresAt: external_exports.string().optional().describe("Expiration date (ISO string)"),
38079
+ bindings: external_exports.array(external_exports.object({
38080
+ role: external_exports.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
38081
+ scopeType: external_exports.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
38082
+ scopeId: external_exports.string().describe("Scope ID (org, team, or project ID)")
38083
+ })).optional().describe("Role bindings (required for personal keys)"),
38084
+ projectIds: external_exports.array(external_exports.string()).optional().describe("Project IDs for service keys (creates ADMIN bindings per project)")
38085
+ },
38086
+ withToolLogging("platform_create_api_key", async (params) => {
38087
+ requireApiKey();
38088
+ const { handleCreateApiKey: handleCreateApiKey2 } = await Promise.resolve().then(() => (init_create_api_key(), create_api_key_exports));
38089
+ return {
38090
+ content: [{ type: "text", text: await handleCreateApiKey2(params) }]
38091
+ };
38092
+ })
38093
+ );
38094
+ server.tool(
38095
+ "platform_revoke_api_key",
38096
+ "Revoke an API key. Once revoked, the key can no longer be used for authentication.",
38097
+ {
38098
+ id: external_exports.string().describe("The API key ID to revoke")
38099
+ },
38100
+ withToolLogging("platform_revoke_api_key", async (params) => {
38101
+ requireApiKey();
38102
+ const { handleRevokeApiKey: handleRevokeApiKey2 } = await Promise.resolve().then(() => (init_revoke_api_key(), revoke_api_key_exports));
38103
+ return {
38104
+ content: [{ type: "text", text: await handleRevokeApiKey2(params) }]
38105
+ };
38106
+ })
38107
+ );
37272
38108
  }
37273
38109
  var modelSchema;
37274
38110
  var init_create_mcp_server = __esm({
@@ -37279,6 +38115,10 @@ var init_create_mcp_server = __esm({
37279
38115
  init_package();
37280
38116
  init_config();
37281
38117
  init_create_dataset();
38118
+ init_get_experiment_results();
38119
+ init_list_experiment_runs();
38120
+ init_list_experiments();
38121
+ init_run_experiment();
37282
38122
  modelSchema = external_exports.string().describe(
37283
38123
  'Model in "provider/model-name" format, e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929"'
37284
38124
  );