@langwatch/mcp-server 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/LICENSE +7 -0
- package/dist/archive-project-HZBRPR2S.js +19 -0
- package/dist/{archive-scenario-YKGLJ4MZ.js → archive-scenario-P5RRITRM.js} +3 -3
- package/dist/{archive-suite-QFNDO3Y3.js → archive-suite-NUUFD2EM.js} +3 -3
- package/dist/{assign-prompt-tag-Y2B4CBBT.js → assign-prompt-tag-FKCCZPML.js} +2 -2
- package/dist/{chunk-YH2IBTEL.js → chunk-42ZHHHIJ.js} +14 -3
- package/dist/chunk-ADSJ2FDA.js +44 -0
- package/dist/{chunk-L6QENXYA.js → chunk-BB7EOKHR.js} +2 -2
- package/dist/chunk-BBJ5WAJB.js +24 -0
- package/dist/{chunk-PZUWCWOS.js → chunk-BYWYM2RP.js} +2 -2
- package/dist/{chunk-NACSR6B7.js → chunk-CCM4ZFGF.js} +2 -2
- package/dist/{chunk-FBRKHHJV.js → chunk-CHKCEZBP.js} +2 -2
- package/dist/{chunk-PO7FM7IX.js → chunk-HKY3GW45.js} +2 -2
- package/dist/{chunk-KZNBCKWG.js → chunk-JHOSVOHT.js} +2 -2
- package/dist/{chunk-B57MZ4WI.js → chunk-JVLJKQJG.js} +657 -88
- package/dist/{chunk-HDJDFKZR.js → chunk-R7C7OUVG.js} +2 -2
- package/dist/{chunk-OVQF3D5R.js → chunk-TROIWSEQ.js} +2 -2
- package/dist/{chunk-CPUPGDNA.js → chunk-X2SEURT2.js} +2 -2
- package/dist/{chunk-YI3OG4PI.js → chunk-XC62KJIT.js} +3 -3
- package/dist/{create-agent-YRSDH7P7.js → create-agent-K5Q7BFGZ.js} +3 -3
- package/dist/{create-annotation-FYQNKMVJ.js → create-annotation-KN3GE3DK.js} +3 -3
- package/dist/create-api-key-OQI5PUNQ.js +26 -0
- package/dist/{create-dashboard-CORBR4YQ.js → create-dashboard-G4QA5AXP.js} +3 -3
- package/dist/{create-dataset-ZC63T6BB.js → create-dataset-WQSC37N7.js} +3 -3
- package/dist/{create-dataset-records-ILVNVKVR.js → create-dataset-records-B4RL3FLC.js} +3 -3
- package/dist/{create-evaluator-2AU3PXFB.js → create-evaluator-YFFQIR7E.js} +3 -3
- package/dist/create-mcp-server.js +2 -1
- package/dist/create-project-CUVQ67VN.js +44 -0
- package/dist/{create-prompt-NKRJ47YY.js → create-prompt-NGXKSP4X.js} +2 -2
- package/dist/{create-prompt-tag-S4JX3ULL.js → create-prompt-tag-IUKYPVHH.js} +2 -2
- package/dist/{create-scenario-BPRF75KF.js → create-scenario-FNBXALH3.js} +3 -3
- package/dist/{create-suite-4YN7D6K2.js → create-suite-DJEPD4ZB.js} +3 -3
- package/dist/{delete-agent-Y72R2YQA.js → delete-agent-TJI5XBBM.js} +3 -3
- package/dist/{delete-annotation-UHWDMHSH.js → delete-annotation-KRP53TUU.js} +3 -3
- package/dist/{delete-dashboard-LPWT36KN.js → delete-dashboard-N5SZJIR3.js} +3 -3
- package/dist/{delete-dataset-GGPQRHKX.js → delete-dataset-KDUOHL2P.js} +3 -3
- package/dist/{delete-dataset-records-LTG5LT26.js → delete-dataset-records-DM4W3PAW.js} +3 -3
- package/dist/{delete-evaluator-F7QLXDHU.js → delete-evaluator-7ZATLJOA.js} +3 -3
- package/dist/{delete-prompt-tag-B5UKZL45.js → delete-prompt-tag-I6HNMSVJ.js} +2 -2
- package/dist/{delete-workflow-4DGUDIDE.js → delete-workflow-QEO74JQR.js} +3 -3
- package/dist/{get-agent-YEVICUX4.js → get-agent-3KDWCP5N.js} +3 -3
- package/dist/{get-analytics-3W5CMU3F.js → get-analytics-GPVJX7SF.js} +2 -2
- package/dist/{get-dashboard-NCQHN6MW.js → get-dashboard-YPTIE5YY.js} +3 -3
- package/dist/{get-dataset-KBK6PH36.js → get-dataset-6DQFYITM.js} +3 -3
- package/dist/{get-evaluator-QDUL5LXR.js → get-evaluator-PCM6XQAU.js} +3 -3
- package/dist/get-project-JCMWXVLU.js +26 -0
- package/dist/{get-prompt-XCSSNRKR.js → get-prompt-5XAGWWBG.js} +2 -2
- package/dist/{get-scenario-FNRKOY7E.js → get-scenario-7XBCDZWX.js} +3 -3
- package/dist/{get-simulation-run-WOEKUF7K.js → get-simulation-run-AV24FJWI.js} +3 -3
- package/dist/{get-suite-U5AOD5TC.js → get-suite-4DBVHFEM.js} +3 -3
- package/dist/{get-trace-MUWEJM2Z.js → get-trace-MRBDS3HJ.js} +2 -2
- package/dist/{get-workflow-SOA7MT7L.js → get-workflow-UR4Y6UFX.js} +3 -3
- package/dist/{http-server-7DMOCB75.js → http-server-DEGNV5EP.js} +3 -2
- package/dist/index.js +1060 -220
- package/dist/{langwatch-api-annotations-Q57NDUVR.js → langwatch-api-annotations-ZJDRB4JH.js} +3 -3
- package/dist/{langwatch-api-dashboards-VBWWXV3Y.js → langwatch-api-dashboards-WARLI6NF.js} +3 -3
- package/dist/{langwatch-api-monitors-L3IY7FDT.js → langwatch-api-monitors-4XKXHYCG.js} +2 -2
- package/dist/{langwatch-api-secrets-S5HHF3XR.js → langwatch-api-secrets-6SHK5G7Q.js} +2 -2
- package/dist/{langwatch-api-triggers-ME33QE26.js → langwatch-api-triggers-5SX6R2WU.js} +2 -2
- package/dist/{langwatch-api-workflows-4CJQAYP2.js → langwatch-api-workflows-NLJENNXE.js} +3 -3
- package/dist/{list-agents-3HGIBJFM.js → list-agents-NECO7MPI.js} +3 -3
- package/dist/{list-annotations-KWXYVEPI.js → list-annotations-3OJEO774.js} +3 -3
- package/dist/list-api-keys-AWW7ROLB.js +37 -0
- package/dist/{list-dashboards-VQ2LH666.js → list-dashboards-DVPVVVKW.js} +3 -3
- package/dist/{list-dataset-records-RNRGROJP.js → list-dataset-records-I4BYORF5.js} +3 -3
- package/dist/{list-datasets-I4A2LT2S.js → list-datasets-WUAE5HGX.js} +3 -3
- package/dist/{list-evaluators-AKKRJ5MX.js → list-evaluators-B6OSVAM2.js} +3 -3
- package/dist/{list-model-providers-DVFEHFHZ.js → list-model-providers-NL6IP4UZ.js} +3 -3
- package/dist/list-projects-5FHMJ4PQ.js +42 -0
- package/dist/{list-prompt-tags-PL5OISUA.js → list-prompt-tags-Z2DD6FLQ.js} +2 -2
- package/dist/{list-prompts-7VTFWE2Z.js → list-prompts-VNXKHK7T.js} +2 -2
- package/dist/{list-scenarios-5BZWZ6ZN.js → list-scenarios-CFZHXL76.js} +3 -3
- package/dist/{list-simulation-runs-WCZ6RLPE.js → list-simulation-runs-AM2BXLBW.js} +3 -3
- package/dist/{list-suites-RRQ2JX5Y.js → list-suites-ZDOHAAXF.js} +3 -3
- package/dist/{list-workflows-UQB5FCX5.js → list-workflows-WBRBKLIT.js} +3 -3
- package/dist/{rename-prompt-tag-TURD6KYU.js → rename-prompt-tag-A4RJUK7Z.js} +2 -2
- package/dist/revoke-api-key-ZSPSPF5G.js +15 -0
- package/dist/{run-agent-BEJ4K4CM.js → run-agent-JFNDY7MR.js} +3 -3
- package/dist/{run-suite-U7V432G2.js → run-suite-TH7LO5TJ.js} +3 -3
- package/dist/{run-workflow-5WX7DK5T.js → run-workflow-CK3JP5U2.js} +3 -3
- package/dist/{search-traces-ZCHNOMTX.js → search-traces-3BBVYPC3.js} +2 -2
- package/dist/{set-model-provider-6CFHMCNE.js → set-model-provider-WOC5FNMV.js} +3 -3
- package/dist/{update-agent-UE6UWLZP.js → update-agent-YMEV5MTL.js} +3 -3
- package/dist/{update-dataset-4XQZZE4S.js → update-dataset-EYT5POFM.js} +3 -3
- package/dist/{update-dataset-record-GNNM5TKM.js → update-dataset-record-VFF5JTX4.js} +3 -3
- package/dist/{update-evaluator-LHVDXIZD.js → update-evaluator-6IJHFG2E.js} +3 -3
- package/dist/update-project-OIT4WPFI.js +24 -0
- package/dist/{update-prompt-3XEHECJ7.js → update-prompt-ZU2DD6KM.js} +2 -2
- package/dist/{update-scenario-DFPREYCD.js → update-scenario-3VUTPDRU.js} +3 -3
- package/dist/{update-suite-RATU5THO.js → update-suite-C6JFDQHL.js} +3 -3
- package/package.json +2 -2
- package/pnpm-workspace.yaml +2 -0
- package/pyproject.toml +3 -0
- package/src/__tests__/experiment-list-tools.unit.test.ts +157 -0
- package/src/__tests__/get-experiment-results.unit.test.ts +222 -0
- package/src/__tests__/project-api-key-tools.unit.test.ts +451 -0
- package/src/create-mcp-server.ts +257 -13
- package/src/langwatch-api-api-keys.ts +54 -0
- package/src/langwatch-api-projects.ts +82 -0
- package/src/langwatch-api.ts +17 -3
- package/src/tools/archive-project.ts +14 -0
- package/src/tools/create-api-key.ts +29 -0
- package/src/tools/create-project.ts +45 -0
- package/src/tools/get-experiment-results.ts +273 -0
- package/src/tools/get-project.ts +20 -0
- package/src/tools/list-api-keys.ts +33 -0
- package/src/tools/list-experiment-runs.ts +142 -0
- package/src/tools/list-experiments.ts +95 -0
- package/src/tools/list-projects.ts +41 -0
- package/src/tools/revoke-api-key.ts +9 -0
- package/src/tools/{run-evaluation.ts → run-experiment.ts} +5 -5
- package/src/tools/update-project.ts +22 -0
- package/uv.lock +7 -3
- package/dist/run-evaluation-AJ7WZNQD.js +0 -64
package/dist/index.js
CHANGED
|
@@ -31109,7 +31109,7 @@ var init_package = __esm({
|
|
|
31109
31109
|
"package.json"() {
|
|
31110
31110
|
package_default = {
|
|
31111
31111
|
name: "@langwatch/mcp-server",
|
|
31112
|
-
version: "0.
|
|
31112
|
+
version: "0.9.0",
|
|
31113
31113
|
description: "An MCP server for Langwatch.",
|
|
31114
31114
|
type: "module",
|
|
31115
31115
|
main: "./dist/index.js",
|
|
@@ -31160,7 +31160,7 @@ var init_package = __esm({
|
|
|
31160
31160
|
"@types/node": "^25.6.0",
|
|
31161
31161
|
"@types/yargs": "^17.0.33",
|
|
31162
31162
|
"@typescript/native-preview": "7.0.0-dev.20250911.1",
|
|
31163
|
-
ai: "^6.0.
|
|
31163
|
+
ai: "^6.0.161",
|
|
31164
31164
|
dotenv: "^17.2.2",
|
|
31165
31165
|
esbuild: "^0.25.0",
|
|
31166
31166
|
eslint: "^10.2.1",
|
|
@@ -31188,6 +31188,7 @@ var init_package = __esm({
|
|
|
31188
31188
|
pnpm: {
|
|
31189
31189
|
overrides: {
|
|
31190
31190
|
protobufjs: ">=7.5.6 <8.0.0 || >=8.0.2",
|
|
31191
|
+
"langsmith@<0.6.0": "0.6.0",
|
|
31191
31192
|
"fast-uri@<3.1.2": "3.1.2"
|
|
31192
31193
|
}
|
|
31193
31194
|
}
|
|
@@ -31227,6 +31228,533 @@ var init_create_dataset = __esm({
|
|
|
31227
31228
|
}
|
|
31228
31229
|
});
|
|
31229
31230
|
|
|
31231
|
+
// src/langwatch-api.ts
|
|
31232
|
+
async function makeRequest(method, path, body) {
|
|
31233
|
+
const config2 = getConfig();
|
|
31234
|
+
const url2 = config2.endpoint + path;
|
|
31235
|
+
const headers = {
|
|
31236
|
+
"X-Auth-Token": requireApiKey()
|
|
31237
|
+
};
|
|
31238
|
+
if (config2.projectId) {
|
|
31239
|
+
headers["X-Project-Id"] = config2.projectId;
|
|
31240
|
+
}
|
|
31241
|
+
if (body !== void 0) {
|
|
31242
|
+
headers["Content-Type"] = "application/json";
|
|
31243
|
+
}
|
|
31244
|
+
const response = await fetch(url2, {
|
|
31245
|
+
method,
|
|
31246
|
+
headers,
|
|
31247
|
+
...body !== void 0 ? { body: JSON.stringify(body) } : {}
|
|
31248
|
+
});
|
|
31249
|
+
if (!response.ok) {
|
|
31250
|
+
const responseBody = await response.text();
|
|
31251
|
+
throw new LangWatchApiError(
|
|
31252
|
+
`LangWatch API error ${response.status}: ${responseBody}`,
|
|
31253
|
+
response.status,
|
|
31254
|
+
responseBody
|
|
31255
|
+
);
|
|
31256
|
+
}
|
|
31257
|
+
if (response.status === 204 || response.headers?.get("content-length") === "0") {
|
|
31258
|
+
return null;
|
|
31259
|
+
}
|
|
31260
|
+
return response.json();
|
|
31261
|
+
}
|
|
31262
|
+
async function searchTraces(params) {
|
|
31263
|
+
const { format: format3 = "digest", ...rest } = params;
|
|
31264
|
+
return makeRequest("POST", "/api/traces/search", {
|
|
31265
|
+
...rest,
|
|
31266
|
+
format: format3
|
|
31267
|
+
});
|
|
31268
|
+
}
|
|
31269
|
+
async function getTraceById(traceId, format3 = "digest") {
|
|
31270
|
+
return makeRequest(
|
|
31271
|
+
"GET",
|
|
31272
|
+
`/api/traces/${encodeURIComponent(traceId)}?format=${format3}`
|
|
31273
|
+
);
|
|
31274
|
+
}
|
|
31275
|
+
async function getAnalyticsTimeseries(params) {
|
|
31276
|
+
return makeRequest(
|
|
31277
|
+
"POST",
|
|
31278
|
+
"/api/analytics/timeseries",
|
|
31279
|
+
params
|
|
31280
|
+
);
|
|
31281
|
+
}
|
|
31282
|
+
async function listPrompts() {
|
|
31283
|
+
return makeRequest("GET", "/api/prompts");
|
|
31284
|
+
}
|
|
31285
|
+
async function getPrompt(idOrHandle, options) {
|
|
31286
|
+
const params = new URLSearchParams();
|
|
31287
|
+
if (options?.version != null) params.set("version", String(options.version));
|
|
31288
|
+
if (options?.tag) params.set("tag", options.tag);
|
|
31289
|
+
const query = params.toString() ? `?${params}` : "";
|
|
31290
|
+
return makeRequest(
|
|
31291
|
+
"GET",
|
|
31292
|
+
`/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
|
|
31293
|
+
);
|
|
31294
|
+
}
|
|
31295
|
+
async function createPrompt(data) {
|
|
31296
|
+
return makeRequest(
|
|
31297
|
+
"POST",
|
|
31298
|
+
"/api/prompts",
|
|
31299
|
+
data
|
|
31300
|
+
);
|
|
31301
|
+
}
|
|
31302
|
+
async function updatePrompt(idOrHandle, data) {
|
|
31303
|
+
return makeRequest(
|
|
31304
|
+
"PUT",
|
|
31305
|
+
`/api/prompts/${encodeURIComponent(idOrHandle)}`,
|
|
31306
|
+
data
|
|
31307
|
+
);
|
|
31308
|
+
}
|
|
31309
|
+
async function assignPromptTag({
|
|
31310
|
+
idOrHandle,
|
|
31311
|
+
tag,
|
|
31312
|
+
versionId
|
|
31313
|
+
}) {
|
|
31314
|
+
return makeRequest(
|
|
31315
|
+
"PUT",
|
|
31316
|
+
`/api/prompts/${encodeURIComponent(idOrHandle)}/tags/${encodeURIComponent(tag)}`,
|
|
31317
|
+
{ versionId }
|
|
31318
|
+
);
|
|
31319
|
+
}
|
|
31320
|
+
async function listPromptTags() {
|
|
31321
|
+
return makeRequest("GET", "/api/prompts/tags");
|
|
31322
|
+
}
|
|
31323
|
+
async function createPromptTag(name) {
|
|
31324
|
+
return makeRequest("POST", "/api/prompts/tags", { name });
|
|
31325
|
+
}
|
|
31326
|
+
async function renamePromptTag({
|
|
31327
|
+
tag,
|
|
31328
|
+
name
|
|
31329
|
+
}) {
|
|
31330
|
+
return makeRequest(
|
|
31331
|
+
"PUT",
|
|
31332
|
+
`/api/prompts/tags/${encodeURIComponent(tag)}`,
|
|
31333
|
+
{ name }
|
|
31334
|
+
);
|
|
31335
|
+
}
|
|
31336
|
+
async function deletePromptTag(tag) {
|
|
31337
|
+
return makeRequest(
|
|
31338
|
+
"DELETE",
|
|
31339
|
+
`/api/prompts/tags/${encodeURIComponent(tag)}`
|
|
31340
|
+
);
|
|
31341
|
+
}
|
|
31342
|
+
var LangWatchApiError;
|
|
31343
|
+
var init_langwatch_api = __esm({
|
|
31344
|
+
"src/langwatch-api.ts"() {
|
|
31345
|
+
"use strict";
|
|
31346
|
+
init_config();
|
|
31347
|
+
LangWatchApiError = class extends Error {
|
|
31348
|
+
constructor(message, status, responseBody) {
|
|
31349
|
+
super(message);
|
|
31350
|
+
this.status = status;
|
|
31351
|
+
this.responseBody = responseBody;
|
|
31352
|
+
this.name = "LangWatchApiError";
|
|
31353
|
+
}
|
|
31354
|
+
};
|
|
31355
|
+
}
|
|
31356
|
+
});
|
|
31357
|
+
|
|
31358
|
+
// src/tools/get-experiment-results.ts
|
|
31359
|
+
async function handleExperimentResults(params) {
|
|
31360
|
+
const filter = params.filter ?? "all";
|
|
31361
|
+
const evaluatorFilter = params.evaluator?.trim();
|
|
31362
|
+
const limit = typeof params.limit === "number" && params.limit > 0 ? Math.min(params.limit, DEFAULT_ROW_CAP) : DEFAULT_ROW_CAP;
|
|
31363
|
+
const search = new URLSearchParams();
|
|
31364
|
+
if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
|
|
31365
|
+
const qs = search.toString() ? `?${search.toString()}` : "";
|
|
31366
|
+
let results;
|
|
31367
|
+
try {
|
|
31368
|
+
results = await makeRequest(
|
|
31369
|
+
"GET",
|
|
31370
|
+
`/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`
|
|
31371
|
+
);
|
|
31372
|
+
} catch (error48) {
|
|
31373
|
+
const message = error48 instanceof Error ? error48.message : String(error48);
|
|
31374
|
+
const status = error48 instanceof LangWatchApiError ? error48.status : error48 && typeof error48 === "object" && "status" in error48 && typeof error48.status === "number" ? error48.status : void 0;
|
|
31375
|
+
if (status === 404 || status === void 0 && /404|not found/i.test(message)) {
|
|
31376
|
+
return [
|
|
31377
|
+
`# Evaluation Results: ${params.runId}`,
|
|
31378
|
+
"",
|
|
31379
|
+
"**Status**: not found",
|
|
31380
|
+
"",
|
|
31381
|
+
`Could not load results for run \`${params.runId}\`. The run may still be in progress, or the id may be incorrect.`,
|
|
31382
|
+
"",
|
|
31383
|
+
"> Try `platform_experiment_status` first to confirm the run id and that it is completed."
|
|
31384
|
+
].join("\n");
|
|
31385
|
+
}
|
|
31386
|
+
throw error48;
|
|
31387
|
+
}
|
|
31388
|
+
if (!results || results.timestamps.finishedAt == null && results.timestamps.stoppedAt == null) {
|
|
31389
|
+
return [
|
|
31390
|
+
`# Evaluation Results: ${params.runId}`,
|
|
31391
|
+
"",
|
|
31392
|
+
"**Status**: results are not yet available",
|
|
31393
|
+
"",
|
|
31394
|
+
`Results for run \`${params.runId}\` are not ready yet.`,
|
|
31395
|
+
"",
|
|
31396
|
+
"> Try `platform_experiment_status` first to confirm the run id and that it is completed."
|
|
31397
|
+
].join("\n");
|
|
31398
|
+
}
|
|
31399
|
+
const evaluationsByRow = /* @__PURE__ */ new Map();
|
|
31400
|
+
for (const evaluation of results.evaluations) {
|
|
31401
|
+
if (evaluatorFilter && evaluation.evaluator !== evaluatorFilter) continue;
|
|
31402
|
+
const key = rowKey(evaluation.index, evaluation.targetId);
|
|
31403
|
+
const list = evaluationsByRow.get(key) ?? [];
|
|
31404
|
+
list.push(evaluation);
|
|
31405
|
+
evaluationsByRow.set(key, list);
|
|
31406
|
+
}
|
|
31407
|
+
const evaluatorNames = Array.from(
|
|
31408
|
+
new Set(
|
|
31409
|
+
(evaluatorFilter ? results.evaluations.filter((e) => e.evaluator === evaluatorFilter) : results.evaluations).map((e) => e.evaluator)
|
|
31410
|
+
)
|
|
31411
|
+
);
|
|
31412
|
+
let rows = results.dataset.map((entry) => ({
|
|
31413
|
+
entry,
|
|
31414
|
+
evaluations: evaluationsByRow.get(rowKey(entry.index, entry.targetId)) ?? []
|
|
31415
|
+
}));
|
|
31416
|
+
if (filter === "failed") {
|
|
31417
|
+
rows = rows.filter(
|
|
31418
|
+
(r) => isFailedRow({ entry: r.entry, evaluations: r.evaluations })
|
|
31419
|
+
);
|
|
31420
|
+
}
|
|
31421
|
+
const totalMatching = rows.length;
|
|
31422
|
+
const rowsForSummary = rows;
|
|
31423
|
+
const truncated = rows.length > limit;
|
|
31424
|
+
rows = rows.slice(0, limit);
|
|
31425
|
+
const evaluatorAverages = /* @__PURE__ */ new Map();
|
|
31426
|
+
for (const r of rowsForSummary) {
|
|
31427
|
+
for (const e of r.evaluations) {
|
|
31428
|
+
const stats = evaluatorAverages.get(e.evaluator) ?? {
|
|
31429
|
+
sum: 0,
|
|
31430
|
+
count: 0,
|
|
31431
|
+
passed: 0,
|
|
31432
|
+
failed: 0,
|
|
31433
|
+
errored: 0
|
|
31434
|
+
};
|
|
31435
|
+
if (typeof e.score === "number") {
|
|
31436
|
+
stats.sum += e.score;
|
|
31437
|
+
stats.count += 1;
|
|
31438
|
+
}
|
|
31439
|
+
if (e.status === "error") stats.errored += 1;
|
|
31440
|
+
else if (e.passed === true) stats.passed += 1;
|
|
31441
|
+
else if (e.passed === false) stats.failed += 1;
|
|
31442
|
+
evaluatorAverages.set(e.evaluator, stats);
|
|
31443
|
+
}
|
|
31444
|
+
}
|
|
31445
|
+
const lines = [];
|
|
31446
|
+
lines.push(`# Evaluation Results: ${results.runId}`);
|
|
31447
|
+
lines.push("");
|
|
31448
|
+
lines.push(`**Experiment**: ${results.experimentId}`);
|
|
31449
|
+
lines.push(`**Total rows**: ${results.dataset.length}`);
|
|
31450
|
+
lines.push(`**Total evaluations**: ${results.evaluations.length}`);
|
|
31451
|
+
if (filter === "failed") {
|
|
31452
|
+
lines.push(`**Filter**: failed only`);
|
|
31453
|
+
}
|
|
31454
|
+
if (evaluatorFilter) {
|
|
31455
|
+
lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
|
|
31456
|
+
}
|
|
31457
|
+
lines.push("");
|
|
31458
|
+
if (evaluatorAverages.size > 0) {
|
|
31459
|
+
lines.push("## Evaluator Summary");
|
|
31460
|
+
lines.push("");
|
|
31461
|
+
lines.push("| Evaluator | Avg Score | Passed | Failed | Errored |");
|
|
31462
|
+
lines.push("| --- | --- | --- | --- | --- |");
|
|
31463
|
+
for (const [name, stats] of evaluatorAverages) {
|
|
31464
|
+
const avg = stats.count > 0 ? (stats.sum / stats.count).toFixed(3) : "\u2014";
|
|
31465
|
+
lines.push(
|
|
31466
|
+
`| ${name} | ${avg} | ${stats.passed} | ${stats.failed} | ${stats.errored} |`
|
|
31467
|
+
);
|
|
31468
|
+
}
|
|
31469
|
+
lines.push("");
|
|
31470
|
+
}
|
|
31471
|
+
if (rows.length === 0) {
|
|
31472
|
+
lines.push("_No rows matched the filter._");
|
|
31473
|
+
return lines.join("\n");
|
|
31474
|
+
}
|
|
31475
|
+
lines.push(
|
|
31476
|
+
`## Rows (${rows.length}${truncated ? ` of ${totalMatching}` : ""})`
|
|
31477
|
+
);
|
|
31478
|
+
lines.push("");
|
|
31479
|
+
for (const { entry, evaluations } of rows) {
|
|
31480
|
+
const summary = summarizeEntry(entry.entry);
|
|
31481
|
+
lines.push(`### Row #${entry.index}${summary ? ` \u2014 ${summary}` : ""}`);
|
|
31482
|
+
if (entry.error) {
|
|
31483
|
+
lines.push(`- **Error**: ${entry.error}`);
|
|
31484
|
+
}
|
|
31485
|
+
if (entry.traceId) {
|
|
31486
|
+
lines.push(`- **Trace ID**: \`${entry.traceId}\``);
|
|
31487
|
+
}
|
|
31488
|
+
if (evaluations.length === 0) {
|
|
31489
|
+
lines.push(
|
|
31490
|
+
evaluatorNames.length === 0 ? "- _No evaluations recorded_" : "- _No evaluations recorded for this row_"
|
|
31491
|
+
);
|
|
31492
|
+
}
|
|
31493
|
+
for (const e of evaluations) {
|
|
31494
|
+
const parts = [`**${e.evaluator}**`];
|
|
31495
|
+
if (e.status === "error") parts.push("status=error");
|
|
31496
|
+
else if (typeof e.score === "number") parts.push(`score=${e.score.toFixed(3)}`);
|
|
31497
|
+
if (typeof e.passed === "boolean") {
|
|
31498
|
+
parts.push(`passed=${e.passed ? "yes" : "no"}`);
|
|
31499
|
+
}
|
|
31500
|
+
if (e.label) parts.push(`label=${e.label}`);
|
|
31501
|
+
lines.push(`- ${parts.join(" \xB7 ")}`);
|
|
31502
|
+
if (e.details) {
|
|
31503
|
+
lines.push(` - details: ${e.details}`);
|
|
31504
|
+
}
|
|
31505
|
+
}
|
|
31506
|
+
lines.push("");
|
|
31507
|
+
}
|
|
31508
|
+
if (truncated) {
|
|
31509
|
+
lines.push(
|
|
31510
|
+
`> Output truncated to ${limit} rows of ${totalMatching} matching to protect the agent's context window. Pass \`limit\` to expand or filter further.`
|
|
31511
|
+
);
|
|
31512
|
+
}
|
|
31513
|
+
return lines.join("\n");
|
|
31514
|
+
}
|
|
31515
|
+
var DEFAULT_ROW_CAP, rowKey, summarizeEntry, isFailedEvaluation, isFailedRow;
|
|
31516
|
+
var init_get_experiment_results = __esm({
|
|
31517
|
+
"src/tools/get-experiment-results.ts"() {
|
|
31518
|
+
"use strict";
|
|
31519
|
+
init_langwatch_api();
|
|
31520
|
+
DEFAULT_ROW_CAP = 50;
|
|
31521
|
+
rowKey = (index, targetId) => `${index}:${targetId ?? ""}`;
|
|
31522
|
+
summarizeEntry = (entry) => {
|
|
31523
|
+
const candidates = ["input", "question", "query", "prompt", "user"];
|
|
31524
|
+
for (const key of candidates) {
|
|
31525
|
+
const value = entry[key];
|
|
31526
|
+
if (typeof value === "string" && value.length > 0) {
|
|
31527
|
+
return value.length > 80 ? `${value.slice(0, 77)}...` : value;
|
|
31528
|
+
}
|
|
31529
|
+
}
|
|
31530
|
+
return "";
|
|
31531
|
+
};
|
|
31532
|
+
isFailedEvaluation = (e) => e.status === "error" || e.passed === false;
|
|
31533
|
+
isFailedRow = ({
|
|
31534
|
+
entry,
|
|
31535
|
+
evaluations
|
|
31536
|
+
}) => Boolean(entry.error) || evaluations.some(isFailedEvaluation);
|
|
31537
|
+
}
|
|
31538
|
+
});
|
|
31539
|
+
|
|
31540
|
+
// src/tools/list-experiment-runs.ts
|
|
31541
|
+
async function handleExperimentListRuns(params) {
|
|
31542
|
+
const requested = typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT;
|
|
31543
|
+
const effectiveLimit = Math.min(requested, MAX_LIMIT);
|
|
31544
|
+
const search = new URLSearchParams();
|
|
31545
|
+
search.set("experimentSlug", params.experimentSlug);
|
|
31546
|
+
search.set("pageSize", String(effectiveLimit));
|
|
31547
|
+
let result;
|
|
31548
|
+
try {
|
|
31549
|
+
result = await makeRequest(
|
|
31550
|
+
"GET",
|
|
31551
|
+
`/api/experiments/runs?${search.toString()}`
|
|
31552
|
+
);
|
|
31553
|
+
} catch (error48) {
|
|
31554
|
+
const message = error48 instanceof Error ? error48.message : String(error48);
|
|
31555
|
+
if (/404|not found/i.test(message)) {
|
|
31556
|
+
return [
|
|
31557
|
+
`# Evaluation Runs: ${params.experimentSlug}`,
|
|
31558
|
+
"",
|
|
31559
|
+
"**Status**: experiment not found",
|
|
31560
|
+
"",
|
|
31561
|
+
`No experiment with slug \`${params.experimentSlug}\` exists in this project.`,
|
|
31562
|
+
"",
|
|
31563
|
+
"> Run `platform_experiment_list` to discover available experiment slugs."
|
|
31564
|
+
].join("\n");
|
|
31565
|
+
}
|
|
31566
|
+
throw error48;
|
|
31567
|
+
}
|
|
31568
|
+
if (result.runs.length === 0) {
|
|
31569
|
+
return [
|
|
31570
|
+
`# Evaluation Runs: ${result.experimentSlug}`,
|
|
31571
|
+
"",
|
|
31572
|
+
"_This experiment has no runs yet._",
|
|
31573
|
+
"",
|
|
31574
|
+
`> Trigger a run with \`platform_run_experiment\` (slug: \`${result.experimentSlug}\`) to populate this list.`
|
|
31575
|
+
].join("\n");
|
|
31576
|
+
}
|
|
31577
|
+
const lines = [];
|
|
31578
|
+
lines.push(`# Evaluation Runs: ${result.experimentSlug}`);
|
|
31579
|
+
lines.push("");
|
|
31580
|
+
lines.push(
|
|
31581
|
+
`Showing ${result.runs.length} of ${result.pagination.totalHits} run${result.pagination.totalHits === 1 ? "" : "s"}.`
|
|
31582
|
+
);
|
|
31583
|
+
if (result.pagination.hasMore) {
|
|
31584
|
+
lines.push(
|
|
31585
|
+
`> More runs exist beyond this limit. Re-run with a higher \`limit\` (max ${MAX_LIMIT}) if needed.`
|
|
31586
|
+
);
|
|
31587
|
+
}
|
|
31588
|
+
lines.push("");
|
|
31589
|
+
lines.push("| Run ID | Status | Started | Finished | Result |");
|
|
31590
|
+
lines.push("|--------|--------|---------|----------|--------|");
|
|
31591
|
+
for (const run of result.runs) {
|
|
31592
|
+
lines.push(
|
|
31593
|
+
`| \`${run.runId}\` | ${runStatus(run)} | ${formatEpoch(run.timestamps.createdAt)} | ${formatEpoch(run.timestamps.finishedAt ?? null)} | ${passRate(run.summary?.evaluations ?? {})} |`
|
|
31594
|
+
);
|
|
31595
|
+
}
|
|
31596
|
+
lines.push("");
|
|
31597
|
+
lines.push(
|
|
31598
|
+
"> Use `platform_experiment_status <runId>` for live progress, or `platform_experiment_results <runId>` for per-row results."
|
|
31599
|
+
);
|
|
31600
|
+
return lines.join("\n");
|
|
31601
|
+
}
|
|
31602
|
+
var DEFAULT_LIMIT, MAX_LIMIT, formatEpoch, passRate, runStatus;
|
|
31603
|
+
var init_list_experiment_runs = __esm({
|
|
31604
|
+
"src/tools/list-experiment-runs.ts"() {
|
|
31605
|
+
"use strict";
|
|
31606
|
+
init_langwatch_api();
|
|
31607
|
+
DEFAULT_LIMIT = 25;
|
|
31608
|
+
MAX_LIMIT = 100;
|
|
31609
|
+
formatEpoch = (epoch) => {
|
|
31610
|
+
if (!epoch || !Number.isFinite(epoch)) return "\u2014";
|
|
31611
|
+
return new Date(epoch).toISOString().replace("T", " ").slice(0, 19) + " UTC";
|
|
31612
|
+
};
|
|
31613
|
+
passRate = (evaluations) => {
|
|
31614
|
+
const entries = Object.values(evaluations ?? {});
|
|
31615
|
+
if (entries.length === 0) return "\u2014";
|
|
31616
|
+
const passed = entries.filter((e) => typeof e.averagePassed === "number");
|
|
31617
|
+
if (passed.length === 0) {
|
|
31618
|
+
const scored = entries.filter((e) => typeof e.averageScore === "number");
|
|
31619
|
+
if (scored.length === 0) return "\u2014";
|
|
31620
|
+
const avg2 = scored.reduce((sum, e) => sum + (e.averageScore ?? 0), 0) / scored.length;
|
|
31621
|
+
return `${avg2.toFixed(2)} avg`;
|
|
31622
|
+
}
|
|
31623
|
+
const avg = passed.reduce((sum, e) => sum + (e.averagePassed ?? 0), 0) / passed.length;
|
|
31624
|
+
return `${(avg * 100).toFixed(0)}% pass`;
|
|
31625
|
+
};
|
|
31626
|
+
runStatus = (run) => {
|
|
31627
|
+
if (run.timestamps.stoppedAt) return "stopped";
|
|
31628
|
+
if (run.timestamps.finishedAt) return "completed";
|
|
31629
|
+
return "running";
|
|
31630
|
+
};
|
|
31631
|
+
}
|
|
31632
|
+
});
|
|
31633
|
+
|
|
31634
|
+
// src/tools/list-experiments.ts
|
|
31635
|
+
async function handleExperimentList(params) {
|
|
31636
|
+
const requested = typeof params.limit === "number" && params.limit > 0 ? params.limit : DEFAULT_LIMIT2;
|
|
31637
|
+
const effectiveLimit = Math.min(requested, MAX_LIMIT2);
|
|
31638
|
+
const search = new URLSearchParams();
|
|
31639
|
+
search.set("pageSize", String(effectiveLimit));
|
|
31640
|
+
const result = await makeRequest(
|
|
31641
|
+
"GET",
|
|
31642
|
+
`/api/experiments?${search.toString()}`
|
|
31643
|
+
);
|
|
31644
|
+
if (result.experiments.length === 0) {
|
|
31645
|
+
return [
|
|
31646
|
+
"# Experiments",
|
|
31647
|
+
"",
|
|
31648
|
+
"_No experiments found in this project._",
|
|
31649
|
+
"",
|
|
31650
|
+
"> Create an experiment in the LangWatch dashboard, then re-run this tool to discover its slug."
|
|
31651
|
+
].join("\n");
|
|
31652
|
+
}
|
|
31653
|
+
const lines = [];
|
|
31654
|
+
lines.push("# Experiments");
|
|
31655
|
+
lines.push("");
|
|
31656
|
+
lines.push(
|
|
31657
|
+
`Showing ${result.experiments.length} of ${result.pagination.totalHits} experiment${result.pagination.totalHits === 1 ? "" : "s"}.`
|
|
31658
|
+
);
|
|
31659
|
+
if (result.pagination.hasMore) {
|
|
31660
|
+
lines.push(
|
|
31661
|
+
`> More experiments exist beyond this limit. Re-run with a higher \`limit\` (max ${MAX_LIMIT2}) if needed.`
|
|
31662
|
+
);
|
|
31663
|
+
}
|
|
31664
|
+
lines.push("");
|
|
31665
|
+
lines.push("| Slug | Name | Runs | Last Run |");
|
|
31666
|
+
lines.push("|------|------|------|----------|");
|
|
31667
|
+
for (const exp of result.experiments) {
|
|
31668
|
+
const name = escapeMarkdownTableCell(exp.name ?? exp.slug);
|
|
31669
|
+
lines.push(
|
|
31670
|
+
`| ${formatInlineCode(exp.slug)} | ${name} | ${exp.runsCount} | ${formatTimestamp(exp.lastRunAt)} |`
|
|
31671
|
+
);
|
|
31672
|
+
}
|
|
31673
|
+
lines.push("");
|
|
31674
|
+
lines.push(
|
|
31675
|
+
"> Use `platform_experiment_list_runs` with one of these slugs to see its runs."
|
|
31676
|
+
);
|
|
31677
|
+
return lines.join("\n");
|
|
31678
|
+
}
|
|
31679
|
+
var DEFAULT_LIMIT2, MAX_LIMIT2, formatTimestamp, escapeMarkdownTableCell, formatInlineCode;
|
|
31680
|
+
var init_list_experiments = __esm({
|
|
31681
|
+
"src/tools/list-experiments.ts"() {
|
|
31682
|
+
"use strict";
|
|
31683
|
+
init_langwatch_api();
|
|
31684
|
+
DEFAULT_LIMIT2 = 25;
|
|
31685
|
+
MAX_LIMIT2 = 100;
|
|
31686
|
+
formatTimestamp = (iso) => {
|
|
31687
|
+
if (!iso) return "\u2014";
|
|
31688
|
+
const d = new Date(iso);
|
|
31689
|
+
if (Number.isNaN(d.getTime())) return iso;
|
|
31690
|
+
return d.toISOString().replace("T", " ").slice(0, 19) + " UTC";
|
|
31691
|
+
};
|
|
31692
|
+
escapeMarkdownTableCell = (value) => value.replace(/[\\|]/g, (c) => `\\${c}`).replace(/\n/g, " ");
|
|
31693
|
+
formatInlineCode = (value) => `\`${value.replace(/[\\`|]/g, (c) => `\\${c}`).replace(/\n/g, " ")}\``;
|
|
31694
|
+
}
|
|
31695
|
+
});
|
|
31696
|
+
|
|
31697
|
+
// src/tools/run-experiment.ts
|
|
31698
|
+
async function handleRunExperiment(params) {
|
|
31699
|
+
const result = await makeRequest(
|
|
31700
|
+
"POST",
|
|
31701
|
+
`/api/experiments/${encodeURIComponent(params.slug)}/run`
|
|
31702
|
+
);
|
|
31703
|
+
const lines = [];
|
|
31704
|
+
lines.push(`Evaluation started!
|
|
31705
|
+
`);
|
|
31706
|
+
lines.push(`**Run ID**: ${result.runId}`);
|
|
31707
|
+
lines.push(`**Status**: ${result.status}`);
|
|
31708
|
+
lines.push(`**Total cells**: ${result.total}`);
|
|
31709
|
+
if (result.runUrl) lines.push(`**View at**: ${result.runUrl}`);
|
|
31710
|
+
lines.push("");
|
|
31711
|
+
lines.push(
|
|
31712
|
+
`> Use \`platform_experiment_status\` with run ID "${result.runId}" to check progress.`
|
|
31713
|
+
);
|
|
31714
|
+
return lines.join("\n");
|
|
31715
|
+
}
|
|
31716
|
+
async function handleExperimentStatus(params) {
|
|
31717
|
+
const status = await makeRequest(
|
|
31718
|
+
"GET",
|
|
31719
|
+
`/api/experiments/runs/${encodeURIComponent(params.runId)}`
|
|
31720
|
+
);
|
|
31721
|
+
const lines = [];
|
|
31722
|
+
lines.push(`# Evaluation Run ${status.runId}
|
|
31723
|
+
`);
|
|
31724
|
+
lines.push(`**Status**: ${status.status}`);
|
|
31725
|
+
lines.push(`**Progress**: ${status.progress}/${status.total} cells`);
|
|
31726
|
+
if (status.startedAt) {
|
|
31727
|
+
lines.push(`**Started**: ${new Date(status.startedAt).toISOString()}`);
|
|
31728
|
+
}
|
|
31729
|
+
if (status.finishedAt) {
|
|
31730
|
+
lines.push(`**Finished**: ${new Date(status.finishedAt).toISOString()}`);
|
|
31731
|
+
}
|
|
31732
|
+
if (status.summary) {
|
|
31733
|
+
lines.push("\n## Summary\n");
|
|
31734
|
+
if (status.summary.completedCells !== void 0) {
|
|
31735
|
+
lines.push(`**Completed**: ${status.summary.completedCells}`);
|
|
31736
|
+
}
|
|
31737
|
+
if (status.summary.failedCells) {
|
|
31738
|
+
lines.push(`**Failed**: ${status.summary.failedCells}`);
|
|
31739
|
+
}
|
|
31740
|
+
if (status.summary.duration) {
|
|
31741
|
+
lines.push(
|
|
31742
|
+
`**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`
|
|
31743
|
+
);
|
|
31744
|
+
}
|
|
31745
|
+
if (status.summary.runUrl) {
|
|
31746
|
+
lines.push(`**View results**: ${status.summary.runUrl}`);
|
|
31747
|
+
}
|
|
31748
|
+
}
|
|
31749
|
+
return lines.join("\n");
|
|
31750
|
+
}
|
|
31751
|
+
var init_run_experiment = __esm({
|
|
31752
|
+
"src/tools/run-experiment.ts"() {
|
|
31753
|
+
"use strict";
|
|
31754
|
+
init_langwatch_api();
|
|
31755
|
+
}
|
|
31756
|
+
});
|
|
31757
|
+
|
|
31230
31758
|
// src/tools/discover-scenario-schema.ts
|
|
31231
31759
|
var discover_scenario_schema_exports = {};
|
|
31232
31760
|
__export(discover_scenario_schema_exports, {
|
|
@@ -33185,122 +33713,6 @@ var init_discover_schema = __esm({
|
|
|
33185
33713
|
}
|
|
33186
33714
|
});
|
|
33187
33715
|
|
|
33188
|
-
// src/langwatch-api.ts
|
|
33189
|
-
async function makeRequest(method, path, body) {
|
|
33190
|
-
const config2 = getConfig();
|
|
33191
|
-
const url2 = config2.endpoint + path;
|
|
33192
|
-
const headers = {
|
|
33193
|
-
"X-Auth-Token": requireApiKey()
|
|
33194
|
-
};
|
|
33195
|
-
if (config2.projectId) {
|
|
33196
|
-
headers["X-Project-Id"] = config2.projectId;
|
|
33197
|
-
}
|
|
33198
|
-
if (body !== void 0) {
|
|
33199
|
-
headers["Content-Type"] = "application/json";
|
|
33200
|
-
}
|
|
33201
|
-
const response = await fetch(url2, {
|
|
33202
|
-
method,
|
|
33203
|
-
headers,
|
|
33204
|
-
...body !== void 0 ? { body: JSON.stringify(body) } : {}
|
|
33205
|
-
});
|
|
33206
|
-
if (!response.ok) {
|
|
33207
|
-
const responseBody = await response.text();
|
|
33208
|
-
throw new Error(
|
|
33209
|
-
`LangWatch API error ${response.status}: ${responseBody}`
|
|
33210
|
-
);
|
|
33211
|
-
}
|
|
33212
|
-
if (response.status === 204 || response.headers?.get("content-length") === "0") {
|
|
33213
|
-
return null;
|
|
33214
|
-
}
|
|
33215
|
-
return response.json();
|
|
33216
|
-
}
|
|
33217
|
-
async function searchTraces(params) {
|
|
33218
|
-
const { format: format3 = "digest", ...rest } = params;
|
|
33219
|
-
return makeRequest("POST", "/api/traces/search", {
|
|
33220
|
-
...rest,
|
|
33221
|
-
format: format3
|
|
33222
|
-
});
|
|
33223
|
-
}
|
|
33224
|
-
async function getTraceById(traceId, format3 = "digest") {
|
|
33225
|
-
return makeRequest(
|
|
33226
|
-
"GET",
|
|
33227
|
-
`/api/traces/${encodeURIComponent(traceId)}?format=${format3}`
|
|
33228
|
-
);
|
|
33229
|
-
}
|
|
33230
|
-
async function getAnalyticsTimeseries(params) {
|
|
33231
|
-
return makeRequest(
|
|
33232
|
-
"POST",
|
|
33233
|
-
"/api/analytics/timeseries",
|
|
33234
|
-
params
|
|
33235
|
-
);
|
|
33236
|
-
}
|
|
33237
|
-
async function listPrompts() {
|
|
33238
|
-
return makeRequest("GET", "/api/prompts");
|
|
33239
|
-
}
|
|
33240
|
-
async function getPrompt(idOrHandle, options) {
|
|
33241
|
-
const params = new URLSearchParams();
|
|
33242
|
-
if (options?.version != null) params.set("version", String(options.version));
|
|
33243
|
-
if (options?.tag) params.set("tag", options.tag);
|
|
33244
|
-
const query = params.toString() ? `?${params}` : "";
|
|
33245
|
-
return makeRequest(
|
|
33246
|
-
"GET",
|
|
33247
|
-
`/api/prompts/${encodeURIComponent(idOrHandle)}${query}`
|
|
33248
|
-
);
|
|
33249
|
-
}
|
|
33250
|
-
async function createPrompt(data) {
|
|
33251
|
-
return makeRequest(
|
|
33252
|
-
"POST",
|
|
33253
|
-
"/api/prompts",
|
|
33254
|
-
data
|
|
33255
|
-
);
|
|
33256
|
-
}
|
|
33257
|
-
async function updatePrompt(idOrHandle, data) {
|
|
33258
|
-
return makeRequest(
|
|
33259
|
-
"PUT",
|
|
33260
|
-
`/api/prompts/${encodeURIComponent(idOrHandle)}`,
|
|
33261
|
-
data
|
|
33262
|
-
);
|
|
33263
|
-
}
|
|
33264
|
-
async function assignPromptTag({
|
|
33265
|
-
idOrHandle,
|
|
33266
|
-
tag,
|
|
33267
|
-
versionId
|
|
33268
|
-
}) {
|
|
33269
|
-
return makeRequest(
|
|
33270
|
-
"PUT",
|
|
33271
|
-
`/api/prompts/${encodeURIComponent(idOrHandle)}/tags/${encodeURIComponent(tag)}`,
|
|
33272
|
-
{ versionId }
|
|
33273
|
-
);
|
|
33274
|
-
}
|
|
33275
|
-
async function listPromptTags() {
|
|
33276
|
-
return makeRequest("GET", "/api/prompts/tags");
|
|
33277
|
-
}
|
|
33278
|
-
async function createPromptTag(name) {
|
|
33279
|
-
return makeRequest("POST", "/api/prompts/tags", { name });
|
|
33280
|
-
}
|
|
33281
|
-
async function renamePromptTag({
|
|
33282
|
-
tag,
|
|
33283
|
-
name
|
|
33284
|
-
}) {
|
|
33285
|
-
return makeRequest(
|
|
33286
|
-
"PUT",
|
|
33287
|
-
`/api/prompts/tags/${encodeURIComponent(tag)}`,
|
|
33288
|
-
{ name }
|
|
33289
|
-
);
|
|
33290
|
-
}
|
|
33291
|
-
async function deletePromptTag(tag) {
|
|
33292
|
-
return makeRequest(
|
|
33293
|
-
"DELETE",
|
|
33294
|
-
`/api/prompts/tags/${encodeURIComponent(tag)}`
|
|
33295
|
-
);
|
|
33296
|
-
}
|
|
33297
|
-
var init_langwatch_api = __esm({
|
|
33298
|
-
"src/langwatch-api.ts"() {
|
|
33299
|
-
"use strict";
|
|
33300
|
-
init_config();
|
|
33301
|
-
}
|
|
33302
|
-
});
|
|
33303
|
-
|
|
33304
33716
|
// src/utils/date-parsing.ts
|
|
33305
33717
|
function parseRelativeDate(input) {
|
|
33306
33718
|
if (input === "now") return Date.now();
|
|
@@ -35355,97 +35767,31 @@ __export(langwatch_api_secrets_exports, {
|
|
|
35355
35767
|
async function listSecrets() {
|
|
35356
35768
|
return makeRequest("GET", "/api/secrets");
|
|
35357
35769
|
}
|
|
35358
|
-
async function getSecret(id) {
|
|
35359
|
-
return makeRequest(
|
|
35360
|
-
"GET",
|
|
35361
|
-
`/api/secrets/${encodeURIComponent(id)}`
|
|
35362
|
-
);
|
|
35363
|
-
}
|
|
35364
|
-
async function createSecret(data) {
|
|
35365
|
-
return makeRequest("POST", "/api/secrets", data);
|
|
35366
|
-
}
|
|
35367
|
-
async function updateSecret(params) {
|
|
35368
|
-
const { id, value } = params;
|
|
35369
|
-
return makeRequest(
|
|
35370
|
-
"PUT",
|
|
35371
|
-
`/api/secrets/${encodeURIComponent(id)}`,
|
|
35372
|
-
{ value }
|
|
35373
|
-
);
|
|
35374
|
-
}
|
|
35375
|
-
async function deleteSecret(id) {
|
|
35376
|
-
return makeRequest(
|
|
35377
|
-
"DELETE",
|
|
35378
|
-
`/api/secrets/${encodeURIComponent(id)}`
|
|
35379
|
-
);
|
|
35380
|
-
}
|
|
35381
|
-
var init_langwatch_api_secrets = __esm({
|
|
35382
|
-
"src/langwatch-api-secrets.ts"() {
|
|
35383
|
-
"use strict";
|
|
35384
|
-
init_langwatch_api();
|
|
35385
|
-
}
|
|
35386
|
-
});
|
|
35387
|
-
|
|
35388
|
-
// src/tools/run-evaluation.ts
|
|
35389
|
-
var run_evaluation_exports = {};
|
|
35390
|
-
__export(run_evaluation_exports, {
|
|
35391
|
-
handleEvaluationStatus: () => handleEvaluationStatus,
|
|
35392
|
-
handleRunEvaluation: () => handleRunEvaluation
|
|
35393
|
-
});
|
|
35394
|
-
async function handleRunEvaluation(params) {
|
|
35395
|
-
const result = await makeRequest(
|
|
35396
|
-
"POST",
|
|
35397
|
-
`/api/evaluations/v3/${encodeURIComponent(params.slug)}/run`
|
|
35398
|
-
);
|
|
35399
|
-
const lines = [];
|
|
35400
|
-
lines.push(`Evaluation started!
|
|
35401
|
-
`);
|
|
35402
|
-
lines.push(`**Run ID**: ${result.runId}`);
|
|
35403
|
-
lines.push(`**Status**: ${result.status}`);
|
|
35404
|
-
lines.push(`**Total cells**: ${result.total}`);
|
|
35405
|
-
if (result.runUrl) lines.push(`**View at**: ${result.runUrl}`);
|
|
35406
|
-
lines.push("");
|
|
35407
|
-
lines.push(
|
|
35408
|
-
`> Use \`platform_evaluation_status\` with run ID "${result.runId}" to check progress.`
|
|
35409
|
-
);
|
|
35410
|
-
return lines.join("\n");
|
|
35411
|
-
}
|
|
35412
|
-
async function handleEvaluationStatus(params) {
|
|
35413
|
-
const status = await makeRequest(
|
|
35414
|
-
"GET",
|
|
35415
|
-
`/api/evaluations/v3/runs/${encodeURIComponent(params.runId)}`
|
|
35416
|
-
);
|
|
35417
|
-
const lines = [];
|
|
35418
|
-
lines.push(`# Evaluation Run ${status.runId}
|
|
35419
|
-
`);
|
|
35420
|
-
lines.push(`**Status**: ${status.status}`);
|
|
35421
|
-
lines.push(`**Progress**: ${status.progress}/${status.total} cells`);
|
|
35422
|
-
if (status.startedAt) {
|
|
35423
|
-
lines.push(`**Started**: ${new Date(status.startedAt).toISOString()}`);
|
|
35424
|
-
}
|
|
35425
|
-
if (status.finishedAt) {
|
|
35426
|
-
lines.push(`**Finished**: ${new Date(status.finishedAt).toISOString()}`);
|
|
35427
|
-
}
|
|
35428
|
-
if (status.summary) {
|
|
35429
|
-
lines.push("\n## Summary\n");
|
|
35430
|
-
if (status.summary.completedCells !== void 0) {
|
|
35431
|
-
lines.push(`**Completed**: ${status.summary.completedCells}`);
|
|
35432
|
-
}
|
|
35433
|
-
if (status.summary.failedCells) {
|
|
35434
|
-
lines.push(`**Failed**: ${status.summary.failedCells}`);
|
|
35435
|
-
}
|
|
35436
|
-
if (status.summary.duration) {
|
|
35437
|
-
lines.push(
|
|
35438
|
-
`**Duration**: ${(status.summary.duration / 1e3).toFixed(1)}s`
|
|
35439
|
-
);
|
|
35440
|
-
}
|
|
35441
|
-
if (status.summary.runUrl) {
|
|
35442
|
-
lines.push(`**View results**: ${status.summary.runUrl}`);
|
|
35443
|
-
}
|
|
35444
|
-
}
|
|
35445
|
-
return lines.join("\n");
|
|
35770
|
+
async function getSecret(id) {
|
|
35771
|
+
return makeRequest(
|
|
35772
|
+
"GET",
|
|
35773
|
+
`/api/secrets/${encodeURIComponent(id)}`
|
|
35774
|
+
);
|
|
35775
|
+
}
|
|
35776
|
+
async function createSecret(data) {
|
|
35777
|
+
return makeRequest("POST", "/api/secrets", data);
|
|
35778
|
+
}
|
|
35779
|
+
async function updateSecret(params) {
|
|
35780
|
+
const { id, value } = params;
|
|
35781
|
+
return makeRequest(
|
|
35782
|
+
"PUT",
|
|
35783
|
+
`/api/secrets/${encodeURIComponent(id)}`,
|
|
35784
|
+
{ value }
|
|
35785
|
+
);
|
|
35446
35786
|
}
|
|
35447
|
-
|
|
35448
|
-
|
|
35787
|
+
async function deleteSecret(id) {
|
|
35788
|
+
return makeRequest(
|
|
35789
|
+
"DELETE",
|
|
35790
|
+
`/api/secrets/${encodeURIComponent(id)}`
|
|
35791
|
+
);
|
|
35792
|
+
}
|
|
35793
|
+
var init_langwatch_api_secrets = __esm({
|
|
35794
|
+
"src/langwatch-api-secrets.ts"() {
|
|
35449
35795
|
"use strict";
|
|
35450
35796
|
init_langwatch_api();
|
|
35451
35797
|
}
|
|
@@ -35826,6 +36172,305 @@ var init_delete_dataset_records = __esm({
|
|
|
35826
36172
|
}
|
|
35827
36173
|
});
|
|
35828
36174
|
|
|
36175
|
+
// src/langwatch-api-projects.ts
|
|
36176
|
+
async function listProjects(params) {
|
|
36177
|
+
const query = new URLSearchParams();
|
|
36178
|
+
if (params?.page) query.set("page", String(params.page));
|
|
36179
|
+
if (params?.limit) query.set("limit", String(params.limit));
|
|
36180
|
+
const qs = query.toString() ? `?${query}` : "";
|
|
36181
|
+
return makeRequest("GET", `/api/projects${qs}`);
|
|
36182
|
+
}
|
|
36183
|
+
async function getProject(id) {
|
|
36184
|
+
return makeRequest(
|
|
36185
|
+
"GET",
|
|
36186
|
+
`/api/projects/${encodeURIComponent(id)}`
|
|
36187
|
+
);
|
|
36188
|
+
}
|
|
36189
|
+
async function createProject(data) {
|
|
36190
|
+
return makeRequest("POST", "/api/projects", data);
|
|
36191
|
+
}
|
|
36192
|
+
async function updateProject(params) {
|
|
36193
|
+
const { id, ...data } = params;
|
|
36194
|
+
return makeRequest(
|
|
36195
|
+
"PATCH",
|
|
36196
|
+
`/api/projects/${encodeURIComponent(id)}`,
|
|
36197
|
+
data
|
|
36198
|
+
);
|
|
36199
|
+
}
|
|
36200
|
+
async function archiveProject(id) {
|
|
36201
|
+
return makeRequest(
|
|
36202
|
+
"DELETE",
|
|
36203
|
+
`/api/projects/${encodeURIComponent(id)}`
|
|
36204
|
+
);
|
|
36205
|
+
}
|
|
36206
|
+
var init_langwatch_api_projects = __esm({
|
|
36207
|
+
"src/langwatch-api-projects.ts"() {
|
|
36208
|
+
"use strict";
|
|
36209
|
+
init_langwatch_api();
|
|
36210
|
+
}
|
|
36211
|
+
});
|
|
36212
|
+
|
|
36213
|
+
// src/tools/list-projects.ts
|
|
36214
|
+
var list_projects_exports = {};
|
|
36215
|
+
__export(list_projects_exports, {
|
|
36216
|
+
handleListProjects: () => handleListProjects
|
|
36217
|
+
});
|
|
36218
|
+
async function handleListProjects(params) {
|
|
36219
|
+
const result = await listProjects({
|
|
36220
|
+
page: params?.page,
|
|
36221
|
+
limit: params?.limit ?? 100
|
|
36222
|
+
});
|
|
36223
|
+
const projects = result.data;
|
|
36224
|
+
if (!Array.isArray(projects) || projects.length === 0) {
|
|
36225
|
+
return "No projects found in this organization.\n\n> Tip: Use `platform_create_project` to create your first project.";
|
|
36226
|
+
}
|
|
36227
|
+
const lines = [];
|
|
36228
|
+
lines.push(`# Projects (${result.pagination.total} total)
|
|
36229
|
+
`);
|
|
36230
|
+
for (const p of projects) {
|
|
36231
|
+
lines.push(`## ${p.name}`);
|
|
36232
|
+
lines.push(`**ID**: ${p.id}`);
|
|
36233
|
+
lines.push(`**Slug**: ${p.slug}`);
|
|
36234
|
+
lines.push(`**Language**: ${p.language}`);
|
|
36235
|
+
lines.push(`**Framework**: ${p.framework}`);
|
|
36236
|
+
lines.push(`**Updated**: ${p.updatedAt}`);
|
|
36237
|
+
lines.push("");
|
|
36238
|
+
}
|
|
36239
|
+
if (result.pagination.totalPages > 1) {
|
|
36240
|
+
lines.push(
|
|
36241
|
+
`> Showing page ${result.pagination.page} of ${result.pagination.totalPages}. Use \`page\` parameter to navigate.`
|
|
36242
|
+
);
|
|
36243
|
+
}
|
|
36244
|
+
lines.push(
|
|
36245
|
+
"> Use `platform_get_project` with the ID to see full project details."
|
|
36246
|
+
);
|
|
36247
|
+
return lines.join("\n");
|
|
36248
|
+
}
|
|
36249
|
+
var init_list_projects = __esm({
|
|
36250
|
+
"src/tools/list-projects.ts"() {
|
|
36251
|
+
"use strict";
|
|
36252
|
+
init_langwatch_api_projects();
|
|
36253
|
+
}
|
|
36254
|
+
});
|
|
36255
|
+
|
|
36256
|
+
// src/tools/get-project.ts
|
|
36257
|
+
var get_project_exports = {};
|
|
36258
|
+
__export(get_project_exports, {
|
|
36259
|
+
handleGetProject: () => handleGetProject
|
|
36260
|
+
});
|
|
36261
|
+
async function handleGetProject(params) {
|
|
36262
|
+
const project = await getProject(params.id);
|
|
36263
|
+
const lines = [];
|
|
36264
|
+
lines.push(`# ${project.name}
|
|
36265
|
+
`);
|
|
36266
|
+
lines.push(`**ID**: ${project.id}`);
|
|
36267
|
+
lines.push(`**Slug**: ${project.slug}`);
|
|
36268
|
+
lines.push(`**Language**: ${project.language}`);
|
|
36269
|
+
lines.push(`**Framework**: ${project.framework}`);
|
|
36270
|
+
lines.push(`**Team ID**: ${project.teamId}`);
|
|
36271
|
+
lines.push(`**PII Redaction**: ${project.piiRedactionLevel}`);
|
|
36272
|
+
lines.push(`**Created**: ${project.createdAt}`);
|
|
36273
|
+
lines.push(`**Updated**: ${project.updatedAt}`);
|
|
36274
|
+
return lines.join("\n");
|
|
36275
|
+
}
|
|
36276
|
+
var init_get_project = __esm({
|
|
36277
|
+
"src/tools/get-project.ts"() {
|
|
36278
|
+
"use strict";
|
|
36279
|
+
init_langwatch_api_projects();
|
|
36280
|
+
}
|
|
36281
|
+
});
|
|
36282
|
+
|
|
36283
|
+
// src/tools/create-project.ts
|
|
36284
|
+
var create_project_exports = {};
|
|
36285
|
+
__export(create_project_exports, {
|
|
36286
|
+
handleCreateProject: () => handleCreateProject
|
|
36287
|
+
});
|
|
36288
|
+
async function handleCreateProject(params) {
|
|
36289
|
+
if (!params.teamId && !params.newTeamName) {
|
|
36290
|
+
return "Error: You must provide either `teamId` (to add to an existing team) or `newTeamName` (to create a new team for this project).";
|
|
36291
|
+
}
|
|
36292
|
+
const result = await createProject({
|
|
36293
|
+
name: params.name,
|
|
36294
|
+
language: params.language,
|
|
36295
|
+
framework: params.framework,
|
|
36296
|
+
teamId: params.teamId,
|
|
36297
|
+
newTeamName: params.newTeamName
|
|
36298
|
+
});
|
|
36299
|
+
if (!result.serviceApiKey) {
|
|
36300
|
+
throw new Error(
|
|
36301
|
+
"API did not return a service API key. The project was created but the key is missing \u2014 check the LangWatch UI."
|
|
36302
|
+
);
|
|
36303
|
+
}
|
|
36304
|
+
const lines = [];
|
|
36305
|
+
lines.push(`Project created successfully!
|
|
36306
|
+
`);
|
|
36307
|
+
lines.push(`**Name**: ${result.name}`);
|
|
36308
|
+
lines.push(`**ID**: ${result.id}`);
|
|
36309
|
+
lines.push(`**Slug**: ${result.slug}`);
|
|
36310
|
+
lines.push(`**Language**: ${result.language}`);
|
|
36311
|
+
lines.push(`**Framework**: ${result.framework}`);
|
|
36312
|
+
lines.push("");
|
|
36313
|
+
lines.push(`**Service API Key**: \`${result.serviceApiKey}\``);
|
|
36314
|
+
lines.push(`**Service API Key ID**: ${result.serviceApiKeyId}`);
|
|
36315
|
+
lines.push("");
|
|
36316
|
+
lines.push(
|
|
36317
|
+
"> \u26A0\uFE0F Save the service API key now \u2014 it will not be shown again. Use it as `LANGWATCH_API_KEY` to authenticate project-scoped operations."
|
|
36318
|
+
);
|
|
36319
|
+
return lines.join("\n");
|
|
36320
|
+
}
|
|
36321
|
+
var init_create_project = __esm({
|
|
36322
|
+
"src/tools/create-project.ts"() {
|
|
36323
|
+
"use strict";
|
|
36324
|
+
init_langwatch_api_projects();
|
|
36325
|
+
}
|
|
36326
|
+
});
|
|
36327
|
+
|
|
36328
|
+
// src/tools/update-project.ts
|
|
36329
|
+
var update_project_exports = {};
|
|
36330
|
+
__export(update_project_exports, {
|
|
36331
|
+
handleUpdateProject: () => handleUpdateProject
|
|
36332
|
+
});
|
|
36333
|
+
async function handleUpdateProject(params) {
|
|
36334
|
+
const project = await updateProject(params);
|
|
36335
|
+
const lines = [];
|
|
36336
|
+
lines.push(`Project updated successfully!
|
|
36337
|
+
`);
|
|
36338
|
+
lines.push(`**Name**: ${project.name}`);
|
|
36339
|
+
lines.push(`**ID**: ${project.id}`);
|
|
36340
|
+
lines.push(`**Slug**: ${project.slug}`);
|
|
36341
|
+
lines.push(`**Language**: ${project.language}`);
|
|
36342
|
+
lines.push(`**Framework**: ${project.framework}`);
|
|
36343
|
+
lines.push(`**PII Redaction**: ${project.piiRedactionLevel}`);
|
|
36344
|
+
return lines.join("\n");
|
|
36345
|
+
}
|
|
36346
|
+
var init_update_project = __esm({
|
|
36347
|
+
"src/tools/update-project.ts"() {
|
|
36348
|
+
"use strict";
|
|
36349
|
+
init_langwatch_api_projects();
|
|
36350
|
+
}
|
|
36351
|
+
});
|
|
36352
|
+
|
|
36353
|
+
// src/tools/archive-project.ts
|
|
36354
|
+
var archive_project_exports = {};
|
|
36355
|
+
__export(archive_project_exports, {
|
|
36356
|
+
handleArchiveProject: () => handleArchiveProject
|
|
36357
|
+
});
|
|
36358
|
+
async function handleArchiveProject(params) {
|
|
36359
|
+
const result = await archiveProject(params.id);
|
|
36360
|
+
return `Project archived successfully.
|
|
36361
|
+
|
|
36362
|
+
**Name**: ${result.name}
|
|
36363
|
+
**ID**: ${result.id}
|
|
36364
|
+
**Archived At**: ${result.archivedAt}`;
|
|
36365
|
+
}
|
|
36366
|
+
var init_archive_project = __esm({
|
|
36367
|
+
"src/tools/archive-project.ts"() {
|
|
36368
|
+
"use strict";
|
|
36369
|
+
init_langwatch_api_projects();
|
|
36370
|
+
}
|
|
36371
|
+
});
|
|
36372
|
+
|
|
36373
|
+
// src/langwatch-api-api-keys.ts
|
|
36374
|
+
async function listApiKeys() {
|
|
36375
|
+
return makeRequest("GET", "/api/api-keys");
|
|
36376
|
+
}
|
|
36377
|
+
async function createApiKey(data) {
|
|
36378
|
+
return makeRequest("POST", "/api/api-keys", data);
|
|
36379
|
+
}
|
|
36380
|
+
async function revokeApiKey(id) {
|
|
36381
|
+
return makeRequest(
|
|
36382
|
+
"DELETE",
|
|
36383
|
+
`/api/api-keys/${encodeURIComponent(id)}`
|
|
36384
|
+
);
|
|
36385
|
+
}
|
|
36386
|
+
var init_langwatch_api_api_keys = __esm({
|
|
36387
|
+
"src/langwatch-api-api-keys.ts"() {
|
|
36388
|
+
"use strict";
|
|
36389
|
+
init_langwatch_api();
|
|
36390
|
+
}
|
|
36391
|
+
});
|
|
36392
|
+
|
|
36393
|
+
// src/tools/list-api-keys.ts
|
|
36394
|
+
var list_api_keys_exports = {};
|
|
36395
|
+
__export(list_api_keys_exports, {
|
|
36396
|
+
handleListApiKeys: () => handleListApiKeys
|
|
36397
|
+
});
|
|
36398
|
+
async function handleListApiKeys() {
|
|
36399
|
+
const result = await listApiKeys();
|
|
36400
|
+
const keys = result.data;
|
|
36401
|
+
if (!Array.isArray(keys) || keys.length === 0) {
|
|
36402
|
+
return "No API keys found.\n\n> Tip: Use `platform_create_api_key` to create a new key.";
|
|
36403
|
+
}
|
|
36404
|
+
const lines = [];
|
|
36405
|
+
lines.push(`# API Keys (${keys.length} total)
|
|
36406
|
+
`);
|
|
36407
|
+
for (const k of keys) {
|
|
36408
|
+
const status = k.revokedAt ? "REVOKED" : k.expiresAt && new Date(k.expiresAt) < /* @__PURE__ */ new Date() ? "EXPIRED" : "ACTIVE";
|
|
36409
|
+
lines.push(`## ${k.name}`);
|
|
36410
|
+
lines.push(`**ID**: ${k.id}`);
|
|
36411
|
+
lines.push(`**Status**: ${status}`);
|
|
36412
|
+
if (k.description) lines.push(`**Description**: ${k.description}`);
|
|
36413
|
+
if (k.expiresAt) lines.push(`**Expires**: ${k.expiresAt}`);
|
|
36414
|
+
if (k.lastUsedAt) lines.push(`**Last Used**: ${k.lastUsedAt}`);
|
|
36415
|
+
lines.push(`**Created**: ${k.createdAt}`);
|
|
36416
|
+
if (k.roleBindings.length > 0) {
|
|
36417
|
+
const bindings = k.roleBindings.map((rb) => `${rb.role} on ${rb.scopeType}:${rb.scopeId}`).join(", ");
|
|
36418
|
+
lines.push(`**Bindings**: ${bindings}`);
|
|
36419
|
+
}
|
|
36420
|
+
lines.push("");
|
|
36421
|
+
}
|
|
36422
|
+
return lines.join("\n");
|
|
36423
|
+
}
|
|
36424
|
+
var init_list_api_keys = __esm({
|
|
36425
|
+
"src/tools/list-api-keys.ts"() {
|
|
36426
|
+
"use strict";
|
|
36427
|
+
init_langwatch_api_api_keys();
|
|
36428
|
+
}
|
|
36429
|
+
});
|
|
36430
|
+
|
|
36431
|
+
// src/tools/create-api-key.ts
|
|
36432
|
+
var create_api_key_exports = {};
|
|
36433
|
+
__export(create_api_key_exports, {
|
|
36434
|
+
handleCreateApiKey: () => handleCreateApiKey
|
|
36435
|
+
});
|
|
36436
|
+
async function handleCreateApiKey(params) {
|
|
36437
|
+
const result = await createApiKey(params);
|
|
36438
|
+
const lines = [];
|
|
36439
|
+
lines.push(`API key created successfully!
|
|
36440
|
+
`);
|
|
36441
|
+
lines.push(`**Name**: ${result.apiKey.name}`);
|
|
36442
|
+
lines.push(`**ID**: ${result.apiKey.id}`);
|
|
36443
|
+
lines.push(`**Token**: \`${result.token}\``);
|
|
36444
|
+
lines.push(`**Created**: ${result.apiKey.createdAt}`);
|
|
36445
|
+
lines.push("");
|
|
36446
|
+
lines.push(
|
|
36447
|
+
"> \u26A0\uFE0F Save this token now \u2014 it will not be shown again."
|
|
36448
|
+
);
|
|
36449
|
+
return lines.join("\n");
|
|
36450
|
+
}
|
|
36451
|
+
var init_create_api_key = __esm({
|
|
36452
|
+
"src/tools/create-api-key.ts"() {
|
|
36453
|
+
"use strict";
|
|
36454
|
+
init_langwatch_api_api_keys();
|
|
36455
|
+
}
|
|
36456
|
+
});
|
|
36457
|
+
|
|
36458
|
+
// src/tools/revoke-api-key.ts
|
|
36459
|
+
var revoke_api_key_exports = {};
|
|
36460
|
+
__export(revoke_api_key_exports, {
|
|
36461
|
+
handleRevokeApiKey: () => handleRevokeApiKey
|
|
36462
|
+
});
|
|
36463
|
+
async function handleRevokeApiKey(params) {
|
|
36464
|
+
await revokeApiKey(params.id);
|
|
36465
|
+
return `API key ${params.id} revoked successfully.`;
|
|
36466
|
+
}
|
|
36467
|
+
var init_revoke_api_key = __esm({
|
|
36468
|
+
"src/tools/revoke-api-key.ts"() {
|
|
36469
|
+
"use strict";
|
|
36470
|
+
init_langwatch_api_api_keys();
|
|
36471
|
+
}
|
|
36472
|
+
});
|
|
36473
|
+
|
|
35829
36474
|
// src/create-mcp-server.ts
|
|
35830
36475
|
function createMcpServer() {
|
|
35831
36476
|
const server = new McpServer({
|
|
@@ -37087,30 +37732,86 @@ ${lines.join("\n")}`
|
|
|
37087
37732
|
})
|
|
37088
37733
|
);
|
|
37089
37734
|
server.tool(
|
|
37090
|
-
"
|
|
37091
|
-
"Start an
|
|
37735
|
+
"platform_run_experiment",
|
|
37736
|
+
"Start an experiment run by slug. Returns a run ID for polling status.",
|
|
37737
|
+
{
|
|
37738
|
+
slug: external_exports.string().describe("The experiment slug to run")
|
|
37739
|
+
},
|
|
37740
|
+
withToolLogging("platform_run_experiment", async (params) => {
|
|
37741
|
+
requireApiKey();
|
|
37742
|
+
return {
|
|
37743
|
+
content: [{ type: "text", text: await handleRunExperiment(params) }]
|
|
37744
|
+
};
|
|
37745
|
+
})
|
|
37746
|
+
);
|
|
37747
|
+
server.tool(
|
|
37748
|
+
"platform_experiment_status",
|
|
37749
|
+
"Check the status of an experiment run. Returns progress and summary when completed.",
|
|
37750
|
+
{
|
|
37751
|
+
runId: external_exports.string().describe("The run ID returned from platform_run_experiment")
|
|
37752
|
+
},
|
|
37753
|
+
withToolLogging("platform_experiment_status", async (params) => {
|
|
37754
|
+
requireApiKey();
|
|
37755
|
+
return {
|
|
37756
|
+
content: [{ type: "text", text: await handleExperimentStatus(params) }]
|
|
37757
|
+
};
|
|
37758
|
+
})
|
|
37759
|
+
);
|
|
37760
|
+
server.tool(
|
|
37761
|
+
"platform_experiment_list",
|
|
37762
|
+
"List experiments configured in the LangWatch project. Returns a markdown table with each experiment's slug, name, last-run timestamp, and run count. Use this to discover experiment slugs before drilling into runs with platform_experiment_list_runs. Output is capped (default 25, max 100) to protect the agent's context window.",
|
|
37763
|
+
{
|
|
37764
|
+
limit: external_exports.number().int().positive().max(100).optional().describe(
|
|
37765
|
+
"Maximum number of experiments to include (default 25, hard-capped at 100)"
|
|
37766
|
+
)
|
|
37767
|
+
},
|
|
37768
|
+
withToolLogging("platform_experiment_list", async (params) => {
|
|
37769
|
+
requireApiKey();
|
|
37770
|
+
return {
|
|
37771
|
+
content: [{ type: "text", text: await handleExperimentList(params) }]
|
|
37772
|
+
};
|
|
37773
|
+
})
|
|
37774
|
+
);
|
|
37775
|
+
server.tool(
|
|
37776
|
+
"platform_experiment_list_runs",
|
|
37777
|
+
"List experiment runs for a specific experiment slug. Returns a markdown table with each run's id, status, started/finished timestamps, and pass-rate summary. Use this to discover run ids before calling platform_experiment_results. Output is capped (default 25, max 100) to protect the agent's context window.",
|
|
37092
37778
|
{
|
|
37093
|
-
|
|
37779
|
+
experimentSlug: external_exports.string().describe(
|
|
37780
|
+
"Experiment slug from platform_experiment_list (e.g. 'checkout-flow')"
|
|
37781
|
+
),
|
|
37782
|
+
limit: external_exports.number().int().positive().max(100).optional().describe(
|
|
37783
|
+
"Maximum number of runs to include (default 25, hard-capped at 100)"
|
|
37784
|
+
)
|
|
37094
37785
|
},
|
|
37095
|
-
withToolLogging("
|
|
37786
|
+
withToolLogging("platform_experiment_list_runs", async (params) => {
|
|
37096
37787
|
requireApiKey();
|
|
37097
|
-
const { handleRunEvaluation: handleRunEvaluation2 } = await Promise.resolve().then(() => (init_run_evaluation(), run_evaluation_exports));
|
|
37098
37788
|
return {
|
|
37099
|
-
content: [{ type: "text", text: await
|
|
37789
|
+
content: [{ type: "text", text: await handleExperimentListRuns(params) }]
|
|
37100
37790
|
};
|
|
37101
37791
|
})
|
|
37102
37792
|
);
|
|
37103
37793
|
server.tool(
|
|
37104
|
-
"
|
|
37105
|
-
"
|
|
37794
|
+
"platform_experiment_results",
|
|
37795
|
+
"Fetch per-row results for a completed experiment run so you can debug evaluator scores and missed rows. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window \u2014 narrow with `filter: 'failed'` or `evaluator` to see what matters.",
|
|
37106
37796
|
{
|
|
37107
|
-
runId: external_exports.string().describe("The run ID returned from
|
|
37797
|
+
runId: external_exports.string().describe("The run ID returned from platform_run_experiment"),
|
|
37798
|
+
experimentSlug: external_exports.string().optional().describe(
|
|
37799
|
+
"Experiment slug \u2014 required when fetching results for runs older than 24h (Redis state may have expired). Use platform_experiment_list_runs to find the slug."
|
|
37800
|
+
),
|
|
37801
|
+
filter: external_exports.enum(["all", "failed"]).optional().describe(
|
|
37802
|
+
"Filter rows: 'failed' shows only rows that errored or failed an evaluation; 'all' (default) shows everything"
|
|
37803
|
+
),
|
|
37804
|
+
evaluator: external_exports.string().optional().describe("Show only this evaluator's results (matched on `evaluator` field)"),
|
|
37805
|
+
limit: external_exports.number().int().positive().max(50).optional().describe(
|
|
37806
|
+
"Maximum number of rows to include in the markdown output (default 50, hard-capped at 50)."
|
|
37807
|
+
)
|
|
37108
37808
|
},
|
|
37109
|
-
withToolLogging("
|
|
37809
|
+
withToolLogging("platform_experiment_results", async (params) => {
|
|
37110
37810
|
requireApiKey();
|
|
37111
|
-
const { handleEvaluationStatus: handleEvaluationStatus2 } = await Promise.resolve().then(() => (init_run_evaluation(), run_evaluation_exports));
|
|
37112
37811
|
return {
|
|
37113
|
-
content: [
|
|
37812
|
+
content: [
|
|
37813
|
+
{ type: "text", text: await handleExperimentResults(params) }
|
|
37814
|
+
]
|
|
37114
37815
|
};
|
|
37115
37816
|
})
|
|
37116
37817
|
);
|
|
@@ -37269,6 +37970,141 @@ ${lines.join("\n")}`
|
|
|
37269
37970
|
};
|
|
37270
37971
|
}
|
|
37271
37972
|
);
|
|
37973
|
+
server.tool(
|
|
37974
|
+
"platform_list_projects",
|
|
37975
|
+
"List all projects in your LangWatch organization. Requires an org-level API key.",
|
|
37976
|
+
{
|
|
37977
|
+
page: external_exports.number().int().positive().optional().describe("Page number (default: 1)"),
|
|
37978
|
+
limit: external_exports.number().int().positive().max(1e3).optional().describe("Results per page (default: 100)")
|
|
37979
|
+
},
|
|
37980
|
+
withToolLogging("platform_list_projects", async (params) => {
|
|
37981
|
+
requireApiKey();
|
|
37982
|
+
const { handleListProjects: handleListProjects2 } = await Promise.resolve().then(() => (init_list_projects(), list_projects_exports));
|
|
37983
|
+
return {
|
|
37984
|
+
content: [{ type: "text", text: await handleListProjects2(params) }]
|
|
37985
|
+
};
|
|
37986
|
+
})
|
|
37987
|
+
);
|
|
37988
|
+
server.tool(
|
|
37989
|
+
"platform_get_project",
|
|
37990
|
+
"Get details of a specific project by its ID.",
|
|
37991
|
+
{
|
|
37992
|
+
id: external_exports.string().describe("The project ID")
|
|
37993
|
+
},
|
|
37994
|
+
withToolLogging("platform_get_project", async (params) => {
|
|
37995
|
+
requireApiKey();
|
|
37996
|
+
const { handleGetProject: handleGetProject2 } = await Promise.resolve().then(() => (init_get_project(), get_project_exports));
|
|
37997
|
+
return {
|
|
37998
|
+
content: [{ type: "text", text: await handleGetProject2(params) }]
|
|
37999
|
+
};
|
|
38000
|
+
})
|
|
38001
|
+
);
|
|
38002
|
+
server.tool(
|
|
38003
|
+
"platform_create_project",
|
|
38004
|
+
`Create a new project in your LangWatch organization. Returns a one-time service API key for the project.
|
|
38005
|
+
|
|
38006
|
+
You must provide either teamId (to add the project to an existing team) or newTeamName (to create a new team for the project).`,
|
|
38007
|
+
{
|
|
38008
|
+
name: external_exports.string().describe("Project name"),
|
|
38009
|
+
language: external_exports.string().describe('Programming language (e.g. "python", "typescript")'),
|
|
38010
|
+
framework: external_exports.string().describe('Framework (e.g. "openai", "langchain", "custom")'),
|
|
38011
|
+
teamId: external_exports.string().optional().describe("ID of an existing team to add the project to"),
|
|
38012
|
+
newTeamName: external_exports.string().optional().describe("Name for a new team to create for this project")
|
|
38013
|
+
},
|
|
38014
|
+
withToolLogging("platform_create_project", async (params) => {
|
|
38015
|
+
requireApiKey();
|
|
38016
|
+
const { handleCreateProject: handleCreateProject2 } = await Promise.resolve().then(() => (init_create_project(), create_project_exports));
|
|
38017
|
+
return {
|
|
38018
|
+
content: [{ type: "text", text: await handleCreateProject2(params) }]
|
|
38019
|
+
};
|
|
38020
|
+
})
|
|
38021
|
+
);
|
|
38022
|
+
server.tool(
|
|
38023
|
+
"platform_update_project",
|
|
38024
|
+
"Update an existing project's name, language, framework, or PII redaction level.",
|
|
38025
|
+
{
|
|
38026
|
+
id: external_exports.string().describe("The project ID to update"),
|
|
38027
|
+
name: external_exports.string().optional().describe("New project name"),
|
|
38028
|
+
language: external_exports.string().optional().describe("New programming language"),
|
|
38029
|
+
framework: external_exports.string().optional().describe("New framework"),
|
|
38030
|
+
piiRedactionLevel: external_exports.enum(["STRICT", "ESSENTIAL", "DISABLED"]).optional().describe("PII redaction level")
|
|
38031
|
+
},
|
|
38032
|
+
withToolLogging("platform_update_project", async (params) => {
|
|
38033
|
+
requireApiKey();
|
|
38034
|
+
const { handleUpdateProject: handleUpdateProject2 } = await Promise.resolve().then(() => (init_update_project(), update_project_exports));
|
|
38035
|
+
return {
|
|
38036
|
+
content: [{ type: "text", text: await handleUpdateProject2(params) }]
|
|
38037
|
+
};
|
|
38038
|
+
})
|
|
38039
|
+
);
|
|
38040
|
+
server.tool(
|
|
38041
|
+
"platform_archive_project",
|
|
38042
|
+
"Archive (soft-delete) a project. The project can be restored later if needed.",
|
|
38043
|
+
{
|
|
38044
|
+
id: external_exports.string().describe("The project ID to archive")
|
|
38045
|
+
},
|
|
38046
|
+
withToolLogging("platform_archive_project", async (params) => {
|
|
38047
|
+
requireApiKey();
|
|
38048
|
+
const { handleArchiveProject: handleArchiveProject2 } = await Promise.resolve().then(() => (init_archive_project(), archive_project_exports));
|
|
38049
|
+
return {
|
|
38050
|
+
content: [{ type: "text", text: await handleArchiveProject2(params) }]
|
|
38051
|
+
};
|
|
38052
|
+
})
|
|
38053
|
+
);
|
|
38054
|
+
server.tool(
|
|
38055
|
+
"platform_list_api_keys",
|
|
38056
|
+
"List all API keys in your LangWatch organization. Shows key metadata, status (active/revoked/expired), and role bindings. API key tokens are never returned.",
|
|
38057
|
+
{},
|
|
38058
|
+
withToolLogging("platform_list_api_keys", async () => {
|
|
38059
|
+
requireApiKey();
|
|
38060
|
+
const { handleListApiKeys: handleListApiKeys2 } = await Promise.resolve().then(() => (init_list_api_keys(), list_api_keys_exports));
|
|
38061
|
+
return {
|
|
38062
|
+
content: [{ type: "text", text: await handleListApiKeys2() }]
|
|
38063
|
+
};
|
|
38064
|
+
})
|
|
38065
|
+
);
|
|
38066
|
+
server.tool(
|
|
38067
|
+
"platform_create_api_key",
|
|
38068
|
+
`Create a new API key. Two types available:
|
|
38069
|
+
|
|
38070
|
+
- **personal**: Tied to a user, requires explicit role bindings (e.g. ADMIN on PROJECT:proj_123).
|
|
38071
|
+
- **service**: Not tied to a user. Use projectIds for project-scoped service keys, or omit for org-wide access.
|
|
38072
|
+
|
|
38073
|
+
The token is returned once and cannot be retrieved again.`,
|
|
38074
|
+
{
|
|
38075
|
+
keyType: external_exports.enum(["personal", "service"]).describe("Key type: 'personal' (user-tied) or 'service' (headless)"),
|
|
38076
|
+
name: external_exports.string().describe("Key name (max 100 chars)"),
|
|
38077
|
+
description: external_exports.string().optional().describe("Key description (max 500 chars)"),
|
|
38078
|
+
expiresAt: external_exports.string().optional().describe("Expiration date (ISO string)"),
|
|
38079
|
+
bindings: external_exports.array(external_exports.object({
|
|
38080
|
+
role: external_exports.enum(["ADMIN", "MEMBER", "VIEWER"]).describe("Role"),
|
|
38081
|
+
scopeType: external_exports.enum(["ORGANIZATION", "TEAM", "PROJECT"]).describe("Scope type"),
|
|
38082
|
+
scopeId: external_exports.string().describe("Scope ID (org, team, or project ID)")
|
|
38083
|
+
})).optional().describe("Role bindings (required for personal keys)"),
|
|
38084
|
+
projectIds: external_exports.array(external_exports.string()).optional().describe("Project IDs for service keys (creates ADMIN bindings per project)")
|
|
38085
|
+
},
|
|
38086
|
+
withToolLogging("platform_create_api_key", async (params) => {
|
|
38087
|
+
requireApiKey();
|
|
38088
|
+
const { handleCreateApiKey: handleCreateApiKey2 } = await Promise.resolve().then(() => (init_create_api_key(), create_api_key_exports));
|
|
38089
|
+
return {
|
|
38090
|
+
content: [{ type: "text", text: await handleCreateApiKey2(params) }]
|
|
38091
|
+
};
|
|
38092
|
+
})
|
|
38093
|
+
);
|
|
38094
|
+
server.tool(
|
|
38095
|
+
"platform_revoke_api_key",
|
|
38096
|
+
"Revoke an API key. Once revoked, the key can no longer be used for authentication.",
|
|
38097
|
+
{
|
|
38098
|
+
id: external_exports.string().describe("The API key ID to revoke")
|
|
38099
|
+
},
|
|
38100
|
+
withToolLogging("platform_revoke_api_key", async (params) => {
|
|
38101
|
+
requireApiKey();
|
|
38102
|
+
const { handleRevokeApiKey: handleRevokeApiKey2 } = await Promise.resolve().then(() => (init_revoke_api_key(), revoke_api_key_exports));
|
|
38103
|
+
return {
|
|
38104
|
+
content: [{ type: "text", text: await handleRevokeApiKey2(params) }]
|
|
38105
|
+
};
|
|
38106
|
+
})
|
|
38107
|
+
);
|
|
37272
38108
|
}
|
|
37273
38109
|
var modelSchema;
|
|
37274
38110
|
var init_create_mcp_server = __esm({
|
|
@@ -37279,6 +38115,10 @@ var init_create_mcp_server = __esm({
|
|
|
37279
38115
|
init_package();
|
|
37280
38116
|
init_config();
|
|
37281
38117
|
init_create_dataset();
|
|
38118
|
+
init_get_experiment_results();
|
|
38119
|
+
init_list_experiment_runs();
|
|
38120
|
+
init_list_experiments();
|
|
38121
|
+
init_run_experiment();
|
|
37282
38122
|
modelSchema = external_exports.string().describe(
|
|
37283
38123
|
'Model in "provider/model-name" format, e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929"'
|
|
37284
38124
|
);
|