harness-mcp-v2 3.2.32 → 3.2.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +6 -5
  2. package/build/data/schemas/v0/pipeline.d.ts.map +1 -1
  3. package/build/data/schemas/v0/pipeline.js +442 -134
  4. package/build/data/schemas/v0/pipeline.js.map +1 -1
  5. package/build/data/schemas/v0/template.d.ts.map +1 -1
  6. package/build/data/schemas/v0/template.js +1338 -1019
  7. package/build/data/schemas/v0/template.js.map +1 -1
  8. package/build/data/schemas/v1/pipeline.d.ts.map +1 -1
  9. package/build/data/schemas/v1/pipeline.js +32 -0
  10. package/build/data/schemas/v1/pipeline.js.map +1 -1
  11. package/build/data/schemas/v1/template.d.ts.map +1 -1
  12. package/build/data/schemas/v1/template.js +32 -0
  13. package/build/data/schemas/v1/template.js.map +1 -1
  14. package/build/registry/extractors.d.ts +7 -0
  15. package/build/registry/extractors.d.ts.map +1 -1
  16. package/build/registry/extractors.js +36 -1
  17. package/build/registry/extractors.js.map +1 -1
  18. package/build/registry/remote-git-preflight.d.ts +29 -0
  19. package/build/registry/remote-git-preflight.d.ts.map +1 -0
  20. package/build/registry/remote-git-preflight.js +148 -0
  21. package/build/registry/remote-git-preflight.js.map +1 -0
  22. package/build/registry/toolsets/ai-evals.d.ts.map +1 -1
  23. package/build/registry/toolsets/ai-evals.js +105 -70
  24. package/build/registry/toolsets/ai-evals.js.map +1 -1
  25. package/build/registry/toolsets/ccm.d.ts.map +1 -1
  26. package/build/registry/toolsets/ccm.js +50 -40
  27. package/build/registry/toolsets/ccm.js.map +1 -1
  28. package/build/registry/toolsets/connectors.d.ts.map +1 -1
  29. package/build/registry/toolsets/connectors.js +53 -0
  30. package/build/registry/toolsets/connectors.js.map +1 -1
  31. package/build/registry/toolsets/incidents.d.ts.map +1 -1
  32. package/build/registry/toolsets/incidents.js +78 -2
  33. package/build/registry/toolsets/incidents.js.map +1 -1
  34. package/build/registry/toolsets/observability-evaluations.d.ts.map +1 -1
  35. package/build/registry/toolsets/observability-evaluations.js +31 -17
  36. package/build/registry/toolsets/observability-evaluations.js.map +1 -1
  37. package/build/registry/toolsets/pipelines.d.ts.map +1 -1
  38. package/build/registry/toolsets/pipelines.js +14 -131
  39. package/build/registry/toolsets/pipelines.js.map +1 -1
  40. package/build/registry/toolsets/templates.d.ts.map +1 -1
  41. package/build/registry/toolsets/templates.js +83 -10
  42. package/build/registry/toolsets/templates.js.map +1 -1
  43. package/build/tools/harness-search.d.ts.map +1 -1
  44. package/build/tools/harness-search.js +4 -1
  45. package/build/tools/harness-search.js.map +1 -1
  46. package/build/utils/ccm-time.d.ts +35 -0
  47. package/build/utils/ccm-time.d.ts.map +1 -0
  48. package/build/utils/ccm-time.js +90 -0
  49. package/build/utils/ccm-time.js.map +1 -0
  50. package/npm-shrinkwrap.json +206 -265
  51. package/package.json +8 -6
@@ -5,6 +5,8 @@ const LLM_CONNECTOR_TYPES = new Set(["OpenAi", "OpenAI", "Anthropic", "AzureOpen
5
5
  const JUDGE_BACKED_METRIC_TYPES = new Set(["llm", "ai_judge"]);
6
6
  const TARGET_TYPES = new Set(["prompt", "agent", "precomputed"]);
7
7
  const AGENT_METHODS = new Set(["GET", "POST", "PUT"]);
8
+ const UUID_404_HINT = "Use the id from harness_list for get, update, delete, and execute actions; identifiers and display names are not accepted. " +
9
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL.";
8
10
  function scopeOrThrow(input, config) {
9
11
  const org = input.org_id ?? config.HARNESS_ORG ?? "";
10
12
  const project = input.project_id ?? config.HARNESS_PROJECT ?? "";
@@ -36,10 +38,28 @@ function requireBody(input, operation) {
36
38
  function requireUuid(value, field) {
37
39
  const id = nonEmptyString(value);
38
40
  if (!id || !UUID_PATTERN.test(id)) {
39
- throw new Error(`${field} must be a UUID returned by the matching AI Evals list/create call; do not invent an identifier.`);
41
+ throw new Error(`${field} must be the id or uuid from harness_list for this AI Evals resource, not its identifier or name.`);
40
42
  }
41
43
  return id;
42
44
  }
45
+ function pathSegment(value, field) {
46
+ const id = nonEmptyString(value);
47
+ if (!id)
48
+ throw new Error(`${field} must be a non-empty path identifier.`);
49
+ return encodeURIComponent(id);
50
+ }
51
+ function uuidPathSegment(input, field) {
52
+ return encodeURIComponent(requireUuid(input[field], field));
53
+ }
54
+ function datasetPath(input, config) {
55
+ const datasetId = nonEmptyString(input.dataset_id);
56
+ if (!datasetId)
57
+ throw new Error("dataset_id must be a non-empty UUID or dataset identifier.");
58
+ const root = `${base(input, config)}/dataset`;
59
+ return UUID_PATTERN.test(datasetId)
60
+ ? `${root}/${encodeURIComponent(datasetId)}`
61
+ : `${root}/by-identifier/${encodeURIComponent(datasetId)}`;
62
+ }
43
63
  function preflightScope({ input, registry }) {
44
64
  const org_id = nonEmptyString(input.org_id) ?? registry.orgId;
45
65
  const project_id = nonEmptyString(input.project_id) ?? registry.projectId;
@@ -1101,7 +1121,9 @@ export const aiEvalsToolset = {
1101
1121
  scopeOptional: true,
1102
1122
  headerBasedScoping: true,
1103
1123
  identifierFields: ["dataset_id"],
1104
- diagnosticHint: "Dataset items require 'input' as a JSON object (e.g. { messages: [{role:'user', content:'...'}] } or { prompt: '...' }). " +
1124
+ diagnosticHint: "For get/update/delete, use the uuid from harness_list (or use a dataset identifier only for get). " +
1125
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
1126
+ "Dataset items require 'input' as a JSON object (e.g. { messages: [{role:'user', content:'...'}] } or { prompt: '...' }). " +
1105
1127
  "Optional fields depend on metric type: 'expected_output' for correctness metrics, 'context' (string array) for RAG/groundedness metrics, " +
1106
1128
  "'expected_tools' for agent tool-use metrics. Items can be added inline on create or managed separately via eval_dataset_item.",
1107
1129
  listFilterFields: [
@@ -1125,10 +1147,10 @@ export const aiEvalsToolset = {
1125
1147
  get: {
1126
1148
  method: "GET",
1127
1149
  path: "",
1128
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}`,
1150
+ pathBuilder: datasetPath,
1129
1151
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1130
1152
  responseExtractor: passthrough,
1131
- description: "Get dataset by UUID",
1153
+ description: "Get dataset by UUID or identifier",
1132
1154
  },
1133
1155
  create: {
1134
1156
  method: "POST",
@@ -1143,7 +1165,7 @@ export const aiEvalsToolset = {
1143
1165
  update: {
1144
1166
  method: "PUT",
1145
1167
  path: "",
1146
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}`,
1168
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}`,
1147
1169
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1148
1170
  bodyBuilder: (input) => input.body ?? {},
1149
1171
  bodySchema: updateDatasetSchema,
@@ -1153,7 +1175,7 @@ export const aiEvalsToolset = {
1153
1175
  delete: {
1154
1176
  method: "DELETE",
1155
1177
  path: "",
1156
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}`,
1178
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}`,
1157
1179
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1158
1180
  responseExtractor: passthrough,
1159
1181
  description: "Delete dataset",
@@ -1163,7 +1185,7 @@ export const aiEvalsToolset = {
1163
1185
  get_by_identifier: {
1164
1186
  method: "GET",
1165
1187
  path: "",
1166
- pathBuilder: (input, config) => `${base(input, config)}/dataset/by-identifier/${encodeURIComponent(input.identifier)}`,
1188
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/by-identifier/${pathSegment(input.identifier, "identifier")}`,
1167
1189
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1168
1190
  responseExtractor: passthrough,
1169
1191
  actionDescription: "Get dataset by its unique identifier slug (not UUID). Pass identifier via params.identifier.",
@@ -1172,7 +1194,7 @@ export const aiEvalsToolset = {
1172
1194
  generate: {
1173
1195
  method: "POST",
1174
1196
  path: "",
1175
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/generate`,
1197
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/generate`,
1176
1198
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
1177
1199
  preflight: validateDatasetGeneration,
1178
1200
  bodyBuilder: bodyFromInput,
@@ -1198,7 +1220,7 @@ export const aiEvalsToolset = {
1198
1220
  list: {
1199
1221
  method: "GET",
1200
1222
  path: "",
1201
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items`,
1223
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items`,
1202
1224
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1203
1225
  queryParams: listQ,
1204
1226
  responseExtractor: aiEvalsListExtract,
@@ -1207,7 +1229,7 @@ export const aiEvalsToolset = {
1207
1229
  get: {
1208
1230
  method: "GET",
1209
1231
  path: "",
1210
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items/${input.item_id}`,
1232
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/${uuidPathSegment(input, "item_id")}`,
1211
1233
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1212
1234
  responseExtractor: passthrough,
1213
1235
  description: "Get item by UUID",
@@ -1215,7 +1237,7 @@ export const aiEvalsToolset = {
1215
1237
  create: {
1216
1238
  method: "POST",
1217
1239
  path: "",
1218
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items`,
1240
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items`,
1219
1241
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
1220
1242
  bodyBuilder: (input) => input.body ?? {},
1221
1243
  bodySchema: createDatasetItemSchema,
@@ -1225,7 +1247,7 @@ export const aiEvalsToolset = {
1225
1247
  update: {
1226
1248
  method: "PUT",
1227
1249
  path: "",
1228
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items/${input.item_id}`,
1250
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/${uuidPathSegment(input, "item_id")}`,
1229
1251
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1230
1252
  bodyBuilder: (input) => input.body ?? {},
1231
1253
  bodySchema: updateDatasetItemSchema,
@@ -1235,7 +1257,7 @@ export const aiEvalsToolset = {
1235
1257
  delete: {
1236
1258
  method: "DELETE",
1237
1259
  path: "",
1238
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items/${input.item_id}`,
1260
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/${uuidPathSegment(input, "item_id")}`,
1239
1261
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1240
1262
  responseExtractor: passthrough,
1241
1263
  description: "Delete item",
@@ -1245,7 +1267,7 @@ export const aiEvalsToolset = {
1245
1267
  bulk_upsert: {
1246
1268
  method: "PATCH",
1247
1269
  path: "",
1248
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items/bulk`,
1270
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/bulk`,
1249
1271
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
1250
1272
  bodyBuilder: bodyFromInput,
1251
1273
  bodySchema: bulkUpsertDatasetItemsSchema,
@@ -1255,7 +1277,7 @@ export const aiEvalsToolset = {
1255
1277
  bulk_delete: {
1256
1278
  method: "POST",
1257
1279
  path: "",
1258
- pathBuilder: (input, config) => `${base(input, config)}/dataset/${input.dataset_id}/items/bulk-delete`,
1280
+ pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/bulk-delete`,
1259
1281
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1260
1282
  preflight: async (ctx) => {
1261
1283
  await getScopedResource(ctx, "eval_dataset", "dataset_id", ctx.input.dataset_id);
@@ -1281,7 +1303,9 @@ export const aiEvalsToolset = {
1281
1303
  scopeOptional: true,
1282
1304
  headerBasedScoping: true,
1283
1305
  identifierFields: ["eval_id"],
1284
- diagnosticHint: "An eval requires three components: dataset_id, target_id, and metric_set_id. " +
1306
+ diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; identifiers and display names are not accepted. " +
1307
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
1308
+ "An eval requires three components: dataset_id, target_id, and metric_set_id. " +
1285
1309
  "Before creating an eval, list existing resources with harness_list for eval_dataset, eval_target, and eval_metric_set. " +
1286
1310
  "Create any missing components first. Managed evaluations cannot be created until all three are set. " +
1287
1311
  "When storage_type='git', omit dataset_id/target_id/metric_set_id (they live in the YAML at git_source.file_path).",
@@ -1315,7 +1339,7 @@ export const aiEvalsToolset = {
1315
1339
  get: {
1316
1340
  method: "GET",
1317
1341
  path: "",
1318
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}`,
1342
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}`,
1319
1343
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1320
1344
  responseExtractor: passthrough,
1321
1345
  description: "Get eval",
@@ -1334,7 +1358,7 @@ export const aiEvalsToolset = {
1334
1358
  update: {
1335
1359
  method: "PATCH",
1336
1360
  path: "",
1337
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}`,
1361
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}`,
1338
1362
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1339
1363
  preflight: async (ctx) => validateManagedEvalComposition(ctx, ctx.input, false),
1340
1364
  bodyBuilder: (input) => input.body ?? {},
@@ -1345,7 +1369,7 @@ export const aiEvalsToolset = {
1345
1369
  delete: {
1346
1370
  method: "DELETE",
1347
1371
  path: "",
1348
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}`,
1372
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}`,
1349
1373
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1350
1374
  responseExtractor: passthrough,
1351
1375
  description: "Hard-delete eval and its runs (409 if referenced by a suite)",
@@ -1355,7 +1379,7 @@ export const aiEvalsToolset = {
1355
1379
  run: {
1356
1380
  method: "POST",
1357
1381
  path: "",
1358
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}/run`,
1382
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/run`,
1359
1383
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
1360
1384
  preflight: validateEvalRun,
1361
1385
  bodyBuilder: bodyFromInput,
@@ -1366,7 +1390,7 @@ export const aiEvalsToolset = {
1366
1390
  clone: {
1367
1391
  method: "POST",
1368
1392
  path: "",
1369
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}/clone`,
1393
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/clone`,
1370
1394
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
1371
1395
  preflight: async (ctx) => {
1372
1396
  const evaluation = await getScopedResource(ctx, "evaluation", "eval_id", ctx.input.eval_id);
@@ -1382,7 +1406,7 @@ export const aiEvalsToolset = {
1382
1406
  item_history: {
1383
1407
  method: "POST",
1384
1408
  path: "",
1385
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}/items/history`,
1409
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/items/history`,
1386
1410
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1387
1411
  preflight: async (ctx) => {
1388
1412
  await getScopedResource(ctx, "evaluation", "eval_id", ctx.input.eval_id);
@@ -1410,7 +1434,7 @@ export const aiEvalsToolset = {
1410
1434
  export_yaml: {
1411
1435
  method: "GET",
1412
1436
  path: "",
1413
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}/export-yaml`,
1437
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/export-yaml`,
1414
1438
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1415
1439
  responseExtractor: passthrough,
1416
1440
  actionDescription: "Export an eval and all referenced entities as a denormalized YAML document.",
@@ -1428,6 +1452,7 @@ export const aiEvalsToolset = {
1428
1452
  scopeOptional: true,
1429
1453
  headerBasedScoping: true,
1430
1454
  identifierFields: ["run_id"],
1455
+ diagnosticHint: UUID_404_HINT,
1431
1456
  listFilterFields: [
1432
1457
  { name: "target_id", description: "Filter runs by target UUID" },
1433
1458
  ],
@@ -1449,7 +1474,7 @@ export const aiEvalsToolset = {
1449
1474
  get: {
1450
1475
  method: "GET",
1451
1476
  path: "",
1452
- pathBuilder: (input, config) => `${base(input, config)}/runs/${input.run_id}`,
1477
+ pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}`,
1453
1478
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1454
1479
  responseExtractor: passthrough,
1455
1480
  description: "Get run",
@@ -1469,7 +1494,7 @@ export const aiEvalsToolset = {
1469
1494
  rescore: {
1470
1495
  method: "POST",
1471
1496
  path: "",
1472
- pathBuilder: (input, config) => `${base(input, config)}/runs/${input.run_id}/rescore`,
1497
+ pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}/rescore`,
1473
1498
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
1474
1499
  preflight: async (ctx) => {
1475
1500
  await validateRunReference(ctx);
@@ -1485,7 +1510,7 @@ export const aiEvalsToolset = {
1485
1510
  recommendations: {
1486
1511
  method: "POST",
1487
1512
  path: "",
1488
- pathBuilder: (input, config) => `${base(input, config)}/runs/${input.run_id}/recommendations`,
1513
+ pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}/recommendations`,
1489
1514
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
1490
1515
  preflight: validateRunReference,
1491
1516
  bodyBuilder: bodyFromInput,
@@ -1509,7 +1534,7 @@ export const aiEvalsToolset = {
1509
1534
  list: {
1510
1535
  method: "GET",
1511
1536
  path: "",
1512
- pathBuilder: (input, config) => `${base(input, config)}/runs/${input.run_id}/items`,
1537
+ pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}/items`,
1513
1538
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1514
1539
  queryParams: listQ,
1515
1540
  responseExtractor: aiEvalsListExtract,
@@ -1531,7 +1556,7 @@ export const aiEvalsToolset = {
1531
1556
  list: {
1532
1557
  method: "GET",
1533
1558
  path: "",
1534
- pathBuilder: (input, config) => `${base(input, config)}/evals/${input.eval_id}/runs`,
1559
+ pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/runs`,
1535
1560
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1536
1561
  queryParams: listQ,
1537
1562
  responseExtractor: aiEvalsListExtract,
@@ -1549,7 +1574,9 @@ export const aiEvalsToolset = {
1549
1574
  scopeOptional: true,
1550
1575
  headerBasedScoping: true,
1551
1576
  identifierFields: ["metric_id"],
1552
- diagnosticHint: "Use the 'suggestions' execute action to discover appropriate metrics for a given target type and dataset shape. " +
1577
+ diagnosticHint: "Use the id from harness_list for get, update, and delete; metric names are not accepted. " +
1578
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
1579
+ "Use the 'suggestions' execute action to discover appropriate metrics for a given target type and dataset shape. " +
1553
1580
  "Metrics are added to metric sets (eval_metric_set) via eval_metric_set_entry, then referenced by evaluations. " +
1554
1581
  "Each metric response includes a 'config_schema' field (JSON Schema) describing available config options for that metric kind — " +
1555
1582
  "use harness_get to inspect a metric's config_schema before setting config on a metric set entry.",
@@ -1574,7 +1601,7 @@ export const aiEvalsToolset = {
1574
1601
  get: {
1575
1602
  method: "GET",
1576
1603
  path: "",
1577
- pathBuilder: (input, config) => `${base(input, config)}/metrics/${input.metric_id}`,
1604
+ pathBuilder: (input, config) => `${base(input, config)}/metrics/${uuidPathSegment(input, "metric_id")}`,
1578
1605
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1579
1606
  responseExtractor: passthrough,
1580
1607
  description: "Get metric",
@@ -1592,7 +1619,7 @@ export const aiEvalsToolset = {
1592
1619
  update: {
1593
1620
  method: "PATCH",
1594
1621
  path: "",
1595
- pathBuilder: (input, config) => `${base(input, config)}/metrics/${input.metric_id}`,
1622
+ pathBuilder: (input, config) => `${base(input, config)}/metrics/${uuidPathSegment(input, "metric_id")}`,
1596
1623
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1597
1624
  bodyBuilder: (input) => input.body ?? {},
1598
1625
  bodySchema: updateMetricSchema,
@@ -1602,7 +1629,7 @@ export const aiEvalsToolset = {
1602
1629
  delete: {
1603
1630
  method: "DELETE",
1604
1631
  path: "",
1605
- pathBuilder: (input, config) => `${base(input, config)}/metrics/${input.metric_id}`,
1632
+ pathBuilder: (input, config) => `${base(input, config)}/metrics/${uuidPathSegment(input, "metric_id")}`,
1606
1633
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1607
1634
  responseExtractor: passthrough,
1608
1635
  description: "Delete metric",
@@ -1636,7 +1663,9 @@ export const aiEvalsToolset = {
1636
1663
  scopeOptional: true,
1637
1664
  headerBasedScoping: true,
1638
1665
  identifierFields: ["set_id"],
1639
- diagnosticHint: "Before creating a metric set, list available metrics with harness_list(resource_type='eval_metric'). " +
1666
+ diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; metric-set names are not accepted. " +
1667
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
1668
+ "Before creating a metric set, list available metrics with harness_list(resource_type='eval_metric'). " +
1640
1669
  "Use harness_execute(resource_type='eval_metric', action='suggestions') to discover metrics appropriate for a target type. " +
1641
1670
  "If using LLM metrics (llm-as-judge), set judge_llm_config to a structured provider configuration. " +
1642
1671
  "judge_llm_connector_ref remains accepted but is DEPRECATED.",
@@ -1662,7 +1691,7 @@ export const aiEvalsToolset = {
1662
1691
  get: {
1663
1692
  method: "GET",
1664
1693
  path: "",
1665
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}`,
1694
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}`,
1666
1695
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1667
1696
  responseExtractor: passthrough,
1668
1697
  description: "Get metric set",
@@ -1681,7 +1710,7 @@ export const aiEvalsToolset = {
1681
1710
  update: {
1682
1711
  method: "PATCH",
1683
1712
  path: "",
1684
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}`,
1713
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}`,
1685
1714
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1686
1715
  preflight: async (ctx) => validateMetricSetWrite(ctx, ctx.input, true),
1687
1716
  bodyBuilder: (input) => input.body ?? {},
@@ -1692,7 +1721,7 @@ export const aiEvalsToolset = {
1692
1721
  delete: {
1693
1722
  method: "DELETE",
1694
1723
  path: "",
1695
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}`,
1724
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}`,
1696
1725
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1697
1726
  responseExtractor: passthrough,
1698
1727
  description: "Delete metric set",
@@ -1702,7 +1731,7 @@ export const aiEvalsToolset = {
1702
1731
  calibrate: {
1703
1732
  method: "POST",
1704
1733
  path: "",
1705
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}/calibrate`,
1734
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/calibrate`,
1706
1735
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
1707
1736
  bodyBuilder: bodyFromInput,
1708
1737
  bodySchema: calibrateSchema,
@@ -1712,7 +1741,7 @@ export const aiEvalsToolset = {
1712
1741
  replace_metrics: {
1713
1742
  method: "PUT",
1714
1743
  path: "",
1715
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}/metrics`,
1744
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics`,
1716
1745
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1717
1746
  preflight: validateMetricSetReplacement,
1718
1747
  bodyBuilder: (input) => {
@@ -1748,7 +1777,7 @@ export const aiEvalsToolset = {
1748
1777
  list: {
1749
1778
  method: "GET",
1750
1779
  path: "",
1751
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}/metrics`,
1780
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics`,
1752
1781
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1753
1782
  responseExtractor: aiEvalsArrayExtract,
1754
1783
  description: "List entries in a metric set",
@@ -1756,7 +1785,7 @@ export const aiEvalsToolset = {
1756
1785
  create: {
1757
1786
  method: "POST",
1758
1787
  path: "",
1759
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}/metrics`,
1788
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics`,
1760
1789
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
1761
1790
  preflight: async (ctx) => validateMetricSetEntryWrite(ctx, false),
1762
1791
  bodyBuilder: (input) => input.body ?? {},
@@ -1767,7 +1796,7 @@ export const aiEvalsToolset = {
1767
1796
  update: {
1768
1797
  method: "PATCH",
1769
1798
  path: "",
1770
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}/metrics/${input.metric_id}`,
1799
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics/${uuidPathSegment(input, "metric_id")}`,
1771
1800
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1772
1801
  preflight: async (ctx) => validateMetricSetEntryWrite(ctx, true),
1773
1802
  bodyBuilder: (input) => input.body ?? {},
@@ -1778,7 +1807,7 @@ export const aiEvalsToolset = {
1778
1807
  delete: {
1779
1808
  method: "DELETE",
1780
1809
  path: "",
1781
- pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input.set_id}/metrics/${input.metric_id}`,
1810
+ pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics/${uuidPathSegment(input, "metric_id")}`,
1782
1811
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1783
1812
  responseExtractor: passthrough,
1784
1813
  description: "Remove metric from set",
@@ -1795,7 +1824,9 @@ export const aiEvalsToolset = {
1795
1824
  scopeOptional: true,
1796
1825
  headerBasedScoping: true,
1797
1826
  identifierFields: ["suite_id"],
1798
- diagnosticHint: "A suite groups evaluations together. First create evaluations (each with dataset + target + metric set), " +
1827
+ diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; suite names are not accepted. " +
1828
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
1829
+ "A suite groups evaluations together. First create evaluations (each with dataset + target + metric set), " +
1799
1830
  "then create the suite and add evaluations via eval_suite_evaluation or the replace_evaluations execute action. " +
1800
1831
  "List existing evaluations with harness_list(resource_type='evaluation').",
1801
1832
  relatedResources: [
@@ -1816,7 +1847,7 @@ export const aiEvalsToolset = {
1816
1847
  get: {
1817
1848
  method: "GET",
1818
1849
  path: "",
1819
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}`,
1850
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}`,
1820
1851
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1821
1852
  responseExtractor: passthrough,
1822
1853
  description: "Get suite",
@@ -1834,7 +1865,7 @@ export const aiEvalsToolset = {
1834
1865
  update: {
1835
1866
  method: "PATCH",
1836
1867
  path: "",
1837
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}`,
1868
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}`,
1838
1869
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1839
1870
  bodyBuilder: (input) => input.body ?? {},
1840
1871
  bodySchema: updateSuiteSchema,
@@ -1844,7 +1875,7 @@ export const aiEvalsToolset = {
1844
1875
  delete: {
1845
1876
  method: "DELETE",
1846
1877
  path: "",
1847
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}`,
1878
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}`,
1848
1879
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1849
1880
  responseExtractor: passthrough,
1850
1881
  description: "Delete suite",
@@ -1854,7 +1885,7 @@ export const aiEvalsToolset = {
1854
1885
  run: {
1855
1886
  method: "POST",
1856
1887
  path: "",
1857
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/run`,
1888
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/run`,
1858
1889
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
1859
1890
  preflight: validateSuiteReference,
1860
1891
  bodyBuilder: bodyFromInput,
@@ -1865,7 +1896,7 @@ export const aiEvalsToolset = {
1865
1896
  replace_evaluations: {
1866
1897
  method: "PUT",
1867
1898
  path: "",
1868
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/evaluations`,
1899
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations`,
1869
1900
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
1870
1901
  bodyBuilder: (input) => input.body ?? {},
1871
1902
  bodySchema: replaceSuiteEntriesSchema,
@@ -1885,7 +1916,7 @@ export const aiEvalsToolset = {
1885
1916
  export_yaml: {
1886
1917
  method: "GET",
1887
1918
  path: "",
1888
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/export-yaml`,
1919
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/export-yaml`,
1889
1920
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1890
1921
  responseExtractor: passthrough,
1891
1922
  actionDescription: "Export a suite and its member evaluations as a denormalized YAML document.",
@@ -1911,7 +1942,7 @@ export const aiEvalsToolset = {
1911
1942
  list: {
1912
1943
  method: "GET",
1913
1944
  path: "",
1914
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/evaluations`,
1945
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations`,
1915
1946
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1916
1947
  responseExtractor: aiEvalsArrayExtract,
1917
1948
  description: "List suite members",
@@ -1919,7 +1950,7 @@ export const aiEvalsToolset = {
1919
1950
  create: {
1920
1951
  method: "POST",
1921
1952
  path: "",
1922
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/evaluations`,
1953
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations`,
1923
1954
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
1924
1955
  bodyBuilder: (input) => input.body ?? {},
1925
1956
  bodySchema: addSuiteEntrySchema,
@@ -1929,7 +1960,7 @@ export const aiEvalsToolset = {
1929
1960
  delete: {
1930
1961
  method: "DELETE",
1931
1962
  path: "",
1932
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/evaluations/${input.evaluation_id}`,
1963
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations/${uuidPathSegment(input, "evaluation_id")}`,
1933
1964
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
1934
1965
  responseExtractor: passthrough,
1935
1966
  description: "Remove evaluation from suite",
@@ -1945,6 +1976,7 @@ export const aiEvalsToolset = {
1945
1976
  scopeOptional: true,
1946
1977
  headerBasedScoping: true,
1947
1978
  identifierFields: ["suite_run_id"],
1979
+ diagnosticHint: UUID_404_HINT,
1948
1980
  relatedResources: [
1949
1981
  { resourceType: "eval_suite", relationship: "belongs_to", description: "Suite run belongs to a suite" },
1950
1982
  { resourceType: "eval_run", relationship: "contains", description: "Suite run spawns child eval runs (filter via suite_run_id on eval_run)" },
@@ -1954,7 +1986,7 @@ export const aiEvalsToolset = {
1954
1986
  list: {
1955
1987
  method: "GET",
1956
1988
  path: "",
1957
- pathBuilder: (input, config) => `${base(input, config)}/suites/${input.suite_id}/runs`,
1989
+ pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/runs`,
1958
1990
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1959
1991
  queryParams: listQ,
1960
1992
  responseExtractor: aiEvalsListExtract,
@@ -1963,7 +1995,7 @@ export const aiEvalsToolset = {
1963
1995
  get: {
1964
1996
  method: "GET",
1965
1997
  path: "",
1966
- pathBuilder: (input, config) => `${base(input, config)}/suite-runs/${input.suite_run_id}`,
1998
+ pathBuilder: (input, config) => `${base(input, config)}/suite-runs/${uuidPathSegment(input, "suite_run_id")}`,
1967
1999
  operationPolicy: { risk: "read", retryPolicy: "safe" },
1968
2000
  responseExtractor: passthrough,
1969
2001
  description: "Get suite run status",
@@ -1980,7 +2012,9 @@ export const aiEvalsToolset = {
1980
2012
  scopeOptional: true,
1981
2013
  headerBasedScoping: true,
1982
2014
  identifierFields: ["target_id"],
1983
- diagnosticHint: "When creating a prompt target, use an LLM connector reference (config.llm_connector_ref) " +
2015
+ diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; target names are not accepted. " +
2016
+ "Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
2017
+ "When creating a prompt target, use an LLM connector reference (config.llm_connector_ref) " +
1984
2018
  "to specify the model credentials. List connectors via harness_list(resource_type='connector', filters={type:'OpenAI'}) (also type:'Anthropic').",
1985
2019
  relatedResources: [
1986
2020
  { resourceType: "evaluation", relationship: "used_by", description: "Evals reference targets via target_id" },
@@ -2002,7 +2036,7 @@ export const aiEvalsToolset = {
2002
2036
  get: {
2003
2037
  method: "GET",
2004
2038
  path: "",
2005
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}`,
2039
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}`,
2006
2040
  operationPolicy: { risk: "read", retryPolicy: "safe" },
2007
2041
  responseExtractor: passthrough,
2008
2042
  description: "Get target",
@@ -2021,7 +2055,7 @@ export const aiEvalsToolset = {
2021
2055
  update: {
2022
2056
  method: "PATCH",
2023
2057
  path: "",
2024
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}`,
2058
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}`,
2025
2059
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
2026
2060
  preflight: async (ctx) => validateTargetWrite(ctx, true),
2027
2061
  bodyBuilder: (input) => input.body ?? {},
@@ -2032,7 +2066,7 @@ export const aiEvalsToolset = {
2032
2066
  delete: {
2033
2067
  method: "DELETE",
2034
2068
  path: "",
2035
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}`,
2069
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}`,
2036
2070
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
2037
2071
  responseExtractor: passthrough,
2038
2072
  description: "Delete target",
@@ -2042,7 +2076,7 @@ export const aiEvalsToolset = {
2042
2076
  test: {
2043
2077
  method: "POST",
2044
2078
  path: "",
2045
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}/test`,
2079
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/test`,
2046
2080
  operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
2047
2081
  preflight: validateTargetTest,
2048
2082
  bodyBuilder: bodyFromInput,
@@ -2053,7 +2087,7 @@ export const aiEvalsToolset = {
2053
2087
  upload_outputs: {
2054
2088
  method: "POST",
2055
2089
  path: "",
2056
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}/outputs`,
2090
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/outputs`,
2057
2091
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
2058
2092
  preflight: validateOutputUpload,
2059
2093
  bodyBuilder: bodyFromInput,
@@ -2064,7 +2098,7 @@ export const aiEvalsToolset = {
2064
2098
  list_outputs: {
2065
2099
  method: "GET",
2066
2100
  path: "",
2067
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}/outputs`,
2101
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/outputs`,
2068
2102
  operationPolicy: { risk: "read", retryPolicy: "safe" },
2069
2103
  queryParams: listQ,
2070
2104
  responseExtractor: aiEvalsListExtract,
@@ -2074,7 +2108,7 @@ export const aiEvalsToolset = {
2074
2108
  export_yaml: {
2075
2109
  method: "GET",
2076
2110
  path: "",
2077
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}/export-yaml`,
2111
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/export-yaml`,
2078
2112
  operationPolicy: { risk: "read", retryPolicy: "safe" },
2079
2113
  responseExtractor: passthrough,
2080
2114
  actionDescription: "Export target config as a standalone YAML document.",
@@ -2083,7 +2117,7 @@ export const aiEvalsToolset = {
2083
2117
  overview: {
2084
2118
  method: "GET",
2085
2119
  path: "",
2086
- pathBuilder: (input, config) => `${base(input, config)}/targets/${input.target_id}/overview`,
2120
+ pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/overview`,
2087
2121
  operationPolicy: { risk: "read", retryPolicy: "safe" },
2088
2122
  responseExtractor: passthrough,
2089
2123
  actionDescription: "Summary metrics and per-eval health trend (total_evals, total_runs, last_run_at, overall_pass_rate, per-eval pass rates).",
@@ -2101,6 +2135,7 @@ export const aiEvalsToolset = {
2101
2135
  scopeOptional: true,
2102
2136
  headerBasedScoping: true,
2103
2137
  identifierFields: ["annotation_id"],
2138
+ diagnosticHint: UUID_404_HINT,
2104
2139
  listFilterFields: [
2105
2140
  { name: "trace_id", description: "Filter by trace id" },
2106
2141
  { name: "label", description: "Filter by label" },
@@ -2124,7 +2159,7 @@ export const aiEvalsToolset = {
2124
2159
  get: {
2125
2160
  method: "GET",
2126
2161
  path: "",
2127
- pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${input.annotation_id}`,
2162
+ pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${uuidPathSegment(input, "annotation_id")}`,
2128
2163
  operationPolicy: { risk: "read", retryPolicy: "safe" },
2129
2164
  responseExtractor: passthrough,
2130
2165
  description: "Get annotation",
@@ -2142,7 +2177,7 @@ export const aiEvalsToolset = {
2142
2177
  update: {
2143
2178
  method: "PATCH",
2144
2179
  path: "",
2145
- pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${input.annotation_id}`,
2180
+ pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${uuidPathSegment(input, "annotation_id")}`,
2146
2181
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
2147
2182
  bodyBuilder: (input) => input.body ?? {},
2148
2183
  bodySchema: updateAnnotationSchema,
@@ -2152,7 +2187,7 @@ export const aiEvalsToolset = {
2152
2187
  delete: {
2153
2188
  method: "DELETE",
2154
2189
  path: "",
2155
- pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${input.annotation_id}`,
2190
+ pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${uuidPathSegment(input, "annotation_id")}`,
2156
2191
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
2157
2192
  responseExtractor: passthrough,
2158
2193
  description: "Delete annotation",
@@ -2196,7 +2231,7 @@ export const aiEvalsToolset = {
2196
2231
  evaluate: {
2197
2232
  method: "POST",
2198
2233
  path: "",
2199
- pathBuilder: (input, config) => `${base(input, config)}/traces/${input.trace_id}/evaluate`,
2234
+ pathBuilder: (input, config) => `${base(input, config)}/traces/${pathSegment(input.trace_id, "trace_id")}/evaluate`,
2200
2235
  operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
2201
2236
  bodyBuilder: bodyFromInput,
2202
2237
  bodySchema: evaluateTraceSchema,
@@ -2323,7 +2358,7 @@ export const aiEvalsToolset = {
2323
2358
  get: {
2324
2359
  method: "GET",
2325
2360
  path: "",
2326
- pathBuilder: (input, config) => `${base(input, config)}/registry/${input.item_id}`,
2361
+ pathBuilder: (input, config) => `${base(input, config)}/registry/${pathSegment(input.item_id, "item_id")}`,
2327
2362
  operationPolicy: { risk: "read", retryPolicy: "safe" },
2328
2363
  queryParams: { type: "type" },
2329
2364
  responseExtractor: passthrough,
@@ -2342,7 +2377,7 @@ export const aiEvalsToolset = {
2342
2377
  update: {
2343
2378
  method: "PATCH",
2344
2379
  path: "",
2345
- pathBuilder: (input, config) => `${base(input, config)}/registry/${input.item_id}`,
2380
+ pathBuilder: (input, config) => `${base(input, config)}/registry/${pathSegment(input.item_id, "item_id")}`,
2346
2381
  operationPolicy: { risk: "low_write", retryPolicy: "safe" },
2347
2382
  bodyBuilder: (input) => input.body ?? {},
2348
2383
  bodySchema: updateRegistryItemSchema,
@@ -2352,7 +2387,7 @@ export const aiEvalsToolset = {
2352
2387
  delete: {
2353
2388
  method: "DELETE",
2354
2389
  path: "",
2355
- pathBuilder: (input, config) => `${base(input, config)}/registry/${input.item_id}`,
2390
+ pathBuilder: (input, config) => `${base(input, config)}/registry/${pathSegment(input.item_id, "item_id")}`,
2356
2391
  operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
2357
2392
  queryParams: { type: "type" },
2358
2393
  responseExtractor: passthrough,