harness-mcp-v2 3.2.32 → 3.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/build/data/schemas/v0/pipeline.d.ts.map +1 -1
- package/build/data/schemas/v0/pipeline.js +442 -134
- package/build/data/schemas/v0/pipeline.js.map +1 -1
- package/build/data/schemas/v0/template.d.ts.map +1 -1
- package/build/data/schemas/v0/template.js +1338 -1019
- package/build/data/schemas/v0/template.js.map +1 -1
- package/build/data/schemas/v1/pipeline.d.ts.map +1 -1
- package/build/data/schemas/v1/pipeline.js +32 -0
- package/build/data/schemas/v1/pipeline.js.map +1 -1
- package/build/data/schemas/v1/template.d.ts.map +1 -1
- package/build/data/schemas/v1/template.js +32 -0
- package/build/data/schemas/v1/template.js.map +1 -1
- package/build/registry/extractors.d.ts +7 -0
- package/build/registry/extractors.d.ts.map +1 -1
- package/build/registry/extractors.js +36 -1
- package/build/registry/extractors.js.map +1 -1
- package/build/registry/remote-git-preflight.d.ts +29 -0
- package/build/registry/remote-git-preflight.d.ts.map +1 -0
- package/build/registry/remote-git-preflight.js +148 -0
- package/build/registry/remote-git-preflight.js.map +1 -0
- package/build/registry/toolsets/ai-evals.d.ts.map +1 -1
- package/build/registry/toolsets/ai-evals.js +105 -70
- package/build/registry/toolsets/ai-evals.js.map +1 -1
- package/build/registry/toolsets/ccm.d.ts.map +1 -1
- package/build/registry/toolsets/ccm.js +50 -40
- package/build/registry/toolsets/ccm.js.map +1 -1
- package/build/registry/toolsets/connectors.d.ts.map +1 -1
- package/build/registry/toolsets/connectors.js +53 -0
- package/build/registry/toolsets/connectors.js.map +1 -1
- package/build/registry/toolsets/incidents.d.ts.map +1 -1
- package/build/registry/toolsets/incidents.js +78 -2
- package/build/registry/toolsets/incidents.js.map +1 -1
- package/build/registry/toolsets/observability-evaluations.d.ts.map +1 -1
- package/build/registry/toolsets/observability-evaluations.js +31 -17
- package/build/registry/toolsets/observability-evaluations.js.map +1 -1
- package/build/registry/toolsets/pipelines.d.ts.map +1 -1
- package/build/registry/toolsets/pipelines.js +14 -131
- package/build/registry/toolsets/pipelines.js.map +1 -1
- package/build/registry/toolsets/templates.d.ts.map +1 -1
- package/build/registry/toolsets/templates.js +83 -10
- package/build/registry/toolsets/templates.js.map +1 -1
- package/build/tools/harness-search.d.ts.map +1 -1
- package/build/tools/harness-search.js +4 -1
- package/build/tools/harness-search.js.map +1 -1
- package/build/utils/ccm-time.d.ts +35 -0
- package/build/utils/ccm-time.d.ts.map +1 -0
- package/build/utils/ccm-time.js +90 -0
- package/build/utils/ccm-time.js.map +1 -0
- package/npm-shrinkwrap.json +206 -265
- package/package.json +8 -6
|
@@ -5,6 +5,8 @@ const LLM_CONNECTOR_TYPES = new Set(["OpenAi", "OpenAI", "Anthropic", "AzureOpen
|
|
|
5
5
|
const JUDGE_BACKED_METRIC_TYPES = new Set(["llm", "ai_judge"]);
|
|
6
6
|
const TARGET_TYPES = new Set(["prompt", "agent", "precomputed"]);
|
|
7
7
|
const AGENT_METHODS = new Set(["GET", "POST", "PUT"]);
|
|
8
|
+
const UUID_404_HINT = "Use the id from harness_list for get, update, delete, and execute actions; identifiers and display names are not accepted. " +
|
|
9
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL.";
|
|
8
10
|
function scopeOrThrow(input, config) {
|
|
9
11
|
const org = input.org_id ?? config.HARNESS_ORG ?? "";
|
|
10
12
|
const project = input.project_id ?? config.HARNESS_PROJECT ?? "";
|
|
@@ -36,10 +38,28 @@ function requireBody(input, operation) {
|
|
|
36
38
|
function requireUuid(value, field) {
|
|
37
39
|
const id = nonEmptyString(value);
|
|
38
40
|
if (!id || !UUID_PATTERN.test(id)) {
|
|
39
|
-
throw new Error(`${field} must be
|
|
41
|
+
throw new Error(`${field} must be the id or uuid from harness_list for this AI Evals resource, not its identifier or name.`);
|
|
40
42
|
}
|
|
41
43
|
return id;
|
|
42
44
|
}
|
|
45
|
+
function pathSegment(value, field) {
|
|
46
|
+
const id = nonEmptyString(value);
|
|
47
|
+
if (!id)
|
|
48
|
+
throw new Error(`${field} must be a non-empty path identifier.`);
|
|
49
|
+
return encodeURIComponent(id);
|
|
50
|
+
}
|
|
51
|
+
function uuidPathSegment(input, field) {
|
|
52
|
+
return encodeURIComponent(requireUuid(input[field], field));
|
|
53
|
+
}
|
|
54
|
+
function datasetPath(input, config) {
|
|
55
|
+
const datasetId = nonEmptyString(input.dataset_id);
|
|
56
|
+
if (!datasetId)
|
|
57
|
+
throw new Error("dataset_id must be a non-empty UUID or dataset identifier.");
|
|
58
|
+
const root = `${base(input, config)}/dataset`;
|
|
59
|
+
return UUID_PATTERN.test(datasetId)
|
|
60
|
+
? `${root}/${encodeURIComponent(datasetId)}`
|
|
61
|
+
: `${root}/by-identifier/${encodeURIComponent(datasetId)}`;
|
|
62
|
+
}
|
|
43
63
|
function preflightScope({ input, registry }) {
|
|
44
64
|
const org_id = nonEmptyString(input.org_id) ?? registry.orgId;
|
|
45
65
|
const project_id = nonEmptyString(input.project_id) ?? registry.projectId;
|
|
@@ -1101,7 +1121,9 @@ export const aiEvalsToolset = {
|
|
|
1101
1121
|
scopeOptional: true,
|
|
1102
1122
|
headerBasedScoping: true,
|
|
1103
1123
|
identifierFields: ["dataset_id"],
|
|
1104
|
-
diagnosticHint: "
|
|
1124
|
+
diagnosticHint: "For get/update/delete, use the uuid from harness_list (or use a dataset identifier only for get). " +
|
|
1125
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
|
|
1126
|
+
"Dataset items require 'input' as a JSON object (e.g. { messages: [{role:'user', content:'...'}] } or { prompt: '...' }). " +
|
|
1105
1127
|
"Optional fields depend on metric type: 'expected_output' for correctness metrics, 'context' (string array) for RAG/groundedness metrics, " +
|
|
1106
1128
|
"'expected_tools' for agent tool-use metrics. Items can be added inline on create or managed separately via eval_dataset_item.",
|
|
1107
1129
|
listFilterFields: [
|
|
@@ -1125,10 +1147,10 @@ export const aiEvalsToolset = {
|
|
|
1125
1147
|
get: {
|
|
1126
1148
|
method: "GET",
|
|
1127
1149
|
path: "",
|
|
1128
|
-
pathBuilder:
|
|
1150
|
+
pathBuilder: datasetPath,
|
|
1129
1151
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1130
1152
|
responseExtractor: passthrough,
|
|
1131
|
-
description: "Get dataset by UUID",
|
|
1153
|
+
description: "Get dataset by UUID or identifier",
|
|
1132
1154
|
},
|
|
1133
1155
|
create: {
|
|
1134
1156
|
method: "POST",
|
|
@@ -1143,7 +1165,7 @@ export const aiEvalsToolset = {
|
|
|
1143
1165
|
update: {
|
|
1144
1166
|
method: "PUT",
|
|
1145
1167
|
path: "",
|
|
1146
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1168
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}`,
|
|
1147
1169
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1148
1170
|
bodyBuilder: (input) => input.body ?? {},
|
|
1149
1171
|
bodySchema: updateDatasetSchema,
|
|
@@ -1153,7 +1175,7 @@ export const aiEvalsToolset = {
|
|
|
1153
1175
|
delete: {
|
|
1154
1176
|
method: "DELETE",
|
|
1155
1177
|
path: "",
|
|
1156
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1178
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}`,
|
|
1157
1179
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1158
1180
|
responseExtractor: passthrough,
|
|
1159
1181
|
description: "Delete dataset",
|
|
@@ -1163,7 +1185,7 @@ export const aiEvalsToolset = {
|
|
|
1163
1185
|
get_by_identifier: {
|
|
1164
1186
|
method: "GET",
|
|
1165
1187
|
path: "",
|
|
1166
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/by-identifier/${
|
|
1188
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/by-identifier/${pathSegment(input.identifier, "identifier")}`,
|
|
1167
1189
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1168
1190
|
responseExtractor: passthrough,
|
|
1169
1191
|
actionDescription: "Get dataset by its unique identifier slug (not UUID). Pass identifier via params.identifier.",
|
|
@@ -1172,7 +1194,7 @@ export const aiEvalsToolset = {
|
|
|
1172
1194
|
generate: {
|
|
1173
1195
|
method: "POST",
|
|
1174
1196
|
path: "",
|
|
1175
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1197
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/generate`,
|
|
1176
1198
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
1177
1199
|
preflight: validateDatasetGeneration,
|
|
1178
1200
|
bodyBuilder: bodyFromInput,
|
|
@@ -1198,7 +1220,7 @@ export const aiEvalsToolset = {
|
|
|
1198
1220
|
list: {
|
|
1199
1221
|
method: "GET",
|
|
1200
1222
|
path: "",
|
|
1201
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1223
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items`,
|
|
1202
1224
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1203
1225
|
queryParams: listQ,
|
|
1204
1226
|
responseExtractor: aiEvalsListExtract,
|
|
@@ -1207,7 +1229,7 @@ export const aiEvalsToolset = {
|
|
|
1207
1229
|
get: {
|
|
1208
1230
|
method: "GET",
|
|
1209
1231
|
path: "",
|
|
1210
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1232
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/${uuidPathSegment(input, "item_id")}`,
|
|
1211
1233
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1212
1234
|
responseExtractor: passthrough,
|
|
1213
1235
|
description: "Get item by UUID",
|
|
@@ -1215,7 +1237,7 @@ export const aiEvalsToolset = {
|
|
|
1215
1237
|
create: {
|
|
1216
1238
|
method: "POST",
|
|
1217
1239
|
path: "",
|
|
1218
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1240
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items`,
|
|
1219
1241
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
1220
1242
|
bodyBuilder: (input) => input.body ?? {},
|
|
1221
1243
|
bodySchema: createDatasetItemSchema,
|
|
@@ -1225,7 +1247,7 @@ export const aiEvalsToolset = {
|
|
|
1225
1247
|
update: {
|
|
1226
1248
|
method: "PUT",
|
|
1227
1249
|
path: "",
|
|
1228
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1250
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/${uuidPathSegment(input, "item_id")}`,
|
|
1229
1251
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1230
1252
|
bodyBuilder: (input) => input.body ?? {},
|
|
1231
1253
|
bodySchema: updateDatasetItemSchema,
|
|
@@ -1235,7 +1257,7 @@ export const aiEvalsToolset = {
|
|
|
1235
1257
|
delete: {
|
|
1236
1258
|
method: "DELETE",
|
|
1237
1259
|
path: "",
|
|
1238
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1260
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/${uuidPathSegment(input, "item_id")}`,
|
|
1239
1261
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1240
1262
|
responseExtractor: passthrough,
|
|
1241
1263
|
description: "Delete item",
|
|
@@ -1245,7 +1267,7 @@ export const aiEvalsToolset = {
|
|
|
1245
1267
|
bulk_upsert: {
|
|
1246
1268
|
method: "PATCH",
|
|
1247
1269
|
path: "",
|
|
1248
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1270
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/bulk`,
|
|
1249
1271
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
1250
1272
|
bodyBuilder: bodyFromInput,
|
|
1251
1273
|
bodySchema: bulkUpsertDatasetItemsSchema,
|
|
@@ -1255,7 +1277,7 @@ export const aiEvalsToolset = {
|
|
|
1255
1277
|
bulk_delete: {
|
|
1256
1278
|
method: "POST",
|
|
1257
1279
|
path: "",
|
|
1258
|
-
pathBuilder: (input, config) => `${base(input, config)}/dataset/${input
|
|
1280
|
+
pathBuilder: (input, config) => `${base(input, config)}/dataset/${uuidPathSegment(input, "dataset_id")}/items/bulk-delete`,
|
|
1259
1281
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1260
1282
|
preflight: async (ctx) => {
|
|
1261
1283
|
await getScopedResource(ctx, "eval_dataset", "dataset_id", ctx.input.dataset_id);
|
|
@@ -1281,7 +1303,9 @@ export const aiEvalsToolset = {
|
|
|
1281
1303
|
scopeOptional: true,
|
|
1282
1304
|
headerBasedScoping: true,
|
|
1283
1305
|
identifierFields: ["eval_id"],
|
|
1284
|
-
diagnosticHint: "
|
|
1306
|
+
diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; identifiers and display names are not accepted. " +
|
|
1307
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
|
|
1308
|
+
"An eval requires three components: dataset_id, target_id, and metric_set_id. " +
|
|
1285
1309
|
"Before creating an eval, list existing resources with harness_list for eval_dataset, eval_target, and eval_metric_set. " +
|
|
1286
1310
|
"Create any missing components first. Managed evaluations cannot be created until all three are set. " +
|
|
1287
1311
|
"When storage_type='git', omit dataset_id/target_id/metric_set_id (they live in the YAML at git_source.file_path).",
|
|
@@ -1315,7 +1339,7 @@ export const aiEvalsToolset = {
|
|
|
1315
1339
|
get: {
|
|
1316
1340
|
method: "GET",
|
|
1317
1341
|
path: "",
|
|
1318
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1342
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}`,
|
|
1319
1343
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1320
1344
|
responseExtractor: passthrough,
|
|
1321
1345
|
description: "Get eval",
|
|
@@ -1334,7 +1358,7 @@ export const aiEvalsToolset = {
|
|
|
1334
1358
|
update: {
|
|
1335
1359
|
method: "PATCH",
|
|
1336
1360
|
path: "",
|
|
1337
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1361
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}`,
|
|
1338
1362
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1339
1363
|
preflight: async (ctx) => validateManagedEvalComposition(ctx, ctx.input, false),
|
|
1340
1364
|
bodyBuilder: (input) => input.body ?? {},
|
|
@@ -1345,7 +1369,7 @@ export const aiEvalsToolset = {
|
|
|
1345
1369
|
delete: {
|
|
1346
1370
|
method: "DELETE",
|
|
1347
1371
|
path: "",
|
|
1348
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1372
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}`,
|
|
1349
1373
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1350
1374
|
responseExtractor: passthrough,
|
|
1351
1375
|
description: "Hard-delete eval and its runs (409 if referenced by a suite)",
|
|
@@ -1355,7 +1379,7 @@ export const aiEvalsToolset = {
|
|
|
1355
1379
|
run: {
|
|
1356
1380
|
method: "POST",
|
|
1357
1381
|
path: "",
|
|
1358
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1382
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/run`,
|
|
1359
1383
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
1360
1384
|
preflight: validateEvalRun,
|
|
1361
1385
|
bodyBuilder: bodyFromInput,
|
|
@@ -1366,7 +1390,7 @@ export const aiEvalsToolset = {
|
|
|
1366
1390
|
clone: {
|
|
1367
1391
|
method: "POST",
|
|
1368
1392
|
path: "",
|
|
1369
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1393
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/clone`,
|
|
1370
1394
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
1371
1395
|
preflight: async (ctx) => {
|
|
1372
1396
|
const evaluation = await getScopedResource(ctx, "evaluation", "eval_id", ctx.input.eval_id);
|
|
@@ -1382,7 +1406,7 @@ export const aiEvalsToolset = {
|
|
|
1382
1406
|
item_history: {
|
|
1383
1407
|
method: "POST",
|
|
1384
1408
|
path: "",
|
|
1385
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1409
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/items/history`,
|
|
1386
1410
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1387
1411
|
preflight: async (ctx) => {
|
|
1388
1412
|
await getScopedResource(ctx, "evaluation", "eval_id", ctx.input.eval_id);
|
|
@@ -1410,7 +1434,7 @@ export const aiEvalsToolset = {
|
|
|
1410
1434
|
export_yaml: {
|
|
1411
1435
|
method: "GET",
|
|
1412
1436
|
path: "",
|
|
1413
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1437
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/export-yaml`,
|
|
1414
1438
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1415
1439
|
responseExtractor: passthrough,
|
|
1416
1440
|
actionDescription: "Export an eval and all referenced entities as a denormalized YAML document.",
|
|
@@ -1428,6 +1452,7 @@ export const aiEvalsToolset = {
|
|
|
1428
1452
|
scopeOptional: true,
|
|
1429
1453
|
headerBasedScoping: true,
|
|
1430
1454
|
identifierFields: ["run_id"],
|
|
1455
|
+
diagnosticHint: UUID_404_HINT,
|
|
1431
1456
|
listFilterFields: [
|
|
1432
1457
|
{ name: "target_id", description: "Filter runs by target UUID" },
|
|
1433
1458
|
],
|
|
@@ -1449,7 +1474,7 @@ export const aiEvalsToolset = {
|
|
|
1449
1474
|
get: {
|
|
1450
1475
|
method: "GET",
|
|
1451
1476
|
path: "",
|
|
1452
|
-
pathBuilder: (input, config) => `${base(input, config)}/runs/${input
|
|
1477
|
+
pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}`,
|
|
1453
1478
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1454
1479
|
responseExtractor: passthrough,
|
|
1455
1480
|
description: "Get run",
|
|
@@ -1469,7 +1494,7 @@ export const aiEvalsToolset = {
|
|
|
1469
1494
|
rescore: {
|
|
1470
1495
|
method: "POST",
|
|
1471
1496
|
path: "",
|
|
1472
|
-
pathBuilder: (input, config) => `${base(input, config)}/runs/${input
|
|
1497
|
+
pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}/rescore`,
|
|
1473
1498
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
1474
1499
|
preflight: async (ctx) => {
|
|
1475
1500
|
await validateRunReference(ctx);
|
|
@@ -1485,7 +1510,7 @@ export const aiEvalsToolset = {
|
|
|
1485
1510
|
recommendations: {
|
|
1486
1511
|
method: "POST",
|
|
1487
1512
|
path: "",
|
|
1488
|
-
pathBuilder: (input, config) => `${base(input, config)}/runs/${input
|
|
1513
|
+
pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}/recommendations`,
|
|
1489
1514
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
1490
1515
|
preflight: validateRunReference,
|
|
1491
1516
|
bodyBuilder: bodyFromInput,
|
|
@@ -1509,7 +1534,7 @@ export const aiEvalsToolset = {
|
|
|
1509
1534
|
list: {
|
|
1510
1535
|
method: "GET",
|
|
1511
1536
|
path: "",
|
|
1512
|
-
pathBuilder: (input, config) => `${base(input, config)}/runs/${input
|
|
1537
|
+
pathBuilder: (input, config) => `${base(input, config)}/runs/${uuidPathSegment(input, "run_id")}/items`,
|
|
1513
1538
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1514
1539
|
queryParams: listQ,
|
|
1515
1540
|
responseExtractor: aiEvalsListExtract,
|
|
@@ -1531,7 +1556,7 @@ export const aiEvalsToolset = {
|
|
|
1531
1556
|
list: {
|
|
1532
1557
|
method: "GET",
|
|
1533
1558
|
path: "",
|
|
1534
|
-
pathBuilder: (input, config) => `${base(input, config)}/evals/${input
|
|
1559
|
+
pathBuilder: (input, config) => `${base(input, config)}/evals/${uuidPathSegment(input, "eval_id")}/runs`,
|
|
1535
1560
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1536
1561
|
queryParams: listQ,
|
|
1537
1562
|
responseExtractor: aiEvalsListExtract,
|
|
@@ -1549,7 +1574,9 @@ export const aiEvalsToolset = {
|
|
|
1549
1574
|
scopeOptional: true,
|
|
1550
1575
|
headerBasedScoping: true,
|
|
1551
1576
|
identifierFields: ["metric_id"],
|
|
1552
|
-
diagnosticHint: "Use the
|
|
1577
|
+
diagnosticHint: "Use the id from harness_list for get, update, and delete; metric names are not accepted. " +
|
|
1578
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
|
|
1579
|
+
"Use the 'suggestions' execute action to discover appropriate metrics for a given target type and dataset shape. " +
|
|
1553
1580
|
"Metrics are added to metric sets (eval_metric_set) via eval_metric_set_entry, then referenced by evaluations. " +
|
|
1554
1581
|
"Each metric response includes a 'config_schema' field (JSON Schema) describing available config options for that metric kind — " +
|
|
1555
1582
|
"use harness_get to inspect a metric's config_schema before setting config on a metric set entry.",
|
|
@@ -1574,7 +1601,7 @@ export const aiEvalsToolset = {
|
|
|
1574
1601
|
get: {
|
|
1575
1602
|
method: "GET",
|
|
1576
1603
|
path: "",
|
|
1577
|
-
pathBuilder: (input, config) => `${base(input, config)}/metrics/${input
|
|
1604
|
+
pathBuilder: (input, config) => `${base(input, config)}/metrics/${uuidPathSegment(input, "metric_id")}`,
|
|
1578
1605
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1579
1606
|
responseExtractor: passthrough,
|
|
1580
1607
|
description: "Get metric",
|
|
@@ -1592,7 +1619,7 @@ export const aiEvalsToolset = {
|
|
|
1592
1619
|
update: {
|
|
1593
1620
|
method: "PATCH",
|
|
1594
1621
|
path: "",
|
|
1595
|
-
pathBuilder: (input, config) => `${base(input, config)}/metrics/${input
|
|
1622
|
+
pathBuilder: (input, config) => `${base(input, config)}/metrics/${uuidPathSegment(input, "metric_id")}`,
|
|
1596
1623
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1597
1624
|
bodyBuilder: (input) => input.body ?? {},
|
|
1598
1625
|
bodySchema: updateMetricSchema,
|
|
@@ -1602,7 +1629,7 @@ export const aiEvalsToolset = {
|
|
|
1602
1629
|
delete: {
|
|
1603
1630
|
method: "DELETE",
|
|
1604
1631
|
path: "",
|
|
1605
|
-
pathBuilder: (input, config) => `${base(input, config)}/metrics/${input
|
|
1632
|
+
pathBuilder: (input, config) => `${base(input, config)}/metrics/${uuidPathSegment(input, "metric_id")}`,
|
|
1606
1633
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1607
1634
|
responseExtractor: passthrough,
|
|
1608
1635
|
description: "Delete metric",
|
|
@@ -1636,7 +1663,9 @@ export const aiEvalsToolset = {
|
|
|
1636
1663
|
scopeOptional: true,
|
|
1637
1664
|
headerBasedScoping: true,
|
|
1638
1665
|
identifierFields: ["set_id"],
|
|
1639
|
-
diagnosticHint: "
|
|
1666
|
+
diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; metric-set names are not accepted. " +
|
|
1667
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
|
|
1668
|
+
"Before creating a metric set, list available metrics with harness_list(resource_type='eval_metric'). " +
|
|
1640
1669
|
"Use harness_execute(resource_type='eval_metric', action='suggestions') to discover metrics appropriate for a target type. " +
|
|
1641
1670
|
"If using LLM metrics (llm-as-judge), set judge_llm_config to a structured provider configuration. " +
|
|
1642
1671
|
"judge_llm_connector_ref remains accepted but is DEPRECATED.",
|
|
@@ -1662,7 +1691,7 @@ export const aiEvalsToolset = {
|
|
|
1662
1691
|
get: {
|
|
1663
1692
|
method: "GET",
|
|
1664
1693
|
path: "",
|
|
1665
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1694
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}`,
|
|
1666
1695
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1667
1696
|
responseExtractor: passthrough,
|
|
1668
1697
|
description: "Get metric set",
|
|
@@ -1681,7 +1710,7 @@ export const aiEvalsToolset = {
|
|
|
1681
1710
|
update: {
|
|
1682
1711
|
method: "PATCH",
|
|
1683
1712
|
path: "",
|
|
1684
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1713
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}`,
|
|
1685
1714
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1686
1715
|
preflight: async (ctx) => validateMetricSetWrite(ctx, ctx.input, true),
|
|
1687
1716
|
bodyBuilder: (input) => input.body ?? {},
|
|
@@ -1692,7 +1721,7 @@ export const aiEvalsToolset = {
|
|
|
1692
1721
|
delete: {
|
|
1693
1722
|
method: "DELETE",
|
|
1694
1723
|
path: "",
|
|
1695
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1724
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}`,
|
|
1696
1725
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1697
1726
|
responseExtractor: passthrough,
|
|
1698
1727
|
description: "Delete metric set",
|
|
@@ -1702,7 +1731,7 @@ export const aiEvalsToolset = {
|
|
|
1702
1731
|
calibrate: {
|
|
1703
1732
|
method: "POST",
|
|
1704
1733
|
path: "",
|
|
1705
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1734
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/calibrate`,
|
|
1706
1735
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
1707
1736
|
bodyBuilder: bodyFromInput,
|
|
1708
1737
|
bodySchema: calibrateSchema,
|
|
@@ -1712,7 +1741,7 @@ export const aiEvalsToolset = {
|
|
|
1712
1741
|
replace_metrics: {
|
|
1713
1742
|
method: "PUT",
|
|
1714
1743
|
path: "",
|
|
1715
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1744
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics`,
|
|
1716
1745
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1717
1746
|
preflight: validateMetricSetReplacement,
|
|
1718
1747
|
bodyBuilder: (input) => {
|
|
@@ -1748,7 +1777,7 @@ export const aiEvalsToolset = {
|
|
|
1748
1777
|
list: {
|
|
1749
1778
|
method: "GET",
|
|
1750
1779
|
path: "",
|
|
1751
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1780
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics`,
|
|
1752
1781
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1753
1782
|
responseExtractor: aiEvalsArrayExtract,
|
|
1754
1783
|
description: "List entries in a metric set",
|
|
@@ -1756,7 +1785,7 @@ export const aiEvalsToolset = {
|
|
|
1756
1785
|
create: {
|
|
1757
1786
|
method: "POST",
|
|
1758
1787
|
path: "",
|
|
1759
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1788
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics`,
|
|
1760
1789
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
1761
1790
|
preflight: async (ctx) => validateMetricSetEntryWrite(ctx, false),
|
|
1762
1791
|
bodyBuilder: (input) => input.body ?? {},
|
|
@@ -1767,7 +1796,7 @@ export const aiEvalsToolset = {
|
|
|
1767
1796
|
update: {
|
|
1768
1797
|
method: "PATCH",
|
|
1769
1798
|
path: "",
|
|
1770
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1799
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics/${uuidPathSegment(input, "metric_id")}`,
|
|
1771
1800
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1772
1801
|
preflight: async (ctx) => validateMetricSetEntryWrite(ctx, true),
|
|
1773
1802
|
bodyBuilder: (input) => input.body ?? {},
|
|
@@ -1778,7 +1807,7 @@ export const aiEvalsToolset = {
|
|
|
1778
1807
|
delete: {
|
|
1779
1808
|
method: "DELETE",
|
|
1780
1809
|
path: "",
|
|
1781
|
-
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${input
|
|
1810
|
+
pathBuilder: (input, config) => `${base(input, config)}/metric-sets/${uuidPathSegment(input, "set_id")}/metrics/${uuidPathSegment(input, "metric_id")}`,
|
|
1782
1811
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1783
1812
|
responseExtractor: passthrough,
|
|
1784
1813
|
description: "Remove metric from set",
|
|
@@ -1795,7 +1824,9 @@ export const aiEvalsToolset = {
|
|
|
1795
1824
|
scopeOptional: true,
|
|
1796
1825
|
headerBasedScoping: true,
|
|
1797
1826
|
identifierFields: ["suite_id"],
|
|
1798
|
-
diagnosticHint: "
|
|
1827
|
+
diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; suite names are not accepted. " +
|
|
1828
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
|
|
1829
|
+
"A suite groups evaluations together. First create evaluations (each with dataset + target + metric set), " +
|
|
1799
1830
|
"then create the suite and add evaluations via eval_suite_evaluation or the replace_evaluations execute action. " +
|
|
1800
1831
|
"List existing evaluations with harness_list(resource_type='evaluation').",
|
|
1801
1832
|
relatedResources: [
|
|
@@ -1816,7 +1847,7 @@ export const aiEvalsToolset = {
|
|
|
1816
1847
|
get: {
|
|
1817
1848
|
method: "GET",
|
|
1818
1849
|
path: "",
|
|
1819
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1850
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}`,
|
|
1820
1851
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1821
1852
|
responseExtractor: passthrough,
|
|
1822
1853
|
description: "Get suite",
|
|
@@ -1834,7 +1865,7 @@ export const aiEvalsToolset = {
|
|
|
1834
1865
|
update: {
|
|
1835
1866
|
method: "PATCH",
|
|
1836
1867
|
path: "",
|
|
1837
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1868
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}`,
|
|
1838
1869
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1839
1870
|
bodyBuilder: (input) => input.body ?? {},
|
|
1840
1871
|
bodySchema: updateSuiteSchema,
|
|
@@ -1844,7 +1875,7 @@ export const aiEvalsToolset = {
|
|
|
1844
1875
|
delete: {
|
|
1845
1876
|
method: "DELETE",
|
|
1846
1877
|
path: "",
|
|
1847
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1878
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}`,
|
|
1848
1879
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1849
1880
|
responseExtractor: passthrough,
|
|
1850
1881
|
description: "Delete suite",
|
|
@@ -1854,7 +1885,7 @@ export const aiEvalsToolset = {
|
|
|
1854
1885
|
run: {
|
|
1855
1886
|
method: "POST",
|
|
1856
1887
|
path: "",
|
|
1857
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1888
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/run`,
|
|
1858
1889
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
1859
1890
|
preflight: validateSuiteReference,
|
|
1860
1891
|
bodyBuilder: bodyFromInput,
|
|
@@ -1865,7 +1896,7 @@ export const aiEvalsToolset = {
|
|
|
1865
1896
|
replace_evaluations: {
|
|
1866
1897
|
method: "PUT",
|
|
1867
1898
|
path: "",
|
|
1868
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1899
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations`,
|
|
1869
1900
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
1870
1901
|
bodyBuilder: (input) => input.body ?? {},
|
|
1871
1902
|
bodySchema: replaceSuiteEntriesSchema,
|
|
@@ -1885,7 +1916,7 @@ export const aiEvalsToolset = {
|
|
|
1885
1916
|
export_yaml: {
|
|
1886
1917
|
method: "GET",
|
|
1887
1918
|
path: "",
|
|
1888
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1919
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/export-yaml`,
|
|
1889
1920
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1890
1921
|
responseExtractor: passthrough,
|
|
1891
1922
|
actionDescription: "Export a suite and its member evaluations as a denormalized YAML document.",
|
|
@@ -1911,7 +1942,7 @@ export const aiEvalsToolset = {
|
|
|
1911
1942
|
list: {
|
|
1912
1943
|
method: "GET",
|
|
1913
1944
|
path: "",
|
|
1914
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1945
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations`,
|
|
1915
1946
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1916
1947
|
responseExtractor: aiEvalsArrayExtract,
|
|
1917
1948
|
description: "List suite members",
|
|
@@ -1919,7 +1950,7 @@ export const aiEvalsToolset = {
|
|
|
1919
1950
|
create: {
|
|
1920
1951
|
method: "POST",
|
|
1921
1952
|
path: "",
|
|
1922
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1953
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations`,
|
|
1923
1954
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
1924
1955
|
bodyBuilder: (input) => input.body ?? {},
|
|
1925
1956
|
bodySchema: addSuiteEntrySchema,
|
|
@@ -1929,7 +1960,7 @@ export const aiEvalsToolset = {
|
|
|
1929
1960
|
delete: {
|
|
1930
1961
|
method: "DELETE",
|
|
1931
1962
|
path: "",
|
|
1932
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1963
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/evaluations/${uuidPathSegment(input, "evaluation_id")}`,
|
|
1933
1964
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
1934
1965
|
responseExtractor: passthrough,
|
|
1935
1966
|
description: "Remove evaluation from suite",
|
|
@@ -1945,6 +1976,7 @@ export const aiEvalsToolset = {
|
|
|
1945
1976
|
scopeOptional: true,
|
|
1946
1977
|
headerBasedScoping: true,
|
|
1947
1978
|
identifierFields: ["suite_run_id"],
|
|
1979
|
+
diagnosticHint: UUID_404_HINT,
|
|
1948
1980
|
relatedResources: [
|
|
1949
1981
|
{ resourceType: "eval_suite", relationship: "belongs_to", description: "Suite run belongs to a suite" },
|
|
1950
1982
|
{ resourceType: "eval_run", relationship: "contains", description: "Suite run spawns child eval runs (filter via suite_run_id on eval_run)" },
|
|
@@ -1954,7 +1986,7 @@ export const aiEvalsToolset = {
|
|
|
1954
1986
|
list: {
|
|
1955
1987
|
method: "GET",
|
|
1956
1988
|
path: "",
|
|
1957
|
-
pathBuilder: (input, config) => `${base(input, config)}/suites/${input
|
|
1989
|
+
pathBuilder: (input, config) => `${base(input, config)}/suites/${uuidPathSegment(input, "suite_id")}/runs`,
|
|
1958
1990
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1959
1991
|
queryParams: listQ,
|
|
1960
1992
|
responseExtractor: aiEvalsListExtract,
|
|
@@ -1963,7 +1995,7 @@ export const aiEvalsToolset = {
|
|
|
1963
1995
|
get: {
|
|
1964
1996
|
method: "GET",
|
|
1965
1997
|
path: "",
|
|
1966
|
-
pathBuilder: (input, config) => `${base(input, config)}/suite-runs/${input
|
|
1998
|
+
pathBuilder: (input, config) => `${base(input, config)}/suite-runs/${uuidPathSegment(input, "suite_run_id")}`,
|
|
1967
1999
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
1968
2000
|
responseExtractor: passthrough,
|
|
1969
2001
|
description: "Get suite run status",
|
|
@@ -1980,7 +2012,9 @@ export const aiEvalsToolset = {
|
|
|
1980
2012
|
scopeOptional: true,
|
|
1981
2013
|
headerBasedScoping: true,
|
|
1982
2014
|
identifierFields: ["target_id"],
|
|
1983
|
-
diagnosticHint: "
|
|
2015
|
+
diagnosticHint: "Use the id from harness_list for get, update, delete, and execute actions; target names are not accepted. " +
|
|
2016
|
+
"Verify org_id and project_id. An HTML nginx 404 means AI Evals is not deployed at this Harness base URL. " +
|
|
2017
|
+
"When creating a prompt target, use an LLM connector reference (config.llm_connector_ref) " +
|
|
1984
2018
|
"to specify the model credentials. List connectors via harness_list(resource_type='connector', filters={type:'OpenAI'}) (also type:'Anthropic').",
|
|
1985
2019
|
relatedResources: [
|
|
1986
2020
|
{ resourceType: "evaluation", relationship: "used_by", description: "Evals reference targets via target_id" },
|
|
@@ -2002,7 +2036,7 @@ export const aiEvalsToolset = {
|
|
|
2002
2036
|
get: {
|
|
2003
2037
|
method: "GET",
|
|
2004
2038
|
path: "",
|
|
2005
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2039
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}`,
|
|
2006
2040
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
2007
2041
|
responseExtractor: passthrough,
|
|
2008
2042
|
description: "Get target",
|
|
@@ -2021,7 +2055,7 @@ export const aiEvalsToolset = {
|
|
|
2021
2055
|
update: {
|
|
2022
2056
|
method: "PATCH",
|
|
2023
2057
|
path: "",
|
|
2024
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2058
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}`,
|
|
2025
2059
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
2026
2060
|
preflight: async (ctx) => validateTargetWrite(ctx, true),
|
|
2027
2061
|
bodyBuilder: (input) => input.body ?? {},
|
|
@@ -2032,7 +2066,7 @@ export const aiEvalsToolset = {
|
|
|
2032
2066
|
delete: {
|
|
2033
2067
|
method: "DELETE",
|
|
2034
2068
|
path: "",
|
|
2035
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2069
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}`,
|
|
2036
2070
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
2037
2071
|
responseExtractor: passthrough,
|
|
2038
2072
|
description: "Delete target",
|
|
@@ -2042,7 +2076,7 @@ export const aiEvalsToolset = {
|
|
|
2042
2076
|
test: {
|
|
2043
2077
|
method: "POST",
|
|
2044
2078
|
path: "",
|
|
2045
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2079
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/test`,
|
|
2046
2080
|
operationPolicy: { risk: "medium_write", retryPolicy: "do_not_retry" },
|
|
2047
2081
|
preflight: validateTargetTest,
|
|
2048
2082
|
bodyBuilder: bodyFromInput,
|
|
@@ -2053,7 +2087,7 @@ export const aiEvalsToolset = {
|
|
|
2053
2087
|
upload_outputs: {
|
|
2054
2088
|
method: "POST",
|
|
2055
2089
|
path: "",
|
|
2056
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2090
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/outputs`,
|
|
2057
2091
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
2058
2092
|
preflight: validateOutputUpload,
|
|
2059
2093
|
bodyBuilder: bodyFromInput,
|
|
@@ -2064,7 +2098,7 @@ export const aiEvalsToolset = {
|
|
|
2064
2098
|
list_outputs: {
|
|
2065
2099
|
method: "GET",
|
|
2066
2100
|
path: "",
|
|
2067
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2101
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/outputs`,
|
|
2068
2102
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
2069
2103
|
queryParams: listQ,
|
|
2070
2104
|
responseExtractor: aiEvalsListExtract,
|
|
@@ -2074,7 +2108,7 @@ export const aiEvalsToolset = {
|
|
|
2074
2108
|
export_yaml: {
|
|
2075
2109
|
method: "GET",
|
|
2076
2110
|
path: "",
|
|
2077
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2111
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/export-yaml`,
|
|
2078
2112
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
2079
2113
|
responseExtractor: passthrough,
|
|
2080
2114
|
actionDescription: "Export target config as a standalone YAML document.",
|
|
@@ -2083,7 +2117,7 @@ export const aiEvalsToolset = {
|
|
|
2083
2117
|
overview: {
|
|
2084
2118
|
method: "GET",
|
|
2085
2119
|
path: "",
|
|
2086
|
-
pathBuilder: (input, config) => `${base(input, config)}/targets/${input
|
|
2120
|
+
pathBuilder: (input, config) => `${base(input, config)}/targets/${uuidPathSegment(input, "target_id")}/overview`,
|
|
2087
2121
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
2088
2122
|
responseExtractor: passthrough,
|
|
2089
2123
|
actionDescription: "Summary metrics and per-eval health trend (total_evals, total_runs, last_run_at, overall_pass_rate, per-eval pass rates).",
|
|
@@ -2101,6 +2135,7 @@ export const aiEvalsToolset = {
|
|
|
2101
2135
|
scopeOptional: true,
|
|
2102
2136
|
headerBasedScoping: true,
|
|
2103
2137
|
identifierFields: ["annotation_id"],
|
|
2138
|
+
diagnosticHint: UUID_404_HINT,
|
|
2104
2139
|
listFilterFields: [
|
|
2105
2140
|
{ name: "trace_id", description: "Filter by trace id" },
|
|
2106
2141
|
{ name: "label", description: "Filter by label" },
|
|
@@ -2124,7 +2159,7 @@ export const aiEvalsToolset = {
|
|
|
2124
2159
|
get: {
|
|
2125
2160
|
method: "GET",
|
|
2126
2161
|
path: "",
|
|
2127
|
-
pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${input
|
|
2162
|
+
pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${uuidPathSegment(input, "annotation_id")}`,
|
|
2128
2163
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
2129
2164
|
responseExtractor: passthrough,
|
|
2130
2165
|
description: "Get annotation",
|
|
@@ -2142,7 +2177,7 @@ export const aiEvalsToolset = {
|
|
|
2142
2177
|
update: {
|
|
2143
2178
|
method: "PATCH",
|
|
2144
2179
|
path: "",
|
|
2145
|
-
pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${input
|
|
2180
|
+
pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${uuidPathSegment(input, "annotation_id")}`,
|
|
2146
2181
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
2147
2182
|
bodyBuilder: (input) => input.body ?? {},
|
|
2148
2183
|
bodySchema: updateAnnotationSchema,
|
|
@@ -2152,7 +2187,7 @@ export const aiEvalsToolset = {
|
|
|
2152
2187
|
delete: {
|
|
2153
2188
|
method: "DELETE",
|
|
2154
2189
|
path: "",
|
|
2155
|
-
pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${input
|
|
2190
|
+
pathBuilder: (input, config) => `${base(input, config)}/observe/annotations/${uuidPathSegment(input, "annotation_id")}`,
|
|
2156
2191
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
2157
2192
|
responseExtractor: passthrough,
|
|
2158
2193
|
description: "Delete annotation",
|
|
@@ -2196,7 +2231,7 @@ export const aiEvalsToolset = {
|
|
|
2196
2231
|
evaluate: {
|
|
2197
2232
|
method: "POST",
|
|
2198
2233
|
path: "",
|
|
2199
|
-
pathBuilder: (input, config) => `${base(input, config)}/traces/${input.trace_id}/evaluate`,
|
|
2234
|
+
pathBuilder: (input, config) => `${base(input, config)}/traces/${pathSegment(input.trace_id, "trace_id")}/evaluate`,
|
|
2200
2235
|
operationPolicy: { risk: "low_write", retryPolicy: "do_not_retry" },
|
|
2201
2236
|
bodyBuilder: bodyFromInput,
|
|
2202
2237
|
bodySchema: evaluateTraceSchema,
|
|
@@ -2323,7 +2358,7 @@ export const aiEvalsToolset = {
|
|
|
2323
2358
|
get: {
|
|
2324
2359
|
method: "GET",
|
|
2325
2360
|
path: "",
|
|
2326
|
-
pathBuilder: (input, config) => `${base(input, config)}/registry/${input.item_id}`,
|
|
2361
|
+
pathBuilder: (input, config) => `${base(input, config)}/registry/${pathSegment(input.item_id, "item_id")}`,
|
|
2327
2362
|
operationPolicy: { risk: "read", retryPolicy: "safe" },
|
|
2328
2363
|
queryParams: { type: "type" },
|
|
2329
2364
|
responseExtractor: passthrough,
|
|
@@ -2342,7 +2377,7 @@ export const aiEvalsToolset = {
|
|
|
2342
2377
|
update: {
|
|
2343
2378
|
method: "PATCH",
|
|
2344
2379
|
path: "",
|
|
2345
|
-
pathBuilder: (input, config) => `${base(input, config)}/registry/${input.item_id}`,
|
|
2380
|
+
pathBuilder: (input, config) => `${base(input, config)}/registry/${pathSegment(input.item_id, "item_id")}`,
|
|
2346
2381
|
operationPolicy: { risk: "low_write", retryPolicy: "safe" },
|
|
2347
2382
|
bodyBuilder: (input) => input.body ?? {},
|
|
2348
2383
|
bodySchema: updateRegistryItemSchema,
|
|
@@ -2352,7 +2387,7 @@ export const aiEvalsToolset = {
|
|
|
2352
2387
|
delete: {
|
|
2353
2388
|
method: "DELETE",
|
|
2354
2389
|
path: "",
|
|
2355
|
-
pathBuilder: (input, config) => `${base(input, config)}/registry/${input.item_id}`,
|
|
2390
|
+
pathBuilder: (input, config) => `${base(input, config)}/registry/${pathSegment(input.item_id, "item_id")}`,
|
|
2356
2391
|
operationPolicy: { risk: "destructive", retryPolicy: "do_not_retry" },
|
|
2357
2392
|
queryParams: { type: "type" },
|
|
2358
2393
|
responseExtractor: passthrough,
|