@alis-build/harness-eval 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +187 -30
- package/dist/adapters/claude-code/index.d.ts +2 -2
- package/dist/adapters/claude-code/index.js +2 -1
- package/dist/adapters/codex/index.d.ts +68 -0
- package/dist/adapters/codex/index.js +3 -0
- package/dist/{claude-code-DZ4Vkgp6.js → claude-code-C_7hxC8z.js} +3 -245
- package/dist/claude-code-C_7hxC8z.js.map +1 -0
- package/dist/cli/bin.js +131 -151
- package/dist/cli/bin.js.map +1 -1
- package/dist/codex-0cHO2te9.js +496 -0
- package/dist/codex-0cHO2te9.js.map +1 -0
- package/dist/config/loader.d.ts +2 -2
- package/dist/config/loader.js +2 -2
- package/dist/{index-V22PrR0p.d.ts → index-C56AEDUr.d.ts} +2 -2
- package/dist/index.d.ts +134 -6
- package/dist/index.js +6 -5
- package/dist/index.js.map +1 -1
- package/dist/{loader-DcI0KfRX.js → loader-CiBm4Kf6.js} +491 -209
- package/dist/loader-CiBm4Kf6.js.map +1 -0
- package/dist/loader-CrmzNwkq.d.ts +107 -0
- package/dist/{projections-BcX7w-f6.js → reporter-BKCJZRYr.js} +1475 -729
- package/dist/reporter-BKCJZRYr.js.map +1 -0
- package/dist/runner/suite.d.ts +1 -1
- package/dist/runner/suite.js +1 -1
- package/dist/{suite-Dlzl-HI0.js → suite-C3-8EjUW.js} +558 -4
- package/dist/suite-C3-8EjUW.js.map +1 -0
- package/dist/{suite-DPJMIEbu.d.ts → suite-qyOGre2g.d.ts} +2 -2
- package/dist/types-Bac8_Ixb.js +246 -0
- package/dist/types-Bac8_Ixb.js.map +1 -0
- package/dist/{types-CD3TwOtZ.d.ts → types-CLt4Yygc.d.ts} +2 -2
- package/dist/{types-B9H4IZtA.d.ts → types-D0HR2WnP.d.ts} +9 -2
- package/dist/types-DFMpv_HJ.d.ts +77 -0
- package/package.json +11 -2
- package/schemas/eval-run-envelope.schema.json +193 -183
- package/dist/claude-code-DZ4Vkgp6.js.map +0 -1
- package/dist/loader-C9yQHUPC.d.ts +0 -50
- package/dist/loader-DcI0KfRX.js.map +0 -1
- package/dist/projections-BcX7w-f6.js.map +0 -1
- package/dist/suite-Dlzl-HI0.js.map +0 -1
|
@@ -652,34 +652,34 @@
|
|
|
652
652
|
"$ref": "#/$defs/__schema123"
|
|
653
653
|
},
|
|
654
654
|
"assertionResults": {
|
|
655
|
-
"$ref": "#/$defs/
|
|
655
|
+
"$ref": "#/$defs/__schema140"
|
|
656
656
|
},
|
|
657
657
|
"outcomeGrades": {
|
|
658
|
-
"$ref": "#/$defs/
|
|
658
|
+
"$ref": "#/$defs/__schema149"
|
|
659
659
|
},
|
|
660
660
|
"externalScores": {
|
|
661
|
-
"$ref": "#/$defs/
|
|
661
|
+
"$ref": "#/$defs/__schema173"
|
|
662
662
|
},
|
|
663
663
|
"artifacts": {
|
|
664
|
-
"$ref": "#/$defs/
|
|
664
|
+
"$ref": "#/$defs/__schema183"
|
|
665
665
|
},
|
|
666
666
|
"evaluationInstance": {
|
|
667
|
-
"$ref": "#/$defs/
|
|
667
|
+
"$ref": "#/$defs/__schema190"
|
|
668
668
|
},
|
|
669
669
|
"trajectoryInstances": {
|
|
670
|
-
"$ref": "#/$defs/
|
|
670
|
+
"$ref": "#/$defs/__schema196"
|
|
671
671
|
},
|
|
672
672
|
"harnessMetrics": {
|
|
673
|
-
"$ref": "#/$defs/
|
|
673
|
+
"$ref": "#/$defs/__schema206"
|
|
674
674
|
},
|
|
675
675
|
"latencySeconds": {
|
|
676
|
-
"$ref": "#/$defs/
|
|
676
|
+
"$ref": "#/$defs/__schema213"
|
|
677
677
|
},
|
|
678
678
|
"failure": {
|
|
679
|
-
"$ref": "#/$defs/
|
|
679
|
+
"$ref": "#/$defs/__schema215"
|
|
680
680
|
},
|
|
681
681
|
"error": {
|
|
682
|
-
"$ref": "#/$defs/
|
|
682
|
+
"$ref": "#/$defs/__schema217"
|
|
683
683
|
}
|
|
684
684
|
},
|
|
685
685
|
"required": [
|
|
@@ -1154,20 +1154,23 @@
|
|
|
1154
1154
|
"exitCode": {
|
|
1155
1155
|
"$ref": "#/$defs/__schema124"
|
|
1156
1156
|
},
|
|
1157
|
-
"
|
|
1157
|
+
"exitCodeDescription": {
|
|
1158
1158
|
"$ref": "#/$defs/__schema126"
|
|
1159
1159
|
},
|
|
1160
|
-
"
|
|
1160
|
+
"signal": {
|
|
1161
1161
|
"$ref": "#/$defs/__schema128"
|
|
1162
1162
|
},
|
|
1163
|
-
"
|
|
1163
|
+
"stderr": {
|
|
1164
1164
|
"$ref": "#/$defs/__schema130"
|
|
1165
1165
|
},
|
|
1166
|
+
"parseErrors": {
|
|
1167
|
+
"$ref": "#/$defs/__schema132"
|
|
1168
|
+
},
|
|
1166
1169
|
"timedOut": {
|
|
1167
|
-
"$ref": "#/$defs/
|
|
1170
|
+
"$ref": "#/$defs/__schema136"
|
|
1168
1171
|
},
|
|
1169
1172
|
"durationMs": {
|
|
1170
|
-
"$ref": "#/$defs/
|
|
1173
|
+
"$ref": "#/$defs/__schema138"
|
|
1171
1174
|
}
|
|
1172
1175
|
},
|
|
1173
1176
|
"additionalProperties": false,
|
|
@@ -1189,10 +1192,17 @@
|
|
|
1189
1192
|
]
|
|
1190
1193
|
},
|
|
1191
1194
|
"__schema126": {
|
|
1192
|
-
"description": "
|
|
1195
|
+
"description": "Human-readable label for a non-zero harness exit code when the adapter provides one.",
|
|
1193
1196
|
"$ref": "#/$defs/__schema127"
|
|
1194
1197
|
},
|
|
1195
1198
|
"__schema127": {
|
|
1199
|
+
"type": "string"
|
|
1200
|
+
},
|
|
1201
|
+
"__schema128": {
|
|
1202
|
+
"description": "Termination signal when the harness process was signaled.",
|
|
1203
|
+
"$ref": "#/$defs/__schema129"
|
|
1204
|
+
},
|
|
1205
|
+
"__schema129": {
|
|
1196
1206
|
"anyOf": [
|
|
1197
1207
|
{
|
|
1198
1208
|
"type": "string"
|
|
@@ -1202,18 +1212,18 @@
|
|
|
1202
1212
|
}
|
|
1203
1213
|
]
|
|
1204
1214
|
},
|
|
1205
|
-
"
|
|
1215
|
+
"__schema130": {
|
|
1206
1216
|
"description": "Captured stderr from the harness process.",
|
|
1207
|
-
"$ref": "#/$defs/
|
|
1217
|
+
"$ref": "#/$defs/__schema131"
|
|
1208
1218
|
},
|
|
1209
|
-
"
|
|
1219
|
+
"__schema131": {
|
|
1210
1220
|
"type": "string"
|
|
1211
1221
|
},
|
|
1212
|
-
"
|
|
1222
|
+
"__schema132": {
|
|
1213
1223
|
"description": "Parse errors from adapter output handling.",
|
|
1214
|
-
"$ref": "#/$defs/
|
|
1224
|
+
"$ref": "#/$defs/__schema133"
|
|
1215
1225
|
},
|
|
1216
|
-
"
|
|
1226
|
+
"__schema133": {
|
|
1217
1227
|
"type": "array",
|
|
1218
1228
|
"items": {
|
|
1219
1229
|
"$ref": "#/$defs/ParseErrorRecord"
|
|
@@ -1223,10 +1233,10 @@
|
|
|
1223
1233
|
"type": "object",
|
|
1224
1234
|
"properties": {
|
|
1225
1235
|
"line": {
|
|
1226
|
-
"$ref": "#/$defs/
|
|
1236
|
+
"$ref": "#/$defs/__schema134"
|
|
1227
1237
|
},
|
|
1228
1238
|
"error": {
|
|
1229
|
-
"$ref": "#/$defs/
|
|
1239
|
+
"$ref": "#/$defs/__schema135"
|
|
1230
1240
|
}
|
|
1231
1241
|
},
|
|
1232
1242
|
"required": [
|
|
@@ -1237,55 +1247,55 @@
|
|
|
1237
1247
|
"title": "ParseErrorRecord",
|
|
1238
1248
|
"description": "One stream-json or adapter output parse failure."
|
|
1239
1249
|
},
|
|
1240
|
-
"
|
|
1250
|
+
"__schema134": {
|
|
1241
1251
|
"type": "string",
|
|
1242
1252
|
"description": "Raw line from harness output that failed to parse."
|
|
1243
1253
|
},
|
|
1244
|
-
"
|
|
1254
|
+
"__schema135": {
|
|
1245
1255
|
"type": "string",
|
|
1246
1256
|
"description": "Parse error message."
|
|
1247
1257
|
},
|
|
1248
|
-
"
|
|
1258
|
+
"__schema136": {
|
|
1249
1259
|
"description": "Whether the harness run hit the configured timeout.",
|
|
1250
|
-
"$ref": "#/$defs/
|
|
1260
|
+
"$ref": "#/$defs/__schema137"
|
|
1251
1261
|
},
|
|
1252
|
-
"
|
|
1262
|
+
"__schema137": {
|
|
1253
1263
|
"type": "boolean"
|
|
1254
1264
|
},
|
|
1255
|
-
"
|
|
1265
|
+
"__schema138": {
|
|
1256
1266
|
"description": "Harness process duration in milliseconds.",
|
|
1257
|
-
"$ref": "#/$defs/
|
|
1267
|
+
"$ref": "#/$defs/__schema139"
|
|
1258
1268
|
},
|
|
1259
|
-
"
|
|
1269
|
+
"__schema139": {
|
|
1260
1270
|
"type": "number"
|
|
1261
1271
|
},
|
|
1262
|
-
"
|
|
1272
|
+
"__schema140": {
|
|
1263
1273
|
"type": "array",
|
|
1264
1274
|
"items": {
|
|
1265
|
-
"$ref": "#/$defs/
|
|
1275
|
+
"$ref": "#/$defs/__schema141"
|
|
1266
1276
|
},
|
|
1267
1277
|
"description": "Deterministic behavioral assertion results for this repetition."
|
|
1268
1278
|
},
|
|
1269
|
-
"
|
|
1279
|
+
"__schema141": {
|
|
1270
1280
|
"$ref": "#/$defs/AssertionResult"
|
|
1271
1281
|
},
|
|
1272
1282
|
"AssertionResult": {
|
|
1273
1283
|
"type": "object",
|
|
1274
1284
|
"properties": {
|
|
1275
1285
|
"passed": {
|
|
1276
|
-
"$ref": "#/$defs/
|
|
1286
|
+
"$ref": "#/$defs/__schema142"
|
|
1277
1287
|
},
|
|
1278
1288
|
"description": {
|
|
1279
|
-
"$ref": "#/$defs/
|
|
1289
|
+
"$ref": "#/$defs/__schema143"
|
|
1280
1290
|
},
|
|
1281
1291
|
"details": {
|
|
1282
|
-
"$ref": "#/$defs/
|
|
1292
|
+
"$ref": "#/$defs/__schema144"
|
|
1283
1293
|
},
|
|
1284
1294
|
"matches": {
|
|
1285
|
-
"$ref": "#/$defs/
|
|
1295
|
+
"$ref": "#/$defs/__schema145"
|
|
1286
1296
|
},
|
|
1287
1297
|
"children": {
|
|
1288
|
-
"$ref": "#/$defs/
|
|
1298
|
+
"$ref": "#/$defs/__schema147"
|
|
1289
1299
|
}
|
|
1290
1300
|
},
|
|
1291
1301
|
"required": [
|
|
@@ -1297,39 +1307,39 @@
|
|
|
1297
1307
|
"title": "AssertionResult",
|
|
1298
1308
|
"description": "Result of evaluating one assertion, optionally with child nodes."
|
|
1299
1309
|
},
|
|
1300
|
-
"
|
|
1310
|
+
"__schema142": {
|
|
1301
1311
|
"type": "boolean",
|
|
1302
1312
|
"description": "Whether this assertion node passed."
|
|
1303
1313
|
},
|
|
1304
|
-
"
|
|
1314
|
+
"__schema143": {
|
|
1305
1315
|
"type": "string",
|
|
1306
1316
|
"description": "Short human-readable name, e.g. called(mcp__api__SearchSkills, >= 1)."
|
|
1307
1317
|
},
|
|
1308
|
-
"
|
|
1318
|
+
"__schema144": {
|
|
1309
1319
|
"type": "string",
|
|
1310
1320
|
"description": "Diagnostic detail explaining pass or fail."
|
|
1311
1321
|
},
|
|
1312
|
-
"
|
|
1322
|
+
"__schema145": {
|
|
1313
1323
|
"description": "Tool calls that satisfied (or could have satisfied) this assertion.",
|
|
1314
|
-
"$ref": "#/$defs/
|
|
1324
|
+
"$ref": "#/$defs/__schema146"
|
|
1315
1325
|
},
|
|
1316
|
-
"
|
|
1326
|
+
"__schema146": {
|
|
1317
1327
|
"type": "array",
|
|
1318
1328
|
"items": {
|
|
1319
1329
|
"$ref": "#/$defs/ToolCall"
|
|
1320
1330
|
}
|
|
1321
1331
|
},
|
|
1322
|
-
"
|
|
1332
|
+
"__schema147": {
|
|
1323
1333
|
"description": "Sub-results for compound assertions (and/or/not).",
|
|
1324
|
-
"$ref": "#/$defs/
|
|
1334
|
+
"$ref": "#/$defs/__schema148"
|
|
1325
1335
|
},
|
|
1326
|
-
"
|
|
1336
|
+
"__schema148": {
|
|
1327
1337
|
"type": "array",
|
|
1328
1338
|
"items": {
|
|
1329
|
-
"$ref": "#/$defs/
|
|
1339
|
+
"$ref": "#/$defs/__schema141"
|
|
1330
1340
|
}
|
|
1331
1341
|
},
|
|
1332
|
-
"
|
|
1342
|
+
"__schema149": {
|
|
1333
1343
|
"description": "LLM or custom judge outcome grades when grading was run.",
|
|
1334
1344
|
"$ref": "#/$defs/OutcomeGrades"
|
|
1335
1345
|
},
|
|
@@ -1337,19 +1347,19 @@
|
|
|
1337
1347
|
"type": "object",
|
|
1338
1348
|
"properties": {
|
|
1339
1349
|
"judge": {
|
|
1340
|
-
"$ref": "#/$defs/
|
|
1350
|
+
"$ref": "#/$defs/__schema150"
|
|
1341
1351
|
},
|
|
1342
1352
|
"expectations": {
|
|
1343
|
-
"$ref": "#/$defs/
|
|
1353
|
+
"$ref": "#/$defs/__schema156"
|
|
1344
1354
|
},
|
|
1345
1355
|
"summary": {
|
|
1346
|
-
"$ref": "#/$defs/
|
|
1356
|
+
"$ref": "#/$defs/__schema160"
|
|
1347
1357
|
},
|
|
1348
1358
|
"evalFeedback": {
|
|
1349
|
-
"$ref": "#/$defs/
|
|
1359
|
+
"$ref": "#/$defs/__schema165"
|
|
1350
1360
|
},
|
|
1351
1361
|
"error": {
|
|
1352
|
-
"$ref": "#/$defs/
|
|
1362
|
+
"$ref": "#/$defs/__schema171"
|
|
1353
1363
|
}
|
|
1354
1364
|
},
|
|
1355
1365
|
"required": [
|
|
@@ -1361,39 +1371,39 @@
|
|
|
1361
1371
|
"title": "OutcomeGrades",
|
|
1362
1372
|
"description": "Outcome grades for one repetition from the built-in or external LLM judge."
|
|
1363
1373
|
},
|
|
1364
|
-
"
|
|
1374
|
+
"__schema150": {
|
|
1365
1375
|
"description": "Judge that produced these grades.",
|
|
1366
1376
|
"$ref": "#/$defs/JudgeInfo"
|
|
1367
1377
|
},
|
|
1368
|
-
"
|
|
1378
|
+
"__schema151": {
|
|
1369
1379
|
"type": "string",
|
|
1370
1380
|
"description": "Stable judge identifier, e.g. harness-eval/claude-grader or langsmith/my-judge."
|
|
1371
1381
|
},
|
|
1372
|
-
"
|
|
1382
|
+
"__schema152": {
|
|
1373
1383
|
"description": "Model used by the judge, when applicable.",
|
|
1374
|
-
"$ref": "#/$defs/
|
|
1384
|
+
"$ref": "#/$defs/__schema153"
|
|
1375
1385
|
},
|
|
1376
|
-
"
|
|
1386
|
+
"__schema153": {
|
|
1377
1387
|
"type": "string"
|
|
1378
1388
|
},
|
|
1379
|
-
"
|
|
1389
|
+
"__schema154": {
|
|
1380
1390
|
"description": "Judge or grader package version.",
|
|
1381
|
-
"$ref": "#/$defs/
|
|
1391
|
+
"$ref": "#/$defs/__schema155"
|
|
1382
1392
|
},
|
|
1383
|
-
"
|
|
1393
|
+
"__schema155": {
|
|
1384
1394
|
"type": "string"
|
|
1385
1395
|
},
|
|
1386
1396
|
"JudgeInfo": {
|
|
1387
1397
|
"type": "object",
|
|
1388
1398
|
"properties": {
|
|
1389
1399
|
"id": {
|
|
1390
|
-
"$ref": "#/$defs/
|
|
1400
|
+
"$ref": "#/$defs/__schema151"
|
|
1391
1401
|
},
|
|
1392
1402
|
"model": {
|
|
1393
|
-
"$ref": "#/$defs/
|
|
1403
|
+
"$ref": "#/$defs/__schema152"
|
|
1394
1404
|
},
|
|
1395
1405
|
"version": {
|
|
1396
|
-
"$ref": "#/$defs/
|
|
1406
|
+
"$ref": "#/$defs/__schema154"
|
|
1397
1407
|
}
|
|
1398
1408
|
},
|
|
1399
1409
|
"required": [
|
|
@@ -1403,7 +1413,7 @@
|
|
|
1403
1413
|
"title": "JudgeInfo",
|
|
1404
1414
|
"description": "Identity of the judge that produced outcome grades."
|
|
1405
1415
|
},
|
|
1406
|
-
"
|
|
1416
|
+
"__schema156": {
|
|
1407
1417
|
"type": "array",
|
|
1408
1418
|
"items": {
|
|
1409
1419
|
"$ref": "#/$defs/GradedExpectation"
|
|
@@ -1414,13 +1424,13 @@
|
|
|
1414
1424
|
"type": "object",
|
|
1415
1425
|
"properties": {
|
|
1416
1426
|
"text": {
|
|
1417
|
-
"$ref": "#/$defs/
|
|
1427
|
+
"$ref": "#/$defs/__schema157"
|
|
1418
1428
|
},
|
|
1419
1429
|
"passed": {
|
|
1420
|
-
"$ref": "#/$defs/
|
|
1430
|
+
"$ref": "#/$defs/__schema158"
|
|
1421
1431
|
},
|
|
1422
1432
|
"evidence": {
|
|
1423
|
-
"$ref": "#/$defs/
|
|
1433
|
+
"$ref": "#/$defs/__schema159"
|
|
1424
1434
|
}
|
|
1425
1435
|
},
|
|
1426
1436
|
"required": [
|
|
@@ -1432,41 +1442,41 @@
|
|
|
1432
1442
|
"title": "GradedExpectation",
|
|
1433
1443
|
"description": "Outcome grade for one natural-language expectation."
|
|
1434
1444
|
},
|
|
1435
|
-
"
|
|
1445
|
+
"__schema157": {
|
|
1436
1446
|
"type": "string",
|
|
1437
1447
|
"description": "Natural-language expectation that was graded."
|
|
1438
1448
|
},
|
|
1439
|
-
"
|
|
1449
|
+
"__schema158": {
|
|
1440
1450
|
"type": "boolean",
|
|
1441
1451
|
"description": "Whether the judge deemed this expectation satisfied."
|
|
1442
1452
|
},
|
|
1443
|
-
"
|
|
1453
|
+
"__schema159": {
|
|
1444
1454
|
"type": "string",
|
|
1445
1455
|
"description": "Judge explanation citing transcript or trajectory evidence."
|
|
1446
1456
|
},
|
|
1447
|
-
"
|
|
1457
|
+
"__schema160": {
|
|
1448
1458
|
"description": "Aggregate pass/fail counts.",
|
|
1449
1459
|
"$ref": "#/$defs/GradingSummary"
|
|
1450
1460
|
},
|
|
1451
|
-
"
|
|
1461
|
+
"__schema161": {
|
|
1452
1462
|
"type": "integer",
|
|
1453
1463
|
"minimum": 0,
|
|
1454
1464
|
"maximum": 9007199254740991,
|
|
1455
1465
|
"description": "Count of expectations that passed."
|
|
1456
1466
|
},
|
|
1457
|
-
"
|
|
1467
|
+
"__schema162": {
|
|
1458
1468
|
"type": "integer",
|
|
1459
1469
|
"minimum": 0,
|
|
1460
1470
|
"maximum": 9007199254740991,
|
|
1461
1471
|
"description": "Count of expectations that failed."
|
|
1462
1472
|
},
|
|
1463
|
-
"
|
|
1473
|
+
"__schema163": {
|
|
1464
1474
|
"type": "integer",
|
|
1465
1475
|
"minimum": 0,
|
|
1466
1476
|
"maximum": 9007199254740991,
|
|
1467
1477
|
"description": "Total expectations graded."
|
|
1468
1478
|
},
|
|
1469
|
-
"
|
|
1479
|
+
"__schema164": {
|
|
1470
1480
|
"type": "number",
|
|
1471
1481
|
"minimum": 0,
|
|
1472
1482
|
"maximum": 1,
|
|
@@ -1476,16 +1486,16 @@
|
|
|
1476
1486
|
"type": "object",
|
|
1477
1487
|
"properties": {
|
|
1478
1488
|
"passed": {
|
|
1479
|
-
"$ref": "#/$defs/
|
|
1489
|
+
"$ref": "#/$defs/__schema161"
|
|
1480
1490
|
},
|
|
1481
1491
|
"failed": {
|
|
1482
|
-
"$ref": "#/$defs/
|
|
1492
|
+
"$ref": "#/$defs/__schema162"
|
|
1483
1493
|
},
|
|
1484
1494
|
"total": {
|
|
1485
|
-
"$ref": "#/$defs/
|
|
1495
|
+
"$ref": "#/$defs/__schema163"
|
|
1486
1496
|
},
|
|
1487
1497
|
"passRate": {
|
|
1488
|
-
"$ref": "#/$defs/
|
|
1498
|
+
"$ref": "#/$defs/__schema164"
|
|
1489
1499
|
}
|
|
1490
1500
|
},
|
|
1491
1501
|
"required": [
|
|
@@ -1498,7 +1508,7 @@
|
|
|
1498
1508
|
"title": "GradingSummary",
|
|
1499
1509
|
"description": "Aggregate counts for a set of graded expectations."
|
|
1500
1510
|
},
|
|
1501
|
-
"
|
|
1511
|
+
"__schema165": {
|
|
1502
1512
|
"description": "Optional structured feedback for suite authors.",
|
|
1503
1513
|
"$ref": "#/$defs/EvalFeedback"
|
|
1504
1514
|
},
|
|
@@ -1506,10 +1516,10 @@
|
|
|
1506
1516
|
"type": "object",
|
|
1507
1517
|
"properties": {
|
|
1508
1518
|
"suggestions": {
|
|
1509
|
-
"$ref": "#/$defs/
|
|
1519
|
+
"$ref": "#/$defs/__schema166"
|
|
1510
1520
|
},
|
|
1511
1521
|
"overall": {
|
|
1512
|
-
"$ref": "#/$defs/
|
|
1522
|
+
"$ref": "#/$defs/__schema170"
|
|
1513
1523
|
}
|
|
1514
1524
|
},
|
|
1515
1525
|
"required": [
|
|
@@ -1520,7 +1530,7 @@
|
|
|
1520
1530
|
"title": "EvalFeedback",
|
|
1521
1531
|
"description": "Structured eval feedback from the outcome judge."
|
|
1522
1532
|
},
|
|
1523
|
-
"
|
|
1533
|
+
"__schema166": {
|
|
1524
1534
|
"type": "array",
|
|
1525
1535
|
"items": {
|
|
1526
1536
|
"$ref": "#/$defs/EvalFeedbackSuggestion"
|
|
@@ -1531,10 +1541,10 @@
|
|
|
1531
1541
|
"type": "object",
|
|
1532
1542
|
"properties": {
|
|
1533
1543
|
"assertion": {
|
|
1534
|
-
"$ref": "#/$defs/
|
|
1544
|
+
"$ref": "#/$defs/__schema167"
|
|
1535
1545
|
},
|
|
1536
1546
|
"reason": {
|
|
1537
|
-
"$ref": "#/$defs/
|
|
1547
|
+
"$ref": "#/$defs/__schema169"
|
|
1538
1548
|
}
|
|
1539
1549
|
},
|
|
1540
1550
|
"required": [
|
|
@@ -1544,33 +1554,33 @@
|
|
|
1544
1554
|
"title": "EvalFeedbackSuggestion",
|
|
1545
1555
|
"description": "Actionable suggestion from eval feedback."
|
|
1546
1556
|
},
|
|
1547
|
-
"
|
|
1557
|
+
"__schema167": {
|
|
1548
1558
|
"description": "Related assertion or expectation text, when the suggestion targets one.",
|
|
1549
|
-
"$ref": "#/$defs/
|
|
1559
|
+
"$ref": "#/$defs/__schema168"
|
|
1550
1560
|
},
|
|
1551
|
-
"
|
|
1561
|
+
"__schema168": {
|
|
1552
1562
|
"type": "string"
|
|
1553
1563
|
},
|
|
1554
|
-
"
|
|
1564
|
+
"__schema169": {
|
|
1555
1565
|
"type": "string",
|
|
1556
1566
|
"description": "Why the judge suggests changing the suite or assertions."
|
|
1557
1567
|
},
|
|
1558
|
-
"
|
|
1568
|
+
"__schema170": {
|
|
1559
1569
|
"type": "string",
|
|
1560
1570
|
"description": "Overall narrative feedback from the judge."
|
|
1561
1571
|
},
|
|
1562
|
-
"
|
|
1572
|
+
"__schema171": {
|
|
1563
1573
|
"description": "Error message when grading failed for this repetition.",
|
|
1564
|
-
"$ref": "#/$defs/
|
|
1574
|
+
"$ref": "#/$defs/__schema172"
|
|
1565
1575
|
},
|
|
1566
|
-
"
|
|
1576
|
+
"__schema172": {
|
|
1567
1577
|
"type": "string"
|
|
1568
1578
|
},
|
|
1569
|
-
"
|
|
1579
|
+
"__schema173": {
|
|
1570
1580
|
"description": "Scores from external eval frameworks (LangSmith, Braintrust, etc.).",
|
|
1571
|
-
"$ref": "#/$defs/
|
|
1581
|
+
"$ref": "#/$defs/__schema174"
|
|
1572
1582
|
},
|
|
1573
|
-
"
|
|
1583
|
+
"__schema174": {
|
|
1574
1584
|
"type": "array",
|
|
1575
1585
|
"items": {
|
|
1576
1586
|
"$ref": "#/$defs/ExternalScore"
|
|
@@ -1580,16 +1590,16 @@
|
|
|
1580
1590
|
"type": "object",
|
|
1581
1591
|
"properties": {
|
|
1582
1592
|
"source": {
|
|
1583
|
-
"$ref": "#/$defs/
|
|
1593
|
+
"$ref": "#/$defs/__schema175"
|
|
1584
1594
|
},
|
|
1585
1595
|
"metric": {
|
|
1586
|
-
"$ref": "#/$defs/
|
|
1596
|
+
"$ref": "#/$defs/__schema176"
|
|
1587
1597
|
},
|
|
1588
1598
|
"value": {
|
|
1589
|
-
"$ref": "#/$defs/
|
|
1599
|
+
"$ref": "#/$defs/__schema177"
|
|
1590
1600
|
},
|
|
1591
1601
|
"metadata": {
|
|
1592
|
-
"$ref": "#/$defs/
|
|
1602
|
+
"$ref": "#/$defs/__schema181"
|
|
1593
1603
|
}
|
|
1594
1604
|
},
|
|
1595
1605
|
"required": [
|
|
@@ -1601,49 +1611,49 @@
|
|
|
1601
1611
|
"title": "ExternalScore",
|
|
1602
1612
|
"description": "Score attached from an external eval platform without replacing OutcomeGrades."
|
|
1603
1613
|
},
|
|
1604
|
-
"
|
|
1614
|
+
"__schema175": {
|
|
1605
1615
|
"type": "string",
|
|
1606
1616
|
"description": "External framework identifier, e.g. langsmith or braintrust."
|
|
1607
1617
|
},
|
|
1608
|
-
"
|
|
1618
|
+
"__schema176": {
|
|
1609
1619
|
"type": "string",
|
|
1610
1620
|
"description": "Metric name within the external framework."
|
|
1611
1621
|
},
|
|
1612
|
-
"
|
|
1622
|
+
"__schema177": {
|
|
1613
1623
|
"anyOf": [
|
|
1614
1624
|
{
|
|
1615
|
-
"$ref": "#/$defs/
|
|
1625
|
+
"$ref": "#/$defs/__schema178"
|
|
1616
1626
|
},
|
|
1617
1627
|
{
|
|
1618
|
-
"$ref": "#/$defs/
|
|
1628
|
+
"$ref": "#/$defs/__schema179"
|
|
1619
1629
|
},
|
|
1620
1630
|
{
|
|
1621
|
-
"$ref": "#/$defs/
|
|
1631
|
+
"$ref": "#/$defs/__schema180"
|
|
1622
1632
|
}
|
|
1623
1633
|
],
|
|
1624
1634
|
"description": "Metric value (numeric score, boolean pass, or categorical label)."
|
|
1625
1635
|
},
|
|
1626
|
-
"
|
|
1636
|
+
"__schema178": {
|
|
1627
1637
|
"type": "number"
|
|
1628
1638
|
},
|
|
1629
|
-
"
|
|
1639
|
+
"__schema179": {
|
|
1630
1640
|
"type": "boolean"
|
|
1631
1641
|
},
|
|
1632
|
-
"
|
|
1642
|
+
"__schema180": {
|
|
1633
1643
|
"type": "string"
|
|
1634
1644
|
},
|
|
1635
|
-
"
|
|
1645
|
+
"__schema181": {
|
|
1636
1646
|
"description": "Framework-specific metadata (run ids, trace urls, etc.).",
|
|
1637
|
-
"$ref": "#/$defs/
|
|
1647
|
+
"$ref": "#/$defs/__schema182"
|
|
1638
1648
|
},
|
|
1639
|
-
"
|
|
1649
|
+
"__schema182": {
|
|
1640
1650
|
"type": "object",
|
|
1641
1651
|
"propertyNames": {
|
|
1642
1652
|
"type": "string"
|
|
1643
1653
|
},
|
|
1644
1654
|
"additionalProperties": {}
|
|
1645
1655
|
},
|
|
1646
|
-
"
|
|
1656
|
+
"__schema183": {
|
|
1647
1657
|
"description": "Optional transcript, raw stream, or OTLP URI artifacts.",
|
|
1648
1658
|
"$ref": "#/$defs/EvalArtifacts"
|
|
1649
1659
|
},
|
|
@@ -1651,42 +1661,42 @@
|
|
|
1651
1661
|
"type": "object",
|
|
1652
1662
|
"properties": {
|
|
1653
1663
|
"rawStreamEvents": {
|
|
1654
|
-
"$ref": "#/$defs/
|
|
1664
|
+
"$ref": "#/$defs/__schema184"
|
|
1655
1665
|
},
|
|
1656
1666
|
"otlpTraceUri": {
|
|
1657
|
-
"$ref": "#/$defs/
|
|
1667
|
+
"$ref": "#/$defs/__schema186"
|
|
1658
1668
|
},
|
|
1659
1669
|
"transcript": {
|
|
1660
|
-
"$ref": "#/$defs/
|
|
1670
|
+
"$ref": "#/$defs/__schema188"
|
|
1661
1671
|
}
|
|
1662
1672
|
},
|
|
1663
1673
|
"additionalProperties": false,
|
|
1664
1674
|
"title": "EvalArtifacts",
|
|
1665
1675
|
"description": "Optional large or vendor-specific blobs. Store by reference in DB when possible."
|
|
1666
1676
|
},
|
|
1667
|
-
"
|
|
1677
|
+
"__schema184": {
|
|
1668
1678
|
"description": "Claude Code stream-json lines — debug only, not cross-harness. Prefer transcript for judges.",
|
|
1669
|
-
"$ref": "#/$defs/
|
|
1679
|
+
"$ref": "#/$defs/__schema185"
|
|
1670
1680
|
},
|
|
1671
|
-
"
|
|
1681
|
+
"__schema185": {
|
|
1672
1682
|
"type": "array",
|
|
1673
1683
|
"items": {}
|
|
1674
1684
|
},
|
|
1675
|
-
"
|
|
1685
|
+
"__schema186": {
|
|
1676
1686
|
"description": "URI to an OTLP trace blob (S3, GCS, etc.) when exported separately.",
|
|
1677
|
-
"$ref": "#/$defs/
|
|
1687
|
+
"$ref": "#/$defs/__schema187"
|
|
1678
1688
|
},
|
|
1679
|
-
"
|
|
1689
|
+
"__schema187": {
|
|
1680
1690
|
"type": "string"
|
|
1681
1691
|
},
|
|
1682
|
-
"
|
|
1692
|
+
"__schema188": {
|
|
1683
1693
|
"description": "Text transcript for judges (trajectoryToTranscript output).",
|
|
1684
|
-
"$ref": "#/$defs/
|
|
1694
|
+
"$ref": "#/$defs/__schema189"
|
|
1685
1695
|
},
|
|
1686
|
-
"
|
|
1696
|
+
"__schema189": {
|
|
1687
1697
|
"type": "string"
|
|
1688
1698
|
},
|
|
1689
|
-
"
|
|
1699
|
+
"__schema190": {
|
|
1690
1700
|
"description": "Vertex EvaluationInstance protojson wire object.",
|
|
1691
1701
|
"$ref": "#/$defs/EvaluationInstanceJson"
|
|
1692
1702
|
},
|
|
@@ -1694,20 +1704,20 @@
|
|
|
1694
1704
|
"type": "object",
|
|
1695
1705
|
"properties": {
|
|
1696
1706
|
"prompt": {
|
|
1697
|
-
"$ref": "#/$defs/
|
|
1707
|
+
"$ref": "#/$defs/__schema191"
|
|
1698
1708
|
},
|
|
1699
1709
|
"response": {
|
|
1700
|
-
"$ref": "#/$defs/
|
|
1710
|
+
"$ref": "#/$defs/__schema194"
|
|
1701
1711
|
},
|
|
1702
1712
|
"reference": {
|
|
1703
|
-
"$ref": "#/$defs/
|
|
1713
|
+
"$ref": "#/$defs/__schema195"
|
|
1704
1714
|
}
|
|
1705
1715
|
},
|
|
1706
1716
|
"additionalProperties": false,
|
|
1707
1717
|
"title": "EvaluationInstanceJson",
|
|
1708
1718
|
"description": "Vertex EvaluationInstance wire format (agentEvalData omitted in v1)."
|
|
1709
1719
|
},
|
|
1710
|
-
"
|
|
1720
|
+
"__schema191": {
|
|
1711
1721
|
"description": "Eval prompt.",
|
|
1712
1722
|
"$ref": "#/$defs/InstanceData"
|
|
1713
1723
|
},
|
|
@@ -1715,29 +1725,29 @@
|
|
|
1715
1725
|
"type": "object",
|
|
1716
1726
|
"properties": {
|
|
1717
1727
|
"text": {
|
|
1718
|
-
"$ref": "#/$defs/
|
|
1728
|
+
"$ref": "#/$defs/__schema192"
|
|
1719
1729
|
}
|
|
1720
1730
|
},
|
|
1721
1731
|
"additionalProperties": false,
|
|
1722
1732
|
"title": "InstanceData",
|
|
1723
1733
|
"description": "EvaluationInstance prompt/response/reference text wrapper."
|
|
1724
1734
|
},
|
|
1725
|
-
"
|
|
1735
|
+
"__schema192": {
|
|
1726
1736
|
"description": "Plain text instance data.",
|
|
1727
|
-
"$ref": "#/$defs/
|
|
1737
|
+
"$ref": "#/$defs/__schema193"
|
|
1728
1738
|
},
|
|
1729
|
-
"
|
|
1739
|
+
"__schema193": {
|
|
1730
1740
|
"type": "string"
|
|
1731
1741
|
},
|
|
1732
|
-
"
|
|
1742
|
+
"__schema194": {
|
|
1733
1743
|
"description": "Final agent response.",
|
|
1734
1744
|
"$ref": "#/$defs/InstanceData"
|
|
1735
1745
|
},
|
|
1736
|
-
"
|
|
1746
|
+
"__schema195": {
|
|
1737
1747
|
"description": "Reference answer text.",
|
|
1738
1748
|
"$ref": "#/$defs/InstanceData"
|
|
1739
1749
|
},
|
|
1740
|
-
"
|
|
1750
|
+
"__schema196": {
|
|
1741
1751
|
"description": "Vertex Trajectory*Instance protojson wire objects keyed by metric.",
|
|
1742
1752
|
"$ref": "#/$defs/TrajectoryInstancesJson"
|
|
1743
1753
|
},
|
|
@@ -1745,29 +1755,29 @@
|
|
|
1745
1755
|
"type": "object",
|
|
1746
1756
|
"properties": {
|
|
1747
1757
|
"exactMatch": {
|
|
1748
|
-
"$ref": "#/$defs/
|
|
1758
|
+
"$ref": "#/$defs/__schema197"
|
|
1749
1759
|
},
|
|
1750
1760
|
"inOrderMatch": {
|
|
1751
|
-
"$ref": "#/$defs/
|
|
1761
|
+
"$ref": "#/$defs/__schema200"
|
|
1752
1762
|
},
|
|
1753
1763
|
"anyOrderMatch": {
|
|
1754
|
-
"$ref": "#/$defs/
|
|
1764
|
+
"$ref": "#/$defs/__schema201"
|
|
1755
1765
|
},
|
|
1756
1766
|
"precision": {
|
|
1757
|
-
"$ref": "#/$defs/
|
|
1767
|
+
"$ref": "#/$defs/__schema202"
|
|
1758
1768
|
},
|
|
1759
1769
|
"recall": {
|
|
1760
|
-
"$ref": "#/$defs/
|
|
1770
|
+
"$ref": "#/$defs/__schema203"
|
|
1761
1771
|
},
|
|
1762
1772
|
"singleToolUse": {
|
|
1763
|
-
"$ref": "#/$defs/
|
|
1773
|
+
"$ref": "#/$defs/__schema204"
|
|
1764
1774
|
}
|
|
1765
1775
|
},
|
|
1766
1776
|
"additionalProperties": false,
|
|
1767
1777
|
"title": "TrajectoryInstancesJson",
|
|
1768
1778
|
"description": "Vertex Trajectory*Instance messages keyed by metric."
|
|
1769
1779
|
},
|
|
1770
|
-
"
|
|
1780
|
+
"__schema197": {
|
|
1771
1781
|
"description": "Exact match instance.",
|
|
1772
1782
|
"$ref": "#/$defs/TrajectoryPairInstanceJson"
|
|
1773
1783
|
},
|
|
@@ -1775,10 +1785,10 @@
|
|
|
1775
1785
|
"type": "object",
|
|
1776
1786
|
"properties": {
|
|
1777
1787
|
"predictedTrajectory": {
|
|
1778
|
-
"$ref": "#/$defs/
|
|
1788
|
+
"$ref": "#/$defs/__schema198"
|
|
1779
1789
|
},
|
|
1780
1790
|
"referenceTrajectory": {
|
|
1781
|
-
"$ref": "#/$defs/
|
|
1791
|
+
"$ref": "#/$defs/__schema199"
|
|
1782
1792
|
}
|
|
1783
1793
|
},
|
|
1784
1794
|
"required": [
|
|
@@ -1789,31 +1799,31 @@
|
|
|
1789
1799
|
"title": "TrajectoryPairInstanceJson",
|
|
1790
1800
|
"description": "Shared shape for Trajectory*Match/Precision/Recall instances."
|
|
1791
1801
|
},
|
|
1792
|
-
"
|
|
1802
|
+
"__schema198": {
|
|
1793
1803
|
"description": "Predicted tool-call trajectory.",
|
|
1794
1804
|
"$ref": "#/$defs/ProtojsonTrajectory"
|
|
1795
1805
|
},
|
|
1796
|
-
"
|
|
1806
|
+
"__schema199": {
|
|
1797
1807
|
"description": "Reference tool-call trajectory.",
|
|
1798
1808
|
"$ref": "#/$defs/ProtojsonTrajectory"
|
|
1799
1809
|
},
|
|
1800
|
-
"
|
|
1810
|
+
"__schema200": {
|
|
1801
1811
|
"description": "In-order match instance.",
|
|
1802
1812
|
"$ref": "#/$defs/TrajectoryPairInstanceJson"
|
|
1803
1813
|
},
|
|
1804
|
-
"
|
|
1814
|
+
"__schema201": {
|
|
1805
1815
|
"description": "Any-order match instance.",
|
|
1806
1816
|
"$ref": "#/$defs/TrajectoryPairInstanceJson"
|
|
1807
1817
|
},
|
|
1808
|
-
"
|
|
1818
|
+
"__schema202": {
|
|
1809
1819
|
"description": "Precision instance.",
|
|
1810
1820
|
"$ref": "#/$defs/TrajectoryPairInstanceJson"
|
|
1811
1821
|
},
|
|
1812
|
-
"
|
|
1822
|
+
"__schema203": {
|
|
1813
1823
|
"description": "Recall instance.",
|
|
1814
1824
|
"$ref": "#/$defs/TrajectoryPairInstanceJson"
|
|
1815
1825
|
},
|
|
1816
|
-
"
|
|
1826
|
+
"__schema204": {
|
|
1817
1827
|
"description": "Single tool use instance.",
|
|
1818
1828
|
"$ref": "#/$defs/TrajectorySingleToolUseInstanceJson"
|
|
1819
1829
|
},
|
|
@@ -1821,7 +1831,7 @@
|
|
|
1821
1831
|
"type": "object",
|
|
1822
1832
|
"properties": {
|
|
1823
1833
|
"predictedTrajectory": {
|
|
1824
|
-
"$ref": "#/$defs/
|
|
1834
|
+
"$ref": "#/$defs/__schema205"
|
|
1825
1835
|
}
|
|
1826
1836
|
},
|
|
1827
1837
|
"required": [
|
|
@@ -1831,33 +1841,33 @@
|
|
|
1831
1841
|
"title": "TrajectorySingleToolUseInstanceJson",
|
|
1832
1842
|
"description": "Vertex TrajectorySingleToolUseInstance wire format."
|
|
1833
1843
|
},
|
|
1834
|
-
"
|
|
1844
|
+
"__schema205": {
|
|
1835
1845
|
"description": "Predicted tool-call trajectory.",
|
|
1836
1846
|
"$ref": "#/$defs/ProtojsonTrajectory"
|
|
1837
1847
|
},
|
|
1838
|
-
"
|
|
1848
|
+
"__schema206": {
|
|
1839
1849
|
"$ref": "#/$defs/HarnessMetrics"
|
|
1840
1850
|
},
|
|
1841
1851
|
"HarnessMetrics": {
|
|
1842
1852
|
"type": "object",
|
|
1843
1853
|
"properties": {
|
|
1844
1854
|
"trajectoryExactMatch": {
|
|
1845
|
-
"$ref": "#/$defs/
|
|
1855
|
+
"$ref": "#/$defs/__schema207"
|
|
1846
1856
|
},
|
|
1847
1857
|
"trajectoryInOrderMatch": {
|
|
1848
|
-
"$ref": "#/$defs/
|
|
1858
|
+
"$ref": "#/$defs/__schema208"
|
|
1849
1859
|
},
|
|
1850
1860
|
"trajectoryAnyOrderMatch": {
|
|
1851
|
-
"$ref": "#/$defs/
|
|
1861
|
+
"$ref": "#/$defs/__schema209"
|
|
1852
1862
|
},
|
|
1853
1863
|
"trajectoryPrecision": {
|
|
1854
|
-
"$ref": "#/$defs/
|
|
1864
|
+
"$ref": "#/$defs/__schema210"
|
|
1855
1865
|
},
|
|
1856
1866
|
"trajectoryRecall": {
|
|
1857
|
-
"$ref": "#/$defs/
|
|
1867
|
+
"$ref": "#/$defs/__schema211"
|
|
1858
1868
|
},
|
|
1859
1869
|
"trajectorySingleToolUse": {
|
|
1860
|
-
"$ref": "#/$defs/
|
|
1870
|
+
"$ref": "#/$defs/__schema212"
|
|
1861
1871
|
}
|
|
1862
1872
|
},
|
|
1863
1873
|
"required": [
|
|
@@ -1872,42 +1882,42 @@
|
|
|
1872
1882
|
"title": "HarnessMetrics",
|
|
1873
1883
|
"description": "Harness-precomputed trajectory metric scores."
|
|
1874
1884
|
},
|
|
1875
|
-
"
|
|
1885
|
+
"__schema207": {
|
|
1876
1886
|
"type": "number",
|
|
1877
1887
|
"description": "Exact trajectory match score (0 or 1)."
|
|
1878
1888
|
},
|
|
1879
|
-
"
|
|
1889
|
+
"__schema208": {
|
|
1880
1890
|
"type": "number",
|
|
1881
1891
|
"description": "In-order trajectory match score (0 or 1)."
|
|
1882
1892
|
},
|
|
1883
|
-
"
|
|
1893
|
+
"__schema209": {
|
|
1884
1894
|
"type": "number",
|
|
1885
1895
|
"description": "Any-order trajectory match score (0 or 1)."
|
|
1886
1896
|
},
|
|
1887
|
-
"
|
|
1897
|
+
"__schema210": {
|
|
1888
1898
|
"type": "number",
|
|
1889
1899
|
"description": "Trajectory precision (0..1)."
|
|
1890
1900
|
},
|
|
1891
|
-
"
|
|
1901
|
+
"__schema211": {
|
|
1892
1902
|
"type": "number",
|
|
1893
1903
|
"description": "Trajectory recall (0..1)."
|
|
1894
1904
|
},
|
|
1895
|
-
"
|
|
1905
|
+
"__schema212": {
|
|
1896
1906
|
"type": "number",
|
|
1897
1907
|
"description": "Single-tool-use match score (0 or 1)."
|
|
1898
1908
|
},
|
|
1899
|
-
"
|
|
1909
|
+
"__schema213": {
|
|
1900
1910
|
"description": "Session latency in seconds.",
|
|
1901
|
-
"$ref": "#/$defs/
|
|
1911
|
+
"$ref": "#/$defs/__schema214"
|
|
1902
1912
|
},
|
|
1903
|
-
"
|
|
1913
|
+
"__schema214": {
|
|
1904
1914
|
"type": "number"
|
|
1905
1915
|
},
|
|
1906
|
-
"
|
|
1916
|
+
"__schema215": {
|
|
1907
1917
|
"description": "1 when the harness run failed, 0 on success.",
|
|
1908
|
-
"$ref": "#/$defs/
|
|
1918
|
+
"$ref": "#/$defs/__schema216"
|
|
1909
1919
|
},
|
|
1910
|
-
"
|
|
1920
|
+
"__schema216": {
|
|
1911
1921
|
"anyOf": [
|
|
1912
1922
|
{
|
|
1913
1923
|
"type": "number",
|
|
@@ -1919,7 +1929,7 @@
|
|
|
1919
1929
|
}
|
|
1920
1930
|
]
|
|
1921
1931
|
},
|
|
1922
|
-
"
|
|
1932
|
+
"__schema217": {
|
|
1923
1933
|
"description": "Present when the harness failed without producing a trajectory.",
|
|
1924
1934
|
"$ref": "#/$defs/RepetitionError"
|
|
1925
1935
|
},
|
|
@@ -1927,10 +1937,10 @@
|
|
|
1927
1937
|
"type": "object",
|
|
1928
1938
|
"properties": {
|
|
1929
1939
|
"message": {
|
|
1930
|
-
"$ref": "#/$defs/
|
|
1940
|
+
"$ref": "#/$defs/__schema218"
|
|
1931
1941
|
},
|
|
1932
1942
|
"diagnostics": {
|
|
1933
|
-
"$ref": "#/$defs/
|
|
1943
|
+
"$ref": "#/$defs/__schema219"
|
|
1934
1944
|
}
|
|
1935
1945
|
},
|
|
1936
1946
|
"required": [
|
|
@@ -1940,11 +1950,11 @@
|
|
|
1940
1950
|
"title": "RepetitionError",
|
|
1941
1951
|
"description": "Harness failure for one repetition without a usable TrajectoryView."
|
|
1942
1952
|
},
|
|
1943
|
-
"
|
|
1953
|
+
"__schema218": {
|
|
1944
1954
|
"type": "string",
|
|
1945
1955
|
"description": "Harness failure message for this repetition."
|
|
1946
1956
|
},
|
|
1947
|
-
"
|
|
1957
|
+
"__schema219": {
|
|
1948
1958
|
"description": "Adapter diagnostics when the harness failed before producing a trajectory.",
|
|
1949
1959
|
"$ref": "#/$defs/AdapterDiagnostics"
|
|
1950
1960
|
}
|