agentv 5.2.0-next.1 → 5.3.1-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +22 -20
  2. package/dist/{artifact-writer-SN6UNL5G.js → artifact-writer-7NBCOAYC.js} +4 -4
  3. package/dist/{chunk-3X357HS4.js → chunk-ASIGJIOJ.js} +107 -31
  4. package/dist/chunk-ASIGJIOJ.js.map +1 -0
  5. package/dist/{chunk-3S6A2RKR.js → chunk-ELCJ23K4.js} +18879 -17538
  6. package/dist/chunk-ELCJ23K4.js.map +1 -0
  7. package/dist/{chunk-T4AD3H3Y.js → chunk-LKGARI3W.js} +399 -32
  8. package/dist/chunk-LKGARI3W.js.map +1 -0
  9. package/dist/{chunk-ALVZQUZP.js → chunk-LXBI3SPX.js} +142 -14
  10. package/dist/chunk-LXBI3SPX.js.map +1 -0
  11. package/dist/{chunk-46K2OET3.js → chunk-RKE7SSET.js} +2 -2
  12. package/dist/cli.js +5 -5
  13. package/dist/dashboard/assets/{index-DNgf3qJ2.js → index-CbEMiJSb.js} +1 -1
  14. package/dist/dashboard/assets/{index-r_jSJmlw.js → index-DTA6-l7q.js} +3 -3
  15. package/dist/dashboard/index.html +1 -1
  16. package/dist/{dist-DEPJOMCA.js → dist-NMXMI5SK.js} +25 -5
  17. package/dist/index.js +5 -5
  18. package/dist/{interactive-36ZNMQB2.js → interactive-BN527UV3.js} +5 -5
  19. package/dist/skills/agentv-bench/SKILL.md +15 -14
  20. package/dist/skills/agentv-bench/agents/analyzer.md +2 -2
  21. package/dist/skills/agentv-bench/agents/comparator.md +3 -3
  22. package/dist/skills/agentv-bench/agents/executor.md +1 -1
  23. package/dist/skills/agentv-bench/agents/grader.md +4 -2
  24. package/dist/skills/agentv-bench/references/autoresearch.md +9 -9
  25. package/dist/skills/agentv-bench/references/description-optimization.md +5 -6
  26. package/dist/skills/agentv-bench/references/environment-adaptation.md +6 -6
  27. package/dist/skills/agentv-bench/references/eval-yaml-spec.md +50 -56
  28. package/dist/skills/agentv-bench/references/schemas.md +44 -60
  29. package/dist/skills/agentv-bench/references/subagent-pipeline.md +20 -18
  30. package/dist/skills/agentv-eval-migrations/SKILL.md +13 -0
  31. package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +138 -22
  32. package/dist/skills/agentv-eval-writer/SKILL.md +103 -63
  33. package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +10 -5
  34. package/dist/skills/agentv-eval-writer/references/eval.schema.json +749 -2158
  35. package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +1 -1
  36. package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js → ts-eval-loader-2RFVZHCT-7CZ3DCDD.js} +8 -4
  37. package/package.json +1 -1
  38. package/dist/chunk-3S6A2RKR.js.map +0 -1
  39. package/dist/chunk-3X357HS4.js.map +0 -1
  40. package/dist/chunk-ALVZQUZP.js.map +0 -1
  41. package/dist/chunk-T4AD3H3Y.js.map +0 -1
  42. /package/dist/{artifact-writer-SN6UNL5G.js.map → artifact-writer-7NBCOAYC.js.map} +0 -0
  43. /package/dist/{chunk-46K2OET3.js.map → chunk-RKE7SSET.js.map} +0 -0
  44. /package/dist/{dist-DEPJOMCA.js.map → dist-NMXMI5SK.js.map} +0 -0
  45. /package/dist/{interactive-36ZNMQB2.js.map → interactive-BN527UV3.js.map} +0 -0
  46. /package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map → ts-eval-loader-2RFVZHCT-7CZ3DCDD.js.map} +0 -0
package/dist/skills/agentv-eval-writer/references/eval.schema.json CHANGED
@@ -2067,6 +2067,12 @@
2067
2067
  }
2068
2068
  ]
2069
2069
  },
2070
+ "providerPromptMap": {
2071
+ "not": {}
2072
+ },
2073
+ "provider_prompt_map": {
2074
+ "not": {}
2075
+ },
2070
2076
  "graders": {
2071
2077
  "anyOf": [
2072
2078
  {
@@ -2120,2018 +2126,164 @@
2120
2126
  ]
2121
2127
  },
2122
2128
  "eval_cases": {
2129
+ "not": {}
2130
+ },
2131
+ "evalcases": {
2132
+ "not": {}
2133
+ },
2134
+ "target": {
2123
2135
  "anyOf": [
2124
2136
  {
2125
- "type": "array",
2126
- "items": {
2127
- "anyOf": [
2128
- {
2129
- "type": "object",
2130
- "properties": {
2131
- "id": {
2132
- "type": "string",
2133
- "minLength": 1
2134
- },
2135
- "description": {
2136
- "type": "string"
2137
- },
2138
- "vars": {
2139
- "type": "object",
2140
- "properties": {},
2141
- "additionalProperties": {}
2142
- },
2143
- "provider": {
2137
+ "type": "string",
2138
+ "minLength": 1
2139
+ },
2140
+ {
2141
+ "type": "object",
2142
+ "properties": {
2143
+ "id": {
2144
+ "type": "string",
2145
+ "minLength": 1
2146
+ },
2147
+ "extends": {
2148
+ "type": "string",
2149
+ "minLength": 1
2150
+ },
2151
+ "provider": {
2152
+ "type": "string",
2153
+ "minLength": 1
2154
+ },
2155
+ "model": {
2156
+ "type": "string",
2157
+ "minLength": 1
2158
+ },
2159
+ "config": {
2160
+ "type": "object",
2161
+ "additionalProperties": {}
2162
+ },
2163
+ "prompts": {
2164
+ "anyOf": [
2165
+ {
2166
+ "anyOf": [
2167
+ {
2168
+ "type": "string"
2169
+ },
2170
+ {
2171
+ "type": "object",
2172
+ "properties": {
2173
+ "command": {
2174
+ "anyOf": [
2175
+ {
2176
+ "type": "string"
2177
+ },
2178
+ {
2179
+ "type": "array",
2180
+ "items": {
2181
+ "type": "string"
2182
+ }
2183
+ }
2184
+ ]
2185
+ },
2186
+ "config": {
2187
+ "type": "object",
2188
+ "additionalProperties": {}
2189
+ }
2190
+ },
2191
+ "required": ["command"],
2192
+ "additionalProperties": false
2193
+ },
2194
+ {
2195
+ "type": "object",
2196
+ "properties": {
2197
+ "id": {
2198
+ "type": "string"
2199
+ },
2200
+ "label": {
2201
+ "type": "string"
2202
+ },
2203
+ "raw": {
2204
+ "type": "string"
2205
+ },
2206
+ "function": {
2207
+ "type": "string"
2208
+ },
2209
+ "function_file": {
2210
+ "type": "string"
2211
+ },
2212
+ "path": {
2213
+ "type": "string"
2214
+ },
2215
+ "prefix": {
2216
+ "type": "string"
2217
+ },
2218
+ "suffix": {
2219
+ "type": "string"
2220
+ },
2221
+ "config": {
2222
+ "type": "object",
2223
+ "additionalProperties": {}
2224
+ }
2225
+ },
2226
+ "additionalProperties": true
2227
+ }
2228
+ ]
2229
+ },
2230
+ {
2231
+ "type": "array",
2232
+ "items": {
2144
2233
  "anyOf": [
2145
2234
  {
2146
- "type": "string",
2147
- "minLength": 1
2235
+ "type": "string"
2236
+ },
2237
+ {
2238
+ "type": "object",
2239
+ "properties": {
2240
+ "command": {
2241
+ "anyOf": [
2242
+ {
2243
+ "type": "string"
2244
+ },
2245
+ {
2246
+ "type": "array",
2247
+ "items": {
2248
+ "type": "string"
2249
+ }
2250
+ }
2251
+ ]
2252
+ },
2253
+ "config": {
2254
+ "type": "object",
2255
+ "additionalProperties": {}
2256
+ }
2257
+ },
2258
+ "required": ["command"],
2259
+ "additionalProperties": false
2148
2260
  },
2149
2261
  {
2150
2262
  "type": "object",
2151
2263
  "properties": {
2152
2264
  "id": {
2153
- "type": "string",
2154
- "minLength": 1
2265
+ "type": "string"
2155
2266
  },
2156
- "extends": {
2157
- "type": "string",
2158
- "minLength": 1
2267
+ "label": {
2268
+ "type": "string"
2159
2269
  },
2160
- "provider": {
2161
- "type": "string",
2162
- "minLength": 1
2270
+ "raw": {
2271
+ "type": "string"
2163
2272
  },
2164
- "model": {
2165
- "type": "string",
2166
- "minLength": 1
2167
- },
2168
- "config": {
2169
- "type": "object",
2170
- "additionalProperties": {}
2171
- },
2172
- "prompts": {
2173
- "anyOf": [
2174
- {
2175
- "anyOf": [
2176
- {
2177
- "type": "string"
2178
- },
2179
- {
2180
- "type": "object",
2181
- "properties": {
2182
- "command": {
2183
- "anyOf": [
2184
- {
2185
- "type": "string"
2186
- },
2187
- {
2188
- "type": "array",
2189
- "items": {
2190
- "type": "string"
2191
- }
2192
- }
2193
- ]
2194
- },
2195
- "config": {
2196
- "type": "object",
2197
- "additionalProperties": {}
2198
- }
2199
- },
2200
- "required": ["command"],
2201
- "additionalProperties": false
2202
- },
2203
- {
2204
- "type": "object",
2205
- "properties": {
2206
- "id": {
2207
- "type": "string"
2208
- },
2209
- "label": {
2210
- "type": "string"
2211
- },
2212
- "raw": {
2213
- "type": "string"
2214
- },
2215
- "function": {
2216
- "type": "string"
2217
- },
2218
- "function_file": {
2219
- "type": "string"
2220
- },
2221
- "path": {
2222
- "type": "string"
2223
- },
2224
- "prefix": {
2225
- "type": "string"
2226
- },
2227
- "suffix": {
2228
- "type": "string"
2229
- },
2230
- "config": {
2231
- "type": "object",
2232
- "additionalProperties": {}
2233
- }
2234
- },
2235
- "additionalProperties": true
2236
- }
2237
- ]
2238
- },
2239
- {
2240
- "type": "array",
2241
- "items": {
2242
- "anyOf": [
2243
- {
2244
- "type": "string"
2245
- },
2246
- {
2247
- "type": "object",
2248
- "properties": {
2249
- "command": {
2250
- "anyOf": [
2251
- {
2252
- "type": "string"
2253
- },
2254
- {
2255
- "type": "array",
2256
- "items": {
2257
- "type": "string"
2258
- }
2259
- }
2260
- ]
2261
- },
2262
- "config": {
2263
- "type": "object",
2264
- "additionalProperties": {}
2265
- }
2266
- },
2267
- "required": ["command"],
2268
- "additionalProperties": false
2269
- },
2270
- {
2271
- "type": "object",
2272
- "properties": {
2273
- "id": {
2274
- "type": "string"
2275
- },
2276
- "label": {
2277
- "type": "string"
2278
- },
2279
- "raw": {
2280
- "type": "string"
2281
- },
2282
- "function": {
2283
- "type": "string"
2284
- },
2285
- "function_file": {
2286
- "type": "string"
2287
- },
2288
- "path": {
2289
- "type": "string"
2290
- },
2291
- "prefix": {
2292
- "type": "string"
2293
- },
2294
- "suffix": {
2295
- "type": "string"
2296
- },
2297
- "config": {
2298
- "type": "object",
2299
- "additionalProperties": {}
2300
- }
2301
- },
2302
- "additionalProperties": true
2303
- }
2304
- ]
2305
- },
2306
- "minItems": 1
2307
- }
2308
- ]
2309
- },
2310
- "transform": {
2311
- "anyOf": [
2312
- {
2313
- "type": "string"
2314
- },
2315
- {
2316
- "type": "object",
2317
- "properties": {},
2318
- "additionalProperties": {}
2319
- }
2320
- ]
2321
- },
2322
- "delay": {
2323
- "type": "number",
2324
- "minimum": 0
2325
- },
2326
- "env": {
2327
- "type": "object",
2328
- "additionalProperties": {
2329
- "type": "string"
2330
- }
2331
- },
2332
- "environment": {
2333
- "not": {}
2334
- },
2335
- "container": {
2336
- "not": {}
2337
- },
2338
- "install": {
2339
- "not": {}
2340
- },
2341
- "reasoning_effort": {
2342
- "type": "string",
2343
- "minLength": 1
2344
- },
2345
- "hooks": {
2346
- "type": "object",
2347
- "properties": {
2348
- "before_all": {
2349
- "type": "object",
2350
- "properties": {
2351
- "command": {
2352
- "anyOf": [
2353
- {
2354
- "type": "string"
2355
- },
2356
- {
2357
- "type": "array",
2358
- "items": {
2359
- "type": "string"
2360
- }
2361
- }
2362
- ]
2363
- },
2364
- "timeout_ms": {
2365
- "type": "number"
2366
- },
2367
- "timeoutMs": {
2368
- "type": "number"
2369
- },
2370
- "cwd": {
2371
- "type": "string"
2372
- },
2373
- "reset": {
2374
- "type": "string",
2375
- "enum": ["none", "fast", "strict"]
2376
- }
2377
- },
2378
- "additionalProperties": false
2379
- },
2380
- "before_each": {
2381
- "type": "object",
2382
- "properties": {
2383
- "command": {
2384
- "anyOf": [
2385
- {
2386
- "type": "string"
2387
- },
2388
- {
2389
- "type": "array",
2390
- "items": {
2391
- "type": "string"
2392
- }
2393
- }
2394
- ]
2395
- },
2396
- "timeout_ms": {
2397
- "type": "number"
2398
- },
2399
- "timeoutMs": {
2400
- "type": "number"
2401
- },
2402
- "cwd": {
2403
- "type": "string"
2404
- },
2405
- "reset": {
2406
- "type": "string",
2407
- "enum": ["none", "fast", "strict"]
2408
- }
2409
- },
2410
- "additionalProperties": false
2411
- },
2412
- "after_each": {
2413
- "type": "object",
2414
- "properties": {
2415
- "command": {
2416
- "anyOf": [
2417
- {
2418
- "type": "string"
2419
- },
2420
- {
2421
- "type": "array",
2422
- "items": {
2423
- "type": "string"
2424
- }
2425
- }
2426
- ]
2427
- },
2428
- "timeout_ms": {
2429
- "type": "number"
2430
- },
2431
- "timeoutMs": {
2432
- "type": "number"
2433
- },
2434
- "cwd": {
2435
- "type": "string"
2436
- },
2437
- "reset": {
2438
- "type": "string",
2439
- "enum": ["none", "fast", "strict"]
2440
- }
2441
- },
2442
- "additionalProperties": false
2443
- },
2444
- "after_all": {
2445
- "type": "object",
2446
- "properties": {
2447
- "command": {
2448
- "anyOf": [
2449
- {
2450
- "type": "string"
2451
- },
2452
- {
2453
- "type": "array",
2454
- "items": {
2455
- "type": "string"
2456
- }
2457
- }
2458
- ]
2459
- },
2460
- "timeout_ms": {
2461
- "type": "number"
2462
- },
2463
- "timeoutMs": {
2464
- "type": "number"
2465
- },
2466
- "cwd": {
2467
- "type": "string"
2468
- },
2469
- "reset": {
2470
- "type": "string",
2471
- "enum": ["none", "fast", "strict"]
2472
- }
2473
- },
2474
- "additionalProperties": false
2475
- }
2476
- },
2477
- "additionalProperties": false
2478
- }
2479
- },
2480
- "additionalProperties": true
2481
- }
2482
- ]
2483
- },
2484
- "providers": {
2485
- "anyOf": [
2486
- {
2487
- "anyOf": [
2488
- {
2489
- "type": "string",
2490
- "minLength": 1
2491
- },
2492
- {
2493
- "type": "object",
2494
- "properties": {
2495
- "id": {
2496
- "type": "string",
2497
- "minLength": 1
2498
- },
2499
- "extends": {
2500
- "type": "string",
2501
- "minLength": 1
2502
- },
2503
- "provider": {
2504
- "type": "string",
2505
- "minLength": 1
2506
- },
2507
- "model": {
2508
- "type": "string",
2509
- "minLength": 1
2510
- },
2511
- "config": {
2512
- "type": "object",
2513
- "additionalProperties": {}
2514
- },
2515
- "prompts": {
2516
- "anyOf": [
2517
- {
2518
- "anyOf": [
2519
- {
2520
- "type": "string"
2521
- },
2522
- {
2523
- "type": "object",
2524
- "properties": {
2525
- "command": {
2526
- "anyOf": [
2527
- {
2528
- "type": "string"
2529
- },
2530
- {
2531
- "type": "array",
2532
- "items": {
2533
- "type": "string"
2534
- }
2535
- }
2536
- ]
2537
- },
2538
- "config": {
2539
- "type": "object",
2540
- "additionalProperties": {}
2541
- }
2542
- },
2543
- "required": ["command"],
2544
- "additionalProperties": false
2545
- },
2546
- {
2547
- "type": "object",
2548
- "properties": {
2549
- "id": {
2550
- "type": "string"
2551
- },
2552
- "label": {
2553
- "type": "string"
2554
- },
2555
- "raw": {
2556
- "type": "string"
2557
- },
2558
- "function": {
2559
- "type": "string"
2560
- },
2561
- "function_file": {
2562
- "type": "string"
2563
- },
2564
- "path": {
2565
- "type": "string"
2566
- },
2567
- "prefix": {
2568
- "type": "string"
2569
- },
2570
- "suffix": {
2571
- "type": "string"
2572
- },
2573
- "config": {
2574
- "type": "object",
2575
- "additionalProperties": {}
2576
- }
2577
- },
2578
- "additionalProperties": true
2579
- }
2580
- ]
2581
- },
2582
- {
2583
- "type": "array",
2584
- "items": {
2585
- "anyOf": [
2586
- {
2587
- "type": "string"
2588
- },
2589
- {
2590
- "type": "object",
2591
- "properties": {
2592
- "command": {
2593
- "anyOf": [
2594
- {
2595
- "type": "string"
2596
- },
2597
- {
2598
- "type": "array",
2599
- "items": {
2600
- "type": "string"
2601
- }
2602
- }
2603
- ]
2604
- },
2605
- "config": {
2606
- "type": "object",
2607
- "additionalProperties": {}
2608
- }
2609
- },
2610
- "required": ["command"],
2611
- "additionalProperties": false
2612
- },
2613
- {
2614
- "type": "object",
2615
- "properties": {
2616
- "id": {
2617
- "type": "string"
2618
- },
2619
- "label": {
2620
- "type": "string"
2621
- },
2622
- "raw": {
2623
- "type": "string"
2624
- },
2625
- "function": {
2626
- "type": "string"
2627
- },
2628
- "function_file": {
2629
- "type": "string"
2630
- },
2631
- "path": {
2632
- "type": "string"
2633
- },
2634
- "prefix": {
2635
- "type": "string"
2636
- },
2637
- "suffix": {
2638
- "type": "string"
2639
- },
2640
- "config": {
2641
- "type": "object",
2642
- "additionalProperties": {}
2643
- }
2644
- },
2645
- "additionalProperties": true
2646
- }
2647
- ]
2648
- },
2649
- "minItems": 1
2650
- }
2651
- ]
2652
- },
2653
- "transform": {
2654
- "anyOf": [
2655
- {
2656
- "type": "string"
2657
- },
2658
- {
2659
- "type": "object",
2660
- "properties": {},
2661
- "additionalProperties": {}
2662
- }
2663
- ]
2664
- },
2665
- "delay": {
2666
- "type": "number",
2667
- "minimum": 0
2668
- },
2669
- "env": {
2670
- "type": "object",
2671
- "additionalProperties": {
2672
- "type": "string"
2673
- }
2674
- },
2675
- "environment": {
2676
- "not": {}
2677
- },
2678
- "container": {
2679
- "not": {}
2680
- },
2681
- "install": {
2682
- "not": {}
2683
- },
2684
- "reasoning_effort": {
2685
- "type": "string",
2686
- "minLength": 1
2687
- },
2688
- "hooks": {
2689
- "type": "object",
2690
- "properties": {
2691
- "before_all": {
2692
- "type": "object",
2693
- "properties": {
2694
- "command": {
2695
- "anyOf": [
2696
- {
2697
- "type": "string"
2698
- },
2699
- {
2700
- "type": "array",
2701
- "items": {
2702
- "type": "string"
2703
- }
2704
- }
2705
- ]
2706
- },
2707
- "timeout_ms": {
2708
- "type": "number"
2709
- },
2710
- "timeoutMs": {
2711
- "type": "number"
2712
- },
2713
- "cwd": {
2714
- "type": "string"
2715
- },
2716
- "reset": {
2717
- "type": "string",
2718
- "enum": ["none", "fast", "strict"]
2719
- }
2720
- },
2721
- "additionalProperties": false
2722
- },
2723
- "before_each": {
2724
- "type": "object",
2725
- "properties": {
2726
- "command": {
2727
- "anyOf": [
2728
- {
2729
- "type": "string"
2730
- },
2731
- {
2732
- "type": "array",
2733
- "items": {
2734
- "type": "string"
2735
- }
2736
- }
2737
- ]
2738
- },
2739
- "timeout_ms": {
2740
- "type": "number"
2741
- },
2742
- "timeoutMs": {
2743
- "type": "number"
2744
- },
2745
- "cwd": {
2746
- "type": "string"
2747
- },
2748
- "reset": {
2749
- "type": "string",
2750
- "enum": ["none", "fast", "strict"]
2751
- }
2752
- },
2753
- "additionalProperties": false
2754
- },
2755
- "after_each": {
2756
- "type": "object",
2757
- "properties": {
2758
- "command": {
2759
- "anyOf": [
2760
- {
2761
- "type": "string"
2762
- },
2763
- {
2764
- "type": "array",
2765
- "items": {
2766
- "type": "string"
2767
- }
2768
- }
2769
- ]
2770
- },
2771
- "timeout_ms": {
2772
- "type": "number"
2773
- },
2774
- "timeoutMs": {
2775
- "type": "number"
2776
- },
2777
- "cwd": {
2778
- "type": "string"
2779
- },
2780
- "reset": {
2781
- "type": "string",
2782
- "enum": ["none", "fast", "strict"]
2783
- }
2784
- },
2785
- "additionalProperties": false
2786
- },
2787
- "after_all": {
2788
- "type": "object",
2789
- "properties": {
2790
- "command": {
2791
- "anyOf": [
2792
- {
2793
- "type": "string"
2794
- },
2795
- {
2796
- "type": "array",
2797
- "items": {
2798
- "type": "string"
2799
- }
2800
- }
2801
- ]
2802
- },
2803
- "timeout_ms": {
2804
- "type": "number"
2805
- },
2806
- "timeoutMs": {
2807
- "type": "number"
2808
- },
2809
- "cwd": {
2810
- "type": "string"
2811
- },
2812
- "reset": {
2813
- "type": "string",
2814
- "enum": ["none", "fast", "strict"]
2815
- }
2816
- },
2817
- "additionalProperties": false
2818
- }
2819
- },
2820
- "additionalProperties": false
2821
- }
2822
- },
2823
- "additionalProperties": true
2824
- }
2825
- ]
2826
- },
2827
- {
2828
- "type": "array",
2829
- "items": {
2830
- "anyOf": [
2831
- {
2832
- "type": "string",
2833
- "minLength": 1
2834
- },
2835
- {
2836
- "type": "object",
2837
- "properties": {
2838
- "id": {
2839
- "type": "string",
2840
- "minLength": 1
2841
- },
2842
- "extends": {
2843
- "type": "string",
2844
- "minLength": 1
2845
- },
2846
- "provider": {
2847
- "type": "string",
2848
- "minLength": 1
2849
- },
2850
- "model": {
2851
- "type": "string",
2852
- "minLength": 1
2853
- },
2854
- "config": {
2855
- "type": "object",
2856
- "additionalProperties": {}
2857
- },
2858
- "prompts": {
2859
- "anyOf": [
2860
- {
2861
- "anyOf": [
2862
- {
2863
- "type": "string"
2864
- },
2865
- {
2866
- "type": "object",
2867
- "properties": {
2868
- "command": {
2869
- "anyOf": [
2870
- {
2871
- "type": "string"
2872
- },
2873
- {
2874
- "type": "array",
2875
- "items": {
2876
- "type": "string"
2877
- }
2878
- }
2879
- ]
2880
- },
2881
- "config": {
2882
- "type": "object",
2883
- "additionalProperties": {}
2884
- }
2885
- },
2886
- "required": ["command"],
2887
- "additionalProperties": false
2888
- },
2889
- {
2890
- "type": "object",
2891
- "properties": {
2892
- "id": {
2893
- "type": "string"
2894
- },
2895
- "label": {
2896
- "type": "string"
2897
- },
2898
- "raw": {
2899
- "type": "string"
2900
- },
2901
- "function": {
2902
- "type": "string"
2903
- },
2904
- "function_file": {
2905
- "type": "string"
2906
- },
2907
- "path": {
2908
- "type": "string"
2909
- },
2910
- "prefix": {
2911
- "type": "string"
2912
- },
2913
- "suffix": {
2914
- "type": "string"
2915
- },
2916
- "config": {
2917
- "type": "object",
2918
- "additionalProperties": {}
2919
- }
2920
- },
2921
- "additionalProperties": true
2922
- }
2923
- ]
2924
- },
2925
- {
2926
- "type": "array",
2927
- "items": {
2928
- "anyOf": [
2929
- {
2930
- "type": "string"
2931
- },
2932
- {
2933
- "type": "object",
2934
- "properties": {
2935
- "command": {
2936
- "anyOf": [
2937
- {
2938
- "type": "string"
2939
- },
2940
- {
2941
- "type": "array",
2942
- "items": {
2943
- "type": "string"
2944
- }
2945
- }
2946
- ]
2947
- },
2948
- "config": {
2949
- "type": "object",
2950
- "additionalProperties": {}
2951
- }
2952
- },
2953
- "required": ["command"],
2954
- "additionalProperties": false
2955
- },
2956
- {
2957
- "type": "object",
2958
- "properties": {
2959
- "id": {
2960
- "type": "string"
2961
- },
2962
- "label": {
2963
- "type": "string"
2964
- },
2965
- "raw": {
2966
- "type": "string"
2967
- },
2968
- "function": {
2969
- "type": "string"
2970
- },
2971
- "function_file": {
2972
- "type": "string"
2973
- },
2974
- "path": {
2975
- "type": "string"
2976
- },
2977
- "prefix": {
2978
- "type": "string"
2979
- },
2980
- "suffix": {
2981
- "type": "string"
2982
- },
2983
- "config": {
2984
- "type": "object",
2985
- "additionalProperties": {}
2986
- }
2987
- },
2988
- "additionalProperties": true
2989
- }
2990
- ]
2991
- },
2992
- "minItems": 1
2993
- }
2994
- ]
2995
- },
2996
- "transform": {
2997
- "anyOf": [
2998
- {
2999
- "type": "string"
3000
- },
3001
- {
3002
- "type": "object",
3003
- "properties": {},
3004
- "additionalProperties": {}
3005
- }
3006
- ]
3007
- },
3008
- "delay": {
3009
- "type": "number",
3010
- "minimum": 0
3011
- },
3012
- "env": {
3013
- "type": "object",
3014
- "additionalProperties": {
3015
- "type": "string"
3016
- }
3017
- },
3018
- "environment": {
3019
- "not": {}
3020
- },
3021
- "container": {
3022
- "not": {}
3023
- },
3024
- "install": {
3025
- "not": {}
3026
- },
3027
- "reasoning_effort": {
3028
- "type": "string",
3029
- "minLength": 1
3030
- },
3031
- "hooks": {
3032
- "type": "object",
3033
- "properties": {
3034
- "before_all": {
3035
- "type": "object",
3036
- "properties": {
3037
- "command": {
3038
- "anyOf": [
3039
- {
3040
- "type": "string"
3041
- },
3042
- {
3043
- "type": "array",
3044
- "items": {
3045
- "type": "string"
3046
- }
3047
- }
3048
- ]
3049
- },
3050
- "timeout_ms": {
3051
- "type": "number"
3052
- },
3053
- "timeoutMs": {
3054
- "type": "number"
3055
- },
3056
- "cwd": {
3057
- "type": "string"
3058
- },
3059
- "reset": {
3060
- "type": "string",
3061
- "enum": ["none", "fast", "strict"]
3062
- }
3063
- },
3064
- "additionalProperties": false
3065
- },
3066
- "before_each": {
3067
- "type": "object",
3068
- "properties": {
3069
- "command": {
3070
- "anyOf": [
3071
- {
3072
- "type": "string"
3073
- },
3074
- {
3075
- "type": "array",
3076
- "items": {
3077
- "type": "string"
3078
- }
3079
- }
3080
- ]
3081
- },
3082
- "timeout_ms": {
3083
- "type": "number"
3084
- },
3085
- "timeoutMs": {
3086
- "type": "number"
3087
- },
3088
- "cwd": {
3089
- "type": "string"
3090
- },
3091
- "reset": {
3092
- "type": "string",
3093
- "enum": ["none", "fast", "strict"]
3094
- }
3095
- },
3096
- "additionalProperties": false
3097
- },
3098
- "after_each": {
3099
- "type": "object",
3100
- "properties": {
3101
- "command": {
3102
- "anyOf": [
3103
- {
3104
- "type": "string"
3105
- },
3106
- {
3107
- "type": "array",
3108
- "items": {
3109
- "type": "string"
3110
- }
3111
- }
3112
- ]
3113
- },
3114
- "timeout_ms": {
3115
- "type": "number"
3116
- },
3117
- "timeoutMs": {
3118
- "type": "number"
3119
- },
3120
- "cwd": {
3121
- "type": "string"
3122
- },
3123
- "reset": {
3124
- "type": "string",
3125
- "enum": ["none", "fast", "strict"]
3126
- }
3127
- },
3128
- "additionalProperties": false
3129
- },
3130
- "after_all": {
3131
- "type": "object",
3132
- "properties": {
3133
- "command": {
3134
- "anyOf": [
3135
- {
3136
- "type": "string"
3137
- },
3138
- {
3139
- "type": "array",
3140
- "items": {
3141
- "type": "string"
3142
- }
3143
- }
3144
- ]
3145
- },
3146
- "timeout_ms": {
3147
- "type": "number"
3148
- },
3149
- "timeoutMs": {
3150
- "type": "number"
3151
- },
3152
- "cwd": {
3153
- "type": "string"
3154
- },
3155
- "reset": {
3156
- "type": "string",
3157
- "enum": ["none", "fast", "strict"]
3158
- }
3159
- },
3160
- "additionalProperties": false
3161
- }
3162
- },
3163
- "additionalProperties": false
3164
- }
3165
- },
3166
- "additionalProperties": true
3167
- }
3168
- ]
3169
- },
3170
- "minItems": 1
3171
- }
3172
- ]
3173
- },
3174
- "prompts": {
3175
- "anyOf": [
3176
- {
3177
- "anyOf": [
3178
- {
3179
- "type": "string"
3180
- },
3181
- {
3182
- "type": "object",
3183
- "properties": {
3184
- "command": {
3185
- "anyOf": [
3186
- {
3187
- "type": "string"
3188
- },
3189
- {
3190
- "type": "array",
3191
- "items": {
3192
- "type": "string"
3193
- }
3194
- }
3195
- ]
3196
- },
3197
- "config": {
3198
- "type": "object",
3199
- "additionalProperties": {}
3200
- }
3201
- },
3202
- "required": ["command"],
3203
- "additionalProperties": false
3204
- },
3205
- {
3206
- "type": "object",
3207
- "properties": {
3208
- "id": {
3209
- "type": "string"
3210
- },
3211
- "label": {
3212
- "type": "string"
3213
- },
3214
- "raw": {
3215
- "type": "string"
3216
- },
3217
- "function": {
3218
- "type": "string"
3219
- },
3220
- "function_file": {
3221
- "type": "string"
3222
- },
3223
- "path": {
3224
- "type": "string"
3225
- },
3226
- "prefix": {
3227
- "type": "string"
3228
- },
3229
- "suffix": {
3230
- "type": "string"
3231
- },
3232
- "config": {
3233
- "type": "object",
3234
- "additionalProperties": {}
3235
- }
3236
- },
3237
- "additionalProperties": true
3238
- }
3239
- ]
3240
- },
3241
- {
3242
- "type": "array",
3243
- "items": {
3244
- "anyOf": [
3245
- {
3246
- "type": "string"
3247
- },
3248
- {
3249
- "type": "object",
3250
- "properties": {
3251
- "command": {
3252
- "anyOf": [
3253
- {
3254
- "type": "string"
3255
- },
3256
- {
3257
- "type": "array",
3258
- "items": {
3259
- "type": "string"
3260
- }
3261
- }
3262
- ]
3263
- },
3264
- "config": {
3265
- "type": "object",
3266
- "additionalProperties": {}
3267
- }
3268
- },
3269
- "required": ["command"],
3270
- "additionalProperties": false
3271
- },
3272
- {
3273
- "type": "object",
3274
- "properties": {
3275
- "id": {
3276
- "type": "string"
3277
- },
3278
- "label": {
3279
- "type": "string"
3280
- },
3281
- "raw": {
3282
- "type": "string"
3283
- },
3284
- "function": {
3285
- "type": "string"
3286
- },
3287
- "function_file": {
3288
- "type": "string"
3289
- },
3290
- "path": {
3291
- "type": "string"
3292
- },
3293
- "prefix": {
3294
- "type": "string"
3295
- },
3296
- "suffix": {
3297
- "type": "string"
3298
- },
3299
- "config": {
3300
- "type": "object",
3301
- "additionalProperties": {}
3302
- }
3303
- },
3304
- "additionalProperties": true
3305
- }
3306
- ]
3307
- },
3308
- "minItems": 1
3309
- }
3310
- ]
3311
- },
3312
- "provider_output": {
3313
- "anyOf": [
3314
- {
3315
- "type": "string"
3316
- },
3317
- {
3318
- "type": "object",
3319
- "properties": {},
3320
- "additionalProperties": {}
3321
- },
3322
- {
3323
- "type": "array",
3324
- "items": {
3325
- "type": "object",
3326
- "properties": {
3327
- "role": {
3328
- "type": "string",
3329
- "enum": ["system", "user", "assistant", "tool"]
3330
- },
3331
- "content": {
3332
- "anyOf": [
3333
- {
3334
- "type": "string"
3335
- },
3336
- {
3337
- "type": "object",
3338
- "properties": {},
3339
- "additionalProperties": {}
3340
- },
3341
- {
3342
- "type": "array",
3343
- "items": {
3344
- "type": "object",
3345
- "properties": {
3346
- "type": {
3347
- "type": "string",
3348
- "enum": ["text", "file", "image"]
3349
- },
3350
- "value": {
3351
- "type": "string"
3352
- }
3353
- },
3354
- "required": ["type", "value"],
3355
- "additionalProperties": false
3356
- }
3357
- }
3358
- ]
3359
- }
3360
- },
3361
- "required": ["role", "content"],
3362
- "additionalProperties": false
3363
- }
3364
- }
3365
- ]
3366
- },
3367
- "input": {
3368
- "not": {}
3369
- },
3370
- "input_files": {
3371
- "type": "array",
3372
- "items": {
3373
- "type": "string"
3374
- }
3375
- },
3376
- "expected_output": {
3377
- "not": {}
3378
- },
3379
- "assert": {
3380
- "type": "array",
3381
- "items": {
3382
- "anyOf": [
3383
- {
3384
- "type": "string"
3385
- },
3386
- {
3387
- "type": "object",
3388
- "properties": {},
3389
- "additionalProperties": {}
3390
- }
3391
- ]
3392
- }
3393
- },
3394
- "assert_scoring_function": {
3395
- "anyOf": [
3396
- {
3397
- "type": "string",
3398
- "minLength": 1
3399
- },
3400
- {
3401
- "type": "object",
3402
- "properties": {},
3403
- "additionalProperties": {}
3404
- }
3405
- ]
3406
- },
3407
- "options": {
3408
- "type": "object",
3409
- "properties": {},
3410
- "additionalProperties": {}
3411
- },
3412
- "threshold": {
3413
- "type": "number",
3414
- "minimum": 0,
3415
- "maximum": 1
3416
- },
3417
- "execution": {
3418
- "type": "object",
3419
- "properties": {
3420
- "workers": {
3421
- "not": {}
3422
- },
3423
- "assert": {
3424
- "type": "array",
3425
- "items": {
3426
- "anyOf": [
3427
- {
3428
- "type": "string"
3429
- },
3430
- {
3431
- "type": "object",
3432
- "properties": {},
3433
- "additionalProperties": {}
3434
- }
3435
- ]
3436
- }
3437
- },
3438
- "skip_defaults": {
3439
- "type": "boolean"
3440
- },
3441
- "cache": {
3442
- "type": "boolean"
3443
- },
3444
- "trials": {
3445
- "not": {}
3446
- },
3447
- "budget_usd": {
3448
- "type": "number",
3449
- "minimum": 0
3450
- },
3451
- "budgetUsd": {
3452
- "type": "number",
3453
- "minimum": 0
3454
- },
3455
- "fail_on_error": {
3456
- "type": "boolean"
3457
- },
3458
- "failOnError": {
3459
- "type": "boolean"
3460
- },
3461
- "threshold": {
3462
- "type": "number",
3463
- "minimum": 0,
3464
- "maximum": 1
3465
- }
3466
- },
3467
- "additionalProperties": false
3468
- },
3469
- "run": {
3470
- "type": "object",
3471
- "properties": {
3472
- "threshold": {
3473
- "type": "number",
3474
- "minimum": 0,
3475
- "maximum": 1
3476
- },
3477
- "repeat": {
3478
- "type": "object",
3479
- "properties": {
3480
- "count": {
3481
- "type": "integer",
3482
- "minimum": 1
3483
- },
3484
- "strategy": {
3485
- "type": "string",
3486
- "enum": ["pass_any", "pass_all", "mean", "confidence_interval"]
3487
- },
3488
- "early_exit": {
3489
- "type": "boolean"
3490
- },
3491
- "cost_limit_usd": {
3492
- "type": "number",
3493
- "minimum": 0
3494
- }
3495
- },
3496
- "required": ["count"],
3497
- "additionalProperties": false
3498
- },
3499
- "timeout_seconds": {
3500
- "type": "number",
3501
- "exclusiveMinimum": true,
3502
- "minimum": 0
3503
- },
3504
- "budget_usd": {
3505
- "type": "number",
3506
- "exclusiveMinimum": true,
3507
- "minimum": 0
3508
- }
3509
- },
3510
- "additionalProperties": false
3511
- },
3512
- "environment": {
3513
- "anyOf": [
3514
- {
3515
- "type": "string",
3516
- "pattern": "^\\s*file:\\/\\/"
3517
- },
3518
- {
3519
- "type": "object",
3520
- "properties": {
3521
- "workdir": {
3522
- "type": "string",
3523
- "minLength": 1
3524
- },
3525
- "setup": {
3526
- "allOf": [
3527
- {},
3528
- {
3529
- "type": "object",
3530
- "properties": {
3531
- "command": {
3532
- "type": "array",
3533
- "items": {
3534
- "type": "string",
3535
- "minLength": 1
3536
- },
3537
- "minItems": 1
3538
- },
3539
- "cwd": {
3540
- "type": "string",
3541
- "minLength": 1
3542
- },
3543
- "timeout_ms": {
3544
- "type": "number",
3545
- "exclusiveMinimum": true,
3546
- "minimum": 0
3547
- }
3548
- },
3549
- "required": ["command"],
3550
- "additionalProperties": false
3551
- }
3552
- ]
3553
- },
3554
- "env": {
3555
- "type": "object",
3556
- "additionalProperties": {
3557
- "type": "string"
3558
- }
3559
- },
3560
- "type": {
3561
- "type": "string",
3562
- "const": "host"
3563
- }
3564
- },
3565
- "required": ["workdir", "type"],
3566
- "additionalProperties": false
3567
- },
3568
- {
3569
- "type": "object",
3570
- "properties": {
3571
- "workdir": {
3572
- "type": "string",
3573
- "minLength": 1
3574
- },
3575
- "setup": {
3576
- "allOf": [
3577
- {},
3578
- {
3579
- "type": "object",
3580
- "properties": {
3581
- "command": {
3582
- "type": "array",
3583
- "items": {
3584
- "type": "string",
3585
- "minLength": 1
3586
- },
3587
- "minItems": 1
3588
- },
3589
- "cwd": {
3590
- "type": "string",
3591
- "minLength": 1
3592
- },
3593
- "timeout_ms": {
3594
- "type": "number",
3595
- "exclusiveMinimum": true,
3596
- "minimum": 0
3597
- }
3598
- },
3599
- "required": ["command"],
3600
- "additionalProperties": false
3601
- }
3602
- ]
3603
- },
3604
- "env": {
3605
- "type": "object",
3606
- "additionalProperties": {
3607
- "type": "string"
3608
- }
3609
- },
3610
- "type": {
3611
- "type": "string",
3612
- "const": "docker"
3613
- },
3614
- "context": {
3615
- "type": "string",
3616
- "minLength": 1
3617
- },
3618
- "dockerfile": {
3619
- "type": "string",
3620
- "minLength": 1
3621
- },
3622
- "image": {
3623
- "type": "string",
3624
- "minLength": 1
3625
- },
3626
- "resources": {
3627
- "type": "object",
3628
- "properties": {
3629
- "cpus": {
3630
- "type": "number",
3631
- "exclusiveMinimum": true,
3632
- "minimum": 0
3633
- },
3634
- "memory": {
3635
- "type": "string",
3636
- "minLength": 1
3637
- },
3638
- "disk": {
3639
- "type": "string",
3640
- "minLength": 1
3641
- },
3642
- "gpu": {
3643
- "anyOf": [
3644
- {
3645
- "type": "boolean"
3646
- },
3647
- {
3648
- "type": "string",
3649
- "minLength": 1
3650
- }
3651
- ]
3652
- }
3653
- },
3654
- "additionalProperties": false
3655
- },
3656
- "mounts": {
3657
- "type": "array",
3658
- "items": {
3659
- "type": "object",
3660
- "properties": {
3661
- "source": {
3662
- "type": "string",
3663
- "minLength": 1
3664
- },
3665
- "target": {
3666
- "type": "string",
3667
- "minLength": 1
3668
- },
3669
- "access": {
3670
- "type": "string",
3671
- "enum": ["ro", "rw"]
3672
- },
3673
- "read_only": {
3674
- "type": "boolean"
3675
- }
3676
- },
3677
- "required": ["source", "target"],
3678
- "additionalProperties": false
3679
- }
3680
- },
3681
- "secrets": {
3682
- "type": "object",
3683
- "additionalProperties": {
3684
- "type": "string"
3685
- }
3686
- }
3687
- },
3688
- "required": ["workdir", "type"],
3689
- "additionalProperties": false
3690
- }
3691
- ]
3692
- },
3693
- "metadata": {
3694
- "type": "object",
3695
- "additionalProperties": {}
3696
- },
3697
- "conversation_id": {
3698
- "type": "string"
3699
- },
3700
- "suite": {
3701
- "type": "string"
3702
- },
3703
- "depends_on": {
3704
- "type": "array",
3705
- "items": {
3706
- "type": "string"
3707
- }
3708
- },
3709
- "on_dependency_failure": {
3710
- "type": "string",
3711
- "enum": ["skip", "fail", "run"]
3712
- },
3713
- "mode": {
3714
- "type": "string",
3715
- "enum": ["conversation"]
3716
- },
3717
- "turns": {
3718
- "type": "array",
3719
- "items": {
3720
- "type": "object",
3721
- "properties": {
3722
- "input": {
3723
- "anyOf": [
3724
- {
3725
- "type": "string"
3726
- },
3727
- {
3728
- "anyOf": [
3729
- {
3730
- "type": "string"
3731
- },
3732
- {
3733
- "type": "object",
3734
- "properties": {},
3735
- "additionalProperties": {}
3736
- },
3737
- {
3738
- "type": "array",
3739
- "items": {
3740
- "type": "object",
3741
- "properties": {
3742
- "type": {
3743
- "type": "string",
3744
- "enum": ["text", "file", "image"]
3745
- },
3746
- "value": {
3747
- "type": "string"
3748
- }
3749
- },
3750
- "required": ["type", "value"],
3751
- "additionalProperties": false
3752
- }
3753
- }
3754
- ]
3755
- }
3756
- ]
3757
- },
3758
- "expected_output": {
3759
- "anyOf": [
3760
- {
3761
- "type": "string"
3762
- },
3763
- {
3764
- "anyOf": [
3765
- {
3766
- "type": "string"
3767
- },
3768
- {
3769
- "type": "object",
3770
- "properties": {},
3771
- "additionalProperties": {}
3772
- },
3773
- {
3774
- "type": "array",
3775
- "items": {
3776
- "type": "object",
3777
- "properties": {
3778
- "type": {
3779
- "type": "string",
3780
- "enum": ["text", "file", "image"]
3781
- },
3782
- "value": {
3783
- "type": "string"
3784
- }
3785
- },
3786
- "required": ["type", "value"],
3787
- "additionalProperties": false
3788
- }
3789
- }
3790
- ]
3791
- }
3792
- ]
3793
- },
3794
- "assert": {
3795
- "type": "array",
3796
- "items": {
3797
- "anyOf": [
3798
- {
3799
- "type": "string"
3800
- },
3801
- {
3802
- "type": "object",
3803
- "properties": {},
3804
- "additionalProperties": {}
3805
- }
3806
- ]
3807
- }
3808
- }
3809
- },
3810
- "required": ["input"],
3811
- "additionalProperties": false
3812
- },
3813
- "minItems": 1
3814
- },
3815
- "aggregation": {
3816
- "type": "string",
3817
- "enum": ["mean", "min", "max"]
3818
- },
3819
- "on_turn_failure": {
3820
- "type": "string",
3821
- "enum": ["continue", "stop"]
3822
- },
3823
- "window_size": {
3824
- "type": "integer",
3825
- "minimum": 1
3826
- }
3827
- },
3828
- "additionalProperties": false
3829
- },
3830
- {
3831
- "type": "object",
3832
- "properties": {
3833
- "include": {
3834
- "type": "string",
3835
- "minLength": 1
3836
- },
3837
- "type": {
3838
- "type": "string",
3839
- "enum": ["suite", "tests"]
3840
- },
3841
- "select": {
3842
- "anyOf": [
3843
- {
3844
- "anyOf": [
3845
- {
3846
- "type": "string",
3847
- "minLength": 1
3848
- },
3849
- {
3850
- "type": "array",
3851
- "items": {
3852
- "type": "string",
3853
- "minLength": 1
3854
- },
3855
- "minItems": 1
3856
- }
3857
- ]
3858
- },
3859
- {
3860
- "type": "object",
3861
- "properties": {
3862
- "test_ids": {
3863
- "anyOf": [
3864
- {
3865
- "type": "string",
3866
- "minLength": 1
3867
- },
3868
- {
3869
- "type": "array",
3870
- "items": {
3871
- "type": "string",
3872
- "minLength": 1
3873
- },
3874
- "minItems": 1
3875
- }
3876
- ]
3877
- },
3878
- "tags": {
3879
- "anyOf": [
3880
- {
3881
- "type": "string",
3882
- "minLength": 1
3883
- },
3884
- {
3885
- "type": "array",
3886
- "items": {
3887
- "type": "string",
3888
- "minLength": 1
3889
- },
3890
- "minItems": 1
3891
- }
3892
- ]
3893
- },
3894
- "metadata": {
3895
- "type": "object",
3896
- "additionalProperties": {
3897
- "anyOf": [
3898
- {
3899
- "type": "string"
3900
- },
3901
- {
3902
- "type": "number"
3903
- },
3904
- {
3905
- "type": "boolean"
3906
- },
3907
- {
3908
- "type": "array",
3909
- "items": {
3910
- "type": ["string", "number", "boolean"]
3911
- },
3912
- "minItems": 1
3913
- }
3914
- ]
3915
- }
3916
- }
3917
- },
3918
- "additionalProperties": false
3919
- }
3920
- ]
3921
- },
3922
- "run": {
3923
- "type": "object",
3924
- "properties": {
3925
- "threshold": {
3926
- "type": "number",
3927
- "minimum": 0,
3928
- "maximum": 1
3929
- },
3930
- "repeat": {
3931
- "type": "object",
3932
- "properties": {
3933
- "count": {
3934
- "type": "integer",
3935
- "minimum": 1
3936
- },
3937
- "strategy": {
3938
- "type": "string",
3939
- "enum": ["pass_any", "pass_all", "mean", "confidence_interval"]
3940
- },
3941
- "early_exit": {
3942
- "type": "boolean"
3943
- },
3944
- "cost_limit_usd": {
3945
- "type": "number",
3946
- "minimum": 0
3947
- }
3948
- },
3949
- "required": ["count"],
3950
- "additionalProperties": false
3951
- },
3952
- "timeout_seconds": {
3953
- "type": "number",
3954
- "exclusiveMinimum": true,
3955
- "minimum": 0
3956
- },
3957
- "budget_usd": {
3958
- "type": "number",
3959
- "exclusiveMinimum": true,
3960
- "minimum": 0
3961
- }
3962
- },
3963
- "additionalProperties": false
3964
- }
3965
- },
3966
- "required": ["include", "type"],
3967
- "additionalProperties": false
3968
- },
3969
- {
3970
- "type": "string",
3971
- "minLength": 1
3972
- }
3973
- ]
3974
- }
3975
- },
3976
- {
3977
- "type": "string",
3978
- "minLength": 1
3979
- }
3980
- ]
3981
- },
3982
- "target": {
3983
- "anyOf": [
3984
- {
3985
- "type": "string",
3986
- "minLength": 1
3987
- },
3988
- {
3989
- "type": "object",
3990
- "properties": {
3991
- "id": {
3992
- "type": "string",
3993
- "minLength": 1
3994
- },
3995
- "extends": {
3996
- "type": "string",
3997
- "minLength": 1
3998
- },
3999
- "provider": {
4000
- "type": "string",
4001
- "minLength": 1
4002
- },
4003
- "model": {
4004
- "type": "string",
4005
- "minLength": 1
4006
- },
4007
- "config": {
4008
- "type": "object",
4009
- "additionalProperties": {}
4010
- },
4011
- "prompts": {
4012
- "anyOf": [
4013
- {
4014
- "anyOf": [
4015
- {
4016
- "type": "string"
4017
- },
4018
- {
4019
- "type": "object",
4020
- "properties": {
4021
- "command": {
4022
- "anyOf": [
4023
- {
4024
- "type": "string"
4025
- },
4026
- {
4027
- "type": "array",
4028
- "items": {
4029
- "type": "string"
4030
- }
4031
- }
4032
- ]
4033
- },
4034
- "config": {
4035
- "type": "object",
4036
- "additionalProperties": {}
4037
- }
4038
- },
4039
- "required": ["command"],
4040
- "additionalProperties": false
4041
- },
4042
- {
4043
- "type": "object",
4044
- "properties": {
4045
- "id": {
4046
- "type": "string"
4047
- },
4048
- "label": {
4049
- "type": "string"
4050
- },
4051
- "raw": {
4052
- "type": "string"
4053
- },
4054
- "function": {
4055
- "type": "string"
4056
- },
4057
- "function_file": {
4058
- "type": "string"
4059
- },
4060
- "path": {
4061
- "type": "string"
4062
- },
4063
- "prefix": {
4064
- "type": "string"
4065
- },
4066
- "suffix": {
4067
- "type": "string"
4068
- },
4069
- "config": {
4070
- "type": "object",
4071
- "additionalProperties": {}
4072
- }
4073
- },
4074
- "additionalProperties": true
4075
- }
4076
- ]
4077
- },
4078
- {
4079
- "type": "array",
4080
- "items": {
4081
- "anyOf": [
4082
- {
4083
- "type": "string"
4084
- },
4085
- {
4086
- "type": "object",
4087
- "properties": {
4088
- "command": {
4089
- "anyOf": [
4090
- {
4091
- "type": "string"
4092
- },
4093
- {
4094
- "type": "array",
4095
- "items": {
4096
- "type": "string"
4097
- }
4098
- }
4099
- ]
4100
- },
4101
- "config": {
4102
- "type": "object",
4103
- "additionalProperties": {}
4104
- }
4105
- },
4106
- "required": ["command"],
4107
- "additionalProperties": false
4108
- },
4109
- {
4110
- "type": "object",
4111
- "properties": {
4112
- "id": {
4113
- "type": "string"
4114
- },
4115
- "label": {
4116
- "type": "string"
4117
- },
4118
- "raw": {
4119
- "type": "string"
4120
- },
4121
- "function": {
4122
- "type": "string"
4123
- },
4124
- "function_file": {
4125
- "type": "string"
4126
- },
4127
- "path": {
4128
- "type": "string"
4129
- },
4130
- "prefix": {
4131
- "type": "string"
4132
- },
4133
- "suffix": {
4134
- "type": "string"
2273
+ "function": {
2274
+ "type": "string"
2275
+ },
2276
+ "function_file": {
2277
+ "type": "string"
2278
+ },
2279
+ "path": {
2280
+ "type": "string"
2281
+ },
2282
+ "prefix": {
2283
+ "type": "string"
2284
+ },
2285
+ "suffix": {
2286
+ "type": "string"
4135
2287
  },
4136
2288
  "config": {
4137
2289
  "type": "object",
@@ -6619,6 +4771,13 @@
6619
4771
  "items": {
6620
4772
  "type": "object",
6621
4773
  "properties": {
4774
+ "id": {
4775
+ "type": "string",
4776
+ "minLength": 1
4777
+ },
4778
+ "description": {
4779
+ "type": "string"
4780
+ },
6622
4781
  "vars": {
6623
4782
  "type": "object",
6624
4783
  "properties": {},
@@ -7651,168 +5810,600 @@
7651
5810
  }
7652
5811
  ]
7653
5812
  },
7654
- "minItems": 1
5813
+ "minItems": 1
5814
+ }
5815
+ ]
5816
+ },
5817
+ "prompts": {
5818
+ "anyOf": [
5819
+ {
5820
+ "anyOf": [
5821
+ {
5822
+ "type": "string"
5823
+ },
5824
+ {
5825
+ "type": "object",
5826
+ "properties": {
5827
+ "command": {
5828
+ "anyOf": [
5829
+ {
5830
+ "type": "string"
5831
+ },
5832
+ {
5833
+ "type": "array",
5834
+ "items": {
5835
+ "type": "string"
5836
+ }
5837
+ }
5838
+ ]
5839
+ },
5840
+ "config": {
5841
+ "type": "object",
5842
+ "additionalProperties": {}
5843
+ }
5844
+ },
5845
+ "required": ["command"],
5846
+ "additionalProperties": false
5847
+ },
5848
+ {
5849
+ "type": "object",
5850
+ "properties": {
5851
+ "id": {
5852
+ "type": "string"
5853
+ },
5854
+ "label": {
5855
+ "type": "string"
5856
+ },
5857
+ "raw": {
5858
+ "type": "string"
5859
+ },
5860
+ "function": {
5861
+ "type": "string"
5862
+ },
5863
+ "function_file": {
5864
+ "type": "string"
5865
+ },
5866
+ "path": {
5867
+ "type": "string"
5868
+ },
5869
+ "prefix": {
5870
+ "type": "string"
5871
+ },
5872
+ "suffix": {
5873
+ "type": "string"
5874
+ },
5875
+ "config": {
5876
+ "type": "object",
5877
+ "additionalProperties": {}
5878
+ }
5879
+ },
5880
+ "additionalProperties": true
5881
+ }
5882
+ ]
5883
+ },
5884
+ {
5885
+ "type": "array",
5886
+ "items": {
5887
+ "anyOf": [
5888
+ {
5889
+ "type": "string"
5890
+ },
5891
+ {
5892
+ "type": "object",
5893
+ "properties": {
5894
+ "command": {
5895
+ "anyOf": [
5896
+ {
5897
+ "type": "string"
5898
+ },
5899
+ {
5900
+ "type": "array",
5901
+ "items": {
5902
+ "type": "string"
5903
+ }
5904
+ }
5905
+ ]
5906
+ },
5907
+ "config": {
5908
+ "type": "object",
5909
+ "additionalProperties": {}
5910
+ }
5911
+ },
5912
+ "required": ["command"],
5913
+ "additionalProperties": false
5914
+ },
5915
+ {
5916
+ "type": "object",
5917
+ "properties": {
5918
+ "id": {
5919
+ "type": "string"
5920
+ },
5921
+ "label": {
5922
+ "type": "string"
5923
+ },
5924
+ "raw": {
5925
+ "type": "string"
5926
+ },
5927
+ "function": {
5928
+ "type": "string"
5929
+ },
5930
+ "function_file": {
5931
+ "type": "string"
5932
+ },
5933
+ "path": {
5934
+ "type": "string"
5935
+ },
5936
+ "prefix": {
5937
+ "type": "string"
5938
+ },
5939
+ "suffix": {
5940
+ "type": "string"
5941
+ },
5942
+ "config": {
5943
+ "type": "object",
5944
+ "additionalProperties": {}
5945
+ }
5946
+ },
5947
+ "additionalProperties": true
5948
+ }
5949
+ ]
5950
+ },
5951
+ "minItems": 1
5952
+ }
5953
+ ]
5954
+ },
5955
+ "provider_output": {
5956
+ "anyOf": [
5957
+ {
5958
+ "type": "string"
5959
+ },
5960
+ {
5961
+ "type": "object",
5962
+ "properties": {},
5963
+ "additionalProperties": {}
5964
+ },
5965
+ {
5966
+ "type": "array",
5967
+ "items": {
5968
+ "type": "object",
5969
+ "properties": {
5970
+ "role": {
5971
+ "type": "string",
5972
+ "enum": ["system", "user", "assistant", "tool"]
5973
+ },
5974
+ "content": {
5975
+ "anyOf": [
5976
+ {
5977
+ "type": "string"
5978
+ },
5979
+ {
5980
+ "type": "object",
5981
+ "properties": {},
5982
+ "additionalProperties": {}
5983
+ },
5984
+ {
5985
+ "type": "array",
5986
+ "items": {
5987
+ "type": "object",
5988
+ "properties": {
5989
+ "type": {
5990
+ "type": "string",
5991
+ "enum": ["text", "file", "image"]
5992
+ },
5993
+ "value": {
5994
+ "type": "string"
5995
+ }
5996
+ },
5997
+ "required": ["type", "value"],
5998
+ "additionalProperties": false
5999
+ }
6000
+ }
6001
+ ]
6002
+ }
6003
+ },
6004
+ "required": ["role", "content"],
6005
+ "additionalProperties": false
6006
+ }
6007
+ }
6008
+ ]
6009
+ },
6010
+ "input": {
6011
+ "not": {}
6012
+ },
6013
+ "input_files": {
6014
+ "type": "array",
6015
+ "items": {
6016
+ "type": "string"
6017
+ }
6018
+ },
6019
+ "expected_output": {
6020
+ "not": {}
6021
+ },
6022
+ "assert": {
6023
+ "type": "array",
6024
+ "items": {
6025
+ "anyOf": [
6026
+ {
6027
+ "type": "string"
6028
+ },
6029
+ {
6030
+ "type": "object",
6031
+ "properties": {},
6032
+ "additionalProperties": {}
6033
+ }
6034
+ ]
6035
+ }
6036
+ },
6037
+ "assert_scoring_function": {
6038
+ "anyOf": [
6039
+ {
6040
+ "type": "string",
6041
+ "minLength": 1
6042
+ },
6043
+ {
6044
+ "type": "object",
6045
+ "properties": {},
6046
+ "additionalProperties": {}
6047
+ }
6048
+ ]
6049
+ },
6050
+ "options": {
6051
+ "type": "object",
6052
+ "properties": {},
6053
+ "additionalProperties": {}
6054
+ },
6055
+ "threshold": {
6056
+ "type": "number",
6057
+ "minimum": 0,
6058
+ "maximum": 1
6059
+ },
6060
+ "execution": {
6061
+ "type": "object",
6062
+ "properties": {
6063
+ "workers": {
6064
+ "not": {}
6065
+ },
6066
+ "assert": {
6067
+ "type": "array",
6068
+ "items": {
6069
+ "anyOf": [
6070
+ {
6071
+ "type": "string"
6072
+ },
6073
+ {
6074
+ "type": "object",
6075
+ "properties": {},
6076
+ "additionalProperties": {}
6077
+ }
6078
+ ]
6079
+ }
6080
+ },
6081
+ "skip_defaults": {
6082
+ "type": "boolean"
6083
+ },
6084
+ "cache": {
6085
+ "type": "boolean"
6086
+ },
6087
+ "trials": {
6088
+ "not": {}
6089
+ },
6090
+ "budget_usd": {
6091
+ "type": "number",
6092
+ "minimum": 0
6093
+ },
6094
+ "budgetUsd": {
6095
+ "type": "number",
6096
+ "minimum": 0
6097
+ },
6098
+ "fail_on_error": {
6099
+ "type": "boolean"
6100
+ },
6101
+ "failOnError": {
6102
+ "type": "boolean"
6103
+ },
6104
+ "threshold": {
6105
+ "type": "number",
6106
+ "minimum": 0,
6107
+ "maximum": 1
6108
+ }
6109
+ },
6110
+ "additionalProperties": false
6111
+ },
6112
+ "run": {
6113
+ "type": "object",
6114
+ "properties": {
6115
+ "threshold": {
6116
+ "type": "number",
6117
+ "minimum": 0,
6118
+ "maximum": 1
6119
+ },
6120
+ "repeat": {
6121
+ "type": "object",
6122
+ "properties": {
6123
+ "count": {
6124
+ "type": "integer",
6125
+ "minimum": 1
6126
+ },
6127
+ "strategy": {
6128
+ "type": "string",
6129
+ "enum": ["pass_any", "pass_all", "mean", "confidence_interval"]
6130
+ },
6131
+ "early_exit": {
6132
+ "type": "boolean"
6133
+ },
6134
+ "cost_limit_usd": {
6135
+ "type": "number",
6136
+ "minimum": 0
6137
+ }
6138
+ },
6139
+ "required": ["count"],
6140
+ "additionalProperties": false
6141
+ },
6142
+ "timeout_seconds": {
6143
+ "type": "number",
6144
+ "exclusiveMinimum": true,
6145
+ "minimum": 0
6146
+ },
6147
+ "budget_usd": {
6148
+ "type": "number",
6149
+ "exclusiveMinimum": true,
6150
+ "minimum": 0
7655
6151
  }
7656
- ]
6152
+ },
6153
+ "additionalProperties": false
7657
6154
  },
7658
- "prompts": {
6155
+ "environment": {
7659
6156
  "anyOf": [
7660
6157
  {
7661
- "anyOf": [
7662
- {
7663
- "type": "string"
6158
+ "type": "string",
6159
+ "pattern": "^\\s*file:\\/\\/"
6160
+ },
6161
+ {
6162
+ "type": "object",
6163
+ "properties": {
6164
+ "workdir": {
6165
+ "type": "string",
6166
+ "minLength": 1
7664
6167
  },
7665
- {
7666
- "type": "object",
7667
- "properties": {
7668
- "command": {
7669
- "anyOf": [
7670
- {
7671
- "type": "string"
7672
- },
7673
- {
6168
+ "setup": {
6169
+ "allOf": [
6170
+ {},
6171
+ {
6172
+ "type": "object",
6173
+ "properties": {
6174
+ "command": {
7674
6175
  "type": "array",
7675
6176
  "items": {
7676
- "type": "string"
7677
- }
6177
+ "type": "string",
6178
+ "minLength": 1
6179
+ },
6180
+ "minItems": 1
6181
+ },
6182
+ "cwd": {
6183
+ "type": "string",
6184
+ "minLength": 1
6185
+ },
6186
+ "timeout_ms": {
6187
+ "type": "number",
6188
+ "exclusiveMinimum": true,
6189
+ "minimum": 0
7678
6190
  }
7679
- ]
7680
- },
7681
- "config": {
6191
+ },
6192
+ "required": ["command"],
6193
+ "additionalProperties": false
6194
+ }
6195
+ ]
6196
+ },
6197
+ "env": {
6198
+ "type": "object",
6199
+ "additionalProperties": {
6200
+ "type": "string"
6201
+ }
6202
+ },
6203
+ "type": {
6204
+ "type": "string",
6205
+ "const": "host"
6206
+ }
6207
+ },
6208
+ "required": ["workdir", "type"],
6209
+ "additionalProperties": false
6210
+ },
6211
+ {
6212
+ "type": "object",
6213
+ "properties": {
6214
+ "workdir": {
6215
+ "type": "string",
6216
+ "minLength": 1
6217
+ },
6218
+ "setup": {
6219
+ "allOf": [
6220
+ {},
6221
+ {
7682
6222
  "type": "object",
7683
- "additionalProperties": {}
6223
+ "properties": {
6224
+ "command": {
6225
+ "type": "array",
6226
+ "items": {
6227
+ "type": "string",
6228
+ "minLength": 1
6229
+ },
6230
+ "minItems": 1
6231
+ },
6232
+ "cwd": {
6233
+ "type": "string",
6234
+ "minLength": 1
6235
+ },
6236
+ "timeout_ms": {
6237
+ "type": "number",
6238
+ "exclusiveMinimum": true,
6239
+ "minimum": 0
6240
+ }
6241
+ },
6242
+ "required": ["command"],
6243
+ "additionalProperties": false
7684
6244
  }
7685
- },
7686
- "required": ["command"],
7687
- "additionalProperties": false
6245
+ ]
7688
6246
  },
7689
- {
6247
+ "env": {
6248
+ "type": "object",
6249
+ "additionalProperties": {
6250
+ "type": "string"
6251
+ }
6252
+ },
6253
+ "type": {
6254
+ "type": "string",
6255
+ "const": "docker"
6256
+ },
6257
+ "context": {
6258
+ "type": "string",
6259
+ "minLength": 1
6260
+ },
6261
+ "dockerfile": {
6262
+ "type": "string",
6263
+ "minLength": 1
6264
+ },
6265
+ "image": {
6266
+ "type": "string",
6267
+ "minLength": 1
6268
+ },
6269
+ "resources": {
7690
6270
  "type": "object",
7691
6271
  "properties": {
7692
- "id": {
7693
- "type": "string"
7694
- },
7695
- "label": {
7696
- "type": "string"
7697
- },
7698
- "raw": {
7699
- "type": "string"
7700
- },
7701
- "function": {
7702
- "type": "string"
7703
- },
7704
- "function_file": {
7705
- "type": "string"
7706
- },
7707
- "path": {
7708
- "type": "string"
6272
+ "cpus": {
6273
+ "type": "number",
6274
+ "exclusiveMinimum": true,
6275
+ "minimum": 0
7709
6276
  },
7710
- "prefix": {
7711
- "type": "string"
6277
+ "memory": {
6278
+ "type": "string",
6279
+ "minLength": 1
7712
6280
  },
7713
- "suffix": {
7714
- "type": "string"
6281
+ "disk": {
6282
+ "type": "string",
6283
+ "minLength": 1
7715
6284
  },
7716
- "config": {
7717
- "type": "object",
7718
- "additionalProperties": {}
6285
+ "gpu": {
6286
+ "anyOf": [
6287
+ {
6288
+ "type": "boolean"
6289
+ },
6290
+ {
6291
+ "type": "string",
6292
+ "minLength": 1
6293
+ }
6294
+ ]
7719
6295
  }
7720
6296
  },
7721
- "additionalProperties": true
7722
- }
7723
- ]
7724
- },
7725
- {
7726
- "type": "array",
7727
- "items": {
7728
- "anyOf": [
7729
- {
7730
- "type": "string"
7731
- },
7732
- {
6297
+ "additionalProperties": false
6298
+ },
6299
+ "mounts": {
6300
+ "type": "array",
6301
+ "items": {
7733
6302
  "type": "object",
7734
6303
  "properties": {
7735
- "command": {
7736
- "anyOf": [
7737
- {
7738
- "type": "string"
7739
- },
7740
- {
7741
- "type": "array",
7742
- "items": {
7743
- "type": "string"
7744
- }
7745
- }
7746
- ]
6304
+ "source": {
6305
+ "type": "string",
6306
+ "minLength": 1
7747
6307
  },
7748
- "config": {
7749
- "type": "object",
7750
- "additionalProperties": {}
6308
+ "target": {
6309
+ "type": "string",
6310
+ "minLength": 1
6311
+ },
6312
+ "access": {
6313
+ "type": "string",
6314
+ "enum": ["ro", "rw"]
6315
+ },
6316
+ "read_only": {
6317
+ "type": "boolean"
7751
6318
  }
7752
6319
  },
7753
- "required": ["command"],
6320
+ "required": ["source", "target"],
7754
6321
  "additionalProperties": false
6322
+ }
6323
+ },
6324
+ "secrets": {
6325
+ "type": "object",
6326
+ "additionalProperties": {
6327
+ "type": "string"
6328
+ }
6329
+ }
6330
+ },
6331
+ "required": ["workdir", "type"],
6332
+ "additionalProperties": false
6333
+ }
6334
+ ]
6335
+ },
6336
+ "metadata": {
6337
+ "type": "object",
6338
+ "additionalProperties": {}
6339
+ },
6340
+ "conversation_id": {
6341
+ "type": "string"
6342
+ },
6343
+ "suite": {
6344
+ "type": "string"
6345
+ },
6346
+ "depends_on": {
6347
+ "type": "array",
6348
+ "items": {
6349
+ "type": "string"
6350
+ }
6351
+ },
6352
+ "on_dependency_failure": {
6353
+ "type": "string",
6354
+ "enum": ["skip", "fail", "run"]
6355
+ },
6356
+ "mode": {
6357
+ "type": "string",
6358
+ "enum": ["conversation"]
6359
+ },
6360
+ "turns": {
6361
+ "type": "array",
6362
+ "items": {
6363
+ "type": "object",
6364
+ "properties": {
6365
+ "input": {
6366
+ "anyOf": [
6367
+ {
6368
+ "type": "string"
7755
6369
  },
7756
6370
  {
7757
- "type": "object",
7758
- "properties": {
7759
- "id": {
7760
- "type": "string"
7761
- },
7762
- "label": {
7763
- "type": "string"
7764
- },
7765
- "raw": {
7766
- "type": "string"
7767
- },
7768
- "function": {
7769
- "type": "string"
7770
- },
7771
- "function_file": {
7772
- "type": "string"
7773
- },
7774
- "path": {
7775
- "type": "string"
7776
- },
7777
- "prefix": {
7778
- "type": "string"
7779
- },
7780
- "suffix": {
6371
+ "anyOf": [
6372
+ {
7781
6373
  "type": "string"
7782
6374
  },
7783
- "config": {
6375
+ {
7784
6376
  "type": "object",
6377
+ "properties": {},
7785
6378
  "additionalProperties": {}
6379
+ },
6380
+ {
6381
+ "type": "array",
6382
+ "items": {
6383
+ "type": "object",
6384
+ "properties": {
6385
+ "type": {
6386
+ "type": "string",
6387
+ "enum": ["text", "file", "image"]
6388
+ },
6389
+ "value": {
6390
+ "type": "string"
6391
+ }
6392
+ },
6393
+ "required": ["type", "value"],
6394
+ "additionalProperties": false
6395
+ }
7786
6396
  }
7787
- },
7788
- "additionalProperties": true
6397
+ ]
7789
6398
  }
7790
6399
  ]
7791
6400
  },
7792
- "minItems": 1
7793
- }
7794
- ]
7795
- },
7796
- "provider_output": {
7797
- "anyOf": [
7798
- {
7799
- "type": "string"
7800
- },
7801
- {
7802
- "type": "object",
7803
- "properties": {},
7804
- "additionalProperties": {}
7805
- },
7806
- {
7807
- "type": "array",
7808
- "items": {
7809
- "type": "object",
7810
- "properties": {
7811
- "role": {
7812
- "type": "string",
7813
- "enum": ["system", "user", "assistant", "tool"]
6401
+ "expected_output": {
6402
+ "anyOf": [
6403
+ {
6404
+ "type": "string"
7814
6405
  },
7815
- "content": {
6406
+ {
7816
6407
  "anyOf": [
7817
6408
  {
7818
6409
  "type": "string"
@@ -7841,44 +6432,43 @@
7841
6432
  }
7842
6433
  ]
7843
6434
  }
7844
- },
7845
- "required": ["role", "content"],
7846
- "additionalProperties": false
7847
- }
7848
- }
7849
- ]
7850
- },
7851
- "assert": {
7852
- "type": "array",
7853
- "items": {
7854
- "anyOf": [
7855
- {
7856
- "type": "string"
6435
+ ]
7857
6436
  },
7858
- {
7859
- "type": "object",
7860
- "properties": {},
7861
- "additionalProperties": {}
6437
+ "assert": {
6438
+ "type": "array",
6439
+ "items": {
6440
+ "anyOf": [
6441
+ {
6442
+ "type": "string"
6443
+ },
6444
+ {
6445
+ "type": "object",
6446
+ "properties": {},
6447
+ "additionalProperties": {}
6448
+ }
6449
+ ]
6450
+ }
7862
6451
  }
7863
- ]
7864
- }
6452
+ },
6453
+ "required": ["input"],
6454
+ "additionalProperties": false
6455
+ },
6456
+ "minItems": 1
7865
6457
  },
7866
- "options": {
7867
- "type": "object",
7868
- "properties": {},
7869
- "additionalProperties": {}
6458
+ "aggregation": {
6459
+ "type": "string",
6460
+ "enum": ["mean", "min", "max"]
7870
6461
  },
7871
- "threshold": {
7872
- "type": "number",
7873
- "minimum": 0,
7874
- "maximum": 1
6462
+ "on_turn_failure": {
6463
+ "type": "string",
6464
+ "enum": ["continue", "stop"]
7875
6465
  },
7876
- "metadata": {
7877
- "type": "object",
7878
- "additionalProperties": {}
6466
+ "window_size": {
6467
+ "type": "integer",
6468
+ "minimum": 1
7879
6469
  }
7880
6470
  },
7881
- "additionalProperties": true
6471
+ "additionalProperties": false
7882
6472
  }
7883
6473
  },
7884
6474
  "tests": {
@@ -9587,6 +8177,7 @@
9587
8177
  }
9588
8178
  }
9589
8179
  },
8180
+ "required": ["config", "tests"],
9590
8181
  "additionalProperties": false
9591
8182
  }
9592
8183
  },