@ora-ai/cli 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  // GENERATED by scripts/generate.mjs — DO NOT EDIT.
2
- // Source: openapi/ora-public.openapi.json (sha256 ee4afa0e56fad3b833b91655bc9a0ebed29c2ff1553c774b274f3fe2d4af2810)
2
+ // Source: openapi/ora-public.openapi.json (sha256 f177bc9eace1b4d72a06b0641d1d816e308c341e56a6ab369ed4d559da77780e)
3
3
  // Regenerate: yarn nx run ora-cli:generate
4
4
  export const SPEC = {
5
5
  "version": "1.0.0",
6
- "contentHash": "ee4afa0e56fad3b833b91655bc9a0ebed29c2ff1553c774b274f3fe2d4af2810",
6
+ "contentHash": "f177bc9eace1b4d72a06b0641d1d816e308c341e56a6ab369ed4d559da77780e",
7
7
  "server": "https://api.ora.ai",
8
8
  "bin": "ora"
9
9
  };
@@ -2072,7 +2072,12 @@ export const OPERATIONS = [
2072
2072
  "keys": [
2073
2073
  {
2074
2074
  "path": "kind",
2075
- "kind": "string"
2075
+ "kind": "string",
2076
+ "values": [
2077
+ "team",
2078
+ "squad",
2079
+ "member"
2080
+ ]
2076
2081
  },
2077
2082
  {
2078
2083
  "path": "id",
@@ -2104,7 +2109,11 @@ export const OPERATIONS = [
2104
2109
  },
2105
2110
  {
2106
2111
  "path": "budgets.onTrip",
2107
- "kind": "string"
2112
+ "kind": "string",
2113
+ "values": [
2114
+ "pause",
2115
+ "stop"
2116
+ ]
2108
2117
  },
2109
2118
  {
2110
2119
  "path": "budgets.mission.usd",
@@ -2395,7 +2404,11 @@ export const OPERATIONS = [
2395
2404
  "keys": [
2396
2405
  {
2397
2406
  "path": "onTrip",
2398
- "kind": "string"
2407
+ "kind": "string",
2408
+ "values": [
2409
+ "pause",
2410
+ "stop"
2411
+ ]
2399
2412
  },
2400
2413
  {
2401
2414
  "path": "mission.usd",
@@ -3095,7 +3108,10 @@ export const OPERATIONS = [
3095
3108
  },
3096
3109
  {
3097
3110
  "path": "tier",
3098
- "kind": "string"
3111
+ "kind": "string",
3112
+ "values": [
3113
+ "core"
3114
+ ]
3099
3115
  },
3100
3116
  {
3101
3117
  "path": "position",
@@ -3166,7 +3182,11 @@ export const OPERATIONS = [
3166
3182
  "keys": [
3167
3183
  {
3168
3184
  "path": "onTrip",
3169
- "kind": "string"
3185
+ "kind": "string",
3186
+ "values": [
3187
+ "pause",
3188
+ "stop"
3189
+ ]
3170
3190
  },
3171
3191
  {
3172
3192
  "path": "team.usd",
@@ -3406,7 +3426,10 @@ export const OPERATIONS = [
3406
3426
  },
3407
3427
  {
3408
3428
  "path": "tier",
3409
- "kind": "string"
3429
+ "kind": "string",
3430
+ "values": [
3431
+ "core"
3432
+ ]
3410
3433
  },
3411
3434
  {
3412
3435
  "path": "position",
@@ -3477,7 +3500,11 @@ export const OPERATIONS = [
3477
3500
  "keys": [
3478
3501
  {
3479
3502
  "path": "onTrip",
3480
- "kind": "string"
3503
+ "kind": "string",
3504
+ "values": [
3505
+ "pause",
3506
+ "stop"
3507
+ ]
3481
3508
  },
3482
3509
  {
3483
3510
  "path": "team.usd",
@@ -3973,7 +4000,11 @@ export const OPERATIONS = [
3973
4000
  "keys": [
3974
4001
  {
3975
4002
  "path": "onTrip",
3976
- "kind": "string"
4003
+ "kind": "string",
4004
+ "values": [
4005
+ "pause",
4006
+ "stop"
4007
+ ]
3977
4008
  },
3978
4009
  {
3979
4010
  "path": "team.usd",
@@ -4502,7 +4533,12 @@ export const OPERATIONS = [
4502
4533
  },
4503
4534
  {
4504
4535
  "path": "params.reasoningEffort",
4505
- "kind": "string"
4536
+ "kind": "string",
4537
+ "values": [
4538
+ "low",
4539
+ "medium",
4540
+ "high"
4541
+ ]
4506
4542
  },
4507
4543
  {
4508
4544
  "path": "params.thinkingBudgetTokens",
@@ -4510,11 +4546,21 @@ export const OPERATIONS = [
4510
4546
  },
4511
4547
  {
4512
4548
  "path": "params.codeMode",
4513
- "kind": "string"
4549
+ "kind": "string",
4550
+ "values": [
4551
+ "auto",
4552
+ "on",
4553
+ "off"
4554
+ ]
4514
4555
  },
4515
4556
  {
4516
4557
  "path": "params.toolSearch",
4517
- "kind": "string"
4558
+ "kind": "string",
4559
+ "values": [
4560
+ "on",
4561
+ "off",
4562
+ "directory"
4563
+ ]
4518
4564
  }
4519
4565
  ]
4520
4566
  },
@@ -4530,6 +4576,7 @@ export const OPERATIONS = [
4530
4576
  "flag": "agent",
4531
4577
  "optionKey": "agent",
4532
4578
  "field": "matrix",
4579
+ "target": "element",
4533
4580
  "pattern": "{harness}/{provider?}/{model}",
4534
4581
  "description": "An agent every group runs: its harness and its model, with the provider between them for a harness that has several. After a comma, the cell's own settings: params.reasoningEffort=high.",
4535
4582
  "keys": [
@@ -4555,7 +4602,12 @@ export const OPERATIONS = [
4555
4602
  },
4556
4603
  {
4557
4604
  "path": "params.reasoningEffort",
4558
- "kind": "string"
4605
+ "kind": "string",
4606
+ "values": [
4607
+ "low",
4608
+ "medium",
4609
+ "high"
4610
+ ]
4559
4611
  },
4560
4612
  {
4561
4613
  "path": "params.thinkingBudgetTokens",
@@ -4563,11 +4615,21 @@ export const OPERATIONS = [
4563
4615
  },
4564
4616
  {
4565
4617
  "path": "params.codeMode",
4566
- "kind": "string"
4618
+ "kind": "string",
4619
+ "values": [
4620
+ "auto",
4621
+ "on",
4622
+ "off"
4623
+ ]
4567
4624
  },
4568
4625
  {
4569
4626
  "path": "params.toolSearch",
4570
- "kind": "string"
4627
+ "kind": "string",
4628
+ "values": [
4629
+ "on",
4630
+ "off",
4631
+ "directory"
4632
+ ]
4571
4633
  }
4572
4634
  ]
4573
4635
  },
@@ -4575,6 +4637,7 @@ export const OPERATIONS = [
4575
4637
  "flag": "task",
4576
4638
  "optionKey": "task",
4577
4639
  "field": "tasks",
4640
+ "target": "element",
4578
4641
  "text": "prompt",
4579
4642
  "description": "An authored task, by its prompt. @file reads the prompt from a file; a .json file holds the whole task (its rubric, its checks, its steps), or a list of them.",
4580
4643
  "keys": [
@@ -4615,7 +4678,7 @@ export const OPERATIONS = [
4615
4678
  "help": {
4616
4679
  "command": "ora benchmarks create --file run.json --watch",
4617
4680
  "body": "{\n \"name\": \"tavily-search A/B\",\n \"subjectKind\": \"addon\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"armConfig\": {\n \"treatment\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n },\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"any_of\",\n \"label\": \"searched, by either route (treatment arm only)\",\n \"arms\": [\n \"skill\"\n ],\n \"of\": [\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"search\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"Node\"\n }\n }\n },\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"call_page_tool\",\n \"arguments\": {\n \"tool\": \"search\"\n }\n }\n ]\n },\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}",
4618
- "flags": "ora benchmarks create --name <name> --agent <harness>/<provider>/<model>"
4681
+ "flags": "ora benchmarks create --name <name> --agent <harness>/<model>"
4619
4682
  }
4620
4683
  },
4621
4684
  {
@@ -5918,7 +5981,12 @@ export const OPERATIONS = [
5918
5981
  },
5919
5982
  {
5920
5983
  "path": "reasoningEffort",
5921
- "kind": "string"
5984
+ "kind": "string",
5985
+ "values": [
5986
+ "low",
5987
+ "medium",
5988
+ "high"
5989
+ ]
5922
5990
  },
5923
5991
  {
5924
5992
  "path": "thinkingBudgetTokens",
@@ -5926,11 +5994,21 @@ export const OPERATIONS = [
5926
5994
  },
5927
5995
  {
5928
5996
  "path": "codeMode",
5929
- "kind": "string"
5997
+ "kind": "string",
5998
+ "values": [
5999
+ "auto",
6000
+ "on",
6001
+ "off"
6002
+ ]
5930
6003
  },
5931
6004
  {
5932
6005
  "path": "toolSearch",
5933
- "kind": "string"
6006
+ "kind": "string",
6007
+ "values": [
6008
+ "on",
6009
+ "off",
6010
+ "directory"
6011
+ ]
5934
6012
  }
5935
6013
  ]
5936
6014
  }
@@ -6800,6 +6878,10 @@ export const OPERATIONS = [
6800
6878
  "name": "impersonators",
6801
6879
  "required": false,
6802
6880
  "kind": "string",
6881
+ "enum": [
6882
+ "flag",
6883
+ "block"
6884
+ ],
6803
6885
  "description": "null: back to flag",
6804
6886
  "placeholder": "<impersonators>"
6805
6887
  },
@@ -6831,19 +6913,35 @@ export const OPERATIONS = [
6831
6913
  "keys": [
6832
6914
  {
6833
6915
  "path": "training_crawl",
6834
- "kind": "string"
6916
+ "kind": "string",
6917
+ "values": [
6918
+ "allow",
6919
+ "block"
6920
+ ]
6835
6921
  },
6836
6922
  {
6837
6923
  "path": "search_index",
6838
- "kind": "string"
6924
+ "kind": "string",
6925
+ "values": [
6926
+ "allow",
6927
+ "block"
6928
+ ]
6839
6929
  },
6840
6930
  {
6841
6931
  "path": "live_fetch",
6842
- "kind": "string"
6932
+ "kind": "string",
6933
+ "values": [
6934
+ "allow",
6935
+ "block"
6936
+ ]
6843
6937
  },
6844
6938
  {
6845
6939
  "path": "agent_action",
6846
- "kind": "string"
6940
+ "kind": "string",
6941
+ "values": [
6942
+ "allow",
6943
+ "block"
6944
+ ]
6847
6945
  }
6848
6946
  ],
6849
6947
  "nullable": true
@@ -8324,7 +8422,12 @@ export const OPERATIONS = [
8324
8422
  },
8325
8423
  {
8326
8424
  "path": "mode",
8327
- "kind": "string"
8425
+ "kind": "string",
8426
+ "values": [
8427
+ "inbox",
8428
+ "deliver",
8429
+ "link"
8430
+ ]
8328
8431
  },
8329
8432
  {
8330
8433
  "path": "title",
@@ -10025,7 +10128,12 @@ export const OPERATIONS = [
10025
10128
  "keys": [
10026
10129
  {
10027
10130
  "path": "mode",
10028
- "kind": "string"
10131
+ "kind": "string",
10132
+ "values": [
10133
+ "inbox",
10134
+ "deliver",
10135
+ "link"
10136
+ ]
10029
10137
  },
10030
10138
  {
10031
10139
  "path": "about",
@@ -14363,7 +14471,13 @@ export const OPERATIONS = [
14363
14471
  },
14364
14472
  {
14365
14473
  "path": "type",
14366
- "kind": "string"
14474
+ "kind": "string",
14475
+ "values": [
14476
+ "A",
14477
+ "AAAA",
14478
+ "CNAME",
14479
+ "ALIAS"
14480
+ ]
14367
14481
  },
14368
14482
  {
14369
14483
  "path": "value",
@@ -14379,11 +14493,20 @@ export const OPERATIONS = [
14379
14493
  },
14380
14494
  {
14381
14495
  "path": "proxyProvider",
14382
- "kind": "string"
14496
+ "kind": "string",
14497
+ "values": [
14498
+ "cloudflare",
14499
+ "other",
14500
+ "unknown"
14501
+ ]
14383
14502
  },
14384
14503
  {
14385
14504
  "path": "route53Alias.recordType",
14386
- "kind": "string"
14505
+ "kind": "string",
14506
+ "values": [
14507
+ "A",
14508
+ "AAAA"
14509
+ ]
14387
14510
  },
14388
14511
  {
14389
14512
  "path": "route53Alias.hostedZoneId",
@@ -17395,7 +17518,15 @@ export const OPERATIONS = [
17395
17518
  "keys": [
17396
17519
  {
17397
17520
  "path": "hostingProvider",
17398
- "kind": "string"
17521
+ "kind": "string",
17522
+ "values": [
17523
+ "wpengine",
17524
+ "kinsta",
17525
+ "webflow",
17526
+ "hostinger",
17527
+ "cloudfront",
17528
+ "self_hosted"
17529
+ ]
17399
17530
  },
17400
17531
  {
17401
17532
  "path": "originUrl",
@@ -17403,7 +17534,14 @@ export const OPERATIONS = [
17403
17534
  },
17404
17535
  {
17405
17536
  "path": "dnsManagement",
17406
- "kind": "string"
17537
+ "kind": "string",
17538
+ "values": [
17539
+ "external",
17540
+ "wpengine",
17541
+ "hostinger",
17542
+ "flattened",
17543
+ "apex_proxy"
17544
+ ]
17407
17545
  },
17408
17546
  {
17409
17547
  "path": "originalDnsRecords",
@@ -34664,7 +34802,12 @@ export const OPERATIONS = [
34664
34802
  },
34665
34803
  {
34666
34804
  "path": "layer",
34667
- "kind": "string"
34805
+ "kind": "string",
34806
+ "values": [
34807
+ "discovery",
34808
+ "access",
34809
+ "use"
34810
+ ]
34668
34811
  },
34669
34812
  {
34670
34813
  "path": "persona",
@@ -34819,7 +34962,18 @@ export const OPERATIONS = [
34819
34962
  },
34820
34963
  {
34821
34964
  "path": "category",
34822
- "kind": "string"
34965
+ "kind": "string",
34966
+ "values": [
34967
+ "recommended",
34968
+ "answered",
34969
+ "accurate",
34970
+ "informed",
34971
+ "capability",
34972
+ "real_steps",
34973
+ "safe",
34974
+ "task_done",
34975
+ "other"
34976
+ ]
34823
34977
  },
34824
34978
  {
34825
34979
  "path": "needsTools",
@@ -35239,7 +35393,11 @@ export const OPERATIONS = [
35239
35393
  },
35240
35394
  {
35241
35395
  "path": "browser.webmcpSurface",
35242
- "kind": "string"
35396
+ "kind": "string",
35397
+ "values": [
35398
+ "native",
35399
+ "shim"
35400
+ ]
35243
35401
  }
35244
35402
  ]
35245
35403
  },
@@ -35860,7 +36018,11 @@ export const OPERATIONS = [
35860
36018
  },
35861
36019
  {
35862
36020
  "path": "webmcpSurface",
35863
- "kind": "string"
36021
+ "kind": "string",
36022
+ "values": [
36023
+ "native",
36024
+ "shim"
36025
+ ]
35864
36026
  }
35865
36027
  ]
35866
36028
  }
@@ -36560,7 +36722,12 @@ export const OPERATIONS = [
36560
36722
  },
36561
36723
  {
36562
36724
  "path": "reasoningEffort",
36563
- "kind": "string"
36725
+ "kind": "string",
36726
+ "values": [
36727
+ "low",
36728
+ "medium",
36729
+ "high"
36730
+ ]
36564
36731
  },
36565
36732
  {
36566
36733
  "path": "thinkingBudgetTokens",
@@ -36568,11 +36735,21 @@ export const OPERATIONS = [
36568
36735
  },
36569
36736
  {
36570
36737
  "path": "codeMode",
36571
- "kind": "string"
36738
+ "kind": "string",
36739
+ "values": [
36740
+ "auto",
36741
+ "on",
36742
+ "off"
36743
+ ]
36572
36744
  },
36573
36745
  {
36574
36746
  "path": "toolSearch",
36575
- "kind": "string"
36747
+ "kind": "string",
36748
+ "values": [
36749
+ "on",
36750
+ "off",
36751
+ "directory"
36752
+ ]
36576
36753
  }
36577
36754
  ]
36578
36755
  }
@@ -36880,7 +37057,12 @@ export const OPERATIONS = [
36880
37057
  },
36881
37058
  {
36882
37059
  "path": "params.reasoningEffort",
36883
- "kind": "string"
37060
+ "kind": "string",
37061
+ "values": [
37062
+ "low",
37063
+ "medium",
37064
+ "high"
37065
+ ]
36884
37066
  },
36885
37067
  {
36886
37068
  "path": "params.thinkingBudgetTokens",
@@ -36888,11 +37070,21 @@ export const OPERATIONS = [
36888
37070
  },
36889
37071
  {
36890
37072
  "path": "params.codeMode",
36891
- "kind": "string"
37073
+ "kind": "string",
37074
+ "values": [
37075
+ "auto",
37076
+ "on",
37077
+ "off"
37078
+ ]
36892
37079
  },
36893
37080
  {
36894
37081
  "path": "params.toolSearch",
36895
- "kind": "string"
37082
+ "kind": "string",
37083
+ "values": [
37084
+ "on",
37085
+ "off",
37086
+ "directory"
37087
+ ]
36896
37088
  }
36897
37089
  ]
36898
37090
  },
@@ -37194,6 +37386,20 @@ export const OPERATIONS = [
37194
37386
  "name": "license",
37195
37387
  "required": false,
37196
37388
  "kind": "string",
37389
+ "enum": [
37390
+ "MIT",
37391
+ "Apache-2.0",
37392
+ "BSD-2-Clause",
37393
+ "BSD-3-Clause",
37394
+ "CC-BY-4.0",
37395
+ "CC0-1.0",
37396
+ "CC-BY-SA-4.0",
37397
+ "CC-BY-NC-4.0",
37398
+ "CC-BY-NC-SA-4.0",
37399
+ "GPL-3.0",
37400
+ "AGPL-3.0",
37401
+ "MPL-2.0"
37402
+ ],
37197
37403
  "placeholder": "<license>"
37198
37404
  },
37199
37405
  {
@@ -37341,7 +37547,11 @@ export const OPERATIONS = [
37341
37547
  },
37342
37548
  {
37343
37549
  "path": "coverage",
37344
- "kind": "string"
37550
+ "kind": "string",
37551
+ "values": [
37552
+ "every-task",
37553
+ "per-task"
37554
+ ]
37345
37555
  }
37346
37556
  ]
37347
37557
  },
@@ -37353,7 +37563,15 @@ export const OPERATIONS = [
37353
37563
  "keys": [
37354
37564
  {
37355
37565
  "path": "kind",
37356
- "kind": "string"
37566
+ "kind": "string",
37567
+ "values": [
37568
+ "none",
37569
+ "workspace",
37570
+ "image",
37571
+ "hosted",
37572
+ "services",
37573
+ "stack"
37574
+ ]
37357
37575
  }
37358
37576
  ]
37359
37577
  },
@@ -37390,15 +37608,28 @@ export const OPERATIONS = [
37390
37608
  },
37391
37609
  {
37392
37610
  "path": "missingAnswer",
37393
- "kind": "string"
37611
+ "kind": "string",
37612
+ "values": [
37613
+ "unmeasured",
37614
+ "wrong",
37615
+ "grade_full_response"
37616
+ ]
37394
37617
  },
37395
37618
  {
37396
37619
  "path": "aggregation",
37397
- "kind": "string"
37620
+ "kind": "string",
37621
+ "values": [
37622
+ "included",
37623
+ "excluded"
37624
+ ]
37398
37625
  },
37399
37626
  {
37400
37627
  "path": "aggregationReason",
37401
- "kind": "string"
37628
+ "kind": "string",
37629
+ "values": [
37630
+ "smoke_test",
37631
+ "suite_of_benchmarks"
37632
+ ]
37402
37633
  },
37403
37634
  {
37404
37635
  "path": "aggregateTaskSample.count",
@@ -37853,7 +38084,11 @@ export const OPERATIONS = [
37853
38084
  },
37854
38085
  {
37855
38086
  "path": "redistribution",
37856
- "kind": "string"
38087
+ "kind": "string",
38088
+ "values": [
38089
+ "allowed",
38090
+ "reference_only"
38091
+ ]
37857
38092
  },
37858
38093
  {
37859
38094
  "path": "sourceUrl",
@@ -38004,6 +38239,20 @@ export const OPERATIONS = [
38004
38239
  "name": "license",
38005
38240
  "required": false,
38006
38241
  "kind": "string",
38242
+ "enum": [
38243
+ "MIT",
38244
+ "Apache-2.0",
38245
+ "BSD-2-Clause",
38246
+ "BSD-3-Clause",
38247
+ "CC-BY-4.0",
38248
+ "CC0-1.0",
38249
+ "CC-BY-SA-4.0",
38250
+ "CC-BY-NC-4.0",
38251
+ "CC-BY-NC-SA-4.0",
38252
+ "GPL-3.0",
38253
+ "AGPL-3.0",
38254
+ "MPL-2.0"
38255
+ ],
38007
38256
  "placeholder": "<license>"
38008
38257
  },
38009
38258
  {
@@ -38145,7 +38394,11 @@ export const OPERATIONS = [
38145
38394
  },
38146
38395
  {
38147
38396
  "path": "coverage",
38148
- "kind": "string"
38397
+ "kind": "string",
38398
+ "values": [
38399
+ "every-task",
38400
+ "per-task"
38401
+ ]
38149
38402
  }
38150
38403
  ]
38151
38404
  },
@@ -38157,7 +38410,15 @@ export const OPERATIONS = [
38157
38410
  "keys": [
38158
38411
  {
38159
38412
  "path": "kind",
38160
- "kind": "string"
38413
+ "kind": "string",
38414
+ "values": [
38415
+ "none",
38416
+ "workspace",
38417
+ "image",
38418
+ "hosted",
38419
+ "services",
38420
+ "stack"
38421
+ ]
38161
38422
  }
38162
38423
  ]
38163
38424
  },
@@ -38194,15 +38455,28 @@ export const OPERATIONS = [
38194
38455
  },
38195
38456
  {
38196
38457
  "path": "missingAnswer",
38197
- "kind": "string"
38458
+ "kind": "string",
38459
+ "values": [
38460
+ "unmeasured",
38461
+ "wrong",
38462
+ "grade_full_response"
38463
+ ]
38198
38464
  },
38199
38465
  {
38200
38466
  "path": "aggregation",
38201
- "kind": "string"
38467
+ "kind": "string",
38468
+ "values": [
38469
+ "included",
38470
+ "excluded"
38471
+ ]
38202
38472
  },
38203
38473
  {
38204
38474
  "path": "aggregationReason",
38205
- "kind": "string"
38475
+ "kind": "string",
38476
+ "values": [
38477
+ "smoke_test",
38478
+ "suite_of_benchmarks"
38479
+ ]
38206
38480
  },
38207
38481
  {
38208
38482
  "path": "aggregateTaskSample.count",
@@ -38971,7 +39245,7 @@ export const OPERATIONS = [
38971
39245
  "name": "modeOptions",
38972
39246
  "required": false,
38973
39247
  "kind": "object",
38974
- "description": "A comparisonMode's private inputs, validated by that mode:",
39248
+ "description": "A comparisonMode's private inputs, validated by that mode. No mode offered here takes any.",
38975
39249
  "placeholder": "{}"
38976
39250
  },
38977
39251
  {
@@ -39055,6 +39329,7 @@ export const OPERATIONS = [
39055
39329
  "name": "runCounting",
39056
39330
  "required": false,
39057
39331
  "kind": "string",
39332
+ "description": "Which of the study's runs its numbers count. Absent or null = every run. 'hold_back_screened' leaves out the runs the named screen flagged at one of `levels`: each is a missing measurement with a named reason, never a fail, and the count held back is stated beside every figure.",
39058
39333
  "placeholder": "<runCounting>"
39059
39334
  },
39060
39335
  {
@@ -39224,7 +39499,11 @@ export const OPERATIONS = [
39224
39499
  },
39225
39500
  {
39226
39501
  "path": "role",
39227
- "kind": "string"
39502
+ "kind": "string",
39503
+ "values": [
39504
+ "reference",
39505
+ "peer"
39506
+ ]
39228
39507
  },
39229
39508
  {
39230
39509
  "path": "config.tools",
@@ -39409,7 +39688,10 @@ export const OPERATIONS = [
39409
39688
  "keys": [
39410
39689
  {
39411
39690
  "path": "from",
39412
- "kind": "string"
39691
+ "kind": "string",
39692
+ "values": [
39693
+ "series"
39694
+ ]
39413
39695
  },
39414
39696
  {
39415
39697
  "path": "maxAgeDays",
@@ -39425,16 +39707,26 @@ export const OPERATIONS = [
39425
39707
  "keys": [
39426
39708
  {
39427
39709
  "path": "policy",
39428
- "kind": "string"
39710
+ "kind": "string",
39711
+ "values": [
39712
+ "hold_back_screened"
39713
+ ]
39429
39714
  },
39430
39715
  {
39431
39716
  "path": "screen",
39432
- "kind": "string"
39717
+ "kind": "string",
39718
+ "values": [
39719
+ "test_awareness"
39720
+ ]
39433
39721
  },
39434
39722
  {
39435
39723
  "path": "levels",
39436
39724
  "kind": "list",
39437
- "item": "string"
39725
+ "item": "string",
39726
+ "values": [
39727
+ "aware",
39728
+ "influenced"
39729
+ ]
39438
39730
  }
39439
39731
  ],
39440
39732
  "nullable": true
@@ -39487,7 +39779,12 @@ export const OPERATIONS = [
39487
39779
  },
39488
39780
  {
39489
39781
  "path": "params.reasoningEffort",
39490
- "kind": "string"
39782
+ "kind": "string",
39783
+ "values": [
39784
+ "low",
39785
+ "medium",
39786
+ "high"
39787
+ ]
39491
39788
  },
39492
39789
  {
39493
39790
  "path": "params.thinkingBudgetTokens",
@@ -39495,11 +39792,21 @@ export const OPERATIONS = [
39495
39792
  },
39496
39793
  {
39497
39794
  "path": "params.codeMode",
39498
- "kind": "string"
39795
+ "kind": "string",
39796
+ "values": [
39797
+ "auto",
39798
+ "on",
39799
+ "off"
39800
+ ]
39499
39801
  },
39500
39802
  {
39501
39803
  "path": "params.toolSearch",
39502
- "kind": "string"
39804
+ "kind": "string",
39805
+ "values": [
39806
+ "on",
39807
+ "off",
39808
+ "directory"
39809
+ ]
39503
39810
  }
39504
39811
  ]
39505
39812
  },
@@ -39515,6 +39822,7 @@ export const OPERATIONS = [
39515
39822
  "flag": "agent",
39516
39823
  "optionKey": "agent",
39517
39824
  "field": "matrix",
39825
+ "target": "element",
39518
39826
  "pattern": "{harness}/{provider?}/{model}",
39519
39827
  "description": "An agent every group runs: its harness and its model, with the provider between them for a harness that has several. After a comma, the cell's own settings: params.reasoningEffort=high.",
39520
39828
  "keys": [
@@ -39540,7 +39848,12 @@ export const OPERATIONS = [
39540
39848
  },
39541
39849
  {
39542
39850
  "path": "params.reasoningEffort",
39543
- "kind": "string"
39851
+ "kind": "string",
39852
+ "values": [
39853
+ "low",
39854
+ "medium",
39855
+ "high"
39856
+ ]
39544
39857
  },
39545
39858
  {
39546
39859
  "path": "params.thinkingBudgetTokens",
@@ -39548,11 +39861,21 @@ export const OPERATIONS = [
39548
39861
  },
39549
39862
  {
39550
39863
  "path": "params.codeMode",
39551
- "kind": "string"
39864
+ "kind": "string",
39865
+ "values": [
39866
+ "auto",
39867
+ "on",
39868
+ "off"
39869
+ ]
39552
39870
  },
39553
39871
  {
39554
39872
  "path": "params.toolSearch",
39555
- "kind": "string"
39873
+ "kind": "string",
39874
+ "values": [
39875
+ "on",
39876
+ "off",
39877
+ "directory"
39878
+ ]
39556
39879
  }
39557
39880
  ]
39558
39881
  },
@@ -39560,6 +39883,7 @@ export const OPERATIONS = [
39560
39883
  "flag": "group",
39561
39884
  "optionKey": "group",
39562
39885
  "field": "arms",
39886
+ "target": "element",
39563
39887
  "pattern": "{label}",
39564
39888
  "defaults": {
39565
39889
  "config": {}
@@ -39576,7 +39900,11 @@ export const OPERATIONS = [
39576
39900
  },
39577
39901
  {
39578
39902
  "path": "role",
39579
- "kind": "string"
39903
+ "kind": "string",
39904
+ "values": [
39905
+ "reference",
39906
+ "peer"
39907
+ ]
39580
39908
  },
39581
39909
  {
39582
39910
  "path": "config.tools",
@@ -39596,6 +39924,7 @@ export const OPERATIONS = [
39596
39924
  "flag": "agent",
39597
39925
  "optionKey": "agent",
39598
39926
  "field": "arms.cells",
39927
+ "target": "element",
39599
39928
  "pattern": "{harness}/{provider?}/{model}",
39600
39929
  "description": "One of a group's own agents.",
39601
39930
  "keys": [
@@ -39621,7 +39950,12 @@ export const OPERATIONS = [
39621
39950
  },
39622
39951
  {
39623
39952
  "path": "params.reasoningEffort",
39624
- "kind": "string"
39953
+ "kind": "string",
39954
+ "values": [
39955
+ "low",
39956
+ "medium",
39957
+ "high"
39958
+ ]
39625
39959
  },
39626
39960
  {
39627
39961
  "path": "params.thinkingBudgetTokens",
@@ -39629,11 +39963,21 @@ export const OPERATIONS = [
39629
39963
  },
39630
39964
  {
39631
39965
  "path": "params.codeMode",
39632
- "kind": "string"
39966
+ "kind": "string",
39967
+ "values": [
39968
+ "auto",
39969
+ "on",
39970
+ "off"
39971
+ ]
39633
39972
  },
39634
39973
  {
39635
39974
  "path": "params.toolSearch",
39636
- "kind": "string"
39975
+ "kind": "string",
39976
+ "values": [
39977
+ "on",
39978
+ "off",
39979
+ "directory"
39980
+ ]
39637
39981
  }
39638
39982
  ]
39639
39983
  },
@@ -39641,6 +39985,7 @@ export const OPERATIONS = [
39641
39985
  "flag": "task",
39642
39986
  "optionKey": "task",
39643
39987
  "field": "tasks",
39988
+ "target": "element",
39644
39989
  "text": "prompt",
39645
39990
  "description": "An authored task, by its prompt. @file reads the prompt from a file; a .json file holds the whole task (its rubric, its checks, its steps), or a list of them.",
39646
39991
  "keys": [
@@ -39683,13 +40028,53 @@ export const OPERATIONS = [
39683
40028
  "kind": "boolean"
39684
40029
  }
39685
40030
  ]
40031
+ },
40032
+ {
40033
+ "flag": "run-counting",
40034
+ "optionKey": "runCounting",
40035
+ "field": "runCounting",
40036
+ "target": "object",
40037
+ "pattern": "{policy?}:{screen?}:{levels}",
40038
+ "defaults": {
40039
+ "policy": "hold_back_screened",
40040
+ "screen": "test_awareness"
40041
+ },
40042
+ "words": {
40043
+ "count_all": null
40044
+ },
40045
+ "description": "The levels of the test_awareness screen's verdict whose runs are held back, joined by , or +: influenced (the agent changed what it did because it knew it was tested; the recommended setting), or aware,influenced (it noticed at all). A held-back run is left out of the study's numbers as not measured, never as a fail. The policy as `ora study run-counting` prints it works too: hold_back_screened:test_awareness:aware+influenced, or count_all to count every run.",
40046
+ "keys": [
40047
+ {
40048
+ "path": "policy",
40049
+ "kind": "string",
40050
+ "values": [
40051
+ "hold_back_screened"
40052
+ ]
40053
+ },
40054
+ {
40055
+ "path": "screen",
40056
+ "kind": "string",
40057
+ "values": [
40058
+ "test_awareness"
40059
+ ]
40060
+ },
40061
+ {
40062
+ "path": "levels",
40063
+ "kind": "list",
40064
+ "item": "string",
40065
+ "values": [
40066
+ "aware",
40067
+ "influenced"
40068
+ ]
40069
+ }
40070
+ ]
39686
40071
  }
39687
40072
  ],
39688
40073
  "watchable": true,
39689
40074
  "help": {
39690
40075
  "command": "ora study create --file run.json --watch",
39691
40076
  "body": "{\n \"name\": \"tavily-search groups\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"role\": \"reference\",\n \"config\": {}\n },\n {\n \"label\": \"With Tavily\",\n \"config\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n }\n ],\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}",
39692
- "flags": "ora study create --name <name> --agent <harness>/<provider>/<model>"
40077
+ "flags": "ora study create --name <name> --agent <harness>/<model>"
39693
40078
  }
39694
40079
  },
39695
40080
  {
@@ -40633,7 +41018,7 @@ export const OPERATIONS = [
40633
41018
  "name": "modeOptions",
40634
41019
  "required": false,
40635
41020
  "kind": "object",
40636
- "description": "A comparisonMode's private inputs, validated by that mode:",
41021
+ "description": "A comparisonMode's private inputs, validated by that mode. No mode offered here takes any.",
40637
41022
  "placeholder": "{}"
40638
41023
  },
40639
41024
  {
@@ -40717,6 +41102,7 @@ export const OPERATIONS = [
40717
41102
  "name": "runCounting",
40718
41103
  "required": false,
40719
41104
  "kind": "string",
41105
+ "description": "Which of the study's runs its numbers count. Absent or null = every run. 'hold_back_screened' leaves out the runs the named screen flagged at one of `levels`: each is a missing measurement with a named reason, never a fail, and the count held back is stated beside every figure.",
40720
41106
  "placeholder": "<runCounting>"
40721
41107
  },
40722
41108
  {
@@ -40886,7 +41272,11 @@ export const OPERATIONS = [
40886
41272
  },
40887
41273
  {
40888
41274
  "path": "role",
40889
- "kind": "string"
41275
+ "kind": "string",
41276
+ "values": [
41277
+ "reference",
41278
+ "peer"
41279
+ ]
40890
41280
  },
40891
41281
  {
40892
41282
  "path": "config.tools",
@@ -41071,7 +41461,10 @@ export const OPERATIONS = [
41071
41461
  "keys": [
41072
41462
  {
41073
41463
  "path": "from",
41074
- "kind": "string"
41464
+ "kind": "string",
41465
+ "values": [
41466
+ "series"
41467
+ ]
41075
41468
  },
41076
41469
  {
41077
41470
  "path": "maxAgeDays",
@@ -41087,16 +41480,26 @@ export const OPERATIONS = [
41087
41480
  "keys": [
41088
41481
  {
41089
41482
  "path": "policy",
41090
- "kind": "string"
41483
+ "kind": "string",
41484
+ "values": [
41485
+ "hold_back_screened"
41486
+ ]
41091
41487
  },
41092
41488
  {
41093
41489
  "path": "screen",
41094
- "kind": "string"
41490
+ "kind": "string",
41491
+ "values": [
41492
+ "test_awareness"
41493
+ ]
41095
41494
  },
41096
41495
  {
41097
41496
  "path": "levels",
41098
41497
  "kind": "list",
41099
- "item": "string"
41498
+ "item": "string",
41499
+ "values": [
41500
+ "aware",
41501
+ "influenced"
41502
+ ]
41100
41503
  }
41101
41504
  ],
41102
41505
  "nullable": true
@@ -41149,7 +41552,12 @@ export const OPERATIONS = [
41149
41552
  },
41150
41553
  {
41151
41554
  "path": "params.reasoningEffort",
41152
- "kind": "string"
41555
+ "kind": "string",
41556
+ "values": [
41557
+ "low",
41558
+ "medium",
41559
+ "high"
41560
+ ]
41153
41561
  },
41154
41562
  {
41155
41563
  "path": "params.thinkingBudgetTokens",
@@ -41157,11 +41565,21 @@ export const OPERATIONS = [
41157
41565
  },
41158
41566
  {
41159
41567
  "path": "params.codeMode",
41160
- "kind": "string"
41568
+ "kind": "string",
41569
+ "values": [
41570
+ "auto",
41571
+ "on",
41572
+ "off"
41573
+ ]
41161
41574
  },
41162
41575
  {
41163
41576
  "path": "params.toolSearch",
41164
- "kind": "string"
41577
+ "kind": "string",
41578
+ "values": [
41579
+ "on",
41580
+ "off",
41581
+ "directory"
41582
+ ]
41165
41583
  }
41166
41584
  ]
41167
41585
  },
@@ -41177,6 +41595,7 @@ export const OPERATIONS = [
41177
41595
  "flag": "agent",
41178
41596
  "optionKey": "agent",
41179
41597
  "field": "matrix",
41598
+ "target": "element",
41180
41599
  "pattern": "{harness}/{provider?}/{model}",
41181
41600
  "description": "An agent every group runs: its harness and its model, with the provider between them for a harness that has several. After a comma, the cell's own settings: params.reasoningEffort=high.",
41182
41601
  "keys": [
@@ -41202,7 +41621,12 @@ export const OPERATIONS = [
41202
41621
  },
41203
41622
  {
41204
41623
  "path": "params.reasoningEffort",
41205
- "kind": "string"
41624
+ "kind": "string",
41625
+ "values": [
41626
+ "low",
41627
+ "medium",
41628
+ "high"
41629
+ ]
41206
41630
  },
41207
41631
  {
41208
41632
  "path": "params.thinkingBudgetTokens",
@@ -41210,11 +41634,21 @@ export const OPERATIONS = [
41210
41634
  },
41211
41635
  {
41212
41636
  "path": "params.codeMode",
41213
- "kind": "string"
41637
+ "kind": "string",
41638
+ "values": [
41639
+ "auto",
41640
+ "on",
41641
+ "off"
41642
+ ]
41214
41643
  },
41215
41644
  {
41216
41645
  "path": "params.toolSearch",
41217
- "kind": "string"
41646
+ "kind": "string",
41647
+ "values": [
41648
+ "on",
41649
+ "off",
41650
+ "directory"
41651
+ ]
41218
41652
  }
41219
41653
  ]
41220
41654
  },
@@ -41222,6 +41656,7 @@ export const OPERATIONS = [
41222
41656
  "flag": "group",
41223
41657
  "optionKey": "group",
41224
41658
  "field": "arms",
41659
+ "target": "element",
41225
41660
  "pattern": "{label}",
41226
41661
  "defaults": {
41227
41662
  "config": {}
@@ -41238,7 +41673,11 @@ export const OPERATIONS = [
41238
41673
  },
41239
41674
  {
41240
41675
  "path": "role",
41241
- "kind": "string"
41676
+ "kind": "string",
41677
+ "values": [
41678
+ "reference",
41679
+ "peer"
41680
+ ]
41242
41681
  },
41243
41682
  {
41244
41683
  "path": "config.tools",
@@ -41258,6 +41697,7 @@ export const OPERATIONS = [
41258
41697
  "flag": "agent",
41259
41698
  "optionKey": "agent",
41260
41699
  "field": "arms.cells",
41700
+ "target": "element",
41261
41701
  "pattern": "{harness}/{provider?}/{model}",
41262
41702
  "description": "One of a group's own agents.",
41263
41703
  "keys": [
@@ -41283,7 +41723,12 @@ export const OPERATIONS = [
41283
41723
  },
41284
41724
  {
41285
41725
  "path": "params.reasoningEffort",
41286
- "kind": "string"
41726
+ "kind": "string",
41727
+ "values": [
41728
+ "low",
41729
+ "medium",
41730
+ "high"
41731
+ ]
41287
41732
  },
41288
41733
  {
41289
41734
  "path": "params.thinkingBudgetTokens",
@@ -41291,11 +41736,21 @@ export const OPERATIONS = [
41291
41736
  },
41292
41737
  {
41293
41738
  "path": "params.codeMode",
41294
- "kind": "string"
41739
+ "kind": "string",
41740
+ "values": [
41741
+ "auto",
41742
+ "on",
41743
+ "off"
41744
+ ]
41295
41745
  },
41296
41746
  {
41297
41747
  "path": "params.toolSearch",
41298
- "kind": "string"
41748
+ "kind": "string",
41749
+ "values": [
41750
+ "on",
41751
+ "off",
41752
+ "directory"
41753
+ ]
41299
41754
  }
41300
41755
  ]
41301
41756
  },
@@ -41303,6 +41758,7 @@ export const OPERATIONS = [
41303
41758
  "flag": "task",
41304
41759
  "optionKey": "task",
41305
41760
  "field": "tasks",
41761
+ "target": "element",
41306
41762
  "text": "prompt",
41307
41763
  "description": "An authored task, by its prompt. @file reads the prompt from a file; a .json file holds the whole task (its rubric, its checks, its steps), or a list of them.",
41308
41764
  "keys": [
@@ -41345,13 +41801,53 @@ export const OPERATIONS = [
41345
41801
  "kind": "boolean"
41346
41802
  }
41347
41803
  ]
41804
+ },
41805
+ {
41806
+ "flag": "run-counting",
41807
+ "optionKey": "runCounting",
41808
+ "field": "runCounting",
41809
+ "target": "object",
41810
+ "pattern": "{policy?}:{screen?}:{levels}",
41811
+ "defaults": {
41812
+ "policy": "hold_back_screened",
41813
+ "screen": "test_awareness"
41814
+ },
41815
+ "words": {
41816
+ "count_all": null
41817
+ },
41818
+ "description": "The levels of the test_awareness screen's verdict whose runs are held back, joined by , or +: influenced (the agent changed what it did because it knew it was tested; the recommended setting), or aware,influenced (it noticed at all). A held-back run is left out of the study's numbers as not measured, never as a fail. The policy as `ora study run-counting` prints it works too: hold_back_screened:test_awareness:aware+influenced, or count_all to count every run.",
41819
+ "keys": [
41820
+ {
41821
+ "path": "policy",
41822
+ "kind": "string",
41823
+ "values": [
41824
+ "hold_back_screened"
41825
+ ]
41826
+ },
41827
+ {
41828
+ "path": "screen",
41829
+ "kind": "string",
41830
+ "values": [
41831
+ "test_awareness"
41832
+ ]
41833
+ },
41834
+ {
41835
+ "path": "levels",
41836
+ "kind": "list",
41837
+ "item": "string",
41838
+ "values": [
41839
+ "aware",
41840
+ "influenced"
41841
+ ]
41842
+ }
41843
+ ]
41348
41844
  }
41349
41845
  ],
41350
41846
  "watchable": false,
41351
41847
  "help": {
41352
41848
  "command": "ora study preflight --file run.json",
41353
41849
  "body": "{\n \"name\": \"Rank agents on my tasks\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"My support tasks\",\n \"promise\": \"Answers a customer's billing question correctly\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"config\": {}\n }\n ],\n \"taskCount\": 10,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ]\n}",
41354
- "flags": "ora study preflight --name <name> --agent <harness>/<provider>/<model>"
41850
+ "flags": "ora study preflight --name <name> --agent <harness>/<model>"
41355
41851
  }
41356
41852
  },
41357
41853
  {
@@ -41901,7 +42397,7 @@ export const OPERATIONS = [
41901
42397
  "name": "modeOptions",
41902
42398
  "required": false,
41903
42399
  "kind": "object",
41904
- "description": "A comparisonMode's private inputs, validated by that mode:",
42400
+ "description": "A comparisonMode's private inputs, validated by that mode. No mode offered here takes any.",
41905
42401
  "placeholder": "{}"
41906
42402
  },
41907
42403
  {
@@ -41985,6 +42481,7 @@ export const OPERATIONS = [
41985
42481
  "name": "runCounting",
41986
42482
  "required": false,
41987
42483
  "kind": "string",
42484
+ "description": "Which of the study's runs its numbers count. Absent or null = every run. 'hold_back_screened' leaves out the runs the named screen flagged at one of `levels`: each is a missing measurement with a named reason, never a fail, and the count held back is stated beside every figure.",
41988
42485
  "placeholder": "<runCounting>"
41989
42486
  },
41990
42487
  {
@@ -42154,7 +42651,11 @@ export const OPERATIONS = [
42154
42651
  },
42155
42652
  {
42156
42653
  "path": "role",
42157
- "kind": "string"
42654
+ "kind": "string",
42655
+ "values": [
42656
+ "reference",
42657
+ "peer"
42658
+ ]
42158
42659
  },
42159
42660
  {
42160
42661
  "path": "config.tools",
@@ -42339,7 +42840,10 @@ export const OPERATIONS = [
42339
42840
  "keys": [
42340
42841
  {
42341
42842
  "path": "from",
42342
- "kind": "string"
42843
+ "kind": "string",
42844
+ "values": [
42845
+ "series"
42846
+ ]
42343
42847
  },
42344
42848
  {
42345
42849
  "path": "maxAgeDays",
@@ -42355,16 +42859,26 @@ export const OPERATIONS = [
42355
42859
  "keys": [
42356
42860
  {
42357
42861
  "path": "policy",
42358
- "kind": "string"
42862
+ "kind": "string",
42863
+ "values": [
42864
+ "hold_back_screened"
42865
+ ]
42359
42866
  },
42360
42867
  {
42361
42868
  "path": "screen",
42362
- "kind": "string"
42869
+ "kind": "string",
42870
+ "values": [
42871
+ "test_awareness"
42872
+ ]
42363
42873
  },
42364
42874
  {
42365
42875
  "path": "levels",
42366
42876
  "kind": "list",
42367
- "item": "string"
42877
+ "item": "string",
42878
+ "values": [
42879
+ "aware",
42880
+ "influenced"
42881
+ ]
42368
42882
  }
42369
42883
  ],
42370
42884
  "nullable": true
@@ -42417,7 +42931,12 @@ export const OPERATIONS = [
42417
42931
  },
42418
42932
  {
42419
42933
  "path": "params.reasoningEffort",
42420
- "kind": "string"
42934
+ "kind": "string",
42935
+ "values": [
42936
+ "low",
42937
+ "medium",
42938
+ "high"
42939
+ ]
42421
42940
  },
42422
42941
  {
42423
42942
  "path": "params.thinkingBudgetTokens",
@@ -42425,11 +42944,21 @@ export const OPERATIONS = [
42425
42944
  },
42426
42945
  {
42427
42946
  "path": "params.codeMode",
42428
- "kind": "string"
42947
+ "kind": "string",
42948
+ "values": [
42949
+ "auto",
42950
+ "on",
42951
+ "off"
42952
+ ]
42429
42953
  },
42430
42954
  {
42431
42955
  "path": "params.toolSearch",
42432
- "kind": "string"
42956
+ "kind": "string",
42957
+ "values": [
42958
+ "on",
42959
+ "off",
42960
+ "directory"
42961
+ ]
42433
42962
  }
42434
42963
  ]
42435
42964
  },
@@ -42445,6 +42974,7 @@ export const OPERATIONS = [
42445
42974
  "flag": "agent",
42446
42975
  "optionKey": "agent",
42447
42976
  "field": "matrix",
42977
+ "target": "element",
42448
42978
  "pattern": "{harness}/{provider?}/{model}",
42449
42979
  "description": "An agent every group runs: its harness and its model, with the provider between them for a harness that has several. After a comma, the cell's own settings: params.reasoningEffort=high.",
42450
42980
  "keys": [
@@ -42470,7 +43000,12 @@ export const OPERATIONS = [
42470
43000
  },
42471
43001
  {
42472
43002
  "path": "params.reasoningEffort",
42473
- "kind": "string"
43003
+ "kind": "string",
43004
+ "values": [
43005
+ "low",
43006
+ "medium",
43007
+ "high"
43008
+ ]
42474
43009
  },
42475
43010
  {
42476
43011
  "path": "params.thinkingBudgetTokens",
@@ -42478,11 +43013,21 @@ export const OPERATIONS = [
42478
43013
  },
42479
43014
  {
42480
43015
  "path": "params.codeMode",
42481
- "kind": "string"
43016
+ "kind": "string",
43017
+ "values": [
43018
+ "auto",
43019
+ "on",
43020
+ "off"
43021
+ ]
42482
43022
  },
42483
43023
  {
42484
43024
  "path": "params.toolSearch",
42485
- "kind": "string"
43025
+ "kind": "string",
43026
+ "values": [
43027
+ "on",
43028
+ "off",
43029
+ "directory"
43030
+ ]
42486
43031
  }
42487
43032
  ]
42488
43033
  },
@@ -42490,6 +43035,7 @@ export const OPERATIONS = [
42490
43035
  "flag": "group",
42491
43036
  "optionKey": "group",
42492
43037
  "field": "arms",
43038
+ "target": "element",
42493
43039
  "pattern": "{label}",
42494
43040
  "defaults": {
42495
43041
  "config": {}
@@ -42506,7 +43052,11 @@ export const OPERATIONS = [
42506
43052
  },
42507
43053
  {
42508
43054
  "path": "role",
42509
- "kind": "string"
43055
+ "kind": "string",
43056
+ "values": [
43057
+ "reference",
43058
+ "peer"
43059
+ ]
42510
43060
  },
42511
43061
  {
42512
43062
  "path": "config.tools",
@@ -42526,6 +43076,7 @@ export const OPERATIONS = [
42526
43076
  "flag": "agent",
42527
43077
  "optionKey": "agent",
42528
43078
  "field": "arms.cells",
43079
+ "target": "element",
42529
43080
  "pattern": "{harness}/{provider?}/{model}",
42530
43081
  "description": "One of a group's own agents.",
42531
43082
  "keys": [
@@ -42551,7 +43102,12 @@ export const OPERATIONS = [
42551
43102
  },
42552
43103
  {
42553
43104
  "path": "params.reasoningEffort",
42554
- "kind": "string"
43105
+ "kind": "string",
43106
+ "values": [
43107
+ "low",
43108
+ "medium",
43109
+ "high"
43110
+ ]
42555
43111
  },
42556
43112
  {
42557
43113
  "path": "params.thinkingBudgetTokens",
@@ -42559,11 +43115,21 @@ export const OPERATIONS = [
42559
43115
  },
42560
43116
  {
42561
43117
  "path": "params.codeMode",
42562
- "kind": "string"
43118
+ "kind": "string",
43119
+ "values": [
43120
+ "auto",
43121
+ "on",
43122
+ "off"
43123
+ ]
42563
43124
  },
42564
43125
  {
42565
43126
  "path": "params.toolSearch",
42566
- "kind": "string"
43127
+ "kind": "string",
43128
+ "values": [
43129
+ "on",
43130
+ "off",
43131
+ "directory"
43132
+ ]
42567
43133
  }
42568
43134
  ]
42569
43135
  },
@@ -42571,6 +43137,7 @@ export const OPERATIONS = [
42571
43137
  "flag": "task",
42572
43138
  "optionKey": "task",
42573
43139
  "field": "tasks",
43140
+ "target": "element",
42574
43141
  "text": "prompt",
42575
43142
  "description": "An authored task, by its prompt. @file reads the prompt from a file; a .json file holds the whole task (its rubric, its checks, its steps), or a list of them.",
42576
43143
  "keys": [
@@ -42613,13 +43180,53 @@ export const OPERATIONS = [
42613
43180
  "kind": "boolean"
42614
43181
  }
42615
43182
  ]
43183
+ },
43184
+ {
43185
+ "flag": "run-counting",
43186
+ "optionKey": "runCounting",
43187
+ "field": "runCounting",
43188
+ "target": "object",
43189
+ "pattern": "{policy?}:{screen?}:{levels}",
43190
+ "defaults": {
43191
+ "policy": "hold_back_screened",
43192
+ "screen": "test_awareness"
43193
+ },
43194
+ "words": {
43195
+ "count_all": null
43196
+ },
43197
+ "description": "The levels of the test_awareness screen's verdict whose runs are held back, joined by , or +: influenced (the agent changed what it did because it knew it was tested; the recommended setting), or aware,influenced (it noticed at all). A held-back run is left out of the study's numbers as not measured, never as a fail. The policy as `ora study run-counting` prints it works too: hold_back_screened:test_awareness:aware+influenced, or count_all to count every run.",
43198
+ "keys": [
43199
+ {
43200
+ "path": "policy",
43201
+ "kind": "string",
43202
+ "values": [
43203
+ "hold_back_screened"
43204
+ ]
43205
+ },
43206
+ {
43207
+ "path": "screen",
43208
+ "kind": "string",
43209
+ "values": [
43210
+ "test_awareness"
43211
+ ]
43212
+ },
43213
+ {
43214
+ "path": "levels",
43215
+ "kind": "list",
43216
+ "item": "string",
43217
+ "values": [
43218
+ "aware",
43219
+ "influenced"
43220
+ ]
43221
+ }
43222
+ ]
42616
43223
  }
42617
43224
  ],
42618
43225
  "watchable": false,
42619
43226
  "help": {
42620
43227
  "command": "ora study rehearse --file run.json",
42621
43228
  "body": "{\n \"name\": \"<name>\",\n \"matrix\": []\n}",
42622
- "flags": "ora study rehearse --name <name> --agent <harness>/<provider>/<model>"
43229
+ "flags": "ora study rehearse --name <name> --agent <harness>/<model>"
42623
43230
  },
42624
43231
  "description": "Takes the SAME body a create takes and launches its smallest real version as a study of its own: the first task (`tasks` up to 3), one agent per group, one repeat. It catches what a finished study hides: an add-on that did not attach, a skill that did not load, a grader with nothing to grade. These are real runs and are metered as such. A rehearsal can never be published, and belongs to no series. Read what it proved with GET /studies/:id/rehearsal. The body's tasks must already exist (`tasks`, or a pinned benchmark version). `mode=setup` and `mode=mock` start no agent: each group's environment boots exactly as for a real run, with a stand-in in the agent's place. They cannot show that a setup was USED; only a live rehearsal can."
42625
43232
  },
@@ -43525,7 +44132,12 @@ export const OPERATIONS = [
43525
44132
  },
43526
44133
  {
43527
44134
  "path": "reasoningEffort",
43528
- "kind": "string"
44135
+ "kind": "string",
44136
+ "values": [
44137
+ "low",
44138
+ "medium",
44139
+ "high"
44140
+ ]
43529
44141
  },
43530
44142
  {
43531
44143
  "path": "thinkingBudgetTokens",
@@ -43533,11 +44145,21 @@ export const OPERATIONS = [
43533
44145
  },
43534
44146
  {
43535
44147
  "path": "codeMode",
43536
- "kind": "string"
44148
+ "kind": "string",
44149
+ "values": [
44150
+ "auto",
44151
+ "on",
44152
+ "off"
44153
+ ]
43537
44154
  },
43538
44155
  {
43539
44156
  "path": "toolSearch",
43540
- "kind": "string"
44157
+ "kind": "string",
44158
+ "values": [
44159
+ "on",
44160
+ "off",
44161
+ "directory"
44162
+ ]
43541
44163
  }
43542
44164
  ]
43543
44165
  }
@@ -43963,7 +44585,10 @@ export const OPERATIONS = [
43963
44585
  "keys": [
43964
44586
  {
43965
44587
  "path": "from",
43966
- "kind": "string"
44588
+ "kind": "string",
44589
+ "values": [
44590
+ "series"
44591
+ ]
43967
44592
  },
43968
44593
  {
43969
44594
  "path": "maxAgeDays",
@@ -43977,7 +44602,7 @@ export const OPERATIONS = [
43977
44602
  "help": {
43978
44603
  "command": "ora study set-carry <id> --file run.json",
43979
44604
  "body": "{\n \"carry\": \"<carry>\"\n}",
43980
- "flags": "ora study set-carry <id> --carry from=<from>,maxAgeDays=<maxAgeDays>"
44605
+ "flags": "ora study set-carry <id> --carry from=series,maxAgeDays=<maxAgeDays>"
43981
44606
  },
43982
44607
  "description": "Sets the study's carry policy while it has not launched. `{ carry: null }` turns carrying off: the launch then runs every job, which is how a study is made to measure everything again (for example to gate a group against the study its runs would have been carried from). `{ carry: { from: 'series', maxAgeDays: 7 } }` turns it on, for a study that is in a series. It answers the plan as the launch would now make it. Once the study has launched its policy is frozen: what it carried is a fact of its runs."
43983
44608
  },
@@ -44028,16 +44653,26 @@ export const OPERATIONS = [
44028
44653
  "keys": [
44029
44654
  {
44030
44655
  "path": "policy",
44031
- "kind": "string"
44656
+ "kind": "string",
44657
+ "values": [
44658
+ "hold_back_screened"
44659
+ ]
44032
44660
  },
44033
44661
  {
44034
44662
  "path": "screen",
44035
- "kind": "string"
44663
+ "kind": "string",
44664
+ "values": [
44665
+ "test_awareness"
44666
+ ]
44036
44667
  },
44037
44668
  {
44038
44669
  "path": "levels",
44039
44670
  "kind": "list",
44040
- "item": "string"
44671
+ "item": "string",
44672
+ "values": [
44673
+ "aware",
44674
+ "influenced"
44675
+ ]
44041
44676
  }
44042
44677
  ],
44043
44678
  "nullable": true
@@ -44049,11 +44684,53 @@ export const OPERATIONS = [
44049
44684
  "shape": "boolean"
44050
44685
  }
44051
44686
  ],
44687
+ "bodyShorthands": [
44688
+ {
44689
+ "flag": "run-counting",
44690
+ "optionKey": "runCounting",
44691
+ "field": "runCounting",
44692
+ "target": "object",
44693
+ "pattern": "{policy?}:{screen?}:{levels}",
44694
+ "defaults": {
44695
+ "policy": "hold_back_screened",
44696
+ "screen": "test_awareness"
44697
+ },
44698
+ "words": {
44699
+ "count_all": null
44700
+ },
44701
+ "description": "The levels of the test_awareness screen's verdict whose runs are held back, joined by , or +: influenced (the agent changed what it did because it knew it was tested; the recommended setting), or aware,influenced (it noticed at all). A held-back run is left out of the study's numbers as not measured, never as a fail. The policy as `ora study run-counting` prints it works too: hold_back_screened:test_awareness:aware+influenced, or count_all to count every run.",
44702
+ "keys": [
44703
+ {
44704
+ "path": "policy",
44705
+ "kind": "string",
44706
+ "values": [
44707
+ "hold_back_screened"
44708
+ ]
44709
+ },
44710
+ {
44711
+ "path": "screen",
44712
+ "kind": "string",
44713
+ "values": [
44714
+ "test_awareness"
44715
+ ]
44716
+ },
44717
+ {
44718
+ "path": "levels",
44719
+ "kind": "list",
44720
+ "item": "string",
44721
+ "values": [
44722
+ "aware",
44723
+ "influenced"
44724
+ ]
44725
+ }
44726
+ ]
44727
+ }
44728
+ ],
44052
44729
  "watchable": false,
44053
44730
  "help": {
44054
44731
  "command": "ora study set-run-counting <id> --file run.json",
44055
44732
  "body": "{\n \"runCounting\": \"<runCounting>\"\n}",
44056
- "flags": "ora study set-run-counting <id> --run-counting policy=<policy>,screen=<screen>"
44733
+ "flags": "ora study set-run-counting <id> --run-counting <levels>"
44057
44734
  },
44058
44735
  "description": "Sets the study's counting policy. `{ policy: 'hold_back_screened', screen: 'test_awareness', levels: ['influenced'] }` leaves the runs that screen flagged at those levels out of every number of the study: the readout, the tables, the gate and the series. A held-back run is a missing measurement, never a fail; a judged pair counts only when both its runs do; a run the screen could not judge is never held back. Nothing is re-run or re-judged, and `null` counts every run again. Every change is recorded with who made it. Send `dryRun: true` first to see what would be held back and each group's figure both ways."
44059
44736
  },