@ora-ai/cli 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  // GENERATED by scripts/generate.mjs — DO NOT EDIT.
2
- // Source: openapi/ora-public.openapi.json (sha256 1d3b25b4c612e53f1e666e93e37bfdd3effc5de3f02ff7967c69cd2341c735c1)
2
+ // Source: openapi/ora-public.openapi.json (sha256 d4ecf4f46a6656b52ec10ed5189c677491d35672b2e88f757d20bba92ae74457)
3
3
  // Regenerate: yarn nx run ora-cli:generate
4
4
  export const SPEC = {
5
5
  "version": "1.0.0",
6
- "contentHash": "1d3b25b4c612e53f1e666e93e37bfdd3effc5de3f02ff7967c69cd2341c735c1",
6
+ "contentHash": "d4ecf4f46a6656b52ec10ed5189c677491d35672b2e88f757d20bba92ae74457",
7
7
  "server": "https://api.agentfront.sh",
8
8
  "bin": "ora"
9
9
  };
@@ -672,7 +672,7 @@ export const OPERATIONS = [
672
672
  ],
673
673
  "method": "post",
674
674
  "pathTemplate": "/experiment/v1/benchmarks/{id}/regrade",
675
- "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
675
+ "summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
676
676
  "scope": "benchmarks:write",
677
677
  "product": "experiment",
678
678
  "pathParams": [
@@ -739,6 +739,48 @@ export const OPERATIONS = [
739
739
  "command": "ora benchmarks unpublish <id>"
740
740
  }
741
741
  },
742
+ {
743
+ "id": "benchmarks.updateDetails",
744
+ "commandPath": [
745
+ "benchmarks",
746
+ "update-details"
747
+ ],
748
+ "method": "patch",
749
+ "pathTemplate": "/experiment/v1/benchmarks/{id}",
750
+ "summary": "Edit a study's display name and/or description at ANY status, published included — a correction or a disclosure written after launch reaches the published report (the name as its heading, the description in full as the publisher's note) within the page's normal cache revalidation; the page title and meta description derive from the subject, not from these fields. Changes those two fields and nothing else: the slug (so every shared link stays live), the publication date, the sharing mode and token, the corpus and every grade are untouched. Each field follows benchmarks.create's own rule; send at least one. The response's `shareToken` is null — benchmarks.get carries a link study's token",
751
+ "scope": "benchmarks:write",
752
+ "product": "experiment",
753
+ "pathParams": [
754
+ {
755
+ "name": "id"
756
+ }
757
+ ],
758
+ "queryParams": [],
759
+ "hasBody": true,
760
+ "bodyRequired": true,
761
+ "isStream": false,
762
+ "bodyFields": [
763
+ {
764
+ "name": "name",
765
+ "required": false,
766
+ "kind": "string",
767
+ "description": "Display name for the study (≤160 chars).",
768
+ "placeholder": "<name>"
769
+ },
770
+ {
771
+ "name": "description",
772
+ "required": false,
773
+ "kind": "string",
774
+ "description": "Optional longer description shown on the study card.",
775
+ "placeholder": "Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration."
776
+ }
777
+ ],
778
+ "watchable": false,
779
+ "help": {
780
+ "command": "ora benchmarks update-details 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
781
+ "body": "{\n \"description\": \"Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration.\"\n}"
782
+ }
783
+ },
742
784
  {
743
785
  "id": "benchmarks.updateLenses",
744
786
  "commandPath": [
@@ -1048,6 +1090,7 @@ export const OPERATIONS = [
1048
1090
  "required": false,
1049
1091
  "kind": "string",
1050
1092
  "enum": [
1093
+ "queued",
1051
1094
  "pending",
1052
1095
  "running",
1053
1096
  "succeeded",
@@ -1321,6 +1364,16 @@ export const OPERATIONS = [
1321
1364
  "kind": "boolean",
1322
1365
  "placeholder": "true"
1323
1366
  },
1367
+ {
1368
+ "name": "prompt_context",
1369
+ "required": false,
1370
+ "kind": "string",
1371
+ "enum": [
1372
+ "none",
1373
+ "target_site"
1374
+ ],
1375
+ "placeholder": "none"
1376
+ },
1324
1377
  {
1325
1378
  "name": "mcp_servers",
1326
1379
  "required": false,
@@ -2089,6 +2142,16 @@ export const OPERATIONS = [
2089
2142
  "required": false,
2090
2143
  "kind": "string",
2091
2144
  "placeholder": "<icpId>"
2145
+ },
2146
+ {
2147
+ "name": "firstMeasurement",
2148
+ "required": false,
2149
+ "kind": "string",
2150
+ "enum": [
2151
+ "written",
2152
+ "intent"
2153
+ ],
2154
+ "placeholder": "written"
2092
2155
  }
2093
2156
  ],
2094
2157
  "watchable": false,
@@ -2484,6 +2547,46 @@ export const OPERATIONS = [
2484
2547
  "body": "{\n \"icpId\": \"<icpId>\"\n}"
2485
2548
  }
2486
2549
  },
2550
+ {
2551
+ "id": "intents.setPromptGoal",
2552
+ "commandPath": [
2553
+ "intents",
2554
+ "set-prompt-goal"
2555
+ ],
2556
+ "method": "patch",
2557
+ "pathTemplate": "/studio/v1/intents/{id}/prompts/{promptId}/goals/{kind}",
2558
+ "summary": "Turn one goal a prompt implies on or off (Goal met stops or resumes counting it)",
2559
+ "scope": "intents:write",
2560
+ "product": "studio",
2561
+ "pathParams": [
2562
+ {
2563
+ "name": "id"
2564
+ },
2565
+ {
2566
+ "name": "promptId"
2567
+ },
2568
+ {
2569
+ "name": "kind"
2570
+ }
2571
+ ],
2572
+ "queryParams": [],
2573
+ "hasBody": true,
2574
+ "bodyRequired": true,
2575
+ "isStream": false,
2576
+ "bodyFields": [
2577
+ {
2578
+ "name": "active",
2579
+ "required": true,
2580
+ "kind": "boolean",
2581
+ "placeholder": "true"
2582
+ }
2583
+ ],
2584
+ "watchable": false,
2585
+ "help": {
2586
+ "command": "ora intents set-prompt-goal <id> <promptId> <kind> --file run.json",
2587
+ "body": "{\n \"active\": true\n}"
2588
+ }
2589
+ },
2487
2590
  {
2488
2591
  "id": "intents.update",
2489
2592
  "commandPath": [
@@ -3058,6 +3161,16 @@ export const OPERATIONS = [
3058
3161
  "kind": "boolean",
3059
3162
  "placeholder": "true"
3060
3163
  },
3164
+ {
3165
+ "name": "prompt_context",
3166
+ "required": false,
3167
+ "kind": "string",
3168
+ "enum": [
3169
+ "none",
3170
+ "target_site"
3171
+ ],
3172
+ "placeholder": "none"
3173
+ },
3061
3174
  {
3062
3175
  "name": "mcp_servers",
3063
3176
  "required": false,
@@ -4242,6 +4355,28 @@ export const OPERATIONS = [
4242
4355
  "command": "ora study catalog"
4243
4356
  }
4244
4357
  },
4358
+ {
4359
+ "id": "study.comparisonModes.list",
4360
+ "commandPath": [
4361
+ "study",
4362
+ "comparisonModes",
4363
+ "list"
4364
+ ],
4365
+ "method": "get",
4366
+ "pathTemplate": "/study/v1/studies/comparison-modes",
4367
+ "summary": "List the comparison modes and which this deployment creates studies of",
4368
+ "scope": "study.studies:read",
4369
+ "product": "study",
4370
+ "pathParams": [],
4371
+ "queryParams": [],
4372
+ "hasBody": false,
4373
+ "bodyRequired": false,
4374
+ "isStream": false,
4375
+ "watchable": false,
4376
+ "help": {
4377
+ "command": "ora study comparisonModes list"
4378
+ }
4379
+ },
4245
4380
  {
4246
4381
  "id": "study.convertWebmcpSuite",
4247
4382
  "commandPath": [
@@ -4302,7 +4437,7 @@ export const OPERATIONS = [
4302
4437
  "required": true,
4303
4438
  "kind": "string",
4304
4439
  "description": "Display name for the study (≤160 chars).",
4305
- "placeholder": "tavily-search A/B"
4440
+ "placeholder": "tavily-search groups"
4306
4441
  },
4307
4442
  {
4308
4443
  "name": "description",
@@ -4322,14 +4457,14 @@ export const OPERATIONS = [
4322
4457
  "custom",
4323
4458
  "benchmark"
4324
4459
  ],
4325
- "description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
4326
- "placeholder": "addon"
4460
+ "description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
4461
+ "placeholder": "custom"
4327
4462
  },
4328
4463
  {
4329
4464
  "name": "skillRef",
4330
4465
  "required": false,
4331
4466
  "kind": "string",
4332
- "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
4467
+ "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4333
4468
  "placeholder": "<skillRef>"
4334
4469
  },
4335
4470
  {
@@ -4350,8 +4485,8 @@ export const OPERATIONS = [
4350
4485
  "name": "armConfig",
4351
4486
  "required": false,
4352
4487
  "kind": "object",
4353
- "description": "Frozen per-arm run-payload deltas. Required for subjectKind 'custom' and comparisonMode 'ab_custom'; for 'tools'/'addon' subjects the treatment side is derived when omitted.",
4354
- "placeholder": "{\"treatment\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}"
4488
+ "description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4489
+ "placeholder": "{}"
4355
4490
  },
4356
4491
  {
4357
4492
  "name": "lenses",
@@ -4381,10 +4516,11 @@ export const OPERATIONS = [
4381
4516
  "enum": [
4382
4517
  "ab_vs_none",
4383
4518
  "ab_custom",
4384
- "single"
4519
+ "single",
4520
+ "multi_arm"
4385
4521
  ],
4386
- "description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
4387
- "placeholder": "ab_vs_none"
4522
+ "description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
4523
+ "placeholder": "multi_arm"
4388
4524
  },
4389
4525
  {
4390
4526
  "name": "subjectBackend",
@@ -4401,21 +4537,46 @@ export const OPERATIONS = [
4401
4537
  "name": "baselineLabel",
4402
4538
  "required": false,
4403
4539
  "kind": "string",
4404
- "description": "Baseline arm's display label (e.g. 'REST API'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
4540
+ "description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4405
4541
  "placeholder": "<baselineLabel>"
4406
4542
  },
4407
4543
  {
4408
4544
  "name": "treatmentLabel",
4409
4545
  "required": false,
4410
4546
  "kind": "string",
4411
- "description": "Treatment arm's display label (e.g. 'Remote MCP'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
4547
+ "description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4412
4548
  "placeholder": "<treatmentLabel>"
4413
4549
  },
4550
+ {
4551
+ "name": "arms",
4552
+ "required": false,
4553
+ "kind": "array",
4554
+ "description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
4555
+ "placeholder": "[{\"label\":\"As published\",\"role\":\"reference\",\"config\":{}},{\"label\":\"With Tavily\",\"config\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}]"
4556
+ },
4557
+ {
4558
+ "name": "contrasts",
4559
+ "required": false,
4560
+ "kind": "string",
4561
+ "enum": [
4562
+ "vs_reference",
4563
+ "all_pairs"
4564
+ ],
4565
+ "description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
4566
+ "placeholder": "vs_reference"
4567
+ },
4568
+ {
4569
+ "name": "modeOptions",
4570
+ "required": false,
4571
+ "kind": "object",
4572
+ "description": "A comparisonMode's private inputs, validated by that mode:",
4573
+ "placeholder": "{}"
4574
+ },
4414
4575
  {
4415
4576
  "name": "taskCount",
4416
4577
  "required": false,
4417
4578
  "kind": "number",
4418
- "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
4579
+ "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
4419
4580
  "placeholder": "1"
4420
4581
  },
4421
4582
  {
@@ -4437,7 +4598,7 @@ export const OPERATIONS = [
4437
4598
  "required": false,
4438
4599
  "kind": "array",
4439
4600
  "description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
4440
- "placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
4601
+ "placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
4441
4602
  },
4442
4603
  {
4443
4604
  "name": "subset",
@@ -4464,7 +4625,7 @@ export const OPERATIONS = [
4464
4625
  "watchable": true,
4465
4626
  "help": {
4466
4627
  "command": "ora study create --file run.json --watch",
4467
- "body": "{\n \"name\": \"tavily-search A/B\",\n \"subjectKind\": \"addon\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"armConfig\": {\n \"treatment\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n },\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"any_of\",\n \"label\": \"searched, by either route (treatment arm only)\",\n \"arms\": [\n \"skill\"\n ],\n \"of\": [\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"search\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"Node\"\n }\n }\n },\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"call_page_tool\",\n \"arguments\": {\n \"tool\": \"search\"\n }\n }\n ]\n },\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
4628
+ "body": "{\n \"name\": \"tavily-search groups\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"role\": \"reference\",\n \"config\": {}\n },\n {\n \"label\": \"With Tavily\",\n \"config\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n }\n ],\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
4468
4629
  }
4469
4630
  },
4470
4631
  {
@@ -4684,6 +4845,7 @@ export const OPERATIONS = [
4684
4845
  "required": false,
4685
4846
  "kind": "string",
4686
4847
  "enum": [
4848
+ "queued",
4687
4849
  "pending",
4688
4850
  "running",
4689
4851
  "succeeded",
@@ -4882,6 +5044,231 @@ export const OPERATIONS = [
4882
5044
  "command": "ora study list-run-answers 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 7c9e6679-7425-40de-944b-e07fc1f90ae7"
4883
5045
  }
4884
5046
  },
5047
+ {
5048
+ "id": "study.preflight",
5049
+ "commandPath": [
5050
+ "study",
5051
+ "preflight"
5052
+ ],
5053
+ "method": "post",
5054
+ "pathTemplate": "/study/v1/studies/preflight",
5055
+ "summary": "Check a study design without creating it: the same body a create takes, answered with the study it would be (its form and name, planned runs and judge comparisons, effective groups, planned comparisons, how it would board) — or the SAME 422 create would answer. Nothing is persisted and nothing is generated.",
5056
+ "scope": "study.studies:write",
5057
+ "product": "study",
5058
+ "pathParams": [],
5059
+ "queryParams": [
5060
+ {
5061
+ "name": "depth",
5062
+ "flag": "depth",
5063
+ "optionKey": "depth",
5064
+ "required": false,
5065
+ "kind": "string",
5066
+ "enum": [
5067
+ "default",
5068
+ "full"
5069
+ ]
5070
+ }
5071
+ ],
5072
+ "hasBody": true,
5073
+ "bodyRequired": true,
5074
+ "isStream": false,
5075
+ "bodyFields": [
5076
+ {
5077
+ "name": "name",
5078
+ "required": true,
5079
+ "kind": "string",
5080
+ "description": "Display name for the study (≤160 chars).",
5081
+ "placeholder": "Rank agents on my tasks"
5082
+ },
5083
+ {
5084
+ "name": "description",
5085
+ "required": false,
5086
+ "kind": "string",
5087
+ "description": "Optional longer description shown on the study card.",
5088
+ "placeholder": "<description>"
5089
+ },
5090
+ {
5091
+ "name": "subjectKind",
5092
+ "required": false,
5093
+ "kind": "string",
5094
+ "enum": [
5095
+ "skill",
5096
+ "tools",
5097
+ "addon",
5098
+ "custom",
5099
+ "benchmark"
5100
+ ],
5101
+ "description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
5102
+ "placeholder": "custom"
5103
+ },
5104
+ {
5105
+ "name": "skillRef",
5106
+ "required": false,
5107
+ "kind": "string",
5108
+ "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5109
+ "placeholder": "<skillRef>"
5110
+ },
5111
+ {
5112
+ "name": "definitionVersionId",
5113
+ "required": false,
5114
+ "kind": "string",
5115
+ "description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
5116
+ "placeholder": "<definitionVersionId>"
5117
+ },
5118
+ {
5119
+ "name": "subjectName",
5120
+ "required": false,
5121
+ "kind": "string",
5122
+ "description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
5123
+ "placeholder": "My support tasks"
5124
+ },
5125
+ {
5126
+ "name": "armConfig",
5127
+ "required": false,
5128
+ "kind": "object",
5129
+ "description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5130
+ "placeholder": "{}"
5131
+ },
5132
+ {
5133
+ "name": "lenses",
5134
+ "required": false,
5135
+ "kind": "array",
5136
+ "description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
5137
+ "placeholder": "[\"judge\"]"
5138
+ },
5139
+ {
5140
+ "name": "judgeSupplementary",
5141
+ "required": false,
5142
+ "kind": "boolean",
5143
+ "description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
5144
+ "placeholder": "true"
5145
+ },
5146
+ {
5147
+ "name": "proposeExpectations",
5148
+ "required": false,
5149
+ "kind": "boolean",
5150
+ "description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
5151
+ "placeholder": "true"
5152
+ },
5153
+ {
5154
+ "name": "comparisonMode",
5155
+ "required": false,
5156
+ "kind": "string",
5157
+ "enum": [
5158
+ "ab_vs_none",
5159
+ "ab_custom",
5160
+ "single",
5161
+ "multi_arm"
5162
+ ],
5163
+ "description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
5164
+ "placeholder": "multi_arm"
5165
+ },
5166
+ {
5167
+ "name": "subjectBackend",
5168
+ "required": false,
5169
+ "kind": "string",
5170
+ "enum": [
5171
+ "model",
5172
+ "harness"
5173
+ ],
5174
+ "description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
5175
+ "placeholder": "model"
5176
+ },
5177
+ {
5178
+ "name": "baselineLabel",
5179
+ "required": false,
5180
+ "kind": "string",
5181
+ "description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5182
+ "placeholder": "<baselineLabel>"
5183
+ },
5184
+ {
5185
+ "name": "treatmentLabel",
5186
+ "required": false,
5187
+ "kind": "string",
5188
+ "description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5189
+ "placeholder": "<treatmentLabel>"
5190
+ },
5191
+ {
5192
+ "name": "arms",
5193
+ "required": false,
5194
+ "kind": "array",
5195
+ "description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
5196
+ "placeholder": "[{\"label\":\"As published\",\"config\":{}}]"
5197
+ },
5198
+ {
5199
+ "name": "contrasts",
5200
+ "required": false,
5201
+ "kind": "string",
5202
+ "enum": [
5203
+ "vs_reference",
5204
+ "all_pairs"
5205
+ ],
5206
+ "description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
5207
+ "placeholder": "vs_reference"
5208
+ },
5209
+ {
5210
+ "name": "modeOptions",
5211
+ "required": false,
5212
+ "kind": "object",
5213
+ "description": "A comparisonMode's private inputs, validated by that mode:",
5214
+ "placeholder": "{}"
5215
+ },
5216
+ {
5217
+ "name": "taskCount",
5218
+ "required": false,
5219
+ "kind": "number",
5220
+ "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
5221
+ "placeholder": "10"
5222
+ },
5223
+ {
5224
+ "name": "variantsPerTask",
5225
+ "required": false,
5226
+ "kind": "number",
5227
+ "description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
5228
+ "placeholder": "1"
5229
+ },
5230
+ {
5231
+ "name": "repeats",
5232
+ "required": false,
5233
+ "kind": "number",
5234
+ "description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
5235
+ "placeholder": "1"
5236
+ },
5237
+ {
5238
+ "name": "tasks",
5239
+ "required": false,
5240
+ "kind": "array",
5241
+ "description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
5242
+ "placeholder": "[]"
5243
+ },
5244
+ {
5245
+ "name": "subset",
5246
+ "required": false,
5247
+ "kind": "object",
5248
+ "description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
5249
+ "placeholder": "{}"
5250
+ },
5251
+ {
5252
+ "name": "matrix",
5253
+ "required": true,
5254
+ "kind": "array",
5255
+ "description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
5256
+ "placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"},{\"harness\":\"codex\",\"model\":\"gpt-5.5\"}]"
5257
+ },
5258
+ {
5259
+ "name": "promise",
5260
+ "required": false,
5261
+ "kind": "string",
5262
+ "description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
5263
+ "placeholder": "Answers a customer's billing question correctly"
5264
+ }
5265
+ ],
5266
+ "watchable": false,
5267
+ "help": {
5268
+ "command": "ora study preflight --file run.json",
5269
+ "body": "{\n \"name\": \"Rank agents on my tasks\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"My support tasks\",\n \"promise\": \"Answers a customer's billing question correctly\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"config\": {}\n }\n ],\n \"taskCount\": 10,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ]\n}"
5270
+ }
5271
+ },
4885
5272
  {
4886
5273
  "id": "study.preview",
4887
5274
  "commandPath": [
@@ -4947,6 +5334,17 @@ export const OPERATIONS = [
4947
5334
  ],
4948
5335
  "description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
4949
5336
  "placeholder": "private"
5337
+ },
5338
+ {
5339
+ "name": "skillNames",
5340
+ "required": false,
5341
+ "kind": "string",
5342
+ "enum": [
5343
+ "withheld",
5344
+ "named"
5345
+ ],
5346
+ "description": "A Groups study only: 'named' shows each group's skills by name on the published page (as they were when the study was created); 'withheld' (the default, also when omitted) keeps them private. Every publish states it, so a re-publish that omits it withholds the names again. Add-on and MCP instance names are never published.",
5347
+ "placeholder": "withheld"
4950
5348
  }
4951
5349
  ],
4952
5350
  "watchable": false,
@@ -5013,7 +5411,7 @@ export const OPERATIONS = [
5013
5411
  ],
5014
5412
  "method": "post",
5015
5413
  "pathTemplate": "/study/v1/studies/{id}/regrade",
5016
- "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
5414
+ "summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
5017
5415
  "scope": "study.studies:write",
5018
5416
  "product": "study",
5019
5417
  "pathParams": [
@@ -5273,6 +5671,16 @@ export const OPERATIONS = [
5273
5671
  "kind": "boolean",
5274
5672
  "placeholder": "true"
5275
5673
  },
5674
+ {
5675
+ "name": "prompt_context",
5676
+ "required": false,
5677
+ "kind": "string",
5678
+ "enum": [
5679
+ "none",
5680
+ "target_site"
5681
+ ],
5682
+ "placeholder": "none"
5683
+ },
5276
5684
  {
5277
5685
  "name": "mcp_servers",
5278
5686
  "required": false,
@@ -5787,6 +6195,48 @@ export const OPERATIONS = [
5787
6195
  "command": "ora study unpublish <id>"
5788
6196
  }
5789
6197
  },
6198
+ {
6199
+ "id": "study.updateDetails",
6200
+ "commandPath": [
6201
+ "study",
6202
+ "update-details"
6203
+ ],
6204
+ "method": "patch",
6205
+ "pathTemplate": "/study/v1/studies/{id}",
6206
+ "summary": "Edit a study's display name and/or description at ANY status, published included — a correction or a disclosure written after launch reaches the published report (the name as its heading, the description in full as the publisher's note) within the page's normal cache revalidation; the page title and meta description derive from the subject, not from these fields. Changes those two fields and nothing else: the slug (so every shared link stays live), the publication date, the sharing mode and token, the corpus and every grade are untouched. Each field follows study.create's own rule; send at least one. The response's `shareToken` is null — study.get carries a link study's token",
6207
+ "scope": "study.studies:write",
6208
+ "product": "study",
6209
+ "pathParams": [
6210
+ {
6211
+ "name": "id"
6212
+ }
6213
+ ],
6214
+ "queryParams": [],
6215
+ "hasBody": true,
6216
+ "bodyRequired": true,
6217
+ "isStream": false,
6218
+ "bodyFields": [
6219
+ {
6220
+ "name": "name",
6221
+ "required": false,
6222
+ "kind": "string",
6223
+ "description": "Display name for the study (≤160 chars).",
6224
+ "placeholder": "<name>"
6225
+ },
6226
+ {
6227
+ "name": "description",
6228
+ "required": false,
6229
+ "kind": "string",
6230
+ "description": "Optional longer description shown on the study card.",
6231
+ "placeholder": "Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration."
6232
+ }
6233
+ ],
6234
+ "watchable": false,
6235
+ "help": {
6236
+ "command": "ora study update-details 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
6237
+ "body": "{\n \"description\": \"Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration.\"\n}"
6238
+ }
6239
+ },
5790
6240
  {
5791
6241
  "id": "study.updateLenses",
5792
6242
  "commandPath": [