@ora-ai/cli 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  // GENERATED by scripts/generate.mjs — DO NOT EDIT.
2
- // Source: openapi/ora-public.openapi.json (sha256 067a219e40d4ab8e05bcd2f6fb9b38e772c34e946852e4ad1ee19264faba71b1)
2
+ // Source: openapi/ora-public.openapi.json (sha256 d4ecf4f46a6656b52ec10ed5189c677491d35672b2e88f757d20bba92ae74457)
3
3
  // Regenerate: yarn nx run ora-cli:generate
4
4
  export const SPEC = {
5
5
  "version": "1.0.0",
6
- "contentHash": "067a219e40d4ab8e05bcd2f6fb9b38e772c34e946852e4ad1ee19264faba71b1",
6
+ "contentHash": "d4ecf4f46a6656b52ec10ed5189c677491d35672b2e88f757d20bba92ae74457",
7
7
  "server": "https://api.agentfront.sh",
8
8
  "bin": "ora"
9
9
  };
@@ -328,6 +328,13 @@ export const OPERATIONS = [
328
328
  "description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
329
329
  "placeholder": "[\"judge\",\"evals\"]"
330
330
  },
331
+ {
332
+ "name": "judgeSupplementary",
333
+ "required": false,
334
+ "kind": "boolean",
335
+ "description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
336
+ "placeholder": "true"
337
+ },
331
338
  {
332
339
  "name": "proposeExpectations",
333
340
  "required": false,
@@ -665,7 +672,7 @@ export const OPERATIONS = [
665
672
  ],
666
673
  "method": "post",
667
674
  "pathTemplate": "/experiment/v1/benchmarks/{id}/regrade",
668
- "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
675
+ "summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
669
676
  "scope": "benchmarks:write",
670
677
  "product": "experiment",
671
678
  "pathParams": [
@@ -732,6 +739,48 @@ export const OPERATIONS = [
732
739
  "command": "ora benchmarks unpublish <id>"
733
740
  }
734
741
  },
742
+ {
743
+ "id": "benchmarks.updateDetails",
744
+ "commandPath": [
745
+ "benchmarks",
746
+ "update-details"
747
+ ],
748
+ "method": "patch",
749
+ "pathTemplate": "/experiment/v1/benchmarks/{id}",
750
+ "summary": "Edit a study's display name and/or description at ANY status, published included — a correction or a disclosure written after launch reaches the published report (the name as its heading, the description in full as the publisher's note) within the page's normal cache revalidation; the page title and meta description derive from the subject, not from these fields. Changes those two fields and nothing else: the slug (so every shared link stays live), the publication date, the sharing mode and token, the corpus and every grade are untouched. Each field follows benchmarks.create's own rule; send at least one. The response's `shareToken` is null — benchmarks.get carries a link study's token",
751
+ "scope": "benchmarks:write",
752
+ "product": "experiment",
753
+ "pathParams": [
754
+ {
755
+ "name": "id"
756
+ }
757
+ ],
758
+ "queryParams": [],
759
+ "hasBody": true,
760
+ "bodyRequired": true,
761
+ "isStream": false,
762
+ "bodyFields": [
763
+ {
764
+ "name": "name",
765
+ "required": false,
766
+ "kind": "string",
767
+ "description": "Display name for the study (≤160 chars).",
768
+ "placeholder": "<name>"
769
+ },
770
+ {
771
+ "name": "description",
772
+ "required": false,
773
+ "kind": "string",
774
+ "description": "Optional longer description shown on the study card.",
775
+ "placeholder": "Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration."
776
+ }
777
+ ],
778
+ "watchable": false,
779
+ "help": {
780
+ "command": "ora benchmarks update-details 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
781
+ "body": "{\n \"description\": \"Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration.\"\n}"
782
+ }
783
+ },
735
784
  {
736
785
  "id": "benchmarks.updateLenses",
737
786
  "commandPath": [
@@ -1041,6 +1090,7 @@ export const OPERATIONS = [
1041
1090
  "required": false,
1042
1091
  "kind": "string",
1043
1092
  "enum": [
1093
+ "queued",
1044
1094
  "pending",
1045
1095
  "running",
1046
1096
  "succeeded",
@@ -1314,6 +1364,16 @@ export const OPERATIONS = [
1314
1364
  "kind": "boolean",
1315
1365
  "placeholder": "true"
1316
1366
  },
1367
+ {
1368
+ "name": "prompt_context",
1369
+ "required": false,
1370
+ "kind": "string",
1371
+ "enum": [
1372
+ "none",
1373
+ "target_site"
1374
+ ],
1375
+ "placeholder": "none"
1376
+ },
1317
1377
  {
1318
1378
  "name": "mcp_servers",
1319
1379
  "required": false,
@@ -2082,6 +2142,16 @@ export const OPERATIONS = [
2082
2142
  "required": false,
2083
2143
  "kind": "string",
2084
2144
  "placeholder": "<icpId>"
2145
+ },
2146
+ {
2147
+ "name": "firstMeasurement",
2148
+ "required": false,
2149
+ "kind": "string",
2150
+ "enum": [
2151
+ "written",
2152
+ "intent"
2153
+ ],
2154
+ "placeholder": "written"
2085
2155
  }
2086
2156
  ],
2087
2157
  "watchable": false,
@@ -2477,6 +2547,46 @@ export const OPERATIONS = [
2477
2547
  "body": "{\n \"icpId\": \"<icpId>\"\n}"
2478
2548
  }
2479
2549
  },
2550
+ {
2551
+ "id": "intents.setPromptGoal",
2552
+ "commandPath": [
2553
+ "intents",
2554
+ "set-prompt-goal"
2555
+ ],
2556
+ "method": "patch",
2557
+ "pathTemplate": "/studio/v1/intents/{id}/prompts/{promptId}/goals/{kind}",
2558
+ "summary": "Turn one goal a prompt implies on or off (Goal met stops or resumes counting it)",
2559
+ "scope": "intents:write",
2560
+ "product": "studio",
2561
+ "pathParams": [
2562
+ {
2563
+ "name": "id"
2564
+ },
2565
+ {
2566
+ "name": "promptId"
2567
+ },
2568
+ {
2569
+ "name": "kind"
2570
+ }
2571
+ ],
2572
+ "queryParams": [],
2573
+ "hasBody": true,
2574
+ "bodyRequired": true,
2575
+ "isStream": false,
2576
+ "bodyFields": [
2577
+ {
2578
+ "name": "active",
2579
+ "required": true,
2580
+ "kind": "boolean",
2581
+ "placeholder": "true"
2582
+ }
2583
+ ],
2584
+ "watchable": false,
2585
+ "help": {
2586
+ "command": "ora intents set-prompt-goal <id> <promptId> <kind> --file run.json",
2587
+ "body": "{\n \"active\": true\n}"
2588
+ }
2589
+ },
2480
2590
  {
2481
2591
  "id": "intents.update",
2482
2592
  "commandPath": [
@@ -3051,6 +3161,16 @@ export const OPERATIONS = [
3051
3161
  "kind": "boolean",
3052
3162
  "placeholder": "true"
3053
3163
  },
3164
+ {
3165
+ "name": "prompt_context",
3166
+ "required": false,
3167
+ "kind": "string",
3168
+ "enum": [
3169
+ "none",
3170
+ "target_site"
3171
+ ],
3172
+ "placeholder": "none"
3173
+ },
3054
3174
  {
3055
3175
  "name": "mcp_servers",
3056
3176
  "required": false,
@@ -4235,6 +4355,28 @@ export const OPERATIONS = [
4235
4355
  "command": "ora study catalog"
4236
4356
  }
4237
4357
  },
4358
+ {
4359
+ "id": "study.comparisonModes.list",
4360
+ "commandPath": [
4361
+ "study",
4362
+ "comparisonModes",
4363
+ "list"
4364
+ ],
4365
+ "method": "get",
4366
+ "pathTemplate": "/study/v1/studies/comparison-modes",
4367
+ "summary": "List the comparison modes and which this deployment creates studies of",
4368
+ "scope": "study.studies:read",
4369
+ "product": "study",
4370
+ "pathParams": [],
4371
+ "queryParams": [],
4372
+ "hasBody": false,
4373
+ "bodyRequired": false,
4374
+ "isStream": false,
4375
+ "watchable": false,
4376
+ "help": {
4377
+ "command": "ora study comparisonModes list"
4378
+ }
4379
+ },
4238
4380
  {
4239
4381
  "id": "study.convertWebmcpSuite",
4240
4382
  "commandPath": [
@@ -4295,7 +4437,7 @@ export const OPERATIONS = [
4295
4437
  "required": true,
4296
4438
  "kind": "string",
4297
4439
  "description": "Display name for the study (≤160 chars).",
4298
- "placeholder": "tavily-search A/B"
4440
+ "placeholder": "tavily-search groups"
4299
4441
  },
4300
4442
  {
4301
4443
  "name": "description",
@@ -4315,14 +4457,14 @@ export const OPERATIONS = [
4315
4457
  "custom",
4316
4458
  "benchmark"
4317
4459
  ],
4318
- "description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
4319
- "placeholder": "addon"
4460
+ "description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
4461
+ "placeholder": "custom"
4320
4462
  },
4321
4463
  {
4322
4464
  "name": "skillRef",
4323
4465
  "required": false,
4324
4466
  "kind": "string",
4325
- "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
4467
+ "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4326
4468
  "placeholder": "<skillRef>"
4327
4469
  },
4328
4470
  {
@@ -4343,8 +4485,8 @@ export const OPERATIONS = [
4343
4485
  "name": "armConfig",
4344
4486
  "required": false,
4345
4487
  "kind": "object",
4346
- "description": "Frozen per-arm run-payload deltas. Required for subjectKind 'custom' and comparisonMode 'ab_custom'; for 'tools'/'addon' subjects the treatment side is derived when omitted.",
4347
- "placeholder": "{\"treatment\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}"
4488
+ "description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4489
+ "placeholder": "{}"
4348
4490
  },
4349
4491
  {
4350
4492
  "name": "lenses",
@@ -4353,6 +4495,13 @@ export const OPERATIONS = [
4353
4495
  "description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
4354
4496
  "placeholder": "[\"judge\",\"evals\"]"
4355
4497
  },
4498
+ {
4499
+ "name": "judgeSupplementary",
4500
+ "required": false,
4501
+ "kind": "boolean",
4502
+ "description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
4503
+ "placeholder": "true"
4504
+ },
4356
4505
  {
4357
4506
  "name": "proposeExpectations",
4358
4507
  "required": false,
@@ -4367,10 +4516,11 @@ export const OPERATIONS = [
4367
4516
  "enum": [
4368
4517
  "ab_vs_none",
4369
4518
  "ab_custom",
4370
- "single"
4519
+ "single",
4520
+ "multi_arm"
4371
4521
  ],
4372
- "description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
4373
- "placeholder": "ab_vs_none"
4522
+ "description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
4523
+ "placeholder": "multi_arm"
4374
4524
  },
4375
4525
  {
4376
4526
  "name": "subjectBackend",
@@ -4387,21 +4537,46 @@ export const OPERATIONS = [
4387
4537
  "name": "baselineLabel",
4388
4538
  "required": false,
4389
4539
  "kind": "string",
4390
- "description": "Baseline arm's display label (e.g. 'REST API'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
4540
+ "description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4391
4541
  "placeholder": "<baselineLabel>"
4392
4542
  },
4393
4543
  {
4394
4544
  "name": "treatmentLabel",
4395
4545
  "required": false,
4396
4546
  "kind": "string",
4397
- "description": "Treatment arm's display label (e.g. 'Remote MCP'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
4547
+ "description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4398
4548
  "placeholder": "<treatmentLabel>"
4399
4549
  },
4550
+ {
4551
+ "name": "arms",
4552
+ "required": false,
4553
+ "kind": "array",
4554
+ "description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
4555
+ "placeholder": "[{\"label\":\"As published\",\"role\":\"reference\",\"config\":{}},{\"label\":\"With Tavily\",\"config\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}]"
4556
+ },
4557
+ {
4558
+ "name": "contrasts",
4559
+ "required": false,
4560
+ "kind": "string",
4561
+ "enum": [
4562
+ "vs_reference",
4563
+ "all_pairs"
4564
+ ],
4565
+ "description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
4566
+ "placeholder": "vs_reference"
4567
+ },
4568
+ {
4569
+ "name": "modeOptions",
4570
+ "required": false,
4571
+ "kind": "object",
4572
+ "description": "A comparisonMode's private inputs, validated by that mode:",
4573
+ "placeholder": "{}"
4574
+ },
4400
4575
  {
4401
4576
  "name": "taskCount",
4402
4577
  "required": false,
4403
4578
  "kind": "number",
4404
- "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
4579
+ "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
4405
4580
  "placeholder": "1"
4406
4581
  },
4407
4582
  {
@@ -4423,7 +4598,7 @@ export const OPERATIONS = [
4423
4598
  "required": false,
4424
4599
  "kind": "array",
4425
4600
  "description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
4426
- "placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
4601
+ "placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
4427
4602
  },
4428
4603
  {
4429
4604
  "name": "subset",
@@ -4450,7 +4625,7 @@ export const OPERATIONS = [
4450
4625
  "watchable": true,
4451
4626
  "help": {
4452
4627
  "command": "ora study create --file run.json --watch",
4453
- "body": "{\n \"name\": \"tavily-search A/B\",\n \"subjectKind\": \"addon\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"armConfig\": {\n \"treatment\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n },\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"any_of\",\n \"label\": \"searched, by either route (treatment arm only)\",\n \"arms\": [\n \"skill\"\n ],\n \"of\": [\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"search\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"Node\"\n }\n }\n },\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"call_page_tool\",\n \"arguments\": {\n \"tool\": \"search\"\n }\n }\n ]\n },\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
4628
+ "body": "{\n \"name\": \"tavily-search groups\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"role\": \"reference\",\n \"config\": {}\n },\n {\n \"label\": \"With Tavily\",\n \"config\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n }\n ],\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
4454
4629
  }
4455
4630
  },
4456
4631
  {
@@ -4670,6 +4845,7 @@ export const OPERATIONS = [
4670
4845
  "required": false,
4671
4846
  "kind": "string",
4672
4847
  "enum": [
4848
+ "queued",
4673
4849
  "pending",
4674
4850
  "running",
4675
4851
  "succeeded",
@@ -4868,6 +5044,231 @@ export const OPERATIONS = [
4868
5044
  "command": "ora study list-run-answers 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 7c9e6679-7425-40de-944b-e07fc1f90ae7"
4869
5045
  }
4870
5046
  },
5047
+ {
5048
+ "id": "study.preflight",
5049
+ "commandPath": [
5050
+ "study",
5051
+ "preflight"
5052
+ ],
5053
+ "method": "post",
5054
+ "pathTemplate": "/study/v1/studies/preflight",
5055
+ "summary": "Check a study design without creating it: the same body a create takes, answered with the study it would be (its form and name, planned runs and judge comparisons, effective groups, planned comparisons, how it would board) — or the SAME 422 create would answer. Nothing is persisted and nothing is generated.",
5056
+ "scope": "study.studies:write",
5057
+ "product": "study",
5058
+ "pathParams": [],
5059
+ "queryParams": [
5060
+ {
5061
+ "name": "depth",
5062
+ "flag": "depth",
5063
+ "optionKey": "depth",
5064
+ "required": false,
5065
+ "kind": "string",
5066
+ "enum": [
5067
+ "default",
5068
+ "full"
5069
+ ]
5070
+ }
5071
+ ],
5072
+ "hasBody": true,
5073
+ "bodyRequired": true,
5074
+ "isStream": false,
5075
+ "bodyFields": [
5076
+ {
5077
+ "name": "name",
5078
+ "required": true,
5079
+ "kind": "string",
5080
+ "description": "Display name for the study (≤160 chars).",
5081
+ "placeholder": "Rank agents on my tasks"
5082
+ },
5083
+ {
5084
+ "name": "description",
5085
+ "required": false,
5086
+ "kind": "string",
5087
+ "description": "Optional longer description shown on the study card.",
5088
+ "placeholder": "<description>"
5089
+ },
5090
+ {
5091
+ "name": "subjectKind",
5092
+ "required": false,
5093
+ "kind": "string",
5094
+ "enum": [
5095
+ "skill",
5096
+ "tools",
5097
+ "addon",
5098
+ "custom",
5099
+ "benchmark"
5100
+ ],
5101
+ "description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
5102
+ "placeholder": "custom"
5103
+ },
5104
+ {
5105
+ "name": "skillRef",
5106
+ "required": false,
5107
+ "kind": "string",
5108
+ "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5109
+ "placeholder": "<skillRef>"
5110
+ },
5111
+ {
5112
+ "name": "definitionVersionId",
5113
+ "required": false,
5114
+ "kind": "string",
5115
+ "description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
5116
+ "placeholder": "<definitionVersionId>"
5117
+ },
5118
+ {
5119
+ "name": "subjectName",
5120
+ "required": false,
5121
+ "kind": "string",
5122
+ "description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
5123
+ "placeholder": "My support tasks"
5124
+ },
5125
+ {
5126
+ "name": "armConfig",
5127
+ "required": false,
5128
+ "kind": "object",
5129
+ "description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5130
+ "placeholder": "{}"
5131
+ },
5132
+ {
5133
+ "name": "lenses",
5134
+ "required": false,
5135
+ "kind": "array",
5136
+ "description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
5137
+ "placeholder": "[\"judge\"]"
5138
+ },
5139
+ {
5140
+ "name": "judgeSupplementary",
5141
+ "required": false,
5142
+ "kind": "boolean",
5143
+ "description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
5144
+ "placeholder": "true"
5145
+ },
5146
+ {
5147
+ "name": "proposeExpectations",
5148
+ "required": false,
5149
+ "kind": "boolean",
5150
+ "description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
5151
+ "placeholder": "true"
5152
+ },
5153
+ {
5154
+ "name": "comparisonMode",
5155
+ "required": false,
5156
+ "kind": "string",
5157
+ "enum": [
5158
+ "ab_vs_none",
5159
+ "ab_custom",
5160
+ "single",
5161
+ "multi_arm"
5162
+ ],
5163
+ "description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
5164
+ "placeholder": "multi_arm"
5165
+ },
5166
+ {
5167
+ "name": "subjectBackend",
5168
+ "required": false,
5169
+ "kind": "string",
5170
+ "enum": [
5171
+ "model",
5172
+ "harness"
5173
+ ],
5174
+ "description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
5175
+ "placeholder": "model"
5176
+ },
5177
+ {
5178
+ "name": "baselineLabel",
5179
+ "required": false,
5180
+ "kind": "string",
5181
+ "description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5182
+ "placeholder": "<baselineLabel>"
5183
+ },
5184
+ {
5185
+ "name": "treatmentLabel",
5186
+ "required": false,
5187
+ "kind": "string",
5188
+ "description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5189
+ "placeholder": "<treatmentLabel>"
5190
+ },
5191
+ {
5192
+ "name": "arms",
5193
+ "required": false,
5194
+ "kind": "array",
5195
+ "description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
5196
+ "placeholder": "[{\"label\":\"As published\",\"config\":{}}]"
5197
+ },
5198
+ {
5199
+ "name": "contrasts",
5200
+ "required": false,
5201
+ "kind": "string",
5202
+ "enum": [
5203
+ "vs_reference",
5204
+ "all_pairs"
5205
+ ],
5206
+ "description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
5207
+ "placeholder": "vs_reference"
5208
+ },
5209
+ {
5210
+ "name": "modeOptions",
5211
+ "required": false,
5212
+ "kind": "object",
5213
+ "description": "A comparisonMode's private inputs, validated by that mode:",
5214
+ "placeholder": "{}"
5215
+ },
5216
+ {
5217
+ "name": "taskCount",
5218
+ "required": false,
5219
+ "kind": "number",
5220
+ "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
5221
+ "placeholder": "10"
5222
+ },
5223
+ {
5224
+ "name": "variantsPerTask",
5225
+ "required": false,
5226
+ "kind": "number",
5227
+ "description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
5228
+ "placeholder": "1"
5229
+ },
5230
+ {
5231
+ "name": "repeats",
5232
+ "required": false,
5233
+ "kind": "number",
5234
+ "description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
5235
+ "placeholder": "1"
5236
+ },
5237
+ {
5238
+ "name": "tasks",
5239
+ "required": false,
5240
+ "kind": "array",
5241
+ "description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
5242
+ "placeholder": "[]"
5243
+ },
5244
+ {
5245
+ "name": "subset",
5246
+ "required": false,
5247
+ "kind": "object",
5248
+ "description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
5249
+ "placeholder": "{}"
5250
+ },
5251
+ {
5252
+ "name": "matrix",
5253
+ "required": true,
5254
+ "kind": "array",
5255
+ "description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
5256
+ "placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"},{\"harness\":\"codex\",\"model\":\"gpt-5.5\"}]"
5257
+ },
5258
+ {
5259
+ "name": "promise",
5260
+ "required": false,
5261
+ "kind": "string",
5262
+ "description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
5263
+ "placeholder": "Answers a customer's billing question correctly"
5264
+ }
5265
+ ],
5266
+ "watchable": false,
5267
+ "help": {
5268
+ "command": "ora study preflight --file run.json",
5269
+ "body": "{\n \"name\": \"Rank agents on my tasks\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"My support tasks\",\n \"promise\": \"Answers a customer's billing question correctly\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"config\": {}\n }\n ],\n \"taskCount\": 10,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ]\n}"
5270
+ }
5271
+ },
4871
5272
  {
4872
5273
  "id": "study.preview",
4873
5274
  "commandPath": [
@@ -4933,6 +5334,17 @@ export const OPERATIONS = [
4933
5334
  ],
4934
5335
  "description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
4935
5336
  "placeholder": "private"
5337
+ },
5338
+ {
5339
+ "name": "skillNames",
5340
+ "required": false,
5341
+ "kind": "string",
5342
+ "enum": [
5343
+ "withheld",
5344
+ "named"
5345
+ ],
5346
+ "description": "A Groups study only: 'named' shows each group's skills by name on the published page (as they were when the study was created); 'withheld' (the default, also when omitted) keeps them private. Every publish states it, so a re-publish that omits it withholds the names again. Add-on and MCP instance names are never published.",
5347
+ "placeholder": "withheld"
4936
5348
  }
4937
5349
  ],
4938
5350
  "watchable": false,
@@ -4999,7 +5411,7 @@ export const OPERATIONS = [
4999
5411
  ],
5000
5412
  "method": "post",
5001
5413
  "pathTemplate": "/study/v1/studies/{id}/regrade",
5002
- "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
5414
+ "summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
5003
5415
  "scope": "study.studies:write",
5004
5416
  "product": "study",
5005
5417
  "pathParams": [
@@ -5259,6 +5671,16 @@ export const OPERATIONS = [
5259
5671
  "kind": "boolean",
5260
5672
  "placeholder": "true"
5261
5673
  },
5674
+ {
5675
+ "name": "prompt_context",
5676
+ "required": false,
5677
+ "kind": "string",
5678
+ "enum": [
5679
+ "none",
5680
+ "target_site"
5681
+ ],
5682
+ "placeholder": "none"
5683
+ },
5262
5684
  {
5263
5685
  "name": "mcp_servers",
5264
5686
  "required": false,
@@ -5773,6 +6195,48 @@ export const OPERATIONS = [
5773
6195
  "command": "ora study unpublish <id>"
5774
6196
  }
5775
6197
  },
6198
+ {
6199
+ "id": "study.updateDetails",
6200
+ "commandPath": [
6201
+ "study",
6202
+ "update-details"
6203
+ ],
6204
+ "method": "patch",
6205
+ "pathTemplate": "/study/v1/studies/{id}",
6206
+ "summary": "Edit a study's display name and/or description at ANY status, published included — a correction or a disclosure written after launch reaches the published report (the name as its heading, the description in full as the publisher's note) within the page's normal cache revalidation; the page title and meta description derive from the subject, not from these fields. Changes those two fields and nothing else: the slug (so every shared link stays live), the publication date, the sharing mode and token, the corpus and every grade are untouched. Each field follows study.create's own rule; send at least one. The response's `shareToken` is null — study.get carries a link study's token",
6207
+ "scope": "study.studies:write",
6208
+ "product": "study",
6209
+ "pathParams": [
6210
+ {
6211
+ "name": "id"
6212
+ }
6213
+ ],
6214
+ "queryParams": [],
6215
+ "hasBody": true,
6216
+ "bodyRequired": true,
6217
+ "isStream": false,
6218
+ "bodyFields": [
6219
+ {
6220
+ "name": "name",
6221
+ "required": false,
6222
+ "kind": "string",
6223
+ "description": "Display name for the study (≤160 chars).",
6224
+ "placeholder": "<name>"
6225
+ },
6226
+ {
6227
+ "name": "description",
6228
+ "required": false,
6229
+ "kind": "string",
6230
+ "description": "Optional longer description shown on the study card.",
6231
+ "placeholder": "Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration."
6232
+ }
6233
+ ],
6234
+ "watchable": false,
6235
+ "help": {
6236
+ "command": "ora study update-details 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
6237
+ "body": "{\n \"description\": \"Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration.\"\n}"
6238
+ }
6239
+ },
5776
6240
  {
5777
6241
  "id": "study.updateLenses",
5778
6242
  "commandPath": [