@ora-ai/cli 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  // GENERATED by scripts/generate.mjs — DO NOT EDIT.
2
- // Source: openapi/ora-public.openapi.json (sha256 1d3b25b4c612e53f1e666e93e37bfdd3effc5de3f02ff7967c69cd2341c735c1)
2
+ // Source: openapi/ora-public.openapi.json (sha256 43cebbdca1928a2f344ae61b881d2ba3871dc2f503e6fbda7c4152e57c385095)
3
3
  // Regenerate: yarn nx run ora-cli:generate
4
4
  export const SPEC = {
5
5
  "version": "1.0.0",
6
- "contentHash": "1d3b25b4c612e53f1e666e93e37bfdd3effc5de3f02ff7967c69cd2341c735c1",
6
+ "contentHash": "43cebbdca1928a2f344ae61b881d2ba3871dc2f503e6fbda7c4152e57c385095",
7
7
  "server": "https://api.agentfront.sh",
8
8
  "bin": "ora"
9
9
  };
@@ -672,7 +672,7 @@ export const OPERATIONS = [
672
672
  ],
673
673
  "method": "post",
674
674
  "pathTemplate": "/experiment/v1/benchmarks/{id}/regrade",
675
- "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
675
+ "summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
676
676
  "scope": "benchmarks:write",
677
677
  "product": "experiment",
678
678
  "pathParams": [
@@ -739,6 +739,48 @@ export const OPERATIONS = [
739
739
  "command": "ora benchmarks unpublish <id>"
740
740
  }
741
741
  },
742
+ {
743
+ "id": "benchmarks.updateDetails",
744
+ "commandPath": [
745
+ "benchmarks",
746
+ "update-details"
747
+ ],
748
+ "method": "patch",
749
+ "pathTemplate": "/experiment/v1/benchmarks/{id}",
750
+ "summary": "Edit a study's display name and/or description at ANY status, published included — a correction or a disclosure written after launch reaches the published report (the name as its heading, the description in full as the publisher's note) within the page's normal cache revalidation; the page title and meta description derive from the subject, not from these fields. Changes those two fields and nothing else: the slug (so every shared link stays live), the publication date, the sharing mode and token, the corpus and every grade are untouched. Each field follows benchmarks.create's own rule; send at least one. The response's `shareToken` is null — benchmarks.get carries a link study's token",
751
+ "scope": "benchmarks:write",
752
+ "product": "experiment",
753
+ "pathParams": [
754
+ {
755
+ "name": "id"
756
+ }
757
+ ],
758
+ "queryParams": [],
759
+ "hasBody": true,
760
+ "bodyRequired": true,
761
+ "isStream": false,
762
+ "bodyFields": [
763
+ {
764
+ "name": "name",
765
+ "required": false,
766
+ "kind": "string",
767
+ "description": "Display name for the study (≤160 chars).",
768
+ "placeholder": "<name>"
769
+ },
770
+ {
771
+ "name": "description",
772
+ "required": false,
773
+ "kind": "string",
774
+ "description": "Optional longer description shown on the study card.",
775
+ "placeholder": "Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration."
776
+ }
777
+ ],
778
+ "watchable": false,
779
+ "help": {
780
+ "command": "ora benchmarks update-details 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
781
+ "body": "{\n \"description\": \"Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration.\"\n}"
782
+ }
783
+ },
742
784
  {
743
785
  "id": "benchmarks.updateLenses",
744
786
  "commandPath": [
@@ -1048,6 +1090,7 @@ export const OPERATIONS = [
1048
1090
  "required": false,
1049
1091
  "kind": "string",
1050
1092
  "enum": [
1093
+ "queued",
1051
1094
  "pending",
1052
1095
  "running",
1053
1096
  "succeeded",
@@ -1321,6 +1364,16 @@ export const OPERATIONS = [
1321
1364
  "kind": "boolean",
1322
1365
  "placeholder": "true"
1323
1366
  },
1367
+ {
1368
+ "name": "prompt_context",
1369
+ "required": false,
1370
+ "kind": "string",
1371
+ "enum": [
1372
+ "none",
1373
+ "target_site"
1374
+ ],
1375
+ "placeholder": "none"
1376
+ },
1324
1377
  {
1325
1378
  "name": "mcp_servers",
1326
1379
  "required": false,
@@ -2089,6 +2142,16 @@ export const OPERATIONS = [
2089
2142
  "required": false,
2090
2143
  "kind": "string",
2091
2144
  "placeholder": "<icpId>"
2145
+ },
2146
+ {
2147
+ "name": "firstMeasurement",
2148
+ "required": false,
2149
+ "kind": "string",
2150
+ "enum": [
2151
+ "written",
2152
+ "intent"
2153
+ ],
2154
+ "placeholder": "written"
2092
2155
  }
2093
2156
  ],
2094
2157
  "watchable": false,
@@ -2484,6 +2547,46 @@ export const OPERATIONS = [
2484
2547
  "body": "{\n \"icpId\": \"<icpId>\"\n}"
2485
2548
  }
2486
2549
  },
2550
+ {
2551
+ "id": "intents.setPromptGoal",
2552
+ "commandPath": [
2553
+ "intents",
2554
+ "set-prompt-goal"
2555
+ ],
2556
+ "method": "patch",
2557
+ "pathTemplate": "/studio/v1/intents/{id}/prompts/{promptId}/goals/{kind}",
2558
+ "summary": "Turn one goal a prompt implies on or off (Goal met stops or resumes counting it)",
2559
+ "scope": "intents:write",
2560
+ "product": "studio",
2561
+ "pathParams": [
2562
+ {
2563
+ "name": "id"
2564
+ },
2565
+ {
2566
+ "name": "promptId"
2567
+ },
2568
+ {
2569
+ "name": "kind"
2570
+ }
2571
+ ],
2572
+ "queryParams": [],
2573
+ "hasBody": true,
2574
+ "bodyRequired": true,
2575
+ "isStream": false,
2576
+ "bodyFields": [
2577
+ {
2578
+ "name": "active",
2579
+ "required": true,
2580
+ "kind": "boolean",
2581
+ "placeholder": "true"
2582
+ }
2583
+ ],
2584
+ "watchable": false,
2585
+ "help": {
2586
+ "command": "ora intents set-prompt-goal <id> <promptId> <kind> --file run.json",
2587
+ "body": "{\n \"active\": true\n}"
2588
+ }
2589
+ },
2487
2590
  {
2488
2591
  "id": "intents.update",
2489
2592
  "commandPath": [
@@ -3058,6 +3161,16 @@ export const OPERATIONS = [
3058
3161
  "kind": "boolean",
3059
3162
  "placeholder": "true"
3060
3163
  },
3164
+ {
3165
+ "name": "prompt_context",
3166
+ "required": false,
3167
+ "kind": "string",
3168
+ "enum": [
3169
+ "none",
3170
+ "target_site"
3171
+ ],
3172
+ "placeholder": "none"
3173
+ },
3061
3174
  {
3062
3175
  "name": "mcp_servers",
3063
3176
  "required": false,
@@ -3311,6 +3424,55 @@ export const OPERATIONS = [
3311
3424
  "command": "ora studio runs trajectory <id>"
3312
3425
  }
3313
3426
  },
3427
+ {
3428
+ "id": "study.addAgents",
3429
+ "commandPath": [
3430
+ "study",
3431
+ "add-agents"
3432
+ ],
3433
+ "method": "post",
3434
+ "pathTemplate": "/study/v1/studies/{id}/agents",
3435
+ "summary": "Add agents to a running or completed-but-unpublished study — to one group when each group runs its own, to every group when they share the matrix — and create exactly the runs they owe, in the same study. Nothing that exists is re-run or re-graded; each comparison keeps its basis and only gains agents. A completed study re-opens until the new runs finish and are graded. Idempotent: an agent already present is reported, never added twice. dryRun answers the plan.",
3436
+ "scope": "study.studies:write",
3437
+ "product": "study",
3438
+ "pathParams": [
3439
+ {
3440
+ "name": "id"
3441
+ }
3442
+ ],
3443
+ "queryParams": [],
3444
+ "hasBody": true,
3445
+ "bodyRequired": true,
3446
+ "isStream": false,
3447
+ "bodyFields": [
3448
+ {
3449
+ "name": "group",
3450
+ "required": false,
3451
+ "kind": "string",
3452
+ "description": "The group the agents join — required when each group runs its own agents, refused when every group runs the shared matrix (then each agent joins every group).",
3453
+ "placeholder": "g2"
3454
+ },
3455
+ {
3456
+ "name": "agents",
3457
+ "required": true,
3458
+ "kind": "array",
3459
+ "description": "The agents to add, in order — each a harness × model (× provider × params) cell the catalog offers. An agent the target already runs is reported in `alreadyPresent` and added once only.",
3460
+ "placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"}]"
3461
+ },
3462
+ {
3463
+ "name": "dryRun",
3464
+ "required": false,
3465
+ "kind": "boolean",
3466
+ "description": "Answer the full plan — the agents, the jobs it would create, each comparison before and after — writing nothing (default false).",
3467
+ "placeholder": "true"
3468
+ }
3469
+ ],
3470
+ "watchable": false,
3471
+ "help": {
3472
+ "command": "ora study add-agents 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
3473
+ "body": "{\n \"group\": \"g2\",\n \"agents\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n }\n ],\n \"dryRun\": true\n}"
3474
+ }
3475
+ },
3314
3476
  {
3315
3477
  "id": "study.addons.list",
3316
3478
  "commandPath": [
@@ -4242,6 +4404,28 @@ export const OPERATIONS = [
4242
4404
  "command": "ora study catalog"
4243
4405
  }
4244
4406
  },
4407
+ {
4408
+ "id": "study.comparisonModes.list",
4409
+ "commandPath": [
4410
+ "study",
4411
+ "comparisonModes",
4412
+ "list"
4413
+ ],
4414
+ "method": "get",
4415
+ "pathTemplate": "/study/v1/studies/comparison-modes",
4416
+ "summary": "List the comparison modes and which this deployment creates studies of",
4417
+ "scope": "study.studies:read",
4418
+ "product": "study",
4419
+ "pathParams": [],
4420
+ "queryParams": [],
4421
+ "hasBody": false,
4422
+ "bodyRequired": false,
4423
+ "isStream": false,
4424
+ "watchable": false,
4425
+ "help": {
4426
+ "command": "ora study comparisonModes list"
4427
+ }
4428
+ },
4245
4429
  {
4246
4430
  "id": "study.convertWebmcpSuite",
4247
4431
  "commandPath": [
@@ -4302,7 +4486,7 @@ export const OPERATIONS = [
4302
4486
  "required": true,
4303
4487
  "kind": "string",
4304
4488
  "description": "Display name for the study (≤160 chars).",
4305
- "placeholder": "tavily-search A/B"
4489
+ "placeholder": "tavily-search groups"
4306
4490
  },
4307
4491
  {
4308
4492
  "name": "description",
@@ -4322,14 +4506,14 @@ export const OPERATIONS = [
4322
4506
  "custom",
4323
4507
  "benchmark"
4324
4508
  ],
4325
- "description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
4326
- "placeholder": "addon"
4509
+ "description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
4510
+ "placeholder": "custom"
4327
4511
  },
4328
4512
  {
4329
4513
  "name": "skillRef",
4330
4514
  "required": false,
4331
4515
  "kind": "string",
4332
- "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
4516
+ "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4333
4517
  "placeholder": "<skillRef>"
4334
4518
  },
4335
4519
  {
@@ -4350,8 +4534,8 @@ export const OPERATIONS = [
4350
4534
  "name": "armConfig",
4351
4535
  "required": false,
4352
4536
  "kind": "object",
4353
- "description": "Frozen per-arm run-payload deltas. Required for subjectKind 'custom' and comparisonMode 'ab_custom'; for 'tools'/'addon' subjects the treatment side is derived when omitted.",
4354
- "placeholder": "{\"treatment\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}"
4537
+ "description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4538
+ "placeholder": "{}"
4355
4539
  },
4356
4540
  {
4357
4541
  "name": "lenses",
@@ -4381,10 +4565,11 @@ export const OPERATIONS = [
4381
4565
  "enum": [
4382
4566
  "ab_vs_none",
4383
4567
  "ab_custom",
4384
- "single"
4568
+ "single",
4569
+ "multi_arm"
4385
4570
  ],
4386
- "description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
4387
- "placeholder": "ab_vs_none"
4571
+ "description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
4572
+ "placeholder": "multi_arm"
4388
4573
  },
4389
4574
  {
4390
4575
  "name": "subjectBackend",
@@ -4401,21 +4586,46 @@ export const OPERATIONS = [
4401
4586
  "name": "baselineLabel",
4402
4587
  "required": false,
4403
4588
  "kind": "string",
4404
- "description": "Baseline arm's display label (e.g. 'REST API'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
4589
+ "description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4405
4590
  "placeholder": "<baselineLabel>"
4406
4591
  },
4407
4592
  {
4408
4593
  "name": "treatmentLabel",
4409
4594
  "required": false,
4410
4595
  "kind": "string",
4411
- "description": "Treatment arm's display label (e.g. 'Remote MCP'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
4596
+ "description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
4412
4597
  "placeholder": "<treatmentLabel>"
4413
4598
  },
4599
+ {
4600
+ "name": "arms",
4601
+ "required": false,
4602
+ "kind": "array",
4603
+ "description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
4604
+ "placeholder": "[{\"label\":\"As published\",\"role\":\"reference\",\"config\":{}},{\"label\":\"With Tavily\",\"config\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}]"
4605
+ },
4606
+ {
4607
+ "name": "contrasts",
4608
+ "required": false,
4609
+ "kind": "string",
4610
+ "enum": [
4611
+ "vs_reference",
4612
+ "all_pairs"
4613
+ ],
4614
+ "description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
4615
+ "placeholder": "vs_reference"
4616
+ },
4617
+ {
4618
+ "name": "modeOptions",
4619
+ "required": false,
4620
+ "kind": "object",
4621
+ "description": "A comparisonMode's private inputs, validated by that mode:",
4622
+ "placeholder": "{}"
4623
+ },
4414
4624
  {
4415
4625
  "name": "taskCount",
4416
4626
  "required": false,
4417
4627
  "kind": "number",
4418
- "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
4628
+ "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
4419
4629
  "placeholder": "1"
4420
4630
  },
4421
4631
  {
@@ -4437,7 +4647,7 @@ export const OPERATIONS = [
4437
4647
  "required": false,
4438
4648
  "kind": "array",
4439
4649
  "description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
4440
- "placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
4650
+ "placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
4441
4651
  },
4442
4652
  {
4443
4653
  "name": "subset",
@@ -4464,7 +4674,7 @@ export const OPERATIONS = [
4464
4674
  "watchable": true,
4465
4675
  "help": {
4466
4676
  "command": "ora study create --file run.json --watch",
4467
- "body": "{\n \"name\": \"tavily-search A/B\",\n \"subjectKind\": \"addon\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"armConfig\": {\n \"treatment\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n },\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"any_of\",\n \"label\": \"searched, by either route (treatment arm only)\",\n \"arms\": [\n \"skill\"\n ],\n \"of\": [\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"search\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"Node\"\n }\n }\n },\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"call_page_tool\",\n \"arguments\": {\n \"tool\": \"search\"\n }\n }\n ]\n },\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
4677
+ "body": "{\n \"name\": \"tavily-search groups\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"role\": \"reference\",\n \"config\": {}\n },\n {\n \"label\": \"With Tavily\",\n \"config\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n }\n ],\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
4468
4678
  }
4469
4679
  },
4470
4680
  {
@@ -4684,6 +4894,7 @@ export const OPERATIONS = [
4684
4894
  "required": false,
4685
4895
  "kind": "string",
4686
4896
  "enum": [
4897
+ "queued",
4687
4898
  "pending",
4688
4899
  "running",
4689
4900
  "succeeded",
@@ -4816,7 +5027,7 @@ export const OPERATIONS = [
4816
5027
  ],
4817
5028
  "method": "post",
4818
5029
  "pathTemplate": "/study/v1/studies/{id}/launch",
4819
- "summary": "Launch: fan out task × matrix-cell × arm runs (async — poll get until status 'completed')",
5030
+ "summary": "Launch: fan out task × matrix-cell × arm runs (async — poll get until status 'completed'). Launching a RUNNING study again creates exactly the planned runs its launch record says are missing (`launch.missing.retryable`, `relaunchable`), or takes over a fan-out that stopped — never a duplicate",
4820
5031
  "scope": "study.studies:write",
4821
5032
  "product": "study",
4822
5033
  "pathParams": [
@@ -4882,6 +5093,231 @@ export const OPERATIONS = [
4882
5093
  "command": "ora study list-run-answers 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 7c9e6679-7425-40de-944b-e07fc1f90ae7"
4883
5094
  }
4884
5095
  },
5096
+ {
5097
+ "id": "study.preflight",
5098
+ "commandPath": [
5099
+ "study",
5100
+ "preflight"
5101
+ ],
5102
+ "method": "post",
5103
+ "pathTemplate": "/study/v1/studies/preflight",
5104
+ "summary": "Check a study design without creating it: the same body a create takes, answered with the study it would be (its form and name, planned runs and judge comparisons, effective groups, planned comparisons, how it would board) — or the SAME 422 create would answer. Nothing is persisted and nothing is generated.",
5105
+ "scope": "study.studies:write",
5106
+ "product": "study",
5107
+ "pathParams": [],
5108
+ "queryParams": [
5109
+ {
5110
+ "name": "depth",
5111
+ "flag": "depth",
5112
+ "optionKey": "depth",
5113
+ "required": false,
5114
+ "kind": "string",
5115
+ "enum": [
5116
+ "default",
5117
+ "full"
5118
+ ]
5119
+ }
5120
+ ],
5121
+ "hasBody": true,
5122
+ "bodyRequired": true,
5123
+ "isStream": false,
5124
+ "bodyFields": [
5125
+ {
5126
+ "name": "name",
5127
+ "required": true,
5128
+ "kind": "string",
5129
+ "description": "Display name for the study (≤160 chars).",
5130
+ "placeholder": "Rank agents on my tasks"
5131
+ },
5132
+ {
5133
+ "name": "description",
5134
+ "required": false,
5135
+ "kind": "string",
5136
+ "description": "Optional longer description shown on the study card.",
5137
+ "placeholder": "<description>"
5138
+ },
5139
+ {
5140
+ "name": "subjectKind",
5141
+ "required": false,
5142
+ "kind": "string",
5143
+ "enum": [
5144
+ "skill",
5145
+ "tools",
5146
+ "addon",
5147
+ "custom",
5148
+ "benchmark"
5149
+ ],
5150
+ "description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
5151
+ "placeholder": "custom"
5152
+ },
5153
+ {
5154
+ "name": "skillRef",
5155
+ "required": false,
5156
+ "kind": "string",
5157
+ "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5158
+ "placeholder": "<skillRef>"
5159
+ },
5160
+ {
5161
+ "name": "definitionVersionId",
5162
+ "required": false,
5163
+ "kind": "string",
5164
+ "description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
5165
+ "placeholder": "<definitionVersionId>"
5166
+ },
5167
+ {
5168
+ "name": "subjectName",
5169
+ "required": false,
5170
+ "kind": "string",
5171
+ "description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
5172
+ "placeholder": "My support tasks"
5173
+ },
5174
+ {
5175
+ "name": "armConfig",
5176
+ "required": false,
5177
+ "kind": "object",
5178
+ "description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5179
+ "placeholder": "{}"
5180
+ },
5181
+ {
5182
+ "name": "lenses",
5183
+ "required": false,
5184
+ "kind": "array",
5185
+ "description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
5186
+ "placeholder": "[\"judge\"]"
5187
+ },
5188
+ {
5189
+ "name": "judgeSupplementary",
5190
+ "required": false,
5191
+ "kind": "boolean",
5192
+ "description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
5193
+ "placeholder": "true"
5194
+ },
5195
+ {
5196
+ "name": "proposeExpectations",
5197
+ "required": false,
5198
+ "kind": "boolean",
5199
+ "description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
5200
+ "placeholder": "true"
5201
+ },
5202
+ {
5203
+ "name": "comparisonMode",
5204
+ "required": false,
5205
+ "kind": "string",
5206
+ "enum": [
5207
+ "ab_vs_none",
5208
+ "ab_custom",
5209
+ "single",
5210
+ "multi_arm"
5211
+ ],
5212
+ "description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
5213
+ "placeholder": "multi_arm"
5214
+ },
5215
+ {
5216
+ "name": "subjectBackend",
5217
+ "required": false,
5218
+ "kind": "string",
5219
+ "enum": [
5220
+ "model",
5221
+ "harness"
5222
+ ],
5223
+ "description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
5224
+ "placeholder": "model"
5225
+ },
5226
+ {
5227
+ "name": "baselineLabel",
5228
+ "required": false,
5229
+ "kind": "string",
5230
+ "description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5231
+ "placeholder": "<baselineLabel>"
5232
+ },
5233
+ {
5234
+ "name": "treatmentLabel",
5235
+ "required": false,
5236
+ "kind": "string",
5237
+ "description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
5238
+ "placeholder": "<treatmentLabel>"
5239
+ },
5240
+ {
5241
+ "name": "arms",
5242
+ "required": false,
5243
+ "kind": "array",
5244
+ "description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
5245
+ "placeholder": "[{\"label\":\"As published\",\"config\":{}}]"
5246
+ },
5247
+ {
5248
+ "name": "contrasts",
5249
+ "required": false,
5250
+ "kind": "string",
5251
+ "enum": [
5252
+ "vs_reference",
5253
+ "all_pairs"
5254
+ ],
5255
+ "description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
5256
+ "placeholder": "vs_reference"
5257
+ },
5258
+ {
5259
+ "name": "modeOptions",
5260
+ "required": false,
5261
+ "kind": "object",
5262
+ "description": "A comparisonMode's private inputs, validated by that mode:",
5263
+ "placeholder": "{}"
5264
+ },
5265
+ {
5266
+ "name": "taskCount",
5267
+ "required": false,
5268
+ "kind": "number",
5269
+ "description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
5270
+ "placeholder": "10"
5271
+ },
5272
+ {
5273
+ "name": "variantsPerTask",
5274
+ "required": false,
5275
+ "kind": "number",
5276
+ "description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
5277
+ "placeholder": "1"
5278
+ },
5279
+ {
5280
+ "name": "repeats",
5281
+ "required": false,
5282
+ "kind": "number",
5283
+ "description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
5284
+ "placeholder": "1"
5285
+ },
5286
+ {
5287
+ "name": "tasks",
5288
+ "required": false,
5289
+ "kind": "array",
5290
+ "description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
5291
+ "placeholder": "[]"
5292
+ },
5293
+ {
5294
+ "name": "subset",
5295
+ "required": false,
5296
+ "kind": "object",
5297
+ "description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
5298
+ "placeholder": "{}"
5299
+ },
5300
+ {
5301
+ "name": "matrix",
5302
+ "required": true,
5303
+ "kind": "array",
5304
+ "description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
5305
+ "placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"},{\"harness\":\"codex\",\"model\":\"gpt-5.5\"}]"
5306
+ },
5307
+ {
5308
+ "name": "promise",
5309
+ "required": false,
5310
+ "kind": "string",
5311
+ "description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
5312
+ "placeholder": "Answers a customer's billing question correctly"
5313
+ }
5314
+ ],
5315
+ "watchable": false,
5316
+ "help": {
5317
+ "command": "ora study preflight --file run.json",
5318
+ "body": "{\n \"name\": \"Rank agents on my tasks\",\n \"subjectKind\": \"custom\",\n \"subjectName\": \"My support tasks\",\n \"promise\": \"Answers a customer's billing question correctly\",\n \"comparisonMode\": \"multi_arm\",\n \"arms\": [\n {\n \"label\": \"As published\",\n \"config\": {}\n }\n ],\n \"taskCount\": 10,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ]\n}"
5319
+ }
5320
+ },
4885
5321
  {
4886
5322
  "id": "study.preview",
4887
5323
  "commandPath": [
@@ -4947,6 +5383,17 @@ export const OPERATIONS = [
4947
5383
  ],
4948
5384
  "description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
4949
5385
  "placeholder": "private"
5386
+ },
5387
+ {
5388
+ "name": "skillNames",
5389
+ "required": false,
5390
+ "kind": "string",
5391
+ "enum": [
5392
+ "withheld",
5393
+ "named"
5394
+ ],
5395
+ "description": "A Groups study only: 'named' shows each group's skills by name on the published page (as they were when the study was created); 'withheld' (the default, also when omitted) keeps them private. Every publish states it, so a re-publish that omits it withholds the names again. Add-on and MCP instance names are never published.",
5396
+ "placeholder": "withheld"
4950
5397
  }
4951
5398
  ],
4952
5399
  "watchable": false,
@@ -5013,7 +5460,7 @@ export const OPERATIONS = [
5013
5460
  ],
5014
5461
  "method": "post",
5015
5462
  "pathTemplate": "/study/v1/studies/{id}/regrade",
5016
- "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
5463
+ "summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
5017
5464
  "scope": "study.studies:write",
5018
5465
  "product": "study",
5019
5466
  "pathParams": [
@@ -5030,6 +5477,80 @@ export const OPERATIONS = [
5030
5477
  "command": "ora study regrade <id> --watch"
5031
5478
  }
5032
5479
  },
5480
+ {
5481
+ "id": "study.rerunAttempts",
5482
+ "commandPath": [
5483
+ "study",
5484
+ "rerun-attempts"
5485
+ ],
5486
+ "method": "get",
5487
+ "pathTemplate": "/study/v1/studies/{id}/rerun-attempts",
5488
+ "summary": "The study's re-run ledger: every in-place re-dispatch of one of its jobs — the run it replaced and how that run ended, why it was re-run (with a declared re-run's evidence), the run it created and whether it dispatched. Oldest first.",
5489
+ "scope": "study.studies:read",
5490
+ "product": "study",
5491
+ "pathParams": [
5492
+ {
5493
+ "name": "id"
5494
+ }
5495
+ ],
5496
+ "queryParams": [],
5497
+ "hasBody": false,
5498
+ "bodyRequired": false,
5499
+ "isStream": false,
5500
+ "watchable": false,
5501
+ "help": {
5502
+ "command": "ora study rerun-attempts 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
5503
+ }
5504
+ },
5505
+ {
5506
+ "id": "study.rerunInfraFailures",
5507
+ "commandPath": [
5508
+ "study",
5509
+ "rerun-infra-failures"
5510
+ ],
5511
+ "method": "post",
5512
+ "pathTemplate": "/study/v1/studies/{id}/rerun-infra-failures",
5513
+ "summary": "Re-run, in place, the jobs of a running or completed-but-unpublished study whose run ora's own infrastructure killed (or that the launch never created): each new run takes its job's place, the failed run stays in the record, at most 2 re-runs per job, never while the job's run is still running. A completed study re-opens until its re-runs finish and are graded. Synchronous; dryRun answers the plan.",
5514
+ "scope": "study.studies:write",
5515
+ "product": "study",
5516
+ "pathParams": [
5517
+ {
5518
+ "name": "id"
5519
+ }
5520
+ ],
5521
+ "queryParams": [],
5522
+ "hasBody": true,
5523
+ "bodyRequired": true,
5524
+ "isStream": false,
5525
+ "bodyFields": [
5526
+ {
5527
+ "name": "dryRun",
5528
+ "required": false,
5529
+ "kind": "boolean",
5530
+ "description": "Answer what WOULD be re-run, dispatching and writing nothing (default false).",
5531
+ "placeholder": "true"
5532
+ },
5533
+ {
5534
+ "name": "limit",
5535
+ "required": false,
5536
+ "kind": "number",
5537
+ "description": "The most jobs this call dispatches (default 200, or every declared run); the rest are counted in `skipped.overLimit` for the next call.",
5538
+ "placeholder": "1"
5539
+ },
5540
+ {
5541
+ "name": "declared",
5542
+ "required": false,
5543
+ "kind": "array",
5544
+ "description": "The DECLARED re-run: exactly these runs are re-run in place (cause declared_infra), each with its evidence — runs ora's own predicate does not flag. Absent: ora's own infra failures and never-created jobs.",
5545
+ "placeholder": "[]"
5546
+ }
5547
+ ],
5548
+ "watchable": false,
5549
+ "help": {
5550
+ "command": "ora study rerun-infra-failures 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
5551
+ "body": "{\n \"dryRun\": true\n}"
5552
+ }
5553
+ },
5033
5554
  {
5034
5555
  "id": "study.rotateShareToken",
5035
5556
  "commandPath": [
@@ -5273,6 +5794,16 @@ export const OPERATIONS = [
5273
5794
  "kind": "boolean",
5274
5795
  "placeholder": "true"
5275
5796
  },
5797
+ {
5798
+ "name": "prompt_context",
5799
+ "required": false,
5800
+ "kind": "string",
5801
+ "enum": [
5802
+ "none",
5803
+ "target_site"
5804
+ ],
5805
+ "placeholder": "none"
5806
+ },
5276
5807
  {
5277
5808
  "name": "mcp_servers",
5278
5809
  "required": false,
@@ -5787,6 +6318,48 @@ export const OPERATIONS = [
5787
6318
  "command": "ora study unpublish <id>"
5788
6319
  }
5789
6320
  },
6321
+ {
6322
+ "id": "study.updateDetails",
6323
+ "commandPath": [
6324
+ "study",
6325
+ "update-details"
6326
+ ],
6327
+ "method": "patch",
6328
+ "pathTemplate": "/study/v1/studies/{id}",
6329
+ "summary": "Edit a study's display name and/or description at ANY status, published included — a correction or a disclosure written after launch reaches the published report (the name as its heading, the description in full as the publisher's note) within the page's normal cache revalidation; the page title and meta description derive from the subject, not from these fields. Changes those two fields and nothing else: the slug (so every shared link stays live), the publication date, the sharing mode and token, the corpus and every grade are untouched. Each field follows study.create's own rule; send at least one. The response's `shareToken` is null — study.get carries a link study's token",
6330
+ "scope": "study.studies:write",
6331
+ "product": "study",
6332
+ "pathParams": [
6333
+ {
6334
+ "name": "id"
6335
+ }
6336
+ ],
6337
+ "queryParams": [],
6338
+ "hasBody": true,
6339
+ "bodyRequired": true,
6340
+ "isStream": false,
6341
+ "bodyFields": [
6342
+ {
6343
+ "name": "name",
6344
+ "required": false,
6345
+ "kind": "string",
6346
+ "description": "Display name for the study (≤160 chars).",
6347
+ "placeholder": "<name>"
6348
+ },
6349
+ {
6350
+ "name": "description",
6351
+ "required": false,
6352
+ "kind": "string",
6353
+ "description": "Optional longer description shown on the study card.",
6354
+ "placeholder": "Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration."
6355
+ }
6356
+ ],
6357
+ "watchable": false,
6358
+ "help": {
6359
+ "command": "ora study update-details 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
6360
+ "body": "{\n \"description\": \"Disclosure (added after publication): one harness ran with its default backend, which has no shell or HTTP tool, so its row measures that configuration.\"\n}"
6361
+ }
6362
+ },
5790
6363
  {
5791
6364
  "id": "study.updateLenses",
5792
6365
  "commandPath": [