@ora-ai/cli 0.1.0 → 0.2.1

package/dist/generated/operations.js

@@ -1,9 +1,9 @@
 // GENERATED by scripts/generate.mjs — DO NOT EDIT.
-// Source: openapi/ora-public.openapi.json (sha256 d1aeca5bf5255cdfbdf7836aace5a10fc4242ce23a30dd16761092e8383291b0)
+// Source: openapi/ora-public.openapi.json (sha256 352c6714b3c9a03fd9709b5d06bd71c0fd7d8e7932375de27624661484bb76f5)
 // Regenerate: yarn nx run ora-cli:generate
 export const SPEC = {
     "version": "1.0.0",
-    "contentHash": "d1aeca5bf5255cdfbdf7836aace5a10fc4242ce23a30dd16761092e8383291b0",
+    "contentHash": "352c6714b3c9a03fd9709b5d06bd71c0fd7d8e7932375de27624661484bb76f5",
     "server": "https://api.agentfront.sh",
     "bin": "ora"
 };
@@ -210,6 +210,394 @@ export const OPERATIONS = [
             "command": "ora analytics overview"
         }
     },
+    {
+        "id": "benchmarks.create",
+        "commandPath": [
+            "benchmarks",
+            "create"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/benchmarks",
+        "summary": "Create a benchmark and start async task generation (poll get until status 'ready', then launch)",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [],
+        "queryParams": [],
+        "hasBody": true,
+        "bodyRequired": true,
+        "isStream": false,
+        "bodyFields": [
+            {
+                "name": "name",
+                "required": true,
+                "kind": "string",
+                "description": "Display name for the benchmark (≤160 chars).",
+                "placeholder": "tavily-search A/B"
+            },
+            {
+                "name": "description",
+                "required": false,
+                "kind": "string",
+                "description": "Optional longer description shown on the benchmark card.",
+                "placeholder": "<description>"
+            },
+            {
+                "name": "subjectKind",
+                "required": false,
+                "kind": "string",
+                "enum": [
+                    "skill",
+                    "tools",
+                    "addon",
+                    "custom"
+                ],
+                "description": "What the benchmark measures — defaults to 'skill'. See the enum description for each kind's rules.",
+                "placeholder": "addon"
+            },
+            {
+                "name": "skillRef",
+                "required": false,
+                "kind": "string",
+                "description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
+                "placeholder": "<skillRef>"
+            },
+            {
+                "name": "subjectName",
+                "required": false,
+                "kind": "string",
+                "description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required iff subjectKind ≠ 'skill'; forbidden for 'skill' (the skill's own name is used).",
+                "placeholder": "Tavily search"
+            },
+            {
+                "name": "armConfig",
+                "required": false,
+                "kind": "object",
+                "description": "Frozen per-arm run-payload deltas. Required for subjectKind 'custom' and comparisonMode 'ab_custom'; for 'tools'/'addon' subjects the treatment side is derived when omitted.",
+                "placeholder": "{\"treatment\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}"
+            },
+            {
+                "name": "comparisonMode",
+                "required": false,
+                "kind": "string",
+                "enum": [
+                    "ab_vs_none",
+                    "ab_custom",
+                    "single"
+                ],
+                "description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
+                "placeholder": "ab_vs_none"
+            },
+            {
+                "name": "baselineLabel",
+                "required": false,
+                "kind": "string",
+                "description": "Baseline arm's display label (e.g. 'REST API'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
+                "placeholder": "<baselineLabel>"
+            },
+            {
+                "name": "treatmentLabel",
+                "required": false,
+                "kind": "string",
+                "description": "Treatment arm's display label (e.g. 'Remote MCP'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
+                "placeholder": "<treatmentLabel>"
+            },
+            {
+                "name": "taskCount",
+                "required": false,
+                "kind": "number",
+                "description": "How many tasks to auto-generate (1–50, default 5). Runs launched = taskCount × matrix cells × arms (2 arms unless 'single').",
+                "placeholder": "5"
+            },
+            {
+                "name": "matrix",
+                "required": true,
+                "kind": "array",
+                "description": "The (harness, model) cells to fan out over — 1 to 12 cells.",
+                "placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"},{\"harness\":\"codex\",\"model\":\"gpt-5.5\"}]"
+            },
+            {
+                "name": "promise",
+                "required": false,
+                "kind": "string",
+                "description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); REQUIRED for every other kind.",
+                "placeholder": "Web search via Tavily produces fresher, better-sourced answers than no web access"
+            }
+        ],
+        "watchable": true,
+        "help": {
+            "command": "ora benchmarks create --file run.json --watch",
+            "body": "{\n  \"name\": \"tavily-search A/B\",\n  \"subjectKind\": \"addon\",\n  \"subjectName\": \"Tavily search\",\n  \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n  \"armConfig\": {\n    \"treatment\": {\n      \"addons\": {\n        \"tavily\": {\n          \"secretId\": \"<org-secret-uuid>\"\n        }\n      }\n    }\n  },\n  \"taskCount\": 5,\n  \"matrix\": [\n    {\n      \"harness\": \"claude-code\",\n      \"model\": \"claude-sonnet-5\"\n    },\n    {\n      \"harness\": \"codex\",\n      \"model\": \"gpt-5.5\"\n    }\n  ]\n}"
+        }
+    },
+    {
+        "id": "benchmarks.delete",
+        "commandPath": [
+            "benchmarks",
+            "delete"
+        ],
+        "method": "delete",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}",
+        "summary": "Delete a benchmark and its tasks/facts (409 while running)",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks delete <id>"
+        }
+    },
+    {
+        "id": "benchmarks.generate",
+        "commandPath": [
+            "benchmarks",
+            "generate"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/generate",
+        "summary": "Regenerate the task corpus (status ready/failed only — wipes existing tasks and their rubrics)",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": true,
+        "help": {
+            "command": "ora benchmarks generate <id> --watch"
+        }
+    },
+    {
+        "id": "benchmarks.get",
+        "commandPath": [
+            "benchmarks",
+            "get"
+        ],
+        "method": "get",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}",
+        "summary": "Benchmark detail: status to poll, tasks + rubrics, per-run facts, live aggregate + verdict",
+        "scope": "benchmarks:read",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks get 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
+        }
+    },
+    {
+        "id": "benchmarks.launch",
+        "commandPath": [
+            "benchmarks",
+            "launch"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/launch",
+        "summary": "Launch: fan out task × matrix-cell × arm runs (async — poll get until status 'completed')",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": true,
+        "help": {
+            "command": "ora benchmarks launch <id> --watch"
+        }
+    },
+    {
+        "id": "benchmarks.list",
+        "commandPath": [
+            "benchmarks",
+            "list"
+        ],
+        "method": "get",
+        "pathTemplate": "/experiment/v1/benchmarks",
+        "summary": "List the org's benchmarks (id, status, verdict, subject)",
+        "scope": "benchmarks:read",
+        "product": "experiment",
+        "pathParams": [],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks list"
+        }
+    },
+    {
+        "id": "benchmarks.publish",
+        "commandPath": [
+            "benchmarks",
+            "publish"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/publish",
+        "summary": "Publish a completed benchmark to the anonymous public catalog (idempotent slug upsert)",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": true,
+        "bodyRequired": true,
+        "isStream": false,
+        "bodyFields": [
+            {
+                "name": "slug",
+                "required": false,
+                "kind": "string",
+                "description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the benchmark's own generated slug. 409 benchmark_slug_taken if another published benchmark holds it.",
+                "placeholder": "tavily-search-ab"
+            }
+        ],
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks publish 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
+            "body": "{\n  \"slug\": \"tavily-search-ab\"\n}"
+        }
+    },
+    {
+        "id": "benchmarks.regrade",
+        "commandPath": [
+            "benchmarks",
+            "regrade"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/regrade",
+        "summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed benchmark",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": true,
+        "help": {
+            "command": "ora benchmarks regrade <id> --watch"
+        }
+    },
+    {
+        "id": "benchmarks.stream",
+        "commandPath": [
+            "benchmarks",
+            "stream"
+        ],
+        "method": "get",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/stream",
+        "summary": "Live SSE stream: status transitions, per-run facts, grading, publish state (terminal status closes it)",
+        "scope": "benchmarks:read",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": true,
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks stream 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
+        }
+    },
+    {
+        "id": "benchmarks.unpublish",
+        "commandPath": [
+            "benchmarks",
+            "unpublish"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/unpublish",
+        "summary": "Remove a benchmark from the public catalog (data kept; re-publish any time)",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks unpublish <id>"
+        }
+    },
+    {
+        "id": "benchmarks.updateTask",
+        "commandPath": [
+            "benchmarks",
+            "update-task"
+        ],
+        "method": "put",
+        "pathTemplate": "/experiment/v1/benchmarks/{id}/tasks/{taskId}",
+        "summary": "Edit a generated task's prompt before launch (status 'ready' only)",
+        "scope": "benchmarks:write",
+        "product": "experiment",
+        "pathParams": [
+            {
+                "name": "id"
+            },
+            {
+                "name": "taskId"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": true,
+        "bodyRequired": true,
+        "isStream": false,
+        "bodyFields": [
+            {
+                "name": "prompt",
+                "required": true,
+                "kind": "string",
+                "description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the benchmark is 'ready' (pre-launch).",
+                "placeholder": "Research the three largest EU battery-storage tenders announced this quarter and cite each source."
+            }
+        ],
+        "watchable": false,
+        "help": {
+            "command": "ora benchmarks update-task 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 8a9b0c1d-2e3f-4a5b-8c7d-6e5f4a3b2c1d --file run.json",
+            "body": "{\n  \"prompt\": \"Research the three largest EU battery-storage tenders announced this quarter and cite each source.\"\n}"
+        }
+    },
     {
         "id": "domains.add",
         "commandPath": [
@@ -286,6 +674,27 @@ export const OPERATIONS = [
             "command": "ora domains remove <id>"
         }
     },
+    {
+        "id": "experiment.catalog",
+        "commandPath": [
+            "experiment",
+            "catalog"
+        ],
+        "method": "get",
+        "pathTemplate": "/experiment/v1/catalog",
+        "summary": "Harness × model catalog: valid matrix cells, defaults, modes, and withheld harnesses with reasons",
+        "scope": null,
+        "product": "experiment",
+        "pathParams": [],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora experiment catalog"
+        }
+    },
     {
         "id": "experiment.runs.create",
         "commandPath": [
@@ -587,6 +996,93 @@ export const OPERATIONS = [
             "command": "ora experiment runs trajectory <id>"
         }
     },
+    {
+        "id": "experiment.secrets.create",
+        "commandPath": [
+            "experiment",
+            "secrets",
+            "create"
+        ],
+        "method": "post",
+        "pathTemplate": "/experiment/v1/secrets",
+        "summary": "Create an org secret (encrypted at rest; the value is never readable back through this surface)",
+        "scope": "secrets:write",
+        "product": "platform",
+        "pathParams": [],
+        "queryParams": [],
+        "hasBody": true,
+        "bodyRequired": true,
+        "isStream": false,
+        "bodyFields": [
+            {
+                "name": "name",
+                "required": true,
+                "kind": "string",
+                "description": "Secret name (letters, numbers, '.', '_', '-') — how add-on configs and env bindings will refer to it in the UI; the wire references use the returned id.",
+                "placeholder": "MONDAY_API_KEY"
+            },
+            {
+                "name": "value",
+                "required": true,
+                "kind": "string",
+                "description": "The secret value (an API token, key, …). Encrypted at rest; NEVER echoed back by any read this surface offers — reads return metadata only.",
+                "placeholder": "<the token — never logged, never echoed>"
+            }
+        ],
+        "watchable": false,
+        "help": {
+            "command": "ora experiment secrets create --file run.json",
+            "body": "{\n  \"name\": \"MONDAY_API_KEY\",\n  \"value\": \"<the token — never logged, never echoed>\"\n}"
+        }
+    },
+    {
+        "id": "experiment.secrets.delete",
+        "commandPath": [
+            "experiment",
+            "secrets",
+            "delete"
+        ],
+        "method": "delete",
+        "pathTemplate": "/experiment/v1/secrets/{id}",
+        "summary": "Delete an org secret (runs already launched keep nothing; future boot-fetches fail closed)",
+        "scope": "secrets:write",
+        "product": "platform",
+        "pathParams": [
+            {
+                "name": "id"
+            }
+        ],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora experiment secrets delete <id>"
+        }
+    },
+    {
+        "id": "experiment.secrets.list",
+        "commandPath": [
+            "experiment",
+            "secrets",
+            "list"
+        ],
+        "method": "get",
+        "pathTemplate": "/experiment/v1/secrets",
+        "summary": "List the org's secrets (metadata only — ids for armConfig/envBindings references)",
+        "scope": "secrets:read",
+        "product": "platform",
+        "pathParams": [],
+        "queryParams": [],
+        "hasBody": false,
+        "bodyRequired": false,
+        "isStream": false,
+        "watchable": false,
+        "help": {
+            "command": "ora experiment secrets list"
+        }
+    },
     {
         "id": "intents.addPrompt",
         "commandPath": [