@ora-ai/cli 0.7.4 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -6
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +3260 -320
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +3872 -60
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/package.json +2 -2
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// GENERATED by scripts/generate.mjs — DO NOT EDIT.
|
|
2
|
-
// Source: openapi/ora-public.openapi.json (sha256
|
|
2
|
+
// Source: openapi/ora-public.openapi.json (sha256 1d3b25b4c612e53f1e666e93e37bfdd3effc5de3f02ff7967c69cd2341c735c1)
|
|
3
3
|
// Regenerate: yarn nx run ora-cli:generate
|
|
4
4
|
export const SPEC = {
|
|
5
5
|
"version": "1.0.0",
|
|
6
|
-
"contentHash": "
|
|
6
|
+
"contentHash": "1d3b25b4c612e53f1e666e93e37bfdd3effc5de3f02ff7967c69cd2341c735c1",
|
|
7
7
|
"server": "https://api.agentfront.sh",
|
|
8
8
|
"bin": "ora"
|
|
9
9
|
};
|
|
@@ -218,7 +218,7 @@ export const OPERATIONS = [
|
|
|
218
218
|
],
|
|
219
219
|
"method": "post",
|
|
220
220
|
"pathTemplate": "/experiment/v1/benchmarks/convert/webmcp",
|
|
221
|
-
"summary": "Convert a webmcp-evals suite JSON into ora
|
|
221
|
+
"summary": "Convert a webmcp-evals suite JSON into ora study tasks + 'evals'-lens expectations — a PURE transform, nothing persisted; resubmit the returned `tasks` through benchmarks.create's `tasks` field",
|
|
222
222
|
"scope": "benchmarks:write",
|
|
223
223
|
"product": "experiment",
|
|
224
224
|
"pathParams": [],
|
|
@@ -256,7 +256,7 @@ export const OPERATIONS = [
|
|
|
256
256
|
],
|
|
257
257
|
"method": "post",
|
|
258
258
|
"pathTemplate": "/experiment/v1/benchmarks",
|
|
259
|
-
"summary": "Create a
|
|
259
|
+
"summary": "Create a study (status 'draft', ZERO LLM spend — an authored `tasks` corpus instead lands 'ready' synchronously); approve generation via POST .../generate",
|
|
260
260
|
"scope": "benchmarks:write",
|
|
261
261
|
"product": "experiment",
|
|
262
262
|
"pathParams": [],
|
|
@@ -269,14 +269,14 @@ export const OPERATIONS = [
|
|
|
269
269
|
"name": "name",
|
|
270
270
|
"required": true,
|
|
271
271
|
"kind": "string",
|
|
272
|
-
"description": "Display name for the
|
|
272
|
+
"description": "Display name for the study (≤160 chars).",
|
|
273
273
|
"placeholder": "tavily-search A/B"
|
|
274
274
|
},
|
|
275
275
|
{
|
|
276
276
|
"name": "description",
|
|
277
277
|
"required": false,
|
|
278
278
|
"kind": "string",
|
|
279
|
-
"description": "Optional longer description shown on the
|
|
279
|
+
"description": "Optional longer description shown on the study card.",
|
|
280
280
|
"placeholder": "<description>"
|
|
281
281
|
},
|
|
282
282
|
{
|
|
@@ -287,9 +287,10 @@ export const OPERATIONS = [
|
|
|
287
287
|
"skill",
|
|
288
288
|
"tools",
|
|
289
289
|
"addon",
|
|
290
|
-
"custom"
|
|
290
|
+
"custom",
|
|
291
|
+
"benchmark"
|
|
291
292
|
],
|
|
292
|
-
"description": "What the
|
|
293
|
+
"description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
|
|
293
294
|
"placeholder": "addon"
|
|
294
295
|
},
|
|
295
296
|
{
|
|
@@ -299,11 +300,18 @@ export const OPERATIONS = [
|
|
|
299
300
|
"description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
|
|
300
301
|
"placeholder": "<skillRef>"
|
|
301
302
|
},
|
|
303
|
+
{
|
|
304
|
+
"name": "definitionVersionId",
|
|
305
|
+
"required": false,
|
|
306
|
+
"kind": "string",
|
|
307
|
+
"description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
|
|
308
|
+
"placeholder": "<definitionVersionId>"
|
|
309
|
+
},
|
|
302
310
|
{
|
|
303
311
|
"name": "subjectName",
|
|
304
312
|
"required": false,
|
|
305
313
|
"kind": "string",
|
|
306
|
-
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required
|
|
314
|
+
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
|
|
307
315
|
"placeholder": "Tavily search"
|
|
308
316
|
},
|
|
309
317
|
{
|
|
@@ -320,11 +328,18 @@ export const OPERATIONS = [
|
|
|
320
328
|
"description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
|
|
321
329
|
"placeholder": "[\"judge\",\"evals\"]"
|
|
322
330
|
},
|
|
331
|
+
{
|
|
332
|
+
"name": "judgeSupplementary",
|
|
333
|
+
"required": false,
|
|
334
|
+
"kind": "boolean",
|
|
335
|
+
"description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
|
|
336
|
+
"placeholder": "true"
|
|
337
|
+
},
|
|
323
338
|
{
|
|
324
339
|
"name": "proposeExpectations",
|
|
325
340
|
"required": false,
|
|
326
341
|
"kind": "boolean",
|
|
327
|
-
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the
|
|
342
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
|
|
328
343
|
"placeholder": "true"
|
|
329
344
|
},
|
|
330
345
|
{
|
|
@@ -339,6 +354,17 @@ export const OPERATIONS = [
|
|
|
339
354
|
"description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
|
|
340
355
|
"placeholder": "ab_vs_none"
|
|
341
356
|
},
|
|
357
|
+
{
|
|
358
|
+
"name": "subjectBackend",
|
|
359
|
+
"required": false,
|
|
360
|
+
"kind": "string",
|
|
361
|
+
"enum": [
|
|
362
|
+
"model",
|
|
363
|
+
"harness"
|
|
364
|
+
],
|
|
365
|
+
"description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
|
|
366
|
+
"placeholder": "model"
|
|
367
|
+
},
|
|
342
368
|
{
|
|
343
369
|
"name": "baselineLabel",
|
|
344
370
|
"required": false,
|
|
@@ -357,7 +383,7 @@ export const OPERATIONS = [
|
|
|
357
383
|
"name": "taskCount",
|
|
358
384
|
"required": false,
|
|
359
385
|
"kind": "number",
|
|
360
|
-
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at
|
|
386
|
+
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
|
|
361
387
|
"placeholder": "1"
|
|
362
388
|
},
|
|
363
389
|
{
|
|
@@ -378,9 +404,16 @@ export const OPERATIONS = [
|
|
|
378
404
|
"name": "tasks",
|
|
379
405
|
"required": false,
|
|
380
406
|
"kind": "array",
|
|
381
|
-
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the
|
|
407
|
+
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
|
|
382
408
|
"placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
|
|
383
409
|
},
|
|
410
|
+
{
|
|
411
|
+
"name": "subset",
|
|
412
|
+
"required": false,
|
|
413
|
+
"kind": "object",
|
|
414
|
+
"description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
|
|
415
|
+
"placeholder": "{}"
|
|
416
|
+
},
|
|
384
417
|
{
|
|
385
418
|
"name": "matrix",
|
|
386
419
|
"required": true,
|
|
@@ -392,7 +425,7 @@ export const OPERATIONS = [
|
|
|
392
425
|
"name": "promise",
|
|
393
426
|
"required": false,
|
|
394
427
|
"kind": "string",
|
|
395
|
-
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); REQUIRED for every other kind.",
|
|
428
|
+
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
|
|
396
429
|
"placeholder": "Web search via Tavily produces fresher, better-sourced answers than no web access"
|
|
397
430
|
}
|
|
398
431
|
],
|
|
@@ -410,7 +443,7 @@ export const OPERATIONS = [
|
|
|
410
443
|
],
|
|
411
444
|
"method": "delete",
|
|
412
445
|
"pathTemplate": "/experiment/v1/benchmarks/{id}",
|
|
413
|
-
"summary": "Delete a
|
|
446
|
+
"summary": "Delete a study and its tasks/facts (409 while running)",
|
|
414
447
|
"scope": "benchmarks:write",
|
|
415
448
|
"product": "experiment",
|
|
416
449
|
"pathParams": [
|
|
@@ -470,7 +503,7 @@ export const OPERATIONS = [
|
|
|
470
503
|
],
|
|
471
504
|
"method": "get",
|
|
472
505
|
"pathTemplate": "/experiment/v1/benchmarks/{id}",
|
|
473
|
-
"summary": "
|
|
506
|
+
"summary": "Study detail: status to poll, tasks + rubrics, per-run facts, live aggregate + verdict",
|
|
474
507
|
"scope": "benchmarks:read",
|
|
475
508
|
"product": "experiment",
|
|
476
509
|
"pathParams": [
|
|
@@ -520,7 +553,7 @@ export const OPERATIONS = [
|
|
|
520
553
|
],
|
|
521
554
|
"method": "get",
|
|
522
555
|
"pathTemplate": "/experiment/v1/benchmarks",
|
|
523
|
-
"summary": "List the org's
|
|
556
|
+
"summary": "List the org's studies (id, status, verdict, subject)",
|
|
524
557
|
"scope": "benchmarks:read",
|
|
525
558
|
"product": "experiment",
|
|
526
559
|
"pathParams": [],
|
|
@@ -541,7 +574,7 @@ export const OPERATIONS = [
|
|
|
541
574
|
],
|
|
542
575
|
"method": "get",
|
|
543
576
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/preview",
|
|
544
|
-
"summary": "Preview the PUBLISHED page's exact payload before publishing — the same projection, aggregate and behavior block
|
|
577
|
+
"summary": "Preview the PUBLISHED page's exact payload before publishing — the same projection, aggregate and behavior block publicStudies.get will serve, org-scoped and available at any status (no visibility requirement)",
|
|
545
578
|
"scope": "benchmarks:read",
|
|
546
579
|
"product": "experiment",
|
|
547
580
|
"pathParams": [
|
|
@@ -566,7 +599,7 @@ export const OPERATIONS = [
|
|
|
566
599
|
],
|
|
567
600
|
"method": "post",
|
|
568
601
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/publish",
|
|
569
|
-
"summary": "Publish a completed
|
|
602
|
+
"summary": "Publish a completed study under a shareable mode: 'public' (listed in the anonymous catalog, the default) or — where the deployment supports sharing — 'link' (unlisted, openable only with the returned share token). Idempotent slug upsert.",
|
|
570
603
|
"scope": "benchmarks:write",
|
|
571
604
|
"product": "experiment",
|
|
572
605
|
"pathParams": [
|
|
@@ -583,8 +616,21 @@ export const OPERATIONS = [
|
|
|
583
616
|
"name": "slug",
|
|
584
617
|
"required": false,
|
|
585
618
|
"kind": "string",
|
|
586
|
-
"description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the
|
|
619
|
+
"description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the study's own generated slug. 409 study_slug_taken if another published study holds it.",
|
|
587
620
|
"placeholder": "tavily-search-ab"
|
|
621
|
+
},
|
|
622
|
+
{
|
|
623
|
+
"name": "visibility",
|
|
624
|
+
"required": false,
|
|
625
|
+
"kind": "string",
|
|
626
|
+
"enum": [
|
|
627
|
+
"private",
|
|
628
|
+
"org",
|
|
629
|
+
"link",
|
|
630
|
+
"public"
|
|
631
|
+
],
|
|
632
|
+
"description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
|
|
633
|
+
"placeholder": "private"
|
|
588
634
|
}
|
|
589
635
|
],
|
|
590
636
|
"watchable": false,
|
|
@@ -626,7 +672,7 @@ export const OPERATIONS = [
|
|
|
626
672
|
],
|
|
627
673
|
"method": "post",
|
|
628
674
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/regrade",
|
|
629
|
-
"summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed
|
|
675
|
+
"summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
|
|
630
676
|
"scope": "benchmarks:write",
|
|
631
677
|
"product": "experiment",
|
|
632
678
|
"pathParams": [
|
|
@@ -676,7 +722,7 @@ export const OPERATIONS = [
|
|
|
676
722
|
],
|
|
677
723
|
"method": "post",
|
|
678
724
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/unpublish",
|
|
679
|
-
"summary": "Remove a
|
|
725
|
+
"summary": "Remove a study from the public catalog (data kept; re-publish any time)",
|
|
680
726
|
"scope": "benchmarks:write",
|
|
681
727
|
"product": "experiment",
|
|
682
728
|
"pathParams": [
|
|
@@ -701,7 +747,7 @@ export const OPERATIONS = [
|
|
|
701
747
|
],
|
|
702
748
|
"method": "put",
|
|
703
749
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/lenses",
|
|
704
|
-
"summary": "Change which grading lenses a
|
|
750
|
+
"summary": "Change which grading lenses a study runs under before launch (status 'ready' only) — the complete replacement set, e.g. add 'evals' to a judge-only study so its tasks can carry expectations",
|
|
705
751
|
"scope": "benchmarks:write",
|
|
706
752
|
"product": "experiment",
|
|
707
753
|
"pathParams": [
|
|
@@ -756,14 +802,14 @@ export const OPERATIONS = [
|
|
|
756
802
|
"name": "prompt",
|
|
757
803
|
"required": true,
|
|
758
804
|
"kind": "string",
|
|
759
|
-
"description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the
|
|
805
|
+
"description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the study is 'ready' (pre-launch).",
|
|
760
806
|
"placeholder": "Research the three largest EU battery-storage tenders announced this quarter and cite each source."
|
|
761
807
|
},
|
|
762
808
|
{
|
|
763
809
|
"name": "expectations",
|
|
764
810
|
"required": false,
|
|
765
811
|
"kind": "array",
|
|
766
|
-
"description": "Replace this task's 'evals'-lens expectations (see
|
|
812
|
+
"description": "Replace this task's 'evals'-lens expectations (see studyCreateBody.tasks[].expectations). Omitted = expectations unchanged. REQUIRES the study's `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); validated the same way as create.",
|
|
767
813
|
"placeholder": "[]"
|
|
768
814
|
}
|
|
769
815
|
],
|
|
@@ -859,7 +905,7 @@ export const OPERATIONS = [
|
|
|
859
905
|
"pathTemplate": "/experiment/v1/catalog",
|
|
860
906
|
"summary": "Harness × model catalog: valid matrix cells, defaults, modes, and withheld harnesses with reasons",
|
|
861
907
|
"scope": null,
|
|
862
|
-
"product": "
|
|
908
|
+
"product": "platform",
|
|
863
909
|
"pathParams": [],
|
|
864
910
|
"queryParams": [],
|
|
865
911
|
"hasBody": false,
|
|
@@ -870,6 +916,262 @@ export const OPERATIONS = [
|
|
|
870
916
|
"command": "ora experiment catalog"
|
|
871
917
|
}
|
|
872
918
|
},
|
|
919
|
+
{
|
|
920
|
+
"id": "experiment.images.builds.cancel",
|
|
921
|
+
"commandPath": [
|
|
922
|
+
"experiment",
|
|
923
|
+
"images",
|
|
924
|
+
"builds",
|
|
925
|
+
"cancel"
|
|
926
|
+
],
|
|
927
|
+
"method": "post",
|
|
928
|
+
"pathTemplate": "/experiment/v1/images/builds/{id}/cancel",
|
|
929
|
+
"summary": "Cancel an in-flight image build",
|
|
930
|
+
"scope": "images:write",
|
|
931
|
+
"product": "platform",
|
|
932
|
+
"pathParams": [
|
|
933
|
+
{
|
|
934
|
+
"name": "id"
|
|
935
|
+
}
|
|
936
|
+
],
|
|
937
|
+
"queryParams": [],
|
|
938
|
+
"hasBody": false,
|
|
939
|
+
"bodyRequired": false,
|
|
940
|
+
"isStream": false,
|
|
941
|
+
"watchable": false,
|
|
942
|
+
"help": {
|
|
943
|
+
"command": "ora experiment images builds cancel <id>"
|
|
944
|
+
}
|
|
945
|
+
},
|
|
946
|
+
{
|
|
947
|
+
"id": "experiment.images.builds.create",
|
|
948
|
+
"commandPath": [
|
|
949
|
+
"experiment",
|
|
950
|
+
"images",
|
|
951
|
+
"builds",
|
|
952
|
+
"create"
|
|
953
|
+
],
|
|
954
|
+
"method": "post",
|
|
955
|
+
"pathTemplate": "/experiment/v1/images/builds",
|
|
956
|
+
"summary": "Submit an image build (idempotent on name + source + Dockerfile + build args)",
|
|
957
|
+
"scope": "images:write",
|
|
958
|
+
"product": "platform",
|
|
959
|
+
"pathParams": [],
|
|
960
|
+
"queryParams": [],
|
|
961
|
+
"hasBody": true,
|
|
962
|
+
"bodyRequired": true,
|
|
963
|
+
"isStream": false,
|
|
964
|
+
"bodyFields": [
|
|
965
|
+
{
|
|
966
|
+
"name": "name",
|
|
967
|
+
"required": true,
|
|
968
|
+
"kind": "string",
|
|
969
|
+
"placeholder": "<name>"
|
|
970
|
+
},
|
|
971
|
+
{
|
|
972
|
+
"name": "source",
|
|
973
|
+
"required": true,
|
|
974
|
+
"kind": "string",
|
|
975
|
+
"placeholder": "<source>"
|
|
976
|
+
},
|
|
977
|
+
{
|
|
978
|
+
"name": "dockerfile",
|
|
979
|
+
"required": false,
|
|
980
|
+
"kind": "object",
|
|
981
|
+
"placeholder": "{}"
|
|
982
|
+
},
|
|
983
|
+
{
|
|
984
|
+
"name": "buildArgs",
|
|
985
|
+
"required": false,
|
|
986
|
+
"kind": "object",
|
|
987
|
+
"placeholder": "{}"
|
|
988
|
+
},
|
|
989
|
+
{
|
|
990
|
+
"name": "platform",
|
|
991
|
+
"required": false,
|
|
992
|
+
"kind": "string",
|
|
993
|
+
"placeholder": "<platform>"
|
|
994
|
+
}
|
|
995
|
+
],
|
|
996
|
+
"watchable": false,
|
|
997
|
+
"help": {
|
|
998
|
+
"command": "ora experiment images builds create --file run.json",
|
|
999
|
+
"body": "{\n \"name\": \"<name>\",\n \"source\": \"<source>\"\n}"
|
|
1000
|
+
}
|
|
1001
|
+
},
|
|
1002
|
+
{
|
|
1003
|
+
"id": "experiment.images.builds.get",
|
|
1004
|
+
"commandPath": [
|
|
1005
|
+
"experiment",
|
|
1006
|
+
"images",
|
|
1007
|
+
"builds",
|
|
1008
|
+
"get"
|
|
1009
|
+
],
|
|
1010
|
+
"method": "get",
|
|
1011
|
+
"pathTemplate": "/experiment/v1/images/builds/{id}",
|
|
1012
|
+
"summary": "Get one image build",
|
|
1013
|
+
"scope": "images:read",
|
|
1014
|
+
"product": "platform",
|
|
1015
|
+
"pathParams": [
|
|
1016
|
+
{
|
|
1017
|
+
"name": "id"
|
|
1018
|
+
}
|
|
1019
|
+
],
|
|
1020
|
+
"queryParams": [],
|
|
1021
|
+
"hasBody": false,
|
|
1022
|
+
"bodyRequired": false,
|
|
1023
|
+
"isStream": false,
|
|
1024
|
+
"watchable": false,
|
|
1025
|
+
"help": {
|
|
1026
|
+
"command": "ora experiment images builds get <id>"
|
|
1027
|
+
}
|
|
1028
|
+
},
|
|
1029
|
+
{
|
|
1030
|
+
"id": "experiment.images.builds.list",
|
|
1031
|
+
"commandPath": [
|
|
1032
|
+
"experiment",
|
|
1033
|
+
"images",
|
|
1034
|
+
"builds",
|
|
1035
|
+
"list"
|
|
1036
|
+
],
|
|
1037
|
+
"method": "get",
|
|
1038
|
+
"pathTemplate": "/experiment/v1/images/builds",
|
|
1039
|
+
"summary": "List the org's image builds, newest first",
|
|
1040
|
+
"scope": "images:read",
|
|
1041
|
+
"product": "platform",
|
|
1042
|
+
"pathParams": [],
|
|
1043
|
+
"queryParams": [
|
|
1044
|
+
{
|
|
1045
|
+
"name": "phase",
|
|
1046
|
+
"flag": "phase",
|
|
1047
|
+
"optionKey": "phase",
|
|
1048
|
+
"required": false,
|
|
1049
|
+
"kind": "string",
|
|
1050
|
+
"enum": [
|
|
1051
|
+
"pending",
|
|
1052
|
+
"running",
|
|
1053
|
+
"succeeded",
|
|
1054
|
+
"failed",
|
|
1055
|
+
"cancelled"
|
|
1056
|
+
]
|
|
1057
|
+
},
|
|
1058
|
+
{
|
|
1059
|
+
"name": "name",
|
|
1060
|
+
"flag": "name",
|
|
1061
|
+
"optionKey": "name",
|
|
1062
|
+
"required": false,
|
|
1063
|
+
"kind": "string"
|
|
1064
|
+
},
|
|
1065
|
+
{
|
|
1066
|
+
"name": "pageSize",
|
|
1067
|
+
"flag": "page-size",
|
|
1068
|
+
"optionKey": "pageSize",
|
|
1069
|
+
"required": false,
|
|
1070
|
+
"kind": "number"
|
|
1071
|
+
},
|
|
1072
|
+
{
|
|
1073
|
+
"name": "pageToken",
|
|
1074
|
+
"flag": "page-token",
|
|
1075
|
+
"optionKey": "pageToken",
|
|
1076
|
+
"required": false,
|
|
1077
|
+
"kind": "string"
|
|
1078
|
+
}
|
|
1079
|
+
],
|
|
1080
|
+
"hasBody": false,
|
|
1081
|
+
"bodyRequired": false,
|
|
1082
|
+
"isStream": false,
|
|
1083
|
+
"watchable": false,
|
|
1084
|
+
"help": {
|
|
1085
|
+
"command": "ora experiment images builds list"
|
|
1086
|
+
}
|
|
1087
|
+
},
|
|
1088
|
+
{
|
|
1089
|
+
"id": "experiment.images.builds.stream",
|
|
1090
|
+
"commandPath": [
|
|
1091
|
+
"experiment",
|
|
1092
|
+
"images",
|
|
1093
|
+
"builds",
|
|
1094
|
+
"stream"
|
|
1095
|
+
],
|
|
1096
|
+
"method": "get",
|
|
1097
|
+
"pathTemplate": "/experiment/v1/images/builds/{id}/stream",
|
|
1098
|
+
"summary": "SSE stream of one image build's row changes",
|
|
1099
|
+
"scope": "images:read",
|
|
1100
|
+
"product": "platform",
|
|
1101
|
+
"pathParams": [
|
|
1102
|
+
{
|
|
1103
|
+
"name": "id"
|
|
1104
|
+
}
|
|
1105
|
+
],
|
|
1106
|
+
"queryParams": [],
|
|
1107
|
+
"hasBody": false,
|
|
1108
|
+
"bodyRequired": false,
|
|
1109
|
+
"isStream": true,
|
|
1110
|
+
"watchable": false,
|
|
1111
|
+
"help": {
|
|
1112
|
+
"command": "ora experiment images builds stream <id>"
|
|
1113
|
+
}
|
|
1114
|
+
},
|
|
1115
|
+
{
|
|
1116
|
+
"id": "experiment.images.runs.create",
|
|
1117
|
+
"commandPath": [
|
|
1118
|
+
"experiment",
|
|
1119
|
+
"images",
|
|
1120
|
+
"runs",
|
|
1121
|
+
"create"
|
|
1122
|
+
],
|
|
1123
|
+
"method": "post",
|
|
1124
|
+
"pathTemplate": "/experiment/v1/images/runs",
|
|
1125
|
+
"summary": "Launch a run (harness or environment) from a built or referenced image",
|
|
1126
|
+
"scope": "images:write",
|
|
1127
|
+
"product": "platform",
|
|
1128
|
+
"pathParams": [],
|
|
1129
|
+
"queryParams": [],
|
|
1130
|
+
"hasBody": true,
|
|
1131
|
+
"bodyRequired": true,
|
|
1132
|
+
"isStream": false,
|
|
1133
|
+
"bodyFields": [
|
|
1134
|
+
{
|
|
1135
|
+
"name": "buildId",
|
|
1136
|
+
"required": false,
|
|
1137
|
+
"kind": "string",
|
|
1138
|
+
"placeholder": "<buildId>"
|
|
1139
|
+
},
|
|
1140
|
+
{
|
|
1141
|
+
"name": "image",
|
|
1142
|
+
"required": false,
|
|
1143
|
+
"kind": "string",
|
|
1144
|
+
"placeholder": "<image>"
|
|
1145
|
+
},
|
|
1146
|
+
{
|
|
1147
|
+
"name": "mode",
|
|
1148
|
+
"required": true,
|
|
1149
|
+
"kind": "string",
|
|
1150
|
+
"enum": [
|
|
1151
|
+
"harness",
|
|
1152
|
+
"environment"
|
|
1153
|
+
],
|
|
1154
|
+
"placeholder": "harness"
|
|
1155
|
+
},
|
|
1156
|
+
{
|
|
1157
|
+
"name": "harness",
|
|
1158
|
+
"required": false,
|
|
1159
|
+
"kind": "string",
|
|
1160
|
+
"placeholder": "<harness>"
|
|
1161
|
+
},
|
|
1162
|
+
{
|
|
1163
|
+
"name": "task",
|
|
1164
|
+
"required": false,
|
|
1165
|
+
"kind": "string",
|
|
1166
|
+
"placeholder": "<task>"
|
|
1167
|
+
}
|
|
1168
|
+
],
|
|
1169
|
+
"watchable": false,
|
|
1170
|
+
"help": {
|
|
1171
|
+
"command": "ora experiment images runs create --file run.json",
|
|
1172
|
+
"body": "{\n \"mode\": \"harness\"\n}"
|
|
1173
|
+
}
|
|
1174
|
+
},
|
|
873
1175
|
{
|
|
874
1176
|
"id": "experiment.runs.cancel",
|
|
875
1177
|
"commandPath": [
|
|
@@ -1025,6 +1327,12 @@ export const OPERATIONS = [
|
|
|
1025
1327
|
"kind": "array",
|
|
1026
1328
|
"placeholder": "[]"
|
|
1027
1329
|
},
|
|
1330
|
+
{
|
|
1331
|
+
"name": "model_params",
|
|
1332
|
+
"required": false,
|
|
1333
|
+
"kind": "object",
|
|
1334
|
+
"placeholder": "{}"
|
|
1335
|
+
},
|
|
1028
1336
|
{
|
|
1029
1337
|
"name": "wait",
|
|
1030
1338
|
"required": false,
|
|
@@ -1354,16 +1662,17 @@ export const OPERATIONS = [
|
|
|
1354
1662
|
}
|
|
1355
1663
|
},
|
|
1356
1664
|
{
|
|
1357
|
-
"id": "
|
|
1665
|
+
"id": "experiment.skills.create",
|
|
1358
1666
|
"commandPath": [
|
|
1359
|
-
"
|
|
1667
|
+
"experiment",
|
|
1668
|
+
"skills",
|
|
1360
1669
|
"create"
|
|
1361
1670
|
],
|
|
1362
1671
|
"method": "post",
|
|
1363
|
-
"pathTemplate": "/
|
|
1364
|
-
"summary": "
|
|
1365
|
-
"scope": "
|
|
1366
|
-
"product": "
|
|
1672
|
+
"pathTemplate": "/experiment/v1/skills",
|
|
1673
|
+
"summary": "Create a skill (custom or git-imported)",
|
|
1674
|
+
"scope": "skills:write",
|
|
1675
|
+
"product": "platform",
|
|
1367
1676
|
"pathParams": [],
|
|
1368
1677
|
"queryParams": [],
|
|
1369
1678
|
"hasBody": true,
|
|
@@ -1371,19 +1680,188 @@ export const OPERATIONS = [
|
|
|
1371
1680
|
"isStream": false,
|
|
1372
1681
|
"bodyFields": [
|
|
1373
1682
|
{
|
|
1374
|
-
"name": "
|
|
1683
|
+
"name": "source",
|
|
1375
1684
|
"required": true,
|
|
1376
1685
|
"kind": "string",
|
|
1377
|
-
"placeholder": "
|
|
1686
|
+
"placeholder": "custom"
|
|
1378
1687
|
},
|
|
1379
1688
|
{
|
|
1380
|
-
"name": "
|
|
1689
|
+
"name": "name",
|
|
1381
1690
|
"required": true,
|
|
1382
1691
|
"kind": "string",
|
|
1383
|
-
"placeholder": "<
|
|
1692
|
+
"placeholder": "<name>"
|
|
1384
1693
|
},
|
|
1385
1694
|
{
|
|
1386
|
-
"name": "
|
|
1695
|
+
"name": "body",
|
|
1696
|
+
"required": true,
|
|
1697
|
+
"kind": "string",
|
|
1698
|
+
"placeholder": "<body>"
|
|
1699
|
+
},
|
|
1700
|
+
{
|
|
1701
|
+
"name": "visibility",
|
|
1702
|
+
"required": false,
|
|
1703
|
+
"kind": "string",
|
|
1704
|
+
"enum": [
|
|
1705
|
+
"user",
|
|
1706
|
+
"org"
|
|
1707
|
+
],
|
|
1708
|
+
"placeholder": "user"
|
|
1709
|
+
}
|
|
1710
|
+
],
|
|
1711
|
+
"watchable": false,
|
|
1712
|
+
"help": {
|
|
1713
|
+
"command": "ora experiment skills create --file run.json",
|
|
1714
|
+
"body": "{\n \"source\": \"custom\",\n \"name\": \"<name>\",\n \"body\": \"<body>\"\n}"
|
|
1715
|
+
}
|
|
1716
|
+
},
|
|
1717
|
+
{
|
|
1718
|
+
"id": "experiment.skills.delete",
|
|
1719
|
+
"commandPath": [
|
|
1720
|
+
"experiment",
|
|
1721
|
+
"skills",
|
|
1722
|
+
"delete"
|
|
1723
|
+
],
|
|
1724
|
+
"method": "delete",
|
|
1725
|
+
"pathTemplate": "/experiment/v1/skills/{id}",
|
|
1726
|
+
"summary": "Delete a skill",
|
|
1727
|
+
"scope": "skills:write",
|
|
1728
|
+
"product": "platform",
|
|
1729
|
+
"pathParams": [
|
|
1730
|
+
{
|
|
1731
|
+
"name": "id"
|
|
1732
|
+
}
|
|
1733
|
+
],
|
|
1734
|
+
"queryParams": [],
|
|
1735
|
+
"hasBody": false,
|
|
1736
|
+
"bodyRequired": false,
|
|
1737
|
+
"isStream": false,
|
|
1738
|
+
"watchable": false,
|
|
1739
|
+
"help": {
|
|
1740
|
+
"command": "ora experiment skills delete <id>"
|
|
1741
|
+
}
|
|
1742
|
+
},
|
|
1743
|
+
{
|
|
1744
|
+
"id": "experiment.skills.list",
|
|
1745
|
+
"commandPath": [
|
|
1746
|
+
"experiment",
|
|
1747
|
+
"skills",
|
|
1748
|
+
"list"
|
|
1749
|
+
],
|
|
1750
|
+
"method": "get",
|
|
1751
|
+
"pathTemplate": "/experiment/v1/skills",
|
|
1752
|
+
"summary": "List the caller's usable skills catalog",
|
|
1753
|
+
"scope": "skills:read",
|
|
1754
|
+
"product": "platform",
|
|
1755
|
+
"pathParams": [],
|
|
1756
|
+
"queryParams": [],
|
|
1757
|
+
"hasBody": false,
|
|
1758
|
+
"bodyRequired": false,
|
|
1759
|
+
"isStream": false,
|
|
1760
|
+
"watchable": false,
|
|
1761
|
+
"help": {
|
|
1762
|
+
"command": "ora experiment skills list"
|
|
1763
|
+
}
|
|
1764
|
+
},
|
|
1765
|
+
{
|
|
1766
|
+
"id": "experiment.skills.update",
|
|
1767
|
+
"commandPath": [
|
|
1768
|
+
"experiment",
|
|
1769
|
+
"skills",
|
|
1770
|
+
"update"
|
|
1771
|
+
],
|
|
1772
|
+
"method": "put",
|
|
1773
|
+
"pathTemplate": "/experiment/v1/skills/{id}",
|
|
1774
|
+
"summary": "Update a skill",
|
|
1775
|
+
"scope": "skills:write",
|
|
1776
|
+
"product": "platform",
|
|
1777
|
+
"pathParams": [
|
|
1778
|
+
{
|
|
1779
|
+
"name": "id"
|
|
1780
|
+
}
|
|
1781
|
+
],
|
|
1782
|
+
"queryParams": [],
|
|
1783
|
+
"hasBody": true,
|
|
1784
|
+
"bodyRequired": true,
|
|
1785
|
+
"isStream": false,
|
|
1786
|
+
"bodyFields": [
|
|
1787
|
+
{
|
|
1788
|
+
"name": "name",
|
|
1789
|
+
"required": false,
|
|
1790
|
+
"kind": "string",
|
|
1791
|
+
"placeholder": "<name>"
|
|
1792
|
+
},
|
|
1793
|
+
{
|
|
1794
|
+
"name": "visibility",
|
|
1795
|
+
"required": false,
|
|
1796
|
+
"kind": "string",
|
|
1797
|
+
"enum": [
|
|
1798
|
+
"user",
|
|
1799
|
+
"org"
|
|
1800
|
+
],
|
|
1801
|
+
"placeholder": "user"
|
|
1802
|
+
},
|
|
1803
|
+
{
|
|
1804
|
+
"name": "body",
|
|
1805
|
+
"required": false,
|
|
1806
|
+
"kind": "string",
|
|
1807
|
+
"placeholder": "<body>"
|
|
1808
|
+
},
|
|
1809
|
+
{
|
|
1810
|
+
"name": "repoUrl",
|
|
1811
|
+
"required": false,
|
|
1812
|
+
"kind": "string",
|
|
1813
|
+
"placeholder": "<repoUrl>"
|
|
1814
|
+
},
|
|
1815
|
+
{
|
|
1816
|
+
"name": "gitRef",
|
|
1817
|
+
"required": false,
|
|
1818
|
+
"kind": "string",
|
|
1819
|
+
"placeholder": "<gitRef>"
|
|
1820
|
+
},
|
|
1821
|
+
{
|
|
1822
|
+
"name": "subpath",
|
|
1823
|
+
"required": false,
|
|
1824
|
+
"kind": "string",
|
|
1825
|
+
"placeholder": "<subpath>"
|
|
1826
|
+
}
|
|
1827
|
+
],
|
|
1828
|
+
"watchable": false,
|
|
1829
|
+
"help": {
|
|
1830
|
+
"command": "ora experiment skills update <id> --file run.json",
|
|
1831
|
+
"body": "{\n \"name\": \"<name>\",\n \"visibility\": \"user\",\n \"body\": \"<body>\"\n}"
|
|
1832
|
+
}
|
|
1833
|
+
},
|
|
1834
|
+
{
|
|
1835
|
+
"id": "icps.create",
|
|
1836
|
+
"commandPath": [
|
|
1837
|
+
"icps",
|
|
1838
|
+
"create"
|
|
1839
|
+
],
|
|
1840
|
+
"method": "post",
|
|
1841
|
+
"pathTemplate": "/studio/v1/icps",
|
|
1842
|
+
"summary": "Record an ICP for a tracked domain (get-or-create on its label)",
|
|
1843
|
+
"scope": "intents:write",
|
|
1844
|
+
"product": "studio",
|
|
1845
|
+
"pathParams": [],
|
|
1846
|
+
"queryParams": [],
|
|
1847
|
+
"hasBody": true,
|
|
1848
|
+
"bodyRequired": true,
|
|
1849
|
+
"isStream": false,
|
|
1850
|
+
"bodyFields": [
|
|
1851
|
+
{
|
|
1852
|
+
"name": "domainId",
|
|
1853
|
+
"required": true,
|
|
1854
|
+
"kind": "string",
|
|
1855
|
+
"placeholder": "<domainId>"
|
|
1856
|
+
},
|
|
1857
|
+
{
|
|
1858
|
+
"name": "description",
|
|
1859
|
+
"required": true,
|
|
1860
|
+
"kind": "string",
|
|
1861
|
+
"placeholder": "<description>"
|
|
1862
|
+
},
|
|
1863
|
+
{
|
|
1864
|
+
"name": "label",
|
|
1387
1865
|
"required": false,
|
|
1388
1866
|
"kind": "string",
|
|
1389
1867
|
"placeholder": "<label>"
|
|
@@ -2165,6 +2643,12 @@ export const OPERATIONS = [
|
|
|
2165
2643
|
"kind": "array",
|
|
2166
2644
|
"placeholder": "[]"
|
|
2167
2645
|
},
|
|
2646
|
+
{
|
|
2647
|
+
"name": "browser",
|
|
2648
|
+
"required": false,
|
|
2649
|
+
"kind": "object",
|
|
2650
|
+
"placeholder": "{}"
|
|
2651
|
+
},
|
|
2168
2652
|
{
|
|
2169
2653
|
"name": "schedule",
|
|
2170
2654
|
"required": false,
|
|
@@ -2426,68 +2910,17 @@ export const OPERATIONS = [
|
|
|
2426
2910
|
}
|
|
2427
2911
|
},
|
|
2428
2912
|
{
|
|
2429
|
-
"id": "
|
|
2913
|
+
"id": "studio.runs.cancel",
|
|
2430
2914
|
"commandPath": [
|
|
2431
|
-
"
|
|
2432
|
-
"
|
|
2915
|
+
"studio",
|
|
2916
|
+
"runs",
|
|
2917
|
+
"cancel"
|
|
2433
2918
|
],
|
|
2434
2919
|
"method": "post",
|
|
2435
|
-
"pathTemplate": "/
|
|
2436
|
-
"summary": "
|
|
2437
|
-
"scope": "
|
|
2438
|
-
"product": "
|
|
2439
|
-
"pathParams": [],
|
|
2440
|
-
"queryParams": [],
|
|
2441
|
-
"hasBody": true,
|
|
2442
|
-
"bodyRequired": true,
|
|
2443
|
-
"isStream": false,
|
|
2444
|
-
"bodyFields": [
|
|
2445
|
-
{
|
|
2446
|
-
"name": "source",
|
|
2447
|
-
"required": true,
|
|
2448
|
-
"kind": "string",
|
|
2449
|
-
"placeholder": "custom"
|
|
2450
|
-
},
|
|
2451
|
-
{
|
|
2452
|
-
"name": "name",
|
|
2453
|
-
"required": true,
|
|
2454
|
-
"kind": "string",
|
|
2455
|
-
"placeholder": "<name>"
|
|
2456
|
-
},
|
|
2457
|
-
{
|
|
2458
|
-
"name": "body",
|
|
2459
|
-
"required": true,
|
|
2460
|
-
"kind": "string",
|
|
2461
|
-
"placeholder": "<body>"
|
|
2462
|
-
},
|
|
2463
|
-
{
|
|
2464
|
-
"name": "visibility",
|
|
2465
|
-
"required": false,
|
|
2466
|
-
"kind": "string",
|
|
2467
|
-
"enum": [
|
|
2468
|
-
"user",
|
|
2469
|
-
"org"
|
|
2470
|
-
],
|
|
2471
|
-
"placeholder": "user"
|
|
2472
|
-
}
|
|
2473
|
-
],
|
|
2474
|
-
"watchable": false,
|
|
2475
|
-
"help": {
|
|
2476
|
-
"command": "ora skills create --file run.json",
|
|
2477
|
-
"body": "{\n \"source\": \"custom\",\n \"name\": \"<name>\",\n \"body\": \"<body>\"\n}"
|
|
2478
|
-
}
|
|
2479
|
-
},
|
|
2480
|
-
{
|
|
2481
|
-
"id": "skills.delete",
|
|
2482
|
-
"commandPath": [
|
|
2483
|
-
"skills",
|
|
2484
|
-
"delete"
|
|
2485
|
-
],
|
|
2486
|
-
"method": "delete",
|
|
2487
|
-
"pathTemplate": "/experiment/v1/skills/{id}",
|
|
2488
|
-
"summary": "Delete a skill",
|
|
2489
|
-
"scope": "skills:write",
|
|
2490
|
-
"product": "experiment",
|
|
2920
|
+
"pathTemplate": "/studio/v1/runs/{id}/cancel",
|
|
2921
|
+
"summary": "Cancel a run (terminate it and stop its engine)",
|
|
2922
|
+
"scope": "runs:write",
|
|
2923
|
+
"product": "platform",
|
|
2491
2924
|
"pathParams": [
|
|
2492
2925
|
{
|
|
2493
2926
|
"name": "id"
|
|
@@ -2499,153 +2932,38 @@ export const OPERATIONS = [
|
|
|
2499
2932
|
"isStream": false,
|
|
2500
2933
|
"watchable": false,
|
|
2501
2934
|
"help": {
|
|
2502
|
-
"command": "ora
|
|
2935
|
+
"command": "ora studio runs cancel <id>"
|
|
2503
2936
|
}
|
|
2504
2937
|
},
|
|
2505
2938
|
{
|
|
2506
|
-
"id": "
|
|
2939
|
+
"id": "studio.runs.cancelBatch",
|
|
2507
2940
|
"commandPath": [
|
|
2508
|
-
"
|
|
2509
|
-
"
|
|
2941
|
+
"studio",
|
|
2942
|
+
"runs",
|
|
2943
|
+
"cancel-batch"
|
|
2510
2944
|
],
|
|
2511
|
-
"method": "
|
|
2512
|
-
"pathTemplate": "/
|
|
2513
|
-
"summary": "
|
|
2514
|
-
"scope": "
|
|
2515
|
-
"product": "
|
|
2945
|
+
"method": "post",
|
|
2946
|
+
"pathTemplate": "/studio/v1/runs/cancel",
|
|
2947
|
+
"summary": "Cancel a batch of runs (terminate them and stop their engines)",
|
|
2948
|
+
"scope": "runs:write",
|
|
2949
|
+
"product": "platform",
|
|
2516
2950
|
"pathParams": [],
|
|
2517
2951
|
"queryParams": [],
|
|
2518
|
-
"hasBody":
|
|
2519
|
-
"bodyRequired":
|
|
2952
|
+
"hasBody": true,
|
|
2953
|
+
"bodyRequired": true,
|
|
2520
2954
|
"isStream": false,
|
|
2955
|
+
"bodyFields": [
|
|
2956
|
+
{
|
|
2957
|
+
"name": "runIds",
|
|
2958
|
+
"required": true,
|
|
2959
|
+
"kind": "array",
|
|
2960
|
+
"placeholder": "[]"
|
|
2961
|
+
}
|
|
2962
|
+
],
|
|
2521
2963
|
"watchable": false,
|
|
2522
2964
|
"help": {
|
|
2523
|
-
"command": "ora
|
|
2524
|
-
|
|
2525
|
-
},
|
|
2526
|
-
{
|
|
2527
|
-
"id": "skills.update",
|
|
2528
|
-
"commandPath": [
|
|
2529
|
-
"skills",
|
|
2530
|
-
"update"
|
|
2531
|
-
],
|
|
2532
|
-
"method": "put",
|
|
2533
|
-
"pathTemplate": "/experiment/v1/skills/{id}",
|
|
2534
|
-
"summary": "Update a skill",
|
|
2535
|
-
"scope": "skills:write",
|
|
2536
|
-
"product": "experiment",
|
|
2537
|
-
"pathParams": [
|
|
2538
|
-
{
|
|
2539
|
-
"name": "id"
|
|
2540
|
-
}
|
|
2541
|
-
],
|
|
2542
|
-
"queryParams": [],
|
|
2543
|
-
"hasBody": true,
|
|
2544
|
-
"bodyRequired": true,
|
|
2545
|
-
"isStream": false,
|
|
2546
|
-
"bodyFields": [
|
|
2547
|
-
{
|
|
2548
|
-
"name": "name",
|
|
2549
|
-
"required": false,
|
|
2550
|
-
"kind": "string",
|
|
2551
|
-
"placeholder": "<name>"
|
|
2552
|
-
},
|
|
2553
|
-
{
|
|
2554
|
-
"name": "visibility",
|
|
2555
|
-
"required": false,
|
|
2556
|
-
"kind": "string",
|
|
2557
|
-
"enum": [
|
|
2558
|
-
"user",
|
|
2559
|
-
"org"
|
|
2560
|
-
],
|
|
2561
|
-
"placeholder": "user"
|
|
2562
|
-
},
|
|
2563
|
-
{
|
|
2564
|
-
"name": "body",
|
|
2565
|
-
"required": false,
|
|
2566
|
-
"kind": "string",
|
|
2567
|
-
"placeholder": "<body>"
|
|
2568
|
-
},
|
|
2569
|
-
{
|
|
2570
|
-
"name": "repoUrl",
|
|
2571
|
-
"required": false,
|
|
2572
|
-
"kind": "string",
|
|
2573
|
-
"placeholder": "<repoUrl>"
|
|
2574
|
-
},
|
|
2575
|
-
{
|
|
2576
|
-
"name": "gitRef",
|
|
2577
|
-
"required": false,
|
|
2578
|
-
"kind": "string",
|
|
2579
|
-
"placeholder": "<gitRef>"
|
|
2580
|
-
},
|
|
2581
|
-
{
|
|
2582
|
-
"name": "subpath",
|
|
2583
|
-
"required": false,
|
|
2584
|
-
"kind": "string",
|
|
2585
|
-
"placeholder": "<subpath>"
|
|
2586
|
-
}
|
|
2587
|
-
],
|
|
2588
|
-
"watchable": false,
|
|
2589
|
-
"help": {
|
|
2590
|
-
"command": "ora skills update <id> --file run.json",
|
|
2591
|
-
"body": "{\n \"name\": \"<name>\",\n \"visibility\": \"user\",\n \"body\": \"<body>\"\n}"
|
|
2592
|
-
}
|
|
2593
|
-
},
|
|
2594
|
-
{
|
|
2595
|
-
"id": "studio.runs.cancel",
|
|
2596
|
-
"commandPath": [
|
|
2597
|
-
"studio",
|
|
2598
|
-
"runs",
|
|
2599
|
-
"cancel"
|
|
2600
|
-
],
|
|
2601
|
-
"method": "post",
|
|
2602
|
-
"pathTemplate": "/studio/v1/runs/{id}/cancel",
|
|
2603
|
-
"summary": "Cancel a run (terminate it and stop its engine)",
|
|
2604
|
-
"scope": "runs:write",
|
|
2605
|
-
"product": "platform",
|
|
2606
|
-
"pathParams": [
|
|
2607
|
-
{
|
|
2608
|
-
"name": "id"
|
|
2609
|
-
}
|
|
2610
|
-
],
|
|
2611
|
-
"queryParams": [],
|
|
2612
|
-
"hasBody": false,
|
|
2613
|
-
"bodyRequired": false,
|
|
2614
|
-
"isStream": false,
|
|
2615
|
-
"watchable": false,
|
|
2616
|
-
"help": {
|
|
2617
|
-
"command": "ora studio runs cancel <id>"
|
|
2618
|
-
}
|
|
2619
|
-
},
|
|
2620
|
-
{
|
|
2621
|
-
"id": "studio.runs.cancelBatch",
|
|
2622
|
-
"commandPath": [
|
|
2623
|
-
"studio",
|
|
2624
|
-
"runs",
|
|
2625
|
-
"cancel-batch"
|
|
2626
|
-
],
|
|
2627
|
-
"method": "post",
|
|
2628
|
-
"pathTemplate": "/studio/v1/runs/cancel",
|
|
2629
|
-
"summary": "Cancel a batch of runs (terminate them and stop their engines)",
|
|
2630
|
-
"scope": "runs:write",
|
|
2631
|
-
"product": "platform",
|
|
2632
|
-
"pathParams": [],
|
|
2633
|
-
"queryParams": [],
|
|
2634
|
-
"hasBody": true,
|
|
2635
|
-
"bodyRequired": true,
|
|
2636
|
-
"isStream": false,
|
|
2637
|
-
"bodyFields": [
|
|
2638
|
-
{
|
|
2639
|
-
"name": "runIds",
|
|
2640
|
-
"required": true,
|
|
2641
|
-
"kind": "array",
|
|
2642
|
-
"placeholder": "[]"
|
|
2643
|
-
}
|
|
2644
|
-
],
|
|
2645
|
-
"watchable": false,
|
|
2646
|
-
"help": {
|
|
2647
|
-
"command": "ora studio runs cancel-batch --file run.json",
|
|
2648
|
-
"body": "{\n \"runIds\": []\n}"
|
|
2965
|
+
"command": "ora studio runs cancel-batch --file run.json",
|
|
2966
|
+
"body": "{\n \"runIds\": []\n}"
|
|
2649
2967
|
}
|
|
2650
2968
|
},
|
|
2651
2969
|
{
|
|
@@ -2746,6 +3064,12 @@ export const OPERATIONS = [
|
|
|
2746
3064
|
"kind": "array",
|
|
2747
3065
|
"placeholder": "[]"
|
|
2748
3066
|
},
|
|
3067
|
+
{
|
|
3068
|
+
"name": "model_params",
|
|
3069
|
+
"required": false,
|
|
3070
|
+
"kind": "object",
|
|
3071
|
+
"placeholder": "{}"
|
|
3072
|
+
},
|
|
2749
3073
|
{
|
|
2750
3074
|
"name": "wait",
|
|
2751
3075
|
"required": false,
|
|
@@ -2988,130 +3312,2686 @@ export const OPERATIONS = [
|
|
|
2988
3312
|
}
|
|
2989
3313
|
},
|
|
2990
3314
|
{
|
|
2991
|
-
"id": "
|
|
3315
|
+
"id": "study.addons.list",
|
|
2992
3316
|
"commandPath": [
|
|
2993
|
-
"
|
|
2994
|
-
"
|
|
3317
|
+
"study",
|
|
3318
|
+
"addons",
|
|
3319
|
+
"list"
|
|
2995
3320
|
],
|
|
2996
|
-
"method": "
|
|
2997
|
-
"pathTemplate": "/
|
|
2998
|
-
"summary": "
|
|
2999
|
-
"scope": "
|
|
3000
|
-
"product": "
|
|
3321
|
+
"method": "get",
|
|
3322
|
+
"pathTemplate": "/study/v1/addons",
|
|
3323
|
+
"summary": "The add-on catalog (harness/surface-filtered)",
|
|
3324
|
+
"scope": "study.addons:read",
|
|
3325
|
+
"product": "study",
|
|
3001
3326
|
"pathParams": [],
|
|
3002
|
-
"queryParams": [
|
|
3003
|
-
"hasBody": true,
|
|
3004
|
-
"bodyRequired": true,
|
|
3005
|
-
"isStream": false,
|
|
3006
|
-
"bodyFields": [
|
|
3007
|
-
{
|
|
3008
|
-
"name": "name",
|
|
3009
|
-
"required": true,
|
|
3010
|
-
"kind": "string",
|
|
3011
|
-
"placeholder": "<name>"
|
|
3012
|
-
},
|
|
3327
|
+
"queryParams": [
|
|
3013
3328
|
{
|
|
3014
|
-
"name": "
|
|
3329
|
+
"name": "harness",
|
|
3330
|
+
"flag": "harness",
|
|
3331
|
+
"optionKey": "harness",
|
|
3015
3332
|
"required": false,
|
|
3016
|
-
"kind": "string"
|
|
3017
|
-
"placeholder": "<productKey>"
|
|
3333
|
+
"kind": "string"
|
|
3018
3334
|
},
|
|
3019
3335
|
{
|
|
3020
|
-
"name": "
|
|
3021
|
-
"
|
|
3336
|
+
"name": "surface",
|
|
3337
|
+
"flag": "surface",
|
|
3338
|
+
"optionKey": "surface",
|
|
3339
|
+
"required": false,
|
|
3022
3340
|
"kind": "string",
|
|
3023
3341
|
"enum": [
|
|
3024
|
-
"
|
|
3025
|
-
"
|
|
3026
|
-
|
|
3027
|
-
],
|
|
3028
|
-
"placeholder": "private"
|
|
3029
|
-
}
|
|
3030
|
-
],
|
|
3031
|
-
"watchable": false,
|
|
3032
|
-
"help": {
|
|
3033
|
-
"command": "ora tunnels create --file run.json",
|
|
3034
|
-
"body": "{\n \"name\": \"<name>\",\n \"access\": \"private\"\n}"
|
|
3035
|
-
}
|
|
3036
|
-
},
|
|
3037
|
-
{
|
|
3038
|
-
"id": "tunnels.delete",
|
|
3039
|
-
"commandPath": [
|
|
3040
|
-
"tunnels",
|
|
3041
|
-
"delete"
|
|
3042
|
-
],
|
|
3043
|
-
"method": "delete",
|
|
3044
|
-
"pathTemplate": "/tunnels/v1/{id}",
|
|
3045
|
-
"summary": "Delete a dev tunnel (revokes any live connection, removes the row)",
|
|
3046
|
-
"scope": "tunnels:write",
|
|
3047
|
-
"product": "platform",
|
|
3048
|
-
"pathParams": [
|
|
3049
|
-
{
|
|
3050
|
-
"name": "id"
|
|
3342
|
+
"run",
|
|
3343
|
+
"session"
|
|
3344
|
+
]
|
|
3051
3345
|
}
|
|
3052
3346
|
],
|
|
3053
|
-
"queryParams": [],
|
|
3054
3347
|
"hasBody": false,
|
|
3055
3348
|
"bodyRequired": false,
|
|
3056
3349
|
"isStream": false,
|
|
3057
3350
|
"watchable": false,
|
|
3058
3351
|
"help": {
|
|
3059
|
-
"command": "ora
|
|
3352
|
+
"command": "ora study addons list"
|
|
3060
3353
|
}
|
|
3061
3354
|
},
|
|
3062
3355
|
{
|
|
3063
|
-
"id": "
|
|
3356
|
+
"id": "study.benchmarkDefinitions.attachCorpus",
|
|
3064
3357
|
"commandPath": [
|
|
3065
|
-
"
|
|
3066
|
-
"
|
|
3358
|
+
"study",
|
|
3359
|
+
"benchmarkDefinitions",
|
|
3360
|
+
"attach-corpus"
|
|
3067
3361
|
],
|
|
3068
|
-
"method": "
|
|
3069
|
-
"pathTemplate": "/
|
|
3070
|
-
"summary": "
|
|
3071
|
-
"scope": "
|
|
3072
|
-
"product": "
|
|
3362
|
+
"method": "post",
|
|
3363
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/attach-corpus",
|
|
3364
|
+
"summary": "Accept an import of your definition's declared source into a NEW immutable version — a JOB, followed over its streamUrl",
|
|
3365
|
+
"scope": "study.studies:write",
|
|
3366
|
+
"product": "study",
|
|
3073
3367
|
"pathParams": [
|
|
3074
3368
|
{
|
|
3075
|
-
"name": "
|
|
3369
|
+
"name": "slug"
|
|
3076
3370
|
}
|
|
3077
3371
|
],
|
|
3078
3372
|
"queryParams": [],
|
|
3079
|
-
"hasBody":
|
|
3080
|
-
"bodyRequired":
|
|
3373
|
+
"hasBody": true,
|
|
3374
|
+
"bodyRequired": true,
|
|
3081
3375
|
"isStream": false,
|
|
3082
|
-
"
|
|
3376
|
+
"bodyFields": [
|
|
3377
|
+
{
|
|
3378
|
+
"name": "version",
|
|
3379
|
+
"required": false,
|
|
3380
|
+
"kind": "number",
|
|
3381
|
+
"description": "Write this version NUMBER. Omit it to take the next free one. A number that already exists is a 409 — a published version is immutable, because studies pin it.",
|
|
3382
|
+
"placeholder": "1"
|
|
3383
|
+
},
|
|
3384
|
+
{
|
|
3385
|
+
"name": "limit",
|
|
3386
|
+
"required": false,
|
|
3387
|
+
"kind": "number",
|
|
3388
|
+
"description": "Import at most this many tasks from the top of the source's own order. Recorded in the version's `notes`, never applied silently — a short corpus with no explanation is a benchmark nobody can reproduce.",
|
|
3389
|
+
"placeholder": "200"
|
|
3390
|
+
},
|
|
3391
|
+
{
|
|
3392
|
+
"name": "changelog",
|
|
3393
|
+
"required": false,
|
|
3394
|
+
"kind": "string",
|
|
3395
|
+
"description": "What changed about this version, in the publisher's own words.",
|
|
3396
|
+
"placeholder": "<changelog>"
|
|
3397
|
+
}
|
|
3398
|
+
],
|
|
3399
|
+
"watchable": true,
|
|
3083
3400
|
"help": {
|
|
3084
|
-
"command": "ora
|
|
3401
|
+
"command": "ora study benchmarkDefinitions attach-corpus windtunnel --file run.json --watch",
|
|
3402
|
+
"body": "{\n \"limit\": 200\n}"
|
|
3085
3403
|
}
|
|
3086
3404
|
},
|
|
3087
3405
|
{
|
|
3088
|
-
"id": "
|
|
3406
|
+
"id": "study.benchmarkDefinitions.create",
|
|
3089
3407
|
"commandPath": [
|
|
3090
|
-
"
|
|
3091
|
-
"
|
|
3408
|
+
"study",
|
|
3409
|
+
"benchmarkDefinitions",
|
|
3410
|
+
"create"
|
|
3092
3411
|
],
|
|
3093
|
-
"method": "
|
|
3094
|
-
"pathTemplate": "/
|
|
3095
|
-
"summary": "
|
|
3096
|
-
"scope": "
|
|
3097
|
-
"product": "
|
|
3412
|
+
"method": "post",
|
|
3413
|
+
"pathTemplate": "/study/v1/benchmark-definitions",
|
|
3414
|
+
"summary": "Author a benchmark definition — born private to your organization, unlisted and licence-unverified",
|
|
3415
|
+
"scope": "study.studies:write",
|
|
3416
|
+
"product": "study",
|
|
3098
3417
|
"pathParams": [],
|
|
3099
3418
|
"queryParams": [],
|
|
3100
|
-
"hasBody":
|
|
3101
|
-
"bodyRequired":
|
|
3419
|
+
"hasBody": true,
|
|
3420
|
+
"bodyRequired": true,
|
|
3102
3421
|
"isStream": false,
|
|
3103
|
-
"
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3422
|
+
"bodyFields": [
|
|
3423
|
+
{
|
|
3424
|
+
"name": "slug",
|
|
3425
|
+
"required": true,
|
|
3426
|
+
"kind": "string",
|
|
3427
|
+
"placeholder": "windtunnel"
|
|
3428
|
+
},
|
|
3429
|
+
{
|
|
3430
|
+
"name": "name",
|
|
3431
|
+
"required": true,
|
|
3432
|
+
"kind": "string",
|
|
3433
|
+
"placeholder": "WindTunnel"
|
|
3434
|
+
},
|
|
3435
|
+
{
|
|
3436
|
+
"name": "description",
|
|
3437
|
+
"required": false,
|
|
3438
|
+
"kind": "string",
|
|
3439
|
+
"placeholder": "<description>"
|
|
3440
|
+
},
|
|
3441
|
+
{
|
|
3442
|
+
"name": "family",
|
|
3443
|
+
"required": true,
|
|
3444
|
+
"kind": "string",
|
|
3445
|
+
"enum": [
|
|
3446
|
+
"static_qa",
|
|
3447
|
+
"code",
|
|
3448
|
+
"agentic",
|
|
3449
|
+
"tools",
|
|
3450
|
+
"web",
|
|
3451
|
+
"knowledge_work",
|
|
3452
|
+
"safety",
|
|
3453
|
+
"suite"
|
|
3454
|
+
],
|
|
3455
|
+
"placeholder": "web"
|
|
3456
|
+
},
|
|
3457
|
+
{
|
|
3458
|
+
"name": "sourceKind",
|
|
3459
|
+
"required": true,
|
|
3460
|
+
"kind": "string",
|
|
3461
|
+
"enum": [
|
|
3462
|
+
"hf-dataset",
|
|
3463
|
+
"jsonl",
|
|
3464
|
+
"git-tasks",
|
|
3465
|
+
"release-file",
|
|
3466
|
+
"authored",
|
|
3467
|
+
"framework"
|
|
3468
|
+
],
|
|
3469
|
+
"placeholder": "git-tasks"
|
|
3470
|
+
},
|
|
3471
|
+
{
|
|
3472
|
+
"name": "sourceConfig",
|
|
3473
|
+
"required": false,
|
|
3474
|
+
"kind": "object",
|
|
3475
|
+
"placeholder": "{}"
|
|
3476
|
+
},
|
|
3477
|
+
{
|
|
3478
|
+
"name": "adapterId",
|
|
3479
|
+
"required": false,
|
|
3480
|
+
"kind": "string",
|
|
3481
|
+
"placeholder": "<adapterId>"
|
|
3482
|
+
},
|
|
3483
|
+
{
|
|
3484
|
+
"name": "headlineMetric",
|
|
3485
|
+
"required": true,
|
|
3486
|
+
"kind": "string",
|
|
3487
|
+
"placeholder": "pass_rate"
|
|
3488
|
+
},
|
|
3489
|
+
{
|
|
3490
|
+
"name": "gradingSpec",
|
|
3491
|
+
"required": false,
|
|
3492
|
+
"kind": "object",
|
|
3493
|
+
"placeholder": "{}"
|
|
3494
|
+
},
|
|
3495
|
+
{
|
|
3496
|
+
"name": "environmentSpec",
|
|
3497
|
+
"required": false,
|
|
3498
|
+
"kind": "object",
|
|
3499
|
+
"placeholder": "{}"
|
|
3500
|
+
},
|
|
3501
|
+
{
|
|
3502
|
+
"name": "methodology",
|
|
3503
|
+
"required": false,
|
|
3504
|
+
"kind": "object",
|
|
3505
|
+
"placeholder": "{}"
|
|
3506
|
+
},
|
|
3507
|
+
{
|
|
3508
|
+
"name": "runnerKind",
|
|
3509
|
+
"required": false,
|
|
3510
|
+
"kind": "string",
|
|
3511
|
+
"enum": [
|
|
3512
|
+
"none",
|
|
3513
|
+
"image",
|
|
3514
|
+
"environment",
|
|
3515
|
+
"framework"
|
|
3516
|
+
],
|
|
3517
|
+
"placeholder": "none"
|
|
3518
|
+
},
|
|
3519
|
+
{
|
|
3520
|
+
"name": "runnerRecipe",
|
|
3521
|
+
"required": false,
|
|
3522
|
+
"kind": "string",
|
|
3523
|
+
"placeholder": "<runnerRecipe>"
|
|
3524
|
+
},
|
|
3525
|
+
{
|
|
3526
|
+
"name": "license",
|
|
3527
|
+
"required": false,
|
|
3528
|
+
"kind": "string",
|
|
3529
|
+
"placeholder": "<license>"
|
|
3530
|
+
},
|
|
3531
|
+
{
|
|
3532
|
+
"name": "licenseText",
|
|
3533
|
+
"required": false,
|
|
3534
|
+
"kind": "string",
|
|
3535
|
+
"placeholder": "<licenseText>"
|
|
3536
|
+
},
|
|
3537
|
+
{
|
|
3538
|
+
"name": "redistribution",
|
|
3539
|
+
"required": false,
|
|
3540
|
+
"kind": "string",
|
|
3541
|
+
"enum": [
|
|
3542
|
+
"allowed",
|
|
3543
|
+
"reference_only"
|
|
3544
|
+
],
|
|
3545
|
+
"placeholder": "allowed"
|
|
3546
|
+
},
|
|
3547
|
+
{
|
|
3548
|
+
"name": "access",
|
|
3549
|
+
"required": false,
|
|
3550
|
+
"kind": "string",
|
|
3551
|
+
"enum": [
|
|
3552
|
+
"open",
|
|
3553
|
+
"gated",
|
|
3554
|
+
"private"
|
|
3555
|
+
],
|
|
3556
|
+
"placeholder": "open"
|
|
3557
|
+
},
|
|
3558
|
+
{
|
|
3559
|
+
"name": "termsUrl",
|
|
3560
|
+
"required": false,
|
|
3561
|
+
"kind": "string",
|
|
3562
|
+
"placeholder": "<termsUrl>"
|
|
3563
|
+
},
|
|
3564
|
+
{
|
|
3565
|
+
"name": "homepageUrl",
|
|
3566
|
+
"required": false,
|
|
3567
|
+
"kind": "string",
|
|
3568
|
+
"placeholder": "<homepageUrl>"
|
|
3569
|
+
},
|
|
3570
|
+
{
|
|
3571
|
+
"name": "paperUrl",
|
|
3572
|
+
"required": false,
|
|
3573
|
+
"kind": "string",
|
|
3574
|
+
"placeholder": "<paperUrl>"
|
|
3575
|
+
},
|
|
3576
|
+
{
|
|
3577
|
+
"name": "defaultConfig",
|
|
3578
|
+
"required": false,
|
|
3579
|
+
"kind": "string",
|
|
3580
|
+
"placeholder": "<defaultConfig>"
|
|
3581
|
+
},
|
|
3582
|
+
{
|
|
3583
|
+
"name": "visibility",
|
|
3584
|
+
"required": false,
|
|
3585
|
+
"kind": "string",
|
|
3586
|
+
"enum": [
|
|
3587
|
+
"private",
|
|
3588
|
+
"org",
|
|
3589
|
+
"public"
|
|
3590
|
+
],
|
|
3591
|
+
"placeholder": "private"
|
|
3592
|
+
}
|
|
3593
|
+
],
|
|
3594
|
+
"watchable": false,
|
|
3595
|
+
"help": {
|
|
3596
|
+
"command": "ora study benchmarkDefinitions create --file run.json",
|
|
3597
|
+
"body": "{\n \"slug\": \"windtunnel\",\n \"name\": \"WindTunnel\",\n \"family\": \"web\",\n \"sourceKind\": \"git-tasks\",\n \"headlineMetric\": \"pass_rate\"\n}"
|
|
3598
|
+
}
|
|
3599
|
+
},
|
|
3600
|
+
{
|
|
3601
|
+
"id": "study.benchmarkDefinitions.get",
|
|
3602
|
+
"commandPath": [
|
|
3603
|
+
"study",
|
|
3604
|
+
"benchmarkDefinitions",
|
|
3605
|
+
"get"
|
|
3606
|
+
],
|
|
3607
|
+
"method": "get",
|
|
3608
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}",
|
|
3609
|
+
"summary": "One benchmark definition: methodology, licensing, environment and whether it can be run today",
|
|
3610
|
+
"scope": "study.studies:read",
|
|
3611
|
+
"product": "study",
|
|
3612
|
+
"pathParams": [
|
|
3613
|
+
{
|
|
3614
|
+
"name": "slug"
|
|
3615
|
+
}
|
|
3616
|
+
],
|
|
3617
|
+
"queryParams": [],
|
|
3618
|
+
"hasBody": false,
|
|
3619
|
+
"bodyRequired": false,
|
|
3620
|
+
"isStream": false,
|
|
3621
|
+
"watchable": false,
|
|
3622
|
+
"help": {
|
|
3623
|
+
"command": "ora study benchmarkDefinitions get mmlu-pro"
|
|
3624
|
+
}
|
|
3625
|
+
},
|
|
3626
|
+
{
|
|
3627
|
+
"id": "study.benchmarkDefinitions.list",
|
|
3628
|
+
"commandPath": [
|
|
3629
|
+
"study",
|
|
3630
|
+
"benchmarkDefinitions",
|
|
3631
|
+
"list"
|
|
3632
|
+
],
|
|
3633
|
+
"method": "get",
|
|
3634
|
+
"pathTemplate": "/study/v1/benchmark-definitions",
|
|
3635
|
+
"summary": "The benchmark catalog: every listed third-party definition plus the org's own, each with its derived runnability",
|
|
3636
|
+
"scope": "study.studies:read",
|
|
3637
|
+
"product": "study",
|
|
3638
|
+
"pathParams": [],
|
|
3639
|
+
"queryParams": [
|
|
3640
|
+
{
|
|
3641
|
+
"name": "family",
|
|
3642
|
+
"flag": "family",
|
|
3643
|
+
"optionKey": "family",
|
|
3644
|
+
"required": false,
|
|
3645
|
+
"kind": "string",
|
|
3646
|
+
"enum": [
|
|
3647
|
+
"static_qa",
|
|
3648
|
+
"code",
|
|
3649
|
+
"agentic",
|
|
3650
|
+
"tools",
|
|
3651
|
+
"web",
|
|
3652
|
+
"knowledge_work",
|
|
3653
|
+
"safety",
|
|
3654
|
+
"suite"
|
|
3655
|
+
]
|
|
3656
|
+
},
|
|
3657
|
+
{
|
|
3658
|
+
"name": "sourceKind",
|
|
3659
|
+
"flag": "source-kind",
|
|
3660
|
+
"optionKey": "sourceKind",
|
|
3661
|
+
"required": false,
|
|
3662
|
+
"kind": "string",
|
|
3663
|
+
"enum": [
|
|
3664
|
+
"hf-dataset",
|
|
3665
|
+
"jsonl",
|
|
3666
|
+
"git-tasks",
|
|
3667
|
+
"release-file",
|
|
3668
|
+
"authored",
|
|
3669
|
+
"framework"
|
|
3670
|
+
]
|
|
3671
|
+
},
|
|
3672
|
+
{
|
|
3673
|
+
"name": "tier",
|
|
3674
|
+
"flag": "tier",
|
|
3675
|
+
"optionKey": "tier",
|
|
3676
|
+
"required": false,
|
|
3677
|
+
"kind": "string",
|
|
3678
|
+
"enum": [
|
|
3679
|
+
"featured",
|
|
3680
|
+
"community",
|
|
3681
|
+
"mine"
|
|
3682
|
+
]
|
|
3683
|
+
},
|
|
3684
|
+
{
|
|
3685
|
+
"name": "runnable",
|
|
3686
|
+
"flag": "runnable",
|
|
3687
|
+
"optionKey": "runnable",
|
|
3688
|
+
"required": false,
|
|
3689
|
+
"kind": "string",
|
|
3690
|
+
"enum": [
|
|
3691
|
+
"true",
|
|
3692
|
+
"false"
|
|
3693
|
+
]
|
|
3694
|
+
},
|
|
3695
|
+
{
|
|
3696
|
+
"name": "search",
|
|
3697
|
+
"flag": "search",
|
|
3698
|
+
"optionKey": "search",
|
|
3699
|
+
"required": false,
|
|
3700
|
+
"kind": "string"
|
|
3701
|
+
},
|
|
3702
|
+
{
|
|
3703
|
+
"name": "pageSize",
|
|
3704
|
+
"flag": "page-size",
|
|
3705
|
+
"optionKey": "pageSize",
|
|
3706
|
+
"required": false,
|
|
3707
|
+
"kind": "number"
|
|
3708
|
+
}
|
|
3709
|
+
],
|
|
3710
|
+
"hasBody": false,
|
|
3711
|
+
"bodyRequired": false,
|
|
3712
|
+
"isStream": false,
|
|
3713
|
+
"watchable": false,
|
|
3714
|
+
"help": {
|
|
3715
|
+
"command": "ora study benchmarkDefinitions list"
|
|
3716
|
+
}
|
|
3717
|
+
},
|
|
3718
|
+
{
|
|
3719
|
+
"id": "study.benchmarkDefinitions.publish",
|
|
3720
|
+
"commandPath": [
|
|
3721
|
+
"study",
|
|
3722
|
+
"benchmarkDefinitions",
|
|
3723
|
+
"publish"
|
|
3724
|
+
],
|
|
3725
|
+
"method": "post",
|
|
3726
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/publish",
|
|
3727
|
+
"summary": "Point your definition at an imported version — a study can pin it from here on",
|
|
3728
|
+
"scope": "study.studies:write",
|
|
3729
|
+
"product": "study",
|
|
3730
|
+
"pathParams": [
|
|
3731
|
+
{
|
|
3732
|
+
"name": "slug"
|
|
3733
|
+
}
|
|
3734
|
+
],
|
|
3735
|
+
"queryParams": [],
|
|
3736
|
+
"hasBody": true,
|
|
3737
|
+
"bodyRequired": true,
|
|
3738
|
+
"isStream": false,
|
|
3739
|
+
"bodyFields": [
|
|
3740
|
+
{
|
|
3741
|
+
"name": "version",
|
|
3742
|
+
"required": false,
|
|
3743
|
+
"kind": "number",
|
|
3744
|
+
"description": "Point the catalog at this version. Omit it to publish the newest imported one.",
|
|
3745
|
+
"placeholder": "1"
|
|
3746
|
+
}
|
|
3747
|
+
],
|
|
3748
|
+
"watchable": false,
|
|
3749
|
+
"help": {
|
|
3750
|
+
"command": "ora study benchmarkDefinitions publish windtunnel --file run.json",
|
|
3751
|
+
"body": "{}"
|
|
3752
|
+
}
|
|
3753
|
+
},
|
|
3754
|
+
{
|
|
3755
|
+
"id": "study.benchmarkDefinitions.setVisibility",
|
|
3756
|
+
"commandPath": [
|
|
3757
|
+
"study",
|
|
3758
|
+
"benchmarkDefinitions",
|
|
3759
|
+
"set-visibility"
|
|
3760
|
+
],
|
|
3761
|
+
"method": "post",
|
|
3762
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/visibility",
|
|
3763
|
+
"summary": "Make it private, org-visible or public — effective immediately, by slug or link",
|
|
3764
|
+
"scope": "study.studies:write",
|
|
3765
|
+
"product": "study",
|
|
3766
|
+
"pathParams": [
|
|
3767
|
+
{
|
|
3768
|
+
"name": "slug"
|
|
3769
|
+
}
|
|
3770
|
+
],
|
|
3771
|
+
"queryParams": [],
|
|
3772
|
+
"hasBody": true,
|
|
3773
|
+
"bodyRequired": true,
|
|
3774
|
+
"isStream": false,
|
|
3775
|
+
"bodyFields": [
|
|
3776
|
+
{
|
|
3777
|
+
"name": "visibility",
|
|
3778
|
+
"required": true,
|
|
3779
|
+
"kind": "string",
|
|
3780
|
+
"enum": [
|
|
3781
|
+
"private",
|
|
3782
|
+
"org",
|
|
3783
|
+
"public"
|
|
3784
|
+
],
|
|
3785
|
+
"placeholder": "public"
|
|
3786
|
+
}
|
|
3787
|
+
],
|
|
3788
|
+
"watchable": false,
|
|
3789
|
+
"help": {
|
|
3790
|
+
"command": "ora study benchmarkDefinitions set-visibility windtunnel --file run.json",
|
|
3791
|
+
"body": "{\n \"visibility\": \"public\"\n}"
|
|
3792
|
+
}
|
|
3793
|
+
},
|
|
3794
|
+
{
|
|
3795
|
+
"id": "study.benchmarkDefinitions.submit",
|
|
3796
|
+
"commandPath": [
|
|
3797
|
+
"study",
|
|
3798
|
+
"benchmarkDefinitions",
|
|
3799
|
+
"submit"
|
|
3800
|
+
],
|
|
3801
|
+
"method": "post",
|
|
3802
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/submit",
|
|
3803
|
+
"summary": "Ask ora to feature it in the public catalog, with a signed licence attestation. A successful review makes the definition PUBLIC and lists it — submitting is asking for both.",
|
|
3804
|
+
"scope": "study.studies:write",
|
|
3805
|
+
"product": "study",
|
|
3806
|
+
"pathParams": [
|
|
3807
|
+
{
|
|
3808
|
+
"name": "slug"
|
|
3809
|
+
}
|
|
3810
|
+
],
|
|
3811
|
+
"queryParams": [],
|
|
3812
|
+
"hasBody": true,
|
|
3813
|
+
"bodyRequired": true,
|
|
3814
|
+
"isStream": false,
|
|
3815
|
+
"bodyFields": [
|
|
3816
|
+
{
|
|
3817
|
+
"name": "attestation",
|
|
3818
|
+
"required": true,
|
|
3819
|
+
"kind": "object",
|
|
3820
|
+
"placeholder": "{\"licenseSpdx\":\"Apache-2.0\",\"redistribution\":\"allowed\",\"sourceUrl\":\"https://github.com/nekuda-ai/WindTunnel\",\"statement\":\"The repository ships an Apache-2.0 LICENSE covering the task files.\"}"
|
|
3821
|
+
}
|
|
3822
|
+
],
|
|
3823
|
+
"watchable": false,
|
|
3824
|
+
"help": {
|
|
3825
|
+
"command": "ora study benchmarkDefinitions submit windtunnel --file run.json",
|
|
3826
|
+
"body": "{\n \"attestation\": {\n \"licenseSpdx\": \"Apache-2.0\",\n \"redistribution\": \"allowed\",\n \"sourceUrl\": \"https://github.com/nekuda-ai/WindTunnel\",\n \"statement\": \"The repository ships an Apache-2.0 LICENSE covering the task files.\"\n }\n}"
|
|
3827
|
+
}
|
|
3828
|
+
},
|
|
3829
|
+
{
|
|
3830
|
+
"id": "study.benchmarkDefinitions.update",
|
|
3831
|
+
"commandPath": [
|
|
3832
|
+
"study",
|
|
3833
|
+
"benchmarkDefinitions",
|
|
3834
|
+
"update"
|
|
3835
|
+
],
|
|
3836
|
+
"method": "patch",
|
|
3837
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}",
|
|
3838
|
+
"summary": "Edit a benchmark definition your organization publishes",
|
|
3839
|
+
"scope": "study.studies:write",
|
|
3840
|
+
"product": "study",
|
|
3841
|
+
"pathParams": [
|
|
3842
|
+
{
|
|
3843
|
+
"name": "slug"
|
|
3844
|
+
}
|
|
3845
|
+
],
|
|
3846
|
+
"queryParams": [],
|
|
3847
|
+
"hasBody": true,
|
|
3848
|
+
"bodyRequired": true,
|
|
3849
|
+
"isStream": false,
|
|
3850
|
+
"bodyFields": [
|
|
3851
|
+
{
|
|
3852
|
+
"name": "name",
|
|
3853
|
+
"required": false,
|
|
3854
|
+
"kind": "string",
|
|
3855
|
+
"placeholder": "<name>"
|
|
3856
|
+
},
|
|
3857
|
+
{
|
|
3858
|
+
"name": "description",
|
|
3859
|
+
"required": false,
|
|
3860
|
+
"kind": "string",
|
|
3861
|
+
"placeholder": "49 browser tasks over 8 self-hosted apps"
|
|
3862
|
+
},
|
|
3863
|
+
{
|
|
3864
|
+
"name": "family",
|
|
3865
|
+
"required": false,
|
|
3866
|
+
"kind": "string",
|
|
3867
|
+
"enum": [
|
|
3868
|
+
"static_qa",
|
|
3869
|
+
"code",
|
|
3870
|
+
"agentic",
|
|
3871
|
+
"tools",
|
|
3872
|
+
"web",
|
|
3873
|
+
"knowledge_work",
|
|
3874
|
+
"safety",
|
|
3875
|
+
"suite"
|
|
3876
|
+
],
|
|
3877
|
+
"placeholder": "static_qa"
|
|
3878
|
+
},
|
|
3879
|
+
{
|
|
3880
|
+
"name": "sourceKind",
|
|
3881
|
+
"required": false,
|
|
3882
|
+
"kind": "string",
|
|
3883
|
+
"enum": [
|
|
3884
|
+
"hf-dataset",
|
|
3885
|
+
"jsonl",
|
|
3886
|
+
"git-tasks",
|
|
3887
|
+
"release-file",
|
|
3888
|
+
"authored",
|
|
3889
|
+
"framework"
|
|
3890
|
+
],
|
|
3891
|
+
"placeholder": "hf-dataset"
|
|
3892
|
+
},
|
|
3893
|
+
{
|
|
3894
|
+
"name": "sourceConfig",
|
|
3895
|
+
"required": false,
|
|
3896
|
+
"kind": "object",
|
|
3897
|
+
"placeholder": "{}"
|
|
3898
|
+
},
|
|
3899
|
+
{
|
|
3900
|
+
"name": "adapterId",
|
|
3901
|
+
"required": false,
|
|
3902
|
+
"kind": "string",
|
|
3903
|
+
"placeholder": "<adapterId>"
|
|
3904
|
+
},
|
|
3905
|
+
{
|
|
3906
|
+
"name": "headlineMetric",
|
|
3907
|
+
"required": false,
|
|
3908
|
+
"kind": "string",
|
|
3909
|
+
"placeholder": "<headlineMetric>"
|
|
3910
|
+
},
|
|
3911
|
+
{
|
|
3912
|
+
"name": "gradingSpec",
|
|
3913
|
+
"required": false,
|
|
3914
|
+
"kind": "object",
|
|
3915
|
+
"placeholder": "{}"
|
|
3916
|
+
},
|
|
3917
|
+
{
|
|
3918
|
+
"name": "environmentSpec",
|
|
3919
|
+
"required": false,
|
|
3920
|
+
"kind": "object",
|
|
3921
|
+
"placeholder": "{}"
|
|
3922
|
+
},
|
|
3923
|
+
{
|
|
3924
|
+
"name": "methodology",
|
|
3925
|
+
"required": false,
|
|
3926
|
+
"kind": "object",
|
|
3927
|
+
"placeholder": "{}"
|
|
3928
|
+
},
|
|
3929
|
+
{
|
|
3930
|
+
"name": "runnerKind",
|
|
3931
|
+
"required": false,
|
|
3932
|
+
"kind": "string",
|
|
3933
|
+
"enum": [
|
|
3934
|
+
"none",
|
|
3935
|
+
"image",
|
|
3936
|
+
"environment",
|
|
3937
|
+
"framework"
|
|
3938
|
+
],
|
|
3939
|
+
"placeholder": "none"
|
|
3940
|
+
},
|
|
3941
|
+
{
|
|
3942
|
+
"name": "runnerRecipe",
|
|
3943
|
+
"required": false,
|
|
3944
|
+
"kind": "string",
|
|
3945
|
+
"placeholder": "<runnerRecipe>"
|
|
3946
|
+
},
|
|
3947
|
+
{
|
|
3948
|
+
"name": "license",
|
|
3949
|
+
"required": false,
|
|
3950
|
+
"kind": "string",
|
|
3951
|
+
"placeholder": "<license>"
|
|
3952
|
+
},
|
|
3953
|
+
{
|
|
3954
|
+
"name": "licenseText",
|
|
3955
|
+
"required": false,
|
|
3956
|
+
"kind": "string",
|
|
3957
|
+
"placeholder": "<licenseText>"
|
|
3958
|
+
},
|
|
3959
|
+
{
|
|
3960
|
+
"name": "redistribution",
|
|
3961
|
+
"required": false,
|
|
3962
|
+
"kind": "string",
|
|
3963
|
+
"enum": [
|
|
3964
|
+
"allowed",
|
|
3965
|
+
"reference_only"
|
|
3966
|
+
],
|
|
3967
|
+
"placeholder": "allowed"
|
|
3968
|
+
},
|
|
3969
|
+
{
|
|
3970
|
+
"name": "access",
|
|
3971
|
+
"required": false,
|
|
3972
|
+
"kind": "string",
|
|
3973
|
+
"enum": [
|
|
3974
|
+
"open",
|
|
3975
|
+
"gated",
|
|
3976
|
+
"private"
|
|
3977
|
+
],
|
|
3978
|
+
"placeholder": "open"
|
|
3979
|
+
},
|
|
3980
|
+
{
|
|
3981
|
+
"name": "termsUrl",
|
|
3982
|
+
"required": false,
|
|
3983
|
+
"kind": "string",
|
|
3984
|
+
"placeholder": "<termsUrl>"
|
|
3985
|
+
},
|
|
3986
|
+
{
|
|
3987
|
+
"name": "homepageUrl",
|
|
3988
|
+
"required": false,
|
|
3989
|
+
"kind": "string",
|
|
3990
|
+
"placeholder": "<homepageUrl>"
|
|
3991
|
+
},
|
|
3992
|
+
{
|
|
3993
|
+
"name": "paperUrl",
|
|
3994
|
+
"required": false,
|
|
3995
|
+
"kind": "string",
|
|
3996
|
+
"placeholder": "<paperUrl>"
|
|
3997
|
+
},
|
|
3998
|
+
{
|
|
3999
|
+
"name": "defaultConfig",
|
|
4000
|
+
"required": false,
|
|
4001
|
+
"kind": "string",
|
|
4002
|
+
"placeholder": "<defaultConfig>"
|
|
4003
|
+
},
|
|
4004
|
+
{
|
|
4005
|
+
"name": "visibility",
|
|
4006
|
+
"required": false,
|
|
4007
|
+
"kind": "string",
|
|
4008
|
+
"enum": [
|
|
4009
|
+
"private",
|
|
4010
|
+
"org",
|
|
4011
|
+
"public"
|
|
4012
|
+
],
|
|
4013
|
+
"placeholder": "private"
|
|
4014
|
+
}
|
|
4015
|
+
],
|
|
4016
|
+
"watchable": false,
|
|
4017
|
+
"help": {
|
|
4018
|
+
"command": "ora study benchmarkDefinitions update windtunnel --file run.json",
|
|
4019
|
+
"body": "{\n \"description\": \"49 browser tasks over 8 self-hosted apps\"\n}"
|
|
4020
|
+
}
|
|
4021
|
+
},
|
|
4022
|
+
{
|
|
4023
|
+
"id": "study.benchmarkDefinitions.withdraw",
|
|
4024
|
+
"commandPath": [
|
|
4025
|
+
"study",
|
|
4026
|
+
"benchmarkDefinitions",
|
|
4027
|
+
"withdraw"
|
|
4028
|
+
],
|
|
4029
|
+
"method": "post",
|
|
4030
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/withdraw",
|
|
4031
|
+
"summary": "Take a pending submission back — the definition returns to draft and stays runnable",
|
|
4032
|
+
"scope": "study.studies:write",
|
|
4033
|
+
"product": "study",
|
|
4034
|
+
"pathParams": [
|
|
4035
|
+
{
|
|
4036
|
+
"name": "slug"
|
|
4037
|
+
}
|
|
4038
|
+
],
|
|
4039
|
+
"queryParams": [],
|
|
4040
|
+
"hasBody": false,
|
|
4041
|
+
"bodyRequired": false,
|
|
4042
|
+
"isStream": false,
|
|
4043
|
+
"watchable": false,
|
|
4044
|
+
"help": {
|
|
4045
|
+
"command": "ora study benchmarkDefinitions withdraw windtunnel"
|
|
4046
|
+
}
|
|
4047
|
+
},
|
|
4048
|
+
{
|
|
4049
|
+
"id": "study.benchmarkImports.get",
|
|
4050
|
+
"commandPath": [
|
|
4051
|
+
"study",
|
|
4052
|
+
"benchmarkImports",
|
|
4053
|
+
"get"
|
|
4054
|
+
],
|
|
4055
|
+
"method": "get",
|
|
4056
|
+
"pathTemplate": "/study/v1/benchmark-imports/{id}",
|
|
4057
|
+
"summary": "An import job — queued, running, then succeeded (with its new version) or failed (with its reason)",
|
|
4058
|
+
"scope": "study.studies:read",
|
|
4059
|
+
"product": "study",
|
|
4060
|
+
"pathParams": [
|
|
4061
|
+
{
|
|
4062
|
+
"name": "id"
|
|
4063
|
+
}
|
|
4064
|
+
],
|
|
4065
|
+
"queryParams": [],
|
|
4066
|
+
"hasBody": false,
|
|
4067
|
+
"bodyRequired": false,
|
|
4068
|
+
"isStream": false,
|
|
4069
|
+
"watchable": false,
|
|
4070
|
+
"help": {
|
|
4071
|
+
"command": "ora study benchmarkImports get 11111111-1111-4111-8111-111111111111"
|
|
4072
|
+
}
|
|
4073
|
+
},
|
|
4074
|
+
{
|
|
4075
|
+
"id": "study.benchmarkImports.stream",
|
|
4076
|
+
"commandPath": [
|
|
4077
|
+
"study",
|
|
4078
|
+
"benchmarkImports",
|
|
4079
|
+
"stream"
|
|
4080
|
+
],
|
|
4081
|
+
"method": "get",
|
|
4082
|
+
"pathTemplate": "/study/v1/benchmark-imports/{id}/stream",
|
|
4083
|
+
"summary": "Live SSE of one import job: its current state at once, then every change; closes after succeeded or failed",
|
|
4084
|
+
"scope": "study.studies:read",
|
|
4085
|
+
"product": "study",
|
|
4086
|
+
"pathParams": [
|
|
4087
|
+
{
|
|
4088
|
+
"name": "id"
|
|
4089
|
+
}
|
|
4090
|
+
],
|
|
4091
|
+
"queryParams": [],
|
|
4092
|
+
"hasBody": false,
|
|
4093
|
+
"bodyRequired": false,
|
|
4094
|
+
"isStream": true,
|
|
4095
|
+
"watchable": false,
|
|
4096
|
+
"help": {
|
|
4097
|
+
"command": "ora study benchmarkImports stream <id>"
|
|
4098
|
+
}
|
|
4099
|
+
},
|
|
4100
|
+
{
|
|
4101
|
+
"id": "study.benchmarkIntake.assist",
|
|
4102
|
+
"commandPath": [
|
|
4103
|
+
"study",
|
|
4104
|
+
"benchmarkIntake",
|
|
4105
|
+
"assist"
|
|
4106
|
+
],
|
|
4107
|
+
"method": "post",
|
|
4108
|
+
"pathTemplate": "/study/v1/benchmark-intake/assist",
|
|
4109
|
+
"summary": "Have an agent propose the rest of the definition from what the probe read — every field with a rationale and a confidence, grounded on this deployment's registries. It proposes; you confirm. Nothing is created.",
|
|
4110
|
+
"scope": "study.studies:write",
|
|
4111
|
+
"product": "study",
|
|
4112
|
+
"pathParams": [],
|
|
4113
|
+
"queryParams": [],
|
|
4114
|
+
"hasBody": true,
|
|
4115
|
+
"bodyRequired": true,
|
|
4116
|
+
"isStream": false,
|
|
4117
|
+
"bodyFields": [
|
|
4118
|
+
{
|
|
4119
|
+
"name": "strategy",
|
|
4120
|
+
"required": true,
|
|
4121
|
+
"kind": "string",
|
|
4122
|
+
"placeholder": "git-repo"
|
|
4123
|
+
},
|
|
4124
|
+
{
|
|
4125
|
+
"name": "config",
|
|
4126
|
+
"required": true,
|
|
4127
|
+
"kind": "object",
|
|
4128
|
+
"placeholder": "{\"repo\":\"https://github.com/nekuda-ai/WindTunnel\"}"
|
|
4129
|
+
},
|
|
4130
|
+
{
|
|
4131
|
+
"name": "probe",
|
|
4132
|
+
"required": false,
|
|
4133
|
+
"kind": "object",
|
|
4134
|
+
"placeholder": "{}"
|
|
4135
|
+
}
|
|
4136
|
+
],
|
|
4137
|
+
"watchable": false,
|
|
4138
|
+
"help": {
|
|
4139
|
+
"command": "ora study benchmarkIntake assist --file run.json",
|
|
4140
|
+
"body": "{\n \"strategy\": \"git-repo\",\n \"config\": {\n \"repo\": \"https://github.com/nekuda-ai/WindTunnel\"\n }\n}"
|
|
4141
|
+
}
|
|
4142
|
+
},
|
|
4143
|
+
{
|
|
4144
|
+
"id": "study.benchmarkIntake.probe",
|
|
4145
|
+
"commandPath": [
|
|
4146
|
+
"study",
|
|
4147
|
+
"benchmarkIntake",
|
|
4148
|
+
"probe"
|
|
4149
|
+
],
|
|
4150
|
+
"method": "post",
|
|
4151
|
+
"pathTemplate": "/study/v1/benchmark-intake/probe",
|
|
4152
|
+
"summary": "Inspect a target and propose a definition — with what it could not determine listed explicitly",
|
|
4153
|
+
"scope": "study.studies:write",
|
|
4154
|
+
"product": "study",
|
|
4155
|
+
"pathParams": [],
|
|
4156
|
+
"queryParams": [],
|
|
4157
|
+
"hasBody": true,
|
|
4158
|
+
"bodyRequired": true,
|
|
4159
|
+
"isStream": false,
|
|
4160
|
+
"bodyFields": [
|
|
4161
|
+
{
|
|
4162
|
+
"name": "strategy",
|
|
4163
|
+
"required": true,
|
|
4164
|
+
"kind": "string",
|
|
4165
|
+
"placeholder": "git-repo"
|
|
4166
|
+
},
|
|
4167
|
+
{
|
|
4168
|
+
"name": "config",
|
|
4169
|
+
"required": true,
|
|
4170
|
+
"kind": "object",
|
|
4171
|
+
"placeholder": "{\"repo\":\"https://github.com/nekuda-ai/WindTunnel\"}"
|
|
4172
|
+
}
|
|
4173
|
+
],
|
|
4174
|
+
"watchable": false,
|
|
4175
|
+
"help": {
|
|
4176
|
+
"command": "ora study benchmarkIntake probe --file run.json",
|
|
4177
|
+
"body": "{\n \"strategy\": \"git-repo\",\n \"config\": {\n \"repo\": \"https://github.com/nekuda-ai/WindTunnel\"\n }\n}"
|
|
4178
|
+
}
|
|
4179
|
+
},
|
|
4180
|
+
{
|
|
4181
|
+
"id": "study.benchmarkIntake.strategies",
|
|
4182
|
+
"commandPath": [
|
|
4183
|
+
"study",
|
|
4184
|
+
"benchmarkIntake",
|
|
4185
|
+
"strategies"
|
|
4186
|
+
],
|
|
4187
|
+
"method": "get",
|
|
4188
|
+
"pathTemplate": "/study/v1/benchmark-intake/strategies",
|
|
4189
|
+
"summary": "The ways a benchmark can be added (a repository, a dataset, a release file, an image, an upload) and the fields each one takes",
|
|
4190
|
+
"scope": "study.studies:write",
|
|
4191
|
+
"product": "study",
|
|
4192
|
+
"pathParams": [],
|
|
4193
|
+
"queryParams": [],
|
|
4194
|
+
"hasBody": false,
|
|
4195
|
+
"bodyRequired": false,
|
|
4196
|
+
"isStream": false,
|
|
4197
|
+
"watchable": false,
|
|
4198
|
+
"help": {
|
|
4199
|
+
"command": "ora study benchmarkIntake strategies"
|
|
4200
|
+
}
|
|
4201
|
+
},
|
|
4202
|
+
{
|
|
4203
|
+
"id": "study.benchmarkTerms.get",
|
|
4204
|
+
"commandPath": [
|
|
4205
|
+
"study",
|
|
4206
|
+
"benchmarkTerms",
|
|
4207
|
+
"get"
|
|
4208
|
+
],
|
|
4209
|
+
"method": "get",
|
|
4210
|
+
"pathTemplate": "/study/v1/benchmark-terms",
|
|
4211
|
+
"summary": "The platform's benchmark-data terms — what an organization accepts once to read the task text of guarded benchmarks — and this organization's acceptance, if any",
|
|
4212
|
+
"scope": "study.studies:read",
|
|
4213
|
+
"product": "study",
|
|
4214
|
+
"pathParams": [],
|
|
4215
|
+
"queryParams": [],
|
|
4216
|
+
"hasBody": false,
|
|
4217
|
+
"bodyRequired": false,
|
|
4218
|
+
"isStream": false,
|
|
4219
|
+
"watchable": false,
|
|
4220
|
+
"help": {
|
|
4221
|
+
"command": "ora study benchmarkTerms get"
|
|
4222
|
+
}
|
|
4223
|
+
},
|
|
4224
|
+
{
|
|
4225
|
+
"id": "study.catalog",
|
|
4226
|
+
"commandPath": [
|
|
4227
|
+
"study",
|
|
4228
|
+
"catalog"
|
|
4229
|
+
],
|
|
4230
|
+
"method": "get",
|
|
4231
|
+
"pathTemplate": "/study/v1/catalog",
|
|
4232
|
+
"summary": "Harness × model catalog: valid matrix cells, defaults, modes, and withheld harnesses with reasons",
|
|
4233
|
+
"scope": null,
|
|
4234
|
+
"product": "platform",
|
|
4235
|
+
"pathParams": [],
|
|
4236
|
+
"queryParams": [],
|
|
4237
|
+
"hasBody": false,
|
|
4238
|
+
"bodyRequired": false,
|
|
4239
|
+
"isStream": false,
|
|
4240
|
+
"watchable": false,
|
|
4241
|
+
"help": {
|
|
4242
|
+
"command": "ora study catalog"
|
|
4243
|
+
}
|
|
4244
|
+
},
|
|
4245
|
+
{
|
|
4246
|
+
"id": "study.convertWebmcpSuite",
|
|
4247
|
+
"commandPath": [
|
|
4248
|
+
"study",
|
|
4249
|
+
"convert-webmcp-suite"
|
|
4250
|
+
],
|
|
4251
|
+
"method": "post",
|
|
4252
|
+
"pathTemplate": "/study/v1/studies/convert/webmcp",
|
|
4253
|
+
"summary": "Convert a webmcp-evals suite JSON into ora study tasks + 'evals'-lens expectations — a PURE transform, nothing persisted; resubmit the returned `tasks` through study.create's `tasks` field",
|
|
4254
|
+
"scope": "study.studies:write",
|
|
4255
|
+
"product": "study",
|
|
4256
|
+
"pathParams": [],
|
|
4257
|
+
"queryParams": [],
|
|
4258
|
+
"hasBody": true,
|
|
4259
|
+
"bodyRequired": true,
|
|
4260
|
+
"isStream": false,
|
|
4261
|
+
"bodyFields": [
|
|
4262
|
+
{
|
|
4263
|
+
"name": "suite",
|
|
4264
|
+
"required": true,
|
|
4265
|
+
"kind": "string",
|
|
4266
|
+
"description": "The foreign webmcp-evals suite JSON (an array of {name, messages, expectedCall} cases) — validated by the converter itself; a malformed suite answers 422 with the converter's own message.",
|
|
4267
|
+
"placeholder": "[object Object]"
|
|
4268
|
+
},
|
|
4269
|
+
{
|
|
4270
|
+
"name": "ordered",
|
|
4271
|
+
"required": false,
|
|
4272
|
+
"kind": "boolean",
|
|
4273
|
+
"description": "When true, a case's multiple expectedCall entries become one ORDERED `sequence` expectation instead of independent ones (mirrors the reference project's evals.json vs evals-unordered.json split). Default false.",
|
|
4274
|
+
"placeholder": "true"
|
|
4275
|
+
}
|
|
4276
|
+
],
|
|
4277
|
+
"watchable": false,
|
|
4278
|
+
"help": {
|
|
4279
|
+
"command": "ora study convert-webmcp-suite --file run.json",
|
|
4280
|
+
"body": "{\n \"suite\": [\n {\n \"name\": \"search finds the refund policy\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"type\": \"message\",\n \"content\": \"Find the refund policy and quote it.\"\n }\n ],\n \"expectedCall\": [\n {\n \"functionName\": \"search_docs\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"refund\"\n }\n }\n }\n ]\n }\n ]\n}"
|
|
4281
|
+
}
|
|
4282
|
+
},
|
|
4283
|
+
{
|
|
4284
|
+
"id": "study.create",
|
|
4285
|
+
"commandPath": [
|
|
4286
|
+
"study",
|
|
4287
|
+
"create"
|
|
4288
|
+
],
|
|
4289
|
+
"method": "post",
|
|
4290
|
+
"pathTemplate": "/study/v1/studies",
|
|
4291
|
+
"summary": "Create a study (status 'draft', ZERO LLM spend — an authored `tasks` corpus instead lands 'ready' synchronously); approve generation via POST .../generate",
|
|
4292
|
+
"scope": "study.studies:write",
|
|
4293
|
+
"product": "study",
|
|
4294
|
+
"pathParams": [],
|
|
4295
|
+
"queryParams": [],
|
|
4296
|
+
"hasBody": true,
|
|
4297
|
+
"bodyRequired": true,
|
|
4298
|
+
"isStream": false,
|
|
4299
|
+
"bodyFields": [
|
|
4300
|
+
{
|
|
4301
|
+
"name": "name",
|
|
4302
|
+
"required": true,
|
|
4303
|
+
"kind": "string",
|
|
4304
|
+
"description": "Display name for the study (≤160 chars).",
|
|
4305
|
+
"placeholder": "tavily-search A/B"
|
|
4306
|
+
},
|
|
4307
|
+
{
|
|
4308
|
+
"name": "description",
|
|
4309
|
+
"required": false,
|
|
4310
|
+
"kind": "string",
|
|
4311
|
+
"description": "Optional longer description shown on the study card.",
|
|
4312
|
+
"placeholder": "<description>"
|
|
4313
|
+
},
|
|
4314
|
+
{
|
|
4315
|
+
"name": "subjectKind",
|
|
4316
|
+
"required": false,
|
|
4317
|
+
"kind": "string",
|
|
4318
|
+
"enum": [
|
|
4319
|
+
"skill",
|
|
4320
|
+
"tools",
|
|
4321
|
+
"addon",
|
|
4322
|
+
"custom",
|
|
4323
|
+
"benchmark"
|
|
4324
|
+
],
|
|
4325
|
+
"description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
|
|
4326
|
+
"placeholder": "addon"
|
|
4327
|
+
},
|
|
4328
|
+
{
|
|
4329
|
+
"name": "skillRef",
|
|
4330
|
+
"required": false,
|
|
4331
|
+
"kind": "string",
|
|
4332
|
+
"description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
|
|
4333
|
+
"placeholder": "<skillRef>"
|
|
4334
|
+
},
|
|
4335
|
+
{
|
|
4336
|
+
"name": "definitionVersionId",
|
|
4337
|
+
"required": false,
|
|
4338
|
+
"kind": "string",
|
|
4339
|
+
"description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
|
|
4340
|
+
"placeholder": "<definitionVersionId>"
|
|
4341
|
+
},
|
|
4342
|
+
{
|
|
4343
|
+
"name": "subjectName",
|
|
4344
|
+
"required": false,
|
|
4345
|
+
"kind": "string",
|
|
4346
|
+
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
|
|
4347
|
+
"placeholder": "Tavily search"
|
|
4348
|
+
},
|
|
4349
|
+
{
|
|
4350
|
+
"name": "armConfig",
|
|
4351
|
+
"required": false,
|
|
4352
|
+
"kind": "object",
|
|
4353
|
+
"description": "Frozen per-arm run-payload deltas. Required for subjectKind 'custom' and comparisonMode 'ab_custom'; for 'tools'/'addon' subjects the treatment side is derived when omitted.",
|
|
4354
|
+
"placeholder": "{\"treatment\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}"
|
|
4355
|
+
},
|
|
4356
|
+
{
|
|
4357
|
+
"name": "lenses",
|
|
4358
|
+
"required": false,
|
|
4359
|
+
"kind": "array",
|
|
4360
|
+
"description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
|
|
4361
|
+
"placeholder": "[\"judge\",\"evals\"]"
|
|
4362
|
+
},
|
|
4363
|
+
{
|
|
4364
|
+
"name": "judgeSupplementary",
|
|
4365
|
+
"required": false,
|
|
4366
|
+
"kind": "boolean",
|
|
4367
|
+
"description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
|
|
4368
|
+
"placeholder": "true"
|
|
4369
|
+
},
|
|
4370
|
+
{
|
|
4371
|
+
"name": "proposeExpectations",
|
|
4372
|
+
"required": false,
|
|
4373
|
+
"kind": "boolean",
|
|
4374
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
|
|
4375
|
+
"placeholder": "true"
|
|
4376
|
+
},
|
|
4377
|
+
{
|
|
4378
|
+
"name": "comparisonMode",
|
|
4379
|
+
"required": false,
|
|
4380
|
+
"kind": "string",
|
|
4381
|
+
"enum": [
|
|
4382
|
+
"ab_vs_none",
|
|
4383
|
+
"ab_custom",
|
|
4384
|
+
"single"
|
|
4385
|
+
],
|
|
4386
|
+
"description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
|
|
4387
|
+
"placeholder": "ab_vs_none"
|
|
4388
|
+
},
|
|
4389
|
+
{
|
|
4390
|
+
"name": "subjectBackend",
|
|
4391
|
+
"required": false,
|
|
4392
|
+
"kind": "string",
|
|
4393
|
+
"enum": [
|
|
4394
|
+
"model",
|
|
4395
|
+
"harness"
|
|
4396
|
+
],
|
|
4397
|
+
"description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
|
|
4398
|
+
"placeholder": "model"
|
|
4399
|
+
},
|
|
4400
|
+
{
|
|
4401
|
+
"name": "baselineLabel",
|
|
4402
|
+
"required": false,
|
|
4403
|
+
"kind": "string",
|
|
4404
|
+
"description": "Baseline arm's display label (e.g. 'REST API'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
|
|
4405
|
+
"placeholder": "<baselineLabel>"
|
|
4406
|
+
},
|
|
4407
|
+
{
|
|
4408
|
+
"name": "treatmentLabel",
|
|
4409
|
+
"required": false,
|
|
4410
|
+
"kind": "string",
|
|
4411
|
+
"description": "Treatment arm's display label (e.g. 'Remote MCP'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
|
|
4412
|
+
"placeholder": "<treatmentLabel>"
|
|
4413
|
+
},
|
|
4414
|
+
{
|
|
4415
|
+
"name": "taskCount",
|
|
4416
|
+
"required": false,
|
|
4417
|
+
"kind": "number",
|
|
4418
|
+
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
|
|
4419
|
+
"placeholder": "1"
|
|
4420
|
+
},
|
|
4421
|
+
{
|
|
4422
|
+
"name": "variantsPerTask",
|
|
4423
|
+
"required": false,
|
|
4424
|
+
"kind": "number",
|
|
4425
|
+
"description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
|
|
4426
|
+
"placeholder": "1"
|
|
4427
|
+
},
|
|
4428
|
+
{
|
|
4429
|
+
"name": "repeats",
|
|
4430
|
+
"required": false,
|
|
4431
|
+
"kind": "number",
|
|
4432
|
+
"description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
|
|
4433
|
+
"placeholder": "2"
|
|
4434
|
+
},
|
|
4435
|
+
{
|
|
4436
|
+
"name": "tasks",
|
|
4437
|
+
"required": false,
|
|
4438
|
+
"kind": "array",
|
|
4439
|
+
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
|
|
4440
|
+
"placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
|
|
4441
|
+
},
|
|
4442
|
+
{
|
|
4443
|
+
"name": "subset",
|
|
4444
|
+
"required": false,
|
|
4445
|
+
"kind": "object",
|
|
4446
|
+
"description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
|
|
4447
|
+
"placeholder": "{}"
|
|
4448
|
+
},
|
|
4449
|
+
{
|
|
4450
|
+
"name": "matrix",
|
|
4451
|
+
"required": true,
|
|
4452
|
+
"kind": "array",
|
|
4453
|
+
"description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
|
|
4454
|
+
"placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"},{\"harness\":\"codex\",\"model\":\"gpt-5.5\"}]"
|
|
4455
|
+
},
|
|
4456
|
+
{
|
|
4457
|
+
"name": "promise",
|
|
4458
|
+
"required": false,
|
|
4459
|
+
"kind": "string",
|
|
4460
|
+
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
|
|
4461
|
+
"placeholder": "Web search via Tavily produces fresher, better-sourced answers than no web access"
|
|
4462
|
+
}
|
|
4463
|
+
],
|
|
4464
|
+
"watchable": true,
|
|
4465
|
+
"help": {
|
|
4466
|
+
"command": "ora study create --file run.json --watch",
|
|
4467
|
+
"body": "{\n \"name\": \"tavily-search A/B\",\n \"subjectKind\": \"addon\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"armConfig\": {\n \"treatment\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n },\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"any_of\",\n \"label\": \"searched, by either route (treatment arm only)\",\n \"arms\": [\n \"skill\"\n ],\n \"of\": [\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"search\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"Node\"\n }\n }\n },\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"call_page_tool\",\n \"arguments\": {\n \"tool\": \"search\"\n }\n }\n ]\n },\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
|
|
4468
|
+
}
|
|
4469
|
+
},
|
|
4470
|
+
{
|
|
4471
|
+
"id": "study.delete",
|
|
4472
|
+
"commandPath": [
|
|
4473
|
+
"study",
|
|
4474
|
+
"delete"
|
|
4475
|
+
],
|
|
4476
|
+
"method": "delete",
|
|
4477
|
+
"pathTemplate": "/study/v1/studies/{id}",
|
|
4478
|
+
"summary": "Delete a study and its tasks/facts (409 while running)",
|
|
4479
|
+
"scope": "study.studies:write",
|
|
4480
|
+
"product": "study",
|
|
4481
|
+
"pathParams": [
|
|
4482
|
+
{
|
|
4483
|
+
"name": "id"
|
|
4484
|
+
}
|
|
4485
|
+
],
|
|
4486
|
+
"queryParams": [],
|
|
4487
|
+
"hasBody": false,
|
|
4488
|
+
"bodyRequired": false,
|
|
4489
|
+
"isStream": false,
|
|
4490
|
+
"watchable": false,
|
|
4491
|
+
"help": {
|
|
4492
|
+
"command": "ora study delete <id>"
|
|
4493
|
+
}
|
|
4494
|
+
},
|
|
4495
|
+
{
|
|
4496
|
+
"id": "study.generate",
|
|
4497
|
+
"commandPath": [
|
|
4498
|
+
"study",
|
|
4499
|
+
"generate"
|
|
4500
|
+
],
|
|
4501
|
+
"method": "post",
|
|
4502
|
+
"pathTemplate": "/study/v1/studies/{id}/generate",
|
|
4503
|
+
"summary": "Approve task generation from 'draft' (first generation), or regenerate from 'ready'/'failed' (wipes existing tasks and their rubrics)",
|
|
4504
|
+
"scope": "study.studies:write",
|
|
4505
|
+
"product": "study",
|
|
4506
|
+
"pathParams": [
|
|
4507
|
+
{
|
|
4508
|
+
"name": "id"
|
|
4509
|
+
}
|
|
4510
|
+
],
|
|
4511
|
+
"queryParams": [],
|
|
4512
|
+
"hasBody": true,
|
|
4513
|
+
"bodyRequired": true,
|
|
4514
|
+
"isStream": false,
|
|
4515
|
+
"bodyFields": [
|
|
4516
|
+
{
|
|
4517
|
+
"name": "proposeExpectations",
|
|
4518
|
+
"required": false,
|
|
4519
|
+
"kind": "boolean",
|
|
4520
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each task this generation produces — the generate-time re-assertion of create's `proposeExpectations` (create-time no longer fires generation itself, so this is where the opt-in actually takes effect). Same REQUIRES/IGNORED rules as create's field.",
|
|
4521
|
+
"placeholder": "true"
|
|
4522
|
+
}
|
|
4523
|
+
],
|
|
4524
|
+
"watchable": true,
|
|
4525
|
+
"help": {
|
|
4526
|
+
"command": "ora study generate <id> --file run.json --watch",
|
|
4527
|
+
"body": "{\n \"proposeExpectations\": true\n}"
|
|
4528
|
+
}
|
|
4529
|
+
},
|
|
4530
|
+
{
|
|
4531
|
+
"id": "study.get",
|
|
4532
|
+
"commandPath": [
|
|
4533
|
+
"study",
|
|
4534
|
+
"get"
|
|
4535
|
+
],
|
|
4536
|
+
"method": "get",
|
|
4537
|
+
"pathTemplate": "/study/v1/studies/{id}",
|
|
4538
|
+
"summary": "Study detail: status to poll, tasks + rubrics, per-run facts, live aggregate + verdict",
|
|
4539
|
+
"scope": "study.studies:read",
|
|
4540
|
+
"product": "study",
|
|
4541
|
+
"pathParams": [
|
|
4542
|
+
{
|
|
4543
|
+
"name": "id"
|
|
4544
|
+
}
|
|
4545
|
+
],
|
|
4546
|
+
"queryParams": [],
|
|
4547
|
+
"hasBody": false,
|
|
4548
|
+
"bodyRequired": false,
|
|
4549
|
+
"isStream": false,
|
|
4550
|
+
"watchable": false,
|
|
4551
|
+
"help": {
|
|
4552
|
+
"command": "ora study get 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4553
|
+
}
|
|
4554
|
+
},
|
|
4555
|
+
{
|
|
4556
|
+
"id": "study.images.builds.cancel",
|
|
4557
|
+
"commandPath": [
|
|
4558
|
+
"study",
|
|
4559
|
+
"images",
|
|
4560
|
+
"builds",
|
|
4561
|
+
"cancel"
|
|
4562
|
+
],
|
|
4563
|
+
"method": "post",
|
|
4564
|
+
"pathTemplate": "/study/v1/images/builds/{id}/cancel",
|
|
4565
|
+
"summary": "Cancel an in-flight image build",
|
|
4566
|
+
"scope": "images:write",
|
|
4567
|
+
"product": "platform",
|
|
4568
|
+
"pathParams": [
|
|
4569
|
+
{
|
|
4570
|
+
"name": "id"
|
|
4571
|
+
}
|
|
4572
|
+
],
|
|
4573
|
+
"queryParams": [],
|
|
4574
|
+
"hasBody": false,
|
|
4575
|
+
"bodyRequired": false,
|
|
4576
|
+
"isStream": false,
|
|
4577
|
+
"watchable": false,
|
|
4578
|
+
"help": {
|
|
4579
|
+
"command": "ora study images builds cancel <id>"
|
|
4580
|
+
}
|
|
4581
|
+
},
|
|
4582
|
+
{
|
|
4583
|
+
"id": "study.images.builds.create",
|
|
4584
|
+
"commandPath": [
|
|
4585
|
+
"study",
|
|
4586
|
+
"images",
|
|
4587
|
+
"builds",
|
|
4588
|
+
"create"
|
|
4589
|
+
],
|
|
4590
|
+
"method": "post",
|
|
4591
|
+
"pathTemplate": "/study/v1/images/builds",
|
|
4592
|
+
"summary": "Submit an image build (idempotent on name + source + Dockerfile + build args)",
|
|
4593
|
+
"scope": "images:write",
|
|
4594
|
+
"product": "platform",
|
|
4595
|
+
"pathParams": [],
|
|
4596
|
+
"queryParams": [],
|
|
4597
|
+
"hasBody": true,
|
|
4598
|
+
"bodyRequired": true,
|
|
4599
|
+
"isStream": false,
|
|
4600
|
+
"bodyFields": [
|
|
4601
|
+
{
|
|
4602
|
+
"name": "name",
|
|
4603
|
+
"required": true,
|
|
4604
|
+
"kind": "string",
|
|
4605
|
+
"placeholder": "<name>"
|
|
4606
|
+
},
|
|
4607
|
+
{
|
|
4608
|
+
"name": "source",
|
|
4609
|
+
"required": true,
|
|
4610
|
+
"kind": "string",
|
|
4611
|
+
"placeholder": "<source>"
|
|
4612
|
+
},
|
|
4613
|
+
{
|
|
4614
|
+
"name": "dockerfile",
|
|
4615
|
+
"required": false,
|
|
4616
|
+
"kind": "object",
|
|
4617
|
+
"placeholder": "{}"
|
|
4618
|
+
},
|
|
4619
|
+
{
|
|
4620
|
+
"name": "buildArgs",
|
|
4621
|
+
"required": false,
|
|
4622
|
+
"kind": "object",
|
|
4623
|
+
"placeholder": "{}"
|
|
4624
|
+
},
|
|
4625
|
+
{
|
|
4626
|
+
"name": "platform",
|
|
4627
|
+
"required": false,
|
|
4628
|
+
"kind": "string",
|
|
4629
|
+
"placeholder": "<platform>"
|
|
4630
|
+
}
|
|
4631
|
+
],
|
|
4632
|
+
"watchable": false,
|
|
4633
|
+
"help": {
|
|
4634
|
+
"command": "ora study images builds create --file run.json",
|
|
4635
|
+
"body": "{\n \"name\": \"<name>\",\n \"source\": \"<source>\"\n}"
|
|
4636
|
+
}
|
|
4637
|
+
},
|
|
4638
|
+
{
|
|
4639
|
+
"id": "study.images.builds.get",
|
|
4640
|
+
"commandPath": [
|
|
4641
|
+
"study",
|
|
4642
|
+
"images",
|
|
4643
|
+
"builds",
|
|
4644
|
+
"get"
|
|
4645
|
+
],
|
|
4646
|
+
"method": "get",
|
|
4647
|
+
"pathTemplate": "/study/v1/images/builds/{id}",
|
|
4648
|
+
"summary": "Get one image build",
|
|
4649
|
+
"scope": "images:read",
|
|
4650
|
+
"product": "platform",
|
|
4651
|
+
"pathParams": [
|
|
4652
|
+
{
|
|
4653
|
+
"name": "id"
|
|
4654
|
+
}
|
|
4655
|
+
],
|
|
4656
|
+
"queryParams": [],
|
|
4657
|
+
"hasBody": false,
|
|
4658
|
+
"bodyRequired": false,
|
|
4659
|
+
"isStream": false,
|
|
4660
|
+
"watchable": false,
|
|
4661
|
+
"help": {
|
|
4662
|
+
"command": "ora study images builds get <id>"
|
|
4663
|
+
}
|
|
4664
|
+
},
|
|
4665
|
+
{
|
|
4666
|
+
"id": "study.images.builds.list",
|
|
4667
|
+
"commandPath": [
|
|
4668
|
+
"study",
|
|
4669
|
+
"images",
|
|
4670
|
+
"builds",
|
|
4671
|
+
"list"
|
|
4672
|
+
],
|
|
4673
|
+
"method": "get",
|
|
4674
|
+
"pathTemplate": "/study/v1/images/builds",
|
|
4675
|
+
"summary": "List the org's image builds, newest first",
|
|
4676
|
+
"scope": "images:read",
|
|
4677
|
+
"product": "platform",
|
|
4678
|
+
"pathParams": [],
|
|
4679
|
+
"queryParams": [
|
|
4680
|
+
{
|
|
4681
|
+
"name": "phase",
|
|
4682
|
+
"flag": "phase",
|
|
4683
|
+
"optionKey": "phase",
|
|
4684
|
+
"required": false,
|
|
4685
|
+
"kind": "string",
|
|
4686
|
+
"enum": [
|
|
4687
|
+
"pending",
|
|
4688
|
+
"running",
|
|
4689
|
+
"succeeded",
|
|
4690
|
+
"failed",
|
|
4691
|
+
"cancelled"
|
|
4692
|
+
]
|
|
4693
|
+
},
|
|
4694
|
+
{
|
|
4695
|
+
"name": "name",
|
|
4696
|
+
"flag": "name",
|
|
4697
|
+
"optionKey": "name",
|
|
4698
|
+
"required": false,
|
|
4699
|
+
"kind": "string"
|
|
4700
|
+
},
|
|
4701
|
+
{
|
|
4702
|
+
"name": "pageSize",
|
|
4703
|
+
"flag": "page-size",
|
|
4704
|
+
"optionKey": "pageSize",
|
|
4705
|
+
"required": false,
|
|
4706
|
+
"kind": "number"
|
|
4707
|
+
},
|
|
4708
|
+
{
|
|
4709
|
+
"name": "pageToken",
|
|
4710
|
+
"flag": "page-token",
|
|
4711
|
+
"optionKey": "pageToken",
|
|
4712
|
+
"required": false,
|
|
4713
|
+
"kind": "string"
|
|
4714
|
+
}
|
|
4715
|
+
],
|
|
4716
|
+
"hasBody": false,
|
|
4717
|
+
"bodyRequired": false,
|
|
4718
|
+
"isStream": false,
|
|
4719
|
+
"watchable": false,
|
|
4720
|
+
"help": {
|
|
4721
|
+
"command": "ora study images builds list"
|
|
4722
|
+
}
|
|
4723
|
+
},
|
|
4724
|
+
{
|
|
4725
|
+
"id": "study.images.builds.stream",
|
|
4726
|
+
"commandPath": [
|
|
4727
|
+
"study",
|
|
4728
|
+
"images",
|
|
4729
|
+
"builds",
|
|
4730
|
+
"stream"
|
|
4731
|
+
],
|
|
4732
|
+
"method": "get",
|
|
4733
|
+
"pathTemplate": "/study/v1/images/builds/{id}/stream",
|
|
4734
|
+
"summary": "SSE stream of one image build's row changes",
|
|
4735
|
+
"scope": "images:read",
|
|
4736
|
+
"product": "platform",
|
|
4737
|
+
"pathParams": [
|
|
4738
|
+
{
|
|
4739
|
+
"name": "id"
|
|
4740
|
+
}
|
|
4741
|
+
],
|
|
4742
|
+
"queryParams": [],
|
|
4743
|
+
"hasBody": false,
|
|
4744
|
+
"bodyRequired": false,
|
|
4745
|
+
"isStream": true,
|
|
4746
|
+
"watchable": false,
|
|
4747
|
+
"help": {
|
|
4748
|
+
"command": "ora study images builds stream <id>"
|
|
4749
|
+
}
|
|
4750
|
+
},
|
|
4751
|
+
{
|
|
4752
|
+
"id": "study.images.runs.create",
|
|
4753
|
+
"commandPath": [
|
|
4754
|
+
"study",
|
|
4755
|
+
"images",
|
|
4756
|
+
"runs",
|
|
4757
|
+
"create"
|
|
4758
|
+
],
|
|
4759
|
+
"method": "post",
|
|
4760
|
+
"pathTemplate": "/study/v1/images/runs",
|
|
4761
|
+
"summary": "Launch a run (harness or environment) from a built or referenced image",
|
|
4762
|
+
"scope": "images:write",
|
|
4763
|
+
"product": "platform",
|
|
4764
|
+
"pathParams": [],
|
|
4765
|
+
"queryParams": [],
|
|
4766
|
+
"hasBody": true,
|
|
4767
|
+
"bodyRequired": true,
|
|
4768
|
+
"isStream": false,
|
|
4769
|
+
"bodyFields": [
|
|
4770
|
+
{
|
|
4771
|
+
"name": "buildId",
|
|
4772
|
+
"required": false,
|
|
4773
|
+
"kind": "string",
|
|
4774
|
+
"placeholder": "<buildId>"
|
|
4775
|
+
},
|
|
4776
|
+
{
|
|
4777
|
+
"name": "image",
|
|
4778
|
+
"required": false,
|
|
4779
|
+
"kind": "string",
|
|
4780
|
+
"placeholder": "<image>"
|
|
4781
|
+
},
|
|
4782
|
+
{
|
|
4783
|
+
"name": "mode",
|
|
4784
|
+
"required": true,
|
|
4785
|
+
"kind": "string",
|
|
4786
|
+
"enum": [
|
|
4787
|
+
"harness",
|
|
4788
|
+
"environment"
|
|
4789
|
+
],
|
|
4790
|
+
"placeholder": "harness"
|
|
4791
|
+
},
|
|
4792
|
+
{
|
|
4793
|
+
"name": "harness",
|
|
4794
|
+
"required": false,
|
|
4795
|
+
"kind": "string",
|
|
4796
|
+
"placeholder": "<harness>"
|
|
4797
|
+
},
|
|
4798
|
+
{
|
|
4799
|
+
"name": "task",
|
|
4800
|
+
"required": false,
|
|
4801
|
+
"kind": "string",
|
|
4802
|
+
"placeholder": "<task>"
|
|
4803
|
+
}
|
|
4804
|
+
],
|
|
4805
|
+
"watchable": false,
|
|
4806
|
+
"help": {
|
|
4807
|
+
"command": "ora study images runs create --file run.json",
|
|
4808
|
+
"body": "{\n \"mode\": \"harness\"\n}"
|
|
4809
|
+
}
|
|
4810
|
+
},
|
|
4811
|
+
{
|
|
4812
|
+
"id": "study.launch",
|
|
4813
|
+
"commandPath": [
|
|
4814
|
+
"study",
|
|
4815
|
+
"launch"
|
|
4816
|
+
],
|
|
4817
|
+
"method": "post",
|
|
4818
|
+
"pathTemplate": "/study/v1/studies/{id}/launch",
|
|
4819
|
+
"summary": "Launch: fan out task × matrix-cell × arm runs (async — poll get until status 'completed')",
|
|
4820
|
+
"scope": "study.studies:write",
|
|
4821
|
+
"product": "study",
|
|
4822
|
+
"pathParams": [
|
|
4823
|
+
{
|
|
4824
|
+
"name": "id"
|
|
4825
|
+
}
|
|
4826
|
+
],
|
|
4827
|
+
"queryParams": [],
|
|
4828
|
+
"hasBody": false,
|
|
4829
|
+
"bodyRequired": false,
|
|
4830
|
+
"isStream": false,
|
|
4831
|
+
"watchable": true,
|
|
4832
|
+
"help": {
|
|
4833
|
+
"command": "ora study launch <id> --watch"
|
|
4834
|
+
}
|
|
4835
|
+
},
|
|
4836
|
+
{
|
|
4837
|
+
"id": "study.list",
|
|
4838
|
+
"commandPath": [
|
|
4839
|
+
"study",
|
|
4840
|
+
"list"
|
|
4841
|
+
],
|
|
4842
|
+
"method": "get",
|
|
4843
|
+
"pathTemplate": "/study/v1/studies",
|
|
4844
|
+
"summary": "List the org's studies (id, status, verdict, subject)",
|
|
4845
|
+
"scope": "study.studies:read",
|
|
4846
|
+
"product": "study",
|
|
4847
|
+
"pathParams": [],
|
|
4848
|
+
"queryParams": [],
|
|
4849
|
+
"hasBody": false,
|
|
4850
|
+
"bodyRequired": false,
|
|
4851
|
+
"isStream": false,
|
|
4852
|
+
"watchable": false,
|
|
4853
|
+
"help": {
|
|
4854
|
+
"command": "ora study list"
|
|
4855
|
+
}
|
|
4856
|
+
},
|
|
4857
|
+
{
|
|
4858
|
+
"id": "study.listRunAnswers",
|
|
4859
|
+
"commandPath": [
|
|
4860
|
+
"study",
|
|
4861
|
+
"list-run-answers"
|
|
4862
|
+
],
|
|
4863
|
+
"method": "get",
|
|
4864
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/answers",
|
|
4865
|
+
"summary": "List the agent runs that answered a framework benchmark run's requests (the 'harness' subject backend), oldest first. Empty for a run that asked for none.",
|
|
4866
|
+
"scope": "study.studies:read",
|
|
4867
|
+
"product": "study",
|
|
4868
|
+
"pathParams": [
|
|
4869
|
+
{
|
|
4870
|
+
"name": "id"
|
|
4871
|
+
},
|
|
4872
|
+
{
|
|
4873
|
+
"name": "runId"
|
|
4874
|
+
}
|
|
4875
|
+
],
|
|
4876
|
+
"queryParams": [],
|
|
4877
|
+
"hasBody": false,
|
|
4878
|
+
"bodyRequired": false,
|
|
4879
|
+
"isStream": false,
|
|
4880
|
+
"watchable": false,
|
|
4881
|
+
"help": {
|
|
4882
|
+
"command": "ora study list-run-answers 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 7c9e6679-7425-40de-944b-e07fc1f90ae7"
|
|
4883
|
+
}
|
|
4884
|
+
},
|
|
4885
|
+
{
|
|
4886
|
+
"id": "study.preview",
|
|
4887
|
+
"commandPath": [
|
|
4888
|
+
"study",
|
|
4889
|
+
"preview"
|
|
4890
|
+
],
|
|
4891
|
+
"method": "get",
|
|
4892
|
+
"pathTemplate": "/study/v1/studies/{id}/preview",
|
|
4893
|
+
"summary": "Preview the PUBLISHED page's exact payload before publishing — the same projection, aggregate and behavior block publicStudies.get will serve, org-scoped and available at any status (no visibility requirement)",
|
|
4894
|
+
"scope": "study.studies:read",
|
|
4895
|
+
"product": "study",
|
|
4896
|
+
"pathParams": [
|
|
4897
|
+
{
|
|
4898
|
+
"name": "id"
|
|
4899
|
+
}
|
|
4900
|
+
],
|
|
4901
|
+
"queryParams": [],
|
|
4902
|
+
"hasBody": false,
|
|
4903
|
+
"bodyRequired": false,
|
|
4904
|
+
"isStream": false,
|
|
4905
|
+
"watchable": false,
|
|
4906
|
+
"help": {
|
|
4907
|
+
"command": "ora study preview 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4908
|
+
}
|
|
4909
|
+
},
|
|
4910
|
+
{
|
|
4911
|
+
"id": "study.publish",
|
|
4912
|
+
"commandPath": [
|
|
4913
|
+
"study",
|
|
4914
|
+
"publish"
|
|
4915
|
+
],
|
|
4916
|
+
"method": "post",
|
|
4917
|
+
"pathTemplate": "/study/v1/studies/{id}/publish",
|
|
4918
|
+
"summary": "Publish a completed study under a shareable mode: 'public' (listed in the anonymous catalog, the default) or — where the deployment supports sharing — 'link' (unlisted, openable only with the returned share token). Idempotent slug upsert.",
|
|
4919
|
+
"scope": "study.studies:write",
|
|
4920
|
+
"product": "study",
|
|
4921
|
+
"pathParams": [
|
|
4922
|
+
{
|
|
4923
|
+
"name": "id"
|
|
4924
|
+
}
|
|
4925
|
+
],
|
|
4926
|
+
"queryParams": [],
|
|
4927
|
+
"hasBody": true,
|
|
4928
|
+
"bodyRequired": true,
|
|
4929
|
+
"isStream": false,
|
|
4930
|
+
"bodyFields": [
|
|
4931
|
+
{
|
|
4932
|
+
"name": "slug",
|
|
4933
|
+
"required": false,
|
|
4934
|
+
"kind": "string",
|
|
4935
|
+
"description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the study's own generated slug. 409 study_slug_taken if another published study holds it.",
|
|
4936
|
+
"placeholder": "tavily-search-ab"
|
|
4937
|
+
},
|
|
4938
|
+
{
|
|
4939
|
+
"name": "visibility",
|
|
4940
|
+
"required": false,
|
|
4941
|
+
"kind": "string",
|
|
4942
|
+
"enum": [
|
|
4943
|
+
"private",
|
|
4944
|
+
"org",
|
|
4945
|
+
"link",
|
|
4946
|
+
"public"
|
|
4947
|
+
],
|
|
4948
|
+
"description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
|
|
4949
|
+
"placeholder": "private"
|
|
4950
|
+
}
|
|
4951
|
+
],
|
|
4952
|
+
"watchable": false,
|
|
4953
|
+
"help": {
|
|
4954
|
+
"command": "ora study publish 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
4955
|
+
"body": "{\n \"slug\": \"tavily-search-ab\"\n}"
|
|
4956
|
+
}
|
|
4957
|
+
},
|
|
4958
|
+
{
|
|
4959
|
+
"id": "study.reconcileRuns",
|
|
4960
|
+
"commandPath": [
|
|
4961
|
+
"study",
|
|
4962
|
+
"reconcile-runs"
|
|
4963
|
+
],
|
|
4964
|
+
"method": "post",
|
|
4965
|
+
"pathTemplate": "/study/v1/studies/{id}/reconcile-runs",
|
|
4966
|
+
"summary": "Re-derive a study's run projection from the authoritative run rows and re-probe completion (platform-admin only; idempotent) — the repair for runs left non-terminal in the projection while the spine says they finished",
|
|
4967
|
+
"scope": "study.studies:write",
|
|
4968
|
+
"product": "study",
|
|
4969
|
+
"pathParams": [
|
|
4970
|
+
{
|
|
4971
|
+
"name": "id"
|
|
4972
|
+
}
|
|
4973
|
+
],
|
|
4974
|
+
"queryParams": [],
|
|
4975
|
+
"hasBody": false,
|
|
4976
|
+
"bodyRequired": false,
|
|
4977
|
+
"isStream": false,
|
|
4978
|
+
"watchable": false,
|
|
4979
|
+
"help": {
|
|
4980
|
+
"command": "ora study reconcile-runs 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4981
|
+
}
|
|
4982
|
+
},
|
|
4983
|
+
{
|
|
4984
|
+
"id": "study.regenerateSummary",
|
|
4985
|
+
"commandPath": [
|
|
4986
|
+
"study",
|
|
4987
|
+
"regenerate-summary"
|
|
4988
|
+
],
|
|
4989
|
+
"method": "post",
|
|
4990
|
+
"pathTemplate": "/study/v1/studies/{id}/exec-summary",
|
|
4991
|
+
"summary": "Regenerate the executive summary from the current aggregate (async; poll get)",
|
|
4992
|
+
"scope": "study.studies:write",
|
|
4993
|
+
"product": "study",
|
|
4994
|
+
"pathParams": [
|
|
4995
|
+
{
|
|
4996
|
+
"name": "id"
|
|
4997
|
+
}
|
|
4998
|
+
],
|
|
4999
|
+
"queryParams": [],
|
|
5000
|
+
"hasBody": false,
|
|
5001
|
+
"bodyRequired": false,
|
|
5002
|
+
"isStream": false,
|
|
5003
|
+
"watchable": false,
|
|
5004
|
+
"help": {
|
|
5005
|
+
"command": "ora study regenerate-summary 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
5006
|
+
}
|
|
5007
|
+
},
|
|
5008
|
+
{
|
|
5009
|
+
"id": "study.regrade",
|
|
5010
|
+
"commandPath": [
|
|
5011
|
+
"study",
|
|
5012
|
+
"regrade"
|
|
5013
|
+
],
|
|
5014
|
+
"method": "post",
|
|
5015
|
+
"pathTemplate": "/study/v1/studies/{id}/regrade",
|
|
5016
|
+
"summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
|
|
5017
|
+
"scope": "study.studies:write",
|
|
5018
|
+
"product": "study",
|
|
5019
|
+
"pathParams": [
|
|
5020
|
+
{
|
|
5021
|
+
"name": "id"
|
|
5022
|
+
}
|
|
5023
|
+
],
|
|
5024
|
+
"queryParams": [],
|
|
5025
|
+
"hasBody": false,
|
|
5026
|
+
"bodyRequired": false,
|
|
5027
|
+
"isStream": false,
|
|
5028
|
+
"watchable": true,
|
|
5029
|
+
"help": {
|
|
5030
|
+
"command": "ora study regrade <id> --watch"
|
|
5031
|
+
}
|
|
5032
|
+
},
|
|
5033
|
+
{
|
|
5034
|
+
"id": "study.rotateShareToken",
|
|
5035
|
+
"commandPath": [
|
|
5036
|
+
"study",
|
|
5037
|
+
"rotate-share-token"
|
|
5038
|
+
],
|
|
5039
|
+
"method": "post",
|
|
5040
|
+
"pathTemplate": "/study/v1/studies/{id}/share-token/rotate",
|
|
5041
|
+
"summary": "Mint a fresh share link for a 'link' study. The previous link stops working IMMEDIATELY — there is no grace window.",
|
|
5042
|
+
"scope": "study.studies:write",
|
|
5043
|
+
"product": "study",
|
|
5044
|
+
"pathParams": [
|
|
5045
|
+
{
|
|
5046
|
+
"name": "id"
|
|
5047
|
+
}
|
|
5048
|
+
],
|
|
5049
|
+
"queryParams": [],
|
|
5050
|
+
"hasBody": false,
|
|
5051
|
+
"bodyRequired": false,
|
|
5052
|
+
"isStream": false,
|
|
5053
|
+
"watchable": false,
|
|
5054
|
+
"help": {
|
|
5055
|
+
"command": "ora study rotate-share-token <id>"
|
|
5056
|
+
}
|
|
5057
|
+
},
|
|
5058
|
+
{
|
|
5059
|
+
"id": "study.runInsight",
|
|
5060
|
+
"commandPath": [
|
|
5061
|
+
"study",
|
|
5062
|
+
"run-insight"
|
|
5063
|
+
],
|
|
5064
|
+
"method": "get",
|
|
5065
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/insight",
|
|
5066
|
+
"summary": "Read a study run's full insight with the study's access permissions",
|
|
5067
|
+
"scope": "study.studies:read",
|
|
5068
|
+
"product": "study",
|
|
5069
|
+
"pathParams": [
|
|
5070
|
+
{
|
|
5071
|
+
"name": "id"
|
|
5072
|
+
},
|
|
5073
|
+
{
|
|
5074
|
+
"name": "runId"
|
|
5075
|
+
}
|
|
5076
|
+
],
|
|
5077
|
+
"queryParams": [],
|
|
5078
|
+
"hasBody": false,
|
|
5079
|
+
"bodyRequired": false,
|
|
5080
|
+
"isStream": false,
|
|
5081
|
+
"watchable": false,
|
|
5082
|
+
"help": {
|
|
5083
|
+
"command": "ora study run-insight <id> <runId>"
|
|
5084
|
+
}
|
|
5085
|
+
},
|
|
5086
|
+
{
|
|
5087
|
+
"id": "study.runTrajectory",
|
|
5088
|
+
"commandPath": [
|
|
5089
|
+
"study",
|
|
5090
|
+
"run-trajectory"
|
|
5091
|
+
],
|
|
5092
|
+
"method": "get",
|
|
5093
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/trajectory",
|
|
5094
|
+
"summary": "Read a study run's full trajectory with the study's access permissions",
|
|
5095
|
+
"scope": "study.studies:read",
|
|
5096
|
+
"product": "study",
|
|
5097
|
+
"pathParams": [
|
|
5098
|
+
{
|
|
5099
|
+
"name": "id"
|
|
5100
|
+
},
|
|
5101
|
+
{
|
|
5102
|
+
"name": "runId"
|
|
5103
|
+
}
|
|
5104
|
+
],
|
|
5105
|
+
"queryParams": [
|
|
5106
|
+
{
|
|
5107
|
+
"name": "profile",
|
|
5108
|
+
"flag": "profile",
|
|
5109
|
+
"optionKey": "profile",
|
|
5110
|
+
"required": false,
|
|
5111
|
+
"kind": "string",
|
|
5112
|
+
"enum": [
|
|
5113
|
+
"raw",
|
|
5114
|
+
"heuristic",
|
|
5115
|
+
"full"
|
|
5116
|
+
]
|
|
5117
|
+
}
|
|
5118
|
+
],
|
|
5119
|
+
"hasBody": false,
|
|
5120
|
+
"bodyRequired": false,
|
|
5121
|
+
"isStream": false,
|
|
5122
|
+
"watchable": false,
|
|
5123
|
+
"help": {
|
|
5124
|
+
"command": "ora study run-trajectory <id> <runId>"
|
|
5125
|
+
}
|
|
5126
|
+
},
|
|
5127
|
+
{
|
|
5128
|
+
"id": "study.runs.cancel",
|
|
5129
|
+
"commandPath": [
|
|
5130
|
+
"study",
|
|
5131
|
+
"runs",
|
|
5132
|
+
"cancel"
|
|
5133
|
+
],
|
|
5134
|
+
"method": "post",
|
|
5135
|
+
"pathTemplate": "/study/v1/runs/{id}/cancel",
|
|
5136
|
+
"summary": "Cancel a run (terminate it and stop its engine)",
|
|
5137
|
+
"scope": "runs:write",
|
|
5138
|
+
"product": "platform",
|
|
5139
|
+
"pathParams": [
|
|
5140
|
+
{
|
|
5141
|
+
"name": "id"
|
|
5142
|
+
}
|
|
5143
|
+
],
|
|
5144
|
+
"queryParams": [],
|
|
5145
|
+
"hasBody": false,
|
|
5146
|
+
"bodyRequired": false,
|
|
5147
|
+
"isStream": false,
|
|
5148
|
+
"watchable": false,
|
|
5149
|
+
"help": {
|
|
5150
|
+
"command": "ora study runs cancel <id>"
|
|
5151
|
+
}
|
|
5152
|
+
},
|
|
5153
|
+
{
|
|
5154
|
+
"id": "study.runs.cancelBatch",
|
|
5155
|
+
"commandPath": [
|
|
5156
|
+
"study",
|
|
5157
|
+
"runs",
|
|
5158
|
+
"cancel-batch"
|
|
5159
|
+
],
|
|
5160
|
+
"method": "post",
|
|
5161
|
+
"pathTemplate": "/study/v1/runs/cancel",
|
|
5162
|
+
"summary": "Cancel a batch of runs (terminate them and stop their engines)",
|
|
5163
|
+
"scope": "runs:write",
|
|
5164
|
+
"product": "platform",
|
|
5165
|
+
"pathParams": [],
|
|
5166
|
+
"queryParams": [],
|
|
5167
|
+
"hasBody": true,
|
|
5168
|
+
"bodyRequired": true,
|
|
5169
|
+
"isStream": false,
|
|
5170
|
+
"bodyFields": [
|
|
5171
|
+
{
|
|
5172
|
+
"name": "runIds",
|
|
5173
|
+
"required": true,
|
|
5174
|
+
"kind": "array",
|
|
5175
|
+
"placeholder": "[]"
|
|
5176
|
+
}
|
|
5177
|
+
],
|
|
5178
|
+
"watchable": false,
|
|
5179
|
+
"help": {
|
|
5180
|
+
"command": "ora study runs cancel-batch --file run.json",
|
|
5181
|
+
"body": "{\n \"runIds\": []\n}"
|
|
5182
|
+
}
|
|
5183
|
+
},
|
|
5184
|
+
{
|
|
5185
|
+
"id": "study.runs.create",
|
|
5186
|
+
"commandPath": [
|
|
5187
|
+
"study",
|
|
5188
|
+
"runs",
|
|
5189
|
+
"create"
|
|
5190
|
+
],
|
|
5191
|
+
"method": "post",
|
|
5192
|
+
"pathTemplate": "/study/v1/runs",
|
|
5193
|
+
"summary": "Create a run",
|
|
5194
|
+
"scope": "runs:write",
|
|
5195
|
+
"product": "platform",
|
|
5196
|
+
"pathParams": [],
|
|
5197
|
+
"queryParams": [],
|
|
5198
|
+
"hasBody": true,
|
|
5199
|
+
"bodyRequired": true,
|
|
5200
|
+
"isStream": false,
|
|
5201
|
+
"bodyFields": [
|
|
5202
|
+
{
|
|
5203
|
+
"name": "product_key",
|
|
5204
|
+
"required": false,
|
|
5205
|
+
"kind": "string",
|
|
5206
|
+
"placeholder": "<product_key>"
|
|
5207
|
+
},
|
|
5208
|
+
{
|
|
5209
|
+
"name": "intent",
|
|
5210
|
+
"required": true,
|
|
5211
|
+
"kind": "string",
|
|
5212
|
+
"placeholder": "check pricing for acme.com"
|
|
5213
|
+
},
|
|
5214
|
+
{
|
|
5215
|
+
"name": "domain",
|
|
5216
|
+
"required": false,
|
|
5217
|
+
"kind": "string",
|
|
5218
|
+
"placeholder": "<domain>"
|
|
5219
|
+
},
|
|
5220
|
+
{
|
|
5221
|
+
"name": "harness",
|
|
5222
|
+
"required": false,
|
|
5223
|
+
"kind": "string",
|
|
5224
|
+
"placeholder": "eve"
|
|
5225
|
+
},
|
|
5226
|
+
{
|
|
5227
|
+
"name": "model",
|
|
5228
|
+
"required": false,
|
|
5229
|
+
"kind": "string",
|
|
5230
|
+
"placeholder": "claude-sonnet-5"
|
|
5231
|
+
},
|
|
5232
|
+
{
|
|
5233
|
+
"name": "mode",
|
|
5234
|
+
"required": false,
|
|
5235
|
+
"kind": "string",
|
|
5236
|
+
"enum": [
|
|
5237
|
+
"task",
|
|
5238
|
+
"session"
|
|
5239
|
+
],
|
|
5240
|
+
"placeholder": "task"
|
|
5241
|
+
},
|
|
5242
|
+
{
|
|
5243
|
+
"name": "addons",
|
|
5244
|
+
"required": false,
|
|
5245
|
+
"kind": "object",
|
|
5246
|
+
"placeholder": "{}"
|
|
5247
|
+
},
|
|
5248
|
+
{
|
|
5249
|
+
"name": "engine",
|
|
5250
|
+
"required": false,
|
|
5251
|
+
"kind": "string",
|
|
5252
|
+
"placeholder": "<engine>"
|
|
5253
|
+
},
|
|
5254
|
+
{
|
|
5255
|
+
"name": "execution_kind",
|
|
5256
|
+
"required": false,
|
|
5257
|
+
"kind": "string",
|
|
5258
|
+
"enum": [
|
|
5259
|
+
"harness",
|
|
5260
|
+
"direct"
|
|
5261
|
+
],
|
|
5262
|
+
"placeholder": "harness"
|
|
5263
|
+
},
|
|
5264
|
+
{
|
|
5265
|
+
"name": "tools",
|
|
5266
|
+
"required": false,
|
|
5267
|
+
"kind": "string",
|
|
5268
|
+
"placeholder": "<tools>"
|
|
5269
|
+
},
|
|
5270
|
+
{
|
|
5271
|
+
"name": "requires_tools",
|
|
5272
|
+
"required": false,
|
|
5273
|
+
"kind": "boolean",
|
|
5274
|
+
"placeholder": "true"
|
|
5275
|
+
},
|
|
5276
|
+
{
|
|
5277
|
+
"name": "mcp_servers",
|
|
5278
|
+
"required": false,
|
|
5279
|
+
"kind": "array",
|
|
5280
|
+
"placeholder": "[]"
|
|
5281
|
+
},
|
|
5282
|
+
{
|
|
5283
|
+
"name": "model_params",
|
|
5284
|
+
"required": false,
|
|
5285
|
+
"kind": "object",
|
|
5286
|
+
"placeholder": "{}"
|
|
5287
|
+
},
|
|
5288
|
+
{
|
|
5289
|
+
"name": "wait",
|
|
5290
|
+
"required": false,
|
|
5291
|
+
"kind": "boolean",
|
|
5292
|
+
"placeholder": "true"
|
|
5293
|
+
}
|
|
5294
|
+
],
|
|
5295
|
+
"watchable": true,
|
|
5296
|
+
"help": {
|
|
5297
|
+
"command": "ora study runs create --file run.json --watch",
|
|
5298
|
+
"body": "{\n \"intent\": \"check pricing for acme.com\",\n \"harness\": \"eve\",\n \"model\": \"claude-sonnet-5\"\n}"
|
|
5299
|
+
}
|
|
5300
|
+
},
|
|
5301
|
+
{
|
|
5302
|
+
"id": "study.runs.directSignals",
|
|
5303
|
+
"commandPath": [
|
|
5304
|
+
"study",
|
|
5305
|
+
"runs",
|
|
5306
|
+
"direct-signals"
|
|
5307
|
+
],
|
|
5308
|
+
"method": "get",
|
|
5309
|
+
"pathTemplate": "/study/v1/runs/{id}/direct-signals",
|
|
5310
|
+
"summary": "Read a direct run's extracted DirectSignals artifact",
|
|
5311
|
+
"scope": "runs:read",
|
|
5312
|
+
"product": "platform",
|
|
5313
|
+
"pathParams": [
|
|
5314
|
+
{
|
|
5315
|
+
"name": "id"
|
|
5316
|
+
}
|
|
5317
|
+
],
|
|
5318
|
+
"queryParams": [],
|
|
5319
|
+
"hasBody": false,
|
|
5320
|
+
"bodyRequired": false,
|
|
5321
|
+
"isStream": false,
|
|
5322
|
+
"watchable": false,
|
|
5323
|
+
"help": {
|
|
5324
|
+
"command": "ora study runs direct-signals <id>"
|
|
5325
|
+
}
|
|
5326
|
+
},
|
|
5327
|
+
{
|
|
5328
|
+
"id": "study.runs.feed",
|
|
5329
|
+
"commandPath": [
|
|
5330
|
+
"study",
|
|
5331
|
+
"runs",
|
|
5332
|
+
"feed"
|
|
5333
|
+
],
|
|
5334
|
+
"method": "get",
|
|
5335
|
+
"pathTemplate": "/study/v1/runs/stream",
|
|
5336
|
+
"summary": "Org-scoped live feed of run status patches (SSE)",
|
|
5337
|
+
"scope": "runs:read",
|
|
5338
|
+
"product": "platform",
|
|
5339
|
+
"pathParams": [],
|
|
5340
|
+
"queryParams": [],
|
|
5341
|
+
"hasBody": false,
|
|
5342
|
+
"bodyRequired": false,
|
|
5343
|
+
"isStream": true,
|
|
5344
|
+
"watchable": false,
|
|
5345
|
+
"help": {
|
|
5346
|
+
"command": "ora study runs feed"
|
|
5347
|
+
}
|
|
5348
|
+
},
|
|
5349
|
+
{
|
|
5350
|
+
"id": "study.runs.insight",
|
|
5351
|
+
"commandPath": [
|
|
5352
|
+
"study",
|
|
5353
|
+
"runs",
|
|
5354
|
+
"insight"
|
|
5355
|
+
],
|
|
5356
|
+
"method": "get",
|
|
5357
|
+
"pathTemplate": "/study/v1/runs/{id}/insight",
|
|
5358
|
+
"summary": "Read a run's stored insight artifact",
|
|
5359
|
+
"scope": "runs:read",
|
|
5360
|
+
"product": "platform",
|
|
5361
|
+
"pathParams": [
|
|
5362
|
+
{
|
|
5363
|
+
"name": "id"
|
|
5364
|
+
}
|
|
5365
|
+
],
|
|
5366
|
+
"queryParams": [],
|
|
5367
|
+
"hasBody": false,
|
|
5368
|
+
"bodyRequired": false,
|
|
5369
|
+
"isStream": false,
|
|
5370
|
+
"watchable": false,
|
|
5371
|
+
"help": {
|
|
5372
|
+
"command": "ora study runs insight <id>"
|
|
5373
|
+
}
|
|
5374
|
+
},
|
|
5375
|
+
{
|
|
5376
|
+
"id": "study.runs.list",
|
|
5377
|
+
"commandPath": [
|
|
5378
|
+
"study",
|
|
5379
|
+
"runs",
|
|
5380
|
+
"list"
|
|
5381
|
+
],
|
|
5382
|
+
"method": "get",
|
|
5383
|
+
"pathTemplate": "/study/v1/runs",
|
|
5384
|
+
"summary": "List runs",
|
|
5385
|
+
"scope": "runs:read",
|
|
5386
|
+
"product": "platform",
|
|
5387
|
+
"pathParams": [],
|
|
5388
|
+
"queryParams": [
|
|
5389
|
+
{
|
|
5390
|
+
"name": "pageSize",
|
|
5391
|
+
"flag": "page-size",
|
|
5392
|
+
"optionKey": "pageSize",
|
|
5393
|
+
"required": false,
|
|
5394
|
+
"kind": "number"
|
|
5395
|
+
},
|
|
5396
|
+
{
|
|
5397
|
+
"name": "pageToken",
|
|
5398
|
+
"flag": "page-token",
|
|
5399
|
+
"optionKey": "pageToken",
|
|
5400
|
+
"required": false,
|
|
5401
|
+
"kind": "string"
|
|
5402
|
+
}
|
|
5403
|
+
],
|
|
5404
|
+
"hasBody": false,
|
|
5405
|
+
"bodyRequired": false,
|
|
5406
|
+
"isStream": false,
|
|
5407
|
+
"watchable": false,
|
|
5408
|
+
"help": {
|
|
5409
|
+
"command": "ora study runs list"
|
|
5410
|
+
}
|
|
5411
|
+
},
|
|
5412
|
+
{
|
|
5413
|
+
"id": "study.runs.rawTrace",
|
|
5414
|
+
"commandPath": [
|
|
5415
|
+
"study",
|
|
5416
|
+
"runs",
|
|
5417
|
+
"raw-trace"
|
|
5418
|
+
],
|
|
5419
|
+
"method": "get",
|
|
5420
|
+
"pathTemplate": "/study/v1/runs/{id}/raw-trace",
|
|
5421
|
+
"summary": "Read a run's full raw frame stream (pre-truncation trace artifact)",
|
|
5422
|
+
"scope": "runs:read",
|
|
5423
|
+
"product": "platform",
|
|
5424
|
+
"pathParams": [
|
|
5425
|
+
{
|
|
5426
|
+
"name": "id"
|
|
5427
|
+
}
|
|
5428
|
+
],
|
|
5429
|
+
"queryParams": [],
|
|
5430
|
+
"hasBody": false,
|
|
5431
|
+
"bodyRequired": false,
|
|
5432
|
+
"isStream": false,
|
|
5433
|
+
"watchable": false,
|
|
5434
|
+
"help": {
|
|
5435
|
+
"command": "ora study runs raw-trace <id>"
|
|
5436
|
+
}
|
|
5437
|
+
},
|
|
5438
|
+
{
|
|
5439
|
+
"id": "study.runs.spans",
|
|
5440
|
+
"commandPath": [
|
|
5441
|
+
"study",
|
|
5442
|
+
"runs",
|
|
5443
|
+
"spans"
|
|
5444
|
+
],
|
|
5445
|
+
"method": "get",
|
|
5446
|
+
"pathTemplate": "/study/v1/runs/{id}/spans",
|
|
5447
|
+
"summary": "Read a run's stored trace spans (CompactSpan[])",
|
|
5448
|
+
"scope": "runs:read",
|
|
5449
|
+
"product": "platform",
|
|
5450
|
+
"pathParams": [
|
|
5451
|
+
{
|
|
5452
|
+
"name": "id"
|
|
5453
|
+
}
|
|
5454
|
+
],
|
|
5455
|
+
"queryParams": [],
|
|
5456
|
+
"hasBody": false,
|
|
5457
|
+
"bodyRequired": false,
|
|
5458
|
+
"isStream": false,
|
|
5459
|
+
"watchable": false,
|
|
5460
|
+
"help": {
|
|
5461
|
+
"command": "ora study runs spans <id>"
|
|
5462
|
+
}
|
|
5463
|
+
},
|
|
5464
|
+
{
|
|
5465
|
+
"id": "study.runs.stream",
|
|
5466
|
+
"commandPath": [
|
|
5467
|
+
"study",
|
|
5468
|
+
"runs",
|
|
5469
|
+
"stream"
|
|
5470
|
+
],
|
|
5471
|
+
"method": "get",
|
|
5472
|
+
"pathTemplate": "/study/v1/runs/{id}/stream",
|
|
5473
|
+
"summary": "Stream a run's live trajectory (SSE)",
|
|
5474
|
+
"scope": "runs:read",
|
|
5475
|
+
"product": "platform",
|
|
5476
|
+
"pathParams": [
|
|
5477
|
+
{
|
|
5478
|
+
"name": "id"
|
|
5479
|
+
}
|
|
5480
|
+
],
|
|
5481
|
+
"queryParams": [],
|
|
5482
|
+
"hasBody": false,
|
|
5483
|
+
"bodyRequired": false,
|
|
5484
|
+
"isStream": true,
|
|
5485
|
+
"watchable": false,
|
|
5486
|
+
"help": {
|
|
5487
|
+
"command": "ora study runs stream <id>"
|
|
5488
|
+
}
|
|
5489
|
+
},
|
|
5490
|
+
{
|
|
5491
|
+
"id": "study.runs.trajectory",
|
|
5492
|
+
"commandPath": [
|
|
5493
|
+
"study",
|
|
5494
|
+
"runs",
|
|
5495
|
+
"trajectory"
|
|
5496
|
+
],
|
|
5497
|
+
"method": "get",
|
|
5498
|
+
"pathTemplate": "/study/v1/runs/{id}/trajectory",
|
|
5499
|
+
"summary": "Read a run's normalized trajectory snapshot",
|
|
5500
|
+
"scope": "runs:read",
|
|
5501
|
+
"product": "platform",
|
|
5502
|
+
"pathParams": [
|
|
5503
|
+
{
|
|
5504
|
+
"name": "id"
|
|
5505
|
+
}
|
|
5506
|
+
],
|
|
5507
|
+
"queryParams": [
|
|
5508
|
+
{
|
|
5509
|
+
"name": "profile",
|
|
5510
|
+
"flag": "profile",
|
|
5511
|
+
"optionKey": "profile",
|
|
5512
|
+
"required": false,
|
|
5513
|
+
"kind": "string",
|
|
5514
|
+
"enum": [
|
|
5515
|
+
"raw",
|
|
5516
|
+
"heuristic",
|
|
5517
|
+
"full"
|
|
5518
|
+
]
|
|
5519
|
+
}
|
|
5520
|
+
],
|
|
5521
|
+
"hasBody": false,
|
|
5522
|
+
"bodyRequired": false,
|
|
5523
|
+
"isStream": false,
|
|
5524
|
+
"watchable": false,
|
|
5525
|
+
"help": {
|
|
5526
|
+
"command": "ora study runs trajectory <id>"
|
|
5527
|
+
}
|
|
5528
|
+
},
|
|
5529
|
+
{
|
|
5530
|
+
"id": "study.setVisibility",
|
|
5531
|
+
"commandPath": [
|
|
5532
|
+
"study",
|
|
5533
|
+
"set-visibility"
|
|
5534
|
+
],
|
|
5535
|
+
"method": "patch",
|
|
5536
|
+
"pathTemplate": "/study/v1/studies/{id}/visibility",
|
|
5537
|
+
"summary": "Set who may read this study: private | org | link | public. Moving INTO 'link' mints a 256-bit share token (returned in `shareUrl`); moving out clears it immediately. The FIRST move to 'link'/'public' publishes the study; later mode changes are not re-publications and never re-stamp its publication date.",
|
|
5538
|
+
"scope": "study.studies:write",
|
|
5539
|
+
"product": "study",
|
|
5540
|
+
"pathParams": [
|
|
5541
|
+
{
|
|
5542
|
+
"name": "id"
|
|
5543
|
+
}
|
|
5544
|
+
],
|
|
5545
|
+
"queryParams": [],
|
|
5546
|
+
"hasBody": true,
|
|
5547
|
+
"bodyRequired": true,
|
|
5548
|
+
"isStream": false,
|
|
5549
|
+
"bodyFields": [
|
|
5550
|
+
{
|
|
5551
|
+
"name": "visibility",
|
|
5552
|
+
"required": true,
|
|
5553
|
+
"kind": "string",
|
|
5554
|
+
"enum": [
|
|
5555
|
+
"private",
|
|
5556
|
+
"org",
|
|
5557
|
+
"link",
|
|
5558
|
+
"public"
|
|
5559
|
+
],
|
|
5560
|
+
"description": "The sharing mode: private (creator + org owners) | org (the whole org) | link (anyone holding the share link) | public (everyone, listed in the catalog). Moving INTO 'link' mints a share token; moving out clears it.",
|
|
5561
|
+
"placeholder": "link"
|
|
5562
|
+
}
|
|
5563
|
+
],
|
|
5564
|
+
"watchable": false,
|
|
5565
|
+
"help": {
|
|
5566
|
+
"command": "ora study set-visibility 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
5567
|
+
"body": "{\n \"visibility\": \"link\"\n}"
|
|
5568
|
+
}
|
|
5569
|
+
},
|
|
5570
|
+
{
|
|
5571
|
+
"id": "study.skills.create",
|
|
5572
|
+
"commandPath": [
|
|
5573
|
+
"study",
|
|
5574
|
+
"skills",
|
|
5575
|
+
"create"
|
|
5576
|
+
],
|
|
5577
|
+
"method": "post",
|
|
5578
|
+
"pathTemplate": "/study/v1/skills",
|
|
5579
|
+
"summary": "Create a skill (custom or git-imported)",
|
|
5580
|
+
"scope": "skills:write",
|
|
5581
|
+
"product": "platform",
|
|
5582
|
+
"pathParams": [],
|
|
5583
|
+
"queryParams": [],
|
|
5584
|
+
"hasBody": true,
|
|
5585
|
+
"bodyRequired": true,
|
|
5586
|
+
"isStream": false,
|
|
5587
|
+
"bodyFields": [
|
|
5588
|
+
{
|
|
5589
|
+
"name": "source",
|
|
5590
|
+
"required": true,
|
|
5591
|
+
"kind": "string",
|
|
5592
|
+
"placeholder": "custom"
|
|
5593
|
+
},
|
|
5594
|
+
{
|
|
5595
|
+
"name": "name",
|
|
5596
|
+
"required": true,
|
|
5597
|
+
"kind": "string",
|
|
5598
|
+
"placeholder": "<name>"
|
|
5599
|
+
},
|
|
5600
|
+
{
|
|
5601
|
+
"name": "body",
|
|
5602
|
+
"required": true,
|
|
5603
|
+
"kind": "string",
|
|
5604
|
+
"placeholder": "<body>"
|
|
5605
|
+
},
|
|
5606
|
+
{
|
|
5607
|
+
"name": "visibility",
|
|
5608
|
+
"required": false,
|
|
5609
|
+
"kind": "string",
|
|
5610
|
+
"enum": [
|
|
5611
|
+
"user",
|
|
5612
|
+
"org"
|
|
5613
|
+
],
|
|
5614
|
+
"placeholder": "user"
|
|
5615
|
+
}
|
|
5616
|
+
],
|
|
5617
|
+
"watchable": false,
|
|
5618
|
+
"help": {
|
|
5619
|
+
"command": "ora study skills create --file run.json",
|
|
5620
|
+
"body": "{\n \"source\": \"custom\",\n \"name\": \"<name>\",\n \"body\": \"<body>\"\n}"
|
|
5621
|
+
}
|
|
5622
|
+
},
|
|
5623
|
+
{
|
|
5624
|
+
"id": "study.skills.delete",
|
|
5625
|
+
"commandPath": [
|
|
5626
|
+
"study",
|
|
5627
|
+
"skills",
|
|
5628
|
+
"delete"
|
|
5629
|
+
],
|
|
5630
|
+
"method": "delete",
|
|
5631
|
+
"pathTemplate": "/study/v1/skills/{id}",
|
|
5632
|
+
"summary": "Delete a skill",
|
|
5633
|
+
"scope": "skills:write",
|
|
5634
|
+
"product": "platform",
|
|
5635
|
+
"pathParams": [
|
|
5636
|
+
{
|
|
5637
|
+
"name": "id"
|
|
5638
|
+
}
|
|
5639
|
+
],
|
|
5640
|
+
"queryParams": [],
|
|
5641
|
+
"hasBody": false,
|
|
5642
|
+
"bodyRequired": false,
|
|
5643
|
+
"isStream": false,
|
|
5644
|
+
"watchable": false,
|
|
5645
|
+
"help": {
|
|
5646
|
+
"command": "ora study skills delete <id>"
|
|
5647
|
+
}
|
|
5648
|
+
},
|
|
5649
|
+
{
|
|
5650
|
+
"id": "study.skills.list",
|
|
5651
|
+
"commandPath": [
|
|
5652
|
+
"study",
|
|
5653
|
+
"skills",
|
|
5654
|
+
"list"
|
|
5655
|
+
],
|
|
5656
|
+
"method": "get",
|
|
5657
|
+
"pathTemplate": "/study/v1/skills",
|
|
5658
|
+
"summary": "List the caller's usable skills catalog",
|
|
5659
|
+
"scope": "skills:read",
|
|
5660
|
+
"product": "platform",
|
|
5661
|
+
"pathParams": [],
|
|
5662
|
+
"queryParams": [],
|
|
5663
|
+
"hasBody": false,
|
|
5664
|
+
"bodyRequired": false,
|
|
5665
|
+
"isStream": false,
|
|
5666
|
+
"watchable": false,
|
|
5667
|
+
"help": {
|
|
5668
|
+
"command": "ora study skills list"
|
|
5669
|
+
}
|
|
5670
|
+
},
|
|
5671
|
+
{
|
|
5672
|
+
"id": "study.skills.update",
|
|
5673
|
+
"commandPath": [
|
|
5674
|
+
"study",
|
|
5675
|
+
"skills",
|
|
5676
|
+
"update"
|
|
5677
|
+
],
|
|
5678
|
+
"method": "put",
|
|
5679
|
+
"pathTemplate": "/study/v1/skills/{id}",
|
|
5680
|
+
"summary": "Update a skill",
|
|
5681
|
+
"scope": "skills:write",
|
|
5682
|
+
"product": "platform",
|
|
5683
|
+
"pathParams": [
|
|
5684
|
+
{
|
|
5685
|
+
"name": "id"
|
|
5686
|
+
}
|
|
5687
|
+
],
|
|
5688
|
+
"queryParams": [],
|
|
5689
|
+
"hasBody": true,
|
|
5690
|
+
"bodyRequired": true,
|
|
5691
|
+
"isStream": false,
|
|
5692
|
+
"bodyFields": [
|
|
5693
|
+
{
|
|
5694
|
+
"name": "name",
|
|
5695
|
+
"required": false,
|
|
5696
|
+
"kind": "string",
|
|
5697
|
+
"placeholder": "<name>"
|
|
5698
|
+
},
|
|
5699
|
+
{
|
|
5700
|
+
"name": "visibility",
|
|
5701
|
+
"required": false,
|
|
5702
|
+
"kind": "string",
|
|
5703
|
+
"enum": [
|
|
5704
|
+
"user",
|
|
5705
|
+
"org"
|
|
5706
|
+
],
|
|
5707
|
+
"placeholder": "user"
|
|
5708
|
+
},
|
|
5709
|
+
{
|
|
5710
|
+
"name": "body",
|
|
5711
|
+
"required": false,
|
|
5712
|
+
"kind": "string",
|
|
5713
|
+
"placeholder": "<body>"
|
|
5714
|
+
},
|
|
5715
|
+
{
|
|
5716
|
+
"name": "repoUrl",
|
|
5717
|
+
"required": false,
|
|
5718
|
+
"kind": "string",
|
|
5719
|
+
"placeholder": "<repoUrl>"
|
|
5720
|
+
},
|
|
5721
|
+
{
|
|
5722
|
+
"name": "gitRef",
|
|
5723
|
+
"required": false,
|
|
5724
|
+
"kind": "string",
|
|
5725
|
+
"placeholder": "<gitRef>"
|
|
5726
|
+
},
|
|
5727
|
+
{
|
|
5728
|
+
"name": "subpath",
|
|
5729
|
+
"required": false,
|
|
5730
|
+
"kind": "string",
|
|
5731
|
+
"placeholder": "<subpath>"
|
|
5732
|
+
}
|
|
5733
|
+
],
|
|
5734
|
+
"watchable": false,
|
|
5735
|
+
"help": {
|
|
5736
|
+
"command": "ora study skills update <id> --file run.json",
|
|
5737
|
+
"body": "{\n \"name\": \"<name>\",\n \"visibility\": \"user\",\n \"body\": \"<body>\"\n}"
|
|
5738
|
+
}
|
|
5739
|
+
},
|
|
5740
|
+
{
|
|
5741
|
+
"id": "study.stream",
|
|
5742
|
+
"commandPath": [
|
|
5743
|
+
"study",
|
|
5744
|
+
"stream"
|
|
5745
|
+
],
|
|
5746
|
+
"method": "get",
|
|
5747
|
+
"pathTemplate": "/study/v1/studies/{id}/stream",
|
|
5748
|
+
"summary": "Live SSE stream: status transitions, per-run facts, grading, publish state (terminal status closes it)",
|
|
5749
|
+
"scope": "study.studies:read",
|
|
5750
|
+
"product": "study",
|
|
5751
|
+
"pathParams": [
|
|
5752
|
+
{
|
|
5753
|
+
"name": "id"
|
|
5754
|
+
}
|
|
5755
|
+
],
|
|
5756
|
+
"queryParams": [],
|
|
5757
|
+
"hasBody": false,
|
|
5758
|
+
"bodyRequired": false,
|
|
5759
|
+
"isStream": true,
|
|
5760
|
+
"watchable": false,
|
|
5761
|
+
"help": {
|
|
5762
|
+
"command": "ora study stream 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
5763
|
+
}
|
|
5764
|
+
},
|
|
5765
|
+
{
|
|
5766
|
+
"id": "study.unpublish",
|
|
5767
|
+
"commandPath": [
|
|
5768
|
+
"study",
|
|
5769
|
+
"unpublish"
|
|
5770
|
+
],
|
|
5771
|
+
"method": "post",
|
|
5772
|
+
"pathTemplate": "/study/v1/studies/{id}/unpublish",
|
|
5773
|
+
"summary": "Remove a study from the public catalog (data kept; re-publish any time)",
|
|
5774
|
+
"scope": "study.studies:write",
|
|
5775
|
+
"product": "study",
|
|
5776
|
+
"pathParams": [
|
|
5777
|
+
{
|
|
5778
|
+
"name": "id"
|
|
5779
|
+
}
|
|
5780
|
+
],
|
|
5781
|
+
"queryParams": [],
|
|
5782
|
+
"hasBody": false,
|
|
5783
|
+
"bodyRequired": false,
|
|
5784
|
+
"isStream": false,
|
|
5785
|
+
"watchable": false,
|
|
5786
|
+
"help": {
|
|
5787
|
+
"command": "ora study unpublish <id>"
|
|
5788
|
+
}
|
|
5789
|
+
},
|
|
5790
|
+
{
|
|
5791
|
+
"id": "study.updateLenses",
|
|
5792
|
+
"commandPath": [
|
|
5793
|
+
"study",
|
|
5794
|
+
"update-lenses"
|
|
5795
|
+
],
|
|
5796
|
+
"method": "put",
|
|
5797
|
+
"pathTemplate": "/study/v1/studies/{id}/lenses",
|
|
5798
|
+
"summary": "Change which grading lenses a study runs under before launch (status 'ready' only) — the complete replacement set, e.g. add 'evals' to a judge-only study so its tasks can carry expectations",
|
|
5799
|
+
"scope": "study.studies:write",
|
|
5800
|
+
"product": "study",
|
|
5801
|
+
"pathParams": [
|
|
5802
|
+
{
|
|
5803
|
+
"name": "id"
|
|
5804
|
+
}
|
|
5805
|
+
],
|
|
5806
|
+
"queryParams": [],
|
|
5807
|
+
"hasBody": true,
|
|
5808
|
+
"bodyRequired": true,
|
|
5809
|
+
"isStream": false,
|
|
5810
|
+
"bodyFields": [
|
|
5811
|
+
{
|
|
5812
|
+
"name": "lenses",
|
|
5813
|
+
"required": true,
|
|
5814
|
+
"kind": "array",
|
|
5815
|
+
"description": "The COMPLETE replacement lens set (not a delta). Editable only while status is 'ready'. Removing 'evals' while a task still carries authored `expectations` is BLOCKED (422 LENS_REMOVAL_BLOCKED, names the task(s)); removing 'judge' while a task carries a `rubric` succeeds but returns a non-blocking warning (the rubric stays stored, just ungraded).",
|
|
5816
|
+
"placeholder": "[\"judge\",\"evals\"]"
|
|
5817
|
+
}
|
|
5818
|
+
],
|
|
5819
|
+
"watchable": false,
|
|
5820
|
+
"help": {
|
|
5821
|
+
"command": "ora study update-lenses 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
5822
|
+
"body": "{\n \"lenses\": [\n \"judge\",\n \"evals\"\n ]\n}"
|
|
5823
|
+
}
|
|
5824
|
+
},
|
|
5825
|
+
{
|
|
5826
|
+
"id": "study.updateTask",
|
|
5827
|
+
"commandPath": [
|
|
5828
|
+
"study",
|
|
5829
|
+
"update-task"
|
|
5830
|
+
],
|
|
5831
|
+
"method": "put",
|
|
5832
|
+
"pathTemplate": "/study/v1/studies/{id}/tasks/{taskId}",
|
|
5833
|
+
"summary": "Edit a generated task's prompt before launch (status 'ready' only)",
|
|
5834
|
+
"scope": "study.studies:write",
|
|
5835
|
+
"product": "study",
|
|
5836
|
+
"pathParams": [
|
|
5837
|
+
{
|
|
5838
|
+
"name": "id"
|
|
5839
|
+
},
|
|
5840
|
+
{
|
|
5841
|
+
"name": "taskId"
|
|
5842
|
+
}
|
|
5843
|
+
],
|
|
5844
|
+
"queryParams": [],
|
|
5845
|
+
"hasBody": true,
|
|
5846
|
+
"bodyRequired": true,
|
|
5847
|
+
"isStream": false,
|
|
5848
|
+
"bodyFields": [
|
|
5849
|
+
{
|
|
5850
|
+
"name": "prompt",
|
|
5851
|
+
"required": true,
|
|
5852
|
+
"kind": "string",
|
|
5853
|
+
"description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the study is 'ready' (pre-launch).",
|
|
5854
|
+
"placeholder": "Research the three largest EU battery-storage tenders announced this quarter and cite each source."
|
|
5855
|
+
},
|
|
5856
|
+
{
|
|
5857
|
+
"name": "expectations",
|
|
5858
|
+
"required": false,
|
|
5859
|
+
"kind": "array",
|
|
5860
|
+
"description": "Replace this task's 'evals'-lens expectations (see studyCreateBody.tasks[].expectations). Omitted = expectations unchanged. REQUIRES the study's `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); validated the same way as create.",
|
|
5861
|
+
"placeholder": "[]"
|
|
5862
|
+
}
|
|
5863
|
+
],
|
|
5864
|
+
"watchable": false,
|
|
5865
|
+
"help": {
|
|
5866
|
+
"command": "ora study update-task 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 8a9b0c1d-2e3f-4a5b-8c7d-6e5f4a3b2c1d --file run.json",
|
|
5867
|
+
"body": "{\n \"prompt\": \"Research the three largest EU battery-storage tenders announced this quarter and cite each source.\"\n}"
|
|
5868
|
+
}
|
|
5869
|
+
},
|
|
5870
|
+
{
|
|
5871
|
+
"id": "tunnels.create",
|
|
5872
|
+
"commandPath": [
|
|
5873
|
+
"tunnels",
|
|
5874
|
+
"create"
|
|
5875
|
+
],
|
|
5876
|
+
"method": "post",
|
|
5877
|
+
"pathTemplate": "/tunnels/v1",
|
|
5878
|
+
"summary": "Create a dev tunnel",
|
|
5879
|
+
"scope": "tunnels:write",
|
|
5880
|
+
"product": "platform",
|
|
5881
|
+
"pathParams": [],
|
|
5882
|
+
"queryParams": [],
|
|
5883
|
+
"hasBody": true,
|
|
5884
|
+
"bodyRequired": true,
|
|
5885
|
+
"isStream": false,
|
|
5886
|
+
"bodyFields": [
|
|
5887
|
+
{
|
|
5888
|
+
"name": "name",
|
|
5889
|
+
"required": true,
|
|
5890
|
+
"kind": "string",
|
|
5891
|
+
"placeholder": "<name>"
|
|
5892
|
+
},
|
|
5893
|
+
{
|
|
5894
|
+
"name": "productKey",
|
|
5895
|
+
"required": false,
|
|
5896
|
+
"kind": "string",
|
|
5897
|
+
"placeholder": "<productKey>"
|
|
5898
|
+
},
|
|
5899
|
+
{
|
|
5900
|
+
"name": "access",
|
|
5901
|
+
"required": true,
|
|
5902
|
+
"kind": "string",
|
|
5903
|
+
"enum": [
|
|
5904
|
+
"private",
|
|
5905
|
+
"protected",
|
|
5906
|
+
"public"
|
|
5907
|
+
],
|
|
5908
|
+
"placeholder": "private"
|
|
5909
|
+
}
|
|
5910
|
+
],
|
|
5911
|
+
"watchable": false,
|
|
5912
|
+
"help": {
|
|
5913
|
+
"command": "ora tunnels create --file run.json",
|
|
5914
|
+
"body": "{\n \"name\": \"<name>\",\n \"access\": \"private\"\n}"
|
|
5915
|
+
}
|
|
5916
|
+
},
|
|
5917
|
+
{
|
|
5918
|
+
"id": "tunnels.delete",
|
|
5919
|
+
"commandPath": [
|
|
5920
|
+
"tunnels",
|
|
5921
|
+
"delete"
|
|
5922
|
+
],
|
|
5923
|
+
"method": "delete",
|
|
5924
|
+
"pathTemplate": "/tunnels/v1/{id}",
|
|
5925
|
+
"summary": "Delete a dev tunnel (revokes any live connection, removes the row)",
|
|
5926
|
+
"scope": "tunnels:write",
|
|
5927
|
+
"product": "platform",
|
|
5928
|
+
"pathParams": [
|
|
5929
|
+
{
|
|
5930
|
+
"name": "id"
|
|
5931
|
+
}
|
|
5932
|
+
],
|
|
5933
|
+
"queryParams": [],
|
|
5934
|
+
"hasBody": false,
|
|
5935
|
+
"bodyRequired": false,
|
|
5936
|
+
"isStream": false,
|
|
5937
|
+
"watchable": false,
|
|
5938
|
+
"help": {
|
|
5939
|
+
"command": "ora tunnels delete <id>"
|
|
5940
|
+
}
|
|
5941
|
+
},
|
|
5942
|
+
{
|
|
5943
|
+
"id": "tunnels.get",
|
|
5944
|
+
"commandPath": [
|
|
5945
|
+
"tunnels",
|
|
5946
|
+
"get"
|
|
5947
|
+
],
|
|
5948
|
+
"method": "get",
|
|
5949
|
+
"pathTemplate": "/tunnels/v1/{id}",
|
|
5950
|
+
"summary": "Get a dev tunnel",
|
|
5951
|
+
"scope": "tunnels:read",
|
|
5952
|
+
"product": "platform",
|
|
5953
|
+
"pathParams": [
|
|
5954
|
+
{
|
|
5955
|
+
"name": "id"
|
|
5956
|
+
}
|
|
5957
|
+
],
|
|
5958
|
+
"queryParams": [],
|
|
5959
|
+
"hasBody": false,
|
|
5960
|
+
"bodyRequired": false,
|
|
5961
|
+
"isStream": false,
|
|
5962
|
+
"watchable": false,
|
|
5963
|
+
"help": {
|
|
5964
|
+
"command": "ora tunnels get <id>"
|
|
5965
|
+
}
|
|
5966
|
+
},
|
|
5967
|
+
{
|
|
5968
|
+
"id": "tunnels.list",
|
|
5969
|
+
"commandPath": [
|
|
5970
|
+
"tunnels",
|
|
5971
|
+
"list"
|
|
5972
|
+
],
|
|
5973
|
+
"method": "get",
|
|
5974
|
+
"pathTemplate": "/tunnels/v1",
|
|
5975
|
+
"summary": "List dev tunnels",
|
|
5976
|
+
"scope": "tunnels:read",
|
|
5977
|
+
"product": "platform",
|
|
5978
|
+
"pathParams": [],
|
|
5979
|
+
"queryParams": [],
|
|
5980
|
+
"hasBody": false,
|
|
5981
|
+
"bodyRequired": false,
|
|
5982
|
+
"isStream": false,
|
|
5983
|
+
"watchable": false,
|
|
5984
|
+
"help": {
|
|
5985
|
+
"command": "ora tunnels list"
|
|
5986
|
+
}
|
|
5987
|
+
},
|
|
5988
|
+
{
|
|
5989
|
+
"id": "tunnels.revoke",
|
|
5990
|
+
"commandPath": [
|
|
5991
|
+
"tunnels",
|
|
5992
|
+
"revoke"
|
|
5993
|
+
],
|
|
5994
|
+
"method": "post",
|
|
3115
5995
|
"pathTemplate": "/tunnels/v1/{id}/revoke",
|
|
3116
5996
|
"summary": "Revoke a dev tunnel (kill the connection, keep the row)",
|
|
3117
5997
|
"scope": "tunnels:write",
|
|
@@ -3224,6 +6104,66 @@ export const OPERATIONS = [
|
|
|
3224
6104
|
"command": "ora webmcp arena create --file run.json --watch",
|
|
3225
6105
|
"body": "{\n \"targetUrl\": \"https://shop.example.com\",\n \"intent\": \"Find the price of the Chronometer and add it to the cart.\",\n \"harness\": \"claude-code\",\n \"model\": \"claude-haiku-4-5-20251001\",\n \"baseline\": \"none\",\n \"treatment\": \"webmcp-native\"\n}"
|
|
3226
6106
|
}
|
|
6107
|
+
},
|
|
6108
|
+
{
|
|
6109
|
+
"id": "webmcp.runs.browserRecording",
|
|
6110
|
+
"commandPath": [
|
|
6111
|
+
"webmcp",
|
|
6112
|
+
"runs",
|
|
6113
|
+
"browser-recording"
|
|
6114
|
+
],
|
|
6115
|
+
"method": "get",
|
|
6116
|
+
"pathTemplate": "/experiment/v1/webmcp/runs/{runId}/browser/recording",
|
|
6117
|
+
"summary": "The run's RECORDED browser screencast (time travel): frames stamped from the recording start, replayable and scrubbable after the run ends",
|
|
6118
|
+
"scope": "benchmarks:read",
|
|
6119
|
+
"product": "experiment",
|
|
6120
|
+
"pathParams": [
|
|
6121
|
+
{
|
|
6122
|
+
"name": "runId"
|
|
6123
|
+
}
|
|
6124
|
+
],
|
|
6125
|
+
"queryParams": [
|
|
6126
|
+
{
|
|
6127
|
+
"name": "maxFrames",
|
|
6128
|
+
"flag": "max-frames",
|
|
6129
|
+
"optionKey": "maxFrames",
|
|
6130
|
+
"required": false,
|
|
6131
|
+
"kind": "number"
|
|
6132
|
+
}
|
|
6133
|
+
],
|
|
6134
|
+
"hasBody": false,
|
|
6135
|
+
"bodyRequired": false,
|
|
6136
|
+
"isStream": false,
|
|
6137
|
+
"watchable": false,
|
|
6138
|
+
"help": {
|
|
6139
|
+
"command": "ora webmcp runs browser-recording <runId>"
|
|
6140
|
+
}
|
|
6141
|
+
},
|
|
6142
|
+
{
|
|
6143
|
+
"id": "webmcp.runs.browserStream",
|
|
6144
|
+
"commandPath": [
|
|
6145
|
+
"webmcp",
|
|
6146
|
+
"runs",
|
|
6147
|
+
"browser-stream"
|
|
6148
|
+
],
|
|
6149
|
+
"method": "get",
|
|
6150
|
+
"pathTemplate": "/experiment/v1/webmcp/runs/{runId}/browser/stream",
|
|
6151
|
+
"summary": "Live browser view of an arena run (SSE JPEG frames relayed from the run pod's driven browser)",
|
|
6152
|
+
"scope": "benchmarks:read",
|
|
6153
|
+
"product": "experiment",
|
|
6154
|
+
"pathParams": [
|
|
6155
|
+
{
|
|
6156
|
+
"name": "runId"
|
|
6157
|
+
}
|
|
6158
|
+
],
|
|
6159
|
+
"queryParams": [],
|
|
6160
|
+
"hasBody": false,
|
|
6161
|
+
"bodyRequired": false,
|
|
6162
|
+
"isStream": true,
|
|
6163
|
+
"watchable": false,
|
|
6164
|
+
"help": {
|
|
6165
|
+
"command": "ora webmcp runs browser-stream <runId>"
|
|
6166
|
+
}
|
|
3227
6167
|
}
|
|
3228
6168
|
];
|
|
3229
6169
|
//# sourceMappingURL=operations.js.map
|