@ora-ai/cli 0.7.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -6
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +3280 -354
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +3837 -58
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/package.json +2 -2
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// GENERATED by scripts/generate.mjs — DO NOT EDIT.
|
|
2
|
-
// Source: openapi/ora-public.openapi.json (sha256
|
|
2
|
+
// Source: openapi/ora-public.openapi.json (sha256 067a219e40d4ab8e05bcd2f6fb9b38e772c34e946852e4ad1ee19264faba71b1)
|
|
3
3
|
// Regenerate: yarn nx run ora-cli:generate
|
|
4
4
|
export const SPEC = {
|
|
5
5
|
"version": "1.0.0",
|
|
6
|
-
"contentHash": "
|
|
6
|
+
"contentHash": "067a219e40d4ab8e05bcd2f6fb9b38e772c34e946852e4ad1ee19264faba71b1",
|
|
7
7
|
"server": "https://api.agentfront.sh",
|
|
8
8
|
"bin": "ora"
|
|
9
9
|
};
|
|
@@ -218,7 +218,7 @@ export const OPERATIONS = [
|
|
|
218
218
|
],
|
|
219
219
|
"method": "post",
|
|
220
220
|
"pathTemplate": "/experiment/v1/benchmarks/convert/webmcp",
|
|
221
|
-
"summary": "Convert a webmcp-evals suite JSON into ora
|
|
221
|
+
"summary": "Convert a webmcp-evals suite JSON into ora study tasks + 'evals'-lens expectations — a PURE transform, nothing persisted; resubmit the returned `tasks` through benchmarks.create's `tasks` field",
|
|
222
222
|
"scope": "benchmarks:write",
|
|
223
223
|
"product": "experiment",
|
|
224
224
|
"pathParams": [],
|
|
@@ -256,7 +256,7 @@ export const OPERATIONS = [
|
|
|
256
256
|
],
|
|
257
257
|
"method": "post",
|
|
258
258
|
"pathTemplate": "/experiment/v1/benchmarks",
|
|
259
|
-
"summary": "Create a
|
|
259
|
+
"summary": "Create a study (status 'draft', ZERO LLM spend — an authored `tasks` corpus instead lands 'ready' synchronously); approve generation via POST .../generate",
|
|
260
260
|
"scope": "benchmarks:write",
|
|
261
261
|
"product": "experiment",
|
|
262
262
|
"pathParams": [],
|
|
@@ -269,14 +269,14 @@ export const OPERATIONS = [
|
|
|
269
269
|
"name": "name",
|
|
270
270
|
"required": true,
|
|
271
271
|
"kind": "string",
|
|
272
|
-
"description": "Display name for the
|
|
272
|
+
"description": "Display name for the study (≤160 chars).",
|
|
273
273
|
"placeholder": "tavily-search A/B"
|
|
274
274
|
},
|
|
275
275
|
{
|
|
276
276
|
"name": "description",
|
|
277
277
|
"required": false,
|
|
278
278
|
"kind": "string",
|
|
279
|
-
"description": "Optional longer description shown on the
|
|
279
|
+
"description": "Optional longer description shown on the study card.",
|
|
280
280
|
"placeholder": "<description>"
|
|
281
281
|
},
|
|
282
282
|
{
|
|
@@ -287,9 +287,10 @@ export const OPERATIONS = [
|
|
|
287
287
|
"skill",
|
|
288
288
|
"tools",
|
|
289
289
|
"addon",
|
|
290
|
-
"custom"
|
|
290
|
+
"custom",
|
|
291
|
+
"benchmark"
|
|
291
292
|
],
|
|
292
|
-
"description": "What the
|
|
293
|
+
"description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
|
|
293
294
|
"placeholder": "addon"
|
|
294
295
|
},
|
|
295
296
|
{
|
|
@@ -299,11 +300,18 @@ export const OPERATIONS = [
|
|
|
299
300
|
"description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
|
|
300
301
|
"placeholder": "<skillRef>"
|
|
301
302
|
},
|
|
303
|
+
{
|
|
304
|
+
"name": "definitionVersionId",
|
|
305
|
+
"required": false,
|
|
306
|
+
"kind": "string",
|
|
307
|
+
"description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
|
|
308
|
+
"placeholder": "<definitionVersionId>"
|
|
309
|
+
},
|
|
302
310
|
{
|
|
303
311
|
"name": "subjectName",
|
|
304
312
|
"required": false,
|
|
305
313
|
"kind": "string",
|
|
306
|
-
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required
|
|
314
|
+
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
|
|
307
315
|
"placeholder": "Tavily search"
|
|
308
316
|
},
|
|
309
317
|
{
|
|
@@ -324,7 +332,7 @@ export const OPERATIONS = [
|
|
|
324
332
|
"name": "proposeExpectations",
|
|
325
333
|
"required": false,
|
|
326
334
|
"kind": "boolean",
|
|
327
|
-
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the
|
|
335
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
|
|
328
336
|
"placeholder": "true"
|
|
329
337
|
},
|
|
330
338
|
{
|
|
@@ -339,6 +347,17 @@ export const OPERATIONS = [
|
|
|
339
347
|
"description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
|
|
340
348
|
"placeholder": "ab_vs_none"
|
|
341
349
|
},
|
|
350
|
+
{
|
|
351
|
+
"name": "subjectBackend",
|
|
352
|
+
"required": false,
|
|
353
|
+
"kind": "string",
|
|
354
|
+
"enum": [
|
|
355
|
+
"model",
|
|
356
|
+
"harness"
|
|
357
|
+
],
|
|
358
|
+
"description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
|
|
359
|
+
"placeholder": "model"
|
|
360
|
+
},
|
|
342
361
|
{
|
|
343
362
|
"name": "baselineLabel",
|
|
344
363
|
"required": false,
|
|
@@ -357,7 +376,7 @@ export const OPERATIONS = [
|
|
|
357
376
|
"name": "taskCount",
|
|
358
377
|
"required": false,
|
|
359
378
|
"kind": "number",
|
|
360
|
-
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at
|
|
379
|
+
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
|
|
361
380
|
"placeholder": "1"
|
|
362
381
|
},
|
|
363
382
|
{
|
|
@@ -378,9 +397,16 @@ export const OPERATIONS = [
|
|
|
378
397
|
"name": "tasks",
|
|
379
398
|
"required": false,
|
|
380
399
|
"kind": "array",
|
|
381
|
-
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the
|
|
400
|
+
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
|
|
382
401
|
"placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
|
|
383
402
|
},
|
|
403
|
+
{
|
|
404
|
+
"name": "subset",
|
|
405
|
+
"required": false,
|
|
406
|
+
"kind": "object",
|
|
407
|
+
"description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
|
|
408
|
+
"placeholder": "{}"
|
|
409
|
+
},
|
|
384
410
|
{
|
|
385
411
|
"name": "matrix",
|
|
386
412
|
"required": true,
|
|
@@ -392,7 +418,7 @@ export const OPERATIONS = [
|
|
|
392
418
|
"name": "promise",
|
|
393
419
|
"required": false,
|
|
394
420
|
"kind": "string",
|
|
395
|
-
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); REQUIRED for every other kind.",
|
|
421
|
+
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
|
|
396
422
|
"placeholder": "Web search via Tavily produces fresher, better-sourced answers than no web access"
|
|
397
423
|
}
|
|
398
424
|
],
|
|
@@ -410,7 +436,7 @@ export const OPERATIONS = [
|
|
|
410
436
|
],
|
|
411
437
|
"method": "delete",
|
|
412
438
|
"pathTemplate": "/experiment/v1/benchmarks/{id}",
|
|
413
|
-
"summary": "Delete a
|
|
439
|
+
"summary": "Delete a study and its tasks/facts (409 while running)",
|
|
414
440
|
"scope": "benchmarks:write",
|
|
415
441
|
"product": "experiment",
|
|
416
442
|
"pathParams": [
|
|
@@ -470,7 +496,7 @@ export const OPERATIONS = [
|
|
|
470
496
|
],
|
|
471
497
|
"method": "get",
|
|
472
498
|
"pathTemplate": "/experiment/v1/benchmarks/{id}",
|
|
473
|
-
"summary": "
|
|
499
|
+
"summary": "Study detail: status to poll, tasks + rubrics, per-run facts, live aggregate + verdict",
|
|
474
500
|
"scope": "benchmarks:read",
|
|
475
501
|
"product": "experiment",
|
|
476
502
|
"pathParams": [
|
|
@@ -520,7 +546,7 @@ export const OPERATIONS = [
|
|
|
520
546
|
],
|
|
521
547
|
"method": "get",
|
|
522
548
|
"pathTemplate": "/experiment/v1/benchmarks",
|
|
523
|
-
"summary": "List the org's
|
|
549
|
+
"summary": "List the org's studies (id, status, verdict, subject)",
|
|
524
550
|
"scope": "benchmarks:read",
|
|
525
551
|
"product": "experiment",
|
|
526
552
|
"pathParams": [],
|
|
@@ -541,7 +567,7 @@ export const OPERATIONS = [
|
|
|
541
567
|
],
|
|
542
568
|
"method": "get",
|
|
543
569
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/preview",
|
|
544
|
-
"summary": "Preview the PUBLISHED page's exact payload before publishing — the same projection, aggregate and behavior block
|
|
570
|
+
"summary": "Preview the PUBLISHED page's exact payload before publishing — the same projection, aggregate and behavior block publicStudies.get will serve, org-scoped and available at any status (no visibility requirement)",
|
|
545
571
|
"scope": "benchmarks:read",
|
|
546
572
|
"product": "experiment",
|
|
547
573
|
"pathParams": [
|
|
@@ -566,7 +592,7 @@ export const OPERATIONS = [
|
|
|
566
592
|
],
|
|
567
593
|
"method": "post",
|
|
568
594
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/publish",
|
|
569
|
-
"summary": "Publish a completed
|
|
595
|
+
"summary": "Publish a completed study under a shareable mode: 'public' (listed in the anonymous catalog, the default) or — where the deployment supports sharing — 'link' (unlisted, openable only with the returned share token). Idempotent slug upsert.",
|
|
570
596
|
"scope": "benchmarks:write",
|
|
571
597
|
"product": "experiment",
|
|
572
598
|
"pathParams": [
|
|
@@ -583,8 +609,21 @@ export const OPERATIONS = [
|
|
|
583
609
|
"name": "slug",
|
|
584
610
|
"required": false,
|
|
585
611
|
"kind": "string",
|
|
586
|
-
"description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the
|
|
612
|
+
"description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the study's own generated slug. 409 study_slug_taken if another published study holds it.",
|
|
587
613
|
"placeholder": "tavily-search-ab"
|
|
614
|
+
},
|
|
615
|
+
{
|
|
616
|
+
"name": "visibility",
|
|
617
|
+
"required": false,
|
|
618
|
+
"kind": "string",
|
|
619
|
+
"enum": [
|
|
620
|
+
"private",
|
|
621
|
+
"org",
|
|
622
|
+
"link",
|
|
623
|
+
"public"
|
|
624
|
+
],
|
|
625
|
+
"description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
|
|
626
|
+
"placeholder": "private"
|
|
588
627
|
}
|
|
589
628
|
],
|
|
590
629
|
"watchable": false,
|
|
@@ -626,7 +665,7 @@ export const OPERATIONS = [
|
|
|
626
665
|
],
|
|
627
666
|
"method": "post",
|
|
628
667
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/regrade",
|
|
629
|
-
"summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed
|
|
668
|
+
"summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
|
|
630
669
|
"scope": "benchmarks:write",
|
|
631
670
|
"product": "experiment",
|
|
632
671
|
"pathParams": [
|
|
@@ -676,7 +715,7 @@ export const OPERATIONS = [
|
|
|
676
715
|
],
|
|
677
716
|
"method": "post",
|
|
678
717
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/unpublish",
|
|
679
|
-
"summary": "Remove a
|
|
718
|
+
"summary": "Remove a study from the public catalog (data kept; re-publish any time)",
|
|
680
719
|
"scope": "benchmarks:write",
|
|
681
720
|
"product": "experiment",
|
|
682
721
|
"pathParams": [
|
|
@@ -701,7 +740,7 @@ export const OPERATIONS = [
|
|
|
701
740
|
],
|
|
702
741
|
"method": "put",
|
|
703
742
|
"pathTemplate": "/experiment/v1/benchmarks/{id}/lenses",
|
|
704
|
-
"summary": "Change which grading lenses a
|
|
743
|
+
"summary": "Change which grading lenses a study runs under before launch (status 'ready' only) — the complete replacement set, e.g. add 'evals' to a judge-only study so its tasks can carry expectations",
|
|
705
744
|
"scope": "benchmarks:write",
|
|
706
745
|
"product": "experiment",
|
|
707
746
|
"pathParams": [
|
|
@@ -756,14 +795,14 @@ export const OPERATIONS = [
|
|
|
756
795
|
"name": "prompt",
|
|
757
796
|
"required": true,
|
|
758
797
|
"kind": "string",
|
|
759
|
-
"description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the
|
|
798
|
+
"description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the study is 'ready' (pre-launch).",
|
|
760
799
|
"placeholder": "Research the three largest EU battery-storage tenders announced this quarter and cite each source."
|
|
761
800
|
},
|
|
762
801
|
{
|
|
763
802
|
"name": "expectations",
|
|
764
803
|
"required": false,
|
|
765
804
|
"kind": "array",
|
|
766
|
-
"description": "Replace this task's 'evals'-lens expectations (see
|
|
805
|
+
"description": "Replace this task's 'evals'-lens expectations (see studyCreateBody.tasks[].expectations). Omitted = expectations unchanged. REQUIRES the study's `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); validated the same way as create.",
|
|
767
806
|
"placeholder": "[]"
|
|
768
807
|
}
|
|
769
808
|
],
|
|
@@ -859,7 +898,7 @@ export const OPERATIONS = [
|
|
|
859
898
|
"pathTemplate": "/experiment/v1/catalog",
|
|
860
899
|
"summary": "Harness × model catalog: valid matrix cells, defaults, modes, and withheld harnesses with reasons",
|
|
861
900
|
"scope": null,
|
|
862
|
-
"product": "
|
|
901
|
+
"product": "platform",
|
|
863
902
|
"pathParams": [],
|
|
864
903
|
"queryParams": [],
|
|
865
904
|
"hasBody": false,
|
|
@@ -870,6 +909,262 @@ export const OPERATIONS = [
|
|
|
870
909
|
"command": "ora experiment catalog"
|
|
871
910
|
}
|
|
872
911
|
},
|
|
912
|
+
{
|
|
913
|
+
"id": "experiment.images.builds.cancel",
|
|
914
|
+
"commandPath": [
|
|
915
|
+
"experiment",
|
|
916
|
+
"images",
|
|
917
|
+
"builds",
|
|
918
|
+
"cancel"
|
|
919
|
+
],
|
|
920
|
+
"method": "post",
|
|
921
|
+
"pathTemplate": "/experiment/v1/images/builds/{id}/cancel",
|
|
922
|
+
"summary": "Cancel an in-flight image build",
|
|
923
|
+
"scope": "images:write",
|
|
924
|
+
"product": "platform",
|
|
925
|
+
"pathParams": [
|
|
926
|
+
{
|
|
927
|
+
"name": "id"
|
|
928
|
+
}
|
|
929
|
+
],
|
|
930
|
+
"queryParams": [],
|
|
931
|
+
"hasBody": false,
|
|
932
|
+
"bodyRequired": false,
|
|
933
|
+
"isStream": false,
|
|
934
|
+
"watchable": false,
|
|
935
|
+
"help": {
|
|
936
|
+
"command": "ora experiment images builds cancel <id>"
|
|
937
|
+
}
|
|
938
|
+
},
|
|
939
|
+
{
|
|
940
|
+
"id": "experiment.images.builds.create",
|
|
941
|
+
"commandPath": [
|
|
942
|
+
"experiment",
|
|
943
|
+
"images",
|
|
944
|
+
"builds",
|
|
945
|
+
"create"
|
|
946
|
+
],
|
|
947
|
+
"method": "post",
|
|
948
|
+
"pathTemplate": "/experiment/v1/images/builds",
|
|
949
|
+
"summary": "Submit an image build (idempotent on name + source + Dockerfile + build args)",
|
|
950
|
+
"scope": "images:write",
|
|
951
|
+
"product": "platform",
|
|
952
|
+
"pathParams": [],
|
|
953
|
+
"queryParams": [],
|
|
954
|
+
"hasBody": true,
|
|
955
|
+
"bodyRequired": true,
|
|
956
|
+
"isStream": false,
|
|
957
|
+
"bodyFields": [
|
|
958
|
+
{
|
|
959
|
+
"name": "name",
|
|
960
|
+
"required": true,
|
|
961
|
+
"kind": "string",
|
|
962
|
+
"placeholder": "<name>"
|
|
963
|
+
},
|
|
964
|
+
{
|
|
965
|
+
"name": "source",
|
|
966
|
+
"required": true,
|
|
967
|
+
"kind": "string",
|
|
968
|
+
"placeholder": "<source>"
|
|
969
|
+
},
|
|
970
|
+
{
|
|
971
|
+
"name": "dockerfile",
|
|
972
|
+
"required": false,
|
|
973
|
+
"kind": "object",
|
|
974
|
+
"placeholder": "{}"
|
|
975
|
+
},
|
|
976
|
+
{
|
|
977
|
+
"name": "buildArgs",
|
|
978
|
+
"required": false,
|
|
979
|
+
"kind": "object",
|
|
980
|
+
"placeholder": "{}"
|
|
981
|
+
},
|
|
982
|
+
{
|
|
983
|
+
"name": "platform",
|
|
984
|
+
"required": false,
|
|
985
|
+
"kind": "string",
|
|
986
|
+
"placeholder": "<platform>"
|
|
987
|
+
}
|
|
988
|
+
],
|
|
989
|
+
"watchable": false,
|
|
990
|
+
"help": {
|
|
991
|
+
"command": "ora experiment images builds create --file run.json",
|
|
992
|
+
"body": "{\n \"name\": \"<name>\",\n \"source\": \"<source>\"\n}"
|
|
993
|
+
}
|
|
994
|
+
},
|
|
995
|
+
{
|
|
996
|
+
"id": "experiment.images.builds.get",
|
|
997
|
+
"commandPath": [
|
|
998
|
+
"experiment",
|
|
999
|
+
"images",
|
|
1000
|
+
"builds",
|
|
1001
|
+
"get"
|
|
1002
|
+
],
|
|
1003
|
+
"method": "get",
|
|
1004
|
+
"pathTemplate": "/experiment/v1/images/builds/{id}",
|
|
1005
|
+
"summary": "Get one image build",
|
|
1006
|
+
"scope": "images:read",
|
|
1007
|
+
"product": "platform",
|
|
1008
|
+
"pathParams": [
|
|
1009
|
+
{
|
|
1010
|
+
"name": "id"
|
|
1011
|
+
}
|
|
1012
|
+
],
|
|
1013
|
+
"queryParams": [],
|
|
1014
|
+
"hasBody": false,
|
|
1015
|
+
"bodyRequired": false,
|
|
1016
|
+
"isStream": false,
|
|
1017
|
+
"watchable": false,
|
|
1018
|
+
"help": {
|
|
1019
|
+
"command": "ora experiment images builds get <id>"
|
|
1020
|
+
}
|
|
1021
|
+
},
|
|
1022
|
+
{
|
|
1023
|
+
"id": "experiment.images.builds.list",
|
|
1024
|
+
"commandPath": [
|
|
1025
|
+
"experiment",
|
|
1026
|
+
"images",
|
|
1027
|
+
"builds",
|
|
1028
|
+
"list"
|
|
1029
|
+
],
|
|
1030
|
+
"method": "get",
|
|
1031
|
+
"pathTemplate": "/experiment/v1/images/builds",
|
|
1032
|
+
"summary": "List the org's image builds, newest first",
|
|
1033
|
+
"scope": "images:read",
|
|
1034
|
+
"product": "platform",
|
|
1035
|
+
"pathParams": [],
|
|
1036
|
+
"queryParams": [
|
|
1037
|
+
{
|
|
1038
|
+
"name": "phase",
|
|
1039
|
+
"flag": "phase",
|
|
1040
|
+
"optionKey": "phase",
|
|
1041
|
+
"required": false,
|
|
1042
|
+
"kind": "string",
|
|
1043
|
+
"enum": [
|
|
1044
|
+
"pending",
|
|
1045
|
+
"running",
|
|
1046
|
+
"succeeded",
|
|
1047
|
+
"failed",
|
|
1048
|
+
"cancelled"
|
|
1049
|
+
]
|
|
1050
|
+
},
|
|
1051
|
+
{
|
|
1052
|
+
"name": "name",
|
|
1053
|
+
"flag": "name",
|
|
1054
|
+
"optionKey": "name",
|
|
1055
|
+
"required": false,
|
|
1056
|
+
"kind": "string"
|
|
1057
|
+
},
|
|
1058
|
+
{
|
|
1059
|
+
"name": "pageSize",
|
|
1060
|
+
"flag": "page-size",
|
|
1061
|
+
"optionKey": "pageSize",
|
|
1062
|
+
"required": false,
|
|
1063
|
+
"kind": "number"
|
|
1064
|
+
},
|
|
1065
|
+
{
|
|
1066
|
+
"name": "pageToken",
|
|
1067
|
+
"flag": "page-token",
|
|
1068
|
+
"optionKey": "pageToken",
|
|
1069
|
+
"required": false,
|
|
1070
|
+
"kind": "string"
|
|
1071
|
+
}
|
|
1072
|
+
],
|
|
1073
|
+
"hasBody": false,
|
|
1074
|
+
"bodyRequired": false,
|
|
1075
|
+
"isStream": false,
|
|
1076
|
+
"watchable": false,
|
|
1077
|
+
"help": {
|
|
1078
|
+
"command": "ora experiment images builds list"
|
|
1079
|
+
}
|
|
1080
|
+
},
|
|
1081
|
+
{
|
|
1082
|
+
"id": "experiment.images.builds.stream",
|
|
1083
|
+
"commandPath": [
|
|
1084
|
+
"experiment",
|
|
1085
|
+
"images",
|
|
1086
|
+
"builds",
|
|
1087
|
+
"stream"
|
|
1088
|
+
],
|
|
1089
|
+
"method": "get",
|
|
1090
|
+
"pathTemplate": "/experiment/v1/images/builds/{id}/stream",
|
|
1091
|
+
"summary": "SSE stream of one image build's row changes",
|
|
1092
|
+
"scope": "images:read",
|
|
1093
|
+
"product": "platform",
|
|
1094
|
+
"pathParams": [
|
|
1095
|
+
{
|
|
1096
|
+
"name": "id"
|
|
1097
|
+
}
|
|
1098
|
+
],
|
|
1099
|
+
"queryParams": [],
|
|
1100
|
+
"hasBody": false,
|
|
1101
|
+
"bodyRequired": false,
|
|
1102
|
+
"isStream": true,
|
|
1103
|
+
"watchable": false,
|
|
1104
|
+
"help": {
|
|
1105
|
+
"command": "ora experiment images builds stream <id>"
|
|
1106
|
+
}
|
|
1107
|
+
},
|
|
1108
|
+
{
|
|
1109
|
+
"id": "experiment.images.runs.create",
|
|
1110
|
+
"commandPath": [
|
|
1111
|
+
"experiment",
|
|
1112
|
+
"images",
|
|
1113
|
+
"runs",
|
|
1114
|
+
"create"
|
|
1115
|
+
],
|
|
1116
|
+
"method": "post",
|
|
1117
|
+
"pathTemplate": "/experiment/v1/images/runs",
|
|
1118
|
+
"summary": "Launch a run (harness or environment) from a built or referenced image",
|
|
1119
|
+
"scope": "images:write",
|
|
1120
|
+
"product": "platform",
|
|
1121
|
+
"pathParams": [],
|
|
1122
|
+
"queryParams": [],
|
|
1123
|
+
"hasBody": true,
|
|
1124
|
+
"bodyRequired": true,
|
|
1125
|
+
"isStream": false,
|
|
1126
|
+
"bodyFields": [
|
|
1127
|
+
{
|
|
1128
|
+
"name": "buildId",
|
|
1129
|
+
"required": false,
|
|
1130
|
+
"kind": "string",
|
|
1131
|
+
"placeholder": "<buildId>"
|
|
1132
|
+
},
|
|
1133
|
+
{
|
|
1134
|
+
"name": "image",
|
|
1135
|
+
"required": false,
|
|
1136
|
+
"kind": "string",
|
|
1137
|
+
"placeholder": "<image>"
|
|
1138
|
+
},
|
|
1139
|
+
{
|
|
1140
|
+
"name": "mode",
|
|
1141
|
+
"required": true,
|
|
1142
|
+
"kind": "string",
|
|
1143
|
+
"enum": [
|
|
1144
|
+
"harness",
|
|
1145
|
+
"environment"
|
|
1146
|
+
],
|
|
1147
|
+
"placeholder": "harness"
|
|
1148
|
+
},
|
|
1149
|
+
{
|
|
1150
|
+
"name": "harness",
|
|
1151
|
+
"required": false,
|
|
1152
|
+
"kind": "string",
|
|
1153
|
+
"placeholder": "<harness>"
|
|
1154
|
+
},
|
|
1155
|
+
{
|
|
1156
|
+
"name": "task",
|
|
1157
|
+
"required": false,
|
|
1158
|
+
"kind": "string",
|
|
1159
|
+
"placeholder": "<task>"
|
|
1160
|
+
}
|
|
1161
|
+
],
|
|
1162
|
+
"watchable": false,
|
|
1163
|
+
"help": {
|
|
1164
|
+
"command": "ora experiment images runs create --file run.json",
|
|
1165
|
+
"body": "{\n \"mode\": \"harness\"\n}"
|
|
1166
|
+
}
|
|
1167
|
+
},
|
|
873
1168
|
{
|
|
874
1169
|
"id": "experiment.runs.cancel",
|
|
875
1170
|
"commandPath": [
|
|
@@ -1025,6 +1320,12 @@ export const OPERATIONS = [
|
|
|
1025
1320
|
"kind": "array",
|
|
1026
1321
|
"placeholder": "[]"
|
|
1027
1322
|
},
|
|
1323
|
+
{
|
|
1324
|
+
"name": "model_params",
|
|
1325
|
+
"required": false,
|
|
1326
|
+
"kind": "object",
|
|
1327
|
+
"placeholder": "{}"
|
|
1328
|
+
},
|
|
1028
1329
|
{
|
|
1029
1330
|
"name": "wait",
|
|
1030
1331
|
"required": false,
|
|
@@ -1354,16 +1655,17 @@ export const OPERATIONS = [
|
|
|
1354
1655
|
}
|
|
1355
1656
|
},
|
|
1356
1657
|
{
|
|
1357
|
-
"id": "
|
|
1658
|
+
"id": "experiment.skills.create",
|
|
1358
1659
|
"commandPath": [
|
|
1359
|
-
"
|
|
1660
|
+
"experiment",
|
|
1661
|
+
"skills",
|
|
1360
1662
|
"create"
|
|
1361
1663
|
],
|
|
1362
1664
|
"method": "post",
|
|
1363
|
-
"pathTemplate": "/
|
|
1364
|
-
"summary": "
|
|
1365
|
-
"scope": "
|
|
1366
|
-
"product": "
|
|
1665
|
+
"pathTemplate": "/experiment/v1/skills",
|
|
1666
|
+
"summary": "Create a skill (custom or git-imported)",
|
|
1667
|
+
"scope": "skills:write",
|
|
1668
|
+
"product": "platform",
|
|
1367
1669
|
"pathParams": [],
|
|
1368
1670
|
"queryParams": [],
|
|
1369
1671
|
"hasBody": true,
|
|
@@ -1371,26 +1673,195 @@ export const OPERATIONS = [
|
|
|
1371
1673
|
"isStream": false,
|
|
1372
1674
|
"bodyFields": [
|
|
1373
1675
|
{
|
|
1374
|
-
"name": "
|
|
1676
|
+
"name": "source",
|
|
1375
1677
|
"required": true,
|
|
1376
1678
|
"kind": "string",
|
|
1377
|
-
"placeholder": "
|
|
1679
|
+
"placeholder": "custom"
|
|
1378
1680
|
},
|
|
1379
1681
|
{
|
|
1380
|
-
"name": "
|
|
1682
|
+
"name": "name",
|
|
1381
1683
|
"required": true,
|
|
1382
1684
|
"kind": "string",
|
|
1383
|
-
"placeholder": "<
|
|
1685
|
+
"placeholder": "<name>"
|
|
1384
1686
|
},
|
|
1385
1687
|
{
|
|
1386
|
-
"name": "
|
|
1688
|
+
"name": "body",
|
|
1689
|
+
"required": true,
|
|
1690
|
+
"kind": "string",
|
|
1691
|
+
"placeholder": "<body>"
|
|
1692
|
+
},
|
|
1693
|
+
{
|
|
1694
|
+
"name": "visibility",
|
|
1387
1695
|
"required": false,
|
|
1388
1696
|
"kind": "string",
|
|
1389
|
-
"
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1697
|
+
"enum": [
|
|
1698
|
+
"user",
|
|
1699
|
+
"org"
|
|
1700
|
+
],
|
|
1701
|
+
"placeholder": "user"
|
|
1702
|
+
}
|
|
1703
|
+
],
|
|
1704
|
+
"watchable": false,
|
|
1705
|
+
"help": {
|
|
1706
|
+
"command": "ora experiment skills create --file run.json",
|
|
1707
|
+
"body": "{\n \"source\": \"custom\",\n \"name\": \"<name>\",\n \"body\": \"<body>\"\n}"
|
|
1708
|
+
}
|
|
1709
|
+
},
|
|
1710
|
+
{
|
|
1711
|
+
"id": "experiment.skills.delete",
|
|
1712
|
+
"commandPath": [
|
|
1713
|
+
"experiment",
|
|
1714
|
+
"skills",
|
|
1715
|
+
"delete"
|
|
1716
|
+
],
|
|
1717
|
+
"method": "delete",
|
|
1718
|
+
"pathTemplate": "/experiment/v1/skills/{id}",
|
|
1719
|
+
"summary": "Delete a skill",
|
|
1720
|
+
"scope": "skills:write",
|
|
1721
|
+
"product": "platform",
|
|
1722
|
+
"pathParams": [
|
|
1723
|
+
{
|
|
1724
|
+
"name": "id"
|
|
1725
|
+
}
|
|
1726
|
+
],
|
|
1727
|
+
"queryParams": [],
|
|
1728
|
+
"hasBody": false,
|
|
1729
|
+
"bodyRequired": false,
|
|
1730
|
+
"isStream": false,
|
|
1731
|
+
"watchable": false,
|
|
1732
|
+
"help": {
|
|
1733
|
+
"command": "ora experiment skills delete <id>"
|
|
1734
|
+
}
|
|
1735
|
+
},
|
|
1736
|
+
{
|
|
1737
|
+
"id": "experiment.skills.list",
|
|
1738
|
+
"commandPath": [
|
|
1739
|
+
"experiment",
|
|
1740
|
+
"skills",
|
|
1741
|
+
"list"
|
|
1742
|
+
],
|
|
1743
|
+
"method": "get",
|
|
1744
|
+
"pathTemplate": "/experiment/v1/skills",
|
|
1745
|
+
"summary": "List the caller's usable skills catalog",
|
|
1746
|
+
"scope": "skills:read",
|
|
1747
|
+
"product": "platform",
|
|
1748
|
+
"pathParams": [],
|
|
1749
|
+
"queryParams": [],
|
|
1750
|
+
"hasBody": false,
|
|
1751
|
+
"bodyRequired": false,
|
|
1752
|
+
"isStream": false,
|
|
1753
|
+
"watchable": false,
|
|
1754
|
+
"help": {
|
|
1755
|
+
"command": "ora experiment skills list"
|
|
1756
|
+
}
|
|
1757
|
+
},
|
|
1758
|
+
{
|
|
1759
|
+
"id": "experiment.skills.update",
|
|
1760
|
+
"commandPath": [
|
|
1761
|
+
"experiment",
|
|
1762
|
+
"skills",
|
|
1763
|
+
"update"
|
|
1764
|
+
],
|
|
1765
|
+
"method": "put",
|
|
1766
|
+
"pathTemplate": "/experiment/v1/skills/{id}",
|
|
1767
|
+
"summary": "Update a skill",
|
|
1768
|
+
"scope": "skills:write",
|
|
1769
|
+
"product": "platform",
|
|
1770
|
+
"pathParams": [
|
|
1771
|
+
{
|
|
1772
|
+
"name": "id"
|
|
1773
|
+
}
|
|
1774
|
+
],
|
|
1775
|
+
"queryParams": [],
|
|
1776
|
+
"hasBody": true,
|
|
1777
|
+
"bodyRequired": true,
|
|
1778
|
+
"isStream": false,
|
|
1779
|
+
"bodyFields": [
|
|
1780
|
+
{
|
|
1781
|
+
"name": "name",
|
|
1782
|
+
"required": false,
|
|
1783
|
+
"kind": "string",
|
|
1784
|
+
"placeholder": "<name>"
|
|
1785
|
+
},
|
|
1786
|
+
{
|
|
1787
|
+
"name": "visibility",
|
|
1788
|
+
"required": false,
|
|
1789
|
+
"kind": "string",
|
|
1790
|
+
"enum": [
|
|
1791
|
+
"user",
|
|
1792
|
+
"org"
|
|
1793
|
+
],
|
|
1794
|
+
"placeholder": "user"
|
|
1795
|
+
},
|
|
1796
|
+
{
|
|
1797
|
+
"name": "body",
|
|
1798
|
+
"required": false,
|
|
1799
|
+
"kind": "string",
|
|
1800
|
+
"placeholder": "<body>"
|
|
1801
|
+
},
|
|
1802
|
+
{
|
|
1803
|
+
"name": "repoUrl",
|
|
1804
|
+
"required": false,
|
|
1805
|
+
"kind": "string",
|
|
1806
|
+
"placeholder": "<repoUrl>"
|
|
1807
|
+
},
|
|
1808
|
+
{
|
|
1809
|
+
"name": "gitRef",
|
|
1810
|
+
"required": false,
|
|
1811
|
+
"kind": "string",
|
|
1812
|
+
"placeholder": "<gitRef>"
|
|
1813
|
+
},
|
|
1814
|
+
{
|
|
1815
|
+
"name": "subpath",
|
|
1816
|
+
"required": false,
|
|
1817
|
+
"kind": "string",
|
|
1818
|
+
"placeholder": "<subpath>"
|
|
1819
|
+
}
|
|
1820
|
+
],
|
|
1821
|
+
"watchable": false,
|
|
1822
|
+
"help": {
|
|
1823
|
+
"command": "ora experiment skills update <id> --file run.json",
|
|
1824
|
+
"body": "{\n \"name\": \"<name>\",\n \"visibility\": \"user\",\n \"body\": \"<body>\"\n}"
|
|
1825
|
+
}
|
|
1826
|
+
},
|
|
1827
|
+
{
|
|
1828
|
+
"id": "icps.create",
|
|
1829
|
+
"commandPath": [
|
|
1830
|
+
"icps",
|
|
1831
|
+
"create"
|
|
1832
|
+
],
|
|
1833
|
+
"method": "post",
|
|
1834
|
+
"pathTemplate": "/studio/v1/icps",
|
|
1835
|
+
"summary": "Record an ICP for a tracked domain (get-or-create on its label)",
|
|
1836
|
+
"scope": "intents:write",
|
|
1837
|
+
"product": "studio",
|
|
1838
|
+
"pathParams": [],
|
|
1839
|
+
"queryParams": [],
|
|
1840
|
+
"hasBody": true,
|
|
1841
|
+
"bodyRequired": true,
|
|
1842
|
+
"isStream": false,
|
|
1843
|
+
"bodyFields": [
|
|
1844
|
+
{
|
|
1845
|
+
"name": "domainId",
|
|
1846
|
+
"required": true,
|
|
1847
|
+
"kind": "string",
|
|
1848
|
+
"placeholder": "<domainId>"
|
|
1849
|
+
},
|
|
1850
|
+
{
|
|
1851
|
+
"name": "description",
|
|
1852
|
+
"required": true,
|
|
1853
|
+
"kind": "string",
|
|
1854
|
+
"placeholder": "<description>"
|
|
1855
|
+
},
|
|
1856
|
+
{
|
|
1857
|
+
"name": "label",
|
|
1858
|
+
"required": false,
|
|
1859
|
+
"kind": "string",
|
|
1860
|
+
"placeholder": "<label>"
|
|
1861
|
+
}
|
|
1862
|
+
],
|
|
1863
|
+
"watchable": false,
|
|
1864
|
+
"help": {
|
|
1394
1865
|
"command": "ora icps create --file run.json",
|
|
1395
1866
|
"body": "{\n \"domainId\": \"<domainId>\",\n \"description\": \"<description>\"\n}"
|
|
1396
1867
|
}
|
|
@@ -2165,6 +2636,12 @@ export const OPERATIONS = [
|
|
|
2165
2636
|
"kind": "array",
|
|
2166
2637
|
"placeholder": "[]"
|
|
2167
2638
|
},
|
|
2639
|
+
{
|
|
2640
|
+
"name": "browser",
|
|
2641
|
+
"required": false,
|
|
2642
|
+
"kind": "object",
|
|
2643
|
+
"placeholder": "{}"
|
|
2644
|
+
},
|
|
2168
2645
|
{
|
|
2169
2646
|
"name": "schedule",
|
|
2170
2647
|
"required": false,
|
|
@@ -2426,16 +2903,43 @@ export const OPERATIONS = [
|
|
|
2426
2903
|
}
|
|
2427
2904
|
},
|
|
2428
2905
|
{
|
|
2429
|
-
"id": "
|
|
2906
|
+
"id": "studio.runs.cancel",
|
|
2430
2907
|
"commandPath": [
|
|
2431
|
-
"
|
|
2432
|
-
"
|
|
2908
|
+
"studio",
|
|
2909
|
+
"runs",
|
|
2910
|
+
"cancel"
|
|
2433
2911
|
],
|
|
2434
2912
|
"method": "post",
|
|
2435
|
-
"pathTemplate": "/
|
|
2436
|
-
"summary": "
|
|
2437
|
-
"scope": "
|
|
2438
|
-
"product": "
|
|
2913
|
+
"pathTemplate": "/studio/v1/runs/{id}/cancel",
|
|
2914
|
+
"summary": "Cancel a run (terminate it and stop its engine)",
|
|
2915
|
+
"scope": "runs:write",
|
|
2916
|
+
"product": "platform",
|
|
2917
|
+
"pathParams": [
|
|
2918
|
+
{
|
|
2919
|
+
"name": "id"
|
|
2920
|
+
}
|
|
2921
|
+
],
|
|
2922
|
+
"queryParams": [],
|
|
2923
|
+
"hasBody": false,
|
|
2924
|
+
"bodyRequired": false,
|
|
2925
|
+
"isStream": false,
|
|
2926
|
+
"watchable": false,
|
|
2927
|
+
"help": {
|
|
2928
|
+
"command": "ora studio runs cancel <id>"
|
|
2929
|
+
}
|
|
2930
|
+
},
|
|
2931
|
+
{
|
|
2932
|
+
"id": "studio.runs.cancelBatch",
|
|
2933
|
+
"commandPath": [
|
|
2934
|
+
"studio",
|
|
2935
|
+
"runs",
|
|
2936
|
+
"cancel-batch"
|
|
2937
|
+
],
|
|
2938
|
+
"method": "post",
|
|
2939
|
+
"pathTemplate": "/studio/v1/runs/cancel",
|
|
2940
|
+
"summary": "Cancel a batch of runs (terminate them and stop their engines)",
|
|
2941
|
+
"scope": "runs:write",
|
|
2942
|
+
"product": "platform",
|
|
2439
2943
|
"pathParams": [],
|
|
2440
2944
|
"queryParams": [],
|
|
2441
2945
|
"hasBody": true,
|
|
@@ -2443,51 +2947,2466 @@ export const OPERATIONS = [
|
|
|
2443
2947
|
"isStream": false,
|
|
2444
2948
|
"bodyFields": [
|
|
2445
2949
|
{
|
|
2446
|
-
"name": "
|
|
2950
|
+
"name": "runIds",
|
|
2447
2951
|
"required": true,
|
|
2952
|
+
"kind": "array",
|
|
2953
|
+
"placeholder": "[]"
|
|
2954
|
+
}
|
|
2955
|
+
],
|
|
2956
|
+
"watchable": false,
|
|
2957
|
+
"help": {
|
|
2958
|
+
"command": "ora studio runs cancel-batch --file run.json",
|
|
2959
|
+
"body": "{\n \"runIds\": []\n}"
|
|
2960
|
+
}
|
|
2961
|
+
},
|
|
2962
|
+
{
|
|
2963
|
+
"id": "studio.runs.create",
|
|
2964
|
+
"commandPath": [
|
|
2965
|
+
"studio",
|
|
2966
|
+
"runs",
|
|
2967
|
+
"create"
|
|
2968
|
+
],
|
|
2969
|
+
"method": "post",
|
|
2970
|
+
"pathTemplate": "/studio/v1/runs",
|
|
2971
|
+
"summary": "Create a run",
|
|
2972
|
+
"scope": "runs:write",
|
|
2973
|
+
"product": "platform",
|
|
2974
|
+
"pathParams": [],
|
|
2975
|
+
"queryParams": [],
|
|
2976
|
+
"hasBody": true,
|
|
2977
|
+
"bodyRequired": true,
|
|
2978
|
+
"isStream": false,
|
|
2979
|
+
"bodyFields": [
|
|
2980
|
+
{
|
|
2981
|
+
"name": "product_key",
|
|
2982
|
+
"required": false,
|
|
2448
2983
|
"kind": "string",
|
|
2449
|
-
"placeholder": "
|
|
2984
|
+
"placeholder": "<product_key>"
|
|
2450
2985
|
},
|
|
2451
2986
|
{
|
|
2452
|
-
"name": "
|
|
2987
|
+
"name": "intent",
|
|
2453
2988
|
"required": true,
|
|
2454
2989
|
"kind": "string",
|
|
2455
|
-
"placeholder": "
|
|
2990
|
+
"placeholder": "check pricing for acme.com"
|
|
2456
2991
|
},
|
|
2457
2992
|
{
|
|
2458
|
-
"name": "
|
|
2459
|
-
"required":
|
|
2993
|
+
"name": "domain",
|
|
2994
|
+
"required": false,
|
|
2460
2995
|
"kind": "string",
|
|
2461
|
-
"placeholder": "<
|
|
2996
|
+
"placeholder": "<domain>"
|
|
2462
2997
|
},
|
|
2463
2998
|
{
|
|
2464
|
-
"name": "
|
|
2999
|
+
"name": "harness",
|
|
2465
3000
|
"required": false,
|
|
2466
3001
|
"kind": "string",
|
|
2467
|
-
"
|
|
2468
|
-
|
|
2469
|
-
|
|
3002
|
+
"placeholder": "eve"
|
|
3003
|
+
},
|
|
3004
|
+
{
|
|
3005
|
+
"name": "model",
|
|
3006
|
+
"required": false,
|
|
3007
|
+
"kind": "string",
|
|
3008
|
+
"placeholder": "claude-sonnet-5"
|
|
3009
|
+
},
|
|
3010
|
+
{
|
|
3011
|
+
"name": "mode",
|
|
3012
|
+
"required": false,
|
|
3013
|
+
"kind": "string",
|
|
3014
|
+
"enum": [
|
|
3015
|
+
"task",
|
|
3016
|
+
"session"
|
|
3017
|
+
],
|
|
3018
|
+
"placeholder": "task"
|
|
3019
|
+
},
|
|
3020
|
+
{
|
|
3021
|
+
"name": "addons",
|
|
3022
|
+
"required": false,
|
|
3023
|
+
"kind": "object",
|
|
3024
|
+
"placeholder": "{}"
|
|
3025
|
+
},
|
|
3026
|
+
{
|
|
3027
|
+
"name": "engine",
|
|
3028
|
+
"required": false,
|
|
3029
|
+
"kind": "string",
|
|
3030
|
+
"placeholder": "<engine>"
|
|
3031
|
+
},
|
|
3032
|
+
{
|
|
3033
|
+
"name": "execution_kind",
|
|
3034
|
+
"required": false,
|
|
3035
|
+
"kind": "string",
|
|
3036
|
+
"enum": [
|
|
3037
|
+
"harness",
|
|
3038
|
+
"direct"
|
|
3039
|
+
],
|
|
3040
|
+
"placeholder": "harness"
|
|
3041
|
+
},
|
|
3042
|
+
{
|
|
3043
|
+
"name": "tools",
|
|
3044
|
+
"required": false,
|
|
3045
|
+
"kind": "string",
|
|
3046
|
+
"placeholder": "<tools>"
|
|
3047
|
+
},
|
|
3048
|
+
{
|
|
3049
|
+
"name": "requires_tools",
|
|
3050
|
+
"required": false,
|
|
3051
|
+
"kind": "boolean",
|
|
3052
|
+
"placeholder": "true"
|
|
3053
|
+
},
|
|
3054
|
+
{
|
|
3055
|
+
"name": "mcp_servers",
|
|
3056
|
+
"required": false,
|
|
3057
|
+
"kind": "array",
|
|
3058
|
+
"placeholder": "[]"
|
|
3059
|
+
},
|
|
3060
|
+
{
|
|
3061
|
+
"name": "model_params",
|
|
3062
|
+
"required": false,
|
|
3063
|
+
"kind": "object",
|
|
3064
|
+
"placeholder": "{}"
|
|
3065
|
+
},
|
|
3066
|
+
{
|
|
3067
|
+
"name": "wait",
|
|
3068
|
+
"required": false,
|
|
3069
|
+
"kind": "boolean",
|
|
3070
|
+
"placeholder": "true"
|
|
3071
|
+
}
|
|
3072
|
+
],
|
|
3073
|
+
"watchable": true,
|
|
3074
|
+
"help": {
|
|
3075
|
+
"command": "ora studio runs create --file run.json --watch",
|
|
3076
|
+
"body": "{\n \"intent\": \"check pricing for acme.com\",\n \"harness\": \"eve\",\n \"model\": \"claude-sonnet-5\"\n}"
|
|
3077
|
+
}
|
|
3078
|
+
},
|
|
3079
|
+
{
|
|
3080
|
+
"id": "studio.runs.directSignals",
|
|
3081
|
+
"commandPath": [
|
|
3082
|
+
"studio",
|
|
3083
|
+
"runs",
|
|
3084
|
+
"direct-signals"
|
|
3085
|
+
],
|
|
3086
|
+
"method": "get",
|
|
3087
|
+
"pathTemplate": "/studio/v1/runs/{id}/direct-signals",
|
|
3088
|
+
"summary": "Read a direct run's extracted DirectSignals artifact",
|
|
3089
|
+
"scope": "runs:read",
|
|
3090
|
+
"product": "platform",
|
|
3091
|
+
"pathParams": [
|
|
3092
|
+
{
|
|
3093
|
+
"name": "id"
|
|
3094
|
+
}
|
|
3095
|
+
],
|
|
3096
|
+
"queryParams": [],
|
|
3097
|
+
"hasBody": false,
|
|
3098
|
+
"bodyRequired": false,
|
|
3099
|
+
"isStream": false,
|
|
3100
|
+
"watchable": false,
|
|
3101
|
+
"help": {
|
|
3102
|
+
"command": "ora studio runs direct-signals <id>"
|
|
3103
|
+
}
|
|
3104
|
+
},
|
|
3105
|
+
{
|
|
3106
|
+
"id": "studio.runs.feed",
|
|
3107
|
+
"commandPath": [
|
|
3108
|
+
"studio",
|
|
3109
|
+
"runs",
|
|
3110
|
+
"feed"
|
|
3111
|
+
],
|
|
3112
|
+
"method": "get",
|
|
3113
|
+
"pathTemplate": "/studio/v1/runs/stream",
|
|
3114
|
+
"summary": "Org-scoped live feed of run status patches (SSE)",
|
|
3115
|
+
"scope": "runs:read",
|
|
3116
|
+
"product": "platform",
|
|
3117
|
+
"pathParams": [],
|
|
3118
|
+
"queryParams": [],
|
|
3119
|
+
"hasBody": false,
|
|
3120
|
+
"bodyRequired": false,
|
|
3121
|
+
"isStream": true,
|
|
3122
|
+
"watchable": false,
|
|
3123
|
+
"help": {
|
|
3124
|
+
"command": "ora studio runs feed"
|
|
3125
|
+
}
|
|
3126
|
+
},
|
|
3127
|
+
{
|
|
3128
|
+
"id": "studio.runs.insight",
|
|
3129
|
+
"commandPath": [
|
|
3130
|
+
"studio",
|
|
3131
|
+
"runs",
|
|
3132
|
+
"insight"
|
|
3133
|
+
],
|
|
3134
|
+
"method": "get",
|
|
3135
|
+
"pathTemplate": "/studio/v1/runs/{id}/insight",
|
|
3136
|
+
"summary": "Read a run's stored insight artifact",
|
|
3137
|
+
"scope": "runs:read",
|
|
3138
|
+
"product": "platform",
|
|
3139
|
+
"pathParams": [
|
|
3140
|
+
{
|
|
3141
|
+
"name": "id"
|
|
3142
|
+
}
|
|
3143
|
+
],
|
|
3144
|
+
"queryParams": [],
|
|
3145
|
+
"hasBody": false,
|
|
3146
|
+
"bodyRequired": false,
|
|
3147
|
+
"isStream": false,
|
|
3148
|
+
"watchable": false,
|
|
3149
|
+
"help": {
|
|
3150
|
+
"command": "ora studio runs insight <id>"
|
|
3151
|
+
}
|
|
3152
|
+
},
|
|
3153
|
+
{
|
|
3154
|
+
"id": "studio.runs.list",
|
|
3155
|
+
"commandPath": [
|
|
3156
|
+
"studio",
|
|
3157
|
+
"runs",
|
|
3158
|
+
"list"
|
|
3159
|
+
],
|
|
3160
|
+
"method": "get",
|
|
3161
|
+
"pathTemplate": "/studio/v1/runs",
|
|
3162
|
+
"summary": "List runs",
|
|
3163
|
+
"scope": "runs:read",
|
|
3164
|
+
"product": "platform",
|
|
3165
|
+
"pathParams": [],
|
|
3166
|
+
"queryParams": [
|
|
3167
|
+
{
|
|
3168
|
+
"name": "pageSize",
|
|
3169
|
+
"flag": "page-size",
|
|
3170
|
+
"optionKey": "pageSize",
|
|
3171
|
+
"required": false,
|
|
3172
|
+
"kind": "number"
|
|
3173
|
+
},
|
|
3174
|
+
{
|
|
3175
|
+
"name": "pageToken",
|
|
3176
|
+
"flag": "page-token",
|
|
3177
|
+
"optionKey": "pageToken",
|
|
3178
|
+
"required": false,
|
|
3179
|
+
"kind": "string"
|
|
3180
|
+
}
|
|
3181
|
+
],
|
|
3182
|
+
"hasBody": false,
|
|
3183
|
+
"bodyRequired": false,
|
|
3184
|
+
"isStream": false,
|
|
3185
|
+
"watchable": false,
|
|
3186
|
+
"help": {
|
|
3187
|
+
"command": "ora studio runs list"
|
|
3188
|
+
}
|
|
3189
|
+
},
|
|
3190
|
+
{
|
|
3191
|
+
"id": "studio.runs.rawTrace",
|
|
3192
|
+
"commandPath": [
|
|
3193
|
+
"studio",
|
|
3194
|
+
"runs",
|
|
3195
|
+
"raw-trace"
|
|
3196
|
+
],
|
|
3197
|
+
"method": "get",
|
|
3198
|
+
"pathTemplate": "/studio/v1/runs/{id}/raw-trace",
|
|
3199
|
+
"summary": "Read a run's full raw frame stream (pre-truncation trace artifact)",
|
|
3200
|
+
"scope": "runs:read",
|
|
3201
|
+
"product": "platform",
|
|
3202
|
+
"pathParams": [
|
|
3203
|
+
{
|
|
3204
|
+
"name": "id"
|
|
3205
|
+
}
|
|
3206
|
+
],
|
|
3207
|
+
"queryParams": [],
|
|
3208
|
+
"hasBody": false,
|
|
3209
|
+
"bodyRequired": false,
|
|
3210
|
+
"isStream": false,
|
|
3211
|
+
"watchable": false,
|
|
3212
|
+
"help": {
|
|
3213
|
+
"command": "ora studio runs raw-trace <id>"
|
|
3214
|
+
}
|
|
3215
|
+
},
|
|
3216
|
+
{
|
|
3217
|
+
"id": "studio.runs.spans",
|
|
3218
|
+
"commandPath": [
|
|
3219
|
+
"studio",
|
|
3220
|
+
"runs",
|
|
3221
|
+
"spans"
|
|
3222
|
+
],
|
|
3223
|
+
"method": "get",
|
|
3224
|
+
"pathTemplate": "/studio/v1/runs/{id}/spans",
|
|
3225
|
+
"summary": "Read a run's stored trace spans (CompactSpan[])",
|
|
3226
|
+
"scope": "runs:read",
|
|
3227
|
+
"product": "platform",
|
|
3228
|
+
"pathParams": [
|
|
3229
|
+
{
|
|
3230
|
+
"name": "id"
|
|
3231
|
+
}
|
|
3232
|
+
],
|
|
3233
|
+
"queryParams": [],
|
|
3234
|
+
"hasBody": false,
|
|
3235
|
+
"bodyRequired": false,
|
|
3236
|
+
"isStream": false,
|
|
3237
|
+
"watchable": false,
|
|
3238
|
+
"help": {
|
|
3239
|
+
"command": "ora studio runs spans <id>"
|
|
3240
|
+
}
|
|
3241
|
+
},
|
|
3242
|
+
{
|
|
3243
|
+
"id": "studio.runs.stream",
|
|
3244
|
+
"commandPath": [
|
|
3245
|
+
"studio",
|
|
3246
|
+
"runs",
|
|
3247
|
+
"stream"
|
|
3248
|
+
],
|
|
3249
|
+
"method": "get",
|
|
3250
|
+
"pathTemplate": "/studio/v1/runs/{id}/stream",
|
|
3251
|
+
"summary": "Stream a run's live trajectory (SSE)",
|
|
3252
|
+
"scope": "runs:read",
|
|
3253
|
+
"product": "platform",
|
|
3254
|
+
"pathParams": [
|
|
3255
|
+
{
|
|
3256
|
+
"name": "id"
|
|
3257
|
+
}
|
|
3258
|
+
],
|
|
3259
|
+
"queryParams": [],
|
|
3260
|
+
"hasBody": false,
|
|
3261
|
+
"bodyRequired": false,
|
|
3262
|
+
"isStream": true,
|
|
3263
|
+
"watchable": false,
|
|
3264
|
+
"help": {
|
|
3265
|
+
"command": "ora studio runs stream <id>"
|
|
3266
|
+
}
|
|
3267
|
+
},
|
|
3268
|
+
{
|
|
3269
|
+
"id": "studio.runs.trajectory",
|
|
3270
|
+
"commandPath": [
|
|
3271
|
+
"studio",
|
|
3272
|
+
"runs",
|
|
3273
|
+
"trajectory"
|
|
3274
|
+
],
|
|
3275
|
+
"method": "get",
|
|
3276
|
+
"pathTemplate": "/studio/v1/runs/{id}/trajectory",
|
|
3277
|
+
"summary": "Read a run's normalized trajectory snapshot",
|
|
3278
|
+
"scope": "runs:read",
|
|
3279
|
+
"product": "platform",
|
|
3280
|
+
"pathParams": [
|
|
3281
|
+
{
|
|
3282
|
+
"name": "id"
|
|
3283
|
+
}
|
|
3284
|
+
],
|
|
3285
|
+
"queryParams": [
|
|
3286
|
+
{
|
|
3287
|
+
"name": "profile",
|
|
3288
|
+
"flag": "profile",
|
|
3289
|
+
"optionKey": "profile",
|
|
3290
|
+
"required": false,
|
|
3291
|
+
"kind": "string",
|
|
3292
|
+
"enum": [
|
|
3293
|
+
"raw",
|
|
3294
|
+
"heuristic",
|
|
3295
|
+
"full"
|
|
3296
|
+
]
|
|
3297
|
+
}
|
|
3298
|
+
],
|
|
3299
|
+
"hasBody": false,
|
|
3300
|
+
"bodyRequired": false,
|
|
3301
|
+
"isStream": false,
|
|
3302
|
+
"watchable": false,
|
|
3303
|
+
"help": {
|
|
3304
|
+
"command": "ora studio runs trajectory <id>"
|
|
3305
|
+
}
|
|
3306
|
+
},
|
|
3307
|
+
{
|
|
3308
|
+
"id": "study.addons.list",
|
|
3309
|
+
"commandPath": [
|
|
3310
|
+
"study",
|
|
3311
|
+
"addons",
|
|
3312
|
+
"list"
|
|
3313
|
+
],
|
|
3314
|
+
"method": "get",
|
|
3315
|
+
"pathTemplate": "/study/v1/addons",
|
|
3316
|
+
"summary": "The add-on catalog (harness/surface-filtered)",
|
|
3317
|
+
"scope": "study.addons:read",
|
|
3318
|
+
"product": "study",
|
|
3319
|
+
"pathParams": [],
|
|
3320
|
+
"queryParams": [
|
|
3321
|
+
{
|
|
3322
|
+
"name": "harness",
|
|
3323
|
+
"flag": "harness",
|
|
3324
|
+
"optionKey": "harness",
|
|
3325
|
+
"required": false,
|
|
3326
|
+
"kind": "string"
|
|
3327
|
+
},
|
|
3328
|
+
{
|
|
3329
|
+
"name": "surface",
|
|
3330
|
+
"flag": "surface",
|
|
3331
|
+
"optionKey": "surface",
|
|
3332
|
+
"required": false,
|
|
3333
|
+
"kind": "string",
|
|
3334
|
+
"enum": [
|
|
3335
|
+
"run",
|
|
3336
|
+
"session"
|
|
3337
|
+
]
|
|
3338
|
+
}
|
|
3339
|
+
],
|
|
3340
|
+
"hasBody": false,
|
|
3341
|
+
"bodyRequired": false,
|
|
3342
|
+
"isStream": false,
|
|
3343
|
+
"watchable": false,
|
|
3344
|
+
"help": {
|
|
3345
|
+
"command": "ora study addons list"
|
|
3346
|
+
}
|
|
3347
|
+
},
|
|
3348
|
+
{
|
|
3349
|
+
"id": "study.benchmarkDefinitions.attachCorpus",
|
|
3350
|
+
"commandPath": [
|
|
3351
|
+
"study",
|
|
3352
|
+
"benchmarkDefinitions",
|
|
3353
|
+
"attach-corpus"
|
|
3354
|
+
],
|
|
3355
|
+
"method": "post",
|
|
3356
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/attach-corpus",
|
|
3357
|
+
"summary": "Accept an import of your definition's declared source into a NEW immutable version — a JOB, followed over its streamUrl",
|
|
3358
|
+
"scope": "study.studies:write",
|
|
3359
|
+
"product": "study",
|
|
3360
|
+
"pathParams": [
|
|
3361
|
+
{
|
|
3362
|
+
"name": "slug"
|
|
3363
|
+
}
|
|
3364
|
+
],
|
|
3365
|
+
"queryParams": [],
|
|
3366
|
+
"hasBody": true,
|
|
3367
|
+
"bodyRequired": true,
|
|
3368
|
+
"isStream": false,
|
|
3369
|
+
"bodyFields": [
|
|
3370
|
+
{
|
|
3371
|
+
"name": "version",
|
|
3372
|
+
"required": false,
|
|
3373
|
+
"kind": "number",
|
|
3374
|
+
"description": "Write this version NUMBER. Omit it to take the next free one. A number that already exists is a 409 — a published version is immutable, because studies pin it.",
|
|
3375
|
+
"placeholder": "1"
|
|
3376
|
+
},
|
|
3377
|
+
{
|
|
3378
|
+
"name": "limit",
|
|
3379
|
+
"required": false,
|
|
3380
|
+
"kind": "number",
|
|
3381
|
+
"description": "Import at most this many tasks from the top of the source's own order. Recorded in the version's `notes`, never applied silently — a short corpus with no explanation is a benchmark nobody can reproduce.",
|
|
3382
|
+
"placeholder": "200"
|
|
3383
|
+
},
|
|
3384
|
+
{
|
|
3385
|
+
"name": "changelog",
|
|
3386
|
+
"required": false,
|
|
3387
|
+
"kind": "string",
|
|
3388
|
+
"description": "What changed about this version, in the publisher's own words.",
|
|
3389
|
+
"placeholder": "<changelog>"
|
|
3390
|
+
}
|
|
3391
|
+
],
|
|
3392
|
+
"watchable": true,
|
|
3393
|
+
"help": {
|
|
3394
|
+
"command": "ora study benchmarkDefinitions attach-corpus windtunnel --file run.json --watch",
|
|
3395
|
+
"body": "{\n \"limit\": 200\n}"
|
|
3396
|
+
}
|
|
3397
|
+
},
|
|
3398
|
+
{
|
|
3399
|
+
"id": "study.benchmarkDefinitions.create",
|
|
3400
|
+
"commandPath": [
|
|
3401
|
+
"study",
|
|
3402
|
+
"benchmarkDefinitions",
|
|
3403
|
+
"create"
|
|
3404
|
+
],
|
|
3405
|
+
"method": "post",
|
|
3406
|
+
"pathTemplate": "/study/v1/benchmark-definitions",
|
|
3407
|
+
"summary": "Author a benchmark definition — born private to your organization, unlisted and licence-unverified",
|
|
3408
|
+
"scope": "study.studies:write",
|
|
3409
|
+
"product": "study",
|
|
3410
|
+
"pathParams": [],
|
|
3411
|
+
"queryParams": [],
|
|
3412
|
+
"hasBody": true,
|
|
3413
|
+
"bodyRequired": true,
|
|
3414
|
+
"isStream": false,
|
|
3415
|
+
"bodyFields": [
|
|
3416
|
+
{
|
|
3417
|
+
"name": "slug",
|
|
3418
|
+
"required": true,
|
|
3419
|
+
"kind": "string",
|
|
3420
|
+
"placeholder": "windtunnel"
|
|
3421
|
+
},
|
|
3422
|
+
{
|
|
3423
|
+
"name": "name",
|
|
3424
|
+
"required": true,
|
|
3425
|
+
"kind": "string",
|
|
3426
|
+
"placeholder": "WindTunnel"
|
|
3427
|
+
},
|
|
3428
|
+
{
|
|
3429
|
+
"name": "description",
|
|
3430
|
+
"required": false,
|
|
3431
|
+
"kind": "string",
|
|
3432
|
+
"placeholder": "<description>"
|
|
3433
|
+
},
|
|
3434
|
+
{
|
|
3435
|
+
"name": "family",
|
|
3436
|
+
"required": true,
|
|
3437
|
+
"kind": "string",
|
|
3438
|
+
"enum": [
|
|
3439
|
+
"static_qa",
|
|
3440
|
+
"code",
|
|
3441
|
+
"agentic",
|
|
3442
|
+
"tools",
|
|
3443
|
+
"web",
|
|
3444
|
+
"knowledge_work",
|
|
3445
|
+
"safety",
|
|
3446
|
+
"suite"
|
|
3447
|
+
],
|
|
3448
|
+
"placeholder": "web"
|
|
3449
|
+
},
|
|
3450
|
+
{
|
|
3451
|
+
"name": "sourceKind",
|
|
3452
|
+
"required": true,
|
|
3453
|
+
"kind": "string",
|
|
3454
|
+
"enum": [
|
|
3455
|
+
"hf-dataset",
|
|
3456
|
+
"jsonl",
|
|
3457
|
+
"git-tasks",
|
|
3458
|
+
"release-file",
|
|
3459
|
+
"authored",
|
|
3460
|
+
"framework"
|
|
3461
|
+
],
|
|
3462
|
+
"placeholder": "git-tasks"
|
|
3463
|
+
},
|
|
3464
|
+
{
|
|
3465
|
+
"name": "sourceConfig",
|
|
3466
|
+
"required": false,
|
|
3467
|
+
"kind": "object",
|
|
3468
|
+
"placeholder": "{}"
|
|
3469
|
+
},
|
|
3470
|
+
{
|
|
3471
|
+
"name": "adapterId",
|
|
3472
|
+
"required": false,
|
|
3473
|
+
"kind": "string",
|
|
3474
|
+
"placeholder": "<adapterId>"
|
|
3475
|
+
},
|
|
3476
|
+
{
|
|
3477
|
+
"name": "headlineMetric",
|
|
3478
|
+
"required": true,
|
|
3479
|
+
"kind": "string",
|
|
3480
|
+
"placeholder": "pass_rate"
|
|
3481
|
+
},
|
|
3482
|
+
{
|
|
3483
|
+
"name": "gradingSpec",
|
|
3484
|
+
"required": false,
|
|
3485
|
+
"kind": "object",
|
|
3486
|
+
"placeholder": "{}"
|
|
3487
|
+
},
|
|
3488
|
+
{
|
|
3489
|
+
"name": "environmentSpec",
|
|
3490
|
+
"required": false,
|
|
3491
|
+
"kind": "object",
|
|
3492
|
+
"placeholder": "{}"
|
|
3493
|
+
},
|
|
3494
|
+
{
|
|
3495
|
+
"name": "methodology",
|
|
3496
|
+
"required": false,
|
|
3497
|
+
"kind": "object",
|
|
3498
|
+
"placeholder": "{}"
|
|
3499
|
+
},
|
|
3500
|
+
{
|
|
3501
|
+
"name": "runnerKind",
|
|
3502
|
+
"required": false,
|
|
3503
|
+
"kind": "string",
|
|
3504
|
+
"enum": [
|
|
3505
|
+
"none",
|
|
3506
|
+
"image",
|
|
3507
|
+
"environment",
|
|
3508
|
+
"framework"
|
|
3509
|
+
],
|
|
3510
|
+
"placeholder": "none"
|
|
3511
|
+
},
|
|
3512
|
+
{
|
|
3513
|
+
"name": "runnerRecipe",
|
|
3514
|
+
"required": false,
|
|
3515
|
+
"kind": "string",
|
|
3516
|
+
"placeholder": "<runnerRecipe>"
|
|
3517
|
+
},
|
|
3518
|
+
{
|
|
3519
|
+
"name": "license",
|
|
3520
|
+
"required": false,
|
|
3521
|
+
"kind": "string",
|
|
3522
|
+
"placeholder": "<license>"
|
|
3523
|
+
},
|
|
3524
|
+
{
|
|
3525
|
+
"name": "licenseText",
|
|
3526
|
+
"required": false,
|
|
3527
|
+
"kind": "string",
|
|
3528
|
+
"placeholder": "<licenseText>"
|
|
3529
|
+
},
|
|
3530
|
+
{
|
|
3531
|
+
"name": "redistribution",
|
|
3532
|
+
"required": false,
|
|
3533
|
+
"kind": "string",
|
|
3534
|
+
"enum": [
|
|
3535
|
+
"allowed",
|
|
3536
|
+
"reference_only"
|
|
3537
|
+
],
|
|
3538
|
+
"placeholder": "allowed"
|
|
3539
|
+
},
|
|
3540
|
+
{
|
|
3541
|
+
"name": "access",
|
|
3542
|
+
"required": false,
|
|
3543
|
+
"kind": "string",
|
|
3544
|
+
"enum": [
|
|
3545
|
+
"open",
|
|
3546
|
+
"gated",
|
|
3547
|
+
"private"
|
|
3548
|
+
],
|
|
3549
|
+
"placeholder": "open"
|
|
3550
|
+
},
|
|
3551
|
+
{
|
|
3552
|
+
"name": "termsUrl",
|
|
3553
|
+
"required": false,
|
|
3554
|
+
"kind": "string",
|
|
3555
|
+
"placeholder": "<termsUrl>"
|
|
3556
|
+
},
|
|
3557
|
+
{
|
|
3558
|
+
"name": "homepageUrl",
|
|
3559
|
+
"required": false,
|
|
3560
|
+
"kind": "string",
|
|
3561
|
+
"placeholder": "<homepageUrl>"
|
|
3562
|
+
},
|
|
3563
|
+
{
|
|
3564
|
+
"name": "paperUrl",
|
|
3565
|
+
"required": false,
|
|
3566
|
+
"kind": "string",
|
|
3567
|
+
"placeholder": "<paperUrl>"
|
|
3568
|
+
},
|
|
3569
|
+
{
|
|
3570
|
+
"name": "defaultConfig",
|
|
3571
|
+
"required": false,
|
|
3572
|
+
"kind": "string",
|
|
3573
|
+
"placeholder": "<defaultConfig>"
|
|
3574
|
+
},
|
|
3575
|
+
{
|
|
3576
|
+
"name": "visibility",
|
|
3577
|
+
"required": false,
|
|
3578
|
+
"kind": "string",
|
|
3579
|
+
"enum": [
|
|
3580
|
+
"private",
|
|
3581
|
+
"org",
|
|
3582
|
+
"public"
|
|
3583
|
+
],
|
|
3584
|
+
"placeholder": "private"
|
|
3585
|
+
}
|
|
3586
|
+
],
|
|
3587
|
+
"watchable": false,
|
|
3588
|
+
"help": {
|
|
3589
|
+
"command": "ora study benchmarkDefinitions create --file run.json",
|
|
3590
|
+
"body": "{\n \"slug\": \"windtunnel\",\n \"name\": \"WindTunnel\",\n \"family\": \"web\",\n \"sourceKind\": \"git-tasks\",\n \"headlineMetric\": \"pass_rate\"\n}"
|
|
3591
|
+
}
|
|
3592
|
+
},
|
|
3593
|
+
{
|
|
3594
|
+
"id": "study.benchmarkDefinitions.get",
|
|
3595
|
+
"commandPath": [
|
|
3596
|
+
"study",
|
|
3597
|
+
"benchmarkDefinitions",
|
|
3598
|
+
"get"
|
|
3599
|
+
],
|
|
3600
|
+
"method": "get",
|
|
3601
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}",
|
|
3602
|
+
"summary": "One benchmark definition: methodology, licensing, environment and whether it can be run today",
|
|
3603
|
+
"scope": "study.studies:read",
|
|
3604
|
+
"product": "study",
|
|
3605
|
+
"pathParams": [
|
|
3606
|
+
{
|
|
3607
|
+
"name": "slug"
|
|
3608
|
+
}
|
|
3609
|
+
],
|
|
3610
|
+
"queryParams": [],
|
|
3611
|
+
"hasBody": false,
|
|
3612
|
+
"bodyRequired": false,
|
|
3613
|
+
"isStream": false,
|
|
3614
|
+
"watchable": false,
|
|
3615
|
+
"help": {
|
|
3616
|
+
"command": "ora study benchmarkDefinitions get mmlu-pro"
|
|
3617
|
+
}
|
|
3618
|
+
},
|
|
3619
|
+
{
|
|
3620
|
+
"id": "study.benchmarkDefinitions.list",
|
|
3621
|
+
"commandPath": [
|
|
3622
|
+
"study",
|
|
3623
|
+
"benchmarkDefinitions",
|
|
3624
|
+
"list"
|
|
3625
|
+
],
|
|
3626
|
+
"method": "get",
|
|
3627
|
+
"pathTemplate": "/study/v1/benchmark-definitions",
|
|
3628
|
+
"summary": "The benchmark catalog: every listed third-party definition plus the org's own, each with its derived runnability",
|
|
3629
|
+
"scope": "study.studies:read",
|
|
3630
|
+
"product": "study",
|
|
3631
|
+
"pathParams": [],
|
|
3632
|
+
"queryParams": [
|
|
3633
|
+
{
|
|
3634
|
+
"name": "family",
|
|
3635
|
+
"flag": "family",
|
|
3636
|
+
"optionKey": "family",
|
|
3637
|
+
"required": false,
|
|
3638
|
+
"kind": "string",
|
|
3639
|
+
"enum": [
|
|
3640
|
+
"static_qa",
|
|
3641
|
+
"code",
|
|
3642
|
+
"agentic",
|
|
3643
|
+
"tools",
|
|
3644
|
+
"web",
|
|
3645
|
+
"knowledge_work",
|
|
3646
|
+
"safety",
|
|
3647
|
+
"suite"
|
|
3648
|
+
]
|
|
3649
|
+
},
|
|
3650
|
+
{
|
|
3651
|
+
"name": "sourceKind",
|
|
3652
|
+
"flag": "source-kind",
|
|
3653
|
+
"optionKey": "sourceKind",
|
|
3654
|
+
"required": false,
|
|
3655
|
+
"kind": "string",
|
|
3656
|
+
"enum": [
|
|
3657
|
+
"hf-dataset",
|
|
3658
|
+
"jsonl",
|
|
3659
|
+
"git-tasks",
|
|
3660
|
+
"release-file",
|
|
3661
|
+
"authored",
|
|
3662
|
+
"framework"
|
|
3663
|
+
]
|
|
3664
|
+
},
|
|
3665
|
+
{
|
|
3666
|
+
"name": "tier",
|
|
3667
|
+
"flag": "tier",
|
|
3668
|
+
"optionKey": "tier",
|
|
3669
|
+
"required": false,
|
|
3670
|
+
"kind": "string",
|
|
3671
|
+
"enum": [
|
|
3672
|
+
"featured",
|
|
3673
|
+
"community",
|
|
3674
|
+
"mine"
|
|
3675
|
+
]
|
|
3676
|
+
},
|
|
3677
|
+
{
|
|
3678
|
+
"name": "runnable",
|
|
3679
|
+
"flag": "runnable",
|
|
3680
|
+
"optionKey": "runnable",
|
|
3681
|
+
"required": false,
|
|
3682
|
+
"kind": "string",
|
|
3683
|
+
"enum": [
|
|
3684
|
+
"true",
|
|
3685
|
+
"false"
|
|
3686
|
+
]
|
|
3687
|
+
},
|
|
3688
|
+
{
|
|
3689
|
+
"name": "search",
|
|
3690
|
+
"flag": "search",
|
|
3691
|
+
"optionKey": "search",
|
|
3692
|
+
"required": false,
|
|
3693
|
+
"kind": "string"
|
|
3694
|
+
},
|
|
3695
|
+
{
|
|
3696
|
+
"name": "pageSize",
|
|
3697
|
+
"flag": "page-size",
|
|
3698
|
+
"optionKey": "pageSize",
|
|
3699
|
+
"required": false,
|
|
3700
|
+
"kind": "number"
|
|
3701
|
+
}
|
|
3702
|
+
],
|
|
3703
|
+
"hasBody": false,
|
|
3704
|
+
"bodyRequired": false,
|
|
3705
|
+
"isStream": false,
|
|
3706
|
+
"watchable": false,
|
|
3707
|
+
"help": {
|
|
3708
|
+
"command": "ora study benchmarkDefinitions list"
|
|
3709
|
+
}
|
|
3710
|
+
},
|
|
3711
|
+
{
|
|
3712
|
+
"id": "study.benchmarkDefinitions.publish",
|
|
3713
|
+
"commandPath": [
|
|
3714
|
+
"study",
|
|
3715
|
+
"benchmarkDefinitions",
|
|
3716
|
+
"publish"
|
|
3717
|
+
],
|
|
3718
|
+
"method": "post",
|
|
3719
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/publish",
|
|
3720
|
+
"summary": "Point your definition at an imported version — a study can pin it from here on",
|
|
3721
|
+
"scope": "study.studies:write",
|
|
3722
|
+
"product": "study",
|
|
3723
|
+
"pathParams": [
|
|
3724
|
+
{
|
|
3725
|
+
"name": "slug"
|
|
3726
|
+
}
|
|
3727
|
+
],
|
|
3728
|
+
"queryParams": [],
|
|
3729
|
+
"hasBody": true,
|
|
3730
|
+
"bodyRequired": true,
|
|
3731
|
+
"isStream": false,
|
|
3732
|
+
"bodyFields": [
|
|
3733
|
+
{
|
|
3734
|
+
"name": "version",
|
|
3735
|
+
"required": false,
|
|
3736
|
+
"kind": "number",
|
|
3737
|
+
"description": "Point the catalog at this version. Omit it to publish the newest imported one.",
|
|
3738
|
+
"placeholder": "1"
|
|
3739
|
+
}
|
|
3740
|
+
],
|
|
3741
|
+
"watchable": false,
|
|
3742
|
+
"help": {
|
|
3743
|
+
"command": "ora study benchmarkDefinitions publish windtunnel --file run.json",
|
|
3744
|
+
"body": "{}"
|
|
3745
|
+
}
|
|
3746
|
+
},
|
|
3747
|
+
{
|
|
3748
|
+
"id": "study.benchmarkDefinitions.setVisibility",
|
|
3749
|
+
"commandPath": [
|
|
3750
|
+
"study",
|
|
3751
|
+
"benchmarkDefinitions",
|
|
3752
|
+
"set-visibility"
|
|
3753
|
+
],
|
|
3754
|
+
"method": "post",
|
|
3755
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/visibility",
|
|
3756
|
+
"summary": "Make it private, org-visible or public — effective immediately, by slug or link",
|
|
3757
|
+
"scope": "study.studies:write",
|
|
3758
|
+
"product": "study",
|
|
3759
|
+
"pathParams": [
|
|
3760
|
+
{
|
|
3761
|
+
"name": "slug"
|
|
3762
|
+
}
|
|
3763
|
+
],
|
|
3764
|
+
"queryParams": [],
|
|
3765
|
+
"hasBody": true,
|
|
3766
|
+
"bodyRequired": true,
|
|
3767
|
+
"isStream": false,
|
|
3768
|
+
"bodyFields": [
|
|
3769
|
+
{
|
|
3770
|
+
"name": "visibility",
|
|
3771
|
+
"required": true,
|
|
3772
|
+
"kind": "string",
|
|
3773
|
+
"enum": [
|
|
3774
|
+
"private",
|
|
3775
|
+
"org",
|
|
3776
|
+
"public"
|
|
3777
|
+
],
|
|
3778
|
+
"placeholder": "public"
|
|
3779
|
+
}
|
|
3780
|
+
],
|
|
3781
|
+
"watchable": false,
|
|
3782
|
+
"help": {
|
|
3783
|
+
"command": "ora study benchmarkDefinitions set-visibility windtunnel --file run.json",
|
|
3784
|
+
"body": "{\n \"visibility\": \"public\"\n}"
|
|
3785
|
+
}
|
|
3786
|
+
},
|
|
3787
|
+
{
|
|
3788
|
+
"id": "study.benchmarkDefinitions.submit",
|
|
3789
|
+
"commandPath": [
|
|
3790
|
+
"study",
|
|
3791
|
+
"benchmarkDefinitions",
|
|
3792
|
+
"submit"
|
|
3793
|
+
],
|
|
3794
|
+
"method": "post",
|
|
3795
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/submit",
|
|
3796
|
+
"summary": "Ask ora to feature it in the public catalog, with a signed licence attestation. A successful review makes the definition PUBLIC and lists it — submitting is asking for both.",
|
|
3797
|
+
"scope": "study.studies:write",
|
|
3798
|
+
"product": "study",
|
|
3799
|
+
"pathParams": [
|
|
3800
|
+
{
|
|
3801
|
+
"name": "slug"
|
|
3802
|
+
}
|
|
3803
|
+
],
|
|
3804
|
+
"queryParams": [],
|
|
3805
|
+
"hasBody": true,
|
|
3806
|
+
"bodyRequired": true,
|
|
3807
|
+
"isStream": false,
|
|
3808
|
+
"bodyFields": [
|
|
3809
|
+
{
|
|
3810
|
+
"name": "attestation",
|
|
3811
|
+
"required": true,
|
|
3812
|
+
"kind": "object",
|
|
3813
|
+
"placeholder": "{\"licenseSpdx\":\"Apache-2.0\",\"redistribution\":\"allowed\",\"sourceUrl\":\"https://github.com/nekuda-ai/WindTunnel\",\"statement\":\"The repository ships an Apache-2.0 LICENSE covering the task files.\"}"
|
|
3814
|
+
}
|
|
3815
|
+
],
|
|
3816
|
+
"watchable": false,
|
|
3817
|
+
"help": {
|
|
3818
|
+
"command": "ora study benchmarkDefinitions submit windtunnel --file run.json",
|
|
3819
|
+
"body": "{\n \"attestation\": {\n \"licenseSpdx\": \"Apache-2.0\",\n \"redistribution\": \"allowed\",\n \"sourceUrl\": \"https://github.com/nekuda-ai/WindTunnel\",\n \"statement\": \"The repository ships an Apache-2.0 LICENSE covering the task files.\"\n }\n}"
|
|
3820
|
+
}
|
|
3821
|
+
},
|
|
3822
|
+
{
|
|
3823
|
+
"id": "study.benchmarkDefinitions.update",
|
|
3824
|
+
"commandPath": [
|
|
3825
|
+
"study",
|
|
3826
|
+
"benchmarkDefinitions",
|
|
3827
|
+
"update"
|
|
3828
|
+
],
|
|
3829
|
+
"method": "patch",
|
|
3830
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}",
|
|
3831
|
+
"summary": "Edit a benchmark definition your organization publishes",
|
|
3832
|
+
"scope": "study.studies:write",
|
|
3833
|
+
"product": "study",
|
|
3834
|
+
"pathParams": [
|
|
3835
|
+
{
|
|
3836
|
+
"name": "slug"
|
|
3837
|
+
}
|
|
3838
|
+
],
|
|
3839
|
+
"queryParams": [],
|
|
3840
|
+
"hasBody": true,
|
|
3841
|
+
"bodyRequired": true,
|
|
3842
|
+
"isStream": false,
|
|
3843
|
+
"bodyFields": [
|
|
3844
|
+
{
|
|
3845
|
+
"name": "name",
|
|
3846
|
+
"required": false,
|
|
3847
|
+
"kind": "string",
|
|
3848
|
+
"placeholder": "<name>"
|
|
3849
|
+
},
|
|
3850
|
+
{
|
|
3851
|
+
"name": "description",
|
|
3852
|
+
"required": false,
|
|
3853
|
+
"kind": "string",
|
|
3854
|
+
"placeholder": "49 browser tasks over 8 self-hosted apps"
|
|
3855
|
+
},
|
|
3856
|
+
{
|
|
3857
|
+
"name": "family",
|
|
3858
|
+
"required": false,
|
|
3859
|
+
"kind": "string",
|
|
3860
|
+
"enum": [
|
|
3861
|
+
"static_qa",
|
|
3862
|
+
"code",
|
|
3863
|
+
"agentic",
|
|
3864
|
+
"tools",
|
|
3865
|
+
"web",
|
|
3866
|
+
"knowledge_work",
|
|
3867
|
+
"safety",
|
|
3868
|
+
"suite"
|
|
3869
|
+
],
|
|
3870
|
+
"placeholder": "static_qa"
|
|
3871
|
+
},
|
|
3872
|
+
{
|
|
3873
|
+
"name": "sourceKind",
|
|
3874
|
+
"required": false,
|
|
3875
|
+
"kind": "string",
|
|
3876
|
+
"enum": [
|
|
3877
|
+
"hf-dataset",
|
|
3878
|
+
"jsonl",
|
|
3879
|
+
"git-tasks",
|
|
3880
|
+
"release-file",
|
|
3881
|
+
"authored",
|
|
3882
|
+
"framework"
|
|
3883
|
+
],
|
|
3884
|
+
"placeholder": "hf-dataset"
|
|
3885
|
+
},
|
|
3886
|
+
{
|
|
3887
|
+
"name": "sourceConfig",
|
|
3888
|
+
"required": false,
|
|
3889
|
+
"kind": "object",
|
|
3890
|
+
"placeholder": "{}"
|
|
3891
|
+
},
|
|
3892
|
+
{
|
|
3893
|
+
"name": "adapterId",
|
|
3894
|
+
"required": false,
|
|
3895
|
+
"kind": "string",
|
|
3896
|
+
"placeholder": "<adapterId>"
|
|
3897
|
+
},
|
|
3898
|
+
{
|
|
3899
|
+
"name": "headlineMetric",
|
|
3900
|
+
"required": false,
|
|
3901
|
+
"kind": "string",
|
|
3902
|
+
"placeholder": "<headlineMetric>"
|
|
3903
|
+
},
|
|
3904
|
+
{
|
|
3905
|
+
"name": "gradingSpec",
|
|
3906
|
+
"required": false,
|
|
3907
|
+
"kind": "object",
|
|
3908
|
+
"placeholder": "{}"
|
|
3909
|
+
},
|
|
3910
|
+
{
|
|
3911
|
+
"name": "environmentSpec",
|
|
3912
|
+
"required": false,
|
|
3913
|
+
"kind": "object",
|
|
3914
|
+
"placeholder": "{}"
|
|
3915
|
+
},
|
|
3916
|
+
{
|
|
3917
|
+
"name": "methodology",
|
|
3918
|
+
"required": false,
|
|
3919
|
+
"kind": "object",
|
|
3920
|
+
"placeholder": "{}"
|
|
3921
|
+
},
|
|
3922
|
+
{
|
|
3923
|
+
"name": "runnerKind",
|
|
3924
|
+
"required": false,
|
|
3925
|
+
"kind": "string",
|
|
3926
|
+
"enum": [
|
|
3927
|
+
"none",
|
|
3928
|
+
"image",
|
|
3929
|
+
"environment",
|
|
3930
|
+
"framework"
|
|
3931
|
+
],
|
|
3932
|
+
"placeholder": "none"
|
|
3933
|
+
},
|
|
3934
|
+
{
|
|
3935
|
+
"name": "runnerRecipe",
|
|
3936
|
+
"required": false,
|
|
3937
|
+
"kind": "string",
|
|
3938
|
+
"placeholder": "<runnerRecipe>"
|
|
3939
|
+
},
|
|
3940
|
+
{
|
|
3941
|
+
"name": "license",
|
|
3942
|
+
"required": false,
|
|
3943
|
+
"kind": "string",
|
|
3944
|
+
"placeholder": "<license>"
|
|
3945
|
+
},
|
|
3946
|
+
{
|
|
3947
|
+
"name": "licenseText",
|
|
3948
|
+
"required": false,
|
|
3949
|
+
"kind": "string",
|
|
3950
|
+
"placeholder": "<licenseText>"
|
|
3951
|
+
},
|
|
3952
|
+
{
|
|
3953
|
+
"name": "redistribution",
|
|
3954
|
+
"required": false,
|
|
3955
|
+
"kind": "string",
|
|
3956
|
+
"enum": [
|
|
3957
|
+
"allowed",
|
|
3958
|
+
"reference_only"
|
|
3959
|
+
],
|
|
3960
|
+
"placeholder": "allowed"
|
|
3961
|
+
},
|
|
3962
|
+
{
|
|
3963
|
+
"name": "access",
|
|
3964
|
+
"required": false,
|
|
3965
|
+
"kind": "string",
|
|
3966
|
+
"enum": [
|
|
3967
|
+
"open",
|
|
3968
|
+
"gated",
|
|
3969
|
+
"private"
|
|
3970
|
+
],
|
|
3971
|
+
"placeholder": "open"
|
|
3972
|
+
},
|
|
3973
|
+
{
|
|
3974
|
+
"name": "termsUrl",
|
|
3975
|
+
"required": false,
|
|
3976
|
+
"kind": "string",
|
|
3977
|
+
"placeholder": "<termsUrl>"
|
|
3978
|
+
},
|
|
3979
|
+
{
|
|
3980
|
+
"name": "homepageUrl",
|
|
3981
|
+
"required": false,
|
|
3982
|
+
"kind": "string",
|
|
3983
|
+
"placeholder": "<homepageUrl>"
|
|
3984
|
+
},
|
|
3985
|
+
{
|
|
3986
|
+
"name": "paperUrl",
|
|
3987
|
+
"required": false,
|
|
3988
|
+
"kind": "string",
|
|
3989
|
+
"placeholder": "<paperUrl>"
|
|
3990
|
+
},
|
|
3991
|
+
{
|
|
3992
|
+
"name": "defaultConfig",
|
|
3993
|
+
"required": false,
|
|
3994
|
+
"kind": "string",
|
|
3995
|
+
"placeholder": "<defaultConfig>"
|
|
3996
|
+
},
|
|
3997
|
+
{
|
|
3998
|
+
"name": "visibility",
|
|
3999
|
+
"required": false,
|
|
4000
|
+
"kind": "string",
|
|
4001
|
+
"enum": [
|
|
4002
|
+
"private",
|
|
4003
|
+
"org",
|
|
4004
|
+
"public"
|
|
4005
|
+
],
|
|
4006
|
+
"placeholder": "private"
|
|
4007
|
+
}
|
|
4008
|
+
],
|
|
4009
|
+
"watchable": false,
|
|
4010
|
+
"help": {
|
|
4011
|
+
"command": "ora study benchmarkDefinitions update windtunnel --file run.json",
|
|
4012
|
+
"body": "{\n \"description\": \"49 browser tasks over 8 self-hosted apps\"\n}"
|
|
4013
|
+
}
|
|
4014
|
+
},
|
|
4015
|
+
{
|
|
4016
|
+
"id": "study.benchmarkDefinitions.withdraw",
|
|
4017
|
+
"commandPath": [
|
|
4018
|
+
"study",
|
|
4019
|
+
"benchmarkDefinitions",
|
|
4020
|
+
"withdraw"
|
|
4021
|
+
],
|
|
4022
|
+
"method": "post",
|
|
4023
|
+
"pathTemplate": "/study/v1/benchmark-definitions/{slug}/withdraw",
|
|
4024
|
+
"summary": "Take a pending submission back — the definition returns to draft and stays runnable",
|
|
4025
|
+
"scope": "study.studies:write",
|
|
4026
|
+
"product": "study",
|
|
4027
|
+
"pathParams": [
|
|
4028
|
+
{
|
|
4029
|
+
"name": "slug"
|
|
4030
|
+
}
|
|
4031
|
+
],
|
|
4032
|
+
"queryParams": [],
|
|
4033
|
+
"hasBody": false,
|
|
4034
|
+
"bodyRequired": false,
|
|
4035
|
+
"isStream": false,
|
|
4036
|
+
"watchable": false,
|
|
4037
|
+
"help": {
|
|
4038
|
+
"command": "ora study benchmarkDefinitions withdraw windtunnel"
|
|
4039
|
+
}
|
|
4040
|
+
},
|
|
4041
|
+
{
|
|
4042
|
+
"id": "study.benchmarkImports.get",
|
|
4043
|
+
"commandPath": [
|
|
4044
|
+
"study",
|
|
4045
|
+
"benchmarkImports",
|
|
4046
|
+
"get"
|
|
4047
|
+
],
|
|
4048
|
+
"method": "get",
|
|
4049
|
+
"pathTemplate": "/study/v1/benchmark-imports/{id}",
|
|
4050
|
+
"summary": "An import job — queued, running, then succeeded (with its new version) or failed (with its reason)",
|
|
4051
|
+
"scope": "study.studies:read",
|
|
4052
|
+
"product": "study",
|
|
4053
|
+
"pathParams": [
|
|
4054
|
+
{
|
|
4055
|
+
"name": "id"
|
|
4056
|
+
}
|
|
4057
|
+
],
|
|
4058
|
+
"queryParams": [],
|
|
4059
|
+
"hasBody": false,
|
|
4060
|
+
"bodyRequired": false,
|
|
4061
|
+
"isStream": false,
|
|
4062
|
+
"watchable": false,
|
|
4063
|
+
"help": {
|
|
4064
|
+
"command": "ora study benchmarkImports get 11111111-1111-4111-8111-111111111111"
|
|
4065
|
+
}
|
|
4066
|
+
},
|
|
4067
|
+
{
|
|
4068
|
+
"id": "study.benchmarkImports.stream",
|
|
4069
|
+
"commandPath": [
|
|
4070
|
+
"study",
|
|
4071
|
+
"benchmarkImports",
|
|
4072
|
+
"stream"
|
|
4073
|
+
],
|
|
4074
|
+
"method": "get",
|
|
4075
|
+
"pathTemplate": "/study/v1/benchmark-imports/{id}/stream",
|
|
4076
|
+
"summary": "Live SSE of one import job: its current state at once, then every change; closes after succeeded or failed",
|
|
4077
|
+
"scope": "study.studies:read",
|
|
4078
|
+
"product": "study",
|
|
4079
|
+
"pathParams": [
|
|
4080
|
+
{
|
|
4081
|
+
"name": "id"
|
|
4082
|
+
}
|
|
4083
|
+
],
|
|
4084
|
+
"queryParams": [],
|
|
4085
|
+
"hasBody": false,
|
|
4086
|
+
"bodyRequired": false,
|
|
4087
|
+
"isStream": true,
|
|
4088
|
+
"watchable": false,
|
|
4089
|
+
"help": {
|
|
4090
|
+
"command": "ora study benchmarkImports stream <id>"
|
|
4091
|
+
}
|
|
4092
|
+
},
|
|
4093
|
+
{
|
|
4094
|
+
"id": "study.benchmarkIntake.assist",
|
|
4095
|
+
"commandPath": [
|
|
4096
|
+
"study",
|
|
4097
|
+
"benchmarkIntake",
|
|
4098
|
+
"assist"
|
|
4099
|
+
],
|
|
4100
|
+
"method": "post",
|
|
4101
|
+
"pathTemplate": "/study/v1/benchmark-intake/assist",
|
|
4102
|
+
"summary": "Have an agent propose the rest of the definition from what the probe read — every field with a rationale and a confidence, grounded on this deployment's registries. It proposes; you confirm. Nothing is created.",
|
|
4103
|
+
"scope": "study.studies:write",
|
|
4104
|
+
"product": "study",
|
|
4105
|
+
"pathParams": [],
|
|
4106
|
+
"queryParams": [],
|
|
4107
|
+
"hasBody": true,
|
|
4108
|
+
"bodyRequired": true,
|
|
4109
|
+
"isStream": false,
|
|
4110
|
+
"bodyFields": [
|
|
4111
|
+
{
|
|
4112
|
+
"name": "strategy",
|
|
4113
|
+
"required": true,
|
|
4114
|
+
"kind": "string",
|
|
4115
|
+
"placeholder": "git-repo"
|
|
4116
|
+
},
|
|
4117
|
+
{
|
|
4118
|
+
"name": "config",
|
|
4119
|
+
"required": true,
|
|
4120
|
+
"kind": "object",
|
|
4121
|
+
"placeholder": "{\"repo\":\"https://github.com/nekuda-ai/WindTunnel\"}"
|
|
4122
|
+
},
|
|
4123
|
+
{
|
|
4124
|
+
"name": "probe",
|
|
4125
|
+
"required": false,
|
|
4126
|
+
"kind": "object",
|
|
4127
|
+
"placeholder": "{}"
|
|
4128
|
+
}
|
|
4129
|
+
],
|
|
4130
|
+
"watchable": false,
|
|
4131
|
+
"help": {
|
|
4132
|
+
"command": "ora study benchmarkIntake assist --file run.json",
|
|
4133
|
+
"body": "{\n \"strategy\": \"git-repo\",\n \"config\": {\n \"repo\": \"https://github.com/nekuda-ai/WindTunnel\"\n }\n}"
|
|
4134
|
+
}
|
|
4135
|
+
},
|
|
4136
|
+
{
|
|
4137
|
+
"id": "study.benchmarkIntake.probe",
|
|
4138
|
+
"commandPath": [
|
|
4139
|
+
"study",
|
|
4140
|
+
"benchmarkIntake",
|
|
4141
|
+
"probe"
|
|
4142
|
+
],
|
|
4143
|
+
"method": "post",
|
|
4144
|
+
"pathTemplate": "/study/v1/benchmark-intake/probe",
|
|
4145
|
+
"summary": "Inspect a target and propose a definition — with what it could not determine listed explicitly",
|
|
4146
|
+
"scope": "study.studies:write",
|
|
4147
|
+
"product": "study",
|
|
4148
|
+
"pathParams": [],
|
|
4149
|
+
"queryParams": [],
|
|
4150
|
+
"hasBody": true,
|
|
4151
|
+
"bodyRequired": true,
|
|
4152
|
+
"isStream": false,
|
|
4153
|
+
"bodyFields": [
|
|
4154
|
+
{
|
|
4155
|
+
"name": "strategy",
|
|
4156
|
+
"required": true,
|
|
4157
|
+
"kind": "string",
|
|
4158
|
+
"placeholder": "git-repo"
|
|
4159
|
+
},
|
|
4160
|
+
{
|
|
4161
|
+
"name": "config",
|
|
4162
|
+
"required": true,
|
|
4163
|
+
"kind": "object",
|
|
4164
|
+
"placeholder": "{\"repo\":\"https://github.com/nekuda-ai/WindTunnel\"}"
|
|
4165
|
+
}
|
|
4166
|
+
],
|
|
4167
|
+
"watchable": false,
|
|
4168
|
+
"help": {
|
|
4169
|
+
"command": "ora study benchmarkIntake probe --file run.json",
|
|
4170
|
+
"body": "{\n \"strategy\": \"git-repo\",\n \"config\": {\n \"repo\": \"https://github.com/nekuda-ai/WindTunnel\"\n }\n}"
|
|
4171
|
+
}
|
|
4172
|
+
},
|
|
4173
|
+
{
|
|
4174
|
+
"id": "study.benchmarkIntake.strategies",
|
|
4175
|
+
"commandPath": [
|
|
4176
|
+
"study",
|
|
4177
|
+
"benchmarkIntake",
|
|
4178
|
+
"strategies"
|
|
4179
|
+
],
|
|
4180
|
+
"method": "get",
|
|
4181
|
+
"pathTemplate": "/study/v1/benchmark-intake/strategies",
|
|
4182
|
+
"summary": "The ways a benchmark can be added (a repository, a dataset, a release file, an image, an upload) and the fields each one takes",
|
|
4183
|
+
"scope": "study.studies:write",
|
|
4184
|
+
"product": "study",
|
|
4185
|
+
"pathParams": [],
|
|
4186
|
+
"queryParams": [],
|
|
4187
|
+
"hasBody": false,
|
|
4188
|
+
"bodyRequired": false,
|
|
4189
|
+
"isStream": false,
|
|
4190
|
+
"watchable": false,
|
|
4191
|
+
"help": {
|
|
4192
|
+
"command": "ora study benchmarkIntake strategies"
|
|
4193
|
+
}
|
|
4194
|
+
},
|
|
4195
|
+
{
|
|
4196
|
+
"id": "study.benchmarkTerms.get",
|
|
4197
|
+
"commandPath": [
|
|
4198
|
+
"study",
|
|
4199
|
+
"benchmarkTerms",
|
|
4200
|
+
"get"
|
|
4201
|
+
],
|
|
4202
|
+
"method": "get",
|
|
4203
|
+
"pathTemplate": "/study/v1/benchmark-terms",
|
|
4204
|
+
"summary": "The platform's benchmark-data terms — what an organization accepts once to read the task text of guarded benchmarks — and this organization's acceptance, if any",
|
|
4205
|
+
"scope": "study.studies:read",
|
|
4206
|
+
"product": "study",
|
|
4207
|
+
"pathParams": [],
|
|
4208
|
+
"queryParams": [],
|
|
4209
|
+
"hasBody": false,
|
|
4210
|
+
"bodyRequired": false,
|
|
4211
|
+
"isStream": false,
|
|
4212
|
+
"watchable": false,
|
|
4213
|
+
"help": {
|
|
4214
|
+
"command": "ora study benchmarkTerms get"
|
|
4215
|
+
}
|
|
4216
|
+
},
|
|
4217
|
+
{
|
|
4218
|
+
"id": "study.catalog",
|
|
4219
|
+
"commandPath": [
|
|
4220
|
+
"study",
|
|
4221
|
+
"catalog"
|
|
4222
|
+
],
|
|
4223
|
+
"method": "get",
|
|
4224
|
+
"pathTemplate": "/study/v1/catalog",
|
|
4225
|
+
"summary": "Harness × model catalog: valid matrix cells, defaults, modes, and withheld harnesses with reasons",
|
|
4226
|
+
"scope": null,
|
|
4227
|
+
"product": "platform",
|
|
4228
|
+
"pathParams": [],
|
|
4229
|
+
"queryParams": [],
|
|
4230
|
+
"hasBody": false,
|
|
4231
|
+
"bodyRequired": false,
|
|
4232
|
+
"isStream": false,
|
|
4233
|
+
"watchable": false,
|
|
4234
|
+
"help": {
|
|
4235
|
+
"command": "ora study catalog"
|
|
4236
|
+
}
|
|
4237
|
+
},
|
|
4238
|
+
{
|
|
4239
|
+
"id": "study.convertWebmcpSuite",
|
|
4240
|
+
"commandPath": [
|
|
4241
|
+
"study",
|
|
4242
|
+
"convert-webmcp-suite"
|
|
4243
|
+
],
|
|
4244
|
+
"method": "post",
|
|
4245
|
+
"pathTemplate": "/study/v1/studies/convert/webmcp",
|
|
4246
|
+
"summary": "Convert a webmcp-evals suite JSON into ora study tasks + 'evals'-lens expectations — a PURE transform, nothing persisted; resubmit the returned `tasks` through study.create's `tasks` field",
|
|
4247
|
+
"scope": "study.studies:write",
|
|
4248
|
+
"product": "study",
|
|
4249
|
+
"pathParams": [],
|
|
4250
|
+
"queryParams": [],
|
|
4251
|
+
"hasBody": true,
|
|
4252
|
+
"bodyRequired": true,
|
|
4253
|
+
"isStream": false,
|
|
4254
|
+
"bodyFields": [
|
|
4255
|
+
{
|
|
4256
|
+
"name": "suite",
|
|
4257
|
+
"required": true,
|
|
4258
|
+
"kind": "string",
|
|
4259
|
+
"description": "The foreign webmcp-evals suite JSON (an array of {name, messages, expectedCall} cases) — validated by the converter itself; a malformed suite answers 422 with the converter's own message.",
|
|
4260
|
+
"placeholder": "[object Object]"
|
|
4261
|
+
},
|
|
4262
|
+
{
|
|
4263
|
+
"name": "ordered",
|
|
4264
|
+
"required": false,
|
|
4265
|
+
"kind": "boolean",
|
|
4266
|
+
"description": "When true, a case's multiple expectedCall entries become one ORDERED `sequence` expectation instead of independent ones (mirrors the reference project's evals.json vs evals-unordered.json split). Default false.",
|
|
4267
|
+
"placeholder": "true"
|
|
4268
|
+
}
|
|
4269
|
+
],
|
|
4270
|
+
"watchable": false,
|
|
4271
|
+
"help": {
|
|
4272
|
+
"command": "ora study convert-webmcp-suite --file run.json",
|
|
4273
|
+
"body": "{\n \"suite\": [\n {\n \"name\": \"search finds the refund policy\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"type\": \"message\",\n \"content\": \"Find the refund policy and quote it.\"\n }\n ],\n \"expectedCall\": [\n {\n \"functionName\": \"search_docs\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"refund\"\n }\n }\n }\n ]\n }\n ]\n}"
|
|
4274
|
+
}
|
|
4275
|
+
},
|
|
4276
|
+
{
|
|
4277
|
+
"id": "study.create",
|
|
4278
|
+
"commandPath": [
|
|
4279
|
+
"study",
|
|
4280
|
+
"create"
|
|
4281
|
+
],
|
|
4282
|
+
"method": "post",
|
|
4283
|
+
"pathTemplate": "/study/v1/studies",
|
|
4284
|
+
"summary": "Create a study (status 'draft', ZERO LLM spend — an authored `tasks` corpus instead lands 'ready' synchronously); approve generation via POST .../generate",
|
|
4285
|
+
"scope": "study.studies:write",
|
|
4286
|
+
"product": "study",
|
|
4287
|
+
"pathParams": [],
|
|
4288
|
+
"queryParams": [],
|
|
4289
|
+
"hasBody": true,
|
|
4290
|
+
"bodyRequired": true,
|
|
4291
|
+
"isStream": false,
|
|
4292
|
+
"bodyFields": [
|
|
4293
|
+
{
|
|
4294
|
+
"name": "name",
|
|
4295
|
+
"required": true,
|
|
4296
|
+
"kind": "string",
|
|
4297
|
+
"description": "Display name for the study (≤160 chars).",
|
|
4298
|
+
"placeholder": "tavily-search A/B"
|
|
4299
|
+
},
|
|
4300
|
+
{
|
|
4301
|
+
"name": "description",
|
|
4302
|
+
"required": false,
|
|
4303
|
+
"kind": "string",
|
|
4304
|
+
"description": "Optional longer description shown on the study card.",
|
|
4305
|
+
"placeholder": "<description>"
|
|
4306
|
+
},
|
|
4307
|
+
{
|
|
4308
|
+
"name": "subjectKind",
|
|
4309
|
+
"required": false,
|
|
4310
|
+
"kind": "string",
|
|
4311
|
+
"enum": [
|
|
4312
|
+
"skill",
|
|
4313
|
+
"tools",
|
|
4314
|
+
"addon",
|
|
4315
|
+
"custom",
|
|
4316
|
+
"benchmark"
|
|
4317
|
+
],
|
|
4318
|
+
"description": "What the study measures — defaults to 'skill'. See the enum description for each kind's rules.",
|
|
4319
|
+
"placeholder": "addon"
|
|
4320
|
+
},
|
|
4321
|
+
{
|
|
4322
|
+
"name": "skillRef",
|
|
4323
|
+
"required": false,
|
|
4324
|
+
"kind": "string",
|
|
4325
|
+
"description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise.",
|
|
4326
|
+
"placeholder": "<skillRef>"
|
|
4327
|
+
},
|
|
4328
|
+
{
|
|
4329
|
+
"name": "definitionVersionId",
|
|
4330
|
+
"required": false,
|
|
4331
|
+
"kind": "string",
|
|
4332
|
+
"description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
|
|
4333
|
+
"placeholder": "<definitionVersionId>"
|
|
4334
|
+
},
|
|
4335
|
+
{
|
|
4336
|
+
"name": "subjectName",
|
|
4337
|
+
"required": false,
|
|
4338
|
+
"kind": "string",
|
|
4339
|
+
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
|
|
4340
|
+
"placeholder": "Tavily search"
|
|
4341
|
+
},
|
|
4342
|
+
{
|
|
4343
|
+
"name": "armConfig",
|
|
4344
|
+
"required": false,
|
|
4345
|
+
"kind": "object",
|
|
4346
|
+
"description": "Frozen per-arm run-payload deltas. Required for subjectKind 'custom' and comparisonMode 'ab_custom'; for 'tools'/'addon' subjects the treatment side is derived when omitted.",
|
|
4347
|
+
"placeholder": "{\"treatment\":{\"addons\":{\"tavily\":{\"secretId\":\"<org-secret-uuid>\"}}}}"
|
|
4348
|
+
},
|
|
4349
|
+
{
|
|
4350
|
+
"name": "lenses",
|
|
4351
|
+
"required": false,
|
|
4352
|
+
"kind": "array",
|
|
4353
|
+
"description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
|
|
4354
|
+
"placeholder": "[\"judge\",\"evals\"]"
|
|
4355
|
+
},
|
|
4356
|
+
{
|
|
4357
|
+
"name": "proposeExpectations",
|
|
4358
|
+
"required": false,
|
|
4359
|
+
"kind": "boolean",
|
|
4360
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
|
|
4361
|
+
"placeholder": "true"
|
|
4362
|
+
},
|
|
4363
|
+
{
|
|
4364
|
+
"name": "comparisonMode",
|
|
4365
|
+
"required": false,
|
|
4366
|
+
"kind": "string",
|
|
4367
|
+
"enum": [
|
|
4368
|
+
"ab_vs_none",
|
|
4369
|
+
"ab_custom",
|
|
4370
|
+
"single"
|
|
4371
|
+
],
|
|
4372
|
+
"description": "The baseline design — defaults to 'ab_vs_none'. See the enum description for each mode.",
|
|
4373
|
+
"placeholder": "ab_vs_none"
|
|
4374
|
+
},
|
|
4375
|
+
{
|
|
4376
|
+
"name": "subjectBackend",
|
|
4377
|
+
"required": false,
|
|
4378
|
+
"kind": "string",
|
|
4379
|
+
"enum": [
|
|
4380
|
+
"model",
|
|
4381
|
+
"harness"
|
|
4382
|
+
],
|
|
4383
|
+
"description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
|
|
4384
|
+
"placeholder": "model"
|
|
4385
|
+
},
|
|
4386
|
+
{
|
|
4387
|
+
"name": "baselineLabel",
|
|
4388
|
+
"required": false,
|
|
4389
|
+
"kind": "string",
|
|
4390
|
+
"description": "Baseline arm's display label (e.g. 'REST API'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
|
|
4391
|
+
"placeholder": "<baselineLabel>"
|
|
4392
|
+
},
|
|
4393
|
+
{
|
|
4394
|
+
"name": "treatmentLabel",
|
|
4395
|
+
"required": false,
|
|
4396
|
+
"kind": "string",
|
|
4397
|
+
"description": "Treatment arm's display label (e.g. 'Remote MCP'). Required iff comparisonMode 'ab_custom'; forbidden otherwise.",
|
|
4398
|
+
"placeholder": "<treatmentLabel>"
|
|
4399
|
+
},
|
|
4400
|
+
{
|
|
4401
|
+
"name": "taskCount",
|
|
4402
|
+
"required": false,
|
|
4403
|
+
"kind": "number",
|
|
4404
|
+
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'); the total is capped at 10000.",
|
|
4405
|
+
"placeholder": "1"
|
|
4406
|
+
},
|
|
4407
|
+
{
|
|
4408
|
+
"name": "variantsPerTask",
|
|
4409
|
+
"required": false,
|
|
4410
|
+
"kind": "number",
|
|
4411
|
+
"description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
|
|
4412
|
+
"placeholder": "1"
|
|
4413
|
+
},
|
|
4414
|
+
{
|
|
4415
|
+
"name": "repeats",
|
|
4416
|
+
"required": false,
|
|
4417
|
+
"kind": "number",
|
|
4418
|
+
"description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
|
|
4419
|
+
"placeholder": "2"
|
|
4420
|
+
},
|
|
4421
|
+
{
|
|
4422
|
+
"name": "tasks",
|
|
4423
|
+
"required": false,
|
|
4424
|
+
"kind": "array",
|
|
4425
|
+
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
|
|
4426
|
+
"placeholder": "[{\"prompt\":\"What is the current stable Node.js LTS version, and where did you get that from?\",\"rubric\":[\"States a specific Node.js LTS major version rather than hedging.\",\"Names the source it used for the version claim.\"],\"expectations\":[{\"kind\":\"any_of\",\"label\":\"searched, by either route (treatment arm only)\",\"arms\":[\"skill\"],\"of\":[{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"search\",\"arguments\":{\"query\":{\"$contains\":\"Node\"}}},{\"kind\":\"mcp_tool_called\",\"server\":\"tavily\",\"tool\":\"call_page_tool\",\"arguments\":{\"tool\":\"search\"}}]},{\"kind\":\"response_matches\",\"label\":\"answer names a version\",\"pattern\":\"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"},{\"kind\":\"token_budget\",\"maxTotalTokens\":40000}]}]"
|
|
4427
|
+
},
|
|
4428
|
+
{
|
|
4429
|
+
"name": "subset",
|
|
4430
|
+
"required": false,
|
|
4431
|
+
"kind": "object",
|
|
4432
|
+
"description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
|
|
4433
|
+
"placeholder": "{}"
|
|
4434
|
+
},
|
|
4435
|
+
{
|
|
4436
|
+
"name": "matrix",
|
|
4437
|
+
"required": true,
|
|
4438
|
+
"kind": "array",
|
|
4439
|
+
"description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
|
|
4440
|
+
"placeholder": "[{\"harness\":\"claude-code\",\"model\":\"claude-sonnet-5\"},{\"harness\":\"codex\",\"model\":\"gpt-5.5\"}]"
|
|
4441
|
+
},
|
|
4442
|
+
{
|
|
4443
|
+
"name": "promise",
|
|
4444
|
+
"required": false,
|
|
4445
|
+
"kind": "string",
|
|
4446
|
+
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
|
|
4447
|
+
"placeholder": "Web search via Tavily produces fresher, better-sourced answers than no web access"
|
|
4448
|
+
}
|
|
4449
|
+
],
|
|
4450
|
+
"watchable": true,
|
|
4451
|
+
"help": {
|
|
4452
|
+
"command": "ora study create --file run.json --watch",
|
|
4453
|
+
"body": "{\n \"name\": \"tavily-search A/B\",\n \"subjectKind\": \"addon\",\n \"subjectName\": \"Tavily search\",\n \"promise\": \"Web search via Tavily produces fresher, better-sourced answers than no web access\",\n \"armConfig\": {\n \"treatment\": {\n \"addons\": {\n \"tavily\": {\n \"secretId\": \"<org-secret-uuid>\"\n }\n }\n }\n },\n \"lenses\": [\n \"judge\",\n \"evals\"\n ],\n \"taskCount\": 1,\n \"variantsPerTask\": 1,\n \"repeats\": 2,\n \"matrix\": [\n {\n \"harness\": \"claude-code\",\n \"model\": \"claude-sonnet-5\"\n },\n {\n \"harness\": \"codex\",\n \"model\": \"gpt-5.5\"\n }\n ],\n \"tasks\": [\n {\n \"prompt\": \"What is the current stable Node.js LTS version, and where did you get that from?\",\n \"rubric\": [\n \"States a specific Node.js LTS major version rather than hedging.\",\n \"Names the source it used for the version claim.\"\n ],\n \"expectations\": [\n {\n \"kind\": \"any_of\",\n \"label\": \"searched, by either route (treatment arm only)\",\n \"arms\": [\n \"skill\"\n ],\n \"of\": [\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"search\",\n \"arguments\": {\n \"query\": {\n \"$contains\": \"Node\"\n }\n }\n },\n {\n \"kind\": \"mcp_tool_called\",\n \"server\": \"tavily\",\n \"tool\": \"call_page_tool\",\n \"arguments\": {\n \"tool\": \"search\"\n }\n }\n ]\n },\n {\n \"kind\": \"response_matches\",\n \"label\": \"answer names a version\",\n \"pattern\": \"\\\\bv?\\\\d+\\\\.\\\\d+\\\\.\\\\d+\\\\b\"\n },\n {\n \"kind\": \"token_budget\",\n \"maxTotalTokens\": 40000\n }\n ]\n }\n ]\n}"
|
|
4454
|
+
}
|
|
4455
|
+
},
|
|
4456
|
+
{
|
|
4457
|
+
"id": "study.delete",
|
|
4458
|
+
"commandPath": [
|
|
4459
|
+
"study",
|
|
4460
|
+
"delete"
|
|
4461
|
+
],
|
|
4462
|
+
"method": "delete",
|
|
4463
|
+
"pathTemplate": "/study/v1/studies/{id}",
|
|
4464
|
+
"summary": "Delete a study and its tasks/facts (409 while running)",
|
|
4465
|
+
"scope": "study.studies:write",
|
|
4466
|
+
"product": "study",
|
|
4467
|
+
"pathParams": [
|
|
4468
|
+
{
|
|
4469
|
+
"name": "id"
|
|
4470
|
+
}
|
|
4471
|
+
],
|
|
4472
|
+
"queryParams": [],
|
|
4473
|
+
"hasBody": false,
|
|
4474
|
+
"bodyRequired": false,
|
|
4475
|
+
"isStream": false,
|
|
4476
|
+
"watchable": false,
|
|
4477
|
+
"help": {
|
|
4478
|
+
"command": "ora study delete <id>"
|
|
4479
|
+
}
|
|
4480
|
+
},
|
|
4481
|
+
{
|
|
4482
|
+
"id": "study.generate",
|
|
4483
|
+
"commandPath": [
|
|
4484
|
+
"study",
|
|
4485
|
+
"generate"
|
|
4486
|
+
],
|
|
4487
|
+
"method": "post",
|
|
4488
|
+
"pathTemplate": "/study/v1/studies/{id}/generate",
|
|
4489
|
+
"summary": "Approve task generation from 'draft' (first generation), or regenerate from 'ready'/'failed' (wipes existing tasks and their rubrics)",
|
|
4490
|
+
"scope": "study.studies:write",
|
|
4491
|
+
"product": "study",
|
|
4492
|
+
"pathParams": [
|
|
4493
|
+
{
|
|
4494
|
+
"name": "id"
|
|
4495
|
+
}
|
|
4496
|
+
],
|
|
4497
|
+
"queryParams": [],
|
|
4498
|
+
"hasBody": true,
|
|
4499
|
+
"bodyRequired": true,
|
|
4500
|
+
"isStream": false,
|
|
4501
|
+
"bodyFields": [
|
|
4502
|
+
{
|
|
4503
|
+
"name": "proposeExpectations",
|
|
4504
|
+
"required": false,
|
|
4505
|
+
"kind": "boolean",
|
|
4506
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each task this generation produces — the generate-time re-assertion of create's `proposeExpectations` (create-time no longer fires generation itself, so this is where the opt-in actually takes effect). Same REQUIRES/IGNORED rules as create's field.",
|
|
4507
|
+
"placeholder": "true"
|
|
4508
|
+
}
|
|
4509
|
+
],
|
|
4510
|
+
"watchable": true,
|
|
4511
|
+
"help": {
|
|
4512
|
+
"command": "ora study generate <id> --file run.json --watch",
|
|
4513
|
+
"body": "{\n \"proposeExpectations\": true\n}"
|
|
4514
|
+
}
|
|
4515
|
+
},
|
|
4516
|
+
{
|
|
4517
|
+
"id": "study.get",
|
|
4518
|
+
"commandPath": [
|
|
4519
|
+
"study",
|
|
4520
|
+
"get"
|
|
4521
|
+
],
|
|
4522
|
+
"method": "get",
|
|
4523
|
+
"pathTemplate": "/study/v1/studies/{id}",
|
|
4524
|
+
"summary": "Study detail: status to poll, tasks + rubrics, per-run facts, live aggregate + verdict",
|
|
4525
|
+
"scope": "study.studies:read",
|
|
4526
|
+
"product": "study",
|
|
4527
|
+
"pathParams": [
|
|
4528
|
+
{
|
|
4529
|
+
"name": "id"
|
|
4530
|
+
}
|
|
4531
|
+
],
|
|
4532
|
+
"queryParams": [],
|
|
4533
|
+
"hasBody": false,
|
|
4534
|
+
"bodyRequired": false,
|
|
4535
|
+
"isStream": false,
|
|
4536
|
+
"watchable": false,
|
|
4537
|
+
"help": {
|
|
4538
|
+
"command": "ora study get 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4539
|
+
}
|
|
4540
|
+
},
|
|
4541
|
+
{
|
|
4542
|
+
"id": "study.images.builds.cancel",
|
|
4543
|
+
"commandPath": [
|
|
4544
|
+
"study",
|
|
4545
|
+
"images",
|
|
4546
|
+
"builds",
|
|
4547
|
+
"cancel"
|
|
4548
|
+
],
|
|
4549
|
+
"method": "post",
|
|
4550
|
+
"pathTemplate": "/study/v1/images/builds/{id}/cancel",
|
|
4551
|
+
"summary": "Cancel an in-flight image build",
|
|
4552
|
+
"scope": "images:write",
|
|
4553
|
+
"product": "platform",
|
|
4554
|
+
"pathParams": [
|
|
4555
|
+
{
|
|
4556
|
+
"name": "id"
|
|
4557
|
+
}
|
|
4558
|
+
],
|
|
4559
|
+
"queryParams": [],
|
|
4560
|
+
"hasBody": false,
|
|
4561
|
+
"bodyRequired": false,
|
|
4562
|
+
"isStream": false,
|
|
4563
|
+
"watchable": false,
|
|
4564
|
+
"help": {
|
|
4565
|
+
"command": "ora study images builds cancel <id>"
|
|
4566
|
+
}
|
|
4567
|
+
},
|
|
4568
|
+
{
|
|
4569
|
+
"id": "study.images.builds.create",
|
|
4570
|
+
"commandPath": [
|
|
4571
|
+
"study",
|
|
4572
|
+
"images",
|
|
4573
|
+
"builds",
|
|
4574
|
+
"create"
|
|
4575
|
+
],
|
|
4576
|
+
"method": "post",
|
|
4577
|
+
"pathTemplate": "/study/v1/images/builds",
|
|
4578
|
+
"summary": "Submit an image build (idempotent on name + source + Dockerfile + build args)",
|
|
4579
|
+
"scope": "images:write",
|
|
4580
|
+
"product": "platform",
|
|
4581
|
+
"pathParams": [],
|
|
4582
|
+
"queryParams": [],
|
|
4583
|
+
"hasBody": true,
|
|
4584
|
+
"bodyRequired": true,
|
|
4585
|
+
"isStream": false,
|
|
4586
|
+
"bodyFields": [
|
|
4587
|
+
{
|
|
4588
|
+
"name": "name",
|
|
4589
|
+
"required": true,
|
|
4590
|
+
"kind": "string",
|
|
4591
|
+
"placeholder": "<name>"
|
|
4592
|
+
},
|
|
4593
|
+
{
|
|
4594
|
+
"name": "source",
|
|
4595
|
+
"required": true,
|
|
4596
|
+
"kind": "string",
|
|
4597
|
+
"placeholder": "<source>"
|
|
4598
|
+
},
|
|
4599
|
+
{
|
|
4600
|
+
"name": "dockerfile",
|
|
4601
|
+
"required": false,
|
|
4602
|
+
"kind": "object",
|
|
4603
|
+
"placeholder": "{}"
|
|
4604
|
+
},
|
|
4605
|
+
{
|
|
4606
|
+
"name": "buildArgs",
|
|
4607
|
+
"required": false,
|
|
4608
|
+
"kind": "object",
|
|
4609
|
+
"placeholder": "{}"
|
|
4610
|
+
},
|
|
4611
|
+
{
|
|
4612
|
+
"name": "platform",
|
|
4613
|
+
"required": false,
|
|
4614
|
+
"kind": "string",
|
|
4615
|
+
"placeholder": "<platform>"
|
|
4616
|
+
}
|
|
4617
|
+
],
|
|
4618
|
+
"watchable": false,
|
|
4619
|
+
"help": {
|
|
4620
|
+
"command": "ora study images builds create --file run.json",
|
|
4621
|
+
"body": "{\n \"name\": \"<name>\",\n \"source\": \"<source>\"\n}"
|
|
4622
|
+
}
|
|
4623
|
+
},
|
|
4624
|
+
{
|
|
4625
|
+
"id": "study.images.builds.get",
|
|
4626
|
+
"commandPath": [
|
|
4627
|
+
"study",
|
|
4628
|
+
"images",
|
|
4629
|
+
"builds",
|
|
4630
|
+
"get"
|
|
4631
|
+
],
|
|
4632
|
+
"method": "get",
|
|
4633
|
+
"pathTemplate": "/study/v1/images/builds/{id}",
|
|
4634
|
+
"summary": "Get one image build",
|
|
4635
|
+
"scope": "images:read",
|
|
4636
|
+
"product": "platform",
|
|
4637
|
+
"pathParams": [
|
|
4638
|
+
{
|
|
4639
|
+
"name": "id"
|
|
4640
|
+
}
|
|
4641
|
+
],
|
|
4642
|
+
"queryParams": [],
|
|
4643
|
+
"hasBody": false,
|
|
4644
|
+
"bodyRequired": false,
|
|
4645
|
+
"isStream": false,
|
|
4646
|
+
"watchable": false,
|
|
4647
|
+
"help": {
|
|
4648
|
+
"command": "ora study images builds get <id>"
|
|
4649
|
+
}
|
|
4650
|
+
},
|
|
4651
|
+
{
|
|
4652
|
+
"id": "study.images.builds.list",
|
|
4653
|
+
"commandPath": [
|
|
4654
|
+
"study",
|
|
4655
|
+
"images",
|
|
4656
|
+
"builds",
|
|
4657
|
+
"list"
|
|
4658
|
+
],
|
|
4659
|
+
"method": "get",
|
|
4660
|
+
"pathTemplate": "/study/v1/images/builds",
|
|
4661
|
+
"summary": "List the org's image builds, newest first",
|
|
4662
|
+
"scope": "images:read",
|
|
4663
|
+
"product": "platform",
|
|
4664
|
+
"pathParams": [],
|
|
4665
|
+
"queryParams": [
|
|
4666
|
+
{
|
|
4667
|
+
"name": "phase",
|
|
4668
|
+
"flag": "phase",
|
|
4669
|
+
"optionKey": "phase",
|
|
4670
|
+
"required": false,
|
|
4671
|
+
"kind": "string",
|
|
4672
|
+
"enum": [
|
|
4673
|
+
"pending",
|
|
4674
|
+
"running",
|
|
4675
|
+
"succeeded",
|
|
4676
|
+
"failed",
|
|
4677
|
+
"cancelled"
|
|
4678
|
+
]
|
|
4679
|
+
},
|
|
4680
|
+
{
|
|
4681
|
+
"name": "name",
|
|
4682
|
+
"flag": "name",
|
|
4683
|
+
"optionKey": "name",
|
|
4684
|
+
"required": false,
|
|
4685
|
+
"kind": "string"
|
|
4686
|
+
},
|
|
4687
|
+
{
|
|
4688
|
+
"name": "pageSize",
|
|
4689
|
+
"flag": "page-size",
|
|
4690
|
+
"optionKey": "pageSize",
|
|
4691
|
+
"required": false,
|
|
4692
|
+
"kind": "number"
|
|
4693
|
+
},
|
|
4694
|
+
{
|
|
4695
|
+
"name": "pageToken",
|
|
4696
|
+
"flag": "page-token",
|
|
4697
|
+
"optionKey": "pageToken",
|
|
4698
|
+
"required": false,
|
|
4699
|
+
"kind": "string"
|
|
4700
|
+
}
|
|
4701
|
+
],
|
|
4702
|
+
"hasBody": false,
|
|
4703
|
+
"bodyRequired": false,
|
|
4704
|
+
"isStream": false,
|
|
4705
|
+
"watchable": false,
|
|
4706
|
+
"help": {
|
|
4707
|
+
"command": "ora study images builds list"
|
|
4708
|
+
}
|
|
4709
|
+
},
|
|
4710
|
+
{
|
|
4711
|
+
"id": "study.images.builds.stream",
|
|
4712
|
+
"commandPath": [
|
|
4713
|
+
"study",
|
|
4714
|
+
"images",
|
|
4715
|
+
"builds",
|
|
4716
|
+
"stream"
|
|
4717
|
+
],
|
|
4718
|
+
"method": "get",
|
|
4719
|
+
"pathTemplate": "/study/v1/images/builds/{id}/stream",
|
|
4720
|
+
"summary": "SSE stream of one image build's row changes",
|
|
4721
|
+
"scope": "images:read",
|
|
4722
|
+
"product": "platform",
|
|
4723
|
+
"pathParams": [
|
|
4724
|
+
{
|
|
4725
|
+
"name": "id"
|
|
4726
|
+
}
|
|
4727
|
+
],
|
|
4728
|
+
"queryParams": [],
|
|
4729
|
+
"hasBody": false,
|
|
4730
|
+
"bodyRequired": false,
|
|
4731
|
+
"isStream": true,
|
|
4732
|
+
"watchable": false,
|
|
4733
|
+
"help": {
|
|
4734
|
+
"command": "ora study images builds stream <id>"
|
|
4735
|
+
}
|
|
4736
|
+
},
|
|
4737
|
+
{
|
|
4738
|
+
"id": "study.images.runs.create",
|
|
4739
|
+
"commandPath": [
|
|
4740
|
+
"study",
|
|
4741
|
+
"images",
|
|
4742
|
+
"runs",
|
|
4743
|
+
"create"
|
|
4744
|
+
],
|
|
4745
|
+
"method": "post",
|
|
4746
|
+
"pathTemplate": "/study/v1/images/runs",
|
|
4747
|
+
"summary": "Launch a run (harness or environment) from a built or referenced image",
|
|
4748
|
+
"scope": "images:write",
|
|
4749
|
+
"product": "platform",
|
|
4750
|
+
"pathParams": [],
|
|
4751
|
+
"queryParams": [],
|
|
4752
|
+
"hasBody": true,
|
|
4753
|
+
"bodyRequired": true,
|
|
4754
|
+
"isStream": false,
|
|
4755
|
+
"bodyFields": [
|
|
4756
|
+
{
|
|
4757
|
+
"name": "buildId",
|
|
4758
|
+
"required": false,
|
|
4759
|
+
"kind": "string",
|
|
4760
|
+
"placeholder": "<buildId>"
|
|
4761
|
+
},
|
|
4762
|
+
{
|
|
4763
|
+
"name": "image",
|
|
4764
|
+
"required": false,
|
|
4765
|
+
"kind": "string",
|
|
4766
|
+
"placeholder": "<image>"
|
|
4767
|
+
},
|
|
4768
|
+
{
|
|
4769
|
+
"name": "mode",
|
|
4770
|
+
"required": true,
|
|
4771
|
+
"kind": "string",
|
|
4772
|
+
"enum": [
|
|
4773
|
+
"harness",
|
|
4774
|
+
"environment"
|
|
4775
|
+
],
|
|
4776
|
+
"placeholder": "harness"
|
|
4777
|
+
},
|
|
4778
|
+
{
|
|
4779
|
+
"name": "harness",
|
|
4780
|
+
"required": false,
|
|
4781
|
+
"kind": "string",
|
|
4782
|
+
"placeholder": "<harness>"
|
|
4783
|
+
},
|
|
4784
|
+
{
|
|
4785
|
+
"name": "task",
|
|
4786
|
+
"required": false,
|
|
4787
|
+
"kind": "string",
|
|
4788
|
+
"placeholder": "<task>"
|
|
4789
|
+
}
|
|
4790
|
+
],
|
|
4791
|
+
"watchable": false,
|
|
4792
|
+
"help": {
|
|
4793
|
+
"command": "ora study images runs create --file run.json",
|
|
4794
|
+
"body": "{\n \"mode\": \"harness\"\n}"
|
|
4795
|
+
}
|
|
4796
|
+
},
|
|
4797
|
+
{
|
|
4798
|
+
"id": "study.launch",
|
|
4799
|
+
"commandPath": [
|
|
4800
|
+
"study",
|
|
4801
|
+
"launch"
|
|
4802
|
+
],
|
|
4803
|
+
"method": "post",
|
|
4804
|
+
"pathTemplate": "/study/v1/studies/{id}/launch",
|
|
4805
|
+
"summary": "Launch: fan out task × matrix-cell × arm runs (async — poll get until status 'completed')",
|
|
4806
|
+
"scope": "study.studies:write",
|
|
4807
|
+
"product": "study",
|
|
4808
|
+
"pathParams": [
|
|
4809
|
+
{
|
|
4810
|
+
"name": "id"
|
|
4811
|
+
}
|
|
4812
|
+
],
|
|
4813
|
+
"queryParams": [],
|
|
4814
|
+
"hasBody": false,
|
|
4815
|
+
"bodyRequired": false,
|
|
4816
|
+
"isStream": false,
|
|
4817
|
+
"watchable": true,
|
|
4818
|
+
"help": {
|
|
4819
|
+
"command": "ora study launch <id> --watch"
|
|
4820
|
+
}
|
|
4821
|
+
},
|
|
4822
|
+
{
|
|
4823
|
+
"id": "study.list",
|
|
4824
|
+
"commandPath": [
|
|
4825
|
+
"study",
|
|
4826
|
+
"list"
|
|
4827
|
+
],
|
|
4828
|
+
"method": "get",
|
|
4829
|
+
"pathTemplate": "/study/v1/studies",
|
|
4830
|
+
"summary": "List the org's studies (id, status, verdict, subject)",
|
|
4831
|
+
"scope": "study.studies:read",
|
|
4832
|
+
"product": "study",
|
|
4833
|
+
"pathParams": [],
|
|
4834
|
+
"queryParams": [],
|
|
4835
|
+
"hasBody": false,
|
|
4836
|
+
"bodyRequired": false,
|
|
4837
|
+
"isStream": false,
|
|
4838
|
+
"watchable": false,
|
|
4839
|
+
"help": {
|
|
4840
|
+
"command": "ora study list"
|
|
4841
|
+
}
|
|
4842
|
+
},
|
|
4843
|
+
{
|
|
4844
|
+
"id": "study.listRunAnswers",
|
|
4845
|
+
"commandPath": [
|
|
4846
|
+
"study",
|
|
4847
|
+
"list-run-answers"
|
|
4848
|
+
],
|
|
4849
|
+
"method": "get",
|
|
4850
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/answers",
|
|
4851
|
+
"summary": "List the agent runs that answered a framework benchmark run's requests (the 'harness' subject backend), oldest first. Empty for a run that asked for none.",
|
|
4852
|
+
"scope": "study.studies:read",
|
|
4853
|
+
"product": "study",
|
|
4854
|
+
"pathParams": [
|
|
4855
|
+
{
|
|
4856
|
+
"name": "id"
|
|
4857
|
+
},
|
|
4858
|
+
{
|
|
4859
|
+
"name": "runId"
|
|
4860
|
+
}
|
|
4861
|
+
],
|
|
4862
|
+
"queryParams": [],
|
|
4863
|
+
"hasBody": false,
|
|
4864
|
+
"bodyRequired": false,
|
|
4865
|
+
"isStream": false,
|
|
4866
|
+
"watchable": false,
|
|
4867
|
+
"help": {
|
|
4868
|
+
"command": "ora study list-run-answers 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 7c9e6679-7425-40de-944b-e07fc1f90ae7"
|
|
4869
|
+
}
|
|
4870
|
+
},
|
|
4871
|
+
{
|
|
4872
|
+
"id": "study.preview",
|
|
4873
|
+
"commandPath": [
|
|
4874
|
+
"study",
|
|
4875
|
+
"preview"
|
|
4876
|
+
],
|
|
4877
|
+
"method": "get",
|
|
4878
|
+
"pathTemplate": "/study/v1/studies/{id}/preview",
|
|
4879
|
+
"summary": "Preview the PUBLISHED page's exact payload before publishing — the same projection, aggregate and behavior block publicStudies.get will serve, org-scoped and available at any status (no visibility requirement)",
|
|
4880
|
+
"scope": "study.studies:read",
|
|
4881
|
+
"product": "study",
|
|
4882
|
+
"pathParams": [
|
|
4883
|
+
{
|
|
4884
|
+
"name": "id"
|
|
4885
|
+
}
|
|
4886
|
+
],
|
|
4887
|
+
"queryParams": [],
|
|
4888
|
+
"hasBody": false,
|
|
4889
|
+
"bodyRequired": false,
|
|
4890
|
+
"isStream": false,
|
|
4891
|
+
"watchable": false,
|
|
4892
|
+
"help": {
|
|
4893
|
+
"command": "ora study preview 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4894
|
+
}
|
|
4895
|
+
},
|
|
4896
|
+
{
|
|
4897
|
+
"id": "study.publish",
|
|
4898
|
+
"commandPath": [
|
|
4899
|
+
"study",
|
|
4900
|
+
"publish"
|
|
4901
|
+
],
|
|
4902
|
+
"method": "post",
|
|
4903
|
+
"pathTemplate": "/study/v1/studies/{id}/publish",
|
|
4904
|
+
"summary": "Publish a completed study under a shareable mode: 'public' (listed in the anonymous catalog, the default) or — where the deployment supports sharing — 'link' (unlisted, openable only with the returned share token). Idempotent slug upsert.",
|
|
4905
|
+
"scope": "study.studies:write",
|
|
4906
|
+
"product": "study",
|
|
4907
|
+
"pathParams": [
|
|
4908
|
+
{
|
|
4909
|
+
"name": "id"
|
|
4910
|
+
}
|
|
4911
|
+
],
|
|
4912
|
+
"queryParams": [],
|
|
4913
|
+
"hasBody": true,
|
|
4914
|
+
"bodyRequired": true,
|
|
4915
|
+
"isStream": false,
|
|
4916
|
+
"bodyFields": [
|
|
4917
|
+
{
|
|
4918
|
+
"name": "slug",
|
|
4919
|
+
"required": false,
|
|
4920
|
+
"kind": "string",
|
|
4921
|
+
"description": "Public catalog URL slug (lowercase [a-z0-9-], ≤80). Omitted → the study's own generated slug. 409 study_slug_taken if another published study holds it.",
|
|
4922
|
+
"placeholder": "tavily-search-ab"
|
|
4923
|
+
},
|
|
4924
|
+
{
|
|
4925
|
+
"name": "visibility",
|
|
4926
|
+
"required": false,
|
|
4927
|
+
"kind": "string",
|
|
4928
|
+
"enum": [
|
|
4929
|
+
"private",
|
|
4930
|
+
"org",
|
|
4931
|
+
"link",
|
|
4932
|
+
"public"
|
|
4933
|
+
],
|
|
4934
|
+
"description": "Publish under this sharing mode: 'public' (listed in the catalog) or 'link' (unlisted; openable only at /studies/<slug>?share=<token>, which the response carries). Omitted → 'public'. 'private'/'org' are 422.",
|
|
4935
|
+
"placeholder": "private"
|
|
4936
|
+
}
|
|
4937
|
+
],
|
|
4938
|
+
"watchable": false,
|
|
4939
|
+
"help": {
|
|
4940
|
+
"command": "ora study publish 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
4941
|
+
"body": "{\n \"slug\": \"tavily-search-ab\"\n}"
|
|
4942
|
+
}
|
|
4943
|
+
},
|
|
4944
|
+
{
|
|
4945
|
+
"id": "study.reconcileRuns",
|
|
4946
|
+
"commandPath": [
|
|
4947
|
+
"study",
|
|
4948
|
+
"reconcile-runs"
|
|
4949
|
+
],
|
|
4950
|
+
"method": "post",
|
|
4951
|
+
"pathTemplate": "/study/v1/studies/{id}/reconcile-runs",
|
|
4952
|
+
"summary": "Re-derive a study's run projection from the authoritative run rows and re-probe completion (platform-admin only; idempotent) — the repair for runs left non-terminal in the projection while the spine says they finished",
|
|
4953
|
+
"scope": "study.studies:write",
|
|
4954
|
+
"product": "study",
|
|
4955
|
+
"pathParams": [
|
|
4956
|
+
{
|
|
4957
|
+
"name": "id"
|
|
4958
|
+
}
|
|
4959
|
+
],
|
|
4960
|
+
"queryParams": [],
|
|
4961
|
+
"hasBody": false,
|
|
4962
|
+
"bodyRequired": false,
|
|
4963
|
+
"isStream": false,
|
|
4964
|
+
"watchable": false,
|
|
4965
|
+
"help": {
|
|
4966
|
+
"command": "ora study reconcile-runs 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4967
|
+
}
|
|
4968
|
+
},
|
|
4969
|
+
{
|
|
4970
|
+
"id": "study.regenerateSummary",
|
|
4971
|
+
"commandPath": [
|
|
4972
|
+
"study",
|
|
4973
|
+
"regenerate-summary"
|
|
4974
|
+
],
|
|
4975
|
+
"method": "post",
|
|
4976
|
+
"pathTemplate": "/study/v1/studies/{id}/exec-summary",
|
|
4977
|
+
"summary": "Regenerate the executive summary from the current aggregate (async; poll get)",
|
|
4978
|
+
"scope": "study.studies:write",
|
|
4979
|
+
"product": "study",
|
|
4980
|
+
"pathParams": [
|
|
4981
|
+
{
|
|
4982
|
+
"name": "id"
|
|
4983
|
+
}
|
|
4984
|
+
],
|
|
4985
|
+
"queryParams": [],
|
|
4986
|
+
"hasBody": false,
|
|
4987
|
+
"bodyRequired": false,
|
|
4988
|
+
"isStream": false,
|
|
4989
|
+
"watchable": false,
|
|
4990
|
+
"help": {
|
|
4991
|
+
"command": "ora study regenerate-summary 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
4992
|
+
}
|
|
4993
|
+
},
|
|
4994
|
+
{
|
|
4995
|
+
"id": "study.regrade",
|
|
4996
|
+
"commandPath": [
|
|
4997
|
+
"study",
|
|
4998
|
+
"regrade"
|
|
4999
|
+
],
|
|
5000
|
+
"method": "post",
|
|
5001
|
+
"pathTemplate": "/study/v1/studies/{id}/regrade",
|
|
5002
|
+
"summary": "Clear and re-run judge-panel grading (rubrics + blind pairwise) for a completed study",
|
|
5003
|
+
"scope": "study.studies:write",
|
|
5004
|
+
"product": "study",
|
|
5005
|
+
"pathParams": [
|
|
5006
|
+
{
|
|
5007
|
+
"name": "id"
|
|
5008
|
+
}
|
|
5009
|
+
],
|
|
5010
|
+
"queryParams": [],
|
|
5011
|
+
"hasBody": false,
|
|
5012
|
+
"bodyRequired": false,
|
|
5013
|
+
"isStream": false,
|
|
5014
|
+
"watchable": true,
|
|
5015
|
+
"help": {
|
|
5016
|
+
"command": "ora study regrade <id> --watch"
|
|
5017
|
+
}
|
|
5018
|
+
},
|
|
5019
|
+
{
|
|
5020
|
+
"id": "study.rotateShareToken",
|
|
5021
|
+
"commandPath": [
|
|
5022
|
+
"study",
|
|
5023
|
+
"rotate-share-token"
|
|
5024
|
+
],
|
|
5025
|
+
"method": "post",
|
|
5026
|
+
"pathTemplate": "/study/v1/studies/{id}/share-token/rotate",
|
|
5027
|
+
"summary": "Mint a fresh share link for a 'link' study. The previous link stops working IMMEDIATELY — there is no grace window.",
|
|
5028
|
+
"scope": "study.studies:write",
|
|
5029
|
+
"product": "study",
|
|
5030
|
+
"pathParams": [
|
|
5031
|
+
{
|
|
5032
|
+
"name": "id"
|
|
5033
|
+
}
|
|
5034
|
+
],
|
|
5035
|
+
"queryParams": [],
|
|
5036
|
+
"hasBody": false,
|
|
5037
|
+
"bodyRequired": false,
|
|
5038
|
+
"isStream": false,
|
|
5039
|
+
"watchable": false,
|
|
5040
|
+
"help": {
|
|
5041
|
+
"command": "ora study rotate-share-token <id>"
|
|
5042
|
+
}
|
|
5043
|
+
},
|
|
5044
|
+
{
|
|
5045
|
+
"id": "study.runInsight",
|
|
5046
|
+
"commandPath": [
|
|
5047
|
+
"study",
|
|
5048
|
+
"run-insight"
|
|
5049
|
+
],
|
|
5050
|
+
"method": "get",
|
|
5051
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/insight",
|
|
5052
|
+
"summary": "Read a study run's full insight with the study's access permissions",
|
|
5053
|
+
"scope": "study.studies:read",
|
|
5054
|
+
"product": "study",
|
|
5055
|
+
"pathParams": [
|
|
5056
|
+
{
|
|
5057
|
+
"name": "id"
|
|
5058
|
+
},
|
|
5059
|
+
{
|
|
5060
|
+
"name": "runId"
|
|
5061
|
+
}
|
|
5062
|
+
],
|
|
5063
|
+
"queryParams": [],
|
|
5064
|
+
"hasBody": false,
|
|
5065
|
+
"bodyRequired": false,
|
|
5066
|
+
"isStream": false,
|
|
5067
|
+
"watchable": false,
|
|
5068
|
+
"help": {
|
|
5069
|
+
"command": "ora study run-insight <id> <runId>"
|
|
5070
|
+
}
|
|
5071
|
+
},
|
|
5072
|
+
{
|
|
5073
|
+
"id": "study.runTrajectory",
|
|
5074
|
+
"commandPath": [
|
|
5075
|
+
"study",
|
|
5076
|
+
"run-trajectory"
|
|
5077
|
+
],
|
|
5078
|
+
"method": "get",
|
|
5079
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/trajectory",
|
|
5080
|
+
"summary": "Read a study run's full trajectory with the study's access permissions",
|
|
5081
|
+
"scope": "study.studies:read",
|
|
5082
|
+
"product": "study",
|
|
5083
|
+
"pathParams": [
|
|
5084
|
+
{
|
|
5085
|
+
"name": "id"
|
|
5086
|
+
},
|
|
5087
|
+
{
|
|
5088
|
+
"name": "runId"
|
|
5089
|
+
}
|
|
5090
|
+
],
|
|
5091
|
+
"queryParams": [
|
|
5092
|
+
{
|
|
5093
|
+
"name": "profile",
|
|
5094
|
+
"flag": "profile",
|
|
5095
|
+
"optionKey": "profile",
|
|
5096
|
+
"required": false,
|
|
5097
|
+
"kind": "string",
|
|
5098
|
+
"enum": [
|
|
5099
|
+
"raw",
|
|
5100
|
+
"heuristic",
|
|
5101
|
+
"full"
|
|
5102
|
+
]
|
|
5103
|
+
}
|
|
5104
|
+
],
|
|
5105
|
+
"hasBody": false,
|
|
5106
|
+
"bodyRequired": false,
|
|
5107
|
+
"isStream": false,
|
|
5108
|
+
"watchable": false,
|
|
5109
|
+
"help": {
|
|
5110
|
+
"command": "ora study run-trajectory <id> <runId>"
|
|
5111
|
+
}
|
|
5112
|
+
},
|
|
5113
|
+
{
|
|
5114
|
+
"id": "study.runs.cancel",
|
|
5115
|
+
"commandPath": [
|
|
5116
|
+
"study",
|
|
5117
|
+
"runs",
|
|
5118
|
+
"cancel"
|
|
5119
|
+
],
|
|
5120
|
+
"method": "post",
|
|
5121
|
+
"pathTemplate": "/study/v1/runs/{id}/cancel",
|
|
5122
|
+
"summary": "Cancel a run (terminate it and stop its engine)",
|
|
5123
|
+
"scope": "runs:write",
|
|
5124
|
+
"product": "platform",
|
|
5125
|
+
"pathParams": [
|
|
5126
|
+
{
|
|
5127
|
+
"name": "id"
|
|
5128
|
+
}
|
|
5129
|
+
],
|
|
5130
|
+
"queryParams": [],
|
|
5131
|
+
"hasBody": false,
|
|
5132
|
+
"bodyRequired": false,
|
|
5133
|
+
"isStream": false,
|
|
5134
|
+
"watchable": false,
|
|
5135
|
+
"help": {
|
|
5136
|
+
"command": "ora study runs cancel <id>"
|
|
5137
|
+
}
|
|
5138
|
+
},
|
|
5139
|
+
{
|
|
5140
|
+
"id": "study.runs.cancelBatch",
|
|
5141
|
+
"commandPath": [
|
|
5142
|
+
"study",
|
|
5143
|
+
"runs",
|
|
5144
|
+
"cancel-batch"
|
|
5145
|
+
],
|
|
5146
|
+
"method": "post",
|
|
5147
|
+
"pathTemplate": "/study/v1/runs/cancel",
|
|
5148
|
+
"summary": "Cancel a batch of runs (terminate them and stop their engines)",
|
|
5149
|
+
"scope": "runs:write",
|
|
5150
|
+
"product": "platform",
|
|
5151
|
+
"pathParams": [],
|
|
5152
|
+
"queryParams": [],
|
|
5153
|
+
"hasBody": true,
|
|
5154
|
+
"bodyRequired": true,
|
|
5155
|
+
"isStream": false,
|
|
5156
|
+
"bodyFields": [
|
|
5157
|
+
{
|
|
5158
|
+
"name": "runIds",
|
|
5159
|
+
"required": true,
|
|
5160
|
+
"kind": "array",
|
|
5161
|
+
"placeholder": "[]"
|
|
5162
|
+
}
|
|
5163
|
+
],
|
|
5164
|
+
"watchable": false,
|
|
5165
|
+
"help": {
|
|
5166
|
+
"command": "ora study runs cancel-batch --file run.json",
|
|
5167
|
+
"body": "{\n \"runIds\": []\n}"
|
|
5168
|
+
}
|
|
5169
|
+
},
|
|
5170
|
+
{
|
|
5171
|
+
"id": "study.runs.create",
|
|
5172
|
+
"commandPath": [
|
|
5173
|
+
"study",
|
|
5174
|
+
"runs",
|
|
5175
|
+
"create"
|
|
5176
|
+
],
|
|
5177
|
+
"method": "post",
|
|
5178
|
+
"pathTemplate": "/study/v1/runs",
|
|
5179
|
+
"summary": "Create a run",
|
|
5180
|
+
"scope": "runs:write",
|
|
5181
|
+
"product": "platform",
|
|
5182
|
+
"pathParams": [],
|
|
5183
|
+
"queryParams": [],
|
|
5184
|
+
"hasBody": true,
|
|
5185
|
+
"bodyRequired": true,
|
|
5186
|
+
"isStream": false,
|
|
5187
|
+
"bodyFields": [
|
|
5188
|
+
{
|
|
5189
|
+
"name": "product_key",
|
|
5190
|
+
"required": false,
|
|
5191
|
+
"kind": "string",
|
|
5192
|
+
"placeholder": "<product_key>"
|
|
5193
|
+
},
|
|
5194
|
+
{
|
|
5195
|
+
"name": "intent",
|
|
5196
|
+
"required": true,
|
|
5197
|
+
"kind": "string",
|
|
5198
|
+
"placeholder": "check pricing for acme.com"
|
|
5199
|
+
},
|
|
5200
|
+
{
|
|
5201
|
+
"name": "domain",
|
|
5202
|
+
"required": false,
|
|
5203
|
+
"kind": "string",
|
|
5204
|
+
"placeholder": "<domain>"
|
|
5205
|
+
},
|
|
5206
|
+
{
|
|
5207
|
+
"name": "harness",
|
|
5208
|
+
"required": false,
|
|
5209
|
+
"kind": "string",
|
|
5210
|
+
"placeholder": "eve"
|
|
5211
|
+
},
|
|
5212
|
+
{
|
|
5213
|
+
"name": "model",
|
|
5214
|
+
"required": false,
|
|
5215
|
+
"kind": "string",
|
|
5216
|
+
"placeholder": "claude-sonnet-5"
|
|
5217
|
+
},
|
|
5218
|
+
{
|
|
5219
|
+
"name": "mode",
|
|
5220
|
+
"required": false,
|
|
5221
|
+
"kind": "string",
|
|
5222
|
+
"enum": [
|
|
5223
|
+
"task",
|
|
5224
|
+
"session"
|
|
2470
5225
|
],
|
|
2471
|
-
"placeholder": "
|
|
5226
|
+
"placeholder": "task"
|
|
5227
|
+
},
|
|
5228
|
+
{
|
|
5229
|
+
"name": "addons",
|
|
5230
|
+
"required": false,
|
|
5231
|
+
"kind": "object",
|
|
5232
|
+
"placeholder": "{}"
|
|
5233
|
+
},
|
|
5234
|
+
{
|
|
5235
|
+
"name": "engine",
|
|
5236
|
+
"required": false,
|
|
5237
|
+
"kind": "string",
|
|
5238
|
+
"placeholder": "<engine>"
|
|
5239
|
+
},
|
|
5240
|
+
{
|
|
5241
|
+
"name": "execution_kind",
|
|
5242
|
+
"required": false,
|
|
5243
|
+
"kind": "string",
|
|
5244
|
+
"enum": [
|
|
5245
|
+
"harness",
|
|
5246
|
+
"direct"
|
|
5247
|
+
],
|
|
5248
|
+
"placeholder": "harness"
|
|
5249
|
+
},
|
|
5250
|
+
{
|
|
5251
|
+
"name": "tools",
|
|
5252
|
+
"required": false,
|
|
5253
|
+
"kind": "string",
|
|
5254
|
+
"placeholder": "<tools>"
|
|
5255
|
+
},
|
|
5256
|
+
{
|
|
5257
|
+
"name": "requires_tools",
|
|
5258
|
+
"required": false,
|
|
5259
|
+
"kind": "boolean",
|
|
5260
|
+
"placeholder": "true"
|
|
5261
|
+
},
|
|
5262
|
+
{
|
|
5263
|
+
"name": "mcp_servers",
|
|
5264
|
+
"required": false,
|
|
5265
|
+
"kind": "array",
|
|
5266
|
+
"placeholder": "[]"
|
|
5267
|
+
},
|
|
5268
|
+
{
|
|
5269
|
+
"name": "model_params",
|
|
5270
|
+
"required": false,
|
|
5271
|
+
"kind": "object",
|
|
5272
|
+
"placeholder": "{}"
|
|
5273
|
+
},
|
|
5274
|
+
{
|
|
5275
|
+
"name": "wait",
|
|
5276
|
+
"required": false,
|
|
5277
|
+
"kind": "boolean",
|
|
5278
|
+
"placeholder": "true"
|
|
5279
|
+
}
|
|
5280
|
+
],
|
|
5281
|
+
"watchable": true,
|
|
5282
|
+
"help": {
|
|
5283
|
+
"command": "ora study runs create --file run.json --watch",
|
|
5284
|
+
"body": "{\n \"intent\": \"check pricing for acme.com\",\n \"harness\": \"eve\",\n \"model\": \"claude-sonnet-5\"\n}"
|
|
5285
|
+
}
|
|
5286
|
+
},
|
|
5287
|
+
{
|
|
5288
|
+
"id": "study.runs.directSignals",
|
|
5289
|
+
"commandPath": [
|
|
5290
|
+
"study",
|
|
5291
|
+
"runs",
|
|
5292
|
+
"direct-signals"
|
|
5293
|
+
],
|
|
5294
|
+
"method": "get",
|
|
5295
|
+
"pathTemplate": "/study/v1/runs/{id}/direct-signals",
|
|
5296
|
+
"summary": "Read a direct run's extracted DirectSignals artifact",
|
|
5297
|
+
"scope": "runs:read",
|
|
5298
|
+
"product": "platform",
|
|
5299
|
+
"pathParams": [
|
|
5300
|
+
{
|
|
5301
|
+
"name": "id"
|
|
5302
|
+
}
|
|
5303
|
+
],
|
|
5304
|
+
"queryParams": [],
|
|
5305
|
+
"hasBody": false,
|
|
5306
|
+
"bodyRequired": false,
|
|
5307
|
+
"isStream": false,
|
|
5308
|
+
"watchable": false,
|
|
5309
|
+
"help": {
|
|
5310
|
+
"command": "ora study runs direct-signals <id>"
|
|
5311
|
+
}
|
|
5312
|
+
},
|
|
5313
|
+
{
|
|
5314
|
+
"id": "study.runs.feed",
|
|
5315
|
+
"commandPath": [
|
|
5316
|
+
"study",
|
|
5317
|
+
"runs",
|
|
5318
|
+
"feed"
|
|
5319
|
+
],
|
|
5320
|
+
"method": "get",
|
|
5321
|
+
"pathTemplate": "/study/v1/runs/stream",
|
|
5322
|
+
"summary": "Org-scoped live feed of run status patches (SSE)",
|
|
5323
|
+
"scope": "runs:read",
|
|
5324
|
+
"product": "platform",
|
|
5325
|
+
"pathParams": [],
|
|
5326
|
+
"queryParams": [],
|
|
5327
|
+
"hasBody": false,
|
|
5328
|
+
"bodyRequired": false,
|
|
5329
|
+
"isStream": true,
|
|
5330
|
+
"watchable": false,
|
|
5331
|
+
"help": {
|
|
5332
|
+
"command": "ora study runs feed"
|
|
5333
|
+
}
|
|
5334
|
+
},
|
|
5335
|
+
{
|
|
5336
|
+
"id": "study.runs.insight",
|
|
5337
|
+
"commandPath": [
|
|
5338
|
+
"study",
|
|
5339
|
+
"runs",
|
|
5340
|
+
"insight"
|
|
5341
|
+
],
|
|
5342
|
+
"method": "get",
|
|
5343
|
+
"pathTemplate": "/study/v1/runs/{id}/insight",
|
|
5344
|
+
"summary": "Read a run's stored insight artifact",
|
|
5345
|
+
"scope": "runs:read",
|
|
5346
|
+
"product": "platform",
|
|
5347
|
+
"pathParams": [
|
|
5348
|
+
{
|
|
5349
|
+
"name": "id"
|
|
5350
|
+
}
|
|
5351
|
+
],
|
|
5352
|
+
"queryParams": [],
|
|
5353
|
+
"hasBody": false,
|
|
5354
|
+
"bodyRequired": false,
|
|
5355
|
+
"isStream": false,
|
|
5356
|
+
"watchable": false,
|
|
5357
|
+
"help": {
|
|
5358
|
+
"command": "ora study runs insight <id>"
|
|
5359
|
+
}
|
|
5360
|
+
},
|
|
5361
|
+
{
|
|
5362
|
+
"id": "study.runs.list",
|
|
5363
|
+
"commandPath": [
|
|
5364
|
+
"study",
|
|
5365
|
+
"runs",
|
|
5366
|
+
"list"
|
|
5367
|
+
],
|
|
5368
|
+
"method": "get",
|
|
5369
|
+
"pathTemplate": "/study/v1/runs",
|
|
5370
|
+
"summary": "List runs",
|
|
5371
|
+
"scope": "runs:read",
|
|
5372
|
+
"product": "platform",
|
|
5373
|
+
"pathParams": [],
|
|
5374
|
+
"queryParams": [
|
|
5375
|
+
{
|
|
5376
|
+
"name": "pageSize",
|
|
5377
|
+
"flag": "page-size",
|
|
5378
|
+
"optionKey": "pageSize",
|
|
5379
|
+
"required": false,
|
|
5380
|
+
"kind": "number"
|
|
5381
|
+
},
|
|
5382
|
+
{
|
|
5383
|
+
"name": "pageToken",
|
|
5384
|
+
"flag": "page-token",
|
|
5385
|
+
"optionKey": "pageToken",
|
|
5386
|
+
"required": false,
|
|
5387
|
+
"kind": "string"
|
|
2472
5388
|
}
|
|
2473
5389
|
],
|
|
5390
|
+
"hasBody": false,
|
|
5391
|
+
"bodyRequired": false,
|
|
5392
|
+
"isStream": false,
|
|
2474
5393
|
"watchable": false,
|
|
2475
5394
|
"help": {
|
|
2476
|
-
"command": "ora
|
|
2477
|
-
"body": "{\n \"source\": \"custom\",\n \"name\": \"<name>\",\n \"body\": \"<body>\"\n}"
|
|
5395
|
+
"command": "ora study runs list"
|
|
2478
5396
|
}
|
|
2479
5397
|
},
|
|
2480
5398
|
{
|
|
2481
|
-
"id": "
|
|
5399
|
+
"id": "study.runs.rawTrace",
|
|
2482
5400
|
"commandPath": [
|
|
2483
|
-
"
|
|
2484
|
-
"
|
|
5401
|
+
"study",
|
|
5402
|
+
"runs",
|
|
5403
|
+
"raw-trace"
|
|
2485
5404
|
],
|
|
2486
|
-
"method": "
|
|
2487
|
-
"pathTemplate": "/
|
|
2488
|
-
"summary": "
|
|
2489
|
-
"scope": "
|
|
2490
|
-
"product": "
|
|
5405
|
+
"method": "get",
|
|
5406
|
+
"pathTemplate": "/study/v1/runs/{id}/raw-trace",
|
|
5407
|
+
"summary": "Read a run's full raw frame stream (pre-truncation trace artifact)",
|
|
5408
|
+
"scope": "runs:read",
|
|
5409
|
+
"product": "platform",
|
|
2491
5410
|
"pathParams": [
|
|
2492
5411
|
{
|
|
2493
5412
|
"name": "id"
|
|
@@ -2499,166 +5418,152 @@ export const OPERATIONS = [
|
|
|
2499
5418
|
"isStream": false,
|
|
2500
5419
|
"watchable": false,
|
|
2501
5420
|
"help": {
|
|
2502
|
-
"command": "ora
|
|
5421
|
+
"command": "ora study runs raw-trace <id>"
|
|
2503
5422
|
}
|
|
2504
5423
|
},
|
|
2505
5424
|
{
|
|
2506
|
-
"id": "
|
|
5425
|
+
"id": "study.runs.spans",
|
|
2507
5426
|
"commandPath": [
|
|
2508
|
-
"
|
|
2509
|
-
"
|
|
5427
|
+
"study",
|
|
5428
|
+
"runs",
|
|
5429
|
+
"spans"
|
|
2510
5430
|
],
|
|
2511
5431
|
"method": "get",
|
|
2512
|
-
"pathTemplate": "/
|
|
2513
|
-
"summary": "
|
|
2514
|
-
"scope": "
|
|
2515
|
-
"product": "
|
|
2516
|
-
"pathParams": [
|
|
5432
|
+
"pathTemplate": "/study/v1/runs/{id}/spans",
|
|
5433
|
+
"summary": "Read a run's stored trace spans (CompactSpan[])",
|
|
5434
|
+
"scope": "runs:read",
|
|
5435
|
+
"product": "platform",
|
|
5436
|
+
"pathParams": [
|
|
5437
|
+
{
|
|
5438
|
+
"name": "id"
|
|
5439
|
+
}
|
|
5440
|
+
],
|
|
2517
5441
|
"queryParams": [],
|
|
2518
5442
|
"hasBody": false,
|
|
2519
5443
|
"bodyRequired": false,
|
|
2520
5444
|
"isStream": false,
|
|
2521
5445
|
"watchable": false,
|
|
2522
5446
|
"help": {
|
|
2523
|
-
"command": "ora
|
|
5447
|
+
"command": "ora study runs spans <id>"
|
|
2524
5448
|
}
|
|
2525
5449
|
},
|
|
2526
5450
|
{
|
|
2527
|
-
"id": "
|
|
5451
|
+
"id": "study.runs.stream",
|
|
2528
5452
|
"commandPath": [
|
|
2529
|
-
"
|
|
2530
|
-
"
|
|
5453
|
+
"study",
|
|
5454
|
+
"runs",
|
|
5455
|
+
"stream"
|
|
2531
5456
|
],
|
|
2532
|
-
"method": "
|
|
2533
|
-
"pathTemplate": "/
|
|
2534
|
-
"summary": "
|
|
2535
|
-
"scope": "
|
|
2536
|
-
"product": "
|
|
5457
|
+
"method": "get",
|
|
5458
|
+
"pathTemplate": "/study/v1/runs/{id}/stream",
|
|
5459
|
+
"summary": "Stream a run's live trajectory (SSE)",
|
|
5460
|
+
"scope": "runs:read",
|
|
5461
|
+
"product": "platform",
|
|
2537
5462
|
"pathParams": [
|
|
2538
5463
|
{
|
|
2539
5464
|
"name": "id"
|
|
2540
5465
|
}
|
|
2541
5466
|
],
|
|
2542
5467
|
"queryParams": [],
|
|
2543
|
-
"hasBody":
|
|
2544
|
-
"bodyRequired":
|
|
2545
|
-
"isStream":
|
|
2546
|
-
"bodyFields": [
|
|
2547
|
-
{
|
|
2548
|
-
"name": "name",
|
|
2549
|
-
"required": false,
|
|
2550
|
-
"kind": "string",
|
|
2551
|
-
"placeholder": "<name>"
|
|
2552
|
-
},
|
|
2553
|
-
{
|
|
2554
|
-
"name": "visibility",
|
|
2555
|
-
"required": false,
|
|
2556
|
-
"kind": "string",
|
|
2557
|
-
"enum": [
|
|
2558
|
-
"user",
|
|
2559
|
-
"org"
|
|
2560
|
-
],
|
|
2561
|
-
"placeholder": "user"
|
|
2562
|
-
},
|
|
2563
|
-
{
|
|
2564
|
-
"name": "body",
|
|
2565
|
-
"required": false,
|
|
2566
|
-
"kind": "string",
|
|
2567
|
-
"placeholder": "<body>"
|
|
2568
|
-
},
|
|
2569
|
-
{
|
|
2570
|
-
"name": "repoUrl",
|
|
2571
|
-
"required": false,
|
|
2572
|
-
"kind": "string",
|
|
2573
|
-
"placeholder": "<repoUrl>"
|
|
2574
|
-
},
|
|
2575
|
-
{
|
|
2576
|
-
"name": "gitRef",
|
|
2577
|
-
"required": false,
|
|
2578
|
-
"kind": "string",
|
|
2579
|
-
"placeholder": "<gitRef>"
|
|
2580
|
-
},
|
|
2581
|
-
{
|
|
2582
|
-
"name": "subpath",
|
|
2583
|
-
"required": false,
|
|
2584
|
-
"kind": "string",
|
|
2585
|
-
"placeholder": "<subpath>"
|
|
2586
|
-
}
|
|
2587
|
-
],
|
|
5468
|
+
"hasBody": false,
|
|
5469
|
+
"bodyRequired": false,
|
|
5470
|
+
"isStream": true,
|
|
2588
5471
|
"watchable": false,
|
|
2589
5472
|
"help": {
|
|
2590
|
-
"command": "ora
|
|
2591
|
-
"body": "{\n \"name\": \"<name>\",\n \"visibility\": \"user\",\n \"body\": \"<body>\"\n}"
|
|
5473
|
+
"command": "ora study runs stream <id>"
|
|
2592
5474
|
}
|
|
2593
5475
|
},
|
|
2594
5476
|
{
|
|
2595
|
-
"id": "
|
|
5477
|
+
"id": "study.runs.trajectory",
|
|
2596
5478
|
"commandPath": [
|
|
2597
|
-
"
|
|
5479
|
+
"study",
|
|
2598
5480
|
"runs",
|
|
2599
|
-
"
|
|
5481
|
+
"trajectory"
|
|
2600
5482
|
],
|
|
2601
|
-
"method": "
|
|
2602
|
-
"pathTemplate": "/
|
|
2603
|
-
"summary": "
|
|
2604
|
-
"scope": "runs:
|
|
5483
|
+
"method": "get",
|
|
5484
|
+
"pathTemplate": "/study/v1/runs/{id}/trajectory",
|
|
5485
|
+
"summary": "Read a run's normalized trajectory snapshot",
|
|
5486
|
+
"scope": "runs:read",
|
|
2605
5487
|
"product": "platform",
|
|
2606
5488
|
"pathParams": [
|
|
2607
5489
|
{
|
|
2608
5490
|
"name": "id"
|
|
2609
5491
|
}
|
|
2610
5492
|
],
|
|
2611
|
-
"queryParams": [
|
|
5493
|
+
"queryParams": [
|
|
5494
|
+
{
|
|
5495
|
+
"name": "profile",
|
|
5496
|
+
"flag": "profile",
|
|
5497
|
+
"optionKey": "profile",
|
|
5498
|
+
"required": false,
|
|
5499
|
+
"kind": "string",
|
|
5500
|
+
"enum": [
|
|
5501
|
+
"raw",
|
|
5502
|
+
"heuristic",
|
|
5503
|
+
"full"
|
|
5504
|
+
]
|
|
5505
|
+
}
|
|
5506
|
+
],
|
|
2612
5507
|
"hasBody": false,
|
|
2613
5508
|
"bodyRequired": false,
|
|
2614
5509
|
"isStream": false,
|
|
2615
5510
|
"watchable": false,
|
|
2616
5511
|
"help": {
|
|
2617
|
-
"command": "ora
|
|
5512
|
+
"command": "ora study runs trajectory <id>"
|
|
2618
5513
|
}
|
|
2619
5514
|
},
|
|
2620
5515
|
{
|
|
2621
|
-
"id": "
|
|
5516
|
+
"id": "study.setVisibility",
|
|
2622
5517
|
"commandPath": [
|
|
2623
|
-
"
|
|
2624
|
-
"
|
|
2625
|
-
|
|
5518
|
+
"study",
|
|
5519
|
+
"set-visibility"
|
|
5520
|
+
],
|
|
5521
|
+
"method": "patch",
|
|
5522
|
+
"pathTemplate": "/study/v1/studies/{id}/visibility",
|
|
5523
|
+
"summary": "Set who may read this study: private | org | link | public. Moving INTO 'link' mints a 256-bit share token (returned in `shareUrl`); moving out clears it immediately. The FIRST move to 'link'/'public' publishes the study; later mode changes are not re-publications and never re-stamp its publication date.",
|
|
5524
|
+
"scope": "study.studies:write",
|
|
5525
|
+
"product": "study",
|
|
5526
|
+
"pathParams": [
|
|
5527
|
+
{
|
|
5528
|
+
"name": "id"
|
|
5529
|
+
}
|
|
2626
5530
|
],
|
|
2627
|
-
"method": "post",
|
|
2628
|
-
"pathTemplate": "/studio/v1/runs/cancel",
|
|
2629
|
-
"summary": "Cancel a batch of runs (terminate them and stop their engines)",
|
|
2630
|
-
"scope": "runs:write",
|
|
2631
|
-
"product": "platform",
|
|
2632
|
-
"pathParams": [],
|
|
2633
5531
|
"queryParams": [],
|
|
2634
5532
|
"hasBody": true,
|
|
2635
5533
|
"bodyRequired": true,
|
|
2636
5534
|
"isStream": false,
|
|
2637
5535
|
"bodyFields": [
|
|
2638
5536
|
{
|
|
2639
|
-
"name": "
|
|
5537
|
+
"name": "visibility",
|
|
2640
5538
|
"required": true,
|
|
2641
|
-
"kind": "
|
|
2642
|
-
"
|
|
5539
|
+
"kind": "string",
|
|
5540
|
+
"enum": [
|
|
5541
|
+
"private",
|
|
5542
|
+
"org",
|
|
5543
|
+
"link",
|
|
5544
|
+
"public"
|
|
5545
|
+
],
|
|
5546
|
+
"description": "The sharing mode: private (creator + org owners) | org (the whole org) | link (anyone holding the share link) | public (everyone, listed in the catalog). Moving INTO 'link' mints a share token; moving out clears it.",
|
|
5547
|
+
"placeholder": "link"
|
|
2643
5548
|
}
|
|
2644
5549
|
],
|
|
2645
5550
|
"watchable": false,
|
|
2646
5551
|
"help": {
|
|
2647
|
-
"command": "ora
|
|
2648
|
-
"body": "{\n \"
|
|
5552
|
+
"command": "ora study set-visibility 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
5553
|
+
"body": "{\n \"visibility\": \"link\"\n}"
|
|
2649
5554
|
}
|
|
2650
5555
|
},
|
|
2651
5556
|
{
|
|
2652
|
-
"id": "
|
|
5557
|
+
"id": "study.skills.create",
|
|
2653
5558
|
"commandPath": [
|
|
2654
|
-
"
|
|
2655
|
-
"
|
|
5559
|
+
"study",
|
|
5560
|
+
"skills",
|
|
2656
5561
|
"create"
|
|
2657
5562
|
],
|
|
2658
5563
|
"method": "post",
|
|
2659
|
-
"pathTemplate": "/
|
|
2660
|
-
"summary": "Create a
|
|
2661
|
-
"scope": "
|
|
5564
|
+
"pathTemplate": "/study/v1/skills",
|
|
5565
|
+
"summary": "Create a skill (custom or git-imported)",
|
|
5566
|
+
"scope": "skills:write",
|
|
2662
5567
|
"product": "platform",
|
|
2663
5568
|
"pathParams": [],
|
|
2664
5569
|
"queryParams": [],
|
|
@@ -2667,109 +5572,51 @@ export const OPERATIONS = [
|
|
|
2667
5572
|
"isStream": false,
|
|
2668
5573
|
"bodyFields": [
|
|
2669
5574
|
{
|
|
2670
|
-
"name": "
|
|
2671
|
-
"required": false,
|
|
2672
|
-
"kind": "string",
|
|
2673
|
-
"placeholder": "<product_key>"
|
|
2674
|
-
},
|
|
2675
|
-
{
|
|
2676
|
-
"name": "intent",
|
|
5575
|
+
"name": "source",
|
|
2677
5576
|
"required": true,
|
|
2678
5577
|
"kind": "string",
|
|
2679
|
-
"placeholder": "
|
|
2680
|
-
},
|
|
2681
|
-
{
|
|
2682
|
-
"name": "domain",
|
|
2683
|
-
"required": false,
|
|
2684
|
-
"kind": "string",
|
|
2685
|
-
"placeholder": "<domain>"
|
|
2686
|
-
},
|
|
2687
|
-
{
|
|
2688
|
-
"name": "harness",
|
|
2689
|
-
"required": false,
|
|
2690
|
-
"kind": "string",
|
|
2691
|
-
"placeholder": "eve"
|
|
2692
|
-
},
|
|
2693
|
-
{
|
|
2694
|
-
"name": "model",
|
|
2695
|
-
"required": false,
|
|
2696
|
-
"kind": "string",
|
|
2697
|
-
"placeholder": "claude-sonnet-5"
|
|
5578
|
+
"placeholder": "custom"
|
|
2698
5579
|
},
|
|
2699
5580
|
{
|
|
2700
|
-
"name": "
|
|
2701
|
-
"required":
|
|
5581
|
+
"name": "name",
|
|
5582
|
+
"required": true,
|
|
2702
5583
|
"kind": "string",
|
|
2703
|
-
"
|
|
2704
|
-
"task",
|
|
2705
|
-
"session"
|
|
2706
|
-
],
|
|
2707
|
-
"placeholder": "task"
|
|
2708
|
-
},
|
|
2709
|
-
{
|
|
2710
|
-
"name": "addons",
|
|
2711
|
-
"required": false,
|
|
2712
|
-
"kind": "object",
|
|
2713
|
-
"placeholder": "{}"
|
|
5584
|
+
"placeholder": "<name>"
|
|
2714
5585
|
},
|
|
2715
5586
|
{
|
|
2716
|
-
"name": "
|
|
2717
|
-
"required":
|
|
5587
|
+
"name": "body",
|
|
5588
|
+
"required": true,
|
|
2718
5589
|
"kind": "string",
|
|
2719
|
-
"placeholder": "<
|
|
5590
|
+
"placeholder": "<body>"
|
|
2720
5591
|
},
|
|
2721
5592
|
{
|
|
2722
|
-
"name": "
|
|
5593
|
+
"name": "visibility",
|
|
2723
5594
|
"required": false,
|
|
2724
5595
|
"kind": "string",
|
|
2725
5596
|
"enum": [
|
|
2726
|
-
"
|
|
2727
|
-
"
|
|
5597
|
+
"user",
|
|
5598
|
+
"org"
|
|
2728
5599
|
],
|
|
2729
|
-
"placeholder": "
|
|
2730
|
-
},
|
|
2731
|
-
{
|
|
2732
|
-
"name": "tools",
|
|
2733
|
-
"required": false,
|
|
2734
|
-
"kind": "string",
|
|
2735
|
-
"placeholder": "<tools>"
|
|
2736
|
-
},
|
|
2737
|
-
{
|
|
2738
|
-
"name": "requires_tools",
|
|
2739
|
-
"required": false,
|
|
2740
|
-
"kind": "boolean",
|
|
2741
|
-
"placeholder": "true"
|
|
2742
|
-
},
|
|
2743
|
-
{
|
|
2744
|
-
"name": "mcp_servers",
|
|
2745
|
-
"required": false,
|
|
2746
|
-
"kind": "array",
|
|
2747
|
-
"placeholder": "[]"
|
|
2748
|
-
},
|
|
2749
|
-
{
|
|
2750
|
-
"name": "wait",
|
|
2751
|
-
"required": false,
|
|
2752
|
-
"kind": "boolean",
|
|
2753
|
-
"placeholder": "true"
|
|
5600
|
+
"placeholder": "user"
|
|
2754
5601
|
}
|
|
2755
5602
|
],
|
|
2756
|
-
"watchable":
|
|
2757
|
-
"help": {
|
|
2758
|
-
"command": "ora
|
|
2759
|
-
"body": "{\n \"
|
|
5603
|
+
"watchable": false,
|
|
5604
|
+
"help": {
|
|
5605
|
+
"command": "ora study skills create --file run.json",
|
|
5606
|
+
"body": "{\n \"source\": \"custom\",\n \"name\": \"<name>\",\n \"body\": \"<body>\"\n}"
|
|
2760
5607
|
}
|
|
2761
5608
|
},
|
|
2762
5609
|
{
|
|
2763
|
-
"id": "
|
|
5610
|
+
"id": "study.skills.delete",
|
|
2764
5611
|
"commandPath": [
|
|
2765
|
-
"
|
|
2766
|
-
"
|
|
2767
|
-
"
|
|
5612
|
+
"study",
|
|
5613
|
+
"skills",
|
|
5614
|
+
"delete"
|
|
2768
5615
|
],
|
|
2769
|
-
"method": "
|
|
2770
|
-
"pathTemplate": "/
|
|
2771
|
-
"summary": "
|
|
2772
|
-
"scope": "
|
|
5616
|
+
"method": "delete",
|
|
5617
|
+
"pathTemplate": "/study/v1/skills/{id}",
|
|
5618
|
+
"summary": "Delete a skill",
|
|
5619
|
+
"scope": "skills:write",
|
|
2773
5620
|
"product": "platform",
|
|
2774
5621
|
"pathParams": [
|
|
2775
5622
|
{
|
|
@@ -2782,42 +5629,42 @@ export const OPERATIONS = [
|
|
|
2782
5629
|
"isStream": false,
|
|
2783
5630
|
"watchable": false,
|
|
2784
5631
|
"help": {
|
|
2785
|
-
"command": "ora
|
|
5632
|
+
"command": "ora study skills delete <id>"
|
|
2786
5633
|
}
|
|
2787
5634
|
},
|
|
2788
5635
|
{
|
|
2789
|
-
"id": "
|
|
5636
|
+
"id": "study.skills.list",
|
|
2790
5637
|
"commandPath": [
|
|
2791
|
-
"
|
|
2792
|
-
"
|
|
2793
|
-
"
|
|
5638
|
+
"study",
|
|
5639
|
+
"skills",
|
|
5640
|
+
"list"
|
|
2794
5641
|
],
|
|
2795
5642
|
"method": "get",
|
|
2796
|
-
"pathTemplate": "/
|
|
2797
|
-
"summary": "
|
|
2798
|
-
"scope": "
|
|
5643
|
+
"pathTemplate": "/study/v1/skills",
|
|
5644
|
+
"summary": "List the caller's usable skills catalog",
|
|
5645
|
+
"scope": "skills:read",
|
|
2799
5646
|
"product": "platform",
|
|
2800
5647
|
"pathParams": [],
|
|
2801
5648
|
"queryParams": [],
|
|
2802
5649
|
"hasBody": false,
|
|
2803
5650
|
"bodyRequired": false,
|
|
2804
|
-
"isStream":
|
|
5651
|
+
"isStream": false,
|
|
2805
5652
|
"watchable": false,
|
|
2806
5653
|
"help": {
|
|
2807
|
-
"command": "ora
|
|
5654
|
+
"command": "ora study skills list"
|
|
2808
5655
|
}
|
|
2809
5656
|
},
|
|
2810
5657
|
{
|
|
2811
|
-
"id": "
|
|
5658
|
+
"id": "study.skills.update",
|
|
2812
5659
|
"commandPath": [
|
|
2813
|
-
"
|
|
2814
|
-
"
|
|
2815
|
-
"
|
|
5660
|
+
"study",
|
|
5661
|
+
"skills",
|
|
5662
|
+
"update"
|
|
2816
5663
|
],
|
|
2817
|
-
"method": "
|
|
2818
|
-
"pathTemplate": "/
|
|
2819
|
-
"summary": "
|
|
2820
|
-
"scope": "
|
|
5664
|
+
"method": "put",
|
|
5665
|
+
"pathTemplate": "/study/v1/skills/{id}",
|
|
5666
|
+
"summary": "Update a skill",
|
|
5667
|
+
"scope": "skills:write",
|
|
2821
5668
|
"product": "platform",
|
|
2822
5669
|
"pathParams": [
|
|
2823
5670
|
{
|
|
@@ -2825,63 +5672,68 @@ export const OPERATIONS = [
|
|
|
2825
5672
|
}
|
|
2826
5673
|
],
|
|
2827
5674
|
"queryParams": [],
|
|
2828
|
-
"hasBody":
|
|
2829
|
-
"bodyRequired":
|
|
5675
|
+
"hasBody": true,
|
|
5676
|
+
"bodyRequired": true,
|
|
2830
5677
|
"isStream": false,
|
|
2831
|
-
"
|
|
2832
|
-
"help": {
|
|
2833
|
-
"command": "ora studio runs insight <id>"
|
|
2834
|
-
}
|
|
2835
|
-
},
|
|
2836
|
-
{
|
|
2837
|
-
"id": "studio.runs.list",
|
|
2838
|
-
"commandPath": [
|
|
2839
|
-
"studio",
|
|
2840
|
-
"runs",
|
|
2841
|
-
"list"
|
|
2842
|
-
],
|
|
2843
|
-
"method": "get",
|
|
2844
|
-
"pathTemplate": "/studio/v1/runs",
|
|
2845
|
-
"summary": "List runs",
|
|
2846
|
-
"scope": "runs:read",
|
|
2847
|
-
"product": "platform",
|
|
2848
|
-
"pathParams": [],
|
|
2849
|
-
"queryParams": [
|
|
5678
|
+
"bodyFields": [
|
|
2850
5679
|
{
|
|
2851
|
-
"name": "
|
|
2852
|
-
"flag": "page-size",
|
|
2853
|
-
"optionKey": "pageSize",
|
|
5680
|
+
"name": "name",
|
|
2854
5681
|
"required": false,
|
|
2855
|
-
"kind": "
|
|
5682
|
+
"kind": "string",
|
|
5683
|
+
"placeholder": "<name>"
|
|
2856
5684
|
},
|
|
2857
5685
|
{
|
|
2858
|
-
"name": "
|
|
2859
|
-
"flag": "page-token",
|
|
2860
|
-
"optionKey": "pageToken",
|
|
5686
|
+
"name": "visibility",
|
|
2861
5687
|
"required": false,
|
|
2862
|
-
"kind": "string"
|
|
5688
|
+
"kind": "string",
|
|
5689
|
+
"enum": [
|
|
5690
|
+
"user",
|
|
5691
|
+
"org"
|
|
5692
|
+
],
|
|
5693
|
+
"placeholder": "user"
|
|
5694
|
+
},
|
|
5695
|
+
{
|
|
5696
|
+
"name": "body",
|
|
5697
|
+
"required": false,
|
|
5698
|
+
"kind": "string",
|
|
5699
|
+
"placeholder": "<body>"
|
|
5700
|
+
},
|
|
5701
|
+
{
|
|
5702
|
+
"name": "repoUrl",
|
|
5703
|
+
"required": false,
|
|
5704
|
+
"kind": "string",
|
|
5705
|
+
"placeholder": "<repoUrl>"
|
|
5706
|
+
},
|
|
5707
|
+
{
|
|
5708
|
+
"name": "gitRef",
|
|
5709
|
+
"required": false,
|
|
5710
|
+
"kind": "string",
|
|
5711
|
+
"placeholder": "<gitRef>"
|
|
5712
|
+
},
|
|
5713
|
+
{
|
|
5714
|
+
"name": "subpath",
|
|
5715
|
+
"required": false,
|
|
5716
|
+
"kind": "string",
|
|
5717
|
+
"placeholder": "<subpath>"
|
|
2863
5718
|
}
|
|
2864
5719
|
],
|
|
2865
|
-
"hasBody": false,
|
|
2866
|
-
"bodyRequired": false,
|
|
2867
|
-
"isStream": false,
|
|
2868
5720
|
"watchable": false,
|
|
2869
5721
|
"help": {
|
|
2870
|
-
"command": "ora
|
|
5722
|
+
"command": "ora study skills update <id> --file run.json",
|
|
5723
|
+
"body": "{\n \"name\": \"<name>\",\n \"visibility\": \"user\",\n \"body\": \"<body>\"\n}"
|
|
2871
5724
|
}
|
|
2872
5725
|
},
|
|
2873
5726
|
{
|
|
2874
|
-
"id": "
|
|
5727
|
+
"id": "study.stream",
|
|
2875
5728
|
"commandPath": [
|
|
2876
|
-
"
|
|
2877
|
-
"
|
|
2878
|
-
"raw-trace"
|
|
5729
|
+
"study",
|
|
5730
|
+
"stream"
|
|
2879
5731
|
],
|
|
2880
5732
|
"method": "get",
|
|
2881
|
-
"pathTemplate": "/
|
|
2882
|
-
"summary": "
|
|
2883
|
-
"scope": "
|
|
2884
|
-
"product": "
|
|
5733
|
+
"pathTemplate": "/study/v1/studies/{id}/stream",
|
|
5734
|
+
"summary": "Live SSE stream: status transitions, per-run facts, grading, publish state (terminal status closes it)",
|
|
5735
|
+
"scope": "study.studies:read",
|
|
5736
|
+
"product": "study",
|
|
2885
5737
|
"pathParams": [
|
|
2886
5738
|
{
|
|
2887
5739
|
"name": "id"
|
|
@@ -2890,24 +5742,23 @@ export const OPERATIONS = [
|
|
|
2890
5742
|
"queryParams": [],
|
|
2891
5743
|
"hasBody": false,
|
|
2892
5744
|
"bodyRequired": false,
|
|
2893
|
-
"isStream":
|
|
5745
|
+
"isStream": true,
|
|
2894
5746
|
"watchable": false,
|
|
2895
5747
|
"help": {
|
|
2896
|
-
"command": "ora
|
|
5748
|
+
"command": "ora study stream 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
2897
5749
|
}
|
|
2898
5750
|
},
|
|
2899
5751
|
{
|
|
2900
|
-
"id": "
|
|
5752
|
+
"id": "study.unpublish",
|
|
2901
5753
|
"commandPath": [
|
|
2902
|
-
"
|
|
2903
|
-
"
|
|
2904
|
-
"spans"
|
|
5754
|
+
"study",
|
|
5755
|
+
"unpublish"
|
|
2905
5756
|
],
|
|
2906
|
-
"method": "
|
|
2907
|
-
"pathTemplate": "/
|
|
2908
|
-
"summary": "
|
|
2909
|
-
"scope": "
|
|
2910
|
-
"product": "
|
|
5757
|
+
"method": "post",
|
|
5758
|
+
"pathTemplate": "/study/v1/studies/{id}/unpublish",
|
|
5759
|
+
"summary": "Remove a study from the public catalog (data kept; re-publish any time)",
|
|
5760
|
+
"scope": "study.studies:write",
|
|
5761
|
+
"product": "study",
|
|
2911
5762
|
"pathParams": [
|
|
2912
5763
|
{
|
|
2913
5764
|
"name": "id"
|
|
@@ -2919,72 +5770,87 @@ export const OPERATIONS = [
|
|
|
2919
5770
|
"isStream": false,
|
|
2920
5771
|
"watchable": false,
|
|
2921
5772
|
"help": {
|
|
2922
|
-
"command": "ora
|
|
5773
|
+
"command": "ora study unpublish <id>"
|
|
2923
5774
|
}
|
|
2924
5775
|
},
|
|
2925
5776
|
{
|
|
2926
|
-
"id": "
|
|
5777
|
+
"id": "study.updateLenses",
|
|
2927
5778
|
"commandPath": [
|
|
2928
|
-
"
|
|
2929
|
-
"
|
|
2930
|
-
"stream"
|
|
5779
|
+
"study",
|
|
5780
|
+
"update-lenses"
|
|
2931
5781
|
],
|
|
2932
|
-
"method": "
|
|
2933
|
-
"pathTemplate": "/
|
|
2934
|
-
"summary": "
|
|
2935
|
-
"scope": "
|
|
2936
|
-
"product": "
|
|
5782
|
+
"method": "put",
|
|
5783
|
+
"pathTemplate": "/study/v1/studies/{id}/lenses",
|
|
5784
|
+
"summary": "Change which grading lenses a study runs under before launch (status 'ready' only) — the complete replacement set, e.g. add 'evals' to a judge-only study so its tasks can carry expectations",
|
|
5785
|
+
"scope": "study.studies:write",
|
|
5786
|
+
"product": "study",
|
|
2937
5787
|
"pathParams": [
|
|
2938
5788
|
{
|
|
2939
5789
|
"name": "id"
|
|
2940
5790
|
}
|
|
2941
5791
|
],
|
|
2942
5792
|
"queryParams": [],
|
|
2943
|
-
"hasBody":
|
|
2944
|
-
"bodyRequired":
|
|
2945
|
-
"isStream":
|
|
5793
|
+
"hasBody": true,
|
|
5794
|
+
"bodyRequired": true,
|
|
5795
|
+
"isStream": false,
|
|
5796
|
+
"bodyFields": [
|
|
5797
|
+
{
|
|
5798
|
+
"name": "lenses",
|
|
5799
|
+
"required": true,
|
|
5800
|
+
"kind": "array",
|
|
5801
|
+
"description": "The COMPLETE replacement lens set (not a delta). Editable only while status is 'ready'. Removing 'evals' while a task still carries authored `expectations` is BLOCKED (422 LENS_REMOVAL_BLOCKED, names the task(s)); removing 'judge' while a task carries a `rubric` succeeds but returns a non-blocking warning (the rubric stays stored, just ungraded).",
|
|
5802
|
+
"placeholder": "[\"judge\",\"evals\"]"
|
|
5803
|
+
}
|
|
5804
|
+
],
|
|
2946
5805
|
"watchable": false,
|
|
2947
5806
|
"help": {
|
|
2948
|
-
"command": "ora
|
|
5807
|
+
"command": "ora study update-lenses 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
5808
|
+
"body": "{\n \"lenses\": [\n \"judge\",\n \"evals\"\n ]\n}"
|
|
2949
5809
|
}
|
|
2950
5810
|
},
|
|
2951
5811
|
{
|
|
2952
|
-
"id": "
|
|
5812
|
+
"id": "study.updateTask",
|
|
2953
5813
|
"commandPath": [
|
|
2954
|
-
"
|
|
2955
|
-
"
|
|
2956
|
-
"trajectory"
|
|
5814
|
+
"study",
|
|
5815
|
+
"update-task"
|
|
2957
5816
|
],
|
|
2958
|
-
"method": "
|
|
2959
|
-
"pathTemplate": "/
|
|
2960
|
-
"summary": "
|
|
2961
|
-
"scope": "
|
|
2962
|
-
"product": "
|
|
5817
|
+
"method": "put",
|
|
5818
|
+
"pathTemplate": "/study/v1/studies/{id}/tasks/{taskId}",
|
|
5819
|
+
"summary": "Edit a generated task's prompt before launch (status 'ready' only)",
|
|
5820
|
+
"scope": "study.studies:write",
|
|
5821
|
+
"product": "study",
|
|
2963
5822
|
"pathParams": [
|
|
2964
5823
|
{
|
|
2965
5824
|
"name": "id"
|
|
5825
|
+
},
|
|
5826
|
+
{
|
|
5827
|
+
"name": "taskId"
|
|
2966
5828
|
}
|
|
2967
5829
|
],
|
|
2968
|
-
"queryParams": [
|
|
5830
|
+
"queryParams": [],
|
|
5831
|
+
"hasBody": true,
|
|
5832
|
+
"bodyRequired": true,
|
|
5833
|
+
"isStream": false,
|
|
5834
|
+
"bodyFields": [
|
|
2969
5835
|
{
|
|
2970
|
-
"name": "
|
|
2971
|
-
"
|
|
2972
|
-
"optionKey": "profile",
|
|
2973
|
-
"required": false,
|
|
5836
|
+
"name": "prompt",
|
|
5837
|
+
"required": true,
|
|
2974
5838
|
"kind": "string",
|
|
2975
|
-
"
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
5839
|
+
"description": "The replacement task prompt, dispatched verbatim as each run's intent. Editable only while the study is 'ready' (pre-launch).",
|
|
5840
|
+
"placeholder": "Research the three largest EU battery-storage tenders announced this quarter and cite each source."
|
|
5841
|
+
},
|
|
5842
|
+
{
|
|
5843
|
+
"name": "expectations",
|
|
5844
|
+
"required": false,
|
|
5845
|
+
"kind": "array",
|
|
5846
|
+
"description": "Replace this task's 'evals'-lens expectations (see studyCreateBody.tasks[].expectations). Omitted = expectations unchanged. REQUIRES the study's `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); validated the same way as create.",
|
|
5847
|
+
"placeholder": "[]"
|
|
2980
5848
|
}
|
|
2981
5849
|
],
|
|
2982
|
-
"hasBody": false,
|
|
2983
|
-
"bodyRequired": false,
|
|
2984
|
-
"isStream": false,
|
|
2985
5850
|
"watchable": false,
|
|
2986
5851
|
"help": {
|
|
2987
|
-
"command": "ora
|
|
5852
|
+
"command": "ora study update-task 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f 8a9b0c1d-2e3f-4a5b-8c7d-6e5f4a3b2c1d --file run.json",
|
|
5853
|
+
"body": "{\n \"prompt\": \"Research the three largest EU battery-storage tenders announced this quarter and cite each source.\"\n}"
|
|
2988
5854
|
}
|
|
2989
5855
|
},
|
|
2990
5856
|
{
|
|
@@ -3224,6 +6090,66 @@ export const OPERATIONS = [
|
|
|
3224
6090
|
"command": "ora webmcp arena create --file run.json --watch",
|
|
3225
6091
|
"body": "{\n \"targetUrl\": \"https://shop.example.com\",\n \"intent\": \"Find the price of the Chronometer and add it to the cart.\",\n \"harness\": \"claude-code\",\n \"model\": \"claude-haiku-4-5-20251001\",\n \"baseline\": \"none\",\n \"treatment\": \"webmcp-native\"\n}"
|
|
3226
6092
|
}
|
|
6093
|
+
},
|
|
6094
|
+
{
|
|
6095
|
+
"id": "webmcp.runs.browserRecording",
|
|
6096
|
+
"commandPath": [
|
|
6097
|
+
"webmcp",
|
|
6098
|
+
"runs",
|
|
6099
|
+
"browser-recording"
|
|
6100
|
+
],
|
|
6101
|
+
"method": "get",
|
|
6102
|
+
"pathTemplate": "/experiment/v1/webmcp/runs/{runId}/browser/recording",
|
|
6103
|
+
"summary": "The run's RECORDED browser screencast (time travel): frames stamped from the recording start, replayable and scrubbable after the run ends",
|
|
6104
|
+
"scope": "benchmarks:read",
|
|
6105
|
+
"product": "experiment",
|
|
6106
|
+
"pathParams": [
|
|
6107
|
+
{
|
|
6108
|
+
"name": "runId"
|
|
6109
|
+
}
|
|
6110
|
+
],
|
|
6111
|
+
"queryParams": [
|
|
6112
|
+
{
|
|
6113
|
+
"name": "maxFrames",
|
|
6114
|
+
"flag": "max-frames",
|
|
6115
|
+
"optionKey": "maxFrames",
|
|
6116
|
+
"required": false,
|
|
6117
|
+
"kind": "number"
|
|
6118
|
+
}
|
|
6119
|
+
],
|
|
6120
|
+
"hasBody": false,
|
|
6121
|
+
"bodyRequired": false,
|
|
6122
|
+
"isStream": false,
|
|
6123
|
+
"watchable": false,
|
|
6124
|
+
"help": {
|
|
6125
|
+
"command": "ora webmcp runs browser-recording <runId>"
|
|
6126
|
+
}
|
|
6127
|
+
},
|
|
6128
|
+
{
|
|
6129
|
+
"id": "webmcp.runs.browserStream",
|
|
6130
|
+
"commandPath": [
|
|
6131
|
+
"webmcp",
|
|
6132
|
+
"runs",
|
|
6133
|
+
"browser-stream"
|
|
6134
|
+
],
|
|
6135
|
+
"method": "get",
|
|
6136
|
+
"pathTemplate": "/experiment/v1/webmcp/runs/{runId}/browser/stream",
|
|
6137
|
+
"summary": "Live browser view of an arena run (SSE JPEG frames relayed from the run pod's driven browser)",
|
|
6138
|
+
"scope": "benchmarks:read",
|
|
6139
|
+
"product": "experiment",
|
|
6140
|
+
"pathParams": [
|
|
6141
|
+
{
|
|
6142
|
+
"name": "runId"
|
|
6143
|
+
}
|
|
6144
|
+
],
|
|
6145
|
+
"queryParams": [],
|
|
6146
|
+
"hasBody": false,
|
|
6147
|
+
"bodyRequired": false,
|
|
6148
|
+
"isStream": true,
|
|
6149
|
+
"watchable": false,
|
|
6150
|
+
"help": {
|
|
6151
|
+
"command": "ora webmcp runs browser-stream <runId>"
|
|
6152
|
+
}
|
|
3227
6153
|
}
|
|
3228
6154
|
];
|
|
3229
6155
|
//# sourceMappingURL=operations.js.map
|