@ora-ai/cli 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -1
- package/dist/auth-commands.d.ts.map +1 -1
- package/dist/auth-commands.js +12 -1
- package/dist/auth-commands.js.map +1 -1
- package/dist/core/auth.d.ts +24 -13
- package/dist/core/auth.d.ts.map +1 -1
- package/dist/core/auth.js +26 -15
- package/dist/core/auth.js.map +1 -1
- package/dist/core/credential-sources.d.ts +45 -0
- package/dist/core/credential-sources.d.ts.map +1 -0
- package/dist/core/credential-sources.js +65 -0
- package/dist/core/credential-sources.js.map +1 -0
- package/dist/core/github-actions-sign-in.d.ts +33 -0
- package/dist/core/github-actions-sign-in.d.ts.map +1 -0
- package/dist/core/github-actions-sign-in.js +54 -0
- package/dist/core/github-actions-sign-in.js.map +1 -0
- package/dist/core/output.js +14 -1
- package/dist/core/output.js.map +1 -1
- package/dist/core/run-operation.d.ts +11 -4
- package/dist/core/run-operation.d.ts.map +1 -1
- package/dist/core/run-operation.js +14 -9
- package/dist/core/run-operation.js.map +1 -1
- package/dist/core/sign-in.d.ts +34 -0
- package/dist/core/sign-in.d.ts.map +1 -0
- package/dist/core/sign-in.js +21 -0
- package/dist/core/sign-in.js.map +1 -0
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +1274 -73
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +1445 -10
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/dist/mcp/mcp-call.d.ts +25 -0
- package/dist/mcp/mcp-call.d.ts.map +1 -0
- package/dist/mcp/mcp-call.js +44 -0
- package/dist/mcp/mcp-call.js.map +1 -0
- package/dist/mcp/mcp-command.d.ts +26 -0
- package/dist/mcp/mcp-command.d.ts.map +1 -0
- package/dist/mcp/mcp-command.js +91 -0
- package/dist/mcp/mcp-command.js.map +1 -0
- package/dist/mcp/mcp-install.d.ts +50 -0
- package/dist/mcp/mcp-install.d.ts.map +1 -0
- package/dist/mcp/mcp-install.js +71 -0
- package/dist/mcp/mcp-install.js.map +1 -0
- package/dist/mcp/mcp-protocol.d.ts +69 -0
- package/dist/mcp/mcp-protocol.d.ts.map +1 -0
- package/dist/mcp/mcp-protocol.js +96 -0
- package/dist/mcp/mcp-protocol.js.map +1 -0
- package/dist/mcp/mcp-stdio.d.ts +17 -0
- package/dist/mcp/mcp-stdio.d.ts.map +1 -0
- package/dist/mcp/mcp-stdio.js +39 -0
- package/dist/mcp/mcp-stdio.js.map +1 -0
- package/dist/mcp/mcp-tools.d.ts +81 -0
- package/dist/mcp/mcp-tools.d.ts.map +1 -0
- package/dist/mcp/mcp-tools.js +134 -0
- package/dist/mcp/mcp-tools.js.map +1 -0
- package/dist/program.d.ts +4 -9
- package/dist/program.d.ts.map +1 -1
- package/dist/program.js +9 -3
- package/dist/program.js.map +1 -1
- package/dist/render/index.js +6 -0
- package/dist/render/index.js.map +1 -1
- package/dist/render/study/counting-views.d.ts +46 -0
- package/dist/render/study/counting-views.d.ts.map +1 -0
- package/dist/render/study/counting-views.js +49 -0
- package/dist/render/study/counting-views.js.map +1 -0
- package/dist/render/study/gate-views.d.ts +56 -0
- package/dist/render/study/gate-views.d.ts.map +1 -0
- package/dist/render/study/gate-views.js +64 -0
- package/dist/render/study/gate-views.js.map +1 -0
- package/dist/render/study/renderers.d.ts +8 -0
- package/dist/render/study/renderers.d.ts.map +1 -0
- package/dist/render/study/renderers.js +31 -0
- package/dist/render/study/renderers.js.map +1 -0
- package/dist/render/study/series-views.d.ts +65 -0
- package/dist/render/study/series-views.d.ts.map +1 -0
- package/dist/render/study/series-views.js +116 -0
- package/dist/render/study/series-views.js.map +1 -0
- package/dist/render/study/study-format.d.ts +24 -0
- package/dist/render/study/study-format.d.ts.map +1 -0
- package/dist/render/study/study-format.js +42 -0
- package/dist/render/study/study-format.js.map +1 -0
- package/dist/study-ci/ci-command.d.ts +40 -0
- package/dist/study-ci/ci-command.d.ts.map +1 -0
- package/dist/study-ci/ci-command.js +151 -0
- package/dist/study-ci/ci-command.js.map +1 -0
- package/dist/study-ci/ci-rehearse-command.d.ts +11 -0
- package/dist/study-ci/ci-rehearse-command.d.ts.map +1 -0
- package/dist/study-ci/ci-rehearse-command.js +83 -0
- package/dist/study-ci/ci-rehearse-command.js.map +1 -0
- package/dist/study-ci/ci-rehearse.d.ts +62 -0
- package/dist/study-ci/ci-rehearse.d.ts.map +1 -0
- package/dist/study-ci/ci-rehearse.js +49 -0
- package/dist/study-ci/ci-rehearse.js.map +1 -0
- package/dist/study-ci/ci-run.d.ts +87 -0
- package/dist/study-ci/ci-run.d.ts.map +1 -0
- package/dist/study-ci/ci-run.js +131 -0
- package/dist/study-ci/ci-run.js.map +1 -0
- package/dist/study-ci/gate-report.d.ts +64 -0
- package/dist/study-ci/gate-report.d.ts.map +1 -0
- package/dist/study-ci/gate-report.js +80 -0
- package/dist/study-ci/gate-report.js.map +1 -0
- package/dist/study-ci/params.d.ts +8 -0
- package/dist/study-ci/params.d.ts.map +1 -0
- package/dist/study-ci/params.js +72 -0
- package/dist/study-ci/params.js.map +1 -0
- package/package.json +1 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// GENERATED by scripts/generate.mjs — DO NOT EDIT.
|
|
2
|
-
// Source: openapi/ora-public.openapi.json (sha256
|
|
2
|
+
// Source: openapi/ora-public.openapi.json (sha256 fe3e7f533202e4ed7f17422627f7d89bb2fe15cf3b2e20c064df03ed0cc6113d)
|
|
3
3
|
// Regenerate: yarn nx run ora-cli:generate
|
|
4
4
|
export const SPEC = {
|
|
5
5
|
"version": "1.0.0",
|
|
6
|
-
"contentHash": "
|
|
6
|
+
"contentHash": "fe3e7f533202e4ed7f17422627f7d89bb2fe15cf3b2e20c064df03ed0cc6113d",
|
|
7
7
|
"server": "https://api.ora.ai",
|
|
8
8
|
"bin": "ora"
|
|
9
9
|
};
|
|
@@ -6750,6 +6750,31 @@ export const OPERATIONS = [
|
|
|
6750
6750
|
"command": "ora study addons list"
|
|
6751
6751
|
}
|
|
6752
6752
|
},
|
|
6753
|
+
{
|
|
6754
|
+
"id": "study.behaviors",
|
|
6755
|
+
"commandPath": [
|
|
6756
|
+
"study",
|
|
6757
|
+
"behaviors"
|
|
6758
|
+
],
|
|
6759
|
+
"method": "get",
|
|
6760
|
+
"pathTemplate": "/study/v1/studies/{id}/behaviors",
|
|
6761
|
+
"summary": "Read a study's custom behaviors: what each found, and how far its verdicts held up under review",
|
|
6762
|
+
"scope": "study.studies:read",
|
|
6763
|
+
"product": "study",
|
|
6764
|
+
"pathParams": [
|
|
6765
|
+
{
|
|
6766
|
+
"name": "id"
|
|
6767
|
+
}
|
|
6768
|
+
],
|
|
6769
|
+
"queryParams": [],
|
|
6770
|
+
"hasBody": false,
|
|
6771
|
+
"bodyRequired": false,
|
|
6772
|
+
"isStream": false,
|
|
6773
|
+
"watchable": false,
|
|
6774
|
+
"help": {
|
|
6775
|
+
"command": "ora study behaviors <id>"
|
|
6776
|
+
}
|
|
6777
|
+
},
|
|
6753
6778
|
{
|
|
6754
6779
|
"id": "study.benchmarkDefinitions.attachCorpus",
|
|
6755
6780
|
"commandPath": [
|
|
@@ -7619,6 +7644,31 @@ export const OPERATIONS = [
|
|
|
7619
7644
|
"command": "ora study benchmarkTerms get"
|
|
7620
7645
|
}
|
|
7621
7646
|
},
|
|
7647
|
+
{
|
|
7648
|
+
"id": "study.carryPlan",
|
|
7649
|
+
"commandPath": [
|
|
7650
|
+
"study",
|
|
7651
|
+
"carry-plan"
|
|
7652
|
+
],
|
|
7653
|
+
"method": "get",
|
|
7654
|
+
"pathTemplate": "/study/v1/studies/{id}/carry-plan",
|
|
7655
|
+
"summary": "What a series study would carry instead of run",
|
|
7656
|
+
"scope": "study.studies:read",
|
|
7657
|
+
"product": "study",
|
|
7658
|
+
"pathParams": [
|
|
7659
|
+
{
|
|
7660
|
+
"name": "id"
|
|
7661
|
+
}
|
|
7662
|
+
],
|
|
7663
|
+
"queryParams": [],
|
|
7664
|
+
"hasBody": false,
|
|
7665
|
+
"bodyRequired": false,
|
|
7666
|
+
"isStream": false,
|
|
7667
|
+
"watchable": false,
|
|
7668
|
+
"help": {
|
|
7669
|
+
"command": "ora study carry-plan <id>"
|
|
7670
|
+
}
|
|
7671
|
+
},
|
|
7622
7672
|
{
|
|
7623
7673
|
"id": "study.catalog",
|
|
7624
7674
|
"commandPath": [
|
|
@@ -7899,6 +7949,47 @@ export const OPERATIONS = [
|
|
|
7899
7949
|
"description": "LAUNCH PACING — start this study's runs gradually instead of all at once, for groups that call a rate-limited service (an API that answers 429 past N requests a minute). Every group's run of one agent on one task sample (a comparison unit) always starts together, in the same wave, so a comparison is never confounded by time; each value must therefore be at least the number of groups that run one agent. Declare maxInFlight, startsPerMinute, or both (both then hold). Frozen at create. Omit it, or send null, for no pacing: every run starts at launch.",
|
|
7900
7950
|
"placeholder": "<pacing>"
|
|
7901
7951
|
},
|
|
7952
|
+
{
|
|
7953
|
+
"name": "screens",
|
|
7954
|
+
"required": false,
|
|
7955
|
+
"kind": "array",
|
|
7956
|
+
"description": "Which run screens this study runs. A screen is a per-run check of whether the run is valid evidence, kept apart from grading: it never changes a grade. Omitted = the deployment's default. Frozen at launch, like lenses. 'test_awareness': after a run ends, a judge panel reads its full trace for signs that the agent noticed it was being tested ('aware') or changed how it worked because of that ('influenced'), and records the quoted evidence with the classifier's version.",
|
|
7957
|
+
"placeholder": "[]"
|
|
7958
|
+
},
|
|
7959
|
+
{
|
|
7960
|
+
"name": "series",
|
|
7961
|
+
"required": false,
|
|
7962
|
+
"kind": "string",
|
|
7963
|
+
"description": "The SERIES this study belongs to: studies that ask the same question at different moments (every commit of a branch). A gate's series baseline is found in it. Immutable. Omit for a study of no series.",
|
|
7964
|
+
"placeholder": "<series>"
|
|
7965
|
+
},
|
|
7966
|
+
{
|
|
7967
|
+
"name": "ref",
|
|
7968
|
+
"required": false,
|
|
7969
|
+
"kind": "object",
|
|
7970
|
+
"description": "Where the study was taken: the commit, branch and pull request. A label, immutable.",
|
|
7971
|
+
"placeholder": "{}"
|
|
7972
|
+
},
|
|
7973
|
+
{
|
|
7974
|
+
"name": "carry",
|
|
7975
|
+
"required": false,
|
|
7976
|
+
"kind": "object",
|
|
7977
|
+
"description": "Carry the runs of jobs that cannot have changed from the series' previous comparable study instead of running them again (needs `series`). Each carried run says so. Omit to run every job.",
|
|
7978
|
+
"placeholder": "{}"
|
|
7979
|
+
},
|
|
7980
|
+
{
|
|
7981
|
+
"name": "runCounting",
|
|
7982
|
+
"required": false,
|
|
7983
|
+
"kind": "string",
|
|
7984
|
+
"placeholder": "<runCounting>"
|
|
7985
|
+
},
|
|
7986
|
+
{
|
|
7987
|
+
"name": "behaviors",
|
|
7988
|
+
"required": false,
|
|
7989
|
+
"kind": "array",
|
|
7990
|
+
"description": "Custom behaviors: up to 5 yes-or-no questions a judge panel answers about every finished run, with quoted evidence. A behavior is disclosure about a run, like a screen: it has no score and never changes a grade. Frozen at launch. Each costs one panel read per run.",
|
|
7991
|
+
"placeholder": "[]"
|
|
7992
|
+
},
|
|
7902
7993
|
{
|
|
7903
7994
|
"name": "matrix",
|
|
7904
7995
|
"required": true,
|
|
@@ -7928,7 +8019,7 @@ export const OPERATIONS = [
|
|
|
7928
8019
|
],
|
|
7929
8020
|
"method": "delete",
|
|
7930
8021
|
"pathTemplate": "/study/v1/studies/{id}",
|
|
7931
|
-
"summary": "
|
|
8022
|
+
"summary": "Archive a study (409 while it is generating, preparing or running). Nothing is removed; restore brings it back. The study leaves the default list (list with archived=only to see it), becomes read-only (every change answers 409 study_archived until it is restored), and is unpublished first if it was shared outside the organisation. Archiving an archived study succeeds and changes nothing.",
|
|
7932
8023
|
"scope": "study.studies:write",
|
|
7933
8024
|
"product": "study",
|
|
7934
8025
|
"pathParams": [
|
|
@@ -7945,6 +8036,129 @@ export const OPERATIONS = [
|
|
|
7945
8036
|
"command": "ora study delete <id>"
|
|
7946
8037
|
}
|
|
7947
8038
|
},
|
|
8039
|
+
{
|
|
8040
|
+
"id": "study.deleteView",
|
|
8041
|
+
"commandPath": [
|
|
8042
|
+
"study",
|
|
8043
|
+
"delete-view"
|
|
8044
|
+
],
|
|
8045
|
+
"method": "delete",
|
|
8046
|
+
"pathTemplate": "/study/v1/studies/{id}/views/{viewId}",
|
|
8047
|
+
"summary": "Delete a saved view",
|
|
8048
|
+
"scope": "study.studies:write",
|
|
8049
|
+
"product": "study",
|
|
8050
|
+
"pathParams": [
|
|
8051
|
+
{
|
|
8052
|
+
"name": "id"
|
|
8053
|
+
},
|
|
8054
|
+
{
|
|
8055
|
+
"name": "viewId"
|
|
8056
|
+
}
|
|
8057
|
+
],
|
|
8058
|
+
"queryParams": [],
|
|
8059
|
+
"hasBody": false,
|
|
8060
|
+
"bodyRequired": false,
|
|
8061
|
+
"isStream": false,
|
|
8062
|
+
"watchable": false,
|
|
8063
|
+
"help": {
|
|
8064
|
+
"command": "ora study delete-view <id> <viewId>"
|
|
8065
|
+
}
|
|
8066
|
+
},
|
|
8067
|
+
{
|
|
8068
|
+
"id": "study.gate",
|
|
8069
|
+
"commandPath": [
|
|
8070
|
+
"study",
|
|
8071
|
+
"gate"
|
|
8072
|
+
],
|
|
8073
|
+
"method": "post",
|
|
8074
|
+
"pathTemplate": "/study/v1/studies/{id}/gate",
|
|
8075
|
+
"summary": "Ask a study whether it passes a gate",
|
|
8076
|
+
"scope": "study.studies:read",
|
|
8077
|
+
"product": "study",
|
|
8078
|
+
"pathParams": [
|
|
8079
|
+
{
|
|
8080
|
+
"name": "id"
|
|
8081
|
+
}
|
|
8082
|
+
],
|
|
8083
|
+
"queryParams": [],
|
|
8084
|
+
"hasBody": true,
|
|
8085
|
+
"bodyRequired": true,
|
|
8086
|
+
"isStream": false,
|
|
8087
|
+
"bodyFields": [
|
|
8088
|
+
{
|
|
8089
|
+
"name": "metric",
|
|
8090
|
+
"required": false,
|
|
8091
|
+
"kind": "string",
|
|
8092
|
+
"enum": [
|
|
8093
|
+
"benchmark",
|
|
8094
|
+
"goal",
|
|
8095
|
+
"evals",
|
|
8096
|
+
"quality",
|
|
8097
|
+
"rubric",
|
|
8098
|
+
"completion"
|
|
8099
|
+
],
|
|
8100
|
+
"description": "The metric the gate reads. Omitted = the study's own correctness measure (benchmark, else goal, else evals…).",
|
|
8101
|
+
"placeholder": "benchmark"
|
|
8102
|
+
},
|
|
8103
|
+
{
|
|
8104
|
+
"name": "group",
|
|
8105
|
+
"required": false,
|
|
8106
|
+
"kind": "string",
|
|
8107
|
+
"description": "The group the gate judges, by key. Omitted = the study's only group, or its one non-reference group.",
|
|
8108
|
+
"placeholder": "<group>"
|
|
8109
|
+
},
|
|
8110
|
+
{
|
|
8111
|
+
"name": "floor",
|
|
8112
|
+
"required": false,
|
|
8113
|
+
"kind": "string",
|
|
8114
|
+
"description": "Pass when the metric's interval lies wholly at or above this. `{ rate: 0.8 }` for a rate, `{ points: 70 }` for a 0-100 score.",
|
|
8115
|
+
"placeholder": "<floor>"
|
|
8116
|
+
},
|
|
8117
|
+
{
|
|
8118
|
+
"name": "maxDrop",
|
|
8119
|
+
"required": false,
|
|
8120
|
+
"kind": "string",
|
|
8121
|
+
"description": "Fail when the metric fell against the baseline by more than this, for certain. `{ pp: 5 }` for a rate, `{ points: 5 }` for a score. Needs `baseline`.",
|
|
8122
|
+
"placeholder": "<maxDrop>"
|
|
8123
|
+
},
|
|
8124
|
+
{
|
|
8125
|
+
"name": "baseline",
|
|
8126
|
+
"required": false,
|
|
8127
|
+
"kind": "string",
|
|
8128
|
+
"placeholder": "<baseline>"
|
|
8129
|
+
},
|
|
8130
|
+
{
|
|
8131
|
+
"name": "minTasks",
|
|
8132
|
+
"required": false,
|
|
8133
|
+
"kind": "number",
|
|
8134
|
+
"description": "Fewer measured tasks than this is 'not enough runs to decide'.",
|
|
8135
|
+
"placeholder": "5"
|
|
8136
|
+
},
|
|
8137
|
+
{
|
|
8138
|
+
"name": "infraCeiling",
|
|
8139
|
+
"required": false,
|
|
8140
|
+
"kind": "number",
|
|
8141
|
+
"description": "The share of the group's runs ora's own infrastructure may have killed before the gate refuses to decide.",
|
|
8142
|
+
"placeholder": "0.2"
|
|
8143
|
+
},
|
|
8144
|
+
{
|
|
8145
|
+
"name": "counting",
|
|
8146
|
+
"required": false,
|
|
8147
|
+
"kind": "string",
|
|
8148
|
+
"enum": [
|
|
8149
|
+
"policy",
|
|
8150
|
+
"all"
|
|
8151
|
+
],
|
|
8152
|
+
"description": "Which runs the gate reads. 'policy' (the default): the runs the study counts, so a study that holds back test-aware runs is judged without them. 'all': every run, held back or not, on both this study and the baseline.",
|
|
8153
|
+
"placeholder": "policy"
|
|
8154
|
+
}
|
|
8155
|
+
],
|
|
8156
|
+
"watchable": false,
|
|
8157
|
+
"help": {
|
|
8158
|
+
"command": "ora study gate <id> --file run.json",
|
|
8159
|
+
"body": "{\n \"metric\": \"benchmark\",\n \"group\": \"<group>\",\n \"floor\": \"<floor>\"\n}"
|
|
8160
|
+
}
|
|
8161
|
+
},
|
|
7948
8162
|
{
|
|
7949
8163
|
"id": "study.generate",
|
|
7950
8164
|
"commandPath": [
|
|
@@ -8005,6 +8219,31 @@ export const OPERATIONS = [
|
|
|
8005
8219
|
"command": "ora study get 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
8006
8220
|
}
|
|
8007
8221
|
},
|
|
8222
|
+
{
|
|
8223
|
+
"id": "study.gradingRevisions",
|
|
8224
|
+
"commandPath": [
|
|
8225
|
+
"study",
|
|
8226
|
+
"grading-revisions"
|
|
8227
|
+
],
|
|
8228
|
+
"method": "get",
|
|
8229
|
+
"pathTemplate": "/study/v1/studies/{id}/grading-revisions",
|
|
8230
|
+
"summary": "List a study's grading revisions",
|
|
8231
|
+
"scope": "study.studies:read",
|
|
8232
|
+
"product": "study",
|
|
8233
|
+
"pathParams": [
|
|
8234
|
+
{
|
|
8235
|
+
"name": "id"
|
|
8236
|
+
}
|
|
8237
|
+
],
|
|
8238
|
+
"queryParams": [],
|
|
8239
|
+
"hasBody": false,
|
|
8240
|
+
"bodyRequired": false,
|
|
8241
|
+
"isStream": false,
|
|
8242
|
+
"watchable": false,
|
|
8243
|
+
"help": {
|
|
8244
|
+
"command": "ora study grading-revisions <id>"
|
|
8245
|
+
}
|
|
8246
|
+
},
|
|
8008
8247
|
{
|
|
8009
8248
|
"id": "study.images.builds.cancel",
|
|
8010
8249
|
"commandPath": [
|
|
@@ -8299,7 +8538,20 @@ export const OPERATIONS = [
|
|
|
8299
8538
|
"scope": "study.studies:read",
|
|
8300
8539
|
"product": "study",
|
|
8301
8540
|
"pathParams": [],
|
|
8302
|
-
"queryParams": [
|
|
8541
|
+
"queryParams": [
|
|
8542
|
+
{
|
|
8543
|
+
"name": "archived",
|
|
8544
|
+
"flag": "archived",
|
|
8545
|
+
"optionKey": "archived",
|
|
8546
|
+
"required": false,
|
|
8547
|
+
"kind": "string",
|
|
8548
|
+
"enum": [
|
|
8549
|
+
"exclude",
|
|
8550
|
+
"only",
|
|
8551
|
+
"include"
|
|
8552
|
+
]
|
|
8553
|
+
}
|
|
8554
|
+
],
|
|
8303
8555
|
"hasBody": false,
|
|
8304
8556
|
"bodyRequired": false,
|
|
8305
8557
|
"isStream": false,
|
|
@@ -8547,6 +8799,47 @@ export const OPERATIONS = [
|
|
|
8547
8799
|
"description": "LAUNCH PACING — start this study's runs gradually instead of all at once, for groups that call a rate-limited service (an API that answers 429 past N requests a minute). Every group's run of one agent on one task sample (a comparison unit) always starts together, in the same wave, so a comparison is never confounded by time; each value must therefore be at least the number of groups that run one agent. Declare maxInFlight, startsPerMinute, or both (both then hold). Frozen at create. Omit it, or send null, for no pacing: every run starts at launch.",
|
|
8548
8800
|
"placeholder": "<pacing>"
|
|
8549
8801
|
},
|
|
8802
|
+
{
|
|
8803
|
+
"name": "screens",
|
|
8804
|
+
"required": false,
|
|
8805
|
+
"kind": "array",
|
|
8806
|
+
"description": "Which run screens this study runs. A screen is a per-run check of whether the run is valid evidence, kept apart from grading: it never changes a grade. Omitted = the deployment's default. Frozen at launch, like lenses. 'test_awareness': after a run ends, a judge panel reads its full trace for signs that the agent noticed it was being tested ('aware') or changed how it worked because of that ('influenced'), and records the quoted evidence with the classifier's version.",
|
|
8807
|
+
"placeholder": "[]"
|
|
8808
|
+
},
|
|
8809
|
+
{
|
|
8810
|
+
"name": "series",
|
|
8811
|
+
"required": false,
|
|
8812
|
+
"kind": "string",
|
|
8813
|
+
"description": "The SERIES this study belongs to: studies that ask the same question at different moments (every commit of a branch). A gate's series baseline is found in it. Immutable. Omit for a study of no series.",
|
|
8814
|
+
"placeholder": "<series>"
|
|
8815
|
+
},
|
|
8816
|
+
{
|
|
8817
|
+
"name": "ref",
|
|
8818
|
+
"required": false,
|
|
8819
|
+
"kind": "object",
|
|
8820
|
+
"description": "Where the study was taken: the commit, branch and pull request. A label, immutable.",
|
|
8821
|
+
"placeholder": "{}"
|
|
8822
|
+
},
|
|
8823
|
+
{
|
|
8824
|
+
"name": "carry",
|
|
8825
|
+
"required": false,
|
|
8826
|
+
"kind": "object",
|
|
8827
|
+
"description": "Carry the runs of jobs that cannot have changed from the series' previous comparable study instead of running them again (needs `series`). Each carried run says so. Omit to run every job.",
|
|
8828
|
+
"placeholder": "{}"
|
|
8829
|
+
},
|
|
8830
|
+
{
|
|
8831
|
+
"name": "runCounting",
|
|
8832
|
+
"required": false,
|
|
8833
|
+
"kind": "string",
|
|
8834
|
+
"placeholder": "<runCounting>"
|
|
8835
|
+
},
|
|
8836
|
+
{
|
|
8837
|
+
"name": "behaviors",
|
|
8838
|
+
"required": false,
|
|
8839
|
+
"kind": "array",
|
|
8840
|
+
"description": "Custom behaviors: up to 5 yes-or-no questions a judge panel answers about every finished run, with quoted evidence. A behavior is disclosure about a run, like a screen: it has no score and never changes a grade. Frozen at launch. Each costs one panel read per run.",
|
|
8841
|
+
"placeholder": "[]"
|
|
8842
|
+
},
|
|
8550
8843
|
{
|
|
8551
8844
|
"name": "matrix",
|
|
8552
8845
|
"required": true,
|
|
@@ -8660,15 +8953,15 @@ export const OPERATIONS = [
|
|
|
8660
8953
|
}
|
|
8661
8954
|
},
|
|
8662
8955
|
{
|
|
8663
|
-
"id": "study.
|
|
8956
|
+
"id": "study.query",
|
|
8664
8957
|
"commandPath": [
|
|
8665
8958
|
"study",
|
|
8666
|
-
"
|
|
8959
|
+
"query"
|
|
8667
8960
|
],
|
|
8668
8961
|
"method": "post",
|
|
8669
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8670
|
-
"summary": "
|
|
8671
|
-
"scope": "study.studies:
|
|
8962
|
+
"pathTemplate": "/study/v1/studies/{id}/query",
|
|
8963
|
+
"summary": "Ask a question of a study's runs: which runs, split how, measured on what",
|
|
8964
|
+
"scope": "study.studies:read",
|
|
8672
8965
|
"product": "study",
|
|
8673
8966
|
"pathParams": [
|
|
8674
8967
|
{
|
|
@@ -8676,24 +8969,53 @@ export const OPERATIONS = [
|
|
|
8676
8969
|
}
|
|
8677
8970
|
],
|
|
8678
8971
|
"queryParams": [],
|
|
8679
|
-
"hasBody":
|
|
8680
|
-
"bodyRequired":
|
|
8972
|
+
"hasBody": true,
|
|
8973
|
+
"bodyRequired": true,
|
|
8681
8974
|
"isStream": false,
|
|
8975
|
+
"bodyFields": [
|
|
8976
|
+
{
|
|
8977
|
+
"name": "metrics",
|
|
8978
|
+
"required": true,
|
|
8979
|
+
"kind": "array",
|
|
8980
|
+
"description": "Metric ids. `GET …/query/vocabulary` lists them.",
|
|
8981
|
+
"placeholder": "[]"
|
|
8982
|
+
},
|
|
8983
|
+
{
|
|
8984
|
+
"name": "groupBy",
|
|
8985
|
+
"required": false,
|
|
8986
|
+
"kind": "array",
|
|
8987
|
+
"description": "Dimension ids to split the runs by. None = one row over every matched run.",
|
|
8988
|
+
"placeholder": "[]"
|
|
8989
|
+
},
|
|
8990
|
+
{
|
|
8991
|
+
"name": "filters",
|
|
8992
|
+
"required": false,
|
|
8993
|
+
"kind": "array",
|
|
8994
|
+
"placeholder": "[]"
|
|
8995
|
+
},
|
|
8996
|
+
{
|
|
8997
|
+
"name": "limit",
|
|
8998
|
+
"required": false,
|
|
8999
|
+
"kind": "number",
|
|
9000
|
+
"placeholder": "200"
|
|
9001
|
+
}
|
|
9002
|
+
],
|
|
8682
9003
|
"watchable": false,
|
|
8683
9004
|
"help": {
|
|
8684
|
-
"command": "ora study
|
|
9005
|
+
"command": "ora study query <id> --file run.json",
|
|
9006
|
+
"body": "{\n \"metrics\": []\n}"
|
|
8685
9007
|
}
|
|
8686
9008
|
},
|
|
8687
9009
|
{
|
|
8688
|
-
"id": "study.
|
|
9010
|
+
"id": "study.queryVocabulary",
|
|
8689
9011
|
"commandPath": [
|
|
8690
9012
|
"study",
|
|
8691
|
-
"
|
|
9013
|
+
"query-vocabulary"
|
|
8692
9014
|
],
|
|
8693
|
-
"method": "
|
|
8694
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8695
|
-
"summary": "
|
|
8696
|
-
"scope": "study.studies:
|
|
9015
|
+
"method": "get",
|
|
9016
|
+
"pathTemplate": "/study/v1/studies/{id}/query/vocabulary",
|
|
9017
|
+
"summary": "List what a study's runs can be split by and measured on",
|
|
9018
|
+
"scope": "study.studies:read",
|
|
8697
9019
|
"product": "study",
|
|
8698
9020
|
"pathParams": [
|
|
8699
9021
|
{
|
|
@@ -8706,17 +9028,67 @@ export const OPERATIONS = [
|
|
|
8706
9028
|
"isStream": false,
|
|
8707
9029
|
"watchable": false,
|
|
8708
9030
|
"help": {
|
|
8709
|
-
"command": "ora study
|
|
9031
|
+
"command": "ora study query-vocabulary <id>"
|
|
8710
9032
|
}
|
|
8711
9033
|
},
|
|
8712
9034
|
{
|
|
8713
|
-
"id": "study.
|
|
9035
|
+
"id": "study.reconcileRuns",
|
|
8714
9036
|
"commandPath": [
|
|
8715
9037
|
"study",
|
|
8716
|
-
"
|
|
9038
|
+
"reconcile-runs"
|
|
8717
9039
|
],
|
|
8718
9040
|
"method": "post",
|
|
8719
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
9041
|
+
"pathTemplate": "/study/v1/studies/{id}/reconcile-runs",
|
|
9042
|
+
"summary": "Re-derive a study's run projection from the authoritative run rows and re-probe completion (platform-admin only; idempotent) — the repair for runs left non-terminal in the projection while the spine says they finished",
|
|
9043
|
+
"scope": "study.studies:write",
|
|
9044
|
+
"product": "study",
|
|
9045
|
+
"pathParams": [
|
|
9046
|
+
{
|
|
9047
|
+
"name": "id"
|
|
9048
|
+
}
|
|
9049
|
+
],
|
|
9050
|
+
"queryParams": [],
|
|
9051
|
+
"hasBody": false,
|
|
9052
|
+
"bodyRequired": false,
|
|
9053
|
+
"isStream": false,
|
|
9054
|
+
"watchable": false,
|
|
9055
|
+
"help": {
|
|
9056
|
+
"command": "ora study reconcile-runs 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9057
|
+
}
|
|
9058
|
+
},
|
|
9059
|
+
{
|
|
9060
|
+
"id": "study.regenerateSummary",
|
|
9061
|
+
"commandPath": [
|
|
9062
|
+
"study",
|
|
9063
|
+
"regenerate-summary"
|
|
9064
|
+
],
|
|
9065
|
+
"method": "post",
|
|
9066
|
+
"pathTemplate": "/study/v1/studies/{id}/exec-summary",
|
|
9067
|
+
"summary": "Regenerate the executive summary from the current aggregate (async; poll get). Where the product keeps review drafts, a PUBLISHED study's lands as `executiveSummaryDraft` and its public summary does not change until a re-publish makes the draft public — `target` in the answer says where it lands",
|
|
9068
|
+
"scope": "study.studies:write",
|
|
9069
|
+
"product": "study",
|
|
9070
|
+
"pathParams": [
|
|
9071
|
+
{
|
|
9072
|
+
"name": "id"
|
|
9073
|
+
}
|
|
9074
|
+
],
|
|
9075
|
+
"queryParams": [],
|
|
9076
|
+
"hasBody": false,
|
|
9077
|
+
"bodyRequired": false,
|
|
9078
|
+
"isStream": false,
|
|
9079
|
+
"watchable": false,
|
|
9080
|
+
"help": {
|
|
9081
|
+
"command": "ora study regenerate-summary 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9082
|
+
}
|
|
9083
|
+
},
|
|
9084
|
+
{
|
|
9085
|
+
"id": "study.regrade",
|
|
9086
|
+
"commandPath": [
|
|
9087
|
+
"study",
|
|
9088
|
+
"regrade"
|
|
9089
|
+
],
|
|
9090
|
+
"method": "post",
|
|
9091
|
+
"pathTemplate": "/study/v1/studies/{id}/regrade",
|
|
8720
9092
|
"summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
|
|
8721
9093
|
"scope": "study.studies:write",
|
|
8722
9094
|
"product": "study",
|
|
@@ -8735,19 +9107,616 @@ export const OPERATIONS = [
|
|
|
8735
9107
|
}
|
|
8736
9108
|
},
|
|
8737
9109
|
{
|
|
8738
|
-
"id": "study.
|
|
9110
|
+
"id": "study.rehearsalReport",
|
|
9111
|
+
"commandPath": [
|
|
9112
|
+
"study",
|
|
9113
|
+
"rehearsal-report"
|
|
9114
|
+
],
|
|
9115
|
+
"method": "get",
|
|
9116
|
+
"pathTemplate": "/study/v1/studies/{id}/rehearsal",
|
|
9117
|
+
"summary": "Read what a rehearsal proved",
|
|
9118
|
+
"scope": "study.studies:read",
|
|
9119
|
+
"product": "study",
|
|
9120
|
+
"pathParams": [
|
|
9121
|
+
{
|
|
9122
|
+
"name": "id"
|
|
9123
|
+
}
|
|
9124
|
+
],
|
|
9125
|
+
"queryParams": [],
|
|
9126
|
+
"hasBody": false,
|
|
9127
|
+
"bodyRequired": false,
|
|
9128
|
+
"isStream": false,
|
|
9129
|
+
"watchable": false,
|
|
9130
|
+
"help": {
|
|
9131
|
+
"command": "ora study rehearsal-report <id>"
|
|
9132
|
+
}
|
|
9133
|
+
},
|
|
9134
|
+
{
|
|
9135
|
+
"id": "study.rehearse",
|
|
9136
|
+
"commandPath": [
|
|
9137
|
+
"study",
|
|
9138
|
+
"rehearse"
|
|
9139
|
+
],
|
|
9140
|
+
"method": "post",
|
|
9141
|
+
"pathTemplate": "/study/v1/studies/rehearsals",
|
|
9142
|
+
"summary": "Rehearse a study design: one real run per group, before the study itself",
|
|
9143
|
+
"scope": "study.studies:write",
|
|
9144
|
+
"product": "study",
|
|
9145
|
+
"pathParams": [],
|
|
9146
|
+
"queryParams": [
|
|
9147
|
+
{
|
|
9148
|
+
"name": "tasks",
|
|
9149
|
+
"flag": "tasks",
|
|
9150
|
+
"optionKey": "tasks",
|
|
9151
|
+
"required": false,
|
|
9152
|
+
"kind": "string",
|
|
9153
|
+
"enum": [
|
|
9154
|
+
"1",
|
|
9155
|
+
"2",
|
|
9156
|
+
"3"
|
|
9157
|
+
]
|
|
9158
|
+
},
|
|
9159
|
+
{
|
|
9160
|
+
"name": "mode",
|
|
9161
|
+
"flag": "mode",
|
|
9162
|
+
"optionKey": "mode",
|
|
9163
|
+
"required": false,
|
|
9164
|
+
"kind": "string",
|
|
9165
|
+
"enum": [
|
|
9166
|
+
"live",
|
|
9167
|
+
"setup",
|
|
9168
|
+
"mock"
|
|
9169
|
+
]
|
|
9170
|
+
}
|
|
9171
|
+
],
|
|
9172
|
+
"hasBody": true,
|
|
9173
|
+
"bodyRequired": true,
|
|
9174
|
+
"isStream": false,
|
|
9175
|
+
"bodyFields": [
|
|
9176
|
+
{
|
|
9177
|
+
"name": "name",
|
|
9178
|
+
"required": true,
|
|
9179
|
+
"kind": "string",
|
|
9180
|
+
"description": "Display name for the study (≤160 chars).",
|
|
9181
|
+
"placeholder": "<name>"
|
|
9182
|
+
},
|
|
9183
|
+
{
|
|
9184
|
+
"name": "description",
|
|
9185
|
+
"required": false,
|
|
9186
|
+
"kind": "string",
|
|
9187
|
+
"description": "Optional longer description shown on the study card.",
|
|
9188
|
+
"placeholder": "<description>"
|
|
9189
|
+
},
|
|
9190
|
+
{
|
|
9191
|
+
"name": "subjectKind",
|
|
9192
|
+
"required": false,
|
|
9193
|
+
"kind": "string",
|
|
9194
|
+
"enum": [
|
|
9195
|
+
"skill",
|
|
9196
|
+
"tools",
|
|
9197
|
+
"addon",
|
|
9198
|
+
"custom",
|
|
9199
|
+
"benchmark"
|
|
9200
|
+
],
|
|
9201
|
+
"description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
|
|
9202
|
+
"placeholder": "custom"
|
|
9203
|
+
},
|
|
9204
|
+
{
|
|
9205
|
+
"name": "skillRef",
|
|
9206
|
+
"required": false,
|
|
9207
|
+
"kind": "string",
|
|
9208
|
+
"description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9209
|
+
"placeholder": "<skillRef>"
|
|
9210
|
+
},
|
|
9211
|
+
{
|
|
9212
|
+
"name": "definitionVersionId",
|
|
9213
|
+
"required": false,
|
|
9214
|
+
"kind": "string",
|
|
9215
|
+
"description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
|
|
9216
|
+
"placeholder": "<definitionVersionId>"
|
|
9217
|
+
},
|
|
9218
|
+
{
|
|
9219
|
+
"name": "subjectName",
|
|
9220
|
+
"required": false,
|
|
9221
|
+
"kind": "string",
|
|
9222
|
+
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
|
|
9223
|
+
"placeholder": "<subjectName>"
|
|
9224
|
+
},
|
|
9225
|
+
{
|
|
9226
|
+
"name": "armConfig",
|
|
9227
|
+
"required": false,
|
|
9228
|
+
"kind": "object",
|
|
9229
|
+
"description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9230
|
+
"placeholder": "{}"
|
|
9231
|
+
},
|
|
9232
|
+
{
|
|
9233
|
+
"name": "lenses",
|
|
9234
|
+
"required": false,
|
|
9235
|
+
"kind": "array",
|
|
9236
|
+
"description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
|
|
9237
|
+
"placeholder": "[\"judge\"]"
|
|
9238
|
+
},
|
|
9239
|
+
{
|
|
9240
|
+
"name": "judgeSupplementary",
|
|
9241
|
+
"required": false,
|
|
9242
|
+
"kind": "boolean",
|
|
9243
|
+
"description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
|
|
9244
|
+
"placeholder": "true"
|
|
9245
|
+
},
|
|
9246
|
+
{
|
|
9247
|
+
"name": "proposeExpectations",
|
|
9248
|
+
"required": false,
|
|
9249
|
+
"kind": "boolean",
|
|
9250
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
|
|
9251
|
+
"placeholder": "true"
|
|
9252
|
+
},
|
|
9253
|
+
{
|
|
9254
|
+
"name": "comparisonMode",
|
|
9255
|
+
"required": false,
|
|
9256
|
+
"kind": "string",
|
|
9257
|
+
"enum": [
|
|
9258
|
+
"ab_vs_none",
|
|
9259
|
+
"ab_custom",
|
|
9260
|
+
"single",
|
|
9261
|
+
"multi_arm"
|
|
9262
|
+
],
|
|
9263
|
+
"description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
|
|
9264
|
+
"placeholder": "multi_arm"
|
|
9265
|
+
},
|
|
9266
|
+
{
|
|
9267
|
+
"name": "subjectBackend",
|
|
9268
|
+
"required": false,
|
|
9269
|
+
"kind": "string",
|
|
9270
|
+
"enum": [
|
|
9271
|
+
"model",
|
|
9272
|
+
"harness"
|
|
9273
|
+
],
|
|
9274
|
+
"description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
|
|
9275
|
+
"placeholder": "model"
|
|
9276
|
+
},
|
|
9277
|
+
{
|
|
9278
|
+
"name": "baselineLabel",
|
|
9279
|
+
"required": false,
|
|
9280
|
+
"kind": "string",
|
|
9281
|
+
"description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9282
|
+
"placeholder": "<baselineLabel>"
|
|
9283
|
+
},
|
|
9284
|
+
{
|
|
9285
|
+
"name": "treatmentLabel",
|
|
9286
|
+
"required": false,
|
|
9287
|
+
"kind": "string",
|
|
9288
|
+
"description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9289
|
+
"placeholder": "<treatmentLabel>"
|
|
9290
|
+
},
|
|
9291
|
+
{
|
|
9292
|
+
"name": "arms",
|
|
9293
|
+
"required": false,
|
|
9294
|
+
"kind": "array",
|
|
9295
|
+
"description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
|
|
9296
|
+
"placeholder": "[]"
|
|
9297
|
+
},
|
|
9298
|
+
{
|
|
9299
|
+
"name": "contrasts",
|
|
9300
|
+
"required": false,
|
|
9301
|
+
"kind": "string",
|
|
9302
|
+
"enum": [
|
|
9303
|
+
"vs_reference",
|
|
9304
|
+
"all_pairs"
|
|
9305
|
+
],
|
|
9306
|
+
"description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
|
|
9307
|
+
"placeholder": "vs_reference"
|
|
9308
|
+
},
|
|
9309
|
+
{
|
|
9310
|
+
"name": "modeOptions",
|
|
9311
|
+
"required": false,
|
|
9312
|
+
"kind": "object",
|
|
9313
|
+
"description": "A comparisonMode's private inputs, validated by that mode:",
|
|
9314
|
+
"placeholder": "{}"
|
|
9315
|
+
},
|
|
9316
|
+
{
|
|
9317
|
+
"name": "taskCount",
|
|
9318
|
+
"required": false,
|
|
9319
|
+
"kind": "number",
|
|
9320
|
+
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
|
|
9321
|
+
"placeholder": "5"
|
|
9322
|
+
},
|
|
9323
|
+
{
|
|
9324
|
+
"name": "variantsPerTask",
|
|
9325
|
+
"required": false,
|
|
9326
|
+
"kind": "number",
|
|
9327
|
+
"description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
|
|
9328
|
+
"placeholder": "1"
|
|
9329
|
+
},
|
|
9330
|
+
{
|
|
9331
|
+
"name": "repeats",
|
|
9332
|
+
"required": false,
|
|
9333
|
+
"kind": "number",
|
|
9334
|
+
"description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
|
|
9335
|
+
"placeholder": "1"
|
|
9336
|
+
},
|
|
9337
|
+
{
|
|
9338
|
+
"name": "tasks",
|
|
9339
|
+
"required": false,
|
|
9340
|
+
"kind": "array",
|
|
9341
|
+
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
|
|
9342
|
+
"placeholder": "[]"
|
|
9343
|
+
},
|
|
9344
|
+
{
|
|
9345
|
+
"name": "subset",
|
|
9346
|
+
"required": false,
|
|
9347
|
+
"kind": "object",
|
|
9348
|
+
"description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
|
|
9349
|
+
"placeholder": "{}"
|
|
9350
|
+
},
|
|
9351
|
+
{
|
|
9352
|
+
"name": "pacing",
|
|
9353
|
+
"required": false,
|
|
9354
|
+
"kind": "string",
|
|
9355
|
+
"description": "LAUNCH PACING — start this study's runs gradually instead of all at once, for groups that call a rate-limited service (an API that answers 429 past N requests a minute). Every group's run of one agent on one task sample (a comparison unit) always starts together, in the same wave, so a comparison is never confounded by time; each value must therefore be at least the number of groups that run one agent. Declare maxInFlight, startsPerMinute, or both (both then hold). Frozen at create. Omit it, or send null, for no pacing: every run starts at launch.",
|
|
9356
|
+
"placeholder": "<pacing>"
|
|
9357
|
+
},
|
|
9358
|
+
{
|
|
9359
|
+
"name": "screens",
|
|
9360
|
+
"required": false,
|
|
9361
|
+
"kind": "array",
|
|
9362
|
+
"description": "Which run screens this study runs. A screen is a per-run check of whether the run is valid evidence, kept apart from grading: it never changes a grade. Omitted = the deployment's default. Frozen at launch, like lenses. 'test_awareness': after a run ends, a judge panel reads its full trace for signs that the agent noticed it was being tested ('aware') or changed how it worked because of that ('influenced'), and records the quoted evidence with the classifier's version.",
|
|
9363
|
+
"placeholder": "[]"
|
|
9364
|
+
},
|
|
9365
|
+
{
|
|
9366
|
+
"name": "series",
|
|
9367
|
+
"required": false,
|
|
9368
|
+
"kind": "string",
|
|
9369
|
+
"description": "The SERIES this study belongs to: studies that ask the same question at different moments (every commit of a branch). A gate's series baseline is found in it. Immutable. Omit for a study of no series.",
|
|
9370
|
+
"placeholder": "<series>"
|
|
9371
|
+
},
|
|
9372
|
+
{
|
|
9373
|
+
"name": "ref",
|
|
9374
|
+
"required": false,
|
|
9375
|
+
"kind": "object",
|
|
9376
|
+
"description": "Where the study was taken: the commit, branch and pull request. A label, immutable.",
|
|
9377
|
+
"placeholder": "{}"
|
|
9378
|
+
},
|
|
9379
|
+
{
|
|
9380
|
+
"name": "carry",
|
|
9381
|
+
"required": false,
|
|
9382
|
+
"kind": "object",
|
|
9383
|
+
"description": "Carry the runs of jobs that cannot have changed from the series' previous comparable study instead of running them again (needs `series`). Each carried run says so. Omit to run every job.",
|
|
9384
|
+
"placeholder": "{}"
|
|
9385
|
+
},
|
|
9386
|
+
{
|
|
9387
|
+
"name": "runCounting",
|
|
9388
|
+
"required": false,
|
|
9389
|
+
"kind": "string",
|
|
9390
|
+
"placeholder": "<runCounting>"
|
|
9391
|
+
},
|
|
9392
|
+
{
|
|
9393
|
+
"name": "behaviors",
|
|
9394
|
+
"required": false,
|
|
9395
|
+
"kind": "array",
|
|
9396
|
+
"description": "Custom behaviors: up to 5 yes-or-no questions a judge panel answers about every finished run, with quoted evidence. A behavior is disclosure about a run, like a screen: it has no score and never changes a grade. Frozen at launch. Each costs one panel read per run.",
|
|
9397
|
+
"placeholder": "[]"
|
|
9398
|
+
},
|
|
9399
|
+
{
|
|
9400
|
+
"name": "matrix",
|
|
9401
|
+
"required": true,
|
|
9402
|
+
"kind": "array",
|
|
9403
|
+
"description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
|
|
9404
|
+
"placeholder": "[]"
|
|
9405
|
+
},
|
|
9406
|
+
{
|
|
9407
|
+
"name": "promise",
|
|
9408
|
+
"required": false,
|
|
9409
|
+
"kind": "string",
|
|
9410
|
+
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
|
|
9411
|
+
"placeholder": "<promise>"
|
|
9412
|
+
}
|
|
9413
|
+
],
|
|
9414
|
+
"watchable": false,
|
|
9415
|
+
"help": {
|
|
9416
|
+
"command": "ora study rehearse --file run.json",
|
|
9417
|
+
"body": "{\n \"name\": \"<name>\",\n \"matrix\": []\n}"
|
|
9418
|
+
}
|
|
9419
|
+
},
|
|
9420
|
+
{
|
|
9421
|
+
"id": "study.rerunAttempts",
|
|
9422
|
+
"commandPath": [
|
|
9423
|
+
"study",
|
|
9424
|
+
"rerun-attempts"
|
|
9425
|
+
],
|
|
9426
|
+
"method": "get",
|
|
9427
|
+
"pathTemplate": "/study/v1/studies/{id}/rerun-attempts",
|
|
9428
|
+
"summary": "The study's re-run ledger: every in-place re-dispatch of one of its jobs — the run it replaced and how that run ended, why it was re-run (with a declared re-run's evidence), the run it created and whether it dispatched. Oldest first.",
|
|
9429
|
+
"scope": "study.studies:read",
|
|
9430
|
+
"product": "study",
|
|
9431
|
+
"pathParams": [
|
|
9432
|
+
{
|
|
9433
|
+
"name": "id"
|
|
9434
|
+
}
|
|
9435
|
+
],
|
|
9436
|
+
"queryParams": [],
|
|
9437
|
+
"hasBody": false,
|
|
9438
|
+
"bodyRequired": false,
|
|
9439
|
+
"isStream": false,
|
|
9440
|
+
"watchable": false,
|
|
9441
|
+
"help": {
|
|
9442
|
+
"command": "ora study rerun-attempts 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9443
|
+
}
|
|
9444
|
+
},
|
|
9445
|
+
{
|
|
9446
|
+
"id": "study.rerunInfraFailures",
|
|
9447
|
+
"commandPath": [
|
|
9448
|
+
"study",
|
|
9449
|
+
"rerun-infra-failures"
|
|
9450
|
+
],
|
|
9451
|
+
"method": "post",
|
|
9452
|
+
"pathTemplate": "/study/v1/studies/{id}/rerun-infra-failures",
|
|
9453
|
+
"summary": "Re-run, in place, the jobs of a running or completed-but-unpublished study whose run ora's own infrastructure killed (or that the launch never created): each new run takes its job's place, the failed run stays in the record, at most 2 re-runs per job, never while the job's run is still running. A completed study re-opens until its re-runs finish and are graded. Synchronous; dryRun answers the plan.",
|
|
9454
|
+
"scope": "study.studies:write",
|
|
9455
|
+
"product": "study",
|
|
9456
|
+
"pathParams": [
|
|
9457
|
+
{
|
|
9458
|
+
"name": "id"
|
|
9459
|
+
}
|
|
9460
|
+
],
|
|
9461
|
+
"queryParams": [],
|
|
9462
|
+
"hasBody": true,
|
|
9463
|
+
"bodyRequired": true,
|
|
9464
|
+
"isStream": false,
|
|
9465
|
+
"bodyFields": [
|
|
9466
|
+
{
|
|
9467
|
+
"name": "dryRun",
|
|
9468
|
+
"required": false,
|
|
9469
|
+
"kind": "boolean",
|
|
9470
|
+
"description": "Answer what WOULD be re-run, dispatching and writing nothing (default false).",
|
|
9471
|
+
"placeholder": "true"
|
|
9472
|
+
},
|
|
9473
|
+
{
|
|
9474
|
+
"name": "limit",
|
|
9475
|
+
"required": false,
|
|
9476
|
+
"kind": "number",
|
|
9477
|
+
"description": "The most jobs this call dispatches (default 200, or every declared run); the rest are counted in `skipped.overLimit` for the next call.",
|
|
9478
|
+
"placeholder": "1"
|
|
9479
|
+
},
|
|
9480
|
+
{
|
|
9481
|
+
"name": "declared",
|
|
9482
|
+
"required": false,
|
|
9483
|
+
"kind": "array",
|
|
9484
|
+
"description": "The DECLARED re-run: exactly these runs are re-run in place (cause declared_infra), each with its evidence — runs ora's own predicate does not flag. Absent: ora's own infra failures and never-created jobs.",
|
|
9485
|
+
"placeholder": "[]"
|
|
9486
|
+
}
|
|
9487
|
+
],
|
|
9488
|
+
"watchable": false,
|
|
9489
|
+
"help": {
|
|
9490
|
+
"command": "ora study rerun-infra-failures 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
9491
|
+
"body": "{\n \"dryRun\": true\n}"
|
|
9492
|
+
}
|
|
9493
|
+
},
|
|
9494
|
+
{
|
|
9495
|
+
"id": "study.restore",
|
|
9496
|
+
"commandPath": [
|
|
9497
|
+
"study",
|
|
9498
|
+
"restore"
|
|
9499
|
+
],
|
|
9500
|
+
"method": "post",
|
|
9501
|
+
"pathTemplate": "/study/v1/studies/{id}/restore",
|
|
9502
|
+
"summary": "Restore an archived study: it returns to the default list and can be changed again. It comes back unpublished (archiving a shared study unpublishes it first); publishing again is a separate, deliberate call. Restoring a study that is not archived succeeds and changes nothing.",
|
|
9503
|
+
"scope": "study.studies:write",
|
|
9504
|
+
"product": "study",
|
|
9505
|
+
"pathParams": [
|
|
9506
|
+
{
|
|
9507
|
+
"name": "id"
|
|
9508
|
+
}
|
|
9509
|
+
],
|
|
9510
|
+
"queryParams": [],
|
|
9511
|
+
"hasBody": false,
|
|
9512
|
+
"bodyRequired": false,
|
|
9513
|
+
"isStream": false,
|
|
9514
|
+
"watchable": false,
|
|
9515
|
+
"help": {
|
|
9516
|
+
"command": "ora study restore 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9517
|
+
}
|
|
9518
|
+
},
|
|
9519
|
+
{
|
|
9520
|
+
"id": "study.retest",
|
|
9521
|
+
"commandPath": [
|
|
9522
|
+
"study",
|
|
9523
|
+
"retest"
|
|
9524
|
+
],
|
|
9525
|
+
"method": "post",
|
|
9526
|
+
"pathTemplate": "/study/v1/studies/{id}/retests",
|
|
9527
|
+
"summary": "Re-check runs' saved files with the benchmark's own checker",
|
|
9528
|
+
"scope": "study.studies:write",
|
|
9529
|
+
"product": "study",
|
|
9530
|
+
"pathParams": [
|
|
9531
|
+
{
|
|
9532
|
+
"name": "id"
|
|
9533
|
+
}
|
|
9534
|
+
],
|
|
9535
|
+
"queryParams": [],
|
|
9536
|
+
"hasBody": true,
|
|
9537
|
+
"bodyRequired": true,
|
|
9538
|
+
"isStream": false,
|
|
9539
|
+
"bodyFields": [
|
|
9540
|
+
{
|
|
9541
|
+
"name": "kind",
|
|
9542
|
+
"required": false,
|
|
9543
|
+
"kind": "string",
|
|
9544
|
+
"enum": [
|
|
9545
|
+
"fill",
|
|
9546
|
+
"audit"
|
|
9547
|
+
],
|
|
9548
|
+
"description": "fill (the default): re-check every run whose verifier verdict was never measured; a verdict the re-check measures becomes the run's grade. audit: re-check runs and record the second verdict beside the first; no grade changes.",
|
|
9549
|
+
"placeholder": "fill"
|
|
9550
|
+
},
|
|
9551
|
+
{
|
|
9552
|
+
"name": "runIds",
|
|
9553
|
+
"required": false,
|
|
9554
|
+
"kind": "array",
|
|
9555
|
+
"description": "audit only: the runs to re-check. Absent = every run with a measured verdict.",
|
|
9556
|
+
"placeholder": "[]"
|
|
9557
|
+
},
|
|
9558
|
+
{
|
|
9559
|
+
"name": "reason",
|
|
9560
|
+
"required": false,
|
|
9561
|
+
"kind": "string",
|
|
9562
|
+
"description": "Why the runs are being re-checked. Kept with the request.",
|
|
9563
|
+
"placeholder": "<reason>"
|
|
9564
|
+
},
|
|
9565
|
+
{
|
|
9566
|
+
"name": "dryRun",
|
|
9567
|
+
"required": false,
|
|
9568
|
+
"kind": "boolean",
|
|
9569
|
+
"description": "true = create nothing: answer which runs would be re-checked, and which could not be (their saved files are gone).",
|
|
9570
|
+
"placeholder": "true"
|
|
9571
|
+
}
|
|
9572
|
+
],
|
|
9573
|
+
"watchable": false,
|
|
9574
|
+
"help": {
|
|
9575
|
+
"command": "ora study retest <id> --file run.json",
|
|
9576
|
+
"body": "{\n \"kind\": \"fill\",\n \"runIds\": [],\n \"reason\": \"<reason>\"\n}"
|
|
9577
|
+
}
|
|
9578
|
+
},
|
|
9579
|
+
{
|
|
9580
|
+
"id": "study.retests",
|
|
9581
|
+
"commandPath": [
|
|
9582
|
+
"study",
|
|
9583
|
+
"retests"
|
|
9584
|
+
],
|
|
9585
|
+
"method": "get",
|
|
9586
|
+
"pathTemplate": "/study/v1/studies/{id}/retests",
|
|
9587
|
+
"summary": "List a study's re-checks",
|
|
9588
|
+
"scope": "study.studies:read",
|
|
9589
|
+
"product": "study",
|
|
9590
|
+
"pathParams": [
|
|
9591
|
+
{
|
|
9592
|
+
"name": "id"
|
|
9593
|
+
}
|
|
9594
|
+
],
|
|
9595
|
+
"queryParams": [],
|
|
9596
|
+
"hasBody": false,
|
|
9597
|
+
"bodyRequired": false,
|
|
9598
|
+
"isStream": false,
|
|
9599
|
+
"watchable": false,
|
|
9600
|
+
"help": {
|
|
9601
|
+
"command": "ora study retests <id>"
|
|
9602
|
+
}
|
|
9603
|
+
},
|
|
9604
|
+
{
|
|
9605
|
+
"id": "study.reviseGrading",
|
|
9606
|
+
"commandPath": [
|
|
9607
|
+
"study",
|
|
9608
|
+
"revise-grading"
|
|
9609
|
+
],
|
|
9610
|
+
"method": "post",
|
|
9611
|
+
"pathTemplate": "/study/v1/studies/{id}/grading-revisions",
|
|
9612
|
+
"summary": "Revise a finished study's checks and re-grade it",
|
|
9613
|
+
"scope": "study.studies:write",
|
|
9614
|
+
"product": "study",
|
|
9615
|
+
"pathParams": [
|
|
9616
|
+
{
|
|
9617
|
+
"name": "id"
|
|
9618
|
+
}
|
|
9619
|
+
],
|
|
9620
|
+
"queryParams": [],
|
|
9621
|
+
"hasBody": true,
|
|
9622
|
+
"bodyRequired": true,
|
|
9623
|
+
"isStream": false,
|
|
9624
|
+
"bodyFields": [
|
|
9625
|
+
{
|
|
9626
|
+
"name": "reason",
|
|
9627
|
+
"required": true,
|
|
9628
|
+
"kind": "string",
|
|
9629
|
+
"description": "Why the checks are changing. Kept in the revision's record and shown beside it.",
|
|
9630
|
+
"placeholder": "<reason>"
|
|
9631
|
+
},
|
|
9632
|
+
{
|
|
9633
|
+
"name": "tasks",
|
|
9634
|
+
"required": true,
|
|
9635
|
+
"kind": "array",
|
|
9636
|
+
"description": "The tasks whose checks change.",
|
|
9637
|
+
"placeholder": "[]"
|
|
9638
|
+
},
|
|
9639
|
+
{
|
|
9640
|
+
"name": "dryRun",
|
|
9641
|
+
"required": false,
|
|
9642
|
+
"kind": "boolean",
|
|
9643
|
+
"description": "true = change nothing: answer what the revision would change, and which runs' deterministic checks would flip.",
|
|
9644
|
+
"placeholder": "true"
|
|
9645
|
+
}
|
|
9646
|
+
],
|
|
9647
|
+
"watchable": false,
|
|
9648
|
+
"help": {
|
|
9649
|
+
"command": "ora study revise-grading <id> --file run.json",
|
|
9650
|
+
"body": "{\n \"reason\": \"<reason>\",\n \"tasks\": []\n}"
|
|
9651
|
+
}
|
|
9652
|
+
},
|
|
9653
|
+
{
|
|
9654
|
+
"id": "study.rotateShareToken",
|
|
9655
|
+
"commandPath": [
|
|
9656
|
+
"study",
|
|
9657
|
+
"rotate-share-token"
|
|
9658
|
+
],
|
|
9659
|
+
"method": "post",
|
|
9660
|
+
"pathTemplate": "/study/v1/studies/{id}/share-token/rotate",
|
|
9661
|
+
"summary": "Replace the study's share link with a fresh one. The previous link stops working IMMEDIATELY — there is no grace window — and never works again. Works in ANY mode: a study that left 'link' keeps its link, switched off (`shareLinkKept`), and replacing it then retires a leaked link without opening it first (the new one opens only once the study is back in 'link'). This is the ONLY way a study's link changes.",
|
|
9662
|
+
"scope": "study.studies:write",
|
|
9663
|
+
"product": "study",
|
|
9664
|
+
"pathParams": [
|
|
9665
|
+
{
|
|
9666
|
+
"name": "id"
|
|
9667
|
+
}
|
|
9668
|
+
],
|
|
9669
|
+
"queryParams": [],
|
|
9670
|
+
"hasBody": false,
|
|
9671
|
+
"bodyRequired": false,
|
|
9672
|
+
"isStream": false,
|
|
9673
|
+
"watchable": false,
|
|
9674
|
+
"help": {
|
|
9675
|
+
"command": "ora study rotate-share-token <id>"
|
|
9676
|
+
}
|
|
9677
|
+
},
|
|
9678
|
+
{
|
|
9679
|
+
"id": "study.runCounting",
|
|
9680
|
+
"commandPath": [
|
|
9681
|
+
"study",
|
|
9682
|
+
"run-counting"
|
|
9683
|
+
],
|
|
9684
|
+
"method": "get",
|
|
9685
|
+
"pathTemplate": "/study/v1/studies/{id}/run-counting",
|
|
9686
|
+
"summary": "Which of a study's runs its numbers count",
|
|
9687
|
+
"scope": "study.studies:read",
|
|
9688
|
+
"product": "study",
|
|
9689
|
+
"pathParams": [
|
|
9690
|
+
{
|
|
9691
|
+
"name": "id"
|
|
9692
|
+
}
|
|
9693
|
+
],
|
|
9694
|
+
"queryParams": [],
|
|
9695
|
+
"hasBody": false,
|
|
9696
|
+
"bodyRequired": false,
|
|
9697
|
+
"isStream": false,
|
|
9698
|
+
"watchable": false,
|
|
9699
|
+
"help": {
|
|
9700
|
+
"command": "ora study run-counting <id>"
|
|
9701
|
+
}
|
|
9702
|
+
},
|
|
9703
|
+
{
|
|
9704
|
+
"id": "study.runDelivered",
|
|
8739
9705
|
"commandPath": [
|
|
8740
9706
|
"study",
|
|
8741
|
-
"
|
|
9707
|
+
"run-delivered"
|
|
8742
9708
|
],
|
|
8743
9709
|
"method": "get",
|
|
8744
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8745
|
-
"summary": "
|
|
9710
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/delivered",
|
|
9711
|
+
"summary": "List what a study run delivered: the files and images ora keeps a copy of",
|
|
8746
9712
|
"scope": "study.studies:read",
|
|
8747
9713
|
"product": "study",
|
|
8748
9714
|
"pathParams": [
|
|
8749
9715
|
{
|
|
8750
9716
|
"name": "id"
|
|
9717
|
+
},
|
|
9718
|
+
{
|
|
9719
|
+
"name": "runId"
|
|
8751
9720
|
}
|
|
8752
9721
|
],
|
|
8753
9722
|
"queryParams": [],
|
|
@@ -8756,72 +9725,54 @@ export const OPERATIONS = [
|
|
|
8756
9725
|
"isStream": false,
|
|
8757
9726
|
"watchable": false,
|
|
8758
9727
|
"help": {
|
|
8759
|
-
"command": "ora study
|
|
9728
|
+
"command": "ora study run-delivered <id> <runId>"
|
|
8760
9729
|
}
|
|
8761
9730
|
},
|
|
8762
9731
|
{
|
|
8763
|
-
"id": "study.
|
|
9732
|
+
"id": "study.runInsight",
|
|
8764
9733
|
"commandPath": [
|
|
8765
9734
|
"study",
|
|
8766
|
-
"
|
|
9735
|
+
"run-insight"
|
|
8767
9736
|
],
|
|
8768
|
-
"method": "
|
|
8769
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8770
|
-
"summary": "
|
|
8771
|
-
"scope": "study.studies:
|
|
9737
|
+
"method": "get",
|
|
9738
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/insight",
|
|
9739
|
+
"summary": "Read a study run's full insight with the study's access permissions",
|
|
9740
|
+
"scope": "study.studies:read",
|
|
8772
9741
|
"product": "study",
|
|
8773
9742
|
"pathParams": [
|
|
8774
9743
|
{
|
|
8775
9744
|
"name": "id"
|
|
8776
|
-
}
|
|
8777
|
-
],
|
|
8778
|
-
"queryParams": [],
|
|
8779
|
-
"hasBody": true,
|
|
8780
|
-
"bodyRequired": true,
|
|
8781
|
-
"isStream": false,
|
|
8782
|
-
"bodyFields": [
|
|
8783
|
-
{
|
|
8784
|
-
"name": "dryRun",
|
|
8785
|
-
"required": false,
|
|
8786
|
-
"kind": "boolean",
|
|
8787
|
-
"description": "Answer what WOULD be re-run, dispatching and writing nothing (default false).",
|
|
8788
|
-
"placeholder": "true"
|
|
8789
9745
|
},
|
|
8790
9746
|
{
|
|
8791
|
-
"name": "
|
|
8792
|
-
"required": false,
|
|
8793
|
-
"kind": "number",
|
|
8794
|
-
"description": "The most jobs this call dispatches (default 200, or every declared run); the rest are counted in `skipped.overLimit` for the next call.",
|
|
8795
|
-
"placeholder": "1"
|
|
8796
|
-
},
|
|
8797
|
-
{
|
|
8798
|
-
"name": "declared",
|
|
8799
|
-
"required": false,
|
|
8800
|
-
"kind": "array",
|
|
8801
|
-
"description": "The DECLARED re-run: exactly these runs are re-run in place (cause declared_infra), each with its evidence — runs ora's own predicate does not flag. Absent: ora's own infra failures and never-created jobs.",
|
|
8802
|
-
"placeholder": "[]"
|
|
9747
|
+
"name": "runId"
|
|
8803
9748
|
}
|
|
8804
9749
|
],
|
|
9750
|
+
"queryParams": [],
|
|
9751
|
+
"hasBody": false,
|
|
9752
|
+
"bodyRequired": false,
|
|
9753
|
+
"isStream": false,
|
|
8805
9754
|
"watchable": false,
|
|
8806
9755
|
"help": {
|
|
8807
|
-
"command": "ora study
|
|
8808
|
-
"body": "{\n \"dryRun\": true\n}"
|
|
9756
|
+
"command": "ora study run-insight <id> <runId>"
|
|
8809
9757
|
}
|
|
8810
9758
|
},
|
|
8811
9759
|
{
|
|
8812
|
-
"id": "study.
|
|
9760
|
+
"id": "study.runScreens",
|
|
8813
9761
|
"commandPath": [
|
|
8814
9762
|
"study",
|
|
8815
|
-
"
|
|
9763
|
+
"run-screens"
|
|
8816
9764
|
],
|
|
8817
|
-
"method": "
|
|
8818
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8819
|
-
"summary": "
|
|
8820
|
-
"scope": "study.studies:
|
|
9765
|
+
"method": "get",
|
|
9766
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/screens",
|
|
9767
|
+
"summary": "Read a study run's screen verdicts with their evidence",
|
|
9768
|
+
"scope": "study.studies:read",
|
|
8821
9769
|
"product": "study",
|
|
8822
9770
|
"pathParams": [
|
|
8823
9771
|
{
|
|
8824
9772
|
"name": "id"
|
|
9773
|
+
},
|
|
9774
|
+
{
|
|
9775
|
+
"name": "runId"
|
|
8825
9776
|
}
|
|
8826
9777
|
],
|
|
8827
9778
|
"queryParams": [],
|
|
@@ -8830,18 +9781,18 @@ export const OPERATIONS = [
|
|
|
8830
9781
|
"isStream": false,
|
|
8831
9782
|
"watchable": false,
|
|
8832
9783
|
"help": {
|
|
8833
|
-
"command": "ora study
|
|
9784
|
+
"command": "ora study run-screens <id> <runId>"
|
|
8834
9785
|
}
|
|
8835
9786
|
},
|
|
8836
9787
|
{
|
|
8837
|
-
"id": "study.
|
|
9788
|
+
"id": "study.runSpans",
|
|
8838
9789
|
"commandPath": [
|
|
8839
9790
|
"study",
|
|
8840
|
-
"run-
|
|
9791
|
+
"run-spans"
|
|
8841
9792
|
],
|
|
8842
9793
|
"method": "get",
|
|
8843
|
-
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/
|
|
8844
|
-
"summary": "Read a study run's
|
|
9794
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/spans",
|
|
9795
|
+
"summary": "Read a study run's stored spans with the study's access permissions",
|
|
8845
9796
|
"scope": "study.studies:read",
|
|
8846
9797
|
"product": "study",
|
|
8847
9798
|
"pathParams": [
|
|
@@ -8852,13 +9803,21 @@ export const OPERATIONS = [
|
|
|
8852
9803
|
"name": "runId"
|
|
8853
9804
|
}
|
|
8854
9805
|
],
|
|
8855
|
-
"queryParams": [
|
|
9806
|
+
"queryParams": [
|
|
9807
|
+
{
|
|
9808
|
+
"name": "categories",
|
|
9809
|
+
"flag": "categories",
|
|
9810
|
+
"optionKey": "categories",
|
|
9811
|
+
"required": false,
|
|
9812
|
+
"kind": "string"
|
|
9813
|
+
}
|
|
9814
|
+
],
|
|
8856
9815
|
"hasBody": false,
|
|
8857
9816
|
"bodyRequired": false,
|
|
8858
9817
|
"isStream": false,
|
|
8859
9818
|
"watchable": false,
|
|
8860
9819
|
"help": {
|
|
8861
|
-
"command": "ora study run-
|
|
9820
|
+
"command": "ora study run-spans <id> <runId>"
|
|
8862
9821
|
}
|
|
8863
9822
|
},
|
|
8864
9823
|
{
|
|
@@ -8902,6 +9861,34 @@ export const OPERATIONS = [
|
|
|
8902
9861
|
"command": "ora study run-trajectory <id> <runId>"
|
|
8903
9862
|
}
|
|
8904
9863
|
},
|
|
9864
|
+
{
|
|
9865
|
+
"id": "study.runView",
|
|
9866
|
+
"commandPath": [
|
|
9867
|
+
"study",
|
|
9868
|
+
"run-view"
|
|
9869
|
+
],
|
|
9870
|
+
"method": "get",
|
|
9871
|
+
"pathTemplate": "/study/v1/studies/{id}/views/{viewId}",
|
|
9872
|
+
"summary": "Open a saved view: its question, answered from the study's runs as they are now",
|
|
9873
|
+
"scope": "study.studies:read",
|
|
9874
|
+
"product": "study",
|
|
9875
|
+
"pathParams": [
|
|
9876
|
+
{
|
|
9877
|
+
"name": "id"
|
|
9878
|
+
},
|
|
9879
|
+
{
|
|
9880
|
+
"name": "viewId"
|
|
9881
|
+
}
|
|
9882
|
+
],
|
|
9883
|
+
"queryParams": [],
|
|
9884
|
+
"hasBody": false,
|
|
9885
|
+
"bodyRequired": false,
|
|
9886
|
+
"isStream": false,
|
|
9887
|
+
"watchable": false,
|
|
9888
|
+
"help": {
|
|
9889
|
+
"command": "ora study run-view <id> <viewId>"
|
|
9890
|
+
}
|
|
9891
|
+
},
|
|
8905
9892
|
{
|
|
8906
9893
|
"id": "study.runs.cancel",
|
|
8907
9894
|
"commandPath": [
|
|
@@ -9272,6 +10259,195 @@ export const OPERATIONS = [
|
|
|
9272
10259
|
"command": "ora study runs trajectory <id>"
|
|
9273
10260
|
}
|
|
9274
10261
|
},
|
|
10262
|
+
{
|
|
10263
|
+
"id": "study.saveView",
|
|
10264
|
+
"commandPath": [
|
|
10265
|
+
"study",
|
|
10266
|
+
"save-view"
|
|
10267
|
+
],
|
|
10268
|
+
"method": "post",
|
|
10269
|
+
"pathTemplate": "/study/v1/studies/{id}/views",
|
|
10270
|
+
"summary": "Save a run query as a named view of the study",
|
|
10271
|
+
"scope": "study.studies:write",
|
|
10272
|
+
"product": "study",
|
|
10273
|
+
"pathParams": [
|
|
10274
|
+
{
|
|
10275
|
+
"name": "id"
|
|
10276
|
+
}
|
|
10277
|
+
],
|
|
10278
|
+
"queryParams": [],
|
|
10279
|
+
"hasBody": true,
|
|
10280
|
+
"bodyRequired": true,
|
|
10281
|
+
"isStream": false,
|
|
10282
|
+
"bodyFields": [
|
|
10283
|
+
{
|
|
10284
|
+
"name": "name",
|
|
10285
|
+
"required": true,
|
|
10286
|
+
"kind": "string",
|
|
10287
|
+
"placeholder": "<name>"
|
|
10288
|
+
},
|
|
10289
|
+
{
|
|
10290
|
+
"name": "query",
|
|
10291
|
+
"required": true,
|
|
10292
|
+
"kind": "object",
|
|
10293
|
+
"placeholder": "{}"
|
|
10294
|
+
}
|
|
10295
|
+
],
|
|
10296
|
+
"watchable": false,
|
|
10297
|
+
"help": {
|
|
10298
|
+
"command": "ora study save-view <id> --file run.json",
|
|
10299
|
+
"body": "{\n \"name\": \"<name>\",\n \"query\": {}\n}"
|
|
10300
|
+
}
|
|
10301
|
+
},
|
|
10302
|
+
{
|
|
10303
|
+
"id": "study.series",
|
|
10304
|
+
"commandPath": [
|
|
10305
|
+
"study",
|
|
10306
|
+
"series"
|
|
10307
|
+
],
|
|
10308
|
+
"method": "get",
|
|
10309
|
+
"pathTemplate": "/study/v1/studies/series",
|
|
10310
|
+
"summary": "List the organization's study series",
|
|
10311
|
+
"scope": "study.studies:read",
|
|
10312
|
+
"product": "study",
|
|
10313
|
+
"pathParams": [],
|
|
10314
|
+
"queryParams": [],
|
|
10315
|
+
"hasBody": false,
|
|
10316
|
+
"bodyRequired": false,
|
|
10317
|
+
"isStream": false,
|
|
10318
|
+
"watchable": false,
|
|
10319
|
+
"help": {
|
|
10320
|
+
"command": "ora study series"
|
|
10321
|
+
}
|
|
10322
|
+
},
|
|
10323
|
+
{
|
|
10324
|
+
"id": "study.seriesTrend",
|
|
10325
|
+
"commandPath": [
|
|
10326
|
+
"study",
|
|
10327
|
+
"series-trend"
|
|
10328
|
+
],
|
|
10329
|
+
"method": "get",
|
|
10330
|
+
"pathTemplate": "/study/v1/studies/series/trend",
|
|
10331
|
+
"summary": "Read one series as a trend: each study's figure per group, split where the studies stop being comparable",
|
|
10332
|
+
"scope": "study.studies:read",
|
|
10333
|
+
"product": "study",
|
|
10334
|
+
"pathParams": [],
|
|
10335
|
+
"queryParams": [
|
|
10336
|
+
{
|
|
10337
|
+
"name": "series",
|
|
10338
|
+
"flag": "series",
|
|
10339
|
+
"optionKey": "series",
|
|
10340
|
+
"required": true,
|
|
10341
|
+
"kind": "string"
|
|
10342
|
+
},
|
|
10343
|
+
{
|
|
10344
|
+
"name": "metric",
|
|
10345
|
+
"flag": "metric",
|
|
10346
|
+
"optionKey": "metric",
|
|
10347
|
+
"required": false,
|
|
10348
|
+
"kind": "string",
|
|
10349
|
+
"enum": [
|
|
10350
|
+
"benchmark",
|
|
10351
|
+
"goal",
|
|
10352
|
+
"evals",
|
|
10353
|
+
"quality",
|
|
10354
|
+
"rubric",
|
|
10355
|
+
"completion"
|
|
10356
|
+
]
|
|
10357
|
+
},
|
|
10358
|
+
{
|
|
10359
|
+
"name": "branch",
|
|
10360
|
+
"flag": "branch",
|
|
10361
|
+
"optionKey": "branch",
|
|
10362
|
+
"required": false,
|
|
10363
|
+
"kind": "string"
|
|
10364
|
+
}
|
|
10365
|
+
],
|
|
10366
|
+
"hasBody": false,
|
|
10367
|
+
"bodyRequired": false,
|
|
10368
|
+
"isStream": false,
|
|
10369
|
+
"watchable": false,
|
|
10370
|
+
"help": {
|
|
10371
|
+
"command": "ora study series-trend --series <series>"
|
|
10372
|
+
}
|
|
10373
|
+
},
|
|
10374
|
+
{
|
|
10375
|
+
"id": "study.setCarry",
|
|
10376
|
+
"commandPath": [
|
|
10377
|
+
"study",
|
|
10378
|
+
"set-carry"
|
|
10379
|
+
],
|
|
10380
|
+
"method": "put",
|
|
10381
|
+
"pathTemplate": "/study/v1/studies/{id}/carry",
|
|
10382
|
+
"summary": "Turn carrying on or off for a study that has not launched",
|
|
10383
|
+
"scope": "study.studies:write",
|
|
10384
|
+
"product": "study",
|
|
10385
|
+
"pathParams": [
|
|
10386
|
+
{
|
|
10387
|
+
"name": "id"
|
|
10388
|
+
}
|
|
10389
|
+
],
|
|
10390
|
+
"queryParams": [],
|
|
10391
|
+
"hasBody": true,
|
|
10392
|
+
"bodyRequired": true,
|
|
10393
|
+
"isStream": false,
|
|
10394
|
+
"bodyFields": [
|
|
10395
|
+
{
|
|
10396
|
+
"name": "carry",
|
|
10397
|
+
"required": true,
|
|
10398
|
+
"kind": "string",
|
|
10399
|
+
"description": "The policy the study takes. null = carry nothing: every job runs when the study launches.",
|
|
10400
|
+
"placeholder": "<carry>"
|
|
10401
|
+
}
|
|
10402
|
+
],
|
|
10403
|
+
"watchable": false,
|
|
10404
|
+
"help": {
|
|
10405
|
+
"command": "ora study set-carry <id> --file run.json",
|
|
10406
|
+
"body": "{\n \"carry\": \"<carry>\"\n}"
|
|
10407
|
+
}
|
|
10408
|
+
},
|
|
10409
|
+
{
|
|
10410
|
+
"id": "study.setRunCounting",
|
|
10411
|
+
"commandPath": [
|
|
10412
|
+
"study",
|
|
10413
|
+
"set-run-counting"
|
|
10414
|
+
],
|
|
10415
|
+
"method": "put",
|
|
10416
|
+
"pathTemplate": "/study/v1/studies/{id}/run-counting",
|
|
10417
|
+
"summary": "Hold back the runs a screen flagged, or count every run again",
|
|
10418
|
+
"scope": "study.studies:write",
|
|
10419
|
+
"product": "study",
|
|
10420
|
+
"pathParams": [
|
|
10421
|
+
{
|
|
10422
|
+
"name": "id"
|
|
10423
|
+
}
|
|
10424
|
+
],
|
|
10425
|
+
"queryParams": [],
|
|
10426
|
+
"hasBody": true,
|
|
10427
|
+
"bodyRequired": true,
|
|
10428
|
+
"isStream": false,
|
|
10429
|
+
"bodyFields": [
|
|
10430
|
+
{
|
|
10431
|
+
"name": "runCounting",
|
|
10432
|
+
"required": true,
|
|
10433
|
+
"kind": "string",
|
|
10434
|
+
"description": "The policy the study takes. null = count every run.",
|
|
10435
|
+
"placeholder": "<runCounting>"
|
|
10436
|
+
},
|
|
10437
|
+
{
|
|
10438
|
+
"name": "dryRun",
|
|
10439
|
+
"required": false,
|
|
10440
|
+
"kind": "boolean",
|
|
10441
|
+
"description": "true = change nothing and answer the state as it would be: what would be held back, and every group's figure both ways.",
|
|
10442
|
+
"placeholder": "true"
|
|
10443
|
+
}
|
|
10444
|
+
],
|
|
10445
|
+
"watchable": false,
|
|
10446
|
+
"help": {
|
|
10447
|
+
"command": "ora study set-run-counting <id> --file run.json",
|
|
10448
|
+
"body": "{\n \"runCounting\": \"<runCounting>\"\n}"
|
|
10449
|
+
}
|
|
10450
|
+
},
|
|
9275
10451
|
{
|
|
9276
10452
|
"id": "study.setVisibility",
|
|
9277
10453
|
"commandPath": [
|
|
@@ -9655,6 +10831,31 @@ export const OPERATIONS = [
|
|
|
9655
10831
|
"body": "{\n \"prompt\": \"Research the three largest EU battery-storage tenders announced this quarter and cite each source.\"\n}"
|
|
9656
10832
|
}
|
|
9657
10833
|
},
|
|
10834
|
+
{
|
|
10835
|
+
"id": "study.views",
|
|
10836
|
+
"commandPath": [
|
|
10837
|
+
"study",
|
|
10838
|
+
"views"
|
|
10839
|
+
],
|
|
10840
|
+
"method": "get",
|
|
10841
|
+
"pathTemplate": "/study/v1/studies/{id}/views",
|
|
10842
|
+
"summary": "List a study's saved run queries (views)",
|
|
10843
|
+
"scope": "study.studies:read",
|
|
10844
|
+
"product": "study",
|
|
10845
|
+
"pathParams": [
|
|
10846
|
+
{
|
|
10847
|
+
"name": "id"
|
|
10848
|
+
}
|
|
10849
|
+
],
|
|
10850
|
+
"queryParams": [],
|
|
10851
|
+
"hasBody": false,
|
|
10852
|
+
"bodyRequired": false,
|
|
10853
|
+
"isStream": false,
|
|
10854
|
+
"watchable": false,
|
|
10855
|
+
"help": {
|
|
10856
|
+
"command": "ora study views <id>"
|
|
10857
|
+
}
|
|
10858
|
+
},
|
|
9658
10859
|
{
|
|
9659
10860
|
"id": "tunnels.create",
|
|
9660
10861
|
"commandPath": [
|