@ora-ai/cli 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -1
- package/dist/auth-commands.d.ts.map +1 -1
- package/dist/auth-commands.js +12 -1
- package/dist/auth-commands.js.map +1 -1
- package/dist/core/auth.d.ts +24 -13
- package/dist/core/auth.d.ts.map +1 -1
- package/dist/core/auth.js +26 -15
- package/dist/core/auth.js.map +1 -1
- package/dist/core/credential-sources.d.ts +45 -0
- package/dist/core/credential-sources.d.ts.map +1 -0
- package/dist/core/credential-sources.js +65 -0
- package/dist/core/credential-sources.js.map +1 -0
- package/dist/core/github-actions-sign-in.d.ts +33 -0
- package/dist/core/github-actions-sign-in.d.ts.map +1 -0
- package/dist/core/github-actions-sign-in.js +54 -0
- package/dist/core/github-actions-sign-in.js.map +1 -0
- package/dist/core/output.js +14 -1
- package/dist/core/output.js.map +1 -1
- package/dist/core/run-operation.d.ts +11 -4
- package/dist/core/run-operation.d.ts.map +1 -1
- package/dist/core/run-operation.js +14 -9
- package/dist/core/run-operation.js.map +1 -1
- package/dist/core/sign-in.d.ts +34 -0
- package/dist/core/sign-in.d.ts.map +1 -0
- package/dist/core/sign-in.js +21 -0
- package/dist/core/sign-in.js.map +1 -0
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +1236 -70
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +1426 -10
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/dist/mcp/mcp-call.d.ts +25 -0
- package/dist/mcp/mcp-call.d.ts.map +1 -0
- package/dist/mcp/mcp-call.js +44 -0
- package/dist/mcp/mcp-call.js.map +1 -0
- package/dist/mcp/mcp-command.d.ts +26 -0
- package/dist/mcp/mcp-command.d.ts.map +1 -0
- package/dist/mcp/mcp-command.js +91 -0
- package/dist/mcp/mcp-command.js.map +1 -0
- package/dist/mcp/mcp-install.d.ts +50 -0
- package/dist/mcp/mcp-install.d.ts.map +1 -0
- package/dist/mcp/mcp-install.js +71 -0
- package/dist/mcp/mcp-install.js.map +1 -0
- package/dist/mcp/mcp-protocol.d.ts +69 -0
- package/dist/mcp/mcp-protocol.d.ts.map +1 -0
- package/dist/mcp/mcp-protocol.js +96 -0
- package/dist/mcp/mcp-protocol.js.map +1 -0
- package/dist/mcp/mcp-stdio.d.ts +17 -0
- package/dist/mcp/mcp-stdio.d.ts.map +1 -0
- package/dist/mcp/mcp-stdio.js +39 -0
- package/dist/mcp/mcp-stdio.js.map +1 -0
- package/dist/mcp/mcp-tools.d.ts +81 -0
- package/dist/mcp/mcp-tools.d.ts.map +1 -0
- package/dist/mcp/mcp-tools.js +134 -0
- package/dist/mcp/mcp-tools.js.map +1 -0
- package/dist/program.d.ts +4 -9
- package/dist/program.d.ts.map +1 -1
- package/dist/program.js +9 -3
- package/dist/program.js.map +1 -1
- package/dist/render/index.js +6 -0
- package/dist/render/index.js.map +1 -1
- package/dist/render/study/counting-views.d.ts +46 -0
- package/dist/render/study/counting-views.d.ts.map +1 -0
- package/dist/render/study/counting-views.js +49 -0
- package/dist/render/study/counting-views.js.map +1 -0
- package/dist/render/study/gate-views.d.ts +56 -0
- package/dist/render/study/gate-views.d.ts.map +1 -0
- package/dist/render/study/gate-views.js +64 -0
- package/dist/render/study/gate-views.js.map +1 -0
- package/dist/render/study/renderers.d.ts +8 -0
- package/dist/render/study/renderers.d.ts.map +1 -0
- package/dist/render/study/renderers.js +31 -0
- package/dist/render/study/renderers.js.map +1 -0
- package/dist/render/study/series-views.d.ts +65 -0
- package/dist/render/study/series-views.d.ts.map +1 -0
- package/dist/render/study/series-views.js +116 -0
- package/dist/render/study/series-views.js.map +1 -0
- package/dist/render/study/study-format.d.ts +24 -0
- package/dist/render/study/study-format.d.ts.map +1 -0
- package/dist/render/study/study-format.js +42 -0
- package/dist/render/study/study-format.js.map +1 -0
- package/dist/study-ci/ci-command.d.ts +31 -0
- package/dist/study-ci/ci-command.d.ts.map +1 -0
- package/dist/study-ci/ci-command.js +134 -0
- package/dist/study-ci/ci-command.js.map +1 -0
- package/dist/study-ci/ci-rehearse-command.d.ts +11 -0
- package/dist/study-ci/ci-rehearse-command.d.ts.map +1 -0
- package/dist/study-ci/ci-rehearse-command.js +83 -0
- package/dist/study-ci/ci-rehearse-command.js.map +1 -0
- package/dist/study-ci/ci-rehearse.d.ts +62 -0
- package/dist/study-ci/ci-rehearse.d.ts.map +1 -0
- package/dist/study-ci/ci-rehearse.js +49 -0
- package/dist/study-ci/ci-rehearse.js.map +1 -0
- package/dist/study-ci/ci-run.d.ts +87 -0
- package/dist/study-ci/ci-run.d.ts.map +1 -0
- package/dist/study-ci/ci-run.js +131 -0
- package/dist/study-ci/ci-run.js.map +1 -0
- package/dist/study-ci/gate-report.d.ts +64 -0
- package/dist/study-ci/gate-report.d.ts.map +1 -0
- package/dist/study-ci/gate-report.js +80 -0
- package/dist/study-ci/gate-report.js.map +1 -0
- package/dist/study-ci/params.d.ts +8 -0
- package/dist/study-ci/params.d.ts.map +1 -0
- package/dist/study-ci/params.js +72 -0
- package/dist/study-ci/params.js.map +1 -0
- package/package.json +1 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// GENERATED by scripts/generate.mjs — DO NOT EDIT.
|
|
2
|
-
// Source: openapi/ora-public.openapi.json (sha256
|
|
2
|
+
// Source: openapi/ora-public.openapi.json (sha256 0f479e1012ff3fef63b1aed942d4aefabae3782a892c336f372e7b03b0bbb9bd)
|
|
3
3
|
// Regenerate: yarn nx run ora-cli:generate
|
|
4
4
|
export const SPEC = {
|
|
5
5
|
"version": "1.0.0",
|
|
6
|
-
"contentHash": "
|
|
6
|
+
"contentHash": "0f479e1012ff3fef63b1aed942d4aefabae3782a892c336f372e7b03b0bbb9bd",
|
|
7
7
|
"server": "https://api.ora.ai",
|
|
8
8
|
"bin": "ora"
|
|
9
9
|
};
|
|
@@ -6750,6 +6750,31 @@ export const OPERATIONS = [
|
|
|
6750
6750
|
"command": "ora study addons list"
|
|
6751
6751
|
}
|
|
6752
6752
|
},
|
|
6753
|
+
{
|
|
6754
|
+
"id": "study.behaviors",
|
|
6755
|
+
"commandPath": [
|
|
6756
|
+
"study",
|
|
6757
|
+
"behaviors"
|
|
6758
|
+
],
|
|
6759
|
+
"method": "get",
|
|
6760
|
+
"pathTemplate": "/study/v1/studies/{id}/behaviors",
|
|
6761
|
+
"summary": "Read a study's custom behaviors: what each found, and how far its verdicts held up under review",
|
|
6762
|
+
"scope": "study.studies:read",
|
|
6763
|
+
"product": "study",
|
|
6764
|
+
"pathParams": [
|
|
6765
|
+
{
|
|
6766
|
+
"name": "id"
|
|
6767
|
+
}
|
|
6768
|
+
],
|
|
6769
|
+
"queryParams": [],
|
|
6770
|
+
"hasBody": false,
|
|
6771
|
+
"bodyRequired": false,
|
|
6772
|
+
"isStream": false,
|
|
6773
|
+
"watchable": false,
|
|
6774
|
+
"help": {
|
|
6775
|
+
"command": "ora study behaviors <id>"
|
|
6776
|
+
}
|
|
6777
|
+
},
|
|
6753
6778
|
{
|
|
6754
6779
|
"id": "study.benchmarkDefinitions.attachCorpus",
|
|
6755
6780
|
"commandPath": [
|
|
@@ -7619,6 +7644,31 @@ export const OPERATIONS = [
|
|
|
7619
7644
|
"command": "ora study benchmarkTerms get"
|
|
7620
7645
|
}
|
|
7621
7646
|
},
|
|
7647
|
+
{
|
|
7648
|
+
"id": "study.carryPlan",
|
|
7649
|
+
"commandPath": [
|
|
7650
|
+
"study",
|
|
7651
|
+
"carry-plan"
|
|
7652
|
+
],
|
|
7653
|
+
"method": "get",
|
|
7654
|
+
"pathTemplate": "/study/v1/studies/{id}/carry-plan",
|
|
7655
|
+
"summary": "What a series study would carry instead of run",
|
|
7656
|
+
"scope": "study.studies:read",
|
|
7657
|
+
"product": "study",
|
|
7658
|
+
"pathParams": [
|
|
7659
|
+
{
|
|
7660
|
+
"name": "id"
|
|
7661
|
+
}
|
|
7662
|
+
],
|
|
7663
|
+
"queryParams": [],
|
|
7664
|
+
"hasBody": false,
|
|
7665
|
+
"bodyRequired": false,
|
|
7666
|
+
"isStream": false,
|
|
7667
|
+
"watchable": false,
|
|
7668
|
+
"help": {
|
|
7669
|
+
"command": "ora study carry-plan <id>"
|
|
7670
|
+
}
|
|
7671
|
+
},
|
|
7622
7672
|
{
|
|
7623
7673
|
"id": "study.catalog",
|
|
7624
7674
|
"commandPath": [
|
|
@@ -7899,6 +7949,47 @@ export const OPERATIONS = [
|
|
|
7899
7949
|
"description": "LAUNCH PACING — start this study's runs gradually instead of all at once, for groups that call a rate-limited service (an API that answers 429 past N requests a minute). Every group's run of one agent on one task sample (a comparison unit) always starts together, in the same wave, so a comparison is never confounded by time; each value must therefore be at least the number of groups that run one agent. Declare maxInFlight, startsPerMinute, or both (both then hold). Frozen at create. Omit it, or send null, for no pacing: every run starts at launch.",
|
|
7900
7950
|
"placeholder": "<pacing>"
|
|
7901
7951
|
},
|
|
7952
|
+
{
|
|
7953
|
+
"name": "screens",
|
|
7954
|
+
"required": false,
|
|
7955
|
+
"kind": "array",
|
|
7956
|
+
"description": "Which run screens this study runs. A screen is a per-run check of whether the run is valid evidence, kept apart from grading: it never changes a grade. Omitted = the deployment's default. Frozen at launch, like lenses. 'test_awareness': after a run ends, a judge panel reads its full trace for signs that the agent noticed it was being tested ('aware') or changed how it worked because of that ('influenced'), and records the quoted evidence with the classifier's version.",
|
|
7957
|
+
"placeholder": "[]"
|
|
7958
|
+
},
|
|
7959
|
+
{
|
|
7960
|
+
"name": "series",
|
|
7961
|
+
"required": false,
|
|
7962
|
+
"kind": "string",
|
|
7963
|
+
"description": "The SERIES this study belongs to: studies that ask the same question at different moments (every commit of a branch). A gate's series baseline is found in it. Immutable. Omit for a study of no series.",
|
|
7964
|
+
"placeholder": "<series>"
|
|
7965
|
+
},
|
|
7966
|
+
{
|
|
7967
|
+
"name": "ref",
|
|
7968
|
+
"required": false,
|
|
7969
|
+
"kind": "object",
|
|
7970
|
+
"description": "Where the study was taken: the commit, branch and pull request. A label, immutable.",
|
|
7971
|
+
"placeholder": "{}"
|
|
7972
|
+
},
|
|
7973
|
+
{
|
|
7974
|
+
"name": "carry",
|
|
7975
|
+
"required": false,
|
|
7976
|
+
"kind": "object",
|
|
7977
|
+
"description": "Carry the runs of jobs that cannot have changed from the series' previous comparable study instead of running them again (needs `series`). Each carried run says so. Omit to run every job.",
|
|
7978
|
+
"placeholder": "{}"
|
|
7979
|
+
},
|
|
7980
|
+
{
|
|
7981
|
+
"name": "runCounting",
|
|
7982
|
+
"required": false,
|
|
7983
|
+
"kind": "string",
|
|
7984
|
+
"placeholder": "<runCounting>"
|
|
7985
|
+
},
|
|
7986
|
+
{
|
|
7987
|
+
"name": "behaviors",
|
|
7988
|
+
"required": false,
|
|
7989
|
+
"kind": "array",
|
|
7990
|
+
"description": "Custom behaviors: up to 5 yes-or-no questions a judge panel answers about every finished run, with quoted evidence. A behavior is disclosure about a run, like a screen: it has no score and never changes a grade. Frozen at launch. Each costs one panel read per run.",
|
|
7991
|
+
"placeholder": "[]"
|
|
7992
|
+
},
|
|
7902
7993
|
{
|
|
7903
7994
|
"name": "matrix",
|
|
7904
7995
|
"required": true,
|
|
@@ -7928,7 +8019,7 @@ export const OPERATIONS = [
|
|
|
7928
8019
|
],
|
|
7929
8020
|
"method": "delete",
|
|
7930
8021
|
"pathTemplate": "/study/v1/studies/{id}",
|
|
7931
|
-
"summary": "
|
|
8022
|
+
"summary": "Archive a study (409 while it is generating, preparing or running). Nothing is removed; restore brings it back. The study leaves the default list (list with archived=only to see it), becomes read-only (every change answers 409 study_archived until it is restored), and is unpublished first if it was shared outside the organisation. Archiving an archived study succeeds and changes nothing.",
|
|
7932
8023
|
"scope": "study.studies:write",
|
|
7933
8024
|
"product": "study",
|
|
7934
8025
|
"pathParams": [
|
|
@@ -7945,6 +8036,129 @@ export const OPERATIONS = [
|
|
|
7945
8036
|
"command": "ora study delete <id>"
|
|
7946
8037
|
}
|
|
7947
8038
|
},
|
|
8039
|
+
{
|
|
8040
|
+
"id": "study.deleteView",
|
|
8041
|
+
"commandPath": [
|
|
8042
|
+
"study",
|
|
8043
|
+
"delete-view"
|
|
8044
|
+
],
|
|
8045
|
+
"method": "delete",
|
|
8046
|
+
"pathTemplate": "/study/v1/studies/{id}/views/{viewId}",
|
|
8047
|
+
"summary": "Delete a saved view",
|
|
8048
|
+
"scope": "study.studies:write",
|
|
8049
|
+
"product": "study",
|
|
8050
|
+
"pathParams": [
|
|
8051
|
+
{
|
|
8052
|
+
"name": "id"
|
|
8053
|
+
},
|
|
8054
|
+
{
|
|
8055
|
+
"name": "viewId"
|
|
8056
|
+
}
|
|
8057
|
+
],
|
|
8058
|
+
"queryParams": [],
|
|
8059
|
+
"hasBody": false,
|
|
8060
|
+
"bodyRequired": false,
|
|
8061
|
+
"isStream": false,
|
|
8062
|
+
"watchable": false,
|
|
8063
|
+
"help": {
|
|
8064
|
+
"command": "ora study delete-view <id> <viewId>"
|
|
8065
|
+
}
|
|
8066
|
+
},
|
|
8067
|
+
{
|
|
8068
|
+
"id": "study.gate",
|
|
8069
|
+
"commandPath": [
|
|
8070
|
+
"study",
|
|
8071
|
+
"gate"
|
|
8072
|
+
],
|
|
8073
|
+
"method": "post",
|
|
8074
|
+
"pathTemplate": "/study/v1/studies/{id}/gate",
|
|
8075
|
+
"summary": "Ask a study whether it passes a gate",
|
|
8076
|
+
"scope": "study.studies:read",
|
|
8077
|
+
"product": "study",
|
|
8078
|
+
"pathParams": [
|
|
8079
|
+
{
|
|
8080
|
+
"name": "id"
|
|
8081
|
+
}
|
|
8082
|
+
],
|
|
8083
|
+
"queryParams": [],
|
|
8084
|
+
"hasBody": true,
|
|
8085
|
+
"bodyRequired": true,
|
|
8086
|
+
"isStream": false,
|
|
8087
|
+
"bodyFields": [
|
|
8088
|
+
{
|
|
8089
|
+
"name": "metric",
|
|
8090
|
+
"required": false,
|
|
8091
|
+
"kind": "string",
|
|
8092
|
+
"enum": [
|
|
8093
|
+
"benchmark",
|
|
8094
|
+
"goal",
|
|
8095
|
+
"evals",
|
|
8096
|
+
"quality",
|
|
8097
|
+
"rubric",
|
|
8098
|
+
"completion"
|
|
8099
|
+
],
|
|
8100
|
+
"description": "The metric the gate reads. Omitted = the study's own correctness measure (benchmark, else goal, else evals…).",
|
|
8101
|
+
"placeholder": "benchmark"
|
|
8102
|
+
},
|
|
8103
|
+
{
|
|
8104
|
+
"name": "group",
|
|
8105
|
+
"required": false,
|
|
8106
|
+
"kind": "string",
|
|
8107
|
+
"description": "The group the gate judges, by key. Omitted = the study's only group, or its one non-reference group.",
|
|
8108
|
+
"placeholder": "<group>"
|
|
8109
|
+
},
|
|
8110
|
+
{
|
|
8111
|
+
"name": "floor",
|
|
8112
|
+
"required": false,
|
|
8113
|
+
"kind": "string",
|
|
8114
|
+
"description": "Pass when the metric's interval lies wholly at or above this. `{ rate: 0.8 }` for a rate, `{ points: 70 }` for a 0-100 score.",
|
|
8115
|
+
"placeholder": "<floor>"
|
|
8116
|
+
},
|
|
8117
|
+
{
|
|
8118
|
+
"name": "maxDrop",
|
|
8119
|
+
"required": false,
|
|
8120
|
+
"kind": "string",
|
|
8121
|
+
"description": "Fail when the metric fell against the baseline by more than this, for certain. `{ pp: 5 }` for a rate, `{ points: 5 }` for a score. Needs `baseline`.",
|
|
8122
|
+
"placeholder": "<maxDrop>"
|
|
8123
|
+
},
|
|
8124
|
+
{
|
|
8125
|
+
"name": "baseline",
|
|
8126
|
+
"required": false,
|
|
8127
|
+
"kind": "string",
|
|
8128
|
+
"placeholder": "<baseline>"
|
|
8129
|
+
},
|
|
8130
|
+
{
|
|
8131
|
+
"name": "minTasks",
|
|
8132
|
+
"required": false,
|
|
8133
|
+
"kind": "number",
|
|
8134
|
+
"description": "Fewer measured tasks than this is 'not enough runs to decide'.",
|
|
8135
|
+
"placeholder": "5"
|
|
8136
|
+
},
|
|
8137
|
+
{
|
|
8138
|
+
"name": "infraCeiling",
|
|
8139
|
+
"required": false,
|
|
8140
|
+
"kind": "number",
|
|
8141
|
+
"description": "The share of the group's runs ora's own infrastructure may have killed before the gate refuses to decide.",
|
|
8142
|
+
"placeholder": "0.2"
|
|
8143
|
+
},
|
|
8144
|
+
{
|
|
8145
|
+
"name": "counting",
|
|
8146
|
+
"required": false,
|
|
8147
|
+
"kind": "string",
|
|
8148
|
+
"enum": [
|
|
8149
|
+
"policy",
|
|
8150
|
+
"all"
|
|
8151
|
+
],
|
|
8152
|
+
"description": "Which runs the gate reads. 'policy' (the default): the runs the study counts, so a study that holds back test-aware runs is judged without them. 'all': every run, held back or not, on both this study and the baseline.",
|
|
8153
|
+
"placeholder": "policy"
|
|
8154
|
+
}
|
|
8155
|
+
],
|
|
8156
|
+
"watchable": false,
|
|
8157
|
+
"help": {
|
|
8158
|
+
"command": "ora study gate <id> --file run.json",
|
|
8159
|
+
"body": "{\n \"metric\": \"benchmark\",\n \"group\": \"<group>\",\n \"floor\": \"<floor>\"\n}"
|
|
8160
|
+
}
|
|
8161
|
+
},
|
|
7948
8162
|
{
|
|
7949
8163
|
"id": "study.generate",
|
|
7950
8164
|
"commandPath": [
|
|
@@ -8005,6 +8219,31 @@ export const OPERATIONS = [
|
|
|
8005
8219
|
"command": "ora study get 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
8006
8220
|
}
|
|
8007
8221
|
},
|
|
8222
|
+
{
|
|
8223
|
+
"id": "study.gradingRevisions",
|
|
8224
|
+
"commandPath": [
|
|
8225
|
+
"study",
|
|
8226
|
+
"grading-revisions"
|
|
8227
|
+
],
|
|
8228
|
+
"method": "get",
|
|
8229
|
+
"pathTemplate": "/study/v1/studies/{id}/grading-revisions",
|
|
8230
|
+
"summary": "List a study's grading revisions",
|
|
8231
|
+
"scope": "study.studies:read",
|
|
8232
|
+
"product": "study",
|
|
8233
|
+
"pathParams": [
|
|
8234
|
+
{
|
|
8235
|
+
"name": "id"
|
|
8236
|
+
}
|
|
8237
|
+
],
|
|
8238
|
+
"queryParams": [],
|
|
8239
|
+
"hasBody": false,
|
|
8240
|
+
"bodyRequired": false,
|
|
8241
|
+
"isStream": false,
|
|
8242
|
+
"watchable": false,
|
|
8243
|
+
"help": {
|
|
8244
|
+
"command": "ora study grading-revisions <id>"
|
|
8245
|
+
}
|
|
8246
|
+
},
|
|
8008
8247
|
{
|
|
8009
8248
|
"id": "study.images.builds.cancel",
|
|
8010
8249
|
"commandPath": [
|
|
@@ -8299,7 +8538,20 @@ export const OPERATIONS = [
|
|
|
8299
8538
|
"scope": "study.studies:read",
|
|
8300
8539
|
"product": "study",
|
|
8301
8540
|
"pathParams": [],
|
|
8302
|
-
"queryParams": [
|
|
8541
|
+
"queryParams": [
|
|
8542
|
+
{
|
|
8543
|
+
"name": "archived",
|
|
8544
|
+
"flag": "archived",
|
|
8545
|
+
"optionKey": "archived",
|
|
8546
|
+
"required": false,
|
|
8547
|
+
"kind": "string",
|
|
8548
|
+
"enum": [
|
|
8549
|
+
"exclude",
|
|
8550
|
+
"only",
|
|
8551
|
+
"include"
|
|
8552
|
+
]
|
|
8553
|
+
}
|
|
8554
|
+
],
|
|
8303
8555
|
"hasBody": false,
|
|
8304
8556
|
"bodyRequired": false,
|
|
8305
8557
|
"isStream": false,
|
|
@@ -8547,6 +8799,47 @@ export const OPERATIONS = [
|
|
|
8547
8799
|
"description": "LAUNCH PACING — start this study's runs gradually instead of all at once, for groups that call a rate-limited service (an API that answers 429 past N requests a minute). Every group's run of one agent on one task sample (a comparison unit) always starts together, in the same wave, so a comparison is never confounded by time; each value must therefore be at least the number of groups that run one agent. Declare maxInFlight, startsPerMinute, or both (both then hold). Frozen at create. Omit it, or send null, for no pacing: every run starts at launch.",
|
|
8548
8800
|
"placeholder": "<pacing>"
|
|
8549
8801
|
},
|
|
8802
|
+
{
|
|
8803
|
+
"name": "screens",
|
|
8804
|
+
"required": false,
|
|
8805
|
+
"kind": "array",
|
|
8806
|
+
"description": "Which run screens this study runs. A screen is a per-run check of whether the run is valid evidence, kept apart from grading: it never changes a grade. Omitted = the deployment's default. Frozen at launch, like lenses. 'test_awareness': after a run ends, a judge panel reads its full trace for signs that the agent noticed it was being tested ('aware') or changed how it worked because of that ('influenced'), and records the quoted evidence with the classifier's version.",
|
|
8807
|
+
"placeholder": "[]"
|
|
8808
|
+
},
|
|
8809
|
+
{
|
|
8810
|
+
"name": "series",
|
|
8811
|
+
"required": false,
|
|
8812
|
+
"kind": "string",
|
|
8813
|
+
"description": "The SERIES this study belongs to: studies that ask the same question at different moments (every commit of a branch). A gate's series baseline is found in it. Immutable. Omit for a study of no series.",
|
|
8814
|
+
"placeholder": "<series>"
|
|
8815
|
+
},
|
|
8816
|
+
{
|
|
8817
|
+
"name": "ref",
|
|
8818
|
+
"required": false,
|
|
8819
|
+
"kind": "object",
|
|
8820
|
+
"description": "Where the study was taken: the commit, branch and pull request. A label, immutable.",
|
|
8821
|
+
"placeholder": "{}"
|
|
8822
|
+
},
|
|
8823
|
+
{
|
|
8824
|
+
"name": "carry",
|
|
8825
|
+
"required": false,
|
|
8826
|
+
"kind": "object",
|
|
8827
|
+
"description": "Carry the runs of jobs that cannot have changed from the series' previous comparable study instead of running them again (needs `series`). Each carried run says so. Omit to run every job.",
|
|
8828
|
+
"placeholder": "{}"
|
|
8829
|
+
},
|
|
8830
|
+
{
|
|
8831
|
+
"name": "runCounting",
|
|
8832
|
+
"required": false,
|
|
8833
|
+
"kind": "string",
|
|
8834
|
+
"placeholder": "<runCounting>"
|
|
8835
|
+
},
|
|
8836
|
+
{
|
|
8837
|
+
"name": "behaviors",
|
|
8838
|
+
"required": false,
|
|
8839
|
+
"kind": "array",
|
|
8840
|
+
"description": "Custom behaviors: up to 5 yes-or-no questions a judge panel answers about every finished run, with quoted evidence. A behavior is disclosure about a run, like a screen: it has no score and never changes a grade. Frozen at launch. Each costs one panel read per run.",
|
|
8841
|
+
"placeholder": "[]"
|
|
8842
|
+
},
|
|
8550
8843
|
{
|
|
8551
8844
|
"name": "matrix",
|
|
8552
8845
|
"required": true,
|
|
@@ -8660,15 +8953,15 @@ export const OPERATIONS = [
|
|
|
8660
8953
|
}
|
|
8661
8954
|
},
|
|
8662
8955
|
{
|
|
8663
|
-
"id": "study.
|
|
8956
|
+
"id": "study.query",
|
|
8664
8957
|
"commandPath": [
|
|
8665
8958
|
"study",
|
|
8666
|
-
"
|
|
8959
|
+
"query"
|
|
8667
8960
|
],
|
|
8668
8961
|
"method": "post",
|
|
8669
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8670
|
-
"summary": "
|
|
8671
|
-
"scope": "study.studies:
|
|
8962
|
+
"pathTemplate": "/study/v1/studies/{id}/query",
|
|
8963
|
+
"summary": "Ask a question of a study's runs: which runs, split how, measured on what",
|
|
8964
|
+
"scope": "study.studies:read",
|
|
8672
8965
|
"product": "study",
|
|
8673
8966
|
"pathParams": [
|
|
8674
8967
|
{
|
|
@@ -8676,24 +8969,53 @@ export const OPERATIONS = [
|
|
|
8676
8969
|
}
|
|
8677
8970
|
],
|
|
8678
8971
|
"queryParams": [],
|
|
8679
|
-
"hasBody":
|
|
8680
|
-
"bodyRequired":
|
|
8972
|
+
"hasBody": true,
|
|
8973
|
+
"bodyRequired": true,
|
|
8681
8974
|
"isStream": false,
|
|
8975
|
+
"bodyFields": [
|
|
8976
|
+
{
|
|
8977
|
+
"name": "metrics",
|
|
8978
|
+
"required": true,
|
|
8979
|
+
"kind": "array",
|
|
8980
|
+
"description": "Metric ids. `GET …/query/vocabulary` lists them.",
|
|
8981
|
+
"placeholder": "[]"
|
|
8982
|
+
},
|
|
8983
|
+
{
|
|
8984
|
+
"name": "groupBy",
|
|
8985
|
+
"required": false,
|
|
8986
|
+
"kind": "array",
|
|
8987
|
+
"description": "Dimension ids to split the runs by. None = one row over every matched run.",
|
|
8988
|
+
"placeholder": "[]"
|
|
8989
|
+
},
|
|
8990
|
+
{
|
|
8991
|
+
"name": "filters",
|
|
8992
|
+
"required": false,
|
|
8993
|
+
"kind": "array",
|
|
8994
|
+
"placeholder": "[]"
|
|
8995
|
+
},
|
|
8996
|
+
{
|
|
8997
|
+
"name": "limit",
|
|
8998
|
+
"required": false,
|
|
8999
|
+
"kind": "number",
|
|
9000
|
+
"placeholder": "200"
|
|
9001
|
+
}
|
|
9002
|
+
],
|
|
8682
9003
|
"watchable": false,
|
|
8683
9004
|
"help": {
|
|
8684
|
-
"command": "ora study
|
|
9005
|
+
"command": "ora study query <id> --file run.json",
|
|
9006
|
+
"body": "{\n \"metrics\": []\n}"
|
|
8685
9007
|
}
|
|
8686
9008
|
},
|
|
8687
9009
|
{
|
|
8688
|
-
"id": "study.
|
|
9010
|
+
"id": "study.queryVocabulary",
|
|
8689
9011
|
"commandPath": [
|
|
8690
9012
|
"study",
|
|
8691
|
-
"
|
|
9013
|
+
"query-vocabulary"
|
|
8692
9014
|
],
|
|
8693
|
-
"method": "
|
|
8694
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8695
|
-
"summary": "
|
|
8696
|
-
"scope": "study.studies:
|
|
9015
|
+
"method": "get",
|
|
9016
|
+
"pathTemplate": "/study/v1/studies/{id}/query/vocabulary",
|
|
9017
|
+
"summary": "List what a study's runs can be split by and measured on",
|
|
9018
|
+
"scope": "study.studies:read",
|
|
8697
9019
|
"product": "study",
|
|
8698
9020
|
"pathParams": [
|
|
8699
9021
|
{
|
|
@@ -8706,17 +9028,67 @@ export const OPERATIONS = [
|
|
|
8706
9028
|
"isStream": false,
|
|
8707
9029
|
"watchable": false,
|
|
8708
9030
|
"help": {
|
|
8709
|
-
"command": "ora study
|
|
9031
|
+
"command": "ora study query-vocabulary <id>"
|
|
8710
9032
|
}
|
|
8711
9033
|
},
|
|
8712
9034
|
{
|
|
8713
|
-
"id": "study.
|
|
9035
|
+
"id": "study.reconcileRuns",
|
|
8714
9036
|
"commandPath": [
|
|
8715
9037
|
"study",
|
|
8716
|
-
"
|
|
9038
|
+
"reconcile-runs"
|
|
8717
9039
|
],
|
|
8718
9040
|
"method": "post",
|
|
8719
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
9041
|
+
"pathTemplate": "/study/v1/studies/{id}/reconcile-runs",
|
|
9042
|
+
"summary": "Re-derive a study's run projection from the authoritative run rows and re-probe completion (platform-admin only; idempotent) — the repair for runs left non-terminal in the projection while the spine says they finished",
|
|
9043
|
+
"scope": "study.studies:write",
|
|
9044
|
+
"product": "study",
|
|
9045
|
+
"pathParams": [
|
|
9046
|
+
{
|
|
9047
|
+
"name": "id"
|
|
9048
|
+
}
|
|
9049
|
+
],
|
|
9050
|
+
"queryParams": [],
|
|
9051
|
+
"hasBody": false,
|
|
9052
|
+
"bodyRequired": false,
|
|
9053
|
+
"isStream": false,
|
|
9054
|
+
"watchable": false,
|
|
9055
|
+
"help": {
|
|
9056
|
+
"command": "ora study reconcile-runs 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9057
|
+
}
|
|
9058
|
+
},
|
|
9059
|
+
{
|
|
9060
|
+
"id": "study.regenerateSummary",
|
|
9061
|
+
"commandPath": [
|
|
9062
|
+
"study",
|
|
9063
|
+
"regenerate-summary"
|
|
9064
|
+
],
|
|
9065
|
+
"method": "post",
|
|
9066
|
+
"pathTemplate": "/study/v1/studies/{id}/exec-summary",
|
|
9067
|
+
"summary": "Regenerate the executive summary from the current aggregate (async; poll get). Where the product keeps review drafts, a PUBLISHED study's lands as `executiveSummaryDraft` and its public summary does not change until a re-publish makes the draft public — `target` in the answer says where it lands",
|
|
9068
|
+
"scope": "study.studies:write",
|
|
9069
|
+
"product": "study",
|
|
9070
|
+
"pathParams": [
|
|
9071
|
+
{
|
|
9072
|
+
"name": "id"
|
|
9073
|
+
}
|
|
9074
|
+
],
|
|
9075
|
+
"queryParams": [],
|
|
9076
|
+
"hasBody": false,
|
|
9077
|
+
"bodyRequired": false,
|
|
9078
|
+
"isStream": false,
|
|
9079
|
+
"watchable": false,
|
|
9080
|
+
"help": {
|
|
9081
|
+
"command": "ora study regenerate-summary 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9082
|
+
}
|
|
9083
|
+
},
|
|
9084
|
+
{
|
|
9085
|
+
"id": "study.regrade",
|
|
9086
|
+
"commandPath": [
|
|
9087
|
+
"study",
|
|
9088
|
+
"regrade"
|
|
9089
|
+
],
|
|
9090
|
+
"method": "post",
|
|
9091
|
+
"pathTemplate": "/study/v1/studies/{id}/regrade",
|
|
8720
9092
|
"summary": "Clear and re-run every grading lens the study declares for a completed study — the judge panel (rubrics + blind pairwise), the evals checks and the benchmark's own graders",
|
|
8721
9093
|
"scope": "study.studies:write",
|
|
8722
9094
|
"product": "study",
|
|
@@ -8729,20 +9101,589 @@ export const OPERATIONS = [
|
|
|
8729
9101
|
"hasBody": false,
|
|
8730
9102
|
"bodyRequired": false,
|
|
8731
9103
|
"isStream": false,
|
|
8732
|
-
"watchable": true,
|
|
9104
|
+
"watchable": true,
|
|
9105
|
+
"help": {
|
|
9106
|
+
"command": "ora study regrade <id> --watch"
|
|
9107
|
+
}
|
|
9108
|
+
},
|
|
9109
|
+
{
|
|
9110
|
+
"id": "study.rehearsalReport",
|
|
9111
|
+
"commandPath": [
|
|
9112
|
+
"study",
|
|
9113
|
+
"rehearsal-report"
|
|
9114
|
+
],
|
|
9115
|
+
"method": "get",
|
|
9116
|
+
"pathTemplate": "/study/v1/studies/{id}/rehearsal",
|
|
9117
|
+
"summary": "Read what a rehearsal proved",
|
|
9118
|
+
"scope": "study.studies:read",
|
|
9119
|
+
"product": "study",
|
|
9120
|
+
"pathParams": [
|
|
9121
|
+
{
|
|
9122
|
+
"name": "id"
|
|
9123
|
+
}
|
|
9124
|
+
],
|
|
9125
|
+
"queryParams": [],
|
|
9126
|
+
"hasBody": false,
|
|
9127
|
+
"bodyRequired": false,
|
|
9128
|
+
"isStream": false,
|
|
9129
|
+
"watchable": false,
|
|
9130
|
+
"help": {
|
|
9131
|
+
"command": "ora study rehearsal-report <id>"
|
|
9132
|
+
}
|
|
9133
|
+
},
|
|
9134
|
+
{
|
|
9135
|
+
"id": "study.rehearse",
|
|
9136
|
+
"commandPath": [
|
|
9137
|
+
"study",
|
|
9138
|
+
"rehearse"
|
|
9139
|
+
],
|
|
9140
|
+
"method": "post",
|
|
9141
|
+
"pathTemplate": "/study/v1/studies/rehearsals",
|
|
9142
|
+
"summary": "Rehearse a study design: one real run per group, before the study itself",
|
|
9143
|
+
"scope": "study.studies:write",
|
|
9144
|
+
"product": "study",
|
|
9145
|
+
"pathParams": [],
|
|
9146
|
+
"queryParams": [
|
|
9147
|
+
{
|
|
9148
|
+
"name": "tasks",
|
|
9149
|
+
"flag": "tasks",
|
|
9150
|
+
"optionKey": "tasks",
|
|
9151
|
+
"required": false,
|
|
9152
|
+
"kind": "string",
|
|
9153
|
+
"enum": [
|
|
9154
|
+
"1",
|
|
9155
|
+
"2",
|
|
9156
|
+
"3"
|
|
9157
|
+
]
|
|
9158
|
+
},
|
|
9159
|
+
{
|
|
9160
|
+
"name": "mode",
|
|
9161
|
+
"flag": "mode",
|
|
9162
|
+
"optionKey": "mode",
|
|
9163
|
+
"required": false,
|
|
9164
|
+
"kind": "string",
|
|
9165
|
+
"enum": [
|
|
9166
|
+
"live",
|
|
9167
|
+
"setup",
|
|
9168
|
+
"mock"
|
|
9169
|
+
]
|
|
9170
|
+
}
|
|
9171
|
+
],
|
|
9172
|
+
"hasBody": true,
|
|
9173
|
+
"bodyRequired": true,
|
|
9174
|
+
"isStream": false,
|
|
9175
|
+
"bodyFields": [
|
|
9176
|
+
{
|
|
9177
|
+
"name": "name",
|
|
9178
|
+
"required": true,
|
|
9179
|
+
"kind": "string",
|
|
9180
|
+
"description": "Display name for the study (≤160 chars).",
|
|
9181
|
+
"placeholder": "<name>"
|
|
9182
|
+
},
|
|
9183
|
+
{
|
|
9184
|
+
"name": "description",
|
|
9185
|
+
"required": false,
|
|
9186
|
+
"kind": "string",
|
|
9187
|
+
"description": "Optional longer description shown on the study card.",
|
|
9188
|
+
"placeholder": "<description>"
|
|
9189
|
+
},
|
|
9190
|
+
{
|
|
9191
|
+
"name": "subjectKind",
|
|
9192
|
+
"required": false,
|
|
9193
|
+
"kind": "string",
|
|
9194
|
+
"enum": [
|
|
9195
|
+
"skill",
|
|
9196
|
+
"tools",
|
|
9197
|
+
"addon",
|
|
9198
|
+
"custom",
|
|
9199
|
+
"benchmark"
|
|
9200
|
+
],
|
|
9201
|
+
"description": "What the study measures — defaults to 'custom', the kind new studies are created with. See the enum description for each kind's rules.",
|
|
9202
|
+
"placeholder": "custom"
|
|
9203
|
+
},
|
|
9204
|
+
{
|
|
9205
|
+
"name": "skillRef",
|
|
9206
|
+
"required": false,
|
|
9207
|
+
"kind": "string",
|
|
9208
|
+
"description": "The skill under test as '<source>:<id>' — sources: 'custom' (an org-authored skill, id from the skills API), 'builtin' (a starter example), 'git' (a git-imported skill). E.g. 'custom:8c1d…'. REQUIRED iff subjectKind 'skill'; forbidden otherwise. Kept for the studies that already exist: subjectKind 'skill' creates no new study here — a group attaches skills through the skills add-on in `arms[].config.addons`; a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9209
|
+
"placeholder": "<skillRef>"
|
|
9210
|
+
},
|
|
9211
|
+
{
|
|
9212
|
+
"name": "definitionVersionId",
|
|
9213
|
+
"required": false,
|
|
9214
|
+
"kind": "string",
|
|
9215
|
+
"description": "The benchmark definition VERSION this study pins — REQUIRED iff subjectKind 'benchmark', forbidden otherwise. Read it from a catalog entry's `currentVersionId` (GET /benchmark-definitions/{slug}). A VERSION id, never a slug: a slug follows the definition's current version, and pinning is what keeps this study's corpus — and the board row it may later earn — reproducible. The version's tasks are copied FROZEN at create, so a later publish never changes them. Validated as a uuid at the wire: the column is a Postgres `uuid`, so a malformed value would otherwise reach Hasura and answer 500 instead of 422.",
|
|
9216
|
+
"placeholder": "<definitionVersionId>"
|
|
9217
|
+
},
|
|
9218
|
+
{
|
|
9219
|
+
"name": "subjectName",
|
|
9220
|
+
"required": false,
|
|
9221
|
+
"kind": "string",
|
|
9222
|
+
"description": "Display name of a NON-skill subject (e.g. 'Tavily search', 'monday.com remote MCP'). Required for subjectKind 'tools' / 'addon' / 'custom'; forbidden for 'skill' (the skill's own name is used) and for 'benchmark' (the pinned definition's name is used).",
|
|
9223
|
+
"placeholder": "<subjectName>"
|
|
9224
|
+
},
|
|
9225
|
+
{
|
|
9226
|
+
"name": "armConfig",
|
|
9227
|
+
"required": false,
|
|
9228
|
+
"kind": "object",
|
|
9229
|
+
"description": "Frozen per-arm run-payload deltas of a fixed pair of arms (comparisonMode 'ab_vs_none' or 'ab_custom' or 'single'), kept for the studies that already exist: new studies carry each group's configuration in `arms`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9230
|
+
"placeholder": "{}"
|
|
9231
|
+
},
|
|
9232
|
+
{
|
|
9233
|
+
"name": "lenses",
|
|
9234
|
+
"required": false,
|
|
9235
|
+
"kind": "array",
|
|
9236
|
+
"description": "Which grading lenses to run: 'judge' (LLM judge panel — the default) and/or 'evals' (deterministic @ora/eval-checks grading against each task's `expectations`). Set ONCE at create — there is no update path. `expectations` on a task requires 'evals' to be present here.",
|
|
9237
|
+
"placeholder": "[\"judge\"]"
|
|
9238
|
+
},
|
|
9239
|
+
{
|
|
9240
|
+
"name": "judgeSupplementary",
|
|
9241
|
+
"required": false,
|
|
9242
|
+
"kind": "boolean",
|
|
9243
|
+
"description": "subjectKind 'benchmark' only: ALSO grade every run with the LLM judge panel. A rubric is authored for each task at create (the study is 'generating' until it is frozen), and runs are then scored for rubric quality and — in an A/B — compared blind, BESIDE the benchmark's own pass rate, which still decides the verdict first. The rubric measures the quality of a response, never its correctness (that is the benchmark's own grader's job). Costs one judge call per run and per pair. Forbidden for every other subject kind, and for a benchmark whose own grading spec already uses the judge.",
|
|
9244
|
+
"placeholder": "true"
|
|
9245
|
+
},
|
|
9246
|
+
{
|
|
9247
|
+
"name": "proposeExpectations",
|
|
9248
|
+
"required": false,
|
|
9249
|
+
"kind": "boolean",
|
|
9250
|
+
"description": "Opt in to an LLM proposing 'evals'-lens `expectations` for each AUTO-GENERATED task (temp-0, re-validated against the eval-checks registry — an invalid proposal is dropped, never stored). REQUIRES `lenses` to include 'evals' (422 LENS_MISMATCH otherwise); also IGNORED when `tasks` is supplied (an authored corpus already carries its own explicit expectations). Proposals land on the task in the study's normal 'ready' state for human review before launch — the same discipline as generated rubrics.",
|
|
9251
|
+
"placeholder": "true"
|
|
9252
|
+
},
|
|
9253
|
+
{
|
|
9254
|
+
"name": "comparisonMode",
|
|
9255
|
+
"required": false,
|
|
9256
|
+
"kind": "string",
|
|
9257
|
+
"enum": [
|
|
9258
|
+
"ab_vs_none",
|
|
9259
|
+
"ab_custom",
|
|
9260
|
+
"single",
|
|
9261
|
+
"multi_arm"
|
|
9262
|
+
],
|
|
9263
|
+
"description": "The study's design — defaults to 'multi_arm', the mode new studies are created in. See the enum description for each mode.",
|
|
9264
|
+
"placeholder": "multi_arm"
|
|
9265
|
+
},
|
|
9266
|
+
{
|
|
9267
|
+
"name": "subjectBackend",
|
|
9268
|
+
"required": false,
|
|
9269
|
+
"kind": "string",
|
|
9270
|
+
"enum": [
|
|
9271
|
+
"model",
|
|
9272
|
+
"harness"
|
|
9273
|
+
],
|
|
9274
|
+
"description": "FRAMEWORK benchmarks only (e.g. HELM). Omit for the framework's own default — 'harness' where the framework supports it. Ignored by a study no framework runs. An ask the framework cannot honour fails the launch by name.",
|
|
9275
|
+
"placeholder": "model"
|
|
9276
|
+
},
|
|
9277
|
+
{
|
|
9278
|
+
"name": "baselineLabel",
|
|
9279
|
+
"required": false,
|
|
9280
|
+
"kind": "string",
|
|
9281
|
+
"description": "Baseline arm's display label (e.g. 'REST API'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9282
|
+
"placeholder": "<baselineLabel>"
|
|
9283
|
+
},
|
|
9284
|
+
{
|
|
9285
|
+
"name": "treatmentLabel",
|
|
9286
|
+
"required": false,
|
|
9287
|
+
"kind": "string",
|
|
9288
|
+
"description": "Treatment arm's display label (e.g. 'Remote MCP'). Kept for the studies that already exist (Required iff comparisonMode 'ab_custom'; forbidden otherwise): new studies name each group in `arms[].label`, and a create body that names no comparisonMode (or a legacy one) and carries it is read as the legacy body it is and answers 422 comparison_mode_removed (docs/studies/comparison-modes/README.md maps each legacy design onto the new ones).",
|
|
9289
|
+
"placeholder": "<treatmentLabel>"
|
|
9290
|
+
},
|
|
9291
|
+
{
|
|
9292
|
+
"name": "arms",
|
|
9293
|
+
"required": false,
|
|
9294
|
+
"kind": "array",
|
|
9295
|
+
"description": "Declared groups, in display order, each with its own label and configuration. The number of groups is the comparisonMode's own: 'multi_arm': 1-6 groups (1 group reads as the agent leaderboard: no reference group and no comparison, and it seeds the public leaderboard — when bare, labelled 'as published' on a model-API agent and 'harness default' on a harness agent, and 'with my setup' when it carries one; 2-6 groups are compared over the planned contrasts (with 2, one comparison, oriented only when a group has role 'reference'); only a bare group seeds the public leaderboard).",
|
|
9296
|
+
"placeholder": "[]"
|
|
9297
|
+
},
|
|
9298
|
+
{
|
|
9299
|
+
"name": "contrasts",
|
|
9300
|
+
"required": false,
|
|
9301
|
+
"kind": "string",
|
|
9302
|
+
"enum": [
|
|
9303
|
+
"vs_reference",
|
|
9304
|
+
"all_pairs"
|
|
9305
|
+
],
|
|
9306
|
+
"description": "Which pairs of declared groups the judge compares. 'vs_reference': every other group against the one group with role 'reference'. 'all_pairs': every pair of groups. 'multi_arm': 'vs_reference' (every other group against the one group with role 'reference'), 'all_pairs' (every pair of groups, at most 4 groups); default 'vs_reference' when a group has role 'reference', else 'all_pairs'.",
|
|
9307
|
+
"placeholder": "vs_reference"
|
|
9308
|
+
},
|
|
9309
|
+
{
|
|
9310
|
+
"name": "modeOptions",
|
|
9311
|
+
"required": false,
|
|
9312
|
+
"kind": "object",
|
|
9313
|
+
"description": "A comparisonMode's private inputs, validated by that mode:",
|
|
9314
|
+
"placeholder": "{}"
|
|
9315
|
+
},
|
|
9316
|
+
{
|
|
9317
|
+
"name": "taskCount",
|
|
9318
|
+
"required": false,
|
|
9319
|
+
"kind": "number",
|
|
9320
|
+
"description": "How many tasks to auto-generate (1–100, default 5; generation is ONE model call, so very large corpora are better AUTHORED via `tasks`, which allows up to 500). IGNORED when `tasks` is supplied (the authored corpus defines the count). Runs launched = taskCount × variantsPerTask × repeats × matrix cells × arms (2 arms unless 'single'; the declared groups for 'multi_arm'); the total is capped at 10000.",
|
|
9321
|
+
"placeholder": "5"
|
|
9322
|
+
},
|
|
9323
|
+
{
|
|
9324
|
+
"name": "variantsPerTask",
|
|
9325
|
+
"required": false,
|
|
9326
|
+
"kind": "number",
|
|
9327
|
+
"description": "Semantics-preserving paraphrases per task (1-10, default 1). Each variant is a SEPARATE sample of the same question, graded against the same frozen rubric, and both arms see the identical variant set. Sampling phrasing as well as stochastic draw is what lets a comparative claim average over both. Runs = tasks x variants x repeats x cells x arms.",
|
|
9328
|
+
"placeholder": "1"
|
|
9329
|
+
},
|
|
9330
|
+
{
|
|
9331
|
+
"name": "repeats",
|
|
9332
|
+
"required": false,
|
|
9333
|
+
"kind": "number",
|
|
9334
|
+
"description": "Identical re-runs of each condition (1-25, default 1). Variants and repeats measure different things: a failed variant could be phrasing sensitivity OR run-to-run instability, and only identical repeats separate them. Set >1 when reliability is a deliverable.",
|
|
9335
|
+
"placeholder": "1"
|
|
9336
|
+
},
|
|
9337
|
+
{
|
|
9338
|
+
"name": "tasks",
|
|
9339
|
+
"required": false,
|
|
9340
|
+
"kind": "array",
|
|
9341
|
+
"description": "AUTHOR the corpus instead of generating it: tasks + rubrics land verbatim and the study goes straight to 'ready' (no LLM generation; the same pre-registration discipline — rubrics are frozen before any run exists). For reference-driven corpora (a ported study's curated tasks). `promise` is still required for non-skill subjects; `taskCount` is derived from this array. Forbidden for subjectKind 'benchmark', whose corpus is copied frozen from the pinned definition version.",
|
|
9342
|
+
"placeholder": "[]"
|
|
9343
|
+
},
|
|
9344
|
+
{
|
|
9345
|
+
"name": "subset",
|
|
9346
|
+
"required": false,
|
|
9347
|
+
"kind": "object",
|
|
9348
|
+
"description": "Run a deterministic random SAMPLE of the corpus instead of all of it — the way a 1,319-item benchmark becomes a study you can afford. The sample is a pure function of (corpus, count, seed): the server records the seed on the study, so the exact sample is re-derivable forever. Omit it to run every task.",
|
|
9349
|
+
"placeholder": "{}"
|
|
9350
|
+
},
|
|
9351
|
+
{
|
|
9352
|
+
"name": "pacing",
|
|
9353
|
+
"required": false,
|
|
9354
|
+
"kind": "string",
|
|
9355
|
+
"description": "LAUNCH PACING — start this study's runs gradually instead of all at once, for groups that call a rate-limited service (an API that answers 429 past N requests a minute). Every group's run of one agent on one task sample (a comparison unit) always starts together, in the same wave, so a comparison is never confounded by time; each value must therefore be at least the number of groups that run one agent. Declare maxInFlight, startsPerMinute, or both (both then hold). Frozen at create. Omit it, or send null, for no pacing: every run starts at launch.",
|
|
9356
|
+
"placeholder": "<pacing>"
|
|
9357
|
+
},
|
|
9358
|
+
{
|
|
9359
|
+
"name": "screens",
|
|
9360
|
+
"required": false,
|
|
9361
|
+
"kind": "array",
|
|
9362
|
+
"description": "Which run screens this study runs. A screen is a per-run check of whether the run is valid evidence, kept apart from grading: it never changes a grade. Omitted = the deployment's default. Frozen at launch, like lenses. 'test_awareness': after a run ends, a judge panel reads its full trace for signs that the agent noticed it was being tested ('aware') or changed how it worked because of that ('influenced'), and records the quoted evidence with the classifier's version.",
|
|
9363
|
+
"placeholder": "[]"
|
|
9364
|
+
},
|
|
9365
|
+
{
|
|
9366
|
+
"name": "series",
|
|
9367
|
+
"required": false,
|
|
9368
|
+
"kind": "string",
|
|
9369
|
+
"description": "The SERIES this study belongs to: studies that ask the same question at different moments (every commit of a branch). A gate's series baseline is found in it. Immutable. Omit for a study of no series.",
|
|
9370
|
+
"placeholder": "<series>"
|
|
9371
|
+
},
|
|
9372
|
+
{
|
|
9373
|
+
"name": "ref",
|
|
9374
|
+
"required": false,
|
|
9375
|
+
"kind": "object",
|
|
9376
|
+
"description": "Where the study was taken: the commit, branch and pull request. A label, immutable.",
|
|
9377
|
+
"placeholder": "{}"
|
|
9378
|
+
},
|
|
9379
|
+
{
|
|
9380
|
+
"name": "carry",
|
|
9381
|
+
"required": false,
|
|
9382
|
+
"kind": "object",
|
|
9383
|
+
"description": "Carry the runs of jobs that cannot have changed from the series' previous comparable study instead of running them again (needs `series`). Each carried run says so. Omit to run every job.",
|
|
9384
|
+
"placeholder": "{}"
|
|
9385
|
+
},
|
|
9386
|
+
{
|
|
9387
|
+
"name": "runCounting",
|
|
9388
|
+
"required": false,
|
|
9389
|
+
"kind": "string",
|
|
9390
|
+
"placeholder": "<runCounting>"
|
|
9391
|
+
},
|
|
9392
|
+
{
|
|
9393
|
+
"name": "behaviors",
|
|
9394
|
+
"required": false,
|
|
9395
|
+
"kind": "array",
|
|
9396
|
+
"description": "Custom behaviors: up to 5 yes-or-no questions a judge panel answers about every finished run, with quoted evidence. A behavior is disclosure about a run, like a screen: it has no score and never changes a grade. Frozen at launch. Each costs one panel read per run.",
|
|
9397
|
+
"placeholder": "[]"
|
|
9398
|
+
},
|
|
9399
|
+
{
|
|
9400
|
+
"name": "matrix",
|
|
9401
|
+
"required": true,
|
|
9402
|
+
"kind": "array",
|
|
9403
|
+
"description": "The (harness, model) cells to fan out over — 1 to 48 cells (a full catalog sweep fits).",
|
|
9404
|
+
"placeholder": "[]"
|
|
9405
|
+
},
|
|
9406
|
+
{
|
|
9407
|
+
"name": "promise",
|
|
9408
|
+
"required": false,
|
|
9409
|
+
"kind": "string",
|
|
9410
|
+
"description": "The claim under test, driving task generation and the pre-registered rubrics (e.g. 'Tavily search finds fresher, better-sourced answers than no web access'). Optional for subjectKind 'skill' (omitted → extracted from the skill body); forbidden for 'benchmark' (derived from the pinned definition's description); REQUIRED for every other kind.",
|
|
9411
|
+
"placeholder": "<promise>"
|
|
9412
|
+
}
|
|
9413
|
+
],
|
|
9414
|
+
"watchable": false,
|
|
9415
|
+
"help": {
|
|
9416
|
+
"command": "ora study rehearse --file run.json",
|
|
9417
|
+
"body": "{\n \"name\": \"<name>\",\n \"matrix\": []\n}"
|
|
9418
|
+
}
|
|
9419
|
+
},
|
|
9420
|
+
{
|
|
9421
|
+
"id": "study.rerunAttempts",
|
|
9422
|
+
"commandPath": [
|
|
9423
|
+
"study",
|
|
9424
|
+
"rerun-attempts"
|
|
9425
|
+
],
|
|
9426
|
+
"method": "get",
|
|
9427
|
+
"pathTemplate": "/study/v1/studies/{id}/rerun-attempts",
|
|
9428
|
+
"summary": "The study's re-run ledger: every in-place re-dispatch of one of its jobs — the run it replaced and how that run ended, why it was re-run (with a declared re-run's evidence), the run it created and whether it dispatched. Oldest first.",
|
|
9429
|
+
"scope": "study.studies:read",
|
|
9430
|
+
"product": "study",
|
|
9431
|
+
"pathParams": [
|
|
9432
|
+
{
|
|
9433
|
+
"name": "id"
|
|
9434
|
+
}
|
|
9435
|
+
],
|
|
9436
|
+
"queryParams": [],
|
|
9437
|
+
"hasBody": false,
|
|
9438
|
+
"bodyRequired": false,
|
|
9439
|
+
"isStream": false,
|
|
9440
|
+
"watchable": false,
|
|
9441
|
+
"help": {
|
|
9442
|
+
"command": "ora study rerun-attempts 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9443
|
+
}
|
|
9444
|
+
},
|
|
9445
|
+
{
|
|
9446
|
+
"id": "study.rerunInfraFailures",
|
|
9447
|
+
"commandPath": [
|
|
9448
|
+
"study",
|
|
9449
|
+
"rerun-infra-failures"
|
|
9450
|
+
],
|
|
9451
|
+
"method": "post",
|
|
9452
|
+
"pathTemplate": "/study/v1/studies/{id}/rerun-infra-failures",
|
|
9453
|
+
"summary": "Re-run, in place, the jobs of a running or completed-but-unpublished study whose run ora's own infrastructure killed (or that the launch never created): each new run takes its job's place, the failed run stays in the record, at most 2 re-runs per job, never while the job's run is still running. A completed study re-opens until its re-runs finish and are graded. Synchronous; dryRun answers the plan.",
|
|
9454
|
+
"scope": "study.studies:write",
|
|
9455
|
+
"product": "study",
|
|
9456
|
+
"pathParams": [
|
|
9457
|
+
{
|
|
9458
|
+
"name": "id"
|
|
9459
|
+
}
|
|
9460
|
+
],
|
|
9461
|
+
"queryParams": [],
|
|
9462
|
+
"hasBody": true,
|
|
9463
|
+
"bodyRequired": true,
|
|
9464
|
+
"isStream": false,
|
|
9465
|
+
"bodyFields": [
|
|
9466
|
+
{
|
|
9467
|
+
"name": "dryRun",
|
|
9468
|
+
"required": false,
|
|
9469
|
+
"kind": "boolean",
|
|
9470
|
+
"description": "Answer what WOULD be re-run, dispatching and writing nothing (default false).",
|
|
9471
|
+
"placeholder": "true"
|
|
9472
|
+
},
|
|
9473
|
+
{
|
|
9474
|
+
"name": "limit",
|
|
9475
|
+
"required": false,
|
|
9476
|
+
"kind": "number",
|
|
9477
|
+
"description": "The most jobs this call dispatches (default 200, or every declared run); the rest are counted in `skipped.overLimit` for the next call.",
|
|
9478
|
+
"placeholder": "1"
|
|
9479
|
+
},
|
|
9480
|
+
{
|
|
9481
|
+
"name": "declared",
|
|
9482
|
+
"required": false,
|
|
9483
|
+
"kind": "array",
|
|
9484
|
+
"description": "The DECLARED re-run: exactly these runs are re-run in place (cause declared_infra), each with its evidence — runs ora's own predicate does not flag. Absent: ora's own infra failures and never-created jobs.",
|
|
9485
|
+
"placeholder": "[]"
|
|
9486
|
+
}
|
|
9487
|
+
],
|
|
9488
|
+
"watchable": false,
|
|
9489
|
+
"help": {
|
|
9490
|
+
"command": "ora study rerun-infra-failures 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f --file run.json",
|
|
9491
|
+
"body": "{\n \"dryRun\": true\n}"
|
|
9492
|
+
}
|
|
9493
|
+
},
|
|
9494
|
+
{
|
|
9495
|
+
"id": "study.restore",
|
|
9496
|
+
"commandPath": [
|
|
9497
|
+
"study",
|
|
9498
|
+
"restore"
|
|
9499
|
+
],
|
|
9500
|
+
"method": "post",
|
|
9501
|
+
"pathTemplate": "/study/v1/studies/{id}/restore",
|
|
9502
|
+
"summary": "Restore an archived study: it returns to the default list and can be changed again. It comes back unpublished (archiving a shared study unpublishes it first); publishing again is a separate, deliberate call. Restoring a study that is not archived succeeds and changes nothing.",
|
|
9503
|
+
"scope": "study.studies:write",
|
|
9504
|
+
"product": "study",
|
|
9505
|
+
"pathParams": [
|
|
9506
|
+
{
|
|
9507
|
+
"name": "id"
|
|
9508
|
+
}
|
|
9509
|
+
],
|
|
9510
|
+
"queryParams": [],
|
|
9511
|
+
"hasBody": false,
|
|
9512
|
+
"bodyRequired": false,
|
|
9513
|
+
"isStream": false,
|
|
9514
|
+
"watchable": false,
|
|
9515
|
+
"help": {
|
|
9516
|
+
"command": "ora study restore 1f7e4d1c-4b2a-4f3e-9d8c-5a6b7c8d9e0f"
|
|
9517
|
+
}
|
|
9518
|
+
},
|
|
9519
|
+
{
|
|
9520
|
+
"id": "study.retest",
|
|
9521
|
+
"commandPath": [
|
|
9522
|
+
"study",
|
|
9523
|
+
"retest"
|
|
9524
|
+
],
|
|
9525
|
+
"method": "post",
|
|
9526
|
+
"pathTemplate": "/study/v1/studies/{id}/retests",
|
|
9527
|
+
"summary": "Re-check runs' saved files with the benchmark's own checker",
|
|
9528
|
+
"scope": "study.studies:write",
|
|
9529
|
+
"product": "study",
|
|
9530
|
+
"pathParams": [
|
|
9531
|
+
{
|
|
9532
|
+
"name": "id"
|
|
9533
|
+
}
|
|
9534
|
+
],
|
|
9535
|
+
"queryParams": [],
|
|
9536
|
+
"hasBody": true,
|
|
9537
|
+
"bodyRequired": true,
|
|
9538
|
+
"isStream": false,
|
|
9539
|
+
"bodyFields": [
|
|
9540
|
+
{
|
|
9541
|
+
"name": "kind",
|
|
9542
|
+
"required": false,
|
|
9543
|
+
"kind": "string",
|
|
9544
|
+
"enum": [
|
|
9545
|
+
"fill",
|
|
9546
|
+
"audit"
|
|
9547
|
+
],
|
|
9548
|
+
"description": "fill (the default): re-check every run whose verifier verdict was never measured; a verdict the re-check measures becomes the run's grade. audit: re-check runs and record the second verdict beside the first; no grade changes.",
|
|
9549
|
+
"placeholder": "fill"
|
|
9550
|
+
},
|
|
9551
|
+
{
|
|
9552
|
+
"name": "runIds",
|
|
9553
|
+
"required": false,
|
|
9554
|
+
"kind": "array",
|
|
9555
|
+
"description": "audit only: the runs to re-check. Absent = every run with a measured verdict.",
|
|
9556
|
+
"placeholder": "[]"
|
|
9557
|
+
},
|
|
9558
|
+
{
|
|
9559
|
+
"name": "reason",
|
|
9560
|
+
"required": false,
|
|
9561
|
+
"kind": "string",
|
|
9562
|
+
"description": "Why the runs are being re-checked. Kept with the request.",
|
|
9563
|
+
"placeholder": "<reason>"
|
|
9564
|
+
},
|
|
9565
|
+
{
|
|
9566
|
+
"name": "dryRun",
|
|
9567
|
+
"required": false,
|
|
9568
|
+
"kind": "boolean",
|
|
9569
|
+
"description": "true = create nothing: answer which runs would be re-checked, and which could not be (their saved files are gone).",
|
|
9570
|
+
"placeholder": "true"
|
|
9571
|
+
}
|
|
9572
|
+
],
|
|
9573
|
+
"watchable": false,
|
|
9574
|
+
"help": {
|
|
9575
|
+
"command": "ora study retest <id> --file run.json",
|
|
9576
|
+
"body": "{\n \"kind\": \"fill\",\n \"runIds\": [],\n \"reason\": \"<reason>\"\n}"
|
|
9577
|
+
}
|
|
9578
|
+
},
|
|
9579
|
+
{
|
|
9580
|
+
"id": "study.retests",
|
|
9581
|
+
"commandPath": [
|
|
9582
|
+
"study",
|
|
9583
|
+
"retests"
|
|
9584
|
+
],
|
|
9585
|
+
"method": "get",
|
|
9586
|
+
"pathTemplate": "/study/v1/studies/{id}/retests",
|
|
9587
|
+
"summary": "List a study's re-checks",
|
|
9588
|
+
"scope": "study.studies:read",
|
|
9589
|
+
"product": "study",
|
|
9590
|
+
"pathParams": [
|
|
9591
|
+
{
|
|
9592
|
+
"name": "id"
|
|
9593
|
+
}
|
|
9594
|
+
],
|
|
9595
|
+
"queryParams": [],
|
|
9596
|
+
"hasBody": false,
|
|
9597
|
+
"bodyRequired": false,
|
|
9598
|
+
"isStream": false,
|
|
9599
|
+
"watchable": false,
|
|
9600
|
+
"help": {
|
|
9601
|
+
"command": "ora study retests <id>"
|
|
9602
|
+
}
|
|
9603
|
+
},
|
|
9604
|
+
{
|
|
9605
|
+
"id": "study.reviseGrading",
|
|
9606
|
+
"commandPath": [
|
|
9607
|
+
"study",
|
|
9608
|
+
"revise-grading"
|
|
9609
|
+
],
|
|
9610
|
+
"method": "post",
|
|
9611
|
+
"pathTemplate": "/study/v1/studies/{id}/grading-revisions",
|
|
9612
|
+
"summary": "Revise a finished study's checks and re-grade it",
|
|
9613
|
+
"scope": "study.studies:write",
|
|
9614
|
+
"product": "study",
|
|
9615
|
+
"pathParams": [
|
|
9616
|
+
{
|
|
9617
|
+
"name": "id"
|
|
9618
|
+
}
|
|
9619
|
+
],
|
|
9620
|
+
"queryParams": [],
|
|
9621
|
+
"hasBody": true,
|
|
9622
|
+
"bodyRequired": true,
|
|
9623
|
+
"isStream": false,
|
|
9624
|
+
"bodyFields": [
|
|
9625
|
+
{
|
|
9626
|
+
"name": "reason",
|
|
9627
|
+
"required": true,
|
|
9628
|
+
"kind": "string",
|
|
9629
|
+
"description": "Why the checks are changing. Kept in the revision's record and shown beside it.",
|
|
9630
|
+
"placeholder": "<reason>"
|
|
9631
|
+
},
|
|
9632
|
+
{
|
|
9633
|
+
"name": "tasks",
|
|
9634
|
+
"required": true,
|
|
9635
|
+
"kind": "array",
|
|
9636
|
+
"description": "The tasks whose checks change.",
|
|
9637
|
+
"placeholder": "[]"
|
|
9638
|
+
},
|
|
9639
|
+
{
|
|
9640
|
+
"name": "dryRun",
|
|
9641
|
+
"required": false,
|
|
9642
|
+
"kind": "boolean",
|
|
9643
|
+
"description": "true = change nothing: answer what the revision would change, and which runs' deterministic checks would flip.",
|
|
9644
|
+
"placeholder": "true"
|
|
9645
|
+
}
|
|
9646
|
+
],
|
|
9647
|
+
"watchable": false,
|
|
9648
|
+
"help": {
|
|
9649
|
+
"command": "ora study revise-grading <id> --file run.json",
|
|
9650
|
+
"body": "{\n \"reason\": \"<reason>\",\n \"tasks\": []\n}"
|
|
9651
|
+
}
|
|
9652
|
+
},
|
|
9653
|
+
{
|
|
9654
|
+
"id": "study.rotateShareToken",
|
|
9655
|
+
"commandPath": [
|
|
9656
|
+
"study",
|
|
9657
|
+
"rotate-share-token"
|
|
9658
|
+
],
|
|
9659
|
+
"method": "post",
|
|
9660
|
+
"pathTemplate": "/study/v1/studies/{id}/share-token/rotate",
|
|
9661
|
+
"summary": "Replace the study's share link with a fresh one. The previous link stops working IMMEDIATELY — there is no grace window — and never works again. Works in ANY mode: a study that left 'link' keeps its link, switched off (`shareLinkKept`), and replacing it then retires a leaked link without opening it first (the new one opens only once the study is back in 'link'). This is the ONLY way a study's link changes.",
|
|
9662
|
+
"scope": "study.studies:write",
|
|
9663
|
+
"product": "study",
|
|
9664
|
+
"pathParams": [
|
|
9665
|
+
{
|
|
9666
|
+
"name": "id"
|
|
9667
|
+
}
|
|
9668
|
+
],
|
|
9669
|
+
"queryParams": [],
|
|
9670
|
+
"hasBody": false,
|
|
9671
|
+
"bodyRequired": false,
|
|
9672
|
+
"isStream": false,
|
|
9673
|
+
"watchable": false,
|
|
8733
9674
|
"help": {
|
|
8734
|
-
"command": "ora study
|
|
9675
|
+
"command": "ora study rotate-share-token <id>"
|
|
8735
9676
|
}
|
|
8736
9677
|
},
|
|
8737
9678
|
{
|
|
8738
|
-
"id": "study.
|
|
9679
|
+
"id": "study.runCounting",
|
|
8739
9680
|
"commandPath": [
|
|
8740
9681
|
"study",
|
|
8741
|
-
"
|
|
9682
|
+
"run-counting"
|
|
8742
9683
|
],
|
|
8743
9684
|
"method": "get",
|
|
8744
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8745
|
-
"summary": "
|
|
9685
|
+
"pathTemplate": "/study/v1/studies/{id}/run-counting",
|
|
9686
|
+
"summary": "Which of a study's runs its numbers count",
|
|
8746
9687
|
"scope": "study.studies:read",
|
|
8747
9688
|
"product": "study",
|
|
8748
9689
|
"pathParams": [
|
|
@@ -8756,72 +9697,82 @@ export const OPERATIONS = [
|
|
|
8756
9697
|
"isStream": false,
|
|
8757
9698
|
"watchable": false,
|
|
8758
9699
|
"help": {
|
|
8759
|
-
"command": "ora study
|
|
9700
|
+
"command": "ora study run-counting <id>"
|
|
8760
9701
|
}
|
|
8761
9702
|
},
|
|
8762
9703
|
{
|
|
8763
|
-
"id": "study.
|
|
9704
|
+
"id": "study.runDelivered",
|
|
8764
9705
|
"commandPath": [
|
|
8765
9706
|
"study",
|
|
8766
|
-
"
|
|
9707
|
+
"run-delivered"
|
|
8767
9708
|
],
|
|
8768
|
-
"method": "
|
|
8769
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8770
|
-
"summary": "
|
|
8771
|
-
"scope": "study.studies:
|
|
9709
|
+
"method": "get",
|
|
9710
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/delivered",
|
|
9711
|
+
"summary": "List what a study run delivered: the files and images ora keeps a copy of",
|
|
9712
|
+
"scope": "study.studies:read",
|
|
8772
9713
|
"product": "study",
|
|
8773
9714
|
"pathParams": [
|
|
8774
9715
|
{
|
|
8775
9716
|
"name": "id"
|
|
9717
|
+
},
|
|
9718
|
+
{
|
|
9719
|
+
"name": "runId"
|
|
8776
9720
|
}
|
|
8777
9721
|
],
|
|
8778
9722
|
"queryParams": [],
|
|
8779
|
-
"hasBody":
|
|
8780
|
-
"bodyRequired":
|
|
9723
|
+
"hasBody": false,
|
|
9724
|
+
"bodyRequired": false,
|
|
8781
9725
|
"isStream": false,
|
|
8782
|
-
"
|
|
8783
|
-
|
|
8784
|
-
|
|
8785
|
-
|
|
8786
|
-
|
|
8787
|
-
|
|
8788
|
-
|
|
8789
|
-
|
|
9726
|
+
"watchable": false,
|
|
9727
|
+
"help": {
|
|
9728
|
+
"command": "ora study run-delivered <id> <runId>"
|
|
9729
|
+
}
|
|
9730
|
+
},
|
|
9731
|
+
{
|
|
9732
|
+
"id": "study.runInsight",
|
|
9733
|
+
"commandPath": [
|
|
9734
|
+
"study",
|
|
9735
|
+
"run-insight"
|
|
9736
|
+
],
|
|
9737
|
+
"method": "get",
|
|
9738
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/insight",
|
|
9739
|
+
"summary": "Read a study run's full insight with the study's access permissions",
|
|
9740
|
+
"scope": "study.studies:read",
|
|
9741
|
+
"product": "study",
|
|
9742
|
+
"pathParams": [
|
|
8790
9743
|
{
|
|
8791
|
-
"name": "
|
|
8792
|
-
"required": false,
|
|
8793
|
-
"kind": "number",
|
|
8794
|
-
"description": "The most jobs this call dispatches (default 200, or every declared run); the rest are counted in `skipped.overLimit` for the next call.",
|
|
8795
|
-
"placeholder": "1"
|
|
9744
|
+
"name": "id"
|
|
8796
9745
|
},
|
|
8797
9746
|
{
|
|
8798
|
-
"name": "
|
|
8799
|
-
"required": false,
|
|
8800
|
-
"kind": "array",
|
|
8801
|
-
"description": "The DECLARED re-run: exactly these runs are re-run in place (cause declared_infra), each with its evidence — runs ora's own predicate does not flag. Absent: ora's own infra failures and never-created jobs.",
|
|
8802
|
-
"placeholder": "[]"
|
|
9747
|
+
"name": "runId"
|
|
8803
9748
|
}
|
|
8804
9749
|
],
|
|
9750
|
+
"queryParams": [],
|
|
9751
|
+
"hasBody": false,
|
|
9752
|
+
"bodyRequired": false,
|
|
9753
|
+
"isStream": false,
|
|
8805
9754
|
"watchable": false,
|
|
8806
9755
|
"help": {
|
|
8807
|
-
"command": "ora study
|
|
8808
|
-
"body": "{\n \"dryRun\": true\n}"
|
|
9756
|
+
"command": "ora study run-insight <id> <runId>"
|
|
8809
9757
|
}
|
|
8810
9758
|
},
|
|
8811
9759
|
{
|
|
8812
|
-
"id": "study.
|
|
9760
|
+
"id": "study.runScreens",
|
|
8813
9761
|
"commandPath": [
|
|
8814
9762
|
"study",
|
|
8815
|
-
"
|
|
9763
|
+
"run-screens"
|
|
8816
9764
|
],
|
|
8817
|
-
"method": "
|
|
8818
|
-
"pathTemplate": "/study/v1/studies/{id}/
|
|
8819
|
-
"summary": "
|
|
8820
|
-
"scope": "study.studies:
|
|
9765
|
+
"method": "get",
|
|
9766
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/screens",
|
|
9767
|
+
"summary": "Read a study run's screen verdicts with their evidence",
|
|
9768
|
+
"scope": "study.studies:read",
|
|
8821
9769
|
"product": "study",
|
|
8822
9770
|
"pathParams": [
|
|
8823
9771
|
{
|
|
8824
9772
|
"name": "id"
|
|
9773
|
+
},
|
|
9774
|
+
{
|
|
9775
|
+
"name": "runId"
|
|
8825
9776
|
}
|
|
8826
9777
|
],
|
|
8827
9778
|
"queryParams": [],
|
|
@@ -8830,18 +9781,18 @@ export const OPERATIONS = [
|
|
|
8830
9781
|
"isStream": false,
|
|
8831
9782
|
"watchable": false,
|
|
8832
9783
|
"help": {
|
|
8833
|
-
"command": "ora study
|
|
9784
|
+
"command": "ora study run-screens <id> <runId>"
|
|
8834
9785
|
}
|
|
8835
9786
|
},
|
|
8836
9787
|
{
|
|
8837
|
-
"id": "study.
|
|
9788
|
+
"id": "study.runSpans",
|
|
8838
9789
|
"commandPath": [
|
|
8839
9790
|
"study",
|
|
8840
|
-
"run-
|
|
9791
|
+
"run-spans"
|
|
8841
9792
|
],
|
|
8842
9793
|
"method": "get",
|
|
8843
|
-
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/
|
|
8844
|
-
"summary": "Read a study run's
|
|
9794
|
+
"pathTemplate": "/study/v1/studies/{id}/runs/{runId}/spans",
|
|
9795
|
+
"summary": "Read a study run's stored spans with the study's access permissions",
|
|
8845
9796
|
"scope": "study.studies:read",
|
|
8846
9797
|
"product": "study",
|
|
8847
9798
|
"pathParams": [
|
|
@@ -8852,13 +9803,21 @@ export const OPERATIONS = [
|
|
|
8852
9803
|
"name": "runId"
|
|
8853
9804
|
}
|
|
8854
9805
|
],
|
|
8855
|
-
"queryParams": [
|
|
9806
|
+
"queryParams": [
|
|
9807
|
+
{
|
|
9808
|
+
"name": "categories",
|
|
9809
|
+
"flag": "categories",
|
|
9810
|
+
"optionKey": "categories",
|
|
9811
|
+
"required": false,
|
|
9812
|
+
"kind": "string"
|
|
9813
|
+
}
|
|
9814
|
+
],
|
|
8856
9815
|
"hasBody": false,
|
|
8857
9816
|
"bodyRequired": false,
|
|
8858
9817
|
"isStream": false,
|
|
8859
9818
|
"watchable": false,
|
|
8860
9819
|
"help": {
|
|
8861
|
-
"command": "ora study run-
|
|
9820
|
+
"command": "ora study run-spans <id> <runId>"
|
|
8862
9821
|
}
|
|
8863
9822
|
},
|
|
8864
9823
|
{
|
|
@@ -8902,6 +9861,34 @@ export const OPERATIONS = [
|
|
|
8902
9861
|
"command": "ora study run-trajectory <id> <runId>"
|
|
8903
9862
|
}
|
|
8904
9863
|
},
|
|
9864
|
+
{
|
|
9865
|
+
"id": "study.runView",
|
|
9866
|
+
"commandPath": [
|
|
9867
|
+
"study",
|
|
9868
|
+
"run-view"
|
|
9869
|
+
],
|
|
9870
|
+
"method": "get",
|
|
9871
|
+
"pathTemplate": "/study/v1/studies/{id}/views/{viewId}",
|
|
9872
|
+
"summary": "Open a saved view: its question, answered from the study's runs as they are now",
|
|
9873
|
+
"scope": "study.studies:read",
|
|
9874
|
+
"product": "study",
|
|
9875
|
+
"pathParams": [
|
|
9876
|
+
{
|
|
9877
|
+
"name": "id"
|
|
9878
|
+
},
|
|
9879
|
+
{
|
|
9880
|
+
"name": "viewId"
|
|
9881
|
+
}
|
|
9882
|
+
],
|
|
9883
|
+
"queryParams": [],
|
|
9884
|
+
"hasBody": false,
|
|
9885
|
+
"bodyRequired": false,
|
|
9886
|
+
"isStream": false,
|
|
9887
|
+
"watchable": false,
|
|
9888
|
+
"help": {
|
|
9889
|
+
"command": "ora study run-view <id> <viewId>"
|
|
9890
|
+
}
|
|
9891
|
+
},
|
|
8905
9892
|
{
|
|
8906
9893
|
"id": "study.runs.cancel",
|
|
8907
9894
|
"commandPath": [
|
|
@@ -9272,6 +10259,160 @@ export const OPERATIONS = [
|
|
|
9272
10259
|
"command": "ora study runs trajectory <id>"
|
|
9273
10260
|
}
|
|
9274
10261
|
},
|
|
10262
|
+
{
|
|
10263
|
+
"id": "study.saveView",
|
|
10264
|
+
"commandPath": [
|
|
10265
|
+
"study",
|
|
10266
|
+
"save-view"
|
|
10267
|
+
],
|
|
10268
|
+
"method": "post",
|
|
10269
|
+
"pathTemplate": "/study/v1/studies/{id}/views",
|
|
10270
|
+
"summary": "Save a run query as a named view of the study",
|
|
10271
|
+
"scope": "study.studies:write",
|
|
10272
|
+
"product": "study",
|
|
10273
|
+
"pathParams": [
|
|
10274
|
+
{
|
|
10275
|
+
"name": "id"
|
|
10276
|
+
}
|
|
10277
|
+
],
|
|
10278
|
+
"queryParams": [],
|
|
10279
|
+
"hasBody": true,
|
|
10280
|
+
"bodyRequired": true,
|
|
10281
|
+
"isStream": false,
|
|
10282
|
+
"bodyFields": [
|
|
10283
|
+
{
|
|
10284
|
+
"name": "name",
|
|
10285
|
+
"required": true,
|
|
10286
|
+
"kind": "string",
|
|
10287
|
+
"placeholder": "<name>"
|
|
10288
|
+
},
|
|
10289
|
+
{
|
|
10290
|
+
"name": "query",
|
|
10291
|
+
"required": true,
|
|
10292
|
+
"kind": "object",
|
|
10293
|
+
"placeholder": "{}"
|
|
10294
|
+
}
|
|
10295
|
+
],
|
|
10296
|
+
"watchable": false,
|
|
10297
|
+
"help": {
|
|
10298
|
+
"command": "ora study save-view <id> --file run.json",
|
|
10299
|
+
"body": "{\n \"name\": \"<name>\",\n \"query\": {}\n}"
|
|
10300
|
+
}
|
|
10301
|
+
},
|
|
10302
|
+
{
|
|
10303
|
+
"id": "study.series",
|
|
10304
|
+
"commandPath": [
|
|
10305
|
+
"study",
|
|
10306
|
+
"series"
|
|
10307
|
+
],
|
|
10308
|
+
"method": "get",
|
|
10309
|
+
"pathTemplate": "/study/v1/studies/series",
|
|
10310
|
+
"summary": "List the organization's study series",
|
|
10311
|
+
"scope": "study.studies:read",
|
|
10312
|
+
"product": "study",
|
|
10313
|
+
"pathParams": [],
|
|
10314
|
+
"queryParams": [],
|
|
10315
|
+
"hasBody": false,
|
|
10316
|
+
"bodyRequired": false,
|
|
10317
|
+
"isStream": false,
|
|
10318
|
+
"watchable": false,
|
|
10319
|
+
"help": {
|
|
10320
|
+
"command": "ora study series"
|
|
10321
|
+
}
|
|
10322
|
+
},
|
|
10323
|
+
{
|
|
10324
|
+
"id": "study.seriesTrend",
|
|
10325
|
+
"commandPath": [
|
|
10326
|
+
"study",
|
|
10327
|
+
"series-trend"
|
|
10328
|
+
],
|
|
10329
|
+
"method": "get",
|
|
10330
|
+
"pathTemplate": "/study/v1/studies/series/trend",
|
|
10331
|
+
"summary": "Read one series as a trend: each study's figure per group, split where the studies stop being comparable",
|
|
10332
|
+
"scope": "study.studies:read",
|
|
10333
|
+
"product": "study",
|
|
10334
|
+
"pathParams": [],
|
|
10335
|
+
"queryParams": [
|
|
10336
|
+
{
|
|
10337
|
+
"name": "series",
|
|
10338
|
+
"flag": "series",
|
|
10339
|
+
"optionKey": "series",
|
|
10340
|
+
"required": true,
|
|
10341
|
+
"kind": "string"
|
|
10342
|
+
},
|
|
10343
|
+
{
|
|
10344
|
+
"name": "metric",
|
|
10345
|
+
"flag": "metric",
|
|
10346
|
+
"optionKey": "metric",
|
|
10347
|
+
"required": false,
|
|
10348
|
+
"kind": "string",
|
|
10349
|
+
"enum": [
|
|
10350
|
+
"benchmark",
|
|
10351
|
+
"goal",
|
|
10352
|
+
"evals",
|
|
10353
|
+
"quality",
|
|
10354
|
+
"rubric",
|
|
10355
|
+
"completion"
|
|
10356
|
+
]
|
|
10357
|
+
},
|
|
10358
|
+
{
|
|
10359
|
+
"name": "branch",
|
|
10360
|
+
"flag": "branch",
|
|
10361
|
+
"optionKey": "branch",
|
|
10362
|
+
"required": false,
|
|
10363
|
+
"kind": "string"
|
|
10364
|
+
}
|
|
10365
|
+
],
|
|
10366
|
+
"hasBody": false,
|
|
10367
|
+
"bodyRequired": false,
|
|
10368
|
+
"isStream": false,
|
|
10369
|
+
"watchable": false,
|
|
10370
|
+
"help": {
|
|
10371
|
+
"command": "ora study series-trend --series <series>"
|
|
10372
|
+
}
|
|
10373
|
+
},
|
|
10374
|
+
{
|
|
10375
|
+
"id": "study.setRunCounting",
|
|
10376
|
+
"commandPath": [
|
|
10377
|
+
"study",
|
|
10378
|
+
"set-run-counting"
|
|
10379
|
+
],
|
|
10380
|
+
"method": "put",
|
|
10381
|
+
"pathTemplate": "/study/v1/studies/{id}/run-counting",
|
|
10382
|
+
"summary": "Hold back the runs a screen flagged, or count every run again",
|
|
10383
|
+
"scope": "study.studies:write",
|
|
10384
|
+
"product": "study",
|
|
10385
|
+
"pathParams": [
|
|
10386
|
+
{
|
|
10387
|
+
"name": "id"
|
|
10388
|
+
}
|
|
10389
|
+
],
|
|
10390
|
+
"queryParams": [],
|
|
10391
|
+
"hasBody": true,
|
|
10392
|
+
"bodyRequired": true,
|
|
10393
|
+
"isStream": false,
|
|
10394
|
+
"bodyFields": [
|
|
10395
|
+
{
|
|
10396
|
+
"name": "runCounting",
|
|
10397
|
+
"required": true,
|
|
10398
|
+
"kind": "string",
|
|
10399
|
+
"description": "The policy the study takes. null = count every run.",
|
|
10400
|
+
"placeholder": "<runCounting>"
|
|
10401
|
+
},
|
|
10402
|
+
{
|
|
10403
|
+
"name": "dryRun",
|
|
10404
|
+
"required": false,
|
|
10405
|
+
"kind": "boolean",
|
|
10406
|
+
"description": "true = change nothing and answer the state as it would be: what would be held back, and every group's figure both ways.",
|
|
10407
|
+
"placeholder": "true"
|
|
10408
|
+
}
|
|
10409
|
+
],
|
|
10410
|
+
"watchable": false,
|
|
10411
|
+
"help": {
|
|
10412
|
+
"command": "ora study set-run-counting <id> --file run.json",
|
|
10413
|
+
"body": "{\n \"runCounting\": \"<runCounting>\"\n}"
|
|
10414
|
+
}
|
|
10415
|
+
},
|
|
9275
10416
|
{
|
|
9276
10417
|
"id": "study.setVisibility",
|
|
9277
10418
|
"commandPath": [
|
|
@@ -9655,6 +10796,31 @@ export const OPERATIONS = [
|
|
|
9655
10796
|
"body": "{\n \"prompt\": \"Research the three largest EU battery-storage tenders announced this quarter and cite each source.\"\n}"
|
|
9656
10797
|
}
|
|
9657
10798
|
},
|
|
10799
|
+
{
|
|
10800
|
+
"id": "study.views",
|
|
10801
|
+
"commandPath": [
|
|
10802
|
+
"study",
|
|
10803
|
+
"views"
|
|
10804
|
+
],
|
|
10805
|
+
"method": "get",
|
|
10806
|
+
"pathTemplate": "/study/v1/studies/{id}/views",
|
|
10807
|
+
"summary": "List a study's saved run queries (views)",
|
|
10808
|
+
"scope": "study.studies:read",
|
|
10809
|
+
"product": "study",
|
|
10810
|
+
"pathParams": [
|
|
10811
|
+
{
|
|
10812
|
+
"name": "id"
|
|
10813
|
+
}
|
|
10814
|
+
],
|
|
10815
|
+
"queryParams": [],
|
|
10816
|
+
"hasBody": false,
|
|
10817
|
+
"bodyRequired": false,
|
|
10818
|
+
"isStream": false,
|
|
10819
|
+
"watchable": false,
|
|
10820
|
+
"help": {
|
|
10821
|
+
"command": "ora study views <id>"
|
|
10822
|
+
}
|
|
10823
|
+
},
|
|
9658
10824
|
{
|
|
9659
10825
|
"id": "tunnels.create",
|
|
9660
10826
|
"commandPath": [
|