@ora-ai/cli 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -1
- package/dist/auth-commands.d.ts.map +1 -1
- package/dist/auth-commands.js +12 -1
- package/dist/auth-commands.js.map +1 -1
- package/dist/core/auth.d.ts +24 -13
- package/dist/core/auth.d.ts.map +1 -1
- package/dist/core/auth.js +26 -15
- package/dist/core/auth.js.map +1 -1
- package/dist/core/credential-sources.d.ts +45 -0
- package/dist/core/credential-sources.d.ts.map +1 -0
- package/dist/core/credential-sources.js +65 -0
- package/dist/core/credential-sources.js.map +1 -0
- package/dist/core/github-actions-sign-in.d.ts +33 -0
- package/dist/core/github-actions-sign-in.d.ts.map +1 -0
- package/dist/core/github-actions-sign-in.js +54 -0
- package/dist/core/github-actions-sign-in.js.map +1 -0
- package/dist/core/output.js +14 -1
- package/dist/core/output.js.map +1 -1
- package/dist/core/run-operation.d.ts +11 -4
- package/dist/core/run-operation.d.ts.map +1 -1
- package/dist/core/run-operation.js +14 -9
- package/dist/core/run-operation.js.map +1 -1
- package/dist/core/sign-in.d.ts +34 -0
- package/dist/core/sign-in.d.ts.map +1 -0
- package/dist/core/sign-in.js +21 -0
- package/dist/core/sign-in.js.map +1 -0
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +1236 -70
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +1579 -112
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/dist/mcp/mcp-call.d.ts +25 -0
- package/dist/mcp/mcp-call.d.ts.map +1 -0
- package/dist/mcp/mcp-call.js +44 -0
- package/dist/mcp/mcp-call.js.map +1 -0
- package/dist/mcp/mcp-command.d.ts +26 -0
- package/dist/mcp/mcp-command.d.ts.map +1 -0
- package/dist/mcp/mcp-command.js +91 -0
- package/dist/mcp/mcp-command.js.map +1 -0
- package/dist/mcp/mcp-install.d.ts +50 -0
- package/dist/mcp/mcp-install.d.ts.map +1 -0
- package/dist/mcp/mcp-install.js +71 -0
- package/dist/mcp/mcp-install.js.map +1 -0
- package/dist/mcp/mcp-protocol.d.ts +69 -0
- package/dist/mcp/mcp-protocol.d.ts.map +1 -0
- package/dist/mcp/mcp-protocol.js +96 -0
- package/dist/mcp/mcp-protocol.js.map +1 -0
- package/dist/mcp/mcp-stdio.d.ts +17 -0
- package/dist/mcp/mcp-stdio.d.ts.map +1 -0
- package/dist/mcp/mcp-stdio.js +39 -0
- package/dist/mcp/mcp-stdio.js.map +1 -0
- package/dist/mcp/mcp-tools.d.ts +81 -0
- package/dist/mcp/mcp-tools.d.ts.map +1 -0
- package/dist/mcp/mcp-tools.js +134 -0
- package/dist/mcp/mcp-tools.js.map +1 -0
- package/dist/program.d.ts +4 -9
- package/dist/program.d.ts.map +1 -1
- package/dist/program.js +9 -3
- package/dist/program.js.map +1 -1
- package/dist/render/index.js +6 -0
- package/dist/render/index.js.map +1 -1
- package/dist/render/study/counting-views.d.ts +46 -0
- package/dist/render/study/counting-views.d.ts.map +1 -0
- package/dist/render/study/counting-views.js +49 -0
- package/dist/render/study/counting-views.js.map +1 -0
- package/dist/render/study/gate-views.d.ts +56 -0
- package/dist/render/study/gate-views.d.ts.map +1 -0
- package/dist/render/study/gate-views.js +64 -0
- package/dist/render/study/gate-views.js.map +1 -0
- package/dist/render/study/renderers.d.ts +8 -0
- package/dist/render/study/renderers.d.ts.map +1 -0
- package/dist/render/study/renderers.js +31 -0
- package/dist/render/study/renderers.js.map +1 -0
- package/dist/render/study/series-views.d.ts +65 -0
- package/dist/render/study/series-views.d.ts.map +1 -0
- package/dist/render/study/series-views.js +116 -0
- package/dist/render/study/series-views.js.map +1 -0
- package/dist/render/study/study-format.d.ts +24 -0
- package/dist/render/study/study-format.d.ts.map +1 -0
- package/dist/render/study/study-format.js +42 -0
- package/dist/render/study/study-format.js.map +1 -0
- package/dist/study-ci/ci-command.d.ts +31 -0
- package/dist/study-ci/ci-command.d.ts.map +1 -0
- package/dist/study-ci/ci-command.js +134 -0
- package/dist/study-ci/ci-command.js.map +1 -0
- package/dist/study-ci/ci-rehearse-command.d.ts +11 -0
- package/dist/study-ci/ci-rehearse-command.d.ts.map +1 -0
- package/dist/study-ci/ci-rehearse-command.js +83 -0
- package/dist/study-ci/ci-rehearse-command.js.map +1 -0
- package/dist/study-ci/ci-rehearse.d.ts +62 -0
- package/dist/study-ci/ci-rehearse.d.ts.map +1 -0
- package/dist/study-ci/ci-rehearse.js +49 -0
- package/dist/study-ci/ci-rehearse.js.map +1 -0
- package/dist/study-ci/ci-run.d.ts +87 -0
- package/dist/study-ci/ci-run.d.ts.map +1 -0
- package/dist/study-ci/ci-run.js +131 -0
- package/dist/study-ci/ci-run.js.map +1 -0
- package/dist/study-ci/gate-report.d.ts +64 -0
- package/dist/study-ci/gate-report.d.ts.map +1 -0
- package/dist/study-ci/gate-report.js +80 -0
- package/dist/study-ci/gate-report.js.map +1 -0
- package/dist/study-ci/params.d.ts +8 -0
- package/dist/study-ci/params.d.ts.map +1 -0
- package/dist/study-ci/params.js +72 -0
- package/dist/study-ci/params.js.map +1 -0
- package/package.json +1 -1
|
@@ -3362,6 +3362,23 @@ export type ExperimentRunsListResponse = {
|
|
|
3362
3362
|
"output_tokens"?: number | null;
|
|
3363
3363
|
"created_at": string;
|
|
3364
3364
|
"finished_at"?: string | null;
|
|
3365
|
+
"phases"?: {
|
|
3366
|
+
"phases": Array<{
|
|
3367
|
+
"id": "waiting" | "boot" | "installations" | "startup" | "run" | "verification" | "teardown";
|
|
3368
|
+
"ms": number | null;
|
|
3369
|
+
"state": "measured" | "unmeasured" | "not_in_run";
|
|
3370
|
+
}>;
|
|
3371
|
+
"runTimeMs": number | null;
|
|
3372
|
+
"totalMs": number | null;
|
|
3373
|
+
"steps": Array<{
|
|
3374
|
+
"name": string;
|
|
3375
|
+
"kind": string;
|
|
3376
|
+
"outcome": string;
|
|
3377
|
+
"startedAt": string | null;
|
|
3378
|
+
"finishedAt": string | null;
|
|
3379
|
+
}>;
|
|
3380
|
+
"modelCalls": number | null;
|
|
3381
|
+
} | null;
|
|
3365
3382
|
"insight"?: {
|
|
3366
3383
|
"visibility_score": number | null;
|
|
3367
3384
|
"intent_satisfied": boolean | null;
|
|
@@ -5592,6 +5609,23 @@ export type StudioRunsListResponse = {
|
|
|
5592
5609
|
"output_tokens"?: number | null;
|
|
5593
5610
|
"created_at": string;
|
|
5594
5611
|
"finished_at"?: string | null;
|
|
5612
|
+
"phases"?: {
|
|
5613
|
+
"phases": Array<{
|
|
5614
|
+
"id": "waiting" | "boot" | "installations" | "startup" | "run" | "verification" | "teardown";
|
|
5615
|
+
"ms": number | null;
|
|
5616
|
+
"state": "measured" | "unmeasured" | "not_in_run";
|
|
5617
|
+
}>;
|
|
5618
|
+
"runTimeMs": number | null;
|
|
5619
|
+
"totalMs": number | null;
|
|
5620
|
+
"steps": Array<{
|
|
5621
|
+
"name": string;
|
|
5622
|
+
"kind": string;
|
|
5623
|
+
"outcome": string;
|
|
5624
|
+
"startedAt": string | null;
|
|
5625
|
+
"finishedAt": string | null;
|
|
5626
|
+
}>;
|
|
5627
|
+
"modelCalls": number | null;
|
|
5628
|
+
} | null;
|
|
5595
5629
|
"insight"?: {
|
|
5596
5630
|
"visibility_score": number | null;
|
|
5597
5631
|
"intent_satisfied": boolean | null;
|
|
@@ -6828,6 +6862,23 @@ export type StudyRunsListResponse = {
|
|
|
6828
6862
|
"output_tokens"?: number | null;
|
|
6829
6863
|
"created_at": string;
|
|
6830
6864
|
"finished_at"?: string | null;
|
|
6865
|
+
"phases"?: {
|
|
6866
|
+
"phases": Array<{
|
|
6867
|
+
"id": "waiting" | "boot" | "installations" | "startup" | "run" | "verification" | "teardown";
|
|
6868
|
+
"ms": number | null;
|
|
6869
|
+
"state": "measured" | "unmeasured" | "not_in_run";
|
|
6870
|
+
}>;
|
|
6871
|
+
"runTimeMs": number | null;
|
|
6872
|
+
"totalMs": number | null;
|
|
6873
|
+
"steps": Array<{
|
|
6874
|
+
"name": string;
|
|
6875
|
+
"kind": string;
|
|
6876
|
+
"outcome": string;
|
|
6877
|
+
"startedAt": string | null;
|
|
6878
|
+
"finishedAt": string | null;
|
|
6879
|
+
}>;
|
|
6880
|
+
"modelCalls": number | null;
|
|
6881
|
+
} | null;
|
|
6831
6882
|
"insight"?: {
|
|
6832
6883
|
"visibility_score": number | null;
|
|
6833
6884
|
"intent_satisfied": boolean | null;
|
|
@@ -7123,6 +7174,8 @@ export type StudyListResponse = {
|
|
|
7123
7174
|
"startsPerMinute"?: number;
|
|
7124
7175
|
} | null;
|
|
7125
7176
|
"relaunchable"?: boolean;
|
|
7177
|
+
"archivedAt"?: string | null;
|
|
7178
|
+
"archivedBy"?: string | null;
|
|
7126
7179
|
"arms"?: Array<{
|
|
7127
7180
|
"key": string;
|
|
7128
7181
|
"position": number;
|
|
@@ -7161,6 +7214,29 @@ export type StudyListResponse = {
|
|
|
7161
7214
|
"cellPlan"?: "crossed" | "per_setup";
|
|
7162
7215
|
"skillDisclosure"?: "withheld" | "named";
|
|
7163
7216
|
"modeOptions"?: Record<string, unknown>;
|
|
7217
|
+
"screens"?: Array<"test_awareness">;
|
|
7218
|
+
"runCounting"?: {
|
|
7219
|
+
"policy": "hold_back_screened";
|
|
7220
|
+
"screen": "test_awareness";
|
|
7221
|
+
"levels": Array<"aware" | "influenced">;
|
|
7222
|
+
} | null;
|
|
7223
|
+
"seriesKey"?: string | null;
|
|
7224
|
+
"ref"?: {
|
|
7225
|
+
"commit"?: string;
|
|
7226
|
+
"branch"?: string;
|
|
7227
|
+
"pr"?: number;
|
|
7228
|
+
"url"?: string;
|
|
7229
|
+
} | null;
|
|
7230
|
+
"gradingRevision"?: number;
|
|
7231
|
+
"gradingRevisedAt"?: string | null;
|
|
7232
|
+
"rehearsal"?: true;
|
|
7233
|
+
"rehearsalMode"?: string;
|
|
7234
|
+
"behaviors"?: Array<{
|
|
7235
|
+
"key": string;
|
|
7236
|
+
"name": string;
|
|
7237
|
+
"question": string;
|
|
7238
|
+
"version": number;
|
|
7239
|
+
}>;
|
|
7164
7240
|
}>;
|
|
7165
7241
|
};
|
|
7166
7242
|
export type StudyCreateRequest = {
|
|
@@ -7306,6 +7382,28 @@ export type StudyCreateRequest = {
|
|
|
7306
7382
|
"maxInFlight"?: number;
|
|
7307
7383
|
"startsPerMinute"?: number;
|
|
7308
7384
|
} | null;
|
|
7385
|
+
"screens"?: Array<"test_awareness">;
|
|
7386
|
+
"series"?: string;
|
|
7387
|
+
"ref"?: {
|
|
7388
|
+
"commit"?: string;
|
|
7389
|
+
"branch"?: string;
|
|
7390
|
+
"pr"?: number;
|
|
7391
|
+
"url"?: string;
|
|
7392
|
+
};
|
|
7393
|
+
"carry"?: {
|
|
7394
|
+
"from": "series";
|
|
7395
|
+
"maxAgeDays"?: number;
|
|
7396
|
+
};
|
|
7397
|
+
"runCounting"?: {
|
|
7398
|
+
"policy": "hold_back_screened";
|
|
7399
|
+
"screen": "test_awareness";
|
|
7400
|
+
"levels": Array<"aware" | "influenced">;
|
|
7401
|
+
} | null;
|
|
7402
|
+
"behaviors"?: Array<{
|
|
7403
|
+
"key": string;
|
|
7404
|
+
"name": string;
|
|
7405
|
+
"question": string;
|
|
7406
|
+
}>;
|
|
7309
7407
|
"matrix": Array<{
|
|
7310
7408
|
"harness": string;
|
|
7311
7409
|
"model": string;
|
|
@@ -7456,6 +7554,8 @@ export type StudyCreateResponse = {
|
|
|
7456
7554
|
"startsPerMinute"?: number;
|
|
7457
7555
|
} | null;
|
|
7458
7556
|
"relaunchable"?: boolean;
|
|
7557
|
+
"archivedAt"?: string | null;
|
|
7558
|
+
"archivedBy"?: string | null;
|
|
7459
7559
|
"arms"?: Array<{
|
|
7460
7560
|
"key": string;
|
|
7461
7561
|
"position": number;
|
|
@@ -7494,6 +7594,29 @@ export type StudyCreateResponse = {
|
|
|
7494
7594
|
"cellPlan"?: "crossed" | "per_setup";
|
|
7495
7595
|
"skillDisclosure"?: "withheld" | "named";
|
|
7496
7596
|
"modeOptions"?: Record<string, unknown>;
|
|
7597
|
+
"screens"?: Array<"test_awareness">;
|
|
7598
|
+
"runCounting"?: {
|
|
7599
|
+
"policy": "hold_back_screened";
|
|
7600
|
+
"screen": "test_awareness";
|
|
7601
|
+
"levels": Array<"aware" | "influenced">;
|
|
7602
|
+
} | null;
|
|
7603
|
+
"seriesKey"?: string | null;
|
|
7604
|
+
"ref"?: {
|
|
7605
|
+
"commit"?: string;
|
|
7606
|
+
"branch"?: string;
|
|
7607
|
+
"pr"?: number;
|
|
7608
|
+
"url"?: string;
|
|
7609
|
+
} | null;
|
|
7610
|
+
"gradingRevision"?: number;
|
|
7611
|
+
"gradingRevisedAt"?: string | null;
|
|
7612
|
+
"rehearsal"?: true;
|
|
7613
|
+
"rehearsalMode"?: string;
|
|
7614
|
+
"behaviors"?: Array<{
|
|
7615
|
+
"key": string;
|
|
7616
|
+
"name": string;
|
|
7617
|
+
"question": string;
|
|
7618
|
+
"version": number;
|
|
7619
|
+
}>;
|
|
7497
7620
|
};
|
|
7498
7621
|
"streamUrl": string;
|
|
7499
7622
|
"warnings"?: Array<{
|
|
@@ -7514,6 +7637,10 @@ export type StudyComparisonModesListResponse = {
|
|
|
7514
7637
|
"id": "crossed" | "per_setup";
|
|
7515
7638
|
"enabled": boolean;
|
|
7516
7639
|
}>;
|
|
7640
|
+
"customBehaviors": {
|
|
7641
|
+
"enabled": boolean;
|
|
7642
|
+
"max": number;
|
|
7643
|
+
};
|
|
7517
7644
|
};
|
|
7518
7645
|
export type StudyConvertWebmcpSuiteRequest = {
|
|
7519
7646
|
"suite": unknown;
|
|
@@ -7670,6 +7797,28 @@ export type StudyPreflightRequest = {
|
|
|
7670
7797
|
"maxInFlight"?: number;
|
|
7671
7798
|
"startsPerMinute"?: number;
|
|
7672
7799
|
} | null;
|
|
7800
|
+
"screens"?: Array<"test_awareness">;
|
|
7801
|
+
"series"?: string;
|
|
7802
|
+
"ref"?: {
|
|
7803
|
+
"commit"?: string;
|
|
7804
|
+
"branch"?: string;
|
|
7805
|
+
"pr"?: number;
|
|
7806
|
+
"url"?: string;
|
|
7807
|
+
};
|
|
7808
|
+
"carry"?: {
|
|
7809
|
+
"from": "series";
|
|
7810
|
+
"maxAgeDays"?: number;
|
|
7811
|
+
};
|
|
7812
|
+
"runCounting"?: {
|
|
7813
|
+
"policy": "hold_back_screened";
|
|
7814
|
+
"screen": "test_awareness";
|
|
7815
|
+
"levels": Array<"aware" | "influenced">;
|
|
7816
|
+
} | null;
|
|
7817
|
+
"behaviors"?: Array<{
|
|
7818
|
+
"key": string;
|
|
7819
|
+
"name": string;
|
|
7820
|
+
"question": string;
|
|
7821
|
+
}>;
|
|
7673
7822
|
"matrix": Array<{
|
|
7674
7823
|
"harness": string;
|
|
7675
7824
|
"model": string;
|
|
@@ -7692,6 +7841,14 @@ export type StudyPreflightResponse = {
|
|
|
7692
7841
|
"label": string;
|
|
7693
7842
|
"plannedRunCount": number;
|
|
7694
7843
|
"plannedJudgeComparisons": number;
|
|
7844
|
+
"screens"?: {
|
|
7845
|
+
"ids": Array<string>;
|
|
7846
|
+
"plannedReads": number;
|
|
7847
|
+
};
|
|
7848
|
+
"behaviors"?: {
|
|
7849
|
+
"keys": Array<string>;
|
|
7850
|
+
"plannedReads": number;
|
|
7851
|
+
};
|
|
7695
7852
|
"runs": {
|
|
7696
7853
|
"tasks": number;
|
|
7697
7854
|
"variants": number;
|
|
@@ -7762,53 +7919,45 @@ export type StudyPreflightResponse = {
|
|
|
7762
7919
|
"allowed": boolean;
|
|
7763
7920
|
} | null;
|
|
7764
7921
|
};
|
|
7765
|
-
export type
|
|
7766
|
-
"
|
|
7767
|
-
|
|
7768
|
-
|
|
7769
|
-
|
|
7770
|
-
|
|
7771
|
-
|
|
7772
|
-
|
|
7773
|
-
"
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
"name": string;
|
|
7777
|
-
"slug": string;
|
|
7778
|
-
"description": string | null;
|
|
7779
|
-
"skillRef": string | null;
|
|
7780
|
-
"skillSlug": string | null;
|
|
7781
|
-
"skillName": string | null;
|
|
7782
|
-
"skillPromise": string;
|
|
7783
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
7784
|
-
"subjectName": string | null;
|
|
7785
|
-
"subjectLabel": string;
|
|
7786
|
-
"armConfig": {
|
|
7787
|
-
"baseline"?: {
|
|
7788
|
-
"tools"?: "default" | "none" | {
|
|
7789
|
-
"tools": Array<string>;
|
|
7790
|
-
};
|
|
7791
|
-
"addons"?: Record<string, unknown>;
|
|
7922
|
+
export type StudyRehearseRequest = {
|
|
7923
|
+
"name": string;
|
|
7924
|
+
"description"?: string;
|
|
7925
|
+
"subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
7926
|
+
"skillRef"?: string;
|
|
7927
|
+
"definitionVersionId"?: string;
|
|
7928
|
+
"subjectName"?: string;
|
|
7929
|
+
"armConfig"?: {
|
|
7930
|
+
"baseline"?: {
|
|
7931
|
+
"tools"?: "default" | "none" | {
|
|
7932
|
+
"tools": Array<string>;
|
|
7792
7933
|
};
|
|
7793
|
-
"
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
"
|
|
7934
|
+
"addons"?: Record<string, unknown>;
|
|
7935
|
+
};
|
|
7936
|
+
"treatment"?: {
|
|
7937
|
+
"tools"?: "default" | "none" | {
|
|
7938
|
+
"tools": Array<string>;
|
|
7798
7939
|
};
|
|
7940
|
+
"addons"?: Record<string, unknown>;
|
|
7799
7941
|
};
|
|
7800
|
-
|
|
7801
|
-
|
|
7802
|
-
|
|
7803
|
-
|
|
7804
|
-
|
|
7805
|
-
|
|
7806
|
-
|
|
7807
|
-
|
|
7808
|
-
|
|
7809
|
-
|
|
7810
|
-
|
|
7811
|
-
"
|
|
7942
|
+
};
|
|
7943
|
+
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
7944
|
+
"judgeSupplementary"?: boolean;
|
|
7945
|
+
"proposeExpectations"?: boolean;
|
|
7946
|
+
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
7947
|
+
"subjectBackend"?: "model" | "harness";
|
|
7948
|
+
"baselineLabel"?: string;
|
|
7949
|
+
"treatmentLabel"?: string;
|
|
7950
|
+
"arms"?: Array<{
|
|
7951
|
+
"key"?: string;
|
|
7952
|
+
"label": string;
|
|
7953
|
+
"role"?: "reference" | "peer";
|
|
7954
|
+
"config": {
|
|
7955
|
+
"tools"?: "default" | "none" | {
|
|
7956
|
+
"tools": Array<string>;
|
|
7957
|
+
};
|
|
7958
|
+
"addons"?: Record<string, unknown>;
|
|
7959
|
+
};
|
|
7960
|
+
"cells"?: Array<{
|
|
7812
7961
|
"harness": string;
|
|
7813
7962
|
"model": string;
|
|
7814
7963
|
"provider"?: string;
|
|
@@ -7821,74 +7970,324 @@ export type StudyGetResponse = {
|
|
|
7821
7970
|
"toolSearch"?: "on" | "off" | "directory";
|
|
7822
7971
|
};
|
|
7823
7972
|
}>;
|
|
7824
|
-
|
|
7825
|
-
|
|
7826
|
-
|
|
7827
|
-
|
|
7828
|
-
|
|
7829
|
-
|
|
7830
|
-
|
|
7831
|
-
|
|
7832
|
-
|
|
7833
|
-
|
|
7834
|
-
"
|
|
7835
|
-
"
|
|
7836
|
-
"
|
|
7837
|
-
|
|
7838
|
-
"
|
|
7839
|
-
"
|
|
7840
|
-
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
"
|
|
7844
|
-
|
|
7845
|
-
|
|
7846
|
-
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
7850
|
-
|
|
7851
|
-
|
|
7852
|
-
|
|
7853
|
-
}>;
|
|
7854
|
-
} | null;
|
|
7855
|
-
"visibility": "private" | "org" | "link" | "public";
|
|
7856
|
-
"visibilityChangedAt"?: string | null;
|
|
7857
|
-
"visibilityChangedBy"?: string | null;
|
|
7858
|
-
"shareToken"?: string | null;
|
|
7859
|
-
"lowSampleOverride": boolean;
|
|
7860
|
-
"publishedAt": string | null;
|
|
7861
|
-
"verdict": unknown;
|
|
7862
|
-
"definitionVersionId": string | null;
|
|
7863
|
-
"runnerKind": "ora-runs" | "batch-job";
|
|
7864
|
-
"createdAt": string;
|
|
7865
|
-
"updatedAt": string;
|
|
7866
|
-
"liveUntil": string | null;
|
|
7867
|
-
"launch"?: {
|
|
7868
|
-
"attempt": number;
|
|
7869
|
-
"state": "fanning_out" | "deferred" | "settled";
|
|
7870
|
-
"startedAt": string;
|
|
7871
|
-
"progressAt": string;
|
|
7872
|
-
"planned"?: number;
|
|
7873
|
-
"created"?: number;
|
|
7874
|
-
"missing"?: {
|
|
7875
|
-
"retryable": number;
|
|
7876
|
-
"refused": number;
|
|
7877
|
-
};
|
|
7878
|
-
"notes"?: Array<string>;
|
|
7879
|
-
"settledAt"?: string;
|
|
7880
|
-
"pacing"?: {
|
|
7881
|
-
"waiting": number;
|
|
7882
|
-
"nextUnit"?: number;
|
|
7883
|
-
"nextWaveAt"?: string;
|
|
7884
|
-
"emptyWaves"?: number;
|
|
7885
|
-
"stopped"?: string;
|
|
7973
|
+
}>;
|
|
7974
|
+
"contrasts"?: "vs_reference" | "all_pairs";
|
|
7975
|
+
"modeOptions"?: Record<string, unknown>;
|
|
7976
|
+
"taskCount"?: number;
|
|
7977
|
+
"variantsPerTask"?: number;
|
|
7978
|
+
"repeats"?: number;
|
|
7979
|
+
"tasks"?: Array<{
|
|
7980
|
+
"prompt": string;
|
|
7981
|
+
"rationale"?: string;
|
|
7982
|
+
"category"?: string;
|
|
7983
|
+
"requiresWeb"?: boolean;
|
|
7984
|
+
"rubric"?: Array<string>;
|
|
7985
|
+
"variants"?: Array<string>;
|
|
7986
|
+
"expectations"?: Array<{
|
|
7987
|
+
"label"?: string;
|
|
7988
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
7989
|
+
"kind": "mcp_tool_called";
|
|
7990
|
+
"server"?: string;
|
|
7991
|
+
"tool": string;
|
|
7992
|
+
"arguments"?: Record<string, unknown>;
|
|
7993
|
+
"minCalls"?: number;
|
|
7994
|
+
} | {
|
|
7995
|
+
"label"?: string;
|
|
7996
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
7997
|
+
"kind": "tool_not_called";
|
|
7998
|
+
"tool"?: string;
|
|
7999
|
+
"mcp"?: {
|
|
8000
|
+
"server"?: string;
|
|
8001
|
+
"tool": string;
|
|
7886
8002
|
};
|
|
7887
|
-
"
|
|
7888
|
-
|
|
7889
|
-
|
|
7890
|
-
|
|
7891
|
-
|
|
8003
|
+
"pattern"?: string;
|
|
8004
|
+
} | {
|
|
8005
|
+
"label"?: string;
|
|
8006
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8007
|
+
"kind": "command_ran";
|
|
8008
|
+
"pattern": string;
|
|
8009
|
+
} | {
|
|
8010
|
+
"label"?: string;
|
|
8011
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8012
|
+
"kind": "skill_loaded";
|
|
8013
|
+
"slug": string;
|
|
8014
|
+
} | {
|
|
8015
|
+
"label"?: string;
|
|
8016
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8017
|
+
"kind": "file_operation";
|
|
8018
|
+
"op"?: "read" | "write" | "edit";
|
|
8019
|
+
"path": unknown;
|
|
8020
|
+
} | {
|
|
8021
|
+
"label"?: string;
|
|
8022
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8023
|
+
"kind": "web_fetched";
|
|
8024
|
+
"urlHost"?: string;
|
|
8025
|
+
"urlPattern"?: string;
|
|
8026
|
+
} | {
|
|
8027
|
+
"label"?: string;
|
|
8028
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8029
|
+
"kind": "response_contains";
|
|
8030
|
+
"text": string;
|
|
8031
|
+
"normalize"?: "prose";
|
|
8032
|
+
} | {
|
|
8033
|
+
"label"?: string;
|
|
8034
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8035
|
+
"kind": "response_matches";
|
|
8036
|
+
"pattern": string;
|
|
8037
|
+
"ignoreCase"?: boolean;
|
|
8038
|
+
} | {
|
|
8039
|
+
"label"?: string;
|
|
8040
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8041
|
+
"kind": "token_budget";
|
|
8042
|
+
"maxOutputTokens"?: number;
|
|
8043
|
+
"maxTotalTokens"?: number;
|
|
8044
|
+
} | {
|
|
8045
|
+
"label"?: string;
|
|
8046
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8047
|
+
"kind": "sequence";
|
|
8048
|
+
"of": Array<unknown>;
|
|
8049
|
+
} | {
|
|
8050
|
+
"label"?: string;
|
|
8051
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
8052
|
+
"kind": "any_of";
|
|
8053
|
+
"of": Array<unknown>;
|
|
8054
|
+
}>;
|
|
8055
|
+
"private"?: boolean;
|
|
8056
|
+
}>;
|
|
8057
|
+
"subset"?: {
|
|
8058
|
+
"count": number;
|
|
8059
|
+
"seed"?: string;
|
|
8060
|
+
};
|
|
8061
|
+
"pacing"?: {
|
|
8062
|
+
"maxInFlight"?: number;
|
|
8063
|
+
"startsPerMinute"?: number;
|
|
8064
|
+
} | null;
|
|
8065
|
+
"screens"?: Array<"test_awareness">;
|
|
8066
|
+
"series"?: string;
|
|
8067
|
+
"ref"?: {
|
|
8068
|
+
"commit"?: string;
|
|
8069
|
+
"branch"?: string;
|
|
8070
|
+
"pr"?: number;
|
|
8071
|
+
"url"?: string;
|
|
8072
|
+
};
|
|
8073
|
+
"carry"?: {
|
|
8074
|
+
"from": "series";
|
|
8075
|
+
"maxAgeDays"?: number;
|
|
8076
|
+
};
|
|
8077
|
+
"runCounting"?: {
|
|
8078
|
+
"policy": "hold_back_screened";
|
|
8079
|
+
"screen": "test_awareness";
|
|
8080
|
+
"levels": Array<"aware" | "influenced">;
|
|
8081
|
+
} | null;
|
|
8082
|
+
"behaviors"?: Array<{
|
|
8083
|
+
"key": string;
|
|
8084
|
+
"name": string;
|
|
8085
|
+
"question": string;
|
|
8086
|
+
}>;
|
|
8087
|
+
"matrix": Array<{
|
|
8088
|
+
"harness": string;
|
|
8089
|
+
"model": string;
|
|
8090
|
+
"provider"?: string;
|
|
8091
|
+
"params"?: {
|
|
8092
|
+
"temperature"?: number;
|
|
8093
|
+
"maxOutputTokens"?: number;
|
|
8094
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
8095
|
+
"thinkingBudgetTokens"?: number;
|
|
8096
|
+
"codeMode"?: "auto" | "on" | "off";
|
|
8097
|
+
"toolSearch"?: "on" | "off" | "directory";
|
|
8098
|
+
};
|
|
8099
|
+
}>;
|
|
8100
|
+
"promise"?: string;
|
|
8101
|
+
};
|
|
8102
|
+
export type StudyRehearseResponse = {
|
|
8103
|
+
"rehearsal": {
|
|
8104
|
+
"studyId": string;
|
|
8105
|
+
"name": string;
|
|
8106
|
+
"status": string;
|
|
8107
|
+
"tasks": number;
|
|
8108
|
+
"mode": string;
|
|
8109
|
+
};
|
|
8110
|
+
};
|
|
8111
|
+
export type StudySeriesResponse = {
|
|
8112
|
+
"series": Array<{
|
|
8113
|
+
"key": string;
|
|
8114
|
+
"studies": number;
|
|
8115
|
+
"newestAt": string;
|
|
8116
|
+
}>;
|
|
8117
|
+
"capped": boolean;
|
|
8118
|
+
};
|
|
8119
|
+
export type StudySeriesTrendResponse = {
|
|
8120
|
+
"series": string;
|
|
8121
|
+
"metric": {
|
|
8122
|
+
"key": string;
|
|
8123
|
+
"label": string;
|
|
8124
|
+
"family": string;
|
|
8125
|
+
};
|
|
8126
|
+
"countingPolicy": string;
|
|
8127
|
+
"studies": Array<{
|
|
8128
|
+
"studyId": string;
|
|
8129
|
+
"name": string;
|
|
8130
|
+
"createdAt": string;
|
|
8131
|
+
"ref": {
|
|
8132
|
+
"commit"?: string;
|
|
8133
|
+
"branch"?: string;
|
|
8134
|
+
"pr"?: number;
|
|
8135
|
+
"url"?: string;
|
|
8136
|
+
} | null;
|
|
8137
|
+
"state": "settled" | "pending";
|
|
8138
|
+
"owed": Array<string>;
|
|
8139
|
+
"segment": number | null;
|
|
8140
|
+
"heldBackRuns"?: number;
|
|
8141
|
+
"groups": Array<{
|
|
8142
|
+
"key": string;
|
|
8143
|
+
"label": string;
|
|
8144
|
+
"value": number | null;
|
|
8145
|
+
"interval": Array<unknown> | null;
|
|
8146
|
+
"tasks": number;
|
|
8147
|
+
"runs": number;
|
|
8148
|
+
"change": {
|
|
8149
|
+
"effect": number;
|
|
8150
|
+
"unit": string;
|
|
8151
|
+
"interval": Array<unknown>;
|
|
8152
|
+
"tasks": number;
|
|
8153
|
+
} | null;
|
|
8154
|
+
}>;
|
|
8155
|
+
}>;
|
|
8156
|
+
"segments": Array<{
|
|
8157
|
+
"index": number;
|
|
8158
|
+
"comparabilityKey": string;
|
|
8159
|
+
"studyIds": Array<string>;
|
|
8160
|
+
"differsFromPrevious": Array<"corpus" | "lenses" | "graders" | "classifiers" | "countingPolicy" | "agents">;
|
|
8161
|
+
}>;
|
|
8162
|
+
"truncated": boolean;
|
|
8163
|
+
};
|
|
8164
|
+
export type StudyGetResponse = {
|
|
8165
|
+
"study": {
|
|
8166
|
+
"id": string;
|
|
8167
|
+
"executiveSummary"?: string | null;
|
|
8168
|
+
"executiveSummaryDraft"?: string | null;
|
|
8169
|
+
"executiveSummaryRequest"?: "live" | "draft" | null;
|
|
8170
|
+
"executiveSummaryRequestedAt"?: string | null;
|
|
8171
|
+
"executiveSummaryUnavailableAt"?: string | null;
|
|
8172
|
+
"plannedRunCount": number;
|
|
8173
|
+
"organizationId": string;
|
|
8174
|
+
"createdBy": string | null;
|
|
8175
|
+
"name": string;
|
|
8176
|
+
"slug": string;
|
|
8177
|
+
"description": string | null;
|
|
8178
|
+
"skillRef": string | null;
|
|
8179
|
+
"skillSlug": string | null;
|
|
8180
|
+
"skillName": string | null;
|
|
8181
|
+
"skillPromise": string;
|
|
8182
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
8183
|
+
"subjectName": string | null;
|
|
8184
|
+
"subjectLabel": string;
|
|
8185
|
+
"armConfig": {
|
|
8186
|
+
"baseline"?: {
|
|
8187
|
+
"tools"?: "default" | "none" | {
|
|
8188
|
+
"tools": Array<string>;
|
|
8189
|
+
};
|
|
8190
|
+
"addons"?: Record<string, unknown>;
|
|
8191
|
+
};
|
|
8192
|
+
"treatment"?: {
|
|
8193
|
+
"tools"?: "default" | "none" | {
|
|
8194
|
+
"tools": Array<string>;
|
|
8195
|
+
};
|
|
8196
|
+
"addons"?: Record<string, unknown>;
|
|
8197
|
+
};
|
|
8198
|
+
};
|
|
8199
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
8200
|
+
"subjectBackend": "model" | "harness" | null;
|
|
8201
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
8202
|
+
"importedFrom"?: string;
|
|
8203
|
+
"baselineLabel": string | null;
|
|
8204
|
+
"treatmentLabel": string | null;
|
|
8205
|
+
"promiseMetrics": Array<{
|
|
8206
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
8207
|
+
"direction": "down" | "up";
|
|
8208
|
+
"label": string;
|
|
8209
|
+
}>;
|
|
8210
|
+
"matrix": Array<{
|
|
8211
|
+
"harness": string;
|
|
8212
|
+
"model": string;
|
|
8213
|
+
"provider"?: string;
|
|
8214
|
+
"params"?: {
|
|
8215
|
+
"temperature"?: number;
|
|
8216
|
+
"maxOutputTokens"?: number;
|
|
8217
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
8218
|
+
"thinkingBudgetTokens"?: number;
|
|
8219
|
+
"codeMode"?: "auto" | "on" | "off";
|
|
8220
|
+
"toolSearch"?: "on" | "off" | "directory";
|
|
8221
|
+
};
|
|
8222
|
+
}>;
|
|
8223
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
8224
|
+
"judgeSupplementary"?: boolean;
|
|
8225
|
+
"taskCount": number;
|
|
8226
|
+
"variantsPerTask": number;
|
|
8227
|
+
"repeats": number;
|
|
8228
|
+
"subset": {
|
|
8229
|
+
"count": number;
|
|
8230
|
+
"seed": string;
|
|
8231
|
+
"of": number;
|
|
8232
|
+
} | null;
|
|
8233
|
+
"status": string;
|
|
8234
|
+
"error": string | null;
|
|
8235
|
+
"preparation"?: {
|
|
8236
|
+
"since": string;
|
|
8237
|
+
"progressAt"?: string;
|
|
8238
|
+
"launchedBy": {
|
|
8239
|
+
"userId"?: string;
|
|
8240
|
+
"role"?: string;
|
|
8241
|
+
};
|
|
8242
|
+
"waitingOn": Array<{
|
|
8243
|
+
"imageKey": string;
|
|
8244
|
+
"buildId": string;
|
|
8245
|
+
"stage"?: "image" | "snapshot";
|
|
8246
|
+
"retry"?: {
|
|
8247
|
+
"failedAttempts": number;
|
|
8248
|
+
"maxAttempts": number;
|
|
8249
|
+
"notBefore": string;
|
|
8250
|
+
"lastFailure": string;
|
|
8251
|
+
};
|
|
8252
|
+
}>;
|
|
8253
|
+
} | null;
|
|
8254
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
8255
|
+
"visibilityChangedAt"?: string | null;
|
|
8256
|
+
"visibilityChangedBy"?: string | null;
|
|
8257
|
+
"shareToken"?: string | null;
|
|
8258
|
+
"lowSampleOverride": boolean;
|
|
8259
|
+
"publishedAt": string | null;
|
|
8260
|
+
"verdict": unknown;
|
|
8261
|
+
"definitionVersionId": string | null;
|
|
8262
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
8263
|
+
"createdAt": string;
|
|
8264
|
+
"updatedAt": string;
|
|
8265
|
+
"liveUntil": string | null;
|
|
8266
|
+
"launch"?: {
|
|
8267
|
+
"attempt": number;
|
|
8268
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
8269
|
+
"startedAt": string;
|
|
8270
|
+
"progressAt": string;
|
|
8271
|
+
"planned"?: number;
|
|
8272
|
+
"created"?: number;
|
|
8273
|
+
"missing"?: {
|
|
8274
|
+
"retryable": number;
|
|
8275
|
+
"refused": number;
|
|
8276
|
+
};
|
|
8277
|
+
"notes"?: Array<string>;
|
|
8278
|
+
"settledAt"?: string;
|
|
8279
|
+
"pacing"?: {
|
|
8280
|
+
"waiting": number;
|
|
8281
|
+
"nextUnit"?: number;
|
|
8282
|
+
"nextWaveAt"?: string;
|
|
8283
|
+
"emptyWaves"?: number;
|
|
8284
|
+
"stopped"?: string;
|
|
8285
|
+
};
|
|
8286
|
+
"recovery"?: {
|
|
8287
|
+
"state": "retrying" | "needs_attention";
|
|
8288
|
+
"attempts": number;
|
|
8289
|
+
"lastAttemptAt"?: string;
|
|
8290
|
+
"nextAttemptAt"?: string;
|
|
7892
8291
|
"reason"?: string;
|
|
7893
8292
|
};
|
|
7894
8293
|
} | null;
|
|
@@ -7897,6 +8296,8 @@ export type StudyGetResponse = {
|
|
|
7897
8296
|
"startsPerMinute"?: number;
|
|
7898
8297
|
} | null;
|
|
7899
8298
|
"relaunchable"?: boolean;
|
|
8299
|
+
"archivedAt"?: string | null;
|
|
8300
|
+
"archivedBy"?: string | null;
|
|
7900
8301
|
"arms"?: Array<{
|
|
7901
8302
|
"key": string;
|
|
7902
8303
|
"position": number;
|
|
@@ -7935,6 +8336,29 @@ export type StudyGetResponse = {
|
|
|
7935
8336
|
"cellPlan"?: "crossed" | "per_setup";
|
|
7936
8337
|
"skillDisclosure"?: "withheld" | "named";
|
|
7937
8338
|
"modeOptions"?: Record<string, unknown>;
|
|
8339
|
+
"screens"?: Array<"test_awareness">;
|
|
8340
|
+
"runCounting"?: {
|
|
8341
|
+
"policy": "hold_back_screened";
|
|
8342
|
+
"screen": "test_awareness";
|
|
8343
|
+
"levels": Array<"aware" | "influenced">;
|
|
8344
|
+
} | null;
|
|
8345
|
+
"seriesKey"?: string | null;
|
|
8346
|
+
"ref"?: {
|
|
8347
|
+
"commit"?: string;
|
|
8348
|
+
"branch"?: string;
|
|
8349
|
+
"pr"?: number;
|
|
8350
|
+
"url"?: string;
|
|
8351
|
+
} | null;
|
|
8352
|
+
"gradingRevision"?: number;
|
|
8353
|
+
"gradingRevisedAt"?: string | null;
|
|
8354
|
+
"rehearsal"?: true;
|
|
8355
|
+
"rehearsalMode"?: string;
|
|
8356
|
+
"behaviors"?: Array<{
|
|
8357
|
+
"key": string;
|
|
8358
|
+
"name": string;
|
|
8359
|
+
"question": string;
|
|
8360
|
+
"version": number;
|
|
8361
|
+
}>;
|
|
7938
8362
|
};
|
|
7939
8363
|
"tasks": Array<{
|
|
7940
8364
|
"id": string;
|
|
@@ -8083,6 +8507,12 @@ export type StudyGetResponse = {
|
|
|
8083
8507
|
}>;
|
|
8084
8508
|
"modelCalls": number | null;
|
|
8085
8509
|
} | null;
|
|
8510
|
+
"screens"?: Record<string, unknown>;
|
|
8511
|
+
"carried"?: {
|
|
8512
|
+
"fromRunId": string;
|
|
8513
|
+
"fromStudyId": string | null;
|
|
8514
|
+
"at": string;
|
|
8515
|
+
} | null;
|
|
8086
8516
|
}>;
|
|
8087
8517
|
"aggregate": {
|
|
8088
8518
|
"summary": {
|
|
@@ -8346,6 +8776,18 @@ export type StudyGetResponse = {
|
|
|
8346
8776
|
"byArm": Record<string, unknown>;
|
|
8347
8777
|
"phasesByArm": Record<string, unknown>;
|
|
8348
8778
|
};
|
|
8779
|
+
"runCounting"?: {
|
|
8780
|
+
"policy": string;
|
|
8781
|
+
"screen": string;
|
|
8782
|
+
"levels": Array<string>;
|
|
8783
|
+
"heldBack": number;
|
|
8784
|
+
"groups": Array<{
|
|
8785
|
+
"group": string;
|
|
8786
|
+
"heldBack": number;
|
|
8787
|
+
"runs": number;
|
|
8788
|
+
}>;
|
|
8789
|
+
"imbalanced": boolean;
|
|
8790
|
+
};
|
|
8349
8791
|
} | {
|
|
8350
8792
|
"shape": "multi-arm";
|
|
8351
8793
|
"summary": {
|
|
@@ -9221,6 +9663,18 @@ export type StudyGetResponse = {
|
|
|
9221
9663
|
"byArm": Record<string, unknown>;
|
|
9222
9664
|
"phasesByArm": Record<string, unknown>;
|
|
9223
9665
|
};
|
|
9666
|
+
"runCounting"?: {
|
|
9667
|
+
"policy": string;
|
|
9668
|
+
"screen": string;
|
|
9669
|
+
"levels": Array<string>;
|
|
9670
|
+
"heldBack": number;
|
|
9671
|
+
"groups": Array<{
|
|
9672
|
+
"group": string;
|
|
9673
|
+
"heldBack": number;
|
|
9674
|
+
"runs": number;
|
|
9675
|
+
}>;
|
|
9676
|
+
"imbalanced": boolean;
|
|
9677
|
+
};
|
|
9224
9678
|
};
|
|
9225
9679
|
"warnings": Array<{
|
|
9226
9680
|
"type": "eval_capability_gap";
|
|
@@ -9231,7 +9685,12 @@ export type StudyGetResponse = {
|
|
|
9231
9685
|
"detail": string;
|
|
9232
9686
|
"taskIds"?: Array<string>;
|
|
9233
9687
|
} | {
|
|
9234
|
-
"type": "
|
|
9688
|
+
"type": "design_lint";
|
|
9689
|
+
"lint": string;
|
|
9690
|
+
"detail": string;
|
|
9691
|
+
"taskIds": Array<string>;
|
|
9692
|
+
} | {
|
|
9693
|
+
"type": "design_aggregate_budget";
|
|
9235
9694
|
"detail": string;
|
|
9236
9695
|
}>;
|
|
9237
9696
|
"taskHealth": {
|
|
@@ -9396,6 +9855,8 @@ export type StudyUpdateDetailsResponse = {
|
|
|
9396
9855
|
"startsPerMinute"?: number;
|
|
9397
9856
|
} | null;
|
|
9398
9857
|
"relaunchable"?: boolean;
|
|
9858
|
+
"archivedAt"?: string | null;
|
|
9859
|
+
"archivedBy"?: string | null;
|
|
9399
9860
|
"arms"?: Array<{
|
|
9400
9861
|
"key": string;
|
|
9401
9862
|
"position": number;
|
|
@@ -9434,6 +9895,29 @@ export type StudyUpdateDetailsResponse = {
|
|
|
9434
9895
|
"cellPlan"?: "crossed" | "per_setup";
|
|
9435
9896
|
"skillDisclosure"?: "withheld" | "named";
|
|
9436
9897
|
"modeOptions"?: Record<string, unknown>;
|
|
9898
|
+
"screens"?: Array<"test_awareness">;
|
|
9899
|
+
"runCounting"?: {
|
|
9900
|
+
"policy": "hold_back_screened";
|
|
9901
|
+
"screen": "test_awareness";
|
|
9902
|
+
"levels": Array<"aware" | "influenced">;
|
|
9903
|
+
} | null;
|
|
9904
|
+
"seriesKey"?: string | null;
|
|
9905
|
+
"ref"?: {
|
|
9906
|
+
"commit"?: string;
|
|
9907
|
+
"branch"?: string;
|
|
9908
|
+
"pr"?: number;
|
|
9909
|
+
"url"?: string;
|
|
9910
|
+
} | null;
|
|
9911
|
+
"gradingRevision"?: number;
|
|
9912
|
+
"gradingRevisedAt"?: string | null;
|
|
9913
|
+
"rehearsal"?: true;
|
|
9914
|
+
"rehearsalMode"?: string;
|
|
9915
|
+
"behaviors"?: Array<{
|
|
9916
|
+
"key": string;
|
|
9917
|
+
"name": string;
|
|
9918
|
+
"question": string;
|
|
9919
|
+
"version": number;
|
|
9920
|
+
}>;
|
|
9437
9921
|
};
|
|
9438
9922
|
};
|
|
9439
9923
|
export type StudyDeleteResponse = unknown;
|
|
@@ -9543,10 +10027,150 @@ export type StudyAddAgentsResponse = {
|
|
|
9543
10027
|
"reason": string | null;
|
|
9544
10028
|
};
|
|
9545
10029
|
};
|
|
10030
|
+
export type StudyBehaviorsResponse = {
|
|
10031
|
+
"behaviors": Array<{
|
|
10032
|
+
"key": string;
|
|
10033
|
+
"name": string;
|
|
10034
|
+
"question": string;
|
|
10035
|
+
"version": number;
|
|
10036
|
+
"screenable": number;
|
|
10037
|
+
"judged": number;
|
|
10038
|
+
"measurement": {
|
|
10039
|
+
"key": string;
|
|
10040
|
+
"version": number;
|
|
10041
|
+
"verdicts": {
|
|
10042
|
+
"present": number;
|
|
10043
|
+
"absent": number;
|
|
10044
|
+
"unknown": number;
|
|
10045
|
+
};
|
|
10046
|
+
"present": {
|
|
10047
|
+
"reviewed": number;
|
|
10048
|
+
"confirmed": number;
|
|
10049
|
+
"precision": {
|
|
10050
|
+
"rate": number;
|
|
10051
|
+
"lower": number;
|
|
10052
|
+
"upper": number;
|
|
10053
|
+
"successes": number;
|
|
10054
|
+
"n": number;
|
|
10055
|
+
} | null;
|
|
10056
|
+
};
|
|
10057
|
+
"absent": {
|
|
10058
|
+
"reviewed": number;
|
|
10059
|
+
"confirmed": number;
|
|
10060
|
+
"precision": {
|
|
10061
|
+
"rate": number;
|
|
10062
|
+
"lower": number;
|
|
10063
|
+
"upper": number;
|
|
10064
|
+
"successes": number;
|
|
10065
|
+
"n": number;
|
|
10066
|
+
} | null;
|
|
10067
|
+
};
|
|
10068
|
+
"contested": number;
|
|
10069
|
+
"agreement": {
|
|
10070
|
+
"runs": number;
|
|
10071
|
+
"agreed": number;
|
|
10072
|
+
"kappa": number | null;
|
|
10073
|
+
} | null;
|
|
10074
|
+
};
|
|
10075
|
+
}>;
|
|
10076
|
+
"reviews": number;
|
|
10077
|
+
"reviewsTruncated": boolean;
|
|
10078
|
+
};
|
|
10079
|
+
export type StudyCarryPlanResponse = {
|
|
10080
|
+
"policy": {
|
|
10081
|
+
"from": "series";
|
|
10082
|
+
"maxAgeDays": number;
|
|
10083
|
+
} | null;
|
|
10084
|
+
"sourceStudyId": string | null;
|
|
10085
|
+
"planned": number;
|
|
10086
|
+
"carried": number;
|
|
10087
|
+
"fresh": Record<string, unknown>;
|
|
10088
|
+
"differences": Array<string>;
|
|
10089
|
+
"carriedRuns": number;
|
|
10090
|
+
};
|
|
9546
10091
|
export type StudyRegenerateSummaryResponse = {
|
|
9547
10092
|
"status": "queued";
|
|
9548
10093
|
"target": "live" | "draft";
|
|
9549
10094
|
};
|
|
10095
|
+
export type StudyGateRequest = {
|
|
10096
|
+
"metric"?: "benchmark" | "goal" | "evals" | "quality" | "rubric" | "completion";
|
|
10097
|
+
"group"?: string;
|
|
10098
|
+
"floor"?: {
|
|
10099
|
+
"rate": number;
|
|
10100
|
+
} | {
|
|
10101
|
+
"points": number;
|
|
10102
|
+
};
|
|
10103
|
+
"maxDrop"?: {
|
|
10104
|
+
"pp": number;
|
|
10105
|
+
} | {
|
|
10106
|
+
"points": number;
|
|
10107
|
+
};
|
|
10108
|
+
"baseline"?: {
|
|
10109
|
+
"studyId": string;
|
|
10110
|
+
} | {
|
|
10111
|
+
"series": {
|
|
10112
|
+
"branch"?: string;
|
|
10113
|
+
};
|
|
10114
|
+
};
|
|
10115
|
+
"minTasks"?: number;
|
|
10116
|
+
"infraCeiling"?: number;
|
|
10117
|
+
"counting"?: "policy" | "all";
|
|
10118
|
+
};
|
|
10119
|
+
export type StudyGateResponse = {
|
|
10120
|
+
"state": "settled" | "pending";
|
|
10121
|
+
"owed": Array<string>;
|
|
10122
|
+
"countingPolicy": string;
|
|
10123
|
+
"metric": {
|
|
10124
|
+
"key": string;
|
|
10125
|
+
"label": string;
|
|
10126
|
+
"family": "rate" | "score";
|
|
10127
|
+
};
|
|
10128
|
+
"group": string;
|
|
10129
|
+
"groups": Array<{
|
|
10130
|
+
"key": string;
|
|
10131
|
+
"label": string;
|
|
10132
|
+
"value": number | null;
|
|
10133
|
+
"runs": number;
|
|
10134
|
+
"tasks": number;
|
|
10135
|
+
"interval": Array<unknown> | null;
|
|
10136
|
+
"carriedRuns"?: number;
|
|
10137
|
+
"heldBackRuns"?: number;
|
|
10138
|
+
}>;
|
|
10139
|
+
"baseline": {
|
|
10140
|
+
"studyId": string;
|
|
10141
|
+
"name": string;
|
|
10142
|
+
"comparable": boolean;
|
|
10143
|
+
"differs": Array<string>;
|
|
10144
|
+
"value": number | null;
|
|
10145
|
+
"interval": Array<unknown> | null;
|
|
10146
|
+
"difference": {
|
|
10147
|
+
"effect": number;
|
|
10148
|
+
"unit": "pp" | "pts";
|
|
10149
|
+
"interval": Array<unknown>;
|
|
10150
|
+
"tasks": number;
|
|
10151
|
+
} | null;
|
|
10152
|
+
} | null;
|
|
10153
|
+
"comparabilityKey": string;
|
|
10154
|
+
"series": {
|
|
10155
|
+
"key": string;
|
|
10156
|
+
"ref": {
|
|
10157
|
+
"commit"?: string;
|
|
10158
|
+
"branch"?: string;
|
|
10159
|
+
"pr"?: number;
|
|
10160
|
+
"url"?: string;
|
|
10161
|
+
} | null;
|
|
10162
|
+
} | null;
|
|
10163
|
+
"gate": {
|
|
10164
|
+
"decision": "pass" | "fail" | "undecided";
|
|
10165
|
+
"refusedBy": string | null;
|
|
10166
|
+
"reason": string;
|
|
10167
|
+
"rules": Array<{
|
|
10168
|
+
"rule": string;
|
|
10169
|
+
"decision": "pass" | "fail" | "undecided";
|
|
10170
|
+
"reason": string;
|
|
10171
|
+
}>;
|
|
10172
|
+
};
|
|
10173
|
+
};
|
|
9550
10174
|
export type StudyGenerateRequest = {
|
|
9551
10175
|
"proposeExpectations"?: boolean;
|
|
9552
10176
|
};
|
|
@@ -9685,6 +10309,8 @@ export type StudyGenerateResponse = {
|
|
|
9685
10309
|
"startsPerMinute"?: number;
|
|
9686
10310
|
} | null;
|
|
9687
10311
|
"relaunchable"?: boolean;
|
|
10312
|
+
"archivedAt"?: string | null;
|
|
10313
|
+
"archivedBy"?: string | null;
|
|
9688
10314
|
"arms"?: Array<{
|
|
9689
10315
|
"key": string;
|
|
9690
10316
|
"position": number;
|
|
@@ -9723,6 +10349,29 @@ export type StudyGenerateResponse = {
|
|
|
9723
10349
|
"cellPlan"?: "crossed" | "per_setup";
|
|
9724
10350
|
"skillDisclosure"?: "withheld" | "named";
|
|
9725
10351
|
"modeOptions"?: Record<string, unknown>;
|
|
10352
|
+
"screens"?: Array<"test_awareness">;
|
|
10353
|
+
"runCounting"?: {
|
|
10354
|
+
"policy": "hold_back_screened";
|
|
10355
|
+
"screen": "test_awareness";
|
|
10356
|
+
"levels": Array<"aware" | "influenced">;
|
|
10357
|
+
} | null;
|
|
10358
|
+
"seriesKey"?: string | null;
|
|
10359
|
+
"ref"?: {
|
|
10360
|
+
"commit"?: string;
|
|
10361
|
+
"branch"?: string;
|
|
10362
|
+
"pr"?: number;
|
|
10363
|
+
"url"?: string;
|
|
10364
|
+
} | null;
|
|
10365
|
+
"gradingRevision"?: number;
|
|
10366
|
+
"gradingRevisedAt"?: string | null;
|
|
10367
|
+
"rehearsal"?: true;
|
|
10368
|
+
"rehearsalMode"?: string;
|
|
10369
|
+
"behaviors"?: Array<{
|
|
10370
|
+
"key": string;
|
|
10371
|
+
"name": string;
|
|
10372
|
+
"question": string;
|
|
10373
|
+
"version": number;
|
|
10374
|
+
}>;
|
|
9726
10375
|
};
|
|
9727
10376
|
"streamUrl": string;
|
|
9728
10377
|
"warnings"?: Array<{
|
|
@@ -9733,6 +10382,68 @@ export type StudyGenerateResponse = {
|
|
|
9733
10382
|
"detail": string;
|
|
9734
10383
|
}>;
|
|
9735
10384
|
};
|
|
10385
|
+
export type StudyGradingRevisionsResponse = {
|
|
10386
|
+
"revision": number;
|
|
10387
|
+
"revisions": Array<{
|
|
10388
|
+
"revision": number;
|
|
10389
|
+
"createdBy": string | null;
|
|
10390
|
+
"createdAt": string;
|
|
10391
|
+
"reason": string;
|
|
10392
|
+
"changes": Array<{
|
|
10393
|
+
"taskId": string;
|
|
10394
|
+
"position": number;
|
|
10395
|
+
"before": {
|
|
10396
|
+
"rubric": Array<string>;
|
|
10397
|
+
"expectations": Array<unknown>;
|
|
10398
|
+
};
|
|
10399
|
+
"after": {
|
|
10400
|
+
"rubric": Array<string>;
|
|
10401
|
+
"expectations": Array<unknown>;
|
|
10402
|
+
};
|
|
10403
|
+
"lenses": Array<"judge" | "evals">;
|
|
10404
|
+
}>;
|
|
10405
|
+
}>;
|
|
10406
|
+
};
|
|
10407
|
+
export type StudyReviseGradingRequest = {
|
|
10408
|
+
"reason": string;
|
|
10409
|
+
"tasks": Array<{
|
|
10410
|
+
"taskId": string;
|
|
10411
|
+
"rubric"?: Array<string>;
|
|
10412
|
+
"expectations"?: Array<unknown>;
|
|
10413
|
+
}>;
|
|
10414
|
+
"dryRun"?: boolean;
|
|
10415
|
+
};
|
|
10416
|
+
export type StudyReviseGradingResponse = {
|
|
10417
|
+
"dryRun": boolean;
|
|
10418
|
+
"plan": {
|
|
10419
|
+
"revision": number;
|
|
10420
|
+
"lenses": Array<"judge" | "evals">;
|
|
10421
|
+
"tasks": number;
|
|
10422
|
+
"evalRegrades": number;
|
|
10423
|
+
"judgeReads": number;
|
|
10424
|
+
};
|
|
10425
|
+
"changes": Array<{
|
|
10426
|
+
"taskId": string;
|
|
10427
|
+
"position": number;
|
|
10428
|
+
"before": {
|
|
10429
|
+
"rubric": Array<string>;
|
|
10430
|
+
"expectations": Array<unknown>;
|
|
10431
|
+
};
|
|
10432
|
+
"after": {
|
|
10433
|
+
"rubric": Array<string>;
|
|
10434
|
+
"expectations": Array<unknown>;
|
|
10435
|
+
};
|
|
10436
|
+
"lenses": Array<"judge" | "evals">;
|
|
10437
|
+
}>;
|
|
10438
|
+
"flips"?: Array<{
|
|
10439
|
+
"runId": string;
|
|
10440
|
+
"taskId": string;
|
|
10441
|
+
"arm": string;
|
|
10442
|
+
"before": number | null;
|
|
10443
|
+
"after": number | null;
|
|
10444
|
+
}>;
|
|
10445
|
+
"flipCount"?: number;
|
|
10446
|
+
};
|
|
9736
10447
|
export type StudyLaunchResponse = {
|
|
9737
10448
|
"study": {
|
|
9738
10449
|
"id": string;
|
|
@@ -9868,6 +10579,8 @@ export type StudyLaunchResponse = {
|
|
|
9868
10579
|
"startsPerMinute"?: number;
|
|
9869
10580
|
} | null;
|
|
9870
10581
|
"relaunchable"?: boolean;
|
|
10582
|
+
"archivedAt"?: string | null;
|
|
10583
|
+
"archivedBy"?: string | null;
|
|
9871
10584
|
"arms"?: Array<{
|
|
9872
10585
|
"key": string;
|
|
9873
10586
|
"position": number;
|
|
@@ -9906,6 +10619,29 @@ export type StudyLaunchResponse = {
|
|
|
9906
10619
|
"cellPlan"?: "crossed" | "per_setup";
|
|
9907
10620
|
"skillDisclosure"?: "withheld" | "named";
|
|
9908
10621
|
"modeOptions"?: Record<string, unknown>;
|
|
10622
|
+
"screens"?: Array<"test_awareness">;
|
|
10623
|
+
"runCounting"?: {
|
|
10624
|
+
"policy": "hold_back_screened";
|
|
10625
|
+
"screen": "test_awareness";
|
|
10626
|
+
"levels": Array<"aware" | "influenced">;
|
|
10627
|
+
} | null;
|
|
10628
|
+
"seriesKey"?: string | null;
|
|
10629
|
+
"ref"?: {
|
|
10630
|
+
"commit"?: string;
|
|
10631
|
+
"branch"?: string;
|
|
10632
|
+
"pr"?: number;
|
|
10633
|
+
"url"?: string;
|
|
10634
|
+
} | null;
|
|
10635
|
+
"gradingRevision"?: number;
|
|
10636
|
+
"gradingRevisedAt"?: string | null;
|
|
10637
|
+
"rehearsal"?: true;
|
|
10638
|
+
"rehearsalMode"?: string;
|
|
10639
|
+
"behaviors"?: Array<{
|
|
10640
|
+
"key": string;
|
|
10641
|
+
"name": string;
|
|
10642
|
+
"question": string;
|
|
10643
|
+
"version": number;
|
|
10644
|
+
}>;
|
|
9909
10645
|
};
|
|
9910
10646
|
"streamUrl": string;
|
|
9911
10647
|
"warnings"?: Array<{
|
|
@@ -10054,6 +10790,8 @@ export type StudyUpdateLensesResponse = {
|
|
|
10054
10790
|
"startsPerMinute"?: number;
|
|
10055
10791
|
} | null;
|
|
10056
10792
|
"relaunchable"?: boolean;
|
|
10793
|
+
"archivedAt"?: string | null;
|
|
10794
|
+
"archivedBy"?: string | null;
|
|
10057
10795
|
"arms"?: Array<{
|
|
10058
10796
|
"key": string;
|
|
10059
10797
|
"position": number;
|
|
@@ -10092,6 +10830,29 @@ export type StudyUpdateLensesResponse = {
|
|
|
10092
10830
|
"cellPlan"?: "crossed" | "per_setup";
|
|
10093
10831
|
"skillDisclosure"?: "withheld" | "named";
|
|
10094
10832
|
"modeOptions"?: Record<string, unknown>;
|
|
10833
|
+
"screens"?: Array<"test_awareness">;
|
|
10834
|
+
"runCounting"?: {
|
|
10835
|
+
"policy": "hold_back_screened";
|
|
10836
|
+
"screen": "test_awareness";
|
|
10837
|
+
"levels": Array<"aware" | "influenced">;
|
|
10838
|
+
} | null;
|
|
10839
|
+
"seriesKey"?: string | null;
|
|
10840
|
+
"ref"?: {
|
|
10841
|
+
"commit"?: string;
|
|
10842
|
+
"branch"?: string;
|
|
10843
|
+
"pr"?: number;
|
|
10844
|
+
"url"?: string;
|
|
10845
|
+
} | null;
|
|
10846
|
+
"gradingRevision"?: number;
|
|
10847
|
+
"gradingRevisedAt"?: string | null;
|
|
10848
|
+
"rehearsal"?: true;
|
|
10849
|
+
"rehearsalMode"?: string;
|
|
10850
|
+
"behaviors"?: Array<{
|
|
10851
|
+
"key": string;
|
|
10852
|
+
"name": string;
|
|
10853
|
+
"question": string;
|
|
10854
|
+
"version": number;
|
|
10855
|
+
}>;
|
|
10095
10856
|
};
|
|
10096
10857
|
"warnings": Array<{
|
|
10097
10858
|
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
@@ -10195,6 +10956,14 @@ export type StudyPreviewResponse = {
|
|
|
10195
10956
|
"label": string;
|
|
10196
10957
|
"skills": Array<string>;
|
|
10197
10958
|
}>;
|
|
10959
|
+
"gradingRevision"?: number;
|
|
10960
|
+
"gradingRevisedAt"?: string | null;
|
|
10961
|
+
"behaviors"?: Array<{
|
|
10962
|
+
"key": string;
|
|
10963
|
+
"name": string;
|
|
10964
|
+
"question": string;
|
|
10965
|
+
"version": number;
|
|
10966
|
+
}>;
|
|
10198
10967
|
};
|
|
10199
10968
|
"tasks": Array<{
|
|
10200
10969
|
"id": string;
|
|
@@ -10255,6 +11024,8 @@ export type StudyPreviewResponse = {
|
|
|
10255
11024
|
"paramsKey"?: string;
|
|
10256
11025
|
"textWithheld"?: boolean;
|
|
10257
11026
|
"runTimeMs"?: number | null;
|
|
11027
|
+
"screens"?: Record<string, unknown>;
|
|
11028
|
+
"carriedRun"?: true;
|
|
10258
11029
|
}>;
|
|
10259
11030
|
"aggregate": {
|
|
10260
11031
|
"summary": {
|
|
@@ -10518,6 +11289,18 @@ export type StudyPreviewResponse = {
|
|
|
10518
11289
|
"byArm": Record<string, unknown>;
|
|
10519
11290
|
"phasesByArm": Record<string, unknown>;
|
|
10520
11291
|
};
|
|
11292
|
+
"runCounting"?: {
|
|
11293
|
+
"policy": string;
|
|
11294
|
+
"screen": string;
|
|
11295
|
+
"levels": Array<string>;
|
|
11296
|
+
"heldBack": number;
|
|
11297
|
+
"groups": Array<{
|
|
11298
|
+
"group": string;
|
|
11299
|
+
"heldBack": number;
|
|
11300
|
+
"runs": number;
|
|
11301
|
+
}>;
|
|
11302
|
+
"imbalanced": boolean;
|
|
11303
|
+
};
|
|
10521
11304
|
} | {
|
|
10522
11305
|
"shape": "multi-arm";
|
|
10523
11306
|
"summary": {
|
|
@@ -11393,6 +12176,18 @@ export type StudyPreviewResponse = {
|
|
|
11393
12176
|
"byArm": Record<string, unknown>;
|
|
11394
12177
|
"phasesByArm": Record<string, unknown>;
|
|
11395
12178
|
};
|
|
12179
|
+
"runCounting"?: {
|
|
12180
|
+
"policy": string;
|
|
12181
|
+
"screen": string;
|
|
12182
|
+
"levels": Array<string>;
|
|
12183
|
+
"heldBack": number;
|
|
12184
|
+
"groups": Array<{
|
|
12185
|
+
"group": string;
|
|
12186
|
+
"heldBack": number;
|
|
12187
|
+
"runs": number;
|
|
12188
|
+
}>;
|
|
12189
|
+
"imbalanced": boolean;
|
|
12190
|
+
};
|
|
11396
12191
|
};
|
|
11397
12192
|
"textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
|
|
11398
12193
|
};
|
|
@@ -11537,6 +12332,8 @@ export type StudyPublishResponse = {
|
|
|
11537
12332
|
"startsPerMinute"?: number;
|
|
11538
12333
|
} | null;
|
|
11539
12334
|
"relaunchable"?: boolean;
|
|
12335
|
+
"archivedAt"?: string | null;
|
|
12336
|
+
"archivedBy"?: string | null;
|
|
11540
12337
|
"arms"?: Array<{
|
|
11541
12338
|
"key": string;
|
|
11542
12339
|
"position": number;
|
|
@@ -11575,8 +12372,72 @@ export type StudyPublishResponse = {
|
|
|
11575
12372
|
"cellPlan"?: "crossed" | "per_setup";
|
|
11576
12373
|
"skillDisclosure"?: "withheld" | "named";
|
|
11577
12374
|
"modeOptions"?: Record<string, unknown>;
|
|
12375
|
+
"screens"?: Array<"test_awareness">;
|
|
12376
|
+
"runCounting"?: {
|
|
12377
|
+
"policy": "hold_back_screened";
|
|
12378
|
+
"screen": "test_awareness";
|
|
12379
|
+
"levels": Array<"aware" | "influenced">;
|
|
12380
|
+
} | null;
|
|
12381
|
+
"seriesKey"?: string | null;
|
|
12382
|
+
"ref"?: {
|
|
12383
|
+
"commit"?: string;
|
|
12384
|
+
"branch"?: string;
|
|
12385
|
+
"pr"?: number;
|
|
12386
|
+
"url"?: string;
|
|
12387
|
+
} | null;
|
|
12388
|
+
"gradingRevision"?: number;
|
|
12389
|
+
"gradingRevisedAt"?: string | null;
|
|
12390
|
+
"rehearsal"?: true;
|
|
12391
|
+
"rehearsalMode"?: string;
|
|
12392
|
+
"behaviors"?: Array<{
|
|
12393
|
+
"key": string;
|
|
12394
|
+
"name": string;
|
|
12395
|
+
"question": string;
|
|
12396
|
+
"version": number;
|
|
12397
|
+
}>;
|
|
11578
12398
|
};
|
|
11579
12399
|
};
|
|
12400
|
+
export type StudyQueryRequest = {
|
|
12401
|
+
"metrics": Array<string>;
|
|
12402
|
+
"groupBy"?: Array<string>;
|
|
12403
|
+
"filters"?: Array<{
|
|
12404
|
+
"field": string;
|
|
12405
|
+
"in": Array<string>;
|
|
12406
|
+
}>;
|
|
12407
|
+
"limit"?: number;
|
|
12408
|
+
};
|
|
12409
|
+
export type StudyQueryResponse = {
|
|
12410
|
+
"basis": "runs";
|
|
12411
|
+
"matchedRuns": number;
|
|
12412
|
+
"groupBy": Array<{
|
|
12413
|
+
"id": string;
|
|
12414
|
+
"label": string;
|
|
12415
|
+
}>;
|
|
12416
|
+
"metrics": Array<{
|
|
12417
|
+
"id": string;
|
|
12418
|
+
"label": string;
|
|
12419
|
+
"unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
|
|
12420
|
+
"fold": string;
|
|
12421
|
+
}>;
|
|
12422
|
+
"rows": Array<{
|
|
12423
|
+
"key": Record<string, unknown>;
|
|
12424
|
+
"runs": number;
|
|
12425
|
+
"cells": Record<string, unknown>;
|
|
12426
|
+
}>;
|
|
12427
|
+
"truncated": boolean;
|
|
12428
|
+
};
|
|
12429
|
+
export type StudyQueryVocabularyResponse = {
|
|
12430
|
+
"dimensions": Array<{
|
|
12431
|
+
"id": string;
|
|
12432
|
+
"label": string;
|
|
12433
|
+
}>;
|
|
12434
|
+
"metrics": Array<{
|
|
12435
|
+
"id": string;
|
|
12436
|
+
"label": string;
|
|
12437
|
+
"unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
|
|
12438
|
+
"fold": string;
|
|
12439
|
+
}>;
|
|
12440
|
+
};
|
|
11580
12441
|
export type StudyReconcileRunsResponse = {
|
|
11581
12442
|
"studyId": string;
|
|
11582
12443
|
"laggingRuns": number;
|
|
@@ -11721,6 +12582,8 @@ export type StudyRegradeResponse = {
|
|
|
11721
12582
|
"startsPerMinute"?: number;
|
|
11722
12583
|
} | null;
|
|
11723
12584
|
"relaunchable"?: boolean;
|
|
12585
|
+
"archivedAt"?: string | null;
|
|
12586
|
+
"archivedBy"?: string | null;
|
|
11724
12587
|
"arms"?: Array<{
|
|
11725
12588
|
"key": string;
|
|
11726
12589
|
"position": number;
|
|
@@ -11759,6 +12622,29 @@ export type StudyRegradeResponse = {
|
|
|
11759
12622
|
"cellPlan"?: "crossed" | "per_setup";
|
|
11760
12623
|
"skillDisclosure"?: "withheld" | "named";
|
|
11761
12624
|
"modeOptions"?: Record<string, unknown>;
|
|
12625
|
+
"screens"?: Array<"test_awareness">;
|
|
12626
|
+
"runCounting"?: {
|
|
12627
|
+
"policy": "hold_back_screened";
|
|
12628
|
+
"screen": "test_awareness";
|
|
12629
|
+
"levels": Array<"aware" | "influenced">;
|
|
12630
|
+
} | null;
|
|
12631
|
+
"seriesKey"?: string | null;
|
|
12632
|
+
"ref"?: {
|
|
12633
|
+
"commit"?: string;
|
|
12634
|
+
"branch"?: string;
|
|
12635
|
+
"pr"?: number;
|
|
12636
|
+
"url"?: string;
|
|
12637
|
+
} | null;
|
|
12638
|
+
"gradingRevision"?: number;
|
|
12639
|
+
"gradingRevisedAt"?: string | null;
|
|
12640
|
+
"rehearsal"?: true;
|
|
12641
|
+
"rehearsalMode"?: string;
|
|
12642
|
+
"behaviors"?: Array<{
|
|
12643
|
+
"key": string;
|
|
12644
|
+
"name": string;
|
|
12645
|
+
"question": string;
|
|
12646
|
+
"version": number;
|
|
12647
|
+
}>;
|
|
11762
12648
|
};
|
|
11763
12649
|
"streamUrl": string;
|
|
11764
12650
|
"warnings"?: Array<{
|
|
@@ -11769,6 +12655,23 @@ export type StudyRegradeResponse = {
|
|
|
11769
12655
|
"detail": string;
|
|
11770
12656
|
}>;
|
|
11771
12657
|
};
|
|
12658
|
+
export type StudyRehearsalReportResponse = {
|
|
12659
|
+
"studyId": string;
|
|
12660
|
+
"name": string;
|
|
12661
|
+
"status": string;
|
|
12662
|
+
"mode": string;
|
|
12663
|
+
"state": "pending" | "ready" | "not_ready";
|
|
12664
|
+
"groups": Array<{
|
|
12665
|
+
"key": string;
|
|
12666
|
+
"label": string;
|
|
12667
|
+
"state": "pending" | "ready" | "not_ready";
|
|
12668
|
+
"checks": Array<{
|
|
12669
|
+
"check": string;
|
|
12670
|
+
"answer": "pass" | "fail" | "pending" | "unknown";
|
|
12671
|
+
"detail": string;
|
|
12672
|
+
}>;
|
|
12673
|
+
}>;
|
|
12674
|
+
};
|
|
11772
12675
|
export type StudyRerunAttemptsResponse = {
|
|
11773
12676
|
"studyId": string;
|
|
11774
12677
|
"attempts": Array<{
|
|
@@ -11837,6 +12740,405 @@ export type StudyRerunInfraFailuresResponse = {
|
|
|
11837
12740
|
"reason": string;
|
|
11838
12741
|
}>;
|
|
11839
12742
|
};
|
|
12743
|
+
export type StudyRestoreResponse = {
|
|
12744
|
+
"study": {
|
|
12745
|
+
"id": string;
|
|
12746
|
+
"executiveSummary"?: string | null;
|
|
12747
|
+
"executiveSummaryDraft"?: string | null;
|
|
12748
|
+
"executiveSummaryRequest"?: "live" | "draft" | null;
|
|
12749
|
+
"executiveSummaryRequestedAt"?: string | null;
|
|
12750
|
+
"executiveSummaryUnavailableAt"?: string | null;
|
|
12751
|
+
"plannedRunCount": number;
|
|
12752
|
+
"organizationId": string;
|
|
12753
|
+
"createdBy": string | null;
|
|
12754
|
+
"name": string;
|
|
12755
|
+
"slug": string;
|
|
12756
|
+
"description": string | null;
|
|
12757
|
+
"skillRef": string | null;
|
|
12758
|
+
"skillSlug": string | null;
|
|
12759
|
+
"skillName": string | null;
|
|
12760
|
+
"skillPromise": string;
|
|
12761
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
12762
|
+
"subjectName": string | null;
|
|
12763
|
+
"subjectLabel": string;
|
|
12764
|
+
"armConfig": {
|
|
12765
|
+
"baseline"?: {
|
|
12766
|
+
"tools"?: "default" | "none" | {
|
|
12767
|
+
"tools": Array<string>;
|
|
12768
|
+
};
|
|
12769
|
+
"addons"?: Record<string, unknown>;
|
|
12770
|
+
};
|
|
12771
|
+
"treatment"?: {
|
|
12772
|
+
"tools"?: "default" | "none" | {
|
|
12773
|
+
"tools": Array<string>;
|
|
12774
|
+
};
|
|
12775
|
+
"addons"?: Record<string, unknown>;
|
|
12776
|
+
};
|
|
12777
|
+
};
|
|
12778
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
12779
|
+
"subjectBackend": "model" | "harness" | null;
|
|
12780
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
12781
|
+
"importedFrom"?: string;
|
|
12782
|
+
"baselineLabel": string | null;
|
|
12783
|
+
"treatmentLabel": string | null;
|
|
12784
|
+
"promiseMetrics": Array<{
|
|
12785
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
12786
|
+
"direction": "down" | "up";
|
|
12787
|
+
"label": string;
|
|
12788
|
+
}>;
|
|
12789
|
+
"matrix": Array<{
|
|
12790
|
+
"harness": string;
|
|
12791
|
+
"model": string;
|
|
12792
|
+
"provider"?: string;
|
|
12793
|
+
"params"?: {
|
|
12794
|
+
"temperature"?: number;
|
|
12795
|
+
"maxOutputTokens"?: number;
|
|
12796
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
12797
|
+
"thinkingBudgetTokens"?: number;
|
|
12798
|
+
"codeMode"?: "auto" | "on" | "off";
|
|
12799
|
+
"toolSearch"?: "on" | "off" | "directory";
|
|
12800
|
+
};
|
|
12801
|
+
}>;
|
|
12802
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
12803
|
+
"judgeSupplementary"?: boolean;
|
|
12804
|
+
"taskCount": number;
|
|
12805
|
+
"variantsPerTask": number;
|
|
12806
|
+
"repeats": number;
|
|
12807
|
+
"subset": {
|
|
12808
|
+
"count": number;
|
|
12809
|
+
"seed": string;
|
|
12810
|
+
"of": number;
|
|
12811
|
+
} | null;
|
|
12812
|
+
"status": string;
|
|
12813
|
+
"error": string | null;
|
|
12814
|
+
"preparation"?: {
|
|
12815
|
+
"since": string;
|
|
12816
|
+
"progressAt"?: string;
|
|
12817
|
+
"launchedBy": {
|
|
12818
|
+
"userId"?: string;
|
|
12819
|
+
"role"?: string;
|
|
12820
|
+
};
|
|
12821
|
+
"waitingOn": Array<{
|
|
12822
|
+
"imageKey": string;
|
|
12823
|
+
"buildId": string;
|
|
12824
|
+
"stage"?: "image" | "snapshot";
|
|
12825
|
+
"retry"?: {
|
|
12826
|
+
"failedAttempts": number;
|
|
12827
|
+
"maxAttempts": number;
|
|
12828
|
+
"notBefore": string;
|
|
12829
|
+
"lastFailure": string;
|
|
12830
|
+
};
|
|
12831
|
+
}>;
|
|
12832
|
+
} | null;
|
|
12833
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
12834
|
+
"visibilityChangedAt"?: string | null;
|
|
12835
|
+
"visibilityChangedBy"?: string | null;
|
|
12836
|
+
"shareToken"?: string | null;
|
|
12837
|
+
"lowSampleOverride": boolean;
|
|
12838
|
+
"publishedAt": string | null;
|
|
12839
|
+
"verdict": unknown;
|
|
12840
|
+
"definitionVersionId": string | null;
|
|
12841
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
12842
|
+
"createdAt": string;
|
|
12843
|
+
"updatedAt": string;
|
|
12844
|
+
"liveUntil": string | null;
|
|
12845
|
+
"launch"?: {
|
|
12846
|
+
"attempt": number;
|
|
12847
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
12848
|
+
"startedAt": string;
|
|
12849
|
+
"progressAt": string;
|
|
12850
|
+
"planned"?: number;
|
|
12851
|
+
"created"?: number;
|
|
12852
|
+
"missing"?: {
|
|
12853
|
+
"retryable": number;
|
|
12854
|
+
"refused": number;
|
|
12855
|
+
};
|
|
12856
|
+
"notes"?: Array<string>;
|
|
12857
|
+
"settledAt"?: string;
|
|
12858
|
+
"pacing"?: {
|
|
12859
|
+
"waiting": number;
|
|
12860
|
+
"nextUnit"?: number;
|
|
12861
|
+
"nextWaveAt"?: string;
|
|
12862
|
+
"emptyWaves"?: number;
|
|
12863
|
+
"stopped"?: string;
|
|
12864
|
+
};
|
|
12865
|
+
"recovery"?: {
|
|
12866
|
+
"state": "retrying" | "needs_attention";
|
|
12867
|
+
"attempts": number;
|
|
12868
|
+
"lastAttemptAt"?: string;
|
|
12869
|
+
"nextAttemptAt"?: string;
|
|
12870
|
+
"reason"?: string;
|
|
12871
|
+
};
|
|
12872
|
+
} | null;
|
|
12873
|
+
"pacing"?: {
|
|
12874
|
+
"maxInFlight"?: number;
|
|
12875
|
+
"startsPerMinute"?: number;
|
|
12876
|
+
} | null;
|
|
12877
|
+
"relaunchable"?: boolean;
|
|
12878
|
+
"archivedAt"?: string | null;
|
|
12879
|
+
"archivedBy"?: string | null;
|
|
12880
|
+
"arms"?: Array<{
|
|
12881
|
+
"key": string;
|
|
12882
|
+
"position": number;
|
|
12883
|
+
"label": string;
|
|
12884
|
+
"role": "reference" | "peer";
|
|
12885
|
+
"payload": {
|
|
12886
|
+
"tools"?: "default" | "none" | {
|
|
12887
|
+
"tools": Array<string>;
|
|
12888
|
+
};
|
|
12889
|
+
"addons"?: Record<string, unknown>;
|
|
12890
|
+
};
|
|
12891
|
+
"cells"?: Array<{
|
|
12892
|
+
"harness": string;
|
|
12893
|
+
"model": string;
|
|
12894
|
+
"provider"?: string;
|
|
12895
|
+
"params"?: {
|
|
12896
|
+
"temperature"?: number;
|
|
12897
|
+
"maxOutputTokens"?: number;
|
|
12898
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
12899
|
+
"thinkingBudgetTokens"?: number;
|
|
12900
|
+
"codeMode"?: "auto" | "on" | "off";
|
|
12901
|
+
"toolSearch"?: "on" | "off" | "directory";
|
|
12902
|
+
};
|
|
12903
|
+
}>;
|
|
12904
|
+
"skills"?: Array<{
|
|
12905
|
+
"ref": string;
|
|
12906
|
+
"name": string;
|
|
12907
|
+
"changedSinceCreate"?: boolean;
|
|
12908
|
+
}>;
|
|
12909
|
+
"addedAgents"?: Array<{
|
|
12910
|
+
"position": number;
|
|
12911
|
+
"addedAt": string;
|
|
12912
|
+
}>;
|
|
12913
|
+
}>;
|
|
12914
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
12915
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
12916
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
12917
|
+
"modeOptions"?: Record<string, unknown>;
|
|
12918
|
+
"screens"?: Array<"test_awareness">;
|
|
12919
|
+
"runCounting"?: {
|
|
12920
|
+
"policy": "hold_back_screened";
|
|
12921
|
+
"screen": "test_awareness";
|
|
12922
|
+
"levels": Array<"aware" | "influenced">;
|
|
12923
|
+
} | null;
|
|
12924
|
+
"seriesKey"?: string | null;
|
|
12925
|
+
"ref"?: {
|
|
12926
|
+
"commit"?: string;
|
|
12927
|
+
"branch"?: string;
|
|
12928
|
+
"pr"?: number;
|
|
12929
|
+
"url"?: string;
|
|
12930
|
+
} | null;
|
|
12931
|
+
"gradingRevision"?: number;
|
|
12932
|
+
"gradingRevisedAt"?: string | null;
|
|
12933
|
+
"rehearsal"?: true;
|
|
12934
|
+
"rehearsalMode"?: string;
|
|
12935
|
+
"behaviors"?: Array<{
|
|
12936
|
+
"key": string;
|
|
12937
|
+
"name": string;
|
|
12938
|
+
"question": string;
|
|
12939
|
+
"version": number;
|
|
12940
|
+
}>;
|
|
12941
|
+
};
|
|
12942
|
+
};
|
|
12943
|
+
export type StudyRetestsResponse = {
|
|
12944
|
+
"retests": Array<{
|
|
12945
|
+
"id": string;
|
|
12946
|
+
"kind": "fill" | "audit";
|
|
12947
|
+
"reason": string | null;
|
|
12948
|
+
"requestedBy": string | null;
|
|
12949
|
+
"createdAt": string;
|
|
12950
|
+
"settledAt": string | null;
|
|
12951
|
+
"summary": {
|
|
12952
|
+
"rechecked": number;
|
|
12953
|
+
"recorded": number;
|
|
12954
|
+
"filled": number;
|
|
12955
|
+
"agreed": number;
|
|
12956
|
+
"disagreements": Array<{
|
|
12957
|
+
"runId": string;
|
|
12958
|
+
"retestRunId": string;
|
|
12959
|
+
"run": {
|
|
12960
|
+
"pass": boolean | null;
|
|
12961
|
+
"status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
|
|
12962
|
+
"reason"?: string;
|
|
12963
|
+
};
|
|
12964
|
+
"recheck": {
|
|
12965
|
+
"pass": boolean | null;
|
|
12966
|
+
"status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
|
|
12967
|
+
"reason"?: string;
|
|
12968
|
+
};
|
|
12969
|
+
}>;
|
|
12970
|
+
};
|
|
12971
|
+
"runs": Array<{
|
|
12972
|
+
"runId": string;
|
|
12973
|
+
"retestRunId": string;
|
|
12974
|
+
"runStatus": string | null;
|
|
12975
|
+
"finishedAt": string | null;
|
|
12976
|
+
"verdict": {
|
|
12977
|
+
"pass": boolean | null;
|
|
12978
|
+
"status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
|
|
12979
|
+
"reason"?: string;
|
|
12980
|
+
} | null;
|
|
12981
|
+
}>;
|
|
12982
|
+
}>;
|
|
12983
|
+
};
|
|
12984
|
+
export type StudyRetestRequest = {
|
|
12985
|
+
"kind"?: "fill" | "audit";
|
|
12986
|
+
"runIds"?: Array<string>;
|
|
12987
|
+
"reason"?: string;
|
|
12988
|
+
"dryRun"?: boolean;
|
|
12989
|
+
};
|
|
12990
|
+
export type StudyRetestResponse = {
|
|
12991
|
+
"dryRun": boolean;
|
|
12992
|
+
"kind": "fill" | "audit";
|
|
12993
|
+
"retestId": string | null;
|
|
12994
|
+
"runs": Array<{
|
|
12995
|
+
"runId": string;
|
|
12996
|
+
"outcome"?: "dispatched" | "failed";
|
|
12997
|
+
"retestRunId"?: string | null;
|
|
12998
|
+
"error"?: string | null;
|
|
12999
|
+
}>;
|
|
13000
|
+
"skipped": Array<{
|
|
13001
|
+
"runId": string;
|
|
13002
|
+
"reason": "not_in_study" | "not_succeeded" | "no_verdict" | "no_saved_inputs" | "saved_files_missing";
|
|
13003
|
+
"detail": string;
|
|
13004
|
+
}>;
|
|
13005
|
+
};
|
|
13006
|
+
export type StudyRunCountingResponse = {
|
|
13007
|
+
"runCounting": {
|
|
13008
|
+
"policy": "hold_back_screened";
|
|
13009
|
+
"screen": "test_awareness";
|
|
13010
|
+
"levels": Array<"aware" | "influenced">;
|
|
13011
|
+
} | null;
|
|
13012
|
+
"policy": string;
|
|
13013
|
+
"offered": boolean;
|
|
13014
|
+
"refusal": {
|
|
13015
|
+
"code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
|
|
13016
|
+
"message": string;
|
|
13017
|
+
} | null;
|
|
13018
|
+
"screen": string;
|
|
13019
|
+
"screenedRuns": number;
|
|
13020
|
+
"unscreenedRuns": number;
|
|
13021
|
+
"flagged": Record<string, unknown>;
|
|
13022
|
+
"heldBack": Array<{
|
|
13023
|
+
"runId": string;
|
|
13024
|
+
"group": string;
|
|
13025
|
+
"taskId": string;
|
|
13026
|
+
"level": string;
|
|
13027
|
+
}>;
|
|
13028
|
+
"groups": Array<{
|
|
13029
|
+
"key": string;
|
|
13030
|
+
"label": string;
|
|
13031
|
+
"runs": number;
|
|
13032
|
+
"heldBack": number;
|
|
13033
|
+
"counted": {
|
|
13034
|
+
"value": number | null;
|
|
13035
|
+
"interval": Array<unknown> | null;
|
|
13036
|
+
"runs": number;
|
|
13037
|
+
"tasks": number;
|
|
13038
|
+
} | null;
|
|
13039
|
+
"all": {
|
|
13040
|
+
"value": number | null;
|
|
13041
|
+
"interval": Array<unknown> | null;
|
|
13042
|
+
"runs": number;
|
|
13043
|
+
"tasks": number;
|
|
13044
|
+
} | null;
|
|
13045
|
+
}>;
|
|
13046
|
+
"imbalanced": boolean;
|
|
13047
|
+
"metric": {
|
|
13048
|
+
"key": string;
|
|
13049
|
+
"label": string;
|
|
13050
|
+
"family": string;
|
|
13051
|
+
};
|
|
13052
|
+
"changes": Array<{
|
|
13053
|
+
"change": number;
|
|
13054
|
+
"createdBy": string | null;
|
|
13055
|
+
"createdAt": string;
|
|
13056
|
+
"previous": {
|
|
13057
|
+
"policy": "hold_back_screened";
|
|
13058
|
+
"screen": "test_awareness";
|
|
13059
|
+
"levels": Array<"aware" | "influenced">;
|
|
13060
|
+
} | null;
|
|
13061
|
+
"runCounting": {
|
|
13062
|
+
"policy": "hold_back_screened";
|
|
13063
|
+
"screen": "test_awareness";
|
|
13064
|
+
"levels": Array<"aware" | "influenced">;
|
|
13065
|
+
} | null;
|
|
13066
|
+
"classifierVersion": string | null;
|
|
13067
|
+
}>;
|
|
13068
|
+
};
|
|
13069
|
+
export type StudySetRunCountingRequest = {
|
|
13070
|
+
"runCounting": {
|
|
13071
|
+
"policy": "hold_back_screened";
|
|
13072
|
+
"screen": "test_awareness";
|
|
13073
|
+
"levels": Array<"aware" | "influenced">;
|
|
13074
|
+
} | null;
|
|
13075
|
+
"dryRun"?: boolean;
|
|
13076
|
+
};
|
|
13077
|
+
export type StudySetRunCountingResponse = {
|
|
13078
|
+
"runCounting": {
|
|
13079
|
+
"policy": "hold_back_screened";
|
|
13080
|
+
"screen": "test_awareness";
|
|
13081
|
+
"levels": Array<"aware" | "influenced">;
|
|
13082
|
+
} | null;
|
|
13083
|
+
"policy": string;
|
|
13084
|
+
"offered": boolean;
|
|
13085
|
+
"refusal": {
|
|
13086
|
+
"code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
|
|
13087
|
+
"message": string;
|
|
13088
|
+
} | null;
|
|
13089
|
+
"screen": string;
|
|
13090
|
+
"screenedRuns": number;
|
|
13091
|
+
"unscreenedRuns": number;
|
|
13092
|
+
"flagged": Record<string, unknown>;
|
|
13093
|
+
"heldBack": Array<{
|
|
13094
|
+
"runId": string;
|
|
13095
|
+
"group": string;
|
|
13096
|
+
"taskId": string;
|
|
13097
|
+
"level": string;
|
|
13098
|
+
}>;
|
|
13099
|
+
"groups": Array<{
|
|
13100
|
+
"key": string;
|
|
13101
|
+
"label": string;
|
|
13102
|
+
"runs": number;
|
|
13103
|
+
"heldBack": number;
|
|
13104
|
+
"counted": {
|
|
13105
|
+
"value": number | null;
|
|
13106
|
+
"interval": Array<unknown> | null;
|
|
13107
|
+
"runs": number;
|
|
13108
|
+
"tasks": number;
|
|
13109
|
+
} | null;
|
|
13110
|
+
"all": {
|
|
13111
|
+
"value": number | null;
|
|
13112
|
+
"interval": Array<unknown> | null;
|
|
13113
|
+
"runs": number;
|
|
13114
|
+
"tasks": number;
|
|
13115
|
+
} | null;
|
|
13116
|
+
}>;
|
|
13117
|
+
"imbalanced": boolean;
|
|
13118
|
+
"metric": {
|
|
13119
|
+
"key": string;
|
|
13120
|
+
"label": string;
|
|
13121
|
+
"family": string;
|
|
13122
|
+
};
|
|
13123
|
+
"changes": Array<{
|
|
13124
|
+
"change": number;
|
|
13125
|
+
"createdBy": string | null;
|
|
13126
|
+
"createdAt": string;
|
|
13127
|
+
"previous": {
|
|
13128
|
+
"policy": "hold_back_screened";
|
|
13129
|
+
"screen": "test_awareness";
|
|
13130
|
+
"levels": Array<"aware" | "influenced">;
|
|
13131
|
+
} | null;
|
|
13132
|
+
"runCounting": {
|
|
13133
|
+
"policy": "hold_back_screened";
|
|
13134
|
+
"screen": "test_awareness";
|
|
13135
|
+
"levels": Array<"aware" | "influenced">;
|
|
13136
|
+
} | null;
|
|
13137
|
+
"classifierVersion": string | null;
|
|
13138
|
+
}>;
|
|
13139
|
+
"dryRun": boolean;
|
|
13140
|
+
"changed": boolean;
|
|
13141
|
+
};
|
|
11840
13142
|
export type StudyListRunAnswersResponse = {
|
|
11841
13143
|
"coordinatorRunId": string;
|
|
11842
13144
|
"answers": Array<{
|
|
@@ -11850,7 +13152,35 @@ export type StudyListRunAnswersResponse = {
|
|
|
11850
13152
|
"finishedAt": string | null;
|
|
11851
13153
|
}>;
|
|
11852
13154
|
};
|
|
13155
|
+
export type StudyRunDeliveredResponse = {
|
|
13156
|
+
"takenAt": string | null;
|
|
13157
|
+
"items": Array<{
|
|
13158
|
+
"id": string;
|
|
13159
|
+
"kind": "file" | "image";
|
|
13160
|
+
"origin": string;
|
|
13161
|
+
"url"?: string;
|
|
13162
|
+
"path"?: string;
|
|
13163
|
+
"status": string;
|
|
13164
|
+
"reason"?: string;
|
|
13165
|
+
"contentType"?: string;
|
|
13166
|
+
"bytes"?: number;
|
|
13167
|
+
"sha256"?: string;
|
|
13168
|
+
"readable"?: boolean;
|
|
13169
|
+
"redactions"?: Array<string>;
|
|
13170
|
+
"truncated"?: boolean;
|
|
13171
|
+
"downloadable": boolean;
|
|
13172
|
+
"filename": string;
|
|
13173
|
+
}>;
|
|
13174
|
+
"workspaceSkipped"?: {
|
|
13175
|
+
"dependencyOrDot": number;
|
|
13176
|
+
"overCap": number;
|
|
13177
|
+
};
|
|
13178
|
+
};
|
|
11853
13179
|
export type StudyRunInsightResponse = unknown;
|
|
13180
|
+
export type StudyRunScreensResponse = {
|
|
13181
|
+
"screens": Record<string, unknown>;
|
|
13182
|
+
};
|
|
13183
|
+
export type StudyRunSpansResponse = unknown;
|
|
11854
13184
|
export type StudyRunTrajectoryResponse = unknown;
|
|
11855
13185
|
export type StudyRotateShareTokenResponse = {
|
|
11856
13186
|
"study": {
|
|
@@ -11987,6 +13317,8 @@ export type StudyRotateShareTokenResponse = {
|
|
|
11987
13317
|
"startsPerMinute"?: number;
|
|
11988
13318
|
} | null;
|
|
11989
13319
|
"relaunchable"?: boolean;
|
|
13320
|
+
"archivedAt"?: string | null;
|
|
13321
|
+
"archivedBy"?: string | null;
|
|
11990
13322
|
"arms"?: Array<{
|
|
11991
13323
|
"key": string;
|
|
11992
13324
|
"position": number;
|
|
@@ -12025,6 +13357,29 @@ export type StudyRotateShareTokenResponse = {
|
|
|
12025
13357
|
"cellPlan"?: "crossed" | "per_setup";
|
|
12026
13358
|
"skillDisclosure"?: "withheld" | "named";
|
|
12027
13359
|
"modeOptions"?: Record<string, unknown>;
|
|
13360
|
+
"screens"?: Array<"test_awareness">;
|
|
13361
|
+
"runCounting"?: {
|
|
13362
|
+
"policy": "hold_back_screened";
|
|
13363
|
+
"screen": "test_awareness";
|
|
13364
|
+
"levels": Array<"aware" | "influenced">;
|
|
13365
|
+
} | null;
|
|
13366
|
+
"seriesKey"?: string | null;
|
|
13367
|
+
"ref"?: {
|
|
13368
|
+
"commit"?: string;
|
|
13369
|
+
"branch"?: string;
|
|
13370
|
+
"pr"?: number;
|
|
13371
|
+
"url"?: string;
|
|
13372
|
+
} | null;
|
|
13373
|
+
"gradingRevision"?: number;
|
|
13374
|
+
"gradingRevisedAt"?: string | null;
|
|
13375
|
+
"rehearsal"?: true;
|
|
13376
|
+
"rehearsalMode"?: string;
|
|
13377
|
+
"behaviors"?: Array<{
|
|
13378
|
+
"key": string;
|
|
13379
|
+
"name": string;
|
|
13380
|
+
"question": string;
|
|
13381
|
+
"version": number;
|
|
13382
|
+
}>;
|
|
12028
13383
|
};
|
|
12029
13384
|
"shareUrl": string | null;
|
|
12030
13385
|
"shareLinkKept": boolean;
|
|
@@ -12251,6 +13606,8 @@ export type StudyUnpublishResponse = {
|
|
|
12251
13606
|
"startsPerMinute"?: number;
|
|
12252
13607
|
} | null;
|
|
12253
13608
|
"relaunchable"?: boolean;
|
|
13609
|
+
"archivedAt"?: string | null;
|
|
13610
|
+
"archivedBy"?: string | null;
|
|
12254
13611
|
"arms"?: Array<{
|
|
12255
13612
|
"key": string;
|
|
12256
13613
|
"position": number;
|
|
@@ -12289,8 +13646,93 @@ export type StudyUnpublishResponse = {
|
|
|
12289
13646
|
"cellPlan"?: "crossed" | "per_setup";
|
|
12290
13647
|
"skillDisclosure"?: "withheld" | "named";
|
|
12291
13648
|
"modeOptions"?: Record<string, unknown>;
|
|
13649
|
+
"screens"?: Array<"test_awareness">;
|
|
13650
|
+
"runCounting"?: {
|
|
13651
|
+
"policy": "hold_back_screened";
|
|
13652
|
+
"screen": "test_awareness";
|
|
13653
|
+
"levels": Array<"aware" | "influenced">;
|
|
13654
|
+
} | null;
|
|
13655
|
+
"seriesKey"?: string | null;
|
|
13656
|
+
"ref"?: {
|
|
13657
|
+
"commit"?: string;
|
|
13658
|
+
"branch"?: string;
|
|
13659
|
+
"pr"?: number;
|
|
13660
|
+
"url"?: string;
|
|
13661
|
+
} | null;
|
|
13662
|
+
"gradingRevision"?: number;
|
|
13663
|
+
"gradingRevisedAt"?: string | null;
|
|
13664
|
+
"rehearsal"?: true;
|
|
13665
|
+
"rehearsalMode"?: string;
|
|
13666
|
+
"behaviors"?: Array<{
|
|
13667
|
+
"key": string;
|
|
13668
|
+
"name": string;
|
|
13669
|
+
"question": string;
|
|
13670
|
+
"version": number;
|
|
13671
|
+
}>;
|
|
13672
|
+
};
|
|
13673
|
+
};
|
|
13674
|
+
export type StudyViewsResponse = {
|
|
13675
|
+
"views": Array<{
|
|
13676
|
+
"id": string;
|
|
13677
|
+
"name": string;
|
|
13678
|
+
"query": unknown;
|
|
13679
|
+
"createdBy": string | null;
|
|
13680
|
+
"createdAt": string;
|
|
13681
|
+
}>;
|
|
13682
|
+
};
|
|
13683
|
+
export type StudySaveViewRequest = {
|
|
13684
|
+
"name": string;
|
|
13685
|
+
"query": {
|
|
13686
|
+
"metrics": Array<string>;
|
|
13687
|
+
"groupBy"?: Array<string>;
|
|
13688
|
+
"filters"?: Array<{
|
|
13689
|
+
"field": string;
|
|
13690
|
+
"in": Array<string>;
|
|
13691
|
+
}>;
|
|
13692
|
+
"limit"?: number;
|
|
12292
13693
|
};
|
|
12293
13694
|
};
|
|
13695
|
+
export type StudySaveViewResponse = {
|
|
13696
|
+
"view": {
|
|
13697
|
+
"id": string;
|
|
13698
|
+
"name": string;
|
|
13699
|
+
"query": unknown;
|
|
13700
|
+
"createdBy": string | null;
|
|
13701
|
+
"createdAt": string;
|
|
13702
|
+
};
|
|
13703
|
+
};
|
|
13704
|
+
export type StudyRunViewResponse = {
|
|
13705
|
+
"view": {
|
|
13706
|
+
"id": string;
|
|
13707
|
+
"name": string;
|
|
13708
|
+
"query": unknown;
|
|
13709
|
+
"createdBy": string | null;
|
|
13710
|
+
"createdAt": string;
|
|
13711
|
+
};
|
|
13712
|
+
"result": {
|
|
13713
|
+
"basis": "runs";
|
|
13714
|
+
"matchedRuns": number;
|
|
13715
|
+
"groupBy": Array<{
|
|
13716
|
+
"id": string;
|
|
13717
|
+
"label": string;
|
|
13718
|
+
}>;
|
|
13719
|
+
"metrics": Array<{
|
|
13720
|
+
"id": string;
|
|
13721
|
+
"label": string;
|
|
13722
|
+
"unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
|
|
13723
|
+
"fold": string;
|
|
13724
|
+
}>;
|
|
13725
|
+
"rows": Array<{
|
|
13726
|
+
"key": Record<string, unknown>;
|
|
13727
|
+
"runs": number;
|
|
13728
|
+
"cells": Record<string, unknown>;
|
|
13729
|
+
}>;
|
|
13730
|
+
"truncated": boolean;
|
|
13731
|
+
};
|
|
13732
|
+
};
|
|
13733
|
+
export type StudyDeleteViewResponse = {
|
|
13734
|
+
"deleted": true;
|
|
13735
|
+
};
|
|
12294
13736
|
export type StudySetVisibilityRequest = {
|
|
12295
13737
|
"visibility": "private" | "org" | "link" | "public";
|
|
12296
13738
|
};
|
|
@@ -12429,6 +13871,8 @@ export type StudySetVisibilityResponse = {
|
|
|
12429
13871
|
"startsPerMinute"?: number;
|
|
12430
13872
|
} | null;
|
|
12431
13873
|
"relaunchable"?: boolean;
|
|
13874
|
+
"archivedAt"?: string | null;
|
|
13875
|
+
"archivedBy"?: string | null;
|
|
12432
13876
|
"arms"?: Array<{
|
|
12433
13877
|
"key": string;
|
|
12434
13878
|
"position": number;
|
|
@@ -12467,6 +13911,29 @@ export type StudySetVisibilityResponse = {
|
|
|
12467
13911
|
"cellPlan"?: "crossed" | "per_setup";
|
|
12468
13912
|
"skillDisclosure"?: "withheld" | "named";
|
|
12469
13913
|
"modeOptions"?: Record<string, unknown>;
|
|
13914
|
+
"screens"?: Array<"test_awareness">;
|
|
13915
|
+
"runCounting"?: {
|
|
13916
|
+
"policy": "hold_back_screened";
|
|
13917
|
+
"screen": "test_awareness";
|
|
13918
|
+
"levels": Array<"aware" | "influenced">;
|
|
13919
|
+
} | null;
|
|
13920
|
+
"seriesKey"?: string | null;
|
|
13921
|
+
"ref"?: {
|
|
13922
|
+
"commit"?: string;
|
|
13923
|
+
"branch"?: string;
|
|
13924
|
+
"pr"?: number;
|
|
13925
|
+
"url"?: string;
|
|
13926
|
+
} | null;
|
|
13927
|
+
"gradingRevision"?: number;
|
|
13928
|
+
"gradingRevisedAt"?: string | null;
|
|
13929
|
+
"rehearsal"?: true;
|
|
13930
|
+
"rehearsalMode"?: string;
|
|
13931
|
+
"behaviors"?: Array<{
|
|
13932
|
+
"key": string;
|
|
13933
|
+
"name": string;
|
|
13934
|
+
"question": string;
|
|
13935
|
+
"version": number;
|
|
13936
|
+
}>;
|
|
12470
13937
|
};
|
|
12471
13938
|
"shareUrl": string | null;
|
|
12472
13939
|
"shareLinkKept": boolean;
|