@ora-ai/cli 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -84,6 +84,7 @@ export type BenchmarksListResponse = {
84
84
  "error": string | null;
85
85
  "preparation"?: {
86
86
  "since": string;
87
+ "progressAt"?: string;
87
88
  "launchedBy": {
88
89
  "userId"?: string;
89
90
  "role"?: string;
@@ -92,6 +93,12 @@ export type BenchmarksListResponse = {
92
93
  "imageKey": string;
93
94
  "buildId": string;
94
95
  "stage"?: "image" | "snapshot";
96
+ "retry"?: {
97
+ "failedAttempts": number;
98
+ "maxAttempts": number;
99
+ "notBefore": string;
100
+ "lastFailure": string;
101
+ };
95
102
  }>;
96
103
  } | null;
97
104
  "visibility": "private" | "org" | "link" | "public";
@@ -232,6 +239,7 @@ export type BenchmarksCreateResponse = {
232
239
  "error": string | null;
233
240
  "preparation"?: {
234
241
  "since": string;
242
+ "progressAt"?: string;
235
243
  "launchedBy": {
236
244
  "userId"?: string;
237
245
  "role"?: string;
@@ -240,6 +248,12 @@ export type BenchmarksCreateResponse = {
240
248
  "imageKey": string;
241
249
  "buildId": string;
242
250
  "stage"?: "image" | "snapshot";
251
+ "retry"?: {
252
+ "failedAttempts": number;
253
+ "maxAttempts": number;
254
+ "notBefore": string;
255
+ "lastFailure": string;
256
+ };
243
257
  }>;
244
258
  } | null;
245
259
  "visibility": "private" | "org" | "link" | "public";
@@ -340,6 +354,7 @@ export type BenchmarksGetResponse = {
340
354
  "error": string | null;
341
355
  "preparation"?: {
342
356
  "since": string;
357
+ "progressAt"?: string;
343
358
  "launchedBy": {
344
359
  "userId"?: string;
345
360
  "role"?: string;
@@ -348,6 +363,12 @@ export type BenchmarksGetResponse = {
348
363
  "imageKey": string;
349
364
  "buildId": string;
350
365
  "stage"?: "image" | "snapshot";
366
+ "retry"?: {
367
+ "failedAttempts": number;
368
+ "maxAttempts": number;
369
+ "notBefore": string;
370
+ "lastFailure": string;
371
+ };
351
372
  }>;
352
373
  } | null;
353
374
  "visibility": "private" | "org" | "link" | "public";
@@ -474,6 +495,9 @@ export type BenchmarksGetResponse = {
474
495
  "gradedAt": string;
475
496
  };
476
497
  } | null;
498
+ "executionBudgetS"?: number | null;
499
+ "harnessImageDigest"?: string | null;
500
+ "subjectEngaged"?: boolean | null;
477
501
  "runCreatedAt": string | null;
478
502
  "startedAt": string | null;
479
503
  "finishedAt": string | null;
@@ -683,6 +707,17 @@ export type BenchmarksGetResponse = {
683
707
  "ci": Array<unknown> | null;
684
708
  "n": number;
685
709
  };
710
+ "abstention"?: {
711
+ "value": number | null;
712
+ "ci": Array<unknown> | null;
713
+ "n": number;
714
+ "parts"?: Array<{
715
+ "id": string;
716
+ "label": string;
717
+ "value": number | null;
718
+ "n": number;
719
+ }>;
720
+ };
686
721
  "tasks": number;
687
722
  "graded": number;
688
723
  "unmeasured": number;
@@ -694,6 +729,17 @@ export type BenchmarksGetResponse = {
694
729
  "ci": Array<unknown> | null;
695
730
  "n": number;
696
731
  };
732
+ "abstention"?: {
733
+ "value": number | null;
734
+ "ci": Array<unknown> | null;
735
+ "n": number;
736
+ "parts"?: Array<{
737
+ "id": string;
738
+ "label": string;
739
+ "value": number | null;
740
+ "n": number;
741
+ }>;
742
+ };
697
743
  "graded": number;
698
744
  "unmeasured": number;
699
745
  "modelJudged": boolean;
@@ -723,9 +769,130 @@ export type BenchmarksGetResponse = {
723
769
  "detail": string;
724
770
  "taskIds"?: Array<string>;
725
771
  }>;
772
+ "taskHealth": {
773
+ "flags": Array<{
774
+ "taskId": string;
775
+ "signal": string;
776
+ "kind": "floor" | "ceiling";
777
+ "conditions": number;
778
+ }>;
779
+ "rubricSplits": Array<{
780
+ "taskId": string;
781
+ "criterionIndex": number;
782
+ "criterion": string;
783
+ "split": number;
784
+ "graded": number;
785
+ }>;
786
+ "signals": Array<{
787
+ "id": string;
788
+ "label": string;
789
+ }>;
790
+ };
726
791
  "shareUrl"?: string | null;
727
792
  "canManageSharing"?: boolean;
728
793
  };
794
+ export type BenchmarksUpdateDetailsRequest = {
795
+ "name"?: string;
796
+ "description"?: string;
797
+ };
798
+ export type BenchmarksUpdateDetailsResponse = {
799
+ "study": {
800
+ "id": string;
801
+ "executiveSummary"?: string | null;
802
+ "plannedRunCount": number;
803
+ "organizationId": string;
804
+ "createdBy": string | null;
805
+ "name": string;
806
+ "slug": string;
807
+ "description": string | null;
808
+ "skillRef": string | null;
809
+ "skillSlug": string | null;
810
+ "skillName": string | null;
811
+ "skillPromise": string;
812
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
813
+ "subjectName": string | null;
814
+ "subjectLabel": string;
815
+ "armConfig": {
816
+ "baseline"?: {
817
+ "tools"?: "default" | "none" | {
818
+ "tools": Array<string>;
819
+ };
820
+ "addons"?: Record<string, unknown>;
821
+ };
822
+ "treatment"?: {
823
+ "tools"?: "default" | "none" | {
824
+ "tools": Array<string>;
825
+ };
826
+ "addons"?: Record<string, unknown>;
827
+ };
828
+ };
829
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
830
+ "subjectBackend": "model" | "harness" | null;
831
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
832
+ "importedFrom"?: string;
833
+ "baselineLabel": string | null;
834
+ "treatmentLabel": string | null;
835
+ "promiseMetrics": Array<{
836
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
837
+ "direction": "down" | "up";
838
+ "label": string;
839
+ }>;
840
+ "matrix": Array<{
841
+ "harness": string;
842
+ "model": string;
843
+ "provider"?: string;
844
+ "params"?: {
845
+ "temperature"?: number;
846
+ "maxOutputTokens"?: number;
847
+ "reasoningEffort"?: "low" | "medium" | "high";
848
+ "thinkingBudgetTokens"?: number;
849
+ };
850
+ }>;
851
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
852
+ "judgeSupplementary"?: boolean;
853
+ "taskCount": number;
854
+ "variantsPerTask": number;
855
+ "repeats": number;
856
+ "subset": {
857
+ "count": number;
858
+ "seed": string;
859
+ "of": number;
860
+ } | null;
861
+ "status": string;
862
+ "error": string | null;
863
+ "preparation"?: {
864
+ "since": string;
865
+ "progressAt"?: string;
866
+ "launchedBy": {
867
+ "userId"?: string;
868
+ "role"?: string;
869
+ };
870
+ "waitingOn": Array<{
871
+ "imageKey": string;
872
+ "buildId": string;
873
+ "stage"?: "image" | "snapshot";
874
+ "retry"?: {
875
+ "failedAttempts": number;
876
+ "maxAttempts": number;
877
+ "notBefore": string;
878
+ "lastFailure": string;
879
+ };
880
+ }>;
881
+ } | null;
882
+ "visibility": "private" | "org" | "link" | "public";
883
+ "visibilityChangedAt"?: string | null;
884
+ "visibilityChangedBy"?: string | null;
885
+ "shareToken"?: string | null;
886
+ "lowSampleOverride": boolean;
887
+ "publishedAt": string | null;
888
+ "verdict": unknown;
889
+ "definitionVersionId": string | null;
890
+ "runnerKind": "ora-runs" | "batch-job";
891
+ "createdAt": string;
892
+ "updatedAt": string;
893
+ "liveUntil": string | null;
894
+ };
895
+ };
729
896
  export type BenchmarksDeleteResponse = unknown;
730
897
  export type BenchmarksRegenerateSummaryResponse = {
731
898
  "status": "queued";
@@ -800,6 +967,7 @@ export type BenchmarksGenerateResponse = {
800
967
  "error": string | null;
801
968
  "preparation"?: {
802
969
  "since": string;
970
+ "progressAt"?: string;
803
971
  "launchedBy": {
804
972
  "userId"?: string;
805
973
  "role"?: string;
@@ -808,6 +976,12 @@ export type BenchmarksGenerateResponse = {
808
976
  "imageKey": string;
809
977
  "buildId": string;
810
978
  "stage"?: "image" | "snapshot";
979
+ "retry"?: {
980
+ "failedAttempts": number;
981
+ "maxAttempts": number;
982
+ "notBefore": string;
983
+ "lastFailure": string;
984
+ };
811
985
  }>;
812
986
  } | null;
813
987
  "visibility": "private" | "org" | "link" | "public";
@@ -896,6 +1070,7 @@ export type BenchmarksLaunchResponse = {
896
1070
  "error": string | null;
897
1071
  "preparation"?: {
898
1072
  "since": string;
1073
+ "progressAt"?: string;
899
1074
  "launchedBy": {
900
1075
  "userId"?: string;
901
1076
  "role"?: string;
@@ -904,6 +1079,12 @@ export type BenchmarksLaunchResponse = {
904
1079
  "imageKey": string;
905
1080
  "buildId": string;
906
1081
  "stage"?: "image" | "snapshot";
1082
+ "retry"?: {
1083
+ "failedAttempts": number;
1084
+ "maxAttempts": number;
1085
+ "notBefore": string;
1086
+ "lastFailure": string;
1087
+ };
907
1088
  }>;
908
1089
  } | null;
909
1090
  "visibility": "private" | "org" | "link" | "public";
@@ -995,6 +1176,7 @@ export type BenchmarksUpdateLensesResponse = {
995
1176
  "error": string | null;
996
1177
  "preparation"?: {
997
1178
  "since": string;
1179
+ "progressAt"?: string;
998
1180
  "launchedBy": {
999
1181
  "userId"?: string;
1000
1182
  "role"?: string;
@@ -1003,6 +1185,12 @@ export type BenchmarksUpdateLensesResponse = {
1003
1185
  "imageKey": string;
1004
1186
  "buildId": string;
1005
1187
  "stage"?: "image" | "snapshot";
1188
+ "retry"?: {
1189
+ "failedAttempts": number;
1190
+ "maxAttempts": number;
1191
+ "notBefore": string;
1192
+ "lastFailure": string;
1193
+ };
1006
1194
  }>;
1007
1195
  } | null;
1008
1196
  "visibility": "private" | "org" | "link" | "public";
@@ -1139,6 +1327,9 @@ export type BenchmarksPreviewResponse = {
1139
1327
  "pairGradedAt": string | null;
1140
1328
  "evalScore": number | null;
1141
1329
  "evalGradedAt": string | null;
1330
+ "executionBudgetS"?: number | null;
1331
+ "harnessImageDigest"?: string | null;
1332
+ "subjectEngaged"?: boolean | null;
1142
1333
  "runCreatedAt": string | null;
1143
1334
  "startedAt": string | null;
1144
1335
  "finishedAt": string | null;
@@ -1359,6 +1550,17 @@ export type BenchmarksPreviewResponse = {
1359
1550
  "ci": Array<unknown> | null;
1360
1551
  "n": number;
1361
1552
  };
1553
+ "abstention"?: {
1554
+ "value": number | null;
1555
+ "ci": Array<unknown> | null;
1556
+ "n": number;
1557
+ "parts"?: Array<{
1558
+ "id": string;
1559
+ "label": string;
1560
+ "value": number | null;
1561
+ "n": number;
1562
+ }>;
1563
+ };
1362
1564
  "tasks": number;
1363
1565
  "graded": number;
1364
1566
  "unmeasured": number;
@@ -1370,6 +1572,17 @@ export type BenchmarksPreviewResponse = {
1370
1572
  "ci": Array<unknown> | null;
1371
1573
  "n": number;
1372
1574
  };
1575
+ "abstention"?: {
1576
+ "value": number | null;
1577
+ "ci": Array<unknown> | null;
1578
+ "n": number;
1579
+ "parts"?: Array<{
1580
+ "id": string;
1581
+ "label": string;
1582
+ "value": number | null;
1583
+ "n": number;
1584
+ }>;
1585
+ };
1373
1586
  "graded": number;
1374
1587
  "unmeasured": number;
1375
1588
  "modelJudged": boolean;
@@ -1463,6 +1676,7 @@ export type BenchmarksPublishResponse = {
1463
1676
  "error": string | null;
1464
1677
  "preparation"?: {
1465
1678
  "since": string;
1679
+ "progressAt"?: string;
1466
1680
  "launchedBy": {
1467
1681
  "userId"?: string;
1468
1682
  "role"?: string;
@@ -1471,6 +1685,12 @@ export type BenchmarksPublishResponse = {
1471
1685
  "imageKey": string;
1472
1686
  "buildId": string;
1473
1687
  "stage"?: "image" | "snapshot";
1688
+ "retry"?: {
1689
+ "failedAttempts": number;
1690
+ "maxAttempts": number;
1691
+ "notBefore": string;
1692
+ "lastFailure": string;
1693
+ };
1474
1694
  }>;
1475
1695
  } | null;
1476
1696
  "visibility": "private" | "org" | "link" | "public";
@@ -1554,6 +1774,7 @@ export type BenchmarksRegradeResponse = {
1554
1774
  "error": string | null;
1555
1775
  "preparation"?: {
1556
1776
  "since": string;
1777
+ "progressAt"?: string;
1557
1778
  "launchedBy": {
1558
1779
  "userId"?: string;
1559
1780
  "role"?: string;
@@ -1562,6 +1783,12 @@ export type BenchmarksRegradeResponse = {
1562
1783
  "imageKey": string;
1563
1784
  "buildId": string;
1564
1785
  "stage"?: "image" | "snapshot";
1786
+ "retry"?: {
1787
+ "failedAttempts": number;
1788
+ "maxAttempts": number;
1789
+ "notBefore": string;
1790
+ "lastFailure": string;
1791
+ };
1565
1792
  }>;
1566
1793
  } | null;
1567
1794
  "visibility": "private" | "org" | "link" | "public";
@@ -1669,6 +1896,7 @@ export type BenchmarksUnpublishResponse = {
1669
1896
  "error": string | null;
1670
1897
  "preparation"?: {
1671
1898
  "since": string;
1899
+ "progressAt"?: string;
1672
1900
  "launchedBy": {
1673
1901
  "userId"?: string;
1674
1902
  "role"?: string;
@@ -1677,6 +1905,12 @@ export type BenchmarksUnpublishResponse = {
1677
1905
  "imageKey": string;
1678
1906
  "buildId": string;
1679
1907
  "stage"?: "image" | "snapshot";
1908
+ "retry"?: {
1909
+ "failedAttempts": number;
1910
+ "maxAttempts": number;
1911
+ "notBefore": string;
1912
+ "lastFailure": string;
1913
+ };
1680
1914
  }>;
1681
1915
  } | null;
1682
1916
  "visibility": "private" | "org" | "link" | "public";
@@ -1734,6 +1968,9 @@ export type ExperimentImagesBuildsListResponse = {
1734
1968
  } | {
1735
1969
  "kind": "inline";
1736
1970
  "fileCount": number;
1971
+ } | {
1972
+ "kind": "asset";
1973
+ "sha256": string;
1737
1974
  };
1738
1975
  "sourceIdentity": string;
1739
1976
  "dockerfileDir": string;
@@ -1741,7 +1978,7 @@ export type ExperimentImagesBuildsListResponse = {
1741
1978
  "buildArgKeys": Array<string>;
1742
1979
  "platform": string;
1743
1980
  "trusted": boolean;
1744
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
1981
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1745
1982
  "image": string;
1746
1983
  "digest": string | null;
1747
1984
  "sizeBytes": number | null;
@@ -1782,7 +2019,7 @@ export type ExperimentImagesBuildsCreateRequest = {
1782
2019
  };
1783
2020
  export type ExperimentImagesBuildsCreateResponse = {
1784
2021
  "id": string;
1785
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
2022
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1786
2023
  };
1787
2024
  export type ExperimentImagesBuildsGetResponse = {
1788
2025
  "build": {
@@ -1804,6 +2041,9 @@ export type ExperimentImagesBuildsGetResponse = {
1804
2041
  } | {
1805
2042
  "kind": "inline";
1806
2043
  "fileCount": number;
2044
+ } | {
2045
+ "kind": "asset";
2046
+ "sha256": string;
1807
2047
  };
1808
2048
  "sourceIdentity": string;
1809
2049
  "dockerfileDir": string;
@@ -1811,7 +2051,7 @@ export type ExperimentImagesBuildsGetResponse = {
1811
2051
  "buildArgKeys": Array<string>;
1812
2052
  "platform": string;
1813
2053
  "trusted": boolean;
1814
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
2054
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1815
2055
  "image": string;
1816
2056
  "digest": string | null;
1817
2057
  "sizeBytes": number | null;
@@ -1825,7 +2065,7 @@ export type ExperimentImagesBuildsGetResponse = {
1825
2065
  };
1826
2066
  export type ExperimentImagesBuildsCancelResponse = {
1827
2067
  "id": string;
1828
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
2068
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1829
2069
  };
1830
2070
  export type ExperimentImagesBuildsStreamResponse = unknown;
1831
2071
  export type ExperimentImagesRunsCreateRequest = {
@@ -1882,6 +2122,7 @@ export type ExperimentRunsCreateRequest = {
1882
2122
  "tools": Array<string>;
1883
2123
  };
1884
2124
  "requires_tools"?: boolean;
2125
+ "prompt_context"?: "none" | "target_site";
1885
2126
  "mcp_servers"?: Array<{
1886
2127
  "name": string;
1887
2128
  "url": string;
@@ -2158,7 +2399,7 @@ export type WebmcpArenaCreateRequest = {
2158
2399
  "name"?: string;
2159
2400
  };
2160
2401
  export type WebmcpArenaCreateResponse = {
2161
- "study": {
2402
+ "benchmark": {
2162
2403
  "id": string;
2163
2404
  "executiveSummary"?: string | null;
2164
2405
  "plannedRunCount": number;
@@ -2224,6 +2465,7 @@ export type WebmcpArenaCreateResponse = {
2224
2465
  "error": string | null;
2225
2466
  "preparation"?: {
2226
2467
  "since": string;
2468
+ "progressAt"?: string;
2227
2469
  "launchedBy": {
2228
2470
  "userId"?: string;
2229
2471
  "role"?: string;
@@ -2232,6 +2474,12 @@ export type WebmcpArenaCreateResponse = {
2232
2474
  "imageKey": string;
2233
2475
  "buildId": string;
2234
2476
  "stage"?: "image" | "snapshot";
2477
+ "retry"?: {
2478
+ "failedAttempts": number;
2479
+ "maxAttempts": number;
2480
+ "notBefore": string;
2481
+ "lastFailure": string;
2482
+ };
2235
2483
  }>;
2236
2484
  } | null;
2237
2485
  "visibility": "private" | "org" | "link" | "public";
@@ -2248,10 +2496,6 @@ export type WebmcpArenaCreateResponse = {
2248
2496
  "liveUntil": string | null;
2249
2497
  };
2250
2498
  "streamUrl": string;
2251
- "warnings"?: Array<{
2252
- "harness": string;
2253
- "kind": string;
2254
- }>;
2255
2499
  };
2256
2500
  export type WebmcpRunsBrowserRecordingResponse = {
2257
2501
  "runId": string;
@@ -2500,6 +2744,18 @@ export type IntentsBulkCreateRequest = {
2500
2744
  "goal"?: {
2501
2745
  "statement": string;
2502
2746
  };
2747
+ "agents"?: Array<{
2748
+ "harness": string;
2749
+ "model": string;
2750
+ "mcp_server_ids"?: Array<string>;
2751
+ "skill_ids"?: Array<string>;
2752
+ "browser"?: {
2753
+ "webdriver"?: boolean;
2754
+ "webmcp"?: boolean;
2755
+ "webmcpTargetUrl"?: string;
2756
+ "webmcpSurface"?: "native" | "shim";
2757
+ };
2758
+ }>;
2503
2759
  "prompts": Array<{
2504
2760
  "text": string;
2505
2761
  "category"?: string;
@@ -2509,6 +2765,7 @@ export type IntentsBulkCreateRequest = {
2509
2765
  }>;
2510
2766
  "domainId"?: string;
2511
2767
  "icpId"?: string;
2768
+ "firstMeasurement"?: "written" | "intent";
2512
2769
  };
2513
2770
  export type IntentsBulkCreateResponse = {
2514
2771
  "success": true;
@@ -2518,6 +2775,13 @@ export type IntentsBulkCreateResponse = {
2518
2775
  "title": string;
2519
2776
  "merged": boolean;
2520
2777
  "promptsCreated": number;
2778
+ "setupsApplied"?: boolean;
2779
+ "measurementStart": {
2780
+ "started": true;
2781
+ } | {
2782
+ "started": false;
2783
+ "reason": "no_attributable_member";
2784
+ };
2521
2785
  }>;
2522
2786
  "truncated": number;
2523
2787
  };
@@ -2713,6 +2977,12 @@ export type IntentsGenerateResponse = {
2713
2977
  "tracked": boolean;
2714
2978
  }>;
2715
2979
  }>;
2980
+ "measurementStart": {
2981
+ "started": true;
2982
+ } | {
2983
+ "started": false;
2984
+ "reason": "no_attributable_member";
2985
+ };
2716
2986
  }>;
2717
2987
  };
2718
2988
  export type IntentsSetIcpRequest = {
@@ -2824,6 +3094,12 @@ export type IntentsAddPromptResponse = {
2824
3094
  "tracked": boolean;
2825
3095
  }>;
2826
3096
  }>;
3097
+ "measurementStart": {
3098
+ "started": true;
3099
+ } | {
3100
+ "started": false;
3101
+ "reason": "no_attributable_member";
3102
+ };
2827
3103
  };
2828
3104
  };
2829
3105
  export type IntentsUpdatePromptRequest = {
@@ -2842,6 +3118,17 @@ export type IntentsDeletePromptResponse = {
2842
3118
  "id": string;
2843
3119
  };
2844
3120
  };
3121
+ export type IntentsSetPromptGoalRequest = {
3122
+ "active": boolean;
3123
+ };
3124
+ export type IntentsSetPromptGoalResponse = {
3125
+ "success": true;
3126
+ "data": {
3127
+ "promptId": string;
3128
+ "kind": "discover" | "choose" | "verify" | "inform" | "guide" | "act" | "other";
3129
+ "active": boolean;
3130
+ };
3131
+ };
2845
3132
  export type IntentsRepresentationResponse = {
2846
3133
  "success": true;
2847
3134
  "data": {
@@ -3063,6 +3350,7 @@ export type StudioRunsCreateRequest = {
3063
3350
  "tools": Array<string>;
3064
3351
  };
3065
3352
  "requires_tools"?: boolean;
3353
+ "prompt_context"?: "none" | "target_site";
3066
3354
  "mcp_servers"?: Array<{
3067
3355
  "name": string;
3068
3356
  "url": string;
@@ -3145,6 +3433,7 @@ export type StudyAddonsListResponse = {
3145
3433
  "surfaces": Array<"run" | "session">;
3146
3434
  "harnesses": Array<string>;
3147
3435
  "multiInstance"?: boolean;
3436
+ "credentialSourceFields"?: Array<string>;
3148
3437
  }>;
3149
3438
  };
3150
3439
  export type StudyBenchmarkDefinitionsListResponse = {
@@ -3163,6 +3452,13 @@ export type StudyBenchmarkDefinitionsListResponse = {
3163
3452
  "modelParams"?: Array<string>;
3164
3453
  "maxPlannedRuns"?: number;
3165
3454
  "sampleFloor"?: number;
3455
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3456
+ "aggregation"?: "included" | "excluded";
3457
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3458
+ "aggregateTaskSample"?: {
3459
+ "count": number;
3460
+ "seed": string;
3461
+ };
3166
3462
  };
3167
3463
  "defaultConfig": {
3168
3464
  "matrix"?: Array<{
@@ -3229,6 +3525,7 @@ export type StudyBenchmarkDefinitionsListResponse = {
3229
3525
  "userId": string;
3230
3526
  "displayName": string | null;
3231
3527
  } | null;
3528
+ "uploadedByPublisher": boolean | null;
3232
3529
  "notices": Array<{
3233
3530
  "kind": string;
3234
3531
  "text": string;
@@ -3266,6 +3563,13 @@ export type StudyBenchmarkDefinitionsCreateRequest = {
3266
3563
  "modelParams"?: Array<string>;
3267
3564
  "maxPlannedRuns"?: number;
3268
3565
  "sampleFloor"?: number;
3566
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3567
+ "aggregation"?: "included" | "excluded";
3568
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3569
+ "aggregateTaskSample"?: {
3570
+ "count": number;
3571
+ "seed": string;
3572
+ };
3269
3573
  };
3270
3574
  "runnerKind"?: "none" | "image" | "environment" | "framework";
3271
3575
  "runnerRecipe"?: Record<string, unknown> | null;
@@ -3308,6 +3612,13 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
3308
3612
  "modelParams"?: Array<string>;
3309
3613
  "maxPlannedRuns"?: number;
3310
3614
  "sampleFloor"?: number;
3615
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3616
+ "aggregation"?: "included" | "excluded";
3617
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3618
+ "aggregateTaskSample"?: {
3619
+ "count": number;
3620
+ "seed": string;
3621
+ };
3311
3622
  };
3312
3623
  "defaultConfig": {
3313
3624
  "matrix"?: Array<{
@@ -3374,6 +3685,7 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
3374
3685
  "userId": string;
3375
3686
  "displayName": string | null;
3376
3687
  } | null;
3688
+ "uploadedByPublisher": boolean | null;
3377
3689
  "notices": Array<{
3378
3690
  "kind": string;
3379
3691
  "text": string;
@@ -3396,6 +3708,13 @@ export type StudyBenchmarkDefinitionsGetResponse = {
3396
3708
  "modelParams"?: Array<string>;
3397
3709
  "maxPlannedRuns"?: number;
3398
3710
  "sampleFloor"?: number;
3711
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3712
+ "aggregation"?: "included" | "excluded";
3713
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3714
+ "aggregateTaskSample"?: {
3715
+ "count": number;
3716
+ "seed": string;
3717
+ };
3399
3718
  };
3400
3719
  "defaultConfig": {
3401
3720
  "matrix"?: Array<{
@@ -3462,6 +3781,7 @@ export type StudyBenchmarkDefinitionsGetResponse = {
3462
3781
  "userId": string;
3463
3782
  "displayName": string | null;
3464
3783
  } | null;
3784
+ "uploadedByPublisher": boolean | null;
3465
3785
  "notices": Array<{
3466
3786
  "kind": string;
3467
3787
  "text": string;
@@ -3493,6 +3813,13 @@ export type StudyBenchmarkDefinitionsUpdateRequest = {
3493
3813
  "modelParams"?: Array<string>;
3494
3814
  "maxPlannedRuns"?: number;
3495
3815
  "sampleFloor"?: number;
3816
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3817
+ "aggregation"?: "included" | "excluded";
3818
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3819
+ "aggregateTaskSample"?: {
3820
+ "count": number;
3821
+ "seed": string;
3822
+ };
3496
3823
  };
3497
3824
  "runnerKind"?: "none" | "image" | "environment" | "framework";
3498
3825
  "runnerRecipe"?: Record<string, unknown> | null;
@@ -3535,6 +3862,13 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
3535
3862
  "modelParams"?: Array<string>;
3536
3863
  "maxPlannedRuns"?: number;
3537
3864
  "sampleFloor"?: number;
3865
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3866
+ "aggregation"?: "included" | "excluded";
3867
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3868
+ "aggregateTaskSample"?: {
3869
+ "count": number;
3870
+ "seed": string;
3871
+ };
3538
3872
  };
3539
3873
  "defaultConfig": {
3540
3874
  "matrix"?: Array<{
@@ -3601,6 +3935,7 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
3601
3935
  "userId": string;
3602
3936
  "displayName": string | null;
3603
3937
  } | null;
3938
+ "uploadedByPublisher": boolean | null;
3604
3939
  "notices": Array<{
3605
3940
  "kind": string;
3606
3941
  "text": string;
@@ -3645,6 +3980,13 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
3645
3980
  "modelParams"?: Array<string>;
3646
3981
  "maxPlannedRuns"?: number;
3647
3982
  "sampleFloor"?: number;
3983
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3984
+ "aggregation"?: "included" | "excluded";
3985
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3986
+ "aggregateTaskSample"?: {
3987
+ "count": number;
3988
+ "seed": string;
3989
+ };
3648
3990
  };
3649
3991
  "defaultConfig": {
3650
3992
  "matrix"?: Array<{
@@ -3711,6 +4053,7 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
3711
4053
  "userId": string;
3712
4054
  "displayName": string | null;
3713
4055
  } | null;
4056
+ "uploadedByPublisher": boolean | null;
3714
4057
  "notices": Array<{
3715
4058
  "kind": string;
3716
4059
  "text": string;
@@ -3736,6 +4079,13 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
3736
4079
  "modelParams"?: Array<string>;
3737
4080
  "maxPlannedRuns"?: number;
3738
4081
  "sampleFloor"?: number;
4082
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
4083
+ "aggregation"?: "included" | "excluded";
4084
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
4085
+ "aggregateTaskSample"?: {
4086
+ "count": number;
4087
+ "seed": string;
4088
+ };
3739
4089
  };
3740
4090
  "defaultConfig": {
3741
4091
  "matrix"?: Array<{
@@ -3802,6 +4152,7 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
3802
4152
  "userId": string;
3803
4153
  "displayName": string | null;
3804
4154
  } | null;
4155
+ "uploadedByPublisher": boolean | null;
3805
4156
  "notices": Array<{
3806
4157
  "kind": string;
3807
4158
  "text": string;
@@ -3824,6 +4175,13 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
3824
4175
  "modelParams"?: Array<string>;
3825
4176
  "maxPlannedRuns"?: number;
3826
4177
  "sampleFloor"?: number;
4178
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
4179
+ "aggregation"?: "included" | "excluded";
4180
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
4181
+ "aggregateTaskSample"?: {
4182
+ "count": number;
4183
+ "seed": string;
4184
+ };
3827
4185
  };
3828
4186
  "defaultConfig": {
3829
4187
  "matrix"?: Array<{
@@ -3890,6 +4248,7 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
3890
4248
  "userId": string;
3891
4249
  "displayName": string | null;
3892
4250
  } | null;
4251
+ "uploadedByPublisher": boolean | null;
3893
4252
  "notices": Array<{
3894
4253
  "kind": string;
3895
4254
  "text": string;
@@ -4068,6 +4427,9 @@ export type StudyImagesBuildsListResponse = {
4068
4427
  } | {
4069
4428
  "kind": "inline";
4070
4429
  "fileCount": number;
4430
+ } | {
4431
+ "kind": "asset";
4432
+ "sha256": string;
4071
4433
  };
4072
4434
  "sourceIdentity": string;
4073
4435
  "dockerfileDir": string;
@@ -4075,7 +4437,7 @@ export type StudyImagesBuildsListResponse = {
4075
4437
  "buildArgKeys": Array<string>;
4076
4438
  "platform": string;
4077
4439
  "trusted": boolean;
4078
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4440
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4079
4441
  "image": string;
4080
4442
  "digest": string | null;
4081
4443
  "sizeBytes": number | null;
@@ -4116,7 +4478,7 @@ export type StudyImagesBuildsCreateRequest = {
4116
4478
  };
4117
4479
  export type StudyImagesBuildsCreateResponse = {
4118
4480
  "id": string;
4119
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4481
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4120
4482
  };
4121
4483
  export type StudyImagesBuildsGetResponse = {
4122
4484
  "build": {
@@ -4138,6 +4500,9 @@ export type StudyImagesBuildsGetResponse = {
4138
4500
  } | {
4139
4501
  "kind": "inline";
4140
4502
  "fileCount": number;
4503
+ } | {
4504
+ "kind": "asset";
4505
+ "sha256": string;
4141
4506
  };
4142
4507
  "sourceIdentity": string;
4143
4508
  "dockerfileDir": string;
@@ -4145,7 +4510,7 @@ export type StudyImagesBuildsGetResponse = {
4145
4510
  "buildArgKeys": Array<string>;
4146
4511
  "platform": string;
4147
4512
  "trusted": boolean;
4148
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4513
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4149
4514
  "image": string;
4150
4515
  "digest": string | null;
4151
4516
  "sizeBytes": number | null;
@@ -4159,7 +4524,7 @@ export type StudyImagesBuildsGetResponse = {
4159
4524
  };
4160
4525
  export type StudyImagesBuildsCancelResponse = {
4161
4526
  "id": string;
4162
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4527
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4163
4528
  };
4164
4529
  export type StudyImagesBuildsStreamResponse = unknown;
4165
4530
  export type StudyImagesRunsCreateRequest = {
@@ -4216,6 +4581,7 @@ export type StudyRunsCreateRequest = {
4216
4581
  "tools": Array<string>;
4217
4582
  };
4218
4583
  "requires_tools"?: boolean;
4584
+ "prompt_context"?: "none" | "target_site";
4219
4585
  "mcp_servers"?: Array<{
4220
4586
  "name": string;
4221
4587
  "url": string;
@@ -4394,7 +4760,7 @@ export type StudyListResponse = {
4394
4760
  "addons"?: Record<string, unknown>;
4395
4761
  };
4396
4762
  };
4397
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
4763
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4398
4764
  "subjectBackend": "model" | "harness" | null;
4399
4765
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4400
4766
  "importedFrom"?: string;
@@ -4430,6 +4796,7 @@ export type StudyListResponse = {
4430
4796
  "error": string | null;
4431
4797
  "preparation"?: {
4432
4798
  "since": string;
4799
+ "progressAt"?: string;
4433
4800
  "launchedBy": {
4434
4801
  "userId"?: string;
4435
4802
  "role"?: string;
@@ -4438,6 +4805,12 @@ export type StudyListResponse = {
4438
4805
  "imageKey": string;
4439
4806
  "buildId": string;
4440
4807
  "stage"?: "image" | "snapshot";
4808
+ "retry"?: {
4809
+ "failedAttempts": number;
4810
+ "maxAttempts": number;
4811
+ "notBefore": string;
4812
+ "lastFailure": string;
4813
+ };
4441
4814
  }>;
4442
4815
  } | null;
4443
4816
  "visibility": "private" | "org" | "link" | "public";
@@ -4452,6 +4825,38 @@ export type StudyListResponse = {
4452
4825
  "createdAt": string;
4453
4826
  "updatedAt": string;
4454
4827
  "liveUntil": string | null;
4828
+ "arms"?: Array<{
4829
+ "key": string;
4830
+ "position": number;
4831
+ "label": string;
4832
+ "role": "reference" | "peer";
4833
+ "payload": {
4834
+ "tools"?: "default" | "none" | {
4835
+ "tools": Array<string>;
4836
+ };
4837
+ "addons"?: Record<string, unknown>;
4838
+ };
4839
+ "cells"?: Array<{
4840
+ "harness": string;
4841
+ "model": string;
4842
+ "provider"?: string;
4843
+ "params"?: {
4844
+ "temperature"?: number;
4845
+ "maxOutputTokens"?: number;
4846
+ "reasoningEffort"?: "low" | "medium" | "high";
4847
+ "thinkingBudgetTokens"?: number;
4848
+ };
4849
+ }>;
4850
+ "skills"?: Array<{
4851
+ "ref": string;
4852
+ "name": string;
4853
+ "changedSinceCreate"?: boolean;
4854
+ }>;
4855
+ }>;
4856
+ "contrastPlan"?: "vs_reference" | "all_pairs";
4857
+ "cellPlan"?: "crossed" | "per_setup";
4858
+ "skillDisclosure"?: "withheld" | "named";
4859
+ "modeOptions"?: Record<string, unknown>;
4455
4860
  }>;
4456
4861
  };
4457
4862
  export type StudyCreateRequest = {
@@ -4478,10 +4883,34 @@ export type StudyCreateRequest = {
4478
4883
  "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
4479
4884
  "judgeSupplementary"?: boolean;
4480
4885
  "proposeExpectations"?: boolean;
4481
- "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
4886
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4482
4887
  "subjectBackend"?: "model" | "harness";
4483
4888
  "baselineLabel"?: string;
4484
4889
  "treatmentLabel"?: string;
4890
+ "arms"?: Array<{
4891
+ "key"?: string;
4892
+ "label": string;
4893
+ "role"?: "reference" | "peer";
4894
+ "config": {
4895
+ "tools"?: "default" | "none" | {
4896
+ "tools": Array<string>;
4897
+ };
4898
+ "addons"?: Record<string, unknown>;
4899
+ };
4900
+ "cells"?: Array<{
4901
+ "harness": string;
4902
+ "model": string;
4903
+ "provider"?: string;
4904
+ "params"?: {
4905
+ "temperature"?: number;
4906
+ "maxOutputTokens"?: number;
4907
+ "reasoningEffort"?: "low" | "medium" | "high";
4908
+ "thinkingBudgetTokens"?: number;
4909
+ };
4910
+ }>;
4911
+ }>;
4912
+ "contrasts"?: "vs_reference" | "all_pairs";
4913
+ "modeOptions"?: Record<string, unknown>;
4485
4914
  "taskCount"?: number;
4486
4915
  "variantsPerTask"?: number;
4487
4916
  "repeats"?: number;
@@ -4492,7 +4921,75 @@ export type StudyCreateRequest = {
4492
4921
  "requiresWeb"?: boolean;
4493
4922
  "rubric"?: Array<string>;
4494
4923
  "variants"?: Array<string>;
4495
- "expectations"?: Array<unknown>;
4924
+ "expectations"?: Array<{
4925
+ "label"?: string;
4926
+ "arms"?: Array<"baseline" | "skill" | string>;
4927
+ "kind": "mcp_tool_called";
4928
+ "server"?: string;
4929
+ "tool": string;
4930
+ "arguments"?: Record<string, unknown>;
4931
+ "minCalls"?: number;
4932
+ } | {
4933
+ "label"?: string;
4934
+ "arms"?: Array<"baseline" | "skill" | string>;
4935
+ "kind": "tool_not_called";
4936
+ "tool"?: string;
4937
+ "mcp"?: {
4938
+ "server"?: string;
4939
+ "tool": string;
4940
+ };
4941
+ "pattern"?: string;
4942
+ } | {
4943
+ "label"?: string;
4944
+ "arms"?: Array<"baseline" | "skill" | string>;
4945
+ "kind": "command_ran";
4946
+ "pattern": string;
4947
+ } | {
4948
+ "label"?: string;
4949
+ "arms"?: Array<"baseline" | "skill" | string>;
4950
+ "kind": "skill_loaded";
4951
+ "slug": string;
4952
+ } | {
4953
+ "label"?: string;
4954
+ "arms"?: Array<"baseline" | "skill" | string>;
4955
+ "kind": "file_operation";
4956
+ "op"?: "read" | "write" | "edit";
4957
+ "path": unknown;
4958
+ } | {
4959
+ "label"?: string;
4960
+ "arms"?: Array<"baseline" | "skill" | string>;
4961
+ "kind": "web_fetched";
4962
+ "urlHost"?: string;
4963
+ "urlPattern"?: string;
4964
+ } | {
4965
+ "label"?: string;
4966
+ "arms"?: Array<"baseline" | "skill" | string>;
4967
+ "kind": "response_contains";
4968
+ "text": string;
4969
+ "normalize"?: "prose";
4970
+ } | {
4971
+ "label"?: string;
4972
+ "arms"?: Array<"baseline" | "skill" | string>;
4973
+ "kind": "response_matches";
4974
+ "pattern": string;
4975
+ "ignoreCase"?: boolean;
4976
+ } | {
4977
+ "label"?: string;
4978
+ "arms"?: Array<"baseline" | "skill" | string>;
4979
+ "kind": "token_budget";
4980
+ "maxOutputTokens"?: number;
4981
+ "maxTotalTokens"?: number;
4982
+ } | {
4983
+ "label"?: string;
4984
+ "arms"?: Array<"baseline" | "skill" | string>;
4985
+ "kind": "sequence";
4986
+ "of": Array<unknown>;
4987
+ } | {
4988
+ "label"?: string;
4989
+ "arms"?: Array<"baseline" | "skill" | string>;
4990
+ "kind": "any_of";
4991
+ "of": Array<unknown>;
4992
+ }>;
4496
4993
  }>;
4497
4994
  "subset"?: {
4498
4995
  "count": number;
@@ -4542,7 +5039,7 @@ export type StudyCreateResponse = {
4542
5039
  "addons"?: Record<string, unknown>;
4543
5040
  };
4544
5041
  };
4545
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5042
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4546
5043
  "subjectBackend": "model" | "harness" | null;
4547
5044
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4548
5045
  "importedFrom"?: string;
@@ -4578,6 +5075,7 @@ export type StudyCreateResponse = {
4578
5075
  "error": string | null;
4579
5076
  "preparation"?: {
4580
5077
  "since": string;
5078
+ "progressAt"?: string;
4581
5079
  "launchedBy": {
4582
5080
  "userId"?: string;
4583
5081
  "role"?: string;
@@ -4586,6 +5084,12 @@ export type StudyCreateResponse = {
4586
5084
  "imageKey": string;
4587
5085
  "buildId": string;
4588
5086
  "stage"?: "image" | "snapshot";
5087
+ "retry"?: {
5088
+ "failedAttempts": number;
5089
+ "maxAttempts": number;
5090
+ "notBefore": string;
5091
+ "lastFailure": string;
5092
+ };
4589
5093
  }>;
4590
5094
  } | null;
4591
5095
  "visibility": "private" | "org" | "link" | "public";
@@ -4600,13 +5104,59 @@ export type StudyCreateResponse = {
4600
5104
  "createdAt": string;
4601
5105
  "updatedAt": string;
4602
5106
  "liveUntil": string | null;
4603
- };
4604
- "streamUrl": string;
4605
- "warnings"?: Array<{
4606
- "harness": string;
4607
- "kind": string;
4608
- }>;
4609
- };
5107
+ "arms"?: Array<{
5108
+ "key": string;
5109
+ "position": number;
5110
+ "label": string;
5111
+ "role": "reference" | "peer";
5112
+ "payload": {
5113
+ "tools"?: "default" | "none" | {
5114
+ "tools": Array<string>;
5115
+ };
5116
+ "addons"?: Record<string, unknown>;
5117
+ };
5118
+ "cells"?: Array<{
5119
+ "harness": string;
5120
+ "model": string;
5121
+ "provider"?: string;
5122
+ "params"?: {
5123
+ "temperature"?: number;
5124
+ "maxOutputTokens"?: number;
5125
+ "reasoningEffort"?: "low" | "medium" | "high";
5126
+ "thinkingBudgetTokens"?: number;
5127
+ };
5128
+ }>;
5129
+ "skills"?: Array<{
5130
+ "ref": string;
5131
+ "name": string;
5132
+ "changedSinceCreate"?: boolean;
5133
+ }>;
5134
+ }>;
5135
+ "contrastPlan"?: "vs_reference" | "all_pairs";
5136
+ "cellPlan"?: "crossed" | "per_setup";
5137
+ "skillDisclosure"?: "withheld" | "named";
5138
+ "modeOptions"?: Record<string, unknown>;
5139
+ };
5140
+ "streamUrl": string;
5141
+ "warnings"?: Array<{
5142
+ "harness": string;
5143
+ "kind": string;
5144
+ } | {
5145
+ "type": "design_aggregate_budget";
5146
+ "detail": string;
5147
+ }>;
5148
+ };
5149
+ export type StudyComparisonModesListResponse = {
5150
+ "modes": Array<{
5151
+ "id": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5152
+ "enabled": boolean;
5153
+ "authoring": boolean;
5154
+ }>;
5155
+ "cellPlans": Array<{
5156
+ "id": "crossed" | "per_setup";
5157
+ "enabled": boolean;
5158
+ }>;
5159
+ };
4610
5160
  export type StudyConvertWebmcpSuiteRequest = {
4611
5161
  "suite": unknown;
4612
5162
  "ordered"?: boolean;
@@ -4619,6 +5169,225 @@ export type StudyConvertWebmcpSuiteResponse = {
4619
5169
  }>;
4620
5170
  "warnings": Array<string>;
4621
5171
  };
5172
+ export type StudyPreflightRequest = {
5173
+ "name": string;
5174
+ "description"?: string;
5175
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
5176
+ "skillRef"?: string;
5177
+ "definitionVersionId"?: string;
5178
+ "subjectName"?: string;
5179
+ "armConfig"?: {
5180
+ "baseline"?: {
5181
+ "tools"?: "default" | "none" | {
5182
+ "tools": Array<string>;
5183
+ };
5184
+ "addons"?: Record<string, unknown>;
5185
+ };
5186
+ "treatment"?: {
5187
+ "tools"?: "default" | "none" | {
5188
+ "tools": Array<string>;
5189
+ };
5190
+ "addons"?: Record<string, unknown>;
5191
+ };
5192
+ };
5193
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5194
+ "judgeSupplementary"?: boolean;
5195
+ "proposeExpectations"?: boolean;
5196
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5197
+ "subjectBackend"?: "model" | "harness";
5198
+ "baselineLabel"?: string;
5199
+ "treatmentLabel"?: string;
5200
+ "arms"?: Array<{
5201
+ "key"?: string;
5202
+ "label": string;
5203
+ "role"?: "reference" | "peer";
5204
+ "config": {
5205
+ "tools"?: "default" | "none" | {
5206
+ "tools": Array<string>;
5207
+ };
5208
+ "addons"?: Record<string, unknown>;
5209
+ };
5210
+ "cells"?: Array<{
5211
+ "harness": string;
5212
+ "model": string;
5213
+ "provider"?: string;
5214
+ "params"?: {
5215
+ "temperature"?: number;
5216
+ "maxOutputTokens"?: number;
5217
+ "reasoningEffort"?: "low" | "medium" | "high";
5218
+ "thinkingBudgetTokens"?: number;
5219
+ };
5220
+ }>;
5221
+ }>;
5222
+ "contrasts"?: "vs_reference" | "all_pairs";
5223
+ "modeOptions"?: Record<string, unknown>;
5224
+ "taskCount"?: number;
5225
+ "variantsPerTask"?: number;
5226
+ "repeats"?: number;
5227
+ "tasks"?: Array<{
5228
+ "prompt": string;
5229
+ "rationale"?: string;
5230
+ "category"?: string;
5231
+ "requiresWeb"?: boolean;
5232
+ "rubric"?: Array<string>;
5233
+ "variants"?: Array<string>;
5234
+ "expectations"?: Array<{
5235
+ "label"?: string;
5236
+ "arms"?: Array<"baseline" | "skill" | string>;
5237
+ "kind": "mcp_tool_called";
5238
+ "server"?: string;
5239
+ "tool": string;
5240
+ "arguments"?: Record<string, unknown>;
5241
+ "minCalls"?: number;
5242
+ } | {
5243
+ "label"?: string;
5244
+ "arms"?: Array<"baseline" | "skill" | string>;
5245
+ "kind": "tool_not_called";
5246
+ "tool"?: string;
5247
+ "mcp"?: {
5248
+ "server"?: string;
5249
+ "tool": string;
5250
+ };
5251
+ "pattern"?: string;
5252
+ } | {
5253
+ "label"?: string;
5254
+ "arms"?: Array<"baseline" | "skill" | string>;
5255
+ "kind": "command_ran";
5256
+ "pattern": string;
5257
+ } | {
5258
+ "label"?: string;
5259
+ "arms"?: Array<"baseline" | "skill" | string>;
5260
+ "kind": "skill_loaded";
5261
+ "slug": string;
5262
+ } | {
5263
+ "label"?: string;
5264
+ "arms"?: Array<"baseline" | "skill" | string>;
5265
+ "kind": "file_operation";
5266
+ "op"?: "read" | "write" | "edit";
5267
+ "path": unknown;
5268
+ } | {
5269
+ "label"?: string;
5270
+ "arms"?: Array<"baseline" | "skill" | string>;
5271
+ "kind": "web_fetched";
5272
+ "urlHost"?: string;
5273
+ "urlPattern"?: string;
5274
+ } | {
5275
+ "label"?: string;
5276
+ "arms"?: Array<"baseline" | "skill" | string>;
5277
+ "kind": "response_contains";
5278
+ "text": string;
5279
+ "normalize"?: "prose";
5280
+ } | {
5281
+ "label"?: string;
5282
+ "arms"?: Array<"baseline" | "skill" | string>;
5283
+ "kind": "response_matches";
5284
+ "pattern": string;
5285
+ "ignoreCase"?: boolean;
5286
+ } | {
5287
+ "label"?: string;
5288
+ "arms"?: Array<"baseline" | "skill" | string>;
5289
+ "kind": "token_budget";
5290
+ "maxOutputTokens"?: number;
5291
+ "maxTotalTokens"?: number;
5292
+ } | {
5293
+ "label"?: string;
5294
+ "arms"?: Array<"baseline" | "skill" | string>;
5295
+ "kind": "sequence";
5296
+ "of": Array<unknown>;
5297
+ } | {
5298
+ "label"?: string;
5299
+ "arms"?: Array<"baseline" | "skill" | string>;
5300
+ "kind": "any_of";
5301
+ "of": Array<unknown>;
5302
+ }>;
5303
+ }>;
5304
+ "subset"?: {
5305
+ "count": number;
5306
+ "seed"?: string;
5307
+ };
5308
+ "matrix": Array<{
5309
+ "harness": string;
5310
+ "model": string;
5311
+ "provider"?: string;
5312
+ "params"?: {
5313
+ "temperature"?: number;
5314
+ "maxOutputTokens"?: number;
5315
+ "reasoningEffort"?: "low" | "medium" | "high";
5316
+ "thinkingBudgetTokens"?: number;
5317
+ };
5318
+ }>;
5319
+ "promise"?: string;
5320
+ };
5321
+ export type StudyPreflightResponse = {
5322
+ "ok": true;
5323
+ "depth": "default" | "full";
5324
+ "form": string;
5325
+ "label": string;
5326
+ "plannedRunCount": number;
5327
+ "plannedJudgeComparisons": number;
5328
+ "runs": {
5329
+ "tasks": number;
5330
+ "variants": number;
5331
+ "repeats": number;
5332
+ "groupAgents": Array<{
5333
+ "group": string;
5334
+ "count": number;
5335
+ }>;
5336
+ };
5337
+ "judgePerComparison": Array<{
5338
+ "reference": string;
5339
+ "subject": string;
5340
+ "count": number;
5341
+ }>;
5342
+ "arms": Array<{
5343
+ "key": string;
5344
+ "label": string | null;
5345
+ "role": string;
5346
+ "payload": Record<string, unknown> | null;
5347
+ "cells"?: Array<{
5348
+ "harness": string;
5349
+ "model": string;
5350
+ "provider"?: string;
5351
+ "params"?: Record<string, unknown>;
5352
+ }>;
5353
+ }>;
5354
+ "comparisons": Array<{
5355
+ "reference": string;
5356
+ "subject": string;
5357
+ "oriented": boolean;
5358
+ "basis": "matched" | "whole_setup";
5359
+ "sharedAgents": Array<{
5360
+ "harness": string;
5361
+ "model": string;
5362
+ "provider"?: string;
5363
+ "params"?: Record<string, unknown>;
5364
+ }>;
5365
+ "excluded": Array<{
5366
+ "group": string;
5367
+ "agents": Array<{
5368
+ "harness": string;
5369
+ "model": string;
5370
+ "provider"?: string;
5371
+ "params"?: Record<string, unknown>;
5372
+ }>;
5373
+ }>;
5374
+ "droppedTasks"?: unknown | null;
5375
+ }>;
5376
+ "boards": Array<{
5377
+ "group": string;
5378
+ "profile": "as-published" | "harness-default" | "with-setup";
5379
+ }>;
5380
+ "issues": Array<{
5381
+ "path": string;
5382
+ "code": string;
5383
+ "message": string;
5384
+ }>;
5385
+ "quota"?: {
5386
+ "dimension": string;
5387
+ "remaining": number | null;
5388
+ "allowed": boolean;
5389
+ } | null;
5390
+ };
4622
5391
  export type StudyGetResponse = {
4623
5392
  "study": {
4624
5393
  "id": string;
@@ -4650,7 +5419,7 @@ export type StudyGetResponse = {
4650
5419
  "addons"?: Record<string, unknown>;
4651
5420
  };
4652
5421
  };
4653
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5422
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4654
5423
  "subjectBackend": "model" | "harness" | null;
4655
5424
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4656
5425
  "importedFrom"?: string;
@@ -4686,6 +5455,7 @@ export type StudyGetResponse = {
4686
5455
  "error": string | null;
4687
5456
  "preparation"?: {
4688
5457
  "since": string;
5458
+ "progressAt"?: string;
4689
5459
  "launchedBy": {
4690
5460
  "userId"?: string;
4691
5461
  "role"?: string;
@@ -4694,6 +5464,12 @@ export type StudyGetResponse = {
4694
5464
  "imageKey": string;
4695
5465
  "buildId": string;
4696
5466
  "stage"?: "image" | "snapshot";
5467
+ "retry"?: {
5468
+ "failedAttempts": number;
5469
+ "maxAttempts": number;
5470
+ "notBefore": string;
5471
+ "lastFailure": string;
5472
+ };
4697
5473
  }>;
4698
5474
  } | null;
4699
5475
  "visibility": "private" | "org" | "link" | "public";
@@ -4708,6 +5484,38 @@ export type StudyGetResponse = {
4708
5484
  "createdAt": string;
4709
5485
  "updatedAt": string;
4710
5486
  "liveUntil": string | null;
5487
+ "arms"?: Array<{
5488
+ "key": string;
5489
+ "position": number;
5490
+ "label": string;
5491
+ "role": "reference" | "peer";
5492
+ "payload": {
5493
+ "tools"?: "default" | "none" | {
5494
+ "tools": Array<string>;
5495
+ };
5496
+ "addons"?: Record<string, unknown>;
5497
+ };
5498
+ "cells"?: Array<{
5499
+ "harness": string;
5500
+ "model": string;
5501
+ "provider"?: string;
5502
+ "params"?: {
5503
+ "temperature"?: number;
5504
+ "maxOutputTokens"?: number;
5505
+ "reasoningEffort"?: "low" | "medium" | "high";
5506
+ "thinkingBudgetTokens"?: number;
5507
+ };
5508
+ }>;
5509
+ "skills"?: Array<{
5510
+ "ref": string;
5511
+ "name": string;
5512
+ "changedSinceCreate"?: boolean;
5513
+ }>;
5514
+ }>;
5515
+ "contrastPlan"?: "vs_reference" | "all_pairs";
5516
+ "cellPlan"?: "crossed" | "per_setup";
5517
+ "skillDisclosure"?: "withheld" | "named";
5518
+ "modeOptions"?: Record<string, unknown>;
4711
5519
  };
4712
5520
  "tasks": Array<{
4713
5521
  "id": string;
@@ -4724,7 +5532,7 @@ export type StudyGetResponse = {
4724
5532
  "runs": Array<{
4725
5533
  "runId": string;
4726
5534
  "taskId": string;
4727
- "arm": "baseline" | "skill";
5535
+ "arm": "baseline" | "skill" | string;
4728
5536
  "harness": string;
4729
5537
  "model": string;
4730
5538
  "provider": string | null;
@@ -4820,9 +5628,24 @@ export type StudyGetResponse = {
4820
5628
  "gradedAt": string;
4821
5629
  };
4822
5630
  } | null;
5631
+ "executionBudgetS"?: number | null;
5632
+ "harnessImageDigest"?: string | null;
5633
+ "subjectEngaged"?: boolean | null;
4823
5634
  "runCreatedAt": string | null;
4824
5635
  "startedAt": string | null;
4825
5636
  "finishedAt": string | null;
5637
+ "contrastVerdicts"?: Array<{
5638
+ "referenceArm": string;
5639
+ "subjectArm": string;
5640
+ "referenceRunId": string;
5641
+ "verdict": "subject" | "reference" | "tie" | "unmeasurable";
5642
+ "votes": Array<{
5643
+ "judge": string;
5644
+ "ordering": "base-first" | "skill-first";
5645
+ "answer": "A" | "B" | "tie";
5646
+ }>;
5647
+ "gradedAt": string;
5648
+ }>;
4826
5649
  }>;
4827
5650
  "aggregate": {
4828
5651
  "summary": {
@@ -5029,6 +5852,17 @@ export type StudyGetResponse = {
5029
5852
  "ci": Array<unknown> | null;
5030
5853
  "n": number;
5031
5854
  };
5855
+ "abstention"?: {
5856
+ "value": number | null;
5857
+ "ci": Array<unknown> | null;
5858
+ "n": number;
5859
+ "parts"?: Array<{
5860
+ "id": string;
5861
+ "label": string;
5862
+ "value": number | null;
5863
+ "n": number;
5864
+ }>;
5865
+ };
5032
5866
  "tasks": number;
5033
5867
  "graded": number;
5034
5868
  "unmeasured": number;
@@ -5040,6 +5874,17 @@ export type StudyGetResponse = {
5040
5874
  "ci": Array<unknown> | null;
5041
5875
  "n": number;
5042
5876
  };
5877
+ "abstention"?: {
5878
+ "value": number | null;
5879
+ "ci": Array<unknown> | null;
5880
+ "n": number;
5881
+ "parts"?: Array<{
5882
+ "id": string;
5883
+ "label": string;
5884
+ "value": number | null;
5885
+ "n": number;
5886
+ }>;
5887
+ };
5043
5888
  "graded": number;
5044
5889
  "unmeasured": number;
5045
5890
  "modelJudged": boolean;
@@ -5059,253 +5904,1363 @@ export type StudyGetResponse = {
5059
5904
  "languages": Array<string>;
5060
5905
  "unavailableReasons": Array<string>;
5061
5906
  };
5062
- };
5063
- "warnings": Array<{
5064
- "type": "eval_capability_gap";
5065
- "harness": string;
5066
- "kind": string;
5067
5907
  } | {
5068
- "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
5069
- "detail": string;
5070
- "taskIds"?: Array<string>;
5071
- }>;
5072
- "shareUrl"?: string | null;
5073
- "canManageSharing"?: boolean;
5074
- };
5075
- export type StudyDeleteResponse = unknown;
5076
- export type StudyRegenerateSummaryResponse = {
5077
- "status": "queued";
5078
- };
5079
- export type StudyGenerateRequest = {
5080
- "proposeExpectations"?: boolean;
5081
- };
5082
- export type StudyGenerateResponse = {
5083
- "study": {
5084
- "id": string;
5085
- "executiveSummary"?: string | null;
5086
- "plannedRunCount": number;
5087
- "organizationId": string;
5088
- "createdBy": string | null;
5089
- "name": string;
5090
- "slug": string;
5091
- "description": string | null;
5092
- "skillRef": string | null;
5093
- "skillSlug": string | null;
5094
- "skillName": string | null;
5095
- "skillPromise": string;
5096
- "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5097
- "subjectName": string | null;
5098
- "subjectLabel": string;
5099
- "armConfig": {
5100
- "baseline"?: {
5101
- "tools"?: "default" | "none" | {
5102
- "tools": Array<string>;
5908
+ "shape": "multi-arm";
5909
+ "summary": {
5910
+ "shape": "multi-arm";
5911
+ "verdict": "measured" | "inconclusive";
5912
+ "groups": Array<{
5913
+ "key": string;
5914
+ "label": string;
5915
+ "role": "reference" | "peer";
5916
+ "composition": {
5917
+ "tools"?: "default" | "none" | {
5918
+ "tools": Array<string>;
5919
+ };
5920
+ "addons"?: Array<string>;
5103
5921
  };
5104
- "addons"?: Record<string, unknown>;
5105
- };
5106
- "treatment"?: {
5107
- "tools"?: "default" | "none" | {
5108
- "tools": Array<string>;
5922
+ }>;
5923
+ "ranking": {
5924
+ "plan": "vs_reference" | "all_pairs" | null;
5925
+ "referenceArm": string | null;
5926
+ "decision": {
5927
+ "leader": string | null;
5928
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5929
+ "reason": string;
5930
+ "separatedFrom": Array<string>;
5931
+ "lostTo": Array<{
5932
+ "arm": string;
5933
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5934
+ }>;
5935
+ "clearLeader": boolean;
5936
+ "correctnessLeader": string | null;
5109
5937
  };
5110
- "addons"?: Record<string, unknown>;
5938
+ "entries": Array<{
5939
+ "arm": string;
5940
+ "runs": number;
5941
+ "completed": number;
5942
+ "completionRate": number | null;
5943
+ "goalRate": number | null;
5944
+ "avgQualityScore": number | null;
5945
+ "evalPassRate": number | null;
5946
+ "benchmarkPassRate"?: number | null;
5947
+ "compositeScore": number | null;
5948
+ "avgTotalTokens": number | null;
5949
+ "avgOutputTokens": number | null;
5950
+ "avgCostUsd": number | null;
5951
+ "avgDurationMs": number | null;
5952
+ "medianCostUsd": number | null;
5953
+ "medianTotalTokens": number | null;
5954
+ "medianDurationMs": number | null;
5955
+ "rank": number;
5956
+ }>;
5957
+ "contrasts": Array<{
5958
+ "contrastId": string;
5959
+ "referenceArm": string;
5960
+ "subjectArm": string;
5961
+ "oriented": boolean;
5962
+ "outcome": "won" | "tie" | "unmeasurable";
5963
+ "winner": string | null;
5964
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5965
+ "explanation": string;
5966
+ }>;
5967
+ "comparisons": number;
5968
+ "headline"?: string;
5111
5969
  };
5112
- };
5113
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5114
- "subjectBackend": "model" | "harness" | null;
5115
- "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5116
- "importedFrom"?: string;
5117
- "baselineLabel": string | null;
5118
- "treatmentLabel": string | null;
5119
- "promiseMetrics": Array<{
5120
- "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5121
- "direction": "down" | "up";
5122
- "label": string;
5123
- }>;
5124
- "matrix": Array<{
5125
- "harness": string;
5126
- "model": string;
5127
- "provider"?: string;
5128
- "params"?: {
5129
- "temperature"?: number;
5130
- "maxOutputTokens"?: number;
5131
- "reasoningEffort"?: "low" | "medium" | "high";
5132
- "thinkingBudgetTokens"?: number;
5970
+ "matrixShape": {
5971
+ "cells": number;
5972
+ "harnesses": number;
5973
+ "models": number;
5974
+ "crossed": boolean;
5133
5975
  };
5134
- }>;
5135
- "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5136
- "judgeSupplementary"?: boolean;
5137
- "taskCount": number;
5138
- "variantsPerTask": number;
5139
- "repeats": number;
5140
- "subset": {
5141
- "count": number;
5142
- "seed": string;
5143
- "of": number;
5144
- } | null;
5145
- "status": string;
5146
- "error": string | null;
5147
- "preparation"?: {
5148
- "since": string;
5149
- "launchedBy": {
5150
- "userId"?: string;
5151
- "role"?: string;
5976
+ "flakiness": {
5977
+ "measured": boolean;
5978
+ "conditions": number;
5979
+ "unstable": number;
5980
+ "worst": {
5981
+ "cell": string;
5982
+ "taskId": string;
5983
+ "arm": string;
5984
+ "variant"?: number;
5985
+ "passRate": number;
5986
+ } | null;
5152
5987
  };
5153
- "waitingOn": Array<{
5154
- "imageKey": string;
5155
- "buildId": string;
5156
- "stage"?: "image" | "snapshot";
5157
- }>;
5158
- } | null;
5159
- "visibility": "private" | "org" | "link" | "public";
5160
- "visibilityChangedAt"?: string | null;
5161
- "visibilityChangedBy"?: string | null;
5162
- "shareToken"?: string | null;
5163
- "lowSampleOverride": boolean;
5164
- "publishedAt": string | null;
5165
- "verdict": unknown;
5166
- "definitionVersionId": string | null;
5167
- "runnerKind": "ora-runs" | "batch-job";
5168
- "createdAt": string;
5169
- "updatedAt": string;
5170
- "liveUntil": string | null;
5171
- };
5172
- "streamUrl": string;
5173
- "warnings"?: Array<{
5174
- "harness": string;
5175
- "kind": string;
5176
- }>;
5177
- };
5178
- export type StudyLaunchResponse = {
5179
- "study": {
5180
- "id": string;
5181
- "executiveSummary"?: string | null;
5182
- "plannedRunCount": number;
5183
- "organizationId": string;
5184
- "createdBy": string | null;
5185
- "name": string;
5186
- "slug": string;
5187
- "description": string | null;
5188
- "skillRef": string | null;
5189
- "skillSlug": string | null;
5190
- "skillName": string | null;
5191
- "skillPromise": string;
5192
- "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5193
- "subjectName": string | null;
5194
- "subjectLabel": string;
5195
- "armConfig": {
5196
- "baseline"?: {
5197
- "tools"?: "default" | "none" | {
5198
- "tools": Array<string>;
5199
- };
5200
- "addons"?: Record<string, unknown>;
5988
+ "totals": {
5989
+ "cells": number;
5990
+ "runs": number;
5991
+ "completed": number;
5992
+ "failed": number;
5993
+ "costUsd": number;
5994
+ "tokens": number;
5201
5995
  };
5202
- "treatment"?: {
5996
+ };
5997
+ "arms": Array<{
5998
+ "key": string;
5999
+ "label": string;
6000
+ "position": number;
6001
+ "role": "reference" | "peer";
6002
+ "composition": {
5203
6003
  "tools"?: "default" | "none" | {
5204
6004
  "tools": Array<string>;
5205
6005
  };
5206
- "addons"?: Record<string, unknown>;
5207
- };
5208
- };
5209
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5210
- "subjectBackend": "model" | "harness" | null;
5211
- "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5212
- "importedFrom"?: string;
5213
- "baselineLabel": string | null;
5214
- "treatmentLabel": string | null;
5215
- "promiseMetrics": Array<{
5216
- "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5217
- "direction": "down" | "up";
5218
- "label": string;
5219
- }>;
5220
- "matrix": Array<{
5221
- "harness": string;
5222
- "model": string;
5223
- "provider"?: string;
5224
- "params"?: {
5225
- "temperature"?: number;
5226
- "maxOutputTokens"?: number;
5227
- "reasoningEffort"?: "low" | "medium" | "high";
5228
- "thinkingBudgetTokens"?: number;
5229
- };
5230
- }>;
5231
- "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5232
- "judgeSupplementary"?: boolean;
5233
- "taskCount": number;
5234
- "variantsPerTask": number;
5235
- "repeats": number;
5236
- "subset": {
5237
- "count": number;
5238
- "seed": string;
5239
- "of": number;
5240
- } | null;
5241
- "status": string;
5242
- "error": string | null;
5243
- "preparation"?: {
5244
- "since": string;
5245
- "launchedBy": {
5246
- "userId"?: string;
5247
- "role"?: string;
6006
+ "addons"?: Array<string>;
5248
6007
  };
5249
- "waitingOn": Array<{
5250
- "imageKey": string;
5251
- "buildId": string;
5252
- "stage"?: "image" | "snapshot";
6008
+ "agg": {
6009
+ "runs": number;
6010
+ "completed": number;
6011
+ "infraFailures": number;
6012
+ "completionRate": number | null;
6013
+ "goalRate": number | null;
6014
+ "goalTally": {
6015
+ "achieved": number;
6016
+ "partial": number;
6017
+ "missed": number;
6018
+ };
6019
+ "avgTotalTokens": number | null;
6020
+ "avgOutputTokens": number | null;
6021
+ "avgCostUsd": number | null;
6022
+ "avgDurationMs": number | null;
6023
+ "avgTurns": number | null;
6024
+ "costCoverage": number | null;
6025
+ "avgQualityScore": number | null;
6026
+ "gradedCoverage": number | null;
6027
+ "medianCostUsd": number | null;
6028
+ "medianTotalTokens": number | null;
6029
+ "medianDurationMs": number | null;
6030
+ "p95CostUsd": number | null;
6031
+ "p95TotalTokens": number | null;
6032
+ "p95DurationMs": number | null;
6033
+ "maxCostUsd": number | null;
6034
+ "maxTotalTokens": number | null;
6035
+ "maxDurationMs": number | null;
6036
+ "evalPassRate"?: number | null;
6037
+ "benchmarkPassRate"?: number | null;
6038
+ "adoptionRate"?: number | null;
6039
+ };
6040
+ "perCell": Array<{
6041
+ "harness": string;
6042
+ "model": string;
6043
+ "provider": string | null;
6044
+ "paramsKey"?: string;
6045
+ "label"?: string;
6046
+ "agg": {
6047
+ "runs": number;
6048
+ "completed": number;
6049
+ "infraFailures": number;
6050
+ "completionRate": number | null;
6051
+ "goalRate": number | null;
6052
+ "goalTally": {
6053
+ "achieved": number;
6054
+ "partial": number;
6055
+ "missed": number;
6056
+ };
6057
+ "avgTotalTokens": number | null;
6058
+ "avgOutputTokens": number | null;
6059
+ "avgCostUsd": number | null;
6060
+ "avgDurationMs": number | null;
6061
+ "avgTurns": number | null;
6062
+ "costCoverage": number | null;
6063
+ "avgQualityScore": number | null;
6064
+ "gradedCoverage": number | null;
6065
+ "medianCostUsd": number | null;
6066
+ "medianTotalTokens": number | null;
6067
+ "medianDurationMs": number | null;
6068
+ "p95CostUsd": number | null;
6069
+ "p95TotalTokens": number | null;
6070
+ "p95DurationMs": number | null;
6071
+ "maxCostUsd": number | null;
6072
+ "maxTotalTokens": number | null;
6073
+ "maxDurationMs": number | null;
6074
+ "evalPassRate"?: number | null;
6075
+ "benchmarkPassRate"?: number | null;
6076
+ "adoptionRate"?: number | null;
6077
+ };
5253
6078
  }>;
5254
- } | null;
5255
- "visibility": "private" | "org" | "link" | "public";
5256
- "visibilityChangedAt"?: string | null;
5257
- "visibilityChangedBy"?: string | null;
5258
- "shareToken"?: string | null;
5259
- "lowSampleOverride": boolean;
5260
- "publishedAt": string | null;
5261
- "verdict": unknown;
5262
- "definitionVersionId": string | null;
5263
- "runnerKind": "ora-runs" | "batch-job";
5264
- "createdAt": string;
5265
- "updatedAt": string;
5266
- "liveUntil": string | null;
5267
- };
5268
- "streamUrl": string;
5269
- "warnings"?: Array<{
5270
- "harness": string;
5271
- "kind": string;
5272
- }>;
5273
- };
5274
- export type StudyUpdateLensesRequest = {
5275
- "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5276
- };
5277
- export type StudyUpdateLensesResponse = {
5278
- "study": {
5279
- "id": string;
5280
- "executiveSummary"?: string | null;
5281
- "plannedRunCount": number;
5282
- "organizationId": string;
5283
- "createdBy": string | null;
5284
- "name": string;
5285
- "slug": string;
5286
- "description": string | null;
5287
- "skillRef": string | null;
5288
- "skillSlug": string | null;
5289
- "skillName": string | null;
5290
- "skillPromise": string;
5291
- "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5292
- "subjectName": string | null;
5293
- "subjectLabel": string;
5294
- "armConfig": {
5295
- "baseline"?: {
5296
- "tools"?: "default" | "none" | {
5297
- "tools": Array<string>;
6079
+ "reference"?: {
6080
+ "cells": Array<{
6081
+ "cell": string;
6082
+ "harness": string;
6083
+ "model": string;
6084
+ "provider": string | null;
6085
+ "paramsKey": string;
6086
+ "label": string;
6087
+ "passRate": {
6088
+ "value": number | null;
6089
+ "ci": Array<unknown> | null;
6090
+ "n": number;
6091
+ };
6092
+ "abstention"?: {
6093
+ "value": number | null;
6094
+ "ci": Array<unknown> | null;
6095
+ "n": number;
6096
+ "parts"?: Array<{
6097
+ "id": string;
6098
+ "label": string;
6099
+ "value": number | null;
6100
+ "n": number;
6101
+ }>;
6102
+ };
6103
+ "tasks": number;
6104
+ "graded": number;
6105
+ "unmeasured": number;
6106
+ "matchers": Array<string>;
6107
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
6108
+ }>;
6109
+ "overall": {
6110
+ "value": number | null;
6111
+ "ci": Array<unknown> | null;
6112
+ "n": number;
5298
6113
  };
5299
- "addons"?: Record<string, unknown>;
6114
+ "abstention"?: {
6115
+ "value": number | null;
6116
+ "ci": Array<unknown> | null;
6117
+ "n": number;
6118
+ "parts"?: Array<{
6119
+ "id": string;
6120
+ "label": string;
6121
+ "value": number | null;
6122
+ "n": number;
6123
+ }>;
6124
+ };
6125
+ "graded": number;
6126
+ "unmeasured": number;
6127
+ "modelJudged": boolean;
5300
6128
  };
5301
- "treatment"?: {
5302
- "tools"?: "default" | "none" | {
5303
- "tools": Array<string>;
6129
+ }>;
6130
+ "contrasts": Array<{
6131
+ "contrastId": string;
6132
+ "referenceArm": string;
6133
+ "subjectArm": string;
6134
+ "oriented": boolean;
6135
+ "basis"?: {
6136
+ "id": "matched" | "whole_setup";
6137
+ "compared": {
6138
+ "reference": Array<{
6139
+ "harness": string;
6140
+ "model": string;
6141
+ "provider"?: string;
6142
+ "params"?: Record<string, unknown>;
6143
+ }>;
6144
+ "subject": Array<{
6145
+ "harness": string;
6146
+ "model": string;
6147
+ "provider"?: string;
6148
+ "params"?: Record<string, unknown>;
6149
+ }>;
5304
6150
  };
5305
- "addons"?: Record<string, unknown>;
6151
+ "excluded": {
6152
+ "reference": Array<{
6153
+ "harness": string;
6154
+ "model": string;
6155
+ "provider"?: string;
6156
+ "params"?: Record<string, unknown>;
6157
+ }>;
6158
+ "subject": Array<{
6159
+ "harness": string;
6160
+ "model": string;
6161
+ "provider"?: string;
6162
+ "params"?: Record<string, unknown>;
6163
+ }>;
6164
+ };
6165
+ "differsIn": Array<string>;
6166
+ "licensedFacet": string | null;
6167
+ "efficiency": "comparable" | "incomparable";
6168
+ "pairedTasks": number | null;
6169
+ "dropped": {
6170
+ "reference": number;
6171
+ "subject": number;
6172
+ } | null;
6173
+ };
6174
+ "magnitudes": "relative";
6175
+ "summary": {
6176
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
6177
+ "headline": Array<{
6178
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6179
+ "label": string;
6180
+ "direction": "down" | "up";
6181
+ "baselineAvg": number | null;
6182
+ "skillAvg": number | null;
6183
+ "deltaPct": number | null;
6184
+ "delta"?: {
6185
+ "unit": "pp";
6186
+ "absolute": number | null;
6187
+ "relativePct": number | null;
6188
+ } | {
6189
+ "unit": "pts";
6190
+ "absolute": number | null;
6191
+ "relativePct"?: number | null;
6192
+ } | {
6193
+ "unit": "%";
6194
+ "relativePct": number | null;
6195
+ };
6196
+ "improved": boolean;
6197
+ }>;
6198
+ "completionRate": {
6199
+ "baseline": number | null;
6200
+ "skill": number | null;
6201
+ };
6202
+ "goalRate": {
6203
+ "baseline": number | null;
6204
+ "skill": number | null;
6205
+ };
6206
+ "pairwise": {
6207
+ "pairs": number;
6208
+ "skillWins": number;
6209
+ "baselineWins": number;
6210
+ "ties": number;
6211
+ "winRate": number | null;
6212
+ };
6213
+ "decision"?: {
6214
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
6215
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
6216
+ "reason": string;
6217
+ "efficiencyDiscounted": boolean;
6218
+ };
6219
+ "concentration": {
6220
+ "perCell": Array<{
6221
+ "cell": string;
6222
+ "goalDeltaPp": number | null;
6223
+ "contributionPct": number | null;
6224
+ }>;
6225
+ "heterogeneity": {
6226
+ "dominantCell": string;
6227
+ "contributionPct": number;
6228
+ } | null;
6229
+ };
6230
+ "matrixShape": {
6231
+ "cells": number;
6232
+ "harnesses": number;
6233
+ "models": number;
6234
+ "crossed": boolean;
6235
+ };
6236
+ "flakiness"?: {
6237
+ "measured": boolean;
6238
+ "conditions": number;
6239
+ "unstable": number;
6240
+ "worst": {
6241
+ "cell": string;
6242
+ "taskId": string;
6243
+ "arm": "baseline" | "skill";
6244
+ "variant"?: number;
6245
+ "passRate": number;
6246
+ } | null;
6247
+ };
6248
+ "arms"?: {
6249
+ "baseline": {
6250
+ "runs": number;
6251
+ "completed": number;
6252
+ "infraFailures": number;
6253
+ "completionRate": number | null;
6254
+ "goalRate": number | null;
6255
+ "goalTally": {
6256
+ "achieved": number;
6257
+ "partial": number;
6258
+ "missed": number;
6259
+ };
6260
+ "avgTotalTokens": number | null;
6261
+ "avgOutputTokens": number | null;
6262
+ "avgCostUsd": number | null;
6263
+ "avgDurationMs": number | null;
6264
+ "avgTurns": number | null;
6265
+ "costCoverage": number | null;
6266
+ "avgQualityScore": number | null;
6267
+ "gradedCoverage": number | null;
6268
+ "medianCostUsd": number | null;
6269
+ "medianTotalTokens": number | null;
6270
+ "medianDurationMs": number | null;
6271
+ "p95CostUsd": number | null;
6272
+ "p95TotalTokens": number | null;
6273
+ "p95DurationMs": number | null;
6274
+ "maxCostUsd": number | null;
6275
+ "maxTotalTokens": number | null;
6276
+ "maxDurationMs": number | null;
6277
+ "evalPassRate"?: number | null;
6278
+ "benchmarkPassRate"?: number | null;
6279
+ "adoptionRate"?: number | null;
6280
+ };
6281
+ "skill": {
6282
+ "runs": number;
6283
+ "completed": number;
6284
+ "infraFailures": number;
6285
+ "completionRate": number | null;
6286
+ "goalRate": number | null;
6287
+ "goalTally": {
6288
+ "achieved": number;
6289
+ "partial": number;
6290
+ "missed": number;
6291
+ };
6292
+ "avgTotalTokens": number | null;
6293
+ "avgOutputTokens": number | null;
6294
+ "avgCostUsd": number | null;
6295
+ "avgDurationMs": number | null;
6296
+ "avgTurns": number | null;
6297
+ "costCoverage": number | null;
6298
+ "avgQualityScore": number | null;
6299
+ "gradedCoverage": number | null;
6300
+ "medianCostUsd": number | null;
6301
+ "medianTotalTokens": number | null;
6302
+ "medianDurationMs": number | null;
6303
+ "p95CostUsd": number | null;
6304
+ "p95TotalTokens": number | null;
6305
+ "p95DurationMs": number | null;
6306
+ "maxCostUsd": number | null;
6307
+ "maxTotalTokens": number | null;
6308
+ "maxDurationMs": number | null;
6309
+ "evalPassRate"?: number | null;
6310
+ "benchmarkPassRate"?: number | null;
6311
+ "adoptionRate"?: number | null;
6312
+ };
6313
+ };
6314
+ "totals": {
6315
+ "cells": number;
6316
+ "runs": number;
6317
+ "completed": number;
6318
+ "failed": number;
6319
+ "costUsd": number;
6320
+ "tokens": number;
6321
+ };
6322
+ };
6323
+ "perCell": Array<{
6324
+ "harness": string;
6325
+ "model": string;
6326
+ "provider": string | null;
6327
+ "paramsKey"?: string;
6328
+ "label"?: string;
6329
+ "baseline": {
6330
+ "runs": number;
6331
+ "completed": number;
6332
+ "infraFailures": number;
6333
+ "completionRate": number | null;
6334
+ "goalRate": number | null;
6335
+ "goalTally": {
6336
+ "achieved": number;
6337
+ "partial": number;
6338
+ "missed": number;
6339
+ };
6340
+ "avgTotalTokens": number | null;
6341
+ "avgOutputTokens": number | null;
6342
+ "avgCostUsd": number | null;
6343
+ "avgDurationMs": number | null;
6344
+ "avgTurns": number | null;
6345
+ "costCoverage": number | null;
6346
+ "avgQualityScore": number | null;
6347
+ "gradedCoverage": number | null;
6348
+ "medianCostUsd": number | null;
6349
+ "medianTotalTokens": number | null;
6350
+ "medianDurationMs": number | null;
6351
+ "p95CostUsd": number | null;
6352
+ "p95TotalTokens": number | null;
6353
+ "p95DurationMs": number | null;
6354
+ "maxCostUsd": number | null;
6355
+ "maxTotalTokens": number | null;
6356
+ "maxDurationMs": number | null;
6357
+ "evalPassRate"?: number | null;
6358
+ "benchmarkPassRate"?: number | null;
6359
+ "adoptionRate"?: number | null;
6360
+ };
6361
+ "skill": {
6362
+ "runs": number;
6363
+ "completed": number;
6364
+ "infraFailures": number;
6365
+ "completionRate": number | null;
6366
+ "goalRate": number | null;
6367
+ "goalTally": {
6368
+ "achieved": number;
6369
+ "partial": number;
6370
+ "missed": number;
6371
+ };
6372
+ "avgTotalTokens": number | null;
6373
+ "avgOutputTokens": number | null;
6374
+ "avgCostUsd": number | null;
6375
+ "avgDurationMs": number | null;
6376
+ "avgTurns": number | null;
6377
+ "costCoverage": number | null;
6378
+ "avgQualityScore": number | null;
6379
+ "gradedCoverage": number | null;
6380
+ "medianCostUsd": number | null;
6381
+ "medianTotalTokens": number | null;
6382
+ "medianDurationMs": number | null;
6383
+ "p95CostUsd": number | null;
6384
+ "p95TotalTokens": number | null;
6385
+ "p95DurationMs": number | null;
6386
+ "maxCostUsd": number | null;
6387
+ "maxTotalTokens": number | null;
6388
+ "maxDurationMs": number | null;
6389
+ "evalPassRate"?: number | null;
6390
+ "benchmarkPassRate"?: number | null;
6391
+ "adoptionRate"?: number | null;
6392
+ };
6393
+ "deltas": {
6394
+ "totalTokensPct": number | null;
6395
+ "outputTokensPct": number | null;
6396
+ "costUsdPct": number | null;
6397
+ "durationMsPct": number | null;
6398
+ "numTurnsPct": number | null;
6399
+ "completionRatePp": number | null;
6400
+ "goalRatePp": number | null;
6401
+ "qualityScorePts": number | null;
6402
+ "evalPassRatePp"?: number | null;
6403
+ };
6404
+ }>;
6405
+ } | {
6406
+ "contrastId": string;
6407
+ "referenceArm": string;
6408
+ "subjectArm": string;
6409
+ "oriented": boolean;
6410
+ "basis"?: {
6411
+ "id": "matched" | "whole_setup";
6412
+ "compared": {
6413
+ "reference": Array<{
6414
+ "harness": string;
6415
+ "model": string;
6416
+ "provider"?: string;
6417
+ "params"?: Record<string, unknown>;
6418
+ }>;
6419
+ "subject": Array<{
6420
+ "harness": string;
6421
+ "model": string;
6422
+ "provider"?: string;
6423
+ "params"?: Record<string, unknown>;
6424
+ }>;
6425
+ };
6426
+ "excluded": {
6427
+ "reference": Array<{
6428
+ "harness": string;
6429
+ "model": string;
6430
+ "provider"?: string;
6431
+ "params"?: Record<string, unknown>;
6432
+ }>;
6433
+ "subject": Array<{
6434
+ "harness": string;
6435
+ "model": string;
6436
+ "provider"?: string;
6437
+ "params"?: Record<string, unknown>;
6438
+ }>;
6439
+ };
6440
+ "differsIn": Array<string>;
6441
+ "licensedFacet": string | null;
6442
+ "efficiency": "comparable" | "incomparable";
6443
+ "pairedTasks": number | null;
6444
+ "dropped": {
6445
+ "reference": number;
6446
+ "subject": number;
6447
+ } | null;
6448
+ };
6449
+ "magnitudes": "absolute";
6450
+ "summary": {
6451
+ "completionRate": {
6452
+ "baseline": number | null;
6453
+ "skill": number | null;
6454
+ };
6455
+ "goalRate": {
6456
+ "baseline": number | null;
6457
+ "skill": number | null;
6458
+ };
6459
+ "pairwise": {
6460
+ "pairs": number;
6461
+ "skillWins": number;
6462
+ "baselineWins": number;
6463
+ "ties": number;
6464
+ "winRate": number | null;
6465
+ };
6466
+ "decision"?: {
6467
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
6468
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
6469
+ "reason": string;
6470
+ "efficiencyDiscounted": boolean;
6471
+ };
6472
+ "concentration": {
6473
+ "perCell": Array<{
6474
+ "cell": string;
6475
+ "goalDeltaPp": number | null;
6476
+ "contributionPct": number | null;
6477
+ }>;
6478
+ "heterogeneity": {
6479
+ "dominantCell": string;
6480
+ "contributionPct": number;
6481
+ } | null;
6482
+ };
6483
+ "matrixShape": {
6484
+ "cells": number;
6485
+ "harnesses": number;
6486
+ "models": number;
6487
+ "crossed": boolean;
6488
+ };
6489
+ "flakiness"?: {
6490
+ "measured": boolean;
6491
+ "conditions": number;
6492
+ "unstable": number;
6493
+ "worst": {
6494
+ "cell": string;
6495
+ "taskId": string;
6496
+ "arm": "baseline" | "skill";
6497
+ "variant"?: number;
6498
+ "passRate": number;
6499
+ } | null;
6500
+ };
6501
+ "arms"?: {
6502
+ "baseline": {
6503
+ "runs": number;
6504
+ "completed": number;
6505
+ "infraFailures": number;
6506
+ "completionRate": number | null;
6507
+ "goalRate": number | null;
6508
+ "goalTally": {
6509
+ "achieved": number;
6510
+ "partial": number;
6511
+ "missed": number;
6512
+ };
6513
+ "avgTotalTokens": number | null;
6514
+ "avgOutputTokens": number | null;
6515
+ "avgCostUsd": number | null;
6516
+ "avgDurationMs": number | null;
6517
+ "avgTurns": number | null;
6518
+ "costCoverage": number | null;
6519
+ "avgQualityScore": number | null;
6520
+ "gradedCoverage": number | null;
6521
+ "medianCostUsd": number | null;
6522
+ "medianTotalTokens": number | null;
6523
+ "medianDurationMs": number | null;
6524
+ "p95CostUsd": number | null;
6525
+ "p95TotalTokens": number | null;
6526
+ "p95DurationMs": number | null;
6527
+ "maxCostUsd": number | null;
6528
+ "maxTotalTokens": number | null;
6529
+ "maxDurationMs": number | null;
6530
+ "evalPassRate"?: number | null;
6531
+ "benchmarkPassRate"?: number | null;
6532
+ "adoptionRate"?: number | null;
6533
+ };
6534
+ "skill": {
6535
+ "runs": number;
6536
+ "completed": number;
6537
+ "infraFailures": number;
6538
+ "completionRate": number | null;
6539
+ "goalRate": number | null;
6540
+ "goalTally": {
6541
+ "achieved": number;
6542
+ "partial": number;
6543
+ "missed": number;
6544
+ };
6545
+ "avgTotalTokens": number | null;
6546
+ "avgOutputTokens": number | null;
6547
+ "avgCostUsd": number | null;
6548
+ "avgDurationMs": number | null;
6549
+ "avgTurns": number | null;
6550
+ "costCoverage": number | null;
6551
+ "avgQualityScore": number | null;
6552
+ "gradedCoverage": number | null;
6553
+ "medianCostUsd": number | null;
6554
+ "medianTotalTokens": number | null;
6555
+ "medianDurationMs": number | null;
6556
+ "p95CostUsd": number | null;
6557
+ "p95TotalTokens": number | null;
6558
+ "p95DurationMs": number | null;
6559
+ "maxCostUsd": number | null;
6560
+ "maxTotalTokens": number | null;
6561
+ "maxDurationMs": number | null;
6562
+ "evalPassRate"?: number | null;
6563
+ "benchmarkPassRate"?: number | null;
6564
+ "adoptionRate"?: number | null;
6565
+ };
6566
+ };
6567
+ "totals": {
6568
+ "cells": number;
6569
+ "runs": number;
6570
+ "completed": number;
6571
+ "failed": number;
6572
+ "costUsd": number;
6573
+ "tokens": number;
6574
+ };
6575
+ };
6576
+ "perCell": Array<{
6577
+ "harness": string;
6578
+ "model": string;
6579
+ "provider": string | null;
6580
+ "paramsKey"?: string;
6581
+ "label"?: string;
6582
+ "baseline": {
6583
+ "runs": number;
6584
+ "completed": number;
6585
+ "infraFailures": number;
6586
+ "completionRate": number | null;
6587
+ "goalRate": number | null;
6588
+ "goalTally": {
6589
+ "achieved": number;
6590
+ "partial": number;
6591
+ "missed": number;
6592
+ };
6593
+ "avgTotalTokens": number | null;
6594
+ "avgOutputTokens": number | null;
6595
+ "avgCostUsd": number | null;
6596
+ "avgDurationMs": number | null;
6597
+ "avgTurns": number | null;
6598
+ "costCoverage": number | null;
6599
+ "avgQualityScore": number | null;
6600
+ "gradedCoverage": number | null;
6601
+ "medianCostUsd": number | null;
6602
+ "medianTotalTokens": number | null;
6603
+ "medianDurationMs": number | null;
6604
+ "p95CostUsd": number | null;
6605
+ "p95TotalTokens": number | null;
6606
+ "p95DurationMs": number | null;
6607
+ "maxCostUsd": number | null;
6608
+ "maxTotalTokens": number | null;
6609
+ "maxDurationMs": number | null;
6610
+ "evalPassRate"?: number | null;
6611
+ "benchmarkPassRate"?: number | null;
6612
+ "adoptionRate"?: number | null;
6613
+ };
6614
+ "skill": {
6615
+ "runs": number;
6616
+ "completed": number;
6617
+ "infraFailures": number;
6618
+ "completionRate": number | null;
6619
+ "goalRate": number | null;
6620
+ "goalTally": {
6621
+ "achieved": number;
6622
+ "partial": number;
6623
+ "missed": number;
6624
+ };
6625
+ "avgTotalTokens": number | null;
6626
+ "avgOutputTokens": number | null;
6627
+ "avgCostUsd": number | null;
6628
+ "avgDurationMs": number | null;
6629
+ "avgTurns": number | null;
6630
+ "costCoverage": number | null;
6631
+ "avgQualityScore": number | null;
6632
+ "gradedCoverage": number | null;
6633
+ "medianCostUsd": number | null;
6634
+ "medianTotalTokens": number | null;
6635
+ "medianDurationMs": number | null;
6636
+ "p95CostUsd": number | null;
6637
+ "p95TotalTokens": number | null;
6638
+ "p95DurationMs": number | null;
6639
+ "maxCostUsd": number | null;
6640
+ "maxTotalTokens": number | null;
6641
+ "maxDurationMs": number | null;
6642
+ "evalPassRate"?: number | null;
6643
+ "benchmarkPassRate"?: number | null;
6644
+ "adoptionRate"?: number | null;
6645
+ };
6646
+ "deltas": {
6647
+ "completionRatePp": number | null;
6648
+ "goalRatePp": number | null;
6649
+ "qualityScorePts": number | null;
6650
+ "evalPassRatePp"?: number | null;
6651
+ };
6652
+ }>;
6653
+ }>;
6654
+ "board": Array<{
6655
+ "harness": string;
6656
+ "model": string;
6657
+ "provider": string | null;
6658
+ "paramsKey"?: string;
6659
+ "label"?: string;
6660
+ "arm": string;
6661
+ "runs": number;
6662
+ "completed": number;
6663
+ "completionRate": number | null;
6664
+ "goalRate": number | null;
6665
+ "avgQualityScore": number | null;
6666
+ "evalPassRate": number | null;
6667
+ "benchmarkPassRate"?: number | null;
6668
+ "compositeScore": number | null;
6669
+ "avgTotalTokens": number | null;
6670
+ "avgOutputTokens": number | null;
6671
+ "avgCostUsd": number | null;
6672
+ "avgDurationMs": number | null;
6673
+ "medianCostUsd": number | null;
6674
+ "medianTotalTokens": number | null;
6675
+ "medianDurationMs": number | null;
6676
+ "rank": number;
6677
+ }>;
6678
+ "setupsBoard"?: Array<{
6679
+ "harness": string;
6680
+ "model": string;
6681
+ "provider": string | null;
6682
+ "paramsKey"?: string;
6683
+ "label"?: string;
6684
+ "arm": string;
6685
+ "runs": number;
6686
+ "completed": number;
6687
+ "completionRate": number | null;
6688
+ "goalRate": number | null;
6689
+ "avgQualityScore": number | null;
6690
+ "evalPassRate": number | null;
6691
+ "benchmarkPassRate"?: number | null;
6692
+ "compositeScore": number | null;
6693
+ "avgTotalTokens": number | null;
6694
+ "avgOutputTokens": number | null;
6695
+ "avgCostUsd": number | null;
6696
+ "avgDurationMs": number | null;
6697
+ "medianCostUsd": number | null;
6698
+ "medianTotalTokens": number | null;
6699
+ "medianDurationMs": number | null;
6700
+ "rank": number;
6701
+ "group": string;
6702
+ "agent": string;
6703
+ "setup": string;
6704
+ }>;
6705
+ "perTask": Array<{
6706
+ "taskId": string;
6707
+ "position": number;
6708
+ "promptPreview": string;
6709
+ "requiresWeb": boolean;
6710
+ "category": string | null;
6711
+ "byArm": Record<string, unknown>;
6712
+ }>;
6713
+ "behavior"?: Array<{
6714
+ "harness": string;
6715
+ "model": string;
6716
+ "provider": string | null;
6717
+ "arm": string;
6718
+ "totalRuns": number;
6719
+ "signals": Array<{
6720
+ "id": string;
6721
+ "label": string;
6722
+ "tone": "good" | "bad" | "neutral";
6723
+ "runCount": number;
6724
+ "succeededCount": number;
6725
+ }>;
6726
+ }>;
6727
+ "execution"?: Record<string, unknown>;
6728
+ "inference"?: {
6729
+ "contrasts": Array<{
6730
+ "contrastId": string;
6731
+ "table": {
6732
+ "columns": Array<{
6733
+ "key": string;
6734
+ "label": string;
6735
+ "family": "rate" | "score" | "magnitude";
6736
+ "betterWhen": "higher" | "lower";
6737
+ }>;
6738
+ "groups": Array<{
6739
+ "kind": "cell" | "harness" | "model" | "field";
6740
+ "title": string;
6741
+ "rows": Array<{
6742
+ "key": string;
6743
+ "kind": "cell" | "harness" | "model" | "field";
6744
+ "harness": string | null;
6745
+ "model": string | null;
6746
+ "paramsKey"?: string;
6747
+ "runs": number;
6748
+ "cells": Record<string, unknown>;
6749
+ }>;
6750
+ }>;
6751
+ "reference": "baseline" | "field";
6752
+ "draws": number;
6753
+ "adjustment": {
6754
+ "scope": "planned-comparisons";
6755
+ "comparisons": number;
6756
+ };
6757
+ } | null;
6758
+ }>;
6759
+ "holm": {
6760
+ "status": "complete";
6761
+ "ready": number;
6762
+ "planned": number;
6763
+ "adjustment": {
6764
+ "scope": "planned-comparisons";
6765
+ "comparisons": number;
6766
+ };
6767
+ } | {
6768
+ "status": "pending";
6769
+ "ready": number;
6770
+ "planned": number;
6771
+ };
6772
+ "claims": {
6773
+ "perContrast": Record<string, unknown>;
6774
+ "ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
6775
+ };
6776
+ };
6777
+ };
6778
+ "warnings": Array<{
6779
+ "type": "eval_capability_gap";
6780
+ "harness": string;
6781
+ "kind": string;
6782
+ } | {
6783
+ "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
6784
+ "detail": string;
6785
+ "taskIds"?: Array<string>;
6786
+ } | {
6787
+ "type": "design_aggregate_budget";
6788
+ "detail": string;
6789
+ }>;
6790
+ "taskHealth": {
6791
+ "flags": Array<{
6792
+ "taskId": string;
6793
+ "signal": string;
6794
+ "kind": "floor" | "ceiling";
6795
+ "conditions": number;
6796
+ }>;
6797
+ "rubricSplits": Array<{
6798
+ "taskId": string;
6799
+ "criterionIndex": number;
6800
+ "criterion": string;
6801
+ "split": number;
6802
+ "graded": number;
6803
+ }>;
6804
+ "signals": Array<{
6805
+ "id": string;
6806
+ "label": string;
6807
+ }>;
6808
+ };
6809
+ "shareUrl"?: string | null;
6810
+ "canManageSharing"?: boolean;
6811
+ };
6812
+ export type StudyUpdateDetailsRequest = {
6813
+ "name"?: string;
6814
+ "description"?: string;
6815
+ };
6816
+ export type StudyUpdateDetailsResponse = {
6817
+ "study": {
6818
+ "id": string;
6819
+ "executiveSummary"?: string | null;
6820
+ "plannedRunCount": number;
6821
+ "organizationId": string;
6822
+ "createdBy": string | null;
6823
+ "name": string;
6824
+ "slug": string;
6825
+ "description": string | null;
6826
+ "skillRef": string | null;
6827
+ "skillSlug": string | null;
6828
+ "skillName": string | null;
6829
+ "skillPromise": string;
6830
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
6831
+ "subjectName": string | null;
6832
+ "subjectLabel": string;
6833
+ "armConfig": {
6834
+ "baseline"?: {
6835
+ "tools"?: "default" | "none" | {
6836
+ "tools": Array<string>;
6837
+ };
6838
+ "addons"?: Record<string, unknown>;
6839
+ };
6840
+ "treatment"?: {
6841
+ "tools"?: "default" | "none" | {
6842
+ "tools": Array<string>;
6843
+ };
6844
+ "addons"?: Record<string, unknown>;
6845
+ };
6846
+ };
6847
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6848
+ "subjectBackend": "model" | "harness" | null;
6849
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6850
+ "importedFrom"?: string;
6851
+ "baselineLabel": string | null;
6852
+ "treatmentLabel": string | null;
6853
+ "promiseMetrics": Array<{
6854
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6855
+ "direction": "down" | "up";
6856
+ "label": string;
6857
+ }>;
6858
+ "matrix": Array<{
6859
+ "harness": string;
6860
+ "model": string;
6861
+ "provider"?: string;
6862
+ "params"?: {
6863
+ "temperature"?: number;
6864
+ "maxOutputTokens"?: number;
6865
+ "reasoningEffort"?: "low" | "medium" | "high";
6866
+ "thinkingBudgetTokens"?: number;
6867
+ };
6868
+ }>;
6869
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
6870
+ "judgeSupplementary"?: boolean;
6871
+ "taskCount": number;
6872
+ "variantsPerTask": number;
6873
+ "repeats": number;
6874
+ "subset": {
6875
+ "count": number;
6876
+ "seed": string;
6877
+ "of": number;
6878
+ } | null;
6879
+ "status": string;
6880
+ "error": string | null;
6881
+ "preparation"?: {
6882
+ "since": string;
6883
+ "progressAt"?: string;
6884
+ "launchedBy": {
6885
+ "userId"?: string;
6886
+ "role"?: string;
6887
+ };
6888
+ "waitingOn": Array<{
6889
+ "imageKey": string;
6890
+ "buildId": string;
6891
+ "stage"?: "image" | "snapshot";
6892
+ "retry"?: {
6893
+ "failedAttempts": number;
6894
+ "maxAttempts": number;
6895
+ "notBefore": string;
6896
+ "lastFailure": string;
6897
+ };
6898
+ }>;
6899
+ } | null;
6900
+ "visibility": "private" | "org" | "link" | "public";
6901
+ "visibilityChangedAt"?: string | null;
6902
+ "visibilityChangedBy"?: string | null;
6903
+ "shareToken"?: string | null;
6904
+ "lowSampleOverride": boolean;
6905
+ "publishedAt": string | null;
6906
+ "verdict": unknown;
6907
+ "definitionVersionId": string | null;
6908
+ "runnerKind": "ora-runs" | "batch-job";
6909
+ "createdAt": string;
6910
+ "updatedAt": string;
6911
+ "liveUntil": string | null;
6912
+ "arms"?: Array<{
6913
+ "key": string;
6914
+ "position": number;
6915
+ "label": string;
6916
+ "role": "reference" | "peer";
6917
+ "payload": {
6918
+ "tools"?: "default" | "none" | {
6919
+ "tools": Array<string>;
6920
+ };
6921
+ "addons"?: Record<string, unknown>;
6922
+ };
6923
+ "cells"?: Array<{
6924
+ "harness": string;
6925
+ "model": string;
6926
+ "provider"?: string;
6927
+ "params"?: {
6928
+ "temperature"?: number;
6929
+ "maxOutputTokens"?: number;
6930
+ "reasoningEffort"?: "low" | "medium" | "high";
6931
+ "thinkingBudgetTokens"?: number;
6932
+ };
6933
+ }>;
6934
+ "skills"?: Array<{
6935
+ "ref": string;
6936
+ "name": string;
6937
+ "changedSinceCreate"?: boolean;
6938
+ }>;
6939
+ }>;
6940
+ "contrastPlan"?: "vs_reference" | "all_pairs";
6941
+ "cellPlan"?: "crossed" | "per_setup";
6942
+ "skillDisclosure"?: "withheld" | "named";
6943
+ "modeOptions"?: Record<string, unknown>;
6944
+ };
6945
+ };
6946
+ export type StudyDeleteResponse = unknown;
6947
+ export type StudyRegenerateSummaryResponse = {
6948
+ "status": "queued";
6949
+ };
6950
+ export type StudyGenerateRequest = {
6951
+ "proposeExpectations"?: boolean;
6952
+ };
6953
+ export type StudyGenerateResponse = {
6954
+ "study": {
6955
+ "id": string;
6956
+ "executiveSummary"?: string | null;
6957
+ "plannedRunCount": number;
6958
+ "organizationId": string;
6959
+ "createdBy": string | null;
6960
+ "name": string;
6961
+ "slug": string;
6962
+ "description": string | null;
6963
+ "skillRef": string | null;
6964
+ "skillSlug": string | null;
6965
+ "skillName": string | null;
6966
+ "skillPromise": string;
6967
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
6968
+ "subjectName": string | null;
6969
+ "subjectLabel": string;
6970
+ "armConfig": {
6971
+ "baseline"?: {
6972
+ "tools"?: "default" | "none" | {
6973
+ "tools": Array<string>;
6974
+ };
6975
+ "addons"?: Record<string, unknown>;
6976
+ };
6977
+ "treatment"?: {
6978
+ "tools"?: "default" | "none" | {
6979
+ "tools": Array<string>;
6980
+ };
6981
+ "addons"?: Record<string, unknown>;
6982
+ };
6983
+ };
6984
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6985
+ "subjectBackend": "model" | "harness" | null;
6986
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6987
+ "importedFrom"?: string;
6988
+ "baselineLabel": string | null;
6989
+ "treatmentLabel": string | null;
6990
+ "promiseMetrics": Array<{
6991
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6992
+ "direction": "down" | "up";
6993
+ "label": string;
6994
+ }>;
6995
+ "matrix": Array<{
6996
+ "harness": string;
6997
+ "model": string;
6998
+ "provider"?: string;
6999
+ "params"?: {
7000
+ "temperature"?: number;
7001
+ "maxOutputTokens"?: number;
7002
+ "reasoningEffort"?: "low" | "medium" | "high";
7003
+ "thinkingBudgetTokens"?: number;
7004
+ };
7005
+ }>;
7006
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7007
+ "judgeSupplementary"?: boolean;
7008
+ "taskCount": number;
7009
+ "variantsPerTask": number;
7010
+ "repeats": number;
7011
+ "subset": {
7012
+ "count": number;
7013
+ "seed": string;
7014
+ "of": number;
7015
+ } | null;
7016
+ "status": string;
7017
+ "error": string | null;
7018
+ "preparation"?: {
7019
+ "since": string;
7020
+ "progressAt"?: string;
7021
+ "launchedBy": {
7022
+ "userId"?: string;
7023
+ "role"?: string;
7024
+ };
7025
+ "waitingOn": Array<{
7026
+ "imageKey": string;
7027
+ "buildId": string;
7028
+ "stage"?: "image" | "snapshot";
7029
+ "retry"?: {
7030
+ "failedAttempts": number;
7031
+ "maxAttempts": number;
7032
+ "notBefore": string;
7033
+ "lastFailure": string;
7034
+ };
7035
+ }>;
7036
+ } | null;
7037
+ "visibility": "private" | "org" | "link" | "public";
7038
+ "visibilityChangedAt"?: string | null;
7039
+ "visibilityChangedBy"?: string | null;
7040
+ "shareToken"?: string | null;
7041
+ "lowSampleOverride": boolean;
7042
+ "publishedAt": string | null;
7043
+ "verdict": unknown;
7044
+ "definitionVersionId": string | null;
7045
+ "runnerKind": "ora-runs" | "batch-job";
7046
+ "createdAt": string;
7047
+ "updatedAt": string;
7048
+ "liveUntil": string | null;
7049
+ "arms"?: Array<{
7050
+ "key": string;
7051
+ "position": number;
7052
+ "label": string;
7053
+ "role": "reference" | "peer";
7054
+ "payload": {
7055
+ "tools"?: "default" | "none" | {
7056
+ "tools": Array<string>;
7057
+ };
7058
+ "addons"?: Record<string, unknown>;
7059
+ };
7060
+ "cells"?: Array<{
7061
+ "harness": string;
7062
+ "model": string;
7063
+ "provider"?: string;
7064
+ "params"?: {
7065
+ "temperature"?: number;
7066
+ "maxOutputTokens"?: number;
7067
+ "reasoningEffort"?: "low" | "medium" | "high";
7068
+ "thinkingBudgetTokens"?: number;
7069
+ };
7070
+ }>;
7071
+ "skills"?: Array<{
7072
+ "ref": string;
7073
+ "name": string;
7074
+ "changedSinceCreate"?: boolean;
7075
+ }>;
7076
+ }>;
7077
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7078
+ "cellPlan"?: "crossed" | "per_setup";
7079
+ "skillDisclosure"?: "withheld" | "named";
7080
+ "modeOptions"?: Record<string, unknown>;
7081
+ };
7082
+ "streamUrl": string;
7083
+ "warnings"?: Array<{
7084
+ "harness": string;
7085
+ "kind": string;
7086
+ } | {
7087
+ "type": "design_aggregate_budget";
7088
+ "detail": string;
7089
+ }>;
7090
+ };
7091
+ export type StudyLaunchResponse = {
7092
+ "study": {
7093
+ "id": string;
7094
+ "executiveSummary"?: string | null;
7095
+ "plannedRunCount": number;
7096
+ "organizationId": string;
7097
+ "createdBy": string | null;
7098
+ "name": string;
7099
+ "slug": string;
7100
+ "description": string | null;
7101
+ "skillRef": string | null;
7102
+ "skillSlug": string | null;
7103
+ "skillName": string | null;
7104
+ "skillPromise": string;
7105
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
7106
+ "subjectName": string | null;
7107
+ "subjectLabel": string;
7108
+ "armConfig": {
7109
+ "baseline"?: {
7110
+ "tools"?: "default" | "none" | {
7111
+ "tools": Array<string>;
7112
+ };
7113
+ "addons"?: Record<string, unknown>;
7114
+ };
7115
+ "treatment"?: {
7116
+ "tools"?: "default" | "none" | {
7117
+ "tools": Array<string>;
7118
+ };
7119
+ "addons"?: Record<string, unknown>;
7120
+ };
7121
+ };
7122
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
7123
+ "subjectBackend": "model" | "harness" | null;
7124
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
7125
+ "importedFrom"?: string;
7126
+ "baselineLabel": string | null;
7127
+ "treatmentLabel": string | null;
7128
+ "promiseMetrics": Array<{
7129
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
7130
+ "direction": "down" | "up";
7131
+ "label": string;
7132
+ }>;
7133
+ "matrix": Array<{
7134
+ "harness": string;
7135
+ "model": string;
7136
+ "provider"?: string;
7137
+ "params"?: {
7138
+ "temperature"?: number;
7139
+ "maxOutputTokens"?: number;
7140
+ "reasoningEffort"?: "low" | "medium" | "high";
7141
+ "thinkingBudgetTokens"?: number;
7142
+ };
7143
+ }>;
7144
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7145
+ "judgeSupplementary"?: boolean;
7146
+ "taskCount": number;
7147
+ "variantsPerTask": number;
7148
+ "repeats": number;
7149
+ "subset": {
7150
+ "count": number;
7151
+ "seed": string;
7152
+ "of": number;
7153
+ } | null;
7154
+ "status": string;
7155
+ "error": string | null;
7156
+ "preparation"?: {
7157
+ "since": string;
7158
+ "progressAt"?: string;
7159
+ "launchedBy": {
7160
+ "userId"?: string;
7161
+ "role"?: string;
7162
+ };
7163
+ "waitingOn": Array<{
7164
+ "imageKey": string;
7165
+ "buildId": string;
7166
+ "stage"?: "image" | "snapshot";
7167
+ "retry"?: {
7168
+ "failedAttempts": number;
7169
+ "maxAttempts": number;
7170
+ "notBefore": string;
7171
+ "lastFailure": string;
7172
+ };
7173
+ }>;
7174
+ } | null;
7175
+ "visibility": "private" | "org" | "link" | "public";
7176
+ "visibilityChangedAt"?: string | null;
7177
+ "visibilityChangedBy"?: string | null;
7178
+ "shareToken"?: string | null;
7179
+ "lowSampleOverride": boolean;
7180
+ "publishedAt": string | null;
7181
+ "verdict": unknown;
7182
+ "definitionVersionId": string | null;
7183
+ "runnerKind": "ora-runs" | "batch-job";
7184
+ "createdAt": string;
7185
+ "updatedAt": string;
7186
+ "liveUntil": string | null;
7187
+ "arms"?: Array<{
7188
+ "key": string;
7189
+ "position": number;
7190
+ "label": string;
7191
+ "role": "reference" | "peer";
7192
+ "payload": {
7193
+ "tools"?: "default" | "none" | {
7194
+ "tools": Array<string>;
7195
+ };
7196
+ "addons"?: Record<string, unknown>;
7197
+ };
7198
+ "cells"?: Array<{
7199
+ "harness": string;
7200
+ "model": string;
7201
+ "provider"?: string;
7202
+ "params"?: {
7203
+ "temperature"?: number;
7204
+ "maxOutputTokens"?: number;
7205
+ "reasoningEffort"?: "low" | "medium" | "high";
7206
+ "thinkingBudgetTokens"?: number;
7207
+ };
7208
+ }>;
7209
+ "skills"?: Array<{
7210
+ "ref": string;
7211
+ "name": string;
7212
+ "changedSinceCreate"?: boolean;
7213
+ }>;
7214
+ }>;
7215
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7216
+ "cellPlan"?: "crossed" | "per_setup";
7217
+ "skillDisclosure"?: "withheld" | "named";
7218
+ "modeOptions"?: Record<string, unknown>;
7219
+ };
7220
+ "streamUrl": string;
7221
+ "warnings"?: Array<{
7222
+ "harness": string;
7223
+ "kind": string;
7224
+ } | {
7225
+ "type": "design_aggregate_budget";
7226
+ "detail": string;
7227
+ }>;
7228
+ };
7229
+ export type StudyUpdateLensesRequest = {
7230
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7231
+ };
7232
+ export type StudyUpdateLensesResponse = {
7233
+ "study": {
7234
+ "id": string;
7235
+ "executiveSummary"?: string | null;
7236
+ "plannedRunCount": number;
7237
+ "organizationId": string;
7238
+ "createdBy": string | null;
7239
+ "name": string;
7240
+ "slug": string;
7241
+ "description": string | null;
7242
+ "skillRef": string | null;
7243
+ "skillSlug": string | null;
7244
+ "skillName": string | null;
7245
+ "skillPromise": string;
7246
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
7247
+ "subjectName": string | null;
7248
+ "subjectLabel": string;
7249
+ "armConfig": {
7250
+ "baseline"?: {
7251
+ "tools"?: "default" | "none" | {
7252
+ "tools": Array<string>;
7253
+ };
7254
+ "addons"?: Record<string, unknown>;
7255
+ };
7256
+ "treatment"?: {
7257
+ "tools"?: "default" | "none" | {
7258
+ "tools": Array<string>;
7259
+ };
7260
+ "addons"?: Record<string, unknown>;
5306
7261
  };
5307
7262
  };
5308
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
7263
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5309
7264
  "subjectBackend": "model" | "harness" | null;
5310
7265
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5311
7266
  "importedFrom"?: string;
@@ -5341,6 +7296,7 @@ export type StudyUpdateLensesResponse = {
5341
7296
  "error": string | null;
5342
7297
  "preparation"?: {
5343
7298
  "since": string;
7299
+ "progressAt"?: string;
5344
7300
  "launchedBy": {
5345
7301
  "userId"?: string;
5346
7302
  "role"?: string;
@@ -5349,6 +7305,12 @@ export type StudyUpdateLensesResponse = {
5349
7305
  "imageKey": string;
5350
7306
  "buildId": string;
5351
7307
  "stage"?: "image" | "snapshot";
7308
+ "retry"?: {
7309
+ "failedAttempts": number;
7310
+ "maxAttempts": number;
7311
+ "notBefore": string;
7312
+ "lastFailure": string;
7313
+ };
5352
7314
  }>;
5353
7315
  } | null;
5354
7316
  "visibility": "private" | "org" | "link" | "public";
@@ -5363,6 +7325,38 @@ export type StudyUpdateLensesResponse = {
5363
7325
  "createdAt": string;
5364
7326
  "updatedAt": string;
5365
7327
  "liveUntil": string | null;
7328
+ "arms"?: Array<{
7329
+ "key": string;
7330
+ "position": number;
7331
+ "label": string;
7332
+ "role": "reference" | "peer";
7333
+ "payload": {
7334
+ "tools"?: "default" | "none" | {
7335
+ "tools": Array<string>;
7336
+ };
7337
+ "addons"?: Record<string, unknown>;
7338
+ };
7339
+ "cells"?: Array<{
7340
+ "harness": string;
7341
+ "model": string;
7342
+ "provider"?: string;
7343
+ "params"?: {
7344
+ "temperature"?: number;
7345
+ "maxOutputTokens"?: number;
7346
+ "reasoningEffort"?: "low" | "medium" | "high";
7347
+ "thinkingBudgetTokens"?: number;
7348
+ };
7349
+ }>;
7350
+ "skills"?: Array<{
7351
+ "ref": string;
7352
+ "name": string;
7353
+ "changedSinceCreate"?: boolean;
7354
+ }>;
7355
+ }>;
7356
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7357
+ "cellPlan"?: "crossed" | "per_setup";
7358
+ "skillDisclosure"?: "withheld" | "named";
7359
+ "modeOptions"?: Record<string, unknown>;
5366
7360
  };
5367
7361
  "warnings": Array<{
5368
7362
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
@@ -5396,7 +7390,7 @@ export type StudyPreviewResponse = {
5396
7390
  "addons"?: Array<string>;
5397
7391
  };
5398
7392
  };
5399
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
7393
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5400
7394
  "subjectBackend"?: "model" | "harness";
5401
7395
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5402
7396
  "baselineLabel": string | null;
@@ -5443,6 +7437,22 @@ export type StudyPreviewResponse = {
5443
7437
  "kind": string;
5444
7438
  "text": string;
5445
7439
  }>;
7440
+ "groupCount"?: number;
7441
+ "group"?: {
7442
+ "label": string;
7443
+ "composition": {
7444
+ "tools"?: "default" | "none" | {
7445
+ "tools": Array<string>;
7446
+ };
7447
+ "addons"?: Array<string>;
7448
+ };
7449
+ };
7450
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7451
+ "groupComposition"?: Array<{
7452
+ "key": string;
7453
+ "label": string;
7454
+ "skills": Array<string>;
7455
+ }>;
5446
7456
  };
5447
7457
  "tasks": Array<{
5448
7458
  "id": string;
@@ -5459,7 +7469,7 @@ export type StudyPreviewResponse = {
5459
7469
  "runs": Array<{
5460
7470
  "runId": string;
5461
7471
  "taskId": string;
5462
- "arm": "baseline" | "skill";
7472
+ "arm": "baseline" | "skill" | string;
5463
7473
  "harness": string;
5464
7474
  "model": string;
5465
7475
  "provider": string | null;
@@ -5485,6 +7495,9 @@ export type StudyPreviewResponse = {
5485
7495
  "pairGradedAt": string | null;
5486
7496
  "evalScore": number | null;
5487
7497
  "evalGradedAt": string | null;
7498
+ "executionBudgetS"?: number | null;
7499
+ "harnessImageDigest"?: string | null;
7500
+ "subjectEngaged"?: boolean | null;
5488
7501
  "runCreatedAt": string | null;
5489
7502
  "startedAt": string | null;
5490
7503
  "finishedAt": string | null;
@@ -5502,80 +7515,936 @@ export type StudyPreviewResponse = {
5502
7515
  }>;
5503
7516
  "aggregate": {
5504
7517
  "summary": {
5505
- "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
5506
- "headline": Array<{
5507
- "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
7518
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
7519
+ "headline": Array<{
7520
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
7521
+ "label": string;
7522
+ "direction": "down" | "up";
7523
+ "baselineAvg": number | null;
7524
+ "skillAvg": number | null;
7525
+ "deltaPct": number | null;
7526
+ "delta"?: {
7527
+ "unit": "pp";
7528
+ "absolute": number | null;
7529
+ "relativePct": number | null;
7530
+ } | {
7531
+ "unit": "pts";
7532
+ "absolute": number | null;
7533
+ "relativePct"?: number | null;
7534
+ } | {
7535
+ "unit": "%";
7536
+ "relativePct": number | null;
7537
+ };
7538
+ "improved": boolean;
7539
+ }>;
7540
+ "completionRate": {
7541
+ "baseline": number | null;
7542
+ "skill": number | null;
7543
+ };
7544
+ "goalRate": {
7545
+ "baseline": number | null;
7546
+ "skill": number | null;
7547
+ };
7548
+ "pairwise": {
7549
+ "pairs": number;
7550
+ "skillWins": number;
7551
+ "baselineWins": number;
7552
+ "ties": number;
7553
+ "winRate": number | null;
7554
+ };
7555
+ "decision"?: {
7556
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
7557
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
7558
+ "reason": string;
7559
+ "efficiencyDiscounted": boolean;
7560
+ };
7561
+ "leaderboard"?: Array<unknown>;
7562
+ "concentration": {
7563
+ "perCell": Array<{
7564
+ "cell": string;
7565
+ "goalDeltaPp": number | null;
7566
+ "contributionPct": number | null;
7567
+ }>;
7568
+ "heterogeneity": {
7569
+ "dominantCell": string;
7570
+ "contributionPct": number;
7571
+ } | null;
7572
+ };
7573
+ "matrixShape": {
7574
+ "cells": number;
7575
+ "harnesses": number;
7576
+ "models": number;
7577
+ "crossed": boolean;
7578
+ };
7579
+ "flakiness"?: {
7580
+ "measured": boolean;
7581
+ "conditions": number;
7582
+ "unstable": number;
7583
+ "worst": {
7584
+ "cell": string;
7585
+ "taskId": string;
7586
+ "arm": "baseline" | "skill";
7587
+ "variant"?: number;
7588
+ "passRate": number;
7589
+ } | null;
7590
+ };
7591
+ "arms"?: {
7592
+ "baseline": {
7593
+ "runs": number;
7594
+ "completed": number;
7595
+ "infraFailures": number;
7596
+ "completionRate": number | null;
7597
+ "goalRate": number | null;
7598
+ "goalTally": {
7599
+ "achieved": number;
7600
+ "partial": number;
7601
+ "missed": number;
7602
+ };
7603
+ "avgTotalTokens": number | null;
7604
+ "avgOutputTokens": number | null;
7605
+ "avgCostUsd": number | null;
7606
+ "avgDurationMs": number | null;
7607
+ "avgTurns": number | null;
7608
+ "costCoverage": number | null;
7609
+ "avgQualityScore": number | null;
7610
+ "gradedCoverage": number | null;
7611
+ "medianCostUsd": number | null;
7612
+ "medianTotalTokens": number | null;
7613
+ "medianDurationMs": number | null;
7614
+ "p95CostUsd": number | null;
7615
+ "p95TotalTokens": number | null;
7616
+ "p95DurationMs": number | null;
7617
+ "maxCostUsd": number | null;
7618
+ "maxTotalTokens": number | null;
7619
+ "maxDurationMs": number | null;
7620
+ "evalPassRate"?: number | null;
7621
+ "benchmarkPassRate"?: number | null;
7622
+ "adoptionRate"?: number | null;
7623
+ };
7624
+ "skill": {
7625
+ "runs": number;
7626
+ "completed": number;
7627
+ "infraFailures": number;
7628
+ "completionRate": number | null;
7629
+ "goalRate": number | null;
7630
+ "goalTally": {
7631
+ "achieved": number;
7632
+ "partial": number;
7633
+ "missed": number;
7634
+ };
7635
+ "avgTotalTokens": number | null;
7636
+ "avgOutputTokens": number | null;
7637
+ "avgCostUsd": number | null;
7638
+ "avgDurationMs": number | null;
7639
+ "avgTurns": number | null;
7640
+ "costCoverage": number | null;
7641
+ "avgQualityScore": number | null;
7642
+ "gradedCoverage": number | null;
7643
+ "medianCostUsd": number | null;
7644
+ "medianTotalTokens": number | null;
7645
+ "medianDurationMs": number | null;
7646
+ "p95CostUsd": number | null;
7647
+ "p95TotalTokens": number | null;
7648
+ "p95DurationMs": number | null;
7649
+ "maxCostUsd": number | null;
7650
+ "maxTotalTokens": number | null;
7651
+ "maxDurationMs": number | null;
7652
+ "evalPassRate"?: number | null;
7653
+ "benchmarkPassRate"?: number | null;
7654
+ "adoptionRate"?: number | null;
7655
+ };
7656
+ };
7657
+ "totals": {
7658
+ "cells": number;
7659
+ "runs": number;
7660
+ "completed": number;
7661
+ "failed": number;
7662
+ "costUsd": number;
7663
+ "tokens": number;
7664
+ };
7665
+ };
7666
+ "perCell": Array<unknown>;
7667
+ "perHarness": Array<unknown>;
7668
+ "perModel": Array<unknown>;
7669
+ "perTask": Array<unknown>;
7670
+ "webSplit": unknown;
7671
+ "inference": {
7672
+ "columns": Array<{
7673
+ "key": string;
7674
+ "label": string;
7675
+ "family": "rate" | "score" | "magnitude";
7676
+ "betterWhen": "higher" | "lower";
7677
+ }>;
7678
+ "groups": Array<{
7679
+ "kind": "cell" | "harness" | "model" | "field";
7680
+ "title": string;
7681
+ "rows": Array<{
7682
+ "key": string;
7683
+ "kind": "cell" | "harness" | "model" | "field";
7684
+ "harness": string | null;
7685
+ "model": string | null;
7686
+ "paramsKey"?: string;
7687
+ "runs": number;
7688
+ "cells": Record<string, unknown>;
7689
+ }>;
7690
+ }>;
7691
+ "reference": "baseline" | "field";
7692
+ "draws": number;
7693
+ } | null;
7694
+ "behavior"?: Array<{
7695
+ "harness": string;
7696
+ "model": string;
7697
+ "provider": string | null;
7698
+ "arm": string;
7699
+ "totalRuns": number;
7700
+ "signals": Array<{
7701
+ "id": string;
7702
+ "label": string;
7703
+ "tone": "good" | "bad" | "neutral";
7704
+ "runCount": number;
7705
+ "succeededCount": number;
7706
+ }>;
7707
+ }>;
7708
+ "reference"?: {
7709
+ "cells": Array<{
7710
+ "cell": string;
7711
+ "harness": string;
7712
+ "model": string;
7713
+ "provider": string | null;
7714
+ "paramsKey": string;
7715
+ "label": string;
7716
+ "passRate": {
7717
+ "value": number | null;
7718
+ "ci": Array<unknown> | null;
7719
+ "n": number;
7720
+ };
7721
+ "abstention"?: {
7722
+ "value": number | null;
7723
+ "ci": Array<unknown> | null;
7724
+ "n": number;
7725
+ "parts"?: Array<{
7726
+ "id": string;
7727
+ "label": string;
7728
+ "value": number | null;
7729
+ "n": number;
7730
+ }>;
7731
+ };
7732
+ "tasks": number;
7733
+ "graded": number;
7734
+ "unmeasured": number;
7735
+ "matchers": Array<string>;
7736
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
7737
+ }>;
7738
+ "overall": {
7739
+ "value": number | null;
7740
+ "ci": Array<unknown> | null;
7741
+ "n": number;
7742
+ };
7743
+ "abstention"?: {
7744
+ "value": number | null;
7745
+ "ci": Array<unknown> | null;
7746
+ "n": number;
7747
+ "parts"?: Array<{
7748
+ "id": string;
7749
+ "label": string;
7750
+ "value": number | null;
7751
+ "n": number;
7752
+ }>;
7753
+ };
7754
+ "graded": number;
7755
+ "unmeasured": number;
7756
+ "modelJudged": boolean;
7757
+ };
7758
+ "execution"?: {
7759
+ "graded": number;
7760
+ "measured": number;
7761
+ "passed": number;
7762
+ "passRate": number | null;
7763
+ "byStatus": {
7764
+ "passed": number;
7765
+ "failed": number;
7766
+ "timeout": number;
7767
+ "error": number;
7768
+ "unavailable": number;
7769
+ };
7770
+ "languages": Array<string>;
7771
+ "unavailableReasons": Array<string>;
7772
+ };
7773
+ } | {
7774
+ "shape": "multi-arm";
7775
+ "summary": {
7776
+ "shape": "multi-arm";
7777
+ "verdict": "measured" | "inconclusive";
7778
+ "groups": Array<{
7779
+ "key": string;
5508
7780
  "label": string;
5509
- "direction": "down" | "up";
5510
- "baselineAvg": number | null;
5511
- "skillAvg": number | null;
5512
- "deltaPct": number | null;
5513
- "delta"?: {
5514
- "unit": "pp";
5515
- "absolute": number | null;
5516
- "relativePct": number | null;
5517
- } | {
5518
- "unit": "pts";
5519
- "absolute": number | null;
5520
- "relativePct"?: number | null;
5521
- } | {
5522
- "unit": "%";
5523
- "relativePct": number | null;
7781
+ "role": "reference" | "peer";
7782
+ "composition": {
7783
+ "tools"?: "default" | "none" | {
7784
+ "tools": Array<string>;
7785
+ };
7786
+ "addons"?: Array<string>;
5524
7787
  };
5525
- "improved": boolean;
5526
7788
  }>;
5527
- "completionRate": {
5528
- "baseline": number | null;
5529
- "skill": number | null;
7789
+ "ranking": {
7790
+ "plan": "vs_reference" | "all_pairs" | null;
7791
+ "referenceArm": string | null;
7792
+ "decision": {
7793
+ "leader": string | null;
7794
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
7795
+ "reason": string;
7796
+ "separatedFrom": Array<string>;
7797
+ "lostTo": Array<{
7798
+ "arm": string;
7799
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
7800
+ }>;
7801
+ "clearLeader": boolean;
7802
+ "correctnessLeader": string | null;
7803
+ };
7804
+ "entries": Array<{
7805
+ "arm": string;
7806
+ "runs": number;
7807
+ "completed": number;
7808
+ "completionRate": number | null;
7809
+ "goalRate": number | null;
7810
+ "avgQualityScore": number | null;
7811
+ "evalPassRate": number | null;
7812
+ "benchmarkPassRate"?: number | null;
7813
+ "compositeScore": number | null;
7814
+ "avgTotalTokens": number | null;
7815
+ "avgOutputTokens": number | null;
7816
+ "avgCostUsd": number | null;
7817
+ "avgDurationMs": number | null;
7818
+ "medianCostUsd": number | null;
7819
+ "medianTotalTokens": number | null;
7820
+ "medianDurationMs": number | null;
7821
+ "rank": number;
7822
+ }>;
7823
+ "contrasts": Array<{
7824
+ "contrastId": string;
7825
+ "referenceArm": string;
7826
+ "subjectArm": string;
7827
+ "oriented": boolean;
7828
+ "outcome": "won" | "tie" | "unmeasurable";
7829
+ "winner": string | null;
7830
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
7831
+ "explanation": string;
7832
+ }>;
7833
+ "comparisons": number;
7834
+ "headline"?: string;
5530
7835
  };
5531
- "goalRate": {
5532
- "baseline": number | null;
5533
- "skill": number | null;
7836
+ "matrixShape": {
7837
+ "cells": number;
7838
+ "harnesses": number;
7839
+ "models": number;
7840
+ "crossed": boolean;
5534
7841
  };
5535
- "pairwise": {
5536
- "pairs": number;
5537
- "skillWins": number;
5538
- "baselineWins": number;
5539
- "ties": number;
5540
- "winRate": number | null;
7842
+ "flakiness": {
7843
+ "measured": boolean;
7844
+ "conditions": number;
7845
+ "unstable": number;
7846
+ "worst": {
7847
+ "cell": string;
7848
+ "taskId": string;
7849
+ "arm": string;
7850
+ "variant"?: number;
7851
+ "passRate": number;
7852
+ } | null;
5541
7853
  };
5542
- "decision"?: {
5543
- "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
5544
- "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5545
- "reason": string;
5546
- "efficiencyDiscounted": boolean;
7854
+ "totals": {
7855
+ "cells": number;
7856
+ "runs": number;
7857
+ "completed": number;
7858
+ "failed": number;
7859
+ "costUsd": number;
7860
+ "tokens": number;
5547
7861
  };
5548
- "leaderboard"?: Array<unknown>;
5549
- "concentration": {
5550
- "perCell": Array<{
7862
+ };
7863
+ "arms": Array<{
7864
+ "key": string;
7865
+ "label": string;
7866
+ "position": number;
7867
+ "role": "reference" | "peer";
7868
+ "composition": {
7869
+ "tools"?: "default" | "none" | {
7870
+ "tools": Array<string>;
7871
+ };
7872
+ "addons"?: Array<string>;
7873
+ };
7874
+ "agg": {
7875
+ "runs": number;
7876
+ "completed": number;
7877
+ "infraFailures": number;
7878
+ "completionRate": number | null;
7879
+ "goalRate": number | null;
7880
+ "goalTally": {
7881
+ "achieved": number;
7882
+ "partial": number;
7883
+ "missed": number;
7884
+ };
7885
+ "avgTotalTokens": number | null;
7886
+ "avgOutputTokens": number | null;
7887
+ "avgCostUsd": number | null;
7888
+ "avgDurationMs": number | null;
7889
+ "avgTurns": number | null;
7890
+ "costCoverage": number | null;
7891
+ "avgQualityScore": number | null;
7892
+ "gradedCoverage": number | null;
7893
+ "medianCostUsd": number | null;
7894
+ "medianTotalTokens": number | null;
7895
+ "medianDurationMs": number | null;
7896
+ "p95CostUsd": number | null;
7897
+ "p95TotalTokens": number | null;
7898
+ "p95DurationMs": number | null;
7899
+ "maxCostUsd": number | null;
7900
+ "maxTotalTokens": number | null;
7901
+ "maxDurationMs": number | null;
7902
+ "evalPassRate"?: number | null;
7903
+ "benchmarkPassRate"?: number | null;
7904
+ "adoptionRate"?: number | null;
7905
+ };
7906
+ "perCell": Array<{
7907
+ "harness": string;
7908
+ "model": string;
7909
+ "provider": string | null;
7910
+ "paramsKey"?: string;
7911
+ "label"?: string;
7912
+ "agg": {
7913
+ "runs": number;
7914
+ "completed": number;
7915
+ "infraFailures": number;
7916
+ "completionRate": number | null;
7917
+ "goalRate": number | null;
7918
+ "goalTally": {
7919
+ "achieved": number;
7920
+ "partial": number;
7921
+ "missed": number;
7922
+ };
7923
+ "avgTotalTokens": number | null;
7924
+ "avgOutputTokens": number | null;
7925
+ "avgCostUsd": number | null;
7926
+ "avgDurationMs": number | null;
7927
+ "avgTurns": number | null;
7928
+ "costCoverage": number | null;
7929
+ "avgQualityScore": number | null;
7930
+ "gradedCoverage": number | null;
7931
+ "medianCostUsd": number | null;
7932
+ "medianTotalTokens": number | null;
7933
+ "medianDurationMs": number | null;
7934
+ "p95CostUsd": number | null;
7935
+ "p95TotalTokens": number | null;
7936
+ "p95DurationMs": number | null;
7937
+ "maxCostUsd": number | null;
7938
+ "maxTotalTokens": number | null;
7939
+ "maxDurationMs": number | null;
7940
+ "evalPassRate"?: number | null;
7941
+ "benchmarkPassRate"?: number | null;
7942
+ "adoptionRate"?: number | null;
7943
+ };
7944
+ }>;
7945
+ "reference"?: {
7946
+ "cells": Array<{
5551
7947
  "cell": string;
5552
- "goalDeltaPp": number | null;
5553
- "contributionPct": number | null;
7948
+ "harness": string;
7949
+ "model": string;
7950
+ "provider": string | null;
7951
+ "paramsKey": string;
7952
+ "label": string;
7953
+ "passRate": {
7954
+ "value": number | null;
7955
+ "ci": Array<unknown> | null;
7956
+ "n": number;
7957
+ };
7958
+ "abstention"?: {
7959
+ "value": number | null;
7960
+ "ci": Array<unknown> | null;
7961
+ "n": number;
7962
+ "parts"?: Array<{
7963
+ "id": string;
7964
+ "label": string;
7965
+ "value": number | null;
7966
+ "n": number;
7967
+ }>;
7968
+ };
7969
+ "tasks": number;
7970
+ "graded": number;
7971
+ "unmeasured": number;
7972
+ "matchers": Array<string>;
7973
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
5554
7974
  }>;
5555
- "heterogeneity": {
5556
- "dominantCell": string;
5557
- "contributionPct": number;
7975
+ "overall": {
7976
+ "value": number | null;
7977
+ "ci": Array<unknown> | null;
7978
+ "n": number;
7979
+ };
7980
+ "abstention"?: {
7981
+ "value": number | null;
7982
+ "ci": Array<unknown> | null;
7983
+ "n": number;
7984
+ "parts"?: Array<{
7985
+ "id": string;
7986
+ "label": string;
7987
+ "value": number | null;
7988
+ "n": number;
7989
+ }>;
7990
+ };
7991
+ "graded": number;
7992
+ "unmeasured": number;
7993
+ "modelJudged": boolean;
7994
+ };
7995
+ }>;
7996
+ "contrasts": Array<{
7997
+ "contrastId": string;
7998
+ "referenceArm": string;
7999
+ "subjectArm": string;
8000
+ "oriented": boolean;
8001
+ "basis"?: {
8002
+ "id": "matched" | "whole_setup";
8003
+ "compared": {
8004
+ "reference": Array<{
8005
+ "harness": string;
8006
+ "model": string;
8007
+ "provider"?: string;
8008
+ "params"?: Record<string, unknown>;
8009
+ }>;
8010
+ "subject": Array<{
8011
+ "harness": string;
8012
+ "model": string;
8013
+ "provider"?: string;
8014
+ "params"?: Record<string, unknown>;
8015
+ }>;
8016
+ };
8017
+ "excluded": {
8018
+ "reference": Array<{
8019
+ "harness": string;
8020
+ "model": string;
8021
+ "provider"?: string;
8022
+ "params"?: Record<string, unknown>;
8023
+ }>;
8024
+ "subject": Array<{
8025
+ "harness": string;
8026
+ "model": string;
8027
+ "provider"?: string;
8028
+ "params"?: Record<string, unknown>;
8029
+ }>;
8030
+ };
8031
+ "differsIn": Array<string>;
8032
+ "licensedFacet": string | null;
8033
+ "efficiency": "comparable" | "incomparable";
8034
+ "pairedTasks": number | null;
8035
+ "dropped": {
8036
+ "reference": number;
8037
+ "subject": number;
8038
+ } | null;
8039
+ };
8040
+ "magnitudes": "relative";
8041
+ "summary": {
8042
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
8043
+ "headline": Array<{
8044
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
8045
+ "label": string;
8046
+ "direction": "down" | "up";
8047
+ "baselineAvg": number | null;
8048
+ "skillAvg": number | null;
8049
+ "deltaPct": number | null;
8050
+ "delta"?: {
8051
+ "unit": "pp";
8052
+ "absolute": number | null;
8053
+ "relativePct": number | null;
8054
+ } | {
8055
+ "unit": "pts";
8056
+ "absolute": number | null;
8057
+ "relativePct"?: number | null;
8058
+ } | {
8059
+ "unit": "%";
8060
+ "relativePct": number | null;
8061
+ };
8062
+ "improved": boolean;
8063
+ }>;
8064
+ "completionRate": {
8065
+ "baseline": number | null;
8066
+ "skill": number | null;
8067
+ };
8068
+ "goalRate": {
8069
+ "baseline": number | null;
8070
+ "skill": number | null;
8071
+ };
8072
+ "pairwise": {
8073
+ "pairs": number;
8074
+ "skillWins": number;
8075
+ "baselineWins": number;
8076
+ "ties": number;
8077
+ "winRate": number | null;
8078
+ };
8079
+ "decision"?: {
8080
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
8081
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8082
+ "reason": string;
8083
+ "efficiencyDiscounted": boolean;
8084
+ };
8085
+ "concentration": {
8086
+ "perCell": Array<{
8087
+ "cell": string;
8088
+ "goalDeltaPp": number | null;
8089
+ "contributionPct": number | null;
8090
+ }>;
8091
+ "heterogeneity": {
8092
+ "dominantCell": string;
8093
+ "contributionPct": number;
8094
+ } | null;
8095
+ };
8096
+ "matrixShape": {
8097
+ "cells": number;
8098
+ "harnesses": number;
8099
+ "models": number;
8100
+ "crossed": boolean;
8101
+ };
8102
+ "flakiness"?: {
8103
+ "measured": boolean;
8104
+ "conditions": number;
8105
+ "unstable": number;
8106
+ "worst": {
8107
+ "cell": string;
8108
+ "taskId": string;
8109
+ "arm": "baseline" | "skill";
8110
+ "variant"?: number;
8111
+ "passRate": number;
8112
+ } | null;
8113
+ };
8114
+ "arms"?: {
8115
+ "baseline": {
8116
+ "runs": number;
8117
+ "completed": number;
8118
+ "infraFailures": number;
8119
+ "completionRate": number | null;
8120
+ "goalRate": number | null;
8121
+ "goalTally": {
8122
+ "achieved": number;
8123
+ "partial": number;
8124
+ "missed": number;
8125
+ };
8126
+ "avgTotalTokens": number | null;
8127
+ "avgOutputTokens": number | null;
8128
+ "avgCostUsd": number | null;
8129
+ "avgDurationMs": number | null;
8130
+ "avgTurns": number | null;
8131
+ "costCoverage": number | null;
8132
+ "avgQualityScore": number | null;
8133
+ "gradedCoverage": number | null;
8134
+ "medianCostUsd": number | null;
8135
+ "medianTotalTokens": number | null;
8136
+ "medianDurationMs": number | null;
8137
+ "p95CostUsd": number | null;
8138
+ "p95TotalTokens": number | null;
8139
+ "p95DurationMs": number | null;
8140
+ "maxCostUsd": number | null;
8141
+ "maxTotalTokens": number | null;
8142
+ "maxDurationMs": number | null;
8143
+ "evalPassRate"?: number | null;
8144
+ "benchmarkPassRate"?: number | null;
8145
+ "adoptionRate"?: number | null;
8146
+ };
8147
+ "skill": {
8148
+ "runs": number;
8149
+ "completed": number;
8150
+ "infraFailures": number;
8151
+ "completionRate": number | null;
8152
+ "goalRate": number | null;
8153
+ "goalTally": {
8154
+ "achieved": number;
8155
+ "partial": number;
8156
+ "missed": number;
8157
+ };
8158
+ "avgTotalTokens": number | null;
8159
+ "avgOutputTokens": number | null;
8160
+ "avgCostUsd": number | null;
8161
+ "avgDurationMs": number | null;
8162
+ "avgTurns": number | null;
8163
+ "costCoverage": number | null;
8164
+ "avgQualityScore": number | null;
8165
+ "gradedCoverage": number | null;
8166
+ "medianCostUsd": number | null;
8167
+ "medianTotalTokens": number | null;
8168
+ "medianDurationMs": number | null;
8169
+ "p95CostUsd": number | null;
8170
+ "p95TotalTokens": number | null;
8171
+ "p95DurationMs": number | null;
8172
+ "maxCostUsd": number | null;
8173
+ "maxTotalTokens": number | null;
8174
+ "maxDurationMs": number | null;
8175
+ "evalPassRate"?: number | null;
8176
+ "benchmarkPassRate"?: number | null;
8177
+ "adoptionRate"?: number | null;
8178
+ };
8179
+ };
8180
+ "totals": {
8181
+ "cells": number;
8182
+ "runs": number;
8183
+ "completed": number;
8184
+ "failed": number;
8185
+ "costUsd": number;
8186
+ "tokens": number;
8187
+ };
8188
+ };
8189
+ "perCell": Array<{
8190
+ "harness": string;
8191
+ "model": string;
8192
+ "provider": string | null;
8193
+ "paramsKey"?: string;
8194
+ "label"?: string;
8195
+ "baseline": {
8196
+ "runs": number;
8197
+ "completed": number;
8198
+ "infraFailures": number;
8199
+ "completionRate": number | null;
8200
+ "goalRate": number | null;
8201
+ "goalTally": {
8202
+ "achieved": number;
8203
+ "partial": number;
8204
+ "missed": number;
8205
+ };
8206
+ "avgTotalTokens": number | null;
8207
+ "avgOutputTokens": number | null;
8208
+ "avgCostUsd": number | null;
8209
+ "avgDurationMs": number | null;
8210
+ "avgTurns": number | null;
8211
+ "costCoverage": number | null;
8212
+ "avgQualityScore": number | null;
8213
+ "gradedCoverage": number | null;
8214
+ "medianCostUsd": number | null;
8215
+ "medianTotalTokens": number | null;
8216
+ "medianDurationMs": number | null;
8217
+ "p95CostUsd": number | null;
8218
+ "p95TotalTokens": number | null;
8219
+ "p95DurationMs": number | null;
8220
+ "maxCostUsd": number | null;
8221
+ "maxTotalTokens": number | null;
8222
+ "maxDurationMs": number | null;
8223
+ "evalPassRate"?: number | null;
8224
+ "benchmarkPassRate"?: number | null;
8225
+ "adoptionRate"?: number | null;
8226
+ };
8227
+ "skill": {
8228
+ "runs": number;
8229
+ "completed": number;
8230
+ "infraFailures": number;
8231
+ "completionRate": number | null;
8232
+ "goalRate": number | null;
8233
+ "goalTally": {
8234
+ "achieved": number;
8235
+ "partial": number;
8236
+ "missed": number;
8237
+ };
8238
+ "avgTotalTokens": number | null;
8239
+ "avgOutputTokens": number | null;
8240
+ "avgCostUsd": number | null;
8241
+ "avgDurationMs": number | null;
8242
+ "avgTurns": number | null;
8243
+ "costCoverage": number | null;
8244
+ "avgQualityScore": number | null;
8245
+ "gradedCoverage": number | null;
8246
+ "medianCostUsd": number | null;
8247
+ "medianTotalTokens": number | null;
8248
+ "medianDurationMs": number | null;
8249
+ "p95CostUsd": number | null;
8250
+ "p95TotalTokens": number | null;
8251
+ "p95DurationMs": number | null;
8252
+ "maxCostUsd": number | null;
8253
+ "maxTotalTokens": number | null;
8254
+ "maxDurationMs": number | null;
8255
+ "evalPassRate"?: number | null;
8256
+ "benchmarkPassRate"?: number | null;
8257
+ "adoptionRate"?: number | null;
8258
+ };
8259
+ "deltas": {
8260
+ "totalTokensPct": number | null;
8261
+ "outputTokensPct": number | null;
8262
+ "costUsdPct": number | null;
8263
+ "durationMsPct": number | null;
8264
+ "numTurnsPct": number | null;
8265
+ "completionRatePp": number | null;
8266
+ "goalRatePp": number | null;
8267
+ "qualityScorePts": number | null;
8268
+ "evalPassRatePp"?: number | null;
8269
+ };
8270
+ }>;
8271
+ } | {
8272
+ "contrastId": string;
8273
+ "referenceArm": string;
8274
+ "subjectArm": string;
8275
+ "oriented": boolean;
8276
+ "basis"?: {
8277
+ "id": "matched" | "whole_setup";
8278
+ "compared": {
8279
+ "reference": Array<{
8280
+ "harness": string;
8281
+ "model": string;
8282
+ "provider"?: string;
8283
+ "params"?: Record<string, unknown>;
8284
+ }>;
8285
+ "subject": Array<{
8286
+ "harness": string;
8287
+ "model": string;
8288
+ "provider"?: string;
8289
+ "params"?: Record<string, unknown>;
8290
+ }>;
8291
+ };
8292
+ "excluded": {
8293
+ "reference": Array<{
8294
+ "harness": string;
8295
+ "model": string;
8296
+ "provider"?: string;
8297
+ "params"?: Record<string, unknown>;
8298
+ }>;
8299
+ "subject": Array<{
8300
+ "harness": string;
8301
+ "model": string;
8302
+ "provider"?: string;
8303
+ "params"?: Record<string, unknown>;
8304
+ }>;
8305
+ };
8306
+ "differsIn": Array<string>;
8307
+ "licensedFacet": string | null;
8308
+ "efficiency": "comparable" | "incomparable";
8309
+ "pairedTasks": number | null;
8310
+ "dropped": {
8311
+ "reference": number;
8312
+ "subject": number;
5558
8313
  } | null;
5559
8314
  };
5560
- "matrixShape": {
5561
- "cells": number;
5562
- "harnesses": number;
5563
- "models": number;
5564
- "crossed": boolean;
5565
- };
5566
- "flakiness"?: {
5567
- "measured": boolean;
5568
- "conditions": number;
5569
- "unstable": number;
5570
- "worst": {
5571
- "cell": string;
5572
- "taskId": string;
5573
- "arm": "baseline" | "skill";
5574
- "variant"?: number;
5575
- "passRate": number;
5576
- } | null;
8315
+ "magnitudes": "absolute";
8316
+ "summary": {
8317
+ "completionRate": {
8318
+ "baseline": number | null;
8319
+ "skill": number | null;
8320
+ };
8321
+ "goalRate": {
8322
+ "baseline": number | null;
8323
+ "skill": number | null;
8324
+ };
8325
+ "pairwise": {
8326
+ "pairs": number;
8327
+ "skillWins": number;
8328
+ "baselineWins": number;
8329
+ "ties": number;
8330
+ "winRate": number | null;
8331
+ };
8332
+ "decision"?: {
8333
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
8334
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8335
+ "reason": string;
8336
+ "efficiencyDiscounted": boolean;
8337
+ };
8338
+ "concentration": {
8339
+ "perCell": Array<{
8340
+ "cell": string;
8341
+ "goalDeltaPp": number | null;
8342
+ "contributionPct": number | null;
8343
+ }>;
8344
+ "heterogeneity": {
8345
+ "dominantCell": string;
8346
+ "contributionPct": number;
8347
+ } | null;
8348
+ };
8349
+ "matrixShape": {
8350
+ "cells": number;
8351
+ "harnesses": number;
8352
+ "models": number;
8353
+ "crossed": boolean;
8354
+ };
8355
+ "flakiness"?: {
8356
+ "measured": boolean;
8357
+ "conditions": number;
8358
+ "unstable": number;
8359
+ "worst": {
8360
+ "cell": string;
8361
+ "taskId": string;
8362
+ "arm": "baseline" | "skill";
8363
+ "variant"?: number;
8364
+ "passRate": number;
8365
+ } | null;
8366
+ };
8367
+ "arms"?: {
8368
+ "baseline": {
8369
+ "runs": number;
8370
+ "completed": number;
8371
+ "infraFailures": number;
8372
+ "completionRate": number | null;
8373
+ "goalRate": number | null;
8374
+ "goalTally": {
8375
+ "achieved": number;
8376
+ "partial": number;
8377
+ "missed": number;
8378
+ };
8379
+ "avgTotalTokens": number | null;
8380
+ "avgOutputTokens": number | null;
8381
+ "avgCostUsd": number | null;
8382
+ "avgDurationMs": number | null;
8383
+ "avgTurns": number | null;
8384
+ "costCoverage": number | null;
8385
+ "avgQualityScore": number | null;
8386
+ "gradedCoverage": number | null;
8387
+ "medianCostUsd": number | null;
8388
+ "medianTotalTokens": number | null;
8389
+ "medianDurationMs": number | null;
8390
+ "p95CostUsd": number | null;
8391
+ "p95TotalTokens": number | null;
8392
+ "p95DurationMs": number | null;
8393
+ "maxCostUsd": number | null;
8394
+ "maxTotalTokens": number | null;
8395
+ "maxDurationMs": number | null;
8396
+ "evalPassRate"?: number | null;
8397
+ "benchmarkPassRate"?: number | null;
8398
+ "adoptionRate"?: number | null;
8399
+ };
8400
+ "skill": {
8401
+ "runs": number;
8402
+ "completed": number;
8403
+ "infraFailures": number;
8404
+ "completionRate": number | null;
8405
+ "goalRate": number | null;
8406
+ "goalTally": {
8407
+ "achieved": number;
8408
+ "partial": number;
8409
+ "missed": number;
8410
+ };
8411
+ "avgTotalTokens": number | null;
8412
+ "avgOutputTokens": number | null;
8413
+ "avgCostUsd": number | null;
8414
+ "avgDurationMs": number | null;
8415
+ "avgTurns": number | null;
8416
+ "costCoverage": number | null;
8417
+ "avgQualityScore": number | null;
8418
+ "gradedCoverage": number | null;
8419
+ "medianCostUsd": number | null;
8420
+ "medianTotalTokens": number | null;
8421
+ "medianDurationMs": number | null;
8422
+ "p95CostUsd": number | null;
8423
+ "p95TotalTokens": number | null;
8424
+ "p95DurationMs": number | null;
8425
+ "maxCostUsd": number | null;
8426
+ "maxTotalTokens": number | null;
8427
+ "maxDurationMs": number | null;
8428
+ "evalPassRate"?: number | null;
8429
+ "benchmarkPassRate"?: number | null;
8430
+ "adoptionRate"?: number | null;
8431
+ };
8432
+ };
8433
+ "totals": {
8434
+ "cells": number;
8435
+ "runs": number;
8436
+ "completed": number;
8437
+ "failed": number;
8438
+ "costUsd": number;
8439
+ "tokens": number;
8440
+ };
5577
8441
  };
5578
- "arms"?: {
8442
+ "perCell": Array<{
8443
+ "harness": string;
8444
+ "model": string;
8445
+ "provider": string | null;
8446
+ "paramsKey"?: string;
8447
+ "label"?: string;
5579
8448
  "baseline": {
5580
8449
  "runs": number;
5581
8450
  "completed": number;
@@ -5640,44 +8509,73 @@ export type StudyPreviewResponse = {
5640
8509
  "benchmarkPassRate"?: number | null;
5641
8510
  "adoptionRate"?: number | null;
5642
8511
  };
5643
- };
5644
- "totals": {
5645
- "cells": number;
5646
- "runs": number;
5647
- "completed": number;
5648
- "failed": number;
5649
- "costUsd": number;
5650
- "tokens": number;
5651
- };
5652
- };
5653
- "perCell": Array<unknown>;
5654
- "perHarness": Array<unknown>;
5655
- "perModel": Array<unknown>;
5656
- "perTask": Array<unknown>;
5657
- "webSplit": unknown;
5658
- "inference": {
5659
- "columns": Array<{
5660
- "key": string;
5661
- "label": string;
5662
- "family": "rate" | "score" | "magnitude";
5663
- "betterWhen": "higher" | "lower";
5664
- }>;
5665
- "groups": Array<{
5666
- "kind": "cell" | "harness" | "model" | "field";
5667
- "title": string;
5668
- "rows": Array<{
5669
- "key": string;
5670
- "kind": "cell" | "harness" | "model" | "field";
5671
- "harness": string | null;
5672
- "model": string | null;
5673
- "paramsKey"?: string;
5674
- "runs": number;
5675
- "cells": Record<string, unknown>;
5676
- }>;
8512
+ "deltas": {
8513
+ "completionRatePp": number | null;
8514
+ "goalRatePp": number | null;
8515
+ "qualityScorePts": number | null;
8516
+ "evalPassRatePp"?: number | null;
8517
+ };
5677
8518
  }>;
5678
- "reference": "baseline" | "field";
5679
- "draws": number;
5680
- } | null;
8519
+ }>;
8520
+ "board": Array<{
8521
+ "harness": string;
8522
+ "model": string;
8523
+ "provider": string | null;
8524
+ "paramsKey"?: string;
8525
+ "label"?: string;
8526
+ "arm": string;
8527
+ "runs": number;
8528
+ "completed": number;
8529
+ "completionRate": number | null;
8530
+ "goalRate": number | null;
8531
+ "avgQualityScore": number | null;
8532
+ "evalPassRate": number | null;
8533
+ "benchmarkPassRate"?: number | null;
8534
+ "compositeScore": number | null;
8535
+ "avgTotalTokens": number | null;
8536
+ "avgOutputTokens": number | null;
8537
+ "avgCostUsd": number | null;
8538
+ "avgDurationMs": number | null;
8539
+ "medianCostUsd": number | null;
8540
+ "medianTotalTokens": number | null;
8541
+ "medianDurationMs": number | null;
8542
+ "rank": number;
8543
+ }>;
8544
+ "setupsBoard"?: Array<{
8545
+ "harness": string;
8546
+ "model": string;
8547
+ "provider": string | null;
8548
+ "paramsKey"?: string;
8549
+ "label"?: string;
8550
+ "arm": string;
8551
+ "runs": number;
8552
+ "completed": number;
8553
+ "completionRate": number | null;
8554
+ "goalRate": number | null;
8555
+ "avgQualityScore": number | null;
8556
+ "evalPassRate": number | null;
8557
+ "benchmarkPassRate"?: number | null;
8558
+ "compositeScore": number | null;
8559
+ "avgTotalTokens": number | null;
8560
+ "avgOutputTokens": number | null;
8561
+ "avgCostUsd": number | null;
8562
+ "avgDurationMs": number | null;
8563
+ "medianCostUsd": number | null;
8564
+ "medianTotalTokens": number | null;
8565
+ "medianDurationMs": number | null;
8566
+ "rank": number;
8567
+ "group": string;
8568
+ "agent": string;
8569
+ "setup": string;
8570
+ }>;
8571
+ "perTask": Array<{
8572
+ "taskId": string;
8573
+ "position": number;
8574
+ "promptPreview": string;
8575
+ "requiresWeb": boolean;
8576
+ "category": string | null;
8577
+ "byArm": Record<string, unknown>;
8578
+ }>;
5681
8579
  "behavior"?: Array<{
5682
8580
  "harness": string;
5683
8581
  "model": string;
@@ -5692,48 +8590,55 @@ export type StudyPreviewResponse = {
5692
8590
  "succeededCount": number;
5693
8591
  }>;
5694
8592
  }>;
5695
- "reference"?: {
5696
- "cells": Array<{
5697
- "cell": string;
5698
- "harness": string;
5699
- "model": string;
5700
- "provider": string | null;
5701
- "paramsKey": string;
5702
- "label": string;
5703
- "passRate": {
5704
- "value": number | null;
5705
- "ci": Array<unknown> | null;
5706
- "n": number;
5707
- };
5708
- "tasks": number;
5709
- "graded": number;
5710
- "unmeasured": number;
5711
- "matchers": Array<string>;
5712
- "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
8593
+ "execution"?: Record<string, unknown>;
8594
+ "inference"?: {
8595
+ "contrasts": Array<{
8596
+ "contrastId": string;
8597
+ "table": {
8598
+ "columns": Array<{
8599
+ "key": string;
8600
+ "label": string;
8601
+ "family": "rate" | "score" | "magnitude";
8602
+ "betterWhen": "higher" | "lower";
8603
+ }>;
8604
+ "groups": Array<{
8605
+ "kind": "cell" | "harness" | "model" | "field";
8606
+ "title": string;
8607
+ "rows": Array<{
8608
+ "key": string;
8609
+ "kind": "cell" | "harness" | "model" | "field";
8610
+ "harness": string | null;
8611
+ "model": string | null;
8612
+ "paramsKey"?: string;
8613
+ "runs": number;
8614
+ "cells": Record<string, unknown>;
8615
+ }>;
8616
+ }>;
8617
+ "reference": "baseline" | "field";
8618
+ "draws": number;
8619
+ "adjustment": {
8620
+ "scope": "planned-comparisons";
8621
+ "comparisons": number;
8622
+ };
8623
+ } | null;
5713
8624
  }>;
5714
- "overall": {
5715
- "value": number | null;
5716
- "ci": Array<unknown> | null;
5717
- "n": number;
8625
+ "holm": {
8626
+ "status": "complete";
8627
+ "ready": number;
8628
+ "planned": number;
8629
+ "adjustment": {
8630
+ "scope": "planned-comparisons";
8631
+ "comparisons": number;
8632
+ };
8633
+ } | {
8634
+ "status": "pending";
8635
+ "ready": number;
8636
+ "planned": number;
5718
8637
  };
5719
- "graded": number;
5720
- "unmeasured": number;
5721
- "modelJudged": boolean;
5722
- };
5723
- "execution"?: {
5724
- "graded": number;
5725
- "measured": number;
5726
- "passed": number;
5727
- "passRate": number | null;
5728
- "byStatus": {
5729
- "passed": number;
5730
- "failed": number;
5731
- "timeout": number;
5732
- "error": number;
5733
- "unavailable": number;
8638
+ "claims": {
8639
+ "perContrast": Record<string, unknown>;
8640
+ "ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
5734
8641
  };
5735
- "languages": Array<string>;
5736
- "unavailableReasons": Array<string>;
5737
8642
  };
5738
8643
  };
5739
8644
  "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
@@ -5741,6 +8646,7 @@ export type StudyPreviewResponse = {
5741
8646
  export type StudyPublishRequest = {
5742
8647
  "slug"?: string;
5743
8648
  "visibility"?: "private" | "org" | "link" | "public";
8649
+ "skillNames"?: "withheld" | "named";
5744
8650
  };
5745
8651
  export type StudyPublishResponse = {
5746
8652
  "study": {
@@ -5773,7 +8679,7 @@ export type StudyPublishResponse = {
5773
8679
  "addons"?: Record<string, unknown>;
5774
8680
  };
5775
8681
  };
5776
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
8682
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5777
8683
  "subjectBackend": "model" | "harness" | null;
5778
8684
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5779
8685
  "importedFrom"?: string;
@@ -5809,6 +8715,7 @@ export type StudyPublishResponse = {
5809
8715
  "error": string | null;
5810
8716
  "preparation"?: {
5811
8717
  "since": string;
8718
+ "progressAt"?: string;
5812
8719
  "launchedBy": {
5813
8720
  "userId"?: string;
5814
8721
  "role"?: string;
@@ -5817,6 +8724,12 @@ export type StudyPublishResponse = {
5817
8724
  "imageKey": string;
5818
8725
  "buildId": string;
5819
8726
  "stage"?: "image" | "snapshot";
8727
+ "retry"?: {
8728
+ "failedAttempts": number;
8729
+ "maxAttempts": number;
8730
+ "notBefore": string;
8731
+ "lastFailure": string;
8732
+ };
5820
8733
  }>;
5821
8734
  } | null;
5822
8735
  "visibility": "private" | "org" | "link" | "public";
@@ -5831,6 +8744,38 @@ export type StudyPublishResponse = {
5831
8744
  "createdAt": string;
5832
8745
  "updatedAt": string;
5833
8746
  "liveUntil": string | null;
8747
+ "arms"?: Array<{
8748
+ "key": string;
8749
+ "position": number;
8750
+ "label": string;
8751
+ "role": "reference" | "peer";
8752
+ "payload": {
8753
+ "tools"?: "default" | "none" | {
8754
+ "tools": Array<string>;
8755
+ };
8756
+ "addons"?: Record<string, unknown>;
8757
+ };
8758
+ "cells"?: Array<{
8759
+ "harness": string;
8760
+ "model": string;
8761
+ "provider"?: string;
8762
+ "params"?: {
8763
+ "temperature"?: number;
8764
+ "maxOutputTokens"?: number;
8765
+ "reasoningEffort"?: "low" | "medium" | "high";
8766
+ "thinkingBudgetTokens"?: number;
8767
+ };
8768
+ }>;
8769
+ "skills"?: Array<{
8770
+ "ref": string;
8771
+ "name": string;
8772
+ "changedSinceCreate"?: boolean;
8773
+ }>;
8774
+ }>;
8775
+ "contrastPlan"?: "vs_reference" | "all_pairs";
8776
+ "cellPlan"?: "crossed" | "per_setup";
8777
+ "skillDisclosure"?: "withheld" | "named";
8778
+ "modeOptions"?: Record<string, unknown>;
5834
8779
  };
5835
8780
  };
5836
8781
  export type StudyReconcileRunsResponse = {
@@ -5873,7 +8818,7 @@ export type StudyRegradeResponse = {
5873
8818
  "addons"?: Record<string, unknown>;
5874
8819
  };
5875
8820
  };
5876
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
8821
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5877
8822
  "subjectBackend": "model" | "harness" | null;
5878
8823
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5879
8824
  "importedFrom"?: string;
@@ -5909,6 +8854,7 @@ export type StudyRegradeResponse = {
5909
8854
  "error": string | null;
5910
8855
  "preparation"?: {
5911
8856
  "since": string;
8857
+ "progressAt"?: string;
5912
8858
  "launchedBy": {
5913
8859
  "userId"?: string;
5914
8860
  "role"?: string;
@@ -5917,6 +8863,12 @@ export type StudyRegradeResponse = {
5917
8863
  "imageKey": string;
5918
8864
  "buildId": string;
5919
8865
  "stage"?: "image" | "snapshot";
8866
+ "retry"?: {
8867
+ "failedAttempts": number;
8868
+ "maxAttempts": number;
8869
+ "notBefore": string;
8870
+ "lastFailure": string;
8871
+ };
5920
8872
  }>;
5921
8873
  } | null;
5922
8874
  "visibility": "private" | "org" | "link" | "public";
@@ -5931,11 +8883,46 @@ export type StudyRegradeResponse = {
5931
8883
  "createdAt": string;
5932
8884
  "updatedAt": string;
5933
8885
  "liveUntil": string | null;
8886
+ "arms"?: Array<{
8887
+ "key": string;
8888
+ "position": number;
8889
+ "label": string;
8890
+ "role": "reference" | "peer";
8891
+ "payload": {
8892
+ "tools"?: "default" | "none" | {
8893
+ "tools": Array<string>;
8894
+ };
8895
+ "addons"?: Record<string, unknown>;
8896
+ };
8897
+ "cells"?: Array<{
8898
+ "harness": string;
8899
+ "model": string;
8900
+ "provider"?: string;
8901
+ "params"?: {
8902
+ "temperature"?: number;
8903
+ "maxOutputTokens"?: number;
8904
+ "reasoningEffort"?: "low" | "medium" | "high";
8905
+ "thinkingBudgetTokens"?: number;
8906
+ };
8907
+ }>;
8908
+ "skills"?: Array<{
8909
+ "ref": string;
8910
+ "name": string;
8911
+ "changedSinceCreate"?: boolean;
8912
+ }>;
8913
+ }>;
8914
+ "contrastPlan"?: "vs_reference" | "all_pairs";
8915
+ "cellPlan"?: "crossed" | "per_setup";
8916
+ "skillDisclosure"?: "withheld" | "named";
8917
+ "modeOptions"?: Record<string, unknown>;
5934
8918
  };
5935
8919
  "streamUrl": string;
5936
8920
  "warnings"?: Array<{
5937
8921
  "harness": string;
5938
8922
  "kind": string;
8923
+ } | {
8924
+ "type": "design_aggregate_budget";
8925
+ "detail": string;
5939
8926
  }>;
5940
8927
  };
5941
8928
  export type StudyListRunAnswersResponse = {
@@ -5984,7 +8971,7 @@ export type StudyRotateShareTokenResponse = {
5984
8971
  "addons"?: Record<string, unknown>;
5985
8972
  };
5986
8973
  };
5987
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
8974
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5988
8975
  "subjectBackend": "model" | "harness" | null;
5989
8976
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5990
8977
  "importedFrom"?: string;
@@ -6020,6 +9007,7 @@ export type StudyRotateShareTokenResponse = {
6020
9007
  "error": string | null;
6021
9008
  "preparation"?: {
6022
9009
  "since": string;
9010
+ "progressAt"?: string;
6023
9011
  "launchedBy": {
6024
9012
  "userId"?: string;
6025
9013
  "role"?: string;
@@ -6028,6 +9016,12 @@ export type StudyRotateShareTokenResponse = {
6028
9016
  "imageKey": string;
6029
9017
  "buildId": string;
6030
9018
  "stage"?: "image" | "snapshot";
9019
+ "retry"?: {
9020
+ "failedAttempts": number;
9021
+ "maxAttempts": number;
9022
+ "notBefore": string;
9023
+ "lastFailure": string;
9024
+ };
6031
9025
  }>;
6032
9026
  } | null;
6033
9027
  "visibility": "private" | "org" | "link" | "public";
@@ -6042,13 +9036,113 @@ export type StudyRotateShareTokenResponse = {
6042
9036
  "createdAt": string;
6043
9037
  "updatedAt": string;
6044
9038
  "liveUntil": string | null;
9039
+ "arms"?: Array<{
9040
+ "key": string;
9041
+ "position": number;
9042
+ "label": string;
9043
+ "role": "reference" | "peer";
9044
+ "payload": {
9045
+ "tools"?: "default" | "none" | {
9046
+ "tools": Array<string>;
9047
+ };
9048
+ "addons"?: Record<string, unknown>;
9049
+ };
9050
+ "cells"?: Array<{
9051
+ "harness": string;
9052
+ "model": string;
9053
+ "provider"?: string;
9054
+ "params"?: {
9055
+ "temperature"?: number;
9056
+ "maxOutputTokens"?: number;
9057
+ "reasoningEffort"?: "low" | "medium" | "high";
9058
+ "thinkingBudgetTokens"?: number;
9059
+ };
9060
+ }>;
9061
+ "skills"?: Array<{
9062
+ "ref": string;
9063
+ "name": string;
9064
+ "changedSinceCreate"?: boolean;
9065
+ }>;
9066
+ }>;
9067
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9068
+ "cellPlan"?: "crossed" | "per_setup";
9069
+ "skillDisclosure"?: "withheld" | "named";
9070
+ "modeOptions"?: Record<string, unknown>;
6045
9071
  };
6046
9072
  "shareUrl": string | null;
6047
9073
  };
6048
9074
  export type StudyStreamResponse = unknown;
6049
9075
  export type StudyUpdateTaskRequest = {
6050
9076
  "prompt": string;
6051
- "expectations"?: Array<unknown>;
9077
+ "expectations"?: Array<{
9078
+ "label"?: string;
9079
+ "arms"?: Array<"baseline" | "skill" | string>;
9080
+ "kind": "mcp_tool_called";
9081
+ "server"?: string;
9082
+ "tool": string;
9083
+ "arguments"?: Record<string, unknown>;
9084
+ "minCalls"?: number;
9085
+ } | {
9086
+ "label"?: string;
9087
+ "arms"?: Array<"baseline" | "skill" | string>;
9088
+ "kind": "tool_not_called";
9089
+ "tool"?: string;
9090
+ "mcp"?: {
9091
+ "server"?: string;
9092
+ "tool": string;
9093
+ };
9094
+ "pattern"?: string;
9095
+ } | {
9096
+ "label"?: string;
9097
+ "arms"?: Array<"baseline" | "skill" | string>;
9098
+ "kind": "command_ran";
9099
+ "pattern": string;
9100
+ } | {
9101
+ "label"?: string;
9102
+ "arms"?: Array<"baseline" | "skill" | string>;
9103
+ "kind": "skill_loaded";
9104
+ "slug": string;
9105
+ } | {
9106
+ "label"?: string;
9107
+ "arms"?: Array<"baseline" | "skill" | string>;
9108
+ "kind": "file_operation";
9109
+ "op"?: "read" | "write" | "edit";
9110
+ "path": unknown;
9111
+ } | {
9112
+ "label"?: string;
9113
+ "arms"?: Array<"baseline" | "skill" | string>;
9114
+ "kind": "web_fetched";
9115
+ "urlHost"?: string;
9116
+ "urlPattern"?: string;
9117
+ } | {
9118
+ "label"?: string;
9119
+ "arms"?: Array<"baseline" | "skill" | string>;
9120
+ "kind": "response_contains";
9121
+ "text": string;
9122
+ "normalize"?: "prose";
9123
+ } | {
9124
+ "label"?: string;
9125
+ "arms"?: Array<"baseline" | "skill" | string>;
9126
+ "kind": "response_matches";
9127
+ "pattern": string;
9128
+ "ignoreCase"?: boolean;
9129
+ } | {
9130
+ "label"?: string;
9131
+ "arms"?: Array<"baseline" | "skill" | string>;
9132
+ "kind": "token_budget";
9133
+ "maxOutputTokens"?: number;
9134
+ "maxTotalTokens"?: number;
9135
+ } | {
9136
+ "label"?: string;
9137
+ "arms"?: Array<"baseline" | "skill" | string>;
9138
+ "kind": "sequence";
9139
+ "of": Array<unknown>;
9140
+ } | {
9141
+ "label"?: string;
9142
+ "arms"?: Array<"baseline" | "skill" | string>;
9143
+ "kind": "any_of";
9144
+ "of": Array<unknown>;
9145
+ }>;
6052
9146
  };
6053
9147
  export type StudyUpdateTaskResponse = {
6054
9148
  "task": {
@@ -6095,7 +9189,7 @@ export type StudyUnpublishResponse = {
6095
9189
  "addons"?: Record<string, unknown>;
6096
9190
  };
6097
9191
  };
6098
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
9192
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6099
9193
  "subjectBackend": "model" | "harness" | null;
6100
9194
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6101
9195
  "importedFrom"?: string;
@@ -6131,6 +9225,7 @@ export type StudyUnpublishResponse = {
6131
9225
  "error": string | null;
6132
9226
  "preparation"?: {
6133
9227
  "since": string;
9228
+ "progressAt"?: string;
6134
9229
  "launchedBy": {
6135
9230
  "userId"?: string;
6136
9231
  "role"?: string;
@@ -6139,6 +9234,12 @@ export type StudyUnpublishResponse = {
6139
9234
  "imageKey": string;
6140
9235
  "buildId": string;
6141
9236
  "stage"?: "image" | "snapshot";
9237
+ "retry"?: {
9238
+ "failedAttempts": number;
9239
+ "maxAttempts": number;
9240
+ "notBefore": string;
9241
+ "lastFailure": string;
9242
+ };
6142
9243
  }>;
6143
9244
  } | null;
6144
9245
  "visibility": "private" | "org" | "link" | "public";
@@ -6153,6 +9254,38 @@ export type StudyUnpublishResponse = {
6153
9254
  "createdAt": string;
6154
9255
  "updatedAt": string;
6155
9256
  "liveUntil": string | null;
9257
+ "arms"?: Array<{
9258
+ "key": string;
9259
+ "position": number;
9260
+ "label": string;
9261
+ "role": "reference" | "peer";
9262
+ "payload": {
9263
+ "tools"?: "default" | "none" | {
9264
+ "tools": Array<string>;
9265
+ };
9266
+ "addons"?: Record<string, unknown>;
9267
+ };
9268
+ "cells"?: Array<{
9269
+ "harness": string;
9270
+ "model": string;
9271
+ "provider"?: string;
9272
+ "params"?: {
9273
+ "temperature"?: number;
9274
+ "maxOutputTokens"?: number;
9275
+ "reasoningEffort"?: "low" | "medium" | "high";
9276
+ "thinkingBudgetTokens"?: number;
9277
+ };
9278
+ }>;
9279
+ "skills"?: Array<{
9280
+ "ref": string;
9281
+ "name": string;
9282
+ "changedSinceCreate"?: boolean;
9283
+ }>;
9284
+ }>;
9285
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9286
+ "cellPlan"?: "crossed" | "per_setup";
9287
+ "skillDisclosure"?: "withheld" | "named";
9288
+ "modeOptions"?: Record<string, unknown>;
6156
9289
  };
6157
9290
  };
6158
9291
  export type StudySetVisibilityRequest = {
@@ -6189,7 +9322,7 @@ export type StudySetVisibilityResponse = {
6189
9322
  "addons"?: Record<string, unknown>;
6190
9323
  };
6191
9324
  };
6192
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
9325
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6193
9326
  "subjectBackend": "model" | "harness" | null;
6194
9327
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6195
9328
  "importedFrom"?: string;
@@ -6225,6 +9358,7 @@ export type StudySetVisibilityResponse = {
6225
9358
  "error": string | null;
6226
9359
  "preparation"?: {
6227
9360
  "since": string;
9361
+ "progressAt"?: string;
6228
9362
  "launchedBy": {
6229
9363
  "userId"?: string;
6230
9364
  "role"?: string;
@@ -6233,6 +9367,12 @@ export type StudySetVisibilityResponse = {
6233
9367
  "imageKey": string;
6234
9368
  "buildId": string;
6235
9369
  "stage"?: "image" | "snapshot";
9370
+ "retry"?: {
9371
+ "failedAttempts": number;
9372
+ "maxAttempts": number;
9373
+ "notBefore": string;
9374
+ "lastFailure": string;
9375
+ };
6236
9376
  }>;
6237
9377
  } | null;
6238
9378
  "visibility": "private" | "org" | "link" | "public";
@@ -6247,6 +9387,38 @@ export type StudySetVisibilityResponse = {
6247
9387
  "createdAt": string;
6248
9388
  "updatedAt": string;
6249
9389
  "liveUntil": string | null;
9390
+ "arms"?: Array<{
9391
+ "key": string;
9392
+ "position": number;
9393
+ "label": string;
9394
+ "role": "reference" | "peer";
9395
+ "payload": {
9396
+ "tools"?: "default" | "none" | {
9397
+ "tools": Array<string>;
9398
+ };
9399
+ "addons"?: Record<string, unknown>;
9400
+ };
9401
+ "cells"?: Array<{
9402
+ "harness": string;
9403
+ "model": string;
9404
+ "provider"?: string;
9405
+ "params"?: {
9406
+ "temperature"?: number;
9407
+ "maxOutputTokens"?: number;
9408
+ "reasoningEffort"?: "low" | "medium" | "high";
9409
+ "thinkingBudgetTokens"?: number;
9410
+ };
9411
+ }>;
9412
+ "skills"?: Array<{
9413
+ "ref": string;
9414
+ "name": string;
9415
+ "changedSinceCreate"?: boolean;
9416
+ }>;
9417
+ }>;
9418
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9419
+ "cellPlan"?: "crossed" | "per_setup";
9420
+ "skillDisclosure"?: "withheld" | "named";
9421
+ "modeOptions"?: Record<string, unknown>;
6250
9422
  };
6251
9423
  "shareUrl": string | null;
6252
9424
  };