@ora-ai/cli 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -84,6 +84,7 @@ export type BenchmarksListResponse = {
84
84
  "error": string | null;
85
85
  "preparation"?: {
86
86
  "since": string;
87
+ "progressAt"?: string;
87
88
  "launchedBy": {
88
89
  "userId"?: string;
89
90
  "role"?: string;
@@ -92,6 +93,12 @@ export type BenchmarksListResponse = {
92
93
  "imageKey": string;
93
94
  "buildId": string;
94
95
  "stage"?: "image" | "snapshot";
96
+ "retry"?: {
97
+ "failedAttempts": number;
98
+ "maxAttempts": number;
99
+ "notBefore": string;
100
+ "lastFailure": string;
101
+ };
95
102
  }>;
96
103
  } | null;
97
104
  "visibility": "private" | "org" | "link" | "public";
@@ -232,6 +239,7 @@ export type BenchmarksCreateResponse = {
232
239
  "error": string | null;
233
240
  "preparation"?: {
234
241
  "since": string;
242
+ "progressAt"?: string;
235
243
  "launchedBy": {
236
244
  "userId"?: string;
237
245
  "role"?: string;
@@ -240,6 +248,12 @@ export type BenchmarksCreateResponse = {
240
248
  "imageKey": string;
241
249
  "buildId": string;
242
250
  "stage"?: "image" | "snapshot";
251
+ "retry"?: {
252
+ "failedAttempts": number;
253
+ "maxAttempts": number;
254
+ "notBefore": string;
255
+ "lastFailure": string;
256
+ };
243
257
  }>;
244
258
  } | null;
245
259
  "visibility": "private" | "org" | "link" | "public";
@@ -340,6 +354,7 @@ export type BenchmarksGetResponse = {
340
354
  "error": string | null;
341
355
  "preparation"?: {
342
356
  "since": string;
357
+ "progressAt"?: string;
343
358
  "launchedBy": {
344
359
  "userId"?: string;
345
360
  "role"?: string;
@@ -348,6 +363,12 @@ export type BenchmarksGetResponse = {
348
363
  "imageKey": string;
349
364
  "buildId": string;
350
365
  "stage"?: "image" | "snapshot";
366
+ "retry"?: {
367
+ "failedAttempts": number;
368
+ "maxAttempts": number;
369
+ "notBefore": string;
370
+ "lastFailure": string;
371
+ };
351
372
  }>;
352
373
  } | null;
353
374
  "visibility": "private" | "org" | "link" | "public";
@@ -474,6 +495,9 @@ export type BenchmarksGetResponse = {
474
495
  "gradedAt": string;
475
496
  };
476
497
  } | null;
498
+ "executionBudgetS"?: number | null;
499
+ "harnessImageDigest"?: string | null;
500
+ "subjectEngaged"?: boolean | null;
477
501
  "runCreatedAt": string | null;
478
502
  "startedAt": string | null;
479
503
  "finishedAt": string | null;
@@ -683,6 +707,17 @@ export type BenchmarksGetResponse = {
683
707
  "ci": Array<unknown> | null;
684
708
  "n": number;
685
709
  };
710
+ "abstention"?: {
711
+ "value": number | null;
712
+ "ci": Array<unknown> | null;
713
+ "n": number;
714
+ "parts"?: Array<{
715
+ "id": string;
716
+ "label": string;
717
+ "value": number | null;
718
+ "n": number;
719
+ }>;
720
+ };
686
721
  "tasks": number;
687
722
  "graded": number;
688
723
  "unmeasured": number;
@@ -694,6 +729,17 @@ export type BenchmarksGetResponse = {
694
729
  "ci": Array<unknown> | null;
695
730
  "n": number;
696
731
  };
732
+ "abstention"?: {
733
+ "value": number | null;
734
+ "ci": Array<unknown> | null;
735
+ "n": number;
736
+ "parts"?: Array<{
737
+ "id": string;
738
+ "label": string;
739
+ "value": number | null;
740
+ "n": number;
741
+ }>;
742
+ };
697
743
  "graded": number;
698
744
  "unmeasured": number;
699
745
  "modelJudged": boolean;
@@ -723,9 +769,130 @@ export type BenchmarksGetResponse = {
723
769
  "detail": string;
724
770
  "taskIds"?: Array<string>;
725
771
  }>;
772
+ "taskHealth": {
773
+ "flags": Array<{
774
+ "taskId": string;
775
+ "signal": string;
776
+ "kind": "floor" | "ceiling";
777
+ "conditions": number;
778
+ }>;
779
+ "rubricSplits": Array<{
780
+ "taskId": string;
781
+ "criterionIndex": number;
782
+ "criterion": string;
783
+ "split": number;
784
+ "graded": number;
785
+ }>;
786
+ "signals": Array<{
787
+ "id": string;
788
+ "label": string;
789
+ }>;
790
+ };
726
791
  "shareUrl"?: string | null;
727
792
  "canManageSharing"?: boolean;
728
793
  };
794
+ export type BenchmarksUpdateDetailsRequest = {
795
+ "name"?: string;
796
+ "description"?: string;
797
+ };
798
+ export type BenchmarksUpdateDetailsResponse = {
799
+ "study": {
800
+ "id": string;
801
+ "executiveSummary"?: string | null;
802
+ "plannedRunCount": number;
803
+ "organizationId": string;
804
+ "createdBy": string | null;
805
+ "name": string;
806
+ "slug": string;
807
+ "description": string | null;
808
+ "skillRef": string | null;
809
+ "skillSlug": string | null;
810
+ "skillName": string | null;
811
+ "skillPromise": string;
812
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
813
+ "subjectName": string | null;
814
+ "subjectLabel": string;
815
+ "armConfig": {
816
+ "baseline"?: {
817
+ "tools"?: "default" | "none" | {
818
+ "tools": Array<string>;
819
+ };
820
+ "addons"?: Record<string, unknown>;
821
+ };
822
+ "treatment"?: {
823
+ "tools"?: "default" | "none" | {
824
+ "tools": Array<string>;
825
+ };
826
+ "addons"?: Record<string, unknown>;
827
+ };
828
+ };
829
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
830
+ "subjectBackend": "model" | "harness" | null;
831
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
832
+ "importedFrom"?: string;
833
+ "baselineLabel": string | null;
834
+ "treatmentLabel": string | null;
835
+ "promiseMetrics": Array<{
836
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
837
+ "direction": "down" | "up";
838
+ "label": string;
839
+ }>;
840
+ "matrix": Array<{
841
+ "harness": string;
842
+ "model": string;
843
+ "provider"?: string;
844
+ "params"?: {
845
+ "temperature"?: number;
846
+ "maxOutputTokens"?: number;
847
+ "reasoningEffort"?: "low" | "medium" | "high";
848
+ "thinkingBudgetTokens"?: number;
849
+ };
850
+ }>;
851
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
852
+ "judgeSupplementary"?: boolean;
853
+ "taskCount": number;
854
+ "variantsPerTask": number;
855
+ "repeats": number;
856
+ "subset": {
857
+ "count": number;
858
+ "seed": string;
859
+ "of": number;
860
+ } | null;
861
+ "status": string;
862
+ "error": string | null;
863
+ "preparation"?: {
864
+ "since": string;
865
+ "progressAt"?: string;
866
+ "launchedBy": {
867
+ "userId"?: string;
868
+ "role"?: string;
869
+ };
870
+ "waitingOn": Array<{
871
+ "imageKey": string;
872
+ "buildId": string;
873
+ "stage"?: "image" | "snapshot";
874
+ "retry"?: {
875
+ "failedAttempts": number;
876
+ "maxAttempts": number;
877
+ "notBefore": string;
878
+ "lastFailure": string;
879
+ };
880
+ }>;
881
+ } | null;
882
+ "visibility": "private" | "org" | "link" | "public";
883
+ "visibilityChangedAt"?: string | null;
884
+ "visibilityChangedBy"?: string | null;
885
+ "shareToken"?: string | null;
886
+ "lowSampleOverride": boolean;
887
+ "publishedAt": string | null;
888
+ "verdict": unknown;
889
+ "definitionVersionId": string | null;
890
+ "runnerKind": "ora-runs" | "batch-job";
891
+ "createdAt": string;
892
+ "updatedAt": string;
893
+ "liveUntil": string | null;
894
+ };
895
+ };
729
896
  export type BenchmarksDeleteResponse = unknown;
730
897
  export type BenchmarksRegenerateSummaryResponse = {
731
898
  "status": "queued";
@@ -800,6 +967,7 @@ export type BenchmarksGenerateResponse = {
800
967
  "error": string | null;
801
968
  "preparation"?: {
802
969
  "since": string;
970
+ "progressAt"?: string;
803
971
  "launchedBy": {
804
972
  "userId"?: string;
805
973
  "role"?: string;
@@ -808,6 +976,12 @@ export type BenchmarksGenerateResponse = {
808
976
  "imageKey": string;
809
977
  "buildId": string;
810
978
  "stage"?: "image" | "snapshot";
979
+ "retry"?: {
980
+ "failedAttempts": number;
981
+ "maxAttempts": number;
982
+ "notBefore": string;
983
+ "lastFailure": string;
984
+ };
811
985
  }>;
812
986
  } | null;
813
987
  "visibility": "private" | "org" | "link" | "public";
@@ -896,6 +1070,7 @@ export type BenchmarksLaunchResponse = {
896
1070
  "error": string | null;
897
1071
  "preparation"?: {
898
1072
  "since": string;
1073
+ "progressAt"?: string;
899
1074
  "launchedBy": {
900
1075
  "userId"?: string;
901
1076
  "role"?: string;
@@ -904,6 +1079,12 @@ export type BenchmarksLaunchResponse = {
904
1079
  "imageKey": string;
905
1080
  "buildId": string;
906
1081
  "stage"?: "image" | "snapshot";
1082
+ "retry"?: {
1083
+ "failedAttempts": number;
1084
+ "maxAttempts": number;
1085
+ "notBefore": string;
1086
+ "lastFailure": string;
1087
+ };
907
1088
  }>;
908
1089
  } | null;
909
1090
  "visibility": "private" | "org" | "link" | "public";
@@ -995,6 +1176,7 @@ export type BenchmarksUpdateLensesResponse = {
995
1176
  "error": string | null;
996
1177
  "preparation"?: {
997
1178
  "since": string;
1179
+ "progressAt"?: string;
998
1180
  "launchedBy": {
999
1181
  "userId"?: string;
1000
1182
  "role"?: string;
@@ -1003,6 +1185,12 @@ export type BenchmarksUpdateLensesResponse = {
1003
1185
  "imageKey": string;
1004
1186
  "buildId": string;
1005
1187
  "stage"?: "image" | "snapshot";
1188
+ "retry"?: {
1189
+ "failedAttempts": number;
1190
+ "maxAttempts": number;
1191
+ "notBefore": string;
1192
+ "lastFailure": string;
1193
+ };
1006
1194
  }>;
1007
1195
  } | null;
1008
1196
  "visibility": "private" | "org" | "link" | "public";
@@ -1139,6 +1327,9 @@ export type BenchmarksPreviewResponse = {
1139
1327
  "pairGradedAt": string | null;
1140
1328
  "evalScore": number | null;
1141
1329
  "evalGradedAt": string | null;
1330
+ "executionBudgetS"?: number | null;
1331
+ "harnessImageDigest"?: string | null;
1332
+ "subjectEngaged"?: boolean | null;
1142
1333
  "runCreatedAt": string | null;
1143
1334
  "startedAt": string | null;
1144
1335
  "finishedAt": string | null;
@@ -1359,6 +1550,17 @@ export type BenchmarksPreviewResponse = {
1359
1550
  "ci": Array<unknown> | null;
1360
1551
  "n": number;
1361
1552
  };
1553
+ "abstention"?: {
1554
+ "value": number | null;
1555
+ "ci": Array<unknown> | null;
1556
+ "n": number;
1557
+ "parts"?: Array<{
1558
+ "id": string;
1559
+ "label": string;
1560
+ "value": number | null;
1561
+ "n": number;
1562
+ }>;
1563
+ };
1362
1564
  "tasks": number;
1363
1565
  "graded": number;
1364
1566
  "unmeasured": number;
@@ -1370,6 +1572,17 @@ export type BenchmarksPreviewResponse = {
1370
1572
  "ci": Array<unknown> | null;
1371
1573
  "n": number;
1372
1574
  };
1575
+ "abstention"?: {
1576
+ "value": number | null;
1577
+ "ci": Array<unknown> | null;
1578
+ "n": number;
1579
+ "parts"?: Array<{
1580
+ "id": string;
1581
+ "label": string;
1582
+ "value": number | null;
1583
+ "n": number;
1584
+ }>;
1585
+ };
1373
1586
  "graded": number;
1374
1587
  "unmeasured": number;
1375
1588
  "modelJudged": boolean;
@@ -1463,6 +1676,7 @@ export type BenchmarksPublishResponse = {
1463
1676
  "error": string | null;
1464
1677
  "preparation"?: {
1465
1678
  "since": string;
1679
+ "progressAt"?: string;
1466
1680
  "launchedBy": {
1467
1681
  "userId"?: string;
1468
1682
  "role"?: string;
@@ -1471,6 +1685,12 @@ export type BenchmarksPublishResponse = {
1471
1685
  "imageKey": string;
1472
1686
  "buildId": string;
1473
1687
  "stage"?: "image" | "snapshot";
1688
+ "retry"?: {
1689
+ "failedAttempts": number;
1690
+ "maxAttempts": number;
1691
+ "notBefore": string;
1692
+ "lastFailure": string;
1693
+ };
1474
1694
  }>;
1475
1695
  } | null;
1476
1696
  "visibility": "private" | "org" | "link" | "public";
@@ -1554,6 +1774,7 @@ export type BenchmarksRegradeResponse = {
1554
1774
  "error": string | null;
1555
1775
  "preparation"?: {
1556
1776
  "since": string;
1777
+ "progressAt"?: string;
1557
1778
  "launchedBy": {
1558
1779
  "userId"?: string;
1559
1780
  "role"?: string;
@@ -1562,6 +1783,12 @@ export type BenchmarksRegradeResponse = {
1562
1783
  "imageKey": string;
1563
1784
  "buildId": string;
1564
1785
  "stage"?: "image" | "snapshot";
1786
+ "retry"?: {
1787
+ "failedAttempts": number;
1788
+ "maxAttempts": number;
1789
+ "notBefore": string;
1790
+ "lastFailure": string;
1791
+ };
1565
1792
  }>;
1566
1793
  } | null;
1567
1794
  "visibility": "private" | "org" | "link" | "public";
@@ -1669,6 +1896,7 @@ export type BenchmarksUnpublishResponse = {
1669
1896
  "error": string | null;
1670
1897
  "preparation"?: {
1671
1898
  "since": string;
1899
+ "progressAt"?: string;
1672
1900
  "launchedBy": {
1673
1901
  "userId"?: string;
1674
1902
  "role"?: string;
@@ -1677,6 +1905,12 @@ export type BenchmarksUnpublishResponse = {
1677
1905
  "imageKey": string;
1678
1906
  "buildId": string;
1679
1907
  "stage"?: "image" | "snapshot";
1908
+ "retry"?: {
1909
+ "failedAttempts": number;
1910
+ "maxAttempts": number;
1911
+ "notBefore": string;
1912
+ "lastFailure": string;
1913
+ };
1680
1914
  }>;
1681
1915
  } | null;
1682
1916
  "visibility": "private" | "org" | "link" | "public";
@@ -1734,6 +1968,9 @@ export type ExperimentImagesBuildsListResponse = {
1734
1968
  } | {
1735
1969
  "kind": "inline";
1736
1970
  "fileCount": number;
1971
+ } | {
1972
+ "kind": "asset";
1973
+ "sha256": string;
1737
1974
  };
1738
1975
  "sourceIdentity": string;
1739
1976
  "dockerfileDir": string;
@@ -1741,7 +1978,7 @@ export type ExperimentImagesBuildsListResponse = {
1741
1978
  "buildArgKeys": Array<string>;
1742
1979
  "platform": string;
1743
1980
  "trusted": boolean;
1744
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
1981
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1745
1982
  "image": string;
1746
1983
  "digest": string | null;
1747
1984
  "sizeBytes": number | null;
@@ -1782,7 +2019,7 @@ export type ExperimentImagesBuildsCreateRequest = {
1782
2019
  };
1783
2020
  export type ExperimentImagesBuildsCreateResponse = {
1784
2021
  "id": string;
1785
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
2022
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1786
2023
  };
1787
2024
  export type ExperimentImagesBuildsGetResponse = {
1788
2025
  "build": {
@@ -1804,6 +2041,9 @@ export type ExperimentImagesBuildsGetResponse = {
1804
2041
  } | {
1805
2042
  "kind": "inline";
1806
2043
  "fileCount": number;
2044
+ } | {
2045
+ "kind": "asset";
2046
+ "sha256": string;
1807
2047
  };
1808
2048
  "sourceIdentity": string;
1809
2049
  "dockerfileDir": string;
@@ -1811,7 +2051,7 @@ export type ExperimentImagesBuildsGetResponse = {
1811
2051
  "buildArgKeys": Array<string>;
1812
2052
  "platform": string;
1813
2053
  "trusted": boolean;
1814
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
2054
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1815
2055
  "image": string;
1816
2056
  "digest": string | null;
1817
2057
  "sizeBytes": number | null;
@@ -1825,7 +2065,7 @@ export type ExperimentImagesBuildsGetResponse = {
1825
2065
  };
1826
2066
  export type ExperimentImagesBuildsCancelResponse = {
1827
2067
  "id": string;
1828
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
2068
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
1829
2069
  };
1830
2070
  export type ExperimentImagesBuildsStreamResponse = unknown;
1831
2071
  export type ExperimentImagesRunsCreateRequest = {
@@ -1882,6 +2122,7 @@ export type ExperimentRunsCreateRequest = {
1882
2122
  "tools": Array<string>;
1883
2123
  };
1884
2124
  "requires_tools"?: boolean;
2125
+ "prompt_context"?: "none" | "target_site";
1885
2126
  "mcp_servers"?: Array<{
1886
2127
  "name": string;
1887
2128
  "url": string;
@@ -2158,7 +2399,7 @@ export type WebmcpArenaCreateRequest = {
2158
2399
  "name"?: string;
2159
2400
  };
2160
2401
  export type WebmcpArenaCreateResponse = {
2161
- "study": {
2402
+ "benchmark": {
2162
2403
  "id": string;
2163
2404
  "executiveSummary"?: string | null;
2164
2405
  "plannedRunCount": number;
@@ -2224,6 +2465,7 @@ export type WebmcpArenaCreateResponse = {
2224
2465
  "error": string | null;
2225
2466
  "preparation"?: {
2226
2467
  "since": string;
2468
+ "progressAt"?: string;
2227
2469
  "launchedBy": {
2228
2470
  "userId"?: string;
2229
2471
  "role"?: string;
@@ -2232,6 +2474,12 @@ export type WebmcpArenaCreateResponse = {
2232
2474
  "imageKey": string;
2233
2475
  "buildId": string;
2234
2476
  "stage"?: "image" | "snapshot";
2477
+ "retry"?: {
2478
+ "failedAttempts": number;
2479
+ "maxAttempts": number;
2480
+ "notBefore": string;
2481
+ "lastFailure": string;
2482
+ };
2235
2483
  }>;
2236
2484
  } | null;
2237
2485
  "visibility": "private" | "org" | "link" | "public";
@@ -2248,10 +2496,6 @@ export type WebmcpArenaCreateResponse = {
2248
2496
  "liveUntil": string | null;
2249
2497
  };
2250
2498
  "streamUrl": string;
2251
- "warnings"?: Array<{
2252
- "harness": string;
2253
- "kind": string;
2254
- }>;
2255
2499
  };
2256
2500
  export type WebmcpRunsBrowserRecordingResponse = {
2257
2501
  "runId": string;
@@ -2500,6 +2744,18 @@ export type IntentsBulkCreateRequest = {
2500
2744
  "goal"?: {
2501
2745
  "statement": string;
2502
2746
  };
2747
+ "agents"?: Array<{
2748
+ "harness": string;
2749
+ "model": string;
2750
+ "mcp_server_ids"?: Array<string>;
2751
+ "skill_ids"?: Array<string>;
2752
+ "browser"?: {
2753
+ "webdriver"?: boolean;
2754
+ "webmcp"?: boolean;
2755
+ "webmcpTargetUrl"?: string;
2756
+ "webmcpSurface"?: "native" | "shim";
2757
+ };
2758
+ }>;
2503
2759
  "prompts": Array<{
2504
2760
  "text": string;
2505
2761
  "category"?: string;
@@ -2509,6 +2765,7 @@ export type IntentsBulkCreateRequest = {
2509
2765
  }>;
2510
2766
  "domainId"?: string;
2511
2767
  "icpId"?: string;
2768
+ "firstMeasurement"?: "written" | "intent";
2512
2769
  };
2513
2770
  export type IntentsBulkCreateResponse = {
2514
2771
  "success": true;
@@ -2518,6 +2775,13 @@ export type IntentsBulkCreateResponse = {
2518
2775
  "title": string;
2519
2776
  "merged": boolean;
2520
2777
  "promptsCreated": number;
2778
+ "setupsApplied"?: boolean;
2779
+ "measurementStart": {
2780
+ "started": true;
2781
+ } | {
2782
+ "started": false;
2783
+ "reason": "no_attributable_member";
2784
+ };
2521
2785
  }>;
2522
2786
  "truncated": number;
2523
2787
  };
@@ -2713,6 +2977,12 @@ export type IntentsGenerateResponse = {
2713
2977
  "tracked": boolean;
2714
2978
  }>;
2715
2979
  }>;
2980
+ "measurementStart": {
2981
+ "started": true;
2982
+ } | {
2983
+ "started": false;
2984
+ "reason": "no_attributable_member";
2985
+ };
2716
2986
  }>;
2717
2987
  };
2718
2988
  export type IntentsSetIcpRequest = {
@@ -2824,6 +3094,12 @@ export type IntentsAddPromptResponse = {
2824
3094
  "tracked": boolean;
2825
3095
  }>;
2826
3096
  }>;
3097
+ "measurementStart": {
3098
+ "started": true;
3099
+ } | {
3100
+ "started": false;
3101
+ "reason": "no_attributable_member";
3102
+ };
2827
3103
  };
2828
3104
  };
2829
3105
  export type IntentsUpdatePromptRequest = {
@@ -2842,6 +3118,17 @@ export type IntentsDeletePromptResponse = {
2842
3118
  "id": string;
2843
3119
  };
2844
3120
  };
3121
+ export type IntentsSetPromptGoalRequest = {
3122
+ "active": boolean;
3123
+ };
3124
+ export type IntentsSetPromptGoalResponse = {
3125
+ "success": true;
3126
+ "data": {
3127
+ "promptId": string;
3128
+ "kind": "discover" | "choose" | "verify" | "inform" | "guide" | "act" | "other";
3129
+ "active": boolean;
3130
+ };
3131
+ };
2845
3132
  export type IntentsRepresentationResponse = {
2846
3133
  "success": true;
2847
3134
  "data": {
@@ -3063,6 +3350,7 @@ export type StudioRunsCreateRequest = {
3063
3350
  "tools": Array<string>;
3064
3351
  };
3065
3352
  "requires_tools"?: boolean;
3353
+ "prompt_context"?: "none" | "target_site";
3066
3354
  "mcp_servers"?: Array<{
3067
3355
  "name": string;
3068
3356
  "url": string;
@@ -3145,6 +3433,7 @@ export type StudyAddonsListResponse = {
3145
3433
  "surfaces": Array<"run" | "session">;
3146
3434
  "harnesses": Array<string>;
3147
3435
  "multiInstance"?: boolean;
3436
+ "credentialSourceFields"?: Array<string>;
3148
3437
  }>;
3149
3438
  };
3150
3439
  export type StudyBenchmarkDefinitionsListResponse = {
@@ -3163,6 +3452,13 @@ export type StudyBenchmarkDefinitionsListResponse = {
3163
3452
  "modelParams"?: Array<string>;
3164
3453
  "maxPlannedRuns"?: number;
3165
3454
  "sampleFloor"?: number;
3455
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3456
+ "aggregation"?: "included" | "excluded";
3457
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3458
+ "aggregateTaskSample"?: {
3459
+ "count": number;
3460
+ "seed": string;
3461
+ };
3166
3462
  };
3167
3463
  "defaultConfig": {
3168
3464
  "matrix"?: Array<{
@@ -3229,6 +3525,7 @@ export type StudyBenchmarkDefinitionsListResponse = {
3229
3525
  "userId": string;
3230
3526
  "displayName": string | null;
3231
3527
  } | null;
3528
+ "uploadedByPublisher": boolean | null;
3232
3529
  "notices": Array<{
3233
3530
  "kind": string;
3234
3531
  "text": string;
@@ -3266,6 +3563,13 @@ export type StudyBenchmarkDefinitionsCreateRequest = {
3266
3563
  "modelParams"?: Array<string>;
3267
3564
  "maxPlannedRuns"?: number;
3268
3565
  "sampleFloor"?: number;
3566
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3567
+ "aggregation"?: "included" | "excluded";
3568
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3569
+ "aggregateTaskSample"?: {
3570
+ "count": number;
3571
+ "seed": string;
3572
+ };
3269
3573
  };
3270
3574
  "runnerKind"?: "none" | "image" | "environment" | "framework";
3271
3575
  "runnerRecipe"?: Record<string, unknown> | null;
@@ -3308,6 +3612,13 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
3308
3612
  "modelParams"?: Array<string>;
3309
3613
  "maxPlannedRuns"?: number;
3310
3614
  "sampleFloor"?: number;
3615
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3616
+ "aggregation"?: "included" | "excluded";
3617
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3618
+ "aggregateTaskSample"?: {
3619
+ "count": number;
3620
+ "seed": string;
3621
+ };
3311
3622
  };
3312
3623
  "defaultConfig": {
3313
3624
  "matrix"?: Array<{
@@ -3374,6 +3685,7 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
3374
3685
  "userId": string;
3375
3686
  "displayName": string | null;
3376
3687
  } | null;
3688
+ "uploadedByPublisher": boolean | null;
3377
3689
  "notices": Array<{
3378
3690
  "kind": string;
3379
3691
  "text": string;
@@ -3396,6 +3708,13 @@ export type StudyBenchmarkDefinitionsGetResponse = {
3396
3708
  "modelParams"?: Array<string>;
3397
3709
  "maxPlannedRuns"?: number;
3398
3710
  "sampleFloor"?: number;
3711
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3712
+ "aggregation"?: "included" | "excluded";
3713
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3714
+ "aggregateTaskSample"?: {
3715
+ "count": number;
3716
+ "seed": string;
3717
+ };
3399
3718
  };
3400
3719
  "defaultConfig": {
3401
3720
  "matrix"?: Array<{
@@ -3462,6 +3781,7 @@ export type StudyBenchmarkDefinitionsGetResponse = {
3462
3781
  "userId": string;
3463
3782
  "displayName": string | null;
3464
3783
  } | null;
3784
+ "uploadedByPublisher": boolean | null;
3465
3785
  "notices": Array<{
3466
3786
  "kind": string;
3467
3787
  "text": string;
@@ -3493,6 +3813,13 @@ export type StudyBenchmarkDefinitionsUpdateRequest = {
3493
3813
  "modelParams"?: Array<string>;
3494
3814
  "maxPlannedRuns"?: number;
3495
3815
  "sampleFloor"?: number;
3816
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3817
+ "aggregation"?: "included" | "excluded";
3818
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3819
+ "aggregateTaskSample"?: {
3820
+ "count": number;
3821
+ "seed": string;
3822
+ };
3496
3823
  };
3497
3824
  "runnerKind"?: "none" | "image" | "environment" | "framework";
3498
3825
  "runnerRecipe"?: Record<string, unknown> | null;
@@ -3535,6 +3862,13 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
3535
3862
  "modelParams"?: Array<string>;
3536
3863
  "maxPlannedRuns"?: number;
3537
3864
  "sampleFloor"?: number;
3865
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3866
+ "aggregation"?: "included" | "excluded";
3867
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3868
+ "aggregateTaskSample"?: {
3869
+ "count": number;
3870
+ "seed": string;
3871
+ };
3538
3872
  };
3539
3873
  "defaultConfig": {
3540
3874
  "matrix"?: Array<{
@@ -3601,6 +3935,7 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
3601
3935
  "userId": string;
3602
3936
  "displayName": string | null;
3603
3937
  } | null;
3938
+ "uploadedByPublisher": boolean | null;
3604
3939
  "notices": Array<{
3605
3940
  "kind": string;
3606
3941
  "text": string;
@@ -3645,6 +3980,13 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
3645
3980
  "modelParams"?: Array<string>;
3646
3981
  "maxPlannedRuns"?: number;
3647
3982
  "sampleFloor"?: number;
3983
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
3984
+ "aggregation"?: "included" | "excluded";
3985
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
3986
+ "aggregateTaskSample"?: {
3987
+ "count": number;
3988
+ "seed": string;
3989
+ };
3648
3990
  };
3649
3991
  "defaultConfig": {
3650
3992
  "matrix"?: Array<{
@@ -3711,6 +4053,7 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
3711
4053
  "userId": string;
3712
4054
  "displayName": string | null;
3713
4055
  } | null;
4056
+ "uploadedByPublisher": boolean | null;
3714
4057
  "notices": Array<{
3715
4058
  "kind": string;
3716
4059
  "text": string;
@@ -3736,6 +4079,13 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
3736
4079
  "modelParams"?: Array<string>;
3737
4080
  "maxPlannedRuns"?: number;
3738
4081
  "sampleFloor"?: number;
4082
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
4083
+ "aggregation"?: "included" | "excluded";
4084
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
4085
+ "aggregateTaskSample"?: {
4086
+ "count": number;
4087
+ "seed": string;
4088
+ };
3739
4089
  };
3740
4090
  "defaultConfig": {
3741
4091
  "matrix"?: Array<{
@@ -3802,6 +4152,7 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
3802
4152
  "userId": string;
3803
4153
  "displayName": string | null;
3804
4154
  } | null;
4155
+ "uploadedByPublisher": boolean | null;
3805
4156
  "notices": Array<{
3806
4157
  "kind": string;
3807
4158
  "text": string;
@@ -3824,6 +4175,13 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
3824
4175
  "modelParams"?: Array<string>;
3825
4176
  "maxPlannedRuns"?: number;
3826
4177
  "sampleFloor"?: number;
4178
+ "missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
4179
+ "aggregation"?: "included" | "excluded";
4180
+ "aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
4181
+ "aggregateTaskSample"?: {
4182
+ "count": number;
4183
+ "seed": string;
4184
+ };
3827
4185
  };
3828
4186
  "defaultConfig": {
3829
4187
  "matrix"?: Array<{
@@ -3890,6 +4248,7 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
3890
4248
  "userId": string;
3891
4249
  "displayName": string | null;
3892
4250
  } | null;
4251
+ "uploadedByPublisher": boolean | null;
3893
4252
  "notices": Array<{
3894
4253
  "kind": string;
3895
4254
  "text": string;
@@ -4068,6 +4427,9 @@ export type StudyImagesBuildsListResponse = {
4068
4427
  } | {
4069
4428
  "kind": "inline";
4070
4429
  "fileCount": number;
4430
+ } | {
4431
+ "kind": "asset";
4432
+ "sha256": string;
4071
4433
  };
4072
4434
  "sourceIdentity": string;
4073
4435
  "dockerfileDir": string;
@@ -4075,7 +4437,7 @@ export type StudyImagesBuildsListResponse = {
4075
4437
  "buildArgKeys": Array<string>;
4076
4438
  "platform": string;
4077
4439
  "trusted": boolean;
4078
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4440
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4079
4441
  "image": string;
4080
4442
  "digest": string | null;
4081
4443
  "sizeBytes": number | null;
@@ -4116,7 +4478,7 @@ export type StudyImagesBuildsCreateRequest = {
4116
4478
  };
4117
4479
  export type StudyImagesBuildsCreateResponse = {
4118
4480
  "id": string;
4119
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4481
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4120
4482
  };
4121
4483
  export type StudyImagesBuildsGetResponse = {
4122
4484
  "build": {
@@ -4138,6 +4500,9 @@ export type StudyImagesBuildsGetResponse = {
4138
4500
  } | {
4139
4501
  "kind": "inline";
4140
4502
  "fileCount": number;
4503
+ } | {
4504
+ "kind": "asset";
4505
+ "sha256": string;
4141
4506
  };
4142
4507
  "sourceIdentity": string;
4143
4508
  "dockerfileDir": string;
@@ -4145,7 +4510,7 @@ export type StudyImagesBuildsGetResponse = {
4145
4510
  "buildArgKeys": Array<string>;
4146
4511
  "platform": string;
4147
4512
  "trusted": boolean;
4148
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4513
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4149
4514
  "image": string;
4150
4515
  "digest": string | null;
4151
4516
  "sizeBytes": number | null;
@@ -4159,7 +4524,7 @@ export type StudyImagesBuildsGetResponse = {
4159
4524
  };
4160
4525
  export type StudyImagesBuildsCancelResponse = {
4161
4526
  "id": string;
4162
- "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4527
+ "phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
4163
4528
  };
4164
4529
  export type StudyImagesBuildsStreamResponse = unknown;
4165
4530
  export type StudyImagesRunsCreateRequest = {
@@ -4216,6 +4581,7 @@ export type StudyRunsCreateRequest = {
4216
4581
  "tools": Array<string>;
4217
4582
  };
4218
4583
  "requires_tools"?: boolean;
4584
+ "prompt_context"?: "none" | "target_site";
4219
4585
  "mcp_servers"?: Array<{
4220
4586
  "name": string;
4221
4587
  "url": string;
@@ -4394,7 +4760,7 @@ export type StudyListResponse = {
4394
4760
  "addons"?: Record<string, unknown>;
4395
4761
  };
4396
4762
  };
4397
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
4763
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4398
4764
  "subjectBackend": "model" | "harness" | null;
4399
4765
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4400
4766
  "importedFrom"?: string;
@@ -4430,6 +4796,7 @@ export type StudyListResponse = {
4430
4796
  "error": string | null;
4431
4797
  "preparation"?: {
4432
4798
  "since": string;
4799
+ "progressAt"?: string;
4433
4800
  "launchedBy": {
4434
4801
  "userId"?: string;
4435
4802
  "role"?: string;
@@ -4438,6 +4805,12 @@ export type StudyListResponse = {
4438
4805
  "imageKey": string;
4439
4806
  "buildId": string;
4440
4807
  "stage"?: "image" | "snapshot";
4808
+ "retry"?: {
4809
+ "failedAttempts": number;
4810
+ "maxAttempts": number;
4811
+ "notBefore": string;
4812
+ "lastFailure": string;
4813
+ };
4441
4814
  }>;
4442
4815
  } | null;
4443
4816
  "visibility": "private" | "org" | "link" | "public";
@@ -4452,6 +4825,57 @@ export type StudyListResponse = {
4452
4825
  "createdAt": string;
4453
4826
  "updatedAt": string;
4454
4827
  "liveUntil": string | null;
4828
+ "launch"?: {
4829
+ "attempt": number;
4830
+ "state": "fanning_out" | "deferred" | "settled";
4831
+ "startedAt": string;
4832
+ "progressAt": string;
4833
+ "planned"?: number;
4834
+ "created"?: number;
4835
+ "missing"?: {
4836
+ "retryable": number;
4837
+ "refused": number;
4838
+ };
4839
+ "notes"?: Array<string>;
4840
+ "settledAt"?: string;
4841
+ } | null;
4842
+ "relaunchable"?: boolean;
4843
+ "arms"?: Array<{
4844
+ "key": string;
4845
+ "position": number;
4846
+ "label": string;
4847
+ "role": "reference" | "peer";
4848
+ "payload": {
4849
+ "tools"?: "default" | "none" | {
4850
+ "tools": Array<string>;
4851
+ };
4852
+ "addons"?: Record<string, unknown>;
4853
+ };
4854
+ "cells"?: Array<{
4855
+ "harness": string;
4856
+ "model": string;
4857
+ "provider"?: string;
4858
+ "params"?: {
4859
+ "temperature"?: number;
4860
+ "maxOutputTokens"?: number;
4861
+ "reasoningEffort"?: "low" | "medium" | "high";
4862
+ "thinkingBudgetTokens"?: number;
4863
+ };
4864
+ }>;
4865
+ "skills"?: Array<{
4866
+ "ref": string;
4867
+ "name": string;
4868
+ "changedSinceCreate"?: boolean;
4869
+ }>;
4870
+ "addedAgents"?: Array<{
4871
+ "position": number;
4872
+ "addedAt": string;
4873
+ }>;
4874
+ }>;
4875
+ "contrastPlan"?: "vs_reference" | "all_pairs";
4876
+ "cellPlan"?: "crossed" | "per_setup";
4877
+ "skillDisclosure"?: "withheld" | "named";
4878
+ "modeOptions"?: Record<string, unknown>;
4455
4879
  }>;
4456
4880
  };
4457
4881
  export type StudyCreateRequest = {
@@ -4478,10 +4902,34 @@ export type StudyCreateRequest = {
4478
4902
  "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
4479
4903
  "judgeSupplementary"?: boolean;
4480
4904
  "proposeExpectations"?: boolean;
4481
- "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
4905
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4482
4906
  "subjectBackend"?: "model" | "harness";
4483
4907
  "baselineLabel"?: string;
4484
4908
  "treatmentLabel"?: string;
4909
+ "arms"?: Array<{
4910
+ "key"?: string;
4911
+ "label": string;
4912
+ "role"?: "reference" | "peer";
4913
+ "config": {
4914
+ "tools"?: "default" | "none" | {
4915
+ "tools": Array<string>;
4916
+ };
4917
+ "addons"?: Record<string, unknown>;
4918
+ };
4919
+ "cells"?: Array<{
4920
+ "harness": string;
4921
+ "model": string;
4922
+ "provider"?: string;
4923
+ "params"?: {
4924
+ "temperature"?: number;
4925
+ "maxOutputTokens"?: number;
4926
+ "reasoningEffort"?: "low" | "medium" | "high";
4927
+ "thinkingBudgetTokens"?: number;
4928
+ };
4929
+ }>;
4930
+ }>;
4931
+ "contrasts"?: "vs_reference" | "all_pairs";
4932
+ "modeOptions"?: Record<string, unknown>;
4485
4933
  "taskCount"?: number;
4486
4934
  "variantsPerTask"?: number;
4487
4935
  "repeats"?: number;
@@ -4492,7 +4940,76 @@ export type StudyCreateRequest = {
4492
4940
  "requiresWeb"?: boolean;
4493
4941
  "rubric"?: Array<string>;
4494
4942
  "variants"?: Array<string>;
4495
- "expectations"?: Array<unknown>;
4943
+ "expectations"?: Array<{
4944
+ "label"?: string;
4945
+ "arms"?: Array<"baseline" | "skill" | string>;
4946
+ "kind": "mcp_tool_called";
4947
+ "server"?: string;
4948
+ "tool": string;
4949
+ "arguments"?: Record<string, unknown>;
4950
+ "minCalls"?: number;
4951
+ } | {
4952
+ "label"?: string;
4953
+ "arms"?: Array<"baseline" | "skill" | string>;
4954
+ "kind": "tool_not_called";
4955
+ "tool"?: string;
4956
+ "mcp"?: {
4957
+ "server"?: string;
4958
+ "tool": string;
4959
+ };
4960
+ "pattern"?: string;
4961
+ } | {
4962
+ "label"?: string;
4963
+ "arms"?: Array<"baseline" | "skill" | string>;
4964
+ "kind": "command_ran";
4965
+ "pattern": string;
4966
+ } | {
4967
+ "label"?: string;
4968
+ "arms"?: Array<"baseline" | "skill" | string>;
4969
+ "kind": "skill_loaded";
4970
+ "slug": string;
4971
+ } | {
4972
+ "label"?: string;
4973
+ "arms"?: Array<"baseline" | "skill" | string>;
4974
+ "kind": "file_operation";
4975
+ "op"?: "read" | "write" | "edit";
4976
+ "path": unknown;
4977
+ } | {
4978
+ "label"?: string;
4979
+ "arms"?: Array<"baseline" | "skill" | string>;
4980
+ "kind": "web_fetched";
4981
+ "urlHost"?: string;
4982
+ "urlPattern"?: string;
4983
+ } | {
4984
+ "label"?: string;
4985
+ "arms"?: Array<"baseline" | "skill" | string>;
4986
+ "kind": "response_contains";
4987
+ "text": string;
4988
+ "normalize"?: "prose";
4989
+ } | {
4990
+ "label"?: string;
4991
+ "arms"?: Array<"baseline" | "skill" | string>;
4992
+ "kind": "response_matches";
4993
+ "pattern": string;
4994
+ "ignoreCase"?: boolean;
4995
+ } | {
4996
+ "label"?: string;
4997
+ "arms"?: Array<"baseline" | "skill" | string>;
4998
+ "kind": "token_budget";
4999
+ "maxOutputTokens"?: number;
5000
+ "maxTotalTokens"?: number;
5001
+ } | {
5002
+ "label"?: string;
5003
+ "arms"?: Array<"baseline" | "skill" | string>;
5004
+ "kind": "sequence";
5005
+ "of": Array<unknown>;
5006
+ } | {
5007
+ "label"?: string;
5008
+ "arms"?: Array<"baseline" | "skill" | string>;
5009
+ "kind": "any_of";
5010
+ "of": Array<unknown>;
5011
+ }>;
5012
+ "private"?: boolean;
4496
5013
  }>;
4497
5014
  "subset"?: {
4498
5015
  "count": number;
@@ -4542,7 +5059,7 @@ export type StudyCreateResponse = {
4542
5059
  "addons"?: Record<string, unknown>;
4543
5060
  };
4544
5061
  };
4545
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5062
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4546
5063
  "subjectBackend": "model" | "harness" | null;
4547
5064
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4548
5065
  "importedFrom"?: string;
@@ -4578,6 +5095,7 @@ export type StudyCreateResponse = {
4578
5095
  "error": string | null;
4579
5096
  "preparation"?: {
4580
5097
  "since": string;
5098
+ "progressAt"?: string;
4581
5099
  "launchedBy": {
4582
5100
  "userId"?: string;
4583
5101
  "role"?: string;
@@ -4586,6 +5104,12 @@ export type StudyCreateResponse = {
4586
5104
  "imageKey": string;
4587
5105
  "buildId": string;
4588
5106
  "stage"?: "image" | "snapshot";
5107
+ "retry"?: {
5108
+ "failedAttempts": number;
5109
+ "maxAttempts": number;
5110
+ "notBefore": string;
5111
+ "lastFailure": string;
5112
+ };
4589
5113
  }>;
4590
5114
  } | null;
4591
5115
  "visibility": "private" | "org" | "link" | "public";
@@ -4600,11 +5124,76 @@ export type StudyCreateResponse = {
4600
5124
  "createdAt": string;
4601
5125
  "updatedAt": string;
4602
5126
  "liveUntil": string | null;
4603
- };
4604
- "streamUrl": string;
4605
- "warnings"?: Array<{
4606
- "harness": string;
4607
- "kind": string;
5127
+ "launch"?: {
5128
+ "attempt": number;
5129
+ "state": "fanning_out" | "deferred" | "settled";
5130
+ "startedAt": string;
5131
+ "progressAt": string;
5132
+ "planned"?: number;
5133
+ "created"?: number;
5134
+ "missing"?: {
5135
+ "retryable": number;
5136
+ "refused": number;
5137
+ };
5138
+ "notes"?: Array<string>;
5139
+ "settledAt"?: string;
5140
+ } | null;
5141
+ "relaunchable"?: boolean;
5142
+ "arms"?: Array<{
5143
+ "key": string;
5144
+ "position": number;
5145
+ "label": string;
5146
+ "role": "reference" | "peer";
5147
+ "payload": {
5148
+ "tools"?: "default" | "none" | {
5149
+ "tools": Array<string>;
5150
+ };
5151
+ "addons"?: Record<string, unknown>;
5152
+ };
5153
+ "cells"?: Array<{
5154
+ "harness": string;
5155
+ "model": string;
5156
+ "provider"?: string;
5157
+ "params"?: {
5158
+ "temperature"?: number;
5159
+ "maxOutputTokens"?: number;
5160
+ "reasoningEffort"?: "low" | "medium" | "high";
5161
+ "thinkingBudgetTokens"?: number;
5162
+ };
5163
+ }>;
5164
+ "skills"?: Array<{
5165
+ "ref": string;
5166
+ "name": string;
5167
+ "changedSinceCreate"?: boolean;
5168
+ }>;
5169
+ "addedAgents"?: Array<{
5170
+ "position": number;
5171
+ "addedAt": string;
5172
+ }>;
5173
+ }>;
5174
+ "contrastPlan"?: "vs_reference" | "all_pairs";
5175
+ "cellPlan"?: "crossed" | "per_setup";
5176
+ "skillDisclosure"?: "withheld" | "named";
5177
+ "modeOptions"?: Record<string, unknown>;
5178
+ };
5179
+ "streamUrl": string;
5180
+ "warnings"?: Array<{
5181
+ "harness": string;
5182
+ "kind": string;
5183
+ } | {
5184
+ "type": "design_aggregate_budget";
5185
+ "detail": string;
5186
+ }>;
5187
+ };
5188
+ export type StudyComparisonModesListResponse = {
5189
+ "modes": Array<{
5190
+ "id": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5191
+ "enabled": boolean;
5192
+ "authoring": boolean;
5193
+ }>;
5194
+ "cellPlans": Array<{
5195
+ "id": "crossed" | "per_setup";
5196
+ "enabled": boolean;
4608
5197
  }>;
4609
5198
  };
4610
5199
  export type StudyConvertWebmcpSuiteRequest = {
@@ -4619,6 +5208,226 @@ export type StudyConvertWebmcpSuiteResponse = {
4619
5208
  }>;
4620
5209
  "warnings": Array<string>;
4621
5210
  };
5211
+ export type StudyPreflightRequest = {
5212
+ "name": string;
5213
+ "description"?: string;
5214
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
5215
+ "skillRef"?: string;
5216
+ "definitionVersionId"?: string;
5217
+ "subjectName"?: string;
5218
+ "armConfig"?: {
5219
+ "baseline"?: {
5220
+ "tools"?: "default" | "none" | {
5221
+ "tools": Array<string>;
5222
+ };
5223
+ "addons"?: Record<string, unknown>;
5224
+ };
5225
+ "treatment"?: {
5226
+ "tools"?: "default" | "none" | {
5227
+ "tools": Array<string>;
5228
+ };
5229
+ "addons"?: Record<string, unknown>;
5230
+ };
5231
+ };
5232
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5233
+ "judgeSupplementary"?: boolean;
5234
+ "proposeExpectations"?: boolean;
5235
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5236
+ "subjectBackend"?: "model" | "harness";
5237
+ "baselineLabel"?: string;
5238
+ "treatmentLabel"?: string;
5239
+ "arms"?: Array<{
5240
+ "key"?: string;
5241
+ "label": string;
5242
+ "role"?: "reference" | "peer";
5243
+ "config": {
5244
+ "tools"?: "default" | "none" | {
5245
+ "tools": Array<string>;
5246
+ };
5247
+ "addons"?: Record<string, unknown>;
5248
+ };
5249
+ "cells"?: Array<{
5250
+ "harness": string;
5251
+ "model": string;
5252
+ "provider"?: string;
5253
+ "params"?: {
5254
+ "temperature"?: number;
5255
+ "maxOutputTokens"?: number;
5256
+ "reasoningEffort"?: "low" | "medium" | "high";
5257
+ "thinkingBudgetTokens"?: number;
5258
+ };
5259
+ }>;
5260
+ }>;
5261
+ "contrasts"?: "vs_reference" | "all_pairs";
5262
+ "modeOptions"?: Record<string, unknown>;
5263
+ "taskCount"?: number;
5264
+ "variantsPerTask"?: number;
5265
+ "repeats"?: number;
5266
+ "tasks"?: Array<{
5267
+ "prompt": string;
5268
+ "rationale"?: string;
5269
+ "category"?: string;
5270
+ "requiresWeb"?: boolean;
5271
+ "rubric"?: Array<string>;
5272
+ "variants"?: Array<string>;
5273
+ "expectations"?: Array<{
5274
+ "label"?: string;
5275
+ "arms"?: Array<"baseline" | "skill" | string>;
5276
+ "kind": "mcp_tool_called";
5277
+ "server"?: string;
5278
+ "tool": string;
5279
+ "arguments"?: Record<string, unknown>;
5280
+ "minCalls"?: number;
5281
+ } | {
5282
+ "label"?: string;
5283
+ "arms"?: Array<"baseline" | "skill" | string>;
5284
+ "kind": "tool_not_called";
5285
+ "tool"?: string;
5286
+ "mcp"?: {
5287
+ "server"?: string;
5288
+ "tool": string;
5289
+ };
5290
+ "pattern"?: string;
5291
+ } | {
5292
+ "label"?: string;
5293
+ "arms"?: Array<"baseline" | "skill" | string>;
5294
+ "kind": "command_ran";
5295
+ "pattern": string;
5296
+ } | {
5297
+ "label"?: string;
5298
+ "arms"?: Array<"baseline" | "skill" | string>;
5299
+ "kind": "skill_loaded";
5300
+ "slug": string;
5301
+ } | {
5302
+ "label"?: string;
5303
+ "arms"?: Array<"baseline" | "skill" | string>;
5304
+ "kind": "file_operation";
5305
+ "op"?: "read" | "write" | "edit";
5306
+ "path": unknown;
5307
+ } | {
5308
+ "label"?: string;
5309
+ "arms"?: Array<"baseline" | "skill" | string>;
5310
+ "kind": "web_fetched";
5311
+ "urlHost"?: string;
5312
+ "urlPattern"?: string;
5313
+ } | {
5314
+ "label"?: string;
5315
+ "arms"?: Array<"baseline" | "skill" | string>;
5316
+ "kind": "response_contains";
5317
+ "text": string;
5318
+ "normalize"?: "prose";
5319
+ } | {
5320
+ "label"?: string;
5321
+ "arms"?: Array<"baseline" | "skill" | string>;
5322
+ "kind": "response_matches";
5323
+ "pattern": string;
5324
+ "ignoreCase"?: boolean;
5325
+ } | {
5326
+ "label"?: string;
5327
+ "arms"?: Array<"baseline" | "skill" | string>;
5328
+ "kind": "token_budget";
5329
+ "maxOutputTokens"?: number;
5330
+ "maxTotalTokens"?: number;
5331
+ } | {
5332
+ "label"?: string;
5333
+ "arms"?: Array<"baseline" | "skill" | string>;
5334
+ "kind": "sequence";
5335
+ "of": Array<unknown>;
5336
+ } | {
5337
+ "label"?: string;
5338
+ "arms"?: Array<"baseline" | "skill" | string>;
5339
+ "kind": "any_of";
5340
+ "of": Array<unknown>;
5341
+ }>;
5342
+ "private"?: boolean;
5343
+ }>;
5344
+ "subset"?: {
5345
+ "count": number;
5346
+ "seed"?: string;
5347
+ };
5348
+ "matrix": Array<{
5349
+ "harness": string;
5350
+ "model": string;
5351
+ "provider"?: string;
5352
+ "params"?: {
5353
+ "temperature"?: number;
5354
+ "maxOutputTokens"?: number;
5355
+ "reasoningEffort"?: "low" | "medium" | "high";
5356
+ "thinkingBudgetTokens"?: number;
5357
+ };
5358
+ }>;
5359
+ "promise"?: string;
5360
+ };
5361
+ export type StudyPreflightResponse = {
5362
+ "ok": true;
5363
+ "depth": "default" | "full";
5364
+ "form": string;
5365
+ "label": string;
5366
+ "plannedRunCount": number;
5367
+ "plannedJudgeComparisons": number;
5368
+ "runs": {
5369
+ "tasks": number;
5370
+ "variants": number;
5371
+ "repeats": number;
5372
+ "groupAgents": Array<{
5373
+ "group": string;
5374
+ "count": number;
5375
+ }>;
5376
+ };
5377
+ "judgePerComparison": Array<{
5378
+ "reference": string;
5379
+ "subject": string;
5380
+ "count": number;
5381
+ }>;
5382
+ "arms": Array<{
5383
+ "key": string;
5384
+ "label": string | null;
5385
+ "role": string;
5386
+ "payload": Record<string, unknown> | null;
5387
+ "cells"?: Array<{
5388
+ "harness": string;
5389
+ "model": string;
5390
+ "provider"?: string;
5391
+ "params"?: Record<string, unknown>;
5392
+ }>;
5393
+ }>;
5394
+ "comparisons": Array<{
5395
+ "reference": string;
5396
+ "subject": string;
5397
+ "oriented": boolean;
5398
+ "basis": "matched" | "whole_setup";
5399
+ "sharedAgents": Array<{
5400
+ "harness": string;
5401
+ "model": string;
5402
+ "provider"?: string;
5403
+ "params"?: Record<string, unknown>;
5404
+ }>;
5405
+ "excluded": Array<{
5406
+ "group": string;
5407
+ "agents": Array<{
5408
+ "harness": string;
5409
+ "model": string;
5410
+ "provider"?: string;
5411
+ "params"?: Record<string, unknown>;
5412
+ }>;
5413
+ }>;
5414
+ "droppedTasks"?: unknown | null;
5415
+ }>;
5416
+ "boards": Array<{
5417
+ "group": string;
5418
+ "profile": "as-published" | "harness-default" | "with-setup";
5419
+ }>;
5420
+ "issues": Array<{
5421
+ "path": string;
5422
+ "code": string;
5423
+ "message": string;
5424
+ }>;
5425
+ "quota"?: {
5426
+ "dimension": string;
5427
+ "remaining": number | null;
5428
+ "allowed": boolean;
5429
+ } | null;
5430
+ };
4622
5431
  export type StudyGetResponse = {
4623
5432
  "study": {
4624
5433
  "id": string;
@@ -4650,7 +5459,7 @@ export type StudyGetResponse = {
4650
5459
  "addons"?: Record<string, unknown>;
4651
5460
  };
4652
5461
  };
4653
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5462
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
4654
5463
  "subjectBackend": "model" | "harness" | null;
4655
5464
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4656
5465
  "importedFrom"?: string;
@@ -4686,6 +5495,7 @@ export type StudyGetResponse = {
4686
5495
  "error": string | null;
4687
5496
  "preparation"?: {
4688
5497
  "since": string;
5498
+ "progressAt"?: string;
4689
5499
  "launchedBy": {
4690
5500
  "userId"?: string;
4691
5501
  "role"?: string;
@@ -4694,6 +5504,12 @@ export type StudyGetResponse = {
4694
5504
  "imageKey": string;
4695
5505
  "buildId": string;
4696
5506
  "stage"?: "image" | "snapshot";
5507
+ "retry"?: {
5508
+ "failedAttempts": number;
5509
+ "maxAttempts": number;
5510
+ "notBefore": string;
5511
+ "lastFailure": string;
5512
+ };
4697
5513
  }>;
4698
5514
  } | null;
4699
5515
  "visibility": "private" | "org" | "link" | "public";
@@ -4708,6 +5524,57 @@ export type StudyGetResponse = {
4708
5524
  "createdAt": string;
4709
5525
  "updatedAt": string;
4710
5526
  "liveUntil": string | null;
5527
+ "launch"?: {
5528
+ "attempt": number;
5529
+ "state": "fanning_out" | "deferred" | "settled";
5530
+ "startedAt": string;
5531
+ "progressAt": string;
5532
+ "planned"?: number;
5533
+ "created"?: number;
5534
+ "missing"?: {
5535
+ "retryable": number;
5536
+ "refused": number;
5537
+ };
5538
+ "notes"?: Array<string>;
5539
+ "settledAt"?: string;
5540
+ } | null;
5541
+ "relaunchable"?: boolean;
5542
+ "arms"?: Array<{
5543
+ "key": string;
5544
+ "position": number;
5545
+ "label": string;
5546
+ "role": "reference" | "peer";
5547
+ "payload": {
5548
+ "tools"?: "default" | "none" | {
5549
+ "tools": Array<string>;
5550
+ };
5551
+ "addons"?: Record<string, unknown>;
5552
+ };
5553
+ "cells"?: Array<{
5554
+ "harness": string;
5555
+ "model": string;
5556
+ "provider"?: string;
5557
+ "params"?: {
5558
+ "temperature"?: number;
5559
+ "maxOutputTokens"?: number;
5560
+ "reasoningEffort"?: "low" | "medium" | "high";
5561
+ "thinkingBudgetTokens"?: number;
5562
+ };
5563
+ }>;
5564
+ "skills"?: Array<{
5565
+ "ref": string;
5566
+ "name": string;
5567
+ "changedSinceCreate"?: boolean;
5568
+ }>;
5569
+ "addedAgents"?: Array<{
5570
+ "position": number;
5571
+ "addedAt": string;
5572
+ }>;
5573
+ }>;
5574
+ "contrastPlan"?: "vs_reference" | "all_pairs";
5575
+ "cellPlan"?: "crossed" | "per_setup";
5576
+ "skillDisclosure"?: "withheld" | "named";
5577
+ "modeOptions"?: Record<string, unknown>;
4711
5578
  };
4712
5579
  "tasks": Array<{
4713
5580
  "id": string;
@@ -4724,7 +5591,7 @@ export type StudyGetResponse = {
4724
5591
  "runs": Array<{
4725
5592
  "runId": string;
4726
5593
  "taskId": string;
4727
- "arm": "baseline" | "skill";
5594
+ "arm": "baseline" | "skill" | string;
4728
5595
  "harness": string;
4729
5596
  "model": string;
4730
5597
  "provider": string | null;
@@ -4820,9 +5687,24 @@ export type StudyGetResponse = {
4820
5687
  "gradedAt": string;
4821
5688
  };
4822
5689
  } | null;
5690
+ "executionBudgetS"?: number | null;
5691
+ "harnessImageDigest"?: string | null;
5692
+ "subjectEngaged"?: boolean | null;
4823
5693
  "runCreatedAt": string | null;
4824
5694
  "startedAt": string | null;
4825
5695
  "finishedAt": string | null;
5696
+ "contrastVerdicts"?: Array<{
5697
+ "referenceArm": string;
5698
+ "subjectArm": string;
5699
+ "referenceRunId": string;
5700
+ "verdict": "subject" | "reference" | "tie" | "unmeasurable";
5701
+ "votes": Array<{
5702
+ "judge": string;
5703
+ "ordering": "base-first" | "skill-first";
5704
+ "answer": "A" | "B" | "tie";
5705
+ }>;
5706
+ "gradedAt": string;
5707
+ }>;
4826
5708
  }>;
4827
5709
  "aggregate": {
4828
5710
  "summary": {
@@ -5029,6 +5911,17 @@ export type StudyGetResponse = {
5029
5911
  "ci": Array<unknown> | null;
5030
5912
  "n": number;
5031
5913
  };
5914
+ "abstention"?: {
5915
+ "value": number | null;
5916
+ "ci": Array<unknown> | null;
5917
+ "n": number;
5918
+ "parts"?: Array<{
5919
+ "id": string;
5920
+ "label": string;
5921
+ "value": number | null;
5922
+ "n": number;
5923
+ }>;
5924
+ };
5032
5925
  "tasks": number;
5033
5926
  "graded": number;
5034
5927
  "unmeasured": number;
@@ -5040,6 +5933,17 @@ export type StudyGetResponse = {
5040
5933
  "ci": Array<unknown> | null;
5041
5934
  "n": number;
5042
5935
  };
5936
+ "abstention"?: {
5937
+ "value": number | null;
5938
+ "ci": Array<unknown> | null;
5939
+ "n": number;
5940
+ "parts"?: Array<{
5941
+ "id": string;
5942
+ "label": string;
5943
+ "value": number | null;
5944
+ "n": number;
5945
+ }>;
5946
+ };
5043
5947
  "graded": number;
5044
5948
  "unmeasured": number;
5045
5949
  "modelJudged": boolean;
@@ -5059,60 +5963,1195 @@ export type StudyGetResponse = {
5059
5963
  "languages": Array<string>;
5060
5964
  "unavailableReasons": Array<string>;
5061
5965
  };
5062
- };
5063
- "warnings": Array<{
5064
- "type": "eval_capability_gap";
5065
- "harness": string;
5066
- "kind": string;
5067
5966
  } | {
5068
- "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
5069
- "detail": string;
5070
- "taskIds"?: Array<string>;
5071
- }>;
5072
- "shareUrl"?: string | null;
5073
- "canManageSharing"?: boolean;
5074
- };
5075
- export type StudyDeleteResponse = unknown;
5076
- export type StudyRegenerateSummaryResponse = {
5077
- "status": "queued";
5078
- };
5079
- export type StudyGenerateRequest = {
5080
- "proposeExpectations"?: boolean;
5081
- };
5082
- export type StudyGenerateResponse = {
5083
- "study": {
5084
- "id": string;
5085
- "executiveSummary"?: string | null;
5086
- "plannedRunCount": number;
5087
- "organizationId": string;
5088
- "createdBy": string | null;
5089
- "name": string;
5090
- "slug": string;
5091
- "description": string | null;
5092
- "skillRef": string | null;
5093
- "skillSlug": string | null;
5094
- "skillName": string | null;
5095
- "skillPromise": string;
5096
- "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5097
- "subjectName": string | null;
5098
- "subjectLabel": string;
5099
- "armConfig": {
5100
- "baseline"?: {
5101
- "tools"?: "default" | "none" | {
5102
- "tools": Array<string>;
5967
+ "shape": "multi-arm";
5968
+ "summary": {
5969
+ "shape": "multi-arm";
5970
+ "verdict": "measured" | "inconclusive";
5971
+ "groups": Array<{
5972
+ "key": string;
5973
+ "label": string;
5974
+ "role": "reference" | "peer";
5975
+ "composition": {
5976
+ "tools"?: "default" | "none" | {
5977
+ "tools": Array<string>;
5978
+ };
5979
+ "addons"?: Array<string>;
5103
5980
  };
5104
- "addons"?: Record<string, unknown>;
5105
- };
5106
- "treatment"?: {
5107
- "tools"?: "default" | "none" | {
5108
- "tools": Array<string>;
5981
+ }>;
5982
+ "ranking": {
5983
+ "plan": "vs_reference" | "all_pairs" | null;
5984
+ "referenceArm": string | null;
5985
+ "decision": {
5986
+ "leader": string | null;
5987
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5988
+ "reason": string;
5989
+ "separatedFrom": Array<string>;
5990
+ "lostTo": Array<{
5991
+ "arm": string;
5992
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5993
+ }>;
5994
+ "clearLeader": boolean;
5995
+ "correctnessLeader": string | null;
5109
5996
  };
5110
- "addons"?: Record<string, unknown>;
5997
+ "entries": Array<{
5998
+ "arm": string;
5999
+ "runs": number;
6000
+ "completed": number;
6001
+ "completionRate": number | null;
6002
+ "goalRate": number | null;
6003
+ "avgQualityScore": number | null;
6004
+ "evalPassRate": number | null;
6005
+ "benchmarkPassRate"?: number | null;
6006
+ "compositeScore": number | null;
6007
+ "avgTotalTokens": number | null;
6008
+ "avgOutputTokens": number | null;
6009
+ "avgCostUsd": number | null;
6010
+ "avgDurationMs": number | null;
6011
+ "medianCostUsd": number | null;
6012
+ "medianTotalTokens": number | null;
6013
+ "medianDurationMs": number | null;
6014
+ "rank": number;
6015
+ }>;
6016
+ "contrasts": Array<{
6017
+ "contrastId": string;
6018
+ "referenceArm": string;
6019
+ "subjectArm": string;
6020
+ "oriented": boolean;
6021
+ "outcome": "won" | "tie" | "unmeasurable";
6022
+ "winner": string | null;
6023
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
6024
+ "explanation": string;
6025
+ }>;
6026
+ "comparisons": number;
6027
+ "headline"?: string;
5111
6028
  };
5112
- };
5113
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5114
- "subjectBackend": "model" | "harness" | null;
5115
- "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6029
+ "matrixShape": {
6030
+ "cells": number;
6031
+ "harnesses": number;
6032
+ "models": number;
6033
+ "crossed": boolean;
6034
+ };
6035
+ "flakiness": {
6036
+ "measured": boolean;
6037
+ "conditions": number;
6038
+ "unstable": number;
6039
+ "worst": {
6040
+ "cell": string;
6041
+ "taskId": string;
6042
+ "arm": string;
6043
+ "variant"?: number;
6044
+ "passRate": number;
6045
+ } | null;
6046
+ };
6047
+ "totals": {
6048
+ "cells": number;
6049
+ "runs": number;
6050
+ "completed": number;
6051
+ "failed": number;
6052
+ "costUsd": number;
6053
+ "tokens": number;
6054
+ };
6055
+ };
6056
+ "arms": Array<{
6057
+ "key": string;
6058
+ "label": string;
6059
+ "position": number;
6060
+ "role": "reference" | "peer";
6061
+ "composition": {
6062
+ "tools"?: "default" | "none" | {
6063
+ "tools": Array<string>;
6064
+ };
6065
+ "addons"?: Array<string>;
6066
+ };
6067
+ "agg": {
6068
+ "runs": number;
6069
+ "completed": number;
6070
+ "infraFailures": number;
6071
+ "completionRate": number | null;
6072
+ "goalRate": number | null;
6073
+ "goalTally": {
6074
+ "achieved": number;
6075
+ "partial": number;
6076
+ "missed": number;
6077
+ };
6078
+ "avgTotalTokens": number | null;
6079
+ "avgOutputTokens": number | null;
6080
+ "avgCostUsd": number | null;
6081
+ "avgDurationMs": number | null;
6082
+ "avgTurns": number | null;
6083
+ "costCoverage": number | null;
6084
+ "avgQualityScore": number | null;
6085
+ "gradedCoverage": number | null;
6086
+ "medianCostUsd": number | null;
6087
+ "medianTotalTokens": number | null;
6088
+ "medianDurationMs": number | null;
6089
+ "p95CostUsd": number | null;
6090
+ "p95TotalTokens": number | null;
6091
+ "p95DurationMs": number | null;
6092
+ "maxCostUsd": number | null;
6093
+ "maxTotalTokens": number | null;
6094
+ "maxDurationMs": number | null;
6095
+ "evalPassRate"?: number | null;
6096
+ "benchmarkPassRate"?: number | null;
6097
+ "adoptionRate"?: number | null;
6098
+ };
6099
+ "perCell": Array<{
6100
+ "harness": string;
6101
+ "model": string;
6102
+ "provider": string | null;
6103
+ "paramsKey"?: string;
6104
+ "label"?: string;
6105
+ "agg": {
6106
+ "runs": number;
6107
+ "completed": number;
6108
+ "infraFailures": number;
6109
+ "completionRate": number | null;
6110
+ "goalRate": number | null;
6111
+ "goalTally": {
6112
+ "achieved": number;
6113
+ "partial": number;
6114
+ "missed": number;
6115
+ };
6116
+ "avgTotalTokens": number | null;
6117
+ "avgOutputTokens": number | null;
6118
+ "avgCostUsd": number | null;
6119
+ "avgDurationMs": number | null;
6120
+ "avgTurns": number | null;
6121
+ "costCoverage": number | null;
6122
+ "avgQualityScore": number | null;
6123
+ "gradedCoverage": number | null;
6124
+ "medianCostUsd": number | null;
6125
+ "medianTotalTokens": number | null;
6126
+ "medianDurationMs": number | null;
6127
+ "p95CostUsd": number | null;
6128
+ "p95TotalTokens": number | null;
6129
+ "p95DurationMs": number | null;
6130
+ "maxCostUsd": number | null;
6131
+ "maxTotalTokens": number | null;
6132
+ "maxDurationMs": number | null;
6133
+ "evalPassRate"?: number | null;
6134
+ "benchmarkPassRate"?: number | null;
6135
+ "adoptionRate"?: number | null;
6136
+ };
6137
+ }>;
6138
+ "reference"?: {
6139
+ "cells": Array<{
6140
+ "cell": string;
6141
+ "harness": string;
6142
+ "model": string;
6143
+ "provider": string | null;
6144
+ "paramsKey": string;
6145
+ "label": string;
6146
+ "passRate": {
6147
+ "value": number | null;
6148
+ "ci": Array<unknown> | null;
6149
+ "n": number;
6150
+ };
6151
+ "abstention"?: {
6152
+ "value": number | null;
6153
+ "ci": Array<unknown> | null;
6154
+ "n": number;
6155
+ "parts"?: Array<{
6156
+ "id": string;
6157
+ "label": string;
6158
+ "value": number | null;
6159
+ "n": number;
6160
+ }>;
6161
+ };
6162
+ "tasks": number;
6163
+ "graded": number;
6164
+ "unmeasured": number;
6165
+ "matchers": Array<string>;
6166
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
6167
+ }>;
6168
+ "overall": {
6169
+ "value": number | null;
6170
+ "ci": Array<unknown> | null;
6171
+ "n": number;
6172
+ };
6173
+ "abstention"?: {
6174
+ "value": number | null;
6175
+ "ci": Array<unknown> | null;
6176
+ "n": number;
6177
+ "parts"?: Array<{
6178
+ "id": string;
6179
+ "label": string;
6180
+ "value": number | null;
6181
+ "n": number;
6182
+ }>;
6183
+ };
6184
+ "graded": number;
6185
+ "unmeasured": number;
6186
+ "modelJudged": boolean;
6187
+ };
6188
+ }>;
6189
+ "contrasts": Array<{
6190
+ "contrastId": string;
6191
+ "referenceArm": string;
6192
+ "subjectArm": string;
6193
+ "oriented": boolean;
6194
+ "basis"?: {
6195
+ "id": "matched" | "whole_setup";
6196
+ "compared": {
6197
+ "reference": Array<{
6198
+ "harness": string;
6199
+ "model": string;
6200
+ "provider"?: string;
6201
+ "params"?: Record<string, unknown>;
6202
+ }>;
6203
+ "subject": Array<{
6204
+ "harness": string;
6205
+ "model": string;
6206
+ "provider"?: string;
6207
+ "params"?: Record<string, unknown>;
6208
+ }>;
6209
+ };
6210
+ "excluded": {
6211
+ "reference": Array<{
6212
+ "harness": string;
6213
+ "model": string;
6214
+ "provider"?: string;
6215
+ "params"?: Record<string, unknown>;
6216
+ }>;
6217
+ "subject": Array<{
6218
+ "harness": string;
6219
+ "model": string;
6220
+ "provider"?: string;
6221
+ "params"?: Record<string, unknown>;
6222
+ }>;
6223
+ };
6224
+ "differsIn": Array<string>;
6225
+ "licensedFacet": string | null;
6226
+ "efficiency": "comparable" | "incomparable";
6227
+ "pairedTasks": number | null;
6228
+ "dropped": {
6229
+ "reference": number;
6230
+ "subject": number;
6231
+ } | null;
6232
+ };
6233
+ "magnitudes": "relative";
6234
+ "summary": {
6235
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
6236
+ "headline": Array<{
6237
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6238
+ "label": string;
6239
+ "direction": "down" | "up";
6240
+ "baselineAvg": number | null;
6241
+ "skillAvg": number | null;
6242
+ "deltaPct": number | null;
6243
+ "delta"?: {
6244
+ "unit": "pp";
6245
+ "absolute": number | null;
6246
+ "relativePct": number | null;
6247
+ } | {
6248
+ "unit": "pts";
6249
+ "absolute": number | null;
6250
+ "relativePct"?: number | null;
6251
+ } | {
6252
+ "unit": "%";
6253
+ "relativePct": number | null;
6254
+ };
6255
+ "improved": boolean;
6256
+ }>;
6257
+ "completionRate": {
6258
+ "baseline": number | null;
6259
+ "skill": number | null;
6260
+ };
6261
+ "goalRate": {
6262
+ "baseline": number | null;
6263
+ "skill": number | null;
6264
+ };
6265
+ "pairwise": {
6266
+ "pairs": number;
6267
+ "skillWins": number;
6268
+ "baselineWins": number;
6269
+ "ties": number;
6270
+ "winRate": number | null;
6271
+ };
6272
+ "decision"?: {
6273
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
6274
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
6275
+ "reason": string;
6276
+ "efficiencyDiscounted": boolean;
6277
+ };
6278
+ "concentration": {
6279
+ "perCell": Array<{
6280
+ "cell": string;
6281
+ "goalDeltaPp": number | null;
6282
+ "contributionPct": number | null;
6283
+ }>;
6284
+ "heterogeneity": {
6285
+ "dominantCell": string;
6286
+ "contributionPct": number;
6287
+ } | null;
6288
+ };
6289
+ "matrixShape": {
6290
+ "cells": number;
6291
+ "harnesses": number;
6292
+ "models": number;
6293
+ "crossed": boolean;
6294
+ };
6295
+ "flakiness"?: {
6296
+ "measured": boolean;
6297
+ "conditions": number;
6298
+ "unstable": number;
6299
+ "worst": {
6300
+ "cell": string;
6301
+ "taskId": string;
6302
+ "arm": "baseline" | "skill";
6303
+ "variant"?: number;
6304
+ "passRate": number;
6305
+ } | null;
6306
+ };
6307
+ "arms"?: {
6308
+ "baseline": {
6309
+ "runs": number;
6310
+ "completed": number;
6311
+ "infraFailures": number;
6312
+ "completionRate": number | null;
6313
+ "goalRate": number | null;
6314
+ "goalTally": {
6315
+ "achieved": number;
6316
+ "partial": number;
6317
+ "missed": number;
6318
+ };
6319
+ "avgTotalTokens": number | null;
6320
+ "avgOutputTokens": number | null;
6321
+ "avgCostUsd": number | null;
6322
+ "avgDurationMs": number | null;
6323
+ "avgTurns": number | null;
6324
+ "costCoverage": number | null;
6325
+ "avgQualityScore": number | null;
6326
+ "gradedCoverage": number | null;
6327
+ "medianCostUsd": number | null;
6328
+ "medianTotalTokens": number | null;
6329
+ "medianDurationMs": number | null;
6330
+ "p95CostUsd": number | null;
6331
+ "p95TotalTokens": number | null;
6332
+ "p95DurationMs": number | null;
6333
+ "maxCostUsd": number | null;
6334
+ "maxTotalTokens": number | null;
6335
+ "maxDurationMs": number | null;
6336
+ "evalPassRate"?: number | null;
6337
+ "benchmarkPassRate"?: number | null;
6338
+ "adoptionRate"?: number | null;
6339
+ };
6340
+ "skill": {
6341
+ "runs": number;
6342
+ "completed": number;
6343
+ "infraFailures": number;
6344
+ "completionRate": number | null;
6345
+ "goalRate": number | null;
6346
+ "goalTally": {
6347
+ "achieved": number;
6348
+ "partial": number;
6349
+ "missed": number;
6350
+ };
6351
+ "avgTotalTokens": number | null;
6352
+ "avgOutputTokens": number | null;
6353
+ "avgCostUsd": number | null;
6354
+ "avgDurationMs": number | null;
6355
+ "avgTurns": number | null;
6356
+ "costCoverage": number | null;
6357
+ "avgQualityScore": number | null;
6358
+ "gradedCoverage": number | null;
6359
+ "medianCostUsd": number | null;
6360
+ "medianTotalTokens": number | null;
6361
+ "medianDurationMs": number | null;
6362
+ "p95CostUsd": number | null;
6363
+ "p95TotalTokens": number | null;
6364
+ "p95DurationMs": number | null;
6365
+ "maxCostUsd": number | null;
6366
+ "maxTotalTokens": number | null;
6367
+ "maxDurationMs": number | null;
6368
+ "evalPassRate"?: number | null;
6369
+ "benchmarkPassRate"?: number | null;
6370
+ "adoptionRate"?: number | null;
6371
+ };
6372
+ };
6373
+ "totals": {
6374
+ "cells": number;
6375
+ "runs": number;
6376
+ "completed": number;
6377
+ "failed": number;
6378
+ "costUsd": number;
6379
+ "tokens": number;
6380
+ };
6381
+ };
6382
+ "perCell": Array<{
6383
+ "harness": string;
6384
+ "model": string;
6385
+ "provider": string | null;
6386
+ "paramsKey"?: string;
6387
+ "label"?: string;
6388
+ "baseline": {
6389
+ "runs": number;
6390
+ "completed": number;
6391
+ "infraFailures": number;
6392
+ "completionRate": number | null;
6393
+ "goalRate": number | null;
6394
+ "goalTally": {
6395
+ "achieved": number;
6396
+ "partial": number;
6397
+ "missed": number;
6398
+ };
6399
+ "avgTotalTokens": number | null;
6400
+ "avgOutputTokens": number | null;
6401
+ "avgCostUsd": number | null;
6402
+ "avgDurationMs": number | null;
6403
+ "avgTurns": number | null;
6404
+ "costCoverage": number | null;
6405
+ "avgQualityScore": number | null;
6406
+ "gradedCoverage": number | null;
6407
+ "medianCostUsd": number | null;
6408
+ "medianTotalTokens": number | null;
6409
+ "medianDurationMs": number | null;
6410
+ "p95CostUsd": number | null;
6411
+ "p95TotalTokens": number | null;
6412
+ "p95DurationMs": number | null;
6413
+ "maxCostUsd": number | null;
6414
+ "maxTotalTokens": number | null;
6415
+ "maxDurationMs": number | null;
6416
+ "evalPassRate"?: number | null;
6417
+ "benchmarkPassRate"?: number | null;
6418
+ "adoptionRate"?: number | null;
6419
+ };
6420
+ "skill": {
6421
+ "runs": number;
6422
+ "completed": number;
6423
+ "infraFailures": number;
6424
+ "completionRate": number | null;
6425
+ "goalRate": number | null;
6426
+ "goalTally": {
6427
+ "achieved": number;
6428
+ "partial": number;
6429
+ "missed": number;
6430
+ };
6431
+ "avgTotalTokens": number | null;
6432
+ "avgOutputTokens": number | null;
6433
+ "avgCostUsd": number | null;
6434
+ "avgDurationMs": number | null;
6435
+ "avgTurns": number | null;
6436
+ "costCoverage": number | null;
6437
+ "avgQualityScore": number | null;
6438
+ "gradedCoverage": number | null;
6439
+ "medianCostUsd": number | null;
6440
+ "medianTotalTokens": number | null;
6441
+ "medianDurationMs": number | null;
6442
+ "p95CostUsd": number | null;
6443
+ "p95TotalTokens": number | null;
6444
+ "p95DurationMs": number | null;
6445
+ "maxCostUsd": number | null;
6446
+ "maxTotalTokens": number | null;
6447
+ "maxDurationMs": number | null;
6448
+ "evalPassRate"?: number | null;
6449
+ "benchmarkPassRate"?: number | null;
6450
+ "adoptionRate"?: number | null;
6451
+ };
6452
+ "deltas": {
6453
+ "totalTokensPct": number | null;
6454
+ "outputTokensPct": number | null;
6455
+ "costUsdPct": number | null;
6456
+ "durationMsPct": number | null;
6457
+ "numTurnsPct": number | null;
6458
+ "completionRatePp": number | null;
6459
+ "goalRatePp": number | null;
6460
+ "qualityScorePts": number | null;
6461
+ "evalPassRatePp"?: number | null;
6462
+ };
6463
+ }>;
6464
+ } | {
6465
+ "contrastId": string;
6466
+ "referenceArm": string;
6467
+ "subjectArm": string;
6468
+ "oriented": boolean;
6469
+ "basis"?: {
6470
+ "id": "matched" | "whole_setup";
6471
+ "compared": {
6472
+ "reference": Array<{
6473
+ "harness": string;
6474
+ "model": string;
6475
+ "provider"?: string;
6476
+ "params"?: Record<string, unknown>;
6477
+ }>;
6478
+ "subject": Array<{
6479
+ "harness": string;
6480
+ "model": string;
6481
+ "provider"?: string;
6482
+ "params"?: Record<string, unknown>;
6483
+ }>;
6484
+ };
6485
+ "excluded": {
6486
+ "reference": Array<{
6487
+ "harness": string;
6488
+ "model": string;
6489
+ "provider"?: string;
6490
+ "params"?: Record<string, unknown>;
6491
+ }>;
6492
+ "subject": Array<{
6493
+ "harness": string;
6494
+ "model": string;
6495
+ "provider"?: string;
6496
+ "params"?: Record<string, unknown>;
6497
+ }>;
6498
+ };
6499
+ "differsIn": Array<string>;
6500
+ "licensedFacet": string | null;
6501
+ "efficiency": "comparable" | "incomparable";
6502
+ "pairedTasks": number | null;
6503
+ "dropped": {
6504
+ "reference": number;
6505
+ "subject": number;
6506
+ } | null;
6507
+ };
6508
+ "magnitudes": "absolute";
6509
+ "summary": {
6510
+ "completionRate": {
6511
+ "baseline": number | null;
6512
+ "skill": number | null;
6513
+ };
6514
+ "goalRate": {
6515
+ "baseline": number | null;
6516
+ "skill": number | null;
6517
+ };
6518
+ "pairwise": {
6519
+ "pairs": number;
6520
+ "skillWins": number;
6521
+ "baselineWins": number;
6522
+ "ties": number;
6523
+ "winRate": number | null;
6524
+ };
6525
+ "decision"?: {
6526
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
6527
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
6528
+ "reason": string;
6529
+ "efficiencyDiscounted": boolean;
6530
+ };
6531
+ "concentration": {
6532
+ "perCell": Array<{
6533
+ "cell": string;
6534
+ "goalDeltaPp": number | null;
6535
+ "contributionPct": number | null;
6536
+ }>;
6537
+ "heterogeneity": {
6538
+ "dominantCell": string;
6539
+ "contributionPct": number;
6540
+ } | null;
6541
+ };
6542
+ "matrixShape": {
6543
+ "cells": number;
6544
+ "harnesses": number;
6545
+ "models": number;
6546
+ "crossed": boolean;
6547
+ };
6548
+ "flakiness"?: {
6549
+ "measured": boolean;
6550
+ "conditions": number;
6551
+ "unstable": number;
6552
+ "worst": {
6553
+ "cell": string;
6554
+ "taskId": string;
6555
+ "arm": "baseline" | "skill";
6556
+ "variant"?: number;
6557
+ "passRate": number;
6558
+ } | null;
6559
+ };
6560
+ "arms"?: {
6561
+ "baseline": {
6562
+ "runs": number;
6563
+ "completed": number;
6564
+ "infraFailures": number;
6565
+ "completionRate": number | null;
6566
+ "goalRate": number | null;
6567
+ "goalTally": {
6568
+ "achieved": number;
6569
+ "partial": number;
6570
+ "missed": number;
6571
+ };
6572
+ "avgTotalTokens": number | null;
6573
+ "avgOutputTokens": number | null;
6574
+ "avgCostUsd": number | null;
6575
+ "avgDurationMs": number | null;
6576
+ "avgTurns": number | null;
6577
+ "costCoverage": number | null;
6578
+ "avgQualityScore": number | null;
6579
+ "gradedCoverage": number | null;
6580
+ "medianCostUsd": number | null;
6581
+ "medianTotalTokens": number | null;
6582
+ "medianDurationMs": number | null;
6583
+ "p95CostUsd": number | null;
6584
+ "p95TotalTokens": number | null;
6585
+ "p95DurationMs": number | null;
6586
+ "maxCostUsd": number | null;
6587
+ "maxTotalTokens": number | null;
6588
+ "maxDurationMs": number | null;
6589
+ "evalPassRate"?: number | null;
6590
+ "benchmarkPassRate"?: number | null;
6591
+ "adoptionRate"?: number | null;
6592
+ };
6593
+ "skill": {
6594
+ "runs": number;
6595
+ "completed": number;
6596
+ "infraFailures": number;
6597
+ "completionRate": number | null;
6598
+ "goalRate": number | null;
6599
+ "goalTally": {
6600
+ "achieved": number;
6601
+ "partial": number;
6602
+ "missed": number;
6603
+ };
6604
+ "avgTotalTokens": number | null;
6605
+ "avgOutputTokens": number | null;
6606
+ "avgCostUsd": number | null;
6607
+ "avgDurationMs": number | null;
6608
+ "avgTurns": number | null;
6609
+ "costCoverage": number | null;
6610
+ "avgQualityScore": number | null;
6611
+ "gradedCoverage": number | null;
6612
+ "medianCostUsd": number | null;
6613
+ "medianTotalTokens": number | null;
6614
+ "medianDurationMs": number | null;
6615
+ "p95CostUsd": number | null;
6616
+ "p95TotalTokens": number | null;
6617
+ "p95DurationMs": number | null;
6618
+ "maxCostUsd": number | null;
6619
+ "maxTotalTokens": number | null;
6620
+ "maxDurationMs": number | null;
6621
+ "evalPassRate"?: number | null;
6622
+ "benchmarkPassRate"?: number | null;
6623
+ "adoptionRate"?: number | null;
6624
+ };
6625
+ };
6626
+ "totals": {
6627
+ "cells": number;
6628
+ "runs": number;
6629
+ "completed": number;
6630
+ "failed": number;
6631
+ "costUsd": number;
6632
+ "tokens": number;
6633
+ };
6634
+ };
6635
+ "perCell": Array<{
6636
+ "harness": string;
6637
+ "model": string;
6638
+ "provider": string | null;
6639
+ "paramsKey"?: string;
6640
+ "label"?: string;
6641
+ "baseline": {
6642
+ "runs": number;
6643
+ "completed": number;
6644
+ "infraFailures": number;
6645
+ "completionRate": number | null;
6646
+ "goalRate": number | null;
6647
+ "goalTally": {
6648
+ "achieved": number;
6649
+ "partial": number;
6650
+ "missed": number;
6651
+ };
6652
+ "avgTotalTokens": number | null;
6653
+ "avgOutputTokens": number | null;
6654
+ "avgCostUsd": number | null;
6655
+ "avgDurationMs": number | null;
6656
+ "avgTurns": number | null;
6657
+ "costCoverage": number | null;
6658
+ "avgQualityScore": number | null;
6659
+ "gradedCoverage": number | null;
6660
+ "medianCostUsd": number | null;
6661
+ "medianTotalTokens": number | null;
6662
+ "medianDurationMs": number | null;
6663
+ "p95CostUsd": number | null;
6664
+ "p95TotalTokens": number | null;
6665
+ "p95DurationMs": number | null;
6666
+ "maxCostUsd": number | null;
6667
+ "maxTotalTokens": number | null;
6668
+ "maxDurationMs": number | null;
6669
+ "evalPassRate"?: number | null;
6670
+ "benchmarkPassRate"?: number | null;
6671
+ "adoptionRate"?: number | null;
6672
+ };
6673
+ "skill": {
6674
+ "runs": number;
6675
+ "completed": number;
6676
+ "infraFailures": number;
6677
+ "completionRate": number | null;
6678
+ "goalRate": number | null;
6679
+ "goalTally": {
6680
+ "achieved": number;
6681
+ "partial": number;
6682
+ "missed": number;
6683
+ };
6684
+ "avgTotalTokens": number | null;
6685
+ "avgOutputTokens": number | null;
6686
+ "avgCostUsd": number | null;
6687
+ "avgDurationMs": number | null;
6688
+ "avgTurns": number | null;
6689
+ "costCoverage": number | null;
6690
+ "avgQualityScore": number | null;
6691
+ "gradedCoverage": number | null;
6692
+ "medianCostUsd": number | null;
6693
+ "medianTotalTokens": number | null;
6694
+ "medianDurationMs": number | null;
6695
+ "p95CostUsd": number | null;
6696
+ "p95TotalTokens": number | null;
6697
+ "p95DurationMs": number | null;
6698
+ "maxCostUsd": number | null;
6699
+ "maxTotalTokens": number | null;
6700
+ "maxDurationMs": number | null;
6701
+ "evalPassRate"?: number | null;
6702
+ "benchmarkPassRate"?: number | null;
6703
+ "adoptionRate"?: number | null;
6704
+ };
6705
+ "deltas": {
6706
+ "completionRatePp": number | null;
6707
+ "goalRatePp": number | null;
6708
+ "qualityScorePts": number | null;
6709
+ "evalPassRatePp"?: number | null;
6710
+ };
6711
+ }>;
6712
+ }>;
6713
+ "board": Array<{
6714
+ "harness": string;
6715
+ "model": string;
6716
+ "provider": string | null;
6717
+ "paramsKey"?: string;
6718
+ "label"?: string;
6719
+ "arm": string;
6720
+ "runs": number;
6721
+ "completed": number;
6722
+ "completionRate": number | null;
6723
+ "goalRate": number | null;
6724
+ "avgQualityScore": number | null;
6725
+ "evalPassRate": number | null;
6726
+ "benchmarkPassRate"?: number | null;
6727
+ "compositeScore": number | null;
6728
+ "avgTotalTokens": number | null;
6729
+ "avgOutputTokens": number | null;
6730
+ "avgCostUsd": number | null;
6731
+ "avgDurationMs": number | null;
6732
+ "medianCostUsd": number | null;
6733
+ "medianTotalTokens": number | null;
6734
+ "medianDurationMs": number | null;
6735
+ "rank": number;
6736
+ }>;
6737
+ "setupsBoard"?: Array<{
6738
+ "harness": string;
6739
+ "model": string;
6740
+ "provider": string | null;
6741
+ "paramsKey"?: string;
6742
+ "label"?: string;
6743
+ "arm": string;
6744
+ "runs": number;
6745
+ "completed": number;
6746
+ "completionRate": number | null;
6747
+ "goalRate": number | null;
6748
+ "avgQualityScore": number | null;
6749
+ "evalPassRate": number | null;
6750
+ "benchmarkPassRate"?: number | null;
6751
+ "compositeScore": number | null;
6752
+ "avgTotalTokens": number | null;
6753
+ "avgOutputTokens": number | null;
6754
+ "avgCostUsd": number | null;
6755
+ "avgDurationMs": number | null;
6756
+ "medianCostUsd": number | null;
6757
+ "medianTotalTokens": number | null;
6758
+ "medianDurationMs": number | null;
6759
+ "rank": number;
6760
+ "group": string;
6761
+ "agent": string;
6762
+ "setup": string;
6763
+ }>;
6764
+ "perTask": Array<{
6765
+ "taskId": string;
6766
+ "position": number;
6767
+ "promptPreview": string;
6768
+ "requiresWeb": boolean;
6769
+ "category": string | null;
6770
+ "byArm": Record<string, unknown>;
6771
+ }>;
6772
+ "behavior"?: Array<{
6773
+ "harness": string;
6774
+ "model": string;
6775
+ "provider": string | null;
6776
+ "arm": string;
6777
+ "totalRuns": number;
6778
+ "signals": Array<{
6779
+ "id": string;
6780
+ "label": string;
6781
+ "tone": "good" | "bad" | "neutral";
6782
+ "runCount": number;
6783
+ "succeededCount": number;
6784
+ }>;
6785
+ }>;
6786
+ "execution"?: Record<string, unknown>;
6787
+ "inference"?: {
6788
+ "contrasts": Array<{
6789
+ "contrastId": string;
6790
+ "table": {
6791
+ "columns": Array<{
6792
+ "key": string;
6793
+ "label": string;
6794
+ "family": "rate" | "score" | "magnitude";
6795
+ "betterWhen": "higher" | "lower";
6796
+ }>;
6797
+ "groups": Array<{
6798
+ "kind": "cell" | "harness" | "model" | "field";
6799
+ "title": string;
6800
+ "rows": Array<{
6801
+ "key": string;
6802
+ "kind": "cell" | "harness" | "model" | "field";
6803
+ "harness": string | null;
6804
+ "model": string | null;
6805
+ "paramsKey"?: string;
6806
+ "runs": number;
6807
+ "cells": Record<string, unknown>;
6808
+ }>;
6809
+ }>;
6810
+ "reference": "baseline" | "field";
6811
+ "draws": number;
6812
+ "adjustment": {
6813
+ "scope": "planned-comparisons";
6814
+ "comparisons": number;
6815
+ };
6816
+ } | null;
6817
+ }>;
6818
+ "holm": {
6819
+ "status": "complete";
6820
+ "ready": number;
6821
+ "planned": number;
6822
+ "adjustment": {
6823
+ "scope": "planned-comparisons";
6824
+ "comparisons": number;
6825
+ };
6826
+ } | {
6827
+ "status": "pending";
6828
+ "ready": number;
6829
+ "planned": number;
6830
+ };
6831
+ "claims": {
6832
+ "perContrast": Record<string, unknown>;
6833
+ "ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
6834
+ };
6835
+ };
6836
+ };
6837
+ "warnings": Array<{
6838
+ "type": "eval_capability_gap";
6839
+ "harness": string;
6840
+ "kind": string;
6841
+ } | {
6842
+ "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
6843
+ "detail": string;
6844
+ "taskIds"?: Array<string>;
6845
+ } | {
6846
+ "type": "design_aggregate_budget";
6847
+ "detail": string;
6848
+ }>;
6849
+ "taskHealth": {
6850
+ "flags": Array<{
6851
+ "taskId": string;
6852
+ "signal": string;
6853
+ "kind": "floor" | "ceiling";
6854
+ "conditions": number;
6855
+ }>;
6856
+ "rubricSplits": Array<{
6857
+ "taskId": string;
6858
+ "criterionIndex": number;
6859
+ "criterion": string;
6860
+ "split": number;
6861
+ "graded": number;
6862
+ }>;
6863
+ "signals": Array<{
6864
+ "id": string;
6865
+ "label": string;
6866
+ }>;
6867
+ };
6868
+ "shareUrl"?: string | null;
6869
+ "canManageSharing"?: boolean;
6870
+ };
6871
+ export type StudyUpdateDetailsRequest = {
6872
+ "name"?: string;
6873
+ "description"?: string;
6874
+ };
6875
+ export type StudyUpdateDetailsResponse = {
6876
+ "study": {
6877
+ "id": string;
6878
+ "executiveSummary"?: string | null;
6879
+ "plannedRunCount": number;
6880
+ "organizationId": string;
6881
+ "createdBy": string | null;
6882
+ "name": string;
6883
+ "slug": string;
6884
+ "description": string | null;
6885
+ "skillRef": string | null;
6886
+ "skillSlug": string | null;
6887
+ "skillName": string | null;
6888
+ "skillPromise": string;
6889
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
6890
+ "subjectName": string | null;
6891
+ "subjectLabel": string;
6892
+ "armConfig": {
6893
+ "baseline"?: {
6894
+ "tools"?: "default" | "none" | {
6895
+ "tools": Array<string>;
6896
+ };
6897
+ "addons"?: Record<string, unknown>;
6898
+ };
6899
+ "treatment"?: {
6900
+ "tools"?: "default" | "none" | {
6901
+ "tools": Array<string>;
6902
+ };
6903
+ "addons"?: Record<string, unknown>;
6904
+ };
6905
+ };
6906
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6907
+ "subjectBackend": "model" | "harness" | null;
6908
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6909
+ "importedFrom"?: string;
6910
+ "baselineLabel": string | null;
6911
+ "treatmentLabel": string | null;
6912
+ "promiseMetrics": Array<{
6913
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6914
+ "direction": "down" | "up";
6915
+ "label": string;
6916
+ }>;
6917
+ "matrix": Array<{
6918
+ "harness": string;
6919
+ "model": string;
6920
+ "provider"?: string;
6921
+ "params"?: {
6922
+ "temperature"?: number;
6923
+ "maxOutputTokens"?: number;
6924
+ "reasoningEffort"?: "low" | "medium" | "high";
6925
+ "thinkingBudgetTokens"?: number;
6926
+ };
6927
+ }>;
6928
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
6929
+ "judgeSupplementary"?: boolean;
6930
+ "taskCount": number;
6931
+ "variantsPerTask": number;
6932
+ "repeats": number;
6933
+ "subset": {
6934
+ "count": number;
6935
+ "seed": string;
6936
+ "of": number;
6937
+ } | null;
6938
+ "status": string;
6939
+ "error": string | null;
6940
+ "preparation"?: {
6941
+ "since": string;
6942
+ "progressAt"?: string;
6943
+ "launchedBy": {
6944
+ "userId"?: string;
6945
+ "role"?: string;
6946
+ };
6947
+ "waitingOn": Array<{
6948
+ "imageKey": string;
6949
+ "buildId": string;
6950
+ "stage"?: "image" | "snapshot";
6951
+ "retry"?: {
6952
+ "failedAttempts": number;
6953
+ "maxAttempts": number;
6954
+ "notBefore": string;
6955
+ "lastFailure": string;
6956
+ };
6957
+ }>;
6958
+ } | null;
6959
+ "visibility": "private" | "org" | "link" | "public";
6960
+ "visibilityChangedAt"?: string | null;
6961
+ "visibilityChangedBy"?: string | null;
6962
+ "shareToken"?: string | null;
6963
+ "lowSampleOverride": boolean;
6964
+ "publishedAt": string | null;
6965
+ "verdict": unknown;
6966
+ "definitionVersionId": string | null;
6967
+ "runnerKind": "ora-runs" | "batch-job";
6968
+ "createdAt": string;
6969
+ "updatedAt": string;
6970
+ "liveUntil": string | null;
6971
+ "launch"?: {
6972
+ "attempt": number;
6973
+ "state": "fanning_out" | "deferred" | "settled";
6974
+ "startedAt": string;
6975
+ "progressAt": string;
6976
+ "planned"?: number;
6977
+ "created"?: number;
6978
+ "missing"?: {
6979
+ "retryable": number;
6980
+ "refused": number;
6981
+ };
6982
+ "notes"?: Array<string>;
6983
+ "settledAt"?: string;
6984
+ } | null;
6985
+ "relaunchable"?: boolean;
6986
+ "arms"?: Array<{
6987
+ "key": string;
6988
+ "position": number;
6989
+ "label": string;
6990
+ "role": "reference" | "peer";
6991
+ "payload": {
6992
+ "tools"?: "default" | "none" | {
6993
+ "tools": Array<string>;
6994
+ };
6995
+ "addons"?: Record<string, unknown>;
6996
+ };
6997
+ "cells"?: Array<{
6998
+ "harness": string;
6999
+ "model": string;
7000
+ "provider"?: string;
7001
+ "params"?: {
7002
+ "temperature"?: number;
7003
+ "maxOutputTokens"?: number;
7004
+ "reasoningEffort"?: "low" | "medium" | "high";
7005
+ "thinkingBudgetTokens"?: number;
7006
+ };
7007
+ }>;
7008
+ "skills"?: Array<{
7009
+ "ref": string;
7010
+ "name": string;
7011
+ "changedSinceCreate"?: boolean;
7012
+ }>;
7013
+ "addedAgents"?: Array<{
7014
+ "position": number;
7015
+ "addedAt": string;
7016
+ }>;
7017
+ }>;
7018
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7019
+ "cellPlan"?: "crossed" | "per_setup";
7020
+ "skillDisclosure"?: "withheld" | "named";
7021
+ "modeOptions"?: Record<string, unknown>;
7022
+ };
7023
+ };
7024
+ export type StudyDeleteResponse = unknown;
7025
+ export type StudyAddAgentsRequest = {
7026
+ "group"?: string;
7027
+ "agents": Array<{
7028
+ "harness": string;
7029
+ "model": string;
7030
+ "provider"?: string;
7031
+ "params"?: {
7032
+ "temperature"?: number;
7033
+ "maxOutputTokens"?: number;
7034
+ "reasoningEffort"?: "low" | "medium" | "high";
7035
+ "thinkingBudgetTokens"?: number;
7036
+ };
7037
+ }>;
7038
+ "dryRun"?: boolean;
7039
+ };
7040
+ export type StudyAddAgentsResponse = {
7041
+ "studyId": string;
7042
+ "dryRun": boolean;
7043
+ "status": string;
7044
+ "reopened": boolean;
7045
+ "cellPlan": string;
7046
+ "added": Array<{
7047
+ "group": string | null;
7048
+ "harness": string;
7049
+ "model": string;
7050
+ "provider": string | null;
7051
+ "paramsKey": string;
7052
+ "position": number;
7053
+ }>;
7054
+ "alreadyPresent": Array<{
7055
+ "group": string | null;
7056
+ "harness": string;
7057
+ "model": string;
7058
+ "provider": string | null;
7059
+ "paramsKey": string;
7060
+ }>;
7061
+ "matrix": {
7062
+ "before": number;
7063
+ "after": number;
7064
+ };
7065
+ "jobs": {
7066
+ "plannedBefore": number;
7067
+ "plannedAfter": number;
7068
+ "toCreate": number;
7069
+ "toCreateByGroup": Record<string, unknown>;
7070
+ };
7071
+ "contrasts": Array<{
7072
+ "reference": string;
7073
+ "subject": string;
7074
+ "before": {
7075
+ "basis": string;
7076
+ "sharedAxes": Array<string>;
7077
+ "compared": {
7078
+ "reference": number;
7079
+ "subject": number;
7080
+ };
7081
+ };
7082
+ "after": {
7083
+ "basis": string;
7084
+ "sharedAxes": Array<string>;
7085
+ "compared": {
7086
+ "reference": number;
7087
+ "subject": number;
7088
+ };
7089
+ };
7090
+ }>;
7091
+ "quota": {
7092
+ "dimension": string;
7093
+ "remaining": number | null;
7094
+ "allowed": boolean;
7095
+ } | null;
7096
+ "launch": {
7097
+ "attempt": number;
7098
+ "state": "fanning_out" | "deferred" | "settled";
7099
+ "startedAt": string;
7100
+ "progressAt": string;
7101
+ "planned"?: number;
7102
+ "created"?: number;
7103
+ "missing"?: {
7104
+ "retryable": number;
7105
+ "refused": number;
7106
+ };
7107
+ "notes"?: Array<string>;
7108
+ "settledAt"?: string;
7109
+ } | null;
7110
+ "relaunch": {
7111
+ "started": boolean;
7112
+ "reason": string | null;
7113
+ };
7114
+ };
7115
+ export type StudyRegenerateSummaryResponse = {
7116
+ "status": "queued";
7117
+ };
7118
+ export type StudyGenerateRequest = {
7119
+ "proposeExpectations"?: boolean;
7120
+ };
7121
+ export type StudyGenerateResponse = {
7122
+ "study": {
7123
+ "id": string;
7124
+ "executiveSummary"?: string | null;
7125
+ "plannedRunCount": number;
7126
+ "organizationId": string;
7127
+ "createdBy": string | null;
7128
+ "name": string;
7129
+ "slug": string;
7130
+ "description": string | null;
7131
+ "skillRef": string | null;
7132
+ "skillSlug": string | null;
7133
+ "skillName": string | null;
7134
+ "skillPromise": string;
7135
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
7136
+ "subjectName": string | null;
7137
+ "subjectLabel": string;
7138
+ "armConfig": {
7139
+ "baseline"?: {
7140
+ "tools"?: "default" | "none" | {
7141
+ "tools": Array<string>;
7142
+ };
7143
+ "addons"?: Record<string, unknown>;
7144
+ };
7145
+ "treatment"?: {
7146
+ "tools"?: "default" | "none" | {
7147
+ "tools": Array<string>;
7148
+ };
7149
+ "addons"?: Record<string, unknown>;
7150
+ };
7151
+ };
7152
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
7153
+ "subjectBackend": "model" | "harness" | null;
7154
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5116
7155
  "importedFrom"?: string;
5117
7156
  "baselineLabel": string | null;
5118
7157
  "treatmentLabel": string | null;
@@ -5146,6 +7185,7 @@ export type StudyGenerateResponse = {
5146
7185
  "error": string | null;
5147
7186
  "preparation"?: {
5148
7187
  "since": string;
7188
+ "progressAt"?: string;
5149
7189
  "launchedBy": {
5150
7190
  "userId"?: string;
5151
7191
  "role"?: string;
@@ -5154,6 +7194,12 @@ export type StudyGenerateResponse = {
5154
7194
  "imageKey": string;
5155
7195
  "buildId": string;
5156
7196
  "stage"?: "image" | "snapshot";
7197
+ "retry"?: {
7198
+ "failedAttempts": number;
7199
+ "maxAttempts": number;
7200
+ "notBefore": string;
7201
+ "lastFailure": string;
7202
+ };
5157
7203
  }>;
5158
7204
  } | null;
5159
7205
  "visibility": "private" | "org" | "link" | "public";
@@ -5168,11 +7214,65 @@ export type StudyGenerateResponse = {
5168
7214
  "createdAt": string;
5169
7215
  "updatedAt": string;
5170
7216
  "liveUntil": string | null;
7217
+ "launch"?: {
7218
+ "attempt": number;
7219
+ "state": "fanning_out" | "deferred" | "settled";
7220
+ "startedAt": string;
7221
+ "progressAt": string;
7222
+ "planned"?: number;
7223
+ "created"?: number;
7224
+ "missing"?: {
7225
+ "retryable": number;
7226
+ "refused": number;
7227
+ };
7228
+ "notes"?: Array<string>;
7229
+ "settledAt"?: string;
7230
+ } | null;
7231
+ "relaunchable"?: boolean;
7232
+ "arms"?: Array<{
7233
+ "key": string;
7234
+ "position": number;
7235
+ "label": string;
7236
+ "role": "reference" | "peer";
7237
+ "payload": {
7238
+ "tools"?: "default" | "none" | {
7239
+ "tools": Array<string>;
7240
+ };
7241
+ "addons"?: Record<string, unknown>;
7242
+ };
7243
+ "cells"?: Array<{
7244
+ "harness": string;
7245
+ "model": string;
7246
+ "provider"?: string;
7247
+ "params"?: {
7248
+ "temperature"?: number;
7249
+ "maxOutputTokens"?: number;
7250
+ "reasoningEffort"?: "low" | "medium" | "high";
7251
+ "thinkingBudgetTokens"?: number;
7252
+ };
7253
+ }>;
7254
+ "skills"?: Array<{
7255
+ "ref": string;
7256
+ "name": string;
7257
+ "changedSinceCreate"?: boolean;
7258
+ }>;
7259
+ "addedAgents"?: Array<{
7260
+ "position": number;
7261
+ "addedAt": string;
7262
+ }>;
7263
+ }>;
7264
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7265
+ "cellPlan"?: "crossed" | "per_setup";
7266
+ "skillDisclosure"?: "withheld" | "named";
7267
+ "modeOptions"?: Record<string, unknown>;
5171
7268
  };
5172
7269
  "streamUrl": string;
5173
7270
  "warnings"?: Array<{
5174
7271
  "harness": string;
5175
7272
  "kind": string;
7273
+ } | {
7274
+ "type": "design_aggregate_budget";
7275
+ "detail": string;
5176
7276
  }>;
5177
7277
  };
5178
7278
  export type StudyLaunchResponse = {
@@ -5206,7 +7306,7 @@ export type StudyLaunchResponse = {
5206
7306
  "addons"?: Record<string, unknown>;
5207
7307
  };
5208
7308
  };
5209
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
7309
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5210
7310
  "subjectBackend": "model" | "harness" | null;
5211
7311
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5212
7312
  "importedFrom"?: string;
@@ -5242,6 +7342,7 @@ export type StudyLaunchResponse = {
5242
7342
  "error": string | null;
5243
7343
  "preparation"?: {
5244
7344
  "since": string;
7345
+ "progressAt"?: string;
5245
7346
  "launchedBy": {
5246
7347
  "userId"?: string;
5247
7348
  "role"?: string;
@@ -5250,6 +7351,12 @@ export type StudyLaunchResponse = {
5250
7351
  "imageKey": string;
5251
7352
  "buildId": string;
5252
7353
  "stage"?: "image" | "snapshot";
7354
+ "retry"?: {
7355
+ "failedAttempts": number;
7356
+ "maxAttempts": number;
7357
+ "notBefore": string;
7358
+ "lastFailure": string;
7359
+ };
5253
7360
  }>;
5254
7361
  } | null;
5255
7362
  "visibility": "private" | "org" | "link" | "public";
@@ -5264,11 +7371,65 @@ export type StudyLaunchResponse = {
5264
7371
  "createdAt": string;
5265
7372
  "updatedAt": string;
5266
7373
  "liveUntil": string | null;
7374
+ "launch"?: {
7375
+ "attempt": number;
7376
+ "state": "fanning_out" | "deferred" | "settled";
7377
+ "startedAt": string;
7378
+ "progressAt": string;
7379
+ "planned"?: number;
7380
+ "created"?: number;
7381
+ "missing"?: {
7382
+ "retryable": number;
7383
+ "refused": number;
7384
+ };
7385
+ "notes"?: Array<string>;
7386
+ "settledAt"?: string;
7387
+ } | null;
7388
+ "relaunchable"?: boolean;
7389
+ "arms"?: Array<{
7390
+ "key": string;
7391
+ "position": number;
7392
+ "label": string;
7393
+ "role": "reference" | "peer";
7394
+ "payload": {
7395
+ "tools"?: "default" | "none" | {
7396
+ "tools": Array<string>;
7397
+ };
7398
+ "addons"?: Record<string, unknown>;
7399
+ };
7400
+ "cells"?: Array<{
7401
+ "harness": string;
7402
+ "model": string;
7403
+ "provider"?: string;
7404
+ "params"?: {
7405
+ "temperature"?: number;
7406
+ "maxOutputTokens"?: number;
7407
+ "reasoningEffort"?: "low" | "medium" | "high";
7408
+ "thinkingBudgetTokens"?: number;
7409
+ };
7410
+ }>;
7411
+ "skills"?: Array<{
7412
+ "ref": string;
7413
+ "name": string;
7414
+ "changedSinceCreate"?: boolean;
7415
+ }>;
7416
+ "addedAgents"?: Array<{
7417
+ "position": number;
7418
+ "addedAt": string;
7419
+ }>;
7420
+ }>;
7421
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7422
+ "cellPlan"?: "crossed" | "per_setup";
7423
+ "skillDisclosure"?: "withheld" | "named";
7424
+ "modeOptions"?: Record<string, unknown>;
5267
7425
  };
5268
7426
  "streamUrl": string;
5269
7427
  "warnings"?: Array<{
5270
7428
  "harness": string;
5271
7429
  "kind": string;
7430
+ } | {
7431
+ "type": "design_aggregate_budget";
7432
+ "detail": string;
5272
7433
  }>;
5273
7434
  };
5274
7435
  export type StudyUpdateLensesRequest = {
@@ -5305,7 +7466,7 @@ export type StudyUpdateLensesResponse = {
5305
7466
  "addons"?: Record<string, unknown>;
5306
7467
  };
5307
7468
  };
5308
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
7469
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5309
7470
  "subjectBackend": "model" | "harness" | null;
5310
7471
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5311
7472
  "importedFrom"?: string;
@@ -5341,6 +7502,7 @@ export type StudyUpdateLensesResponse = {
5341
7502
  "error": string | null;
5342
7503
  "preparation"?: {
5343
7504
  "since": string;
7505
+ "progressAt"?: string;
5344
7506
  "launchedBy": {
5345
7507
  "userId"?: string;
5346
7508
  "role"?: string;
@@ -5349,6 +7511,12 @@ export type StudyUpdateLensesResponse = {
5349
7511
  "imageKey": string;
5350
7512
  "buildId": string;
5351
7513
  "stage"?: "image" | "snapshot";
7514
+ "retry"?: {
7515
+ "failedAttempts": number;
7516
+ "maxAttempts": number;
7517
+ "notBefore": string;
7518
+ "lastFailure": string;
7519
+ };
5352
7520
  }>;
5353
7521
  } | null;
5354
7522
  "visibility": "private" | "org" | "link" | "public";
@@ -5363,6 +7531,57 @@ export type StudyUpdateLensesResponse = {
5363
7531
  "createdAt": string;
5364
7532
  "updatedAt": string;
5365
7533
  "liveUntil": string | null;
7534
+ "launch"?: {
7535
+ "attempt": number;
7536
+ "state": "fanning_out" | "deferred" | "settled";
7537
+ "startedAt": string;
7538
+ "progressAt": string;
7539
+ "planned"?: number;
7540
+ "created"?: number;
7541
+ "missing"?: {
7542
+ "retryable": number;
7543
+ "refused": number;
7544
+ };
7545
+ "notes"?: Array<string>;
7546
+ "settledAt"?: string;
7547
+ } | null;
7548
+ "relaunchable"?: boolean;
7549
+ "arms"?: Array<{
7550
+ "key": string;
7551
+ "position": number;
7552
+ "label": string;
7553
+ "role": "reference" | "peer";
7554
+ "payload": {
7555
+ "tools"?: "default" | "none" | {
7556
+ "tools": Array<string>;
7557
+ };
7558
+ "addons"?: Record<string, unknown>;
7559
+ };
7560
+ "cells"?: Array<{
7561
+ "harness": string;
7562
+ "model": string;
7563
+ "provider"?: string;
7564
+ "params"?: {
7565
+ "temperature"?: number;
7566
+ "maxOutputTokens"?: number;
7567
+ "reasoningEffort"?: "low" | "medium" | "high";
7568
+ "thinkingBudgetTokens"?: number;
7569
+ };
7570
+ }>;
7571
+ "skills"?: Array<{
7572
+ "ref": string;
7573
+ "name": string;
7574
+ "changedSinceCreate"?: boolean;
7575
+ }>;
7576
+ "addedAgents"?: Array<{
7577
+ "position": number;
7578
+ "addedAt": string;
7579
+ }>;
7580
+ }>;
7581
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7582
+ "cellPlan"?: "crossed" | "per_setup";
7583
+ "skillDisclosure"?: "withheld" | "named";
7584
+ "modeOptions"?: Record<string, unknown>;
5366
7585
  };
5367
7586
  "warnings": Array<{
5368
7587
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
@@ -5396,7 +7615,7 @@ export type StudyPreviewResponse = {
5396
7615
  "addons"?: Array<string>;
5397
7616
  };
5398
7617
  };
5399
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
7618
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5400
7619
  "subjectBackend"?: "model" | "harness";
5401
7620
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5402
7621
  "baselineLabel": string | null;
@@ -5443,6 +7662,27 @@ export type StudyPreviewResponse = {
5443
7662
  "kind": string;
5444
7663
  "text": string;
5445
7664
  }>;
7665
+ "groupCount"?: number;
7666
+ "group"?: {
7667
+ "label": string;
7668
+ "composition": {
7669
+ "tools"?: "default" | "none" | {
7670
+ "tools": Array<string>;
7671
+ };
7672
+ "addons"?: Array<string>;
7673
+ };
7674
+ };
7675
+ "contrastPlan"?: "vs_reference" | "all_pairs";
7676
+ "agentsAdded"?: Array<{
7677
+ "group": string;
7678
+ "agents": number;
7679
+ "addedAt": string;
7680
+ }>;
7681
+ "groupComposition"?: Array<{
7682
+ "key": string;
7683
+ "label": string;
7684
+ "skills": Array<string>;
7685
+ }>;
5446
7686
  };
5447
7687
  "tasks": Array<{
5448
7688
  "id": string;
@@ -5459,7 +7699,7 @@ export type StudyPreviewResponse = {
5459
7699
  "runs": Array<{
5460
7700
  "runId": string;
5461
7701
  "taskId": string;
5462
- "arm": "baseline" | "skill";
7702
+ "arm": "baseline" | "skill" | string;
5463
7703
  "harness": string;
5464
7704
  "model": string;
5465
7705
  "provider": string | null;
@@ -5485,6 +7725,9 @@ export type StudyPreviewResponse = {
5485
7725
  "pairGradedAt": string | null;
5486
7726
  "evalScore": number | null;
5487
7727
  "evalGradedAt": string | null;
7728
+ "executionBudgetS"?: number | null;
7729
+ "harnessImageDigest"?: string | null;
7730
+ "subjectEngaged"?: boolean | null;
5488
7731
  "runCreatedAt": string | null;
5489
7732
  "startedAt": string | null;
5490
7733
  "finishedAt": string | null;
@@ -5522,40 +7765,303 @@ export type StudyPreviewResponse = {
5522
7765
  "unit": "%";
5523
7766
  "relativePct": number | null;
5524
7767
  };
5525
- "improved": boolean;
7768
+ "improved": boolean;
7769
+ }>;
7770
+ "completionRate": {
7771
+ "baseline": number | null;
7772
+ "skill": number | null;
7773
+ };
7774
+ "goalRate": {
7775
+ "baseline": number | null;
7776
+ "skill": number | null;
7777
+ };
7778
+ "pairwise": {
7779
+ "pairs": number;
7780
+ "skillWins": number;
7781
+ "baselineWins": number;
7782
+ "ties": number;
7783
+ "winRate": number | null;
7784
+ };
7785
+ "decision"?: {
7786
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
7787
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
7788
+ "reason": string;
7789
+ "efficiencyDiscounted": boolean;
7790
+ };
7791
+ "leaderboard"?: Array<unknown>;
7792
+ "concentration": {
7793
+ "perCell": Array<{
7794
+ "cell": string;
7795
+ "goalDeltaPp": number | null;
7796
+ "contributionPct": number | null;
7797
+ }>;
7798
+ "heterogeneity": {
7799
+ "dominantCell": string;
7800
+ "contributionPct": number;
7801
+ } | null;
7802
+ };
7803
+ "matrixShape": {
7804
+ "cells": number;
7805
+ "harnesses": number;
7806
+ "models": number;
7807
+ "crossed": boolean;
7808
+ };
7809
+ "flakiness"?: {
7810
+ "measured": boolean;
7811
+ "conditions": number;
7812
+ "unstable": number;
7813
+ "worst": {
7814
+ "cell": string;
7815
+ "taskId": string;
7816
+ "arm": "baseline" | "skill";
7817
+ "variant"?: number;
7818
+ "passRate": number;
7819
+ } | null;
7820
+ };
7821
+ "arms"?: {
7822
+ "baseline": {
7823
+ "runs": number;
7824
+ "completed": number;
7825
+ "infraFailures": number;
7826
+ "completionRate": number | null;
7827
+ "goalRate": number | null;
7828
+ "goalTally": {
7829
+ "achieved": number;
7830
+ "partial": number;
7831
+ "missed": number;
7832
+ };
7833
+ "avgTotalTokens": number | null;
7834
+ "avgOutputTokens": number | null;
7835
+ "avgCostUsd": number | null;
7836
+ "avgDurationMs": number | null;
7837
+ "avgTurns": number | null;
7838
+ "costCoverage": number | null;
7839
+ "avgQualityScore": number | null;
7840
+ "gradedCoverage": number | null;
7841
+ "medianCostUsd": number | null;
7842
+ "medianTotalTokens": number | null;
7843
+ "medianDurationMs": number | null;
7844
+ "p95CostUsd": number | null;
7845
+ "p95TotalTokens": number | null;
7846
+ "p95DurationMs": number | null;
7847
+ "maxCostUsd": number | null;
7848
+ "maxTotalTokens": number | null;
7849
+ "maxDurationMs": number | null;
7850
+ "evalPassRate"?: number | null;
7851
+ "benchmarkPassRate"?: number | null;
7852
+ "adoptionRate"?: number | null;
7853
+ };
7854
+ "skill": {
7855
+ "runs": number;
7856
+ "completed": number;
7857
+ "infraFailures": number;
7858
+ "completionRate": number | null;
7859
+ "goalRate": number | null;
7860
+ "goalTally": {
7861
+ "achieved": number;
7862
+ "partial": number;
7863
+ "missed": number;
7864
+ };
7865
+ "avgTotalTokens": number | null;
7866
+ "avgOutputTokens": number | null;
7867
+ "avgCostUsd": number | null;
7868
+ "avgDurationMs": number | null;
7869
+ "avgTurns": number | null;
7870
+ "costCoverage": number | null;
7871
+ "avgQualityScore": number | null;
7872
+ "gradedCoverage": number | null;
7873
+ "medianCostUsd": number | null;
7874
+ "medianTotalTokens": number | null;
7875
+ "medianDurationMs": number | null;
7876
+ "p95CostUsd": number | null;
7877
+ "p95TotalTokens": number | null;
7878
+ "p95DurationMs": number | null;
7879
+ "maxCostUsd": number | null;
7880
+ "maxTotalTokens": number | null;
7881
+ "maxDurationMs": number | null;
7882
+ "evalPassRate"?: number | null;
7883
+ "benchmarkPassRate"?: number | null;
7884
+ "adoptionRate"?: number | null;
7885
+ };
7886
+ };
7887
+ "totals": {
7888
+ "cells": number;
7889
+ "runs": number;
7890
+ "completed": number;
7891
+ "failed": number;
7892
+ "costUsd": number;
7893
+ "tokens": number;
7894
+ };
7895
+ };
7896
+ "perCell": Array<unknown>;
7897
+ "perHarness": Array<unknown>;
7898
+ "perModel": Array<unknown>;
7899
+ "perTask": Array<unknown>;
7900
+ "webSplit": unknown;
7901
+ "inference": {
7902
+ "columns": Array<{
7903
+ "key": string;
7904
+ "label": string;
7905
+ "family": "rate" | "score" | "magnitude";
7906
+ "betterWhen": "higher" | "lower";
7907
+ }>;
7908
+ "groups": Array<{
7909
+ "kind": "cell" | "harness" | "model" | "field";
7910
+ "title": string;
7911
+ "rows": Array<{
7912
+ "key": string;
7913
+ "kind": "cell" | "harness" | "model" | "field";
7914
+ "harness": string | null;
7915
+ "model": string | null;
7916
+ "paramsKey"?: string;
7917
+ "runs": number;
7918
+ "cells": Record<string, unknown>;
7919
+ }>;
7920
+ }>;
7921
+ "reference": "baseline" | "field";
7922
+ "draws": number;
7923
+ } | null;
7924
+ "behavior"?: Array<{
7925
+ "harness": string;
7926
+ "model": string;
7927
+ "provider": string | null;
7928
+ "arm": string;
7929
+ "totalRuns": number;
7930
+ "signals": Array<{
7931
+ "id": string;
7932
+ "label": string;
7933
+ "tone": "good" | "bad" | "neutral";
7934
+ "runCount": number;
7935
+ "succeededCount": number;
7936
+ }>;
7937
+ }>;
7938
+ "reference"?: {
7939
+ "cells": Array<{
7940
+ "cell": string;
7941
+ "harness": string;
7942
+ "model": string;
7943
+ "provider": string | null;
7944
+ "paramsKey": string;
7945
+ "label": string;
7946
+ "passRate": {
7947
+ "value": number | null;
7948
+ "ci": Array<unknown> | null;
7949
+ "n": number;
7950
+ };
7951
+ "abstention"?: {
7952
+ "value": number | null;
7953
+ "ci": Array<unknown> | null;
7954
+ "n": number;
7955
+ "parts"?: Array<{
7956
+ "id": string;
7957
+ "label": string;
7958
+ "value": number | null;
7959
+ "n": number;
7960
+ }>;
7961
+ };
7962
+ "tasks": number;
7963
+ "graded": number;
7964
+ "unmeasured": number;
7965
+ "matchers": Array<string>;
7966
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
5526
7967
  }>;
5527
- "completionRate": {
5528
- "baseline": number | null;
5529
- "skill": number | null;
5530
- };
5531
- "goalRate": {
5532
- "baseline": number | null;
5533
- "skill": number | null;
7968
+ "overall": {
7969
+ "value": number | null;
7970
+ "ci": Array<unknown> | null;
7971
+ "n": number;
5534
7972
  };
5535
- "pairwise": {
5536
- "pairs": number;
5537
- "skillWins": number;
5538
- "baselineWins": number;
5539
- "ties": number;
5540
- "winRate": number | null;
7973
+ "abstention"?: {
7974
+ "value": number | null;
7975
+ "ci": Array<unknown> | null;
7976
+ "n": number;
7977
+ "parts"?: Array<{
7978
+ "id": string;
7979
+ "label": string;
7980
+ "value": number | null;
7981
+ "n": number;
7982
+ }>;
5541
7983
  };
5542
- "decision"?: {
5543
- "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
5544
- "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5545
- "reason": string;
5546
- "efficiencyDiscounted": boolean;
7984
+ "graded": number;
7985
+ "unmeasured": number;
7986
+ "modelJudged": boolean;
7987
+ };
7988
+ "execution"?: {
7989
+ "graded": number;
7990
+ "measured": number;
7991
+ "passed": number;
7992
+ "passRate": number | null;
7993
+ "byStatus": {
7994
+ "passed": number;
7995
+ "failed": number;
7996
+ "timeout": number;
7997
+ "error": number;
7998
+ "unavailable": number;
5547
7999
  };
5548
- "leaderboard"?: Array<unknown>;
5549
- "concentration": {
5550
- "perCell": Array<{
5551
- "cell": string;
5552
- "goalDeltaPp": number | null;
5553
- "contributionPct": number | null;
8000
+ "languages": Array<string>;
8001
+ "unavailableReasons": Array<string>;
8002
+ };
8003
+ } | {
8004
+ "shape": "multi-arm";
8005
+ "summary": {
8006
+ "shape": "multi-arm";
8007
+ "verdict": "measured" | "inconclusive";
8008
+ "groups": Array<{
8009
+ "key": string;
8010
+ "label": string;
8011
+ "role": "reference" | "peer";
8012
+ "composition": {
8013
+ "tools"?: "default" | "none" | {
8014
+ "tools": Array<string>;
8015
+ };
8016
+ "addons"?: Array<string>;
8017
+ };
8018
+ }>;
8019
+ "ranking": {
8020
+ "plan": "vs_reference" | "all_pairs" | null;
8021
+ "referenceArm": string | null;
8022
+ "decision": {
8023
+ "leader": string | null;
8024
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8025
+ "reason": string;
8026
+ "separatedFrom": Array<string>;
8027
+ "lostTo": Array<{
8028
+ "arm": string;
8029
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8030
+ }>;
8031
+ "clearLeader": boolean;
8032
+ "correctnessLeader": string | null;
8033
+ };
8034
+ "entries": Array<{
8035
+ "arm": string;
8036
+ "runs": number;
8037
+ "completed": number;
8038
+ "completionRate": number | null;
8039
+ "goalRate": number | null;
8040
+ "avgQualityScore": number | null;
8041
+ "evalPassRate": number | null;
8042
+ "benchmarkPassRate"?: number | null;
8043
+ "compositeScore": number | null;
8044
+ "avgTotalTokens": number | null;
8045
+ "avgOutputTokens": number | null;
8046
+ "avgCostUsd": number | null;
8047
+ "avgDurationMs": number | null;
8048
+ "medianCostUsd": number | null;
8049
+ "medianTotalTokens": number | null;
8050
+ "medianDurationMs": number | null;
8051
+ "rank": number;
5554
8052
  }>;
5555
- "heterogeneity": {
5556
- "dominantCell": string;
5557
- "contributionPct": number;
5558
- } | null;
8053
+ "contrasts": Array<{
8054
+ "contrastId": string;
8055
+ "referenceArm": string;
8056
+ "subjectArm": string;
8057
+ "oriented": boolean;
8058
+ "outcome": "won" | "tie" | "unmeasurable";
8059
+ "winner": string | null;
8060
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8061
+ "explanation": string;
8062
+ }>;
8063
+ "comparisons": number;
8064
+ "headline"?: string;
5559
8065
  };
5560
8066
  "matrixShape": {
5561
8067
  "cells": number;
@@ -5563,19 +8069,612 @@ export type StudyPreviewResponse = {
5563
8069
  "models": number;
5564
8070
  "crossed": boolean;
5565
8071
  };
5566
- "flakiness"?: {
8072
+ "flakiness": {
5567
8073
  "measured": boolean;
5568
8074
  "conditions": number;
5569
8075
  "unstable": number;
5570
8076
  "worst": {
5571
8077
  "cell": string;
5572
8078
  "taskId": string;
5573
- "arm": "baseline" | "skill";
8079
+ "arm": string;
5574
8080
  "variant"?: number;
5575
8081
  "passRate": number;
5576
8082
  } | null;
5577
8083
  };
5578
- "arms"?: {
8084
+ "totals": {
8085
+ "cells": number;
8086
+ "runs": number;
8087
+ "completed": number;
8088
+ "failed": number;
8089
+ "costUsd": number;
8090
+ "tokens": number;
8091
+ };
8092
+ };
8093
+ "arms": Array<{
8094
+ "key": string;
8095
+ "label": string;
8096
+ "position": number;
8097
+ "role": "reference" | "peer";
8098
+ "composition": {
8099
+ "tools"?: "default" | "none" | {
8100
+ "tools": Array<string>;
8101
+ };
8102
+ "addons"?: Array<string>;
8103
+ };
8104
+ "agg": {
8105
+ "runs": number;
8106
+ "completed": number;
8107
+ "infraFailures": number;
8108
+ "completionRate": number | null;
8109
+ "goalRate": number | null;
8110
+ "goalTally": {
8111
+ "achieved": number;
8112
+ "partial": number;
8113
+ "missed": number;
8114
+ };
8115
+ "avgTotalTokens": number | null;
8116
+ "avgOutputTokens": number | null;
8117
+ "avgCostUsd": number | null;
8118
+ "avgDurationMs": number | null;
8119
+ "avgTurns": number | null;
8120
+ "costCoverage": number | null;
8121
+ "avgQualityScore": number | null;
8122
+ "gradedCoverage": number | null;
8123
+ "medianCostUsd": number | null;
8124
+ "medianTotalTokens": number | null;
8125
+ "medianDurationMs": number | null;
8126
+ "p95CostUsd": number | null;
8127
+ "p95TotalTokens": number | null;
8128
+ "p95DurationMs": number | null;
8129
+ "maxCostUsd": number | null;
8130
+ "maxTotalTokens": number | null;
8131
+ "maxDurationMs": number | null;
8132
+ "evalPassRate"?: number | null;
8133
+ "benchmarkPassRate"?: number | null;
8134
+ "adoptionRate"?: number | null;
8135
+ };
8136
+ "perCell": Array<{
8137
+ "harness": string;
8138
+ "model": string;
8139
+ "provider": string | null;
8140
+ "paramsKey"?: string;
8141
+ "label"?: string;
8142
+ "agg": {
8143
+ "runs": number;
8144
+ "completed": number;
8145
+ "infraFailures": number;
8146
+ "completionRate": number | null;
8147
+ "goalRate": number | null;
8148
+ "goalTally": {
8149
+ "achieved": number;
8150
+ "partial": number;
8151
+ "missed": number;
8152
+ };
8153
+ "avgTotalTokens": number | null;
8154
+ "avgOutputTokens": number | null;
8155
+ "avgCostUsd": number | null;
8156
+ "avgDurationMs": number | null;
8157
+ "avgTurns": number | null;
8158
+ "costCoverage": number | null;
8159
+ "avgQualityScore": number | null;
8160
+ "gradedCoverage": number | null;
8161
+ "medianCostUsd": number | null;
8162
+ "medianTotalTokens": number | null;
8163
+ "medianDurationMs": number | null;
8164
+ "p95CostUsd": number | null;
8165
+ "p95TotalTokens": number | null;
8166
+ "p95DurationMs": number | null;
8167
+ "maxCostUsd": number | null;
8168
+ "maxTotalTokens": number | null;
8169
+ "maxDurationMs": number | null;
8170
+ "evalPassRate"?: number | null;
8171
+ "benchmarkPassRate"?: number | null;
8172
+ "adoptionRate"?: number | null;
8173
+ };
8174
+ }>;
8175
+ "reference"?: {
8176
+ "cells": Array<{
8177
+ "cell": string;
8178
+ "harness": string;
8179
+ "model": string;
8180
+ "provider": string | null;
8181
+ "paramsKey": string;
8182
+ "label": string;
8183
+ "passRate": {
8184
+ "value": number | null;
8185
+ "ci": Array<unknown> | null;
8186
+ "n": number;
8187
+ };
8188
+ "abstention"?: {
8189
+ "value": number | null;
8190
+ "ci": Array<unknown> | null;
8191
+ "n": number;
8192
+ "parts"?: Array<{
8193
+ "id": string;
8194
+ "label": string;
8195
+ "value": number | null;
8196
+ "n": number;
8197
+ }>;
8198
+ };
8199
+ "tasks": number;
8200
+ "graded": number;
8201
+ "unmeasured": number;
8202
+ "matchers": Array<string>;
8203
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
8204
+ }>;
8205
+ "overall": {
8206
+ "value": number | null;
8207
+ "ci": Array<unknown> | null;
8208
+ "n": number;
8209
+ };
8210
+ "abstention"?: {
8211
+ "value": number | null;
8212
+ "ci": Array<unknown> | null;
8213
+ "n": number;
8214
+ "parts"?: Array<{
8215
+ "id": string;
8216
+ "label": string;
8217
+ "value": number | null;
8218
+ "n": number;
8219
+ }>;
8220
+ };
8221
+ "graded": number;
8222
+ "unmeasured": number;
8223
+ "modelJudged": boolean;
8224
+ };
8225
+ }>;
8226
+ "contrasts": Array<{
8227
+ "contrastId": string;
8228
+ "referenceArm": string;
8229
+ "subjectArm": string;
8230
+ "oriented": boolean;
8231
+ "basis"?: {
8232
+ "id": "matched" | "whole_setup";
8233
+ "compared": {
8234
+ "reference": Array<{
8235
+ "harness": string;
8236
+ "model": string;
8237
+ "provider"?: string;
8238
+ "params"?: Record<string, unknown>;
8239
+ }>;
8240
+ "subject": Array<{
8241
+ "harness": string;
8242
+ "model": string;
8243
+ "provider"?: string;
8244
+ "params"?: Record<string, unknown>;
8245
+ }>;
8246
+ };
8247
+ "excluded": {
8248
+ "reference": Array<{
8249
+ "harness": string;
8250
+ "model": string;
8251
+ "provider"?: string;
8252
+ "params"?: Record<string, unknown>;
8253
+ }>;
8254
+ "subject": Array<{
8255
+ "harness": string;
8256
+ "model": string;
8257
+ "provider"?: string;
8258
+ "params"?: Record<string, unknown>;
8259
+ }>;
8260
+ };
8261
+ "differsIn": Array<string>;
8262
+ "licensedFacet": string | null;
8263
+ "efficiency": "comparable" | "incomparable";
8264
+ "pairedTasks": number | null;
8265
+ "dropped": {
8266
+ "reference": number;
8267
+ "subject": number;
8268
+ } | null;
8269
+ };
8270
+ "magnitudes": "relative";
8271
+ "summary": {
8272
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
8273
+ "headline": Array<{
8274
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
8275
+ "label": string;
8276
+ "direction": "down" | "up";
8277
+ "baselineAvg": number | null;
8278
+ "skillAvg": number | null;
8279
+ "deltaPct": number | null;
8280
+ "delta"?: {
8281
+ "unit": "pp";
8282
+ "absolute": number | null;
8283
+ "relativePct": number | null;
8284
+ } | {
8285
+ "unit": "pts";
8286
+ "absolute": number | null;
8287
+ "relativePct"?: number | null;
8288
+ } | {
8289
+ "unit": "%";
8290
+ "relativePct": number | null;
8291
+ };
8292
+ "improved": boolean;
8293
+ }>;
8294
+ "completionRate": {
8295
+ "baseline": number | null;
8296
+ "skill": number | null;
8297
+ };
8298
+ "goalRate": {
8299
+ "baseline": number | null;
8300
+ "skill": number | null;
8301
+ };
8302
+ "pairwise": {
8303
+ "pairs": number;
8304
+ "skillWins": number;
8305
+ "baselineWins": number;
8306
+ "ties": number;
8307
+ "winRate": number | null;
8308
+ };
8309
+ "decision"?: {
8310
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
8311
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8312
+ "reason": string;
8313
+ "efficiencyDiscounted": boolean;
8314
+ };
8315
+ "concentration": {
8316
+ "perCell": Array<{
8317
+ "cell": string;
8318
+ "goalDeltaPp": number | null;
8319
+ "contributionPct": number | null;
8320
+ }>;
8321
+ "heterogeneity": {
8322
+ "dominantCell": string;
8323
+ "contributionPct": number;
8324
+ } | null;
8325
+ };
8326
+ "matrixShape": {
8327
+ "cells": number;
8328
+ "harnesses": number;
8329
+ "models": number;
8330
+ "crossed": boolean;
8331
+ };
8332
+ "flakiness"?: {
8333
+ "measured": boolean;
8334
+ "conditions": number;
8335
+ "unstable": number;
8336
+ "worst": {
8337
+ "cell": string;
8338
+ "taskId": string;
8339
+ "arm": "baseline" | "skill";
8340
+ "variant"?: number;
8341
+ "passRate": number;
8342
+ } | null;
8343
+ };
8344
+ "arms"?: {
8345
+ "baseline": {
8346
+ "runs": number;
8347
+ "completed": number;
8348
+ "infraFailures": number;
8349
+ "completionRate": number | null;
8350
+ "goalRate": number | null;
8351
+ "goalTally": {
8352
+ "achieved": number;
8353
+ "partial": number;
8354
+ "missed": number;
8355
+ };
8356
+ "avgTotalTokens": number | null;
8357
+ "avgOutputTokens": number | null;
8358
+ "avgCostUsd": number | null;
8359
+ "avgDurationMs": number | null;
8360
+ "avgTurns": number | null;
8361
+ "costCoverage": number | null;
8362
+ "avgQualityScore": number | null;
8363
+ "gradedCoverage": number | null;
8364
+ "medianCostUsd": number | null;
8365
+ "medianTotalTokens": number | null;
8366
+ "medianDurationMs": number | null;
8367
+ "p95CostUsd": number | null;
8368
+ "p95TotalTokens": number | null;
8369
+ "p95DurationMs": number | null;
8370
+ "maxCostUsd": number | null;
8371
+ "maxTotalTokens": number | null;
8372
+ "maxDurationMs": number | null;
8373
+ "evalPassRate"?: number | null;
8374
+ "benchmarkPassRate"?: number | null;
8375
+ "adoptionRate"?: number | null;
8376
+ };
8377
+ "skill": {
8378
+ "runs": number;
8379
+ "completed": number;
8380
+ "infraFailures": number;
8381
+ "completionRate": number | null;
8382
+ "goalRate": number | null;
8383
+ "goalTally": {
8384
+ "achieved": number;
8385
+ "partial": number;
8386
+ "missed": number;
8387
+ };
8388
+ "avgTotalTokens": number | null;
8389
+ "avgOutputTokens": number | null;
8390
+ "avgCostUsd": number | null;
8391
+ "avgDurationMs": number | null;
8392
+ "avgTurns": number | null;
8393
+ "costCoverage": number | null;
8394
+ "avgQualityScore": number | null;
8395
+ "gradedCoverage": number | null;
8396
+ "medianCostUsd": number | null;
8397
+ "medianTotalTokens": number | null;
8398
+ "medianDurationMs": number | null;
8399
+ "p95CostUsd": number | null;
8400
+ "p95TotalTokens": number | null;
8401
+ "p95DurationMs": number | null;
8402
+ "maxCostUsd": number | null;
8403
+ "maxTotalTokens": number | null;
8404
+ "maxDurationMs": number | null;
8405
+ "evalPassRate"?: number | null;
8406
+ "benchmarkPassRate"?: number | null;
8407
+ "adoptionRate"?: number | null;
8408
+ };
8409
+ };
8410
+ "totals": {
8411
+ "cells": number;
8412
+ "runs": number;
8413
+ "completed": number;
8414
+ "failed": number;
8415
+ "costUsd": number;
8416
+ "tokens": number;
8417
+ };
8418
+ };
8419
+ "perCell": Array<{
8420
+ "harness": string;
8421
+ "model": string;
8422
+ "provider": string | null;
8423
+ "paramsKey"?: string;
8424
+ "label"?: string;
8425
+ "baseline": {
8426
+ "runs": number;
8427
+ "completed": number;
8428
+ "infraFailures": number;
8429
+ "completionRate": number | null;
8430
+ "goalRate": number | null;
8431
+ "goalTally": {
8432
+ "achieved": number;
8433
+ "partial": number;
8434
+ "missed": number;
8435
+ };
8436
+ "avgTotalTokens": number | null;
8437
+ "avgOutputTokens": number | null;
8438
+ "avgCostUsd": number | null;
8439
+ "avgDurationMs": number | null;
8440
+ "avgTurns": number | null;
8441
+ "costCoverage": number | null;
8442
+ "avgQualityScore": number | null;
8443
+ "gradedCoverage": number | null;
8444
+ "medianCostUsd": number | null;
8445
+ "medianTotalTokens": number | null;
8446
+ "medianDurationMs": number | null;
8447
+ "p95CostUsd": number | null;
8448
+ "p95TotalTokens": number | null;
8449
+ "p95DurationMs": number | null;
8450
+ "maxCostUsd": number | null;
8451
+ "maxTotalTokens": number | null;
8452
+ "maxDurationMs": number | null;
8453
+ "evalPassRate"?: number | null;
8454
+ "benchmarkPassRate"?: number | null;
8455
+ "adoptionRate"?: number | null;
8456
+ };
8457
+ "skill": {
8458
+ "runs": number;
8459
+ "completed": number;
8460
+ "infraFailures": number;
8461
+ "completionRate": number | null;
8462
+ "goalRate": number | null;
8463
+ "goalTally": {
8464
+ "achieved": number;
8465
+ "partial": number;
8466
+ "missed": number;
8467
+ };
8468
+ "avgTotalTokens": number | null;
8469
+ "avgOutputTokens": number | null;
8470
+ "avgCostUsd": number | null;
8471
+ "avgDurationMs": number | null;
8472
+ "avgTurns": number | null;
8473
+ "costCoverage": number | null;
8474
+ "avgQualityScore": number | null;
8475
+ "gradedCoverage": number | null;
8476
+ "medianCostUsd": number | null;
8477
+ "medianTotalTokens": number | null;
8478
+ "medianDurationMs": number | null;
8479
+ "p95CostUsd": number | null;
8480
+ "p95TotalTokens": number | null;
8481
+ "p95DurationMs": number | null;
8482
+ "maxCostUsd": number | null;
8483
+ "maxTotalTokens": number | null;
8484
+ "maxDurationMs": number | null;
8485
+ "evalPassRate"?: number | null;
8486
+ "benchmarkPassRate"?: number | null;
8487
+ "adoptionRate"?: number | null;
8488
+ };
8489
+ "deltas": {
8490
+ "totalTokensPct": number | null;
8491
+ "outputTokensPct": number | null;
8492
+ "costUsdPct": number | null;
8493
+ "durationMsPct": number | null;
8494
+ "numTurnsPct": number | null;
8495
+ "completionRatePp": number | null;
8496
+ "goalRatePp": number | null;
8497
+ "qualityScorePts": number | null;
8498
+ "evalPassRatePp"?: number | null;
8499
+ };
8500
+ }>;
8501
+ } | {
8502
+ "contrastId": string;
8503
+ "referenceArm": string;
8504
+ "subjectArm": string;
8505
+ "oriented": boolean;
8506
+ "basis"?: {
8507
+ "id": "matched" | "whole_setup";
8508
+ "compared": {
8509
+ "reference": Array<{
8510
+ "harness": string;
8511
+ "model": string;
8512
+ "provider"?: string;
8513
+ "params"?: Record<string, unknown>;
8514
+ }>;
8515
+ "subject": Array<{
8516
+ "harness": string;
8517
+ "model": string;
8518
+ "provider"?: string;
8519
+ "params"?: Record<string, unknown>;
8520
+ }>;
8521
+ };
8522
+ "excluded": {
8523
+ "reference": Array<{
8524
+ "harness": string;
8525
+ "model": string;
8526
+ "provider"?: string;
8527
+ "params"?: Record<string, unknown>;
8528
+ }>;
8529
+ "subject": Array<{
8530
+ "harness": string;
8531
+ "model": string;
8532
+ "provider"?: string;
8533
+ "params"?: Record<string, unknown>;
8534
+ }>;
8535
+ };
8536
+ "differsIn": Array<string>;
8537
+ "licensedFacet": string | null;
8538
+ "efficiency": "comparable" | "incomparable";
8539
+ "pairedTasks": number | null;
8540
+ "dropped": {
8541
+ "reference": number;
8542
+ "subject": number;
8543
+ } | null;
8544
+ };
8545
+ "magnitudes": "absolute";
8546
+ "summary": {
8547
+ "completionRate": {
8548
+ "baseline": number | null;
8549
+ "skill": number | null;
8550
+ };
8551
+ "goalRate": {
8552
+ "baseline": number | null;
8553
+ "skill": number | null;
8554
+ };
8555
+ "pairwise": {
8556
+ "pairs": number;
8557
+ "skillWins": number;
8558
+ "baselineWins": number;
8559
+ "ties": number;
8560
+ "winRate": number | null;
8561
+ };
8562
+ "decision"?: {
8563
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
8564
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
8565
+ "reason": string;
8566
+ "efficiencyDiscounted": boolean;
8567
+ };
8568
+ "concentration": {
8569
+ "perCell": Array<{
8570
+ "cell": string;
8571
+ "goalDeltaPp": number | null;
8572
+ "contributionPct": number | null;
8573
+ }>;
8574
+ "heterogeneity": {
8575
+ "dominantCell": string;
8576
+ "contributionPct": number;
8577
+ } | null;
8578
+ };
8579
+ "matrixShape": {
8580
+ "cells": number;
8581
+ "harnesses": number;
8582
+ "models": number;
8583
+ "crossed": boolean;
8584
+ };
8585
+ "flakiness"?: {
8586
+ "measured": boolean;
8587
+ "conditions": number;
8588
+ "unstable": number;
8589
+ "worst": {
8590
+ "cell": string;
8591
+ "taskId": string;
8592
+ "arm": "baseline" | "skill";
8593
+ "variant"?: number;
8594
+ "passRate": number;
8595
+ } | null;
8596
+ };
8597
+ "arms"?: {
8598
+ "baseline": {
8599
+ "runs": number;
8600
+ "completed": number;
8601
+ "infraFailures": number;
8602
+ "completionRate": number | null;
8603
+ "goalRate": number | null;
8604
+ "goalTally": {
8605
+ "achieved": number;
8606
+ "partial": number;
8607
+ "missed": number;
8608
+ };
8609
+ "avgTotalTokens": number | null;
8610
+ "avgOutputTokens": number | null;
8611
+ "avgCostUsd": number | null;
8612
+ "avgDurationMs": number | null;
8613
+ "avgTurns": number | null;
8614
+ "costCoverage": number | null;
8615
+ "avgQualityScore": number | null;
8616
+ "gradedCoverage": number | null;
8617
+ "medianCostUsd": number | null;
8618
+ "medianTotalTokens": number | null;
8619
+ "medianDurationMs": number | null;
8620
+ "p95CostUsd": number | null;
8621
+ "p95TotalTokens": number | null;
8622
+ "p95DurationMs": number | null;
8623
+ "maxCostUsd": number | null;
8624
+ "maxTotalTokens": number | null;
8625
+ "maxDurationMs": number | null;
8626
+ "evalPassRate"?: number | null;
8627
+ "benchmarkPassRate"?: number | null;
8628
+ "adoptionRate"?: number | null;
8629
+ };
8630
+ "skill": {
8631
+ "runs": number;
8632
+ "completed": number;
8633
+ "infraFailures": number;
8634
+ "completionRate": number | null;
8635
+ "goalRate": number | null;
8636
+ "goalTally": {
8637
+ "achieved": number;
8638
+ "partial": number;
8639
+ "missed": number;
8640
+ };
8641
+ "avgTotalTokens": number | null;
8642
+ "avgOutputTokens": number | null;
8643
+ "avgCostUsd": number | null;
8644
+ "avgDurationMs": number | null;
8645
+ "avgTurns": number | null;
8646
+ "costCoverage": number | null;
8647
+ "avgQualityScore": number | null;
8648
+ "gradedCoverage": number | null;
8649
+ "medianCostUsd": number | null;
8650
+ "medianTotalTokens": number | null;
8651
+ "medianDurationMs": number | null;
8652
+ "p95CostUsd": number | null;
8653
+ "p95TotalTokens": number | null;
8654
+ "p95DurationMs": number | null;
8655
+ "maxCostUsd": number | null;
8656
+ "maxTotalTokens": number | null;
8657
+ "maxDurationMs": number | null;
8658
+ "evalPassRate"?: number | null;
8659
+ "benchmarkPassRate"?: number | null;
8660
+ "adoptionRate"?: number | null;
8661
+ };
8662
+ };
8663
+ "totals": {
8664
+ "cells": number;
8665
+ "runs": number;
8666
+ "completed": number;
8667
+ "failed": number;
8668
+ "costUsd": number;
8669
+ "tokens": number;
8670
+ };
8671
+ };
8672
+ "perCell": Array<{
8673
+ "harness": string;
8674
+ "model": string;
8675
+ "provider": string | null;
8676
+ "paramsKey"?: string;
8677
+ "label"?: string;
5579
8678
  "baseline": {
5580
8679
  "runs": number;
5581
8680
  "completed": number;
@@ -5640,44 +8739,73 @@ export type StudyPreviewResponse = {
5640
8739
  "benchmarkPassRate"?: number | null;
5641
8740
  "adoptionRate"?: number | null;
5642
8741
  };
5643
- };
5644
- "totals": {
5645
- "cells": number;
5646
- "runs": number;
5647
- "completed": number;
5648
- "failed": number;
5649
- "costUsd": number;
5650
- "tokens": number;
5651
- };
5652
- };
5653
- "perCell": Array<unknown>;
5654
- "perHarness": Array<unknown>;
5655
- "perModel": Array<unknown>;
5656
- "perTask": Array<unknown>;
5657
- "webSplit": unknown;
5658
- "inference": {
5659
- "columns": Array<{
5660
- "key": string;
5661
- "label": string;
5662
- "family": "rate" | "score" | "magnitude";
5663
- "betterWhen": "higher" | "lower";
5664
- }>;
5665
- "groups": Array<{
5666
- "kind": "cell" | "harness" | "model" | "field";
5667
- "title": string;
5668
- "rows": Array<{
5669
- "key": string;
5670
- "kind": "cell" | "harness" | "model" | "field";
5671
- "harness": string | null;
5672
- "model": string | null;
5673
- "paramsKey"?: string;
5674
- "runs": number;
5675
- "cells": Record<string, unknown>;
5676
- }>;
8742
+ "deltas": {
8743
+ "completionRatePp": number | null;
8744
+ "goalRatePp": number | null;
8745
+ "qualityScorePts": number | null;
8746
+ "evalPassRatePp"?: number | null;
8747
+ };
5677
8748
  }>;
5678
- "reference": "baseline" | "field";
5679
- "draws": number;
5680
- } | null;
8749
+ }>;
8750
+ "board": Array<{
8751
+ "harness": string;
8752
+ "model": string;
8753
+ "provider": string | null;
8754
+ "paramsKey"?: string;
8755
+ "label"?: string;
8756
+ "arm": string;
8757
+ "runs": number;
8758
+ "completed": number;
8759
+ "completionRate": number | null;
8760
+ "goalRate": number | null;
8761
+ "avgQualityScore": number | null;
8762
+ "evalPassRate": number | null;
8763
+ "benchmarkPassRate"?: number | null;
8764
+ "compositeScore": number | null;
8765
+ "avgTotalTokens": number | null;
8766
+ "avgOutputTokens": number | null;
8767
+ "avgCostUsd": number | null;
8768
+ "avgDurationMs": number | null;
8769
+ "medianCostUsd": number | null;
8770
+ "medianTotalTokens": number | null;
8771
+ "medianDurationMs": number | null;
8772
+ "rank": number;
8773
+ }>;
8774
+ "setupsBoard"?: Array<{
8775
+ "harness": string;
8776
+ "model": string;
8777
+ "provider": string | null;
8778
+ "paramsKey"?: string;
8779
+ "label"?: string;
8780
+ "arm": string;
8781
+ "runs": number;
8782
+ "completed": number;
8783
+ "completionRate": number | null;
8784
+ "goalRate": number | null;
8785
+ "avgQualityScore": number | null;
8786
+ "evalPassRate": number | null;
8787
+ "benchmarkPassRate"?: number | null;
8788
+ "compositeScore": number | null;
8789
+ "avgTotalTokens": number | null;
8790
+ "avgOutputTokens": number | null;
8791
+ "avgCostUsd": number | null;
8792
+ "avgDurationMs": number | null;
8793
+ "medianCostUsd": number | null;
8794
+ "medianTotalTokens": number | null;
8795
+ "medianDurationMs": number | null;
8796
+ "rank": number;
8797
+ "group": string;
8798
+ "agent": string;
8799
+ "setup": string;
8800
+ }>;
8801
+ "perTask": Array<{
8802
+ "taskId": string;
8803
+ "position": number;
8804
+ "promptPreview": string;
8805
+ "requiresWeb": boolean;
8806
+ "category": string | null;
8807
+ "byArm": Record<string, unknown>;
8808
+ }>;
5681
8809
  "behavior"?: Array<{
5682
8810
  "harness": string;
5683
8811
  "model": string;
@@ -5692,48 +8820,55 @@ export type StudyPreviewResponse = {
5692
8820
  "succeededCount": number;
5693
8821
  }>;
5694
8822
  }>;
5695
- "reference"?: {
5696
- "cells": Array<{
5697
- "cell": string;
5698
- "harness": string;
5699
- "model": string;
5700
- "provider": string | null;
5701
- "paramsKey": string;
5702
- "label": string;
5703
- "passRate": {
5704
- "value": number | null;
5705
- "ci": Array<unknown> | null;
5706
- "n": number;
5707
- };
5708
- "tasks": number;
5709
- "graded": number;
5710
- "unmeasured": number;
5711
- "matchers": Array<string>;
5712
- "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
8823
+ "execution"?: Record<string, unknown>;
8824
+ "inference"?: {
8825
+ "contrasts": Array<{
8826
+ "contrastId": string;
8827
+ "table": {
8828
+ "columns": Array<{
8829
+ "key": string;
8830
+ "label": string;
8831
+ "family": "rate" | "score" | "magnitude";
8832
+ "betterWhen": "higher" | "lower";
8833
+ }>;
8834
+ "groups": Array<{
8835
+ "kind": "cell" | "harness" | "model" | "field";
8836
+ "title": string;
8837
+ "rows": Array<{
8838
+ "key": string;
8839
+ "kind": "cell" | "harness" | "model" | "field";
8840
+ "harness": string | null;
8841
+ "model": string | null;
8842
+ "paramsKey"?: string;
8843
+ "runs": number;
8844
+ "cells": Record<string, unknown>;
8845
+ }>;
8846
+ }>;
8847
+ "reference": "baseline" | "field";
8848
+ "draws": number;
8849
+ "adjustment": {
8850
+ "scope": "planned-comparisons";
8851
+ "comparisons": number;
8852
+ };
8853
+ } | null;
5713
8854
  }>;
5714
- "overall": {
5715
- "value": number | null;
5716
- "ci": Array<unknown> | null;
5717
- "n": number;
8855
+ "holm": {
8856
+ "status": "complete";
8857
+ "ready": number;
8858
+ "planned": number;
8859
+ "adjustment": {
8860
+ "scope": "planned-comparisons";
8861
+ "comparisons": number;
8862
+ };
8863
+ } | {
8864
+ "status": "pending";
8865
+ "ready": number;
8866
+ "planned": number;
5718
8867
  };
5719
- "graded": number;
5720
- "unmeasured": number;
5721
- "modelJudged": boolean;
5722
- };
5723
- "execution"?: {
5724
- "graded": number;
5725
- "measured": number;
5726
- "passed": number;
5727
- "passRate": number | null;
5728
- "byStatus": {
5729
- "passed": number;
5730
- "failed": number;
5731
- "timeout": number;
5732
- "error": number;
5733
- "unavailable": number;
8868
+ "claims": {
8869
+ "perContrast": Record<string, unknown>;
8870
+ "ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
5734
8871
  };
5735
- "languages": Array<string>;
5736
- "unavailableReasons": Array<string>;
5737
8872
  };
5738
8873
  };
5739
8874
  "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
@@ -5741,6 +8876,7 @@ export type StudyPreviewResponse = {
5741
8876
  export type StudyPublishRequest = {
5742
8877
  "slug"?: string;
5743
8878
  "visibility"?: "private" | "org" | "link" | "public";
8879
+ "skillNames"?: "withheld" | "named";
5744
8880
  };
5745
8881
  export type StudyPublishResponse = {
5746
8882
  "study": {
@@ -5773,7 +8909,7 @@ export type StudyPublishResponse = {
5773
8909
  "addons"?: Record<string, unknown>;
5774
8910
  };
5775
8911
  };
5776
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
8912
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5777
8913
  "subjectBackend": "model" | "harness" | null;
5778
8914
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5779
8915
  "importedFrom"?: string;
@@ -5809,6 +8945,7 @@ export type StudyPublishResponse = {
5809
8945
  "error": string | null;
5810
8946
  "preparation"?: {
5811
8947
  "since": string;
8948
+ "progressAt"?: string;
5812
8949
  "launchedBy": {
5813
8950
  "userId"?: string;
5814
8951
  "role"?: string;
@@ -5817,6 +8954,12 @@ export type StudyPublishResponse = {
5817
8954
  "imageKey": string;
5818
8955
  "buildId": string;
5819
8956
  "stage"?: "image" | "snapshot";
8957
+ "retry"?: {
8958
+ "failedAttempts": number;
8959
+ "maxAttempts": number;
8960
+ "notBefore": string;
8961
+ "lastFailure": string;
8962
+ };
5820
8963
  }>;
5821
8964
  } | null;
5822
8965
  "visibility": "private" | "org" | "link" | "public";
@@ -5831,6 +8974,57 @@ export type StudyPublishResponse = {
5831
8974
  "createdAt": string;
5832
8975
  "updatedAt": string;
5833
8976
  "liveUntil": string | null;
8977
+ "launch"?: {
8978
+ "attempt": number;
8979
+ "state": "fanning_out" | "deferred" | "settled";
8980
+ "startedAt": string;
8981
+ "progressAt": string;
8982
+ "planned"?: number;
8983
+ "created"?: number;
8984
+ "missing"?: {
8985
+ "retryable": number;
8986
+ "refused": number;
8987
+ };
8988
+ "notes"?: Array<string>;
8989
+ "settledAt"?: string;
8990
+ } | null;
8991
+ "relaunchable"?: boolean;
8992
+ "arms"?: Array<{
8993
+ "key": string;
8994
+ "position": number;
8995
+ "label": string;
8996
+ "role": "reference" | "peer";
8997
+ "payload": {
8998
+ "tools"?: "default" | "none" | {
8999
+ "tools": Array<string>;
9000
+ };
9001
+ "addons"?: Record<string, unknown>;
9002
+ };
9003
+ "cells"?: Array<{
9004
+ "harness": string;
9005
+ "model": string;
9006
+ "provider"?: string;
9007
+ "params"?: {
9008
+ "temperature"?: number;
9009
+ "maxOutputTokens"?: number;
9010
+ "reasoningEffort"?: "low" | "medium" | "high";
9011
+ "thinkingBudgetTokens"?: number;
9012
+ };
9013
+ }>;
9014
+ "skills"?: Array<{
9015
+ "ref": string;
9016
+ "name": string;
9017
+ "changedSinceCreate"?: boolean;
9018
+ }>;
9019
+ "addedAgents"?: Array<{
9020
+ "position": number;
9021
+ "addedAt": string;
9022
+ }>;
9023
+ }>;
9024
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9025
+ "cellPlan"?: "crossed" | "per_setup";
9026
+ "skillDisclosure"?: "withheld" | "named";
9027
+ "modeOptions"?: Record<string, unknown>;
5834
9028
  };
5835
9029
  };
5836
9030
  export type StudyReconcileRunsResponse = {
@@ -5873,7 +9067,7 @@ export type StudyRegradeResponse = {
5873
9067
  "addons"?: Record<string, unknown>;
5874
9068
  };
5875
9069
  };
5876
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
9070
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5877
9071
  "subjectBackend": "model" | "harness" | null;
5878
9072
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5879
9073
  "importedFrom"?: string;
@@ -5909,6 +9103,7 @@ export type StudyRegradeResponse = {
5909
9103
  "error": string | null;
5910
9104
  "preparation"?: {
5911
9105
  "since": string;
9106
+ "progressAt"?: string;
5912
9107
  "launchedBy": {
5913
9108
  "userId"?: string;
5914
9109
  "role"?: string;
@@ -5917,6 +9112,12 @@ export type StudyRegradeResponse = {
5917
9112
  "imageKey": string;
5918
9113
  "buildId": string;
5919
9114
  "stage"?: "image" | "snapshot";
9115
+ "retry"?: {
9116
+ "failedAttempts": number;
9117
+ "maxAttempts": number;
9118
+ "notBefore": string;
9119
+ "lastFailure": string;
9120
+ };
5920
9121
  }>;
5921
9122
  } | null;
5922
9123
  "visibility": "private" | "org" | "link" | "public";
@@ -5931,11 +9132,133 @@ export type StudyRegradeResponse = {
5931
9132
  "createdAt": string;
5932
9133
  "updatedAt": string;
5933
9134
  "liveUntil": string | null;
9135
+ "launch"?: {
9136
+ "attempt": number;
9137
+ "state": "fanning_out" | "deferred" | "settled";
9138
+ "startedAt": string;
9139
+ "progressAt": string;
9140
+ "planned"?: number;
9141
+ "created"?: number;
9142
+ "missing"?: {
9143
+ "retryable": number;
9144
+ "refused": number;
9145
+ };
9146
+ "notes"?: Array<string>;
9147
+ "settledAt"?: string;
9148
+ } | null;
9149
+ "relaunchable"?: boolean;
9150
+ "arms"?: Array<{
9151
+ "key": string;
9152
+ "position": number;
9153
+ "label": string;
9154
+ "role": "reference" | "peer";
9155
+ "payload": {
9156
+ "tools"?: "default" | "none" | {
9157
+ "tools": Array<string>;
9158
+ };
9159
+ "addons"?: Record<string, unknown>;
9160
+ };
9161
+ "cells"?: Array<{
9162
+ "harness": string;
9163
+ "model": string;
9164
+ "provider"?: string;
9165
+ "params"?: {
9166
+ "temperature"?: number;
9167
+ "maxOutputTokens"?: number;
9168
+ "reasoningEffort"?: "low" | "medium" | "high";
9169
+ "thinkingBudgetTokens"?: number;
9170
+ };
9171
+ }>;
9172
+ "skills"?: Array<{
9173
+ "ref": string;
9174
+ "name": string;
9175
+ "changedSinceCreate"?: boolean;
9176
+ }>;
9177
+ "addedAgents"?: Array<{
9178
+ "position": number;
9179
+ "addedAt": string;
9180
+ }>;
9181
+ }>;
9182
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9183
+ "cellPlan"?: "crossed" | "per_setup";
9184
+ "skillDisclosure"?: "withheld" | "named";
9185
+ "modeOptions"?: Record<string, unknown>;
5934
9186
  };
5935
9187
  "streamUrl": string;
5936
9188
  "warnings"?: Array<{
5937
9189
  "harness": string;
5938
9190
  "kind": string;
9191
+ } | {
9192
+ "type": "design_aggregate_budget";
9193
+ "detail": string;
9194
+ }>;
9195
+ };
9196
+ export type StudyRerunAttemptsResponse = {
9197
+ "studyId": string;
9198
+ "attempts": Array<{
9199
+ "taskId": string;
9200
+ "variantIndex": number;
9201
+ "repeatIndex": number;
9202
+ "arm": string;
9203
+ "harness": string;
9204
+ "model": string;
9205
+ "paramsKey": string;
9206
+ "attempt": number;
9207
+ "cause": "infra_failure" | "missing" | "declared_infra";
9208
+ "evidence": string | null;
9209
+ "replacedRunId": string | null;
9210
+ "replacedStatus": string | null;
9211
+ "replacedOutcomeReason": string | null;
9212
+ "replacedStartedAt": string | null;
9213
+ "replacedFinishedAt": string | null;
9214
+ "runId": string | null;
9215
+ "outcome": "dispatched" | "failed";
9216
+ "error": string | null;
9217
+ "requestedBy": string | null;
9218
+ "createdAt": string;
9219
+ }>;
9220
+ };
9221
+ export type StudyRerunInfraFailuresRequest = {
9222
+ "dryRun"?: boolean;
9223
+ "limit"?: number;
9224
+ "declared"?: Array<{
9225
+ "runId": string;
9226
+ "evidence": string;
9227
+ }>;
9228
+ };
9229
+ export type StudyRerunInfraFailuresResponse = {
9230
+ "studyId": string;
9231
+ "status": string;
9232
+ "dryRun": boolean;
9233
+ "reopened": boolean;
9234
+ "maxAttemptsPerJob": number;
9235
+ "jobs": Array<{
9236
+ "taskId": string;
9237
+ "variantIndex": number;
9238
+ "repeatIndex": number;
9239
+ "arm": string;
9240
+ "harness": string;
9241
+ "model": string;
9242
+ "paramsKey": string;
9243
+ "cause": "infra_failure" | "missing" | "declared_infra";
9244
+ "evidence": string | null;
9245
+ "attempt": number;
9246
+ "replacedRunId": string | null;
9247
+ "runId": string | null;
9248
+ "outcome": "planned" | "dispatched" | "failed";
9249
+ "error": string | null;
9250
+ }>;
9251
+ "skipped": {
9252
+ "live": number;
9253
+ "exhausted": number;
9254
+ "missingDeferred": number;
9255
+ "overLimit": number;
9256
+ };
9257
+ "notes": Array<string>;
9258
+ "byGroup": Record<string, unknown>;
9259
+ "declaredRejected": Array<{
9260
+ "runId": string;
9261
+ "reason": string;
5939
9262
  }>;
5940
9263
  };
5941
9264
  export type StudyListRunAnswersResponse = {
@@ -5984,7 +9307,7 @@ export type StudyRotateShareTokenResponse = {
5984
9307
  "addons"?: Record<string, unknown>;
5985
9308
  };
5986
9309
  };
5987
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
9310
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
5988
9311
  "subjectBackend": "model" | "harness" | null;
5989
9312
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5990
9313
  "importedFrom"?: string;
@@ -6020,6 +9343,7 @@ export type StudyRotateShareTokenResponse = {
6020
9343
  "error": string | null;
6021
9344
  "preparation"?: {
6022
9345
  "since": string;
9346
+ "progressAt"?: string;
6023
9347
  "launchedBy": {
6024
9348
  "userId"?: string;
6025
9349
  "role"?: string;
@@ -6028,6 +9352,12 @@ export type StudyRotateShareTokenResponse = {
6028
9352
  "imageKey": string;
6029
9353
  "buildId": string;
6030
9354
  "stage"?: "image" | "snapshot";
9355
+ "retry"?: {
9356
+ "failedAttempts": number;
9357
+ "maxAttempts": number;
9358
+ "notBefore": string;
9359
+ "lastFailure": string;
9360
+ };
6031
9361
  }>;
6032
9362
  } | null;
6033
9363
  "visibility": "private" | "org" | "link" | "public";
@@ -6042,13 +9372,132 @@ export type StudyRotateShareTokenResponse = {
6042
9372
  "createdAt": string;
6043
9373
  "updatedAt": string;
6044
9374
  "liveUntil": string | null;
9375
+ "launch"?: {
9376
+ "attempt": number;
9377
+ "state": "fanning_out" | "deferred" | "settled";
9378
+ "startedAt": string;
9379
+ "progressAt": string;
9380
+ "planned"?: number;
9381
+ "created"?: number;
9382
+ "missing"?: {
9383
+ "retryable": number;
9384
+ "refused": number;
9385
+ };
9386
+ "notes"?: Array<string>;
9387
+ "settledAt"?: string;
9388
+ } | null;
9389
+ "relaunchable"?: boolean;
9390
+ "arms"?: Array<{
9391
+ "key": string;
9392
+ "position": number;
9393
+ "label": string;
9394
+ "role": "reference" | "peer";
9395
+ "payload": {
9396
+ "tools"?: "default" | "none" | {
9397
+ "tools": Array<string>;
9398
+ };
9399
+ "addons"?: Record<string, unknown>;
9400
+ };
9401
+ "cells"?: Array<{
9402
+ "harness": string;
9403
+ "model": string;
9404
+ "provider"?: string;
9405
+ "params"?: {
9406
+ "temperature"?: number;
9407
+ "maxOutputTokens"?: number;
9408
+ "reasoningEffort"?: "low" | "medium" | "high";
9409
+ "thinkingBudgetTokens"?: number;
9410
+ };
9411
+ }>;
9412
+ "skills"?: Array<{
9413
+ "ref": string;
9414
+ "name": string;
9415
+ "changedSinceCreate"?: boolean;
9416
+ }>;
9417
+ "addedAgents"?: Array<{
9418
+ "position": number;
9419
+ "addedAt": string;
9420
+ }>;
9421
+ }>;
9422
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9423
+ "cellPlan"?: "crossed" | "per_setup";
9424
+ "skillDisclosure"?: "withheld" | "named";
9425
+ "modeOptions"?: Record<string, unknown>;
6045
9426
  };
6046
9427
  "shareUrl": string | null;
6047
9428
  };
6048
9429
  export type StudyStreamResponse = unknown;
6049
9430
  export type StudyUpdateTaskRequest = {
6050
9431
  "prompt": string;
6051
- "expectations"?: Array<unknown>;
9432
+ "expectations"?: Array<{
9433
+ "label"?: string;
9434
+ "arms"?: Array<"baseline" | "skill" | string>;
9435
+ "kind": "mcp_tool_called";
9436
+ "server"?: string;
9437
+ "tool": string;
9438
+ "arguments"?: Record<string, unknown>;
9439
+ "minCalls"?: number;
9440
+ } | {
9441
+ "label"?: string;
9442
+ "arms"?: Array<"baseline" | "skill" | string>;
9443
+ "kind": "tool_not_called";
9444
+ "tool"?: string;
9445
+ "mcp"?: {
9446
+ "server"?: string;
9447
+ "tool": string;
9448
+ };
9449
+ "pattern"?: string;
9450
+ } | {
9451
+ "label"?: string;
9452
+ "arms"?: Array<"baseline" | "skill" | string>;
9453
+ "kind": "command_ran";
9454
+ "pattern": string;
9455
+ } | {
9456
+ "label"?: string;
9457
+ "arms"?: Array<"baseline" | "skill" | string>;
9458
+ "kind": "skill_loaded";
9459
+ "slug": string;
9460
+ } | {
9461
+ "label"?: string;
9462
+ "arms"?: Array<"baseline" | "skill" | string>;
9463
+ "kind": "file_operation";
9464
+ "op"?: "read" | "write" | "edit";
9465
+ "path": unknown;
9466
+ } | {
9467
+ "label"?: string;
9468
+ "arms"?: Array<"baseline" | "skill" | string>;
9469
+ "kind": "web_fetched";
9470
+ "urlHost"?: string;
9471
+ "urlPattern"?: string;
9472
+ } | {
9473
+ "label"?: string;
9474
+ "arms"?: Array<"baseline" | "skill" | string>;
9475
+ "kind": "response_contains";
9476
+ "text": string;
9477
+ "normalize"?: "prose";
9478
+ } | {
9479
+ "label"?: string;
9480
+ "arms"?: Array<"baseline" | "skill" | string>;
9481
+ "kind": "response_matches";
9482
+ "pattern": string;
9483
+ "ignoreCase"?: boolean;
9484
+ } | {
9485
+ "label"?: string;
9486
+ "arms"?: Array<"baseline" | "skill" | string>;
9487
+ "kind": "token_budget";
9488
+ "maxOutputTokens"?: number;
9489
+ "maxTotalTokens"?: number;
9490
+ } | {
9491
+ "label"?: string;
9492
+ "arms"?: Array<"baseline" | "skill" | string>;
9493
+ "kind": "sequence";
9494
+ "of": Array<unknown>;
9495
+ } | {
9496
+ "label"?: string;
9497
+ "arms"?: Array<"baseline" | "skill" | string>;
9498
+ "kind": "any_of";
9499
+ "of": Array<unknown>;
9500
+ }>;
6052
9501
  };
6053
9502
  export type StudyUpdateTaskResponse = {
6054
9503
  "task": {
@@ -6095,7 +9544,7 @@ export type StudyUnpublishResponse = {
6095
9544
  "addons"?: Record<string, unknown>;
6096
9545
  };
6097
9546
  };
6098
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
9547
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6099
9548
  "subjectBackend": "model" | "harness" | null;
6100
9549
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6101
9550
  "importedFrom"?: string;
@@ -6131,6 +9580,7 @@ export type StudyUnpublishResponse = {
6131
9580
  "error": string | null;
6132
9581
  "preparation"?: {
6133
9582
  "since": string;
9583
+ "progressAt"?: string;
6134
9584
  "launchedBy": {
6135
9585
  "userId"?: string;
6136
9586
  "role"?: string;
@@ -6139,6 +9589,12 @@ export type StudyUnpublishResponse = {
6139
9589
  "imageKey": string;
6140
9590
  "buildId": string;
6141
9591
  "stage"?: "image" | "snapshot";
9592
+ "retry"?: {
9593
+ "failedAttempts": number;
9594
+ "maxAttempts": number;
9595
+ "notBefore": string;
9596
+ "lastFailure": string;
9597
+ };
6142
9598
  }>;
6143
9599
  } | null;
6144
9600
  "visibility": "private" | "org" | "link" | "public";
@@ -6153,6 +9609,57 @@ export type StudyUnpublishResponse = {
6153
9609
  "createdAt": string;
6154
9610
  "updatedAt": string;
6155
9611
  "liveUntil": string | null;
9612
+ "launch"?: {
9613
+ "attempt": number;
9614
+ "state": "fanning_out" | "deferred" | "settled";
9615
+ "startedAt": string;
9616
+ "progressAt": string;
9617
+ "planned"?: number;
9618
+ "created"?: number;
9619
+ "missing"?: {
9620
+ "retryable": number;
9621
+ "refused": number;
9622
+ };
9623
+ "notes"?: Array<string>;
9624
+ "settledAt"?: string;
9625
+ } | null;
9626
+ "relaunchable"?: boolean;
9627
+ "arms"?: Array<{
9628
+ "key": string;
9629
+ "position": number;
9630
+ "label": string;
9631
+ "role": "reference" | "peer";
9632
+ "payload": {
9633
+ "tools"?: "default" | "none" | {
9634
+ "tools": Array<string>;
9635
+ };
9636
+ "addons"?: Record<string, unknown>;
9637
+ };
9638
+ "cells"?: Array<{
9639
+ "harness": string;
9640
+ "model": string;
9641
+ "provider"?: string;
9642
+ "params"?: {
9643
+ "temperature"?: number;
9644
+ "maxOutputTokens"?: number;
9645
+ "reasoningEffort"?: "low" | "medium" | "high";
9646
+ "thinkingBudgetTokens"?: number;
9647
+ };
9648
+ }>;
9649
+ "skills"?: Array<{
9650
+ "ref": string;
9651
+ "name": string;
9652
+ "changedSinceCreate"?: boolean;
9653
+ }>;
9654
+ "addedAgents"?: Array<{
9655
+ "position": number;
9656
+ "addedAt": string;
9657
+ }>;
9658
+ }>;
9659
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9660
+ "cellPlan"?: "crossed" | "per_setup";
9661
+ "skillDisclosure"?: "withheld" | "named";
9662
+ "modeOptions"?: Record<string, unknown>;
6156
9663
  };
6157
9664
  };
6158
9665
  export type StudySetVisibilityRequest = {
@@ -6189,7 +9696,7 @@ export type StudySetVisibilityResponse = {
6189
9696
  "addons"?: Record<string, unknown>;
6190
9697
  };
6191
9698
  };
6192
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
9699
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
6193
9700
  "subjectBackend": "model" | "harness" | null;
6194
9701
  "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6195
9702
  "importedFrom"?: string;
@@ -6225,6 +9732,7 @@ export type StudySetVisibilityResponse = {
6225
9732
  "error": string | null;
6226
9733
  "preparation"?: {
6227
9734
  "since": string;
9735
+ "progressAt"?: string;
6228
9736
  "launchedBy": {
6229
9737
  "userId"?: string;
6230
9738
  "role"?: string;
@@ -6233,6 +9741,12 @@ export type StudySetVisibilityResponse = {
6233
9741
  "imageKey": string;
6234
9742
  "buildId": string;
6235
9743
  "stage"?: "image" | "snapshot";
9744
+ "retry"?: {
9745
+ "failedAttempts": number;
9746
+ "maxAttempts": number;
9747
+ "notBefore": string;
9748
+ "lastFailure": string;
9749
+ };
6236
9750
  }>;
6237
9751
  } | null;
6238
9752
  "visibility": "private" | "org" | "link" | "public";
@@ -6247,6 +9761,57 @@ export type StudySetVisibilityResponse = {
6247
9761
  "createdAt": string;
6248
9762
  "updatedAt": string;
6249
9763
  "liveUntil": string | null;
9764
+ "launch"?: {
9765
+ "attempt": number;
9766
+ "state": "fanning_out" | "deferred" | "settled";
9767
+ "startedAt": string;
9768
+ "progressAt": string;
9769
+ "planned"?: number;
9770
+ "created"?: number;
9771
+ "missing"?: {
9772
+ "retryable": number;
9773
+ "refused": number;
9774
+ };
9775
+ "notes"?: Array<string>;
9776
+ "settledAt"?: string;
9777
+ } | null;
9778
+ "relaunchable"?: boolean;
9779
+ "arms"?: Array<{
9780
+ "key": string;
9781
+ "position": number;
9782
+ "label": string;
9783
+ "role": "reference" | "peer";
9784
+ "payload": {
9785
+ "tools"?: "default" | "none" | {
9786
+ "tools": Array<string>;
9787
+ };
9788
+ "addons"?: Record<string, unknown>;
9789
+ };
9790
+ "cells"?: Array<{
9791
+ "harness": string;
9792
+ "model": string;
9793
+ "provider"?: string;
9794
+ "params"?: {
9795
+ "temperature"?: number;
9796
+ "maxOutputTokens"?: number;
9797
+ "reasoningEffort"?: "low" | "medium" | "high";
9798
+ "thinkingBudgetTokens"?: number;
9799
+ };
9800
+ }>;
9801
+ "skills"?: Array<{
9802
+ "ref": string;
9803
+ "name": string;
9804
+ "changedSinceCreate"?: boolean;
9805
+ }>;
9806
+ "addedAgents"?: Array<{
9807
+ "position": number;
9808
+ "addedAt": string;
9809
+ }>;
9810
+ }>;
9811
+ "contrastPlan"?: "vs_reference" | "all_pairs";
9812
+ "cellPlan"?: "crossed" | "per_setup";
9813
+ "skillDisclosure"?: "withheld" | "named";
9814
+ "modeOptions"?: Record<string, unknown>;
6250
9815
  };
6251
9816
  "shareUrl": string | null;
6252
9817
  };