@ora-ai/cli 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +592 -19
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +3757 -192
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/dist/render/insight-lines.d.ts.map +1 -1
- package/dist/render/insight-lines.js +14 -3
- package/dist/render/insight-lines.js.map +1 -1
- package/package.json +1 -1
|
@@ -84,6 +84,7 @@ export type BenchmarksListResponse = {
|
|
|
84
84
|
"error": string | null;
|
|
85
85
|
"preparation"?: {
|
|
86
86
|
"since": string;
|
|
87
|
+
"progressAt"?: string;
|
|
87
88
|
"launchedBy": {
|
|
88
89
|
"userId"?: string;
|
|
89
90
|
"role"?: string;
|
|
@@ -92,6 +93,12 @@ export type BenchmarksListResponse = {
|
|
|
92
93
|
"imageKey": string;
|
|
93
94
|
"buildId": string;
|
|
94
95
|
"stage"?: "image" | "snapshot";
|
|
96
|
+
"retry"?: {
|
|
97
|
+
"failedAttempts": number;
|
|
98
|
+
"maxAttempts": number;
|
|
99
|
+
"notBefore": string;
|
|
100
|
+
"lastFailure": string;
|
|
101
|
+
};
|
|
95
102
|
}>;
|
|
96
103
|
} | null;
|
|
97
104
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -232,6 +239,7 @@ export type BenchmarksCreateResponse = {
|
|
|
232
239
|
"error": string | null;
|
|
233
240
|
"preparation"?: {
|
|
234
241
|
"since": string;
|
|
242
|
+
"progressAt"?: string;
|
|
235
243
|
"launchedBy": {
|
|
236
244
|
"userId"?: string;
|
|
237
245
|
"role"?: string;
|
|
@@ -240,6 +248,12 @@ export type BenchmarksCreateResponse = {
|
|
|
240
248
|
"imageKey": string;
|
|
241
249
|
"buildId": string;
|
|
242
250
|
"stage"?: "image" | "snapshot";
|
|
251
|
+
"retry"?: {
|
|
252
|
+
"failedAttempts": number;
|
|
253
|
+
"maxAttempts": number;
|
|
254
|
+
"notBefore": string;
|
|
255
|
+
"lastFailure": string;
|
|
256
|
+
};
|
|
243
257
|
}>;
|
|
244
258
|
} | null;
|
|
245
259
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -340,6 +354,7 @@ export type BenchmarksGetResponse = {
|
|
|
340
354
|
"error": string | null;
|
|
341
355
|
"preparation"?: {
|
|
342
356
|
"since": string;
|
|
357
|
+
"progressAt"?: string;
|
|
343
358
|
"launchedBy": {
|
|
344
359
|
"userId"?: string;
|
|
345
360
|
"role"?: string;
|
|
@@ -348,6 +363,12 @@ export type BenchmarksGetResponse = {
|
|
|
348
363
|
"imageKey": string;
|
|
349
364
|
"buildId": string;
|
|
350
365
|
"stage"?: "image" | "snapshot";
|
|
366
|
+
"retry"?: {
|
|
367
|
+
"failedAttempts": number;
|
|
368
|
+
"maxAttempts": number;
|
|
369
|
+
"notBefore": string;
|
|
370
|
+
"lastFailure": string;
|
|
371
|
+
};
|
|
351
372
|
}>;
|
|
352
373
|
} | null;
|
|
353
374
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -474,6 +495,9 @@ export type BenchmarksGetResponse = {
|
|
|
474
495
|
"gradedAt": string;
|
|
475
496
|
};
|
|
476
497
|
} | null;
|
|
498
|
+
"executionBudgetS"?: number | null;
|
|
499
|
+
"harnessImageDigest"?: string | null;
|
|
500
|
+
"subjectEngaged"?: boolean | null;
|
|
477
501
|
"runCreatedAt": string | null;
|
|
478
502
|
"startedAt": string | null;
|
|
479
503
|
"finishedAt": string | null;
|
|
@@ -683,6 +707,17 @@ export type BenchmarksGetResponse = {
|
|
|
683
707
|
"ci": Array<unknown> | null;
|
|
684
708
|
"n": number;
|
|
685
709
|
};
|
|
710
|
+
"abstention"?: {
|
|
711
|
+
"value": number | null;
|
|
712
|
+
"ci": Array<unknown> | null;
|
|
713
|
+
"n": number;
|
|
714
|
+
"parts"?: Array<{
|
|
715
|
+
"id": string;
|
|
716
|
+
"label": string;
|
|
717
|
+
"value": number | null;
|
|
718
|
+
"n": number;
|
|
719
|
+
}>;
|
|
720
|
+
};
|
|
686
721
|
"tasks": number;
|
|
687
722
|
"graded": number;
|
|
688
723
|
"unmeasured": number;
|
|
@@ -694,6 +729,17 @@ export type BenchmarksGetResponse = {
|
|
|
694
729
|
"ci": Array<unknown> | null;
|
|
695
730
|
"n": number;
|
|
696
731
|
};
|
|
732
|
+
"abstention"?: {
|
|
733
|
+
"value": number | null;
|
|
734
|
+
"ci": Array<unknown> | null;
|
|
735
|
+
"n": number;
|
|
736
|
+
"parts"?: Array<{
|
|
737
|
+
"id": string;
|
|
738
|
+
"label": string;
|
|
739
|
+
"value": number | null;
|
|
740
|
+
"n": number;
|
|
741
|
+
}>;
|
|
742
|
+
};
|
|
697
743
|
"graded": number;
|
|
698
744
|
"unmeasured": number;
|
|
699
745
|
"modelJudged": boolean;
|
|
@@ -723,9 +769,130 @@ export type BenchmarksGetResponse = {
|
|
|
723
769
|
"detail": string;
|
|
724
770
|
"taskIds"?: Array<string>;
|
|
725
771
|
}>;
|
|
772
|
+
"taskHealth": {
|
|
773
|
+
"flags": Array<{
|
|
774
|
+
"taskId": string;
|
|
775
|
+
"signal": string;
|
|
776
|
+
"kind": "floor" | "ceiling";
|
|
777
|
+
"conditions": number;
|
|
778
|
+
}>;
|
|
779
|
+
"rubricSplits": Array<{
|
|
780
|
+
"taskId": string;
|
|
781
|
+
"criterionIndex": number;
|
|
782
|
+
"criterion": string;
|
|
783
|
+
"split": number;
|
|
784
|
+
"graded": number;
|
|
785
|
+
}>;
|
|
786
|
+
"signals": Array<{
|
|
787
|
+
"id": string;
|
|
788
|
+
"label": string;
|
|
789
|
+
}>;
|
|
790
|
+
};
|
|
726
791
|
"shareUrl"?: string | null;
|
|
727
792
|
"canManageSharing"?: boolean;
|
|
728
793
|
};
|
|
794
|
+
export type BenchmarksUpdateDetailsRequest = {
|
|
795
|
+
"name"?: string;
|
|
796
|
+
"description"?: string;
|
|
797
|
+
};
|
|
798
|
+
export type BenchmarksUpdateDetailsResponse = {
|
|
799
|
+
"study": {
|
|
800
|
+
"id": string;
|
|
801
|
+
"executiveSummary"?: string | null;
|
|
802
|
+
"plannedRunCount": number;
|
|
803
|
+
"organizationId": string;
|
|
804
|
+
"createdBy": string | null;
|
|
805
|
+
"name": string;
|
|
806
|
+
"slug": string;
|
|
807
|
+
"description": string | null;
|
|
808
|
+
"skillRef": string | null;
|
|
809
|
+
"skillSlug": string | null;
|
|
810
|
+
"skillName": string | null;
|
|
811
|
+
"skillPromise": string;
|
|
812
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
813
|
+
"subjectName": string | null;
|
|
814
|
+
"subjectLabel": string;
|
|
815
|
+
"armConfig": {
|
|
816
|
+
"baseline"?: {
|
|
817
|
+
"tools"?: "default" | "none" | {
|
|
818
|
+
"tools": Array<string>;
|
|
819
|
+
};
|
|
820
|
+
"addons"?: Record<string, unknown>;
|
|
821
|
+
};
|
|
822
|
+
"treatment"?: {
|
|
823
|
+
"tools"?: "default" | "none" | {
|
|
824
|
+
"tools": Array<string>;
|
|
825
|
+
};
|
|
826
|
+
"addons"?: Record<string, unknown>;
|
|
827
|
+
};
|
|
828
|
+
};
|
|
829
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
830
|
+
"subjectBackend": "model" | "harness" | null;
|
|
831
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
832
|
+
"importedFrom"?: string;
|
|
833
|
+
"baselineLabel": string | null;
|
|
834
|
+
"treatmentLabel": string | null;
|
|
835
|
+
"promiseMetrics": Array<{
|
|
836
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
837
|
+
"direction": "down" | "up";
|
|
838
|
+
"label": string;
|
|
839
|
+
}>;
|
|
840
|
+
"matrix": Array<{
|
|
841
|
+
"harness": string;
|
|
842
|
+
"model": string;
|
|
843
|
+
"provider"?: string;
|
|
844
|
+
"params"?: {
|
|
845
|
+
"temperature"?: number;
|
|
846
|
+
"maxOutputTokens"?: number;
|
|
847
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
848
|
+
"thinkingBudgetTokens"?: number;
|
|
849
|
+
};
|
|
850
|
+
}>;
|
|
851
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
852
|
+
"judgeSupplementary"?: boolean;
|
|
853
|
+
"taskCount": number;
|
|
854
|
+
"variantsPerTask": number;
|
|
855
|
+
"repeats": number;
|
|
856
|
+
"subset": {
|
|
857
|
+
"count": number;
|
|
858
|
+
"seed": string;
|
|
859
|
+
"of": number;
|
|
860
|
+
} | null;
|
|
861
|
+
"status": string;
|
|
862
|
+
"error": string | null;
|
|
863
|
+
"preparation"?: {
|
|
864
|
+
"since": string;
|
|
865
|
+
"progressAt"?: string;
|
|
866
|
+
"launchedBy": {
|
|
867
|
+
"userId"?: string;
|
|
868
|
+
"role"?: string;
|
|
869
|
+
};
|
|
870
|
+
"waitingOn": Array<{
|
|
871
|
+
"imageKey": string;
|
|
872
|
+
"buildId": string;
|
|
873
|
+
"stage"?: "image" | "snapshot";
|
|
874
|
+
"retry"?: {
|
|
875
|
+
"failedAttempts": number;
|
|
876
|
+
"maxAttempts": number;
|
|
877
|
+
"notBefore": string;
|
|
878
|
+
"lastFailure": string;
|
|
879
|
+
};
|
|
880
|
+
}>;
|
|
881
|
+
} | null;
|
|
882
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
883
|
+
"visibilityChangedAt"?: string | null;
|
|
884
|
+
"visibilityChangedBy"?: string | null;
|
|
885
|
+
"shareToken"?: string | null;
|
|
886
|
+
"lowSampleOverride": boolean;
|
|
887
|
+
"publishedAt": string | null;
|
|
888
|
+
"verdict": unknown;
|
|
889
|
+
"definitionVersionId": string | null;
|
|
890
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
891
|
+
"createdAt": string;
|
|
892
|
+
"updatedAt": string;
|
|
893
|
+
"liveUntil": string | null;
|
|
894
|
+
};
|
|
895
|
+
};
|
|
729
896
|
export type BenchmarksDeleteResponse = unknown;
|
|
730
897
|
export type BenchmarksRegenerateSummaryResponse = {
|
|
731
898
|
"status": "queued";
|
|
@@ -800,6 +967,7 @@ export type BenchmarksGenerateResponse = {
|
|
|
800
967
|
"error": string | null;
|
|
801
968
|
"preparation"?: {
|
|
802
969
|
"since": string;
|
|
970
|
+
"progressAt"?: string;
|
|
803
971
|
"launchedBy": {
|
|
804
972
|
"userId"?: string;
|
|
805
973
|
"role"?: string;
|
|
@@ -808,6 +976,12 @@ export type BenchmarksGenerateResponse = {
|
|
|
808
976
|
"imageKey": string;
|
|
809
977
|
"buildId": string;
|
|
810
978
|
"stage"?: "image" | "snapshot";
|
|
979
|
+
"retry"?: {
|
|
980
|
+
"failedAttempts": number;
|
|
981
|
+
"maxAttempts": number;
|
|
982
|
+
"notBefore": string;
|
|
983
|
+
"lastFailure": string;
|
|
984
|
+
};
|
|
811
985
|
}>;
|
|
812
986
|
} | null;
|
|
813
987
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -896,6 +1070,7 @@ export type BenchmarksLaunchResponse = {
|
|
|
896
1070
|
"error": string | null;
|
|
897
1071
|
"preparation"?: {
|
|
898
1072
|
"since": string;
|
|
1073
|
+
"progressAt"?: string;
|
|
899
1074
|
"launchedBy": {
|
|
900
1075
|
"userId"?: string;
|
|
901
1076
|
"role"?: string;
|
|
@@ -904,6 +1079,12 @@ export type BenchmarksLaunchResponse = {
|
|
|
904
1079
|
"imageKey": string;
|
|
905
1080
|
"buildId": string;
|
|
906
1081
|
"stage"?: "image" | "snapshot";
|
|
1082
|
+
"retry"?: {
|
|
1083
|
+
"failedAttempts": number;
|
|
1084
|
+
"maxAttempts": number;
|
|
1085
|
+
"notBefore": string;
|
|
1086
|
+
"lastFailure": string;
|
|
1087
|
+
};
|
|
907
1088
|
}>;
|
|
908
1089
|
} | null;
|
|
909
1090
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -995,6 +1176,7 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
995
1176
|
"error": string | null;
|
|
996
1177
|
"preparation"?: {
|
|
997
1178
|
"since": string;
|
|
1179
|
+
"progressAt"?: string;
|
|
998
1180
|
"launchedBy": {
|
|
999
1181
|
"userId"?: string;
|
|
1000
1182
|
"role"?: string;
|
|
@@ -1003,6 +1185,12 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
1003
1185
|
"imageKey": string;
|
|
1004
1186
|
"buildId": string;
|
|
1005
1187
|
"stage"?: "image" | "snapshot";
|
|
1188
|
+
"retry"?: {
|
|
1189
|
+
"failedAttempts": number;
|
|
1190
|
+
"maxAttempts": number;
|
|
1191
|
+
"notBefore": string;
|
|
1192
|
+
"lastFailure": string;
|
|
1193
|
+
};
|
|
1006
1194
|
}>;
|
|
1007
1195
|
} | null;
|
|
1008
1196
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1139,6 +1327,9 @@ export type BenchmarksPreviewResponse = {
|
|
|
1139
1327
|
"pairGradedAt": string | null;
|
|
1140
1328
|
"evalScore": number | null;
|
|
1141
1329
|
"evalGradedAt": string | null;
|
|
1330
|
+
"executionBudgetS"?: number | null;
|
|
1331
|
+
"harnessImageDigest"?: string | null;
|
|
1332
|
+
"subjectEngaged"?: boolean | null;
|
|
1142
1333
|
"runCreatedAt": string | null;
|
|
1143
1334
|
"startedAt": string | null;
|
|
1144
1335
|
"finishedAt": string | null;
|
|
@@ -1359,6 +1550,17 @@ export type BenchmarksPreviewResponse = {
|
|
|
1359
1550
|
"ci": Array<unknown> | null;
|
|
1360
1551
|
"n": number;
|
|
1361
1552
|
};
|
|
1553
|
+
"abstention"?: {
|
|
1554
|
+
"value": number | null;
|
|
1555
|
+
"ci": Array<unknown> | null;
|
|
1556
|
+
"n": number;
|
|
1557
|
+
"parts"?: Array<{
|
|
1558
|
+
"id": string;
|
|
1559
|
+
"label": string;
|
|
1560
|
+
"value": number | null;
|
|
1561
|
+
"n": number;
|
|
1562
|
+
}>;
|
|
1563
|
+
};
|
|
1362
1564
|
"tasks": number;
|
|
1363
1565
|
"graded": number;
|
|
1364
1566
|
"unmeasured": number;
|
|
@@ -1370,6 +1572,17 @@ export type BenchmarksPreviewResponse = {
|
|
|
1370
1572
|
"ci": Array<unknown> | null;
|
|
1371
1573
|
"n": number;
|
|
1372
1574
|
};
|
|
1575
|
+
"abstention"?: {
|
|
1576
|
+
"value": number | null;
|
|
1577
|
+
"ci": Array<unknown> | null;
|
|
1578
|
+
"n": number;
|
|
1579
|
+
"parts"?: Array<{
|
|
1580
|
+
"id": string;
|
|
1581
|
+
"label": string;
|
|
1582
|
+
"value": number | null;
|
|
1583
|
+
"n": number;
|
|
1584
|
+
}>;
|
|
1585
|
+
};
|
|
1373
1586
|
"graded": number;
|
|
1374
1587
|
"unmeasured": number;
|
|
1375
1588
|
"modelJudged": boolean;
|
|
@@ -1463,6 +1676,7 @@ export type BenchmarksPublishResponse = {
|
|
|
1463
1676
|
"error": string | null;
|
|
1464
1677
|
"preparation"?: {
|
|
1465
1678
|
"since": string;
|
|
1679
|
+
"progressAt"?: string;
|
|
1466
1680
|
"launchedBy": {
|
|
1467
1681
|
"userId"?: string;
|
|
1468
1682
|
"role"?: string;
|
|
@@ -1471,6 +1685,12 @@ export type BenchmarksPublishResponse = {
|
|
|
1471
1685
|
"imageKey": string;
|
|
1472
1686
|
"buildId": string;
|
|
1473
1687
|
"stage"?: "image" | "snapshot";
|
|
1688
|
+
"retry"?: {
|
|
1689
|
+
"failedAttempts": number;
|
|
1690
|
+
"maxAttempts": number;
|
|
1691
|
+
"notBefore": string;
|
|
1692
|
+
"lastFailure": string;
|
|
1693
|
+
};
|
|
1474
1694
|
}>;
|
|
1475
1695
|
} | null;
|
|
1476
1696
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1554,6 +1774,7 @@ export type BenchmarksRegradeResponse = {
|
|
|
1554
1774
|
"error": string | null;
|
|
1555
1775
|
"preparation"?: {
|
|
1556
1776
|
"since": string;
|
|
1777
|
+
"progressAt"?: string;
|
|
1557
1778
|
"launchedBy": {
|
|
1558
1779
|
"userId"?: string;
|
|
1559
1780
|
"role"?: string;
|
|
@@ -1562,6 +1783,12 @@ export type BenchmarksRegradeResponse = {
|
|
|
1562
1783
|
"imageKey": string;
|
|
1563
1784
|
"buildId": string;
|
|
1564
1785
|
"stage"?: "image" | "snapshot";
|
|
1786
|
+
"retry"?: {
|
|
1787
|
+
"failedAttempts": number;
|
|
1788
|
+
"maxAttempts": number;
|
|
1789
|
+
"notBefore": string;
|
|
1790
|
+
"lastFailure": string;
|
|
1791
|
+
};
|
|
1565
1792
|
}>;
|
|
1566
1793
|
} | null;
|
|
1567
1794
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1669,6 +1896,7 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1669
1896
|
"error": string | null;
|
|
1670
1897
|
"preparation"?: {
|
|
1671
1898
|
"since": string;
|
|
1899
|
+
"progressAt"?: string;
|
|
1672
1900
|
"launchedBy": {
|
|
1673
1901
|
"userId"?: string;
|
|
1674
1902
|
"role"?: string;
|
|
@@ -1677,6 +1905,12 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1677
1905
|
"imageKey": string;
|
|
1678
1906
|
"buildId": string;
|
|
1679
1907
|
"stage"?: "image" | "snapshot";
|
|
1908
|
+
"retry"?: {
|
|
1909
|
+
"failedAttempts": number;
|
|
1910
|
+
"maxAttempts": number;
|
|
1911
|
+
"notBefore": string;
|
|
1912
|
+
"lastFailure": string;
|
|
1913
|
+
};
|
|
1680
1914
|
}>;
|
|
1681
1915
|
} | null;
|
|
1682
1916
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1734,6 +1968,9 @@ export type ExperimentImagesBuildsListResponse = {
|
|
|
1734
1968
|
} | {
|
|
1735
1969
|
"kind": "inline";
|
|
1736
1970
|
"fileCount": number;
|
|
1971
|
+
} | {
|
|
1972
|
+
"kind": "asset";
|
|
1973
|
+
"sha256": string;
|
|
1737
1974
|
};
|
|
1738
1975
|
"sourceIdentity": string;
|
|
1739
1976
|
"dockerfileDir": string;
|
|
@@ -1741,7 +1978,7 @@ export type ExperimentImagesBuildsListResponse = {
|
|
|
1741
1978
|
"buildArgKeys": Array<string>;
|
|
1742
1979
|
"platform": string;
|
|
1743
1980
|
"trusted": boolean;
|
|
1744
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1981
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1745
1982
|
"image": string;
|
|
1746
1983
|
"digest": string | null;
|
|
1747
1984
|
"sizeBytes": number | null;
|
|
@@ -1782,7 +2019,7 @@ export type ExperimentImagesBuildsCreateRequest = {
|
|
|
1782
2019
|
};
|
|
1783
2020
|
export type ExperimentImagesBuildsCreateResponse = {
|
|
1784
2021
|
"id": string;
|
|
1785
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
2022
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1786
2023
|
};
|
|
1787
2024
|
export type ExperimentImagesBuildsGetResponse = {
|
|
1788
2025
|
"build": {
|
|
@@ -1804,6 +2041,9 @@ export type ExperimentImagesBuildsGetResponse = {
|
|
|
1804
2041
|
} | {
|
|
1805
2042
|
"kind": "inline";
|
|
1806
2043
|
"fileCount": number;
|
|
2044
|
+
} | {
|
|
2045
|
+
"kind": "asset";
|
|
2046
|
+
"sha256": string;
|
|
1807
2047
|
};
|
|
1808
2048
|
"sourceIdentity": string;
|
|
1809
2049
|
"dockerfileDir": string;
|
|
@@ -1811,7 +2051,7 @@ export type ExperimentImagesBuildsGetResponse = {
|
|
|
1811
2051
|
"buildArgKeys": Array<string>;
|
|
1812
2052
|
"platform": string;
|
|
1813
2053
|
"trusted": boolean;
|
|
1814
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
2054
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1815
2055
|
"image": string;
|
|
1816
2056
|
"digest": string | null;
|
|
1817
2057
|
"sizeBytes": number | null;
|
|
@@ -1825,7 +2065,7 @@ export type ExperimentImagesBuildsGetResponse = {
|
|
|
1825
2065
|
};
|
|
1826
2066
|
export type ExperimentImagesBuildsCancelResponse = {
|
|
1827
2067
|
"id": string;
|
|
1828
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
2068
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1829
2069
|
};
|
|
1830
2070
|
export type ExperimentImagesBuildsStreamResponse = unknown;
|
|
1831
2071
|
export type ExperimentImagesRunsCreateRequest = {
|
|
@@ -1882,6 +2122,7 @@ export type ExperimentRunsCreateRequest = {
|
|
|
1882
2122
|
"tools": Array<string>;
|
|
1883
2123
|
};
|
|
1884
2124
|
"requires_tools"?: boolean;
|
|
2125
|
+
"prompt_context"?: "none" | "target_site";
|
|
1885
2126
|
"mcp_servers"?: Array<{
|
|
1886
2127
|
"name": string;
|
|
1887
2128
|
"url": string;
|
|
@@ -2158,7 +2399,7 @@ export type WebmcpArenaCreateRequest = {
|
|
|
2158
2399
|
"name"?: string;
|
|
2159
2400
|
};
|
|
2160
2401
|
export type WebmcpArenaCreateResponse = {
|
|
2161
|
-
"
|
|
2402
|
+
"benchmark": {
|
|
2162
2403
|
"id": string;
|
|
2163
2404
|
"executiveSummary"?: string | null;
|
|
2164
2405
|
"plannedRunCount": number;
|
|
@@ -2224,6 +2465,7 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2224
2465
|
"error": string | null;
|
|
2225
2466
|
"preparation"?: {
|
|
2226
2467
|
"since": string;
|
|
2468
|
+
"progressAt"?: string;
|
|
2227
2469
|
"launchedBy": {
|
|
2228
2470
|
"userId"?: string;
|
|
2229
2471
|
"role"?: string;
|
|
@@ -2232,6 +2474,12 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2232
2474
|
"imageKey": string;
|
|
2233
2475
|
"buildId": string;
|
|
2234
2476
|
"stage"?: "image" | "snapshot";
|
|
2477
|
+
"retry"?: {
|
|
2478
|
+
"failedAttempts": number;
|
|
2479
|
+
"maxAttempts": number;
|
|
2480
|
+
"notBefore": string;
|
|
2481
|
+
"lastFailure": string;
|
|
2482
|
+
};
|
|
2235
2483
|
}>;
|
|
2236
2484
|
} | null;
|
|
2237
2485
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -2248,10 +2496,6 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2248
2496
|
"liveUntil": string | null;
|
|
2249
2497
|
};
|
|
2250
2498
|
"streamUrl": string;
|
|
2251
|
-
"warnings"?: Array<{
|
|
2252
|
-
"harness": string;
|
|
2253
|
-
"kind": string;
|
|
2254
|
-
}>;
|
|
2255
2499
|
};
|
|
2256
2500
|
export type WebmcpRunsBrowserRecordingResponse = {
|
|
2257
2501
|
"runId": string;
|
|
@@ -2500,6 +2744,18 @@ export type IntentsBulkCreateRequest = {
|
|
|
2500
2744
|
"goal"?: {
|
|
2501
2745
|
"statement": string;
|
|
2502
2746
|
};
|
|
2747
|
+
"agents"?: Array<{
|
|
2748
|
+
"harness": string;
|
|
2749
|
+
"model": string;
|
|
2750
|
+
"mcp_server_ids"?: Array<string>;
|
|
2751
|
+
"skill_ids"?: Array<string>;
|
|
2752
|
+
"browser"?: {
|
|
2753
|
+
"webdriver"?: boolean;
|
|
2754
|
+
"webmcp"?: boolean;
|
|
2755
|
+
"webmcpTargetUrl"?: string;
|
|
2756
|
+
"webmcpSurface"?: "native" | "shim";
|
|
2757
|
+
};
|
|
2758
|
+
}>;
|
|
2503
2759
|
"prompts": Array<{
|
|
2504
2760
|
"text": string;
|
|
2505
2761
|
"category"?: string;
|
|
@@ -2509,6 +2765,7 @@ export type IntentsBulkCreateRequest = {
|
|
|
2509
2765
|
}>;
|
|
2510
2766
|
"domainId"?: string;
|
|
2511
2767
|
"icpId"?: string;
|
|
2768
|
+
"firstMeasurement"?: "written" | "intent";
|
|
2512
2769
|
};
|
|
2513
2770
|
export type IntentsBulkCreateResponse = {
|
|
2514
2771
|
"success": true;
|
|
@@ -2518,6 +2775,13 @@ export type IntentsBulkCreateResponse = {
|
|
|
2518
2775
|
"title": string;
|
|
2519
2776
|
"merged": boolean;
|
|
2520
2777
|
"promptsCreated": number;
|
|
2778
|
+
"setupsApplied"?: boolean;
|
|
2779
|
+
"measurementStart": {
|
|
2780
|
+
"started": true;
|
|
2781
|
+
} | {
|
|
2782
|
+
"started": false;
|
|
2783
|
+
"reason": "no_attributable_member";
|
|
2784
|
+
};
|
|
2521
2785
|
}>;
|
|
2522
2786
|
"truncated": number;
|
|
2523
2787
|
};
|
|
@@ -2713,6 +2977,12 @@ export type IntentsGenerateResponse = {
|
|
|
2713
2977
|
"tracked": boolean;
|
|
2714
2978
|
}>;
|
|
2715
2979
|
}>;
|
|
2980
|
+
"measurementStart": {
|
|
2981
|
+
"started": true;
|
|
2982
|
+
} | {
|
|
2983
|
+
"started": false;
|
|
2984
|
+
"reason": "no_attributable_member";
|
|
2985
|
+
};
|
|
2716
2986
|
}>;
|
|
2717
2987
|
};
|
|
2718
2988
|
export type IntentsSetIcpRequest = {
|
|
@@ -2824,6 +3094,12 @@ export type IntentsAddPromptResponse = {
|
|
|
2824
3094
|
"tracked": boolean;
|
|
2825
3095
|
}>;
|
|
2826
3096
|
}>;
|
|
3097
|
+
"measurementStart": {
|
|
3098
|
+
"started": true;
|
|
3099
|
+
} | {
|
|
3100
|
+
"started": false;
|
|
3101
|
+
"reason": "no_attributable_member";
|
|
3102
|
+
};
|
|
2827
3103
|
};
|
|
2828
3104
|
};
|
|
2829
3105
|
export type IntentsUpdatePromptRequest = {
|
|
@@ -2842,6 +3118,17 @@ export type IntentsDeletePromptResponse = {
|
|
|
2842
3118
|
"id": string;
|
|
2843
3119
|
};
|
|
2844
3120
|
};
|
|
3121
|
+
export type IntentsSetPromptGoalRequest = {
|
|
3122
|
+
"active": boolean;
|
|
3123
|
+
};
|
|
3124
|
+
export type IntentsSetPromptGoalResponse = {
|
|
3125
|
+
"success": true;
|
|
3126
|
+
"data": {
|
|
3127
|
+
"promptId": string;
|
|
3128
|
+
"kind": "discover" | "choose" | "verify" | "inform" | "guide" | "act" | "other";
|
|
3129
|
+
"active": boolean;
|
|
3130
|
+
};
|
|
3131
|
+
};
|
|
2845
3132
|
export type IntentsRepresentationResponse = {
|
|
2846
3133
|
"success": true;
|
|
2847
3134
|
"data": {
|
|
@@ -3063,6 +3350,7 @@ export type StudioRunsCreateRequest = {
|
|
|
3063
3350
|
"tools": Array<string>;
|
|
3064
3351
|
};
|
|
3065
3352
|
"requires_tools"?: boolean;
|
|
3353
|
+
"prompt_context"?: "none" | "target_site";
|
|
3066
3354
|
"mcp_servers"?: Array<{
|
|
3067
3355
|
"name": string;
|
|
3068
3356
|
"url": string;
|
|
@@ -3145,6 +3433,7 @@ export type StudyAddonsListResponse = {
|
|
|
3145
3433
|
"surfaces": Array<"run" | "session">;
|
|
3146
3434
|
"harnesses": Array<string>;
|
|
3147
3435
|
"multiInstance"?: boolean;
|
|
3436
|
+
"credentialSourceFields"?: Array<string>;
|
|
3148
3437
|
}>;
|
|
3149
3438
|
};
|
|
3150
3439
|
export type StudyBenchmarkDefinitionsListResponse = {
|
|
@@ -3163,6 +3452,13 @@ export type StudyBenchmarkDefinitionsListResponse = {
|
|
|
3163
3452
|
"modelParams"?: Array<string>;
|
|
3164
3453
|
"maxPlannedRuns"?: number;
|
|
3165
3454
|
"sampleFloor"?: number;
|
|
3455
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3456
|
+
"aggregation"?: "included" | "excluded";
|
|
3457
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3458
|
+
"aggregateTaskSample"?: {
|
|
3459
|
+
"count": number;
|
|
3460
|
+
"seed": string;
|
|
3461
|
+
};
|
|
3166
3462
|
};
|
|
3167
3463
|
"defaultConfig": {
|
|
3168
3464
|
"matrix"?: Array<{
|
|
@@ -3229,6 +3525,7 @@ export type StudyBenchmarkDefinitionsListResponse = {
|
|
|
3229
3525
|
"userId": string;
|
|
3230
3526
|
"displayName": string | null;
|
|
3231
3527
|
} | null;
|
|
3528
|
+
"uploadedByPublisher": boolean | null;
|
|
3232
3529
|
"notices": Array<{
|
|
3233
3530
|
"kind": string;
|
|
3234
3531
|
"text": string;
|
|
@@ -3266,6 +3563,13 @@ export type StudyBenchmarkDefinitionsCreateRequest = {
|
|
|
3266
3563
|
"modelParams"?: Array<string>;
|
|
3267
3564
|
"maxPlannedRuns"?: number;
|
|
3268
3565
|
"sampleFloor"?: number;
|
|
3566
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3567
|
+
"aggregation"?: "included" | "excluded";
|
|
3568
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3569
|
+
"aggregateTaskSample"?: {
|
|
3570
|
+
"count": number;
|
|
3571
|
+
"seed": string;
|
|
3572
|
+
};
|
|
3269
3573
|
};
|
|
3270
3574
|
"runnerKind"?: "none" | "image" | "environment" | "framework";
|
|
3271
3575
|
"runnerRecipe"?: Record<string, unknown> | null;
|
|
@@ -3308,6 +3612,13 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
|
|
|
3308
3612
|
"modelParams"?: Array<string>;
|
|
3309
3613
|
"maxPlannedRuns"?: number;
|
|
3310
3614
|
"sampleFloor"?: number;
|
|
3615
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3616
|
+
"aggregation"?: "included" | "excluded";
|
|
3617
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3618
|
+
"aggregateTaskSample"?: {
|
|
3619
|
+
"count": number;
|
|
3620
|
+
"seed": string;
|
|
3621
|
+
};
|
|
3311
3622
|
};
|
|
3312
3623
|
"defaultConfig": {
|
|
3313
3624
|
"matrix"?: Array<{
|
|
@@ -3374,6 +3685,7 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
|
|
|
3374
3685
|
"userId": string;
|
|
3375
3686
|
"displayName": string | null;
|
|
3376
3687
|
} | null;
|
|
3688
|
+
"uploadedByPublisher": boolean | null;
|
|
3377
3689
|
"notices": Array<{
|
|
3378
3690
|
"kind": string;
|
|
3379
3691
|
"text": string;
|
|
@@ -3396,6 +3708,13 @@ export type StudyBenchmarkDefinitionsGetResponse = {
|
|
|
3396
3708
|
"modelParams"?: Array<string>;
|
|
3397
3709
|
"maxPlannedRuns"?: number;
|
|
3398
3710
|
"sampleFloor"?: number;
|
|
3711
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3712
|
+
"aggregation"?: "included" | "excluded";
|
|
3713
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3714
|
+
"aggregateTaskSample"?: {
|
|
3715
|
+
"count": number;
|
|
3716
|
+
"seed": string;
|
|
3717
|
+
};
|
|
3399
3718
|
};
|
|
3400
3719
|
"defaultConfig": {
|
|
3401
3720
|
"matrix"?: Array<{
|
|
@@ -3462,6 +3781,7 @@ export type StudyBenchmarkDefinitionsGetResponse = {
|
|
|
3462
3781
|
"userId": string;
|
|
3463
3782
|
"displayName": string | null;
|
|
3464
3783
|
} | null;
|
|
3784
|
+
"uploadedByPublisher": boolean | null;
|
|
3465
3785
|
"notices": Array<{
|
|
3466
3786
|
"kind": string;
|
|
3467
3787
|
"text": string;
|
|
@@ -3493,6 +3813,13 @@ export type StudyBenchmarkDefinitionsUpdateRequest = {
|
|
|
3493
3813
|
"modelParams"?: Array<string>;
|
|
3494
3814
|
"maxPlannedRuns"?: number;
|
|
3495
3815
|
"sampleFloor"?: number;
|
|
3816
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3817
|
+
"aggregation"?: "included" | "excluded";
|
|
3818
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3819
|
+
"aggregateTaskSample"?: {
|
|
3820
|
+
"count": number;
|
|
3821
|
+
"seed": string;
|
|
3822
|
+
};
|
|
3496
3823
|
};
|
|
3497
3824
|
"runnerKind"?: "none" | "image" | "environment" | "framework";
|
|
3498
3825
|
"runnerRecipe"?: Record<string, unknown> | null;
|
|
@@ -3535,6 +3862,13 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
|
|
|
3535
3862
|
"modelParams"?: Array<string>;
|
|
3536
3863
|
"maxPlannedRuns"?: number;
|
|
3537
3864
|
"sampleFloor"?: number;
|
|
3865
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3866
|
+
"aggregation"?: "included" | "excluded";
|
|
3867
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3868
|
+
"aggregateTaskSample"?: {
|
|
3869
|
+
"count": number;
|
|
3870
|
+
"seed": string;
|
|
3871
|
+
};
|
|
3538
3872
|
};
|
|
3539
3873
|
"defaultConfig": {
|
|
3540
3874
|
"matrix"?: Array<{
|
|
@@ -3601,6 +3935,7 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
|
|
|
3601
3935
|
"userId": string;
|
|
3602
3936
|
"displayName": string | null;
|
|
3603
3937
|
} | null;
|
|
3938
|
+
"uploadedByPublisher": boolean | null;
|
|
3604
3939
|
"notices": Array<{
|
|
3605
3940
|
"kind": string;
|
|
3606
3941
|
"text": string;
|
|
@@ -3645,6 +3980,13 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
|
|
|
3645
3980
|
"modelParams"?: Array<string>;
|
|
3646
3981
|
"maxPlannedRuns"?: number;
|
|
3647
3982
|
"sampleFloor"?: number;
|
|
3983
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3984
|
+
"aggregation"?: "included" | "excluded";
|
|
3985
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3986
|
+
"aggregateTaskSample"?: {
|
|
3987
|
+
"count": number;
|
|
3988
|
+
"seed": string;
|
|
3989
|
+
};
|
|
3648
3990
|
};
|
|
3649
3991
|
"defaultConfig": {
|
|
3650
3992
|
"matrix"?: Array<{
|
|
@@ -3711,6 +4053,7 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
|
|
|
3711
4053
|
"userId": string;
|
|
3712
4054
|
"displayName": string | null;
|
|
3713
4055
|
} | null;
|
|
4056
|
+
"uploadedByPublisher": boolean | null;
|
|
3714
4057
|
"notices": Array<{
|
|
3715
4058
|
"kind": string;
|
|
3716
4059
|
"text": string;
|
|
@@ -3736,6 +4079,13 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
|
|
|
3736
4079
|
"modelParams"?: Array<string>;
|
|
3737
4080
|
"maxPlannedRuns"?: number;
|
|
3738
4081
|
"sampleFloor"?: number;
|
|
4082
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
4083
|
+
"aggregation"?: "included" | "excluded";
|
|
4084
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
4085
|
+
"aggregateTaskSample"?: {
|
|
4086
|
+
"count": number;
|
|
4087
|
+
"seed": string;
|
|
4088
|
+
};
|
|
3739
4089
|
};
|
|
3740
4090
|
"defaultConfig": {
|
|
3741
4091
|
"matrix"?: Array<{
|
|
@@ -3802,6 +4152,7 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
|
|
|
3802
4152
|
"userId": string;
|
|
3803
4153
|
"displayName": string | null;
|
|
3804
4154
|
} | null;
|
|
4155
|
+
"uploadedByPublisher": boolean | null;
|
|
3805
4156
|
"notices": Array<{
|
|
3806
4157
|
"kind": string;
|
|
3807
4158
|
"text": string;
|
|
@@ -3824,6 +4175,13 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
|
|
|
3824
4175
|
"modelParams"?: Array<string>;
|
|
3825
4176
|
"maxPlannedRuns"?: number;
|
|
3826
4177
|
"sampleFloor"?: number;
|
|
4178
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
4179
|
+
"aggregation"?: "included" | "excluded";
|
|
4180
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
4181
|
+
"aggregateTaskSample"?: {
|
|
4182
|
+
"count": number;
|
|
4183
|
+
"seed": string;
|
|
4184
|
+
};
|
|
3827
4185
|
};
|
|
3828
4186
|
"defaultConfig": {
|
|
3829
4187
|
"matrix"?: Array<{
|
|
@@ -3890,6 +4248,7 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
|
|
|
3890
4248
|
"userId": string;
|
|
3891
4249
|
"displayName": string | null;
|
|
3892
4250
|
} | null;
|
|
4251
|
+
"uploadedByPublisher": boolean | null;
|
|
3893
4252
|
"notices": Array<{
|
|
3894
4253
|
"kind": string;
|
|
3895
4254
|
"text": string;
|
|
@@ -4068,6 +4427,9 @@ export type StudyImagesBuildsListResponse = {
|
|
|
4068
4427
|
} | {
|
|
4069
4428
|
"kind": "inline";
|
|
4070
4429
|
"fileCount": number;
|
|
4430
|
+
} | {
|
|
4431
|
+
"kind": "asset";
|
|
4432
|
+
"sha256": string;
|
|
4071
4433
|
};
|
|
4072
4434
|
"sourceIdentity": string;
|
|
4073
4435
|
"dockerfileDir": string;
|
|
@@ -4075,7 +4437,7 @@ export type StudyImagesBuildsListResponse = {
|
|
|
4075
4437
|
"buildArgKeys": Array<string>;
|
|
4076
4438
|
"platform": string;
|
|
4077
4439
|
"trusted": boolean;
|
|
4078
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4440
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4079
4441
|
"image": string;
|
|
4080
4442
|
"digest": string | null;
|
|
4081
4443
|
"sizeBytes": number | null;
|
|
@@ -4116,7 +4478,7 @@ export type StudyImagesBuildsCreateRequest = {
|
|
|
4116
4478
|
};
|
|
4117
4479
|
export type StudyImagesBuildsCreateResponse = {
|
|
4118
4480
|
"id": string;
|
|
4119
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4481
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4120
4482
|
};
|
|
4121
4483
|
export type StudyImagesBuildsGetResponse = {
|
|
4122
4484
|
"build": {
|
|
@@ -4138,6 +4500,9 @@ export type StudyImagesBuildsGetResponse = {
|
|
|
4138
4500
|
} | {
|
|
4139
4501
|
"kind": "inline";
|
|
4140
4502
|
"fileCount": number;
|
|
4503
|
+
} | {
|
|
4504
|
+
"kind": "asset";
|
|
4505
|
+
"sha256": string;
|
|
4141
4506
|
};
|
|
4142
4507
|
"sourceIdentity": string;
|
|
4143
4508
|
"dockerfileDir": string;
|
|
@@ -4145,7 +4510,7 @@ export type StudyImagesBuildsGetResponse = {
|
|
|
4145
4510
|
"buildArgKeys": Array<string>;
|
|
4146
4511
|
"platform": string;
|
|
4147
4512
|
"trusted": boolean;
|
|
4148
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4513
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4149
4514
|
"image": string;
|
|
4150
4515
|
"digest": string | null;
|
|
4151
4516
|
"sizeBytes": number | null;
|
|
@@ -4159,7 +4524,7 @@ export type StudyImagesBuildsGetResponse = {
|
|
|
4159
4524
|
};
|
|
4160
4525
|
export type StudyImagesBuildsCancelResponse = {
|
|
4161
4526
|
"id": string;
|
|
4162
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4527
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4163
4528
|
};
|
|
4164
4529
|
export type StudyImagesBuildsStreamResponse = unknown;
|
|
4165
4530
|
export type StudyImagesRunsCreateRequest = {
|
|
@@ -4216,6 +4581,7 @@ export type StudyRunsCreateRequest = {
|
|
|
4216
4581
|
"tools": Array<string>;
|
|
4217
4582
|
};
|
|
4218
4583
|
"requires_tools"?: boolean;
|
|
4584
|
+
"prompt_context"?: "none" | "target_site";
|
|
4219
4585
|
"mcp_servers"?: Array<{
|
|
4220
4586
|
"name": string;
|
|
4221
4587
|
"url": string;
|
|
@@ -4394,7 +4760,7 @@ export type StudyListResponse = {
|
|
|
4394
4760
|
"addons"?: Record<string, unknown>;
|
|
4395
4761
|
};
|
|
4396
4762
|
};
|
|
4397
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
4763
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4398
4764
|
"subjectBackend": "model" | "harness" | null;
|
|
4399
4765
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4400
4766
|
"importedFrom"?: string;
|
|
@@ -4430,6 +4796,7 @@ export type StudyListResponse = {
|
|
|
4430
4796
|
"error": string | null;
|
|
4431
4797
|
"preparation"?: {
|
|
4432
4798
|
"since": string;
|
|
4799
|
+
"progressAt"?: string;
|
|
4433
4800
|
"launchedBy": {
|
|
4434
4801
|
"userId"?: string;
|
|
4435
4802
|
"role"?: string;
|
|
@@ -4438,6 +4805,12 @@ export type StudyListResponse = {
|
|
|
4438
4805
|
"imageKey": string;
|
|
4439
4806
|
"buildId": string;
|
|
4440
4807
|
"stage"?: "image" | "snapshot";
|
|
4808
|
+
"retry"?: {
|
|
4809
|
+
"failedAttempts": number;
|
|
4810
|
+
"maxAttempts": number;
|
|
4811
|
+
"notBefore": string;
|
|
4812
|
+
"lastFailure": string;
|
|
4813
|
+
};
|
|
4441
4814
|
}>;
|
|
4442
4815
|
} | null;
|
|
4443
4816
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -4452,6 +4825,57 @@ export type StudyListResponse = {
|
|
|
4452
4825
|
"createdAt": string;
|
|
4453
4826
|
"updatedAt": string;
|
|
4454
4827
|
"liveUntil": string | null;
|
|
4828
|
+
"launch"?: {
|
|
4829
|
+
"attempt": number;
|
|
4830
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
4831
|
+
"startedAt": string;
|
|
4832
|
+
"progressAt": string;
|
|
4833
|
+
"planned"?: number;
|
|
4834
|
+
"created"?: number;
|
|
4835
|
+
"missing"?: {
|
|
4836
|
+
"retryable": number;
|
|
4837
|
+
"refused": number;
|
|
4838
|
+
};
|
|
4839
|
+
"notes"?: Array<string>;
|
|
4840
|
+
"settledAt"?: string;
|
|
4841
|
+
} | null;
|
|
4842
|
+
"relaunchable"?: boolean;
|
|
4843
|
+
"arms"?: Array<{
|
|
4844
|
+
"key": string;
|
|
4845
|
+
"position": number;
|
|
4846
|
+
"label": string;
|
|
4847
|
+
"role": "reference" | "peer";
|
|
4848
|
+
"payload": {
|
|
4849
|
+
"tools"?: "default" | "none" | {
|
|
4850
|
+
"tools": Array<string>;
|
|
4851
|
+
};
|
|
4852
|
+
"addons"?: Record<string, unknown>;
|
|
4853
|
+
};
|
|
4854
|
+
"cells"?: Array<{
|
|
4855
|
+
"harness": string;
|
|
4856
|
+
"model": string;
|
|
4857
|
+
"provider"?: string;
|
|
4858
|
+
"params"?: {
|
|
4859
|
+
"temperature"?: number;
|
|
4860
|
+
"maxOutputTokens"?: number;
|
|
4861
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4862
|
+
"thinkingBudgetTokens"?: number;
|
|
4863
|
+
};
|
|
4864
|
+
}>;
|
|
4865
|
+
"skills"?: Array<{
|
|
4866
|
+
"ref": string;
|
|
4867
|
+
"name": string;
|
|
4868
|
+
"changedSinceCreate"?: boolean;
|
|
4869
|
+
}>;
|
|
4870
|
+
"addedAgents"?: Array<{
|
|
4871
|
+
"position": number;
|
|
4872
|
+
"addedAt": string;
|
|
4873
|
+
}>;
|
|
4874
|
+
}>;
|
|
4875
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
4876
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
4877
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
4878
|
+
"modeOptions"?: Record<string, unknown>;
|
|
4455
4879
|
}>;
|
|
4456
4880
|
};
|
|
4457
4881
|
export type StudyCreateRequest = {
|
|
@@ -4478,10 +4902,34 @@ export type StudyCreateRequest = {
|
|
|
4478
4902
|
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4479
4903
|
"judgeSupplementary"?: boolean;
|
|
4480
4904
|
"proposeExpectations"?: boolean;
|
|
4481
|
-
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
4905
|
+
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4482
4906
|
"subjectBackend"?: "model" | "harness";
|
|
4483
4907
|
"baselineLabel"?: string;
|
|
4484
4908
|
"treatmentLabel"?: string;
|
|
4909
|
+
"arms"?: Array<{
|
|
4910
|
+
"key"?: string;
|
|
4911
|
+
"label": string;
|
|
4912
|
+
"role"?: "reference" | "peer";
|
|
4913
|
+
"config": {
|
|
4914
|
+
"tools"?: "default" | "none" | {
|
|
4915
|
+
"tools": Array<string>;
|
|
4916
|
+
};
|
|
4917
|
+
"addons"?: Record<string, unknown>;
|
|
4918
|
+
};
|
|
4919
|
+
"cells"?: Array<{
|
|
4920
|
+
"harness": string;
|
|
4921
|
+
"model": string;
|
|
4922
|
+
"provider"?: string;
|
|
4923
|
+
"params"?: {
|
|
4924
|
+
"temperature"?: number;
|
|
4925
|
+
"maxOutputTokens"?: number;
|
|
4926
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4927
|
+
"thinkingBudgetTokens"?: number;
|
|
4928
|
+
};
|
|
4929
|
+
}>;
|
|
4930
|
+
}>;
|
|
4931
|
+
"contrasts"?: "vs_reference" | "all_pairs";
|
|
4932
|
+
"modeOptions"?: Record<string, unknown>;
|
|
4485
4933
|
"taskCount"?: number;
|
|
4486
4934
|
"variantsPerTask"?: number;
|
|
4487
4935
|
"repeats"?: number;
|
|
@@ -4492,7 +4940,76 @@ export type StudyCreateRequest = {
|
|
|
4492
4940
|
"requiresWeb"?: boolean;
|
|
4493
4941
|
"rubric"?: Array<string>;
|
|
4494
4942
|
"variants"?: Array<string>;
|
|
4495
|
-
"expectations"?: Array<
|
|
4943
|
+
"expectations"?: Array<{
|
|
4944
|
+
"label"?: string;
|
|
4945
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4946
|
+
"kind": "mcp_tool_called";
|
|
4947
|
+
"server"?: string;
|
|
4948
|
+
"tool": string;
|
|
4949
|
+
"arguments"?: Record<string, unknown>;
|
|
4950
|
+
"minCalls"?: number;
|
|
4951
|
+
} | {
|
|
4952
|
+
"label"?: string;
|
|
4953
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4954
|
+
"kind": "tool_not_called";
|
|
4955
|
+
"tool"?: string;
|
|
4956
|
+
"mcp"?: {
|
|
4957
|
+
"server"?: string;
|
|
4958
|
+
"tool": string;
|
|
4959
|
+
};
|
|
4960
|
+
"pattern"?: string;
|
|
4961
|
+
} | {
|
|
4962
|
+
"label"?: string;
|
|
4963
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4964
|
+
"kind": "command_ran";
|
|
4965
|
+
"pattern": string;
|
|
4966
|
+
} | {
|
|
4967
|
+
"label"?: string;
|
|
4968
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4969
|
+
"kind": "skill_loaded";
|
|
4970
|
+
"slug": string;
|
|
4971
|
+
} | {
|
|
4972
|
+
"label"?: string;
|
|
4973
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4974
|
+
"kind": "file_operation";
|
|
4975
|
+
"op"?: "read" | "write" | "edit";
|
|
4976
|
+
"path": unknown;
|
|
4977
|
+
} | {
|
|
4978
|
+
"label"?: string;
|
|
4979
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4980
|
+
"kind": "web_fetched";
|
|
4981
|
+
"urlHost"?: string;
|
|
4982
|
+
"urlPattern"?: string;
|
|
4983
|
+
} | {
|
|
4984
|
+
"label"?: string;
|
|
4985
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4986
|
+
"kind": "response_contains";
|
|
4987
|
+
"text": string;
|
|
4988
|
+
"normalize"?: "prose";
|
|
4989
|
+
} | {
|
|
4990
|
+
"label"?: string;
|
|
4991
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4992
|
+
"kind": "response_matches";
|
|
4993
|
+
"pattern": string;
|
|
4994
|
+
"ignoreCase"?: boolean;
|
|
4995
|
+
} | {
|
|
4996
|
+
"label"?: string;
|
|
4997
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4998
|
+
"kind": "token_budget";
|
|
4999
|
+
"maxOutputTokens"?: number;
|
|
5000
|
+
"maxTotalTokens"?: number;
|
|
5001
|
+
} | {
|
|
5002
|
+
"label"?: string;
|
|
5003
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5004
|
+
"kind": "sequence";
|
|
5005
|
+
"of": Array<unknown>;
|
|
5006
|
+
} | {
|
|
5007
|
+
"label"?: string;
|
|
5008
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5009
|
+
"kind": "any_of";
|
|
5010
|
+
"of": Array<unknown>;
|
|
5011
|
+
}>;
|
|
5012
|
+
"private"?: boolean;
|
|
4496
5013
|
}>;
|
|
4497
5014
|
"subset"?: {
|
|
4498
5015
|
"count": number;
|
|
@@ -4542,7 +5059,7 @@ export type StudyCreateResponse = {
|
|
|
4542
5059
|
"addons"?: Record<string, unknown>;
|
|
4543
5060
|
};
|
|
4544
5061
|
};
|
|
4545
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5062
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4546
5063
|
"subjectBackend": "model" | "harness" | null;
|
|
4547
5064
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4548
5065
|
"importedFrom"?: string;
|
|
@@ -4578,6 +5095,7 @@ export type StudyCreateResponse = {
|
|
|
4578
5095
|
"error": string | null;
|
|
4579
5096
|
"preparation"?: {
|
|
4580
5097
|
"since": string;
|
|
5098
|
+
"progressAt"?: string;
|
|
4581
5099
|
"launchedBy": {
|
|
4582
5100
|
"userId"?: string;
|
|
4583
5101
|
"role"?: string;
|
|
@@ -4586,6 +5104,12 @@ export type StudyCreateResponse = {
|
|
|
4586
5104
|
"imageKey": string;
|
|
4587
5105
|
"buildId": string;
|
|
4588
5106
|
"stage"?: "image" | "snapshot";
|
|
5107
|
+
"retry"?: {
|
|
5108
|
+
"failedAttempts": number;
|
|
5109
|
+
"maxAttempts": number;
|
|
5110
|
+
"notBefore": string;
|
|
5111
|
+
"lastFailure": string;
|
|
5112
|
+
};
|
|
4589
5113
|
}>;
|
|
4590
5114
|
} | null;
|
|
4591
5115
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -4600,11 +5124,76 @@ export type StudyCreateResponse = {
|
|
|
4600
5124
|
"createdAt": string;
|
|
4601
5125
|
"updatedAt": string;
|
|
4602
5126
|
"liveUntil": string | null;
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
5127
|
+
"launch"?: {
|
|
5128
|
+
"attempt": number;
|
|
5129
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
5130
|
+
"startedAt": string;
|
|
5131
|
+
"progressAt": string;
|
|
5132
|
+
"planned"?: number;
|
|
5133
|
+
"created"?: number;
|
|
5134
|
+
"missing"?: {
|
|
5135
|
+
"retryable": number;
|
|
5136
|
+
"refused": number;
|
|
5137
|
+
};
|
|
5138
|
+
"notes"?: Array<string>;
|
|
5139
|
+
"settledAt"?: string;
|
|
5140
|
+
} | null;
|
|
5141
|
+
"relaunchable"?: boolean;
|
|
5142
|
+
"arms"?: Array<{
|
|
5143
|
+
"key": string;
|
|
5144
|
+
"position": number;
|
|
5145
|
+
"label": string;
|
|
5146
|
+
"role": "reference" | "peer";
|
|
5147
|
+
"payload": {
|
|
5148
|
+
"tools"?: "default" | "none" | {
|
|
5149
|
+
"tools": Array<string>;
|
|
5150
|
+
};
|
|
5151
|
+
"addons"?: Record<string, unknown>;
|
|
5152
|
+
};
|
|
5153
|
+
"cells"?: Array<{
|
|
5154
|
+
"harness": string;
|
|
5155
|
+
"model": string;
|
|
5156
|
+
"provider"?: string;
|
|
5157
|
+
"params"?: {
|
|
5158
|
+
"temperature"?: number;
|
|
5159
|
+
"maxOutputTokens"?: number;
|
|
5160
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5161
|
+
"thinkingBudgetTokens"?: number;
|
|
5162
|
+
};
|
|
5163
|
+
}>;
|
|
5164
|
+
"skills"?: Array<{
|
|
5165
|
+
"ref": string;
|
|
5166
|
+
"name": string;
|
|
5167
|
+
"changedSinceCreate"?: boolean;
|
|
5168
|
+
}>;
|
|
5169
|
+
"addedAgents"?: Array<{
|
|
5170
|
+
"position": number;
|
|
5171
|
+
"addedAt": string;
|
|
5172
|
+
}>;
|
|
5173
|
+
}>;
|
|
5174
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
5175
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
5176
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
5177
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5178
|
+
};
|
|
5179
|
+
"streamUrl": string;
|
|
5180
|
+
"warnings"?: Array<{
|
|
5181
|
+
"harness": string;
|
|
5182
|
+
"kind": string;
|
|
5183
|
+
} | {
|
|
5184
|
+
"type": "design_aggregate_budget";
|
|
5185
|
+
"detail": string;
|
|
5186
|
+
}>;
|
|
5187
|
+
};
|
|
5188
|
+
export type StudyComparisonModesListResponse = {
|
|
5189
|
+
"modes": Array<{
|
|
5190
|
+
"id": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5191
|
+
"enabled": boolean;
|
|
5192
|
+
"authoring": boolean;
|
|
5193
|
+
}>;
|
|
5194
|
+
"cellPlans": Array<{
|
|
5195
|
+
"id": "crossed" | "per_setup";
|
|
5196
|
+
"enabled": boolean;
|
|
4608
5197
|
}>;
|
|
4609
5198
|
};
|
|
4610
5199
|
export type StudyConvertWebmcpSuiteRequest = {
|
|
@@ -4619,6 +5208,226 @@ export type StudyConvertWebmcpSuiteResponse = {
|
|
|
4619
5208
|
}>;
|
|
4620
5209
|
"warnings": Array<string>;
|
|
4621
5210
|
};
|
|
5211
|
+
export type StudyPreflightRequest = {
|
|
5212
|
+
"name": string;
|
|
5213
|
+
"description"?: string;
|
|
5214
|
+
"subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5215
|
+
"skillRef"?: string;
|
|
5216
|
+
"definitionVersionId"?: string;
|
|
5217
|
+
"subjectName"?: string;
|
|
5218
|
+
"armConfig"?: {
|
|
5219
|
+
"baseline"?: {
|
|
5220
|
+
"tools"?: "default" | "none" | {
|
|
5221
|
+
"tools": Array<string>;
|
|
5222
|
+
};
|
|
5223
|
+
"addons"?: Record<string, unknown>;
|
|
5224
|
+
};
|
|
5225
|
+
"treatment"?: {
|
|
5226
|
+
"tools"?: "default" | "none" | {
|
|
5227
|
+
"tools": Array<string>;
|
|
5228
|
+
};
|
|
5229
|
+
"addons"?: Record<string, unknown>;
|
|
5230
|
+
};
|
|
5231
|
+
};
|
|
5232
|
+
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5233
|
+
"judgeSupplementary"?: boolean;
|
|
5234
|
+
"proposeExpectations"?: boolean;
|
|
5235
|
+
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5236
|
+
"subjectBackend"?: "model" | "harness";
|
|
5237
|
+
"baselineLabel"?: string;
|
|
5238
|
+
"treatmentLabel"?: string;
|
|
5239
|
+
"arms"?: Array<{
|
|
5240
|
+
"key"?: string;
|
|
5241
|
+
"label": string;
|
|
5242
|
+
"role"?: "reference" | "peer";
|
|
5243
|
+
"config": {
|
|
5244
|
+
"tools"?: "default" | "none" | {
|
|
5245
|
+
"tools": Array<string>;
|
|
5246
|
+
};
|
|
5247
|
+
"addons"?: Record<string, unknown>;
|
|
5248
|
+
};
|
|
5249
|
+
"cells"?: Array<{
|
|
5250
|
+
"harness": string;
|
|
5251
|
+
"model": string;
|
|
5252
|
+
"provider"?: string;
|
|
5253
|
+
"params"?: {
|
|
5254
|
+
"temperature"?: number;
|
|
5255
|
+
"maxOutputTokens"?: number;
|
|
5256
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5257
|
+
"thinkingBudgetTokens"?: number;
|
|
5258
|
+
};
|
|
5259
|
+
}>;
|
|
5260
|
+
}>;
|
|
5261
|
+
"contrasts"?: "vs_reference" | "all_pairs";
|
|
5262
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5263
|
+
"taskCount"?: number;
|
|
5264
|
+
"variantsPerTask"?: number;
|
|
5265
|
+
"repeats"?: number;
|
|
5266
|
+
"tasks"?: Array<{
|
|
5267
|
+
"prompt": string;
|
|
5268
|
+
"rationale"?: string;
|
|
5269
|
+
"category"?: string;
|
|
5270
|
+
"requiresWeb"?: boolean;
|
|
5271
|
+
"rubric"?: Array<string>;
|
|
5272
|
+
"variants"?: Array<string>;
|
|
5273
|
+
"expectations"?: Array<{
|
|
5274
|
+
"label"?: string;
|
|
5275
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5276
|
+
"kind": "mcp_tool_called";
|
|
5277
|
+
"server"?: string;
|
|
5278
|
+
"tool": string;
|
|
5279
|
+
"arguments"?: Record<string, unknown>;
|
|
5280
|
+
"minCalls"?: number;
|
|
5281
|
+
} | {
|
|
5282
|
+
"label"?: string;
|
|
5283
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5284
|
+
"kind": "tool_not_called";
|
|
5285
|
+
"tool"?: string;
|
|
5286
|
+
"mcp"?: {
|
|
5287
|
+
"server"?: string;
|
|
5288
|
+
"tool": string;
|
|
5289
|
+
};
|
|
5290
|
+
"pattern"?: string;
|
|
5291
|
+
} | {
|
|
5292
|
+
"label"?: string;
|
|
5293
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5294
|
+
"kind": "command_ran";
|
|
5295
|
+
"pattern": string;
|
|
5296
|
+
} | {
|
|
5297
|
+
"label"?: string;
|
|
5298
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5299
|
+
"kind": "skill_loaded";
|
|
5300
|
+
"slug": string;
|
|
5301
|
+
} | {
|
|
5302
|
+
"label"?: string;
|
|
5303
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5304
|
+
"kind": "file_operation";
|
|
5305
|
+
"op"?: "read" | "write" | "edit";
|
|
5306
|
+
"path": unknown;
|
|
5307
|
+
} | {
|
|
5308
|
+
"label"?: string;
|
|
5309
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5310
|
+
"kind": "web_fetched";
|
|
5311
|
+
"urlHost"?: string;
|
|
5312
|
+
"urlPattern"?: string;
|
|
5313
|
+
} | {
|
|
5314
|
+
"label"?: string;
|
|
5315
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5316
|
+
"kind": "response_contains";
|
|
5317
|
+
"text": string;
|
|
5318
|
+
"normalize"?: "prose";
|
|
5319
|
+
} | {
|
|
5320
|
+
"label"?: string;
|
|
5321
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5322
|
+
"kind": "response_matches";
|
|
5323
|
+
"pattern": string;
|
|
5324
|
+
"ignoreCase"?: boolean;
|
|
5325
|
+
} | {
|
|
5326
|
+
"label"?: string;
|
|
5327
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5328
|
+
"kind": "token_budget";
|
|
5329
|
+
"maxOutputTokens"?: number;
|
|
5330
|
+
"maxTotalTokens"?: number;
|
|
5331
|
+
} | {
|
|
5332
|
+
"label"?: string;
|
|
5333
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5334
|
+
"kind": "sequence";
|
|
5335
|
+
"of": Array<unknown>;
|
|
5336
|
+
} | {
|
|
5337
|
+
"label"?: string;
|
|
5338
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5339
|
+
"kind": "any_of";
|
|
5340
|
+
"of": Array<unknown>;
|
|
5341
|
+
}>;
|
|
5342
|
+
"private"?: boolean;
|
|
5343
|
+
}>;
|
|
5344
|
+
"subset"?: {
|
|
5345
|
+
"count": number;
|
|
5346
|
+
"seed"?: string;
|
|
5347
|
+
};
|
|
5348
|
+
"matrix": Array<{
|
|
5349
|
+
"harness": string;
|
|
5350
|
+
"model": string;
|
|
5351
|
+
"provider"?: string;
|
|
5352
|
+
"params"?: {
|
|
5353
|
+
"temperature"?: number;
|
|
5354
|
+
"maxOutputTokens"?: number;
|
|
5355
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5356
|
+
"thinkingBudgetTokens"?: number;
|
|
5357
|
+
};
|
|
5358
|
+
}>;
|
|
5359
|
+
"promise"?: string;
|
|
5360
|
+
};
|
|
5361
|
+
export type StudyPreflightResponse = {
|
|
5362
|
+
"ok": true;
|
|
5363
|
+
"depth": "default" | "full";
|
|
5364
|
+
"form": string;
|
|
5365
|
+
"label": string;
|
|
5366
|
+
"plannedRunCount": number;
|
|
5367
|
+
"plannedJudgeComparisons": number;
|
|
5368
|
+
"runs": {
|
|
5369
|
+
"tasks": number;
|
|
5370
|
+
"variants": number;
|
|
5371
|
+
"repeats": number;
|
|
5372
|
+
"groupAgents": Array<{
|
|
5373
|
+
"group": string;
|
|
5374
|
+
"count": number;
|
|
5375
|
+
}>;
|
|
5376
|
+
};
|
|
5377
|
+
"judgePerComparison": Array<{
|
|
5378
|
+
"reference": string;
|
|
5379
|
+
"subject": string;
|
|
5380
|
+
"count": number;
|
|
5381
|
+
}>;
|
|
5382
|
+
"arms": Array<{
|
|
5383
|
+
"key": string;
|
|
5384
|
+
"label": string | null;
|
|
5385
|
+
"role": string;
|
|
5386
|
+
"payload": Record<string, unknown> | null;
|
|
5387
|
+
"cells"?: Array<{
|
|
5388
|
+
"harness": string;
|
|
5389
|
+
"model": string;
|
|
5390
|
+
"provider"?: string;
|
|
5391
|
+
"params"?: Record<string, unknown>;
|
|
5392
|
+
}>;
|
|
5393
|
+
}>;
|
|
5394
|
+
"comparisons": Array<{
|
|
5395
|
+
"reference": string;
|
|
5396
|
+
"subject": string;
|
|
5397
|
+
"oriented": boolean;
|
|
5398
|
+
"basis": "matched" | "whole_setup";
|
|
5399
|
+
"sharedAgents": Array<{
|
|
5400
|
+
"harness": string;
|
|
5401
|
+
"model": string;
|
|
5402
|
+
"provider"?: string;
|
|
5403
|
+
"params"?: Record<string, unknown>;
|
|
5404
|
+
}>;
|
|
5405
|
+
"excluded": Array<{
|
|
5406
|
+
"group": string;
|
|
5407
|
+
"agents": Array<{
|
|
5408
|
+
"harness": string;
|
|
5409
|
+
"model": string;
|
|
5410
|
+
"provider"?: string;
|
|
5411
|
+
"params"?: Record<string, unknown>;
|
|
5412
|
+
}>;
|
|
5413
|
+
}>;
|
|
5414
|
+
"droppedTasks"?: unknown | null;
|
|
5415
|
+
}>;
|
|
5416
|
+
"boards": Array<{
|
|
5417
|
+
"group": string;
|
|
5418
|
+
"profile": "as-published" | "harness-default" | "with-setup";
|
|
5419
|
+
}>;
|
|
5420
|
+
"issues": Array<{
|
|
5421
|
+
"path": string;
|
|
5422
|
+
"code": string;
|
|
5423
|
+
"message": string;
|
|
5424
|
+
}>;
|
|
5425
|
+
"quota"?: {
|
|
5426
|
+
"dimension": string;
|
|
5427
|
+
"remaining": number | null;
|
|
5428
|
+
"allowed": boolean;
|
|
5429
|
+
} | null;
|
|
5430
|
+
};
|
|
4622
5431
|
export type StudyGetResponse = {
|
|
4623
5432
|
"study": {
|
|
4624
5433
|
"id": string;
|
|
@@ -4650,7 +5459,7 @@ export type StudyGetResponse = {
|
|
|
4650
5459
|
"addons"?: Record<string, unknown>;
|
|
4651
5460
|
};
|
|
4652
5461
|
};
|
|
4653
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5462
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4654
5463
|
"subjectBackend": "model" | "harness" | null;
|
|
4655
5464
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4656
5465
|
"importedFrom"?: string;
|
|
@@ -4686,6 +5495,7 @@ export type StudyGetResponse = {
|
|
|
4686
5495
|
"error": string | null;
|
|
4687
5496
|
"preparation"?: {
|
|
4688
5497
|
"since": string;
|
|
5498
|
+
"progressAt"?: string;
|
|
4689
5499
|
"launchedBy": {
|
|
4690
5500
|
"userId"?: string;
|
|
4691
5501
|
"role"?: string;
|
|
@@ -4694,6 +5504,12 @@ export type StudyGetResponse = {
|
|
|
4694
5504
|
"imageKey": string;
|
|
4695
5505
|
"buildId": string;
|
|
4696
5506
|
"stage"?: "image" | "snapshot";
|
|
5507
|
+
"retry"?: {
|
|
5508
|
+
"failedAttempts": number;
|
|
5509
|
+
"maxAttempts": number;
|
|
5510
|
+
"notBefore": string;
|
|
5511
|
+
"lastFailure": string;
|
|
5512
|
+
};
|
|
4697
5513
|
}>;
|
|
4698
5514
|
} | null;
|
|
4699
5515
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -4708,6 +5524,57 @@ export type StudyGetResponse = {
|
|
|
4708
5524
|
"createdAt": string;
|
|
4709
5525
|
"updatedAt": string;
|
|
4710
5526
|
"liveUntil": string | null;
|
|
5527
|
+
"launch"?: {
|
|
5528
|
+
"attempt": number;
|
|
5529
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
5530
|
+
"startedAt": string;
|
|
5531
|
+
"progressAt": string;
|
|
5532
|
+
"planned"?: number;
|
|
5533
|
+
"created"?: number;
|
|
5534
|
+
"missing"?: {
|
|
5535
|
+
"retryable": number;
|
|
5536
|
+
"refused": number;
|
|
5537
|
+
};
|
|
5538
|
+
"notes"?: Array<string>;
|
|
5539
|
+
"settledAt"?: string;
|
|
5540
|
+
} | null;
|
|
5541
|
+
"relaunchable"?: boolean;
|
|
5542
|
+
"arms"?: Array<{
|
|
5543
|
+
"key": string;
|
|
5544
|
+
"position": number;
|
|
5545
|
+
"label": string;
|
|
5546
|
+
"role": "reference" | "peer";
|
|
5547
|
+
"payload": {
|
|
5548
|
+
"tools"?: "default" | "none" | {
|
|
5549
|
+
"tools": Array<string>;
|
|
5550
|
+
};
|
|
5551
|
+
"addons"?: Record<string, unknown>;
|
|
5552
|
+
};
|
|
5553
|
+
"cells"?: Array<{
|
|
5554
|
+
"harness": string;
|
|
5555
|
+
"model": string;
|
|
5556
|
+
"provider"?: string;
|
|
5557
|
+
"params"?: {
|
|
5558
|
+
"temperature"?: number;
|
|
5559
|
+
"maxOutputTokens"?: number;
|
|
5560
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5561
|
+
"thinkingBudgetTokens"?: number;
|
|
5562
|
+
};
|
|
5563
|
+
}>;
|
|
5564
|
+
"skills"?: Array<{
|
|
5565
|
+
"ref": string;
|
|
5566
|
+
"name": string;
|
|
5567
|
+
"changedSinceCreate"?: boolean;
|
|
5568
|
+
}>;
|
|
5569
|
+
"addedAgents"?: Array<{
|
|
5570
|
+
"position": number;
|
|
5571
|
+
"addedAt": string;
|
|
5572
|
+
}>;
|
|
5573
|
+
}>;
|
|
5574
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
5575
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
5576
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
5577
|
+
"modeOptions"?: Record<string, unknown>;
|
|
4711
5578
|
};
|
|
4712
5579
|
"tasks": Array<{
|
|
4713
5580
|
"id": string;
|
|
@@ -4724,7 +5591,7 @@ export type StudyGetResponse = {
|
|
|
4724
5591
|
"runs": Array<{
|
|
4725
5592
|
"runId": string;
|
|
4726
5593
|
"taskId": string;
|
|
4727
|
-
"arm": "baseline" | "skill";
|
|
5594
|
+
"arm": "baseline" | "skill" | string;
|
|
4728
5595
|
"harness": string;
|
|
4729
5596
|
"model": string;
|
|
4730
5597
|
"provider": string | null;
|
|
@@ -4820,9 +5687,24 @@ export type StudyGetResponse = {
|
|
|
4820
5687
|
"gradedAt": string;
|
|
4821
5688
|
};
|
|
4822
5689
|
} | null;
|
|
5690
|
+
"executionBudgetS"?: number | null;
|
|
5691
|
+
"harnessImageDigest"?: string | null;
|
|
5692
|
+
"subjectEngaged"?: boolean | null;
|
|
4823
5693
|
"runCreatedAt": string | null;
|
|
4824
5694
|
"startedAt": string | null;
|
|
4825
5695
|
"finishedAt": string | null;
|
|
5696
|
+
"contrastVerdicts"?: Array<{
|
|
5697
|
+
"referenceArm": string;
|
|
5698
|
+
"subjectArm": string;
|
|
5699
|
+
"referenceRunId": string;
|
|
5700
|
+
"verdict": "subject" | "reference" | "tie" | "unmeasurable";
|
|
5701
|
+
"votes": Array<{
|
|
5702
|
+
"judge": string;
|
|
5703
|
+
"ordering": "base-first" | "skill-first";
|
|
5704
|
+
"answer": "A" | "B" | "tie";
|
|
5705
|
+
}>;
|
|
5706
|
+
"gradedAt": string;
|
|
5707
|
+
}>;
|
|
4826
5708
|
}>;
|
|
4827
5709
|
"aggregate": {
|
|
4828
5710
|
"summary": {
|
|
@@ -5029,6 +5911,17 @@ export type StudyGetResponse = {
|
|
|
5029
5911
|
"ci": Array<unknown> | null;
|
|
5030
5912
|
"n": number;
|
|
5031
5913
|
};
|
|
5914
|
+
"abstention"?: {
|
|
5915
|
+
"value": number | null;
|
|
5916
|
+
"ci": Array<unknown> | null;
|
|
5917
|
+
"n": number;
|
|
5918
|
+
"parts"?: Array<{
|
|
5919
|
+
"id": string;
|
|
5920
|
+
"label": string;
|
|
5921
|
+
"value": number | null;
|
|
5922
|
+
"n": number;
|
|
5923
|
+
}>;
|
|
5924
|
+
};
|
|
5032
5925
|
"tasks": number;
|
|
5033
5926
|
"graded": number;
|
|
5034
5927
|
"unmeasured": number;
|
|
@@ -5040,6 +5933,17 @@ export type StudyGetResponse = {
|
|
|
5040
5933
|
"ci": Array<unknown> | null;
|
|
5041
5934
|
"n": number;
|
|
5042
5935
|
};
|
|
5936
|
+
"abstention"?: {
|
|
5937
|
+
"value": number | null;
|
|
5938
|
+
"ci": Array<unknown> | null;
|
|
5939
|
+
"n": number;
|
|
5940
|
+
"parts"?: Array<{
|
|
5941
|
+
"id": string;
|
|
5942
|
+
"label": string;
|
|
5943
|
+
"value": number | null;
|
|
5944
|
+
"n": number;
|
|
5945
|
+
}>;
|
|
5946
|
+
};
|
|
5043
5947
|
"graded": number;
|
|
5044
5948
|
"unmeasured": number;
|
|
5045
5949
|
"modelJudged": boolean;
|
|
@@ -5059,60 +5963,1195 @@ export type StudyGetResponse = {
|
|
|
5059
5963
|
"languages": Array<string>;
|
|
5060
5964
|
"unavailableReasons": Array<string>;
|
|
5061
5965
|
};
|
|
5062
|
-
};
|
|
5063
|
-
"warnings": Array<{
|
|
5064
|
-
"type": "eval_capability_gap";
|
|
5065
|
-
"harness": string;
|
|
5066
|
-
"kind": string;
|
|
5067
5966
|
} | {
|
|
5068
|
-
"
|
|
5069
|
-
"
|
|
5070
|
-
|
|
5071
|
-
|
|
5072
|
-
|
|
5073
|
-
|
|
5074
|
-
|
|
5075
|
-
|
|
5076
|
-
|
|
5077
|
-
|
|
5078
|
-
|
|
5079
|
-
|
|
5080
|
-
|
|
5081
|
-
};
|
|
5082
|
-
export type StudyGenerateResponse = {
|
|
5083
|
-
"study": {
|
|
5084
|
-
"id": string;
|
|
5085
|
-
"executiveSummary"?: string | null;
|
|
5086
|
-
"plannedRunCount": number;
|
|
5087
|
-
"organizationId": string;
|
|
5088
|
-
"createdBy": string | null;
|
|
5089
|
-
"name": string;
|
|
5090
|
-
"slug": string;
|
|
5091
|
-
"description": string | null;
|
|
5092
|
-
"skillRef": string | null;
|
|
5093
|
-
"skillSlug": string | null;
|
|
5094
|
-
"skillName": string | null;
|
|
5095
|
-
"skillPromise": string;
|
|
5096
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5097
|
-
"subjectName": string | null;
|
|
5098
|
-
"subjectLabel": string;
|
|
5099
|
-
"armConfig": {
|
|
5100
|
-
"baseline"?: {
|
|
5101
|
-
"tools"?: "default" | "none" | {
|
|
5102
|
-
"tools": Array<string>;
|
|
5967
|
+
"shape": "multi-arm";
|
|
5968
|
+
"summary": {
|
|
5969
|
+
"shape": "multi-arm";
|
|
5970
|
+
"verdict": "measured" | "inconclusive";
|
|
5971
|
+
"groups": Array<{
|
|
5972
|
+
"key": string;
|
|
5973
|
+
"label": string;
|
|
5974
|
+
"role": "reference" | "peer";
|
|
5975
|
+
"composition": {
|
|
5976
|
+
"tools"?: "default" | "none" | {
|
|
5977
|
+
"tools": Array<string>;
|
|
5978
|
+
};
|
|
5979
|
+
"addons"?: Array<string>;
|
|
5103
5980
|
};
|
|
5104
|
-
|
|
5105
|
-
|
|
5106
|
-
|
|
5107
|
-
"
|
|
5108
|
-
|
|
5981
|
+
}>;
|
|
5982
|
+
"ranking": {
|
|
5983
|
+
"plan": "vs_reference" | "all_pairs" | null;
|
|
5984
|
+
"referenceArm": string | null;
|
|
5985
|
+
"decision": {
|
|
5986
|
+
"leader": string | null;
|
|
5987
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5988
|
+
"reason": string;
|
|
5989
|
+
"separatedFrom": Array<string>;
|
|
5990
|
+
"lostTo": Array<{
|
|
5991
|
+
"arm": string;
|
|
5992
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5993
|
+
}>;
|
|
5994
|
+
"clearLeader": boolean;
|
|
5995
|
+
"correctnessLeader": string | null;
|
|
5109
5996
|
};
|
|
5110
|
-
"
|
|
5997
|
+
"entries": Array<{
|
|
5998
|
+
"arm": string;
|
|
5999
|
+
"runs": number;
|
|
6000
|
+
"completed": number;
|
|
6001
|
+
"completionRate": number | null;
|
|
6002
|
+
"goalRate": number | null;
|
|
6003
|
+
"avgQualityScore": number | null;
|
|
6004
|
+
"evalPassRate": number | null;
|
|
6005
|
+
"benchmarkPassRate"?: number | null;
|
|
6006
|
+
"compositeScore": number | null;
|
|
6007
|
+
"avgTotalTokens": number | null;
|
|
6008
|
+
"avgOutputTokens": number | null;
|
|
6009
|
+
"avgCostUsd": number | null;
|
|
6010
|
+
"avgDurationMs": number | null;
|
|
6011
|
+
"medianCostUsd": number | null;
|
|
6012
|
+
"medianTotalTokens": number | null;
|
|
6013
|
+
"medianDurationMs": number | null;
|
|
6014
|
+
"rank": number;
|
|
6015
|
+
}>;
|
|
6016
|
+
"contrasts": Array<{
|
|
6017
|
+
"contrastId": string;
|
|
6018
|
+
"referenceArm": string;
|
|
6019
|
+
"subjectArm": string;
|
|
6020
|
+
"oriented": boolean;
|
|
6021
|
+
"outcome": "won" | "tie" | "unmeasurable";
|
|
6022
|
+
"winner": string | null;
|
|
6023
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
6024
|
+
"explanation": string;
|
|
6025
|
+
}>;
|
|
6026
|
+
"comparisons": number;
|
|
6027
|
+
"headline"?: string;
|
|
5111
6028
|
};
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
6029
|
+
"matrixShape": {
|
|
6030
|
+
"cells": number;
|
|
6031
|
+
"harnesses": number;
|
|
6032
|
+
"models": number;
|
|
6033
|
+
"crossed": boolean;
|
|
6034
|
+
};
|
|
6035
|
+
"flakiness": {
|
|
6036
|
+
"measured": boolean;
|
|
6037
|
+
"conditions": number;
|
|
6038
|
+
"unstable": number;
|
|
6039
|
+
"worst": {
|
|
6040
|
+
"cell": string;
|
|
6041
|
+
"taskId": string;
|
|
6042
|
+
"arm": string;
|
|
6043
|
+
"variant"?: number;
|
|
6044
|
+
"passRate": number;
|
|
6045
|
+
} | null;
|
|
6046
|
+
};
|
|
6047
|
+
"totals": {
|
|
6048
|
+
"cells": number;
|
|
6049
|
+
"runs": number;
|
|
6050
|
+
"completed": number;
|
|
6051
|
+
"failed": number;
|
|
6052
|
+
"costUsd": number;
|
|
6053
|
+
"tokens": number;
|
|
6054
|
+
};
|
|
6055
|
+
};
|
|
6056
|
+
"arms": Array<{
|
|
6057
|
+
"key": string;
|
|
6058
|
+
"label": string;
|
|
6059
|
+
"position": number;
|
|
6060
|
+
"role": "reference" | "peer";
|
|
6061
|
+
"composition": {
|
|
6062
|
+
"tools"?: "default" | "none" | {
|
|
6063
|
+
"tools": Array<string>;
|
|
6064
|
+
};
|
|
6065
|
+
"addons"?: Array<string>;
|
|
6066
|
+
};
|
|
6067
|
+
"agg": {
|
|
6068
|
+
"runs": number;
|
|
6069
|
+
"completed": number;
|
|
6070
|
+
"infraFailures": number;
|
|
6071
|
+
"completionRate": number | null;
|
|
6072
|
+
"goalRate": number | null;
|
|
6073
|
+
"goalTally": {
|
|
6074
|
+
"achieved": number;
|
|
6075
|
+
"partial": number;
|
|
6076
|
+
"missed": number;
|
|
6077
|
+
};
|
|
6078
|
+
"avgTotalTokens": number | null;
|
|
6079
|
+
"avgOutputTokens": number | null;
|
|
6080
|
+
"avgCostUsd": number | null;
|
|
6081
|
+
"avgDurationMs": number | null;
|
|
6082
|
+
"avgTurns": number | null;
|
|
6083
|
+
"costCoverage": number | null;
|
|
6084
|
+
"avgQualityScore": number | null;
|
|
6085
|
+
"gradedCoverage": number | null;
|
|
6086
|
+
"medianCostUsd": number | null;
|
|
6087
|
+
"medianTotalTokens": number | null;
|
|
6088
|
+
"medianDurationMs": number | null;
|
|
6089
|
+
"p95CostUsd": number | null;
|
|
6090
|
+
"p95TotalTokens": number | null;
|
|
6091
|
+
"p95DurationMs": number | null;
|
|
6092
|
+
"maxCostUsd": number | null;
|
|
6093
|
+
"maxTotalTokens": number | null;
|
|
6094
|
+
"maxDurationMs": number | null;
|
|
6095
|
+
"evalPassRate"?: number | null;
|
|
6096
|
+
"benchmarkPassRate"?: number | null;
|
|
6097
|
+
"adoptionRate"?: number | null;
|
|
6098
|
+
};
|
|
6099
|
+
"perCell": Array<{
|
|
6100
|
+
"harness": string;
|
|
6101
|
+
"model": string;
|
|
6102
|
+
"provider": string | null;
|
|
6103
|
+
"paramsKey"?: string;
|
|
6104
|
+
"label"?: string;
|
|
6105
|
+
"agg": {
|
|
6106
|
+
"runs": number;
|
|
6107
|
+
"completed": number;
|
|
6108
|
+
"infraFailures": number;
|
|
6109
|
+
"completionRate": number | null;
|
|
6110
|
+
"goalRate": number | null;
|
|
6111
|
+
"goalTally": {
|
|
6112
|
+
"achieved": number;
|
|
6113
|
+
"partial": number;
|
|
6114
|
+
"missed": number;
|
|
6115
|
+
};
|
|
6116
|
+
"avgTotalTokens": number | null;
|
|
6117
|
+
"avgOutputTokens": number | null;
|
|
6118
|
+
"avgCostUsd": number | null;
|
|
6119
|
+
"avgDurationMs": number | null;
|
|
6120
|
+
"avgTurns": number | null;
|
|
6121
|
+
"costCoverage": number | null;
|
|
6122
|
+
"avgQualityScore": number | null;
|
|
6123
|
+
"gradedCoverage": number | null;
|
|
6124
|
+
"medianCostUsd": number | null;
|
|
6125
|
+
"medianTotalTokens": number | null;
|
|
6126
|
+
"medianDurationMs": number | null;
|
|
6127
|
+
"p95CostUsd": number | null;
|
|
6128
|
+
"p95TotalTokens": number | null;
|
|
6129
|
+
"p95DurationMs": number | null;
|
|
6130
|
+
"maxCostUsd": number | null;
|
|
6131
|
+
"maxTotalTokens": number | null;
|
|
6132
|
+
"maxDurationMs": number | null;
|
|
6133
|
+
"evalPassRate"?: number | null;
|
|
6134
|
+
"benchmarkPassRate"?: number | null;
|
|
6135
|
+
"adoptionRate"?: number | null;
|
|
6136
|
+
};
|
|
6137
|
+
}>;
|
|
6138
|
+
"reference"?: {
|
|
6139
|
+
"cells": Array<{
|
|
6140
|
+
"cell": string;
|
|
6141
|
+
"harness": string;
|
|
6142
|
+
"model": string;
|
|
6143
|
+
"provider": string | null;
|
|
6144
|
+
"paramsKey": string;
|
|
6145
|
+
"label": string;
|
|
6146
|
+
"passRate": {
|
|
6147
|
+
"value": number | null;
|
|
6148
|
+
"ci": Array<unknown> | null;
|
|
6149
|
+
"n": number;
|
|
6150
|
+
};
|
|
6151
|
+
"abstention"?: {
|
|
6152
|
+
"value": number | null;
|
|
6153
|
+
"ci": Array<unknown> | null;
|
|
6154
|
+
"n": number;
|
|
6155
|
+
"parts"?: Array<{
|
|
6156
|
+
"id": string;
|
|
6157
|
+
"label": string;
|
|
6158
|
+
"value": number | null;
|
|
6159
|
+
"n": number;
|
|
6160
|
+
}>;
|
|
6161
|
+
};
|
|
6162
|
+
"tasks": number;
|
|
6163
|
+
"graded": number;
|
|
6164
|
+
"unmeasured": number;
|
|
6165
|
+
"matchers": Array<string>;
|
|
6166
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
6167
|
+
}>;
|
|
6168
|
+
"overall": {
|
|
6169
|
+
"value": number | null;
|
|
6170
|
+
"ci": Array<unknown> | null;
|
|
6171
|
+
"n": number;
|
|
6172
|
+
};
|
|
6173
|
+
"abstention"?: {
|
|
6174
|
+
"value": number | null;
|
|
6175
|
+
"ci": Array<unknown> | null;
|
|
6176
|
+
"n": number;
|
|
6177
|
+
"parts"?: Array<{
|
|
6178
|
+
"id": string;
|
|
6179
|
+
"label": string;
|
|
6180
|
+
"value": number | null;
|
|
6181
|
+
"n": number;
|
|
6182
|
+
}>;
|
|
6183
|
+
};
|
|
6184
|
+
"graded": number;
|
|
6185
|
+
"unmeasured": number;
|
|
6186
|
+
"modelJudged": boolean;
|
|
6187
|
+
};
|
|
6188
|
+
}>;
|
|
6189
|
+
"contrasts": Array<{
|
|
6190
|
+
"contrastId": string;
|
|
6191
|
+
"referenceArm": string;
|
|
6192
|
+
"subjectArm": string;
|
|
6193
|
+
"oriented": boolean;
|
|
6194
|
+
"basis"?: {
|
|
6195
|
+
"id": "matched" | "whole_setup";
|
|
6196
|
+
"compared": {
|
|
6197
|
+
"reference": Array<{
|
|
6198
|
+
"harness": string;
|
|
6199
|
+
"model": string;
|
|
6200
|
+
"provider"?: string;
|
|
6201
|
+
"params"?: Record<string, unknown>;
|
|
6202
|
+
}>;
|
|
6203
|
+
"subject": Array<{
|
|
6204
|
+
"harness": string;
|
|
6205
|
+
"model": string;
|
|
6206
|
+
"provider"?: string;
|
|
6207
|
+
"params"?: Record<string, unknown>;
|
|
6208
|
+
}>;
|
|
6209
|
+
};
|
|
6210
|
+
"excluded": {
|
|
6211
|
+
"reference": Array<{
|
|
6212
|
+
"harness": string;
|
|
6213
|
+
"model": string;
|
|
6214
|
+
"provider"?: string;
|
|
6215
|
+
"params"?: Record<string, unknown>;
|
|
6216
|
+
}>;
|
|
6217
|
+
"subject": Array<{
|
|
6218
|
+
"harness": string;
|
|
6219
|
+
"model": string;
|
|
6220
|
+
"provider"?: string;
|
|
6221
|
+
"params"?: Record<string, unknown>;
|
|
6222
|
+
}>;
|
|
6223
|
+
};
|
|
6224
|
+
"differsIn": Array<string>;
|
|
6225
|
+
"licensedFacet": string | null;
|
|
6226
|
+
"efficiency": "comparable" | "incomparable";
|
|
6227
|
+
"pairedTasks": number | null;
|
|
6228
|
+
"dropped": {
|
|
6229
|
+
"reference": number;
|
|
6230
|
+
"subject": number;
|
|
6231
|
+
} | null;
|
|
6232
|
+
};
|
|
6233
|
+
"magnitudes": "relative";
|
|
6234
|
+
"summary": {
|
|
6235
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
6236
|
+
"headline": Array<{
|
|
6237
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6238
|
+
"label": string;
|
|
6239
|
+
"direction": "down" | "up";
|
|
6240
|
+
"baselineAvg": number | null;
|
|
6241
|
+
"skillAvg": number | null;
|
|
6242
|
+
"deltaPct": number | null;
|
|
6243
|
+
"delta"?: {
|
|
6244
|
+
"unit": "pp";
|
|
6245
|
+
"absolute": number | null;
|
|
6246
|
+
"relativePct": number | null;
|
|
6247
|
+
} | {
|
|
6248
|
+
"unit": "pts";
|
|
6249
|
+
"absolute": number | null;
|
|
6250
|
+
"relativePct"?: number | null;
|
|
6251
|
+
} | {
|
|
6252
|
+
"unit": "%";
|
|
6253
|
+
"relativePct": number | null;
|
|
6254
|
+
};
|
|
6255
|
+
"improved": boolean;
|
|
6256
|
+
}>;
|
|
6257
|
+
"completionRate": {
|
|
6258
|
+
"baseline": number | null;
|
|
6259
|
+
"skill": number | null;
|
|
6260
|
+
};
|
|
6261
|
+
"goalRate": {
|
|
6262
|
+
"baseline": number | null;
|
|
6263
|
+
"skill": number | null;
|
|
6264
|
+
};
|
|
6265
|
+
"pairwise": {
|
|
6266
|
+
"pairs": number;
|
|
6267
|
+
"skillWins": number;
|
|
6268
|
+
"baselineWins": number;
|
|
6269
|
+
"ties": number;
|
|
6270
|
+
"winRate": number | null;
|
|
6271
|
+
};
|
|
6272
|
+
"decision"?: {
|
|
6273
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
6274
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
6275
|
+
"reason": string;
|
|
6276
|
+
"efficiencyDiscounted": boolean;
|
|
6277
|
+
};
|
|
6278
|
+
"concentration": {
|
|
6279
|
+
"perCell": Array<{
|
|
6280
|
+
"cell": string;
|
|
6281
|
+
"goalDeltaPp": number | null;
|
|
6282
|
+
"contributionPct": number | null;
|
|
6283
|
+
}>;
|
|
6284
|
+
"heterogeneity": {
|
|
6285
|
+
"dominantCell": string;
|
|
6286
|
+
"contributionPct": number;
|
|
6287
|
+
} | null;
|
|
6288
|
+
};
|
|
6289
|
+
"matrixShape": {
|
|
6290
|
+
"cells": number;
|
|
6291
|
+
"harnesses": number;
|
|
6292
|
+
"models": number;
|
|
6293
|
+
"crossed": boolean;
|
|
6294
|
+
};
|
|
6295
|
+
"flakiness"?: {
|
|
6296
|
+
"measured": boolean;
|
|
6297
|
+
"conditions": number;
|
|
6298
|
+
"unstable": number;
|
|
6299
|
+
"worst": {
|
|
6300
|
+
"cell": string;
|
|
6301
|
+
"taskId": string;
|
|
6302
|
+
"arm": "baseline" | "skill";
|
|
6303
|
+
"variant"?: number;
|
|
6304
|
+
"passRate": number;
|
|
6305
|
+
} | null;
|
|
6306
|
+
};
|
|
6307
|
+
"arms"?: {
|
|
6308
|
+
"baseline": {
|
|
6309
|
+
"runs": number;
|
|
6310
|
+
"completed": number;
|
|
6311
|
+
"infraFailures": number;
|
|
6312
|
+
"completionRate": number | null;
|
|
6313
|
+
"goalRate": number | null;
|
|
6314
|
+
"goalTally": {
|
|
6315
|
+
"achieved": number;
|
|
6316
|
+
"partial": number;
|
|
6317
|
+
"missed": number;
|
|
6318
|
+
};
|
|
6319
|
+
"avgTotalTokens": number | null;
|
|
6320
|
+
"avgOutputTokens": number | null;
|
|
6321
|
+
"avgCostUsd": number | null;
|
|
6322
|
+
"avgDurationMs": number | null;
|
|
6323
|
+
"avgTurns": number | null;
|
|
6324
|
+
"costCoverage": number | null;
|
|
6325
|
+
"avgQualityScore": number | null;
|
|
6326
|
+
"gradedCoverage": number | null;
|
|
6327
|
+
"medianCostUsd": number | null;
|
|
6328
|
+
"medianTotalTokens": number | null;
|
|
6329
|
+
"medianDurationMs": number | null;
|
|
6330
|
+
"p95CostUsd": number | null;
|
|
6331
|
+
"p95TotalTokens": number | null;
|
|
6332
|
+
"p95DurationMs": number | null;
|
|
6333
|
+
"maxCostUsd": number | null;
|
|
6334
|
+
"maxTotalTokens": number | null;
|
|
6335
|
+
"maxDurationMs": number | null;
|
|
6336
|
+
"evalPassRate"?: number | null;
|
|
6337
|
+
"benchmarkPassRate"?: number | null;
|
|
6338
|
+
"adoptionRate"?: number | null;
|
|
6339
|
+
};
|
|
6340
|
+
"skill": {
|
|
6341
|
+
"runs": number;
|
|
6342
|
+
"completed": number;
|
|
6343
|
+
"infraFailures": number;
|
|
6344
|
+
"completionRate": number | null;
|
|
6345
|
+
"goalRate": number | null;
|
|
6346
|
+
"goalTally": {
|
|
6347
|
+
"achieved": number;
|
|
6348
|
+
"partial": number;
|
|
6349
|
+
"missed": number;
|
|
6350
|
+
};
|
|
6351
|
+
"avgTotalTokens": number | null;
|
|
6352
|
+
"avgOutputTokens": number | null;
|
|
6353
|
+
"avgCostUsd": number | null;
|
|
6354
|
+
"avgDurationMs": number | null;
|
|
6355
|
+
"avgTurns": number | null;
|
|
6356
|
+
"costCoverage": number | null;
|
|
6357
|
+
"avgQualityScore": number | null;
|
|
6358
|
+
"gradedCoverage": number | null;
|
|
6359
|
+
"medianCostUsd": number | null;
|
|
6360
|
+
"medianTotalTokens": number | null;
|
|
6361
|
+
"medianDurationMs": number | null;
|
|
6362
|
+
"p95CostUsd": number | null;
|
|
6363
|
+
"p95TotalTokens": number | null;
|
|
6364
|
+
"p95DurationMs": number | null;
|
|
6365
|
+
"maxCostUsd": number | null;
|
|
6366
|
+
"maxTotalTokens": number | null;
|
|
6367
|
+
"maxDurationMs": number | null;
|
|
6368
|
+
"evalPassRate"?: number | null;
|
|
6369
|
+
"benchmarkPassRate"?: number | null;
|
|
6370
|
+
"adoptionRate"?: number | null;
|
|
6371
|
+
};
|
|
6372
|
+
};
|
|
6373
|
+
"totals": {
|
|
6374
|
+
"cells": number;
|
|
6375
|
+
"runs": number;
|
|
6376
|
+
"completed": number;
|
|
6377
|
+
"failed": number;
|
|
6378
|
+
"costUsd": number;
|
|
6379
|
+
"tokens": number;
|
|
6380
|
+
};
|
|
6381
|
+
};
|
|
6382
|
+
"perCell": Array<{
|
|
6383
|
+
"harness": string;
|
|
6384
|
+
"model": string;
|
|
6385
|
+
"provider": string | null;
|
|
6386
|
+
"paramsKey"?: string;
|
|
6387
|
+
"label"?: string;
|
|
6388
|
+
"baseline": {
|
|
6389
|
+
"runs": number;
|
|
6390
|
+
"completed": number;
|
|
6391
|
+
"infraFailures": number;
|
|
6392
|
+
"completionRate": number | null;
|
|
6393
|
+
"goalRate": number | null;
|
|
6394
|
+
"goalTally": {
|
|
6395
|
+
"achieved": number;
|
|
6396
|
+
"partial": number;
|
|
6397
|
+
"missed": number;
|
|
6398
|
+
};
|
|
6399
|
+
"avgTotalTokens": number | null;
|
|
6400
|
+
"avgOutputTokens": number | null;
|
|
6401
|
+
"avgCostUsd": number | null;
|
|
6402
|
+
"avgDurationMs": number | null;
|
|
6403
|
+
"avgTurns": number | null;
|
|
6404
|
+
"costCoverage": number | null;
|
|
6405
|
+
"avgQualityScore": number | null;
|
|
6406
|
+
"gradedCoverage": number | null;
|
|
6407
|
+
"medianCostUsd": number | null;
|
|
6408
|
+
"medianTotalTokens": number | null;
|
|
6409
|
+
"medianDurationMs": number | null;
|
|
6410
|
+
"p95CostUsd": number | null;
|
|
6411
|
+
"p95TotalTokens": number | null;
|
|
6412
|
+
"p95DurationMs": number | null;
|
|
6413
|
+
"maxCostUsd": number | null;
|
|
6414
|
+
"maxTotalTokens": number | null;
|
|
6415
|
+
"maxDurationMs": number | null;
|
|
6416
|
+
"evalPassRate"?: number | null;
|
|
6417
|
+
"benchmarkPassRate"?: number | null;
|
|
6418
|
+
"adoptionRate"?: number | null;
|
|
6419
|
+
};
|
|
6420
|
+
"skill": {
|
|
6421
|
+
"runs": number;
|
|
6422
|
+
"completed": number;
|
|
6423
|
+
"infraFailures": number;
|
|
6424
|
+
"completionRate": number | null;
|
|
6425
|
+
"goalRate": number | null;
|
|
6426
|
+
"goalTally": {
|
|
6427
|
+
"achieved": number;
|
|
6428
|
+
"partial": number;
|
|
6429
|
+
"missed": number;
|
|
6430
|
+
};
|
|
6431
|
+
"avgTotalTokens": number | null;
|
|
6432
|
+
"avgOutputTokens": number | null;
|
|
6433
|
+
"avgCostUsd": number | null;
|
|
6434
|
+
"avgDurationMs": number | null;
|
|
6435
|
+
"avgTurns": number | null;
|
|
6436
|
+
"costCoverage": number | null;
|
|
6437
|
+
"avgQualityScore": number | null;
|
|
6438
|
+
"gradedCoverage": number | null;
|
|
6439
|
+
"medianCostUsd": number | null;
|
|
6440
|
+
"medianTotalTokens": number | null;
|
|
6441
|
+
"medianDurationMs": number | null;
|
|
6442
|
+
"p95CostUsd": number | null;
|
|
6443
|
+
"p95TotalTokens": number | null;
|
|
6444
|
+
"p95DurationMs": number | null;
|
|
6445
|
+
"maxCostUsd": number | null;
|
|
6446
|
+
"maxTotalTokens": number | null;
|
|
6447
|
+
"maxDurationMs": number | null;
|
|
6448
|
+
"evalPassRate"?: number | null;
|
|
6449
|
+
"benchmarkPassRate"?: number | null;
|
|
6450
|
+
"adoptionRate"?: number | null;
|
|
6451
|
+
};
|
|
6452
|
+
"deltas": {
|
|
6453
|
+
"totalTokensPct": number | null;
|
|
6454
|
+
"outputTokensPct": number | null;
|
|
6455
|
+
"costUsdPct": number | null;
|
|
6456
|
+
"durationMsPct": number | null;
|
|
6457
|
+
"numTurnsPct": number | null;
|
|
6458
|
+
"completionRatePp": number | null;
|
|
6459
|
+
"goalRatePp": number | null;
|
|
6460
|
+
"qualityScorePts": number | null;
|
|
6461
|
+
"evalPassRatePp"?: number | null;
|
|
6462
|
+
};
|
|
6463
|
+
}>;
|
|
6464
|
+
} | {
|
|
6465
|
+
"contrastId": string;
|
|
6466
|
+
"referenceArm": string;
|
|
6467
|
+
"subjectArm": string;
|
|
6468
|
+
"oriented": boolean;
|
|
6469
|
+
"basis"?: {
|
|
6470
|
+
"id": "matched" | "whole_setup";
|
|
6471
|
+
"compared": {
|
|
6472
|
+
"reference": Array<{
|
|
6473
|
+
"harness": string;
|
|
6474
|
+
"model": string;
|
|
6475
|
+
"provider"?: string;
|
|
6476
|
+
"params"?: Record<string, unknown>;
|
|
6477
|
+
}>;
|
|
6478
|
+
"subject": Array<{
|
|
6479
|
+
"harness": string;
|
|
6480
|
+
"model": string;
|
|
6481
|
+
"provider"?: string;
|
|
6482
|
+
"params"?: Record<string, unknown>;
|
|
6483
|
+
}>;
|
|
6484
|
+
};
|
|
6485
|
+
"excluded": {
|
|
6486
|
+
"reference": Array<{
|
|
6487
|
+
"harness": string;
|
|
6488
|
+
"model": string;
|
|
6489
|
+
"provider"?: string;
|
|
6490
|
+
"params"?: Record<string, unknown>;
|
|
6491
|
+
}>;
|
|
6492
|
+
"subject": Array<{
|
|
6493
|
+
"harness": string;
|
|
6494
|
+
"model": string;
|
|
6495
|
+
"provider"?: string;
|
|
6496
|
+
"params"?: Record<string, unknown>;
|
|
6497
|
+
}>;
|
|
6498
|
+
};
|
|
6499
|
+
"differsIn": Array<string>;
|
|
6500
|
+
"licensedFacet": string | null;
|
|
6501
|
+
"efficiency": "comparable" | "incomparable";
|
|
6502
|
+
"pairedTasks": number | null;
|
|
6503
|
+
"dropped": {
|
|
6504
|
+
"reference": number;
|
|
6505
|
+
"subject": number;
|
|
6506
|
+
} | null;
|
|
6507
|
+
};
|
|
6508
|
+
"magnitudes": "absolute";
|
|
6509
|
+
"summary": {
|
|
6510
|
+
"completionRate": {
|
|
6511
|
+
"baseline": number | null;
|
|
6512
|
+
"skill": number | null;
|
|
6513
|
+
};
|
|
6514
|
+
"goalRate": {
|
|
6515
|
+
"baseline": number | null;
|
|
6516
|
+
"skill": number | null;
|
|
6517
|
+
};
|
|
6518
|
+
"pairwise": {
|
|
6519
|
+
"pairs": number;
|
|
6520
|
+
"skillWins": number;
|
|
6521
|
+
"baselineWins": number;
|
|
6522
|
+
"ties": number;
|
|
6523
|
+
"winRate": number | null;
|
|
6524
|
+
};
|
|
6525
|
+
"decision"?: {
|
|
6526
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
6527
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
6528
|
+
"reason": string;
|
|
6529
|
+
"efficiencyDiscounted": boolean;
|
|
6530
|
+
};
|
|
6531
|
+
"concentration": {
|
|
6532
|
+
"perCell": Array<{
|
|
6533
|
+
"cell": string;
|
|
6534
|
+
"goalDeltaPp": number | null;
|
|
6535
|
+
"contributionPct": number | null;
|
|
6536
|
+
}>;
|
|
6537
|
+
"heterogeneity": {
|
|
6538
|
+
"dominantCell": string;
|
|
6539
|
+
"contributionPct": number;
|
|
6540
|
+
} | null;
|
|
6541
|
+
};
|
|
6542
|
+
"matrixShape": {
|
|
6543
|
+
"cells": number;
|
|
6544
|
+
"harnesses": number;
|
|
6545
|
+
"models": number;
|
|
6546
|
+
"crossed": boolean;
|
|
6547
|
+
};
|
|
6548
|
+
"flakiness"?: {
|
|
6549
|
+
"measured": boolean;
|
|
6550
|
+
"conditions": number;
|
|
6551
|
+
"unstable": number;
|
|
6552
|
+
"worst": {
|
|
6553
|
+
"cell": string;
|
|
6554
|
+
"taskId": string;
|
|
6555
|
+
"arm": "baseline" | "skill";
|
|
6556
|
+
"variant"?: number;
|
|
6557
|
+
"passRate": number;
|
|
6558
|
+
} | null;
|
|
6559
|
+
};
|
|
6560
|
+
"arms"?: {
|
|
6561
|
+
"baseline": {
|
|
6562
|
+
"runs": number;
|
|
6563
|
+
"completed": number;
|
|
6564
|
+
"infraFailures": number;
|
|
6565
|
+
"completionRate": number | null;
|
|
6566
|
+
"goalRate": number | null;
|
|
6567
|
+
"goalTally": {
|
|
6568
|
+
"achieved": number;
|
|
6569
|
+
"partial": number;
|
|
6570
|
+
"missed": number;
|
|
6571
|
+
};
|
|
6572
|
+
"avgTotalTokens": number | null;
|
|
6573
|
+
"avgOutputTokens": number | null;
|
|
6574
|
+
"avgCostUsd": number | null;
|
|
6575
|
+
"avgDurationMs": number | null;
|
|
6576
|
+
"avgTurns": number | null;
|
|
6577
|
+
"costCoverage": number | null;
|
|
6578
|
+
"avgQualityScore": number | null;
|
|
6579
|
+
"gradedCoverage": number | null;
|
|
6580
|
+
"medianCostUsd": number | null;
|
|
6581
|
+
"medianTotalTokens": number | null;
|
|
6582
|
+
"medianDurationMs": number | null;
|
|
6583
|
+
"p95CostUsd": number | null;
|
|
6584
|
+
"p95TotalTokens": number | null;
|
|
6585
|
+
"p95DurationMs": number | null;
|
|
6586
|
+
"maxCostUsd": number | null;
|
|
6587
|
+
"maxTotalTokens": number | null;
|
|
6588
|
+
"maxDurationMs": number | null;
|
|
6589
|
+
"evalPassRate"?: number | null;
|
|
6590
|
+
"benchmarkPassRate"?: number | null;
|
|
6591
|
+
"adoptionRate"?: number | null;
|
|
6592
|
+
};
|
|
6593
|
+
"skill": {
|
|
6594
|
+
"runs": number;
|
|
6595
|
+
"completed": number;
|
|
6596
|
+
"infraFailures": number;
|
|
6597
|
+
"completionRate": number | null;
|
|
6598
|
+
"goalRate": number | null;
|
|
6599
|
+
"goalTally": {
|
|
6600
|
+
"achieved": number;
|
|
6601
|
+
"partial": number;
|
|
6602
|
+
"missed": number;
|
|
6603
|
+
};
|
|
6604
|
+
"avgTotalTokens": number | null;
|
|
6605
|
+
"avgOutputTokens": number | null;
|
|
6606
|
+
"avgCostUsd": number | null;
|
|
6607
|
+
"avgDurationMs": number | null;
|
|
6608
|
+
"avgTurns": number | null;
|
|
6609
|
+
"costCoverage": number | null;
|
|
6610
|
+
"avgQualityScore": number | null;
|
|
6611
|
+
"gradedCoverage": number | null;
|
|
6612
|
+
"medianCostUsd": number | null;
|
|
6613
|
+
"medianTotalTokens": number | null;
|
|
6614
|
+
"medianDurationMs": number | null;
|
|
6615
|
+
"p95CostUsd": number | null;
|
|
6616
|
+
"p95TotalTokens": number | null;
|
|
6617
|
+
"p95DurationMs": number | null;
|
|
6618
|
+
"maxCostUsd": number | null;
|
|
6619
|
+
"maxTotalTokens": number | null;
|
|
6620
|
+
"maxDurationMs": number | null;
|
|
6621
|
+
"evalPassRate"?: number | null;
|
|
6622
|
+
"benchmarkPassRate"?: number | null;
|
|
6623
|
+
"adoptionRate"?: number | null;
|
|
6624
|
+
};
|
|
6625
|
+
};
|
|
6626
|
+
"totals": {
|
|
6627
|
+
"cells": number;
|
|
6628
|
+
"runs": number;
|
|
6629
|
+
"completed": number;
|
|
6630
|
+
"failed": number;
|
|
6631
|
+
"costUsd": number;
|
|
6632
|
+
"tokens": number;
|
|
6633
|
+
};
|
|
6634
|
+
};
|
|
6635
|
+
"perCell": Array<{
|
|
6636
|
+
"harness": string;
|
|
6637
|
+
"model": string;
|
|
6638
|
+
"provider": string | null;
|
|
6639
|
+
"paramsKey"?: string;
|
|
6640
|
+
"label"?: string;
|
|
6641
|
+
"baseline": {
|
|
6642
|
+
"runs": number;
|
|
6643
|
+
"completed": number;
|
|
6644
|
+
"infraFailures": number;
|
|
6645
|
+
"completionRate": number | null;
|
|
6646
|
+
"goalRate": number | null;
|
|
6647
|
+
"goalTally": {
|
|
6648
|
+
"achieved": number;
|
|
6649
|
+
"partial": number;
|
|
6650
|
+
"missed": number;
|
|
6651
|
+
};
|
|
6652
|
+
"avgTotalTokens": number | null;
|
|
6653
|
+
"avgOutputTokens": number | null;
|
|
6654
|
+
"avgCostUsd": number | null;
|
|
6655
|
+
"avgDurationMs": number | null;
|
|
6656
|
+
"avgTurns": number | null;
|
|
6657
|
+
"costCoverage": number | null;
|
|
6658
|
+
"avgQualityScore": number | null;
|
|
6659
|
+
"gradedCoverage": number | null;
|
|
6660
|
+
"medianCostUsd": number | null;
|
|
6661
|
+
"medianTotalTokens": number | null;
|
|
6662
|
+
"medianDurationMs": number | null;
|
|
6663
|
+
"p95CostUsd": number | null;
|
|
6664
|
+
"p95TotalTokens": number | null;
|
|
6665
|
+
"p95DurationMs": number | null;
|
|
6666
|
+
"maxCostUsd": number | null;
|
|
6667
|
+
"maxTotalTokens": number | null;
|
|
6668
|
+
"maxDurationMs": number | null;
|
|
6669
|
+
"evalPassRate"?: number | null;
|
|
6670
|
+
"benchmarkPassRate"?: number | null;
|
|
6671
|
+
"adoptionRate"?: number | null;
|
|
6672
|
+
};
|
|
6673
|
+
"skill": {
|
|
6674
|
+
"runs": number;
|
|
6675
|
+
"completed": number;
|
|
6676
|
+
"infraFailures": number;
|
|
6677
|
+
"completionRate": number | null;
|
|
6678
|
+
"goalRate": number | null;
|
|
6679
|
+
"goalTally": {
|
|
6680
|
+
"achieved": number;
|
|
6681
|
+
"partial": number;
|
|
6682
|
+
"missed": number;
|
|
6683
|
+
};
|
|
6684
|
+
"avgTotalTokens": number | null;
|
|
6685
|
+
"avgOutputTokens": number | null;
|
|
6686
|
+
"avgCostUsd": number | null;
|
|
6687
|
+
"avgDurationMs": number | null;
|
|
6688
|
+
"avgTurns": number | null;
|
|
6689
|
+
"costCoverage": number | null;
|
|
6690
|
+
"avgQualityScore": number | null;
|
|
6691
|
+
"gradedCoverage": number | null;
|
|
6692
|
+
"medianCostUsd": number | null;
|
|
6693
|
+
"medianTotalTokens": number | null;
|
|
6694
|
+
"medianDurationMs": number | null;
|
|
6695
|
+
"p95CostUsd": number | null;
|
|
6696
|
+
"p95TotalTokens": number | null;
|
|
6697
|
+
"p95DurationMs": number | null;
|
|
6698
|
+
"maxCostUsd": number | null;
|
|
6699
|
+
"maxTotalTokens": number | null;
|
|
6700
|
+
"maxDurationMs": number | null;
|
|
6701
|
+
"evalPassRate"?: number | null;
|
|
6702
|
+
"benchmarkPassRate"?: number | null;
|
|
6703
|
+
"adoptionRate"?: number | null;
|
|
6704
|
+
};
|
|
6705
|
+
"deltas": {
|
|
6706
|
+
"completionRatePp": number | null;
|
|
6707
|
+
"goalRatePp": number | null;
|
|
6708
|
+
"qualityScorePts": number | null;
|
|
6709
|
+
"evalPassRatePp"?: number | null;
|
|
6710
|
+
};
|
|
6711
|
+
}>;
|
|
6712
|
+
}>;
|
|
6713
|
+
"board": Array<{
|
|
6714
|
+
"harness": string;
|
|
6715
|
+
"model": string;
|
|
6716
|
+
"provider": string | null;
|
|
6717
|
+
"paramsKey"?: string;
|
|
6718
|
+
"label"?: string;
|
|
6719
|
+
"arm": string;
|
|
6720
|
+
"runs": number;
|
|
6721
|
+
"completed": number;
|
|
6722
|
+
"completionRate": number | null;
|
|
6723
|
+
"goalRate": number | null;
|
|
6724
|
+
"avgQualityScore": number | null;
|
|
6725
|
+
"evalPassRate": number | null;
|
|
6726
|
+
"benchmarkPassRate"?: number | null;
|
|
6727
|
+
"compositeScore": number | null;
|
|
6728
|
+
"avgTotalTokens": number | null;
|
|
6729
|
+
"avgOutputTokens": number | null;
|
|
6730
|
+
"avgCostUsd": number | null;
|
|
6731
|
+
"avgDurationMs": number | null;
|
|
6732
|
+
"medianCostUsd": number | null;
|
|
6733
|
+
"medianTotalTokens": number | null;
|
|
6734
|
+
"medianDurationMs": number | null;
|
|
6735
|
+
"rank": number;
|
|
6736
|
+
}>;
|
|
6737
|
+
"setupsBoard"?: Array<{
|
|
6738
|
+
"harness": string;
|
|
6739
|
+
"model": string;
|
|
6740
|
+
"provider": string | null;
|
|
6741
|
+
"paramsKey"?: string;
|
|
6742
|
+
"label"?: string;
|
|
6743
|
+
"arm": string;
|
|
6744
|
+
"runs": number;
|
|
6745
|
+
"completed": number;
|
|
6746
|
+
"completionRate": number | null;
|
|
6747
|
+
"goalRate": number | null;
|
|
6748
|
+
"avgQualityScore": number | null;
|
|
6749
|
+
"evalPassRate": number | null;
|
|
6750
|
+
"benchmarkPassRate"?: number | null;
|
|
6751
|
+
"compositeScore": number | null;
|
|
6752
|
+
"avgTotalTokens": number | null;
|
|
6753
|
+
"avgOutputTokens": number | null;
|
|
6754
|
+
"avgCostUsd": number | null;
|
|
6755
|
+
"avgDurationMs": number | null;
|
|
6756
|
+
"medianCostUsd": number | null;
|
|
6757
|
+
"medianTotalTokens": number | null;
|
|
6758
|
+
"medianDurationMs": number | null;
|
|
6759
|
+
"rank": number;
|
|
6760
|
+
"group": string;
|
|
6761
|
+
"agent": string;
|
|
6762
|
+
"setup": string;
|
|
6763
|
+
}>;
|
|
6764
|
+
"perTask": Array<{
|
|
6765
|
+
"taskId": string;
|
|
6766
|
+
"position": number;
|
|
6767
|
+
"promptPreview": string;
|
|
6768
|
+
"requiresWeb": boolean;
|
|
6769
|
+
"category": string | null;
|
|
6770
|
+
"byArm": Record<string, unknown>;
|
|
6771
|
+
}>;
|
|
6772
|
+
"behavior"?: Array<{
|
|
6773
|
+
"harness": string;
|
|
6774
|
+
"model": string;
|
|
6775
|
+
"provider": string | null;
|
|
6776
|
+
"arm": string;
|
|
6777
|
+
"totalRuns": number;
|
|
6778
|
+
"signals": Array<{
|
|
6779
|
+
"id": string;
|
|
6780
|
+
"label": string;
|
|
6781
|
+
"tone": "good" | "bad" | "neutral";
|
|
6782
|
+
"runCount": number;
|
|
6783
|
+
"succeededCount": number;
|
|
6784
|
+
}>;
|
|
6785
|
+
}>;
|
|
6786
|
+
"execution"?: Record<string, unknown>;
|
|
6787
|
+
"inference"?: {
|
|
6788
|
+
"contrasts": Array<{
|
|
6789
|
+
"contrastId": string;
|
|
6790
|
+
"table": {
|
|
6791
|
+
"columns": Array<{
|
|
6792
|
+
"key": string;
|
|
6793
|
+
"label": string;
|
|
6794
|
+
"family": "rate" | "score" | "magnitude";
|
|
6795
|
+
"betterWhen": "higher" | "lower";
|
|
6796
|
+
}>;
|
|
6797
|
+
"groups": Array<{
|
|
6798
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
6799
|
+
"title": string;
|
|
6800
|
+
"rows": Array<{
|
|
6801
|
+
"key": string;
|
|
6802
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
6803
|
+
"harness": string | null;
|
|
6804
|
+
"model": string | null;
|
|
6805
|
+
"paramsKey"?: string;
|
|
6806
|
+
"runs": number;
|
|
6807
|
+
"cells": Record<string, unknown>;
|
|
6808
|
+
}>;
|
|
6809
|
+
}>;
|
|
6810
|
+
"reference": "baseline" | "field";
|
|
6811
|
+
"draws": number;
|
|
6812
|
+
"adjustment": {
|
|
6813
|
+
"scope": "planned-comparisons";
|
|
6814
|
+
"comparisons": number;
|
|
6815
|
+
};
|
|
6816
|
+
} | null;
|
|
6817
|
+
}>;
|
|
6818
|
+
"holm": {
|
|
6819
|
+
"status": "complete";
|
|
6820
|
+
"ready": number;
|
|
6821
|
+
"planned": number;
|
|
6822
|
+
"adjustment": {
|
|
6823
|
+
"scope": "planned-comparisons";
|
|
6824
|
+
"comparisons": number;
|
|
6825
|
+
};
|
|
6826
|
+
} | {
|
|
6827
|
+
"status": "pending";
|
|
6828
|
+
"ready": number;
|
|
6829
|
+
"planned": number;
|
|
6830
|
+
};
|
|
6831
|
+
"claims": {
|
|
6832
|
+
"perContrast": Record<string, unknown>;
|
|
6833
|
+
"ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
|
|
6834
|
+
};
|
|
6835
|
+
};
|
|
6836
|
+
};
|
|
6837
|
+
"warnings": Array<{
|
|
6838
|
+
"type": "eval_capability_gap";
|
|
6839
|
+
"harness": string;
|
|
6840
|
+
"kind": string;
|
|
6841
|
+
} | {
|
|
6842
|
+
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
6843
|
+
"detail": string;
|
|
6844
|
+
"taskIds"?: Array<string>;
|
|
6845
|
+
} | {
|
|
6846
|
+
"type": "design_aggregate_budget";
|
|
6847
|
+
"detail": string;
|
|
6848
|
+
}>;
|
|
6849
|
+
"taskHealth": {
|
|
6850
|
+
"flags": Array<{
|
|
6851
|
+
"taskId": string;
|
|
6852
|
+
"signal": string;
|
|
6853
|
+
"kind": "floor" | "ceiling";
|
|
6854
|
+
"conditions": number;
|
|
6855
|
+
}>;
|
|
6856
|
+
"rubricSplits": Array<{
|
|
6857
|
+
"taskId": string;
|
|
6858
|
+
"criterionIndex": number;
|
|
6859
|
+
"criterion": string;
|
|
6860
|
+
"split": number;
|
|
6861
|
+
"graded": number;
|
|
6862
|
+
}>;
|
|
6863
|
+
"signals": Array<{
|
|
6864
|
+
"id": string;
|
|
6865
|
+
"label": string;
|
|
6866
|
+
}>;
|
|
6867
|
+
};
|
|
6868
|
+
"shareUrl"?: string | null;
|
|
6869
|
+
"canManageSharing"?: boolean;
|
|
6870
|
+
};
|
|
6871
|
+
export type StudyUpdateDetailsRequest = {
|
|
6872
|
+
"name"?: string;
|
|
6873
|
+
"description"?: string;
|
|
6874
|
+
};
|
|
6875
|
+
export type StudyUpdateDetailsResponse = {
|
|
6876
|
+
"study": {
|
|
6877
|
+
"id": string;
|
|
6878
|
+
"executiveSummary"?: string | null;
|
|
6879
|
+
"plannedRunCount": number;
|
|
6880
|
+
"organizationId": string;
|
|
6881
|
+
"createdBy": string | null;
|
|
6882
|
+
"name": string;
|
|
6883
|
+
"slug": string;
|
|
6884
|
+
"description": string | null;
|
|
6885
|
+
"skillRef": string | null;
|
|
6886
|
+
"skillSlug": string | null;
|
|
6887
|
+
"skillName": string | null;
|
|
6888
|
+
"skillPromise": string;
|
|
6889
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
6890
|
+
"subjectName": string | null;
|
|
6891
|
+
"subjectLabel": string;
|
|
6892
|
+
"armConfig": {
|
|
6893
|
+
"baseline"?: {
|
|
6894
|
+
"tools"?: "default" | "none" | {
|
|
6895
|
+
"tools": Array<string>;
|
|
6896
|
+
};
|
|
6897
|
+
"addons"?: Record<string, unknown>;
|
|
6898
|
+
};
|
|
6899
|
+
"treatment"?: {
|
|
6900
|
+
"tools"?: "default" | "none" | {
|
|
6901
|
+
"tools": Array<string>;
|
|
6902
|
+
};
|
|
6903
|
+
"addons"?: Record<string, unknown>;
|
|
6904
|
+
};
|
|
6905
|
+
};
|
|
6906
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6907
|
+
"subjectBackend": "model" | "harness" | null;
|
|
6908
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6909
|
+
"importedFrom"?: string;
|
|
6910
|
+
"baselineLabel": string | null;
|
|
6911
|
+
"treatmentLabel": string | null;
|
|
6912
|
+
"promiseMetrics": Array<{
|
|
6913
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6914
|
+
"direction": "down" | "up";
|
|
6915
|
+
"label": string;
|
|
6916
|
+
}>;
|
|
6917
|
+
"matrix": Array<{
|
|
6918
|
+
"harness": string;
|
|
6919
|
+
"model": string;
|
|
6920
|
+
"provider"?: string;
|
|
6921
|
+
"params"?: {
|
|
6922
|
+
"temperature"?: number;
|
|
6923
|
+
"maxOutputTokens"?: number;
|
|
6924
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
6925
|
+
"thinkingBudgetTokens"?: number;
|
|
6926
|
+
};
|
|
6927
|
+
}>;
|
|
6928
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6929
|
+
"judgeSupplementary"?: boolean;
|
|
6930
|
+
"taskCount": number;
|
|
6931
|
+
"variantsPerTask": number;
|
|
6932
|
+
"repeats": number;
|
|
6933
|
+
"subset": {
|
|
6934
|
+
"count": number;
|
|
6935
|
+
"seed": string;
|
|
6936
|
+
"of": number;
|
|
6937
|
+
} | null;
|
|
6938
|
+
"status": string;
|
|
6939
|
+
"error": string | null;
|
|
6940
|
+
"preparation"?: {
|
|
6941
|
+
"since": string;
|
|
6942
|
+
"progressAt"?: string;
|
|
6943
|
+
"launchedBy": {
|
|
6944
|
+
"userId"?: string;
|
|
6945
|
+
"role"?: string;
|
|
6946
|
+
};
|
|
6947
|
+
"waitingOn": Array<{
|
|
6948
|
+
"imageKey": string;
|
|
6949
|
+
"buildId": string;
|
|
6950
|
+
"stage"?: "image" | "snapshot";
|
|
6951
|
+
"retry"?: {
|
|
6952
|
+
"failedAttempts": number;
|
|
6953
|
+
"maxAttempts": number;
|
|
6954
|
+
"notBefore": string;
|
|
6955
|
+
"lastFailure": string;
|
|
6956
|
+
};
|
|
6957
|
+
}>;
|
|
6958
|
+
} | null;
|
|
6959
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
6960
|
+
"visibilityChangedAt"?: string | null;
|
|
6961
|
+
"visibilityChangedBy"?: string | null;
|
|
6962
|
+
"shareToken"?: string | null;
|
|
6963
|
+
"lowSampleOverride": boolean;
|
|
6964
|
+
"publishedAt": string | null;
|
|
6965
|
+
"verdict": unknown;
|
|
6966
|
+
"definitionVersionId": string | null;
|
|
6967
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
6968
|
+
"createdAt": string;
|
|
6969
|
+
"updatedAt": string;
|
|
6970
|
+
"liveUntil": string | null;
|
|
6971
|
+
"launch"?: {
|
|
6972
|
+
"attempt": number;
|
|
6973
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
6974
|
+
"startedAt": string;
|
|
6975
|
+
"progressAt": string;
|
|
6976
|
+
"planned"?: number;
|
|
6977
|
+
"created"?: number;
|
|
6978
|
+
"missing"?: {
|
|
6979
|
+
"retryable": number;
|
|
6980
|
+
"refused": number;
|
|
6981
|
+
};
|
|
6982
|
+
"notes"?: Array<string>;
|
|
6983
|
+
"settledAt"?: string;
|
|
6984
|
+
} | null;
|
|
6985
|
+
"relaunchable"?: boolean;
|
|
6986
|
+
"arms"?: Array<{
|
|
6987
|
+
"key": string;
|
|
6988
|
+
"position": number;
|
|
6989
|
+
"label": string;
|
|
6990
|
+
"role": "reference" | "peer";
|
|
6991
|
+
"payload": {
|
|
6992
|
+
"tools"?: "default" | "none" | {
|
|
6993
|
+
"tools": Array<string>;
|
|
6994
|
+
};
|
|
6995
|
+
"addons"?: Record<string, unknown>;
|
|
6996
|
+
};
|
|
6997
|
+
"cells"?: Array<{
|
|
6998
|
+
"harness": string;
|
|
6999
|
+
"model": string;
|
|
7000
|
+
"provider"?: string;
|
|
7001
|
+
"params"?: {
|
|
7002
|
+
"temperature"?: number;
|
|
7003
|
+
"maxOutputTokens"?: number;
|
|
7004
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7005
|
+
"thinkingBudgetTokens"?: number;
|
|
7006
|
+
};
|
|
7007
|
+
}>;
|
|
7008
|
+
"skills"?: Array<{
|
|
7009
|
+
"ref": string;
|
|
7010
|
+
"name": string;
|
|
7011
|
+
"changedSinceCreate"?: boolean;
|
|
7012
|
+
}>;
|
|
7013
|
+
"addedAgents"?: Array<{
|
|
7014
|
+
"position": number;
|
|
7015
|
+
"addedAt": string;
|
|
7016
|
+
}>;
|
|
7017
|
+
}>;
|
|
7018
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7019
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7020
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7021
|
+
"modeOptions"?: Record<string, unknown>;
|
|
7022
|
+
};
|
|
7023
|
+
};
|
|
7024
|
+
export type StudyDeleteResponse = unknown;
|
|
7025
|
+
export type StudyAddAgentsRequest = {
|
|
7026
|
+
"group"?: string;
|
|
7027
|
+
"agents": Array<{
|
|
7028
|
+
"harness": string;
|
|
7029
|
+
"model": string;
|
|
7030
|
+
"provider"?: string;
|
|
7031
|
+
"params"?: {
|
|
7032
|
+
"temperature"?: number;
|
|
7033
|
+
"maxOutputTokens"?: number;
|
|
7034
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7035
|
+
"thinkingBudgetTokens"?: number;
|
|
7036
|
+
};
|
|
7037
|
+
}>;
|
|
7038
|
+
"dryRun"?: boolean;
|
|
7039
|
+
};
|
|
7040
|
+
export type StudyAddAgentsResponse = {
|
|
7041
|
+
"studyId": string;
|
|
7042
|
+
"dryRun": boolean;
|
|
7043
|
+
"status": string;
|
|
7044
|
+
"reopened": boolean;
|
|
7045
|
+
"cellPlan": string;
|
|
7046
|
+
"added": Array<{
|
|
7047
|
+
"group": string | null;
|
|
7048
|
+
"harness": string;
|
|
7049
|
+
"model": string;
|
|
7050
|
+
"provider": string | null;
|
|
7051
|
+
"paramsKey": string;
|
|
7052
|
+
"position": number;
|
|
7053
|
+
}>;
|
|
7054
|
+
"alreadyPresent": Array<{
|
|
7055
|
+
"group": string | null;
|
|
7056
|
+
"harness": string;
|
|
7057
|
+
"model": string;
|
|
7058
|
+
"provider": string | null;
|
|
7059
|
+
"paramsKey": string;
|
|
7060
|
+
}>;
|
|
7061
|
+
"matrix": {
|
|
7062
|
+
"before": number;
|
|
7063
|
+
"after": number;
|
|
7064
|
+
};
|
|
7065
|
+
"jobs": {
|
|
7066
|
+
"plannedBefore": number;
|
|
7067
|
+
"plannedAfter": number;
|
|
7068
|
+
"toCreate": number;
|
|
7069
|
+
"toCreateByGroup": Record<string, unknown>;
|
|
7070
|
+
};
|
|
7071
|
+
"contrasts": Array<{
|
|
7072
|
+
"reference": string;
|
|
7073
|
+
"subject": string;
|
|
7074
|
+
"before": {
|
|
7075
|
+
"basis": string;
|
|
7076
|
+
"sharedAxes": Array<string>;
|
|
7077
|
+
"compared": {
|
|
7078
|
+
"reference": number;
|
|
7079
|
+
"subject": number;
|
|
7080
|
+
};
|
|
7081
|
+
};
|
|
7082
|
+
"after": {
|
|
7083
|
+
"basis": string;
|
|
7084
|
+
"sharedAxes": Array<string>;
|
|
7085
|
+
"compared": {
|
|
7086
|
+
"reference": number;
|
|
7087
|
+
"subject": number;
|
|
7088
|
+
};
|
|
7089
|
+
};
|
|
7090
|
+
}>;
|
|
7091
|
+
"quota": {
|
|
7092
|
+
"dimension": string;
|
|
7093
|
+
"remaining": number | null;
|
|
7094
|
+
"allowed": boolean;
|
|
7095
|
+
} | null;
|
|
7096
|
+
"launch": {
|
|
7097
|
+
"attempt": number;
|
|
7098
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
7099
|
+
"startedAt": string;
|
|
7100
|
+
"progressAt": string;
|
|
7101
|
+
"planned"?: number;
|
|
7102
|
+
"created"?: number;
|
|
7103
|
+
"missing"?: {
|
|
7104
|
+
"retryable": number;
|
|
7105
|
+
"refused": number;
|
|
7106
|
+
};
|
|
7107
|
+
"notes"?: Array<string>;
|
|
7108
|
+
"settledAt"?: string;
|
|
7109
|
+
} | null;
|
|
7110
|
+
"relaunch": {
|
|
7111
|
+
"started": boolean;
|
|
7112
|
+
"reason": string | null;
|
|
7113
|
+
};
|
|
7114
|
+
};
|
|
7115
|
+
export type StudyRegenerateSummaryResponse = {
|
|
7116
|
+
"status": "queued";
|
|
7117
|
+
};
|
|
7118
|
+
export type StudyGenerateRequest = {
|
|
7119
|
+
"proposeExpectations"?: boolean;
|
|
7120
|
+
};
|
|
7121
|
+
export type StudyGenerateResponse = {
|
|
7122
|
+
"study": {
|
|
7123
|
+
"id": string;
|
|
7124
|
+
"executiveSummary"?: string | null;
|
|
7125
|
+
"plannedRunCount": number;
|
|
7126
|
+
"organizationId": string;
|
|
7127
|
+
"createdBy": string | null;
|
|
7128
|
+
"name": string;
|
|
7129
|
+
"slug": string;
|
|
7130
|
+
"description": string | null;
|
|
7131
|
+
"skillRef": string | null;
|
|
7132
|
+
"skillSlug": string | null;
|
|
7133
|
+
"skillName": string | null;
|
|
7134
|
+
"skillPromise": string;
|
|
7135
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
7136
|
+
"subjectName": string | null;
|
|
7137
|
+
"subjectLabel": string;
|
|
7138
|
+
"armConfig": {
|
|
7139
|
+
"baseline"?: {
|
|
7140
|
+
"tools"?: "default" | "none" | {
|
|
7141
|
+
"tools": Array<string>;
|
|
7142
|
+
};
|
|
7143
|
+
"addons"?: Record<string, unknown>;
|
|
7144
|
+
};
|
|
7145
|
+
"treatment"?: {
|
|
7146
|
+
"tools"?: "default" | "none" | {
|
|
7147
|
+
"tools": Array<string>;
|
|
7148
|
+
};
|
|
7149
|
+
"addons"?: Record<string, unknown>;
|
|
7150
|
+
};
|
|
7151
|
+
};
|
|
7152
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
7153
|
+
"subjectBackend": "model" | "harness" | null;
|
|
7154
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5116
7155
|
"importedFrom"?: string;
|
|
5117
7156
|
"baselineLabel": string | null;
|
|
5118
7157
|
"treatmentLabel": string | null;
|
|
@@ -5146,6 +7185,7 @@ export type StudyGenerateResponse = {
|
|
|
5146
7185
|
"error": string | null;
|
|
5147
7186
|
"preparation"?: {
|
|
5148
7187
|
"since": string;
|
|
7188
|
+
"progressAt"?: string;
|
|
5149
7189
|
"launchedBy": {
|
|
5150
7190
|
"userId"?: string;
|
|
5151
7191
|
"role"?: string;
|
|
@@ -5154,6 +7194,12 @@ export type StudyGenerateResponse = {
|
|
|
5154
7194
|
"imageKey": string;
|
|
5155
7195
|
"buildId": string;
|
|
5156
7196
|
"stage"?: "image" | "snapshot";
|
|
7197
|
+
"retry"?: {
|
|
7198
|
+
"failedAttempts": number;
|
|
7199
|
+
"maxAttempts": number;
|
|
7200
|
+
"notBefore": string;
|
|
7201
|
+
"lastFailure": string;
|
|
7202
|
+
};
|
|
5157
7203
|
}>;
|
|
5158
7204
|
} | null;
|
|
5159
7205
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5168,11 +7214,65 @@ export type StudyGenerateResponse = {
|
|
|
5168
7214
|
"createdAt": string;
|
|
5169
7215
|
"updatedAt": string;
|
|
5170
7216
|
"liveUntil": string | null;
|
|
7217
|
+
"launch"?: {
|
|
7218
|
+
"attempt": number;
|
|
7219
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
7220
|
+
"startedAt": string;
|
|
7221
|
+
"progressAt": string;
|
|
7222
|
+
"planned"?: number;
|
|
7223
|
+
"created"?: number;
|
|
7224
|
+
"missing"?: {
|
|
7225
|
+
"retryable": number;
|
|
7226
|
+
"refused": number;
|
|
7227
|
+
};
|
|
7228
|
+
"notes"?: Array<string>;
|
|
7229
|
+
"settledAt"?: string;
|
|
7230
|
+
} | null;
|
|
7231
|
+
"relaunchable"?: boolean;
|
|
7232
|
+
"arms"?: Array<{
|
|
7233
|
+
"key": string;
|
|
7234
|
+
"position": number;
|
|
7235
|
+
"label": string;
|
|
7236
|
+
"role": "reference" | "peer";
|
|
7237
|
+
"payload": {
|
|
7238
|
+
"tools"?: "default" | "none" | {
|
|
7239
|
+
"tools": Array<string>;
|
|
7240
|
+
};
|
|
7241
|
+
"addons"?: Record<string, unknown>;
|
|
7242
|
+
};
|
|
7243
|
+
"cells"?: Array<{
|
|
7244
|
+
"harness": string;
|
|
7245
|
+
"model": string;
|
|
7246
|
+
"provider"?: string;
|
|
7247
|
+
"params"?: {
|
|
7248
|
+
"temperature"?: number;
|
|
7249
|
+
"maxOutputTokens"?: number;
|
|
7250
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7251
|
+
"thinkingBudgetTokens"?: number;
|
|
7252
|
+
};
|
|
7253
|
+
}>;
|
|
7254
|
+
"skills"?: Array<{
|
|
7255
|
+
"ref": string;
|
|
7256
|
+
"name": string;
|
|
7257
|
+
"changedSinceCreate"?: boolean;
|
|
7258
|
+
}>;
|
|
7259
|
+
"addedAgents"?: Array<{
|
|
7260
|
+
"position": number;
|
|
7261
|
+
"addedAt": string;
|
|
7262
|
+
}>;
|
|
7263
|
+
}>;
|
|
7264
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7265
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7266
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7267
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5171
7268
|
};
|
|
5172
7269
|
"streamUrl": string;
|
|
5173
7270
|
"warnings"?: Array<{
|
|
5174
7271
|
"harness": string;
|
|
5175
7272
|
"kind": string;
|
|
7273
|
+
} | {
|
|
7274
|
+
"type": "design_aggregate_budget";
|
|
7275
|
+
"detail": string;
|
|
5176
7276
|
}>;
|
|
5177
7277
|
};
|
|
5178
7278
|
export type StudyLaunchResponse = {
|
|
@@ -5206,7 +7306,7 @@ export type StudyLaunchResponse = {
|
|
|
5206
7306
|
"addons"?: Record<string, unknown>;
|
|
5207
7307
|
};
|
|
5208
7308
|
};
|
|
5209
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
7309
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5210
7310
|
"subjectBackend": "model" | "harness" | null;
|
|
5211
7311
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5212
7312
|
"importedFrom"?: string;
|
|
@@ -5242,6 +7342,7 @@ export type StudyLaunchResponse = {
|
|
|
5242
7342
|
"error": string | null;
|
|
5243
7343
|
"preparation"?: {
|
|
5244
7344
|
"since": string;
|
|
7345
|
+
"progressAt"?: string;
|
|
5245
7346
|
"launchedBy": {
|
|
5246
7347
|
"userId"?: string;
|
|
5247
7348
|
"role"?: string;
|
|
@@ -5250,6 +7351,12 @@ export type StudyLaunchResponse = {
|
|
|
5250
7351
|
"imageKey": string;
|
|
5251
7352
|
"buildId": string;
|
|
5252
7353
|
"stage"?: "image" | "snapshot";
|
|
7354
|
+
"retry"?: {
|
|
7355
|
+
"failedAttempts": number;
|
|
7356
|
+
"maxAttempts": number;
|
|
7357
|
+
"notBefore": string;
|
|
7358
|
+
"lastFailure": string;
|
|
7359
|
+
};
|
|
5253
7360
|
}>;
|
|
5254
7361
|
} | null;
|
|
5255
7362
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5264,11 +7371,65 @@ export type StudyLaunchResponse = {
|
|
|
5264
7371
|
"createdAt": string;
|
|
5265
7372
|
"updatedAt": string;
|
|
5266
7373
|
"liveUntil": string | null;
|
|
7374
|
+
"launch"?: {
|
|
7375
|
+
"attempt": number;
|
|
7376
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
7377
|
+
"startedAt": string;
|
|
7378
|
+
"progressAt": string;
|
|
7379
|
+
"planned"?: number;
|
|
7380
|
+
"created"?: number;
|
|
7381
|
+
"missing"?: {
|
|
7382
|
+
"retryable": number;
|
|
7383
|
+
"refused": number;
|
|
7384
|
+
};
|
|
7385
|
+
"notes"?: Array<string>;
|
|
7386
|
+
"settledAt"?: string;
|
|
7387
|
+
} | null;
|
|
7388
|
+
"relaunchable"?: boolean;
|
|
7389
|
+
"arms"?: Array<{
|
|
7390
|
+
"key": string;
|
|
7391
|
+
"position": number;
|
|
7392
|
+
"label": string;
|
|
7393
|
+
"role": "reference" | "peer";
|
|
7394
|
+
"payload": {
|
|
7395
|
+
"tools"?: "default" | "none" | {
|
|
7396
|
+
"tools": Array<string>;
|
|
7397
|
+
};
|
|
7398
|
+
"addons"?: Record<string, unknown>;
|
|
7399
|
+
};
|
|
7400
|
+
"cells"?: Array<{
|
|
7401
|
+
"harness": string;
|
|
7402
|
+
"model": string;
|
|
7403
|
+
"provider"?: string;
|
|
7404
|
+
"params"?: {
|
|
7405
|
+
"temperature"?: number;
|
|
7406
|
+
"maxOutputTokens"?: number;
|
|
7407
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7408
|
+
"thinkingBudgetTokens"?: number;
|
|
7409
|
+
};
|
|
7410
|
+
}>;
|
|
7411
|
+
"skills"?: Array<{
|
|
7412
|
+
"ref": string;
|
|
7413
|
+
"name": string;
|
|
7414
|
+
"changedSinceCreate"?: boolean;
|
|
7415
|
+
}>;
|
|
7416
|
+
"addedAgents"?: Array<{
|
|
7417
|
+
"position": number;
|
|
7418
|
+
"addedAt": string;
|
|
7419
|
+
}>;
|
|
7420
|
+
}>;
|
|
7421
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7422
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7423
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7424
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5267
7425
|
};
|
|
5268
7426
|
"streamUrl": string;
|
|
5269
7427
|
"warnings"?: Array<{
|
|
5270
7428
|
"harness": string;
|
|
5271
7429
|
"kind": string;
|
|
7430
|
+
} | {
|
|
7431
|
+
"type": "design_aggregate_budget";
|
|
7432
|
+
"detail": string;
|
|
5272
7433
|
}>;
|
|
5273
7434
|
};
|
|
5274
7435
|
export type StudyUpdateLensesRequest = {
|
|
@@ -5305,7 +7466,7 @@ export type StudyUpdateLensesResponse = {
|
|
|
5305
7466
|
"addons"?: Record<string, unknown>;
|
|
5306
7467
|
};
|
|
5307
7468
|
};
|
|
5308
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
7469
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5309
7470
|
"subjectBackend": "model" | "harness" | null;
|
|
5310
7471
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5311
7472
|
"importedFrom"?: string;
|
|
@@ -5341,6 +7502,7 @@ export type StudyUpdateLensesResponse = {
|
|
|
5341
7502
|
"error": string | null;
|
|
5342
7503
|
"preparation"?: {
|
|
5343
7504
|
"since": string;
|
|
7505
|
+
"progressAt"?: string;
|
|
5344
7506
|
"launchedBy": {
|
|
5345
7507
|
"userId"?: string;
|
|
5346
7508
|
"role"?: string;
|
|
@@ -5349,6 +7511,12 @@ export type StudyUpdateLensesResponse = {
|
|
|
5349
7511
|
"imageKey": string;
|
|
5350
7512
|
"buildId": string;
|
|
5351
7513
|
"stage"?: "image" | "snapshot";
|
|
7514
|
+
"retry"?: {
|
|
7515
|
+
"failedAttempts": number;
|
|
7516
|
+
"maxAttempts": number;
|
|
7517
|
+
"notBefore": string;
|
|
7518
|
+
"lastFailure": string;
|
|
7519
|
+
};
|
|
5352
7520
|
}>;
|
|
5353
7521
|
} | null;
|
|
5354
7522
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5363,6 +7531,57 @@ export type StudyUpdateLensesResponse = {
|
|
|
5363
7531
|
"createdAt": string;
|
|
5364
7532
|
"updatedAt": string;
|
|
5365
7533
|
"liveUntil": string | null;
|
|
7534
|
+
"launch"?: {
|
|
7535
|
+
"attempt": number;
|
|
7536
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
7537
|
+
"startedAt": string;
|
|
7538
|
+
"progressAt": string;
|
|
7539
|
+
"planned"?: number;
|
|
7540
|
+
"created"?: number;
|
|
7541
|
+
"missing"?: {
|
|
7542
|
+
"retryable": number;
|
|
7543
|
+
"refused": number;
|
|
7544
|
+
};
|
|
7545
|
+
"notes"?: Array<string>;
|
|
7546
|
+
"settledAt"?: string;
|
|
7547
|
+
} | null;
|
|
7548
|
+
"relaunchable"?: boolean;
|
|
7549
|
+
"arms"?: Array<{
|
|
7550
|
+
"key": string;
|
|
7551
|
+
"position": number;
|
|
7552
|
+
"label": string;
|
|
7553
|
+
"role": "reference" | "peer";
|
|
7554
|
+
"payload": {
|
|
7555
|
+
"tools"?: "default" | "none" | {
|
|
7556
|
+
"tools": Array<string>;
|
|
7557
|
+
};
|
|
7558
|
+
"addons"?: Record<string, unknown>;
|
|
7559
|
+
};
|
|
7560
|
+
"cells"?: Array<{
|
|
7561
|
+
"harness": string;
|
|
7562
|
+
"model": string;
|
|
7563
|
+
"provider"?: string;
|
|
7564
|
+
"params"?: {
|
|
7565
|
+
"temperature"?: number;
|
|
7566
|
+
"maxOutputTokens"?: number;
|
|
7567
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7568
|
+
"thinkingBudgetTokens"?: number;
|
|
7569
|
+
};
|
|
7570
|
+
}>;
|
|
7571
|
+
"skills"?: Array<{
|
|
7572
|
+
"ref": string;
|
|
7573
|
+
"name": string;
|
|
7574
|
+
"changedSinceCreate"?: boolean;
|
|
7575
|
+
}>;
|
|
7576
|
+
"addedAgents"?: Array<{
|
|
7577
|
+
"position": number;
|
|
7578
|
+
"addedAt": string;
|
|
7579
|
+
}>;
|
|
7580
|
+
}>;
|
|
7581
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7582
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7583
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7584
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5366
7585
|
};
|
|
5367
7586
|
"warnings": Array<{
|
|
5368
7587
|
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
@@ -5396,7 +7615,7 @@ export type StudyPreviewResponse = {
|
|
|
5396
7615
|
"addons"?: Array<string>;
|
|
5397
7616
|
};
|
|
5398
7617
|
};
|
|
5399
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
7618
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5400
7619
|
"subjectBackend"?: "model" | "harness";
|
|
5401
7620
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5402
7621
|
"baselineLabel": string | null;
|
|
@@ -5443,6 +7662,27 @@ export type StudyPreviewResponse = {
|
|
|
5443
7662
|
"kind": string;
|
|
5444
7663
|
"text": string;
|
|
5445
7664
|
}>;
|
|
7665
|
+
"groupCount"?: number;
|
|
7666
|
+
"group"?: {
|
|
7667
|
+
"label": string;
|
|
7668
|
+
"composition": {
|
|
7669
|
+
"tools"?: "default" | "none" | {
|
|
7670
|
+
"tools": Array<string>;
|
|
7671
|
+
};
|
|
7672
|
+
"addons"?: Array<string>;
|
|
7673
|
+
};
|
|
7674
|
+
};
|
|
7675
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7676
|
+
"agentsAdded"?: Array<{
|
|
7677
|
+
"group": string;
|
|
7678
|
+
"agents": number;
|
|
7679
|
+
"addedAt": string;
|
|
7680
|
+
}>;
|
|
7681
|
+
"groupComposition"?: Array<{
|
|
7682
|
+
"key": string;
|
|
7683
|
+
"label": string;
|
|
7684
|
+
"skills": Array<string>;
|
|
7685
|
+
}>;
|
|
5446
7686
|
};
|
|
5447
7687
|
"tasks": Array<{
|
|
5448
7688
|
"id": string;
|
|
@@ -5459,7 +7699,7 @@ export type StudyPreviewResponse = {
|
|
|
5459
7699
|
"runs": Array<{
|
|
5460
7700
|
"runId": string;
|
|
5461
7701
|
"taskId": string;
|
|
5462
|
-
"arm": "baseline" | "skill";
|
|
7702
|
+
"arm": "baseline" | "skill" | string;
|
|
5463
7703
|
"harness": string;
|
|
5464
7704
|
"model": string;
|
|
5465
7705
|
"provider": string | null;
|
|
@@ -5485,6 +7725,9 @@ export type StudyPreviewResponse = {
|
|
|
5485
7725
|
"pairGradedAt": string | null;
|
|
5486
7726
|
"evalScore": number | null;
|
|
5487
7727
|
"evalGradedAt": string | null;
|
|
7728
|
+
"executionBudgetS"?: number | null;
|
|
7729
|
+
"harnessImageDigest"?: string | null;
|
|
7730
|
+
"subjectEngaged"?: boolean | null;
|
|
5488
7731
|
"runCreatedAt": string | null;
|
|
5489
7732
|
"startedAt": string | null;
|
|
5490
7733
|
"finishedAt": string | null;
|
|
@@ -5522,40 +7765,303 @@ export type StudyPreviewResponse = {
|
|
|
5522
7765
|
"unit": "%";
|
|
5523
7766
|
"relativePct": number | null;
|
|
5524
7767
|
};
|
|
5525
|
-
"improved": boolean;
|
|
7768
|
+
"improved": boolean;
|
|
7769
|
+
}>;
|
|
7770
|
+
"completionRate": {
|
|
7771
|
+
"baseline": number | null;
|
|
7772
|
+
"skill": number | null;
|
|
7773
|
+
};
|
|
7774
|
+
"goalRate": {
|
|
7775
|
+
"baseline": number | null;
|
|
7776
|
+
"skill": number | null;
|
|
7777
|
+
};
|
|
7778
|
+
"pairwise": {
|
|
7779
|
+
"pairs": number;
|
|
7780
|
+
"skillWins": number;
|
|
7781
|
+
"baselineWins": number;
|
|
7782
|
+
"ties": number;
|
|
7783
|
+
"winRate": number | null;
|
|
7784
|
+
};
|
|
7785
|
+
"decision"?: {
|
|
7786
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
7787
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
7788
|
+
"reason": string;
|
|
7789
|
+
"efficiencyDiscounted": boolean;
|
|
7790
|
+
};
|
|
7791
|
+
"leaderboard"?: Array<unknown>;
|
|
7792
|
+
"concentration": {
|
|
7793
|
+
"perCell": Array<{
|
|
7794
|
+
"cell": string;
|
|
7795
|
+
"goalDeltaPp": number | null;
|
|
7796
|
+
"contributionPct": number | null;
|
|
7797
|
+
}>;
|
|
7798
|
+
"heterogeneity": {
|
|
7799
|
+
"dominantCell": string;
|
|
7800
|
+
"contributionPct": number;
|
|
7801
|
+
} | null;
|
|
7802
|
+
};
|
|
7803
|
+
"matrixShape": {
|
|
7804
|
+
"cells": number;
|
|
7805
|
+
"harnesses": number;
|
|
7806
|
+
"models": number;
|
|
7807
|
+
"crossed": boolean;
|
|
7808
|
+
};
|
|
7809
|
+
"flakiness"?: {
|
|
7810
|
+
"measured": boolean;
|
|
7811
|
+
"conditions": number;
|
|
7812
|
+
"unstable": number;
|
|
7813
|
+
"worst": {
|
|
7814
|
+
"cell": string;
|
|
7815
|
+
"taskId": string;
|
|
7816
|
+
"arm": "baseline" | "skill";
|
|
7817
|
+
"variant"?: number;
|
|
7818
|
+
"passRate": number;
|
|
7819
|
+
} | null;
|
|
7820
|
+
};
|
|
7821
|
+
"arms"?: {
|
|
7822
|
+
"baseline": {
|
|
7823
|
+
"runs": number;
|
|
7824
|
+
"completed": number;
|
|
7825
|
+
"infraFailures": number;
|
|
7826
|
+
"completionRate": number | null;
|
|
7827
|
+
"goalRate": number | null;
|
|
7828
|
+
"goalTally": {
|
|
7829
|
+
"achieved": number;
|
|
7830
|
+
"partial": number;
|
|
7831
|
+
"missed": number;
|
|
7832
|
+
};
|
|
7833
|
+
"avgTotalTokens": number | null;
|
|
7834
|
+
"avgOutputTokens": number | null;
|
|
7835
|
+
"avgCostUsd": number | null;
|
|
7836
|
+
"avgDurationMs": number | null;
|
|
7837
|
+
"avgTurns": number | null;
|
|
7838
|
+
"costCoverage": number | null;
|
|
7839
|
+
"avgQualityScore": number | null;
|
|
7840
|
+
"gradedCoverage": number | null;
|
|
7841
|
+
"medianCostUsd": number | null;
|
|
7842
|
+
"medianTotalTokens": number | null;
|
|
7843
|
+
"medianDurationMs": number | null;
|
|
7844
|
+
"p95CostUsd": number | null;
|
|
7845
|
+
"p95TotalTokens": number | null;
|
|
7846
|
+
"p95DurationMs": number | null;
|
|
7847
|
+
"maxCostUsd": number | null;
|
|
7848
|
+
"maxTotalTokens": number | null;
|
|
7849
|
+
"maxDurationMs": number | null;
|
|
7850
|
+
"evalPassRate"?: number | null;
|
|
7851
|
+
"benchmarkPassRate"?: number | null;
|
|
7852
|
+
"adoptionRate"?: number | null;
|
|
7853
|
+
};
|
|
7854
|
+
"skill": {
|
|
7855
|
+
"runs": number;
|
|
7856
|
+
"completed": number;
|
|
7857
|
+
"infraFailures": number;
|
|
7858
|
+
"completionRate": number | null;
|
|
7859
|
+
"goalRate": number | null;
|
|
7860
|
+
"goalTally": {
|
|
7861
|
+
"achieved": number;
|
|
7862
|
+
"partial": number;
|
|
7863
|
+
"missed": number;
|
|
7864
|
+
};
|
|
7865
|
+
"avgTotalTokens": number | null;
|
|
7866
|
+
"avgOutputTokens": number | null;
|
|
7867
|
+
"avgCostUsd": number | null;
|
|
7868
|
+
"avgDurationMs": number | null;
|
|
7869
|
+
"avgTurns": number | null;
|
|
7870
|
+
"costCoverage": number | null;
|
|
7871
|
+
"avgQualityScore": number | null;
|
|
7872
|
+
"gradedCoverage": number | null;
|
|
7873
|
+
"medianCostUsd": number | null;
|
|
7874
|
+
"medianTotalTokens": number | null;
|
|
7875
|
+
"medianDurationMs": number | null;
|
|
7876
|
+
"p95CostUsd": number | null;
|
|
7877
|
+
"p95TotalTokens": number | null;
|
|
7878
|
+
"p95DurationMs": number | null;
|
|
7879
|
+
"maxCostUsd": number | null;
|
|
7880
|
+
"maxTotalTokens": number | null;
|
|
7881
|
+
"maxDurationMs": number | null;
|
|
7882
|
+
"evalPassRate"?: number | null;
|
|
7883
|
+
"benchmarkPassRate"?: number | null;
|
|
7884
|
+
"adoptionRate"?: number | null;
|
|
7885
|
+
};
|
|
7886
|
+
};
|
|
7887
|
+
"totals": {
|
|
7888
|
+
"cells": number;
|
|
7889
|
+
"runs": number;
|
|
7890
|
+
"completed": number;
|
|
7891
|
+
"failed": number;
|
|
7892
|
+
"costUsd": number;
|
|
7893
|
+
"tokens": number;
|
|
7894
|
+
};
|
|
7895
|
+
};
|
|
7896
|
+
"perCell": Array<unknown>;
|
|
7897
|
+
"perHarness": Array<unknown>;
|
|
7898
|
+
"perModel": Array<unknown>;
|
|
7899
|
+
"perTask": Array<unknown>;
|
|
7900
|
+
"webSplit": unknown;
|
|
7901
|
+
"inference": {
|
|
7902
|
+
"columns": Array<{
|
|
7903
|
+
"key": string;
|
|
7904
|
+
"label": string;
|
|
7905
|
+
"family": "rate" | "score" | "magnitude";
|
|
7906
|
+
"betterWhen": "higher" | "lower";
|
|
7907
|
+
}>;
|
|
7908
|
+
"groups": Array<{
|
|
7909
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
7910
|
+
"title": string;
|
|
7911
|
+
"rows": Array<{
|
|
7912
|
+
"key": string;
|
|
7913
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
7914
|
+
"harness": string | null;
|
|
7915
|
+
"model": string | null;
|
|
7916
|
+
"paramsKey"?: string;
|
|
7917
|
+
"runs": number;
|
|
7918
|
+
"cells": Record<string, unknown>;
|
|
7919
|
+
}>;
|
|
7920
|
+
}>;
|
|
7921
|
+
"reference": "baseline" | "field";
|
|
7922
|
+
"draws": number;
|
|
7923
|
+
} | null;
|
|
7924
|
+
"behavior"?: Array<{
|
|
7925
|
+
"harness": string;
|
|
7926
|
+
"model": string;
|
|
7927
|
+
"provider": string | null;
|
|
7928
|
+
"arm": string;
|
|
7929
|
+
"totalRuns": number;
|
|
7930
|
+
"signals": Array<{
|
|
7931
|
+
"id": string;
|
|
7932
|
+
"label": string;
|
|
7933
|
+
"tone": "good" | "bad" | "neutral";
|
|
7934
|
+
"runCount": number;
|
|
7935
|
+
"succeededCount": number;
|
|
7936
|
+
}>;
|
|
7937
|
+
}>;
|
|
7938
|
+
"reference"?: {
|
|
7939
|
+
"cells": Array<{
|
|
7940
|
+
"cell": string;
|
|
7941
|
+
"harness": string;
|
|
7942
|
+
"model": string;
|
|
7943
|
+
"provider": string | null;
|
|
7944
|
+
"paramsKey": string;
|
|
7945
|
+
"label": string;
|
|
7946
|
+
"passRate": {
|
|
7947
|
+
"value": number | null;
|
|
7948
|
+
"ci": Array<unknown> | null;
|
|
7949
|
+
"n": number;
|
|
7950
|
+
};
|
|
7951
|
+
"abstention"?: {
|
|
7952
|
+
"value": number | null;
|
|
7953
|
+
"ci": Array<unknown> | null;
|
|
7954
|
+
"n": number;
|
|
7955
|
+
"parts"?: Array<{
|
|
7956
|
+
"id": string;
|
|
7957
|
+
"label": string;
|
|
7958
|
+
"value": number | null;
|
|
7959
|
+
"n": number;
|
|
7960
|
+
}>;
|
|
7961
|
+
};
|
|
7962
|
+
"tasks": number;
|
|
7963
|
+
"graded": number;
|
|
7964
|
+
"unmeasured": number;
|
|
7965
|
+
"matchers": Array<string>;
|
|
7966
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
5526
7967
|
}>;
|
|
5527
|
-
"
|
|
5528
|
-
"
|
|
5529
|
-
"
|
|
5530
|
-
|
|
5531
|
-
"goalRate": {
|
|
5532
|
-
"baseline": number | null;
|
|
5533
|
-
"skill": number | null;
|
|
7968
|
+
"overall": {
|
|
7969
|
+
"value": number | null;
|
|
7970
|
+
"ci": Array<unknown> | null;
|
|
7971
|
+
"n": number;
|
|
5534
7972
|
};
|
|
5535
|
-
"
|
|
5536
|
-
"
|
|
5537
|
-
"
|
|
5538
|
-
"
|
|
5539
|
-
"
|
|
5540
|
-
|
|
7973
|
+
"abstention"?: {
|
|
7974
|
+
"value": number | null;
|
|
7975
|
+
"ci": Array<unknown> | null;
|
|
7976
|
+
"n": number;
|
|
7977
|
+
"parts"?: Array<{
|
|
7978
|
+
"id": string;
|
|
7979
|
+
"label": string;
|
|
7980
|
+
"value": number | null;
|
|
7981
|
+
"n": number;
|
|
7982
|
+
}>;
|
|
5541
7983
|
};
|
|
5542
|
-
"
|
|
5543
|
-
|
|
5544
|
-
|
|
5545
|
-
|
|
5546
|
-
|
|
7984
|
+
"graded": number;
|
|
7985
|
+
"unmeasured": number;
|
|
7986
|
+
"modelJudged": boolean;
|
|
7987
|
+
};
|
|
7988
|
+
"execution"?: {
|
|
7989
|
+
"graded": number;
|
|
7990
|
+
"measured": number;
|
|
7991
|
+
"passed": number;
|
|
7992
|
+
"passRate": number | null;
|
|
7993
|
+
"byStatus": {
|
|
7994
|
+
"passed": number;
|
|
7995
|
+
"failed": number;
|
|
7996
|
+
"timeout": number;
|
|
7997
|
+
"error": number;
|
|
7998
|
+
"unavailable": number;
|
|
5547
7999
|
};
|
|
5548
|
-
"
|
|
5549
|
-
"
|
|
5550
|
-
|
|
5551
|
-
|
|
5552
|
-
|
|
5553
|
-
|
|
8000
|
+
"languages": Array<string>;
|
|
8001
|
+
"unavailableReasons": Array<string>;
|
|
8002
|
+
};
|
|
8003
|
+
} | {
|
|
8004
|
+
"shape": "multi-arm";
|
|
8005
|
+
"summary": {
|
|
8006
|
+
"shape": "multi-arm";
|
|
8007
|
+
"verdict": "measured" | "inconclusive";
|
|
8008
|
+
"groups": Array<{
|
|
8009
|
+
"key": string;
|
|
8010
|
+
"label": string;
|
|
8011
|
+
"role": "reference" | "peer";
|
|
8012
|
+
"composition": {
|
|
8013
|
+
"tools"?: "default" | "none" | {
|
|
8014
|
+
"tools": Array<string>;
|
|
8015
|
+
};
|
|
8016
|
+
"addons"?: Array<string>;
|
|
8017
|
+
};
|
|
8018
|
+
}>;
|
|
8019
|
+
"ranking": {
|
|
8020
|
+
"plan": "vs_reference" | "all_pairs" | null;
|
|
8021
|
+
"referenceArm": string | null;
|
|
8022
|
+
"decision": {
|
|
8023
|
+
"leader": string | null;
|
|
8024
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8025
|
+
"reason": string;
|
|
8026
|
+
"separatedFrom": Array<string>;
|
|
8027
|
+
"lostTo": Array<{
|
|
8028
|
+
"arm": string;
|
|
8029
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8030
|
+
}>;
|
|
8031
|
+
"clearLeader": boolean;
|
|
8032
|
+
"correctnessLeader": string | null;
|
|
8033
|
+
};
|
|
8034
|
+
"entries": Array<{
|
|
8035
|
+
"arm": string;
|
|
8036
|
+
"runs": number;
|
|
8037
|
+
"completed": number;
|
|
8038
|
+
"completionRate": number | null;
|
|
8039
|
+
"goalRate": number | null;
|
|
8040
|
+
"avgQualityScore": number | null;
|
|
8041
|
+
"evalPassRate": number | null;
|
|
8042
|
+
"benchmarkPassRate"?: number | null;
|
|
8043
|
+
"compositeScore": number | null;
|
|
8044
|
+
"avgTotalTokens": number | null;
|
|
8045
|
+
"avgOutputTokens": number | null;
|
|
8046
|
+
"avgCostUsd": number | null;
|
|
8047
|
+
"avgDurationMs": number | null;
|
|
8048
|
+
"medianCostUsd": number | null;
|
|
8049
|
+
"medianTotalTokens": number | null;
|
|
8050
|
+
"medianDurationMs": number | null;
|
|
8051
|
+
"rank": number;
|
|
5554
8052
|
}>;
|
|
5555
|
-
"
|
|
5556
|
-
"
|
|
5557
|
-
"
|
|
5558
|
-
|
|
8053
|
+
"contrasts": Array<{
|
|
8054
|
+
"contrastId": string;
|
|
8055
|
+
"referenceArm": string;
|
|
8056
|
+
"subjectArm": string;
|
|
8057
|
+
"oriented": boolean;
|
|
8058
|
+
"outcome": "won" | "tie" | "unmeasurable";
|
|
8059
|
+
"winner": string | null;
|
|
8060
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8061
|
+
"explanation": string;
|
|
8062
|
+
}>;
|
|
8063
|
+
"comparisons": number;
|
|
8064
|
+
"headline"?: string;
|
|
5559
8065
|
};
|
|
5560
8066
|
"matrixShape": {
|
|
5561
8067
|
"cells": number;
|
|
@@ -5563,19 +8069,612 @@ export type StudyPreviewResponse = {
|
|
|
5563
8069
|
"models": number;
|
|
5564
8070
|
"crossed": boolean;
|
|
5565
8071
|
};
|
|
5566
|
-
"flakiness"
|
|
8072
|
+
"flakiness": {
|
|
5567
8073
|
"measured": boolean;
|
|
5568
8074
|
"conditions": number;
|
|
5569
8075
|
"unstable": number;
|
|
5570
8076
|
"worst": {
|
|
5571
8077
|
"cell": string;
|
|
5572
8078
|
"taskId": string;
|
|
5573
|
-
"arm":
|
|
8079
|
+
"arm": string;
|
|
5574
8080
|
"variant"?: number;
|
|
5575
8081
|
"passRate": number;
|
|
5576
8082
|
} | null;
|
|
5577
8083
|
};
|
|
5578
|
-
"
|
|
8084
|
+
"totals": {
|
|
8085
|
+
"cells": number;
|
|
8086
|
+
"runs": number;
|
|
8087
|
+
"completed": number;
|
|
8088
|
+
"failed": number;
|
|
8089
|
+
"costUsd": number;
|
|
8090
|
+
"tokens": number;
|
|
8091
|
+
};
|
|
8092
|
+
};
|
|
8093
|
+
"arms": Array<{
|
|
8094
|
+
"key": string;
|
|
8095
|
+
"label": string;
|
|
8096
|
+
"position": number;
|
|
8097
|
+
"role": "reference" | "peer";
|
|
8098
|
+
"composition": {
|
|
8099
|
+
"tools"?: "default" | "none" | {
|
|
8100
|
+
"tools": Array<string>;
|
|
8101
|
+
};
|
|
8102
|
+
"addons"?: Array<string>;
|
|
8103
|
+
};
|
|
8104
|
+
"agg": {
|
|
8105
|
+
"runs": number;
|
|
8106
|
+
"completed": number;
|
|
8107
|
+
"infraFailures": number;
|
|
8108
|
+
"completionRate": number | null;
|
|
8109
|
+
"goalRate": number | null;
|
|
8110
|
+
"goalTally": {
|
|
8111
|
+
"achieved": number;
|
|
8112
|
+
"partial": number;
|
|
8113
|
+
"missed": number;
|
|
8114
|
+
};
|
|
8115
|
+
"avgTotalTokens": number | null;
|
|
8116
|
+
"avgOutputTokens": number | null;
|
|
8117
|
+
"avgCostUsd": number | null;
|
|
8118
|
+
"avgDurationMs": number | null;
|
|
8119
|
+
"avgTurns": number | null;
|
|
8120
|
+
"costCoverage": number | null;
|
|
8121
|
+
"avgQualityScore": number | null;
|
|
8122
|
+
"gradedCoverage": number | null;
|
|
8123
|
+
"medianCostUsd": number | null;
|
|
8124
|
+
"medianTotalTokens": number | null;
|
|
8125
|
+
"medianDurationMs": number | null;
|
|
8126
|
+
"p95CostUsd": number | null;
|
|
8127
|
+
"p95TotalTokens": number | null;
|
|
8128
|
+
"p95DurationMs": number | null;
|
|
8129
|
+
"maxCostUsd": number | null;
|
|
8130
|
+
"maxTotalTokens": number | null;
|
|
8131
|
+
"maxDurationMs": number | null;
|
|
8132
|
+
"evalPassRate"?: number | null;
|
|
8133
|
+
"benchmarkPassRate"?: number | null;
|
|
8134
|
+
"adoptionRate"?: number | null;
|
|
8135
|
+
};
|
|
8136
|
+
"perCell": Array<{
|
|
8137
|
+
"harness": string;
|
|
8138
|
+
"model": string;
|
|
8139
|
+
"provider": string | null;
|
|
8140
|
+
"paramsKey"?: string;
|
|
8141
|
+
"label"?: string;
|
|
8142
|
+
"agg": {
|
|
8143
|
+
"runs": number;
|
|
8144
|
+
"completed": number;
|
|
8145
|
+
"infraFailures": number;
|
|
8146
|
+
"completionRate": number | null;
|
|
8147
|
+
"goalRate": number | null;
|
|
8148
|
+
"goalTally": {
|
|
8149
|
+
"achieved": number;
|
|
8150
|
+
"partial": number;
|
|
8151
|
+
"missed": number;
|
|
8152
|
+
};
|
|
8153
|
+
"avgTotalTokens": number | null;
|
|
8154
|
+
"avgOutputTokens": number | null;
|
|
8155
|
+
"avgCostUsd": number | null;
|
|
8156
|
+
"avgDurationMs": number | null;
|
|
8157
|
+
"avgTurns": number | null;
|
|
8158
|
+
"costCoverage": number | null;
|
|
8159
|
+
"avgQualityScore": number | null;
|
|
8160
|
+
"gradedCoverage": number | null;
|
|
8161
|
+
"medianCostUsd": number | null;
|
|
8162
|
+
"medianTotalTokens": number | null;
|
|
8163
|
+
"medianDurationMs": number | null;
|
|
8164
|
+
"p95CostUsd": number | null;
|
|
8165
|
+
"p95TotalTokens": number | null;
|
|
8166
|
+
"p95DurationMs": number | null;
|
|
8167
|
+
"maxCostUsd": number | null;
|
|
8168
|
+
"maxTotalTokens": number | null;
|
|
8169
|
+
"maxDurationMs": number | null;
|
|
8170
|
+
"evalPassRate"?: number | null;
|
|
8171
|
+
"benchmarkPassRate"?: number | null;
|
|
8172
|
+
"adoptionRate"?: number | null;
|
|
8173
|
+
};
|
|
8174
|
+
}>;
|
|
8175
|
+
"reference"?: {
|
|
8176
|
+
"cells": Array<{
|
|
8177
|
+
"cell": string;
|
|
8178
|
+
"harness": string;
|
|
8179
|
+
"model": string;
|
|
8180
|
+
"provider": string | null;
|
|
8181
|
+
"paramsKey": string;
|
|
8182
|
+
"label": string;
|
|
8183
|
+
"passRate": {
|
|
8184
|
+
"value": number | null;
|
|
8185
|
+
"ci": Array<unknown> | null;
|
|
8186
|
+
"n": number;
|
|
8187
|
+
};
|
|
8188
|
+
"abstention"?: {
|
|
8189
|
+
"value": number | null;
|
|
8190
|
+
"ci": Array<unknown> | null;
|
|
8191
|
+
"n": number;
|
|
8192
|
+
"parts"?: Array<{
|
|
8193
|
+
"id": string;
|
|
8194
|
+
"label": string;
|
|
8195
|
+
"value": number | null;
|
|
8196
|
+
"n": number;
|
|
8197
|
+
}>;
|
|
8198
|
+
};
|
|
8199
|
+
"tasks": number;
|
|
8200
|
+
"graded": number;
|
|
8201
|
+
"unmeasured": number;
|
|
8202
|
+
"matchers": Array<string>;
|
|
8203
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
8204
|
+
}>;
|
|
8205
|
+
"overall": {
|
|
8206
|
+
"value": number | null;
|
|
8207
|
+
"ci": Array<unknown> | null;
|
|
8208
|
+
"n": number;
|
|
8209
|
+
};
|
|
8210
|
+
"abstention"?: {
|
|
8211
|
+
"value": number | null;
|
|
8212
|
+
"ci": Array<unknown> | null;
|
|
8213
|
+
"n": number;
|
|
8214
|
+
"parts"?: Array<{
|
|
8215
|
+
"id": string;
|
|
8216
|
+
"label": string;
|
|
8217
|
+
"value": number | null;
|
|
8218
|
+
"n": number;
|
|
8219
|
+
}>;
|
|
8220
|
+
};
|
|
8221
|
+
"graded": number;
|
|
8222
|
+
"unmeasured": number;
|
|
8223
|
+
"modelJudged": boolean;
|
|
8224
|
+
};
|
|
8225
|
+
}>;
|
|
8226
|
+
"contrasts": Array<{
|
|
8227
|
+
"contrastId": string;
|
|
8228
|
+
"referenceArm": string;
|
|
8229
|
+
"subjectArm": string;
|
|
8230
|
+
"oriented": boolean;
|
|
8231
|
+
"basis"?: {
|
|
8232
|
+
"id": "matched" | "whole_setup";
|
|
8233
|
+
"compared": {
|
|
8234
|
+
"reference": Array<{
|
|
8235
|
+
"harness": string;
|
|
8236
|
+
"model": string;
|
|
8237
|
+
"provider"?: string;
|
|
8238
|
+
"params"?: Record<string, unknown>;
|
|
8239
|
+
}>;
|
|
8240
|
+
"subject": Array<{
|
|
8241
|
+
"harness": string;
|
|
8242
|
+
"model": string;
|
|
8243
|
+
"provider"?: string;
|
|
8244
|
+
"params"?: Record<string, unknown>;
|
|
8245
|
+
}>;
|
|
8246
|
+
};
|
|
8247
|
+
"excluded": {
|
|
8248
|
+
"reference": Array<{
|
|
8249
|
+
"harness": string;
|
|
8250
|
+
"model": string;
|
|
8251
|
+
"provider"?: string;
|
|
8252
|
+
"params"?: Record<string, unknown>;
|
|
8253
|
+
}>;
|
|
8254
|
+
"subject": Array<{
|
|
8255
|
+
"harness": string;
|
|
8256
|
+
"model": string;
|
|
8257
|
+
"provider"?: string;
|
|
8258
|
+
"params"?: Record<string, unknown>;
|
|
8259
|
+
}>;
|
|
8260
|
+
};
|
|
8261
|
+
"differsIn": Array<string>;
|
|
8262
|
+
"licensedFacet": string | null;
|
|
8263
|
+
"efficiency": "comparable" | "incomparable";
|
|
8264
|
+
"pairedTasks": number | null;
|
|
8265
|
+
"dropped": {
|
|
8266
|
+
"reference": number;
|
|
8267
|
+
"subject": number;
|
|
8268
|
+
} | null;
|
|
8269
|
+
};
|
|
8270
|
+
"magnitudes": "relative";
|
|
8271
|
+
"summary": {
|
|
8272
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
8273
|
+
"headline": Array<{
|
|
8274
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
8275
|
+
"label": string;
|
|
8276
|
+
"direction": "down" | "up";
|
|
8277
|
+
"baselineAvg": number | null;
|
|
8278
|
+
"skillAvg": number | null;
|
|
8279
|
+
"deltaPct": number | null;
|
|
8280
|
+
"delta"?: {
|
|
8281
|
+
"unit": "pp";
|
|
8282
|
+
"absolute": number | null;
|
|
8283
|
+
"relativePct": number | null;
|
|
8284
|
+
} | {
|
|
8285
|
+
"unit": "pts";
|
|
8286
|
+
"absolute": number | null;
|
|
8287
|
+
"relativePct"?: number | null;
|
|
8288
|
+
} | {
|
|
8289
|
+
"unit": "%";
|
|
8290
|
+
"relativePct": number | null;
|
|
8291
|
+
};
|
|
8292
|
+
"improved": boolean;
|
|
8293
|
+
}>;
|
|
8294
|
+
"completionRate": {
|
|
8295
|
+
"baseline": number | null;
|
|
8296
|
+
"skill": number | null;
|
|
8297
|
+
};
|
|
8298
|
+
"goalRate": {
|
|
8299
|
+
"baseline": number | null;
|
|
8300
|
+
"skill": number | null;
|
|
8301
|
+
};
|
|
8302
|
+
"pairwise": {
|
|
8303
|
+
"pairs": number;
|
|
8304
|
+
"skillWins": number;
|
|
8305
|
+
"baselineWins": number;
|
|
8306
|
+
"ties": number;
|
|
8307
|
+
"winRate": number | null;
|
|
8308
|
+
};
|
|
8309
|
+
"decision"?: {
|
|
8310
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
8311
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8312
|
+
"reason": string;
|
|
8313
|
+
"efficiencyDiscounted": boolean;
|
|
8314
|
+
};
|
|
8315
|
+
"concentration": {
|
|
8316
|
+
"perCell": Array<{
|
|
8317
|
+
"cell": string;
|
|
8318
|
+
"goalDeltaPp": number | null;
|
|
8319
|
+
"contributionPct": number | null;
|
|
8320
|
+
}>;
|
|
8321
|
+
"heterogeneity": {
|
|
8322
|
+
"dominantCell": string;
|
|
8323
|
+
"contributionPct": number;
|
|
8324
|
+
} | null;
|
|
8325
|
+
};
|
|
8326
|
+
"matrixShape": {
|
|
8327
|
+
"cells": number;
|
|
8328
|
+
"harnesses": number;
|
|
8329
|
+
"models": number;
|
|
8330
|
+
"crossed": boolean;
|
|
8331
|
+
};
|
|
8332
|
+
"flakiness"?: {
|
|
8333
|
+
"measured": boolean;
|
|
8334
|
+
"conditions": number;
|
|
8335
|
+
"unstable": number;
|
|
8336
|
+
"worst": {
|
|
8337
|
+
"cell": string;
|
|
8338
|
+
"taskId": string;
|
|
8339
|
+
"arm": "baseline" | "skill";
|
|
8340
|
+
"variant"?: number;
|
|
8341
|
+
"passRate": number;
|
|
8342
|
+
} | null;
|
|
8343
|
+
};
|
|
8344
|
+
"arms"?: {
|
|
8345
|
+
"baseline": {
|
|
8346
|
+
"runs": number;
|
|
8347
|
+
"completed": number;
|
|
8348
|
+
"infraFailures": number;
|
|
8349
|
+
"completionRate": number | null;
|
|
8350
|
+
"goalRate": number | null;
|
|
8351
|
+
"goalTally": {
|
|
8352
|
+
"achieved": number;
|
|
8353
|
+
"partial": number;
|
|
8354
|
+
"missed": number;
|
|
8355
|
+
};
|
|
8356
|
+
"avgTotalTokens": number | null;
|
|
8357
|
+
"avgOutputTokens": number | null;
|
|
8358
|
+
"avgCostUsd": number | null;
|
|
8359
|
+
"avgDurationMs": number | null;
|
|
8360
|
+
"avgTurns": number | null;
|
|
8361
|
+
"costCoverage": number | null;
|
|
8362
|
+
"avgQualityScore": number | null;
|
|
8363
|
+
"gradedCoverage": number | null;
|
|
8364
|
+
"medianCostUsd": number | null;
|
|
8365
|
+
"medianTotalTokens": number | null;
|
|
8366
|
+
"medianDurationMs": number | null;
|
|
8367
|
+
"p95CostUsd": number | null;
|
|
8368
|
+
"p95TotalTokens": number | null;
|
|
8369
|
+
"p95DurationMs": number | null;
|
|
8370
|
+
"maxCostUsd": number | null;
|
|
8371
|
+
"maxTotalTokens": number | null;
|
|
8372
|
+
"maxDurationMs": number | null;
|
|
8373
|
+
"evalPassRate"?: number | null;
|
|
8374
|
+
"benchmarkPassRate"?: number | null;
|
|
8375
|
+
"adoptionRate"?: number | null;
|
|
8376
|
+
};
|
|
8377
|
+
"skill": {
|
|
8378
|
+
"runs": number;
|
|
8379
|
+
"completed": number;
|
|
8380
|
+
"infraFailures": number;
|
|
8381
|
+
"completionRate": number | null;
|
|
8382
|
+
"goalRate": number | null;
|
|
8383
|
+
"goalTally": {
|
|
8384
|
+
"achieved": number;
|
|
8385
|
+
"partial": number;
|
|
8386
|
+
"missed": number;
|
|
8387
|
+
};
|
|
8388
|
+
"avgTotalTokens": number | null;
|
|
8389
|
+
"avgOutputTokens": number | null;
|
|
8390
|
+
"avgCostUsd": number | null;
|
|
8391
|
+
"avgDurationMs": number | null;
|
|
8392
|
+
"avgTurns": number | null;
|
|
8393
|
+
"costCoverage": number | null;
|
|
8394
|
+
"avgQualityScore": number | null;
|
|
8395
|
+
"gradedCoverage": number | null;
|
|
8396
|
+
"medianCostUsd": number | null;
|
|
8397
|
+
"medianTotalTokens": number | null;
|
|
8398
|
+
"medianDurationMs": number | null;
|
|
8399
|
+
"p95CostUsd": number | null;
|
|
8400
|
+
"p95TotalTokens": number | null;
|
|
8401
|
+
"p95DurationMs": number | null;
|
|
8402
|
+
"maxCostUsd": number | null;
|
|
8403
|
+
"maxTotalTokens": number | null;
|
|
8404
|
+
"maxDurationMs": number | null;
|
|
8405
|
+
"evalPassRate"?: number | null;
|
|
8406
|
+
"benchmarkPassRate"?: number | null;
|
|
8407
|
+
"adoptionRate"?: number | null;
|
|
8408
|
+
};
|
|
8409
|
+
};
|
|
8410
|
+
"totals": {
|
|
8411
|
+
"cells": number;
|
|
8412
|
+
"runs": number;
|
|
8413
|
+
"completed": number;
|
|
8414
|
+
"failed": number;
|
|
8415
|
+
"costUsd": number;
|
|
8416
|
+
"tokens": number;
|
|
8417
|
+
};
|
|
8418
|
+
};
|
|
8419
|
+
"perCell": Array<{
|
|
8420
|
+
"harness": string;
|
|
8421
|
+
"model": string;
|
|
8422
|
+
"provider": string | null;
|
|
8423
|
+
"paramsKey"?: string;
|
|
8424
|
+
"label"?: string;
|
|
8425
|
+
"baseline": {
|
|
8426
|
+
"runs": number;
|
|
8427
|
+
"completed": number;
|
|
8428
|
+
"infraFailures": number;
|
|
8429
|
+
"completionRate": number | null;
|
|
8430
|
+
"goalRate": number | null;
|
|
8431
|
+
"goalTally": {
|
|
8432
|
+
"achieved": number;
|
|
8433
|
+
"partial": number;
|
|
8434
|
+
"missed": number;
|
|
8435
|
+
};
|
|
8436
|
+
"avgTotalTokens": number | null;
|
|
8437
|
+
"avgOutputTokens": number | null;
|
|
8438
|
+
"avgCostUsd": number | null;
|
|
8439
|
+
"avgDurationMs": number | null;
|
|
8440
|
+
"avgTurns": number | null;
|
|
8441
|
+
"costCoverage": number | null;
|
|
8442
|
+
"avgQualityScore": number | null;
|
|
8443
|
+
"gradedCoverage": number | null;
|
|
8444
|
+
"medianCostUsd": number | null;
|
|
8445
|
+
"medianTotalTokens": number | null;
|
|
8446
|
+
"medianDurationMs": number | null;
|
|
8447
|
+
"p95CostUsd": number | null;
|
|
8448
|
+
"p95TotalTokens": number | null;
|
|
8449
|
+
"p95DurationMs": number | null;
|
|
8450
|
+
"maxCostUsd": number | null;
|
|
8451
|
+
"maxTotalTokens": number | null;
|
|
8452
|
+
"maxDurationMs": number | null;
|
|
8453
|
+
"evalPassRate"?: number | null;
|
|
8454
|
+
"benchmarkPassRate"?: number | null;
|
|
8455
|
+
"adoptionRate"?: number | null;
|
|
8456
|
+
};
|
|
8457
|
+
"skill": {
|
|
8458
|
+
"runs": number;
|
|
8459
|
+
"completed": number;
|
|
8460
|
+
"infraFailures": number;
|
|
8461
|
+
"completionRate": number | null;
|
|
8462
|
+
"goalRate": number | null;
|
|
8463
|
+
"goalTally": {
|
|
8464
|
+
"achieved": number;
|
|
8465
|
+
"partial": number;
|
|
8466
|
+
"missed": number;
|
|
8467
|
+
};
|
|
8468
|
+
"avgTotalTokens": number | null;
|
|
8469
|
+
"avgOutputTokens": number | null;
|
|
8470
|
+
"avgCostUsd": number | null;
|
|
8471
|
+
"avgDurationMs": number | null;
|
|
8472
|
+
"avgTurns": number | null;
|
|
8473
|
+
"costCoverage": number | null;
|
|
8474
|
+
"avgQualityScore": number | null;
|
|
8475
|
+
"gradedCoverage": number | null;
|
|
8476
|
+
"medianCostUsd": number | null;
|
|
8477
|
+
"medianTotalTokens": number | null;
|
|
8478
|
+
"medianDurationMs": number | null;
|
|
8479
|
+
"p95CostUsd": number | null;
|
|
8480
|
+
"p95TotalTokens": number | null;
|
|
8481
|
+
"p95DurationMs": number | null;
|
|
8482
|
+
"maxCostUsd": number | null;
|
|
8483
|
+
"maxTotalTokens": number | null;
|
|
8484
|
+
"maxDurationMs": number | null;
|
|
8485
|
+
"evalPassRate"?: number | null;
|
|
8486
|
+
"benchmarkPassRate"?: number | null;
|
|
8487
|
+
"adoptionRate"?: number | null;
|
|
8488
|
+
};
|
|
8489
|
+
"deltas": {
|
|
8490
|
+
"totalTokensPct": number | null;
|
|
8491
|
+
"outputTokensPct": number | null;
|
|
8492
|
+
"costUsdPct": number | null;
|
|
8493
|
+
"durationMsPct": number | null;
|
|
8494
|
+
"numTurnsPct": number | null;
|
|
8495
|
+
"completionRatePp": number | null;
|
|
8496
|
+
"goalRatePp": number | null;
|
|
8497
|
+
"qualityScorePts": number | null;
|
|
8498
|
+
"evalPassRatePp"?: number | null;
|
|
8499
|
+
};
|
|
8500
|
+
}>;
|
|
8501
|
+
} | {
|
|
8502
|
+
"contrastId": string;
|
|
8503
|
+
"referenceArm": string;
|
|
8504
|
+
"subjectArm": string;
|
|
8505
|
+
"oriented": boolean;
|
|
8506
|
+
"basis"?: {
|
|
8507
|
+
"id": "matched" | "whole_setup";
|
|
8508
|
+
"compared": {
|
|
8509
|
+
"reference": Array<{
|
|
8510
|
+
"harness": string;
|
|
8511
|
+
"model": string;
|
|
8512
|
+
"provider"?: string;
|
|
8513
|
+
"params"?: Record<string, unknown>;
|
|
8514
|
+
}>;
|
|
8515
|
+
"subject": Array<{
|
|
8516
|
+
"harness": string;
|
|
8517
|
+
"model": string;
|
|
8518
|
+
"provider"?: string;
|
|
8519
|
+
"params"?: Record<string, unknown>;
|
|
8520
|
+
}>;
|
|
8521
|
+
};
|
|
8522
|
+
"excluded": {
|
|
8523
|
+
"reference": Array<{
|
|
8524
|
+
"harness": string;
|
|
8525
|
+
"model": string;
|
|
8526
|
+
"provider"?: string;
|
|
8527
|
+
"params"?: Record<string, unknown>;
|
|
8528
|
+
}>;
|
|
8529
|
+
"subject": Array<{
|
|
8530
|
+
"harness": string;
|
|
8531
|
+
"model": string;
|
|
8532
|
+
"provider"?: string;
|
|
8533
|
+
"params"?: Record<string, unknown>;
|
|
8534
|
+
}>;
|
|
8535
|
+
};
|
|
8536
|
+
"differsIn": Array<string>;
|
|
8537
|
+
"licensedFacet": string | null;
|
|
8538
|
+
"efficiency": "comparable" | "incomparable";
|
|
8539
|
+
"pairedTasks": number | null;
|
|
8540
|
+
"dropped": {
|
|
8541
|
+
"reference": number;
|
|
8542
|
+
"subject": number;
|
|
8543
|
+
} | null;
|
|
8544
|
+
};
|
|
8545
|
+
"magnitudes": "absolute";
|
|
8546
|
+
"summary": {
|
|
8547
|
+
"completionRate": {
|
|
8548
|
+
"baseline": number | null;
|
|
8549
|
+
"skill": number | null;
|
|
8550
|
+
};
|
|
8551
|
+
"goalRate": {
|
|
8552
|
+
"baseline": number | null;
|
|
8553
|
+
"skill": number | null;
|
|
8554
|
+
};
|
|
8555
|
+
"pairwise": {
|
|
8556
|
+
"pairs": number;
|
|
8557
|
+
"skillWins": number;
|
|
8558
|
+
"baselineWins": number;
|
|
8559
|
+
"ties": number;
|
|
8560
|
+
"winRate": number | null;
|
|
8561
|
+
};
|
|
8562
|
+
"decision"?: {
|
|
8563
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
8564
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8565
|
+
"reason": string;
|
|
8566
|
+
"efficiencyDiscounted": boolean;
|
|
8567
|
+
};
|
|
8568
|
+
"concentration": {
|
|
8569
|
+
"perCell": Array<{
|
|
8570
|
+
"cell": string;
|
|
8571
|
+
"goalDeltaPp": number | null;
|
|
8572
|
+
"contributionPct": number | null;
|
|
8573
|
+
}>;
|
|
8574
|
+
"heterogeneity": {
|
|
8575
|
+
"dominantCell": string;
|
|
8576
|
+
"contributionPct": number;
|
|
8577
|
+
} | null;
|
|
8578
|
+
};
|
|
8579
|
+
"matrixShape": {
|
|
8580
|
+
"cells": number;
|
|
8581
|
+
"harnesses": number;
|
|
8582
|
+
"models": number;
|
|
8583
|
+
"crossed": boolean;
|
|
8584
|
+
};
|
|
8585
|
+
"flakiness"?: {
|
|
8586
|
+
"measured": boolean;
|
|
8587
|
+
"conditions": number;
|
|
8588
|
+
"unstable": number;
|
|
8589
|
+
"worst": {
|
|
8590
|
+
"cell": string;
|
|
8591
|
+
"taskId": string;
|
|
8592
|
+
"arm": "baseline" | "skill";
|
|
8593
|
+
"variant"?: number;
|
|
8594
|
+
"passRate": number;
|
|
8595
|
+
} | null;
|
|
8596
|
+
};
|
|
8597
|
+
"arms"?: {
|
|
8598
|
+
"baseline": {
|
|
8599
|
+
"runs": number;
|
|
8600
|
+
"completed": number;
|
|
8601
|
+
"infraFailures": number;
|
|
8602
|
+
"completionRate": number | null;
|
|
8603
|
+
"goalRate": number | null;
|
|
8604
|
+
"goalTally": {
|
|
8605
|
+
"achieved": number;
|
|
8606
|
+
"partial": number;
|
|
8607
|
+
"missed": number;
|
|
8608
|
+
};
|
|
8609
|
+
"avgTotalTokens": number | null;
|
|
8610
|
+
"avgOutputTokens": number | null;
|
|
8611
|
+
"avgCostUsd": number | null;
|
|
8612
|
+
"avgDurationMs": number | null;
|
|
8613
|
+
"avgTurns": number | null;
|
|
8614
|
+
"costCoverage": number | null;
|
|
8615
|
+
"avgQualityScore": number | null;
|
|
8616
|
+
"gradedCoverage": number | null;
|
|
8617
|
+
"medianCostUsd": number | null;
|
|
8618
|
+
"medianTotalTokens": number | null;
|
|
8619
|
+
"medianDurationMs": number | null;
|
|
8620
|
+
"p95CostUsd": number | null;
|
|
8621
|
+
"p95TotalTokens": number | null;
|
|
8622
|
+
"p95DurationMs": number | null;
|
|
8623
|
+
"maxCostUsd": number | null;
|
|
8624
|
+
"maxTotalTokens": number | null;
|
|
8625
|
+
"maxDurationMs": number | null;
|
|
8626
|
+
"evalPassRate"?: number | null;
|
|
8627
|
+
"benchmarkPassRate"?: number | null;
|
|
8628
|
+
"adoptionRate"?: number | null;
|
|
8629
|
+
};
|
|
8630
|
+
"skill": {
|
|
8631
|
+
"runs": number;
|
|
8632
|
+
"completed": number;
|
|
8633
|
+
"infraFailures": number;
|
|
8634
|
+
"completionRate": number | null;
|
|
8635
|
+
"goalRate": number | null;
|
|
8636
|
+
"goalTally": {
|
|
8637
|
+
"achieved": number;
|
|
8638
|
+
"partial": number;
|
|
8639
|
+
"missed": number;
|
|
8640
|
+
};
|
|
8641
|
+
"avgTotalTokens": number | null;
|
|
8642
|
+
"avgOutputTokens": number | null;
|
|
8643
|
+
"avgCostUsd": number | null;
|
|
8644
|
+
"avgDurationMs": number | null;
|
|
8645
|
+
"avgTurns": number | null;
|
|
8646
|
+
"costCoverage": number | null;
|
|
8647
|
+
"avgQualityScore": number | null;
|
|
8648
|
+
"gradedCoverage": number | null;
|
|
8649
|
+
"medianCostUsd": number | null;
|
|
8650
|
+
"medianTotalTokens": number | null;
|
|
8651
|
+
"medianDurationMs": number | null;
|
|
8652
|
+
"p95CostUsd": number | null;
|
|
8653
|
+
"p95TotalTokens": number | null;
|
|
8654
|
+
"p95DurationMs": number | null;
|
|
8655
|
+
"maxCostUsd": number | null;
|
|
8656
|
+
"maxTotalTokens": number | null;
|
|
8657
|
+
"maxDurationMs": number | null;
|
|
8658
|
+
"evalPassRate"?: number | null;
|
|
8659
|
+
"benchmarkPassRate"?: number | null;
|
|
8660
|
+
"adoptionRate"?: number | null;
|
|
8661
|
+
};
|
|
8662
|
+
};
|
|
8663
|
+
"totals": {
|
|
8664
|
+
"cells": number;
|
|
8665
|
+
"runs": number;
|
|
8666
|
+
"completed": number;
|
|
8667
|
+
"failed": number;
|
|
8668
|
+
"costUsd": number;
|
|
8669
|
+
"tokens": number;
|
|
8670
|
+
};
|
|
8671
|
+
};
|
|
8672
|
+
"perCell": Array<{
|
|
8673
|
+
"harness": string;
|
|
8674
|
+
"model": string;
|
|
8675
|
+
"provider": string | null;
|
|
8676
|
+
"paramsKey"?: string;
|
|
8677
|
+
"label"?: string;
|
|
5579
8678
|
"baseline": {
|
|
5580
8679
|
"runs": number;
|
|
5581
8680
|
"completed": number;
|
|
@@ -5640,44 +8739,73 @@ export type StudyPreviewResponse = {
|
|
|
5640
8739
|
"benchmarkPassRate"?: number | null;
|
|
5641
8740
|
"adoptionRate"?: number | null;
|
|
5642
8741
|
};
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
"costUsd": number;
|
|
5650
|
-
"tokens": number;
|
|
5651
|
-
};
|
|
5652
|
-
};
|
|
5653
|
-
"perCell": Array<unknown>;
|
|
5654
|
-
"perHarness": Array<unknown>;
|
|
5655
|
-
"perModel": Array<unknown>;
|
|
5656
|
-
"perTask": Array<unknown>;
|
|
5657
|
-
"webSplit": unknown;
|
|
5658
|
-
"inference": {
|
|
5659
|
-
"columns": Array<{
|
|
5660
|
-
"key": string;
|
|
5661
|
-
"label": string;
|
|
5662
|
-
"family": "rate" | "score" | "magnitude";
|
|
5663
|
-
"betterWhen": "higher" | "lower";
|
|
5664
|
-
}>;
|
|
5665
|
-
"groups": Array<{
|
|
5666
|
-
"kind": "cell" | "harness" | "model" | "field";
|
|
5667
|
-
"title": string;
|
|
5668
|
-
"rows": Array<{
|
|
5669
|
-
"key": string;
|
|
5670
|
-
"kind": "cell" | "harness" | "model" | "field";
|
|
5671
|
-
"harness": string | null;
|
|
5672
|
-
"model": string | null;
|
|
5673
|
-
"paramsKey"?: string;
|
|
5674
|
-
"runs": number;
|
|
5675
|
-
"cells": Record<string, unknown>;
|
|
5676
|
-
}>;
|
|
8742
|
+
"deltas": {
|
|
8743
|
+
"completionRatePp": number | null;
|
|
8744
|
+
"goalRatePp": number | null;
|
|
8745
|
+
"qualityScorePts": number | null;
|
|
8746
|
+
"evalPassRatePp"?: number | null;
|
|
8747
|
+
};
|
|
5677
8748
|
}>;
|
|
5678
|
-
|
|
5679
|
-
|
|
5680
|
-
|
|
8749
|
+
}>;
|
|
8750
|
+
"board": Array<{
|
|
8751
|
+
"harness": string;
|
|
8752
|
+
"model": string;
|
|
8753
|
+
"provider": string | null;
|
|
8754
|
+
"paramsKey"?: string;
|
|
8755
|
+
"label"?: string;
|
|
8756
|
+
"arm": string;
|
|
8757
|
+
"runs": number;
|
|
8758
|
+
"completed": number;
|
|
8759
|
+
"completionRate": number | null;
|
|
8760
|
+
"goalRate": number | null;
|
|
8761
|
+
"avgQualityScore": number | null;
|
|
8762
|
+
"evalPassRate": number | null;
|
|
8763
|
+
"benchmarkPassRate"?: number | null;
|
|
8764
|
+
"compositeScore": number | null;
|
|
8765
|
+
"avgTotalTokens": number | null;
|
|
8766
|
+
"avgOutputTokens": number | null;
|
|
8767
|
+
"avgCostUsd": number | null;
|
|
8768
|
+
"avgDurationMs": number | null;
|
|
8769
|
+
"medianCostUsd": number | null;
|
|
8770
|
+
"medianTotalTokens": number | null;
|
|
8771
|
+
"medianDurationMs": number | null;
|
|
8772
|
+
"rank": number;
|
|
8773
|
+
}>;
|
|
8774
|
+
"setupsBoard"?: Array<{
|
|
8775
|
+
"harness": string;
|
|
8776
|
+
"model": string;
|
|
8777
|
+
"provider": string | null;
|
|
8778
|
+
"paramsKey"?: string;
|
|
8779
|
+
"label"?: string;
|
|
8780
|
+
"arm": string;
|
|
8781
|
+
"runs": number;
|
|
8782
|
+
"completed": number;
|
|
8783
|
+
"completionRate": number | null;
|
|
8784
|
+
"goalRate": number | null;
|
|
8785
|
+
"avgQualityScore": number | null;
|
|
8786
|
+
"evalPassRate": number | null;
|
|
8787
|
+
"benchmarkPassRate"?: number | null;
|
|
8788
|
+
"compositeScore": number | null;
|
|
8789
|
+
"avgTotalTokens": number | null;
|
|
8790
|
+
"avgOutputTokens": number | null;
|
|
8791
|
+
"avgCostUsd": number | null;
|
|
8792
|
+
"avgDurationMs": number | null;
|
|
8793
|
+
"medianCostUsd": number | null;
|
|
8794
|
+
"medianTotalTokens": number | null;
|
|
8795
|
+
"medianDurationMs": number | null;
|
|
8796
|
+
"rank": number;
|
|
8797
|
+
"group": string;
|
|
8798
|
+
"agent": string;
|
|
8799
|
+
"setup": string;
|
|
8800
|
+
}>;
|
|
8801
|
+
"perTask": Array<{
|
|
8802
|
+
"taskId": string;
|
|
8803
|
+
"position": number;
|
|
8804
|
+
"promptPreview": string;
|
|
8805
|
+
"requiresWeb": boolean;
|
|
8806
|
+
"category": string | null;
|
|
8807
|
+
"byArm": Record<string, unknown>;
|
|
8808
|
+
}>;
|
|
5681
8809
|
"behavior"?: Array<{
|
|
5682
8810
|
"harness": string;
|
|
5683
8811
|
"model": string;
|
|
@@ -5692,48 +8820,55 @@ export type StudyPreviewResponse = {
|
|
|
5692
8820
|
"succeededCount": number;
|
|
5693
8821
|
}>;
|
|
5694
8822
|
}>;
|
|
5695
|
-
"
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
"
|
|
5699
|
-
"
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5703
|
-
|
|
5704
|
-
|
|
5705
|
-
|
|
5706
|
-
"
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
8823
|
+
"execution"?: Record<string, unknown>;
|
|
8824
|
+
"inference"?: {
|
|
8825
|
+
"contrasts": Array<{
|
|
8826
|
+
"contrastId": string;
|
|
8827
|
+
"table": {
|
|
8828
|
+
"columns": Array<{
|
|
8829
|
+
"key": string;
|
|
8830
|
+
"label": string;
|
|
8831
|
+
"family": "rate" | "score" | "magnitude";
|
|
8832
|
+
"betterWhen": "higher" | "lower";
|
|
8833
|
+
}>;
|
|
8834
|
+
"groups": Array<{
|
|
8835
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
8836
|
+
"title": string;
|
|
8837
|
+
"rows": Array<{
|
|
8838
|
+
"key": string;
|
|
8839
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
8840
|
+
"harness": string | null;
|
|
8841
|
+
"model": string | null;
|
|
8842
|
+
"paramsKey"?: string;
|
|
8843
|
+
"runs": number;
|
|
8844
|
+
"cells": Record<string, unknown>;
|
|
8845
|
+
}>;
|
|
8846
|
+
}>;
|
|
8847
|
+
"reference": "baseline" | "field";
|
|
8848
|
+
"draws": number;
|
|
8849
|
+
"adjustment": {
|
|
8850
|
+
"scope": "planned-comparisons";
|
|
8851
|
+
"comparisons": number;
|
|
8852
|
+
};
|
|
8853
|
+
} | null;
|
|
5713
8854
|
}>;
|
|
5714
|
-
"
|
|
5715
|
-
"
|
|
5716
|
-
"
|
|
5717
|
-
"
|
|
8855
|
+
"holm": {
|
|
8856
|
+
"status": "complete";
|
|
8857
|
+
"ready": number;
|
|
8858
|
+
"planned": number;
|
|
8859
|
+
"adjustment": {
|
|
8860
|
+
"scope": "planned-comparisons";
|
|
8861
|
+
"comparisons": number;
|
|
8862
|
+
};
|
|
8863
|
+
} | {
|
|
8864
|
+
"status": "pending";
|
|
8865
|
+
"ready": number;
|
|
8866
|
+
"planned": number;
|
|
5718
8867
|
};
|
|
5719
|
-
"
|
|
5720
|
-
|
|
5721
|
-
|
|
5722
|
-
};
|
|
5723
|
-
"execution"?: {
|
|
5724
|
-
"graded": number;
|
|
5725
|
-
"measured": number;
|
|
5726
|
-
"passed": number;
|
|
5727
|
-
"passRate": number | null;
|
|
5728
|
-
"byStatus": {
|
|
5729
|
-
"passed": number;
|
|
5730
|
-
"failed": number;
|
|
5731
|
-
"timeout": number;
|
|
5732
|
-
"error": number;
|
|
5733
|
-
"unavailable": number;
|
|
8868
|
+
"claims": {
|
|
8869
|
+
"perContrast": Record<string, unknown>;
|
|
8870
|
+
"ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
|
|
5734
8871
|
};
|
|
5735
|
-
"languages": Array<string>;
|
|
5736
|
-
"unavailableReasons": Array<string>;
|
|
5737
8872
|
};
|
|
5738
8873
|
};
|
|
5739
8874
|
"textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
|
|
@@ -5741,6 +8876,7 @@ export type StudyPreviewResponse = {
|
|
|
5741
8876
|
export type StudyPublishRequest = {
|
|
5742
8877
|
"slug"?: string;
|
|
5743
8878
|
"visibility"?: "private" | "org" | "link" | "public";
|
|
8879
|
+
"skillNames"?: "withheld" | "named";
|
|
5744
8880
|
};
|
|
5745
8881
|
export type StudyPublishResponse = {
|
|
5746
8882
|
"study": {
|
|
@@ -5773,7 +8909,7 @@ export type StudyPublishResponse = {
|
|
|
5773
8909
|
"addons"?: Record<string, unknown>;
|
|
5774
8910
|
};
|
|
5775
8911
|
};
|
|
5776
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
8912
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5777
8913
|
"subjectBackend": "model" | "harness" | null;
|
|
5778
8914
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5779
8915
|
"importedFrom"?: string;
|
|
@@ -5809,6 +8945,7 @@ export type StudyPublishResponse = {
|
|
|
5809
8945
|
"error": string | null;
|
|
5810
8946
|
"preparation"?: {
|
|
5811
8947
|
"since": string;
|
|
8948
|
+
"progressAt"?: string;
|
|
5812
8949
|
"launchedBy": {
|
|
5813
8950
|
"userId"?: string;
|
|
5814
8951
|
"role"?: string;
|
|
@@ -5817,6 +8954,12 @@ export type StudyPublishResponse = {
|
|
|
5817
8954
|
"imageKey": string;
|
|
5818
8955
|
"buildId": string;
|
|
5819
8956
|
"stage"?: "image" | "snapshot";
|
|
8957
|
+
"retry"?: {
|
|
8958
|
+
"failedAttempts": number;
|
|
8959
|
+
"maxAttempts": number;
|
|
8960
|
+
"notBefore": string;
|
|
8961
|
+
"lastFailure": string;
|
|
8962
|
+
};
|
|
5820
8963
|
}>;
|
|
5821
8964
|
} | null;
|
|
5822
8965
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5831,6 +8974,57 @@ export type StudyPublishResponse = {
|
|
|
5831
8974
|
"createdAt": string;
|
|
5832
8975
|
"updatedAt": string;
|
|
5833
8976
|
"liveUntil": string | null;
|
|
8977
|
+
"launch"?: {
|
|
8978
|
+
"attempt": number;
|
|
8979
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
8980
|
+
"startedAt": string;
|
|
8981
|
+
"progressAt": string;
|
|
8982
|
+
"planned"?: number;
|
|
8983
|
+
"created"?: number;
|
|
8984
|
+
"missing"?: {
|
|
8985
|
+
"retryable": number;
|
|
8986
|
+
"refused": number;
|
|
8987
|
+
};
|
|
8988
|
+
"notes"?: Array<string>;
|
|
8989
|
+
"settledAt"?: string;
|
|
8990
|
+
} | null;
|
|
8991
|
+
"relaunchable"?: boolean;
|
|
8992
|
+
"arms"?: Array<{
|
|
8993
|
+
"key": string;
|
|
8994
|
+
"position": number;
|
|
8995
|
+
"label": string;
|
|
8996
|
+
"role": "reference" | "peer";
|
|
8997
|
+
"payload": {
|
|
8998
|
+
"tools"?: "default" | "none" | {
|
|
8999
|
+
"tools": Array<string>;
|
|
9000
|
+
};
|
|
9001
|
+
"addons"?: Record<string, unknown>;
|
|
9002
|
+
};
|
|
9003
|
+
"cells"?: Array<{
|
|
9004
|
+
"harness": string;
|
|
9005
|
+
"model": string;
|
|
9006
|
+
"provider"?: string;
|
|
9007
|
+
"params"?: {
|
|
9008
|
+
"temperature"?: number;
|
|
9009
|
+
"maxOutputTokens"?: number;
|
|
9010
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9011
|
+
"thinkingBudgetTokens"?: number;
|
|
9012
|
+
};
|
|
9013
|
+
}>;
|
|
9014
|
+
"skills"?: Array<{
|
|
9015
|
+
"ref": string;
|
|
9016
|
+
"name": string;
|
|
9017
|
+
"changedSinceCreate"?: boolean;
|
|
9018
|
+
}>;
|
|
9019
|
+
"addedAgents"?: Array<{
|
|
9020
|
+
"position": number;
|
|
9021
|
+
"addedAt": string;
|
|
9022
|
+
}>;
|
|
9023
|
+
}>;
|
|
9024
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9025
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9026
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9027
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5834
9028
|
};
|
|
5835
9029
|
};
|
|
5836
9030
|
export type StudyReconcileRunsResponse = {
|
|
@@ -5873,7 +9067,7 @@ export type StudyRegradeResponse = {
|
|
|
5873
9067
|
"addons"?: Record<string, unknown>;
|
|
5874
9068
|
};
|
|
5875
9069
|
};
|
|
5876
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
9070
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5877
9071
|
"subjectBackend": "model" | "harness" | null;
|
|
5878
9072
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5879
9073
|
"importedFrom"?: string;
|
|
@@ -5909,6 +9103,7 @@ export type StudyRegradeResponse = {
|
|
|
5909
9103
|
"error": string | null;
|
|
5910
9104
|
"preparation"?: {
|
|
5911
9105
|
"since": string;
|
|
9106
|
+
"progressAt"?: string;
|
|
5912
9107
|
"launchedBy": {
|
|
5913
9108
|
"userId"?: string;
|
|
5914
9109
|
"role"?: string;
|
|
@@ -5917,6 +9112,12 @@ export type StudyRegradeResponse = {
|
|
|
5917
9112
|
"imageKey": string;
|
|
5918
9113
|
"buildId": string;
|
|
5919
9114
|
"stage"?: "image" | "snapshot";
|
|
9115
|
+
"retry"?: {
|
|
9116
|
+
"failedAttempts": number;
|
|
9117
|
+
"maxAttempts": number;
|
|
9118
|
+
"notBefore": string;
|
|
9119
|
+
"lastFailure": string;
|
|
9120
|
+
};
|
|
5920
9121
|
}>;
|
|
5921
9122
|
} | null;
|
|
5922
9123
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5931,11 +9132,133 @@ export type StudyRegradeResponse = {
|
|
|
5931
9132
|
"createdAt": string;
|
|
5932
9133
|
"updatedAt": string;
|
|
5933
9134
|
"liveUntil": string | null;
|
|
9135
|
+
"launch"?: {
|
|
9136
|
+
"attempt": number;
|
|
9137
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
9138
|
+
"startedAt": string;
|
|
9139
|
+
"progressAt": string;
|
|
9140
|
+
"planned"?: number;
|
|
9141
|
+
"created"?: number;
|
|
9142
|
+
"missing"?: {
|
|
9143
|
+
"retryable": number;
|
|
9144
|
+
"refused": number;
|
|
9145
|
+
};
|
|
9146
|
+
"notes"?: Array<string>;
|
|
9147
|
+
"settledAt"?: string;
|
|
9148
|
+
} | null;
|
|
9149
|
+
"relaunchable"?: boolean;
|
|
9150
|
+
"arms"?: Array<{
|
|
9151
|
+
"key": string;
|
|
9152
|
+
"position": number;
|
|
9153
|
+
"label": string;
|
|
9154
|
+
"role": "reference" | "peer";
|
|
9155
|
+
"payload": {
|
|
9156
|
+
"tools"?: "default" | "none" | {
|
|
9157
|
+
"tools": Array<string>;
|
|
9158
|
+
};
|
|
9159
|
+
"addons"?: Record<string, unknown>;
|
|
9160
|
+
};
|
|
9161
|
+
"cells"?: Array<{
|
|
9162
|
+
"harness": string;
|
|
9163
|
+
"model": string;
|
|
9164
|
+
"provider"?: string;
|
|
9165
|
+
"params"?: {
|
|
9166
|
+
"temperature"?: number;
|
|
9167
|
+
"maxOutputTokens"?: number;
|
|
9168
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9169
|
+
"thinkingBudgetTokens"?: number;
|
|
9170
|
+
};
|
|
9171
|
+
}>;
|
|
9172
|
+
"skills"?: Array<{
|
|
9173
|
+
"ref": string;
|
|
9174
|
+
"name": string;
|
|
9175
|
+
"changedSinceCreate"?: boolean;
|
|
9176
|
+
}>;
|
|
9177
|
+
"addedAgents"?: Array<{
|
|
9178
|
+
"position": number;
|
|
9179
|
+
"addedAt": string;
|
|
9180
|
+
}>;
|
|
9181
|
+
}>;
|
|
9182
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9183
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9184
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9185
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5934
9186
|
};
|
|
5935
9187
|
"streamUrl": string;
|
|
5936
9188
|
"warnings"?: Array<{
|
|
5937
9189
|
"harness": string;
|
|
5938
9190
|
"kind": string;
|
|
9191
|
+
} | {
|
|
9192
|
+
"type": "design_aggregate_budget";
|
|
9193
|
+
"detail": string;
|
|
9194
|
+
}>;
|
|
9195
|
+
};
|
|
9196
|
+
export type StudyRerunAttemptsResponse = {
|
|
9197
|
+
"studyId": string;
|
|
9198
|
+
"attempts": Array<{
|
|
9199
|
+
"taskId": string;
|
|
9200
|
+
"variantIndex": number;
|
|
9201
|
+
"repeatIndex": number;
|
|
9202
|
+
"arm": string;
|
|
9203
|
+
"harness": string;
|
|
9204
|
+
"model": string;
|
|
9205
|
+
"paramsKey": string;
|
|
9206
|
+
"attempt": number;
|
|
9207
|
+
"cause": "infra_failure" | "missing" | "declared_infra";
|
|
9208
|
+
"evidence": string | null;
|
|
9209
|
+
"replacedRunId": string | null;
|
|
9210
|
+
"replacedStatus": string | null;
|
|
9211
|
+
"replacedOutcomeReason": string | null;
|
|
9212
|
+
"replacedStartedAt": string | null;
|
|
9213
|
+
"replacedFinishedAt": string | null;
|
|
9214
|
+
"runId": string | null;
|
|
9215
|
+
"outcome": "dispatched" | "failed";
|
|
9216
|
+
"error": string | null;
|
|
9217
|
+
"requestedBy": string | null;
|
|
9218
|
+
"createdAt": string;
|
|
9219
|
+
}>;
|
|
9220
|
+
};
|
|
9221
|
+
export type StudyRerunInfraFailuresRequest = {
|
|
9222
|
+
"dryRun"?: boolean;
|
|
9223
|
+
"limit"?: number;
|
|
9224
|
+
"declared"?: Array<{
|
|
9225
|
+
"runId": string;
|
|
9226
|
+
"evidence": string;
|
|
9227
|
+
}>;
|
|
9228
|
+
};
|
|
9229
|
+
export type StudyRerunInfraFailuresResponse = {
|
|
9230
|
+
"studyId": string;
|
|
9231
|
+
"status": string;
|
|
9232
|
+
"dryRun": boolean;
|
|
9233
|
+
"reopened": boolean;
|
|
9234
|
+
"maxAttemptsPerJob": number;
|
|
9235
|
+
"jobs": Array<{
|
|
9236
|
+
"taskId": string;
|
|
9237
|
+
"variantIndex": number;
|
|
9238
|
+
"repeatIndex": number;
|
|
9239
|
+
"arm": string;
|
|
9240
|
+
"harness": string;
|
|
9241
|
+
"model": string;
|
|
9242
|
+
"paramsKey": string;
|
|
9243
|
+
"cause": "infra_failure" | "missing" | "declared_infra";
|
|
9244
|
+
"evidence": string | null;
|
|
9245
|
+
"attempt": number;
|
|
9246
|
+
"replacedRunId": string | null;
|
|
9247
|
+
"runId": string | null;
|
|
9248
|
+
"outcome": "planned" | "dispatched" | "failed";
|
|
9249
|
+
"error": string | null;
|
|
9250
|
+
}>;
|
|
9251
|
+
"skipped": {
|
|
9252
|
+
"live": number;
|
|
9253
|
+
"exhausted": number;
|
|
9254
|
+
"missingDeferred": number;
|
|
9255
|
+
"overLimit": number;
|
|
9256
|
+
};
|
|
9257
|
+
"notes": Array<string>;
|
|
9258
|
+
"byGroup": Record<string, unknown>;
|
|
9259
|
+
"declaredRejected": Array<{
|
|
9260
|
+
"runId": string;
|
|
9261
|
+
"reason": string;
|
|
5939
9262
|
}>;
|
|
5940
9263
|
};
|
|
5941
9264
|
export type StudyListRunAnswersResponse = {
|
|
@@ -5984,7 +9307,7 @@ export type StudyRotateShareTokenResponse = {
|
|
|
5984
9307
|
"addons"?: Record<string, unknown>;
|
|
5985
9308
|
};
|
|
5986
9309
|
};
|
|
5987
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
9310
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5988
9311
|
"subjectBackend": "model" | "harness" | null;
|
|
5989
9312
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5990
9313
|
"importedFrom"?: string;
|
|
@@ -6020,6 +9343,7 @@ export type StudyRotateShareTokenResponse = {
|
|
|
6020
9343
|
"error": string | null;
|
|
6021
9344
|
"preparation"?: {
|
|
6022
9345
|
"since": string;
|
|
9346
|
+
"progressAt"?: string;
|
|
6023
9347
|
"launchedBy": {
|
|
6024
9348
|
"userId"?: string;
|
|
6025
9349
|
"role"?: string;
|
|
@@ -6028,6 +9352,12 @@ export type StudyRotateShareTokenResponse = {
|
|
|
6028
9352
|
"imageKey": string;
|
|
6029
9353
|
"buildId": string;
|
|
6030
9354
|
"stage"?: "image" | "snapshot";
|
|
9355
|
+
"retry"?: {
|
|
9356
|
+
"failedAttempts": number;
|
|
9357
|
+
"maxAttempts": number;
|
|
9358
|
+
"notBefore": string;
|
|
9359
|
+
"lastFailure": string;
|
|
9360
|
+
};
|
|
6031
9361
|
}>;
|
|
6032
9362
|
} | null;
|
|
6033
9363
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -6042,13 +9372,132 @@ export type StudyRotateShareTokenResponse = {
|
|
|
6042
9372
|
"createdAt": string;
|
|
6043
9373
|
"updatedAt": string;
|
|
6044
9374
|
"liveUntil": string | null;
|
|
9375
|
+
"launch"?: {
|
|
9376
|
+
"attempt": number;
|
|
9377
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
9378
|
+
"startedAt": string;
|
|
9379
|
+
"progressAt": string;
|
|
9380
|
+
"planned"?: number;
|
|
9381
|
+
"created"?: number;
|
|
9382
|
+
"missing"?: {
|
|
9383
|
+
"retryable": number;
|
|
9384
|
+
"refused": number;
|
|
9385
|
+
};
|
|
9386
|
+
"notes"?: Array<string>;
|
|
9387
|
+
"settledAt"?: string;
|
|
9388
|
+
} | null;
|
|
9389
|
+
"relaunchable"?: boolean;
|
|
9390
|
+
"arms"?: Array<{
|
|
9391
|
+
"key": string;
|
|
9392
|
+
"position": number;
|
|
9393
|
+
"label": string;
|
|
9394
|
+
"role": "reference" | "peer";
|
|
9395
|
+
"payload": {
|
|
9396
|
+
"tools"?: "default" | "none" | {
|
|
9397
|
+
"tools": Array<string>;
|
|
9398
|
+
};
|
|
9399
|
+
"addons"?: Record<string, unknown>;
|
|
9400
|
+
};
|
|
9401
|
+
"cells"?: Array<{
|
|
9402
|
+
"harness": string;
|
|
9403
|
+
"model": string;
|
|
9404
|
+
"provider"?: string;
|
|
9405
|
+
"params"?: {
|
|
9406
|
+
"temperature"?: number;
|
|
9407
|
+
"maxOutputTokens"?: number;
|
|
9408
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9409
|
+
"thinkingBudgetTokens"?: number;
|
|
9410
|
+
};
|
|
9411
|
+
}>;
|
|
9412
|
+
"skills"?: Array<{
|
|
9413
|
+
"ref": string;
|
|
9414
|
+
"name": string;
|
|
9415
|
+
"changedSinceCreate"?: boolean;
|
|
9416
|
+
}>;
|
|
9417
|
+
"addedAgents"?: Array<{
|
|
9418
|
+
"position": number;
|
|
9419
|
+
"addedAt": string;
|
|
9420
|
+
}>;
|
|
9421
|
+
}>;
|
|
9422
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9423
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9424
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9425
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6045
9426
|
};
|
|
6046
9427
|
"shareUrl": string | null;
|
|
6047
9428
|
};
|
|
6048
9429
|
export type StudyStreamResponse = unknown;
|
|
6049
9430
|
export type StudyUpdateTaskRequest = {
|
|
6050
9431
|
"prompt": string;
|
|
6051
|
-
"expectations"?: Array<
|
|
9432
|
+
"expectations"?: Array<{
|
|
9433
|
+
"label"?: string;
|
|
9434
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9435
|
+
"kind": "mcp_tool_called";
|
|
9436
|
+
"server"?: string;
|
|
9437
|
+
"tool": string;
|
|
9438
|
+
"arguments"?: Record<string, unknown>;
|
|
9439
|
+
"minCalls"?: number;
|
|
9440
|
+
} | {
|
|
9441
|
+
"label"?: string;
|
|
9442
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9443
|
+
"kind": "tool_not_called";
|
|
9444
|
+
"tool"?: string;
|
|
9445
|
+
"mcp"?: {
|
|
9446
|
+
"server"?: string;
|
|
9447
|
+
"tool": string;
|
|
9448
|
+
};
|
|
9449
|
+
"pattern"?: string;
|
|
9450
|
+
} | {
|
|
9451
|
+
"label"?: string;
|
|
9452
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9453
|
+
"kind": "command_ran";
|
|
9454
|
+
"pattern": string;
|
|
9455
|
+
} | {
|
|
9456
|
+
"label"?: string;
|
|
9457
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9458
|
+
"kind": "skill_loaded";
|
|
9459
|
+
"slug": string;
|
|
9460
|
+
} | {
|
|
9461
|
+
"label"?: string;
|
|
9462
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9463
|
+
"kind": "file_operation";
|
|
9464
|
+
"op"?: "read" | "write" | "edit";
|
|
9465
|
+
"path": unknown;
|
|
9466
|
+
} | {
|
|
9467
|
+
"label"?: string;
|
|
9468
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9469
|
+
"kind": "web_fetched";
|
|
9470
|
+
"urlHost"?: string;
|
|
9471
|
+
"urlPattern"?: string;
|
|
9472
|
+
} | {
|
|
9473
|
+
"label"?: string;
|
|
9474
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9475
|
+
"kind": "response_contains";
|
|
9476
|
+
"text": string;
|
|
9477
|
+
"normalize"?: "prose";
|
|
9478
|
+
} | {
|
|
9479
|
+
"label"?: string;
|
|
9480
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9481
|
+
"kind": "response_matches";
|
|
9482
|
+
"pattern": string;
|
|
9483
|
+
"ignoreCase"?: boolean;
|
|
9484
|
+
} | {
|
|
9485
|
+
"label"?: string;
|
|
9486
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9487
|
+
"kind": "token_budget";
|
|
9488
|
+
"maxOutputTokens"?: number;
|
|
9489
|
+
"maxTotalTokens"?: number;
|
|
9490
|
+
} | {
|
|
9491
|
+
"label"?: string;
|
|
9492
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9493
|
+
"kind": "sequence";
|
|
9494
|
+
"of": Array<unknown>;
|
|
9495
|
+
} | {
|
|
9496
|
+
"label"?: string;
|
|
9497
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9498
|
+
"kind": "any_of";
|
|
9499
|
+
"of": Array<unknown>;
|
|
9500
|
+
}>;
|
|
6052
9501
|
};
|
|
6053
9502
|
export type StudyUpdateTaskResponse = {
|
|
6054
9503
|
"task": {
|
|
@@ -6095,7 +9544,7 @@ export type StudyUnpublishResponse = {
|
|
|
6095
9544
|
"addons"?: Record<string, unknown>;
|
|
6096
9545
|
};
|
|
6097
9546
|
};
|
|
6098
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
9547
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6099
9548
|
"subjectBackend": "model" | "harness" | null;
|
|
6100
9549
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6101
9550
|
"importedFrom"?: string;
|
|
@@ -6131,6 +9580,7 @@ export type StudyUnpublishResponse = {
|
|
|
6131
9580
|
"error": string | null;
|
|
6132
9581
|
"preparation"?: {
|
|
6133
9582
|
"since": string;
|
|
9583
|
+
"progressAt"?: string;
|
|
6134
9584
|
"launchedBy": {
|
|
6135
9585
|
"userId"?: string;
|
|
6136
9586
|
"role"?: string;
|
|
@@ -6139,6 +9589,12 @@ export type StudyUnpublishResponse = {
|
|
|
6139
9589
|
"imageKey": string;
|
|
6140
9590
|
"buildId": string;
|
|
6141
9591
|
"stage"?: "image" | "snapshot";
|
|
9592
|
+
"retry"?: {
|
|
9593
|
+
"failedAttempts": number;
|
|
9594
|
+
"maxAttempts": number;
|
|
9595
|
+
"notBefore": string;
|
|
9596
|
+
"lastFailure": string;
|
|
9597
|
+
};
|
|
6142
9598
|
}>;
|
|
6143
9599
|
} | null;
|
|
6144
9600
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -6153,6 +9609,57 @@ export type StudyUnpublishResponse = {
|
|
|
6153
9609
|
"createdAt": string;
|
|
6154
9610
|
"updatedAt": string;
|
|
6155
9611
|
"liveUntil": string | null;
|
|
9612
|
+
"launch"?: {
|
|
9613
|
+
"attempt": number;
|
|
9614
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
9615
|
+
"startedAt": string;
|
|
9616
|
+
"progressAt": string;
|
|
9617
|
+
"planned"?: number;
|
|
9618
|
+
"created"?: number;
|
|
9619
|
+
"missing"?: {
|
|
9620
|
+
"retryable": number;
|
|
9621
|
+
"refused": number;
|
|
9622
|
+
};
|
|
9623
|
+
"notes"?: Array<string>;
|
|
9624
|
+
"settledAt"?: string;
|
|
9625
|
+
} | null;
|
|
9626
|
+
"relaunchable"?: boolean;
|
|
9627
|
+
"arms"?: Array<{
|
|
9628
|
+
"key": string;
|
|
9629
|
+
"position": number;
|
|
9630
|
+
"label": string;
|
|
9631
|
+
"role": "reference" | "peer";
|
|
9632
|
+
"payload": {
|
|
9633
|
+
"tools"?: "default" | "none" | {
|
|
9634
|
+
"tools": Array<string>;
|
|
9635
|
+
};
|
|
9636
|
+
"addons"?: Record<string, unknown>;
|
|
9637
|
+
};
|
|
9638
|
+
"cells"?: Array<{
|
|
9639
|
+
"harness": string;
|
|
9640
|
+
"model": string;
|
|
9641
|
+
"provider"?: string;
|
|
9642
|
+
"params"?: {
|
|
9643
|
+
"temperature"?: number;
|
|
9644
|
+
"maxOutputTokens"?: number;
|
|
9645
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9646
|
+
"thinkingBudgetTokens"?: number;
|
|
9647
|
+
};
|
|
9648
|
+
}>;
|
|
9649
|
+
"skills"?: Array<{
|
|
9650
|
+
"ref": string;
|
|
9651
|
+
"name": string;
|
|
9652
|
+
"changedSinceCreate"?: boolean;
|
|
9653
|
+
}>;
|
|
9654
|
+
"addedAgents"?: Array<{
|
|
9655
|
+
"position": number;
|
|
9656
|
+
"addedAt": string;
|
|
9657
|
+
}>;
|
|
9658
|
+
}>;
|
|
9659
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9660
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9661
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9662
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6156
9663
|
};
|
|
6157
9664
|
};
|
|
6158
9665
|
export type StudySetVisibilityRequest = {
|
|
@@ -6189,7 +9696,7 @@ export type StudySetVisibilityResponse = {
|
|
|
6189
9696
|
"addons"?: Record<string, unknown>;
|
|
6190
9697
|
};
|
|
6191
9698
|
};
|
|
6192
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
9699
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6193
9700
|
"subjectBackend": "model" | "harness" | null;
|
|
6194
9701
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6195
9702
|
"importedFrom"?: string;
|
|
@@ -6225,6 +9732,7 @@ export type StudySetVisibilityResponse = {
|
|
|
6225
9732
|
"error": string | null;
|
|
6226
9733
|
"preparation"?: {
|
|
6227
9734
|
"since": string;
|
|
9735
|
+
"progressAt"?: string;
|
|
6228
9736
|
"launchedBy": {
|
|
6229
9737
|
"userId"?: string;
|
|
6230
9738
|
"role"?: string;
|
|
@@ -6233,6 +9741,12 @@ export type StudySetVisibilityResponse = {
|
|
|
6233
9741
|
"imageKey": string;
|
|
6234
9742
|
"buildId": string;
|
|
6235
9743
|
"stage"?: "image" | "snapshot";
|
|
9744
|
+
"retry"?: {
|
|
9745
|
+
"failedAttempts": number;
|
|
9746
|
+
"maxAttempts": number;
|
|
9747
|
+
"notBefore": string;
|
|
9748
|
+
"lastFailure": string;
|
|
9749
|
+
};
|
|
6236
9750
|
}>;
|
|
6237
9751
|
} | null;
|
|
6238
9752
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -6247,6 +9761,57 @@ export type StudySetVisibilityResponse = {
|
|
|
6247
9761
|
"createdAt": string;
|
|
6248
9762
|
"updatedAt": string;
|
|
6249
9763
|
"liveUntil": string | null;
|
|
9764
|
+
"launch"?: {
|
|
9765
|
+
"attempt": number;
|
|
9766
|
+
"state": "fanning_out" | "deferred" | "settled";
|
|
9767
|
+
"startedAt": string;
|
|
9768
|
+
"progressAt": string;
|
|
9769
|
+
"planned"?: number;
|
|
9770
|
+
"created"?: number;
|
|
9771
|
+
"missing"?: {
|
|
9772
|
+
"retryable": number;
|
|
9773
|
+
"refused": number;
|
|
9774
|
+
};
|
|
9775
|
+
"notes"?: Array<string>;
|
|
9776
|
+
"settledAt"?: string;
|
|
9777
|
+
} | null;
|
|
9778
|
+
"relaunchable"?: boolean;
|
|
9779
|
+
"arms"?: Array<{
|
|
9780
|
+
"key": string;
|
|
9781
|
+
"position": number;
|
|
9782
|
+
"label": string;
|
|
9783
|
+
"role": "reference" | "peer";
|
|
9784
|
+
"payload": {
|
|
9785
|
+
"tools"?: "default" | "none" | {
|
|
9786
|
+
"tools": Array<string>;
|
|
9787
|
+
};
|
|
9788
|
+
"addons"?: Record<string, unknown>;
|
|
9789
|
+
};
|
|
9790
|
+
"cells"?: Array<{
|
|
9791
|
+
"harness": string;
|
|
9792
|
+
"model": string;
|
|
9793
|
+
"provider"?: string;
|
|
9794
|
+
"params"?: {
|
|
9795
|
+
"temperature"?: number;
|
|
9796
|
+
"maxOutputTokens"?: number;
|
|
9797
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9798
|
+
"thinkingBudgetTokens"?: number;
|
|
9799
|
+
};
|
|
9800
|
+
}>;
|
|
9801
|
+
"skills"?: Array<{
|
|
9802
|
+
"ref": string;
|
|
9803
|
+
"name": string;
|
|
9804
|
+
"changedSinceCreate"?: boolean;
|
|
9805
|
+
}>;
|
|
9806
|
+
"addedAgents"?: Array<{
|
|
9807
|
+
"position": number;
|
|
9808
|
+
"addedAt": string;
|
|
9809
|
+
}>;
|
|
9810
|
+
}>;
|
|
9811
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9812
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9813
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9814
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6250
9815
|
};
|
|
6251
9816
|
"shareUrl": string | null;
|
|
6252
9817
|
};
|