@ora-ai/cli 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +468 -18
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +3574 -402
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/dist/render/insight-lines.d.ts.map +1 -1
- package/dist/render/insight-lines.js +14 -3
- package/dist/render/insight-lines.js.map +1 -1
- package/package.json +1 -1
|
@@ -84,6 +84,7 @@ export type BenchmarksListResponse = {
|
|
|
84
84
|
"error": string | null;
|
|
85
85
|
"preparation"?: {
|
|
86
86
|
"since": string;
|
|
87
|
+
"progressAt"?: string;
|
|
87
88
|
"launchedBy": {
|
|
88
89
|
"userId"?: string;
|
|
89
90
|
"role"?: string;
|
|
@@ -92,6 +93,12 @@ export type BenchmarksListResponse = {
|
|
|
92
93
|
"imageKey": string;
|
|
93
94
|
"buildId": string;
|
|
94
95
|
"stage"?: "image" | "snapshot";
|
|
96
|
+
"retry"?: {
|
|
97
|
+
"failedAttempts": number;
|
|
98
|
+
"maxAttempts": number;
|
|
99
|
+
"notBefore": string;
|
|
100
|
+
"lastFailure": string;
|
|
101
|
+
};
|
|
95
102
|
}>;
|
|
96
103
|
} | null;
|
|
97
104
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -232,6 +239,7 @@ export type BenchmarksCreateResponse = {
|
|
|
232
239
|
"error": string | null;
|
|
233
240
|
"preparation"?: {
|
|
234
241
|
"since": string;
|
|
242
|
+
"progressAt"?: string;
|
|
235
243
|
"launchedBy": {
|
|
236
244
|
"userId"?: string;
|
|
237
245
|
"role"?: string;
|
|
@@ -240,6 +248,12 @@ export type BenchmarksCreateResponse = {
|
|
|
240
248
|
"imageKey": string;
|
|
241
249
|
"buildId": string;
|
|
242
250
|
"stage"?: "image" | "snapshot";
|
|
251
|
+
"retry"?: {
|
|
252
|
+
"failedAttempts": number;
|
|
253
|
+
"maxAttempts": number;
|
|
254
|
+
"notBefore": string;
|
|
255
|
+
"lastFailure": string;
|
|
256
|
+
};
|
|
243
257
|
}>;
|
|
244
258
|
} | null;
|
|
245
259
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -340,6 +354,7 @@ export type BenchmarksGetResponse = {
|
|
|
340
354
|
"error": string | null;
|
|
341
355
|
"preparation"?: {
|
|
342
356
|
"since": string;
|
|
357
|
+
"progressAt"?: string;
|
|
343
358
|
"launchedBy": {
|
|
344
359
|
"userId"?: string;
|
|
345
360
|
"role"?: string;
|
|
@@ -348,6 +363,12 @@ export type BenchmarksGetResponse = {
|
|
|
348
363
|
"imageKey": string;
|
|
349
364
|
"buildId": string;
|
|
350
365
|
"stage"?: "image" | "snapshot";
|
|
366
|
+
"retry"?: {
|
|
367
|
+
"failedAttempts": number;
|
|
368
|
+
"maxAttempts": number;
|
|
369
|
+
"notBefore": string;
|
|
370
|
+
"lastFailure": string;
|
|
371
|
+
};
|
|
351
372
|
}>;
|
|
352
373
|
} | null;
|
|
353
374
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -474,6 +495,9 @@ export type BenchmarksGetResponse = {
|
|
|
474
495
|
"gradedAt": string;
|
|
475
496
|
};
|
|
476
497
|
} | null;
|
|
498
|
+
"executionBudgetS"?: number | null;
|
|
499
|
+
"harnessImageDigest"?: string | null;
|
|
500
|
+
"subjectEngaged"?: boolean | null;
|
|
477
501
|
"runCreatedAt": string | null;
|
|
478
502
|
"startedAt": string | null;
|
|
479
503
|
"finishedAt": string | null;
|
|
@@ -683,6 +707,17 @@ export type BenchmarksGetResponse = {
|
|
|
683
707
|
"ci": Array<unknown> | null;
|
|
684
708
|
"n": number;
|
|
685
709
|
};
|
|
710
|
+
"abstention"?: {
|
|
711
|
+
"value": number | null;
|
|
712
|
+
"ci": Array<unknown> | null;
|
|
713
|
+
"n": number;
|
|
714
|
+
"parts"?: Array<{
|
|
715
|
+
"id": string;
|
|
716
|
+
"label": string;
|
|
717
|
+
"value": number | null;
|
|
718
|
+
"n": number;
|
|
719
|
+
}>;
|
|
720
|
+
};
|
|
686
721
|
"tasks": number;
|
|
687
722
|
"graded": number;
|
|
688
723
|
"unmeasured": number;
|
|
@@ -694,6 +729,17 @@ export type BenchmarksGetResponse = {
|
|
|
694
729
|
"ci": Array<unknown> | null;
|
|
695
730
|
"n": number;
|
|
696
731
|
};
|
|
732
|
+
"abstention"?: {
|
|
733
|
+
"value": number | null;
|
|
734
|
+
"ci": Array<unknown> | null;
|
|
735
|
+
"n": number;
|
|
736
|
+
"parts"?: Array<{
|
|
737
|
+
"id": string;
|
|
738
|
+
"label": string;
|
|
739
|
+
"value": number | null;
|
|
740
|
+
"n": number;
|
|
741
|
+
}>;
|
|
742
|
+
};
|
|
697
743
|
"graded": number;
|
|
698
744
|
"unmeasured": number;
|
|
699
745
|
"modelJudged": boolean;
|
|
@@ -723,9 +769,130 @@ export type BenchmarksGetResponse = {
|
|
|
723
769
|
"detail": string;
|
|
724
770
|
"taskIds"?: Array<string>;
|
|
725
771
|
}>;
|
|
772
|
+
"taskHealth": {
|
|
773
|
+
"flags": Array<{
|
|
774
|
+
"taskId": string;
|
|
775
|
+
"signal": string;
|
|
776
|
+
"kind": "floor" | "ceiling";
|
|
777
|
+
"conditions": number;
|
|
778
|
+
}>;
|
|
779
|
+
"rubricSplits": Array<{
|
|
780
|
+
"taskId": string;
|
|
781
|
+
"criterionIndex": number;
|
|
782
|
+
"criterion": string;
|
|
783
|
+
"split": number;
|
|
784
|
+
"graded": number;
|
|
785
|
+
}>;
|
|
786
|
+
"signals": Array<{
|
|
787
|
+
"id": string;
|
|
788
|
+
"label": string;
|
|
789
|
+
}>;
|
|
790
|
+
};
|
|
726
791
|
"shareUrl"?: string | null;
|
|
727
792
|
"canManageSharing"?: boolean;
|
|
728
793
|
};
|
|
794
|
+
export type BenchmarksUpdateDetailsRequest = {
|
|
795
|
+
"name"?: string;
|
|
796
|
+
"description"?: string;
|
|
797
|
+
};
|
|
798
|
+
export type BenchmarksUpdateDetailsResponse = {
|
|
799
|
+
"study": {
|
|
800
|
+
"id": string;
|
|
801
|
+
"executiveSummary"?: string | null;
|
|
802
|
+
"plannedRunCount": number;
|
|
803
|
+
"organizationId": string;
|
|
804
|
+
"createdBy": string | null;
|
|
805
|
+
"name": string;
|
|
806
|
+
"slug": string;
|
|
807
|
+
"description": string | null;
|
|
808
|
+
"skillRef": string | null;
|
|
809
|
+
"skillSlug": string | null;
|
|
810
|
+
"skillName": string | null;
|
|
811
|
+
"skillPromise": string;
|
|
812
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
813
|
+
"subjectName": string | null;
|
|
814
|
+
"subjectLabel": string;
|
|
815
|
+
"armConfig": {
|
|
816
|
+
"baseline"?: {
|
|
817
|
+
"tools"?: "default" | "none" | {
|
|
818
|
+
"tools": Array<string>;
|
|
819
|
+
};
|
|
820
|
+
"addons"?: Record<string, unknown>;
|
|
821
|
+
};
|
|
822
|
+
"treatment"?: {
|
|
823
|
+
"tools"?: "default" | "none" | {
|
|
824
|
+
"tools": Array<string>;
|
|
825
|
+
};
|
|
826
|
+
"addons"?: Record<string, unknown>;
|
|
827
|
+
};
|
|
828
|
+
};
|
|
829
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
830
|
+
"subjectBackend": "model" | "harness" | null;
|
|
831
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
832
|
+
"importedFrom"?: string;
|
|
833
|
+
"baselineLabel": string | null;
|
|
834
|
+
"treatmentLabel": string | null;
|
|
835
|
+
"promiseMetrics": Array<{
|
|
836
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
837
|
+
"direction": "down" | "up";
|
|
838
|
+
"label": string;
|
|
839
|
+
}>;
|
|
840
|
+
"matrix": Array<{
|
|
841
|
+
"harness": string;
|
|
842
|
+
"model": string;
|
|
843
|
+
"provider"?: string;
|
|
844
|
+
"params"?: {
|
|
845
|
+
"temperature"?: number;
|
|
846
|
+
"maxOutputTokens"?: number;
|
|
847
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
848
|
+
"thinkingBudgetTokens"?: number;
|
|
849
|
+
};
|
|
850
|
+
}>;
|
|
851
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
852
|
+
"judgeSupplementary"?: boolean;
|
|
853
|
+
"taskCount": number;
|
|
854
|
+
"variantsPerTask": number;
|
|
855
|
+
"repeats": number;
|
|
856
|
+
"subset": {
|
|
857
|
+
"count": number;
|
|
858
|
+
"seed": string;
|
|
859
|
+
"of": number;
|
|
860
|
+
} | null;
|
|
861
|
+
"status": string;
|
|
862
|
+
"error": string | null;
|
|
863
|
+
"preparation"?: {
|
|
864
|
+
"since": string;
|
|
865
|
+
"progressAt"?: string;
|
|
866
|
+
"launchedBy": {
|
|
867
|
+
"userId"?: string;
|
|
868
|
+
"role"?: string;
|
|
869
|
+
};
|
|
870
|
+
"waitingOn": Array<{
|
|
871
|
+
"imageKey": string;
|
|
872
|
+
"buildId": string;
|
|
873
|
+
"stage"?: "image" | "snapshot";
|
|
874
|
+
"retry"?: {
|
|
875
|
+
"failedAttempts": number;
|
|
876
|
+
"maxAttempts": number;
|
|
877
|
+
"notBefore": string;
|
|
878
|
+
"lastFailure": string;
|
|
879
|
+
};
|
|
880
|
+
}>;
|
|
881
|
+
} | null;
|
|
882
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
883
|
+
"visibilityChangedAt"?: string | null;
|
|
884
|
+
"visibilityChangedBy"?: string | null;
|
|
885
|
+
"shareToken"?: string | null;
|
|
886
|
+
"lowSampleOverride": boolean;
|
|
887
|
+
"publishedAt": string | null;
|
|
888
|
+
"verdict": unknown;
|
|
889
|
+
"definitionVersionId": string | null;
|
|
890
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
891
|
+
"createdAt": string;
|
|
892
|
+
"updatedAt": string;
|
|
893
|
+
"liveUntil": string | null;
|
|
894
|
+
};
|
|
895
|
+
};
|
|
729
896
|
export type BenchmarksDeleteResponse = unknown;
|
|
730
897
|
export type BenchmarksRegenerateSummaryResponse = {
|
|
731
898
|
"status": "queued";
|
|
@@ -800,6 +967,7 @@ export type BenchmarksGenerateResponse = {
|
|
|
800
967
|
"error": string | null;
|
|
801
968
|
"preparation"?: {
|
|
802
969
|
"since": string;
|
|
970
|
+
"progressAt"?: string;
|
|
803
971
|
"launchedBy": {
|
|
804
972
|
"userId"?: string;
|
|
805
973
|
"role"?: string;
|
|
@@ -808,6 +976,12 @@ export type BenchmarksGenerateResponse = {
|
|
|
808
976
|
"imageKey": string;
|
|
809
977
|
"buildId": string;
|
|
810
978
|
"stage"?: "image" | "snapshot";
|
|
979
|
+
"retry"?: {
|
|
980
|
+
"failedAttempts": number;
|
|
981
|
+
"maxAttempts": number;
|
|
982
|
+
"notBefore": string;
|
|
983
|
+
"lastFailure": string;
|
|
984
|
+
};
|
|
811
985
|
}>;
|
|
812
986
|
} | null;
|
|
813
987
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -896,6 +1070,7 @@ export type BenchmarksLaunchResponse = {
|
|
|
896
1070
|
"error": string | null;
|
|
897
1071
|
"preparation"?: {
|
|
898
1072
|
"since": string;
|
|
1073
|
+
"progressAt"?: string;
|
|
899
1074
|
"launchedBy": {
|
|
900
1075
|
"userId"?: string;
|
|
901
1076
|
"role"?: string;
|
|
@@ -904,6 +1079,12 @@ export type BenchmarksLaunchResponse = {
|
|
|
904
1079
|
"imageKey": string;
|
|
905
1080
|
"buildId": string;
|
|
906
1081
|
"stage"?: "image" | "snapshot";
|
|
1082
|
+
"retry"?: {
|
|
1083
|
+
"failedAttempts": number;
|
|
1084
|
+
"maxAttempts": number;
|
|
1085
|
+
"notBefore": string;
|
|
1086
|
+
"lastFailure": string;
|
|
1087
|
+
};
|
|
907
1088
|
}>;
|
|
908
1089
|
} | null;
|
|
909
1090
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -995,6 +1176,7 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
995
1176
|
"error": string | null;
|
|
996
1177
|
"preparation"?: {
|
|
997
1178
|
"since": string;
|
|
1179
|
+
"progressAt"?: string;
|
|
998
1180
|
"launchedBy": {
|
|
999
1181
|
"userId"?: string;
|
|
1000
1182
|
"role"?: string;
|
|
@@ -1003,6 +1185,12 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
1003
1185
|
"imageKey": string;
|
|
1004
1186
|
"buildId": string;
|
|
1005
1187
|
"stage"?: "image" | "snapshot";
|
|
1188
|
+
"retry"?: {
|
|
1189
|
+
"failedAttempts": number;
|
|
1190
|
+
"maxAttempts": number;
|
|
1191
|
+
"notBefore": string;
|
|
1192
|
+
"lastFailure": string;
|
|
1193
|
+
};
|
|
1006
1194
|
}>;
|
|
1007
1195
|
} | null;
|
|
1008
1196
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1139,6 +1327,9 @@ export type BenchmarksPreviewResponse = {
|
|
|
1139
1327
|
"pairGradedAt": string | null;
|
|
1140
1328
|
"evalScore": number | null;
|
|
1141
1329
|
"evalGradedAt": string | null;
|
|
1330
|
+
"executionBudgetS"?: number | null;
|
|
1331
|
+
"harnessImageDigest"?: string | null;
|
|
1332
|
+
"subjectEngaged"?: boolean | null;
|
|
1142
1333
|
"runCreatedAt": string | null;
|
|
1143
1334
|
"startedAt": string | null;
|
|
1144
1335
|
"finishedAt": string | null;
|
|
@@ -1359,6 +1550,17 @@ export type BenchmarksPreviewResponse = {
|
|
|
1359
1550
|
"ci": Array<unknown> | null;
|
|
1360
1551
|
"n": number;
|
|
1361
1552
|
};
|
|
1553
|
+
"abstention"?: {
|
|
1554
|
+
"value": number | null;
|
|
1555
|
+
"ci": Array<unknown> | null;
|
|
1556
|
+
"n": number;
|
|
1557
|
+
"parts"?: Array<{
|
|
1558
|
+
"id": string;
|
|
1559
|
+
"label": string;
|
|
1560
|
+
"value": number | null;
|
|
1561
|
+
"n": number;
|
|
1562
|
+
}>;
|
|
1563
|
+
};
|
|
1362
1564
|
"tasks": number;
|
|
1363
1565
|
"graded": number;
|
|
1364
1566
|
"unmeasured": number;
|
|
@@ -1370,6 +1572,17 @@ export type BenchmarksPreviewResponse = {
|
|
|
1370
1572
|
"ci": Array<unknown> | null;
|
|
1371
1573
|
"n": number;
|
|
1372
1574
|
};
|
|
1575
|
+
"abstention"?: {
|
|
1576
|
+
"value": number | null;
|
|
1577
|
+
"ci": Array<unknown> | null;
|
|
1578
|
+
"n": number;
|
|
1579
|
+
"parts"?: Array<{
|
|
1580
|
+
"id": string;
|
|
1581
|
+
"label": string;
|
|
1582
|
+
"value": number | null;
|
|
1583
|
+
"n": number;
|
|
1584
|
+
}>;
|
|
1585
|
+
};
|
|
1373
1586
|
"graded": number;
|
|
1374
1587
|
"unmeasured": number;
|
|
1375
1588
|
"modelJudged": boolean;
|
|
@@ -1463,6 +1676,7 @@ export type BenchmarksPublishResponse = {
|
|
|
1463
1676
|
"error": string | null;
|
|
1464
1677
|
"preparation"?: {
|
|
1465
1678
|
"since": string;
|
|
1679
|
+
"progressAt"?: string;
|
|
1466
1680
|
"launchedBy": {
|
|
1467
1681
|
"userId"?: string;
|
|
1468
1682
|
"role"?: string;
|
|
@@ -1471,6 +1685,12 @@ export type BenchmarksPublishResponse = {
|
|
|
1471
1685
|
"imageKey": string;
|
|
1472
1686
|
"buildId": string;
|
|
1473
1687
|
"stage"?: "image" | "snapshot";
|
|
1688
|
+
"retry"?: {
|
|
1689
|
+
"failedAttempts": number;
|
|
1690
|
+
"maxAttempts": number;
|
|
1691
|
+
"notBefore": string;
|
|
1692
|
+
"lastFailure": string;
|
|
1693
|
+
};
|
|
1474
1694
|
}>;
|
|
1475
1695
|
} | null;
|
|
1476
1696
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1554,6 +1774,7 @@ export type BenchmarksRegradeResponse = {
|
|
|
1554
1774
|
"error": string | null;
|
|
1555
1775
|
"preparation"?: {
|
|
1556
1776
|
"since": string;
|
|
1777
|
+
"progressAt"?: string;
|
|
1557
1778
|
"launchedBy": {
|
|
1558
1779
|
"userId"?: string;
|
|
1559
1780
|
"role"?: string;
|
|
@@ -1562,6 +1783,12 @@ export type BenchmarksRegradeResponse = {
|
|
|
1562
1783
|
"imageKey": string;
|
|
1563
1784
|
"buildId": string;
|
|
1564
1785
|
"stage"?: "image" | "snapshot";
|
|
1786
|
+
"retry"?: {
|
|
1787
|
+
"failedAttempts": number;
|
|
1788
|
+
"maxAttempts": number;
|
|
1789
|
+
"notBefore": string;
|
|
1790
|
+
"lastFailure": string;
|
|
1791
|
+
};
|
|
1565
1792
|
}>;
|
|
1566
1793
|
} | null;
|
|
1567
1794
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1669,6 +1896,7 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1669
1896
|
"error": string | null;
|
|
1670
1897
|
"preparation"?: {
|
|
1671
1898
|
"since": string;
|
|
1899
|
+
"progressAt"?: string;
|
|
1672
1900
|
"launchedBy": {
|
|
1673
1901
|
"userId"?: string;
|
|
1674
1902
|
"role"?: string;
|
|
@@ -1677,6 +1905,12 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1677
1905
|
"imageKey": string;
|
|
1678
1906
|
"buildId": string;
|
|
1679
1907
|
"stage"?: "image" | "snapshot";
|
|
1908
|
+
"retry"?: {
|
|
1909
|
+
"failedAttempts": number;
|
|
1910
|
+
"maxAttempts": number;
|
|
1911
|
+
"notBefore": string;
|
|
1912
|
+
"lastFailure": string;
|
|
1913
|
+
};
|
|
1680
1914
|
}>;
|
|
1681
1915
|
} | null;
|
|
1682
1916
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -1734,6 +1968,9 @@ export type ExperimentImagesBuildsListResponse = {
|
|
|
1734
1968
|
} | {
|
|
1735
1969
|
"kind": "inline";
|
|
1736
1970
|
"fileCount": number;
|
|
1971
|
+
} | {
|
|
1972
|
+
"kind": "asset";
|
|
1973
|
+
"sha256": string;
|
|
1737
1974
|
};
|
|
1738
1975
|
"sourceIdentity": string;
|
|
1739
1976
|
"dockerfileDir": string;
|
|
@@ -1741,7 +1978,7 @@ export type ExperimentImagesBuildsListResponse = {
|
|
|
1741
1978
|
"buildArgKeys": Array<string>;
|
|
1742
1979
|
"platform": string;
|
|
1743
1980
|
"trusted": boolean;
|
|
1744
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1981
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1745
1982
|
"image": string;
|
|
1746
1983
|
"digest": string | null;
|
|
1747
1984
|
"sizeBytes": number | null;
|
|
@@ -1782,7 +2019,7 @@ export type ExperimentImagesBuildsCreateRequest = {
|
|
|
1782
2019
|
};
|
|
1783
2020
|
export type ExperimentImagesBuildsCreateResponse = {
|
|
1784
2021
|
"id": string;
|
|
1785
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
2022
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1786
2023
|
};
|
|
1787
2024
|
export type ExperimentImagesBuildsGetResponse = {
|
|
1788
2025
|
"build": {
|
|
@@ -1804,6 +2041,9 @@ export type ExperimentImagesBuildsGetResponse = {
|
|
|
1804
2041
|
} | {
|
|
1805
2042
|
"kind": "inline";
|
|
1806
2043
|
"fileCount": number;
|
|
2044
|
+
} | {
|
|
2045
|
+
"kind": "asset";
|
|
2046
|
+
"sha256": string;
|
|
1807
2047
|
};
|
|
1808
2048
|
"sourceIdentity": string;
|
|
1809
2049
|
"dockerfileDir": string;
|
|
@@ -1811,7 +2051,7 @@ export type ExperimentImagesBuildsGetResponse = {
|
|
|
1811
2051
|
"buildArgKeys": Array<string>;
|
|
1812
2052
|
"platform": string;
|
|
1813
2053
|
"trusted": boolean;
|
|
1814
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
2054
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1815
2055
|
"image": string;
|
|
1816
2056
|
"digest": string | null;
|
|
1817
2057
|
"sizeBytes": number | null;
|
|
@@ -1825,7 +2065,7 @@ export type ExperimentImagesBuildsGetResponse = {
|
|
|
1825
2065
|
};
|
|
1826
2066
|
export type ExperimentImagesBuildsCancelResponse = {
|
|
1827
2067
|
"id": string;
|
|
1828
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
2068
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1829
2069
|
};
|
|
1830
2070
|
export type ExperimentImagesBuildsStreamResponse = unknown;
|
|
1831
2071
|
export type ExperimentImagesRunsCreateRequest = {
|
|
@@ -1882,6 +2122,7 @@ export type ExperimentRunsCreateRequest = {
|
|
|
1882
2122
|
"tools": Array<string>;
|
|
1883
2123
|
};
|
|
1884
2124
|
"requires_tools"?: boolean;
|
|
2125
|
+
"prompt_context"?: "none" | "target_site";
|
|
1885
2126
|
"mcp_servers"?: Array<{
|
|
1886
2127
|
"name": string;
|
|
1887
2128
|
"url": string;
|
|
@@ -2158,7 +2399,7 @@ export type WebmcpArenaCreateRequest = {
|
|
|
2158
2399
|
"name"?: string;
|
|
2159
2400
|
};
|
|
2160
2401
|
export type WebmcpArenaCreateResponse = {
|
|
2161
|
-
"
|
|
2402
|
+
"benchmark": {
|
|
2162
2403
|
"id": string;
|
|
2163
2404
|
"executiveSummary"?: string | null;
|
|
2164
2405
|
"plannedRunCount": number;
|
|
@@ -2224,6 +2465,7 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2224
2465
|
"error": string | null;
|
|
2225
2466
|
"preparation"?: {
|
|
2226
2467
|
"since": string;
|
|
2468
|
+
"progressAt"?: string;
|
|
2227
2469
|
"launchedBy": {
|
|
2228
2470
|
"userId"?: string;
|
|
2229
2471
|
"role"?: string;
|
|
@@ -2232,6 +2474,12 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2232
2474
|
"imageKey": string;
|
|
2233
2475
|
"buildId": string;
|
|
2234
2476
|
"stage"?: "image" | "snapshot";
|
|
2477
|
+
"retry"?: {
|
|
2478
|
+
"failedAttempts": number;
|
|
2479
|
+
"maxAttempts": number;
|
|
2480
|
+
"notBefore": string;
|
|
2481
|
+
"lastFailure": string;
|
|
2482
|
+
};
|
|
2235
2483
|
}>;
|
|
2236
2484
|
} | null;
|
|
2237
2485
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -2248,10 +2496,6 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2248
2496
|
"liveUntil": string | null;
|
|
2249
2497
|
};
|
|
2250
2498
|
"streamUrl": string;
|
|
2251
|
-
"warnings"?: Array<{
|
|
2252
|
-
"harness": string;
|
|
2253
|
-
"kind": string;
|
|
2254
|
-
}>;
|
|
2255
2499
|
};
|
|
2256
2500
|
export type WebmcpRunsBrowserRecordingResponse = {
|
|
2257
2501
|
"runId": string;
|
|
@@ -2500,6 +2744,18 @@ export type IntentsBulkCreateRequest = {
|
|
|
2500
2744
|
"goal"?: {
|
|
2501
2745
|
"statement": string;
|
|
2502
2746
|
};
|
|
2747
|
+
"agents"?: Array<{
|
|
2748
|
+
"harness": string;
|
|
2749
|
+
"model": string;
|
|
2750
|
+
"mcp_server_ids"?: Array<string>;
|
|
2751
|
+
"skill_ids"?: Array<string>;
|
|
2752
|
+
"browser"?: {
|
|
2753
|
+
"webdriver"?: boolean;
|
|
2754
|
+
"webmcp"?: boolean;
|
|
2755
|
+
"webmcpTargetUrl"?: string;
|
|
2756
|
+
"webmcpSurface"?: "native" | "shim";
|
|
2757
|
+
};
|
|
2758
|
+
}>;
|
|
2503
2759
|
"prompts": Array<{
|
|
2504
2760
|
"text": string;
|
|
2505
2761
|
"category"?: string;
|
|
@@ -2509,6 +2765,7 @@ export type IntentsBulkCreateRequest = {
|
|
|
2509
2765
|
}>;
|
|
2510
2766
|
"domainId"?: string;
|
|
2511
2767
|
"icpId"?: string;
|
|
2768
|
+
"firstMeasurement"?: "written" | "intent";
|
|
2512
2769
|
};
|
|
2513
2770
|
export type IntentsBulkCreateResponse = {
|
|
2514
2771
|
"success": true;
|
|
@@ -2518,6 +2775,13 @@ export type IntentsBulkCreateResponse = {
|
|
|
2518
2775
|
"title": string;
|
|
2519
2776
|
"merged": boolean;
|
|
2520
2777
|
"promptsCreated": number;
|
|
2778
|
+
"setupsApplied"?: boolean;
|
|
2779
|
+
"measurementStart": {
|
|
2780
|
+
"started": true;
|
|
2781
|
+
} | {
|
|
2782
|
+
"started": false;
|
|
2783
|
+
"reason": "no_attributable_member";
|
|
2784
|
+
};
|
|
2521
2785
|
}>;
|
|
2522
2786
|
"truncated": number;
|
|
2523
2787
|
};
|
|
@@ -2713,6 +2977,12 @@ export type IntentsGenerateResponse = {
|
|
|
2713
2977
|
"tracked": boolean;
|
|
2714
2978
|
}>;
|
|
2715
2979
|
}>;
|
|
2980
|
+
"measurementStart": {
|
|
2981
|
+
"started": true;
|
|
2982
|
+
} | {
|
|
2983
|
+
"started": false;
|
|
2984
|
+
"reason": "no_attributable_member";
|
|
2985
|
+
};
|
|
2716
2986
|
}>;
|
|
2717
2987
|
};
|
|
2718
2988
|
export type IntentsSetIcpRequest = {
|
|
@@ -2824,6 +3094,12 @@ export type IntentsAddPromptResponse = {
|
|
|
2824
3094
|
"tracked": boolean;
|
|
2825
3095
|
}>;
|
|
2826
3096
|
}>;
|
|
3097
|
+
"measurementStart": {
|
|
3098
|
+
"started": true;
|
|
3099
|
+
} | {
|
|
3100
|
+
"started": false;
|
|
3101
|
+
"reason": "no_attributable_member";
|
|
3102
|
+
};
|
|
2827
3103
|
};
|
|
2828
3104
|
};
|
|
2829
3105
|
export type IntentsUpdatePromptRequest = {
|
|
@@ -2842,6 +3118,17 @@ export type IntentsDeletePromptResponse = {
|
|
|
2842
3118
|
"id": string;
|
|
2843
3119
|
};
|
|
2844
3120
|
};
|
|
3121
|
+
export type IntentsSetPromptGoalRequest = {
|
|
3122
|
+
"active": boolean;
|
|
3123
|
+
};
|
|
3124
|
+
export type IntentsSetPromptGoalResponse = {
|
|
3125
|
+
"success": true;
|
|
3126
|
+
"data": {
|
|
3127
|
+
"promptId": string;
|
|
3128
|
+
"kind": "discover" | "choose" | "verify" | "inform" | "guide" | "act" | "other";
|
|
3129
|
+
"active": boolean;
|
|
3130
|
+
};
|
|
3131
|
+
};
|
|
2845
3132
|
export type IntentsRepresentationResponse = {
|
|
2846
3133
|
"success": true;
|
|
2847
3134
|
"data": {
|
|
@@ -3063,6 +3350,7 @@ export type StudioRunsCreateRequest = {
|
|
|
3063
3350
|
"tools": Array<string>;
|
|
3064
3351
|
};
|
|
3065
3352
|
"requires_tools"?: boolean;
|
|
3353
|
+
"prompt_context"?: "none" | "target_site";
|
|
3066
3354
|
"mcp_servers"?: Array<{
|
|
3067
3355
|
"name": string;
|
|
3068
3356
|
"url": string;
|
|
@@ -3145,6 +3433,7 @@ export type StudyAddonsListResponse = {
|
|
|
3145
3433
|
"surfaces": Array<"run" | "session">;
|
|
3146
3434
|
"harnesses": Array<string>;
|
|
3147
3435
|
"multiInstance"?: boolean;
|
|
3436
|
+
"credentialSourceFields"?: Array<string>;
|
|
3148
3437
|
}>;
|
|
3149
3438
|
};
|
|
3150
3439
|
export type StudyBenchmarkDefinitionsListResponse = {
|
|
@@ -3163,6 +3452,13 @@ export type StudyBenchmarkDefinitionsListResponse = {
|
|
|
3163
3452
|
"modelParams"?: Array<string>;
|
|
3164
3453
|
"maxPlannedRuns"?: number;
|
|
3165
3454
|
"sampleFloor"?: number;
|
|
3455
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3456
|
+
"aggregation"?: "included" | "excluded";
|
|
3457
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3458
|
+
"aggregateTaskSample"?: {
|
|
3459
|
+
"count": number;
|
|
3460
|
+
"seed": string;
|
|
3461
|
+
};
|
|
3166
3462
|
};
|
|
3167
3463
|
"defaultConfig": {
|
|
3168
3464
|
"matrix"?: Array<{
|
|
@@ -3229,6 +3525,7 @@ export type StudyBenchmarkDefinitionsListResponse = {
|
|
|
3229
3525
|
"userId": string;
|
|
3230
3526
|
"displayName": string | null;
|
|
3231
3527
|
} | null;
|
|
3528
|
+
"uploadedByPublisher": boolean | null;
|
|
3232
3529
|
"notices": Array<{
|
|
3233
3530
|
"kind": string;
|
|
3234
3531
|
"text": string;
|
|
@@ -3266,6 +3563,13 @@ export type StudyBenchmarkDefinitionsCreateRequest = {
|
|
|
3266
3563
|
"modelParams"?: Array<string>;
|
|
3267
3564
|
"maxPlannedRuns"?: number;
|
|
3268
3565
|
"sampleFloor"?: number;
|
|
3566
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3567
|
+
"aggregation"?: "included" | "excluded";
|
|
3568
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3569
|
+
"aggregateTaskSample"?: {
|
|
3570
|
+
"count": number;
|
|
3571
|
+
"seed": string;
|
|
3572
|
+
};
|
|
3269
3573
|
};
|
|
3270
3574
|
"runnerKind"?: "none" | "image" | "environment" | "framework";
|
|
3271
3575
|
"runnerRecipe"?: Record<string, unknown> | null;
|
|
@@ -3308,6 +3612,13 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
|
|
|
3308
3612
|
"modelParams"?: Array<string>;
|
|
3309
3613
|
"maxPlannedRuns"?: number;
|
|
3310
3614
|
"sampleFloor"?: number;
|
|
3615
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3616
|
+
"aggregation"?: "included" | "excluded";
|
|
3617
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3618
|
+
"aggregateTaskSample"?: {
|
|
3619
|
+
"count": number;
|
|
3620
|
+
"seed": string;
|
|
3621
|
+
};
|
|
3311
3622
|
};
|
|
3312
3623
|
"defaultConfig": {
|
|
3313
3624
|
"matrix"?: Array<{
|
|
@@ -3374,6 +3685,7 @@ export type StudyBenchmarkDefinitionsCreateResponse = {
|
|
|
3374
3685
|
"userId": string;
|
|
3375
3686
|
"displayName": string | null;
|
|
3376
3687
|
} | null;
|
|
3688
|
+
"uploadedByPublisher": boolean | null;
|
|
3377
3689
|
"notices": Array<{
|
|
3378
3690
|
"kind": string;
|
|
3379
3691
|
"text": string;
|
|
@@ -3396,6 +3708,13 @@ export type StudyBenchmarkDefinitionsGetResponse = {
|
|
|
3396
3708
|
"modelParams"?: Array<string>;
|
|
3397
3709
|
"maxPlannedRuns"?: number;
|
|
3398
3710
|
"sampleFloor"?: number;
|
|
3711
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3712
|
+
"aggregation"?: "included" | "excluded";
|
|
3713
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3714
|
+
"aggregateTaskSample"?: {
|
|
3715
|
+
"count": number;
|
|
3716
|
+
"seed": string;
|
|
3717
|
+
};
|
|
3399
3718
|
};
|
|
3400
3719
|
"defaultConfig": {
|
|
3401
3720
|
"matrix"?: Array<{
|
|
@@ -3462,6 +3781,7 @@ export type StudyBenchmarkDefinitionsGetResponse = {
|
|
|
3462
3781
|
"userId": string;
|
|
3463
3782
|
"displayName": string | null;
|
|
3464
3783
|
} | null;
|
|
3784
|
+
"uploadedByPublisher": boolean | null;
|
|
3465
3785
|
"notices": Array<{
|
|
3466
3786
|
"kind": string;
|
|
3467
3787
|
"text": string;
|
|
@@ -3493,6 +3813,13 @@ export type StudyBenchmarkDefinitionsUpdateRequest = {
|
|
|
3493
3813
|
"modelParams"?: Array<string>;
|
|
3494
3814
|
"maxPlannedRuns"?: number;
|
|
3495
3815
|
"sampleFloor"?: number;
|
|
3816
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3817
|
+
"aggregation"?: "included" | "excluded";
|
|
3818
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3819
|
+
"aggregateTaskSample"?: {
|
|
3820
|
+
"count": number;
|
|
3821
|
+
"seed": string;
|
|
3822
|
+
};
|
|
3496
3823
|
};
|
|
3497
3824
|
"runnerKind"?: "none" | "image" | "environment" | "framework";
|
|
3498
3825
|
"runnerRecipe"?: Record<string, unknown> | null;
|
|
@@ -3535,6 +3862,13 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
|
|
|
3535
3862
|
"modelParams"?: Array<string>;
|
|
3536
3863
|
"maxPlannedRuns"?: number;
|
|
3537
3864
|
"sampleFloor"?: number;
|
|
3865
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3866
|
+
"aggregation"?: "included" | "excluded";
|
|
3867
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3868
|
+
"aggregateTaskSample"?: {
|
|
3869
|
+
"count": number;
|
|
3870
|
+
"seed": string;
|
|
3871
|
+
};
|
|
3538
3872
|
};
|
|
3539
3873
|
"defaultConfig": {
|
|
3540
3874
|
"matrix"?: Array<{
|
|
@@ -3601,6 +3935,7 @@ export type StudyBenchmarkDefinitionsUpdateResponse = {
|
|
|
3601
3935
|
"userId": string;
|
|
3602
3936
|
"displayName": string | null;
|
|
3603
3937
|
} | null;
|
|
3938
|
+
"uploadedByPublisher": boolean | null;
|
|
3604
3939
|
"notices": Array<{
|
|
3605
3940
|
"kind": string;
|
|
3606
3941
|
"text": string;
|
|
@@ -3645,6 +3980,13 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
|
|
|
3645
3980
|
"modelParams"?: Array<string>;
|
|
3646
3981
|
"maxPlannedRuns"?: number;
|
|
3647
3982
|
"sampleFloor"?: number;
|
|
3983
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
3984
|
+
"aggregation"?: "included" | "excluded";
|
|
3985
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
3986
|
+
"aggregateTaskSample"?: {
|
|
3987
|
+
"count": number;
|
|
3988
|
+
"seed": string;
|
|
3989
|
+
};
|
|
3648
3990
|
};
|
|
3649
3991
|
"defaultConfig": {
|
|
3650
3992
|
"matrix"?: Array<{
|
|
@@ -3711,6 +4053,7 @@ export type StudyBenchmarkDefinitionsSubmitResponse = {
|
|
|
3711
4053
|
"userId": string;
|
|
3712
4054
|
"displayName": string | null;
|
|
3713
4055
|
} | null;
|
|
4056
|
+
"uploadedByPublisher": boolean | null;
|
|
3714
4057
|
"notices": Array<{
|
|
3715
4058
|
"kind": string;
|
|
3716
4059
|
"text": string;
|
|
@@ -3736,6 +4079,13 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
|
|
|
3736
4079
|
"modelParams"?: Array<string>;
|
|
3737
4080
|
"maxPlannedRuns"?: number;
|
|
3738
4081
|
"sampleFloor"?: number;
|
|
4082
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
4083
|
+
"aggregation"?: "included" | "excluded";
|
|
4084
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
4085
|
+
"aggregateTaskSample"?: {
|
|
4086
|
+
"count": number;
|
|
4087
|
+
"seed": string;
|
|
4088
|
+
};
|
|
3739
4089
|
};
|
|
3740
4090
|
"defaultConfig": {
|
|
3741
4091
|
"matrix"?: Array<{
|
|
@@ -3802,6 +4152,7 @@ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
|
|
|
3802
4152
|
"userId": string;
|
|
3803
4153
|
"displayName": string | null;
|
|
3804
4154
|
} | null;
|
|
4155
|
+
"uploadedByPublisher": boolean | null;
|
|
3805
4156
|
"notices": Array<{
|
|
3806
4157
|
"kind": string;
|
|
3807
4158
|
"text": string;
|
|
@@ -3824,6 +4175,13 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
|
|
|
3824
4175
|
"modelParams"?: Array<string>;
|
|
3825
4176
|
"maxPlannedRuns"?: number;
|
|
3826
4177
|
"sampleFloor"?: number;
|
|
4178
|
+
"missingAnswer"?: "unmeasured" | "wrong" | "grade_full_response";
|
|
4179
|
+
"aggregation"?: "included" | "excluded";
|
|
4180
|
+
"aggregationReason"?: "smoke_test" | "suite_of_benchmarks";
|
|
4181
|
+
"aggregateTaskSample"?: {
|
|
4182
|
+
"count": number;
|
|
4183
|
+
"seed": string;
|
|
4184
|
+
};
|
|
3827
4185
|
};
|
|
3828
4186
|
"defaultConfig": {
|
|
3829
4187
|
"matrix"?: Array<{
|
|
@@ -3890,6 +4248,7 @@ export type StudyBenchmarkDefinitionsWithdrawResponse = {
|
|
|
3890
4248
|
"userId": string;
|
|
3891
4249
|
"displayName": string | null;
|
|
3892
4250
|
} | null;
|
|
4251
|
+
"uploadedByPublisher": boolean | null;
|
|
3893
4252
|
"notices": Array<{
|
|
3894
4253
|
"kind": string;
|
|
3895
4254
|
"text": string;
|
|
@@ -4068,6 +4427,9 @@ export type StudyImagesBuildsListResponse = {
|
|
|
4068
4427
|
} | {
|
|
4069
4428
|
"kind": "inline";
|
|
4070
4429
|
"fileCount": number;
|
|
4430
|
+
} | {
|
|
4431
|
+
"kind": "asset";
|
|
4432
|
+
"sha256": string;
|
|
4071
4433
|
};
|
|
4072
4434
|
"sourceIdentity": string;
|
|
4073
4435
|
"dockerfileDir": string;
|
|
@@ -4075,7 +4437,7 @@ export type StudyImagesBuildsListResponse = {
|
|
|
4075
4437
|
"buildArgKeys": Array<string>;
|
|
4076
4438
|
"platform": string;
|
|
4077
4439
|
"trusted": boolean;
|
|
4078
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4440
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4079
4441
|
"image": string;
|
|
4080
4442
|
"digest": string | null;
|
|
4081
4443
|
"sizeBytes": number | null;
|
|
@@ -4116,7 +4478,7 @@ export type StudyImagesBuildsCreateRequest = {
|
|
|
4116
4478
|
};
|
|
4117
4479
|
export type StudyImagesBuildsCreateResponse = {
|
|
4118
4480
|
"id": string;
|
|
4119
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4481
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4120
4482
|
};
|
|
4121
4483
|
export type StudyImagesBuildsGetResponse = {
|
|
4122
4484
|
"build": {
|
|
@@ -4138,6 +4500,9 @@ export type StudyImagesBuildsGetResponse = {
|
|
|
4138
4500
|
} | {
|
|
4139
4501
|
"kind": "inline";
|
|
4140
4502
|
"fileCount": number;
|
|
4503
|
+
} | {
|
|
4504
|
+
"kind": "asset";
|
|
4505
|
+
"sha256": string;
|
|
4141
4506
|
};
|
|
4142
4507
|
"sourceIdentity": string;
|
|
4143
4508
|
"dockerfileDir": string;
|
|
@@ -4145,7 +4510,7 @@ export type StudyImagesBuildsGetResponse = {
|
|
|
4145
4510
|
"buildArgKeys": Array<string>;
|
|
4146
4511
|
"platform": string;
|
|
4147
4512
|
"trusted": boolean;
|
|
4148
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4513
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4149
4514
|
"image": string;
|
|
4150
4515
|
"digest": string | null;
|
|
4151
4516
|
"sizeBytes": number | null;
|
|
@@ -4159,7 +4524,7 @@ export type StudyImagesBuildsGetResponse = {
|
|
|
4159
4524
|
};
|
|
4160
4525
|
export type StudyImagesBuildsCancelResponse = {
|
|
4161
4526
|
"id": string;
|
|
4162
|
-
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4527
|
+
"phase": "queued" | "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4163
4528
|
};
|
|
4164
4529
|
export type StudyImagesBuildsStreamResponse = unknown;
|
|
4165
4530
|
export type StudyImagesRunsCreateRequest = {
|
|
@@ -4216,6 +4581,7 @@ export type StudyRunsCreateRequest = {
|
|
|
4216
4581
|
"tools": Array<string>;
|
|
4217
4582
|
};
|
|
4218
4583
|
"requires_tools"?: boolean;
|
|
4584
|
+
"prompt_context"?: "none" | "target_site";
|
|
4219
4585
|
"mcp_servers"?: Array<{
|
|
4220
4586
|
"name": string;
|
|
4221
4587
|
"url": string;
|
|
@@ -4394,7 +4760,7 @@ export type StudyListResponse = {
|
|
|
4394
4760
|
"addons"?: Record<string, unknown>;
|
|
4395
4761
|
};
|
|
4396
4762
|
};
|
|
4397
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
4763
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4398
4764
|
"subjectBackend": "model" | "harness" | null;
|
|
4399
4765
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4400
4766
|
"importedFrom"?: string;
|
|
@@ -4430,6 +4796,7 @@ export type StudyListResponse = {
|
|
|
4430
4796
|
"error": string | null;
|
|
4431
4797
|
"preparation"?: {
|
|
4432
4798
|
"since": string;
|
|
4799
|
+
"progressAt"?: string;
|
|
4433
4800
|
"launchedBy": {
|
|
4434
4801
|
"userId"?: string;
|
|
4435
4802
|
"role"?: string;
|
|
@@ -4438,6 +4805,12 @@ export type StudyListResponse = {
|
|
|
4438
4805
|
"imageKey": string;
|
|
4439
4806
|
"buildId": string;
|
|
4440
4807
|
"stage"?: "image" | "snapshot";
|
|
4808
|
+
"retry"?: {
|
|
4809
|
+
"failedAttempts": number;
|
|
4810
|
+
"maxAttempts": number;
|
|
4811
|
+
"notBefore": string;
|
|
4812
|
+
"lastFailure": string;
|
|
4813
|
+
};
|
|
4441
4814
|
}>;
|
|
4442
4815
|
} | null;
|
|
4443
4816
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -4452,6 +4825,38 @@ export type StudyListResponse = {
|
|
|
4452
4825
|
"createdAt": string;
|
|
4453
4826
|
"updatedAt": string;
|
|
4454
4827
|
"liveUntil": string | null;
|
|
4828
|
+
"arms"?: Array<{
|
|
4829
|
+
"key": string;
|
|
4830
|
+
"position": number;
|
|
4831
|
+
"label": string;
|
|
4832
|
+
"role": "reference" | "peer";
|
|
4833
|
+
"payload": {
|
|
4834
|
+
"tools"?: "default" | "none" | {
|
|
4835
|
+
"tools": Array<string>;
|
|
4836
|
+
};
|
|
4837
|
+
"addons"?: Record<string, unknown>;
|
|
4838
|
+
};
|
|
4839
|
+
"cells"?: Array<{
|
|
4840
|
+
"harness": string;
|
|
4841
|
+
"model": string;
|
|
4842
|
+
"provider"?: string;
|
|
4843
|
+
"params"?: {
|
|
4844
|
+
"temperature"?: number;
|
|
4845
|
+
"maxOutputTokens"?: number;
|
|
4846
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4847
|
+
"thinkingBudgetTokens"?: number;
|
|
4848
|
+
};
|
|
4849
|
+
}>;
|
|
4850
|
+
"skills"?: Array<{
|
|
4851
|
+
"ref": string;
|
|
4852
|
+
"name": string;
|
|
4853
|
+
"changedSinceCreate"?: boolean;
|
|
4854
|
+
}>;
|
|
4855
|
+
}>;
|
|
4856
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
4857
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
4858
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
4859
|
+
"modeOptions"?: Record<string, unknown>;
|
|
4455
4860
|
}>;
|
|
4456
4861
|
};
|
|
4457
4862
|
export type StudyCreateRequest = {
|
|
@@ -4478,10 +4883,34 @@ export type StudyCreateRequest = {
|
|
|
4478
4883
|
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4479
4884
|
"judgeSupplementary"?: boolean;
|
|
4480
4885
|
"proposeExpectations"?: boolean;
|
|
4481
|
-
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
4886
|
+
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4482
4887
|
"subjectBackend"?: "model" | "harness";
|
|
4483
4888
|
"baselineLabel"?: string;
|
|
4484
4889
|
"treatmentLabel"?: string;
|
|
4890
|
+
"arms"?: Array<{
|
|
4891
|
+
"key"?: string;
|
|
4892
|
+
"label": string;
|
|
4893
|
+
"role"?: "reference" | "peer";
|
|
4894
|
+
"config": {
|
|
4895
|
+
"tools"?: "default" | "none" | {
|
|
4896
|
+
"tools": Array<string>;
|
|
4897
|
+
};
|
|
4898
|
+
"addons"?: Record<string, unknown>;
|
|
4899
|
+
};
|
|
4900
|
+
"cells"?: Array<{
|
|
4901
|
+
"harness": string;
|
|
4902
|
+
"model": string;
|
|
4903
|
+
"provider"?: string;
|
|
4904
|
+
"params"?: {
|
|
4905
|
+
"temperature"?: number;
|
|
4906
|
+
"maxOutputTokens"?: number;
|
|
4907
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4908
|
+
"thinkingBudgetTokens"?: number;
|
|
4909
|
+
};
|
|
4910
|
+
}>;
|
|
4911
|
+
}>;
|
|
4912
|
+
"contrasts"?: "vs_reference" | "all_pairs";
|
|
4913
|
+
"modeOptions"?: Record<string, unknown>;
|
|
4485
4914
|
"taskCount"?: number;
|
|
4486
4915
|
"variantsPerTask"?: number;
|
|
4487
4916
|
"repeats"?: number;
|
|
@@ -4492,7 +4921,75 @@ export type StudyCreateRequest = {
|
|
|
4492
4921
|
"requiresWeb"?: boolean;
|
|
4493
4922
|
"rubric"?: Array<string>;
|
|
4494
4923
|
"variants"?: Array<string>;
|
|
4495
|
-
"expectations"?: Array<
|
|
4924
|
+
"expectations"?: Array<{
|
|
4925
|
+
"label"?: string;
|
|
4926
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4927
|
+
"kind": "mcp_tool_called";
|
|
4928
|
+
"server"?: string;
|
|
4929
|
+
"tool": string;
|
|
4930
|
+
"arguments"?: Record<string, unknown>;
|
|
4931
|
+
"minCalls"?: number;
|
|
4932
|
+
} | {
|
|
4933
|
+
"label"?: string;
|
|
4934
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4935
|
+
"kind": "tool_not_called";
|
|
4936
|
+
"tool"?: string;
|
|
4937
|
+
"mcp"?: {
|
|
4938
|
+
"server"?: string;
|
|
4939
|
+
"tool": string;
|
|
4940
|
+
};
|
|
4941
|
+
"pattern"?: string;
|
|
4942
|
+
} | {
|
|
4943
|
+
"label"?: string;
|
|
4944
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4945
|
+
"kind": "command_ran";
|
|
4946
|
+
"pattern": string;
|
|
4947
|
+
} | {
|
|
4948
|
+
"label"?: string;
|
|
4949
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4950
|
+
"kind": "skill_loaded";
|
|
4951
|
+
"slug": string;
|
|
4952
|
+
} | {
|
|
4953
|
+
"label"?: string;
|
|
4954
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4955
|
+
"kind": "file_operation";
|
|
4956
|
+
"op"?: "read" | "write" | "edit";
|
|
4957
|
+
"path": unknown;
|
|
4958
|
+
} | {
|
|
4959
|
+
"label"?: string;
|
|
4960
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4961
|
+
"kind": "web_fetched";
|
|
4962
|
+
"urlHost"?: string;
|
|
4963
|
+
"urlPattern"?: string;
|
|
4964
|
+
} | {
|
|
4965
|
+
"label"?: string;
|
|
4966
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4967
|
+
"kind": "response_contains";
|
|
4968
|
+
"text": string;
|
|
4969
|
+
"normalize"?: "prose";
|
|
4970
|
+
} | {
|
|
4971
|
+
"label"?: string;
|
|
4972
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4973
|
+
"kind": "response_matches";
|
|
4974
|
+
"pattern": string;
|
|
4975
|
+
"ignoreCase"?: boolean;
|
|
4976
|
+
} | {
|
|
4977
|
+
"label"?: string;
|
|
4978
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4979
|
+
"kind": "token_budget";
|
|
4980
|
+
"maxOutputTokens"?: number;
|
|
4981
|
+
"maxTotalTokens"?: number;
|
|
4982
|
+
} | {
|
|
4983
|
+
"label"?: string;
|
|
4984
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4985
|
+
"kind": "sequence";
|
|
4986
|
+
"of": Array<unknown>;
|
|
4987
|
+
} | {
|
|
4988
|
+
"label"?: string;
|
|
4989
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
4990
|
+
"kind": "any_of";
|
|
4991
|
+
"of": Array<unknown>;
|
|
4992
|
+
}>;
|
|
4496
4993
|
}>;
|
|
4497
4994
|
"subset"?: {
|
|
4498
4995
|
"count": number;
|
|
@@ -4542,7 +5039,7 @@ export type StudyCreateResponse = {
|
|
|
4542
5039
|
"addons"?: Record<string, unknown>;
|
|
4543
5040
|
};
|
|
4544
5041
|
};
|
|
4545
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5042
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4546
5043
|
"subjectBackend": "model" | "harness" | null;
|
|
4547
5044
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4548
5045
|
"importedFrom"?: string;
|
|
@@ -4578,6 +5075,7 @@ export type StudyCreateResponse = {
|
|
|
4578
5075
|
"error": string | null;
|
|
4579
5076
|
"preparation"?: {
|
|
4580
5077
|
"since": string;
|
|
5078
|
+
"progressAt"?: string;
|
|
4581
5079
|
"launchedBy": {
|
|
4582
5080
|
"userId"?: string;
|
|
4583
5081
|
"role"?: string;
|
|
@@ -4586,6 +5084,12 @@ export type StudyCreateResponse = {
|
|
|
4586
5084
|
"imageKey": string;
|
|
4587
5085
|
"buildId": string;
|
|
4588
5086
|
"stage"?: "image" | "snapshot";
|
|
5087
|
+
"retry"?: {
|
|
5088
|
+
"failedAttempts": number;
|
|
5089
|
+
"maxAttempts": number;
|
|
5090
|
+
"notBefore": string;
|
|
5091
|
+
"lastFailure": string;
|
|
5092
|
+
};
|
|
4589
5093
|
}>;
|
|
4590
5094
|
} | null;
|
|
4591
5095
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -4600,13 +5104,59 @@ export type StudyCreateResponse = {
|
|
|
4600
5104
|
"createdAt": string;
|
|
4601
5105
|
"updatedAt": string;
|
|
4602
5106
|
"liveUntil": string | null;
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
5107
|
+
"arms"?: Array<{
|
|
5108
|
+
"key": string;
|
|
5109
|
+
"position": number;
|
|
5110
|
+
"label": string;
|
|
5111
|
+
"role": "reference" | "peer";
|
|
5112
|
+
"payload": {
|
|
5113
|
+
"tools"?: "default" | "none" | {
|
|
5114
|
+
"tools": Array<string>;
|
|
5115
|
+
};
|
|
5116
|
+
"addons"?: Record<string, unknown>;
|
|
5117
|
+
};
|
|
5118
|
+
"cells"?: Array<{
|
|
5119
|
+
"harness": string;
|
|
5120
|
+
"model": string;
|
|
5121
|
+
"provider"?: string;
|
|
5122
|
+
"params"?: {
|
|
5123
|
+
"temperature"?: number;
|
|
5124
|
+
"maxOutputTokens"?: number;
|
|
5125
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5126
|
+
"thinkingBudgetTokens"?: number;
|
|
5127
|
+
};
|
|
5128
|
+
}>;
|
|
5129
|
+
"skills"?: Array<{
|
|
5130
|
+
"ref": string;
|
|
5131
|
+
"name": string;
|
|
5132
|
+
"changedSinceCreate"?: boolean;
|
|
5133
|
+
}>;
|
|
5134
|
+
}>;
|
|
5135
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
5136
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
5137
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
5138
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5139
|
+
};
|
|
5140
|
+
"streamUrl": string;
|
|
5141
|
+
"warnings"?: Array<{
|
|
5142
|
+
"harness": string;
|
|
5143
|
+
"kind": string;
|
|
5144
|
+
} | {
|
|
5145
|
+
"type": "design_aggregate_budget";
|
|
5146
|
+
"detail": string;
|
|
5147
|
+
}>;
|
|
5148
|
+
};
|
|
5149
|
+
export type StudyComparisonModesListResponse = {
|
|
5150
|
+
"modes": Array<{
|
|
5151
|
+
"id": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5152
|
+
"enabled": boolean;
|
|
5153
|
+
"authoring": boolean;
|
|
5154
|
+
}>;
|
|
5155
|
+
"cellPlans": Array<{
|
|
5156
|
+
"id": "crossed" | "per_setup";
|
|
5157
|
+
"enabled": boolean;
|
|
5158
|
+
}>;
|
|
5159
|
+
};
|
|
4610
5160
|
export type StudyConvertWebmcpSuiteRequest = {
|
|
4611
5161
|
"suite": unknown;
|
|
4612
5162
|
"ordered"?: boolean;
|
|
@@ -4619,6 +5169,225 @@ export type StudyConvertWebmcpSuiteResponse = {
|
|
|
4619
5169
|
}>;
|
|
4620
5170
|
"warnings": Array<string>;
|
|
4621
5171
|
};
|
|
5172
|
+
export type StudyPreflightRequest = {
|
|
5173
|
+
"name": string;
|
|
5174
|
+
"description"?: string;
|
|
5175
|
+
"subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5176
|
+
"skillRef"?: string;
|
|
5177
|
+
"definitionVersionId"?: string;
|
|
5178
|
+
"subjectName"?: string;
|
|
5179
|
+
"armConfig"?: {
|
|
5180
|
+
"baseline"?: {
|
|
5181
|
+
"tools"?: "default" | "none" | {
|
|
5182
|
+
"tools": Array<string>;
|
|
5183
|
+
};
|
|
5184
|
+
"addons"?: Record<string, unknown>;
|
|
5185
|
+
};
|
|
5186
|
+
"treatment"?: {
|
|
5187
|
+
"tools"?: "default" | "none" | {
|
|
5188
|
+
"tools": Array<string>;
|
|
5189
|
+
};
|
|
5190
|
+
"addons"?: Record<string, unknown>;
|
|
5191
|
+
};
|
|
5192
|
+
};
|
|
5193
|
+
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5194
|
+
"judgeSupplementary"?: boolean;
|
|
5195
|
+
"proposeExpectations"?: boolean;
|
|
5196
|
+
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5197
|
+
"subjectBackend"?: "model" | "harness";
|
|
5198
|
+
"baselineLabel"?: string;
|
|
5199
|
+
"treatmentLabel"?: string;
|
|
5200
|
+
"arms"?: Array<{
|
|
5201
|
+
"key"?: string;
|
|
5202
|
+
"label": string;
|
|
5203
|
+
"role"?: "reference" | "peer";
|
|
5204
|
+
"config": {
|
|
5205
|
+
"tools"?: "default" | "none" | {
|
|
5206
|
+
"tools": Array<string>;
|
|
5207
|
+
};
|
|
5208
|
+
"addons"?: Record<string, unknown>;
|
|
5209
|
+
};
|
|
5210
|
+
"cells"?: Array<{
|
|
5211
|
+
"harness": string;
|
|
5212
|
+
"model": string;
|
|
5213
|
+
"provider"?: string;
|
|
5214
|
+
"params"?: {
|
|
5215
|
+
"temperature"?: number;
|
|
5216
|
+
"maxOutputTokens"?: number;
|
|
5217
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5218
|
+
"thinkingBudgetTokens"?: number;
|
|
5219
|
+
};
|
|
5220
|
+
}>;
|
|
5221
|
+
}>;
|
|
5222
|
+
"contrasts"?: "vs_reference" | "all_pairs";
|
|
5223
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5224
|
+
"taskCount"?: number;
|
|
5225
|
+
"variantsPerTask"?: number;
|
|
5226
|
+
"repeats"?: number;
|
|
5227
|
+
"tasks"?: Array<{
|
|
5228
|
+
"prompt": string;
|
|
5229
|
+
"rationale"?: string;
|
|
5230
|
+
"category"?: string;
|
|
5231
|
+
"requiresWeb"?: boolean;
|
|
5232
|
+
"rubric"?: Array<string>;
|
|
5233
|
+
"variants"?: Array<string>;
|
|
5234
|
+
"expectations"?: Array<{
|
|
5235
|
+
"label"?: string;
|
|
5236
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5237
|
+
"kind": "mcp_tool_called";
|
|
5238
|
+
"server"?: string;
|
|
5239
|
+
"tool": string;
|
|
5240
|
+
"arguments"?: Record<string, unknown>;
|
|
5241
|
+
"minCalls"?: number;
|
|
5242
|
+
} | {
|
|
5243
|
+
"label"?: string;
|
|
5244
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5245
|
+
"kind": "tool_not_called";
|
|
5246
|
+
"tool"?: string;
|
|
5247
|
+
"mcp"?: {
|
|
5248
|
+
"server"?: string;
|
|
5249
|
+
"tool": string;
|
|
5250
|
+
};
|
|
5251
|
+
"pattern"?: string;
|
|
5252
|
+
} | {
|
|
5253
|
+
"label"?: string;
|
|
5254
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5255
|
+
"kind": "command_ran";
|
|
5256
|
+
"pattern": string;
|
|
5257
|
+
} | {
|
|
5258
|
+
"label"?: string;
|
|
5259
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5260
|
+
"kind": "skill_loaded";
|
|
5261
|
+
"slug": string;
|
|
5262
|
+
} | {
|
|
5263
|
+
"label"?: string;
|
|
5264
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5265
|
+
"kind": "file_operation";
|
|
5266
|
+
"op"?: "read" | "write" | "edit";
|
|
5267
|
+
"path": unknown;
|
|
5268
|
+
} | {
|
|
5269
|
+
"label"?: string;
|
|
5270
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5271
|
+
"kind": "web_fetched";
|
|
5272
|
+
"urlHost"?: string;
|
|
5273
|
+
"urlPattern"?: string;
|
|
5274
|
+
} | {
|
|
5275
|
+
"label"?: string;
|
|
5276
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5277
|
+
"kind": "response_contains";
|
|
5278
|
+
"text": string;
|
|
5279
|
+
"normalize"?: "prose";
|
|
5280
|
+
} | {
|
|
5281
|
+
"label"?: string;
|
|
5282
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5283
|
+
"kind": "response_matches";
|
|
5284
|
+
"pattern": string;
|
|
5285
|
+
"ignoreCase"?: boolean;
|
|
5286
|
+
} | {
|
|
5287
|
+
"label"?: string;
|
|
5288
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5289
|
+
"kind": "token_budget";
|
|
5290
|
+
"maxOutputTokens"?: number;
|
|
5291
|
+
"maxTotalTokens"?: number;
|
|
5292
|
+
} | {
|
|
5293
|
+
"label"?: string;
|
|
5294
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5295
|
+
"kind": "sequence";
|
|
5296
|
+
"of": Array<unknown>;
|
|
5297
|
+
} | {
|
|
5298
|
+
"label"?: string;
|
|
5299
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
5300
|
+
"kind": "any_of";
|
|
5301
|
+
"of": Array<unknown>;
|
|
5302
|
+
}>;
|
|
5303
|
+
}>;
|
|
5304
|
+
"subset"?: {
|
|
5305
|
+
"count": number;
|
|
5306
|
+
"seed"?: string;
|
|
5307
|
+
};
|
|
5308
|
+
"matrix": Array<{
|
|
5309
|
+
"harness": string;
|
|
5310
|
+
"model": string;
|
|
5311
|
+
"provider"?: string;
|
|
5312
|
+
"params"?: {
|
|
5313
|
+
"temperature"?: number;
|
|
5314
|
+
"maxOutputTokens"?: number;
|
|
5315
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5316
|
+
"thinkingBudgetTokens"?: number;
|
|
5317
|
+
};
|
|
5318
|
+
}>;
|
|
5319
|
+
"promise"?: string;
|
|
5320
|
+
};
|
|
5321
|
+
export type StudyPreflightResponse = {
|
|
5322
|
+
"ok": true;
|
|
5323
|
+
"depth": "default" | "full";
|
|
5324
|
+
"form": string;
|
|
5325
|
+
"label": string;
|
|
5326
|
+
"plannedRunCount": number;
|
|
5327
|
+
"plannedJudgeComparisons": number;
|
|
5328
|
+
"runs": {
|
|
5329
|
+
"tasks": number;
|
|
5330
|
+
"variants": number;
|
|
5331
|
+
"repeats": number;
|
|
5332
|
+
"groupAgents": Array<{
|
|
5333
|
+
"group": string;
|
|
5334
|
+
"count": number;
|
|
5335
|
+
}>;
|
|
5336
|
+
};
|
|
5337
|
+
"judgePerComparison": Array<{
|
|
5338
|
+
"reference": string;
|
|
5339
|
+
"subject": string;
|
|
5340
|
+
"count": number;
|
|
5341
|
+
}>;
|
|
5342
|
+
"arms": Array<{
|
|
5343
|
+
"key": string;
|
|
5344
|
+
"label": string | null;
|
|
5345
|
+
"role": string;
|
|
5346
|
+
"payload": Record<string, unknown> | null;
|
|
5347
|
+
"cells"?: Array<{
|
|
5348
|
+
"harness": string;
|
|
5349
|
+
"model": string;
|
|
5350
|
+
"provider"?: string;
|
|
5351
|
+
"params"?: Record<string, unknown>;
|
|
5352
|
+
}>;
|
|
5353
|
+
}>;
|
|
5354
|
+
"comparisons": Array<{
|
|
5355
|
+
"reference": string;
|
|
5356
|
+
"subject": string;
|
|
5357
|
+
"oriented": boolean;
|
|
5358
|
+
"basis": "matched" | "whole_setup";
|
|
5359
|
+
"sharedAgents": Array<{
|
|
5360
|
+
"harness": string;
|
|
5361
|
+
"model": string;
|
|
5362
|
+
"provider"?: string;
|
|
5363
|
+
"params"?: Record<string, unknown>;
|
|
5364
|
+
}>;
|
|
5365
|
+
"excluded": Array<{
|
|
5366
|
+
"group": string;
|
|
5367
|
+
"agents": Array<{
|
|
5368
|
+
"harness": string;
|
|
5369
|
+
"model": string;
|
|
5370
|
+
"provider"?: string;
|
|
5371
|
+
"params"?: Record<string, unknown>;
|
|
5372
|
+
}>;
|
|
5373
|
+
}>;
|
|
5374
|
+
"droppedTasks"?: unknown | null;
|
|
5375
|
+
}>;
|
|
5376
|
+
"boards": Array<{
|
|
5377
|
+
"group": string;
|
|
5378
|
+
"profile": "as-published" | "harness-default" | "with-setup";
|
|
5379
|
+
}>;
|
|
5380
|
+
"issues": Array<{
|
|
5381
|
+
"path": string;
|
|
5382
|
+
"code": string;
|
|
5383
|
+
"message": string;
|
|
5384
|
+
}>;
|
|
5385
|
+
"quota"?: {
|
|
5386
|
+
"dimension": string;
|
|
5387
|
+
"remaining": number | null;
|
|
5388
|
+
"allowed": boolean;
|
|
5389
|
+
} | null;
|
|
5390
|
+
};
|
|
4622
5391
|
export type StudyGetResponse = {
|
|
4623
5392
|
"study": {
|
|
4624
5393
|
"id": string;
|
|
@@ -4650,7 +5419,7 @@ export type StudyGetResponse = {
|
|
|
4650
5419
|
"addons"?: Record<string, unknown>;
|
|
4651
5420
|
};
|
|
4652
5421
|
};
|
|
4653
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5422
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
4654
5423
|
"subjectBackend": "model" | "harness" | null;
|
|
4655
5424
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4656
5425
|
"importedFrom"?: string;
|
|
@@ -4686,6 +5455,7 @@ export type StudyGetResponse = {
|
|
|
4686
5455
|
"error": string | null;
|
|
4687
5456
|
"preparation"?: {
|
|
4688
5457
|
"since": string;
|
|
5458
|
+
"progressAt"?: string;
|
|
4689
5459
|
"launchedBy": {
|
|
4690
5460
|
"userId"?: string;
|
|
4691
5461
|
"role"?: string;
|
|
@@ -4694,6 +5464,12 @@ export type StudyGetResponse = {
|
|
|
4694
5464
|
"imageKey": string;
|
|
4695
5465
|
"buildId": string;
|
|
4696
5466
|
"stage"?: "image" | "snapshot";
|
|
5467
|
+
"retry"?: {
|
|
5468
|
+
"failedAttempts": number;
|
|
5469
|
+
"maxAttempts": number;
|
|
5470
|
+
"notBefore": string;
|
|
5471
|
+
"lastFailure": string;
|
|
5472
|
+
};
|
|
4697
5473
|
}>;
|
|
4698
5474
|
} | null;
|
|
4699
5475
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -4708,6 +5484,38 @@ export type StudyGetResponse = {
|
|
|
4708
5484
|
"createdAt": string;
|
|
4709
5485
|
"updatedAt": string;
|
|
4710
5486
|
"liveUntil": string | null;
|
|
5487
|
+
"arms"?: Array<{
|
|
5488
|
+
"key": string;
|
|
5489
|
+
"position": number;
|
|
5490
|
+
"label": string;
|
|
5491
|
+
"role": "reference" | "peer";
|
|
5492
|
+
"payload": {
|
|
5493
|
+
"tools"?: "default" | "none" | {
|
|
5494
|
+
"tools": Array<string>;
|
|
5495
|
+
};
|
|
5496
|
+
"addons"?: Record<string, unknown>;
|
|
5497
|
+
};
|
|
5498
|
+
"cells"?: Array<{
|
|
5499
|
+
"harness": string;
|
|
5500
|
+
"model": string;
|
|
5501
|
+
"provider"?: string;
|
|
5502
|
+
"params"?: {
|
|
5503
|
+
"temperature"?: number;
|
|
5504
|
+
"maxOutputTokens"?: number;
|
|
5505
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5506
|
+
"thinkingBudgetTokens"?: number;
|
|
5507
|
+
};
|
|
5508
|
+
}>;
|
|
5509
|
+
"skills"?: Array<{
|
|
5510
|
+
"ref": string;
|
|
5511
|
+
"name": string;
|
|
5512
|
+
"changedSinceCreate"?: boolean;
|
|
5513
|
+
}>;
|
|
5514
|
+
}>;
|
|
5515
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
5516
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
5517
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
5518
|
+
"modeOptions"?: Record<string, unknown>;
|
|
4711
5519
|
};
|
|
4712
5520
|
"tasks": Array<{
|
|
4713
5521
|
"id": string;
|
|
@@ -4724,7 +5532,7 @@ export type StudyGetResponse = {
|
|
|
4724
5532
|
"runs": Array<{
|
|
4725
5533
|
"runId": string;
|
|
4726
5534
|
"taskId": string;
|
|
4727
|
-
"arm": "baseline" | "skill";
|
|
5535
|
+
"arm": "baseline" | "skill" | string;
|
|
4728
5536
|
"harness": string;
|
|
4729
5537
|
"model": string;
|
|
4730
5538
|
"provider": string | null;
|
|
@@ -4820,9 +5628,24 @@ export type StudyGetResponse = {
|
|
|
4820
5628
|
"gradedAt": string;
|
|
4821
5629
|
};
|
|
4822
5630
|
} | null;
|
|
5631
|
+
"executionBudgetS"?: number | null;
|
|
5632
|
+
"harnessImageDigest"?: string | null;
|
|
5633
|
+
"subjectEngaged"?: boolean | null;
|
|
4823
5634
|
"runCreatedAt": string | null;
|
|
4824
5635
|
"startedAt": string | null;
|
|
4825
5636
|
"finishedAt": string | null;
|
|
5637
|
+
"contrastVerdicts"?: Array<{
|
|
5638
|
+
"referenceArm": string;
|
|
5639
|
+
"subjectArm": string;
|
|
5640
|
+
"referenceRunId": string;
|
|
5641
|
+
"verdict": "subject" | "reference" | "tie" | "unmeasurable";
|
|
5642
|
+
"votes": Array<{
|
|
5643
|
+
"judge": string;
|
|
5644
|
+
"ordering": "base-first" | "skill-first";
|
|
5645
|
+
"answer": "A" | "B" | "tie";
|
|
5646
|
+
}>;
|
|
5647
|
+
"gradedAt": string;
|
|
5648
|
+
}>;
|
|
4826
5649
|
}>;
|
|
4827
5650
|
"aggregate": {
|
|
4828
5651
|
"summary": {
|
|
@@ -5029,6 +5852,17 @@ export type StudyGetResponse = {
|
|
|
5029
5852
|
"ci": Array<unknown> | null;
|
|
5030
5853
|
"n": number;
|
|
5031
5854
|
};
|
|
5855
|
+
"abstention"?: {
|
|
5856
|
+
"value": number | null;
|
|
5857
|
+
"ci": Array<unknown> | null;
|
|
5858
|
+
"n": number;
|
|
5859
|
+
"parts"?: Array<{
|
|
5860
|
+
"id": string;
|
|
5861
|
+
"label": string;
|
|
5862
|
+
"value": number | null;
|
|
5863
|
+
"n": number;
|
|
5864
|
+
}>;
|
|
5865
|
+
};
|
|
5032
5866
|
"tasks": number;
|
|
5033
5867
|
"graded": number;
|
|
5034
5868
|
"unmeasured": number;
|
|
@@ -5040,6 +5874,17 @@ export type StudyGetResponse = {
|
|
|
5040
5874
|
"ci": Array<unknown> | null;
|
|
5041
5875
|
"n": number;
|
|
5042
5876
|
};
|
|
5877
|
+
"abstention"?: {
|
|
5878
|
+
"value": number | null;
|
|
5879
|
+
"ci": Array<unknown> | null;
|
|
5880
|
+
"n": number;
|
|
5881
|
+
"parts"?: Array<{
|
|
5882
|
+
"id": string;
|
|
5883
|
+
"label": string;
|
|
5884
|
+
"value": number | null;
|
|
5885
|
+
"n": number;
|
|
5886
|
+
}>;
|
|
5887
|
+
};
|
|
5043
5888
|
"graded": number;
|
|
5044
5889
|
"unmeasured": number;
|
|
5045
5890
|
"modelJudged": boolean;
|
|
@@ -5059,253 +5904,1363 @@ export type StudyGetResponse = {
|
|
|
5059
5904
|
"languages": Array<string>;
|
|
5060
5905
|
"unavailableReasons": Array<string>;
|
|
5061
5906
|
};
|
|
5062
|
-
};
|
|
5063
|
-
"warnings": Array<{
|
|
5064
|
-
"type": "eval_capability_gap";
|
|
5065
|
-
"harness": string;
|
|
5066
|
-
"kind": string;
|
|
5067
5907
|
} | {
|
|
5068
|
-
"
|
|
5069
|
-
"
|
|
5070
|
-
|
|
5071
|
-
|
|
5072
|
-
|
|
5073
|
-
|
|
5074
|
-
|
|
5075
|
-
|
|
5076
|
-
|
|
5077
|
-
|
|
5078
|
-
|
|
5079
|
-
|
|
5080
|
-
|
|
5081
|
-
};
|
|
5082
|
-
export type StudyGenerateResponse = {
|
|
5083
|
-
"study": {
|
|
5084
|
-
"id": string;
|
|
5085
|
-
"executiveSummary"?: string | null;
|
|
5086
|
-
"plannedRunCount": number;
|
|
5087
|
-
"organizationId": string;
|
|
5088
|
-
"createdBy": string | null;
|
|
5089
|
-
"name": string;
|
|
5090
|
-
"slug": string;
|
|
5091
|
-
"description": string | null;
|
|
5092
|
-
"skillRef": string | null;
|
|
5093
|
-
"skillSlug": string | null;
|
|
5094
|
-
"skillName": string | null;
|
|
5095
|
-
"skillPromise": string;
|
|
5096
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5097
|
-
"subjectName": string | null;
|
|
5098
|
-
"subjectLabel": string;
|
|
5099
|
-
"armConfig": {
|
|
5100
|
-
"baseline"?: {
|
|
5101
|
-
"tools"?: "default" | "none" | {
|
|
5102
|
-
"tools": Array<string>;
|
|
5908
|
+
"shape": "multi-arm";
|
|
5909
|
+
"summary": {
|
|
5910
|
+
"shape": "multi-arm";
|
|
5911
|
+
"verdict": "measured" | "inconclusive";
|
|
5912
|
+
"groups": Array<{
|
|
5913
|
+
"key": string;
|
|
5914
|
+
"label": string;
|
|
5915
|
+
"role": "reference" | "peer";
|
|
5916
|
+
"composition": {
|
|
5917
|
+
"tools"?: "default" | "none" | {
|
|
5918
|
+
"tools": Array<string>;
|
|
5919
|
+
};
|
|
5920
|
+
"addons"?: Array<string>;
|
|
5103
5921
|
};
|
|
5104
|
-
|
|
5105
|
-
|
|
5106
|
-
|
|
5107
|
-
"
|
|
5108
|
-
|
|
5922
|
+
}>;
|
|
5923
|
+
"ranking": {
|
|
5924
|
+
"plan": "vs_reference" | "all_pairs" | null;
|
|
5925
|
+
"referenceArm": string | null;
|
|
5926
|
+
"decision": {
|
|
5927
|
+
"leader": string | null;
|
|
5928
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5929
|
+
"reason": string;
|
|
5930
|
+
"separatedFrom": Array<string>;
|
|
5931
|
+
"lostTo": Array<{
|
|
5932
|
+
"arm": string;
|
|
5933
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5934
|
+
}>;
|
|
5935
|
+
"clearLeader": boolean;
|
|
5936
|
+
"correctnessLeader": string | null;
|
|
5109
5937
|
};
|
|
5110
|
-
"
|
|
5938
|
+
"entries": Array<{
|
|
5939
|
+
"arm": string;
|
|
5940
|
+
"runs": number;
|
|
5941
|
+
"completed": number;
|
|
5942
|
+
"completionRate": number | null;
|
|
5943
|
+
"goalRate": number | null;
|
|
5944
|
+
"avgQualityScore": number | null;
|
|
5945
|
+
"evalPassRate": number | null;
|
|
5946
|
+
"benchmarkPassRate"?: number | null;
|
|
5947
|
+
"compositeScore": number | null;
|
|
5948
|
+
"avgTotalTokens": number | null;
|
|
5949
|
+
"avgOutputTokens": number | null;
|
|
5950
|
+
"avgCostUsd": number | null;
|
|
5951
|
+
"avgDurationMs": number | null;
|
|
5952
|
+
"medianCostUsd": number | null;
|
|
5953
|
+
"medianTotalTokens": number | null;
|
|
5954
|
+
"medianDurationMs": number | null;
|
|
5955
|
+
"rank": number;
|
|
5956
|
+
}>;
|
|
5957
|
+
"contrasts": Array<{
|
|
5958
|
+
"contrastId": string;
|
|
5959
|
+
"referenceArm": string;
|
|
5960
|
+
"subjectArm": string;
|
|
5961
|
+
"oriented": boolean;
|
|
5962
|
+
"outcome": "won" | "tie" | "unmeasurable";
|
|
5963
|
+
"winner": string | null;
|
|
5964
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5965
|
+
"explanation": string;
|
|
5966
|
+
}>;
|
|
5967
|
+
"comparisons": number;
|
|
5968
|
+
"headline"?: string;
|
|
5111
5969
|
};
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
"baselineLabel": string | null;
|
|
5118
|
-
"treatmentLabel": string | null;
|
|
5119
|
-
"promiseMetrics": Array<{
|
|
5120
|
-
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5121
|
-
"direction": "down" | "up";
|
|
5122
|
-
"label": string;
|
|
5123
|
-
}>;
|
|
5124
|
-
"matrix": Array<{
|
|
5125
|
-
"harness": string;
|
|
5126
|
-
"model": string;
|
|
5127
|
-
"provider"?: string;
|
|
5128
|
-
"params"?: {
|
|
5129
|
-
"temperature"?: number;
|
|
5130
|
-
"maxOutputTokens"?: number;
|
|
5131
|
-
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5132
|
-
"thinkingBudgetTokens"?: number;
|
|
5970
|
+
"matrixShape": {
|
|
5971
|
+
"cells": number;
|
|
5972
|
+
"harnesses": number;
|
|
5973
|
+
"models": number;
|
|
5974
|
+
"crossed": boolean;
|
|
5133
5975
|
};
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
"status": string;
|
|
5146
|
-
"error": string | null;
|
|
5147
|
-
"preparation"?: {
|
|
5148
|
-
"since": string;
|
|
5149
|
-
"launchedBy": {
|
|
5150
|
-
"userId"?: string;
|
|
5151
|
-
"role"?: string;
|
|
5976
|
+
"flakiness": {
|
|
5977
|
+
"measured": boolean;
|
|
5978
|
+
"conditions": number;
|
|
5979
|
+
"unstable": number;
|
|
5980
|
+
"worst": {
|
|
5981
|
+
"cell": string;
|
|
5982
|
+
"taskId": string;
|
|
5983
|
+
"arm": string;
|
|
5984
|
+
"variant"?: number;
|
|
5985
|
+
"passRate": number;
|
|
5986
|
+
} | null;
|
|
5152
5987
|
};
|
|
5153
|
-
"
|
|
5154
|
-
"
|
|
5155
|
-
"
|
|
5156
|
-
"
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
"visibilityChangedAt"?: string | null;
|
|
5161
|
-
"visibilityChangedBy"?: string | null;
|
|
5162
|
-
"shareToken"?: string | null;
|
|
5163
|
-
"lowSampleOverride": boolean;
|
|
5164
|
-
"publishedAt": string | null;
|
|
5165
|
-
"verdict": unknown;
|
|
5166
|
-
"definitionVersionId": string | null;
|
|
5167
|
-
"runnerKind": "ora-runs" | "batch-job";
|
|
5168
|
-
"createdAt": string;
|
|
5169
|
-
"updatedAt": string;
|
|
5170
|
-
"liveUntil": string | null;
|
|
5171
|
-
};
|
|
5172
|
-
"streamUrl": string;
|
|
5173
|
-
"warnings"?: Array<{
|
|
5174
|
-
"harness": string;
|
|
5175
|
-
"kind": string;
|
|
5176
|
-
}>;
|
|
5177
|
-
};
|
|
5178
|
-
export type StudyLaunchResponse = {
|
|
5179
|
-
"study": {
|
|
5180
|
-
"id": string;
|
|
5181
|
-
"executiveSummary"?: string | null;
|
|
5182
|
-
"plannedRunCount": number;
|
|
5183
|
-
"organizationId": string;
|
|
5184
|
-
"createdBy": string | null;
|
|
5185
|
-
"name": string;
|
|
5186
|
-
"slug": string;
|
|
5187
|
-
"description": string | null;
|
|
5188
|
-
"skillRef": string | null;
|
|
5189
|
-
"skillSlug": string | null;
|
|
5190
|
-
"skillName": string | null;
|
|
5191
|
-
"skillPromise": string;
|
|
5192
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5193
|
-
"subjectName": string | null;
|
|
5194
|
-
"subjectLabel": string;
|
|
5195
|
-
"armConfig": {
|
|
5196
|
-
"baseline"?: {
|
|
5197
|
-
"tools"?: "default" | "none" | {
|
|
5198
|
-
"tools": Array<string>;
|
|
5199
|
-
};
|
|
5200
|
-
"addons"?: Record<string, unknown>;
|
|
5988
|
+
"totals": {
|
|
5989
|
+
"cells": number;
|
|
5990
|
+
"runs": number;
|
|
5991
|
+
"completed": number;
|
|
5992
|
+
"failed": number;
|
|
5993
|
+
"costUsd": number;
|
|
5994
|
+
"tokens": number;
|
|
5201
5995
|
};
|
|
5202
|
-
|
|
5996
|
+
};
|
|
5997
|
+
"arms": Array<{
|
|
5998
|
+
"key": string;
|
|
5999
|
+
"label": string;
|
|
6000
|
+
"position": number;
|
|
6001
|
+
"role": "reference" | "peer";
|
|
6002
|
+
"composition": {
|
|
5203
6003
|
"tools"?: "default" | "none" | {
|
|
5204
6004
|
"tools": Array<string>;
|
|
5205
6005
|
};
|
|
5206
|
-
"addons"?:
|
|
5207
|
-
};
|
|
5208
|
-
};
|
|
5209
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5210
|
-
"subjectBackend": "model" | "harness" | null;
|
|
5211
|
-
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5212
|
-
"importedFrom"?: string;
|
|
5213
|
-
"baselineLabel": string | null;
|
|
5214
|
-
"treatmentLabel": string | null;
|
|
5215
|
-
"promiseMetrics": Array<{
|
|
5216
|
-
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5217
|
-
"direction": "down" | "up";
|
|
5218
|
-
"label": string;
|
|
5219
|
-
}>;
|
|
5220
|
-
"matrix": Array<{
|
|
5221
|
-
"harness": string;
|
|
5222
|
-
"model": string;
|
|
5223
|
-
"provider"?: string;
|
|
5224
|
-
"params"?: {
|
|
5225
|
-
"temperature"?: number;
|
|
5226
|
-
"maxOutputTokens"?: number;
|
|
5227
|
-
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5228
|
-
"thinkingBudgetTokens"?: number;
|
|
5229
|
-
};
|
|
5230
|
-
}>;
|
|
5231
|
-
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5232
|
-
"judgeSupplementary"?: boolean;
|
|
5233
|
-
"taskCount": number;
|
|
5234
|
-
"variantsPerTask": number;
|
|
5235
|
-
"repeats": number;
|
|
5236
|
-
"subset": {
|
|
5237
|
-
"count": number;
|
|
5238
|
-
"seed": string;
|
|
5239
|
-
"of": number;
|
|
5240
|
-
} | null;
|
|
5241
|
-
"status": string;
|
|
5242
|
-
"error": string | null;
|
|
5243
|
-
"preparation"?: {
|
|
5244
|
-
"since": string;
|
|
5245
|
-
"launchedBy": {
|
|
5246
|
-
"userId"?: string;
|
|
5247
|
-
"role"?: string;
|
|
6006
|
+
"addons"?: Array<string>;
|
|
5248
6007
|
};
|
|
5249
|
-
"
|
|
5250
|
-
"
|
|
5251
|
-
"
|
|
5252
|
-
"
|
|
6008
|
+
"agg": {
|
|
6009
|
+
"runs": number;
|
|
6010
|
+
"completed": number;
|
|
6011
|
+
"infraFailures": number;
|
|
6012
|
+
"completionRate": number | null;
|
|
6013
|
+
"goalRate": number | null;
|
|
6014
|
+
"goalTally": {
|
|
6015
|
+
"achieved": number;
|
|
6016
|
+
"partial": number;
|
|
6017
|
+
"missed": number;
|
|
6018
|
+
};
|
|
6019
|
+
"avgTotalTokens": number | null;
|
|
6020
|
+
"avgOutputTokens": number | null;
|
|
6021
|
+
"avgCostUsd": number | null;
|
|
6022
|
+
"avgDurationMs": number | null;
|
|
6023
|
+
"avgTurns": number | null;
|
|
6024
|
+
"costCoverage": number | null;
|
|
6025
|
+
"avgQualityScore": number | null;
|
|
6026
|
+
"gradedCoverage": number | null;
|
|
6027
|
+
"medianCostUsd": number | null;
|
|
6028
|
+
"medianTotalTokens": number | null;
|
|
6029
|
+
"medianDurationMs": number | null;
|
|
6030
|
+
"p95CostUsd": number | null;
|
|
6031
|
+
"p95TotalTokens": number | null;
|
|
6032
|
+
"p95DurationMs": number | null;
|
|
6033
|
+
"maxCostUsd": number | null;
|
|
6034
|
+
"maxTotalTokens": number | null;
|
|
6035
|
+
"maxDurationMs": number | null;
|
|
6036
|
+
"evalPassRate"?: number | null;
|
|
6037
|
+
"benchmarkPassRate"?: number | null;
|
|
6038
|
+
"adoptionRate"?: number | null;
|
|
6039
|
+
};
|
|
6040
|
+
"perCell": Array<{
|
|
6041
|
+
"harness": string;
|
|
6042
|
+
"model": string;
|
|
6043
|
+
"provider": string | null;
|
|
6044
|
+
"paramsKey"?: string;
|
|
6045
|
+
"label"?: string;
|
|
6046
|
+
"agg": {
|
|
6047
|
+
"runs": number;
|
|
6048
|
+
"completed": number;
|
|
6049
|
+
"infraFailures": number;
|
|
6050
|
+
"completionRate": number | null;
|
|
6051
|
+
"goalRate": number | null;
|
|
6052
|
+
"goalTally": {
|
|
6053
|
+
"achieved": number;
|
|
6054
|
+
"partial": number;
|
|
6055
|
+
"missed": number;
|
|
6056
|
+
};
|
|
6057
|
+
"avgTotalTokens": number | null;
|
|
6058
|
+
"avgOutputTokens": number | null;
|
|
6059
|
+
"avgCostUsd": number | null;
|
|
6060
|
+
"avgDurationMs": number | null;
|
|
6061
|
+
"avgTurns": number | null;
|
|
6062
|
+
"costCoverage": number | null;
|
|
6063
|
+
"avgQualityScore": number | null;
|
|
6064
|
+
"gradedCoverage": number | null;
|
|
6065
|
+
"medianCostUsd": number | null;
|
|
6066
|
+
"medianTotalTokens": number | null;
|
|
6067
|
+
"medianDurationMs": number | null;
|
|
6068
|
+
"p95CostUsd": number | null;
|
|
6069
|
+
"p95TotalTokens": number | null;
|
|
6070
|
+
"p95DurationMs": number | null;
|
|
6071
|
+
"maxCostUsd": number | null;
|
|
6072
|
+
"maxTotalTokens": number | null;
|
|
6073
|
+
"maxDurationMs": number | null;
|
|
6074
|
+
"evalPassRate"?: number | null;
|
|
6075
|
+
"benchmarkPassRate"?: number | null;
|
|
6076
|
+
"adoptionRate"?: number | null;
|
|
6077
|
+
};
|
|
5253
6078
|
}>;
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5269
|
-
|
|
5270
|
-
|
|
5271
|
-
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
5275
|
-
|
|
5276
|
-
}
|
|
5277
|
-
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
5282
|
-
|
|
5283
|
-
|
|
5284
|
-
|
|
5285
|
-
|
|
5286
|
-
|
|
5287
|
-
|
|
5288
|
-
"skillSlug": string | null;
|
|
5289
|
-
"skillName": string | null;
|
|
5290
|
-
"skillPromise": string;
|
|
5291
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5292
|
-
"subjectName": string | null;
|
|
5293
|
-
"subjectLabel": string;
|
|
5294
|
-
"armConfig": {
|
|
5295
|
-
"baseline"?: {
|
|
5296
|
-
"tools"?: "default" | "none" | {
|
|
5297
|
-
"tools": Array<string>;
|
|
6079
|
+
"reference"?: {
|
|
6080
|
+
"cells": Array<{
|
|
6081
|
+
"cell": string;
|
|
6082
|
+
"harness": string;
|
|
6083
|
+
"model": string;
|
|
6084
|
+
"provider": string | null;
|
|
6085
|
+
"paramsKey": string;
|
|
6086
|
+
"label": string;
|
|
6087
|
+
"passRate": {
|
|
6088
|
+
"value": number | null;
|
|
6089
|
+
"ci": Array<unknown> | null;
|
|
6090
|
+
"n": number;
|
|
6091
|
+
};
|
|
6092
|
+
"abstention"?: {
|
|
6093
|
+
"value": number | null;
|
|
6094
|
+
"ci": Array<unknown> | null;
|
|
6095
|
+
"n": number;
|
|
6096
|
+
"parts"?: Array<{
|
|
6097
|
+
"id": string;
|
|
6098
|
+
"label": string;
|
|
6099
|
+
"value": number | null;
|
|
6100
|
+
"n": number;
|
|
6101
|
+
}>;
|
|
6102
|
+
};
|
|
6103
|
+
"tasks": number;
|
|
6104
|
+
"graded": number;
|
|
6105
|
+
"unmeasured": number;
|
|
6106
|
+
"matchers": Array<string>;
|
|
6107
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
6108
|
+
}>;
|
|
6109
|
+
"overall": {
|
|
6110
|
+
"value": number | null;
|
|
6111
|
+
"ci": Array<unknown> | null;
|
|
6112
|
+
"n": number;
|
|
5298
6113
|
};
|
|
5299
|
-
"
|
|
6114
|
+
"abstention"?: {
|
|
6115
|
+
"value": number | null;
|
|
6116
|
+
"ci": Array<unknown> | null;
|
|
6117
|
+
"n": number;
|
|
6118
|
+
"parts"?: Array<{
|
|
6119
|
+
"id": string;
|
|
6120
|
+
"label": string;
|
|
6121
|
+
"value": number | null;
|
|
6122
|
+
"n": number;
|
|
6123
|
+
}>;
|
|
6124
|
+
};
|
|
6125
|
+
"graded": number;
|
|
6126
|
+
"unmeasured": number;
|
|
6127
|
+
"modelJudged": boolean;
|
|
5300
6128
|
};
|
|
5301
|
-
|
|
5302
|
-
|
|
5303
|
-
|
|
6129
|
+
}>;
|
|
6130
|
+
"contrasts": Array<{
|
|
6131
|
+
"contrastId": string;
|
|
6132
|
+
"referenceArm": string;
|
|
6133
|
+
"subjectArm": string;
|
|
6134
|
+
"oriented": boolean;
|
|
6135
|
+
"basis"?: {
|
|
6136
|
+
"id": "matched" | "whole_setup";
|
|
6137
|
+
"compared": {
|
|
6138
|
+
"reference": Array<{
|
|
6139
|
+
"harness": string;
|
|
6140
|
+
"model": string;
|
|
6141
|
+
"provider"?: string;
|
|
6142
|
+
"params"?: Record<string, unknown>;
|
|
6143
|
+
}>;
|
|
6144
|
+
"subject": Array<{
|
|
6145
|
+
"harness": string;
|
|
6146
|
+
"model": string;
|
|
6147
|
+
"provider"?: string;
|
|
6148
|
+
"params"?: Record<string, unknown>;
|
|
6149
|
+
}>;
|
|
5304
6150
|
};
|
|
5305
|
-
"
|
|
6151
|
+
"excluded": {
|
|
6152
|
+
"reference": Array<{
|
|
6153
|
+
"harness": string;
|
|
6154
|
+
"model": string;
|
|
6155
|
+
"provider"?: string;
|
|
6156
|
+
"params"?: Record<string, unknown>;
|
|
6157
|
+
}>;
|
|
6158
|
+
"subject": Array<{
|
|
6159
|
+
"harness": string;
|
|
6160
|
+
"model": string;
|
|
6161
|
+
"provider"?: string;
|
|
6162
|
+
"params"?: Record<string, unknown>;
|
|
6163
|
+
}>;
|
|
6164
|
+
};
|
|
6165
|
+
"differsIn": Array<string>;
|
|
6166
|
+
"licensedFacet": string | null;
|
|
6167
|
+
"efficiency": "comparable" | "incomparable";
|
|
6168
|
+
"pairedTasks": number | null;
|
|
6169
|
+
"dropped": {
|
|
6170
|
+
"reference": number;
|
|
6171
|
+
"subject": number;
|
|
6172
|
+
} | null;
|
|
6173
|
+
};
|
|
6174
|
+
"magnitudes": "relative";
|
|
6175
|
+
"summary": {
|
|
6176
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
6177
|
+
"headline": Array<{
|
|
6178
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6179
|
+
"label": string;
|
|
6180
|
+
"direction": "down" | "up";
|
|
6181
|
+
"baselineAvg": number | null;
|
|
6182
|
+
"skillAvg": number | null;
|
|
6183
|
+
"deltaPct": number | null;
|
|
6184
|
+
"delta"?: {
|
|
6185
|
+
"unit": "pp";
|
|
6186
|
+
"absolute": number | null;
|
|
6187
|
+
"relativePct": number | null;
|
|
6188
|
+
} | {
|
|
6189
|
+
"unit": "pts";
|
|
6190
|
+
"absolute": number | null;
|
|
6191
|
+
"relativePct"?: number | null;
|
|
6192
|
+
} | {
|
|
6193
|
+
"unit": "%";
|
|
6194
|
+
"relativePct": number | null;
|
|
6195
|
+
};
|
|
6196
|
+
"improved": boolean;
|
|
6197
|
+
}>;
|
|
6198
|
+
"completionRate": {
|
|
6199
|
+
"baseline": number | null;
|
|
6200
|
+
"skill": number | null;
|
|
6201
|
+
};
|
|
6202
|
+
"goalRate": {
|
|
6203
|
+
"baseline": number | null;
|
|
6204
|
+
"skill": number | null;
|
|
6205
|
+
};
|
|
6206
|
+
"pairwise": {
|
|
6207
|
+
"pairs": number;
|
|
6208
|
+
"skillWins": number;
|
|
6209
|
+
"baselineWins": number;
|
|
6210
|
+
"ties": number;
|
|
6211
|
+
"winRate": number | null;
|
|
6212
|
+
};
|
|
6213
|
+
"decision"?: {
|
|
6214
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
6215
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
6216
|
+
"reason": string;
|
|
6217
|
+
"efficiencyDiscounted": boolean;
|
|
6218
|
+
};
|
|
6219
|
+
"concentration": {
|
|
6220
|
+
"perCell": Array<{
|
|
6221
|
+
"cell": string;
|
|
6222
|
+
"goalDeltaPp": number | null;
|
|
6223
|
+
"contributionPct": number | null;
|
|
6224
|
+
}>;
|
|
6225
|
+
"heterogeneity": {
|
|
6226
|
+
"dominantCell": string;
|
|
6227
|
+
"contributionPct": number;
|
|
6228
|
+
} | null;
|
|
6229
|
+
};
|
|
6230
|
+
"matrixShape": {
|
|
6231
|
+
"cells": number;
|
|
6232
|
+
"harnesses": number;
|
|
6233
|
+
"models": number;
|
|
6234
|
+
"crossed": boolean;
|
|
6235
|
+
};
|
|
6236
|
+
"flakiness"?: {
|
|
6237
|
+
"measured": boolean;
|
|
6238
|
+
"conditions": number;
|
|
6239
|
+
"unstable": number;
|
|
6240
|
+
"worst": {
|
|
6241
|
+
"cell": string;
|
|
6242
|
+
"taskId": string;
|
|
6243
|
+
"arm": "baseline" | "skill";
|
|
6244
|
+
"variant"?: number;
|
|
6245
|
+
"passRate": number;
|
|
6246
|
+
} | null;
|
|
6247
|
+
};
|
|
6248
|
+
"arms"?: {
|
|
6249
|
+
"baseline": {
|
|
6250
|
+
"runs": number;
|
|
6251
|
+
"completed": number;
|
|
6252
|
+
"infraFailures": number;
|
|
6253
|
+
"completionRate": number | null;
|
|
6254
|
+
"goalRate": number | null;
|
|
6255
|
+
"goalTally": {
|
|
6256
|
+
"achieved": number;
|
|
6257
|
+
"partial": number;
|
|
6258
|
+
"missed": number;
|
|
6259
|
+
};
|
|
6260
|
+
"avgTotalTokens": number | null;
|
|
6261
|
+
"avgOutputTokens": number | null;
|
|
6262
|
+
"avgCostUsd": number | null;
|
|
6263
|
+
"avgDurationMs": number | null;
|
|
6264
|
+
"avgTurns": number | null;
|
|
6265
|
+
"costCoverage": number | null;
|
|
6266
|
+
"avgQualityScore": number | null;
|
|
6267
|
+
"gradedCoverage": number | null;
|
|
6268
|
+
"medianCostUsd": number | null;
|
|
6269
|
+
"medianTotalTokens": number | null;
|
|
6270
|
+
"medianDurationMs": number | null;
|
|
6271
|
+
"p95CostUsd": number | null;
|
|
6272
|
+
"p95TotalTokens": number | null;
|
|
6273
|
+
"p95DurationMs": number | null;
|
|
6274
|
+
"maxCostUsd": number | null;
|
|
6275
|
+
"maxTotalTokens": number | null;
|
|
6276
|
+
"maxDurationMs": number | null;
|
|
6277
|
+
"evalPassRate"?: number | null;
|
|
6278
|
+
"benchmarkPassRate"?: number | null;
|
|
6279
|
+
"adoptionRate"?: number | null;
|
|
6280
|
+
};
|
|
6281
|
+
"skill": {
|
|
6282
|
+
"runs": number;
|
|
6283
|
+
"completed": number;
|
|
6284
|
+
"infraFailures": number;
|
|
6285
|
+
"completionRate": number | null;
|
|
6286
|
+
"goalRate": number | null;
|
|
6287
|
+
"goalTally": {
|
|
6288
|
+
"achieved": number;
|
|
6289
|
+
"partial": number;
|
|
6290
|
+
"missed": number;
|
|
6291
|
+
};
|
|
6292
|
+
"avgTotalTokens": number | null;
|
|
6293
|
+
"avgOutputTokens": number | null;
|
|
6294
|
+
"avgCostUsd": number | null;
|
|
6295
|
+
"avgDurationMs": number | null;
|
|
6296
|
+
"avgTurns": number | null;
|
|
6297
|
+
"costCoverage": number | null;
|
|
6298
|
+
"avgQualityScore": number | null;
|
|
6299
|
+
"gradedCoverage": number | null;
|
|
6300
|
+
"medianCostUsd": number | null;
|
|
6301
|
+
"medianTotalTokens": number | null;
|
|
6302
|
+
"medianDurationMs": number | null;
|
|
6303
|
+
"p95CostUsd": number | null;
|
|
6304
|
+
"p95TotalTokens": number | null;
|
|
6305
|
+
"p95DurationMs": number | null;
|
|
6306
|
+
"maxCostUsd": number | null;
|
|
6307
|
+
"maxTotalTokens": number | null;
|
|
6308
|
+
"maxDurationMs": number | null;
|
|
6309
|
+
"evalPassRate"?: number | null;
|
|
6310
|
+
"benchmarkPassRate"?: number | null;
|
|
6311
|
+
"adoptionRate"?: number | null;
|
|
6312
|
+
};
|
|
6313
|
+
};
|
|
6314
|
+
"totals": {
|
|
6315
|
+
"cells": number;
|
|
6316
|
+
"runs": number;
|
|
6317
|
+
"completed": number;
|
|
6318
|
+
"failed": number;
|
|
6319
|
+
"costUsd": number;
|
|
6320
|
+
"tokens": number;
|
|
6321
|
+
};
|
|
6322
|
+
};
|
|
6323
|
+
"perCell": Array<{
|
|
6324
|
+
"harness": string;
|
|
6325
|
+
"model": string;
|
|
6326
|
+
"provider": string | null;
|
|
6327
|
+
"paramsKey"?: string;
|
|
6328
|
+
"label"?: string;
|
|
6329
|
+
"baseline": {
|
|
6330
|
+
"runs": number;
|
|
6331
|
+
"completed": number;
|
|
6332
|
+
"infraFailures": number;
|
|
6333
|
+
"completionRate": number | null;
|
|
6334
|
+
"goalRate": number | null;
|
|
6335
|
+
"goalTally": {
|
|
6336
|
+
"achieved": number;
|
|
6337
|
+
"partial": number;
|
|
6338
|
+
"missed": number;
|
|
6339
|
+
};
|
|
6340
|
+
"avgTotalTokens": number | null;
|
|
6341
|
+
"avgOutputTokens": number | null;
|
|
6342
|
+
"avgCostUsd": number | null;
|
|
6343
|
+
"avgDurationMs": number | null;
|
|
6344
|
+
"avgTurns": number | null;
|
|
6345
|
+
"costCoverage": number | null;
|
|
6346
|
+
"avgQualityScore": number | null;
|
|
6347
|
+
"gradedCoverage": number | null;
|
|
6348
|
+
"medianCostUsd": number | null;
|
|
6349
|
+
"medianTotalTokens": number | null;
|
|
6350
|
+
"medianDurationMs": number | null;
|
|
6351
|
+
"p95CostUsd": number | null;
|
|
6352
|
+
"p95TotalTokens": number | null;
|
|
6353
|
+
"p95DurationMs": number | null;
|
|
6354
|
+
"maxCostUsd": number | null;
|
|
6355
|
+
"maxTotalTokens": number | null;
|
|
6356
|
+
"maxDurationMs": number | null;
|
|
6357
|
+
"evalPassRate"?: number | null;
|
|
6358
|
+
"benchmarkPassRate"?: number | null;
|
|
6359
|
+
"adoptionRate"?: number | null;
|
|
6360
|
+
};
|
|
6361
|
+
"skill": {
|
|
6362
|
+
"runs": number;
|
|
6363
|
+
"completed": number;
|
|
6364
|
+
"infraFailures": number;
|
|
6365
|
+
"completionRate": number | null;
|
|
6366
|
+
"goalRate": number | null;
|
|
6367
|
+
"goalTally": {
|
|
6368
|
+
"achieved": number;
|
|
6369
|
+
"partial": number;
|
|
6370
|
+
"missed": number;
|
|
6371
|
+
};
|
|
6372
|
+
"avgTotalTokens": number | null;
|
|
6373
|
+
"avgOutputTokens": number | null;
|
|
6374
|
+
"avgCostUsd": number | null;
|
|
6375
|
+
"avgDurationMs": number | null;
|
|
6376
|
+
"avgTurns": number | null;
|
|
6377
|
+
"costCoverage": number | null;
|
|
6378
|
+
"avgQualityScore": number | null;
|
|
6379
|
+
"gradedCoverage": number | null;
|
|
6380
|
+
"medianCostUsd": number | null;
|
|
6381
|
+
"medianTotalTokens": number | null;
|
|
6382
|
+
"medianDurationMs": number | null;
|
|
6383
|
+
"p95CostUsd": number | null;
|
|
6384
|
+
"p95TotalTokens": number | null;
|
|
6385
|
+
"p95DurationMs": number | null;
|
|
6386
|
+
"maxCostUsd": number | null;
|
|
6387
|
+
"maxTotalTokens": number | null;
|
|
6388
|
+
"maxDurationMs": number | null;
|
|
6389
|
+
"evalPassRate"?: number | null;
|
|
6390
|
+
"benchmarkPassRate"?: number | null;
|
|
6391
|
+
"adoptionRate"?: number | null;
|
|
6392
|
+
};
|
|
6393
|
+
"deltas": {
|
|
6394
|
+
"totalTokensPct": number | null;
|
|
6395
|
+
"outputTokensPct": number | null;
|
|
6396
|
+
"costUsdPct": number | null;
|
|
6397
|
+
"durationMsPct": number | null;
|
|
6398
|
+
"numTurnsPct": number | null;
|
|
6399
|
+
"completionRatePp": number | null;
|
|
6400
|
+
"goalRatePp": number | null;
|
|
6401
|
+
"qualityScorePts": number | null;
|
|
6402
|
+
"evalPassRatePp"?: number | null;
|
|
6403
|
+
};
|
|
6404
|
+
}>;
|
|
6405
|
+
} | {
|
|
6406
|
+
"contrastId": string;
|
|
6407
|
+
"referenceArm": string;
|
|
6408
|
+
"subjectArm": string;
|
|
6409
|
+
"oriented": boolean;
|
|
6410
|
+
"basis"?: {
|
|
6411
|
+
"id": "matched" | "whole_setup";
|
|
6412
|
+
"compared": {
|
|
6413
|
+
"reference": Array<{
|
|
6414
|
+
"harness": string;
|
|
6415
|
+
"model": string;
|
|
6416
|
+
"provider"?: string;
|
|
6417
|
+
"params"?: Record<string, unknown>;
|
|
6418
|
+
}>;
|
|
6419
|
+
"subject": Array<{
|
|
6420
|
+
"harness": string;
|
|
6421
|
+
"model": string;
|
|
6422
|
+
"provider"?: string;
|
|
6423
|
+
"params"?: Record<string, unknown>;
|
|
6424
|
+
}>;
|
|
6425
|
+
};
|
|
6426
|
+
"excluded": {
|
|
6427
|
+
"reference": Array<{
|
|
6428
|
+
"harness": string;
|
|
6429
|
+
"model": string;
|
|
6430
|
+
"provider"?: string;
|
|
6431
|
+
"params"?: Record<string, unknown>;
|
|
6432
|
+
}>;
|
|
6433
|
+
"subject": Array<{
|
|
6434
|
+
"harness": string;
|
|
6435
|
+
"model": string;
|
|
6436
|
+
"provider"?: string;
|
|
6437
|
+
"params"?: Record<string, unknown>;
|
|
6438
|
+
}>;
|
|
6439
|
+
};
|
|
6440
|
+
"differsIn": Array<string>;
|
|
6441
|
+
"licensedFacet": string | null;
|
|
6442
|
+
"efficiency": "comparable" | "incomparable";
|
|
6443
|
+
"pairedTasks": number | null;
|
|
6444
|
+
"dropped": {
|
|
6445
|
+
"reference": number;
|
|
6446
|
+
"subject": number;
|
|
6447
|
+
} | null;
|
|
6448
|
+
};
|
|
6449
|
+
"magnitudes": "absolute";
|
|
6450
|
+
"summary": {
|
|
6451
|
+
"completionRate": {
|
|
6452
|
+
"baseline": number | null;
|
|
6453
|
+
"skill": number | null;
|
|
6454
|
+
};
|
|
6455
|
+
"goalRate": {
|
|
6456
|
+
"baseline": number | null;
|
|
6457
|
+
"skill": number | null;
|
|
6458
|
+
};
|
|
6459
|
+
"pairwise": {
|
|
6460
|
+
"pairs": number;
|
|
6461
|
+
"skillWins": number;
|
|
6462
|
+
"baselineWins": number;
|
|
6463
|
+
"ties": number;
|
|
6464
|
+
"winRate": number | null;
|
|
6465
|
+
};
|
|
6466
|
+
"decision"?: {
|
|
6467
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
6468
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
6469
|
+
"reason": string;
|
|
6470
|
+
"efficiencyDiscounted": boolean;
|
|
6471
|
+
};
|
|
6472
|
+
"concentration": {
|
|
6473
|
+
"perCell": Array<{
|
|
6474
|
+
"cell": string;
|
|
6475
|
+
"goalDeltaPp": number | null;
|
|
6476
|
+
"contributionPct": number | null;
|
|
6477
|
+
}>;
|
|
6478
|
+
"heterogeneity": {
|
|
6479
|
+
"dominantCell": string;
|
|
6480
|
+
"contributionPct": number;
|
|
6481
|
+
} | null;
|
|
6482
|
+
};
|
|
6483
|
+
"matrixShape": {
|
|
6484
|
+
"cells": number;
|
|
6485
|
+
"harnesses": number;
|
|
6486
|
+
"models": number;
|
|
6487
|
+
"crossed": boolean;
|
|
6488
|
+
};
|
|
6489
|
+
"flakiness"?: {
|
|
6490
|
+
"measured": boolean;
|
|
6491
|
+
"conditions": number;
|
|
6492
|
+
"unstable": number;
|
|
6493
|
+
"worst": {
|
|
6494
|
+
"cell": string;
|
|
6495
|
+
"taskId": string;
|
|
6496
|
+
"arm": "baseline" | "skill";
|
|
6497
|
+
"variant"?: number;
|
|
6498
|
+
"passRate": number;
|
|
6499
|
+
} | null;
|
|
6500
|
+
};
|
|
6501
|
+
"arms"?: {
|
|
6502
|
+
"baseline": {
|
|
6503
|
+
"runs": number;
|
|
6504
|
+
"completed": number;
|
|
6505
|
+
"infraFailures": number;
|
|
6506
|
+
"completionRate": number | null;
|
|
6507
|
+
"goalRate": number | null;
|
|
6508
|
+
"goalTally": {
|
|
6509
|
+
"achieved": number;
|
|
6510
|
+
"partial": number;
|
|
6511
|
+
"missed": number;
|
|
6512
|
+
};
|
|
6513
|
+
"avgTotalTokens": number | null;
|
|
6514
|
+
"avgOutputTokens": number | null;
|
|
6515
|
+
"avgCostUsd": number | null;
|
|
6516
|
+
"avgDurationMs": number | null;
|
|
6517
|
+
"avgTurns": number | null;
|
|
6518
|
+
"costCoverage": number | null;
|
|
6519
|
+
"avgQualityScore": number | null;
|
|
6520
|
+
"gradedCoverage": number | null;
|
|
6521
|
+
"medianCostUsd": number | null;
|
|
6522
|
+
"medianTotalTokens": number | null;
|
|
6523
|
+
"medianDurationMs": number | null;
|
|
6524
|
+
"p95CostUsd": number | null;
|
|
6525
|
+
"p95TotalTokens": number | null;
|
|
6526
|
+
"p95DurationMs": number | null;
|
|
6527
|
+
"maxCostUsd": number | null;
|
|
6528
|
+
"maxTotalTokens": number | null;
|
|
6529
|
+
"maxDurationMs": number | null;
|
|
6530
|
+
"evalPassRate"?: number | null;
|
|
6531
|
+
"benchmarkPassRate"?: number | null;
|
|
6532
|
+
"adoptionRate"?: number | null;
|
|
6533
|
+
};
|
|
6534
|
+
"skill": {
|
|
6535
|
+
"runs": number;
|
|
6536
|
+
"completed": number;
|
|
6537
|
+
"infraFailures": number;
|
|
6538
|
+
"completionRate": number | null;
|
|
6539
|
+
"goalRate": number | null;
|
|
6540
|
+
"goalTally": {
|
|
6541
|
+
"achieved": number;
|
|
6542
|
+
"partial": number;
|
|
6543
|
+
"missed": number;
|
|
6544
|
+
};
|
|
6545
|
+
"avgTotalTokens": number | null;
|
|
6546
|
+
"avgOutputTokens": number | null;
|
|
6547
|
+
"avgCostUsd": number | null;
|
|
6548
|
+
"avgDurationMs": number | null;
|
|
6549
|
+
"avgTurns": number | null;
|
|
6550
|
+
"costCoverage": number | null;
|
|
6551
|
+
"avgQualityScore": number | null;
|
|
6552
|
+
"gradedCoverage": number | null;
|
|
6553
|
+
"medianCostUsd": number | null;
|
|
6554
|
+
"medianTotalTokens": number | null;
|
|
6555
|
+
"medianDurationMs": number | null;
|
|
6556
|
+
"p95CostUsd": number | null;
|
|
6557
|
+
"p95TotalTokens": number | null;
|
|
6558
|
+
"p95DurationMs": number | null;
|
|
6559
|
+
"maxCostUsd": number | null;
|
|
6560
|
+
"maxTotalTokens": number | null;
|
|
6561
|
+
"maxDurationMs": number | null;
|
|
6562
|
+
"evalPassRate"?: number | null;
|
|
6563
|
+
"benchmarkPassRate"?: number | null;
|
|
6564
|
+
"adoptionRate"?: number | null;
|
|
6565
|
+
};
|
|
6566
|
+
};
|
|
6567
|
+
"totals": {
|
|
6568
|
+
"cells": number;
|
|
6569
|
+
"runs": number;
|
|
6570
|
+
"completed": number;
|
|
6571
|
+
"failed": number;
|
|
6572
|
+
"costUsd": number;
|
|
6573
|
+
"tokens": number;
|
|
6574
|
+
};
|
|
6575
|
+
};
|
|
6576
|
+
"perCell": Array<{
|
|
6577
|
+
"harness": string;
|
|
6578
|
+
"model": string;
|
|
6579
|
+
"provider": string | null;
|
|
6580
|
+
"paramsKey"?: string;
|
|
6581
|
+
"label"?: string;
|
|
6582
|
+
"baseline": {
|
|
6583
|
+
"runs": number;
|
|
6584
|
+
"completed": number;
|
|
6585
|
+
"infraFailures": number;
|
|
6586
|
+
"completionRate": number | null;
|
|
6587
|
+
"goalRate": number | null;
|
|
6588
|
+
"goalTally": {
|
|
6589
|
+
"achieved": number;
|
|
6590
|
+
"partial": number;
|
|
6591
|
+
"missed": number;
|
|
6592
|
+
};
|
|
6593
|
+
"avgTotalTokens": number | null;
|
|
6594
|
+
"avgOutputTokens": number | null;
|
|
6595
|
+
"avgCostUsd": number | null;
|
|
6596
|
+
"avgDurationMs": number | null;
|
|
6597
|
+
"avgTurns": number | null;
|
|
6598
|
+
"costCoverage": number | null;
|
|
6599
|
+
"avgQualityScore": number | null;
|
|
6600
|
+
"gradedCoverage": number | null;
|
|
6601
|
+
"medianCostUsd": number | null;
|
|
6602
|
+
"medianTotalTokens": number | null;
|
|
6603
|
+
"medianDurationMs": number | null;
|
|
6604
|
+
"p95CostUsd": number | null;
|
|
6605
|
+
"p95TotalTokens": number | null;
|
|
6606
|
+
"p95DurationMs": number | null;
|
|
6607
|
+
"maxCostUsd": number | null;
|
|
6608
|
+
"maxTotalTokens": number | null;
|
|
6609
|
+
"maxDurationMs": number | null;
|
|
6610
|
+
"evalPassRate"?: number | null;
|
|
6611
|
+
"benchmarkPassRate"?: number | null;
|
|
6612
|
+
"adoptionRate"?: number | null;
|
|
6613
|
+
};
|
|
6614
|
+
"skill": {
|
|
6615
|
+
"runs": number;
|
|
6616
|
+
"completed": number;
|
|
6617
|
+
"infraFailures": number;
|
|
6618
|
+
"completionRate": number | null;
|
|
6619
|
+
"goalRate": number | null;
|
|
6620
|
+
"goalTally": {
|
|
6621
|
+
"achieved": number;
|
|
6622
|
+
"partial": number;
|
|
6623
|
+
"missed": number;
|
|
6624
|
+
};
|
|
6625
|
+
"avgTotalTokens": number | null;
|
|
6626
|
+
"avgOutputTokens": number | null;
|
|
6627
|
+
"avgCostUsd": number | null;
|
|
6628
|
+
"avgDurationMs": number | null;
|
|
6629
|
+
"avgTurns": number | null;
|
|
6630
|
+
"costCoverage": number | null;
|
|
6631
|
+
"avgQualityScore": number | null;
|
|
6632
|
+
"gradedCoverage": number | null;
|
|
6633
|
+
"medianCostUsd": number | null;
|
|
6634
|
+
"medianTotalTokens": number | null;
|
|
6635
|
+
"medianDurationMs": number | null;
|
|
6636
|
+
"p95CostUsd": number | null;
|
|
6637
|
+
"p95TotalTokens": number | null;
|
|
6638
|
+
"p95DurationMs": number | null;
|
|
6639
|
+
"maxCostUsd": number | null;
|
|
6640
|
+
"maxTotalTokens": number | null;
|
|
6641
|
+
"maxDurationMs": number | null;
|
|
6642
|
+
"evalPassRate"?: number | null;
|
|
6643
|
+
"benchmarkPassRate"?: number | null;
|
|
6644
|
+
"adoptionRate"?: number | null;
|
|
6645
|
+
};
|
|
6646
|
+
"deltas": {
|
|
6647
|
+
"completionRatePp": number | null;
|
|
6648
|
+
"goalRatePp": number | null;
|
|
6649
|
+
"qualityScorePts": number | null;
|
|
6650
|
+
"evalPassRatePp"?: number | null;
|
|
6651
|
+
};
|
|
6652
|
+
}>;
|
|
6653
|
+
}>;
|
|
6654
|
+
"board": Array<{
|
|
6655
|
+
"harness": string;
|
|
6656
|
+
"model": string;
|
|
6657
|
+
"provider": string | null;
|
|
6658
|
+
"paramsKey"?: string;
|
|
6659
|
+
"label"?: string;
|
|
6660
|
+
"arm": string;
|
|
6661
|
+
"runs": number;
|
|
6662
|
+
"completed": number;
|
|
6663
|
+
"completionRate": number | null;
|
|
6664
|
+
"goalRate": number | null;
|
|
6665
|
+
"avgQualityScore": number | null;
|
|
6666
|
+
"evalPassRate": number | null;
|
|
6667
|
+
"benchmarkPassRate"?: number | null;
|
|
6668
|
+
"compositeScore": number | null;
|
|
6669
|
+
"avgTotalTokens": number | null;
|
|
6670
|
+
"avgOutputTokens": number | null;
|
|
6671
|
+
"avgCostUsd": number | null;
|
|
6672
|
+
"avgDurationMs": number | null;
|
|
6673
|
+
"medianCostUsd": number | null;
|
|
6674
|
+
"medianTotalTokens": number | null;
|
|
6675
|
+
"medianDurationMs": number | null;
|
|
6676
|
+
"rank": number;
|
|
6677
|
+
}>;
|
|
6678
|
+
"setupsBoard"?: Array<{
|
|
6679
|
+
"harness": string;
|
|
6680
|
+
"model": string;
|
|
6681
|
+
"provider": string | null;
|
|
6682
|
+
"paramsKey"?: string;
|
|
6683
|
+
"label"?: string;
|
|
6684
|
+
"arm": string;
|
|
6685
|
+
"runs": number;
|
|
6686
|
+
"completed": number;
|
|
6687
|
+
"completionRate": number | null;
|
|
6688
|
+
"goalRate": number | null;
|
|
6689
|
+
"avgQualityScore": number | null;
|
|
6690
|
+
"evalPassRate": number | null;
|
|
6691
|
+
"benchmarkPassRate"?: number | null;
|
|
6692
|
+
"compositeScore": number | null;
|
|
6693
|
+
"avgTotalTokens": number | null;
|
|
6694
|
+
"avgOutputTokens": number | null;
|
|
6695
|
+
"avgCostUsd": number | null;
|
|
6696
|
+
"avgDurationMs": number | null;
|
|
6697
|
+
"medianCostUsd": number | null;
|
|
6698
|
+
"medianTotalTokens": number | null;
|
|
6699
|
+
"medianDurationMs": number | null;
|
|
6700
|
+
"rank": number;
|
|
6701
|
+
"group": string;
|
|
6702
|
+
"agent": string;
|
|
6703
|
+
"setup": string;
|
|
6704
|
+
}>;
|
|
6705
|
+
"perTask": Array<{
|
|
6706
|
+
"taskId": string;
|
|
6707
|
+
"position": number;
|
|
6708
|
+
"promptPreview": string;
|
|
6709
|
+
"requiresWeb": boolean;
|
|
6710
|
+
"category": string | null;
|
|
6711
|
+
"byArm": Record<string, unknown>;
|
|
6712
|
+
}>;
|
|
6713
|
+
"behavior"?: Array<{
|
|
6714
|
+
"harness": string;
|
|
6715
|
+
"model": string;
|
|
6716
|
+
"provider": string | null;
|
|
6717
|
+
"arm": string;
|
|
6718
|
+
"totalRuns": number;
|
|
6719
|
+
"signals": Array<{
|
|
6720
|
+
"id": string;
|
|
6721
|
+
"label": string;
|
|
6722
|
+
"tone": "good" | "bad" | "neutral";
|
|
6723
|
+
"runCount": number;
|
|
6724
|
+
"succeededCount": number;
|
|
6725
|
+
}>;
|
|
6726
|
+
}>;
|
|
6727
|
+
"execution"?: Record<string, unknown>;
|
|
6728
|
+
"inference"?: {
|
|
6729
|
+
"contrasts": Array<{
|
|
6730
|
+
"contrastId": string;
|
|
6731
|
+
"table": {
|
|
6732
|
+
"columns": Array<{
|
|
6733
|
+
"key": string;
|
|
6734
|
+
"label": string;
|
|
6735
|
+
"family": "rate" | "score" | "magnitude";
|
|
6736
|
+
"betterWhen": "higher" | "lower";
|
|
6737
|
+
}>;
|
|
6738
|
+
"groups": Array<{
|
|
6739
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
6740
|
+
"title": string;
|
|
6741
|
+
"rows": Array<{
|
|
6742
|
+
"key": string;
|
|
6743
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
6744
|
+
"harness": string | null;
|
|
6745
|
+
"model": string | null;
|
|
6746
|
+
"paramsKey"?: string;
|
|
6747
|
+
"runs": number;
|
|
6748
|
+
"cells": Record<string, unknown>;
|
|
6749
|
+
}>;
|
|
6750
|
+
}>;
|
|
6751
|
+
"reference": "baseline" | "field";
|
|
6752
|
+
"draws": number;
|
|
6753
|
+
"adjustment": {
|
|
6754
|
+
"scope": "planned-comparisons";
|
|
6755
|
+
"comparisons": number;
|
|
6756
|
+
};
|
|
6757
|
+
} | null;
|
|
6758
|
+
}>;
|
|
6759
|
+
"holm": {
|
|
6760
|
+
"status": "complete";
|
|
6761
|
+
"ready": number;
|
|
6762
|
+
"planned": number;
|
|
6763
|
+
"adjustment": {
|
|
6764
|
+
"scope": "planned-comparisons";
|
|
6765
|
+
"comparisons": number;
|
|
6766
|
+
};
|
|
6767
|
+
} | {
|
|
6768
|
+
"status": "pending";
|
|
6769
|
+
"ready": number;
|
|
6770
|
+
"planned": number;
|
|
6771
|
+
};
|
|
6772
|
+
"claims": {
|
|
6773
|
+
"perContrast": Record<string, unknown>;
|
|
6774
|
+
"ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
|
|
6775
|
+
};
|
|
6776
|
+
};
|
|
6777
|
+
};
|
|
6778
|
+
"warnings": Array<{
|
|
6779
|
+
"type": "eval_capability_gap";
|
|
6780
|
+
"harness": string;
|
|
6781
|
+
"kind": string;
|
|
6782
|
+
} | {
|
|
6783
|
+
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
6784
|
+
"detail": string;
|
|
6785
|
+
"taskIds"?: Array<string>;
|
|
6786
|
+
} | {
|
|
6787
|
+
"type": "design_aggregate_budget";
|
|
6788
|
+
"detail": string;
|
|
6789
|
+
}>;
|
|
6790
|
+
"taskHealth": {
|
|
6791
|
+
"flags": Array<{
|
|
6792
|
+
"taskId": string;
|
|
6793
|
+
"signal": string;
|
|
6794
|
+
"kind": "floor" | "ceiling";
|
|
6795
|
+
"conditions": number;
|
|
6796
|
+
}>;
|
|
6797
|
+
"rubricSplits": Array<{
|
|
6798
|
+
"taskId": string;
|
|
6799
|
+
"criterionIndex": number;
|
|
6800
|
+
"criterion": string;
|
|
6801
|
+
"split": number;
|
|
6802
|
+
"graded": number;
|
|
6803
|
+
}>;
|
|
6804
|
+
"signals": Array<{
|
|
6805
|
+
"id": string;
|
|
6806
|
+
"label": string;
|
|
6807
|
+
}>;
|
|
6808
|
+
};
|
|
6809
|
+
"shareUrl"?: string | null;
|
|
6810
|
+
"canManageSharing"?: boolean;
|
|
6811
|
+
};
|
|
6812
|
+
export type StudyUpdateDetailsRequest = {
|
|
6813
|
+
"name"?: string;
|
|
6814
|
+
"description"?: string;
|
|
6815
|
+
};
|
|
6816
|
+
export type StudyUpdateDetailsResponse = {
|
|
6817
|
+
"study": {
|
|
6818
|
+
"id": string;
|
|
6819
|
+
"executiveSummary"?: string | null;
|
|
6820
|
+
"plannedRunCount": number;
|
|
6821
|
+
"organizationId": string;
|
|
6822
|
+
"createdBy": string | null;
|
|
6823
|
+
"name": string;
|
|
6824
|
+
"slug": string;
|
|
6825
|
+
"description": string | null;
|
|
6826
|
+
"skillRef": string | null;
|
|
6827
|
+
"skillSlug": string | null;
|
|
6828
|
+
"skillName": string | null;
|
|
6829
|
+
"skillPromise": string;
|
|
6830
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
6831
|
+
"subjectName": string | null;
|
|
6832
|
+
"subjectLabel": string;
|
|
6833
|
+
"armConfig": {
|
|
6834
|
+
"baseline"?: {
|
|
6835
|
+
"tools"?: "default" | "none" | {
|
|
6836
|
+
"tools": Array<string>;
|
|
6837
|
+
};
|
|
6838
|
+
"addons"?: Record<string, unknown>;
|
|
6839
|
+
};
|
|
6840
|
+
"treatment"?: {
|
|
6841
|
+
"tools"?: "default" | "none" | {
|
|
6842
|
+
"tools": Array<string>;
|
|
6843
|
+
};
|
|
6844
|
+
"addons"?: Record<string, unknown>;
|
|
6845
|
+
};
|
|
6846
|
+
};
|
|
6847
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6848
|
+
"subjectBackend": "model" | "harness" | null;
|
|
6849
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6850
|
+
"importedFrom"?: string;
|
|
6851
|
+
"baselineLabel": string | null;
|
|
6852
|
+
"treatmentLabel": string | null;
|
|
6853
|
+
"promiseMetrics": Array<{
|
|
6854
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6855
|
+
"direction": "down" | "up";
|
|
6856
|
+
"label": string;
|
|
6857
|
+
}>;
|
|
6858
|
+
"matrix": Array<{
|
|
6859
|
+
"harness": string;
|
|
6860
|
+
"model": string;
|
|
6861
|
+
"provider"?: string;
|
|
6862
|
+
"params"?: {
|
|
6863
|
+
"temperature"?: number;
|
|
6864
|
+
"maxOutputTokens"?: number;
|
|
6865
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
6866
|
+
"thinkingBudgetTokens"?: number;
|
|
6867
|
+
};
|
|
6868
|
+
}>;
|
|
6869
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6870
|
+
"judgeSupplementary"?: boolean;
|
|
6871
|
+
"taskCount": number;
|
|
6872
|
+
"variantsPerTask": number;
|
|
6873
|
+
"repeats": number;
|
|
6874
|
+
"subset": {
|
|
6875
|
+
"count": number;
|
|
6876
|
+
"seed": string;
|
|
6877
|
+
"of": number;
|
|
6878
|
+
} | null;
|
|
6879
|
+
"status": string;
|
|
6880
|
+
"error": string | null;
|
|
6881
|
+
"preparation"?: {
|
|
6882
|
+
"since": string;
|
|
6883
|
+
"progressAt"?: string;
|
|
6884
|
+
"launchedBy": {
|
|
6885
|
+
"userId"?: string;
|
|
6886
|
+
"role"?: string;
|
|
6887
|
+
};
|
|
6888
|
+
"waitingOn": Array<{
|
|
6889
|
+
"imageKey": string;
|
|
6890
|
+
"buildId": string;
|
|
6891
|
+
"stage"?: "image" | "snapshot";
|
|
6892
|
+
"retry"?: {
|
|
6893
|
+
"failedAttempts": number;
|
|
6894
|
+
"maxAttempts": number;
|
|
6895
|
+
"notBefore": string;
|
|
6896
|
+
"lastFailure": string;
|
|
6897
|
+
};
|
|
6898
|
+
}>;
|
|
6899
|
+
} | null;
|
|
6900
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
6901
|
+
"visibilityChangedAt"?: string | null;
|
|
6902
|
+
"visibilityChangedBy"?: string | null;
|
|
6903
|
+
"shareToken"?: string | null;
|
|
6904
|
+
"lowSampleOverride": boolean;
|
|
6905
|
+
"publishedAt": string | null;
|
|
6906
|
+
"verdict": unknown;
|
|
6907
|
+
"definitionVersionId": string | null;
|
|
6908
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
6909
|
+
"createdAt": string;
|
|
6910
|
+
"updatedAt": string;
|
|
6911
|
+
"liveUntil": string | null;
|
|
6912
|
+
"arms"?: Array<{
|
|
6913
|
+
"key": string;
|
|
6914
|
+
"position": number;
|
|
6915
|
+
"label": string;
|
|
6916
|
+
"role": "reference" | "peer";
|
|
6917
|
+
"payload": {
|
|
6918
|
+
"tools"?: "default" | "none" | {
|
|
6919
|
+
"tools": Array<string>;
|
|
6920
|
+
};
|
|
6921
|
+
"addons"?: Record<string, unknown>;
|
|
6922
|
+
};
|
|
6923
|
+
"cells"?: Array<{
|
|
6924
|
+
"harness": string;
|
|
6925
|
+
"model": string;
|
|
6926
|
+
"provider"?: string;
|
|
6927
|
+
"params"?: {
|
|
6928
|
+
"temperature"?: number;
|
|
6929
|
+
"maxOutputTokens"?: number;
|
|
6930
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
6931
|
+
"thinkingBudgetTokens"?: number;
|
|
6932
|
+
};
|
|
6933
|
+
}>;
|
|
6934
|
+
"skills"?: Array<{
|
|
6935
|
+
"ref": string;
|
|
6936
|
+
"name": string;
|
|
6937
|
+
"changedSinceCreate"?: boolean;
|
|
6938
|
+
}>;
|
|
6939
|
+
}>;
|
|
6940
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
6941
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
6942
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
6943
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6944
|
+
};
|
|
6945
|
+
};
|
|
6946
|
+
export type StudyDeleteResponse = unknown;
|
|
6947
|
+
export type StudyRegenerateSummaryResponse = {
|
|
6948
|
+
"status": "queued";
|
|
6949
|
+
};
|
|
6950
|
+
export type StudyGenerateRequest = {
|
|
6951
|
+
"proposeExpectations"?: boolean;
|
|
6952
|
+
};
|
|
6953
|
+
export type StudyGenerateResponse = {
|
|
6954
|
+
"study": {
|
|
6955
|
+
"id": string;
|
|
6956
|
+
"executiveSummary"?: string | null;
|
|
6957
|
+
"plannedRunCount": number;
|
|
6958
|
+
"organizationId": string;
|
|
6959
|
+
"createdBy": string | null;
|
|
6960
|
+
"name": string;
|
|
6961
|
+
"slug": string;
|
|
6962
|
+
"description": string | null;
|
|
6963
|
+
"skillRef": string | null;
|
|
6964
|
+
"skillSlug": string | null;
|
|
6965
|
+
"skillName": string | null;
|
|
6966
|
+
"skillPromise": string;
|
|
6967
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
6968
|
+
"subjectName": string | null;
|
|
6969
|
+
"subjectLabel": string;
|
|
6970
|
+
"armConfig": {
|
|
6971
|
+
"baseline"?: {
|
|
6972
|
+
"tools"?: "default" | "none" | {
|
|
6973
|
+
"tools": Array<string>;
|
|
6974
|
+
};
|
|
6975
|
+
"addons"?: Record<string, unknown>;
|
|
6976
|
+
};
|
|
6977
|
+
"treatment"?: {
|
|
6978
|
+
"tools"?: "default" | "none" | {
|
|
6979
|
+
"tools": Array<string>;
|
|
6980
|
+
};
|
|
6981
|
+
"addons"?: Record<string, unknown>;
|
|
6982
|
+
};
|
|
6983
|
+
};
|
|
6984
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6985
|
+
"subjectBackend": "model" | "harness" | null;
|
|
6986
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6987
|
+
"importedFrom"?: string;
|
|
6988
|
+
"baselineLabel": string | null;
|
|
6989
|
+
"treatmentLabel": string | null;
|
|
6990
|
+
"promiseMetrics": Array<{
|
|
6991
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6992
|
+
"direction": "down" | "up";
|
|
6993
|
+
"label": string;
|
|
6994
|
+
}>;
|
|
6995
|
+
"matrix": Array<{
|
|
6996
|
+
"harness": string;
|
|
6997
|
+
"model": string;
|
|
6998
|
+
"provider"?: string;
|
|
6999
|
+
"params"?: {
|
|
7000
|
+
"temperature"?: number;
|
|
7001
|
+
"maxOutputTokens"?: number;
|
|
7002
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7003
|
+
"thinkingBudgetTokens"?: number;
|
|
7004
|
+
};
|
|
7005
|
+
}>;
|
|
7006
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
7007
|
+
"judgeSupplementary"?: boolean;
|
|
7008
|
+
"taskCount": number;
|
|
7009
|
+
"variantsPerTask": number;
|
|
7010
|
+
"repeats": number;
|
|
7011
|
+
"subset": {
|
|
7012
|
+
"count": number;
|
|
7013
|
+
"seed": string;
|
|
7014
|
+
"of": number;
|
|
7015
|
+
} | null;
|
|
7016
|
+
"status": string;
|
|
7017
|
+
"error": string | null;
|
|
7018
|
+
"preparation"?: {
|
|
7019
|
+
"since": string;
|
|
7020
|
+
"progressAt"?: string;
|
|
7021
|
+
"launchedBy": {
|
|
7022
|
+
"userId"?: string;
|
|
7023
|
+
"role"?: string;
|
|
7024
|
+
};
|
|
7025
|
+
"waitingOn": Array<{
|
|
7026
|
+
"imageKey": string;
|
|
7027
|
+
"buildId": string;
|
|
7028
|
+
"stage"?: "image" | "snapshot";
|
|
7029
|
+
"retry"?: {
|
|
7030
|
+
"failedAttempts": number;
|
|
7031
|
+
"maxAttempts": number;
|
|
7032
|
+
"notBefore": string;
|
|
7033
|
+
"lastFailure": string;
|
|
7034
|
+
};
|
|
7035
|
+
}>;
|
|
7036
|
+
} | null;
|
|
7037
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
7038
|
+
"visibilityChangedAt"?: string | null;
|
|
7039
|
+
"visibilityChangedBy"?: string | null;
|
|
7040
|
+
"shareToken"?: string | null;
|
|
7041
|
+
"lowSampleOverride": boolean;
|
|
7042
|
+
"publishedAt": string | null;
|
|
7043
|
+
"verdict": unknown;
|
|
7044
|
+
"definitionVersionId": string | null;
|
|
7045
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
7046
|
+
"createdAt": string;
|
|
7047
|
+
"updatedAt": string;
|
|
7048
|
+
"liveUntil": string | null;
|
|
7049
|
+
"arms"?: Array<{
|
|
7050
|
+
"key": string;
|
|
7051
|
+
"position": number;
|
|
7052
|
+
"label": string;
|
|
7053
|
+
"role": "reference" | "peer";
|
|
7054
|
+
"payload": {
|
|
7055
|
+
"tools"?: "default" | "none" | {
|
|
7056
|
+
"tools": Array<string>;
|
|
7057
|
+
};
|
|
7058
|
+
"addons"?: Record<string, unknown>;
|
|
7059
|
+
};
|
|
7060
|
+
"cells"?: Array<{
|
|
7061
|
+
"harness": string;
|
|
7062
|
+
"model": string;
|
|
7063
|
+
"provider"?: string;
|
|
7064
|
+
"params"?: {
|
|
7065
|
+
"temperature"?: number;
|
|
7066
|
+
"maxOutputTokens"?: number;
|
|
7067
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7068
|
+
"thinkingBudgetTokens"?: number;
|
|
7069
|
+
};
|
|
7070
|
+
}>;
|
|
7071
|
+
"skills"?: Array<{
|
|
7072
|
+
"ref": string;
|
|
7073
|
+
"name": string;
|
|
7074
|
+
"changedSinceCreate"?: boolean;
|
|
7075
|
+
}>;
|
|
7076
|
+
}>;
|
|
7077
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7078
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7079
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7080
|
+
"modeOptions"?: Record<string, unknown>;
|
|
7081
|
+
};
|
|
7082
|
+
"streamUrl": string;
|
|
7083
|
+
"warnings"?: Array<{
|
|
7084
|
+
"harness": string;
|
|
7085
|
+
"kind": string;
|
|
7086
|
+
} | {
|
|
7087
|
+
"type": "design_aggregate_budget";
|
|
7088
|
+
"detail": string;
|
|
7089
|
+
}>;
|
|
7090
|
+
};
|
|
7091
|
+
export type StudyLaunchResponse = {
|
|
7092
|
+
"study": {
|
|
7093
|
+
"id": string;
|
|
7094
|
+
"executiveSummary"?: string | null;
|
|
7095
|
+
"plannedRunCount": number;
|
|
7096
|
+
"organizationId": string;
|
|
7097
|
+
"createdBy": string | null;
|
|
7098
|
+
"name": string;
|
|
7099
|
+
"slug": string;
|
|
7100
|
+
"description": string | null;
|
|
7101
|
+
"skillRef": string | null;
|
|
7102
|
+
"skillSlug": string | null;
|
|
7103
|
+
"skillName": string | null;
|
|
7104
|
+
"skillPromise": string;
|
|
7105
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
7106
|
+
"subjectName": string | null;
|
|
7107
|
+
"subjectLabel": string;
|
|
7108
|
+
"armConfig": {
|
|
7109
|
+
"baseline"?: {
|
|
7110
|
+
"tools"?: "default" | "none" | {
|
|
7111
|
+
"tools": Array<string>;
|
|
7112
|
+
};
|
|
7113
|
+
"addons"?: Record<string, unknown>;
|
|
7114
|
+
};
|
|
7115
|
+
"treatment"?: {
|
|
7116
|
+
"tools"?: "default" | "none" | {
|
|
7117
|
+
"tools": Array<string>;
|
|
7118
|
+
};
|
|
7119
|
+
"addons"?: Record<string, unknown>;
|
|
7120
|
+
};
|
|
7121
|
+
};
|
|
7122
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
7123
|
+
"subjectBackend": "model" | "harness" | null;
|
|
7124
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
7125
|
+
"importedFrom"?: string;
|
|
7126
|
+
"baselineLabel": string | null;
|
|
7127
|
+
"treatmentLabel": string | null;
|
|
7128
|
+
"promiseMetrics": Array<{
|
|
7129
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
7130
|
+
"direction": "down" | "up";
|
|
7131
|
+
"label": string;
|
|
7132
|
+
}>;
|
|
7133
|
+
"matrix": Array<{
|
|
7134
|
+
"harness": string;
|
|
7135
|
+
"model": string;
|
|
7136
|
+
"provider"?: string;
|
|
7137
|
+
"params"?: {
|
|
7138
|
+
"temperature"?: number;
|
|
7139
|
+
"maxOutputTokens"?: number;
|
|
7140
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7141
|
+
"thinkingBudgetTokens"?: number;
|
|
7142
|
+
};
|
|
7143
|
+
}>;
|
|
7144
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
7145
|
+
"judgeSupplementary"?: boolean;
|
|
7146
|
+
"taskCount": number;
|
|
7147
|
+
"variantsPerTask": number;
|
|
7148
|
+
"repeats": number;
|
|
7149
|
+
"subset": {
|
|
7150
|
+
"count": number;
|
|
7151
|
+
"seed": string;
|
|
7152
|
+
"of": number;
|
|
7153
|
+
} | null;
|
|
7154
|
+
"status": string;
|
|
7155
|
+
"error": string | null;
|
|
7156
|
+
"preparation"?: {
|
|
7157
|
+
"since": string;
|
|
7158
|
+
"progressAt"?: string;
|
|
7159
|
+
"launchedBy": {
|
|
7160
|
+
"userId"?: string;
|
|
7161
|
+
"role"?: string;
|
|
7162
|
+
};
|
|
7163
|
+
"waitingOn": Array<{
|
|
7164
|
+
"imageKey": string;
|
|
7165
|
+
"buildId": string;
|
|
7166
|
+
"stage"?: "image" | "snapshot";
|
|
7167
|
+
"retry"?: {
|
|
7168
|
+
"failedAttempts": number;
|
|
7169
|
+
"maxAttempts": number;
|
|
7170
|
+
"notBefore": string;
|
|
7171
|
+
"lastFailure": string;
|
|
7172
|
+
};
|
|
7173
|
+
}>;
|
|
7174
|
+
} | null;
|
|
7175
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
7176
|
+
"visibilityChangedAt"?: string | null;
|
|
7177
|
+
"visibilityChangedBy"?: string | null;
|
|
7178
|
+
"shareToken"?: string | null;
|
|
7179
|
+
"lowSampleOverride": boolean;
|
|
7180
|
+
"publishedAt": string | null;
|
|
7181
|
+
"verdict": unknown;
|
|
7182
|
+
"definitionVersionId": string | null;
|
|
7183
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
7184
|
+
"createdAt": string;
|
|
7185
|
+
"updatedAt": string;
|
|
7186
|
+
"liveUntil": string | null;
|
|
7187
|
+
"arms"?: Array<{
|
|
7188
|
+
"key": string;
|
|
7189
|
+
"position": number;
|
|
7190
|
+
"label": string;
|
|
7191
|
+
"role": "reference" | "peer";
|
|
7192
|
+
"payload": {
|
|
7193
|
+
"tools"?: "default" | "none" | {
|
|
7194
|
+
"tools": Array<string>;
|
|
7195
|
+
};
|
|
7196
|
+
"addons"?: Record<string, unknown>;
|
|
7197
|
+
};
|
|
7198
|
+
"cells"?: Array<{
|
|
7199
|
+
"harness": string;
|
|
7200
|
+
"model": string;
|
|
7201
|
+
"provider"?: string;
|
|
7202
|
+
"params"?: {
|
|
7203
|
+
"temperature"?: number;
|
|
7204
|
+
"maxOutputTokens"?: number;
|
|
7205
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7206
|
+
"thinkingBudgetTokens"?: number;
|
|
7207
|
+
};
|
|
7208
|
+
}>;
|
|
7209
|
+
"skills"?: Array<{
|
|
7210
|
+
"ref": string;
|
|
7211
|
+
"name": string;
|
|
7212
|
+
"changedSinceCreate"?: boolean;
|
|
7213
|
+
}>;
|
|
7214
|
+
}>;
|
|
7215
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7216
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7217
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7218
|
+
"modeOptions"?: Record<string, unknown>;
|
|
7219
|
+
};
|
|
7220
|
+
"streamUrl": string;
|
|
7221
|
+
"warnings"?: Array<{
|
|
7222
|
+
"harness": string;
|
|
7223
|
+
"kind": string;
|
|
7224
|
+
} | {
|
|
7225
|
+
"type": "design_aggregate_budget";
|
|
7226
|
+
"detail": string;
|
|
7227
|
+
}>;
|
|
7228
|
+
};
|
|
7229
|
+
export type StudyUpdateLensesRequest = {
|
|
7230
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
7231
|
+
};
|
|
7232
|
+
export type StudyUpdateLensesResponse = {
|
|
7233
|
+
"study": {
|
|
7234
|
+
"id": string;
|
|
7235
|
+
"executiveSummary"?: string | null;
|
|
7236
|
+
"plannedRunCount": number;
|
|
7237
|
+
"organizationId": string;
|
|
7238
|
+
"createdBy": string | null;
|
|
7239
|
+
"name": string;
|
|
7240
|
+
"slug": string;
|
|
7241
|
+
"description": string | null;
|
|
7242
|
+
"skillRef": string | null;
|
|
7243
|
+
"skillSlug": string | null;
|
|
7244
|
+
"skillName": string | null;
|
|
7245
|
+
"skillPromise": string;
|
|
7246
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
7247
|
+
"subjectName": string | null;
|
|
7248
|
+
"subjectLabel": string;
|
|
7249
|
+
"armConfig": {
|
|
7250
|
+
"baseline"?: {
|
|
7251
|
+
"tools"?: "default" | "none" | {
|
|
7252
|
+
"tools": Array<string>;
|
|
7253
|
+
};
|
|
7254
|
+
"addons"?: Record<string, unknown>;
|
|
7255
|
+
};
|
|
7256
|
+
"treatment"?: {
|
|
7257
|
+
"tools"?: "default" | "none" | {
|
|
7258
|
+
"tools": Array<string>;
|
|
7259
|
+
};
|
|
7260
|
+
"addons"?: Record<string, unknown>;
|
|
5306
7261
|
};
|
|
5307
7262
|
};
|
|
5308
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
7263
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5309
7264
|
"subjectBackend": "model" | "harness" | null;
|
|
5310
7265
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5311
7266
|
"importedFrom"?: string;
|
|
@@ -5341,6 +7296,7 @@ export type StudyUpdateLensesResponse = {
|
|
|
5341
7296
|
"error": string | null;
|
|
5342
7297
|
"preparation"?: {
|
|
5343
7298
|
"since": string;
|
|
7299
|
+
"progressAt"?: string;
|
|
5344
7300
|
"launchedBy": {
|
|
5345
7301
|
"userId"?: string;
|
|
5346
7302
|
"role"?: string;
|
|
@@ -5349,6 +7305,12 @@ export type StudyUpdateLensesResponse = {
|
|
|
5349
7305
|
"imageKey": string;
|
|
5350
7306
|
"buildId": string;
|
|
5351
7307
|
"stage"?: "image" | "snapshot";
|
|
7308
|
+
"retry"?: {
|
|
7309
|
+
"failedAttempts": number;
|
|
7310
|
+
"maxAttempts": number;
|
|
7311
|
+
"notBefore": string;
|
|
7312
|
+
"lastFailure": string;
|
|
7313
|
+
};
|
|
5352
7314
|
}>;
|
|
5353
7315
|
} | null;
|
|
5354
7316
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5363,6 +7325,38 @@ export type StudyUpdateLensesResponse = {
|
|
|
5363
7325
|
"createdAt": string;
|
|
5364
7326
|
"updatedAt": string;
|
|
5365
7327
|
"liveUntil": string | null;
|
|
7328
|
+
"arms"?: Array<{
|
|
7329
|
+
"key": string;
|
|
7330
|
+
"position": number;
|
|
7331
|
+
"label": string;
|
|
7332
|
+
"role": "reference" | "peer";
|
|
7333
|
+
"payload": {
|
|
7334
|
+
"tools"?: "default" | "none" | {
|
|
7335
|
+
"tools": Array<string>;
|
|
7336
|
+
};
|
|
7337
|
+
"addons"?: Record<string, unknown>;
|
|
7338
|
+
};
|
|
7339
|
+
"cells"?: Array<{
|
|
7340
|
+
"harness": string;
|
|
7341
|
+
"model": string;
|
|
7342
|
+
"provider"?: string;
|
|
7343
|
+
"params"?: {
|
|
7344
|
+
"temperature"?: number;
|
|
7345
|
+
"maxOutputTokens"?: number;
|
|
7346
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
7347
|
+
"thinkingBudgetTokens"?: number;
|
|
7348
|
+
};
|
|
7349
|
+
}>;
|
|
7350
|
+
"skills"?: Array<{
|
|
7351
|
+
"ref": string;
|
|
7352
|
+
"name": string;
|
|
7353
|
+
"changedSinceCreate"?: boolean;
|
|
7354
|
+
}>;
|
|
7355
|
+
}>;
|
|
7356
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7357
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
7358
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
7359
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5366
7360
|
};
|
|
5367
7361
|
"warnings": Array<{
|
|
5368
7362
|
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
@@ -5396,7 +7390,7 @@ export type StudyPreviewResponse = {
|
|
|
5396
7390
|
"addons"?: Array<string>;
|
|
5397
7391
|
};
|
|
5398
7392
|
};
|
|
5399
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
7393
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5400
7394
|
"subjectBackend"?: "model" | "harness";
|
|
5401
7395
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5402
7396
|
"baselineLabel": string | null;
|
|
@@ -5443,6 +7437,22 @@ export type StudyPreviewResponse = {
|
|
|
5443
7437
|
"kind": string;
|
|
5444
7438
|
"text": string;
|
|
5445
7439
|
}>;
|
|
7440
|
+
"groupCount"?: number;
|
|
7441
|
+
"group"?: {
|
|
7442
|
+
"label": string;
|
|
7443
|
+
"composition": {
|
|
7444
|
+
"tools"?: "default" | "none" | {
|
|
7445
|
+
"tools": Array<string>;
|
|
7446
|
+
};
|
|
7447
|
+
"addons"?: Array<string>;
|
|
7448
|
+
};
|
|
7449
|
+
};
|
|
7450
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
7451
|
+
"groupComposition"?: Array<{
|
|
7452
|
+
"key": string;
|
|
7453
|
+
"label": string;
|
|
7454
|
+
"skills": Array<string>;
|
|
7455
|
+
}>;
|
|
5446
7456
|
};
|
|
5447
7457
|
"tasks": Array<{
|
|
5448
7458
|
"id": string;
|
|
@@ -5459,7 +7469,7 @@ export type StudyPreviewResponse = {
|
|
|
5459
7469
|
"runs": Array<{
|
|
5460
7470
|
"runId": string;
|
|
5461
7471
|
"taskId": string;
|
|
5462
|
-
"arm": "baseline" | "skill";
|
|
7472
|
+
"arm": "baseline" | "skill" | string;
|
|
5463
7473
|
"harness": string;
|
|
5464
7474
|
"model": string;
|
|
5465
7475
|
"provider": string | null;
|
|
@@ -5485,6 +7495,9 @@ export type StudyPreviewResponse = {
|
|
|
5485
7495
|
"pairGradedAt": string | null;
|
|
5486
7496
|
"evalScore": number | null;
|
|
5487
7497
|
"evalGradedAt": string | null;
|
|
7498
|
+
"executionBudgetS"?: number | null;
|
|
7499
|
+
"harnessImageDigest"?: string | null;
|
|
7500
|
+
"subjectEngaged"?: boolean | null;
|
|
5488
7501
|
"runCreatedAt": string | null;
|
|
5489
7502
|
"startedAt": string | null;
|
|
5490
7503
|
"finishedAt": string | null;
|
|
@@ -5502,80 +7515,936 @@ export type StudyPreviewResponse = {
|
|
|
5502
7515
|
}>;
|
|
5503
7516
|
"aggregate": {
|
|
5504
7517
|
"summary": {
|
|
5505
|
-
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
5506
|
-
"headline": Array<{
|
|
5507
|
-
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
7518
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
7519
|
+
"headline": Array<{
|
|
7520
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
7521
|
+
"label": string;
|
|
7522
|
+
"direction": "down" | "up";
|
|
7523
|
+
"baselineAvg": number | null;
|
|
7524
|
+
"skillAvg": number | null;
|
|
7525
|
+
"deltaPct": number | null;
|
|
7526
|
+
"delta"?: {
|
|
7527
|
+
"unit": "pp";
|
|
7528
|
+
"absolute": number | null;
|
|
7529
|
+
"relativePct": number | null;
|
|
7530
|
+
} | {
|
|
7531
|
+
"unit": "pts";
|
|
7532
|
+
"absolute": number | null;
|
|
7533
|
+
"relativePct"?: number | null;
|
|
7534
|
+
} | {
|
|
7535
|
+
"unit": "%";
|
|
7536
|
+
"relativePct": number | null;
|
|
7537
|
+
};
|
|
7538
|
+
"improved": boolean;
|
|
7539
|
+
}>;
|
|
7540
|
+
"completionRate": {
|
|
7541
|
+
"baseline": number | null;
|
|
7542
|
+
"skill": number | null;
|
|
7543
|
+
};
|
|
7544
|
+
"goalRate": {
|
|
7545
|
+
"baseline": number | null;
|
|
7546
|
+
"skill": number | null;
|
|
7547
|
+
};
|
|
7548
|
+
"pairwise": {
|
|
7549
|
+
"pairs": number;
|
|
7550
|
+
"skillWins": number;
|
|
7551
|
+
"baselineWins": number;
|
|
7552
|
+
"ties": number;
|
|
7553
|
+
"winRate": number | null;
|
|
7554
|
+
};
|
|
7555
|
+
"decision"?: {
|
|
7556
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
7557
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
7558
|
+
"reason": string;
|
|
7559
|
+
"efficiencyDiscounted": boolean;
|
|
7560
|
+
};
|
|
7561
|
+
"leaderboard"?: Array<unknown>;
|
|
7562
|
+
"concentration": {
|
|
7563
|
+
"perCell": Array<{
|
|
7564
|
+
"cell": string;
|
|
7565
|
+
"goalDeltaPp": number | null;
|
|
7566
|
+
"contributionPct": number | null;
|
|
7567
|
+
}>;
|
|
7568
|
+
"heterogeneity": {
|
|
7569
|
+
"dominantCell": string;
|
|
7570
|
+
"contributionPct": number;
|
|
7571
|
+
} | null;
|
|
7572
|
+
};
|
|
7573
|
+
"matrixShape": {
|
|
7574
|
+
"cells": number;
|
|
7575
|
+
"harnesses": number;
|
|
7576
|
+
"models": number;
|
|
7577
|
+
"crossed": boolean;
|
|
7578
|
+
};
|
|
7579
|
+
"flakiness"?: {
|
|
7580
|
+
"measured": boolean;
|
|
7581
|
+
"conditions": number;
|
|
7582
|
+
"unstable": number;
|
|
7583
|
+
"worst": {
|
|
7584
|
+
"cell": string;
|
|
7585
|
+
"taskId": string;
|
|
7586
|
+
"arm": "baseline" | "skill";
|
|
7587
|
+
"variant"?: number;
|
|
7588
|
+
"passRate": number;
|
|
7589
|
+
} | null;
|
|
7590
|
+
};
|
|
7591
|
+
"arms"?: {
|
|
7592
|
+
"baseline": {
|
|
7593
|
+
"runs": number;
|
|
7594
|
+
"completed": number;
|
|
7595
|
+
"infraFailures": number;
|
|
7596
|
+
"completionRate": number | null;
|
|
7597
|
+
"goalRate": number | null;
|
|
7598
|
+
"goalTally": {
|
|
7599
|
+
"achieved": number;
|
|
7600
|
+
"partial": number;
|
|
7601
|
+
"missed": number;
|
|
7602
|
+
};
|
|
7603
|
+
"avgTotalTokens": number | null;
|
|
7604
|
+
"avgOutputTokens": number | null;
|
|
7605
|
+
"avgCostUsd": number | null;
|
|
7606
|
+
"avgDurationMs": number | null;
|
|
7607
|
+
"avgTurns": number | null;
|
|
7608
|
+
"costCoverage": number | null;
|
|
7609
|
+
"avgQualityScore": number | null;
|
|
7610
|
+
"gradedCoverage": number | null;
|
|
7611
|
+
"medianCostUsd": number | null;
|
|
7612
|
+
"medianTotalTokens": number | null;
|
|
7613
|
+
"medianDurationMs": number | null;
|
|
7614
|
+
"p95CostUsd": number | null;
|
|
7615
|
+
"p95TotalTokens": number | null;
|
|
7616
|
+
"p95DurationMs": number | null;
|
|
7617
|
+
"maxCostUsd": number | null;
|
|
7618
|
+
"maxTotalTokens": number | null;
|
|
7619
|
+
"maxDurationMs": number | null;
|
|
7620
|
+
"evalPassRate"?: number | null;
|
|
7621
|
+
"benchmarkPassRate"?: number | null;
|
|
7622
|
+
"adoptionRate"?: number | null;
|
|
7623
|
+
};
|
|
7624
|
+
"skill": {
|
|
7625
|
+
"runs": number;
|
|
7626
|
+
"completed": number;
|
|
7627
|
+
"infraFailures": number;
|
|
7628
|
+
"completionRate": number | null;
|
|
7629
|
+
"goalRate": number | null;
|
|
7630
|
+
"goalTally": {
|
|
7631
|
+
"achieved": number;
|
|
7632
|
+
"partial": number;
|
|
7633
|
+
"missed": number;
|
|
7634
|
+
};
|
|
7635
|
+
"avgTotalTokens": number | null;
|
|
7636
|
+
"avgOutputTokens": number | null;
|
|
7637
|
+
"avgCostUsd": number | null;
|
|
7638
|
+
"avgDurationMs": number | null;
|
|
7639
|
+
"avgTurns": number | null;
|
|
7640
|
+
"costCoverage": number | null;
|
|
7641
|
+
"avgQualityScore": number | null;
|
|
7642
|
+
"gradedCoverage": number | null;
|
|
7643
|
+
"medianCostUsd": number | null;
|
|
7644
|
+
"medianTotalTokens": number | null;
|
|
7645
|
+
"medianDurationMs": number | null;
|
|
7646
|
+
"p95CostUsd": number | null;
|
|
7647
|
+
"p95TotalTokens": number | null;
|
|
7648
|
+
"p95DurationMs": number | null;
|
|
7649
|
+
"maxCostUsd": number | null;
|
|
7650
|
+
"maxTotalTokens": number | null;
|
|
7651
|
+
"maxDurationMs": number | null;
|
|
7652
|
+
"evalPassRate"?: number | null;
|
|
7653
|
+
"benchmarkPassRate"?: number | null;
|
|
7654
|
+
"adoptionRate"?: number | null;
|
|
7655
|
+
};
|
|
7656
|
+
};
|
|
7657
|
+
"totals": {
|
|
7658
|
+
"cells": number;
|
|
7659
|
+
"runs": number;
|
|
7660
|
+
"completed": number;
|
|
7661
|
+
"failed": number;
|
|
7662
|
+
"costUsd": number;
|
|
7663
|
+
"tokens": number;
|
|
7664
|
+
};
|
|
7665
|
+
};
|
|
7666
|
+
"perCell": Array<unknown>;
|
|
7667
|
+
"perHarness": Array<unknown>;
|
|
7668
|
+
"perModel": Array<unknown>;
|
|
7669
|
+
"perTask": Array<unknown>;
|
|
7670
|
+
"webSplit": unknown;
|
|
7671
|
+
"inference": {
|
|
7672
|
+
"columns": Array<{
|
|
7673
|
+
"key": string;
|
|
7674
|
+
"label": string;
|
|
7675
|
+
"family": "rate" | "score" | "magnitude";
|
|
7676
|
+
"betterWhen": "higher" | "lower";
|
|
7677
|
+
}>;
|
|
7678
|
+
"groups": Array<{
|
|
7679
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
7680
|
+
"title": string;
|
|
7681
|
+
"rows": Array<{
|
|
7682
|
+
"key": string;
|
|
7683
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
7684
|
+
"harness": string | null;
|
|
7685
|
+
"model": string | null;
|
|
7686
|
+
"paramsKey"?: string;
|
|
7687
|
+
"runs": number;
|
|
7688
|
+
"cells": Record<string, unknown>;
|
|
7689
|
+
}>;
|
|
7690
|
+
}>;
|
|
7691
|
+
"reference": "baseline" | "field";
|
|
7692
|
+
"draws": number;
|
|
7693
|
+
} | null;
|
|
7694
|
+
"behavior"?: Array<{
|
|
7695
|
+
"harness": string;
|
|
7696
|
+
"model": string;
|
|
7697
|
+
"provider": string | null;
|
|
7698
|
+
"arm": string;
|
|
7699
|
+
"totalRuns": number;
|
|
7700
|
+
"signals": Array<{
|
|
7701
|
+
"id": string;
|
|
7702
|
+
"label": string;
|
|
7703
|
+
"tone": "good" | "bad" | "neutral";
|
|
7704
|
+
"runCount": number;
|
|
7705
|
+
"succeededCount": number;
|
|
7706
|
+
}>;
|
|
7707
|
+
}>;
|
|
7708
|
+
"reference"?: {
|
|
7709
|
+
"cells": Array<{
|
|
7710
|
+
"cell": string;
|
|
7711
|
+
"harness": string;
|
|
7712
|
+
"model": string;
|
|
7713
|
+
"provider": string | null;
|
|
7714
|
+
"paramsKey": string;
|
|
7715
|
+
"label": string;
|
|
7716
|
+
"passRate": {
|
|
7717
|
+
"value": number | null;
|
|
7718
|
+
"ci": Array<unknown> | null;
|
|
7719
|
+
"n": number;
|
|
7720
|
+
};
|
|
7721
|
+
"abstention"?: {
|
|
7722
|
+
"value": number | null;
|
|
7723
|
+
"ci": Array<unknown> | null;
|
|
7724
|
+
"n": number;
|
|
7725
|
+
"parts"?: Array<{
|
|
7726
|
+
"id": string;
|
|
7727
|
+
"label": string;
|
|
7728
|
+
"value": number | null;
|
|
7729
|
+
"n": number;
|
|
7730
|
+
}>;
|
|
7731
|
+
};
|
|
7732
|
+
"tasks": number;
|
|
7733
|
+
"graded": number;
|
|
7734
|
+
"unmeasured": number;
|
|
7735
|
+
"matchers": Array<string>;
|
|
7736
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
7737
|
+
}>;
|
|
7738
|
+
"overall": {
|
|
7739
|
+
"value": number | null;
|
|
7740
|
+
"ci": Array<unknown> | null;
|
|
7741
|
+
"n": number;
|
|
7742
|
+
};
|
|
7743
|
+
"abstention"?: {
|
|
7744
|
+
"value": number | null;
|
|
7745
|
+
"ci": Array<unknown> | null;
|
|
7746
|
+
"n": number;
|
|
7747
|
+
"parts"?: Array<{
|
|
7748
|
+
"id": string;
|
|
7749
|
+
"label": string;
|
|
7750
|
+
"value": number | null;
|
|
7751
|
+
"n": number;
|
|
7752
|
+
}>;
|
|
7753
|
+
};
|
|
7754
|
+
"graded": number;
|
|
7755
|
+
"unmeasured": number;
|
|
7756
|
+
"modelJudged": boolean;
|
|
7757
|
+
};
|
|
7758
|
+
"execution"?: {
|
|
7759
|
+
"graded": number;
|
|
7760
|
+
"measured": number;
|
|
7761
|
+
"passed": number;
|
|
7762
|
+
"passRate": number | null;
|
|
7763
|
+
"byStatus": {
|
|
7764
|
+
"passed": number;
|
|
7765
|
+
"failed": number;
|
|
7766
|
+
"timeout": number;
|
|
7767
|
+
"error": number;
|
|
7768
|
+
"unavailable": number;
|
|
7769
|
+
};
|
|
7770
|
+
"languages": Array<string>;
|
|
7771
|
+
"unavailableReasons": Array<string>;
|
|
7772
|
+
};
|
|
7773
|
+
} | {
|
|
7774
|
+
"shape": "multi-arm";
|
|
7775
|
+
"summary": {
|
|
7776
|
+
"shape": "multi-arm";
|
|
7777
|
+
"verdict": "measured" | "inconclusive";
|
|
7778
|
+
"groups": Array<{
|
|
7779
|
+
"key": string;
|
|
5508
7780
|
"label": string;
|
|
5509
|
-
"
|
|
5510
|
-
"
|
|
5511
|
-
|
|
5512
|
-
|
|
5513
|
-
|
|
5514
|
-
"
|
|
5515
|
-
"absolute": number | null;
|
|
5516
|
-
"relativePct": number | null;
|
|
5517
|
-
} | {
|
|
5518
|
-
"unit": "pts";
|
|
5519
|
-
"absolute": number | null;
|
|
5520
|
-
"relativePct"?: number | null;
|
|
5521
|
-
} | {
|
|
5522
|
-
"unit": "%";
|
|
5523
|
-
"relativePct": number | null;
|
|
7781
|
+
"role": "reference" | "peer";
|
|
7782
|
+
"composition": {
|
|
7783
|
+
"tools"?: "default" | "none" | {
|
|
7784
|
+
"tools": Array<string>;
|
|
7785
|
+
};
|
|
7786
|
+
"addons"?: Array<string>;
|
|
5524
7787
|
};
|
|
5525
|
-
"improved": boolean;
|
|
5526
7788
|
}>;
|
|
5527
|
-
"
|
|
5528
|
-
"
|
|
5529
|
-
"
|
|
7789
|
+
"ranking": {
|
|
7790
|
+
"plan": "vs_reference" | "all_pairs" | null;
|
|
7791
|
+
"referenceArm": string | null;
|
|
7792
|
+
"decision": {
|
|
7793
|
+
"leader": string | null;
|
|
7794
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
7795
|
+
"reason": string;
|
|
7796
|
+
"separatedFrom": Array<string>;
|
|
7797
|
+
"lostTo": Array<{
|
|
7798
|
+
"arm": string;
|
|
7799
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
7800
|
+
}>;
|
|
7801
|
+
"clearLeader": boolean;
|
|
7802
|
+
"correctnessLeader": string | null;
|
|
7803
|
+
};
|
|
7804
|
+
"entries": Array<{
|
|
7805
|
+
"arm": string;
|
|
7806
|
+
"runs": number;
|
|
7807
|
+
"completed": number;
|
|
7808
|
+
"completionRate": number | null;
|
|
7809
|
+
"goalRate": number | null;
|
|
7810
|
+
"avgQualityScore": number | null;
|
|
7811
|
+
"evalPassRate": number | null;
|
|
7812
|
+
"benchmarkPassRate"?: number | null;
|
|
7813
|
+
"compositeScore": number | null;
|
|
7814
|
+
"avgTotalTokens": number | null;
|
|
7815
|
+
"avgOutputTokens": number | null;
|
|
7816
|
+
"avgCostUsd": number | null;
|
|
7817
|
+
"avgDurationMs": number | null;
|
|
7818
|
+
"medianCostUsd": number | null;
|
|
7819
|
+
"medianTotalTokens": number | null;
|
|
7820
|
+
"medianDurationMs": number | null;
|
|
7821
|
+
"rank": number;
|
|
7822
|
+
}>;
|
|
7823
|
+
"contrasts": Array<{
|
|
7824
|
+
"contrastId": string;
|
|
7825
|
+
"referenceArm": string;
|
|
7826
|
+
"subjectArm": string;
|
|
7827
|
+
"oriented": boolean;
|
|
7828
|
+
"outcome": "won" | "tie" | "unmeasurable";
|
|
7829
|
+
"winner": string | null;
|
|
7830
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
7831
|
+
"explanation": string;
|
|
7832
|
+
}>;
|
|
7833
|
+
"comparisons": number;
|
|
7834
|
+
"headline"?: string;
|
|
5530
7835
|
};
|
|
5531
|
-
"
|
|
5532
|
-
"
|
|
5533
|
-
"
|
|
7836
|
+
"matrixShape": {
|
|
7837
|
+
"cells": number;
|
|
7838
|
+
"harnesses": number;
|
|
7839
|
+
"models": number;
|
|
7840
|
+
"crossed": boolean;
|
|
5534
7841
|
};
|
|
5535
|
-
"
|
|
5536
|
-
"
|
|
5537
|
-
"
|
|
5538
|
-
"
|
|
5539
|
-
"
|
|
5540
|
-
|
|
7842
|
+
"flakiness": {
|
|
7843
|
+
"measured": boolean;
|
|
7844
|
+
"conditions": number;
|
|
7845
|
+
"unstable": number;
|
|
7846
|
+
"worst": {
|
|
7847
|
+
"cell": string;
|
|
7848
|
+
"taskId": string;
|
|
7849
|
+
"arm": string;
|
|
7850
|
+
"variant"?: number;
|
|
7851
|
+
"passRate": number;
|
|
7852
|
+
} | null;
|
|
5541
7853
|
};
|
|
5542
|
-
"
|
|
5543
|
-
"
|
|
5544
|
-
"
|
|
5545
|
-
"
|
|
5546
|
-
"
|
|
7854
|
+
"totals": {
|
|
7855
|
+
"cells": number;
|
|
7856
|
+
"runs": number;
|
|
7857
|
+
"completed": number;
|
|
7858
|
+
"failed": number;
|
|
7859
|
+
"costUsd": number;
|
|
7860
|
+
"tokens": number;
|
|
5547
7861
|
};
|
|
5548
|
-
|
|
5549
|
-
|
|
5550
|
-
|
|
7862
|
+
};
|
|
7863
|
+
"arms": Array<{
|
|
7864
|
+
"key": string;
|
|
7865
|
+
"label": string;
|
|
7866
|
+
"position": number;
|
|
7867
|
+
"role": "reference" | "peer";
|
|
7868
|
+
"composition": {
|
|
7869
|
+
"tools"?: "default" | "none" | {
|
|
7870
|
+
"tools": Array<string>;
|
|
7871
|
+
};
|
|
7872
|
+
"addons"?: Array<string>;
|
|
7873
|
+
};
|
|
7874
|
+
"agg": {
|
|
7875
|
+
"runs": number;
|
|
7876
|
+
"completed": number;
|
|
7877
|
+
"infraFailures": number;
|
|
7878
|
+
"completionRate": number | null;
|
|
7879
|
+
"goalRate": number | null;
|
|
7880
|
+
"goalTally": {
|
|
7881
|
+
"achieved": number;
|
|
7882
|
+
"partial": number;
|
|
7883
|
+
"missed": number;
|
|
7884
|
+
};
|
|
7885
|
+
"avgTotalTokens": number | null;
|
|
7886
|
+
"avgOutputTokens": number | null;
|
|
7887
|
+
"avgCostUsd": number | null;
|
|
7888
|
+
"avgDurationMs": number | null;
|
|
7889
|
+
"avgTurns": number | null;
|
|
7890
|
+
"costCoverage": number | null;
|
|
7891
|
+
"avgQualityScore": number | null;
|
|
7892
|
+
"gradedCoverage": number | null;
|
|
7893
|
+
"medianCostUsd": number | null;
|
|
7894
|
+
"medianTotalTokens": number | null;
|
|
7895
|
+
"medianDurationMs": number | null;
|
|
7896
|
+
"p95CostUsd": number | null;
|
|
7897
|
+
"p95TotalTokens": number | null;
|
|
7898
|
+
"p95DurationMs": number | null;
|
|
7899
|
+
"maxCostUsd": number | null;
|
|
7900
|
+
"maxTotalTokens": number | null;
|
|
7901
|
+
"maxDurationMs": number | null;
|
|
7902
|
+
"evalPassRate"?: number | null;
|
|
7903
|
+
"benchmarkPassRate"?: number | null;
|
|
7904
|
+
"adoptionRate"?: number | null;
|
|
7905
|
+
};
|
|
7906
|
+
"perCell": Array<{
|
|
7907
|
+
"harness": string;
|
|
7908
|
+
"model": string;
|
|
7909
|
+
"provider": string | null;
|
|
7910
|
+
"paramsKey"?: string;
|
|
7911
|
+
"label"?: string;
|
|
7912
|
+
"agg": {
|
|
7913
|
+
"runs": number;
|
|
7914
|
+
"completed": number;
|
|
7915
|
+
"infraFailures": number;
|
|
7916
|
+
"completionRate": number | null;
|
|
7917
|
+
"goalRate": number | null;
|
|
7918
|
+
"goalTally": {
|
|
7919
|
+
"achieved": number;
|
|
7920
|
+
"partial": number;
|
|
7921
|
+
"missed": number;
|
|
7922
|
+
};
|
|
7923
|
+
"avgTotalTokens": number | null;
|
|
7924
|
+
"avgOutputTokens": number | null;
|
|
7925
|
+
"avgCostUsd": number | null;
|
|
7926
|
+
"avgDurationMs": number | null;
|
|
7927
|
+
"avgTurns": number | null;
|
|
7928
|
+
"costCoverage": number | null;
|
|
7929
|
+
"avgQualityScore": number | null;
|
|
7930
|
+
"gradedCoverage": number | null;
|
|
7931
|
+
"medianCostUsd": number | null;
|
|
7932
|
+
"medianTotalTokens": number | null;
|
|
7933
|
+
"medianDurationMs": number | null;
|
|
7934
|
+
"p95CostUsd": number | null;
|
|
7935
|
+
"p95TotalTokens": number | null;
|
|
7936
|
+
"p95DurationMs": number | null;
|
|
7937
|
+
"maxCostUsd": number | null;
|
|
7938
|
+
"maxTotalTokens": number | null;
|
|
7939
|
+
"maxDurationMs": number | null;
|
|
7940
|
+
"evalPassRate"?: number | null;
|
|
7941
|
+
"benchmarkPassRate"?: number | null;
|
|
7942
|
+
"adoptionRate"?: number | null;
|
|
7943
|
+
};
|
|
7944
|
+
}>;
|
|
7945
|
+
"reference"?: {
|
|
7946
|
+
"cells": Array<{
|
|
5551
7947
|
"cell": string;
|
|
5552
|
-
"
|
|
5553
|
-
"
|
|
7948
|
+
"harness": string;
|
|
7949
|
+
"model": string;
|
|
7950
|
+
"provider": string | null;
|
|
7951
|
+
"paramsKey": string;
|
|
7952
|
+
"label": string;
|
|
7953
|
+
"passRate": {
|
|
7954
|
+
"value": number | null;
|
|
7955
|
+
"ci": Array<unknown> | null;
|
|
7956
|
+
"n": number;
|
|
7957
|
+
};
|
|
7958
|
+
"abstention"?: {
|
|
7959
|
+
"value": number | null;
|
|
7960
|
+
"ci": Array<unknown> | null;
|
|
7961
|
+
"n": number;
|
|
7962
|
+
"parts"?: Array<{
|
|
7963
|
+
"id": string;
|
|
7964
|
+
"label": string;
|
|
7965
|
+
"value": number | null;
|
|
7966
|
+
"n": number;
|
|
7967
|
+
}>;
|
|
7968
|
+
};
|
|
7969
|
+
"tasks": number;
|
|
7970
|
+
"graded": number;
|
|
7971
|
+
"unmeasured": number;
|
|
7972
|
+
"matchers": Array<string>;
|
|
7973
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
5554
7974
|
}>;
|
|
5555
|
-
"
|
|
5556
|
-
"
|
|
5557
|
-
"
|
|
7975
|
+
"overall": {
|
|
7976
|
+
"value": number | null;
|
|
7977
|
+
"ci": Array<unknown> | null;
|
|
7978
|
+
"n": number;
|
|
7979
|
+
};
|
|
7980
|
+
"abstention"?: {
|
|
7981
|
+
"value": number | null;
|
|
7982
|
+
"ci": Array<unknown> | null;
|
|
7983
|
+
"n": number;
|
|
7984
|
+
"parts"?: Array<{
|
|
7985
|
+
"id": string;
|
|
7986
|
+
"label": string;
|
|
7987
|
+
"value": number | null;
|
|
7988
|
+
"n": number;
|
|
7989
|
+
}>;
|
|
7990
|
+
};
|
|
7991
|
+
"graded": number;
|
|
7992
|
+
"unmeasured": number;
|
|
7993
|
+
"modelJudged": boolean;
|
|
7994
|
+
};
|
|
7995
|
+
}>;
|
|
7996
|
+
"contrasts": Array<{
|
|
7997
|
+
"contrastId": string;
|
|
7998
|
+
"referenceArm": string;
|
|
7999
|
+
"subjectArm": string;
|
|
8000
|
+
"oriented": boolean;
|
|
8001
|
+
"basis"?: {
|
|
8002
|
+
"id": "matched" | "whole_setup";
|
|
8003
|
+
"compared": {
|
|
8004
|
+
"reference": Array<{
|
|
8005
|
+
"harness": string;
|
|
8006
|
+
"model": string;
|
|
8007
|
+
"provider"?: string;
|
|
8008
|
+
"params"?: Record<string, unknown>;
|
|
8009
|
+
}>;
|
|
8010
|
+
"subject": Array<{
|
|
8011
|
+
"harness": string;
|
|
8012
|
+
"model": string;
|
|
8013
|
+
"provider"?: string;
|
|
8014
|
+
"params"?: Record<string, unknown>;
|
|
8015
|
+
}>;
|
|
8016
|
+
};
|
|
8017
|
+
"excluded": {
|
|
8018
|
+
"reference": Array<{
|
|
8019
|
+
"harness": string;
|
|
8020
|
+
"model": string;
|
|
8021
|
+
"provider"?: string;
|
|
8022
|
+
"params"?: Record<string, unknown>;
|
|
8023
|
+
}>;
|
|
8024
|
+
"subject": Array<{
|
|
8025
|
+
"harness": string;
|
|
8026
|
+
"model": string;
|
|
8027
|
+
"provider"?: string;
|
|
8028
|
+
"params"?: Record<string, unknown>;
|
|
8029
|
+
}>;
|
|
8030
|
+
};
|
|
8031
|
+
"differsIn": Array<string>;
|
|
8032
|
+
"licensedFacet": string | null;
|
|
8033
|
+
"efficiency": "comparable" | "incomparable";
|
|
8034
|
+
"pairedTasks": number | null;
|
|
8035
|
+
"dropped": {
|
|
8036
|
+
"reference": number;
|
|
8037
|
+
"subject": number;
|
|
8038
|
+
} | null;
|
|
8039
|
+
};
|
|
8040
|
+
"magnitudes": "relative";
|
|
8041
|
+
"summary": {
|
|
8042
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
8043
|
+
"headline": Array<{
|
|
8044
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
8045
|
+
"label": string;
|
|
8046
|
+
"direction": "down" | "up";
|
|
8047
|
+
"baselineAvg": number | null;
|
|
8048
|
+
"skillAvg": number | null;
|
|
8049
|
+
"deltaPct": number | null;
|
|
8050
|
+
"delta"?: {
|
|
8051
|
+
"unit": "pp";
|
|
8052
|
+
"absolute": number | null;
|
|
8053
|
+
"relativePct": number | null;
|
|
8054
|
+
} | {
|
|
8055
|
+
"unit": "pts";
|
|
8056
|
+
"absolute": number | null;
|
|
8057
|
+
"relativePct"?: number | null;
|
|
8058
|
+
} | {
|
|
8059
|
+
"unit": "%";
|
|
8060
|
+
"relativePct": number | null;
|
|
8061
|
+
};
|
|
8062
|
+
"improved": boolean;
|
|
8063
|
+
}>;
|
|
8064
|
+
"completionRate": {
|
|
8065
|
+
"baseline": number | null;
|
|
8066
|
+
"skill": number | null;
|
|
8067
|
+
};
|
|
8068
|
+
"goalRate": {
|
|
8069
|
+
"baseline": number | null;
|
|
8070
|
+
"skill": number | null;
|
|
8071
|
+
};
|
|
8072
|
+
"pairwise": {
|
|
8073
|
+
"pairs": number;
|
|
8074
|
+
"skillWins": number;
|
|
8075
|
+
"baselineWins": number;
|
|
8076
|
+
"ties": number;
|
|
8077
|
+
"winRate": number | null;
|
|
8078
|
+
};
|
|
8079
|
+
"decision"?: {
|
|
8080
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
8081
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8082
|
+
"reason": string;
|
|
8083
|
+
"efficiencyDiscounted": boolean;
|
|
8084
|
+
};
|
|
8085
|
+
"concentration": {
|
|
8086
|
+
"perCell": Array<{
|
|
8087
|
+
"cell": string;
|
|
8088
|
+
"goalDeltaPp": number | null;
|
|
8089
|
+
"contributionPct": number | null;
|
|
8090
|
+
}>;
|
|
8091
|
+
"heterogeneity": {
|
|
8092
|
+
"dominantCell": string;
|
|
8093
|
+
"contributionPct": number;
|
|
8094
|
+
} | null;
|
|
8095
|
+
};
|
|
8096
|
+
"matrixShape": {
|
|
8097
|
+
"cells": number;
|
|
8098
|
+
"harnesses": number;
|
|
8099
|
+
"models": number;
|
|
8100
|
+
"crossed": boolean;
|
|
8101
|
+
};
|
|
8102
|
+
"flakiness"?: {
|
|
8103
|
+
"measured": boolean;
|
|
8104
|
+
"conditions": number;
|
|
8105
|
+
"unstable": number;
|
|
8106
|
+
"worst": {
|
|
8107
|
+
"cell": string;
|
|
8108
|
+
"taskId": string;
|
|
8109
|
+
"arm": "baseline" | "skill";
|
|
8110
|
+
"variant"?: number;
|
|
8111
|
+
"passRate": number;
|
|
8112
|
+
} | null;
|
|
8113
|
+
};
|
|
8114
|
+
"arms"?: {
|
|
8115
|
+
"baseline": {
|
|
8116
|
+
"runs": number;
|
|
8117
|
+
"completed": number;
|
|
8118
|
+
"infraFailures": number;
|
|
8119
|
+
"completionRate": number | null;
|
|
8120
|
+
"goalRate": number | null;
|
|
8121
|
+
"goalTally": {
|
|
8122
|
+
"achieved": number;
|
|
8123
|
+
"partial": number;
|
|
8124
|
+
"missed": number;
|
|
8125
|
+
};
|
|
8126
|
+
"avgTotalTokens": number | null;
|
|
8127
|
+
"avgOutputTokens": number | null;
|
|
8128
|
+
"avgCostUsd": number | null;
|
|
8129
|
+
"avgDurationMs": number | null;
|
|
8130
|
+
"avgTurns": number | null;
|
|
8131
|
+
"costCoverage": number | null;
|
|
8132
|
+
"avgQualityScore": number | null;
|
|
8133
|
+
"gradedCoverage": number | null;
|
|
8134
|
+
"medianCostUsd": number | null;
|
|
8135
|
+
"medianTotalTokens": number | null;
|
|
8136
|
+
"medianDurationMs": number | null;
|
|
8137
|
+
"p95CostUsd": number | null;
|
|
8138
|
+
"p95TotalTokens": number | null;
|
|
8139
|
+
"p95DurationMs": number | null;
|
|
8140
|
+
"maxCostUsd": number | null;
|
|
8141
|
+
"maxTotalTokens": number | null;
|
|
8142
|
+
"maxDurationMs": number | null;
|
|
8143
|
+
"evalPassRate"?: number | null;
|
|
8144
|
+
"benchmarkPassRate"?: number | null;
|
|
8145
|
+
"adoptionRate"?: number | null;
|
|
8146
|
+
};
|
|
8147
|
+
"skill": {
|
|
8148
|
+
"runs": number;
|
|
8149
|
+
"completed": number;
|
|
8150
|
+
"infraFailures": number;
|
|
8151
|
+
"completionRate": number | null;
|
|
8152
|
+
"goalRate": number | null;
|
|
8153
|
+
"goalTally": {
|
|
8154
|
+
"achieved": number;
|
|
8155
|
+
"partial": number;
|
|
8156
|
+
"missed": number;
|
|
8157
|
+
};
|
|
8158
|
+
"avgTotalTokens": number | null;
|
|
8159
|
+
"avgOutputTokens": number | null;
|
|
8160
|
+
"avgCostUsd": number | null;
|
|
8161
|
+
"avgDurationMs": number | null;
|
|
8162
|
+
"avgTurns": number | null;
|
|
8163
|
+
"costCoverage": number | null;
|
|
8164
|
+
"avgQualityScore": number | null;
|
|
8165
|
+
"gradedCoverage": number | null;
|
|
8166
|
+
"medianCostUsd": number | null;
|
|
8167
|
+
"medianTotalTokens": number | null;
|
|
8168
|
+
"medianDurationMs": number | null;
|
|
8169
|
+
"p95CostUsd": number | null;
|
|
8170
|
+
"p95TotalTokens": number | null;
|
|
8171
|
+
"p95DurationMs": number | null;
|
|
8172
|
+
"maxCostUsd": number | null;
|
|
8173
|
+
"maxTotalTokens": number | null;
|
|
8174
|
+
"maxDurationMs": number | null;
|
|
8175
|
+
"evalPassRate"?: number | null;
|
|
8176
|
+
"benchmarkPassRate"?: number | null;
|
|
8177
|
+
"adoptionRate"?: number | null;
|
|
8178
|
+
};
|
|
8179
|
+
};
|
|
8180
|
+
"totals": {
|
|
8181
|
+
"cells": number;
|
|
8182
|
+
"runs": number;
|
|
8183
|
+
"completed": number;
|
|
8184
|
+
"failed": number;
|
|
8185
|
+
"costUsd": number;
|
|
8186
|
+
"tokens": number;
|
|
8187
|
+
};
|
|
8188
|
+
};
|
|
8189
|
+
"perCell": Array<{
|
|
8190
|
+
"harness": string;
|
|
8191
|
+
"model": string;
|
|
8192
|
+
"provider": string | null;
|
|
8193
|
+
"paramsKey"?: string;
|
|
8194
|
+
"label"?: string;
|
|
8195
|
+
"baseline": {
|
|
8196
|
+
"runs": number;
|
|
8197
|
+
"completed": number;
|
|
8198
|
+
"infraFailures": number;
|
|
8199
|
+
"completionRate": number | null;
|
|
8200
|
+
"goalRate": number | null;
|
|
8201
|
+
"goalTally": {
|
|
8202
|
+
"achieved": number;
|
|
8203
|
+
"partial": number;
|
|
8204
|
+
"missed": number;
|
|
8205
|
+
};
|
|
8206
|
+
"avgTotalTokens": number | null;
|
|
8207
|
+
"avgOutputTokens": number | null;
|
|
8208
|
+
"avgCostUsd": number | null;
|
|
8209
|
+
"avgDurationMs": number | null;
|
|
8210
|
+
"avgTurns": number | null;
|
|
8211
|
+
"costCoverage": number | null;
|
|
8212
|
+
"avgQualityScore": number | null;
|
|
8213
|
+
"gradedCoverage": number | null;
|
|
8214
|
+
"medianCostUsd": number | null;
|
|
8215
|
+
"medianTotalTokens": number | null;
|
|
8216
|
+
"medianDurationMs": number | null;
|
|
8217
|
+
"p95CostUsd": number | null;
|
|
8218
|
+
"p95TotalTokens": number | null;
|
|
8219
|
+
"p95DurationMs": number | null;
|
|
8220
|
+
"maxCostUsd": number | null;
|
|
8221
|
+
"maxTotalTokens": number | null;
|
|
8222
|
+
"maxDurationMs": number | null;
|
|
8223
|
+
"evalPassRate"?: number | null;
|
|
8224
|
+
"benchmarkPassRate"?: number | null;
|
|
8225
|
+
"adoptionRate"?: number | null;
|
|
8226
|
+
};
|
|
8227
|
+
"skill": {
|
|
8228
|
+
"runs": number;
|
|
8229
|
+
"completed": number;
|
|
8230
|
+
"infraFailures": number;
|
|
8231
|
+
"completionRate": number | null;
|
|
8232
|
+
"goalRate": number | null;
|
|
8233
|
+
"goalTally": {
|
|
8234
|
+
"achieved": number;
|
|
8235
|
+
"partial": number;
|
|
8236
|
+
"missed": number;
|
|
8237
|
+
};
|
|
8238
|
+
"avgTotalTokens": number | null;
|
|
8239
|
+
"avgOutputTokens": number | null;
|
|
8240
|
+
"avgCostUsd": number | null;
|
|
8241
|
+
"avgDurationMs": number | null;
|
|
8242
|
+
"avgTurns": number | null;
|
|
8243
|
+
"costCoverage": number | null;
|
|
8244
|
+
"avgQualityScore": number | null;
|
|
8245
|
+
"gradedCoverage": number | null;
|
|
8246
|
+
"medianCostUsd": number | null;
|
|
8247
|
+
"medianTotalTokens": number | null;
|
|
8248
|
+
"medianDurationMs": number | null;
|
|
8249
|
+
"p95CostUsd": number | null;
|
|
8250
|
+
"p95TotalTokens": number | null;
|
|
8251
|
+
"p95DurationMs": number | null;
|
|
8252
|
+
"maxCostUsd": number | null;
|
|
8253
|
+
"maxTotalTokens": number | null;
|
|
8254
|
+
"maxDurationMs": number | null;
|
|
8255
|
+
"evalPassRate"?: number | null;
|
|
8256
|
+
"benchmarkPassRate"?: number | null;
|
|
8257
|
+
"adoptionRate"?: number | null;
|
|
8258
|
+
};
|
|
8259
|
+
"deltas": {
|
|
8260
|
+
"totalTokensPct": number | null;
|
|
8261
|
+
"outputTokensPct": number | null;
|
|
8262
|
+
"costUsdPct": number | null;
|
|
8263
|
+
"durationMsPct": number | null;
|
|
8264
|
+
"numTurnsPct": number | null;
|
|
8265
|
+
"completionRatePp": number | null;
|
|
8266
|
+
"goalRatePp": number | null;
|
|
8267
|
+
"qualityScorePts": number | null;
|
|
8268
|
+
"evalPassRatePp"?: number | null;
|
|
8269
|
+
};
|
|
8270
|
+
}>;
|
|
8271
|
+
} | {
|
|
8272
|
+
"contrastId": string;
|
|
8273
|
+
"referenceArm": string;
|
|
8274
|
+
"subjectArm": string;
|
|
8275
|
+
"oriented": boolean;
|
|
8276
|
+
"basis"?: {
|
|
8277
|
+
"id": "matched" | "whole_setup";
|
|
8278
|
+
"compared": {
|
|
8279
|
+
"reference": Array<{
|
|
8280
|
+
"harness": string;
|
|
8281
|
+
"model": string;
|
|
8282
|
+
"provider"?: string;
|
|
8283
|
+
"params"?: Record<string, unknown>;
|
|
8284
|
+
}>;
|
|
8285
|
+
"subject": Array<{
|
|
8286
|
+
"harness": string;
|
|
8287
|
+
"model": string;
|
|
8288
|
+
"provider"?: string;
|
|
8289
|
+
"params"?: Record<string, unknown>;
|
|
8290
|
+
}>;
|
|
8291
|
+
};
|
|
8292
|
+
"excluded": {
|
|
8293
|
+
"reference": Array<{
|
|
8294
|
+
"harness": string;
|
|
8295
|
+
"model": string;
|
|
8296
|
+
"provider"?: string;
|
|
8297
|
+
"params"?: Record<string, unknown>;
|
|
8298
|
+
}>;
|
|
8299
|
+
"subject": Array<{
|
|
8300
|
+
"harness": string;
|
|
8301
|
+
"model": string;
|
|
8302
|
+
"provider"?: string;
|
|
8303
|
+
"params"?: Record<string, unknown>;
|
|
8304
|
+
}>;
|
|
8305
|
+
};
|
|
8306
|
+
"differsIn": Array<string>;
|
|
8307
|
+
"licensedFacet": string | null;
|
|
8308
|
+
"efficiency": "comparable" | "incomparable";
|
|
8309
|
+
"pairedTasks": number | null;
|
|
8310
|
+
"dropped": {
|
|
8311
|
+
"reference": number;
|
|
8312
|
+
"subject": number;
|
|
5558
8313
|
} | null;
|
|
5559
8314
|
};
|
|
5560
|
-
"
|
|
5561
|
-
|
|
5562
|
-
"
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
|
|
5569
|
-
|
|
5570
|
-
"
|
|
5571
|
-
"
|
|
5572
|
-
"
|
|
5573
|
-
"
|
|
5574
|
-
"
|
|
5575
|
-
"
|
|
5576
|
-
}
|
|
8315
|
+
"magnitudes": "absolute";
|
|
8316
|
+
"summary": {
|
|
8317
|
+
"completionRate": {
|
|
8318
|
+
"baseline": number | null;
|
|
8319
|
+
"skill": number | null;
|
|
8320
|
+
};
|
|
8321
|
+
"goalRate": {
|
|
8322
|
+
"baseline": number | null;
|
|
8323
|
+
"skill": number | null;
|
|
8324
|
+
};
|
|
8325
|
+
"pairwise": {
|
|
8326
|
+
"pairs": number;
|
|
8327
|
+
"skillWins": number;
|
|
8328
|
+
"baselineWins": number;
|
|
8329
|
+
"ties": number;
|
|
8330
|
+
"winRate": number | null;
|
|
8331
|
+
};
|
|
8332
|
+
"decision"?: {
|
|
8333
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
8334
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
8335
|
+
"reason": string;
|
|
8336
|
+
"efficiencyDiscounted": boolean;
|
|
8337
|
+
};
|
|
8338
|
+
"concentration": {
|
|
8339
|
+
"perCell": Array<{
|
|
8340
|
+
"cell": string;
|
|
8341
|
+
"goalDeltaPp": number | null;
|
|
8342
|
+
"contributionPct": number | null;
|
|
8343
|
+
}>;
|
|
8344
|
+
"heterogeneity": {
|
|
8345
|
+
"dominantCell": string;
|
|
8346
|
+
"contributionPct": number;
|
|
8347
|
+
} | null;
|
|
8348
|
+
};
|
|
8349
|
+
"matrixShape": {
|
|
8350
|
+
"cells": number;
|
|
8351
|
+
"harnesses": number;
|
|
8352
|
+
"models": number;
|
|
8353
|
+
"crossed": boolean;
|
|
8354
|
+
};
|
|
8355
|
+
"flakiness"?: {
|
|
8356
|
+
"measured": boolean;
|
|
8357
|
+
"conditions": number;
|
|
8358
|
+
"unstable": number;
|
|
8359
|
+
"worst": {
|
|
8360
|
+
"cell": string;
|
|
8361
|
+
"taskId": string;
|
|
8362
|
+
"arm": "baseline" | "skill";
|
|
8363
|
+
"variant"?: number;
|
|
8364
|
+
"passRate": number;
|
|
8365
|
+
} | null;
|
|
8366
|
+
};
|
|
8367
|
+
"arms"?: {
|
|
8368
|
+
"baseline": {
|
|
8369
|
+
"runs": number;
|
|
8370
|
+
"completed": number;
|
|
8371
|
+
"infraFailures": number;
|
|
8372
|
+
"completionRate": number | null;
|
|
8373
|
+
"goalRate": number | null;
|
|
8374
|
+
"goalTally": {
|
|
8375
|
+
"achieved": number;
|
|
8376
|
+
"partial": number;
|
|
8377
|
+
"missed": number;
|
|
8378
|
+
};
|
|
8379
|
+
"avgTotalTokens": number | null;
|
|
8380
|
+
"avgOutputTokens": number | null;
|
|
8381
|
+
"avgCostUsd": number | null;
|
|
8382
|
+
"avgDurationMs": number | null;
|
|
8383
|
+
"avgTurns": number | null;
|
|
8384
|
+
"costCoverage": number | null;
|
|
8385
|
+
"avgQualityScore": number | null;
|
|
8386
|
+
"gradedCoverage": number | null;
|
|
8387
|
+
"medianCostUsd": number | null;
|
|
8388
|
+
"medianTotalTokens": number | null;
|
|
8389
|
+
"medianDurationMs": number | null;
|
|
8390
|
+
"p95CostUsd": number | null;
|
|
8391
|
+
"p95TotalTokens": number | null;
|
|
8392
|
+
"p95DurationMs": number | null;
|
|
8393
|
+
"maxCostUsd": number | null;
|
|
8394
|
+
"maxTotalTokens": number | null;
|
|
8395
|
+
"maxDurationMs": number | null;
|
|
8396
|
+
"evalPassRate"?: number | null;
|
|
8397
|
+
"benchmarkPassRate"?: number | null;
|
|
8398
|
+
"adoptionRate"?: number | null;
|
|
8399
|
+
};
|
|
8400
|
+
"skill": {
|
|
8401
|
+
"runs": number;
|
|
8402
|
+
"completed": number;
|
|
8403
|
+
"infraFailures": number;
|
|
8404
|
+
"completionRate": number | null;
|
|
8405
|
+
"goalRate": number | null;
|
|
8406
|
+
"goalTally": {
|
|
8407
|
+
"achieved": number;
|
|
8408
|
+
"partial": number;
|
|
8409
|
+
"missed": number;
|
|
8410
|
+
};
|
|
8411
|
+
"avgTotalTokens": number | null;
|
|
8412
|
+
"avgOutputTokens": number | null;
|
|
8413
|
+
"avgCostUsd": number | null;
|
|
8414
|
+
"avgDurationMs": number | null;
|
|
8415
|
+
"avgTurns": number | null;
|
|
8416
|
+
"costCoverage": number | null;
|
|
8417
|
+
"avgQualityScore": number | null;
|
|
8418
|
+
"gradedCoverage": number | null;
|
|
8419
|
+
"medianCostUsd": number | null;
|
|
8420
|
+
"medianTotalTokens": number | null;
|
|
8421
|
+
"medianDurationMs": number | null;
|
|
8422
|
+
"p95CostUsd": number | null;
|
|
8423
|
+
"p95TotalTokens": number | null;
|
|
8424
|
+
"p95DurationMs": number | null;
|
|
8425
|
+
"maxCostUsd": number | null;
|
|
8426
|
+
"maxTotalTokens": number | null;
|
|
8427
|
+
"maxDurationMs": number | null;
|
|
8428
|
+
"evalPassRate"?: number | null;
|
|
8429
|
+
"benchmarkPassRate"?: number | null;
|
|
8430
|
+
"adoptionRate"?: number | null;
|
|
8431
|
+
};
|
|
8432
|
+
};
|
|
8433
|
+
"totals": {
|
|
8434
|
+
"cells": number;
|
|
8435
|
+
"runs": number;
|
|
8436
|
+
"completed": number;
|
|
8437
|
+
"failed": number;
|
|
8438
|
+
"costUsd": number;
|
|
8439
|
+
"tokens": number;
|
|
8440
|
+
};
|
|
5577
8441
|
};
|
|
5578
|
-
"
|
|
8442
|
+
"perCell": Array<{
|
|
8443
|
+
"harness": string;
|
|
8444
|
+
"model": string;
|
|
8445
|
+
"provider": string | null;
|
|
8446
|
+
"paramsKey"?: string;
|
|
8447
|
+
"label"?: string;
|
|
5579
8448
|
"baseline": {
|
|
5580
8449
|
"runs": number;
|
|
5581
8450
|
"completed": number;
|
|
@@ -5640,44 +8509,73 @@ export type StudyPreviewResponse = {
|
|
|
5640
8509
|
"benchmarkPassRate"?: number | null;
|
|
5641
8510
|
"adoptionRate"?: number | null;
|
|
5642
8511
|
};
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
"costUsd": number;
|
|
5650
|
-
"tokens": number;
|
|
5651
|
-
};
|
|
5652
|
-
};
|
|
5653
|
-
"perCell": Array<unknown>;
|
|
5654
|
-
"perHarness": Array<unknown>;
|
|
5655
|
-
"perModel": Array<unknown>;
|
|
5656
|
-
"perTask": Array<unknown>;
|
|
5657
|
-
"webSplit": unknown;
|
|
5658
|
-
"inference": {
|
|
5659
|
-
"columns": Array<{
|
|
5660
|
-
"key": string;
|
|
5661
|
-
"label": string;
|
|
5662
|
-
"family": "rate" | "score" | "magnitude";
|
|
5663
|
-
"betterWhen": "higher" | "lower";
|
|
5664
|
-
}>;
|
|
5665
|
-
"groups": Array<{
|
|
5666
|
-
"kind": "cell" | "harness" | "model" | "field";
|
|
5667
|
-
"title": string;
|
|
5668
|
-
"rows": Array<{
|
|
5669
|
-
"key": string;
|
|
5670
|
-
"kind": "cell" | "harness" | "model" | "field";
|
|
5671
|
-
"harness": string | null;
|
|
5672
|
-
"model": string | null;
|
|
5673
|
-
"paramsKey"?: string;
|
|
5674
|
-
"runs": number;
|
|
5675
|
-
"cells": Record<string, unknown>;
|
|
5676
|
-
}>;
|
|
8512
|
+
"deltas": {
|
|
8513
|
+
"completionRatePp": number | null;
|
|
8514
|
+
"goalRatePp": number | null;
|
|
8515
|
+
"qualityScorePts": number | null;
|
|
8516
|
+
"evalPassRatePp"?: number | null;
|
|
8517
|
+
};
|
|
5677
8518
|
}>;
|
|
5678
|
-
|
|
5679
|
-
|
|
5680
|
-
|
|
8519
|
+
}>;
|
|
8520
|
+
"board": Array<{
|
|
8521
|
+
"harness": string;
|
|
8522
|
+
"model": string;
|
|
8523
|
+
"provider": string | null;
|
|
8524
|
+
"paramsKey"?: string;
|
|
8525
|
+
"label"?: string;
|
|
8526
|
+
"arm": string;
|
|
8527
|
+
"runs": number;
|
|
8528
|
+
"completed": number;
|
|
8529
|
+
"completionRate": number | null;
|
|
8530
|
+
"goalRate": number | null;
|
|
8531
|
+
"avgQualityScore": number | null;
|
|
8532
|
+
"evalPassRate": number | null;
|
|
8533
|
+
"benchmarkPassRate"?: number | null;
|
|
8534
|
+
"compositeScore": number | null;
|
|
8535
|
+
"avgTotalTokens": number | null;
|
|
8536
|
+
"avgOutputTokens": number | null;
|
|
8537
|
+
"avgCostUsd": number | null;
|
|
8538
|
+
"avgDurationMs": number | null;
|
|
8539
|
+
"medianCostUsd": number | null;
|
|
8540
|
+
"medianTotalTokens": number | null;
|
|
8541
|
+
"medianDurationMs": number | null;
|
|
8542
|
+
"rank": number;
|
|
8543
|
+
}>;
|
|
8544
|
+
"setupsBoard"?: Array<{
|
|
8545
|
+
"harness": string;
|
|
8546
|
+
"model": string;
|
|
8547
|
+
"provider": string | null;
|
|
8548
|
+
"paramsKey"?: string;
|
|
8549
|
+
"label"?: string;
|
|
8550
|
+
"arm": string;
|
|
8551
|
+
"runs": number;
|
|
8552
|
+
"completed": number;
|
|
8553
|
+
"completionRate": number | null;
|
|
8554
|
+
"goalRate": number | null;
|
|
8555
|
+
"avgQualityScore": number | null;
|
|
8556
|
+
"evalPassRate": number | null;
|
|
8557
|
+
"benchmarkPassRate"?: number | null;
|
|
8558
|
+
"compositeScore": number | null;
|
|
8559
|
+
"avgTotalTokens": number | null;
|
|
8560
|
+
"avgOutputTokens": number | null;
|
|
8561
|
+
"avgCostUsd": number | null;
|
|
8562
|
+
"avgDurationMs": number | null;
|
|
8563
|
+
"medianCostUsd": number | null;
|
|
8564
|
+
"medianTotalTokens": number | null;
|
|
8565
|
+
"medianDurationMs": number | null;
|
|
8566
|
+
"rank": number;
|
|
8567
|
+
"group": string;
|
|
8568
|
+
"agent": string;
|
|
8569
|
+
"setup": string;
|
|
8570
|
+
}>;
|
|
8571
|
+
"perTask": Array<{
|
|
8572
|
+
"taskId": string;
|
|
8573
|
+
"position": number;
|
|
8574
|
+
"promptPreview": string;
|
|
8575
|
+
"requiresWeb": boolean;
|
|
8576
|
+
"category": string | null;
|
|
8577
|
+
"byArm": Record<string, unknown>;
|
|
8578
|
+
}>;
|
|
5681
8579
|
"behavior"?: Array<{
|
|
5682
8580
|
"harness": string;
|
|
5683
8581
|
"model": string;
|
|
@@ -5692,48 +8590,55 @@ export type StudyPreviewResponse = {
|
|
|
5692
8590
|
"succeededCount": number;
|
|
5693
8591
|
}>;
|
|
5694
8592
|
}>;
|
|
5695
|
-
"
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
"
|
|
5699
|
-
"
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5703
|
-
|
|
5704
|
-
|
|
5705
|
-
|
|
5706
|
-
"
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
8593
|
+
"execution"?: Record<string, unknown>;
|
|
8594
|
+
"inference"?: {
|
|
8595
|
+
"contrasts": Array<{
|
|
8596
|
+
"contrastId": string;
|
|
8597
|
+
"table": {
|
|
8598
|
+
"columns": Array<{
|
|
8599
|
+
"key": string;
|
|
8600
|
+
"label": string;
|
|
8601
|
+
"family": "rate" | "score" | "magnitude";
|
|
8602
|
+
"betterWhen": "higher" | "lower";
|
|
8603
|
+
}>;
|
|
8604
|
+
"groups": Array<{
|
|
8605
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
8606
|
+
"title": string;
|
|
8607
|
+
"rows": Array<{
|
|
8608
|
+
"key": string;
|
|
8609
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
8610
|
+
"harness": string | null;
|
|
8611
|
+
"model": string | null;
|
|
8612
|
+
"paramsKey"?: string;
|
|
8613
|
+
"runs": number;
|
|
8614
|
+
"cells": Record<string, unknown>;
|
|
8615
|
+
}>;
|
|
8616
|
+
}>;
|
|
8617
|
+
"reference": "baseline" | "field";
|
|
8618
|
+
"draws": number;
|
|
8619
|
+
"adjustment": {
|
|
8620
|
+
"scope": "planned-comparisons";
|
|
8621
|
+
"comparisons": number;
|
|
8622
|
+
};
|
|
8623
|
+
} | null;
|
|
5713
8624
|
}>;
|
|
5714
|
-
"
|
|
5715
|
-
"
|
|
5716
|
-
"
|
|
5717
|
-
"
|
|
8625
|
+
"holm": {
|
|
8626
|
+
"status": "complete";
|
|
8627
|
+
"ready": number;
|
|
8628
|
+
"planned": number;
|
|
8629
|
+
"adjustment": {
|
|
8630
|
+
"scope": "planned-comparisons";
|
|
8631
|
+
"comparisons": number;
|
|
8632
|
+
};
|
|
8633
|
+
} | {
|
|
8634
|
+
"status": "pending";
|
|
8635
|
+
"ready": number;
|
|
8636
|
+
"planned": number;
|
|
5718
8637
|
};
|
|
5719
|
-
"
|
|
5720
|
-
|
|
5721
|
-
|
|
5722
|
-
};
|
|
5723
|
-
"execution"?: {
|
|
5724
|
-
"graded": number;
|
|
5725
|
-
"measured": number;
|
|
5726
|
-
"passed": number;
|
|
5727
|
-
"passRate": number | null;
|
|
5728
|
-
"byStatus": {
|
|
5729
|
-
"passed": number;
|
|
5730
|
-
"failed": number;
|
|
5731
|
-
"timeout": number;
|
|
5732
|
-
"error": number;
|
|
5733
|
-
"unavailable": number;
|
|
8638
|
+
"claims": {
|
|
8639
|
+
"perContrast": Record<string, unknown>;
|
|
8640
|
+
"ranking": "beat-all" | "won-all-on-ladder" | "ranks-first" | "composite-with-loss" | "pending" | "none";
|
|
5734
8641
|
};
|
|
5735
|
-
"languages": Array<string>;
|
|
5736
|
-
"unavailableReasons": Array<string>;
|
|
5737
8642
|
};
|
|
5738
8643
|
};
|
|
5739
8644
|
"textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
|
|
@@ -5741,6 +8646,7 @@ export type StudyPreviewResponse = {
|
|
|
5741
8646
|
export type StudyPublishRequest = {
|
|
5742
8647
|
"slug"?: string;
|
|
5743
8648
|
"visibility"?: "private" | "org" | "link" | "public";
|
|
8649
|
+
"skillNames"?: "withheld" | "named";
|
|
5744
8650
|
};
|
|
5745
8651
|
export type StudyPublishResponse = {
|
|
5746
8652
|
"study": {
|
|
@@ -5773,7 +8679,7 @@ export type StudyPublishResponse = {
|
|
|
5773
8679
|
"addons"?: Record<string, unknown>;
|
|
5774
8680
|
};
|
|
5775
8681
|
};
|
|
5776
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
8682
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5777
8683
|
"subjectBackend": "model" | "harness" | null;
|
|
5778
8684
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5779
8685
|
"importedFrom"?: string;
|
|
@@ -5809,6 +8715,7 @@ export type StudyPublishResponse = {
|
|
|
5809
8715
|
"error": string | null;
|
|
5810
8716
|
"preparation"?: {
|
|
5811
8717
|
"since": string;
|
|
8718
|
+
"progressAt"?: string;
|
|
5812
8719
|
"launchedBy": {
|
|
5813
8720
|
"userId"?: string;
|
|
5814
8721
|
"role"?: string;
|
|
@@ -5817,6 +8724,12 @@ export type StudyPublishResponse = {
|
|
|
5817
8724
|
"imageKey": string;
|
|
5818
8725
|
"buildId": string;
|
|
5819
8726
|
"stage"?: "image" | "snapshot";
|
|
8727
|
+
"retry"?: {
|
|
8728
|
+
"failedAttempts": number;
|
|
8729
|
+
"maxAttempts": number;
|
|
8730
|
+
"notBefore": string;
|
|
8731
|
+
"lastFailure": string;
|
|
8732
|
+
};
|
|
5820
8733
|
}>;
|
|
5821
8734
|
} | null;
|
|
5822
8735
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5831,6 +8744,38 @@ export type StudyPublishResponse = {
|
|
|
5831
8744
|
"createdAt": string;
|
|
5832
8745
|
"updatedAt": string;
|
|
5833
8746
|
"liveUntil": string | null;
|
|
8747
|
+
"arms"?: Array<{
|
|
8748
|
+
"key": string;
|
|
8749
|
+
"position": number;
|
|
8750
|
+
"label": string;
|
|
8751
|
+
"role": "reference" | "peer";
|
|
8752
|
+
"payload": {
|
|
8753
|
+
"tools"?: "default" | "none" | {
|
|
8754
|
+
"tools": Array<string>;
|
|
8755
|
+
};
|
|
8756
|
+
"addons"?: Record<string, unknown>;
|
|
8757
|
+
};
|
|
8758
|
+
"cells"?: Array<{
|
|
8759
|
+
"harness": string;
|
|
8760
|
+
"model": string;
|
|
8761
|
+
"provider"?: string;
|
|
8762
|
+
"params"?: {
|
|
8763
|
+
"temperature"?: number;
|
|
8764
|
+
"maxOutputTokens"?: number;
|
|
8765
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
8766
|
+
"thinkingBudgetTokens"?: number;
|
|
8767
|
+
};
|
|
8768
|
+
}>;
|
|
8769
|
+
"skills"?: Array<{
|
|
8770
|
+
"ref": string;
|
|
8771
|
+
"name": string;
|
|
8772
|
+
"changedSinceCreate"?: boolean;
|
|
8773
|
+
}>;
|
|
8774
|
+
}>;
|
|
8775
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
8776
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
8777
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
8778
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5834
8779
|
};
|
|
5835
8780
|
};
|
|
5836
8781
|
export type StudyReconcileRunsResponse = {
|
|
@@ -5873,7 +8818,7 @@ export type StudyRegradeResponse = {
|
|
|
5873
8818
|
"addons"?: Record<string, unknown>;
|
|
5874
8819
|
};
|
|
5875
8820
|
};
|
|
5876
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
8821
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5877
8822
|
"subjectBackend": "model" | "harness" | null;
|
|
5878
8823
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5879
8824
|
"importedFrom"?: string;
|
|
@@ -5909,6 +8854,7 @@ export type StudyRegradeResponse = {
|
|
|
5909
8854
|
"error": string | null;
|
|
5910
8855
|
"preparation"?: {
|
|
5911
8856
|
"since": string;
|
|
8857
|
+
"progressAt"?: string;
|
|
5912
8858
|
"launchedBy": {
|
|
5913
8859
|
"userId"?: string;
|
|
5914
8860
|
"role"?: string;
|
|
@@ -5917,6 +8863,12 @@ export type StudyRegradeResponse = {
|
|
|
5917
8863
|
"imageKey": string;
|
|
5918
8864
|
"buildId": string;
|
|
5919
8865
|
"stage"?: "image" | "snapshot";
|
|
8866
|
+
"retry"?: {
|
|
8867
|
+
"failedAttempts": number;
|
|
8868
|
+
"maxAttempts": number;
|
|
8869
|
+
"notBefore": string;
|
|
8870
|
+
"lastFailure": string;
|
|
8871
|
+
};
|
|
5920
8872
|
}>;
|
|
5921
8873
|
} | null;
|
|
5922
8874
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -5931,11 +8883,46 @@ export type StudyRegradeResponse = {
|
|
|
5931
8883
|
"createdAt": string;
|
|
5932
8884
|
"updatedAt": string;
|
|
5933
8885
|
"liveUntil": string | null;
|
|
8886
|
+
"arms"?: Array<{
|
|
8887
|
+
"key": string;
|
|
8888
|
+
"position": number;
|
|
8889
|
+
"label": string;
|
|
8890
|
+
"role": "reference" | "peer";
|
|
8891
|
+
"payload": {
|
|
8892
|
+
"tools"?: "default" | "none" | {
|
|
8893
|
+
"tools": Array<string>;
|
|
8894
|
+
};
|
|
8895
|
+
"addons"?: Record<string, unknown>;
|
|
8896
|
+
};
|
|
8897
|
+
"cells"?: Array<{
|
|
8898
|
+
"harness": string;
|
|
8899
|
+
"model": string;
|
|
8900
|
+
"provider"?: string;
|
|
8901
|
+
"params"?: {
|
|
8902
|
+
"temperature"?: number;
|
|
8903
|
+
"maxOutputTokens"?: number;
|
|
8904
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
8905
|
+
"thinkingBudgetTokens"?: number;
|
|
8906
|
+
};
|
|
8907
|
+
}>;
|
|
8908
|
+
"skills"?: Array<{
|
|
8909
|
+
"ref": string;
|
|
8910
|
+
"name": string;
|
|
8911
|
+
"changedSinceCreate"?: boolean;
|
|
8912
|
+
}>;
|
|
8913
|
+
}>;
|
|
8914
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
8915
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
8916
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
8917
|
+
"modeOptions"?: Record<string, unknown>;
|
|
5934
8918
|
};
|
|
5935
8919
|
"streamUrl": string;
|
|
5936
8920
|
"warnings"?: Array<{
|
|
5937
8921
|
"harness": string;
|
|
5938
8922
|
"kind": string;
|
|
8923
|
+
} | {
|
|
8924
|
+
"type": "design_aggregate_budget";
|
|
8925
|
+
"detail": string;
|
|
5939
8926
|
}>;
|
|
5940
8927
|
};
|
|
5941
8928
|
export type StudyListRunAnswersResponse = {
|
|
@@ -5984,7 +8971,7 @@ export type StudyRotateShareTokenResponse = {
|
|
|
5984
8971
|
"addons"?: Record<string, unknown>;
|
|
5985
8972
|
};
|
|
5986
8973
|
};
|
|
5987
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
8974
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
5988
8975
|
"subjectBackend": "model" | "harness" | null;
|
|
5989
8976
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5990
8977
|
"importedFrom"?: string;
|
|
@@ -6020,6 +9007,7 @@ export type StudyRotateShareTokenResponse = {
|
|
|
6020
9007
|
"error": string | null;
|
|
6021
9008
|
"preparation"?: {
|
|
6022
9009
|
"since": string;
|
|
9010
|
+
"progressAt"?: string;
|
|
6023
9011
|
"launchedBy": {
|
|
6024
9012
|
"userId"?: string;
|
|
6025
9013
|
"role"?: string;
|
|
@@ -6028,6 +9016,12 @@ export type StudyRotateShareTokenResponse = {
|
|
|
6028
9016
|
"imageKey": string;
|
|
6029
9017
|
"buildId": string;
|
|
6030
9018
|
"stage"?: "image" | "snapshot";
|
|
9019
|
+
"retry"?: {
|
|
9020
|
+
"failedAttempts": number;
|
|
9021
|
+
"maxAttempts": number;
|
|
9022
|
+
"notBefore": string;
|
|
9023
|
+
"lastFailure": string;
|
|
9024
|
+
};
|
|
6031
9025
|
}>;
|
|
6032
9026
|
} | null;
|
|
6033
9027
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -6042,13 +9036,113 @@ export type StudyRotateShareTokenResponse = {
|
|
|
6042
9036
|
"createdAt": string;
|
|
6043
9037
|
"updatedAt": string;
|
|
6044
9038
|
"liveUntil": string | null;
|
|
9039
|
+
"arms"?: Array<{
|
|
9040
|
+
"key": string;
|
|
9041
|
+
"position": number;
|
|
9042
|
+
"label": string;
|
|
9043
|
+
"role": "reference" | "peer";
|
|
9044
|
+
"payload": {
|
|
9045
|
+
"tools"?: "default" | "none" | {
|
|
9046
|
+
"tools": Array<string>;
|
|
9047
|
+
};
|
|
9048
|
+
"addons"?: Record<string, unknown>;
|
|
9049
|
+
};
|
|
9050
|
+
"cells"?: Array<{
|
|
9051
|
+
"harness": string;
|
|
9052
|
+
"model": string;
|
|
9053
|
+
"provider"?: string;
|
|
9054
|
+
"params"?: {
|
|
9055
|
+
"temperature"?: number;
|
|
9056
|
+
"maxOutputTokens"?: number;
|
|
9057
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9058
|
+
"thinkingBudgetTokens"?: number;
|
|
9059
|
+
};
|
|
9060
|
+
}>;
|
|
9061
|
+
"skills"?: Array<{
|
|
9062
|
+
"ref": string;
|
|
9063
|
+
"name": string;
|
|
9064
|
+
"changedSinceCreate"?: boolean;
|
|
9065
|
+
}>;
|
|
9066
|
+
}>;
|
|
9067
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9068
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9069
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9070
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6045
9071
|
};
|
|
6046
9072
|
"shareUrl": string | null;
|
|
6047
9073
|
};
|
|
6048
9074
|
export type StudyStreamResponse = unknown;
|
|
6049
9075
|
export type StudyUpdateTaskRequest = {
|
|
6050
9076
|
"prompt": string;
|
|
6051
|
-
"expectations"?: Array<
|
|
9077
|
+
"expectations"?: Array<{
|
|
9078
|
+
"label"?: string;
|
|
9079
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9080
|
+
"kind": "mcp_tool_called";
|
|
9081
|
+
"server"?: string;
|
|
9082
|
+
"tool": string;
|
|
9083
|
+
"arguments"?: Record<string, unknown>;
|
|
9084
|
+
"minCalls"?: number;
|
|
9085
|
+
} | {
|
|
9086
|
+
"label"?: string;
|
|
9087
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9088
|
+
"kind": "tool_not_called";
|
|
9089
|
+
"tool"?: string;
|
|
9090
|
+
"mcp"?: {
|
|
9091
|
+
"server"?: string;
|
|
9092
|
+
"tool": string;
|
|
9093
|
+
};
|
|
9094
|
+
"pattern"?: string;
|
|
9095
|
+
} | {
|
|
9096
|
+
"label"?: string;
|
|
9097
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9098
|
+
"kind": "command_ran";
|
|
9099
|
+
"pattern": string;
|
|
9100
|
+
} | {
|
|
9101
|
+
"label"?: string;
|
|
9102
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9103
|
+
"kind": "skill_loaded";
|
|
9104
|
+
"slug": string;
|
|
9105
|
+
} | {
|
|
9106
|
+
"label"?: string;
|
|
9107
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9108
|
+
"kind": "file_operation";
|
|
9109
|
+
"op"?: "read" | "write" | "edit";
|
|
9110
|
+
"path": unknown;
|
|
9111
|
+
} | {
|
|
9112
|
+
"label"?: string;
|
|
9113
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9114
|
+
"kind": "web_fetched";
|
|
9115
|
+
"urlHost"?: string;
|
|
9116
|
+
"urlPattern"?: string;
|
|
9117
|
+
} | {
|
|
9118
|
+
"label"?: string;
|
|
9119
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9120
|
+
"kind": "response_contains";
|
|
9121
|
+
"text": string;
|
|
9122
|
+
"normalize"?: "prose";
|
|
9123
|
+
} | {
|
|
9124
|
+
"label"?: string;
|
|
9125
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9126
|
+
"kind": "response_matches";
|
|
9127
|
+
"pattern": string;
|
|
9128
|
+
"ignoreCase"?: boolean;
|
|
9129
|
+
} | {
|
|
9130
|
+
"label"?: string;
|
|
9131
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9132
|
+
"kind": "token_budget";
|
|
9133
|
+
"maxOutputTokens"?: number;
|
|
9134
|
+
"maxTotalTokens"?: number;
|
|
9135
|
+
} | {
|
|
9136
|
+
"label"?: string;
|
|
9137
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9138
|
+
"kind": "sequence";
|
|
9139
|
+
"of": Array<unknown>;
|
|
9140
|
+
} | {
|
|
9141
|
+
"label"?: string;
|
|
9142
|
+
"arms"?: Array<"baseline" | "skill" | string>;
|
|
9143
|
+
"kind": "any_of";
|
|
9144
|
+
"of": Array<unknown>;
|
|
9145
|
+
}>;
|
|
6052
9146
|
};
|
|
6053
9147
|
export type StudyUpdateTaskResponse = {
|
|
6054
9148
|
"task": {
|
|
@@ -6095,7 +9189,7 @@ export type StudyUnpublishResponse = {
|
|
|
6095
9189
|
"addons"?: Record<string, unknown>;
|
|
6096
9190
|
};
|
|
6097
9191
|
};
|
|
6098
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
9192
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6099
9193
|
"subjectBackend": "model" | "harness" | null;
|
|
6100
9194
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6101
9195
|
"importedFrom"?: string;
|
|
@@ -6131,6 +9225,7 @@ export type StudyUnpublishResponse = {
|
|
|
6131
9225
|
"error": string | null;
|
|
6132
9226
|
"preparation"?: {
|
|
6133
9227
|
"since": string;
|
|
9228
|
+
"progressAt"?: string;
|
|
6134
9229
|
"launchedBy": {
|
|
6135
9230
|
"userId"?: string;
|
|
6136
9231
|
"role"?: string;
|
|
@@ -6139,6 +9234,12 @@ export type StudyUnpublishResponse = {
|
|
|
6139
9234
|
"imageKey": string;
|
|
6140
9235
|
"buildId": string;
|
|
6141
9236
|
"stage"?: "image" | "snapshot";
|
|
9237
|
+
"retry"?: {
|
|
9238
|
+
"failedAttempts": number;
|
|
9239
|
+
"maxAttempts": number;
|
|
9240
|
+
"notBefore": string;
|
|
9241
|
+
"lastFailure": string;
|
|
9242
|
+
};
|
|
6142
9243
|
}>;
|
|
6143
9244
|
} | null;
|
|
6144
9245
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -6153,6 +9254,38 @@ export type StudyUnpublishResponse = {
|
|
|
6153
9254
|
"createdAt": string;
|
|
6154
9255
|
"updatedAt": string;
|
|
6155
9256
|
"liveUntil": string | null;
|
|
9257
|
+
"arms"?: Array<{
|
|
9258
|
+
"key": string;
|
|
9259
|
+
"position": number;
|
|
9260
|
+
"label": string;
|
|
9261
|
+
"role": "reference" | "peer";
|
|
9262
|
+
"payload": {
|
|
9263
|
+
"tools"?: "default" | "none" | {
|
|
9264
|
+
"tools": Array<string>;
|
|
9265
|
+
};
|
|
9266
|
+
"addons"?: Record<string, unknown>;
|
|
9267
|
+
};
|
|
9268
|
+
"cells"?: Array<{
|
|
9269
|
+
"harness": string;
|
|
9270
|
+
"model": string;
|
|
9271
|
+
"provider"?: string;
|
|
9272
|
+
"params"?: {
|
|
9273
|
+
"temperature"?: number;
|
|
9274
|
+
"maxOutputTokens"?: number;
|
|
9275
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9276
|
+
"thinkingBudgetTokens"?: number;
|
|
9277
|
+
};
|
|
9278
|
+
}>;
|
|
9279
|
+
"skills"?: Array<{
|
|
9280
|
+
"ref": string;
|
|
9281
|
+
"name": string;
|
|
9282
|
+
"changedSinceCreate"?: boolean;
|
|
9283
|
+
}>;
|
|
9284
|
+
}>;
|
|
9285
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9286
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9287
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9288
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6156
9289
|
};
|
|
6157
9290
|
};
|
|
6158
9291
|
export type StudySetVisibilityRequest = {
|
|
@@ -6189,7 +9322,7 @@ export type StudySetVisibilityResponse = {
|
|
|
6189
9322
|
"addons"?: Record<string, unknown>;
|
|
6190
9323
|
};
|
|
6191
9324
|
};
|
|
6192
|
-
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
9325
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
|
|
6193
9326
|
"subjectBackend": "model" | "harness" | null;
|
|
6194
9327
|
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6195
9328
|
"importedFrom"?: string;
|
|
@@ -6225,6 +9358,7 @@ export type StudySetVisibilityResponse = {
|
|
|
6225
9358
|
"error": string | null;
|
|
6226
9359
|
"preparation"?: {
|
|
6227
9360
|
"since": string;
|
|
9361
|
+
"progressAt"?: string;
|
|
6228
9362
|
"launchedBy": {
|
|
6229
9363
|
"userId"?: string;
|
|
6230
9364
|
"role"?: string;
|
|
@@ -6233,6 +9367,12 @@ export type StudySetVisibilityResponse = {
|
|
|
6233
9367
|
"imageKey": string;
|
|
6234
9368
|
"buildId": string;
|
|
6235
9369
|
"stage"?: "image" | "snapshot";
|
|
9370
|
+
"retry"?: {
|
|
9371
|
+
"failedAttempts": number;
|
|
9372
|
+
"maxAttempts": number;
|
|
9373
|
+
"notBefore": string;
|
|
9374
|
+
"lastFailure": string;
|
|
9375
|
+
};
|
|
6236
9376
|
}>;
|
|
6237
9377
|
} | null;
|
|
6238
9378
|
"visibility": "private" | "org" | "link" | "public";
|
|
@@ -6247,6 +9387,38 @@ export type StudySetVisibilityResponse = {
|
|
|
6247
9387
|
"createdAt": string;
|
|
6248
9388
|
"updatedAt": string;
|
|
6249
9389
|
"liveUntil": string | null;
|
|
9390
|
+
"arms"?: Array<{
|
|
9391
|
+
"key": string;
|
|
9392
|
+
"position": number;
|
|
9393
|
+
"label": string;
|
|
9394
|
+
"role": "reference" | "peer";
|
|
9395
|
+
"payload": {
|
|
9396
|
+
"tools"?: "default" | "none" | {
|
|
9397
|
+
"tools": Array<string>;
|
|
9398
|
+
};
|
|
9399
|
+
"addons"?: Record<string, unknown>;
|
|
9400
|
+
};
|
|
9401
|
+
"cells"?: Array<{
|
|
9402
|
+
"harness": string;
|
|
9403
|
+
"model": string;
|
|
9404
|
+
"provider"?: string;
|
|
9405
|
+
"params"?: {
|
|
9406
|
+
"temperature"?: number;
|
|
9407
|
+
"maxOutputTokens"?: number;
|
|
9408
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
9409
|
+
"thinkingBudgetTokens"?: number;
|
|
9410
|
+
};
|
|
9411
|
+
}>;
|
|
9412
|
+
"skills"?: Array<{
|
|
9413
|
+
"ref": string;
|
|
9414
|
+
"name": string;
|
|
9415
|
+
"changedSinceCreate"?: boolean;
|
|
9416
|
+
}>;
|
|
9417
|
+
}>;
|
|
9418
|
+
"contrastPlan"?: "vs_reference" | "all_pairs";
|
|
9419
|
+
"cellPlan"?: "crossed" | "per_setup";
|
|
9420
|
+
"skillDisclosure"?: "withheld" | "named";
|
|
9421
|
+
"modeOptions"?: Record<string, unknown>;
|
|
6250
9422
|
};
|
|
6251
9423
|
"shareUrl": string | null;
|
|
6252
9424
|
};
|