@ora-ai/cli 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +16 -2
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +37 -4
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/package.json +1 -1
|
@@ -71,6 +71,7 @@ export type BenchmarksListResponse = {
|
|
|
71
71
|
};
|
|
72
72
|
}>;
|
|
73
73
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
74
|
+
"judgeSupplementary"?: boolean;
|
|
74
75
|
"taskCount": number;
|
|
75
76
|
"variantsPerTask": number;
|
|
76
77
|
"repeats": number;
|
|
@@ -129,6 +130,7 @@ export type BenchmarksCreateRequest = {
|
|
|
129
130
|
};
|
|
130
131
|
};
|
|
131
132
|
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
133
|
+
"judgeSupplementary"?: boolean;
|
|
132
134
|
"proposeExpectations"?: boolean;
|
|
133
135
|
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
134
136
|
"subjectBackend"?: "model" | "harness";
|
|
@@ -217,6 +219,7 @@ export type BenchmarksCreateResponse = {
|
|
|
217
219
|
};
|
|
218
220
|
}>;
|
|
219
221
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
222
|
+
"judgeSupplementary"?: boolean;
|
|
220
223
|
"taskCount": number;
|
|
221
224
|
"variantsPerTask": number;
|
|
222
225
|
"repeats": number;
|
|
@@ -324,6 +327,7 @@ export type BenchmarksGetResponse = {
|
|
|
324
327
|
};
|
|
325
328
|
}>;
|
|
326
329
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
330
|
+
"judgeSupplementary"?: boolean;
|
|
327
331
|
"taskCount": number;
|
|
328
332
|
"variantsPerTask": number;
|
|
329
333
|
"repeats": number;
|
|
@@ -515,7 +519,7 @@ export type BenchmarksGetResponse = {
|
|
|
515
519
|
};
|
|
516
520
|
"decision"?: {
|
|
517
521
|
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
518
|
-
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
522
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
519
523
|
"reason": string;
|
|
520
524
|
"efficiencyDiscounted": boolean;
|
|
521
525
|
};
|
|
@@ -579,6 +583,7 @@ export type BenchmarksGetResponse = {
|
|
|
579
583
|
"maxTotalTokens": number | null;
|
|
580
584
|
"maxDurationMs": number | null;
|
|
581
585
|
"evalPassRate"?: number | null;
|
|
586
|
+
"benchmarkPassRate"?: number | null;
|
|
582
587
|
"adoptionRate"?: number | null;
|
|
583
588
|
};
|
|
584
589
|
"skill": {
|
|
@@ -610,6 +615,7 @@ export type BenchmarksGetResponse = {
|
|
|
610
615
|
"maxTotalTokens": number | null;
|
|
611
616
|
"maxDurationMs": number | null;
|
|
612
617
|
"evalPassRate"?: number | null;
|
|
618
|
+
"benchmarkPassRate"?: number | null;
|
|
613
619
|
"adoptionRate"?: number | null;
|
|
614
620
|
};
|
|
615
621
|
};
|
|
@@ -781,6 +787,7 @@ export type BenchmarksGenerateResponse = {
|
|
|
781
787
|
};
|
|
782
788
|
}>;
|
|
783
789
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
790
|
+
"judgeSupplementary"?: boolean;
|
|
784
791
|
"taskCount": number;
|
|
785
792
|
"variantsPerTask": number;
|
|
786
793
|
"repeats": number;
|
|
@@ -876,6 +883,7 @@ export type BenchmarksLaunchResponse = {
|
|
|
876
883
|
};
|
|
877
884
|
}>;
|
|
878
885
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
886
|
+
"judgeSupplementary"?: boolean;
|
|
879
887
|
"taskCount": number;
|
|
880
888
|
"variantsPerTask": number;
|
|
881
889
|
"repeats": number;
|
|
@@ -974,6 +982,7 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
974
982
|
};
|
|
975
983
|
}>;
|
|
976
984
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
985
|
+
"judgeSupplementary"?: boolean;
|
|
977
986
|
"taskCount": number;
|
|
978
987
|
"variantsPerTask": number;
|
|
979
988
|
"repeats": number;
|
|
@@ -1047,6 +1056,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
1047
1056
|
"baselineLabel": string | null;
|
|
1048
1057
|
"treatmentLabel": string | null;
|
|
1049
1058
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1059
|
+
"judgeSupplementary"?: true;
|
|
1050
1060
|
"promiseMetrics": Array<{
|
|
1051
1061
|
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
1052
1062
|
"direction": "down" | "up";
|
|
@@ -1185,7 +1195,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
1185
1195
|
};
|
|
1186
1196
|
"decision"?: {
|
|
1187
1197
|
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
1188
|
-
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
1198
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
1189
1199
|
"reason": string;
|
|
1190
1200
|
"efficiencyDiscounted": boolean;
|
|
1191
1201
|
};
|
|
@@ -1249,6 +1259,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
1249
1259
|
"maxTotalTokens": number | null;
|
|
1250
1260
|
"maxDurationMs": number | null;
|
|
1251
1261
|
"evalPassRate"?: number | null;
|
|
1262
|
+
"benchmarkPassRate"?: number | null;
|
|
1252
1263
|
"adoptionRate"?: number | null;
|
|
1253
1264
|
};
|
|
1254
1265
|
"skill": {
|
|
@@ -1280,6 +1291,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
1280
1291
|
"maxTotalTokens": number | null;
|
|
1281
1292
|
"maxDurationMs": number | null;
|
|
1282
1293
|
"evalPassRate"?: number | null;
|
|
1294
|
+
"benchmarkPassRate"?: number | null;
|
|
1283
1295
|
"adoptionRate"?: number | null;
|
|
1284
1296
|
};
|
|
1285
1297
|
};
|
|
@@ -1438,6 +1450,7 @@ export type BenchmarksPublishResponse = {
|
|
|
1438
1450
|
};
|
|
1439
1451
|
}>;
|
|
1440
1452
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1453
|
+
"judgeSupplementary"?: boolean;
|
|
1441
1454
|
"taskCount": number;
|
|
1442
1455
|
"variantsPerTask": number;
|
|
1443
1456
|
"repeats": number;
|
|
@@ -1528,6 +1541,7 @@ export type BenchmarksRegradeResponse = {
|
|
|
1528
1541
|
};
|
|
1529
1542
|
}>;
|
|
1530
1543
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1544
|
+
"judgeSupplementary"?: boolean;
|
|
1531
1545
|
"taskCount": number;
|
|
1532
1546
|
"variantsPerTask": number;
|
|
1533
1547
|
"repeats": number;
|
|
@@ -1642,6 +1656,7 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1642
1656
|
};
|
|
1643
1657
|
}>;
|
|
1644
1658
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1659
|
+
"judgeSupplementary"?: boolean;
|
|
1645
1660
|
"taskCount": number;
|
|
1646
1661
|
"variantsPerTask": number;
|
|
1647
1662
|
"repeats": number;
|
|
@@ -2196,6 +2211,7 @@ export type WebmcpArenaCreateResponse = {
|
|
|
2196
2211
|
};
|
|
2197
2212
|
}>;
|
|
2198
2213
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
2214
|
+
"judgeSupplementary"?: boolean;
|
|
2199
2215
|
"taskCount": number;
|
|
2200
2216
|
"variantsPerTask": number;
|
|
2201
2217
|
"repeats": number;
|
|
@@ -4401,6 +4417,7 @@ export type StudyListResponse = {
|
|
|
4401
4417
|
};
|
|
4402
4418
|
}>;
|
|
4403
4419
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4420
|
+
"judgeSupplementary"?: boolean;
|
|
4404
4421
|
"taskCount": number;
|
|
4405
4422
|
"variantsPerTask": number;
|
|
4406
4423
|
"repeats": number;
|
|
@@ -4459,6 +4476,7 @@ export type StudyCreateRequest = {
|
|
|
4459
4476
|
};
|
|
4460
4477
|
};
|
|
4461
4478
|
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4479
|
+
"judgeSupplementary"?: boolean;
|
|
4462
4480
|
"proposeExpectations"?: boolean;
|
|
4463
4481
|
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
4464
4482
|
"subjectBackend"?: "model" | "harness";
|
|
@@ -4547,6 +4565,7 @@ export type StudyCreateResponse = {
|
|
|
4547
4565
|
};
|
|
4548
4566
|
}>;
|
|
4549
4567
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4568
|
+
"judgeSupplementary"?: boolean;
|
|
4550
4569
|
"taskCount": number;
|
|
4551
4570
|
"variantsPerTask": number;
|
|
4552
4571
|
"repeats": number;
|
|
@@ -4654,6 +4673,7 @@ export type StudyGetResponse = {
|
|
|
4654
4673
|
};
|
|
4655
4674
|
}>;
|
|
4656
4675
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4676
|
+
"judgeSupplementary"?: boolean;
|
|
4657
4677
|
"taskCount": number;
|
|
4658
4678
|
"variantsPerTask": number;
|
|
4659
4679
|
"repeats": number;
|
|
@@ -4845,7 +4865,7 @@ export type StudyGetResponse = {
|
|
|
4845
4865
|
};
|
|
4846
4866
|
"decision"?: {
|
|
4847
4867
|
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
4848
|
-
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
4868
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
4849
4869
|
"reason": string;
|
|
4850
4870
|
"efficiencyDiscounted": boolean;
|
|
4851
4871
|
};
|
|
@@ -4909,6 +4929,7 @@ export type StudyGetResponse = {
|
|
|
4909
4929
|
"maxTotalTokens": number | null;
|
|
4910
4930
|
"maxDurationMs": number | null;
|
|
4911
4931
|
"evalPassRate"?: number | null;
|
|
4932
|
+
"benchmarkPassRate"?: number | null;
|
|
4912
4933
|
"adoptionRate"?: number | null;
|
|
4913
4934
|
};
|
|
4914
4935
|
"skill": {
|
|
@@ -4940,6 +4961,7 @@ export type StudyGetResponse = {
|
|
|
4940
4961
|
"maxTotalTokens": number | null;
|
|
4941
4962
|
"maxDurationMs": number | null;
|
|
4942
4963
|
"evalPassRate"?: number | null;
|
|
4964
|
+
"benchmarkPassRate"?: number | null;
|
|
4943
4965
|
"adoptionRate"?: number | null;
|
|
4944
4966
|
};
|
|
4945
4967
|
};
|
|
@@ -5111,6 +5133,7 @@ export type StudyGenerateResponse = {
|
|
|
5111
5133
|
};
|
|
5112
5134
|
}>;
|
|
5113
5135
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5136
|
+
"judgeSupplementary"?: boolean;
|
|
5114
5137
|
"taskCount": number;
|
|
5115
5138
|
"variantsPerTask": number;
|
|
5116
5139
|
"repeats": number;
|
|
@@ -5206,6 +5229,7 @@ export type StudyLaunchResponse = {
|
|
|
5206
5229
|
};
|
|
5207
5230
|
}>;
|
|
5208
5231
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5232
|
+
"judgeSupplementary"?: boolean;
|
|
5209
5233
|
"taskCount": number;
|
|
5210
5234
|
"variantsPerTask": number;
|
|
5211
5235
|
"repeats": number;
|
|
@@ -5304,6 +5328,7 @@ export type StudyUpdateLensesResponse = {
|
|
|
5304
5328
|
};
|
|
5305
5329
|
}>;
|
|
5306
5330
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5331
|
+
"judgeSupplementary"?: boolean;
|
|
5307
5332
|
"taskCount": number;
|
|
5308
5333
|
"variantsPerTask": number;
|
|
5309
5334
|
"repeats": number;
|
|
@@ -5377,6 +5402,7 @@ export type StudyPreviewResponse = {
|
|
|
5377
5402
|
"baselineLabel": string | null;
|
|
5378
5403
|
"treatmentLabel": string | null;
|
|
5379
5404
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5405
|
+
"judgeSupplementary"?: true;
|
|
5380
5406
|
"promiseMetrics": Array<{
|
|
5381
5407
|
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5382
5408
|
"direction": "down" | "up";
|
|
@@ -5515,7 +5541,7 @@ export type StudyPreviewResponse = {
|
|
|
5515
5541
|
};
|
|
5516
5542
|
"decision"?: {
|
|
5517
5543
|
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
5518
|
-
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5544
|
+
"basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5519
5545
|
"reason": string;
|
|
5520
5546
|
"efficiencyDiscounted": boolean;
|
|
5521
5547
|
};
|
|
@@ -5579,6 +5605,7 @@ export type StudyPreviewResponse = {
|
|
|
5579
5605
|
"maxTotalTokens": number | null;
|
|
5580
5606
|
"maxDurationMs": number | null;
|
|
5581
5607
|
"evalPassRate"?: number | null;
|
|
5608
|
+
"benchmarkPassRate"?: number | null;
|
|
5582
5609
|
"adoptionRate"?: number | null;
|
|
5583
5610
|
};
|
|
5584
5611
|
"skill": {
|
|
@@ -5610,6 +5637,7 @@ export type StudyPreviewResponse = {
|
|
|
5610
5637
|
"maxTotalTokens": number | null;
|
|
5611
5638
|
"maxDurationMs": number | null;
|
|
5612
5639
|
"evalPassRate"?: number | null;
|
|
5640
|
+
"benchmarkPassRate"?: number | null;
|
|
5613
5641
|
"adoptionRate"?: number | null;
|
|
5614
5642
|
};
|
|
5615
5643
|
};
|
|
@@ -5768,6 +5796,7 @@ export type StudyPublishResponse = {
|
|
|
5768
5796
|
};
|
|
5769
5797
|
}>;
|
|
5770
5798
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5799
|
+
"judgeSupplementary"?: boolean;
|
|
5771
5800
|
"taskCount": number;
|
|
5772
5801
|
"variantsPerTask": number;
|
|
5773
5802
|
"repeats": number;
|
|
@@ -5867,6 +5896,7 @@ export type StudyRegradeResponse = {
|
|
|
5867
5896
|
};
|
|
5868
5897
|
}>;
|
|
5869
5898
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5899
|
+
"judgeSupplementary"?: boolean;
|
|
5870
5900
|
"taskCount": number;
|
|
5871
5901
|
"variantsPerTask": number;
|
|
5872
5902
|
"repeats": number;
|
|
@@ -5977,6 +6007,7 @@ export type StudyRotateShareTokenResponse = {
|
|
|
5977
6007
|
};
|
|
5978
6008
|
}>;
|
|
5979
6009
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6010
|
+
"judgeSupplementary"?: boolean;
|
|
5980
6011
|
"taskCount": number;
|
|
5981
6012
|
"variantsPerTask": number;
|
|
5982
6013
|
"repeats": number;
|
|
@@ -6087,6 +6118,7 @@ export type StudyUnpublishResponse = {
|
|
|
6087
6118
|
};
|
|
6088
6119
|
}>;
|
|
6089
6120
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6121
|
+
"judgeSupplementary"?: boolean;
|
|
6090
6122
|
"taskCount": number;
|
|
6091
6123
|
"variantsPerTask": number;
|
|
6092
6124
|
"repeats": number;
|
|
@@ -6180,6 +6212,7 @@ export type StudySetVisibilityResponse = {
|
|
|
6180
6212
|
};
|
|
6181
6213
|
}>;
|
|
6182
6214
|
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6215
|
+
"judgeSupplementary"?: boolean;
|
|
6183
6216
|
"taskCount": number;
|
|
6184
6217
|
"variantsPerTask": number;
|
|
6185
6218
|
"repeats": number;
|