@ora-ai/cli 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +205 -10
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +443 -5
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/package.json +1 -1
|
@@ -60,10 +60,14 @@ export type BenchmarksListResponse = {
|
|
|
60
60
|
"model": string;
|
|
61
61
|
"provider"?: string;
|
|
62
62
|
}>;
|
|
63
|
+
"lenses": Array<"judge" | "evals">;
|
|
63
64
|
"taskCount": number;
|
|
65
|
+
"variantsPerTask": number;
|
|
66
|
+
"repeats": number;
|
|
64
67
|
"status": string;
|
|
65
68
|
"error": string | null;
|
|
66
69
|
"visibility": "private" | "public";
|
|
70
|
+
"lowSampleOverride": boolean;
|
|
67
71
|
"publishedAt": string | null;
|
|
68
72
|
"verdict": unknown;
|
|
69
73
|
"createdAt": string;
|
|
@@ -90,16 +94,22 @@ export type BenchmarksCreateRequest = {
|
|
|
90
94
|
"addons"?: Record<string, unknown>;
|
|
91
95
|
};
|
|
92
96
|
};
|
|
97
|
+
"lenses"?: Array<"judge" | "evals">;
|
|
98
|
+
"proposeExpectations"?: boolean;
|
|
93
99
|
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
94
100
|
"baselineLabel"?: string;
|
|
95
101
|
"treatmentLabel"?: string;
|
|
96
102
|
"taskCount"?: number;
|
|
103
|
+
"variantsPerTask"?: number;
|
|
104
|
+
"repeats"?: number;
|
|
97
105
|
"tasks"?: Array<{
|
|
98
106
|
"prompt": string;
|
|
99
107
|
"rationale"?: string;
|
|
100
108
|
"category"?: string;
|
|
101
109
|
"requiresWeb"?: boolean;
|
|
102
|
-
"rubric"
|
|
110
|
+
"rubric"?: Array<string>;
|
|
111
|
+
"variants"?: Array<string>;
|
|
112
|
+
"expectations"?: Array<unknown>;
|
|
103
113
|
}>;
|
|
104
114
|
"matrix": Array<{
|
|
105
115
|
"harness": string;
|
|
@@ -152,16 +162,36 @@ export type BenchmarksCreateResponse = {
|
|
|
152
162
|
"model": string;
|
|
153
163
|
"provider"?: string;
|
|
154
164
|
}>;
|
|
165
|
+
"lenses": Array<"judge" | "evals">;
|
|
155
166
|
"taskCount": number;
|
|
167
|
+
"variantsPerTask": number;
|
|
168
|
+
"repeats": number;
|
|
156
169
|
"status": string;
|
|
157
170
|
"error": string | null;
|
|
158
171
|
"visibility": "private" | "public";
|
|
172
|
+
"lowSampleOverride": boolean;
|
|
159
173
|
"publishedAt": string | null;
|
|
160
174
|
"verdict": unknown;
|
|
161
175
|
"createdAt": string;
|
|
162
176
|
"updatedAt": string;
|
|
163
177
|
};
|
|
164
178
|
"streamUrl": string;
|
|
179
|
+
"warnings"?: Array<{
|
|
180
|
+
"harness": string;
|
|
181
|
+
"kind": string;
|
|
182
|
+
}>;
|
|
183
|
+
};
|
|
184
|
+
export type BenchmarksConvertWebmcpSuiteRequest = {
|
|
185
|
+
"suite": unknown;
|
|
186
|
+
"ordered"?: boolean;
|
|
187
|
+
};
|
|
188
|
+
export type BenchmarksConvertWebmcpSuiteResponse = {
|
|
189
|
+
"tasks": Array<{
|
|
190
|
+
"title": string;
|
|
191
|
+
"prompt": string;
|
|
192
|
+
"expectations": Array<unknown>;
|
|
193
|
+
}>;
|
|
194
|
+
"warnings": Array<string>;
|
|
165
195
|
};
|
|
166
196
|
export type BenchmarksGetResponse = {
|
|
167
197
|
"benchmark": {
|
|
@@ -207,10 +237,14 @@ export type BenchmarksGetResponse = {
|
|
|
207
237
|
"model": string;
|
|
208
238
|
"provider"?: string;
|
|
209
239
|
}>;
|
|
240
|
+
"lenses": Array<"judge" | "evals">;
|
|
210
241
|
"taskCount": number;
|
|
242
|
+
"variantsPerTask": number;
|
|
243
|
+
"repeats": number;
|
|
211
244
|
"status": string;
|
|
212
245
|
"error": string | null;
|
|
213
246
|
"visibility": "private" | "public";
|
|
247
|
+
"lowSampleOverride": boolean;
|
|
214
248
|
"publishedAt": string | null;
|
|
215
249
|
"verdict": unknown;
|
|
216
250
|
"createdAt": string;
|
|
@@ -225,6 +259,8 @@ export type BenchmarksGetResponse = {
|
|
|
225
259
|
"category": string | null;
|
|
226
260
|
"requiresWeb": boolean;
|
|
227
261
|
"rubric": Array<string>;
|
|
262
|
+
"variants"?: Array<string> | null;
|
|
263
|
+
"expectations"?: Array<unknown> | null;
|
|
228
264
|
}>;
|
|
229
265
|
"runs": Array<{
|
|
230
266
|
"runId": string;
|
|
@@ -233,6 +269,8 @@ export type BenchmarksGetResponse = {
|
|
|
233
269
|
"harness": string;
|
|
234
270
|
"model": string;
|
|
235
271
|
"provider": string | null;
|
|
272
|
+
"variantIndex"?: number;
|
|
273
|
+
"repeatIndex"?: number;
|
|
236
274
|
"status": string;
|
|
237
275
|
"outcome": string | null;
|
|
238
276
|
"outcomeReason": string | null;
|
|
@@ -255,12 +293,25 @@ export type BenchmarksGetResponse = {
|
|
|
255
293
|
"votes"?: {
|
|
256
294
|
"passed": number;
|
|
257
295
|
"judges": number;
|
|
296
|
+
"byJudge"?: Array<{
|
|
297
|
+
"judge": string;
|
|
298
|
+
"passed": boolean;
|
|
299
|
+
"note"?: string;
|
|
300
|
+
}>;
|
|
258
301
|
};
|
|
259
302
|
}> | null;
|
|
260
303
|
"gradedAt": string | null;
|
|
261
304
|
"judgeModel": string | null;
|
|
262
305
|
"pairVerdict": "skill" | "baseline" | "tie" | "unmeasurable" | null;
|
|
306
|
+
"pairVotes": Array<{
|
|
307
|
+
"judge": string;
|
|
308
|
+
"ordering": "base-first" | "skill-first";
|
|
309
|
+
"answer": "A" | "B" | "tie";
|
|
310
|
+
}> | null;
|
|
263
311
|
"pairGradedAt": string | null;
|
|
312
|
+
"evalResults": unknown | null;
|
|
313
|
+
"evalScore": number | null;
|
|
314
|
+
"evalGradedAt": string | null;
|
|
264
315
|
"runCreatedAt": string | null;
|
|
265
316
|
"startedAt": string | null;
|
|
266
317
|
"finishedAt": string | null;
|
|
@@ -268,7 +319,27 @@ export type BenchmarksGetResponse = {
|
|
|
268
319
|
"aggregate": {
|
|
269
320
|
"summary": {
|
|
270
321
|
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
271
|
-
"headline": Array<
|
|
322
|
+
"headline": Array<{
|
|
323
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
324
|
+
"label": string;
|
|
325
|
+
"direction": "down" | "up";
|
|
326
|
+
"baselineAvg": number | null;
|
|
327
|
+
"skillAvg": number | null;
|
|
328
|
+
"deltaPct": number | null;
|
|
329
|
+
"delta"?: {
|
|
330
|
+
"unit": "pp";
|
|
331
|
+
"absolute": number | null;
|
|
332
|
+
"relativePct": number | null;
|
|
333
|
+
} | {
|
|
334
|
+
"unit": "pts";
|
|
335
|
+
"absolute": number | null;
|
|
336
|
+
"relativePct"?: number | null;
|
|
337
|
+
} | {
|
|
338
|
+
"unit": "%";
|
|
339
|
+
"relativePct": number | null;
|
|
340
|
+
};
|
|
341
|
+
"improved": boolean;
|
|
342
|
+
}>;
|
|
272
343
|
"completionRate": {
|
|
273
344
|
"baseline": number | null;
|
|
274
345
|
"skill": number | null;
|
|
@@ -284,7 +355,106 @@ export type BenchmarksGetResponse = {
|
|
|
284
355
|
"ties": number;
|
|
285
356
|
"winRate": number | null;
|
|
286
357
|
};
|
|
358
|
+
"decision"?: {
|
|
359
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
360
|
+
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
361
|
+
"reason": string;
|
|
362
|
+
"efficiencyDiscounted": boolean;
|
|
363
|
+
};
|
|
287
364
|
"leaderboard"?: Array<unknown>;
|
|
365
|
+
"concentration": {
|
|
366
|
+
"perCell": Array<{
|
|
367
|
+
"cell": string;
|
|
368
|
+
"goalDeltaPp": number | null;
|
|
369
|
+
"contributionPct": number | null;
|
|
370
|
+
}>;
|
|
371
|
+
"heterogeneity": {
|
|
372
|
+
"dominantCell": string;
|
|
373
|
+
"contributionPct": number;
|
|
374
|
+
} | null;
|
|
375
|
+
};
|
|
376
|
+
"matrixShape": {
|
|
377
|
+
"cells": number;
|
|
378
|
+
"harnesses": number;
|
|
379
|
+
"models": number;
|
|
380
|
+
"crossed": boolean;
|
|
381
|
+
};
|
|
382
|
+
"flakiness"?: {
|
|
383
|
+
"measured": boolean;
|
|
384
|
+
"conditions": number;
|
|
385
|
+
"unstable": number;
|
|
386
|
+
"worst": {
|
|
387
|
+
"cell": string;
|
|
388
|
+
"taskId": string;
|
|
389
|
+
"arm": "baseline" | "skill";
|
|
390
|
+
"variant"?: number;
|
|
391
|
+
"passRate": number;
|
|
392
|
+
} | null;
|
|
393
|
+
};
|
|
394
|
+
"arms"?: {
|
|
395
|
+
"baseline": {
|
|
396
|
+
"runs": number;
|
|
397
|
+
"completed": number;
|
|
398
|
+
"infraFailures": number;
|
|
399
|
+
"completionRate": number | null;
|
|
400
|
+
"goalRate": number | null;
|
|
401
|
+
"goalTally": {
|
|
402
|
+
"achieved": number;
|
|
403
|
+
"partial": number;
|
|
404
|
+
"missed": number;
|
|
405
|
+
};
|
|
406
|
+
"avgTotalTokens": number | null;
|
|
407
|
+
"avgOutputTokens": number | null;
|
|
408
|
+
"avgCostUsd": number | null;
|
|
409
|
+
"avgDurationMs": number | null;
|
|
410
|
+
"avgTurns": number | null;
|
|
411
|
+
"costCoverage": number | null;
|
|
412
|
+
"avgQualityScore": number | null;
|
|
413
|
+
"gradedCoverage": number | null;
|
|
414
|
+
"medianCostUsd": number | null;
|
|
415
|
+
"medianTotalTokens": number | null;
|
|
416
|
+
"medianDurationMs": number | null;
|
|
417
|
+
"p95CostUsd": number | null;
|
|
418
|
+
"p95TotalTokens": number | null;
|
|
419
|
+
"p95DurationMs": number | null;
|
|
420
|
+
"maxCostUsd": number | null;
|
|
421
|
+
"maxTotalTokens": number | null;
|
|
422
|
+
"maxDurationMs": number | null;
|
|
423
|
+
"evalPassRate"?: number | null;
|
|
424
|
+
"adoptionRate"?: number | null;
|
|
425
|
+
};
|
|
426
|
+
"skill": {
|
|
427
|
+
"runs": number;
|
|
428
|
+
"completed": number;
|
|
429
|
+
"infraFailures": number;
|
|
430
|
+
"completionRate": number | null;
|
|
431
|
+
"goalRate": number | null;
|
|
432
|
+
"goalTally": {
|
|
433
|
+
"achieved": number;
|
|
434
|
+
"partial": number;
|
|
435
|
+
"missed": number;
|
|
436
|
+
};
|
|
437
|
+
"avgTotalTokens": number | null;
|
|
438
|
+
"avgOutputTokens": number | null;
|
|
439
|
+
"avgCostUsd": number | null;
|
|
440
|
+
"avgDurationMs": number | null;
|
|
441
|
+
"avgTurns": number | null;
|
|
442
|
+
"costCoverage": number | null;
|
|
443
|
+
"avgQualityScore": number | null;
|
|
444
|
+
"gradedCoverage": number | null;
|
|
445
|
+
"medianCostUsd": number | null;
|
|
446
|
+
"medianTotalTokens": number | null;
|
|
447
|
+
"medianDurationMs": number | null;
|
|
448
|
+
"p95CostUsd": number | null;
|
|
449
|
+
"p95TotalTokens": number | null;
|
|
450
|
+
"p95DurationMs": number | null;
|
|
451
|
+
"maxCostUsd": number | null;
|
|
452
|
+
"maxTotalTokens": number | null;
|
|
453
|
+
"maxDurationMs": number | null;
|
|
454
|
+
"evalPassRate"?: number | null;
|
|
455
|
+
"adoptionRate"?: number | null;
|
|
456
|
+
};
|
|
457
|
+
};
|
|
288
458
|
"totals": {
|
|
289
459
|
"cells": number;
|
|
290
460
|
"runs": number;
|
|
@@ -314,8 +484,23 @@ export type BenchmarksGetResponse = {
|
|
|
314
484
|
}>;
|
|
315
485
|
}>;
|
|
316
486
|
};
|
|
487
|
+
"warnings": Array<{
|
|
488
|
+
"type": "eval_capability_gap";
|
|
489
|
+
"harness": string;
|
|
490
|
+
"kind": string;
|
|
491
|
+
} | {
|
|
492
|
+
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
493
|
+
"detail": string;
|
|
494
|
+
"taskIds"?: Array<string>;
|
|
495
|
+
}>;
|
|
317
496
|
};
|
|
318
497
|
export type BenchmarksDeleteResponse = unknown;
|
|
498
|
+
export type BenchmarksRegenerateSummaryResponse = {
|
|
499
|
+
"status": "queued";
|
|
500
|
+
};
|
|
501
|
+
export type BenchmarksGenerateRequest = {
|
|
502
|
+
"proposeExpectations"?: boolean;
|
|
503
|
+
};
|
|
319
504
|
export type BenchmarksGenerateResponse = {
|
|
320
505
|
"benchmark": {
|
|
321
506
|
"id": string;
|
|
@@ -360,16 +545,24 @@ export type BenchmarksGenerateResponse = {
|
|
|
360
545
|
"model": string;
|
|
361
546
|
"provider"?: string;
|
|
362
547
|
}>;
|
|
548
|
+
"lenses": Array<"judge" | "evals">;
|
|
363
549
|
"taskCount": number;
|
|
550
|
+
"variantsPerTask": number;
|
|
551
|
+
"repeats": number;
|
|
364
552
|
"status": string;
|
|
365
553
|
"error": string | null;
|
|
366
554
|
"visibility": "private" | "public";
|
|
555
|
+
"lowSampleOverride": boolean;
|
|
367
556
|
"publishedAt": string | null;
|
|
368
557
|
"verdict": unknown;
|
|
369
558
|
"createdAt": string;
|
|
370
559
|
"updatedAt": string;
|
|
371
560
|
};
|
|
372
561
|
"streamUrl": string;
|
|
562
|
+
"warnings"?: Array<{
|
|
563
|
+
"harness": string;
|
|
564
|
+
"kind": string;
|
|
565
|
+
}>;
|
|
373
566
|
};
|
|
374
567
|
export type BenchmarksLaunchResponse = {
|
|
375
568
|
"benchmark": {
|
|
@@ -415,16 +608,90 @@ export type BenchmarksLaunchResponse = {
|
|
|
415
608
|
"model": string;
|
|
416
609
|
"provider"?: string;
|
|
417
610
|
}>;
|
|
611
|
+
"lenses": Array<"judge" | "evals">;
|
|
418
612
|
"taskCount": number;
|
|
613
|
+
"variantsPerTask": number;
|
|
614
|
+
"repeats": number;
|
|
419
615
|
"status": string;
|
|
420
616
|
"error": string | null;
|
|
421
617
|
"visibility": "private" | "public";
|
|
618
|
+
"lowSampleOverride": boolean;
|
|
422
619
|
"publishedAt": string | null;
|
|
423
620
|
"verdict": unknown;
|
|
424
621
|
"createdAt": string;
|
|
425
622
|
"updatedAt": string;
|
|
426
623
|
};
|
|
427
624
|
"streamUrl": string;
|
|
625
|
+
"warnings"?: Array<{
|
|
626
|
+
"harness": string;
|
|
627
|
+
"kind": string;
|
|
628
|
+
}>;
|
|
629
|
+
};
|
|
630
|
+
export type BenchmarksUpdateLensesRequest = {
|
|
631
|
+
"lenses": Array<"judge" | "evals">;
|
|
632
|
+
};
|
|
633
|
+
export type BenchmarksUpdateLensesResponse = {
|
|
634
|
+
"benchmark": {
|
|
635
|
+
"id": string;
|
|
636
|
+
"executiveSummary"?: string | null;
|
|
637
|
+
"plannedRunCount": number;
|
|
638
|
+
"organizationId": string;
|
|
639
|
+
"createdBy": string | null;
|
|
640
|
+
"name": string;
|
|
641
|
+
"slug": string;
|
|
642
|
+
"description": string | null;
|
|
643
|
+
"skillRef": string | null;
|
|
644
|
+
"skillSlug": string | null;
|
|
645
|
+
"skillName": string | null;
|
|
646
|
+
"skillPromise": string;
|
|
647
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
648
|
+
"subjectName": string | null;
|
|
649
|
+
"subjectLabel": string;
|
|
650
|
+
"armConfig": {
|
|
651
|
+
"baseline"?: {
|
|
652
|
+
"tools"?: "default" | "none" | {
|
|
653
|
+
"tools": Array<string>;
|
|
654
|
+
};
|
|
655
|
+
"addons"?: Record<string, unknown>;
|
|
656
|
+
};
|
|
657
|
+
"treatment"?: {
|
|
658
|
+
"tools"?: "default" | "none" | {
|
|
659
|
+
"tools": Array<string>;
|
|
660
|
+
};
|
|
661
|
+
"addons"?: Record<string, unknown>;
|
|
662
|
+
};
|
|
663
|
+
};
|
|
664
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
665
|
+
"baselineLabel": string | null;
|
|
666
|
+
"treatmentLabel": string | null;
|
|
667
|
+
"promiseMetrics": Array<{
|
|
668
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
669
|
+
"direction": "down" | "up";
|
|
670
|
+
"label": string;
|
|
671
|
+
}>;
|
|
672
|
+
"matrix": Array<{
|
|
673
|
+
"harness": string;
|
|
674
|
+
"model": string;
|
|
675
|
+
"provider"?: string;
|
|
676
|
+
}>;
|
|
677
|
+
"lenses": Array<"judge" | "evals">;
|
|
678
|
+
"taskCount": number;
|
|
679
|
+
"variantsPerTask": number;
|
|
680
|
+
"repeats": number;
|
|
681
|
+
"status": string;
|
|
682
|
+
"error": string | null;
|
|
683
|
+
"visibility": "private" | "public";
|
|
684
|
+
"lowSampleOverride": boolean;
|
|
685
|
+
"publishedAt": string | null;
|
|
686
|
+
"verdict": unknown;
|
|
687
|
+
"createdAt": string;
|
|
688
|
+
"updatedAt": string;
|
|
689
|
+
};
|
|
690
|
+
"warnings": Array<{
|
|
691
|
+
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
692
|
+
"detail": string;
|
|
693
|
+
"taskIds"?: Array<string>;
|
|
694
|
+
}>;
|
|
428
695
|
};
|
|
429
696
|
export type BenchmarksPreviewResponse = {
|
|
430
697
|
"benchmark": {
|
|
@@ -455,6 +722,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
455
722
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
456
723
|
"baselineLabel": string | null;
|
|
457
724
|
"treatmentLabel": string | null;
|
|
725
|
+
"lenses": Array<"judge" | "evals">;
|
|
458
726
|
"promiseMetrics": Array<{
|
|
459
727
|
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
460
728
|
"direction": "down" | "up";
|
|
@@ -466,6 +734,8 @@ export type BenchmarksPreviewResponse = {
|
|
|
466
734
|
"provider"?: string;
|
|
467
735
|
}>;
|
|
468
736
|
"taskCount": number;
|
|
737
|
+
"variantsPerTask": number;
|
|
738
|
+
"repeats": number;
|
|
469
739
|
"status": string;
|
|
470
740
|
"publishedAt": string | null;
|
|
471
741
|
"createdAt": string;
|
|
@@ -478,6 +748,8 @@ export type BenchmarksPreviewResponse = {
|
|
|
478
748
|
"category": string | null;
|
|
479
749
|
"requiresWeb": boolean;
|
|
480
750
|
"rubric": Array<string>;
|
|
751
|
+
"variants"?: Array<string> | null;
|
|
752
|
+
"expectations"?: Array<unknown> | null;
|
|
481
753
|
}>;
|
|
482
754
|
"runs": Array<{
|
|
483
755
|
"runId": string;
|
|
@@ -486,6 +758,8 @@ export type BenchmarksPreviewResponse = {
|
|
|
486
758
|
"harness": string;
|
|
487
759
|
"model": string;
|
|
488
760
|
"provider": string | null;
|
|
761
|
+
"variantIndex"?: number;
|
|
762
|
+
"repeatIndex"?: number;
|
|
489
763
|
"status": string;
|
|
490
764
|
"outcome": string | null;
|
|
491
765
|
"outcomeReason": string | null;
|
|
@@ -508,12 +782,25 @@ export type BenchmarksPreviewResponse = {
|
|
|
508
782
|
"votes"?: {
|
|
509
783
|
"passed": number;
|
|
510
784
|
"judges": number;
|
|
785
|
+
"byJudge"?: Array<{
|
|
786
|
+
"judge": string;
|
|
787
|
+
"passed": boolean;
|
|
788
|
+
"note"?: string;
|
|
789
|
+
}>;
|
|
511
790
|
};
|
|
512
791
|
}> | null;
|
|
513
792
|
"gradedAt": string | null;
|
|
514
793
|
"judgeModel": string | null;
|
|
515
794
|
"pairVerdict": "skill" | "baseline" | "tie" | "unmeasurable" | null;
|
|
795
|
+
"pairVotes": Array<{
|
|
796
|
+
"judge": string;
|
|
797
|
+
"ordering": "base-first" | "skill-first";
|
|
798
|
+
"answer": "A" | "B" | "tie";
|
|
799
|
+
}> | null;
|
|
516
800
|
"pairGradedAt": string | null;
|
|
801
|
+
"evalResults": unknown | null;
|
|
802
|
+
"evalScore": number | null;
|
|
803
|
+
"evalGradedAt": string | null;
|
|
517
804
|
"runCreatedAt": string | null;
|
|
518
805
|
"startedAt": string | null;
|
|
519
806
|
"finishedAt": string | null;
|
|
@@ -521,7 +808,27 @@ export type BenchmarksPreviewResponse = {
|
|
|
521
808
|
"aggregate": {
|
|
522
809
|
"summary": {
|
|
523
810
|
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
524
|
-
"headline": Array<
|
|
811
|
+
"headline": Array<{
|
|
812
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
813
|
+
"label": string;
|
|
814
|
+
"direction": "down" | "up";
|
|
815
|
+
"baselineAvg": number | null;
|
|
816
|
+
"skillAvg": number | null;
|
|
817
|
+
"deltaPct": number | null;
|
|
818
|
+
"delta"?: {
|
|
819
|
+
"unit": "pp";
|
|
820
|
+
"absolute": number | null;
|
|
821
|
+
"relativePct": number | null;
|
|
822
|
+
} | {
|
|
823
|
+
"unit": "pts";
|
|
824
|
+
"absolute": number | null;
|
|
825
|
+
"relativePct"?: number | null;
|
|
826
|
+
} | {
|
|
827
|
+
"unit": "%";
|
|
828
|
+
"relativePct": number | null;
|
|
829
|
+
};
|
|
830
|
+
"improved": boolean;
|
|
831
|
+
}>;
|
|
525
832
|
"completionRate": {
|
|
526
833
|
"baseline": number | null;
|
|
527
834
|
"skill": number | null;
|
|
@@ -537,7 +844,106 @@ export type BenchmarksPreviewResponse = {
|
|
|
537
844
|
"ties": number;
|
|
538
845
|
"winRate": number | null;
|
|
539
846
|
};
|
|
847
|
+
"decision"?: {
|
|
848
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
849
|
+
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
850
|
+
"reason": string;
|
|
851
|
+
"efficiencyDiscounted": boolean;
|
|
852
|
+
};
|
|
540
853
|
"leaderboard"?: Array<unknown>;
|
|
854
|
+
"concentration": {
|
|
855
|
+
"perCell": Array<{
|
|
856
|
+
"cell": string;
|
|
857
|
+
"goalDeltaPp": number | null;
|
|
858
|
+
"contributionPct": number | null;
|
|
859
|
+
}>;
|
|
860
|
+
"heterogeneity": {
|
|
861
|
+
"dominantCell": string;
|
|
862
|
+
"contributionPct": number;
|
|
863
|
+
} | null;
|
|
864
|
+
};
|
|
865
|
+
"matrixShape": {
|
|
866
|
+
"cells": number;
|
|
867
|
+
"harnesses": number;
|
|
868
|
+
"models": number;
|
|
869
|
+
"crossed": boolean;
|
|
870
|
+
};
|
|
871
|
+
"flakiness"?: {
|
|
872
|
+
"measured": boolean;
|
|
873
|
+
"conditions": number;
|
|
874
|
+
"unstable": number;
|
|
875
|
+
"worst": {
|
|
876
|
+
"cell": string;
|
|
877
|
+
"taskId": string;
|
|
878
|
+
"arm": "baseline" | "skill";
|
|
879
|
+
"variant"?: number;
|
|
880
|
+
"passRate": number;
|
|
881
|
+
} | null;
|
|
882
|
+
};
|
|
883
|
+
"arms"?: {
|
|
884
|
+
"baseline": {
|
|
885
|
+
"runs": number;
|
|
886
|
+
"completed": number;
|
|
887
|
+
"infraFailures": number;
|
|
888
|
+
"completionRate": number | null;
|
|
889
|
+
"goalRate": number | null;
|
|
890
|
+
"goalTally": {
|
|
891
|
+
"achieved": number;
|
|
892
|
+
"partial": number;
|
|
893
|
+
"missed": number;
|
|
894
|
+
};
|
|
895
|
+
"avgTotalTokens": number | null;
|
|
896
|
+
"avgOutputTokens": number | null;
|
|
897
|
+
"avgCostUsd": number | null;
|
|
898
|
+
"avgDurationMs": number | null;
|
|
899
|
+
"avgTurns": number | null;
|
|
900
|
+
"costCoverage": number | null;
|
|
901
|
+
"avgQualityScore": number | null;
|
|
902
|
+
"gradedCoverage": number | null;
|
|
903
|
+
"medianCostUsd": number | null;
|
|
904
|
+
"medianTotalTokens": number | null;
|
|
905
|
+
"medianDurationMs": number | null;
|
|
906
|
+
"p95CostUsd": number | null;
|
|
907
|
+
"p95TotalTokens": number | null;
|
|
908
|
+
"p95DurationMs": number | null;
|
|
909
|
+
"maxCostUsd": number | null;
|
|
910
|
+
"maxTotalTokens": number | null;
|
|
911
|
+
"maxDurationMs": number | null;
|
|
912
|
+
"evalPassRate"?: number | null;
|
|
913
|
+
"adoptionRate"?: number | null;
|
|
914
|
+
};
|
|
915
|
+
"skill": {
|
|
916
|
+
"runs": number;
|
|
917
|
+
"completed": number;
|
|
918
|
+
"infraFailures": number;
|
|
919
|
+
"completionRate": number | null;
|
|
920
|
+
"goalRate": number | null;
|
|
921
|
+
"goalTally": {
|
|
922
|
+
"achieved": number;
|
|
923
|
+
"partial": number;
|
|
924
|
+
"missed": number;
|
|
925
|
+
};
|
|
926
|
+
"avgTotalTokens": number | null;
|
|
927
|
+
"avgOutputTokens": number | null;
|
|
928
|
+
"avgCostUsd": number | null;
|
|
929
|
+
"avgDurationMs": number | null;
|
|
930
|
+
"avgTurns": number | null;
|
|
931
|
+
"costCoverage": number | null;
|
|
932
|
+
"avgQualityScore": number | null;
|
|
933
|
+
"gradedCoverage": number | null;
|
|
934
|
+
"medianCostUsd": number | null;
|
|
935
|
+
"medianTotalTokens": number | null;
|
|
936
|
+
"medianDurationMs": number | null;
|
|
937
|
+
"p95CostUsd": number | null;
|
|
938
|
+
"p95TotalTokens": number | null;
|
|
939
|
+
"p95DurationMs": number | null;
|
|
940
|
+
"maxCostUsd": number | null;
|
|
941
|
+
"maxTotalTokens": number | null;
|
|
942
|
+
"maxDurationMs": number | null;
|
|
943
|
+
"evalPassRate"?: number | null;
|
|
944
|
+
"adoptionRate"?: number | null;
|
|
945
|
+
};
|
|
946
|
+
};
|
|
541
947
|
"totals": {
|
|
542
948
|
"cells": number;
|
|
543
949
|
"runs": number;
|
|
@@ -615,10 +1021,14 @@ export type BenchmarksPublishResponse = {
|
|
|
615
1021
|
"model": string;
|
|
616
1022
|
"provider"?: string;
|
|
617
1023
|
}>;
|
|
1024
|
+
"lenses": Array<"judge" | "evals">;
|
|
618
1025
|
"taskCount": number;
|
|
1026
|
+
"variantsPerTask": number;
|
|
1027
|
+
"repeats": number;
|
|
619
1028
|
"status": string;
|
|
620
1029
|
"error": string | null;
|
|
621
1030
|
"visibility": "private" | "public";
|
|
1031
|
+
"lowSampleOverride": boolean;
|
|
622
1032
|
"publishedAt": string | null;
|
|
623
1033
|
"verdict": unknown;
|
|
624
1034
|
"createdAt": string;
|
|
@@ -669,20 +1079,29 @@ export type BenchmarksRegradeResponse = {
|
|
|
669
1079
|
"model": string;
|
|
670
1080
|
"provider"?: string;
|
|
671
1081
|
}>;
|
|
1082
|
+
"lenses": Array<"judge" | "evals">;
|
|
672
1083
|
"taskCount": number;
|
|
1084
|
+
"variantsPerTask": number;
|
|
1085
|
+
"repeats": number;
|
|
673
1086
|
"status": string;
|
|
674
1087
|
"error": string | null;
|
|
675
1088
|
"visibility": "private" | "public";
|
|
1089
|
+
"lowSampleOverride": boolean;
|
|
676
1090
|
"publishedAt": string | null;
|
|
677
1091
|
"verdict": unknown;
|
|
678
1092
|
"createdAt": string;
|
|
679
1093
|
"updatedAt": string;
|
|
680
1094
|
};
|
|
681
1095
|
"streamUrl": string;
|
|
1096
|
+
"warnings"?: Array<{
|
|
1097
|
+
"harness": string;
|
|
1098
|
+
"kind": string;
|
|
1099
|
+
}>;
|
|
682
1100
|
};
|
|
683
1101
|
export type BenchmarksStreamResponse = unknown;
|
|
684
1102
|
export type BenchmarksUpdateTaskRequest = {
|
|
685
1103
|
"prompt": string;
|
|
1104
|
+
"expectations"?: Array<unknown>;
|
|
686
1105
|
};
|
|
687
1106
|
export type BenchmarksUpdateTaskResponse = {
|
|
688
1107
|
"task": {
|
|
@@ -694,6 +1113,8 @@ export type BenchmarksUpdateTaskResponse = {
|
|
|
694
1113
|
"category": string | null;
|
|
695
1114
|
"requiresWeb": boolean;
|
|
696
1115
|
"rubric": Array<string>;
|
|
1116
|
+
"variants"?: Array<string> | null;
|
|
1117
|
+
"expectations"?: Array<unknown> | null;
|
|
697
1118
|
};
|
|
698
1119
|
};
|
|
699
1120
|
export type BenchmarksUnpublishResponse = {
|
|
@@ -740,10 +1161,14 @@ export type BenchmarksUnpublishResponse = {
|
|
|
740
1161
|
"model": string;
|
|
741
1162
|
"provider"?: string;
|
|
742
1163
|
}>;
|
|
1164
|
+
"lenses": Array<"judge" | "evals">;
|
|
743
1165
|
"taskCount": number;
|
|
1166
|
+
"variantsPerTask": number;
|
|
1167
|
+
"repeats": number;
|
|
744
1168
|
"status": string;
|
|
745
1169
|
"error": string | null;
|
|
746
1170
|
"visibility": "private" | "public";
|
|
1171
|
+
"lowSampleOverride": boolean;
|
|
747
1172
|
"publishedAt": string | null;
|
|
748
1173
|
"verdict": unknown;
|
|
749
1174
|
"createdAt": string;
|
|
@@ -761,6 +1186,8 @@ export type ExperimentCatalogResponse = {
|
|
|
761
1186
|
"defaultProvider": string | null;
|
|
762
1187
|
"providers": Array<{
|
|
763
1188
|
"id": string;
|
|
1189
|
+
"offerable": boolean;
|
|
1190
|
+
"reason": string | null;
|
|
764
1191
|
"models": Array<{
|
|
765
1192
|
"id": string;
|
|
766
1193
|
"name": string;
|
|
@@ -860,6 +1287,7 @@ export type ExperimentRunsDirectSignalsResponse = {
|
|
|
860
1287
|
};
|
|
861
1288
|
export type ExperimentRunsInsightResponse = unknown;
|
|
862
1289
|
export type ExperimentRunsRawTraceResponse = unknown;
|
|
1290
|
+
export type ExperimentRunsSpansResponse = unknown;
|
|
863
1291
|
export type ExperimentRunsStreamResponse = unknown;
|
|
864
1292
|
export type ExperimentRunsTrajectoryResponse = unknown;
|
|
865
1293
|
export type ExperimentSecretsListResponse = {
|
|
@@ -1056,8 +1484,8 @@ export type WebmcpArenaCreateRequest = {
|
|
|
1056
1484
|
"harness": string;
|
|
1057
1485
|
"model": string;
|
|
1058
1486
|
"provider"?: string;
|
|
1059
|
-
"baseline"?: "
|
|
1060
|
-
"treatment"?: "webdriver" | "webdriver-webmcp" | "devtools" | "devtools-webmcp" | "webmcp-
|
|
1487
|
+
"baseline"?: "webdriver" | "none" | "devtools" | "browserbase" | "oxylabs";
|
|
1488
|
+
"treatment"?: "webmcp-native" | "webdriver" | "webdriver-webmcp" | "devtools" | "devtools-webmcp" | "webmcp-shim";
|
|
1061
1489
|
"name"?: string;
|
|
1062
1490
|
};
|
|
1063
1491
|
export type WebmcpArenaCreateResponse = {
|
|
@@ -1104,16 +1532,24 @@ export type WebmcpArenaCreateResponse = {
|
|
|
1104
1532
|
"model": string;
|
|
1105
1533
|
"provider"?: string;
|
|
1106
1534
|
}>;
|
|
1535
|
+
"lenses": Array<"judge" | "evals">;
|
|
1107
1536
|
"taskCount": number;
|
|
1537
|
+
"variantsPerTask": number;
|
|
1538
|
+
"repeats": number;
|
|
1108
1539
|
"status": string;
|
|
1109
1540
|
"error": string | null;
|
|
1110
1541
|
"visibility": "private" | "public";
|
|
1542
|
+
"lowSampleOverride": boolean;
|
|
1111
1543
|
"publishedAt": string | null;
|
|
1112
1544
|
"verdict": unknown;
|
|
1113
1545
|
"createdAt": string;
|
|
1114
1546
|
"updatedAt": string;
|
|
1115
1547
|
};
|
|
1116
1548
|
"streamUrl": string;
|
|
1549
|
+
"warnings"?: Array<{
|
|
1550
|
+
"harness": string;
|
|
1551
|
+
"kind": string;
|
|
1552
|
+
}>;
|
|
1117
1553
|
};
|
|
1118
1554
|
export type ActivityListResponse = {
|
|
1119
1555
|
"success": true;
|
|
@@ -1346,6 +1782,7 @@ export type IntentsGetResponse = {
|
|
|
1346
1782
|
"share": number;
|
|
1347
1783
|
"tracked": boolean;
|
|
1348
1784
|
"domain"?: string | null;
|
|
1785
|
+
"curated"?: boolean;
|
|
1349
1786
|
}>;
|
|
1350
1787
|
"sources": Array<{
|
|
1351
1788
|
"host": string;
|
|
@@ -1815,6 +2252,7 @@ export type StudioRunsDirectSignalsResponse = {
|
|
|
1815
2252
|
};
|
|
1816
2253
|
export type StudioRunsInsightResponse = unknown;
|
|
1817
2254
|
export type StudioRunsRawTraceResponse = unknown;
|
|
2255
|
+
export type StudioRunsSpansResponse = unknown;
|
|
1818
2256
|
export type StudioRunsStreamResponse = unknown;
|
|
1819
2257
|
export type StudioRunsTrajectoryResponse = unknown;
|
|
1820
2258
|
export type TunnelsListResponse = {
|