@ora-ai/cli 0.7.4 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,10 +14,11 @@ export type AddonsListResponse = {
14
14
  "configSchema": Record<string, unknown>;
15
15
  "surfaces": Array<"run" | "session">;
16
16
  "harnesses": Array<string>;
17
+ "multiInstance"?: boolean;
17
18
  }>;
18
19
  };
19
20
  export type BenchmarksListResponse = {
20
- "benchmarks": Array<{
21
+ "studies": Array<{
21
22
  "id": string;
22
23
  "executiveSummary"?: string | null;
23
24
  "plannedRunCount": number;
@@ -30,7 +31,7 @@ export type BenchmarksListResponse = {
30
31
  "skillSlug": string | null;
31
32
  "skillName": string | null;
32
33
  "skillPromise": string;
33
- "subjectKind": "skill" | "tools" | "addon" | "custom";
34
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
34
35
  "subjectName": string | null;
35
36
  "subjectLabel": string;
36
37
  "armConfig": {
@@ -48,6 +49,9 @@ export type BenchmarksListResponse = {
48
49
  };
49
50
  };
50
51
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
52
+ "subjectBackend": "model" | "harness" | null;
53
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
54
+ "importedFrom"?: string;
51
55
  "baselineLabel": string | null;
52
56
  "treatmentLabel": string | null;
53
57
  "promiseMetrics": Array<{
@@ -59,26 +63,57 @@ export type BenchmarksListResponse = {
59
63
  "harness": string;
60
64
  "model": string;
61
65
  "provider"?: string;
66
+ "params"?: {
67
+ "temperature"?: number;
68
+ "maxOutputTokens"?: number;
69
+ "reasoningEffort"?: "low" | "medium" | "high";
70
+ "thinkingBudgetTokens"?: number;
71
+ };
62
72
  }>;
63
- "lenses": Array<"judge" | "evals">;
73
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
74
+ "judgeSupplementary"?: boolean;
64
75
  "taskCount": number;
65
76
  "variantsPerTask": number;
66
77
  "repeats": number;
78
+ "subset": {
79
+ "count": number;
80
+ "seed": string;
81
+ "of": number;
82
+ } | null;
67
83
  "status": string;
68
84
  "error": string | null;
69
- "visibility": "private" | "public";
85
+ "preparation"?: {
86
+ "since": string;
87
+ "launchedBy": {
88
+ "userId"?: string;
89
+ "role"?: string;
90
+ };
91
+ "waitingOn": Array<{
92
+ "imageKey": string;
93
+ "buildId": string;
94
+ "stage"?: "image" | "snapshot";
95
+ }>;
96
+ } | null;
97
+ "visibility": "private" | "org" | "link" | "public";
98
+ "visibilityChangedAt"?: string | null;
99
+ "visibilityChangedBy"?: string | null;
100
+ "shareToken"?: string | null;
70
101
  "lowSampleOverride": boolean;
71
102
  "publishedAt": string | null;
72
103
  "verdict": unknown;
104
+ "definitionVersionId": string | null;
105
+ "runnerKind": "ora-runs" | "batch-job";
73
106
  "createdAt": string;
74
107
  "updatedAt": string;
108
+ "liveUntil": string | null;
75
109
  }>;
76
110
  };
77
111
  export type BenchmarksCreateRequest = {
78
112
  "name": string;
79
113
  "description"?: string;
80
- "subjectKind"?: "skill" | "tools" | "addon" | "custom";
114
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
81
115
  "skillRef"?: string;
116
+ "definitionVersionId"?: string;
82
117
  "subjectName"?: string;
83
118
  "armConfig"?: {
84
119
  "baseline"?: {
@@ -94,9 +129,11 @@ export type BenchmarksCreateRequest = {
94
129
  "addons"?: Record<string, unknown>;
95
130
  };
96
131
  };
97
- "lenses"?: Array<"judge" | "evals">;
132
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
133
+ "judgeSupplementary"?: boolean;
98
134
  "proposeExpectations"?: boolean;
99
135
  "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
136
+ "subjectBackend"?: "model" | "harness";
100
137
  "baselineLabel"?: string;
101
138
  "treatmentLabel"?: string;
102
139
  "taskCount"?: number;
@@ -111,15 +148,25 @@ export type BenchmarksCreateRequest = {
111
148
  "variants"?: Array<string>;
112
149
  "expectations"?: Array<unknown>;
113
150
  }>;
151
+ "subset"?: {
152
+ "count": number;
153
+ "seed"?: string;
154
+ };
114
155
  "matrix": Array<{
115
156
  "harness": string;
116
157
  "model": string;
117
158
  "provider"?: string;
159
+ "params"?: {
160
+ "temperature"?: number;
161
+ "maxOutputTokens"?: number;
162
+ "reasoningEffort"?: "low" | "medium" | "high";
163
+ "thinkingBudgetTokens"?: number;
164
+ };
118
165
  }>;
119
166
  "promise"?: string;
120
167
  };
121
168
  export type BenchmarksCreateResponse = {
122
- "benchmark": {
169
+ "study": {
123
170
  "id": string;
124
171
  "executiveSummary"?: string | null;
125
172
  "plannedRunCount": number;
@@ -132,7 +179,7 @@ export type BenchmarksCreateResponse = {
132
179
  "skillSlug": string | null;
133
180
  "skillName": string | null;
134
181
  "skillPromise": string;
135
- "subjectKind": "skill" | "tools" | "addon" | "custom";
182
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
136
183
  "subjectName": string | null;
137
184
  "subjectLabel": string;
138
185
  "armConfig": {
@@ -150,6 +197,9 @@ export type BenchmarksCreateResponse = {
150
197
  };
151
198
  };
152
199
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
200
+ "subjectBackend": "model" | "harness" | null;
201
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
202
+ "importedFrom"?: string;
153
203
  "baselineLabel": string | null;
154
204
  "treatmentLabel": string | null;
155
205
  "promiseMetrics": Array<{
@@ -161,19 +211,49 @@ export type BenchmarksCreateResponse = {
161
211
  "harness": string;
162
212
  "model": string;
163
213
  "provider"?: string;
214
+ "params"?: {
215
+ "temperature"?: number;
216
+ "maxOutputTokens"?: number;
217
+ "reasoningEffort"?: "low" | "medium" | "high";
218
+ "thinkingBudgetTokens"?: number;
219
+ };
164
220
  }>;
165
- "lenses": Array<"judge" | "evals">;
221
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
222
+ "judgeSupplementary"?: boolean;
166
223
  "taskCount": number;
167
224
  "variantsPerTask": number;
168
225
  "repeats": number;
226
+ "subset": {
227
+ "count": number;
228
+ "seed": string;
229
+ "of": number;
230
+ } | null;
169
231
  "status": string;
170
232
  "error": string | null;
171
- "visibility": "private" | "public";
233
+ "preparation"?: {
234
+ "since": string;
235
+ "launchedBy": {
236
+ "userId"?: string;
237
+ "role"?: string;
238
+ };
239
+ "waitingOn": Array<{
240
+ "imageKey": string;
241
+ "buildId": string;
242
+ "stage"?: "image" | "snapshot";
243
+ }>;
244
+ } | null;
245
+ "visibility": "private" | "org" | "link" | "public";
246
+ "visibilityChangedAt"?: string | null;
247
+ "visibilityChangedBy"?: string | null;
248
+ "shareToken"?: string | null;
172
249
  "lowSampleOverride": boolean;
173
250
  "publishedAt": string | null;
174
251
  "verdict": unknown;
252
+ "definitionVersionId": string | null;
253
+ "runnerKind": "ora-runs" | "batch-job";
175
254
  "createdAt": string;
176
255
  "updatedAt": string;
256
+ "liveUntil": string | null;
177
257
  };
178
258
  "streamUrl": string;
179
259
  "warnings"?: Array<{
@@ -194,7 +274,7 @@ export type BenchmarksConvertWebmcpSuiteResponse = {
194
274
  "warnings": Array<string>;
195
275
  };
196
276
  export type BenchmarksGetResponse = {
197
- "benchmark": {
277
+ "study": {
198
278
  "id": string;
199
279
  "executiveSummary"?: string | null;
200
280
  "plannedRunCount": number;
@@ -207,7 +287,7 @@ export type BenchmarksGetResponse = {
207
287
  "skillSlug": string | null;
208
288
  "skillName": string | null;
209
289
  "skillPromise": string;
210
- "subjectKind": "skill" | "tools" | "addon" | "custom";
290
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
211
291
  "subjectName": string | null;
212
292
  "subjectLabel": string;
213
293
  "armConfig": {
@@ -225,6 +305,9 @@ export type BenchmarksGetResponse = {
225
305
  };
226
306
  };
227
307
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
308
+ "subjectBackend": "model" | "harness" | null;
309
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
310
+ "importedFrom"?: string;
228
311
  "baselineLabel": string | null;
229
312
  "treatmentLabel": string | null;
230
313
  "promiseMetrics": Array<{
@@ -236,23 +319,53 @@ export type BenchmarksGetResponse = {
236
319
  "harness": string;
237
320
  "model": string;
238
321
  "provider"?: string;
322
+ "params"?: {
323
+ "temperature"?: number;
324
+ "maxOutputTokens"?: number;
325
+ "reasoningEffort"?: "low" | "medium" | "high";
326
+ "thinkingBudgetTokens"?: number;
327
+ };
239
328
  }>;
240
- "lenses": Array<"judge" | "evals">;
329
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
330
+ "judgeSupplementary"?: boolean;
241
331
  "taskCount": number;
242
332
  "variantsPerTask": number;
243
333
  "repeats": number;
334
+ "subset": {
335
+ "count": number;
336
+ "seed": string;
337
+ "of": number;
338
+ } | null;
244
339
  "status": string;
245
340
  "error": string | null;
246
- "visibility": "private" | "public";
341
+ "preparation"?: {
342
+ "since": string;
343
+ "launchedBy": {
344
+ "userId"?: string;
345
+ "role"?: string;
346
+ };
347
+ "waitingOn": Array<{
348
+ "imageKey": string;
349
+ "buildId": string;
350
+ "stage"?: "image" | "snapshot";
351
+ }>;
352
+ } | null;
353
+ "visibility": "private" | "org" | "link" | "public";
354
+ "visibilityChangedAt"?: string | null;
355
+ "visibilityChangedBy"?: string | null;
356
+ "shareToken"?: string | null;
247
357
  "lowSampleOverride": boolean;
248
358
  "publishedAt": string | null;
249
359
  "verdict": unknown;
360
+ "definitionVersionId": string | null;
361
+ "runnerKind": "ora-runs" | "batch-job";
250
362
  "createdAt": string;
251
363
  "updatedAt": string;
364
+ "liveUntil": string | null;
252
365
  };
253
366
  "tasks": Array<{
254
367
  "id": string;
255
- "benchmarkId": string;
368
+ "studyId": string;
256
369
  "position": number;
257
370
  "prompt": string;
258
371
  "rationale": string | null;
@@ -312,6 +425,55 @@ export type BenchmarksGetResponse = {
312
425
  "evalResults": unknown | null;
313
426
  "evalScore": number | null;
314
427
  "evalGradedAt": string | null;
428
+ "lensGrades"?: {
429
+ "reference"?: {
430
+ "pass": boolean | null;
431
+ "extracted": string | null;
432
+ "extraction": string;
433
+ "matcher": string;
434
+ "votes"?: Array<{
435
+ "judge": string;
436
+ "answer": "equivalent" | "different" | "not_attempted" | null;
437
+ }>;
438
+ "reason"?: string;
439
+ "gradedAt": string;
440
+ };
441
+ "framework"?: {
442
+ "framework": string;
443
+ "primary": {
444
+ "metricId": string;
445
+ "provenance": string;
446
+ "value": number;
447
+ "scenario": string;
448
+ "count": number;
449
+ } | null;
450
+ "secondary": Array<{
451
+ "label": string;
452
+ "name": string;
453
+ "scenario": string;
454
+ "value": number | null;
455
+ "count": number;
456
+ "split": string | null;
457
+ }>;
458
+ "errors": Array<{
459
+ "scenario": string;
460
+ "metric": string;
461
+ "reason": string;
462
+ }>;
463
+ "gradedAt": string;
464
+ };
465
+ "execution"?: {
466
+ "pass": boolean | null;
467
+ "status": "passed" | "failed" | "timeout" | "error" | "unavailable";
468
+ "language": string;
469
+ "passedTests": number | null;
470
+ "failedTests": number | null;
471
+ "stdout": string;
472
+ "stdoutTruncated": boolean;
473
+ "reason"?: string;
474
+ "gradedAt": string;
475
+ };
476
+ } | null;
315
477
  "runCreatedAt": string | null;
316
478
  "startedAt": string | null;
317
479
  "finishedAt": string | null;
@@ -357,7 +519,7 @@ export type BenchmarksGetResponse = {
357
519
  };
358
520
  "decision"?: {
359
521
  "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
360
- "basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
522
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
361
523
  "reason": string;
362
524
  "efficiencyDiscounted": boolean;
363
525
  };
@@ -421,6 +583,7 @@ export type BenchmarksGetResponse = {
421
583
  "maxTotalTokens": number | null;
422
584
  "maxDurationMs": number | null;
423
585
  "evalPassRate"?: number | null;
586
+ "benchmarkPassRate"?: number | null;
424
587
  "adoptionRate"?: number | null;
425
588
  };
426
589
  "skill": {
@@ -452,6 +615,7 @@ export type BenchmarksGetResponse = {
452
615
  "maxTotalTokens": number | null;
453
616
  "maxDurationMs": number | null;
454
617
  "evalPassRate"?: number | null;
618
+ "benchmarkPassRate"?: number | null;
455
619
  "adoptionRate"?: number | null;
456
620
  };
457
621
  };
@@ -484,6 +648,7 @@ export type BenchmarksGetResponse = {
484
648
  "kind": "cell" | "harness" | "model" | "field";
485
649
  "harness": string | null;
486
650
  "model": string | null;
651
+ "paramsKey"?: string;
487
652
  "runs": number;
488
653
  "cells": Record<string, unknown>;
489
654
  }>;
@@ -505,6 +670,49 @@ export type BenchmarksGetResponse = {
505
670
  "succeededCount": number;
506
671
  }>;
507
672
  }>;
673
+ "reference"?: {
674
+ "cells": Array<{
675
+ "cell": string;
676
+ "harness": string;
677
+ "model": string;
678
+ "provider": string | null;
679
+ "paramsKey": string;
680
+ "label": string;
681
+ "passRate": {
682
+ "value": number | null;
683
+ "ci": Array<unknown> | null;
684
+ "n": number;
685
+ };
686
+ "tasks": number;
687
+ "graded": number;
688
+ "unmeasured": number;
689
+ "matchers": Array<string>;
690
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
691
+ }>;
692
+ "overall": {
693
+ "value": number | null;
694
+ "ci": Array<unknown> | null;
695
+ "n": number;
696
+ };
697
+ "graded": number;
698
+ "unmeasured": number;
699
+ "modelJudged": boolean;
700
+ };
701
+ "execution"?: {
702
+ "graded": number;
703
+ "measured": number;
704
+ "passed": number;
705
+ "passRate": number | null;
706
+ "byStatus": {
707
+ "passed": number;
708
+ "failed": number;
709
+ "timeout": number;
710
+ "error": number;
711
+ "unavailable": number;
712
+ };
713
+ "languages": Array<string>;
714
+ "unavailableReasons": Array<string>;
715
+ };
508
716
  };
509
717
  "warnings": Array<{
510
718
  "type": "eval_capability_gap";
@@ -515,6 +723,8 @@ export type BenchmarksGetResponse = {
515
723
  "detail": string;
516
724
  "taskIds"?: Array<string>;
517
725
  }>;
726
+ "shareUrl"?: string | null;
727
+ "canManageSharing"?: boolean;
518
728
  };
519
729
  export type BenchmarksDeleteResponse = unknown;
520
730
  export type BenchmarksRegenerateSummaryResponse = {
@@ -524,7 +734,7 @@ export type BenchmarksGenerateRequest = {
524
734
  "proposeExpectations"?: boolean;
525
735
  };
526
736
  export type BenchmarksGenerateResponse = {
527
- "benchmark": {
737
+ "study": {
528
738
  "id": string;
529
739
  "executiveSummary"?: string | null;
530
740
  "plannedRunCount": number;
@@ -537,7 +747,7 @@ export type BenchmarksGenerateResponse = {
537
747
  "skillSlug": string | null;
538
748
  "skillName": string | null;
539
749
  "skillPromise": string;
540
- "subjectKind": "skill" | "tools" | "addon" | "custom";
750
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
541
751
  "subjectName": string | null;
542
752
  "subjectLabel": string;
543
753
  "armConfig": {
@@ -555,6 +765,9 @@ export type BenchmarksGenerateResponse = {
555
765
  };
556
766
  };
557
767
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
768
+ "subjectBackend": "model" | "harness" | null;
769
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
770
+ "importedFrom"?: string;
558
771
  "baselineLabel": string | null;
559
772
  "treatmentLabel": string | null;
560
773
  "promiseMetrics": Array<{
@@ -566,19 +779,49 @@ export type BenchmarksGenerateResponse = {
566
779
  "harness": string;
567
780
  "model": string;
568
781
  "provider"?: string;
782
+ "params"?: {
783
+ "temperature"?: number;
784
+ "maxOutputTokens"?: number;
785
+ "reasoningEffort"?: "low" | "medium" | "high";
786
+ "thinkingBudgetTokens"?: number;
787
+ };
569
788
  }>;
570
- "lenses": Array<"judge" | "evals">;
789
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
790
+ "judgeSupplementary"?: boolean;
571
791
  "taskCount": number;
572
792
  "variantsPerTask": number;
573
793
  "repeats": number;
794
+ "subset": {
795
+ "count": number;
796
+ "seed": string;
797
+ "of": number;
798
+ } | null;
574
799
  "status": string;
575
800
  "error": string | null;
576
- "visibility": "private" | "public";
801
+ "preparation"?: {
802
+ "since": string;
803
+ "launchedBy": {
804
+ "userId"?: string;
805
+ "role"?: string;
806
+ };
807
+ "waitingOn": Array<{
808
+ "imageKey": string;
809
+ "buildId": string;
810
+ "stage"?: "image" | "snapshot";
811
+ }>;
812
+ } | null;
813
+ "visibility": "private" | "org" | "link" | "public";
814
+ "visibilityChangedAt"?: string | null;
815
+ "visibilityChangedBy"?: string | null;
816
+ "shareToken"?: string | null;
577
817
  "lowSampleOverride": boolean;
578
818
  "publishedAt": string | null;
579
819
  "verdict": unknown;
820
+ "definitionVersionId": string | null;
821
+ "runnerKind": "ora-runs" | "batch-job";
580
822
  "createdAt": string;
581
823
  "updatedAt": string;
824
+ "liveUntil": string | null;
582
825
  };
583
826
  "streamUrl": string;
584
827
  "warnings"?: Array<{
@@ -587,7 +830,7 @@ export type BenchmarksGenerateResponse = {
587
830
  }>;
588
831
  };
589
832
  export type BenchmarksLaunchResponse = {
590
- "benchmark": {
833
+ "study": {
591
834
  "id": string;
592
835
  "executiveSummary"?: string | null;
593
836
  "plannedRunCount": number;
@@ -600,7 +843,7 @@ export type BenchmarksLaunchResponse = {
600
843
  "skillSlug": string | null;
601
844
  "skillName": string | null;
602
845
  "skillPromise": string;
603
- "subjectKind": "skill" | "tools" | "addon" | "custom";
846
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
604
847
  "subjectName": string | null;
605
848
  "subjectLabel": string;
606
849
  "armConfig": {
@@ -618,6 +861,9 @@ export type BenchmarksLaunchResponse = {
618
861
  };
619
862
  };
620
863
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
864
+ "subjectBackend": "model" | "harness" | null;
865
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
866
+ "importedFrom"?: string;
621
867
  "baselineLabel": string | null;
622
868
  "treatmentLabel": string | null;
623
869
  "promiseMetrics": Array<{
@@ -629,19 +875,49 @@ export type BenchmarksLaunchResponse = {
629
875
  "harness": string;
630
876
  "model": string;
631
877
  "provider"?: string;
878
+ "params"?: {
879
+ "temperature"?: number;
880
+ "maxOutputTokens"?: number;
881
+ "reasoningEffort"?: "low" | "medium" | "high";
882
+ "thinkingBudgetTokens"?: number;
883
+ };
632
884
  }>;
633
- "lenses": Array<"judge" | "evals">;
885
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
886
+ "judgeSupplementary"?: boolean;
634
887
  "taskCount": number;
635
888
  "variantsPerTask": number;
636
889
  "repeats": number;
890
+ "subset": {
891
+ "count": number;
892
+ "seed": string;
893
+ "of": number;
894
+ } | null;
637
895
  "status": string;
638
896
  "error": string | null;
639
- "visibility": "private" | "public";
897
+ "preparation"?: {
898
+ "since": string;
899
+ "launchedBy": {
900
+ "userId"?: string;
901
+ "role"?: string;
902
+ };
903
+ "waitingOn": Array<{
904
+ "imageKey": string;
905
+ "buildId": string;
906
+ "stage"?: "image" | "snapshot";
907
+ }>;
908
+ } | null;
909
+ "visibility": "private" | "org" | "link" | "public";
910
+ "visibilityChangedAt"?: string | null;
911
+ "visibilityChangedBy"?: string | null;
912
+ "shareToken"?: string | null;
640
913
  "lowSampleOverride": boolean;
641
914
  "publishedAt": string | null;
642
915
  "verdict": unknown;
916
+ "definitionVersionId": string | null;
917
+ "runnerKind": "ora-runs" | "batch-job";
643
918
  "createdAt": string;
644
919
  "updatedAt": string;
920
+ "liveUntil": string | null;
645
921
  };
646
922
  "streamUrl": string;
647
923
  "warnings"?: Array<{
@@ -650,10 +926,10 @@ export type BenchmarksLaunchResponse = {
650
926
  }>;
651
927
  };
652
928
  export type BenchmarksUpdateLensesRequest = {
653
- "lenses": Array<"judge" | "evals">;
929
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
654
930
  };
655
931
  export type BenchmarksUpdateLensesResponse = {
656
- "benchmark": {
932
+ "study": {
657
933
  "id": string;
658
934
  "executiveSummary"?: string | null;
659
935
  "plannedRunCount": number;
@@ -666,7 +942,7 @@ export type BenchmarksUpdateLensesResponse = {
666
942
  "skillSlug": string | null;
667
943
  "skillName": string | null;
668
944
  "skillPromise": string;
669
- "subjectKind": "skill" | "tools" | "addon" | "custom";
945
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
670
946
  "subjectName": string | null;
671
947
  "subjectLabel": string;
672
948
  "armConfig": {
@@ -684,6 +960,9 @@ export type BenchmarksUpdateLensesResponse = {
684
960
  };
685
961
  };
686
962
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
963
+ "subjectBackend": "model" | "harness" | null;
964
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
965
+ "importedFrom"?: string;
687
966
  "baselineLabel": string | null;
688
967
  "treatmentLabel": string | null;
689
968
  "promiseMetrics": Array<{
@@ -695,19 +974,49 @@ export type BenchmarksUpdateLensesResponse = {
695
974
  "harness": string;
696
975
  "model": string;
697
976
  "provider"?: string;
977
+ "params"?: {
978
+ "temperature"?: number;
979
+ "maxOutputTokens"?: number;
980
+ "reasoningEffort"?: "low" | "medium" | "high";
981
+ "thinkingBudgetTokens"?: number;
982
+ };
698
983
  }>;
699
- "lenses": Array<"judge" | "evals">;
984
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
985
+ "judgeSupplementary"?: boolean;
700
986
  "taskCount": number;
701
987
  "variantsPerTask": number;
702
988
  "repeats": number;
989
+ "subset": {
990
+ "count": number;
991
+ "seed": string;
992
+ "of": number;
993
+ } | null;
703
994
  "status": string;
704
995
  "error": string | null;
705
- "visibility": "private" | "public";
996
+ "preparation"?: {
997
+ "since": string;
998
+ "launchedBy": {
999
+ "userId"?: string;
1000
+ "role"?: string;
1001
+ };
1002
+ "waitingOn": Array<{
1003
+ "imageKey": string;
1004
+ "buildId": string;
1005
+ "stage"?: "image" | "snapshot";
1006
+ }>;
1007
+ } | null;
1008
+ "visibility": "private" | "org" | "link" | "public";
1009
+ "visibilityChangedAt"?: string | null;
1010
+ "visibilityChangedBy"?: string | null;
1011
+ "shareToken"?: string | null;
706
1012
  "lowSampleOverride": boolean;
707
1013
  "publishedAt": string | null;
708
1014
  "verdict": unknown;
1015
+ "definitionVersionId": string | null;
1016
+ "runnerKind": "ora-runs" | "batch-job";
709
1017
  "createdAt": string;
710
1018
  "updatedAt": string;
1019
+ "liveUntil": string | null;
711
1020
  };
712
1021
  "warnings": Array<{
713
1022
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
@@ -716,7 +1025,7 @@ export type BenchmarksUpdateLensesResponse = {
716
1025
  }>;
717
1026
  };
718
1027
  export type BenchmarksPreviewResponse = {
719
- "benchmark": {
1028
+ "study": {
720
1029
  "executiveSummary"?: string | null;
721
1030
  "slug": string;
722
1031
  "name": string;
@@ -724,7 +1033,7 @@ export type BenchmarksPreviewResponse = {
724
1033
  "skillName": string | null;
725
1034
  "skillSlug": string | null;
726
1035
  "skillPromise": string;
727
- "subjectKind": "skill" | "tools" | "addon" | "custom";
1036
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
728
1037
  "subjectName": string | null;
729
1038
  "subjectLabel": string;
730
1039
  "armConfig": {
@@ -742,9 +1051,12 @@ export type BenchmarksPreviewResponse = {
742
1051
  };
743
1052
  };
744
1053
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
1054
+ "subjectBackend"?: "model" | "harness";
1055
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
745
1056
  "baselineLabel": string | null;
746
1057
  "treatmentLabel": string | null;
747
- "lenses": Array<"judge" | "evals">;
1058
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
1059
+ "judgeSupplementary"?: true;
748
1060
  "promiseMetrics": Array<{
749
1061
  "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
750
1062
  "direction": "down" | "up";
@@ -754,6 +1066,12 @@ export type BenchmarksPreviewResponse = {
754
1066
  "harness": string;
755
1067
  "model": string;
756
1068
  "provider"?: string;
1069
+ "params"?: {
1070
+ "temperature"?: number;
1071
+ "maxOutputTokens"?: number;
1072
+ "reasoningEffort"?: "low" | "medium" | "high";
1073
+ "thinkingBudgetTokens"?: number;
1074
+ };
757
1075
  }>;
758
1076
  "taskCount": number;
759
1077
  "variantsPerTask": number;
@@ -761,17 +1079,36 @@ export type BenchmarksPreviewResponse = {
761
1079
  "status": string;
762
1080
  "publishedAt": string | null;
763
1081
  "createdAt": string;
1082
+ "viewCount"?: number;
1083
+ "boardEligibility"?: {
1084
+ "eligible": boolean;
1085
+ "refusedBy": string | null;
1086
+ "reason": string | null;
1087
+ "failures": Array<{
1088
+ "ruleId": string;
1089
+ "reason": string;
1090
+ }>;
1091
+ "evaluatedAt": string;
1092
+ };
1093
+ "subjectLicenseSpdx"?: string | null;
1094
+ "subjectSourceUrl"?: string | null;
1095
+ "subjectWithheldReason"?: string | null;
1096
+ "subjectNotices"?: Array<{
1097
+ "kind": string;
1098
+ "text": string;
1099
+ }>;
764
1100
  };
765
1101
  "tasks": Array<{
766
1102
  "id": string;
767
1103
  "position": number;
768
- "prompt": string;
769
- "rationale": string | null;
1104
+ "prompt"?: string;
1105
+ "rationale"?: string | null;
770
1106
  "category": string | null;
771
1107
  "requiresWeb": boolean;
772
- "rubric": Array<string>;
1108
+ "rubric"?: Array<string>;
773
1109
  "variants"?: Array<string> | null;
774
1110
  "expectations"?: Array<unknown> | null;
1111
+ "textWithheld"?: boolean;
775
1112
  }>;
776
1113
  "runs": Array<{
777
1114
  "runId": string;
@@ -805,6 +1142,7 @@ export type BenchmarksPreviewResponse = {
805
1142
  "runCreatedAt": string | null;
806
1143
  "startedAt": string | null;
807
1144
  "finishedAt": string | null;
1145
+ "lensGrades"?: Record<string, unknown>;
808
1146
  "rubricPassed": Array<boolean> | null;
809
1147
  "evalOutcomes": Array<{
810
1148
  "kind": string;
@@ -813,6 +1151,8 @@ export type BenchmarksPreviewResponse = {
813
1151
  "abstained"?: boolean;
814
1152
  "decisive"?: boolean;
815
1153
  }> | null;
1154
+ "paramsKey"?: string;
1155
+ "textWithheld"?: boolean;
816
1156
  }>;
817
1157
  "aggregate": {
818
1158
  "summary": {
@@ -855,7 +1195,7 @@ export type BenchmarksPreviewResponse = {
855
1195
  };
856
1196
  "decision"?: {
857
1197
  "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
858
- "basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
1198
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
859
1199
  "reason": string;
860
1200
  "efficiencyDiscounted": boolean;
861
1201
  };
@@ -919,6 +1259,7 @@ export type BenchmarksPreviewResponse = {
919
1259
  "maxTotalTokens": number | null;
920
1260
  "maxDurationMs": number | null;
921
1261
  "evalPassRate"?: number | null;
1262
+ "benchmarkPassRate"?: number | null;
922
1263
  "adoptionRate"?: number | null;
923
1264
  };
924
1265
  "skill": {
@@ -950,6 +1291,7 @@ export type BenchmarksPreviewResponse = {
950
1291
  "maxTotalTokens": number | null;
951
1292
  "maxDurationMs": number | null;
952
1293
  "evalPassRate"?: number | null;
1294
+ "benchmarkPassRate"?: number | null;
953
1295
  "adoptionRate"?: number | null;
954
1296
  };
955
1297
  };
@@ -982,6 +1324,7 @@ export type BenchmarksPreviewResponse = {
982
1324
  "kind": "cell" | "harness" | "model" | "field";
983
1325
  "harness": string | null;
984
1326
  "model": string | null;
1327
+ "paramsKey"?: string;
985
1328
  "runs": number;
986
1329
  "cells": Record<string, unknown>;
987
1330
  }>;
@@ -1003,13 +1346,58 @@ export type BenchmarksPreviewResponse = {
1003
1346
  "succeededCount": number;
1004
1347
  }>;
1005
1348
  }>;
1349
+ "reference"?: {
1350
+ "cells": Array<{
1351
+ "cell": string;
1352
+ "harness": string;
1353
+ "model": string;
1354
+ "provider": string | null;
1355
+ "paramsKey": string;
1356
+ "label": string;
1357
+ "passRate": {
1358
+ "value": number | null;
1359
+ "ci": Array<unknown> | null;
1360
+ "n": number;
1361
+ };
1362
+ "tasks": number;
1363
+ "graded": number;
1364
+ "unmeasured": number;
1365
+ "matchers": Array<string>;
1366
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
1367
+ }>;
1368
+ "overall": {
1369
+ "value": number | null;
1370
+ "ci": Array<unknown> | null;
1371
+ "n": number;
1372
+ };
1373
+ "graded": number;
1374
+ "unmeasured": number;
1375
+ "modelJudged": boolean;
1376
+ };
1377
+ "execution"?: {
1378
+ "graded": number;
1379
+ "measured": number;
1380
+ "passed": number;
1381
+ "passRate": number | null;
1382
+ "byStatus": {
1383
+ "passed": number;
1384
+ "failed": number;
1385
+ "timeout": number;
1386
+ "error": number;
1387
+ "unavailable": number;
1388
+ };
1389
+ "languages": Array<string>;
1390
+ "unavailableReasons": Array<string>;
1391
+ };
1006
1392
  };
1393
+ "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
1007
1394
  };
1008
1395
  export type BenchmarksPublishRequest = {
1009
1396
  "slug"?: string;
1397
+ "visibility"?: "private" | "org" | "link" | "public";
1010
1398
  };
1011
1399
  export type BenchmarksPublishResponse = {
1012
- "benchmark": {
1400
+ "study": {
1013
1401
  "id": string;
1014
1402
  "executiveSummary"?: string | null;
1015
1403
  "plannedRunCount": number;
@@ -1022,7 +1410,7 @@ export type BenchmarksPublishResponse = {
1022
1410
  "skillSlug": string | null;
1023
1411
  "skillName": string | null;
1024
1412
  "skillPromise": string;
1025
- "subjectKind": "skill" | "tools" | "addon" | "custom";
1413
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
1026
1414
  "subjectName": string | null;
1027
1415
  "subjectLabel": string;
1028
1416
  "armConfig": {
@@ -1040,6 +1428,9 @@ export type BenchmarksPublishResponse = {
1040
1428
  };
1041
1429
  };
1042
1430
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
1431
+ "subjectBackend": "model" | "harness" | null;
1432
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
1433
+ "importedFrom"?: string;
1043
1434
  "baselineLabel": string | null;
1044
1435
  "treatmentLabel": string | null;
1045
1436
  "promiseMetrics": Array<{
@@ -1051,23 +1442,53 @@ export type BenchmarksPublishResponse = {
1051
1442
  "harness": string;
1052
1443
  "model": string;
1053
1444
  "provider"?: string;
1445
+ "params"?: {
1446
+ "temperature"?: number;
1447
+ "maxOutputTokens"?: number;
1448
+ "reasoningEffort"?: "low" | "medium" | "high";
1449
+ "thinkingBudgetTokens"?: number;
1450
+ };
1054
1451
  }>;
1055
- "lenses": Array<"judge" | "evals">;
1452
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
1453
+ "judgeSupplementary"?: boolean;
1056
1454
  "taskCount": number;
1057
1455
  "variantsPerTask": number;
1058
1456
  "repeats": number;
1457
+ "subset": {
1458
+ "count": number;
1459
+ "seed": string;
1460
+ "of": number;
1461
+ } | null;
1059
1462
  "status": string;
1060
1463
  "error": string | null;
1061
- "visibility": "private" | "public";
1464
+ "preparation"?: {
1465
+ "since": string;
1466
+ "launchedBy": {
1467
+ "userId"?: string;
1468
+ "role"?: string;
1469
+ };
1470
+ "waitingOn": Array<{
1471
+ "imageKey": string;
1472
+ "buildId": string;
1473
+ "stage"?: "image" | "snapshot";
1474
+ }>;
1475
+ } | null;
1476
+ "visibility": "private" | "org" | "link" | "public";
1477
+ "visibilityChangedAt"?: string | null;
1478
+ "visibilityChangedBy"?: string | null;
1479
+ "shareToken"?: string | null;
1062
1480
  "lowSampleOverride": boolean;
1063
1481
  "publishedAt": string | null;
1064
1482
  "verdict": unknown;
1483
+ "definitionVersionId": string | null;
1484
+ "runnerKind": "ora-runs" | "batch-job";
1065
1485
  "createdAt": string;
1066
1486
  "updatedAt": string;
1487
+ "liveUntil": string | null;
1067
1488
  };
1068
1489
  };
1069
1490
  export type BenchmarksRegradeResponse = {
1070
- "benchmark": {
1491
+ "study": {
1071
1492
  "id": string;
1072
1493
  "executiveSummary"?: string | null;
1073
1494
  "plannedRunCount": number;
@@ -1080,7 +1501,7 @@ export type BenchmarksRegradeResponse = {
1080
1501
  "skillSlug": string | null;
1081
1502
  "skillName": string | null;
1082
1503
  "skillPromise": string;
1083
- "subjectKind": "skill" | "tools" | "addon" | "custom";
1504
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
1084
1505
  "subjectName": string | null;
1085
1506
  "subjectLabel": string;
1086
1507
  "armConfig": {
@@ -1098,6 +1519,9 @@ export type BenchmarksRegradeResponse = {
1098
1519
  };
1099
1520
  };
1100
1521
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
1522
+ "subjectBackend": "model" | "harness" | null;
1523
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
1524
+ "importedFrom"?: string;
1101
1525
  "baselineLabel": string | null;
1102
1526
  "treatmentLabel": string | null;
1103
1527
  "promiseMetrics": Array<{
@@ -1109,19 +1533,49 @@ export type BenchmarksRegradeResponse = {
1109
1533
  "harness": string;
1110
1534
  "model": string;
1111
1535
  "provider"?: string;
1536
+ "params"?: {
1537
+ "temperature"?: number;
1538
+ "maxOutputTokens"?: number;
1539
+ "reasoningEffort"?: "low" | "medium" | "high";
1540
+ "thinkingBudgetTokens"?: number;
1541
+ };
1112
1542
  }>;
1113
- "lenses": Array<"judge" | "evals">;
1543
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
1544
+ "judgeSupplementary"?: boolean;
1114
1545
  "taskCount": number;
1115
1546
  "variantsPerTask": number;
1116
1547
  "repeats": number;
1548
+ "subset": {
1549
+ "count": number;
1550
+ "seed": string;
1551
+ "of": number;
1552
+ } | null;
1117
1553
  "status": string;
1118
1554
  "error": string | null;
1119
- "visibility": "private" | "public";
1555
+ "preparation"?: {
1556
+ "since": string;
1557
+ "launchedBy": {
1558
+ "userId"?: string;
1559
+ "role"?: string;
1560
+ };
1561
+ "waitingOn": Array<{
1562
+ "imageKey": string;
1563
+ "buildId": string;
1564
+ "stage"?: "image" | "snapshot";
1565
+ }>;
1566
+ } | null;
1567
+ "visibility": "private" | "org" | "link" | "public";
1568
+ "visibilityChangedAt"?: string | null;
1569
+ "visibilityChangedBy"?: string | null;
1570
+ "shareToken"?: string | null;
1120
1571
  "lowSampleOverride": boolean;
1121
1572
  "publishedAt": string | null;
1122
1573
  "verdict": unknown;
1574
+ "definitionVersionId": string | null;
1575
+ "runnerKind": "ora-runs" | "batch-job";
1123
1576
  "createdAt": string;
1124
1577
  "updatedAt": string;
1578
+ "liveUntil": string | null;
1125
1579
  };
1126
1580
  "streamUrl": string;
1127
1581
  "warnings"?: Array<{
@@ -1137,7 +1591,7 @@ export type BenchmarksUpdateTaskRequest = {
1137
1591
  export type BenchmarksUpdateTaskResponse = {
1138
1592
  "task": {
1139
1593
  "id": string;
1140
- "benchmarkId": string;
1594
+ "studyId": string;
1141
1595
  "position": number;
1142
1596
  "prompt": string;
1143
1597
  "rationale": string | null;
@@ -1149,7 +1603,7 @@ export type BenchmarksUpdateTaskResponse = {
1149
1603
  };
1150
1604
  };
1151
1605
  export type BenchmarksUnpublishResponse = {
1152
- "benchmark": {
1606
+ "study": {
1153
1607
  "id": string;
1154
1608
  "executiveSummary"?: string | null;
1155
1609
  "plannedRunCount": number;
@@ -1162,7 +1616,7 @@ export type BenchmarksUnpublishResponse = {
1162
1616
  "skillSlug": string | null;
1163
1617
  "skillName": string | null;
1164
1618
  "skillPromise": string;
1165
- "subjectKind": "skill" | "tools" | "addon" | "custom";
1619
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
1166
1620
  "subjectName": string | null;
1167
1621
  "subjectLabel": string;
1168
1622
  "armConfig": {
@@ -1180,6 +1634,9 @@ export type BenchmarksUnpublishResponse = {
1180
1634
  };
1181
1635
  };
1182
1636
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
1637
+ "subjectBackend": "model" | "harness" | null;
1638
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
1639
+ "importedFrom"?: string;
1183
1640
  "baselineLabel": string | null;
1184
1641
  "treatmentLabel": string | null;
1185
1642
  "promiseMetrics": Array<{
@@ -1191,19 +1648,49 @@ export type BenchmarksUnpublishResponse = {
1191
1648
  "harness": string;
1192
1649
  "model": string;
1193
1650
  "provider"?: string;
1651
+ "params"?: {
1652
+ "temperature"?: number;
1653
+ "maxOutputTokens"?: number;
1654
+ "reasoningEffort"?: "low" | "medium" | "high";
1655
+ "thinkingBudgetTokens"?: number;
1656
+ };
1194
1657
  }>;
1195
- "lenses": Array<"judge" | "evals">;
1658
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
1659
+ "judgeSupplementary"?: boolean;
1196
1660
  "taskCount": number;
1197
1661
  "variantsPerTask": number;
1198
1662
  "repeats": number;
1663
+ "subset": {
1664
+ "count": number;
1665
+ "seed": string;
1666
+ "of": number;
1667
+ } | null;
1199
1668
  "status": string;
1200
1669
  "error": string | null;
1201
- "visibility": "private" | "public";
1670
+ "preparation"?: {
1671
+ "since": string;
1672
+ "launchedBy": {
1673
+ "userId"?: string;
1674
+ "role"?: string;
1675
+ };
1676
+ "waitingOn": Array<{
1677
+ "imageKey": string;
1678
+ "buildId": string;
1679
+ "stage"?: "image" | "snapshot";
1680
+ }>;
1681
+ } | null;
1682
+ "visibility": "private" | "org" | "link" | "public";
1683
+ "visibilityChangedAt"?: string | null;
1684
+ "visibilityChangedBy"?: string | null;
1685
+ "shareToken"?: string | null;
1202
1686
  "lowSampleOverride": boolean;
1203
1687
  "publishedAt": string | null;
1204
1688
  "verdict": unknown;
1689
+ "definitionVersionId": string | null;
1690
+ "runnerKind": "ora-runs" | "batch-job";
1205
1691
  "createdAt": string;
1206
1692
  "updatedAt": string;
1693
+ "liveUntil": string | null;
1207
1694
  };
1208
1695
  };
1209
1696
  export type ExperimentCatalogResponse = {
@@ -1227,6 +1714,130 @@ export type ExperimentCatalogResponse = {
1227
1714
  }>;
1228
1715
  }>;
1229
1716
  };
1717
+ export type ExperimentImagesBuildsListResponse = {
1718
+ "builds": Array<{
1719
+ "id": string;
1720
+ "organizationId": string;
1721
+ "productKey": string;
1722
+ "requestedByUserId": string | null;
1723
+ "name": string;
1724
+ "source": {
1725
+ "kind": "archive";
1726
+ "sha256": string;
1727
+ } | {
1728
+ "kind": "upload";
1729
+ "fileId": string;
1730
+ } | {
1731
+ "kind": "git";
1732
+ "ref": string;
1733
+ "subdir"?: string;
1734
+ } | {
1735
+ "kind": "inline";
1736
+ "fileCount": number;
1737
+ };
1738
+ "sourceIdentity": string;
1739
+ "dockerfileDir": string;
1740
+ "dockerfileName": string;
1741
+ "buildArgKeys": Array<string>;
1742
+ "platform": string;
1743
+ "trusted": boolean;
1744
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
1745
+ "image": string;
1746
+ "digest": string | null;
1747
+ "sizeBytes": number | null;
1748
+ "reason": string | null;
1749
+ "logRef": string | null;
1750
+ "startedAt": string | null;
1751
+ "finishedAt": string | null;
1752
+ "createdAt": string;
1753
+ "updatedAt": string;
1754
+ }>;
1755
+ "nextPageToken"?: string;
1756
+ };
1757
+ export type ExperimentImagesBuildsCreateRequest = {
1758
+ "name": string;
1759
+ "source": {
1760
+ "kind": "archive";
1761
+ "url": string;
1762
+ "sha256": string;
1763
+ } | {
1764
+ "kind": "upload";
1765
+ "fileId": string;
1766
+ } | {
1767
+ "kind": "git";
1768
+ "url": string;
1769
+ "ref": string;
1770
+ "subdir"?: string;
1771
+ } | {
1772
+ "kind": "inline";
1773
+ "dockerfile": string;
1774
+ "files"?: Record<string, unknown>;
1775
+ };
1776
+ "dockerfile"?: {
1777
+ "dir"?: string;
1778
+ "name"?: string;
1779
+ };
1780
+ "buildArgs"?: Record<string, unknown>;
1781
+ "platform"?: string;
1782
+ };
1783
+ export type ExperimentImagesBuildsCreateResponse = {
1784
+ "id": string;
1785
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
1786
+ };
1787
+ export type ExperimentImagesBuildsGetResponse = {
1788
+ "build": {
1789
+ "id": string;
1790
+ "organizationId": string;
1791
+ "productKey": string;
1792
+ "requestedByUserId": string | null;
1793
+ "name": string;
1794
+ "source": {
1795
+ "kind": "archive";
1796
+ "sha256": string;
1797
+ } | {
1798
+ "kind": "upload";
1799
+ "fileId": string;
1800
+ } | {
1801
+ "kind": "git";
1802
+ "ref": string;
1803
+ "subdir"?: string;
1804
+ } | {
1805
+ "kind": "inline";
1806
+ "fileCount": number;
1807
+ };
1808
+ "sourceIdentity": string;
1809
+ "dockerfileDir": string;
1810
+ "dockerfileName": string;
1811
+ "buildArgKeys": Array<string>;
1812
+ "platform": string;
1813
+ "trusted": boolean;
1814
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
1815
+ "image": string;
1816
+ "digest": string | null;
1817
+ "sizeBytes": number | null;
1818
+ "reason": string | null;
1819
+ "logRef": string | null;
1820
+ "startedAt": string | null;
1821
+ "finishedAt": string | null;
1822
+ "createdAt": string;
1823
+ "updatedAt": string;
1824
+ };
1825
+ };
1826
+ export type ExperimentImagesBuildsCancelResponse = {
1827
+ "id": string;
1828
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
1829
+ };
1830
+ export type ExperimentImagesBuildsStreamResponse = unknown;
1831
+ export type ExperimentImagesRunsCreateRequest = {
1832
+ "buildId"?: string;
1833
+ "image"?: string;
1834
+ "mode": "harness" | "environment";
1835
+ "harness"?: string;
1836
+ "task"?: string;
1837
+ };
1838
+ export type ExperimentImagesRunsCreateResponse = {
1839
+ "runId": string;
1840
+ };
1230
1841
  export type ExperimentRunsListResponse = {
1231
1842
  "results": Array<{
1232
1843
  "id": string;
@@ -1276,6 +1887,12 @@ export type ExperimentRunsCreateRequest = {
1276
1887
  "url": string;
1277
1888
  "secretId"?: string;
1278
1889
  }>;
1890
+ "model_params"?: {
1891
+ "temperature"?: number;
1892
+ "maxOutputTokens"?: number;
1893
+ "reasoningEffort"?: "low" | "medium" | "high";
1894
+ "thinkingBudgetTokens"?: number;
1895
+ };
1279
1896
  "wait"?: boolean;
1280
1897
  };
1281
1898
  export type ExperimentRunsCreateResponse = {
@@ -1454,7 +2071,7 @@ export type SessionsWakeResponse = {
1454
2071
  "updatedAt": string;
1455
2072
  };
1456
2073
  };
1457
- export type SkillsListResponse = {
2074
+ export type ExperimentSkillsListResponse = {
1458
2075
  "skills": Array<{
1459
2076
  "ref": string;
1460
2077
  "source": "builtin" | "custom" | "git";
@@ -1472,7 +2089,7 @@ export type SkillsListResponse = {
1472
2089
  "updatedAt"?: string;
1473
2090
  }>;
1474
2091
  };
1475
- export type SkillsCreateRequest = {
2092
+ export type ExperimentSkillsCreateRequest = {
1476
2093
  "source": "custom";
1477
2094
  "name": string;
1478
2095
  "body": string;
@@ -1485,7 +2102,7 @@ export type SkillsCreateRequest = {
1485
2102
  "subpath"?: string | null;
1486
2103
  "visibility"?: "user" | "org";
1487
2104
  };
1488
- export type SkillsCreateResponse = {
2105
+ export type ExperimentSkillsCreateResponse = {
1489
2106
  "skill": {
1490
2107
  "ref": string;
1491
2108
  "source": "builtin" | "custom" | "git";
@@ -1503,7 +2120,7 @@ export type SkillsCreateResponse = {
1503
2120
  "updatedAt"?: string;
1504
2121
  };
1505
2122
  };
1506
- export type SkillsUpdateRequest = {
2123
+ export type ExperimentSkillsUpdateRequest = {
1507
2124
  "name"?: string;
1508
2125
  "visibility"?: "user" | "org";
1509
2126
  "body"?: string;
@@ -1511,7 +2128,7 @@ export type SkillsUpdateRequest = {
1511
2128
  "gitRef"?: string;
1512
2129
  "subpath"?: string | null;
1513
2130
  };
1514
- export type SkillsUpdateResponse = {
2131
+ export type ExperimentSkillsUpdateResponse = {
1515
2132
  "skill": {
1516
2133
  "ref": string;
1517
2134
  "source": "builtin" | "custom" | "git";
@@ -1529,7 +2146,7 @@ export type SkillsUpdateResponse = {
1529
2146
  "updatedAt"?: string;
1530
2147
  };
1531
2148
  };
1532
- export type SkillsDeleteResponse = unknown;
2149
+ export type ExperimentSkillsDeleteResponse = unknown;
1533
2150
  export type WebmcpArenaCreateRequest = {
1534
2151
  "targetUrl": string;
1535
2152
  "intent": string;
@@ -1541,7 +2158,7 @@ export type WebmcpArenaCreateRequest = {
1541
2158
  "name"?: string;
1542
2159
  };
1543
2160
  export type WebmcpArenaCreateResponse = {
1544
- "benchmark": {
2161
+ "study": {
1545
2162
  "id": string;
1546
2163
  "executiveSummary"?: string | null;
1547
2164
  "plannedRunCount": number;
@@ -1554,7 +2171,7 @@ export type WebmcpArenaCreateResponse = {
1554
2171
  "skillSlug": string | null;
1555
2172
  "skillName": string | null;
1556
2173
  "skillPromise": string;
1557
- "subjectKind": "skill" | "tools" | "addon" | "custom";
2174
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
1558
2175
  "subjectName": string | null;
1559
2176
  "subjectLabel": string;
1560
2177
  "armConfig": {
@@ -1572,6 +2189,9 @@ export type WebmcpArenaCreateResponse = {
1572
2189
  };
1573
2190
  };
1574
2191
  "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
2192
+ "subjectBackend": "model" | "harness" | null;
2193
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
2194
+ "importedFrom"?: string;
1575
2195
  "baselineLabel": string | null;
1576
2196
  "treatmentLabel": string | null;
1577
2197
  "promiseMetrics": Array<{
@@ -1583,19 +2203,49 @@ export type WebmcpArenaCreateResponse = {
1583
2203
  "harness": string;
1584
2204
  "model": string;
1585
2205
  "provider"?: string;
2206
+ "params"?: {
2207
+ "temperature"?: number;
2208
+ "maxOutputTokens"?: number;
2209
+ "reasoningEffort"?: "low" | "medium" | "high";
2210
+ "thinkingBudgetTokens"?: number;
2211
+ };
1586
2212
  }>;
1587
- "lenses": Array<"judge" | "evals">;
2213
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
2214
+ "judgeSupplementary"?: boolean;
1588
2215
  "taskCount": number;
1589
2216
  "variantsPerTask": number;
1590
2217
  "repeats": number;
2218
+ "subset": {
2219
+ "count": number;
2220
+ "seed": string;
2221
+ "of": number;
2222
+ } | null;
1591
2223
  "status": string;
1592
2224
  "error": string | null;
1593
- "visibility": "private" | "public";
2225
+ "preparation"?: {
2226
+ "since": string;
2227
+ "launchedBy": {
2228
+ "userId"?: string;
2229
+ "role"?: string;
2230
+ };
2231
+ "waitingOn": Array<{
2232
+ "imageKey": string;
2233
+ "buildId": string;
2234
+ "stage"?: "image" | "snapshot";
2235
+ }>;
2236
+ } | null;
2237
+ "visibility": "private" | "org" | "link" | "public";
2238
+ "visibilityChangedAt"?: string | null;
2239
+ "visibilityChangedBy"?: string | null;
2240
+ "shareToken"?: string | null;
1594
2241
  "lowSampleOverride": boolean;
1595
2242
  "publishedAt": string | null;
1596
2243
  "verdict": unknown;
2244
+ "definitionVersionId": string | null;
2245
+ "runnerKind": "ora-runs" | "batch-job";
1597
2246
  "createdAt": string;
1598
2247
  "updatedAt": string;
2248
+ "liveUntil": string | null;
1599
2249
  };
1600
2250
  "streamUrl": string;
1601
2251
  "warnings"?: Array<{
@@ -1603,6 +2253,20 @@ export type WebmcpArenaCreateResponse = {
1603
2253
  "kind": string;
1604
2254
  }>;
1605
2255
  };
2256
+ export type WebmcpRunsBrowserRecordingResponse = {
2257
+ "runId": string;
2258
+ "startedAt": string | null;
2259
+ "durationMs": number;
2260
+ "frames": Array<{
2261
+ "t": number;
2262
+ "data": string;
2263
+ "url"?: string;
2264
+ }>;
2265
+ "truncated"?: boolean;
2266
+ "decimated"?: boolean;
2267
+ "totalFrames"?: number;
2268
+ };
2269
+ export type WebmcpRunsBrowserStreamResponse = unknown;
1606
2270
  export type ActivityListResponse = {
1607
2271
  "success": true;
1608
2272
  "data": Array<{
@@ -1778,8 +2442,28 @@ export type IntentsListResponse = {
1778
2442
  };
1779
2443
  "lowSample": boolean;
1780
2444
  "measurementPolicy": {
1781
- "cadence": "daily" | "weekly" | "manual";
2445
+ "cadence": "daily" | "weekly" | "monthly" | "manual";
1782
2446
  "mcp_server_ids": Array<string>;
2447
+ "agents": Array<{
2448
+ "harness": string;
2449
+ "model": string;
2450
+ "mcp_server_ids": Array<string>;
2451
+ "skill_ids": Array<string>;
2452
+ "browser"?: {
2453
+ "webdriver"?: boolean;
2454
+ "webmcp"?: boolean;
2455
+ "webmcpTargetUrl"?: string;
2456
+ "webmcpSurface"?: "native" | "shim";
2457
+ };
2458
+ }>;
2459
+ "skill_ids": Array<string>;
2460
+ "browser"?: {
2461
+ "webdriver"?: boolean;
2462
+ "webmcp"?: boolean;
2463
+ "webmcpTargetUrl"?: string;
2464
+ "webmcpSurface"?: "native" | "shim";
2465
+ };
2466
+ "competitors": Array<string>;
1783
2467
  };
1784
2468
  }>;
1785
2469
  };
@@ -2212,6 +2896,12 @@ export type JourneysCreateRequest = {
2212
2896
  "domains"?: Array<string>;
2213
2897
  "competitors"?: Array<string>;
2214
2898
  "skillIds"?: Array<string>;
2899
+ "browser"?: {
2900
+ "webdriver"?: boolean;
2901
+ "webmcp"?: boolean;
2902
+ "webmcpTargetUrl"?: string;
2903
+ "webmcpSurface"?: "native" | "shim";
2904
+ };
2215
2905
  "schedule"?: unknown;
2216
2906
  };
2217
2907
  export type JourneysCreateResponse = {
@@ -2378,6 +3068,12 @@ export type StudioRunsCreateRequest = {
2378
3068
  "url": string;
2379
3069
  "secretId"?: string;
2380
3070
  }>;
3071
+ "model_params"?: {
3072
+ "temperature"?: number;
3073
+ "maxOutputTokens"?: number;
3074
+ "reasoningEffort"?: "low" | "medium" | "high";
3075
+ "thinkingBudgetTokens"?: number;
3076
+ };
2381
3077
  "wait"?: boolean;
2382
3078
  };
2383
3079
  export type StudioRunsCreateResponse = {
@@ -2438,6 +3134,3122 @@ export type StudioRunsRawTraceResponse = unknown;
2438
3134
  export type StudioRunsSpansResponse = unknown;
2439
3135
  export type StudioRunsStreamResponse = unknown;
2440
3136
  export type StudioRunsTrajectoryResponse = unknown;
3137
+ export type StudyAddonsListResponse = {
3138
+ "addons": Array<{
3139
+ "id": string;
3140
+ "kind": "capability" | "tool" | "skill";
3141
+ "category"?: string;
3142
+ "displayName": string;
3143
+ "description": string;
3144
+ "configSchema": Record<string, unknown>;
3145
+ "surfaces": Array<"run" | "session">;
3146
+ "harnesses": Array<string>;
3147
+ "multiInstance"?: boolean;
3148
+ }>;
3149
+ };
3150
+ export type StudyBenchmarkDefinitionsListResponse = {
3151
+ "definitions": Array<{
3152
+ "id": string;
3153
+ "slug": string;
3154
+ "name": string;
3155
+ "description": string | null;
3156
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3157
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3158
+ "headlineMetric": string;
3159
+ "methodology": {
3160
+ "variantsAllowed"?: boolean;
3161
+ "recommendedRepeats"?: number;
3162
+ "officialFidelity"?: boolean;
3163
+ "modelParams"?: Array<string>;
3164
+ "maxPlannedRuns"?: number;
3165
+ "sampleFloor"?: number;
3166
+ };
3167
+ "defaultConfig": {
3168
+ "matrix"?: Array<{
3169
+ "harness": string;
3170
+ "model": string;
3171
+ "provider"?: string;
3172
+ "params"?: Record<string, unknown>;
3173
+ }>;
3174
+ "skills"?: Array<string>;
3175
+ "tools"?: "default" | "none" | {
3176
+ "tools": Array<string>;
3177
+ };
3178
+ "addons"?: Record<string, unknown>;
3179
+ "params"?: Record<string, unknown>;
3180
+ } | null;
3181
+ "runnerKind": "none" | "image" | "environment" | "framework";
3182
+ "license": string | null;
3183
+ "licenseText": string | null;
3184
+ "licenseVerified": boolean;
3185
+ "licenseVerifiedByPolicy": boolean;
3186
+ "redistribution": "allowed" | "reference_only";
3187
+ "disclosure": "open" | "guarded";
3188
+ "disclosureReason": string | null;
3189
+ "access": "open" | "gated" | "private";
3190
+ "termsUrl": string | null;
3191
+ "homepageUrl": string | null;
3192
+ "paperUrl": string | null;
3193
+ "publishedAt": string | null;
3194
+ "currentVersion": number;
3195
+ "currentVersionId": string | null;
3196
+ "currentVersionTaskCount": number | null;
3197
+ "lenses": Array<string>;
3198
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3199
+ "runnability": {
3200
+ "kind": "runnable";
3201
+ } | {
3202
+ "kind": "corpus-missing";
3203
+ "reason": string;
3204
+ } | {
3205
+ "kind": "needs-import";
3206
+ "reason": string;
3207
+ } | {
3208
+ "kind": "needs-token";
3209
+ "reason": string;
3210
+ } | {
3211
+ "kind": "runner-unsupported";
3212
+ "reason": string;
3213
+ };
3214
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3215
+ "taskTextAvailable": boolean;
3216
+ "hubTier": "featured" | "community" | "mine";
3217
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3218
+ "listed": boolean;
3219
+ "visibility": "private" | "org" | "public";
3220
+ "attestation": {
3221
+ "licenseSpdx": string;
3222
+ "redistribution": "allowed" | "reference_only";
3223
+ "sourceUrl": string;
3224
+ "statement": string;
3225
+ "attestedAt": string;
3226
+ } | null;
3227
+ "rejectionReason": string | null;
3228
+ "uploadedBy": {
3229
+ "userId": string;
3230
+ "displayName": string | null;
3231
+ } | null;
3232
+ "notices": Array<{
3233
+ "kind": string;
3234
+ "text": string;
3235
+ }>;
3236
+ }>;
3237
+ "facets": {
3238
+ "families": Array<"static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite">;
3239
+ "sourceKinds": Array<"hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework">;
3240
+ "hubTiers": Array<"featured" | "community" | "mine">;
3241
+ };
3242
+ };
3243
+ export type StudyBenchmarkDefinitionsCreateRequest = {
3244
+ "slug": string;
3245
+ "name": string;
3246
+ "description"?: string | null;
3247
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3248
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3249
+ "sourceConfig"?: Record<string, unknown>;
3250
+ "adapterId"?: string | null;
3251
+ "headlineMetric": string;
3252
+ "gradingSpec"?: {
3253
+ "lenses": Array<{
3254
+ "id": string;
3255
+ "config"?: Record<string, unknown>;
3256
+ }>;
3257
+ "coverage"?: "every-task" | "per-task";
3258
+ };
3259
+ "environmentSpec"?: {
3260
+ "kind": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3261
+ };
3262
+ "methodology"?: {
3263
+ "variantsAllowed"?: boolean;
3264
+ "recommendedRepeats"?: number;
3265
+ "officialFidelity"?: boolean;
3266
+ "modelParams"?: Array<string>;
3267
+ "maxPlannedRuns"?: number;
3268
+ "sampleFloor"?: number;
3269
+ };
3270
+ "runnerKind"?: "none" | "image" | "environment" | "framework";
3271
+ "runnerRecipe"?: Record<string, unknown> | null;
3272
+ "license"?: "MIT" | "Apache-2.0" | "BSD-2-Clause" | "BSD-3-Clause" | "CC-BY-4.0" | "CC0-1.0" | "CC-BY-SA-4.0" | "CC-BY-NC-4.0" | "CC-BY-NC-SA-4.0" | "GPL-3.0" | "AGPL-3.0" | "MPL-2.0" | null;
3273
+ "licenseText"?: string | null;
3274
+ "redistribution"?: "allowed" | "reference_only";
3275
+ "access"?: "open" | "gated" | "private";
3276
+ "termsUrl"?: string | null;
3277
+ "homepageUrl"?: string | null;
3278
+ "paperUrl"?: string | null;
3279
+ "defaultConfig"?: {
3280
+ "matrix"?: Array<{
3281
+ "harness": string;
3282
+ "model": string;
3283
+ "provider"?: string;
3284
+ "params"?: Record<string, unknown>;
3285
+ }>;
3286
+ "skills"?: Array<string>;
3287
+ "tools"?: "default" | "none" | {
3288
+ "tools": Array<string>;
3289
+ };
3290
+ "addons"?: Record<string, unknown>;
3291
+ "params"?: Record<string, unknown>;
3292
+ } | null;
3293
+ "visibility"?: "private" | "org" | "public";
3294
+ };
3295
+ export type StudyBenchmarkDefinitionsCreateResponse = {
3296
+ "definition": {
3297
+ "id": string;
3298
+ "slug": string;
3299
+ "name": string;
3300
+ "description": string | null;
3301
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3302
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3303
+ "headlineMetric": string;
3304
+ "methodology": {
3305
+ "variantsAllowed"?: boolean;
3306
+ "recommendedRepeats"?: number;
3307
+ "officialFidelity"?: boolean;
3308
+ "modelParams"?: Array<string>;
3309
+ "maxPlannedRuns"?: number;
3310
+ "sampleFloor"?: number;
3311
+ };
3312
+ "defaultConfig": {
3313
+ "matrix"?: Array<{
3314
+ "harness": string;
3315
+ "model": string;
3316
+ "provider"?: string;
3317
+ "params"?: Record<string, unknown>;
3318
+ }>;
3319
+ "skills"?: Array<string>;
3320
+ "tools"?: "default" | "none" | {
3321
+ "tools": Array<string>;
3322
+ };
3323
+ "addons"?: Record<string, unknown>;
3324
+ "params"?: Record<string, unknown>;
3325
+ } | null;
3326
+ "runnerKind": "none" | "image" | "environment" | "framework";
3327
+ "license": string | null;
3328
+ "licenseText": string | null;
3329
+ "licenseVerified": boolean;
3330
+ "licenseVerifiedByPolicy": boolean;
3331
+ "redistribution": "allowed" | "reference_only";
3332
+ "disclosure": "open" | "guarded";
3333
+ "disclosureReason": string | null;
3334
+ "access": "open" | "gated" | "private";
3335
+ "termsUrl": string | null;
3336
+ "homepageUrl": string | null;
3337
+ "paperUrl": string | null;
3338
+ "publishedAt": string | null;
3339
+ "currentVersion": number;
3340
+ "currentVersionId": string | null;
3341
+ "currentVersionTaskCount": number | null;
3342
+ "lenses": Array<string>;
3343
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3344
+ "runnability": {
3345
+ "kind": "runnable";
3346
+ } | {
3347
+ "kind": "corpus-missing";
3348
+ "reason": string;
3349
+ } | {
3350
+ "kind": "needs-import";
3351
+ "reason": string;
3352
+ } | {
3353
+ "kind": "needs-token";
3354
+ "reason": string;
3355
+ } | {
3356
+ "kind": "runner-unsupported";
3357
+ "reason": string;
3358
+ };
3359
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3360
+ "taskTextAvailable": boolean;
3361
+ "hubTier": "featured" | "community" | "mine";
3362
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3363
+ "listed": boolean;
3364
+ "visibility": "private" | "org" | "public";
3365
+ "attestation": {
3366
+ "licenseSpdx": string;
3367
+ "redistribution": "allowed" | "reference_only";
3368
+ "sourceUrl": string;
3369
+ "statement": string;
3370
+ "attestedAt": string;
3371
+ } | null;
3372
+ "rejectionReason": string | null;
3373
+ "uploadedBy": {
3374
+ "userId": string;
3375
+ "displayName": string | null;
3376
+ } | null;
3377
+ "notices": Array<{
3378
+ "kind": string;
3379
+ "text": string;
3380
+ }>;
3381
+ };
3382
+ };
3383
+ export type StudyBenchmarkDefinitionsGetResponse = {
3384
+ "definition": {
3385
+ "id": string;
3386
+ "slug": string;
3387
+ "name": string;
3388
+ "description": string | null;
3389
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3390
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3391
+ "headlineMetric": string;
3392
+ "methodology": {
3393
+ "variantsAllowed"?: boolean;
3394
+ "recommendedRepeats"?: number;
3395
+ "officialFidelity"?: boolean;
3396
+ "modelParams"?: Array<string>;
3397
+ "maxPlannedRuns"?: number;
3398
+ "sampleFloor"?: number;
3399
+ };
3400
+ "defaultConfig": {
3401
+ "matrix"?: Array<{
3402
+ "harness": string;
3403
+ "model": string;
3404
+ "provider"?: string;
3405
+ "params"?: Record<string, unknown>;
3406
+ }>;
3407
+ "skills"?: Array<string>;
3408
+ "tools"?: "default" | "none" | {
3409
+ "tools": Array<string>;
3410
+ };
3411
+ "addons"?: Record<string, unknown>;
3412
+ "params"?: Record<string, unknown>;
3413
+ } | null;
3414
+ "runnerKind": "none" | "image" | "environment" | "framework";
3415
+ "license": string | null;
3416
+ "licenseText": string | null;
3417
+ "licenseVerified": boolean;
3418
+ "licenseVerifiedByPolicy": boolean;
3419
+ "redistribution": "allowed" | "reference_only";
3420
+ "disclosure": "open" | "guarded";
3421
+ "disclosureReason": string | null;
3422
+ "access": "open" | "gated" | "private";
3423
+ "termsUrl": string | null;
3424
+ "homepageUrl": string | null;
3425
+ "paperUrl": string | null;
3426
+ "publishedAt": string | null;
3427
+ "currentVersion": number;
3428
+ "currentVersionId": string | null;
3429
+ "currentVersionTaskCount": number | null;
3430
+ "lenses": Array<string>;
3431
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3432
+ "runnability": {
3433
+ "kind": "runnable";
3434
+ } | {
3435
+ "kind": "corpus-missing";
3436
+ "reason": string;
3437
+ } | {
3438
+ "kind": "needs-import";
3439
+ "reason": string;
3440
+ } | {
3441
+ "kind": "needs-token";
3442
+ "reason": string;
3443
+ } | {
3444
+ "kind": "runner-unsupported";
3445
+ "reason": string;
3446
+ };
3447
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3448
+ "taskTextAvailable": boolean;
3449
+ "hubTier": "featured" | "community" | "mine";
3450
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3451
+ "listed": boolean;
3452
+ "visibility": "private" | "org" | "public";
3453
+ "attestation": {
3454
+ "licenseSpdx": string;
3455
+ "redistribution": "allowed" | "reference_only";
3456
+ "sourceUrl": string;
3457
+ "statement": string;
3458
+ "attestedAt": string;
3459
+ } | null;
3460
+ "rejectionReason": string | null;
3461
+ "uploadedBy": {
3462
+ "userId": string;
3463
+ "displayName": string | null;
3464
+ } | null;
3465
+ "notices": Array<{
3466
+ "kind": string;
3467
+ "text": string;
3468
+ }>;
3469
+ };
3470
+ };
3471
+ export type StudyBenchmarkDefinitionsUpdateRequest = {
3472
+ "name"?: string;
3473
+ "description"?: string | null;
3474
+ "family"?: "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3475
+ "sourceKind"?: "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3476
+ "sourceConfig"?: Record<string, unknown>;
3477
+ "adapterId"?: string | null;
3478
+ "headlineMetric"?: string;
3479
+ "gradingSpec"?: {
3480
+ "lenses": Array<{
3481
+ "id": string;
3482
+ "config"?: Record<string, unknown>;
3483
+ }>;
3484
+ "coverage"?: "every-task" | "per-task";
3485
+ };
3486
+ "environmentSpec"?: {
3487
+ "kind": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3488
+ };
3489
+ "methodology"?: {
3490
+ "variantsAllowed"?: boolean;
3491
+ "recommendedRepeats"?: number;
3492
+ "officialFidelity"?: boolean;
3493
+ "modelParams"?: Array<string>;
3494
+ "maxPlannedRuns"?: number;
3495
+ "sampleFloor"?: number;
3496
+ };
3497
+ "runnerKind"?: "none" | "image" | "environment" | "framework";
3498
+ "runnerRecipe"?: Record<string, unknown> | null;
3499
+ "license"?: "MIT" | "Apache-2.0" | "BSD-2-Clause" | "BSD-3-Clause" | "CC-BY-4.0" | "CC0-1.0" | "CC-BY-SA-4.0" | "CC-BY-NC-4.0" | "CC-BY-NC-SA-4.0" | "GPL-3.0" | "AGPL-3.0" | "MPL-2.0" | null;
3500
+ "licenseText"?: string | null;
3501
+ "redistribution"?: "allowed" | "reference_only";
3502
+ "access"?: "open" | "gated" | "private";
3503
+ "termsUrl"?: string | null;
3504
+ "homepageUrl"?: string | null;
3505
+ "paperUrl"?: string | null;
3506
+ "defaultConfig"?: {
3507
+ "matrix"?: Array<{
3508
+ "harness": string;
3509
+ "model": string;
3510
+ "provider"?: string;
3511
+ "params"?: Record<string, unknown>;
3512
+ }>;
3513
+ "skills"?: Array<string>;
3514
+ "tools"?: "default" | "none" | {
3515
+ "tools": Array<string>;
3516
+ };
3517
+ "addons"?: Record<string, unknown>;
3518
+ "params"?: Record<string, unknown>;
3519
+ } | null;
3520
+ "visibility"?: "private" | "org" | "public";
3521
+ };
3522
+ export type StudyBenchmarkDefinitionsUpdateResponse = {
3523
+ "definition": {
3524
+ "id": string;
3525
+ "slug": string;
3526
+ "name": string;
3527
+ "description": string | null;
3528
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3529
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3530
+ "headlineMetric": string;
3531
+ "methodology": {
3532
+ "variantsAllowed"?: boolean;
3533
+ "recommendedRepeats"?: number;
3534
+ "officialFidelity"?: boolean;
3535
+ "modelParams"?: Array<string>;
3536
+ "maxPlannedRuns"?: number;
3537
+ "sampleFloor"?: number;
3538
+ };
3539
+ "defaultConfig": {
3540
+ "matrix"?: Array<{
3541
+ "harness": string;
3542
+ "model": string;
3543
+ "provider"?: string;
3544
+ "params"?: Record<string, unknown>;
3545
+ }>;
3546
+ "skills"?: Array<string>;
3547
+ "tools"?: "default" | "none" | {
3548
+ "tools": Array<string>;
3549
+ };
3550
+ "addons"?: Record<string, unknown>;
3551
+ "params"?: Record<string, unknown>;
3552
+ } | null;
3553
+ "runnerKind": "none" | "image" | "environment" | "framework";
3554
+ "license": string | null;
3555
+ "licenseText": string | null;
3556
+ "licenseVerified": boolean;
3557
+ "licenseVerifiedByPolicy": boolean;
3558
+ "redistribution": "allowed" | "reference_only";
3559
+ "disclosure": "open" | "guarded";
3560
+ "disclosureReason": string | null;
3561
+ "access": "open" | "gated" | "private";
3562
+ "termsUrl": string | null;
3563
+ "homepageUrl": string | null;
3564
+ "paperUrl": string | null;
3565
+ "publishedAt": string | null;
3566
+ "currentVersion": number;
3567
+ "currentVersionId": string | null;
3568
+ "currentVersionTaskCount": number | null;
3569
+ "lenses": Array<string>;
3570
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3571
+ "runnability": {
3572
+ "kind": "runnable";
3573
+ } | {
3574
+ "kind": "corpus-missing";
3575
+ "reason": string;
3576
+ } | {
3577
+ "kind": "needs-import";
3578
+ "reason": string;
3579
+ } | {
3580
+ "kind": "needs-token";
3581
+ "reason": string;
3582
+ } | {
3583
+ "kind": "runner-unsupported";
3584
+ "reason": string;
3585
+ };
3586
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3587
+ "taskTextAvailable": boolean;
3588
+ "hubTier": "featured" | "community" | "mine";
3589
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3590
+ "listed": boolean;
3591
+ "visibility": "private" | "org" | "public";
3592
+ "attestation": {
3593
+ "licenseSpdx": string;
3594
+ "redistribution": "allowed" | "reference_only";
3595
+ "sourceUrl": string;
3596
+ "statement": string;
3597
+ "attestedAt": string;
3598
+ } | null;
3599
+ "rejectionReason": string | null;
3600
+ "uploadedBy": {
3601
+ "userId": string;
3602
+ "displayName": string | null;
3603
+ } | null;
3604
+ "notices": Array<{
3605
+ "kind": string;
3606
+ "text": string;
3607
+ }>;
3608
+ };
3609
+ };
3610
+ export type StudyBenchmarkDefinitionsAttachCorpusRequest = {
3611
+ "version"?: number;
3612
+ "limit"?: number;
3613
+ "changelog"?: string;
3614
+ };
3615
+ export type StudyBenchmarkDefinitionsPublishRequest = {
3616
+ "version"?: number;
3617
+ };
3618
+ export type StudyBenchmarkDefinitionsPublishResponse = {
3619
+ "slug": string;
3620
+ "currentVersion": number;
3621
+ "versionId": string;
3622
+ "taskCount": number;
3623
+ };
3624
+ export type StudyBenchmarkDefinitionsSubmitRequest = {
3625
+ "attestation": {
3626
+ "licenseSpdx": "MIT" | "Apache-2.0" | "BSD-2-Clause" | "BSD-3-Clause" | "CC-BY-4.0" | "CC0-1.0" | "CC-BY-SA-4.0" | "CC-BY-NC-4.0" | "CC-BY-NC-SA-4.0" | "GPL-3.0" | "AGPL-3.0" | "MPL-2.0" | "";
3627
+ "redistribution": "allowed" | "reference_only";
3628
+ "sourceUrl": string;
3629
+ "statement": string;
3630
+ };
3631
+ };
3632
+ export type StudyBenchmarkDefinitionsSubmitResponse = {
3633
+ "definition": {
3634
+ "id": string;
3635
+ "slug": string;
3636
+ "name": string;
3637
+ "description": string | null;
3638
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3639
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3640
+ "headlineMetric": string;
3641
+ "methodology": {
3642
+ "variantsAllowed"?: boolean;
3643
+ "recommendedRepeats"?: number;
3644
+ "officialFidelity"?: boolean;
3645
+ "modelParams"?: Array<string>;
3646
+ "maxPlannedRuns"?: number;
3647
+ "sampleFloor"?: number;
3648
+ };
3649
+ "defaultConfig": {
3650
+ "matrix"?: Array<{
3651
+ "harness": string;
3652
+ "model": string;
3653
+ "provider"?: string;
3654
+ "params"?: Record<string, unknown>;
3655
+ }>;
3656
+ "skills"?: Array<string>;
3657
+ "tools"?: "default" | "none" | {
3658
+ "tools": Array<string>;
3659
+ };
3660
+ "addons"?: Record<string, unknown>;
3661
+ "params"?: Record<string, unknown>;
3662
+ } | null;
3663
+ "runnerKind": "none" | "image" | "environment" | "framework";
3664
+ "license": string | null;
3665
+ "licenseText": string | null;
3666
+ "licenseVerified": boolean;
3667
+ "licenseVerifiedByPolicy": boolean;
3668
+ "redistribution": "allowed" | "reference_only";
3669
+ "disclosure": "open" | "guarded";
3670
+ "disclosureReason": string | null;
3671
+ "access": "open" | "gated" | "private";
3672
+ "termsUrl": string | null;
3673
+ "homepageUrl": string | null;
3674
+ "paperUrl": string | null;
3675
+ "publishedAt": string | null;
3676
+ "currentVersion": number;
3677
+ "currentVersionId": string | null;
3678
+ "currentVersionTaskCount": number | null;
3679
+ "lenses": Array<string>;
3680
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3681
+ "runnability": {
3682
+ "kind": "runnable";
3683
+ } | {
3684
+ "kind": "corpus-missing";
3685
+ "reason": string;
3686
+ } | {
3687
+ "kind": "needs-import";
3688
+ "reason": string;
3689
+ } | {
3690
+ "kind": "needs-token";
3691
+ "reason": string;
3692
+ } | {
3693
+ "kind": "runner-unsupported";
3694
+ "reason": string;
3695
+ };
3696
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3697
+ "taskTextAvailable": boolean;
3698
+ "hubTier": "featured" | "community" | "mine";
3699
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3700
+ "listed": boolean;
3701
+ "visibility": "private" | "org" | "public";
3702
+ "attestation": {
3703
+ "licenseSpdx": string;
3704
+ "redistribution": "allowed" | "reference_only";
3705
+ "sourceUrl": string;
3706
+ "statement": string;
3707
+ "attestedAt": string;
3708
+ } | null;
3709
+ "rejectionReason": string | null;
3710
+ "uploadedBy": {
3711
+ "userId": string;
3712
+ "displayName": string | null;
3713
+ } | null;
3714
+ "notices": Array<{
3715
+ "kind": string;
3716
+ "text": string;
3717
+ }>;
3718
+ };
3719
+ };
3720
+ export type StudyBenchmarkDefinitionsSetVisibilityRequest = {
3721
+ "visibility": "private" | "org" | "public";
3722
+ };
3723
+ export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
3724
+ "definition": {
3725
+ "id": string;
3726
+ "slug": string;
3727
+ "name": string;
3728
+ "description": string | null;
3729
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3730
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3731
+ "headlineMetric": string;
3732
+ "methodology": {
3733
+ "variantsAllowed"?: boolean;
3734
+ "recommendedRepeats"?: number;
3735
+ "officialFidelity"?: boolean;
3736
+ "modelParams"?: Array<string>;
3737
+ "maxPlannedRuns"?: number;
3738
+ "sampleFloor"?: number;
3739
+ };
3740
+ "defaultConfig": {
3741
+ "matrix"?: Array<{
3742
+ "harness": string;
3743
+ "model": string;
3744
+ "provider"?: string;
3745
+ "params"?: Record<string, unknown>;
3746
+ }>;
3747
+ "skills"?: Array<string>;
3748
+ "tools"?: "default" | "none" | {
3749
+ "tools": Array<string>;
3750
+ };
3751
+ "addons"?: Record<string, unknown>;
3752
+ "params"?: Record<string, unknown>;
3753
+ } | null;
3754
+ "runnerKind": "none" | "image" | "environment" | "framework";
3755
+ "license": string | null;
3756
+ "licenseText": string | null;
3757
+ "licenseVerified": boolean;
3758
+ "licenseVerifiedByPolicy": boolean;
3759
+ "redistribution": "allowed" | "reference_only";
3760
+ "disclosure": "open" | "guarded";
3761
+ "disclosureReason": string | null;
3762
+ "access": "open" | "gated" | "private";
3763
+ "termsUrl": string | null;
3764
+ "homepageUrl": string | null;
3765
+ "paperUrl": string | null;
3766
+ "publishedAt": string | null;
3767
+ "currentVersion": number;
3768
+ "currentVersionId": string | null;
3769
+ "currentVersionTaskCount": number | null;
3770
+ "lenses": Array<string>;
3771
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3772
+ "runnability": {
3773
+ "kind": "runnable";
3774
+ } | {
3775
+ "kind": "corpus-missing";
3776
+ "reason": string;
3777
+ } | {
3778
+ "kind": "needs-import";
3779
+ "reason": string;
3780
+ } | {
3781
+ "kind": "needs-token";
3782
+ "reason": string;
3783
+ } | {
3784
+ "kind": "runner-unsupported";
3785
+ "reason": string;
3786
+ };
3787
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3788
+ "taskTextAvailable": boolean;
3789
+ "hubTier": "featured" | "community" | "mine";
3790
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3791
+ "listed": boolean;
3792
+ "visibility": "private" | "org" | "public";
3793
+ "attestation": {
3794
+ "licenseSpdx": string;
3795
+ "redistribution": "allowed" | "reference_only";
3796
+ "sourceUrl": string;
3797
+ "statement": string;
3798
+ "attestedAt": string;
3799
+ } | null;
3800
+ "rejectionReason": string | null;
3801
+ "uploadedBy": {
3802
+ "userId": string;
3803
+ "displayName": string | null;
3804
+ } | null;
3805
+ "notices": Array<{
3806
+ "kind": string;
3807
+ "text": string;
3808
+ }>;
3809
+ };
3810
+ };
3811
+ export type StudyBenchmarkDefinitionsWithdrawResponse = {
3812
+ "definition": {
3813
+ "id": string;
3814
+ "slug": string;
3815
+ "name": string;
3816
+ "description": string | null;
3817
+ "family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
3818
+ "sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
3819
+ "headlineMetric": string;
3820
+ "methodology": {
3821
+ "variantsAllowed"?: boolean;
3822
+ "recommendedRepeats"?: number;
3823
+ "officialFidelity"?: boolean;
3824
+ "modelParams"?: Array<string>;
3825
+ "maxPlannedRuns"?: number;
3826
+ "sampleFloor"?: number;
3827
+ };
3828
+ "defaultConfig": {
3829
+ "matrix"?: Array<{
3830
+ "harness": string;
3831
+ "model": string;
3832
+ "provider"?: string;
3833
+ "params"?: Record<string, unknown>;
3834
+ }>;
3835
+ "skills"?: Array<string>;
3836
+ "tools"?: "default" | "none" | {
3837
+ "tools": Array<string>;
3838
+ };
3839
+ "addons"?: Record<string, unknown>;
3840
+ "params"?: Record<string, unknown>;
3841
+ } | null;
3842
+ "runnerKind": "none" | "image" | "environment" | "framework";
3843
+ "license": string | null;
3844
+ "licenseText": string | null;
3845
+ "licenseVerified": boolean;
3846
+ "licenseVerifiedByPolicy": boolean;
3847
+ "redistribution": "allowed" | "reference_only";
3848
+ "disclosure": "open" | "guarded";
3849
+ "disclosureReason": string | null;
3850
+ "access": "open" | "gated" | "private";
3851
+ "termsUrl": string | null;
3852
+ "homepageUrl": string | null;
3853
+ "paperUrl": string | null;
3854
+ "publishedAt": string | null;
3855
+ "currentVersion": number;
3856
+ "currentVersionId": string | null;
3857
+ "currentVersionTaskCount": number | null;
3858
+ "lenses": Array<string>;
3859
+ "environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
3860
+ "runnability": {
3861
+ "kind": "runnable";
3862
+ } | {
3863
+ "kind": "corpus-missing";
3864
+ "reason": string;
3865
+ } | {
3866
+ "kind": "needs-import";
3867
+ "reason": string;
3868
+ } | {
3869
+ "kind": "needs-token";
3870
+ "reason": string;
3871
+ } | {
3872
+ "kind": "runner-unsupported";
3873
+ "reason": string;
3874
+ };
3875
+ "runnerImage": "not-needed" | "not-built" | "building" | "ready";
3876
+ "taskTextAvailable": boolean;
3877
+ "hubTier": "featured" | "community" | "mine";
3878
+ "hubStatus": "draft" | "submitted" | "verified" | "rejected";
3879
+ "listed": boolean;
3880
+ "visibility": "private" | "org" | "public";
3881
+ "attestation": {
3882
+ "licenseSpdx": string;
3883
+ "redistribution": "allowed" | "reference_only";
3884
+ "sourceUrl": string;
3885
+ "statement": string;
3886
+ "attestedAt": string;
3887
+ } | null;
3888
+ "rejectionReason": string | null;
3889
+ "uploadedBy": {
3890
+ "userId": string;
3891
+ "displayName": string | null;
3892
+ } | null;
3893
+ "notices": Array<{
3894
+ "kind": string;
3895
+ "text": string;
3896
+ }>;
3897
+ };
3898
+ };
3899
+ export type StudyBenchmarkImportsGetResponse = {
3900
+ "job": {
3901
+ "id": string;
3902
+ "definitionId": string;
3903
+ "status": "queued" | "running" | "succeeded" | "failed";
3904
+ "door": "platform" | "organization";
3905
+ "request": Record<string, unknown>;
3906
+ "result": Record<string, unknown> | null;
3907
+ "error": {
3908
+ "code": string;
3909
+ "message": string;
3910
+ "detail"?: unknown;
3911
+ } | null;
3912
+ "executor": string;
3913
+ "worker": string | null;
3914
+ "createdAt": string;
3915
+ "startedAt": string | null;
3916
+ "finishedAt": string | null;
3917
+ };
3918
+ };
3919
+ export type StudyBenchmarkImportsStreamResponse = unknown;
3920
+ export type StudyBenchmarkIntakeAssistRequest = {
3921
+ "strategy": string;
3922
+ "config": Record<string, unknown>;
3923
+ "probe"?: {
3924
+ "strategy": string;
3925
+ "proposals": Array<{
3926
+ "field": "slug" | "name" | "description" | "family" | "sourceKind" | "sourceConfig" | "adapterId" | "headlineMetric" | "gradingSpec" | "environmentSpec" | "methodology" | "runnerKind" | "runnerRecipe" | "license" | "licenseText" | "redistribution" | "access" | "termsUrl" | "homepageUrl" | "paperUrl" | "defaultConfig" | "visibility";
3927
+ "value": unknown;
3928
+ "confidence": "high" | "medium" | "low";
3929
+ "rationale": string;
3930
+ "evidencePath"?: string;
3931
+ }>;
3932
+ "gaps": Array<{
3933
+ "field": "slug" | "name" | "description" | "family" | "sourceKind" | "sourceConfig" | "adapterId" | "headlineMetric" | "gradingSpec" | "environmentSpec" | "methodology" | "runnerKind" | "runnerRecipe" | "license" | "licenseText" | "redistribution" | "access" | "termsUrl" | "homepageUrl" | "paperUrl" | "defaultConfig" | "visibility";
3934
+ "reason": string;
3935
+ }>;
3936
+ "evidence": Array<{
3937
+ "kind": "file" | "metadata";
3938
+ "path": string;
3939
+ "excerpt": string;
3940
+ "bytes": number;
3941
+ }>;
3942
+ "notes": Array<string>;
3943
+ };
3944
+ };
3945
+ export type StudyBenchmarkIntakeAssistResponse = {
3946
+ "strategy": string;
3947
+ "draft": Record<string, unknown>;
3948
+ "fields": Array<{
3949
+ "field": string;
3950
+ "value": unknown;
3951
+ "confidence": "high" | "medium" | "low";
3952
+ "rationale": string;
3953
+ "evidencePath"?: string;
3954
+ "source": "probe" | "agent";
3955
+ }>;
3956
+ "unresolved": Array<{
3957
+ "field": string;
3958
+ "reason": string;
3959
+ }>;
3960
+ "gaps": Array<{
3961
+ "field": string;
3962
+ "reason": string;
3963
+ }>;
3964
+ "evidence": Array<{
3965
+ "kind": "file" | "metadata";
3966
+ "path": string;
3967
+ "excerpt": string;
3968
+ "bytes": number;
3969
+ }>;
3970
+ "notes": Array<string>;
3971
+ "model": string | null;
3972
+ "usage": {
3973
+ "inputTokens": number;
3974
+ "outputTokens": number;
3975
+ } | null;
3976
+ };
3977
+ export type StudyBenchmarkIntakeProbeRequest = {
3978
+ "strategy": string;
3979
+ "config": Record<string, unknown>;
3980
+ };
3981
+ export type StudyBenchmarkIntakeProbeResponse = {
3982
+ "strategy": string;
3983
+ "draft": Record<string, unknown>;
3984
+ "proposals": Array<{
3985
+ "field": string;
3986
+ "value": unknown;
3987
+ "confidence": "high" | "medium" | "low";
3988
+ "rationale": string;
3989
+ "evidencePath"?: string;
3990
+ }>;
3991
+ "gaps": Array<{
3992
+ "field": string;
3993
+ "reason": string;
3994
+ }>;
3995
+ "evidence": Array<{
3996
+ "kind": "file" | "metadata";
3997
+ "path": string;
3998
+ "excerpt": string;
3999
+ "bytes": number;
4000
+ }>;
4001
+ "notes": Array<string>;
4002
+ };
4003
+ export type StudyBenchmarkIntakeStrategiesResponse = {
4004
+ "strategies": Array<{
4005
+ "id": string;
4006
+ "label": string;
4007
+ "summary": string;
4008
+ "fields": Array<{
4009
+ "name": string;
4010
+ "flag": string;
4011
+ "required": boolean;
4012
+ "kind": "string" | "number" | "boolean" | "array" | "enum" | "object";
4013
+ "description"?: string;
4014
+ "options"?: Array<string>;
4015
+ "defaultValue"?: unknown;
4016
+ }>;
4017
+ }>;
4018
+ };
4019
+ export type StudyBenchmarkTermsGetResponse = {
4020
+ "scope": string;
4021
+ "version": string;
4022
+ "text": Array<string>;
4023
+ "acceptance": {
4024
+ "organizationId": string;
4025
+ "scope": string;
4026
+ "acceptedBy": string;
4027
+ "acceptedAt": string;
4028
+ } | null;
4029
+ };
4030
+ export type StudyCatalogResponse = {
4031
+ "harnesses": Array<{
4032
+ "id": string;
4033
+ "name": string;
4034
+ "maker": string;
4035
+ "offerable": boolean;
4036
+ "reason": string | null;
4037
+ "modes": Array<"task" | "session">;
4038
+ "defaultProvider": string | null;
4039
+ "providers": Array<{
4040
+ "id": string;
4041
+ "offerable": boolean;
4042
+ "reason": string | null;
4043
+ "models": Array<{
4044
+ "id": string;
4045
+ "name": string;
4046
+ }>;
4047
+ "defaultModel": string | null;
4048
+ }>;
4049
+ }>;
4050
+ };
4051
+ export type StudyImagesBuildsListResponse = {
4052
+ "builds": Array<{
4053
+ "id": string;
4054
+ "organizationId": string;
4055
+ "productKey": string;
4056
+ "requestedByUserId": string | null;
4057
+ "name": string;
4058
+ "source": {
4059
+ "kind": "archive";
4060
+ "sha256": string;
4061
+ } | {
4062
+ "kind": "upload";
4063
+ "fileId": string;
4064
+ } | {
4065
+ "kind": "git";
4066
+ "ref": string;
4067
+ "subdir"?: string;
4068
+ } | {
4069
+ "kind": "inline";
4070
+ "fileCount": number;
4071
+ };
4072
+ "sourceIdentity": string;
4073
+ "dockerfileDir": string;
4074
+ "dockerfileName": string;
4075
+ "buildArgKeys": Array<string>;
4076
+ "platform": string;
4077
+ "trusted": boolean;
4078
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4079
+ "image": string;
4080
+ "digest": string | null;
4081
+ "sizeBytes": number | null;
4082
+ "reason": string | null;
4083
+ "logRef": string | null;
4084
+ "startedAt": string | null;
4085
+ "finishedAt": string | null;
4086
+ "createdAt": string;
4087
+ "updatedAt": string;
4088
+ }>;
4089
+ "nextPageToken"?: string;
4090
+ };
4091
+ export type StudyImagesBuildsCreateRequest = {
4092
+ "name": string;
4093
+ "source": {
4094
+ "kind": "archive";
4095
+ "url": string;
4096
+ "sha256": string;
4097
+ } | {
4098
+ "kind": "upload";
4099
+ "fileId": string;
4100
+ } | {
4101
+ "kind": "git";
4102
+ "url": string;
4103
+ "ref": string;
4104
+ "subdir"?: string;
4105
+ } | {
4106
+ "kind": "inline";
4107
+ "dockerfile": string;
4108
+ "files"?: Record<string, unknown>;
4109
+ };
4110
+ "dockerfile"?: {
4111
+ "dir"?: string;
4112
+ "name"?: string;
4113
+ };
4114
+ "buildArgs"?: Record<string, unknown>;
4115
+ "platform"?: string;
4116
+ };
4117
+ export type StudyImagesBuildsCreateResponse = {
4118
+ "id": string;
4119
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4120
+ };
4121
+ export type StudyImagesBuildsGetResponse = {
4122
+ "build": {
4123
+ "id": string;
4124
+ "organizationId": string;
4125
+ "productKey": string;
4126
+ "requestedByUserId": string | null;
4127
+ "name": string;
4128
+ "source": {
4129
+ "kind": "archive";
4130
+ "sha256": string;
4131
+ } | {
4132
+ "kind": "upload";
4133
+ "fileId": string;
4134
+ } | {
4135
+ "kind": "git";
4136
+ "ref": string;
4137
+ "subdir"?: string;
4138
+ } | {
4139
+ "kind": "inline";
4140
+ "fileCount": number;
4141
+ };
4142
+ "sourceIdentity": string;
4143
+ "dockerfileDir": string;
4144
+ "dockerfileName": string;
4145
+ "buildArgKeys": Array<string>;
4146
+ "platform": string;
4147
+ "trusted": boolean;
4148
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4149
+ "image": string;
4150
+ "digest": string | null;
4151
+ "sizeBytes": number | null;
4152
+ "reason": string | null;
4153
+ "logRef": string | null;
4154
+ "startedAt": string | null;
4155
+ "finishedAt": string | null;
4156
+ "createdAt": string;
4157
+ "updatedAt": string;
4158
+ };
4159
+ };
4160
+ export type StudyImagesBuildsCancelResponse = {
4161
+ "id": string;
4162
+ "phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
4163
+ };
4164
+ export type StudyImagesBuildsStreamResponse = unknown;
4165
+ export type StudyImagesRunsCreateRequest = {
4166
+ "buildId"?: string;
4167
+ "image"?: string;
4168
+ "mode": "harness" | "environment";
4169
+ "harness"?: string;
4170
+ "task"?: string;
4171
+ };
4172
+ export type StudyImagesRunsCreateResponse = {
4173
+ "runId": string;
4174
+ };
4175
+ export type StudyRunsListResponse = {
4176
+ "results": Array<{
4177
+ "id": string;
4178
+ "harness": string;
4179
+ "model"?: string;
4180
+ "provider"?: string;
4181
+ "tools"?: "default" | "none" | {
4182
+ "tools": Array<string>;
4183
+ } | null;
4184
+ "intent_text"?: string;
4185
+ "target_domain"?: string;
4186
+ "status": string;
4187
+ "outcome"?: string;
4188
+ "cost_usd"?: number;
4189
+ "num_turns"?: number;
4190
+ "duration_ms"?: number;
4191
+ "total_tokens"?: number;
4192
+ "input_tokens"?: number;
4193
+ "cache_read_tokens"?: number;
4194
+ "cache_creation_tokens"?: number;
4195
+ "output_tokens"?: number;
4196
+ "created_at": string;
4197
+ "finished_at"?: string;
4198
+ "insight"?: {
4199
+ "visibility_score": number;
4200
+ "intent_satisfied": boolean;
4201
+ };
4202
+ }>;
4203
+ "pageToken"?: string;
4204
+ };
4205
+ export type StudyRunsCreateRequest = {
4206
+ "product_key"?: string;
4207
+ "intent": string;
4208
+ "domain"?: string;
4209
+ "harness"?: string;
4210
+ "model"?: string;
4211
+ "mode"?: "task" | "session";
4212
+ "addons"?: Record<string, unknown>;
4213
+ "engine"?: string;
4214
+ "execution_kind"?: "harness" | "direct";
4215
+ "tools"?: "default" | "none" | {
4216
+ "tools": Array<string>;
4217
+ };
4218
+ "requires_tools"?: boolean;
4219
+ "mcp_servers"?: Array<{
4220
+ "name": string;
4221
+ "url": string;
4222
+ "secretId"?: string;
4223
+ }>;
4224
+ "model_params"?: {
4225
+ "temperature"?: number;
4226
+ "maxOutputTokens"?: number;
4227
+ "reasoningEffort"?: "low" | "medium" | "high";
4228
+ "thinkingBudgetTokens"?: number;
4229
+ };
4230
+ "wait"?: boolean;
4231
+ };
4232
+ export type StudyRunsCreateResponse = {
4233
+ "runId": string;
4234
+ "status": "succeeded";
4235
+ "answer"?: string;
4236
+ "signals"?: unknown;
4237
+ } | {
4238
+ "runId": string;
4239
+ "status": "error" | "pending" | "queued";
4240
+ "streamUrl": string;
4241
+ "harness"?: string;
4242
+ "model"?: string;
4243
+ "provider"?: string;
4244
+ };
4245
+ export type StudyRunsCancelBatchRequest = {
4246
+ "runIds": Array<string>;
4247
+ };
4248
+ export type StudyRunsCancelBatchResponse = {
4249
+ "requested": number;
4250
+ "canceled": Array<string>;
4251
+ "alreadyTerminal": Array<string>;
4252
+ "skipped": Array<string>;
4253
+ };
4254
+ export type StudyRunsFeedResponse = unknown;
4255
+ export type StudyRunsCancelResponse = {
4256
+ "runId": string;
4257
+ "status": string;
4258
+ "alreadyTerminal": boolean;
4259
+ };
4260
+ export type StudyRunsDirectSignalsResponse = {
4261
+ "brands": Array<{
4262
+ "name": string;
4263
+ "domain"?: string;
4264
+ "status": "recommended" | "mentioned";
4265
+ "position"?: number;
4266
+ "sentiment"?: "positive" | "neutral" | "negative";
4267
+ "reason"?: string;
4268
+ "caveat"?: string;
4269
+ "backed_by"?: Array<string>;
4270
+ }>;
4271
+ "cited_domains": Array<{
4272
+ "host": string;
4273
+ "url"?: string;
4274
+ "rank"?: number;
4275
+ }>;
4276
+ "made_recommendations": boolean;
4277
+ "usage"?: {
4278
+ "input_tokens": number;
4279
+ "output_tokens": number;
4280
+ };
4281
+ "cost_usd"?: number;
4282
+ "execution_mode"?: "live" | "closed";
4283
+ "inconclusive": boolean;
4284
+ };
4285
+ export type StudyRunsInsightResponse = unknown;
4286
+ export type StudyRunsRawTraceResponse = unknown;
4287
+ export type StudyRunsSpansResponse = unknown;
4288
+ export type StudyRunsStreamResponse = unknown;
4289
+ export type StudyRunsTrajectoryResponse = unknown;
4290
+ export type StudySkillsListResponse = {
4291
+ "skills": Array<{
4292
+ "ref": string;
4293
+ "source": "builtin" | "custom" | "git";
4294
+ "id": string;
4295
+ "name": string;
4296
+ "slug": string;
4297
+ "isExample"?: boolean;
4298
+ "description"?: string;
4299
+ "visibility"?: "user" | "org";
4300
+ "authorUserId"?: string;
4301
+ "body"?: string;
4302
+ "repoUrl"?: string;
4303
+ "gitRef"?: string;
4304
+ "subpath"?: string;
4305
+ "updatedAt"?: string;
4306
+ }>;
4307
+ };
4308
+ export type StudySkillsCreateRequest = {
4309
+ "source": "custom";
4310
+ "name": string;
4311
+ "body": string;
4312
+ "visibility"?: "user" | "org";
4313
+ } | {
4314
+ "source": "git";
4315
+ "name": string;
4316
+ "repoUrl": string;
4317
+ "gitRef": string;
4318
+ "subpath"?: string | null;
4319
+ "visibility"?: "user" | "org";
4320
+ };
4321
+ export type StudySkillsCreateResponse = {
4322
+ "skill": {
4323
+ "ref": string;
4324
+ "source": "builtin" | "custom" | "git";
4325
+ "id": string;
4326
+ "name": string;
4327
+ "slug": string;
4328
+ "isExample"?: boolean;
4329
+ "description"?: string;
4330
+ "visibility"?: "user" | "org";
4331
+ "authorUserId"?: string;
4332
+ "body"?: string;
4333
+ "repoUrl"?: string;
4334
+ "gitRef"?: string;
4335
+ "subpath"?: string;
4336
+ "updatedAt"?: string;
4337
+ };
4338
+ };
4339
+ export type StudySkillsUpdateRequest = {
4340
+ "name"?: string;
4341
+ "visibility"?: "user" | "org";
4342
+ "body"?: string;
4343
+ "repoUrl"?: string;
4344
+ "gitRef"?: string;
4345
+ "subpath"?: string | null;
4346
+ };
4347
+ export type StudySkillsUpdateResponse = {
4348
+ "skill": {
4349
+ "ref": string;
4350
+ "source": "builtin" | "custom" | "git";
4351
+ "id": string;
4352
+ "name": string;
4353
+ "slug": string;
4354
+ "isExample"?: boolean;
4355
+ "description"?: string;
4356
+ "visibility"?: "user" | "org";
4357
+ "authorUserId"?: string;
4358
+ "body"?: string;
4359
+ "repoUrl"?: string;
4360
+ "gitRef"?: string;
4361
+ "subpath"?: string;
4362
+ "updatedAt"?: string;
4363
+ };
4364
+ };
4365
+ export type StudySkillsDeleteResponse = unknown;
4366
+ export type StudyListResponse = {
4367
+ "studies": Array<{
4368
+ "id": string;
4369
+ "executiveSummary"?: string | null;
4370
+ "plannedRunCount": number;
4371
+ "organizationId": string;
4372
+ "createdBy": string | null;
4373
+ "name": string;
4374
+ "slug": string;
4375
+ "description": string | null;
4376
+ "skillRef": string | null;
4377
+ "skillSlug": string | null;
4378
+ "skillName": string | null;
4379
+ "skillPromise": string;
4380
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
4381
+ "subjectName": string | null;
4382
+ "subjectLabel": string;
4383
+ "armConfig": {
4384
+ "baseline"?: {
4385
+ "tools"?: "default" | "none" | {
4386
+ "tools": Array<string>;
4387
+ };
4388
+ "addons"?: Record<string, unknown>;
4389
+ };
4390
+ "treatment"?: {
4391
+ "tools"?: "default" | "none" | {
4392
+ "tools": Array<string>;
4393
+ };
4394
+ "addons"?: Record<string, unknown>;
4395
+ };
4396
+ };
4397
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
4398
+ "subjectBackend": "model" | "harness" | null;
4399
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4400
+ "importedFrom"?: string;
4401
+ "baselineLabel": string | null;
4402
+ "treatmentLabel": string | null;
4403
+ "promiseMetrics": Array<{
4404
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
4405
+ "direction": "down" | "up";
4406
+ "label": string;
4407
+ }>;
4408
+ "matrix": Array<{
4409
+ "harness": string;
4410
+ "model": string;
4411
+ "provider"?: string;
4412
+ "params"?: {
4413
+ "temperature"?: number;
4414
+ "maxOutputTokens"?: number;
4415
+ "reasoningEffort"?: "low" | "medium" | "high";
4416
+ "thinkingBudgetTokens"?: number;
4417
+ };
4418
+ }>;
4419
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
4420
+ "judgeSupplementary"?: boolean;
4421
+ "taskCount": number;
4422
+ "variantsPerTask": number;
4423
+ "repeats": number;
4424
+ "subset": {
4425
+ "count": number;
4426
+ "seed": string;
4427
+ "of": number;
4428
+ } | null;
4429
+ "status": string;
4430
+ "error": string | null;
4431
+ "preparation"?: {
4432
+ "since": string;
4433
+ "launchedBy": {
4434
+ "userId"?: string;
4435
+ "role"?: string;
4436
+ };
4437
+ "waitingOn": Array<{
4438
+ "imageKey": string;
4439
+ "buildId": string;
4440
+ "stage"?: "image" | "snapshot";
4441
+ }>;
4442
+ } | null;
4443
+ "visibility": "private" | "org" | "link" | "public";
4444
+ "visibilityChangedAt"?: string | null;
4445
+ "visibilityChangedBy"?: string | null;
4446
+ "shareToken"?: string | null;
4447
+ "lowSampleOverride": boolean;
4448
+ "publishedAt": string | null;
4449
+ "verdict": unknown;
4450
+ "definitionVersionId": string | null;
4451
+ "runnerKind": "ora-runs" | "batch-job";
4452
+ "createdAt": string;
4453
+ "updatedAt": string;
4454
+ "liveUntil": string | null;
4455
+ }>;
4456
+ };
4457
+ export type StudyCreateRequest = {
4458
+ "name": string;
4459
+ "description"?: string;
4460
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
4461
+ "skillRef"?: string;
4462
+ "definitionVersionId"?: string;
4463
+ "subjectName"?: string;
4464
+ "armConfig"?: {
4465
+ "baseline"?: {
4466
+ "tools"?: "default" | "none" | {
4467
+ "tools": Array<string>;
4468
+ };
4469
+ "addons"?: Record<string, unknown>;
4470
+ };
4471
+ "treatment"?: {
4472
+ "tools"?: "default" | "none" | {
4473
+ "tools": Array<string>;
4474
+ };
4475
+ "addons"?: Record<string, unknown>;
4476
+ };
4477
+ };
4478
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
4479
+ "judgeSupplementary"?: boolean;
4480
+ "proposeExpectations"?: boolean;
4481
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
4482
+ "subjectBackend"?: "model" | "harness";
4483
+ "baselineLabel"?: string;
4484
+ "treatmentLabel"?: string;
4485
+ "taskCount"?: number;
4486
+ "variantsPerTask"?: number;
4487
+ "repeats"?: number;
4488
+ "tasks"?: Array<{
4489
+ "prompt": string;
4490
+ "rationale"?: string;
4491
+ "category"?: string;
4492
+ "requiresWeb"?: boolean;
4493
+ "rubric"?: Array<string>;
4494
+ "variants"?: Array<string>;
4495
+ "expectations"?: Array<unknown>;
4496
+ }>;
4497
+ "subset"?: {
4498
+ "count": number;
4499
+ "seed"?: string;
4500
+ };
4501
+ "matrix": Array<{
4502
+ "harness": string;
4503
+ "model": string;
4504
+ "provider"?: string;
4505
+ "params"?: {
4506
+ "temperature"?: number;
4507
+ "maxOutputTokens"?: number;
4508
+ "reasoningEffort"?: "low" | "medium" | "high";
4509
+ "thinkingBudgetTokens"?: number;
4510
+ };
4511
+ }>;
4512
+ "promise"?: string;
4513
+ };
4514
+ export type StudyCreateResponse = {
4515
+ "study": {
4516
+ "id": string;
4517
+ "executiveSummary"?: string | null;
4518
+ "plannedRunCount": number;
4519
+ "organizationId": string;
4520
+ "createdBy": string | null;
4521
+ "name": string;
4522
+ "slug": string;
4523
+ "description": string | null;
4524
+ "skillRef": string | null;
4525
+ "skillSlug": string | null;
4526
+ "skillName": string | null;
4527
+ "skillPromise": string;
4528
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
4529
+ "subjectName": string | null;
4530
+ "subjectLabel": string;
4531
+ "armConfig": {
4532
+ "baseline"?: {
4533
+ "tools"?: "default" | "none" | {
4534
+ "tools": Array<string>;
4535
+ };
4536
+ "addons"?: Record<string, unknown>;
4537
+ };
4538
+ "treatment"?: {
4539
+ "tools"?: "default" | "none" | {
4540
+ "tools": Array<string>;
4541
+ };
4542
+ "addons"?: Record<string, unknown>;
4543
+ };
4544
+ };
4545
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
4546
+ "subjectBackend": "model" | "harness" | null;
4547
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4548
+ "importedFrom"?: string;
4549
+ "baselineLabel": string | null;
4550
+ "treatmentLabel": string | null;
4551
+ "promiseMetrics": Array<{
4552
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
4553
+ "direction": "down" | "up";
4554
+ "label": string;
4555
+ }>;
4556
+ "matrix": Array<{
4557
+ "harness": string;
4558
+ "model": string;
4559
+ "provider"?: string;
4560
+ "params"?: {
4561
+ "temperature"?: number;
4562
+ "maxOutputTokens"?: number;
4563
+ "reasoningEffort"?: "low" | "medium" | "high";
4564
+ "thinkingBudgetTokens"?: number;
4565
+ };
4566
+ }>;
4567
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
4568
+ "judgeSupplementary"?: boolean;
4569
+ "taskCount": number;
4570
+ "variantsPerTask": number;
4571
+ "repeats": number;
4572
+ "subset": {
4573
+ "count": number;
4574
+ "seed": string;
4575
+ "of": number;
4576
+ } | null;
4577
+ "status": string;
4578
+ "error": string | null;
4579
+ "preparation"?: {
4580
+ "since": string;
4581
+ "launchedBy": {
4582
+ "userId"?: string;
4583
+ "role"?: string;
4584
+ };
4585
+ "waitingOn": Array<{
4586
+ "imageKey": string;
4587
+ "buildId": string;
4588
+ "stage"?: "image" | "snapshot";
4589
+ }>;
4590
+ } | null;
4591
+ "visibility": "private" | "org" | "link" | "public";
4592
+ "visibilityChangedAt"?: string | null;
4593
+ "visibilityChangedBy"?: string | null;
4594
+ "shareToken"?: string | null;
4595
+ "lowSampleOverride": boolean;
4596
+ "publishedAt": string | null;
4597
+ "verdict": unknown;
4598
+ "definitionVersionId": string | null;
4599
+ "runnerKind": "ora-runs" | "batch-job";
4600
+ "createdAt": string;
4601
+ "updatedAt": string;
4602
+ "liveUntil": string | null;
4603
+ };
4604
+ "streamUrl": string;
4605
+ "warnings"?: Array<{
4606
+ "harness": string;
4607
+ "kind": string;
4608
+ }>;
4609
+ };
4610
+ export type StudyConvertWebmcpSuiteRequest = {
4611
+ "suite": unknown;
4612
+ "ordered"?: boolean;
4613
+ };
4614
+ export type StudyConvertWebmcpSuiteResponse = {
4615
+ "tasks": Array<{
4616
+ "title": string;
4617
+ "prompt": string;
4618
+ "expectations": Array<unknown>;
4619
+ }>;
4620
+ "warnings": Array<string>;
4621
+ };
4622
+ export type StudyGetResponse = {
4623
+ "study": {
4624
+ "id": string;
4625
+ "executiveSummary"?: string | null;
4626
+ "plannedRunCount": number;
4627
+ "organizationId": string;
4628
+ "createdBy": string | null;
4629
+ "name": string;
4630
+ "slug": string;
4631
+ "description": string | null;
4632
+ "skillRef": string | null;
4633
+ "skillSlug": string | null;
4634
+ "skillName": string | null;
4635
+ "skillPromise": string;
4636
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
4637
+ "subjectName": string | null;
4638
+ "subjectLabel": string;
4639
+ "armConfig": {
4640
+ "baseline"?: {
4641
+ "tools"?: "default" | "none" | {
4642
+ "tools": Array<string>;
4643
+ };
4644
+ "addons"?: Record<string, unknown>;
4645
+ };
4646
+ "treatment"?: {
4647
+ "tools"?: "default" | "none" | {
4648
+ "tools": Array<string>;
4649
+ };
4650
+ "addons"?: Record<string, unknown>;
4651
+ };
4652
+ };
4653
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
4654
+ "subjectBackend": "model" | "harness" | null;
4655
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
4656
+ "importedFrom"?: string;
4657
+ "baselineLabel": string | null;
4658
+ "treatmentLabel": string | null;
4659
+ "promiseMetrics": Array<{
4660
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
4661
+ "direction": "down" | "up";
4662
+ "label": string;
4663
+ }>;
4664
+ "matrix": Array<{
4665
+ "harness": string;
4666
+ "model": string;
4667
+ "provider"?: string;
4668
+ "params"?: {
4669
+ "temperature"?: number;
4670
+ "maxOutputTokens"?: number;
4671
+ "reasoningEffort"?: "low" | "medium" | "high";
4672
+ "thinkingBudgetTokens"?: number;
4673
+ };
4674
+ }>;
4675
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
4676
+ "judgeSupplementary"?: boolean;
4677
+ "taskCount": number;
4678
+ "variantsPerTask": number;
4679
+ "repeats": number;
4680
+ "subset": {
4681
+ "count": number;
4682
+ "seed": string;
4683
+ "of": number;
4684
+ } | null;
4685
+ "status": string;
4686
+ "error": string | null;
4687
+ "preparation"?: {
4688
+ "since": string;
4689
+ "launchedBy": {
4690
+ "userId"?: string;
4691
+ "role"?: string;
4692
+ };
4693
+ "waitingOn": Array<{
4694
+ "imageKey": string;
4695
+ "buildId": string;
4696
+ "stage"?: "image" | "snapshot";
4697
+ }>;
4698
+ } | null;
4699
+ "visibility": "private" | "org" | "link" | "public";
4700
+ "visibilityChangedAt"?: string | null;
4701
+ "visibilityChangedBy"?: string | null;
4702
+ "shareToken"?: string | null;
4703
+ "lowSampleOverride": boolean;
4704
+ "publishedAt": string | null;
4705
+ "verdict": unknown;
4706
+ "definitionVersionId": string | null;
4707
+ "runnerKind": "ora-runs" | "batch-job";
4708
+ "createdAt": string;
4709
+ "updatedAt": string;
4710
+ "liveUntil": string | null;
4711
+ };
4712
+ "tasks": Array<{
4713
+ "id": string;
4714
+ "studyId": string;
4715
+ "position": number;
4716
+ "prompt": string;
4717
+ "rationale": string | null;
4718
+ "category": string | null;
4719
+ "requiresWeb": boolean;
4720
+ "rubric": Array<string>;
4721
+ "variants"?: Array<string> | null;
4722
+ "expectations"?: Array<unknown> | null;
4723
+ }>;
4724
+ "runs": Array<{
4725
+ "runId": string;
4726
+ "taskId": string;
4727
+ "arm": "baseline" | "skill";
4728
+ "harness": string;
4729
+ "model": string;
4730
+ "provider": string | null;
4731
+ "variantIndex"?: number;
4732
+ "repeatIndex"?: number;
4733
+ "status": string;
4734
+ "outcome": string | null;
4735
+ "outcomeReason": string | null;
4736
+ "totalTokens": number | null;
4737
+ "inputTokens": number | null;
4738
+ "outputTokens": number | null;
4739
+ "cacheReadTokens": number | null;
4740
+ "cacheCreationTokens": number | null;
4741
+ "costUsd": number | null;
4742
+ "numTurns": number | null;
4743
+ "durationMs": number | null;
4744
+ "intentSatisfied": boolean | null;
4745
+ "frictionCount": number | null;
4746
+ "insightSummary": string | null;
4747
+ "qualityScore": number | null;
4748
+ "rubricResults": Array<{
4749
+ "criterion": string;
4750
+ "passed": boolean;
4751
+ "note"?: string;
4752
+ "votes"?: {
4753
+ "passed": number;
4754
+ "judges": number;
4755
+ "byJudge"?: Array<{
4756
+ "judge": string;
4757
+ "passed": boolean;
4758
+ "note"?: string;
4759
+ }>;
4760
+ };
4761
+ }> | null;
4762
+ "gradedAt": string | null;
4763
+ "judgeModel": string | null;
4764
+ "pairVerdict": "skill" | "baseline" | "tie" | "unmeasurable" | null;
4765
+ "pairVotes": Array<{
4766
+ "judge": string;
4767
+ "ordering": "base-first" | "skill-first";
4768
+ "answer": "A" | "B" | "tie";
4769
+ }> | null;
4770
+ "pairGradedAt": string | null;
4771
+ "evalResults": unknown | null;
4772
+ "evalScore": number | null;
4773
+ "evalGradedAt": string | null;
4774
+ "lensGrades"?: {
4775
+ "reference"?: {
4776
+ "pass": boolean | null;
4777
+ "extracted": string | null;
4778
+ "extraction": string;
4779
+ "matcher": string;
4780
+ "votes"?: Array<{
4781
+ "judge": string;
4782
+ "answer": "equivalent" | "different" | "not_attempted" | null;
4783
+ }>;
4784
+ "reason"?: string;
4785
+ "gradedAt": string;
4786
+ };
4787
+ "framework"?: {
4788
+ "framework": string;
4789
+ "primary": {
4790
+ "metricId": string;
4791
+ "provenance": string;
4792
+ "value": number;
4793
+ "scenario": string;
4794
+ "count": number;
4795
+ } | null;
4796
+ "secondary": Array<{
4797
+ "label": string;
4798
+ "name": string;
4799
+ "scenario": string;
4800
+ "value": number | null;
4801
+ "count": number;
4802
+ "split": string | null;
4803
+ }>;
4804
+ "errors": Array<{
4805
+ "scenario": string;
4806
+ "metric": string;
4807
+ "reason": string;
4808
+ }>;
4809
+ "gradedAt": string;
4810
+ };
4811
+ "execution"?: {
4812
+ "pass": boolean | null;
4813
+ "status": "passed" | "failed" | "timeout" | "error" | "unavailable";
4814
+ "language": string;
4815
+ "passedTests": number | null;
4816
+ "failedTests": number | null;
4817
+ "stdout": string;
4818
+ "stdoutTruncated": boolean;
4819
+ "reason"?: string;
4820
+ "gradedAt": string;
4821
+ };
4822
+ } | null;
4823
+ "runCreatedAt": string | null;
4824
+ "startedAt": string | null;
4825
+ "finishedAt": string | null;
4826
+ }>;
4827
+ "aggregate": {
4828
+ "summary": {
4829
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
4830
+ "headline": Array<{
4831
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
4832
+ "label": string;
4833
+ "direction": "down" | "up";
4834
+ "baselineAvg": number | null;
4835
+ "skillAvg": number | null;
4836
+ "deltaPct": number | null;
4837
+ "delta"?: {
4838
+ "unit": "pp";
4839
+ "absolute": number | null;
4840
+ "relativePct": number | null;
4841
+ } | {
4842
+ "unit": "pts";
4843
+ "absolute": number | null;
4844
+ "relativePct"?: number | null;
4845
+ } | {
4846
+ "unit": "%";
4847
+ "relativePct": number | null;
4848
+ };
4849
+ "improved": boolean;
4850
+ }>;
4851
+ "completionRate": {
4852
+ "baseline": number | null;
4853
+ "skill": number | null;
4854
+ };
4855
+ "goalRate": {
4856
+ "baseline": number | null;
4857
+ "skill": number | null;
4858
+ };
4859
+ "pairwise": {
4860
+ "pairs": number;
4861
+ "skillWins": number;
4862
+ "baselineWins": number;
4863
+ "ties": number;
4864
+ "winRate": number | null;
4865
+ };
4866
+ "decision"?: {
4867
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
4868
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
4869
+ "reason": string;
4870
+ "efficiencyDiscounted": boolean;
4871
+ };
4872
+ "leaderboard"?: Array<unknown>;
4873
+ "concentration": {
4874
+ "perCell": Array<{
4875
+ "cell": string;
4876
+ "goalDeltaPp": number | null;
4877
+ "contributionPct": number | null;
4878
+ }>;
4879
+ "heterogeneity": {
4880
+ "dominantCell": string;
4881
+ "contributionPct": number;
4882
+ } | null;
4883
+ };
4884
+ "matrixShape": {
4885
+ "cells": number;
4886
+ "harnesses": number;
4887
+ "models": number;
4888
+ "crossed": boolean;
4889
+ };
4890
+ "flakiness"?: {
4891
+ "measured": boolean;
4892
+ "conditions": number;
4893
+ "unstable": number;
4894
+ "worst": {
4895
+ "cell": string;
4896
+ "taskId": string;
4897
+ "arm": "baseline" | "skill";
4898
+ "variant"?: number;
4899
+ "passRate": number;
4900
+ } | null;
4901
+ };
4902
+ "arms"?: {
4903
+ "baseline": {
4904
+ "runs": number;
4905
+ "completed": number;
4906
+ "infraFailures": number;
4907
+ "completionRate": number | null;
4908
+ "goalRate": number | null;
4909
+ "goalTally": {
4910
+ "achieved": number;
4911
+ "partial": number;
4912
+ "missed": number;
4913
+ };
4914
+ "avgTotalTokens": number | null;
4915
+ "avgOutputTokens": number | null;
4916
+ "avgCostUsd": number | null;
4917
+ "avgDurationMs": number | null;
4918
+ "avgTurns": number | null;
4919
+ "costCoverage": number | null;
4920
+ "avgQualityScore": number | null;
4921
+ "gradedCoverage": number | null;
4922
+ "medianCostUsd": number | null;
4923
+ "medianTotalTokens": number | null;
4924
+ "medianDurationMs": number | null;
4925
+ "p95CostUsd": number | null;
4926
+ "p95TotalTokens": number | null;
4927
+ "p95DurationMs": number | null;
4928
+ "maxCostUsd": number | null;
4929
+ "maxTotalTokens": number | null;
4930
+ "maxDurationMs": number | null;
4931
+ "evalPassRate"?: number | null;
4932
+ "benchmarkPassRate"?: number | null;
4933
+ "adoptionRate"?: number | null;
4934
+ };
4935
+ "skill": {
4936
+ "runs": number;
4937
+ "completed": number;
4938
+ "infraFailures": number;
4939
+ "completionRate": number | null;
4940
+ "goalRate": number | null;
4941
+ "goalTally": {
4942
+ "achieved": number;
4943
+ "partial": number;
4944
+ "missed": number;
4945
+ };
4946
+ "avgTotalTokens": number | null;
4947
+ "avgOutputTokens": number | null;
4948
+ "avgCostUsd": number | null;
4949
+ "avgDurationMs": number | null;
4950
+ "avgTurns": number | null;
4951
+ "costCoverage": number | null;
4952
+ "avgQualityScore": number | null;
4953
+ "gradedCoverage": number | null;
4954
+ "medianCostUsd": number | null;
4955
+ "medianTotalTokens": number | null;
4956
+ "medianDurationMs": number | null;
4957
+ "p95CostUsd": number | null;
4958
+ "p95TotalTokens": number | null;
4959
+ "p95DurationMs": number | null;
4960
+ "maxCostUsd": number | null;
4961
+ "maxTotalTokens": number | null;
4962
+ "maxDurationMs": number | null;
4963
+ "evalPassRate"?: number | null;
4964
+ "benchmarkPassRate"?: number | null;
4965
+ "adoptionRate"?: number | null;
4966
+ };
4967
+ };
4968
+ "totals": {
4969
+ "cells": number;
4970
+ "runs": number;
4971
+ "completed": number;
4972
+ "failed": number;
4973
+ "costUsd": number;
4974
+ "tokens": number;
4975
+ };
4976
+ };
4977
+ "perCell": Array<unknown>;
4978
+ "perHarness": Array<unknown>;
4979
+ "perModel": Array<unknown>;
4980
+ "perTask": Array<unknown>;
4981
+ "webSplit": unknown;
4982
+ "inference": {
4983
+ "columns": Array<{
4984
+ "key": string;
4985
+ "label": string;
4986
+ "family": "rate" | "score" | "magnitude";
4987
+ "betterWhen": "higher" | "lower";
4988
+ }>;
4989
+ "groups": Array<{
4990
+ "kind": "cell" | "harness" | "model" | "field";
4991
+ "title": string;
4992
+ "rows": Array<{
4993
+ "key": string;
4994
+ "kind": "cell" | "harness" | "model" | "field";
4995
+ "harness": string | null;
4996
+ "model": string | null;
4997
+ "paramsKey"?: string;
4998
+ "runs": number;
4999
+ "cells": Record<string, unknown>;
5000
+ }>;
5001
+ }>;
5002
+ "reference": "baseline" | "field";
5003
+ "draws": number;
5004
+ } | null;
5005
+ "behavior"?: Array<{
5006
+ "harness": string;
5007
+ "model": string;
5008
+ "provider": string | null;
5009
+ "arm": string;
5010
+ "totalRuns": number;
5011
+ "signals": Array<{
5012
+ "id": string;
5013
+ "label": string;
5014
+ "tone": "good" | "bad" | "neutral";
5015
+ "runCount": number;
5016
+ "succeededCount": number;
5017
+ }>;
5018
+ }>;
5019
+ "reference"?: {
5020
+ "cells": Array<{
5021
+ "cell": string;
5022
+ "harness": string;
5023
+ "model": string;
5024
+ "provider": string | null;
5025
+ "paramsKey": string;
5026
+ "label": string;
5027
+ "passRate": {
5028
+ "value": number | null;
5029
+ "ci": Array<unknown> | null;
5030
+ "n": number;
5031
+ };
5032
+ "tasks": number;
5033
+ "graded": number;
5034
+ "unmeasured": number;
5035
+ "matchers": Array<string>;
5036
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
5037
+ }>;
5038
+ "overall": {
5039
+ "value": number | null;
5040
+ "ci": Array<unknown> | null;
5041
+ "n": number;
5042
+ };
5043
+ "graded": number;
5044
+ "unmeasured": number;
5045
+ "modelJudged": boolean;
5046
+ };
5047
+ "execution"?: {
5048
+ "graded": number;
5049
+ "measured": number;
5050
+ "passed": number;
5051
+ "passRate": number | null;
5052
+ "byStatus": {
5053
+ "passed": number;
5054
+ "failed": number;
5055
+ "timeout": number;
5056
+ "error": number;
5057
+ "unavailable": number;
5058
+ };
5059
+ "languages": Array<string>;
5060
+ "unavailableReasons": Array<string>;
5061
+ };
5062
+ };
5063
+ "warnings": Array<{
5064
+ "type": "eval_capability_gap";
5065
+ "harness": string;
5066
+ "kind": string;
5067
+ } | {
5068
+ "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
5069
+ "detail": string;
5070
+ "taskIds"?: Array<string>;
5071
+ }>;
5072
+ "shareUrl"?: string | null;
5073
+ "canManageSharing"?: boolean;
5074
+ };
5075
+ export type StudyDeleteResponse = unknown;
5076
+ export type StudyRegenerateSummaryResponse = {
5077
+ "status": "queued";
5078
+ };
5079
+ export type StudyGenerateRequest = {
5080
+ "proposeExpectations"?: boolean;
5081
+ };
5082
+ export type StudyGenerateResponse = {
5083
+ "study": {
5084
+ "id": string;
5085
+ "executiveSummary"?: string | null;
5086
+ "plannedRunCount": number;
5087
+ "organizationId": string;
5088
+ "createdBy": string | null;
5089
+ "name": string;
5090
+ "slug": string;
5091
+ "description": string | null;
5092
+ "skillRef": string | null;
5093
+ "skillSlug": string | null;
5094
+ "skillName": string | null;
5095
+ "skillPromise": string;
5096
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5097
+ "subjectName": string | null;
5098
+ "subjectLabel": string;
5099
+ "armConfig": {
5100
+ "baseline"?: {
5101
+ "tools"?: "default" | "none" | {
5102
+ "tools": Array<string>;
5103
+ };
5104
+ "addons"?: Record<string, unknown>;
5105
+ };
5106
+ "treatment"?: {
5107
+ "tools"?: "default" | "none" | {
5108
+ "tools": Array<string>;
5109
+ };
5110
+ "addons"?: Record<string, unknown>;
5111
+ };
5112
+ };
5113
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5114
+ "subjectBackend": "model" | "harness" | null;
5115
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5116
+ "importedFrom"?: string;
5117
+ "baselineLabel": string | null;
5118
+ "treatmentLabel": string | null;
5119
+ "promiseMetrics": Array<{
5120
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5121
+ "direction": "down" | "up";
5122
+ "label": string;
5123
+ }>;
5124
+ "matrix": Array<{
5125
+ "harness": string;
5126
+ "model": string;
5127
+ "provider"?: string;
5128
+ "params"?: {
5129
+ "temperature"?: number;
5130
+ "maxOutputTokens"?: number;
5131
+ "reasoningEffort"?: "low" | "medium" | "high";
5132
+ "thinkingBudgetTokens"?: number;
5133
+ };
5134
+ }>;
5135
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5136
+ "judgeSupplementary"?: boolean;
5137
+ "taskCount": number;
5138
+ "variantsPerTask": number;
5139
+ "repeats": number;
5140
+ "subset": {
5141
+ "count": number;
5142
+ "seed": string;
5143
+ "of": number;
5144
+ } | null;
5145
+ "status": string;
5146
+ "error": string | null;
5147
+ "preparation"?: {
5148
+ "since": string;
5149
+ "launchedBy": {
5150
+ "userId"?: string;
5151
+ "role"?: string;
5152
+ };
5153
+ "waitingOn": Array<{
5154
+ "imageKey": string;
5155
+ "buildId": string;
5156
+ "stage"?: "image" | "snapshot";
5157
+ }>;
5158
+ } | null;
5159
+ "visibility": "private" | "org" | "link" | "public";
5160
+ "visibilityChangedAt"?: string | null;
5161
+ "visibilityChangedBy"?: string | null;
5162
+ "shareToken"?: string | null;
5163
+ "lowSampleOverride": boolean;
5164
+ "publishedAt": string | null;
5165
+ "verdict": unknown;
5166
+ "definitionVersionId": string | null;
5167
+ "runnerKind": "ora-runs" | "batch-job";
5168
+ "createdAt": string;
5169
+ "updatedAt": string;
5170
+ "liveUntil": string | null;
5171
+ };
5172
+ "streamUrl": string;
5173
+ "warnings"?: Array<{
5174
+ "harness": string;
5175
+ "kind": string;
5176
+ }>;
5177
+ };
5178
+ export type StudyLaunchResponse = {
5179
+ "study": {
5180
+ "id": string;
5181
+ "executiveSummary"?: string | null;
5182
+ "plannedRunCount": number;
5183
+ "organizationId": string;
5184
+ "createdBy": string | null;
5185
+ "name": string;
5186
+ "slug": string;
5187
+ "description": string | null;
5188
+ "skillRef": string | null;
5189
+ "skillSlug": string | null;
5190
+ "skillName": string | null;
5191
+ "skillPromise": string;
5192
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5193
+ "subjectName": string | null;
5194
+ "subjectLabel": string;
5195
+ "armConfig": {
5196
+ "baseline"?: {
5197
+ "tools"?: "default" | "none" | {
5198
+ "tools": Array<string>;
5199
+ };
5200
+ "addons"?: Record<string, unknown>;
5201
+ };
5202
+ "treatment"?: {
5203
+ "tools"?: "default" | "none" | {
5204
+ "tools": Array<string>;
5205
+ };
5206
+ "addons"?: Record<string, unknown>;
5207
+ };
5208
+ };
5209
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5210
+ "subjectBackend": "model" | "harness" | null;
5211
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5212
+ "importedFrom"?: string;
5213
+ "baselineLabel": string | null;
5214
+ "treatmentLabel": string | null;
5215
+ "promiseMetrics": Array<{
5216
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5217
+ "direction": "down" | "up";
5218
+ "label": string;
5219
+ }>;
5220
+ "matrix": Array<{
5221
+ "harness": string;
5222
+ "model": string;
5223
+ "provider"?: string;
5224
+ "params"?: {
5225
+ "temperature"?: number;
5226
+ "maxOutputTokens"?: number;
5227
+ "reasoningEffort"?: "low" | "medium" | "high";
5228
+ "thinkingBudgetTokens"?: number;
5229
+ };
5230
+ }>;
5231
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5232
+ "judgeSupplementary"?: boolean;
5233
+ "taskCount": number;
5234
+ "variantsPerTask": number;
5235
+ "repeats": number;
5236
+ "subset": {
5237
+ "count": number;
5238
+ "seed": string;
5239
+ "of": number;
5240
+ } | null;
5241
+ "status": string;
5242
+ "error": string | null;
5243
+ "preparation"?: {
5244
+ "since": string;
5245
+ "launchedBy": {
5246
+ "userId"?: string;
5247
+ "role"?: string;
5248
+ };
5249
+ "waitingOn": Array<{
5250
+ "imageKey": string;
5251
+ "buildId": string;
5252
+ "stage"?: "image" | "snapshot";
5253
+ }>;
5254
+ } | null;
5255
+ "visibility": "private" | "org" | "link" | "public";
5256
+ "visibilityChangedAt"?: string | null;
5257
+ "visibilityChangedBy"?: string | null;
5258
+ "shareToken"?: string | null;
5259
+ "lowSampleOverride": boolean;
5260
+ "publishedAt": string | null;
5261
+ "verdict": unknown;
5262
+ "definitionVersionId": string | null;
5263
+ "runnerKind": "ora-runs" | "batch-job";
5264
+ "createdAt": string;
5265
+ "updatedAt": string;
5266
+ "liveUntil": string | null;
5267
+ };
5268
+ "streamUrl": string;
5269
+ "warnings"?: Array<{
5270
+ "harness": string;
5271
+ "kind": string;
5272
+ }>;
5273
+ };
5274
+ export type StudyUpdateLensesRequest = {
5275
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5276
+ };
5277
+ export type StudyUpdateLensesResponse = {
5278
+ "study": {
5279
+ "id": string;
5280
+ "executiveSummary"?: string | null;
5281
+ "plannedRunCount": number;
5282
+ "organizationId": string;
5283
+ "createdBy": string | null;
5284
+ "name": string;
5285
+ "slug": string;
5286
+ "description": string | null;
5287
+ "skillRef": string | null;
5288
+ "skillSlug": string | null;
5289
+ "skillName": string | null;
5290
+ "skillPromise": string;
5291
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5292
+ "subjectName": string | null;
5293
+ "subjectLabel": string;
5294
+ "armConfig": {
5295
+ "baseline"?: {
5296
+ "tools"?: "default" | "none" | {
5297
+ "tools": Array<string>;
5298
+ };
5299
+ "addons"?: Record<string, unknown>;
5300
+ };
5301
+ "treatment"?: {
5302
+ "tools"?: "default" | "none" | {
5303
+ "tools": Array<string>;
5304
+ };
5305
+ "addons"?: Record<string, unknown>;
5306
+ };
5307
+ };
5308
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5309
+ "subjectBackend": "model" | "harness" | null;
5310
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5311
+ "importedFrom"?: string;
5312
+ "baselineLabel": string | null;
5313
+ "treatmentLabel": string | null;
5314
+ "promiseMetrics": Array<{
5315
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5316
+ "direction": "down" | "up";
5317
+ "label": string;
5318
+ }>;
5319
+ "matrix": Array<{
5320
+ "harness": string;
5321
+ "model": string;
5322
+ "provider"?: string;
5323
+ "params"?: {
5324
+ "temperature"?: number;
5325
+ "maxOutputTokens"?: number;
5326
+ "reasoningEffort"?: "low" | "medium" | "high";
5327
+ "thinkingBudgetTokens"?: number;
5328
+ };
5329
+ }>;
5330
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5331
+ "judgeSupplementary"?: boolean;
5332
+ "taskCount": number;
5333
+ "variantsPerTask": number;
5334
+ "repeats": number;
5335
+ "subset": {
5336
+ "count": number;
5337
+ "seed": string;
5338
+ "of": number;
5339
+ } | null;
5340
+ "status": string;
5341
+ "error": string | null;
5342
+ "preparation"?: {
5343
+ "since": string;
5344
+ "launchedBy": {
5345
+ "userId"?: string;
5346
+ "role"?: string;
5347
+ };
5348
+ "waitingOn": Array<{
5349
+ "imageKey": string;
5350
+ "buildId": string;
5351
+ "stage"?: "image" | "snapshot";
5352
+ }>;
5353
+ } | null;
5354
+ "visibility": "private" | "org" | "link" | "public";
5355
+ "visibilityChangedAt"?: string | null;
5356
+ "visibilityChangedBy"?: string | null;
5357
+ "shareToken"?: string | null;
5358
+ "lowSampleOverride": boolean;
5359
+ "publishedAt": string | null;
5360
+ "verdict": unknown;
5361
+ "definitionVersionId": string | null;
5362
+ "runnerKind": "ora-runs" | "batch-job";
5363
+ "createdAt": string;
5364
+ "updatedAt": string;
5365
+ "liveUntil": string | null;
5366
+ };
5367
+ "warnings": Array<{
5368
+ "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
5369
+ "detail": string;
5370
+ "taskIds"?: Array<string>;
5371
+ }>;
5372
+ };
5373
+ export type StudyPreviewResponse = {
5374
+ "study": {
5375
+ "executiveSummary"?: string | null;
5376
+ "slug": string;
5377
+ "name": string;
5378
+ "description": string | null;
5379
+ "skillName": string | null;
5380
+ "skillSlug": string | null;
5381
+ "skillPromise": string;
5382
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5383
+ "subjectName": string | null;
5384
+ "subjectLabel": string;
5385
+ "armConfig": {
5386
+ "baseline"?: {
5387
+ "tools"?: "default" | "none" | {
5388
+ "tools": Array<string>;
5389
+ };
5390
+ "addons"?: Array<string>;
5391
+ };
5392
+ "treatment"?: {
5393
+ "tools"?: "default" | "none" | {
5394
+ "tools": Array<string>;
5395
+ };
5396
+ "addons"?: Array<string>;
5397
+ };
5398
+ };
5399
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5400
+ "subjectBackend"?: "model" | "harness";
5401
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5402
+ "baselineLabel": string | null;
5403
+ "treatmentLabel": string | null;
5404
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5405
+ "judgeSupplementary"?: true;
5406
+ "promiseMetrics": Array<{
5407
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5408
+ "direction": "down" | "up";
5409
+ "label": string;
5410
+ }>;
5411
+ "matrix": Array<{
5412
+ "harness": string;
5413
+ "model": string;
5414
+ "provider"?: string;
5415
+ "params"?: {
5416
+ "temperature"?: number;
5417
+ "maxOutputTokens"?: number;
5418
+ "reasoningEffort"?: "low" | "medium" | "high";
5419
+ "thinkingBudgetTokens"?: number;
5420
+ };
5421
+ }>;
5422
+ "taskCount": number;
5423
+ "variantsPerTask": number;
5424
+ "repeats": number;
5425
+ "status": string;
5426
+ "publishedAt": string | null;
5427
+ "createdAt": string;
5428
+ "viewCount"?: number;
5429
+ "boardEligibility"?: {
5430
+ "eligible": boolean;
5431
+ "refusedBy": string | null;
5432
+ "reason": string | null;
5433
+ "failures": Array<{
5434
+ "ruleId": string;
5435
+ "reason": string;
5436
+ }>;
5437
+ "evaluatedAt": string;
5438
+ };
5439
+ "subjectLicenseSpdx"?: string | null;
5440
+ "subjectSourceUrl"?: string | null;
5441
+ "subjectWithheldReason"?: string | null;
5442
+ "subjectNotices"?: Array<{
5443
+ "kind": string;
5444
+ "text": string;
5445
+ }>;
5446
+ };
5447
+ "tasks": Array<{
5448
+ "id": string;
5449
+ "position": number;
5450
+ "prompt"?: string;
5451
+ "rationale"?: string | null;
5452
+ "category": string | null;
5453
+ "requiresWeb": boolean;
5454
+ "rubric"?: Array<string>;
5455
+ "variants"?: Array<string> | null;
5456
+ "expectations"?: Array<unknown> | null;
5457
+ "textWithheld"?: boolean;
5458
+ }>;
5459
+ "runs": Array<{
5460
+ "runId": string;
5461
+ "taskId": string;
5462
+ "arm": "baseline" | "skill";
5463
+ "harness": string;
5464
+ "model": string;
5465
+ "provider": string | null;
5466
+ "variantIndex"?: number;
5467
+ "repeatIndex"?: number;
5468
+ "status": string;
5469
+ "outcome": string | null;
5470
+ "outcomeReason": string | null;
5471
+ "totalTokens": number | null;
5472
+ "inputTokens": number | null;
5473
+ "outputTokens": number | null;
5474
+ "cacheReadTokens": number | null;
5475
+ "cacheCreationTokens": number | null;
5476
+ "costUsd": number | null;
5477
+ "numTurns": number | null;
5478
+ "durationMs": number | null;
5479
+ "intentSatisfied": boolean | null;
5480
+ "frictionCount": number | null;
5481
+ "qualityScore": number | null;
5482
+ "gradedAt": string | null;
5483
+ "judgeModel": string | null;
5484
+ "pairVerdict": "skill" | "baseline" | "tie" | "unmeasurable" | null;
5485
+ "pairGradedAt": string | null;
5486
+ "evalScore": number | null;
5487
+ "evalGradedAt": string | null;
5488
+ "runCreatedAt": string | null;
5489
+ "startedAt": string | null;
5490
+ "finishedAt": string | null;
5491
+ "lensGrades"?: Record<string, unknown>;
5492
+ "rubricPassed": Array<boolean> | null;
5493
+ "evalOutcomes": Array<{
5494
+ "kind": string;
5495
+ "label"?: string;
5496
+ "passed": boolean;
5497
+ "abstained"?: boolean;
5498
+ "decisive"?: boolean;
5499
+ }> | null;
5500
+ "paramsKey"?: string;
5501
+ "textWithheld"?: boolean;
5502
+ }>;
5503
+ "aggregate": {
5504
+ "summary": {
5505
+ "verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
5506
+ "headline": Array<{
5507
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5508
+ "label": string;
5509
+ "direction": "down" | "up";
5510
+ "baselineAvg": number | null;
5511
+ "skillAvg": number | null;
5512
+ "deltaPct": number | null;
5513
+ "delta"?: {
5514
+ "unit": "pp";
5515
+ "absolute": number | null;
5516
+ "relativePct": number | null;
5517
+ } | {
5518
+ "unit": "pts";
5519
+ "absolute": number | null;
5520
+ "relativePct"?: number | null;
5521
+ } | {
5522
+ "unit": "%";
5523
+ "relativePct": number | null;
5524
+ };
5525
+ "improved": boolean;
5526
+ }>;
5527
+ "completionRate": {
5528
+ "baseline": number | null;
5529
+ "skill": number | null;
5530
+ };
5531
+ "goalRate": {
5532
+ "baseline": number | null;
5533
+ "skill": number | null;
5534
+ };
5535
+ "pairwise": {
5536
+ "pairs": number;
5537
+ "skillWins": number;
5538
+ "baselineWins": number;
5539
+ "ties": number;
5540
+ "winRate": number | null;
5541
+ };
5542
+ "decision"?: {
5543
+ "winner": "treatment" | "baseline" | "tie" | "unmeasurable";
5544
+ "basis": "infra" | "goal" | "evals" | "benchmark" | "quality" | "efficiency" | "judge" | "insufficient";
5545
+ "reason": string;
5546
+ "efficiencyDiscounted": boolean;
5547
+ };
5548
+ "leaderboard"?: Array<unknown>;
5549
+ "concentration": {
5550
+ "perCell": Array<{
5551
+ "cell": string;
5552
+ "goalDeltaPp": number | null;
5553
+ "contributionPct": number | null;
5554
+ }>;
5555
+ "heterogeneity": {
5556
+ "dominantCell": string;
5557
+ "contributionPct": number;
5558
+ } | null;
5559
+ };
5560
+ "matrixShape": {
5561
+ "cells": number;
5562
+ "harnesses": number;
5563
+ "models": number;
5564
+ "crossed": boolean;
5565
+ };
5566
+ "flakiness"?: {
5567
+ "measured": boolean;
5568
+ "conditions": number;
5569
+ "unstable": number;
5570
+ "worst": {
5571
+ "cell": string;
5572
+ "taskId": string;
5573
+ "arm": "baseline" | "skill";
5574
+ "variant"?: number;
5575
+ "passRate": number;
5576
+ } | null;
5577
+ };
5578
+ "arms"?: {
5579
+ "baseline": {
5580
+ "runs": number;
5581
+ "completed": number;
5582
+ "infraFailures": number;
5583
+ "completionRate": number | null;
5584
+ "goalRate": number | null;
5585
+ "goalTally": {
5586
+ "achieved": number;
5587
+ "partial": number;
5588
+ "missed": number;
5589
+ };
5590
+ "avgTotalTokens": number | null;
5591
+ "avgOutputTokens": number | null;
5592
+ "avgCostUsd": number | null;
5593
+ "avgDurationMs": number | null;
5594
+ "avgTurns": number | null;
5595
+ "costCoverage": number | null;
5596
+ "avgQualityScore": number | null;
5597
+ "gradedCoverage": number | null;
5598
+ "medianCostUsd": number | null;
5599
+ "medianTotalTokens": number | null;
5600
+ "medianDurationMs": number | null;
5601
+ "p95CostUsd": number | null;
5602
+ "p95TotalTokens": number | null;
5603
+ "p95DurationMs": number | null;
5604
+ "maxCostUsd": number | null;
5605
+ "maxTotalTokens": number | null;
5606
+ "maxDurationMs": number | null;
5607
+ "evalPassRate"?: number | null;
5608
+ "benchmarkPassRate"?: number | null;
5609
+ "adoptionRate"?: number | null;
5610
+ };
5611
+ "skill": {
5612
+ "runs": number;
5613
+ "completed": number;
5614
+ "infraFailures": number;
5615
+ "completionRate": number | null;
5616
+ "goalRate": number | null;
5617
+ "goalTally": {
5618
+ "achieved": number;
5619
+ "partial": number;
5620
+ "missed": number;
5621
+ };
5622
+ "avgTotalTokens": number | null;
5623
+ "avgOutputTokens": number | null;
5624
+ "avgCostUsd": number | null;
5625
+ "avgDurationMs": number | null;
5626
+ "avgTurns": number | null;
5627
+ "costCoverage": number | null;
5628
+ "avgQualityScore": number | null;
5629
+ "gradedCoverage": number | null;
5630
+ "medianCostUsd": number | null;
5631
+ "medianTotalTokens": number | null;
5632
+ "medianDurationMs": number | null;
5633
+ "p95CostUsd": number | null;
5634
+ "p95TotalTokens": number | null;
5635
+ "p95DurationMs": number | null;
5636
+ "maxCostUsd": number | null;
5637
+ "maxTotalTokens": number | null;
5638
+ "maxDurationMs": number | null;
5639
+ "evalPassRate"?: number | null;
5640
+ "benchmarkPassRate"?: number | null;
5641
+ "adoptionRate"?: number | null;
5642
+ };
5643
+ };
5644
+ "totals": {
5645
+ "cells": number;
5646
+ "runs": number;
5647
+ "completed": number;
5648
+ "failed": number;
5649
+ "costUsd": number;
5650
+ "tokens": number;
5651
+ };
5652
+ };
5653
+ "perCell": Array<unknown>;
5654
+ "perHarness": Array<unknown>;
5655
+ "perModel": Array<unknown>;
5656
+ "perTask": Array<unknown>;
5657
+ "webSplit": unknown;
5658
+ "inference": {
5659
+ "columns": Array<{
5660
+ "key": string;
5661
+ "label": string;
5662
+ "family": "rate" | "score" | "magnitude";
5663
+ "betterWhen": "higher" | "lower";
5664
+ }>;
5665
+ "groups": Array<{
5666
+ "kind": "cell" | "harness" | "model" | "field";
5667
+ "title": string;
5668
+ "rows": Array<{
5669
+ "key": string;
5670
+ "kind": "cell" | "harness" | "model" | "field";
5671
+ "harness": string | null;
5672
+ "model": string | null;
5673
+ "paramsKey"?: string;
5674
+ "runs": number;
5675
+ "cells": Record<string, unknown>;
5676
+ }>;
5677
+ }>;
5678
+ "reference": "baseline" | "field";
5679
+ "draws": number;
5680
+ } | null;
5681
+ "behavior"?: Array<{
5682
+ "harness": string;
5683
+ "model": string;
5684
+ "provider": string | null;
5685
+ "arm": string;
5686
+ "totalRuns": number;
5687
+ "signals": Array<{
5688
+ "id": string;
5689
+ "label": string;
5690
+ "tone": "good" | "bad" | "neutral";
5691
+ "runCount": number;
5692
+ "succeededCount": number;
5693
+ }>;
5694
+ }>;
5695
+ "reference"?: {
5696
+ "cells": Array<{
5697
+ "cell": string;
5698
+ "harness": string;
5699
+ "model": string;
5700
+ "provider": string | null;
5701
+ "paramsKey": string;
5702
+ "label": string;
5703
+ "passRate": {
5704
+ "value": number | null;
5705
+ "ci": Array<unknown> | null;
5706
+ "n": number;
5707
+ };
5708
+ "tasks": number;
5709
+ "graded": number;
5710
+ "unmeasured": number;
5711
+ "matchers": Array<string>;
5712
+ "launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
5713
+ }>;
5714
+ "overall": {
5715
+ "value": number | null;
5716
+ "ci": Array<unknown> | null;
5717
+ "n": number;
5718
+ };
5719
+ "graded": number;
5720
+ "unmeasured": number;
5721
+ "modelJudged": boolean;
5722
+ };
5723
+ "execution"?: {
5724
+ "graded": number;
5725
+ "measured": number;
5726
+ "passed": number;
5727
+ "passRate": number | null;
5728
+ "byStatus": {
5729
+ "passed": number;
5730
+ "failed": number;
5731
+ "timeout": number;
5732
+ "error": number;
5733
+ "unavailable": number;
5734
+ };
5735
+ "languages": Array<string>;
5736
+ "unavailableReasons": Array<string>;
5737
+ };
5738
+ };
5739
+ "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
5740
+ };
5741
+ export type StudyPublishRequest = {
5742
+ "slug"?: string;
5743
+ "visibility"?: "private" | "org" | "link" | "public";
5744
+ };
5745
+ export type StudyPublishResponse = {
5746
+ "study": {
5747
+ "id": string;
5748
+ "executiveSummary"?: string | null;
5749
+ "plannedRunCount": number;
5750
+ "organizationId": string;
5751
+ "createdBy": string | null;
5752
+ "name": string;
5753
+ "slug": string;
5754
+ "description": string | null;
5755
+ "skillRef": string | null;
5756
+ "skillSlug": string | null;
5757
+ "skillName": string | null;
5758
+ "skillPromise": string;
5759
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5760
+ "subjectName": string | null;
5761
+ "subjectLabel": string;
5762
+ "armConfig": {
5763
+ "baseline"?: {
5764
+ "tools"?: "default" | "none" | {
5765
+ "tools": Array<string>;
5766
+ };
5767
+ "addons"?: Record<string, unknown>;
5768
+ };
5769
+ "treatment"?: {
5770
+ "tools"?: "default" | "none" | {
5771
+ "tools": Array<string>;
5772
+ };
5773
+ "addons"?: Record<string, unknown>;
5774
+ };
5775
+ };
5776
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5777
+ "subjectBackend": "model" | "harness" | null;
5778
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5779
+ "importedFrom"?: string;
5780
+ "baselineLabel": string | null;
5781
+ "treatmentLabel": string | null;
5782
+ "promiseMetrics": Array<{
5783
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5784
+ "direction": "down" | "up";
5785
+ "label": string;
5786
+ }>;
5787
+ "matrix": Array<{
5788
+ "harness": string;
5789
+ "model": string;
5790
+ "provider"?: string;
5791
+ "params"?: {
5792
+ "temperature"?: number;
5793
+ "maxOutputTokens"?: number;
5794
+ "reasoningEffort"?: "low" | "medium" | "high";
5795
+ "thinkingBudgetTokens"?: number;
5796
+ };
5797
+ }>;
5798
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5799
+ "judgeSupplementary"?: boolean;
5800
+ "taskCount": number;
5801
+ "variantsPerTask": number;
5802
+ "repeats": number;
5803
+ "subset": {
5804
+ "count": number;
5805
+ "seed": string;
5806
+ "of": number;
5807
+ } | null;
5808
+ "status": string;
5809
+ "error": string | null;
5810
+ "preparation"?: {
5811
+ "since": string;
5812
+ "launchedBy": {
5813
+ "userId"?: string;
5814
+ "role"?: string;
5815
+ };
5816
+ "waitingOn": Array<{
5817
+ "imageKey": string;
5818
+ "buildId": string;
5819
+ "stage"?: "image" | "snapshot";
5820
+ }>;
5821
+ } | null;
5822
+ "visibility": "private" | "org" | "link" | "public";
5823
+ "visibilityChangedAt"?: string | null;
5824
+ "visibilityChangedBy"?: string | null;
5825
+ "shareToken"?: string | null;
5826
+ "lowSampleOverride": boolean;
5827
+ "publishedAt": string | null;
5828
+ "verdict": unknown;
5829
+ "definitionVersionId": string | null;
5830
+ "runnerKind": "ora-runs" | "batch-job";
5831
+ "createdAt": string;
5832
+ "updatedAt": string;
5833
+ "liveUntil": string | null;
5834
+ };
5835
+ };
5836
+ export type StudyReconcileRunsResponse = {
5837
+ "studyId": string;
5838
+ "laggingRuns": number;
5839
+ "projected": number;
5840
+ "refinedInsights": number;
5841
+ "redrivenLensGrades": Record<string, unknown>;
5842
+ "skipped": boolean;
5843
+ "status": string;
5844
+ };
5845
+ export type StudyRegradeResponse = {
5846
+ "study": {
5847
+ "id": string;
5848
+ "executiveSummary"?: string | null;
5849
+ "plannedRunCount": number;
5850
+ "organizationId": string;
5851
+ "createdBy": string | null;
5852
+ "name": string;
5853
+ "slug": string;
5854
+ "description": string | null;
5855
+ "skillRef": string | null;
5856
+ "skillSlug": string | null;
5857
+ "skillName": string | null;
5858
+ "skillPromise": string;
5859
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5860
+ "subjectName": string | null;
5861
+ "subjectLabel": string;
5862
+ "armConfig": {
5863
+ "baseline"?: {
5864
+ "tools"?: "default" | "none" | {
5865
+ "tools": Array<string>;
5866
+ };
5867
+ "addons"?: Record<string, unknown>;
5868
+ };
5869
+ "treatment"?: {
5870
+ "tools"?: "default" | "none" | {
5871
+ "tools": Array<string>;
5872
+ };
5873
+ "addons"?: Record<string, unknown>;
5874
+ };
5875
+ };
5876
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5877
+ "subjectBackend": "model" | "harness" | null;
5878
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5879
+ "importedFrom"?: string;
5880
+ "baselineLabel": string | null;
5881
+ "treatmentLabel": string | null;
5882
+ "promiseMetrics": Array<{
5883
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5884
+ "direction": "down" | "up";
5885
+ "label": string;
5886
+ }>;
5887
+ "matrix": Array<{
5888
+ "harness": string;
5889
+ "model": string;
5890
+ "provider"?: string;
5891
+ "params"?: {
5892
+ "temperature"?: number;
5893
+ "maxOutputTokens"?: number;
5894
+ "reasoningEffort"?: "low" | "medium" | "high";
5895
+ "thinkingBudgetTokens"?: number;
5896
+ };
5897
+ }>;
5898
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
5899
+ "judgeSupplementary"?: boolean;
5900
+ "taskCount": number;
5901
+ "variantsPerTask": number;
5902
+ "repeats": number;
5903
+ "subset": {
5904
+ "count": number;
5905
+ "seed": string;
5906
+ "of": number;
5907
+ } | null;
5908
+ "status": string;
5909
+ "error": string | null;
5910
+ "preparation"?: {
5911
+ "since": string;
5912
+ "launchedBy": {
5913
+ "userId"?: string;
5914
+ "role"?: string;
5915
+ };
5916
+ "waitingOn": Array<{
5917
+ "imageKey": string;
5918
+ "buildId": string;
5919
+ "stage"?: "image" | "snapshot";
5920
+ }>;
5921
+ } | null;
5922
+ "visibility": "private" | "org" | "link" | "public";
5923
+ "visibilityChangedAt"?: string | null;
5924
+ "visibilityChangedBy"?: string | null;
5925
+ "shareToken"?: string | null;
5926
+ "lowSampleOverride": boolean;
5927
+ "publishedAt": string | null;
5928
+ "verdict": unknown;
5929
+ "definitionVersionId": string | null;
5930
+ "runnerKind": "ora-runs" | "batch-job";
5931
+ "createdAt": string;
5932
+ "updatedAt": string;
5933
+ "liveUntil": string | null;
5934
+ };
5935
+ "streamUrl": string;
5936
+ "warnings"?: Array<{
5937
+ "harness": string;
5938
+ "kind": string;
5939
+ }>;
5940
+ };
5941
+ export type StudyListRunAnswersResponse = {
5942
+ "coordinatorRunId": string;
5943
+ "answers": Array<{
5944
+ "runId": string;
5945
+ "createdAt": string;
5946
+ "status": string | null;
5947
+ "outcome": string | null;
5948
+ "totalTokens": number | null;
5949
+ "costUsd": number | null;
5950
+ "startedAt": string | null;
5951
+ "finishedAt": string | null;
5952
+ }>;
5953
+ };
5954
+ export type StudyRunInsightResponse = unknown;
5955
+ export type StudyRunTrajectoryResponse = unknown;
5956
+ export type StudyRotateShareTokenResponse = {
5957
+ "study": {
5958
+ "id": string;
5959
+ "executiveSummary"?: string | null;
5960
+ "plannedRunCount": number;
5961
+ "organizationId": string;
5962
+ "createdBy": string | null;
5963
+ "name": string;
5964
+ "slug": string;
5965
+ "description": string | null;
5966
+ "skillRef": string | null;
5967
+ "skillSlug": string | null;
5968
+ "skillName": string | null;
5969
+ "skillPromise": string;
5970
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
5971
+ "subjectName": string | null;
5972
+ "subjectLabel": string;
5973
+ "armConfig": {
5974
+ "baseline"?: {
5975
+ "tools"?: "default" | "none" | {
5976
+ "tools": Array<string>;
5977
+ };
5978
+ "addons"?: Record<string, unknown>;
5979
+ };
5980
+ "treatment"?: {
5981
+ "tools"?: "default" | "none" | {
5982
+ "tools": Array<string>;
5983
+ };
5984
+ "addons"?: Record<string, unknown>;
5985
+ };
5986
+ };
5987
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
5988
+ "subjectBackend": "model" | "harness" | null;
5989
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
5990
+ "importedFrom"?: string;
5991
+ "baselineLabel": string | null;
5992
+ "treatmentLabel": string | null;
5993
+ "promiseMetrics": Array<{
5994
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
5995
+ "direction": "down" | "up";
5996
+ "label": string;
5997
+ }>;
5998
+ "matrix": Array<{
5999
+ "harness": string;
6000
+ "model": string;
6001
+ "provider"?: string;
6002
+ "params"?: {
6003
+ "temperature"?: number;
6004
+ "maxOutputTokens"?: number;
6005
+ "reasoningEffort"?: "low" | "medium" | "high";
6006
+ "thinkingBudgetTokens"?: number;
6007
+ };
6008
+ }>;
6009
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
6010
+ "judgeSupplementary"?: boolean;
6011
+ "taskCount": number;
6012
+ "variantsPerTask": number;
6013
+ "repeats": number;
6014
+ "subset": {
6015
+ "count": number;
6016
+ "seed": string;
6017
+ "of": number;
6018
+ } | null;
6019
+ "status": string;
6020
+ "error": string | null;
6021
+ "preparation"?: {
6022
+ "since": string;
6023
+ "launchedBy": {
6024
+ "userId"?: string;
6025
+ "role"?: string;
6026
+ };
6027
+ "waitingOn": Array<{
6028
+ "imageKey": string;
6029
+ "buildId": string;
6030
+ "stage"?: "image" | "snapshot";
6031
+ }>;
6032
+ } | null;
6033
+ "visibility": "private" | "org" | "link" | "public";
6034
+ "visibilityChangedAt"?: string | null;
6035
+ "visibilityChangedBy"?: string | null;
6036
+ "shareToken"?: string | null;
6037
+ "lowSampleOverride": boolean;
6038
+ "publishedAt": string | null;
6039
+ "verdict": unknown;
6040
+ "definitionVersionId": string | null;
6041
+ "runnerKind": "ora-runs" | "batch-job";
6042
+ "createdAt": string;
6043
+ "updatedAt": string;
6044
+ "liveUntil": string | null;
6045
+ };
6046
+ "shareUrl": string | null;
6047
+ };
6048
+ export type StudyStreamResponse = unknown;
6049
+ export type StudyUpdateTaskRequest = {
6050
+ "prompt": string;
6051
+ "expectations"?: Array<unknown>;
6052
+ };
6053
+ export type StudyUpdateTaskResponse = {
6054
+ "task": {
6055
+ "id": string;
6056
+ "studyId": string;
6057
+ "position": number;
6058
+ "prompt": string;
6059
+ "rationale": string | null;
6060
+ "category": string | null;
6061
+ "requiresWeb": boolean;
6062
+ "rubric": Array<string>;
6063
+ "variants"?: Array<string> | null;
6064
+ "expectations"?: Array<unknown> | null;
6065
+ };
6066
+ };
6067
+ export type StudyUnpublishResponse = {
6068
+ "study": {
6069
+ "id": string;
6070
+ "executiveSummary"?: string | null;
6071
+ "plannedRunCount": number;
6072
+ "organizationId": string;
6073
+ "createdBy": string | null;
6074
+ "name": string;
6075
+ "slug": string;
6076
+ "description": string | null;
6077
+ "skillRef": string | null;
6078
+ "skillSlug": string | null;
6079
+ "skillName": string | null;
6080
+ "skillPromise": string;
6081
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
6082
+ "subjectName": string | null;
6083
+ "subjectLabel": string;
6084
+ "armConfig": {
6085
+ "baseline"?: {
6086
+ "tools"?: "default" | "none" | {
6087
+ "tools": Array<string>;
6088
+ };
6089
+ "addons"?: Record<string, unknown>;
6090
+ };
6091
+ "treatment"?: {
6092
+ "tools"?: "default" | "none" | {
6093
+ "tools": Array<string>;
6094
+ };
6095
+ "addons"?: Record<string, unknown>;
6096
+ };
6097
+ };
6098
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
6099
+ "subjectBackend": "model" | "harness" | null;
6100
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6101
+ "importedFrom"?: string;
6102
+ "baselineLabel": string | null;
6103
+ "treatmentLabel": string | null;
6104
+ "promiseMetrics": Array<{
6105
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6106
+ "direction": "down" | "up";
6107
+ "label": string;
6108
+ }>;
6109
+ "matrix": Array<{
6110
+ "harness": string;
6111
+ "model": string;
6112
+ "provider"?: string;
6113
+ "params"?: {
6114
+ "temperature"?: number;
6115
+ "maxOutputTokens"?: number;
6116
+ "reasoningEffort"?: "low" | "medium" | "high";
6117
+ "thinkingBudgetTokens"?: number;
6118
+ };
6119
+ }>;
6120
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
6121
+ "judgeSupplementary"?: boolean;
6122
+ "taskCount": number;
6123
+ "variantsPerTask": number;
6124
+ "repeats": number;
6125
+ "subset": {
6126
+ "count": number;
6127
+ "seed": string;
6128
+ "of": number;
6129
+ } | null;
6130
+ "status": string;
6131
+ "error": string | null;
6132
+ "preparation"?: {
6133
+ "since": string;
6134
+ "launchedBy": {
6135
+ "userId"?: string;
6136
+ "role"?: string;
6137
+ };
6138
+ "waitingOn": Array<{
6139
+ "imageKey": string;
6140
+ "buildId": string;
6141
+ "stage"?: "image" | "snapshot";
6142
+ }>;
6143
+ } | null;
6144
+ "visibility": "private" | "org" | "link" | "public";
6145
+ "visibilityChangedAt"?: string | null;
6146
+ "visibilityChangedBy"?: string | null;
6147
+ "shareToken"?: string | null;
6148
+ "lowSampleOverride": boolean;
6149
+ "publishedAt": string | null;
6150
+ "verdict": unknown;
6151
+ "definitionVersionId": string | null;
6152
+ "runnerKind": "ora-runs" | "batch-job";
6153
+ "createdAt": string;
6154
+ "updatedAt": string;
6155
+ "liveUntil": string | null;
6156
+ };
6157
+ };
6158
+ export type StudySetVisibilityRequest = {
6159
+ "visibility": "private" | "org" | "link" | "public";
6160
+ };
6161
+ export type StudySetVisibilityResponse = {
6162
+ "study": {
6163
+ "id": string;
6164
+ "executiveSummary"?: string | null;
6165
+ "plannedRunCount": number;
6166
+ "organizationId": string;
6167
+ "createdBy": string | null;
6168
+ "name": string;
6169
+ "slug": string;
6170
+ "description": string | null;
6171
+ "skillRef": string | null;
6172
+ "skillSlug": string | null;
6173
+ "skillName": string | null;
6174
+ "skillPromise": string;
6175
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
6176
+ "subjectName": string | null;
6177
+ "subjectLabel": string;
6178
+ "armConfig": {
6179
+ "baseline"?: {
6180
+ "tools"?: "default" | "none" | {
6181
+ "tools": Array<string>;
6182
+ };
6183
+ "addons"?: Record<string, unknown>;
6184
+ };
6185
+ "treatment"?: {
6186
+ "tools"?: "default" | "none" | {
6187
+ "tools": Array<string>;
6188
+ };
6189
+ "addons"?: Record<string, unknown>;
6190
+ };
6191
+ };
6192
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single";
6193
+ "subjectBackend": "model" | "harness" | null;
6194
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
6195
+ "importedFrom"?: string;
6196
+ "baselineLabel": string | null;
6197
+ "treatmentLabel": string | null;
6198
+ "promiseMetrics": Array<{
6199
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
6200
+ "direction": "down" | "up";
6201
+ "label": string;
6202
+ }>;
6203
+ "matrix": Array<{
6204
+ "harness": string;
6205
+ "model": string;
6206
+ "provider"?: string;
6207
+ "params"?: {
6208
+ "temperature"?: number;
6209
+ "maxOutputTokens"?: number;
6210
+ "reasoningEffort"?: "low" | "medium" | "high";
6211
+ "thinkingBudgetTokens"?: number;
6212
+ };
6213
+ }>;
6214
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
6215
+ "judgeSupplementary"?: boolean;
6216
+ "taskCount": number;
6217
+ "variantsPerTask": number;
6218
+ "repeats": number;
6219
+ "subset": {
6220
+ "count": number;
6221
+ "seed": string;
6222
+ "of": number;
6223
+ } | null;
6224
+ "status": string;
6225
+ "error": string | null;
6226
+ "preparation"?: {
6227
+ "since": string;
6228
+ "launchedBy": {
6229
+ "userId"?: string;
6230
+ "role"?: string;
6231
+ };
6232
+ "waitingOn": Array<{
6233
+ "imageKey": string;
6234
+ "buildId": string;
6235
+ "stage"?: "image" | "snapshot";
6236
+ }>;
6237
+ } | null;
6238
+ "visibility": "private" | "org" | "link" | "public";
6239
+ "visibilityChangedAt"?: string | null;
6240
+ "visibilityChangedBy"?: string | null;
6241
+ "shareToken"?: string | null;
6242
+ "lowSampleOverride": boolean;
6243
+ "publishedAt": string | null;
6244
+ "verdict": unknown;
6245
+ "definitionVersionId": string | null;
6246
+ "runnerKind": "ora-runs" | "batch-job";
6247
+ "createdAt": string;
6248
+ "updatedAt": string;
6249
+ "liveUntil": string | null;
6250
+ };
6251
+ "shareUrl": string | null;
6252
+ };
2441
6253
  export type TunnelsListResponse = {
2442
6254
  "tunnels": Array<{
2443
6255
  "id": string;