@ora-ai/cli 0.7.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -6
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +3280 -354
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/types.d.ts +3837 -58
- package/dist/generated/types.d.ts.map +1 -1
- package/dist/generated/types.js +1 -1
- package/package.json +2 -2
|
@@ -14,10 +14,11 @@ export type AddonsListResponse = {
|
|
|
14
14
|
"configSchema": Record<string, unknown>;
|
|
15
15
|
"surfaces": Array<"run" | "session">;
|
|
16
16
|
"harnesses": Array<string>;
|
|
17
|
+
"multiInstance"?: boolean;
|
|
17
18
|
}>;
|
|
18
19
|
};
|
|
19
20
|
export type BenchmarksListResponse = {
|
|
20
|
-
"
|
|
21
|
+
"studies": Array<{
|
|
21
22
|
"id": string;
|
|
22
23
|
"executiveSummary"?: string | null;
|
|
23
24
|
"plannedRunCount": number;
|
|
@@ -30,7 +31,7 @@ export type BenchmarksListResponse = {
|
|
|
30
31
|
"skillSlug": string | null;
|
|
31
32
|
"skillName": string | null;
|
|
32
33
|
"skillPromise": string;
|
|
33
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
34
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
34
35
|
"subjectName": string | null;
|
|
35
36
|
"subjectLabel": string;
|
|
36
37
|
"armConfig": {
|
|
@@ -48,6 +49,9 @@ export type BenchmarksListResponse = {
|
|
|
48
49
|
};
|
|
49
50
|
};
|
|
50
51
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
52
|
+
"subjectBackend": "model" | "harness" | null;
|
|
53
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
54
|
+
"importedFrom"?: string;
|
|
51
55
|
"baselineLabel": string | null;
|
|
52
56
|
"treatmentLabel": string | null;
|
|
53
57
|
"promiseMetrics": Array<{
|
|
@@ -59,26 +63,56 @@ export type BenchmarksListResponse = {
|
|
|
59
63
|
"harness": string;
|
|
60
64
|
"model": string;
|
|
61
65
|
"provider"?: string;
|
|
66
|
+
"params"?: {
|
|
67
|
+
"temperature"?: number;
|
|
68
|
+
"maxOutputTokens"?: number;
|
|
69
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
70
|
+
"thinkingBudgetTokens"?: number;
|
|
71
|
+
};
|
|
62
72
|
}>;
|
|
63
|
-
"lenses": Array<"judge" | "evals">;
|
|
73
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
64
74
|
"taskCount": number;
|
|
65
75
|
"variantsPerTask": number;
|
|
66
76
|
"repeats": number;
|
|
77
|
+
"subset": {
|
|
78
|
+
"count": number;
|
|
79
|
+
"seed": string;
|
|
80
|
+
"of": number;
|
|
81
|
+
} | null;
|
|
67
82
|
"status": string;
|
|
68
83
|
"error": string | null;
|
|
69
|
-
"
|
|
84
|
+
"preparation"?: {
|
|
85
|
+
"since": string;
|
|
86
|
+
"launchedBy": {
|
|
87
|
+
"userId"?: string;
|
|
88
|
+
"role"?: string;
|
|
89
|
+
};
|
|
90
|
+
"waitingOn": Array<{
|
|
91
|
+
"imageKey": string;
|
|
92
|
+
"buildId": string;
|
|
93
|
+
"stage"?: "image" | "snapshot";
|
|
94
|
+
}>;
|
|
95
|
+
} | null;
|
|
96
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
97
|
+
"visibilityChangedAt"?: string | null;
|
|
98
|
+
"visibilityChangedBy"?: string | null;
|
|
99
|
+
"shareToken"?: string | null;
|
|
70
100
|
"lowSampleOverride": boolean;
|
|
71
101
|
"publishedAt": string | null;
|
|
72
102
|
"verdict": unknown;
|
|
103
|
+
"definitionVersionId": string | null;
|
|
104
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
73
105
|
"createdAt": string;
|
|
74
106
|
"updatedAt": string;
|
|
107
|
+
"liveUntil": string | null;
|
|
75
108
|
}>;
|
|
76
109
|
};
|
|
77
110
|
export type BenchmarksCreateRequest = {
|
|
78
111
|
"name": string;
|
|
79
112
|
"description"?: string;
|
|
80
|
-
"subjectKind"?: "skill" | "tools" | "addon" | "custom";
|
|
113
|
+
"subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
81
114
|
"skillRef"?: string;
|
|
115
|
+
"definitionVersionId"?: string;
|
|
82
116
|
"subjectName"?: string;
|
|
83
117
|
"armConfig"?: {
|
|
84
118
|
"baseline"?: {
|
|
@@ -94,9 +128,10 @@ export type BenchmarksCreateRequest = {
|
|
|
94
128
|
"addons"?: Record<string, unknown>;
|
|
95
129
|
};
|
|
96
130
|
};
|
|
97
|
-
"lenses"?: Array<"judge" | "evals">;
|
|
131
|
+
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
98
132
|
"proposeExpectations"?: boolean;
|
|
99
133
|
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
134
|
+
"subjectBackend"?: "model" | "harness";
|
|
100
135
|
"baselineLabel"?: string;
|
|
101
136
|
"treatmentLabel"?: string;
|
|
102
137
|
"taskCount"?: number;
|
|
@@ -111,15 +146,25 @@ export type BenchmarksCreateRequest = {
|
|
|
111
146
|
"variants"?: Array<string>;
|
|
112
147
|
"expectations"?: Array<unknown>;
|
|
113
148
|
}>;
|
|
149
|
+
"subset"?: {
|
|
150
|
+
"count": number;
|
|
151
|
+
"seed"?: string;
|
|
152
|
+
};
|
|
114
153
|
"matrix": Array<{
|
|
115
154
|
"harness": string;
|
|
116
155
|
"model": string;
|
|
117
156
|
"provider"?: string;
|
|
157
|
+
"params"?: {
|
|
158
|
+
"temperature"?: number;
|
|
159
|
+
"maxOutputTokens"?: number;
|
|
160
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
161
|
+
"thinkingBudgetTokens"?: number;
|
|
162
|
+
};
|
|
118
163
|
}>;
|
|
119
164
|
"promise"?: string;
|
|
120
165
|
};
|
|
121
166
|
export type BenchmarksCreateResponse = {
|
|
122
|
-
"
|
|
167
|
+
"study": {
|
|
123
168
|
"id": string;
|
|
124
169
|
"executiveSummary"?: string | null;
|
|
125
170
|
"plannedRunCount": number;
|
|
@@ -132,7 +177,7 @@ export type BenchmarksCreateResponse = {
|
|
|
132
177
|
"skillSlug": string | null;
|
|
133
178
|
"skillName": string | null;
|
|
134
179
|
"skillPromise": string;
|
|
135
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
180
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
136
181
|
"subjectName": string | null;
|
|
137
182
|
"subjectLabel": string;
|
|
138
183
|
"armConfig": {
|
|
@@ -150,6 +195,9 @@ export type BenchmarksCreateResponse = {
|
|
|
150
195
|
};
|
|
151
196
|
};
|
|
152
197
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
198
|
+
"subjectBackend": "model" | "harness" | null;
|
|
199
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
200
|
+
"importedFrom"?: string;
|
|
153
201
|
"baselineLabel": string | null;
|
|
154
202
|
"treatmentLabel": string | null;
|
|
155
203
|
"promiseMetrics": Array<{
|
|
@@ -161,19 +209,48 @@ export type BenchmarksCreateResponse = {
|
|
|
161
209
|
"harness": string;
|
|
162
210
|
"model": string;
|
|
163
211
|
"provider"?: string;
|
|
212
|
+
"params"?: {
|
|
213
|
+
"temperature"?: number;
|
|
214
|
+
"maxOutputTokens"?: number;
|
|
215
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
216
|
+
"thinkingBudgetTokens"?: number;
|
|
217
|
+
};
|
|
164
218
|
}>;
|
|
165
|
-
"lenses": Array<"judge" | "evals">;
|
|
219
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
166
220
|
"taskCount": number;
|
|
167
221
|
"variantsPerTask": number;
|
|
168
222
|
"repeats": number;
|
|
223
|
+
"subset": {
|
|
224
|
+
"count": number;
|
|
225
|
+
"seed": string;
|
|
226
|
+
"of": number;
|
|
227
|
+
} | null;
|
|
169
228
|
"status": string;
|
|
170
229
|
"error": string | null;
|
|
171
|
-
"
|
|
230
|
+
"preparation"?: {
|
|
231
|
+
"since": string;
|
|
232
|
+
"launchedBy": {
|
|
233
|
+
"userId"?: string;
|
|
234
|
+
"role"?: string;
|
|
235
|
+
};
|
|
236
|
+
"waitingOn": Array<{
|
|
237
|
+
"imageKey": string;
|
|
238
|
+
"buildId": string;
|
|
239
|
+
"stage"?: "image" | "snapshot";
|
|
240
|
+
}>;
|
|
241
|
+
} | null;
|
|
242
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
243
|
+
"visibilityChangedAt"?: string | null;
|
|
244
|
+
"visibilityChangedBy"?: string | null;
|
|
245
|
+
"shareToken"?: string | null;
|
|
172
246
|
"lowSampleOverride": boolean;
|
|
173
247
|
"publishedAt": string | null;
|
|
174
248
|
"verdict": unknown;
|
|
249
|
+
"definitionVersionId": string | null;
|
|
250
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
175
251
|
"createdAt": string;
|
|
176
252
|
"updatedAt": string;
|
|
253
|
+
"liveUntil": string | null;
|
|
177
254
|
};
|
|
178
255
|
"streamUrl": string;
|
|
179
256
|
"warnings"?: Array<{
|
|
@@ -194,7 +271,7 @@ export type BenchmarksConvertWebmcpSuiteResponse = {
|
|
|
194
271
|
"warnings": Array<string>;
|
|
195
272
|
};
|
|
196
273
|
export type BenchmarksGetResponse = {
|
|
197
|
-
"
|
|
274
|
+
"study": {
|
|
198
275
|
"id": string;
|
|
199
276
|
"executiveSummary"?: string | null;
|
|
200
277
|
"plannedRunCount": number;
|
|
@@ -207,7 +284,7 @@ export type BenchmarksGetResponse = {
|
|
|
207
284
|
"skillSlug": string | null;
|
|
208
285
|
"skillName": string | null;
|
|
209
286
|
"skillPromise": string;
|
|
210
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
287
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
211
288
|
"subjectName": string | null;
|
|
212
289
|
"subjectLabel": string;
|
|
213
290
|
"armConfig": {
|
|
@@ -225,6 +302,9 @@ export type BenchmarksGetResponse = {
|
|
|
225
302
|
};
|
|
226
303
|
};
|
|
227
304
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
305
|
+
"subjectBackend": "model" | "harness" | null;
|
|
306
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
307
|
+
"importedFrom"?: string;
|
|
228
308
|
"baselineLabel": string | null;
|
|
229
309
|
"treatmentLabel": string | null;
|
|
230
310
|
"promiseMetrics": Array<{
|
|
@@ -236,23 +316,52 @@ export type BenchmarksGetResponse = {
|
|
|
236
316
|
"harness": string;
|
|
237
317
|
"model": string;
|
|
238
318
|
"provider"?: string;
|
|
319
|
+
"params"?: {
|
|
320
|
+
"temperature"?: number;
|
|
321
|
+
"maxOutputTokens"?: number;
|
|
322
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
323
|
+
"thinkingBudgetTokens"?: number;
|
|
324
|
+
};
|
|
239
325
|
}>;
|
|
240
|
-
"lenses": Array<"judge" | "evals">;
|
|
326
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
241
327
|
"taskCount": number;
|
|
242
328
|
"variantsPerTask": number;
|
|
243
329
|
"repeats": number;
|
|
330
|
+
"subset": {
|
|
331
|
+
"count": number;
|
|
332
|
+
"seed": string;
|
|
333
|
+
"of": number;
|
|
334
|
+
} | null;
|
|
244
335
|
"status": string;
|
|
245
336
|
"error": string | null;
|
|
246
|
-
"
|
|
337
|
+
"preparation"?: {
|
|
338
|
+
"since": string;
|
|
339
|
+
"launchedBy": {
|
|
340
|
+
"userId"?: string;
|
|
341
|
+
"role"?: string;
|
|
342
|
+
};
|
|
343
|
+
"waitingOn": Array<{
|
|
344
|
+
"imageKey": string;
|
|
345
|
+
"buildId": string;
|
|
346
|
+
"stage"?: "image" | "snapshot";
|
|
347
|
+
}>;
|
|
348
|
+
} | null;
|
|
349
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
350
|
+
"visibilityChangedAt"?: string | null;
|
|
351
|
+
"visibilityChangedBy"?: string | null;
|
|
352
|
+
"shareToken"?: string | null;
|
|
247
353
|
"lowSampleOverride": boolean;
|
|
248
354
|
"publishedAt": string | null;
|
|
249
355
|
"verdict": unknown;
|
|
356
|
+
"definitionVersionId": string | null;
|
|
357
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
250
358
|
"createdAt": string;
|
|
251
359
|
"updatedAt": string;
|
|
360
|
+
"liveUntil": string | null;
|
|
252
361
|
};
|
|
253
362
|
"tasks": Array<{
|
|
254
363
|
"id": string;
|
|
255
|
-
"
|
|
364
|
+
"studyId": string;
|
|
256
365
|
"position": number;
|
|
257
366
|
"prompt": string;
|
|
258
367
|
"rationale": string | null;
|
|
@@ -312,6 +421,55 @@ export type BenchmarksGetResponse = {
|
|
|
312
421
|
"evalResults": unknown | null;
|
|
313
422
|
"evalScore": number | null;
|
|
314
423
|
"evalGradedAt": string | null;
|
|
424
|
+
"lensGrades"?: {
|
|
425
|
+
"reference"?: {
|
|
426
|
+
"pass": boolean | null;
|
|
427
|
+
"extracted": string | null;
|
|
428
|
+
"extraction": string;
|
|
429
|
+
"matcher": string;
|
|
430
|
+
"votes"?: Array<{
|
|
431
|
+
"judge": string;
|
|
432
|
+
"answer": "equivalent" | "different" | "not_attempted" | null;
|
|
433
|
+
}>;
|
|
434
|
+
"reason"?: string;
|
|
435
|
+
"gradedAt": string;
|
|
436
|
+
};
|
|
437
|
+
"framework"?: {
|
|
438
|
+
"framework": string;
|
|
439
|
+
"primary": {
|
|
440
|
+
"metricId": string;
|
|
441
|
+
"provenance": string;
|
|
442
|
+
"value": number;
|
|
443
|
+
"scenario": string;
|
|
444
|
+
"count": number;
|
|
445
|
+
} | null;
|
|
446
|
+
"secondary": Array<{
|
|
447
|
+
"label": string;
|
|
448
|
+
"name": string;
|
|
449
|
+
"scenario": string;
|
|
450
|
+
"value": number | null;
|
|
451
|
+
"count": number;
|
|
452
|
+
"split": string | null;
|
|
453
|
+
}>;
|
|
454
|
+
"errors": Array<{
|
|
455
|
+
"scenario": string;
|
|
456
|
+
"metric": string;
|
|
457
|
+
"reason": string;
|
|
458
|
+
}>;
|
|
459
|
+
"gradedAt": string;
|
|
460
|
+
};
|
|
461
|
+
"execution"?: {
|
|
462
|
+
"pass": boolean | null;
|
|
463
|
+
"status": "passed" | "failed" | "timeout" | "error" | "unavailable";
|
|
464
|
+
"language": string;
|
|
465
|
+
"passedTests": number | null;
|
|
466
|
+
"failedTests": number | null;
|
|
467
|
+
"stdout": string;
|
|
468
|
+
"stdoutTruncated": boolean;
|
|
469
|
+
"reason"?: string;
|
|
470
|
+
"gradedAt": string;
|
|
471
|
+
};
|
|
472
|
+
} | null;
|
|
315
473
|
"runCreatedAt": string | null;
|
|
316
474
|
"startedAt": string | null;
|
|
317
475
|
"finishedAt": string | null;
|
|
@@ -484,6 +642,7 @@ export type BenchmarksGetResponse = {
|
|
|
484
642
|
"kind": "cell" | "harness" | "model" | "field";
|
|
485
643
|
"harness": string | null;
|
|
486
644
|
"model": string | null;
|
|
645
|
+
"paramsKey"?: string;
|
|
487
646
|
"runs": number;
|
|
488
647
|
"cells": Record<string, unknown>;
|
|
489
648
|
}>;
|
|
@@ -505,6 +664,49 @@ export type BenchmarksGetResponse = {
|
|
|
505
664
|
"succeededCount": number;
|
|
506
665
|
}>;
|
|
507
666
|
}>;
|
|
667
|
+
"reference"?: {
|
|
668
|
+
"cells": Array<{
|
|
669
|
+
"cell": string;
|
|
670
|
+
"harness": string;
|
|
671
|
+
"model": string;
|
|
672
|
+
"provider": string | null;
|
|
673
|
+
"paramsKey": string;
|
|
674
|
+
"label": string;
|
|
675
|
+
"passRate": {
|
|
676
|
+
"value": number | null;
|
|
677
|
+
"ci": Array<unknown> | null;
|
|
678
|
+
"n": number;
|
|
679
|
+
};
|
|
680
|
+
"tasks": number;
|
|
681
|
+
"graded": number;
|
|
682
|
+
"unmeasured": number;
|
|
683
|
+
"matchers": Array<string>;
|
|
684
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
685
|
+
}>;
|
|
686
|
+
"overall": {
|
|
687
|
+
"value": number | null;
|
|
688
|
+
"ci": Array<unknown> | null;
|
|
689
|
+
"n": number;
|
|
690
|
+
};
|
|
691
|
+
"graded": number;
|
|
692
|
+
"unmeasured": number;
|
|
693
|
+
"modelJudged": boolean;
|
|
694
|
+
};
|
|
695
|
+
"execution"?: {
|
|
696
|
+
"graded": number;
|
|
697
|
+
"measured": number;
|
|
698
|
+
"passed": number;
|
|
699
|
+
"passRate": number | null;
|
|
700
|
+
"byStatus": {
|
|
701
|
+
"passed": number;
|
|
702
|
+
"failed": number;
|
|
703
|
+
"timeout": number;
|
|
704
|
+
"error": number;
|
|
705
|
+
"unavailable": number;
|
|
706
|
+
};
|
|
707
|
+
"languages": Array<string>;
|
|
708
|
+
"unavailableReasons": Array<string>;
|
|
709
|
+
};
|
|
508
710
|
};
|
|
509
711
|
"warnings": Array<{
|
|
510
712
|
"type": "eval_capability_gap";
|
|
@@ -515,6 +717,8 @@ export type BenchmarksGetResponse = {
|
|
|
515
717
|
"detail": string;
|
|
516
718
|
"taskIds"?: Array<string>;
|
|
517
719
|
}>;
|
|
720
|
+
"shareUrl"?: string | null;
|
|
721
|
+
"canManageSharing"?: boolean;
|
|
518
722
|
};
|
|
519
723
|
export type BenchmarksDeleteResponse = unknown;
|
|
520
724
|
export type BenchmarksRegenerateSummaryResponse = {
|
|
@@ -524,7 +728,7 @@ export type BenchmarksGenerateRequest = {
|
|
|
524
728
|
"proposeExpectations"?: boolean;
|
|
525
729
|
};
|
|
526
730
|
export type BenchmarksGenerateResponse = {
|
|
527
|
-
"
|
|
731
|
+
"study": {
|
|
528
732
|
"id": string;
|
|
529
733
|
"executiveSummary"?: string | null;
|
|
530
734
|
"plannedRunCount": number;
|
|
@@ -537,7 +741,7 @@ export type BenchmarksGenerateResponse = {
|
|
|
537
741
|
"skillSlug": string | null;
|
|
538
742
|
"skillName": string | null;
|
|
539
743
|
"skillPromise": string;
|
|
540
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
744
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
541
745
|
"subjectName": string | null;
|
|
542
746
|
"subjectLabel": string;
|
|
543
747
|
"armConfig": {
|
|
@@ -555,6 +759,9 @@ export type BenchmarksGenerateResponse = {
|
|
|
555
759
|
};
|
|
556
760
|
};
|
|
557
761
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
762
|
+
"subjectBackend": "model" | "harness" | null;
|
|
763
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
764
|
+
"importedFrom"?: string;
|
|
558
765
|
"baselineLabel": string | null;
|
|
559
766
|
"treatmentLabel": string | null;
|
|
560
767
|
"promiseMetrics": Array<{
|
|
@@ -566,19 +773,48 @@ export type BenchmarksGenerateResponse = {
|
|
|
566
773
|
"harness": string;
|
|
567
774
|
"model": string;
|
|
568
775
|
"provider"?: string;
|
|
776
|
+
"params"?: {
|
|
777
|
+
"temperature"?: number;
|
|
778
|
+
"maxOutputTokens"?: number;
|
|
779
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
780
|
+
"thinkingBudgetTokens"?: number;
|
|
781
|
+
};
|
|
569
782
|
}>;
|
|
570
|
-
"lenses": Array<"judge" | "evals">;
|
|
783
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
571
784
|
"taskCount": number;
|
|
572
785
|
"variantsPerTask": number;
|
|
573
786
|
"repeats": number;
|
|
787
|
+
"subset": {
|
|
788
|
+
"count": number;
|
|
789
|
+
"seed": string;
|
|
790
|
+
"of": number;
|
|
791
|
+
} | null;
|
|
574
792
|
"status": string;
|
|
575
793
|
"error": string | null;
|
|
576
|
-
"
|
|
794
|
+
"preparation"?: {
|
|
795
|
+
"since": string;
|
|
796
|
+
"launchedBy": {
|
|
797
|
+
"userId"?: string;
|
|
798
|
+
"role"?: string;
|
|
799
|
+
};
|
|
800
|
+
"waitingOn": Array<{
|
|
801
|
+
"imageKey": string;
|
|
802
|
+
"buildId": string;
|
|
803
|
+
"stage"?: "image" | "snapshot";
|
|
804
|
+
}>;
|
|
805
|
+
} | null;
|
|
806
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
807
|
+
"visibilityChangedAt"?: string | null;
|
|
808
|
+
"visibilityChangedBy"?: string | null;
|
|
809
|
+
"shareToken"?: string | null;
|
|
577
810
|
"lowSampleOverride": boolean;
|
|
578
811
|
"publishedAt": string | null;
|
|
579
812
|
"verdict": unknown;
|
|
813
|
+
"definitionVersionId": string | null;
|
|
814
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
580
815
|
"createdAt": string;
|
|
581
816
|
"updatedAt": string;
|
|
817
|
+
"liveUntil": string | null;
|
|
582
818
|
};
|
|
583
819
|
"streamUrl": string;
|
|
584
820
|
"warnings"?: Array<{
|
|
@@ -587,7 +823,7 @@ export type BenchmarksGenerateResponse = {
|
|
|
587
823
|
}>;
|
|
588
824
|
};
|
|
589
825
|
export type BenchmarksLaunchResponse = {
|
|
590
|
-
"
|
|
826
|
+
"study": {
|
|
591
827
|
"id": string;
|
|
592
828
|
"executiveSummary"?: string | null;
|
|
593
829
|
"plannedRunCount": number;
|
|
@@ -600,7 +836,7 @@ export type BenchmarksLaunchResponse = {
|
|
|
600
836
|
"skillSlug": string | null;
|
|
601
837
|
"skillName": string | null;
|
|
602
838
|
"skillPromise": string;
|
|
603
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
839
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
604
840
|
"subjectName": string | null;
|
|
605
841
|
"subjectLabel": string;
|
|
606
842
|
"armConfig": {
|
|
@@ -618,6 +854,9 @@ export type BenchmarksLaunchResponse = {
|
|
|
618
854
|
};
|
|
619
855
|
};
|
|
620
856
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
857
|
+
"subjectBackend": "model" | "harness" | null;
|
|
858
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
859
|
+
"importedFrom"?: string;
|
|
621
860
|
"baselineLabel": string | null;
|
|
622
861
|
"treatmentLabel": string | null;
|
|
623
862
|
"promiseMetrics": Array<{
|
|
@@ -629,19 +868,48 @@ export type BenchmarksLaunchResponse = {
|
|
|
629
868
|
"harness": string;
|
|
630
869
|
"model": string;
|
|
631
870
|
"provider"?: string;
|
|
871
|
+
"params"?: {
|
|
872
|
+
"temperature"?: number;
|
|
873
|
+
"maxOutputTokens"?: number;
|
|
874
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
875
|
+
"thinkingBudgetTokens"?: number;
|
|
876
|
+
};
|
|
632
877
|
}>;
|
|
633
|
-
"lenses": Array<"judge" | "evals">;
|
|
878
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
634
879
|
"taskCount": number;
|
|
635
880
|
"variantsPerTask": number;
|
|
636
881
|
"repeats": number;
|
|
882
|
+
"subset": {
|
|
883
|
+
"count": number;
|
|
884
|
+
"seed": string;
|
|
885
|
+
"of": number;
|
|
886
|
+
} | null;
|
|
637
887
|
"status": string;
|
|
638
888
|
"error": string | null;
|
|
639
|
-
"
|
|
889
|
+
"preparation"?: {
|
|
890
|
+
"since": string;
|
|
891
|
+
"launchedBy": {
|
|
892
|
+
"userId"?: string;
|
|
893
|
+
"role"?: string;
|
|
894
|
+
};
|
|
895
|
+
"waitingOn": Array<{
|
|
896
|
+
"imageKey": string;
|
|
897
|
+
"buildId": string;
|
|
898
|
+
"stage"?: "image" | "snapshot";
|
|
899
|
+
}>;
|
|
900
|
+
} | null;
|
|
901
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
902
|
+
"visibilityChangedAt"?: string | null;
|
|
903
|
+
"visibilityChangedBy"?: string | null;
|
|
904
|
+
"shareToken"?: string | null;
|
|
640
905
|
"lowSampleOverride": boolean;
|
|
641
906
|
"publishedAt": string | null;
|
|
642
907
|
"verdict": unknown;
|
|
908
|
+
"definitionVersionId": string | null;
|
|
909
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
643
910
|
"createdAt": string;
|
|
644
911
|
"updatedAt": string;
|
|
912
|
+
"liveUntil": string | null;
|
|
645
913
|
};
|
|
646
914
|
"streamUrl": string;
|
|
647
915
|
"warnings"?: Array<{
|
|
@@ -650,10 +918,10 @@ export type BenchmarksLaunchResponse = {
|
|
|
650
918
|
}>;
|
|
651
919
|
};
|
|
652
920
|
export type BenchmarksUpdateLensesRequest = {
|
|
653
|
-
"lenses": Array<"judge" | "evals">;
|
|
921
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
654
922
|
};
|
|
655
923
|
export type BenchmarksUpdateLensesResponse = {
|
|
656
|
-
"
|
|
924
|
+
"study": {
|
|
657
925
|
"id": string;
|
|
658
926
|
"executiveSummary"?: string | null;
|
|
659
927
|
"plannedRunCount": number;
|
|
@@ -666,7 +934,7 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
666
934
|
"skillSlug": string | null;
|
|
667
935
|
"skillName": string | null;
|
|
668
936
|
"skillPromise": string;
|
|
669
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
937
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
670
938
|
"subjectName": string | null;
|
|
671
939
|
"subjectLabel": string;
|
|
672
940
|
"armConfig": {
|
|
@@ -684,6 +952,9 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
684
952
|
};
|
|
685
953
|
};
|
|
686
954
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
955
|
+
"subjectBackend": "model" | "harness" | null;
|
|
956
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
957
|
+
"importedFrom"?: string;
|
|
687
958
|
"baselineLabel": string | null;
|
|
688
959
|
"treatmentLabel": string | null;
|
|
689
960
|
"promiseMetrics": Array<{
|
|
@@ -695,19 +966,48 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
695
966
|
"harness": string;
|
|
696
967
|
"model": string;
|
|
697
968
|
"provider"?: string;
|
|
969
|
+
"params"?: {
|
|
970
|
+
"temperature"?: number;
|
|
971
|
+
"maxOutputTokens"?: number;
|
|
972
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
973
|
+
"thinkingBudgetTokens"?: number;
|
|
974
|
+
};
|
|
698
975
|
}>;
|
|
699
|
-
"lenses": Array<"judge" | "evals">;
|
|
976
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
700
977
|
"taskCount": number;
|
|
701
978
|
"variantsPerTask": number;
|
|
702
979
|
"repeats": number;
|
|
980
|
+
"subset": {
|
|
981
|
+
"count": number;
|
|
982
|
+
"seed": string;
|
|
983
|
+
"of": number;
|
|
984
|
+
} | null;
|
|
703
985
|
"status": string;
|
|
704
986
|
"error": string | null;
|
|
705
|
-
"
|
|
987
|
+
"preparation"?: {
|
|
988
|
+
"since": string;
|
|
989
|
+
"launchedBy": {
|
|
990
|
+
"userId"?: string;
|
|
991
|
+
"role"?: string;
|
|
992
|
+
};
|
|
993
|
+
"waitingOn": Array<{
|
|
994
|
+
"imageKey": string;
|
|
995
|
+
"buildId": string;
|
|
996
|
+
"stage"?: "image" | "snapshot";
|
|
997
|
+
}>;
|
|
998
|
+
} | null;
|
|
999
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
1000
|
+
"visibilityChangedAt"?: string | null;
|
|
1001
|
+
"visibilityChangedBy"?: string | null;
|
|
1002
|
+
"shareToken"?: string | null;
|
|
706
1003
|
"lowSampleOverride": boolean;
|
|
707
1004
|
"publishedAt": string | null;
|
|
708
1005
|
"verdict": unknown;
|
|
1006
|
+
"definitionVersionId": string | null;
|
|
1007
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
709
1008
|
"createdAt": string;
|
|
710
1009
|
"updatedAt": string;
|
|
1010
|
+
"liveUntil": string | null;
|
|
711
1011
|
};
|
|
712
1012
|
"warnings": Array<{
|
|
713
1013
|
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
@@ -716,7 +1016,7 @@ export type BenchmarksUpdateLensesResponse = {
|
|
|
716
1016
|
}>;
|
|
717
1017
|
};
|
|
718
1018
|
export type BenchmarksPreviewResponse = {
|
|
719
|
-
"
|
|
1019
|
+
"study": {
|
|
720
1020
|
"executiveSummary"?: string | null;
|
|
721
1021
|
"slug": string;
|
|
722
1022
|
"name": string;
|
|
@@ -724,7 +1024,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
724
1024
|
"skillName": string | null;
|
|
725
1025
|
"skillSlug": string | null;
|
|
726
1026
|
"skillPromise": string;
|
|
727
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
1027
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
728
1028
|
"subjectName": string | null;
|
|
729
1029
|
"subjectLabel": string;
|
|
730
1030
|
"armConfig": {
|
|
@@ -742,9 +1042,11 @@ export type BenchmarksPreviewResponse = {
|
|
|
742
1042
|
};
|
|
743
1043
|
};
|
|
744
1044
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
1045
|
+
"subjectBackend"?: "model" | "harness";
|
|
1046
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
745
1047
|
"baselineLabel": string | null;
|
|
746
1048
|
"treatmentLabel": string | null;
|
|
747
|
-
"lenses": Array<"judge" | "evals">;
|
|
1049
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
748
1050
|
"promiseMetrics": Array<{
|
|
749
1051
|
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
750
1052
|
"direction": "down" | "up";
|
|
@@ -754,6 +1056,12 @@ export type BenchmarksPreviewResponse = {
|
|
|
754
1056
|
"harness": string;
|
|
755
1057
|
"model": string;
|
|
756
1058
|
"provider"?: string;
|
|
1059
|
+
"params"?: {
|
|
1060
|
+
"temperature"?: number;
|
|
1061
|
+
"maxOutputTokens"?: number;
|
|
1062
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
1063
|
+
"thinkingBudgetTokens"?: number;
|
|
1064
|
+
};
|
|
757
1065
|
}>;
|
|
758
1066
|
"taskCount": number;
|
|
759
1067
|
"variantsPerTask": number;
|
|
@@ -761,17 +1069,36 @@ export type BenchmarksPreviewResponse = {
|
|
|
761
1069
|
"status": string;
|
|
762
1070
|
"publishedAt": string | null;
|
|
763
1071
|
"createdAt": string;
|
|
1072
|
+
"viewCount"?: number;
|
|
1073
|
+
"boardEligibility"?: {
|
|
1074
|
+
"eligible": boolean;
|
|
1075
|
+
"refusedBy": string | null;
|
|
1076
|
+
"reason": string | null;
|
|
1077
|
+
"failures": Array<{
|
|
1078
|
+
"ruleId": string;
|
|
1079
|
+
"reason": string;
|
|
1080
|
+
}>;
|
|
1081
|
+
"evaluatedAt": string;
|
|
1082
|
+
};
|
|
1083
|
+
"subjectLicenseSpdx"?: string | null;
|
|
1084
|
+
"subjectSourceUrl"?: string | null;
|
|
1085
|
+
"subjectWithheldReason"?: string | null;
|
|
1086
|
+
"subjectNotices"?: Array<{
|
|
1087
|
+
"kind": string;
|
|
1088
|
+
"text": string;
|
|
1089
|
+
}>;
|
|
764
1090
|
};
|
|
765
1091
|
"tasks": Array<{
|
|
766
1092
|
"id": string;
|
|
767
1093
|
"position": number;
|
|
768
|
-
"prompt"
|
|
769
|
-
"rationale"
|
|
1094
|
+
"prompt"?: string;
|
|
1095
|
+
"rationale"?: string | null;
|
|
770
1096
|
"category": string | null;
|
|
771
1097
|
"requiresWeb": boolean;
|
|
772
|
-
"rubric"
|
|
1098
|
+
"rubric"?: Array<string>;
|
|
773
1099
|
"variants"?: Array<string> | null;
|
|
774
1100
|
"expectations"?: Array<unknown> | null;
|
|
1101
|
+
"textWithheld"?: boolean;
|
|
775
1102
|
}>;
|
|
776
1103
|
"runs": Array<{
|
|
777
1104
|
"runId": string;
|
|
@@ -805,6 +1132,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
805
1132
|
"runCreatedAt": string | null;
|
|
806
1133
|
"startedAt": string | null;
|
|
807
1134
|
"finishedAt": string | null;
|
|
1135
|
+
"lensGrades"?: Record<string, unknown>;
|
|
808
1136
|
"rubricPassed": Array<boolean> | null;
|
|
809
1137
|
"evalOutcomes": Array<{
|
|
810
1138
|
"kind": string;
|
|
@@ -813,6 +1141,8 @@ export type BenchmarksPreviewResponse = {
|
|
|
813
1141
|
"abstained"?: boolean;
|
|
814
1142
|
"decisive"?: boolean;
|
|
815
1143
|
}> | null;
|
|
1144
|
+
"paramsKey"?: string;
|
|
1145
|
+
"textWithheld"?: boolean;
|
|
816
1146
|
}>;
|
|
817
1147
|
"aggregate": {
|
|
818
1148
|
"summary": {
|
|
@@ -982,6 +1312,7 @@ export type BenchmarksPreviewResponse = {
|
|
|
982
1312
|
"kind": "cell" | "harness" | "model" | "field";
|
|
983
1313
|
"harness": string | null;
|
|
984
1314
|
"model": string | null;
|
|
1315
|
+
"paramsKey"?: string;
|
|
985
1316
|
"runs": number;
|
|
986
1317
|
"cells": Record<string, unknown>;
|
|
987
1318
|
}>;
|
|
@@ -1003,13 +1334,58 @@ export type BenchmarksPreviewResponse = {
|
|
|
1003
1334
|
"succeededCount": number;
|
|
1004
1335
|
}>;
|
|
1005
1336
|
}>;
|
|
1337
|
+
"reference"?: {
|
|
1338
|
+
"cells": Array<{
|
|
1339
|
+
"cell": string;
|
|
1340
|
+
"harness": string;
|
|
1341
|
+
"model": string;
|
|
1342
|
+
"provider": string | null;
|
|
1343
|
+
"paramsKey": string;
|
|
1344
|
+
"label": string;
|
|
1345
|
+
"passRate": {
|
|
1346
|
+
"value": number | null;
|
|
1347
|
+
"ci": Array<unknown> | null;
|
|
1348
|
+
"n": number;
|
|
1349
|
+
};
|
|
1350
|
+
"tasks": number;
|
|
1351
|
+
"graded": number;
|
|
1352
|
+
"unmeasured": number;
|
|
1353
|
+
"matchers": Array<string>;
|
|
1354
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
1355
|
+
}>;
|
|
1356
|
+
"overall": {
|
|
1357
|
+
"value": number | null;
|
|
1358
|
+
"ci": Array<unknown> | null;
|
|
1359
|
+
"n": number;
|
|
1360
|
+
};
|
|
1361
|
+
"graded": number;
|
|
1362
|
+
"unmeasured": number;
|
|
1363
|
+
"modelJudged": boolean;
|
|
1364
|
+
};
|
|
1365
|
+
"execution"?: {
|
|
1366
|
+
"graded": number;
|
|
1367
|
+
"measured": number;
|
|
1368
|
+
"passed": number;
|
|
1369
|
+
"passRate": number | null;
|
|
1370
|
+
"byStatus": {
|
|
1371
|
+
"passed": number;
|
|
1372
|
+
"failed": number;
|
|
1373
|
+
"timeout": number;
|
|
1374
|
+
"error": number;
|
|
1375
|
+
"unavailable": number;
|
|
1376
|
+
};
|
|
1377
|
+
"languages": Array<string>;
|
|
1378
|
+
"unavailableReasons": Array<string>;
|
|
1379
|
+
};
|
|
1006
1380
|
};
|
|
1381
|
+
"textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
|
|
1007
1382
|
};
|
|
1008
1383
|
export type BenchmarksPublishRequest = {
|
|
1009
1384
|
"slug"?: string;
|
|
1385
|
+
"visibility"?: "private" | "org" | "link" | "public";
|
|
1010
1386
|
};
|
|
1011
1387
|
export type BenchmarksPublishResponse = {
|
|
1012
|
-
"
|
|
1388
|
+
"study": {
|
|
1013
1389
|
"id": string;
|
|
1014
1390
|
"executiveSummary"?: string | null;
|
|
1015
1391
|
"plannedRunCount": number;
|
|
@@ -1022,7 +1398,7 @@ export type BenchmarksPublishResponse = {
|
|
|
1022
1398
|
"skillSlug": string | null;
|
|
1023
1399
|
"skillName": string | null;
|
|
1024
1400
|
"skillPromise": string;
|
|
1025
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
1401
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
1026
1402
|
"subjectName": string | null;
|
|
1027
1403
|
"subjectLabel": string;
|
|
1028
1404
|
"armConfig": {
|
|
@@ -1040,6 +1416,9 @@ export type BenchmarksPublishResponse = {
|
|
|
1040
1416
|
};
|
|
1041
1417
|
};
|
|
1042
1418
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
1419
|
+
"subjectBackend": "model" | "harness" | null;
|
|
1420
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
1421
|
+
"importedFrom"?: string;
|
|
1043
1422
|
"baselineLabel": string | null;
|
|
1044
1423
|
"treatmentLabel": string | null;
|
|
1045
1424
|
"promiseMetrics": Array<{
|
|
@@ -1051,23 +1430,52 @@ export type BenchmarksPublishResponse = {
|
|
|
1051
1430
|
"harness": string;
|
|
1052
1431
|
"model": string;
|
|
1053
1432
|
"provider"?: string;
|
|
1433
|
+
"params"?: {
|
|
1434
|
+
"temperature"?: number;
|
|
1435
|
+
"maxOutputTokens"?: number;
|
|
1436
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
1437
|
+
"thinkingBudgetTokens"?: number;
|
|
1438
|
+
};
|
|
1054
1439
|
}>;
|
|
1055
|
-
"lenses": Array<"judge" | "evals">;
|
|
1440
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1056
1441
|
"taskCount": number;
|
|
1057
1442
|
"variantsPerTask": number;
|
|
1058
1443
|
"repeats": number;
|
|
1444
|
+
"subset": {
|
|
1445
|
+
"count": number;
|
|
1446
|
+
"seed": string;
|
|
1447
|
+
"of": number;
|
|
1448
|
+
} | null;
|
|
1059
1449
|
"status": string;
|
|
1060
1450
|
"error": string | null;
|
|
1061
|
-
"
|
|
1451
|
+
"preparation"?: {
|
|
1452
|
+
"since": string;
|
|
1453
|
+
"launchedBy": {
|
|
1454
|
+
"userId"?: string;
|
|
1455
|
+
"role"?: string;
|
|
1456
|
+
};
|
|
1457
|
+
"waitingOn": Array<{
|
|
1458
|
+
"imageKey": string;
|
|
1459
|
+
"buildId": string;
|
|
1460
|
+
"stage"?: "image" | "snapshot";
|
|
1461
|
+
}>;
|
|
1462
|
+
} | null;
|
|
1463
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
1464
|
+
"visibilityChangedAt"?: string | null;
|
|
1465
|
+
"visibilityChangedBy"?: string | null;
|
|
1466
|
+
"shareToken"?: string | null;
|
|
1062
1467
|
"lowSampleOverride": boolean;
|
|
1063
1468
|
"publishedAt": string | null;
|
|
1064
1469
|
"verdict": unknown;
|
|
1470
|
+
"definitionVersionId": string | null;
|
|
1471
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
1065
1472
|
"createdAt": string;
|
|
1066
1473
|
"updatedAt": string;
|
|
1474
|
+
"liveUntil": string | null;
|
|
1067
1475
|
};
|
|
1068
1476
|
};
|
|
1069
1477
|
export type BenchmarksRegradeResponse = {
|
|
1070
|
-
"
|
|
1478
|
+
"study": {
|
|
1071
1479
|
"id": string;
|
|
1072
1480
|
"executiveSummary"?: string | null;
|
|
1073
1481
|
"plannedRunCount": number;
|
|
@@ -1080,7 +1488,7 @@ export type BenchmarksRegradeResponse = {
|
|
|
1080
1488
|
"skillSlug": string | null;
|
|
1081
1489
|
"skillName": string | null;
|
|
1082
1490
|
"skillPromise": string;
|
|
1083
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
1491
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
1084
1492
|
"subjectName": string | null;
|
|
1085
1493
|
"subjectLabel": string;
|
|
1086
1494
|
"armConfig": {
|
|
@@ -1098,6 +1506,9 @@ export type BenchmarksRegradeResponse = {
|
|
|
1098
1506
|
};
|
|
1099
1507
|
};
|
|
1100
1508
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
1509
|
+
"subjectBackend": "model" | "harness" | null;
|
|
1510
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
1511
|
+
"importedFrom"?: string;
|
|
1101
1512
|
"baselineLabel": string | null;
|
|
1102
1513
|
"treatmentLabel": string | null;
|
|
1103
1514
|
"promiseMetrics": Array<{
|
|
@@ -1109,19 +1520,48 @@ export type BenchmarksRegradeResponse = {
|
|
|
1109
1520
|
"harness": string;
|
|
1110
1521
|
"model": string;
|
|
1111
1522
|
"provider"?: string;
|
|
1523
|
+
"params"?: {
|
|
1524
|
+
"temperature"?: number;
|
|
1525
|
+
"maxOutputTokens"?: number;
|
|
1526
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
1527
|
+
"thinkingBudgetTokens"?: number;
|
|
1528
|
+
};
|
|
1112
1529
|
}>;
|
|
1113
|
-
"lenses": Array<"judge" | "evals">;
|
|
1530
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1114
1531
|
"taskCount": number;
|
|
1115
1532
|
"variantsPerTask": number;
|
|
1116
1533
|
"repeats": number;
|
|
1534
|
+
"subset": {
|
|
1535
|
+
"count": number;
|
|
1536
|
+
"seed": string;
|
|
1537
|
+
"of": number;
|
|
1538
|
+
} | null;
|
|
1117
1539
|
"status": string;
|
|
1118
1540
|
"error": string | null;
|
|
1119
|
-
"
|
|
1541
|
+
"preparation"?: {
|
|
1542
|
+
"since": string;
|
|
1543
|
+
"launchedBy": {
|
|
1544
|
+
"userId"?: string;
|
|
1545
|
+
"role"?: string;
|
|
1546
|
+
};
|
|
1547
|
+
"waitingOn": Array<{
|
|
1548
|
+
"imageKey": string;
|
|
1549
|
+
"buildId": string;
|
|
1550
|
+
"stage"?: "image" | "snapshot";
|
|
1551
|
+
}>;
|
|
1552
|
+
} | null;
|
|
1553
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
1554
|
+
"visibilityChangedAt"?: string | null;
|
|
1555
|
+
"visibilityChangedBy"?: string | null;
|
|
1556
|
+
"shareToken"?: string | null;
|
|
1120
1557
|
"lowSampleOverride": boolean;
|
|
1121
1558
|
"publishedAt": string | null;
|
|
1122
1559
|
"verdict": unknown;
|
|
1560
|
+
"definitionVersionId": string | null;
|
|
1561
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
1123
1562
|
"createdAt": string;
|
|
1124
1563
|
"updatedAt": string;
|
|
1564
|
+
"liveUntil": string | null;
|
|
1125
1565
|
};
|
|
1126
1566
|
"streamUrl": string;
|
|
1127
1567
|
"warnings"?: Array<{
|
|
@@ -1137,7 +1577,7 @@ export type BenchmarksUpdateTaskRequest = {
|
|
|
1137
1577
|
export type BenchmarksUpdateTaskResponse = {
|
|
1138
1578
|
"task": {
|
|
1139
1579
|
"id": string;
|
|
1140
|
-
"
|
|
1580
|
+
"studyId": string;
|
|
1141
1581
|
"position": number;
|
|
1142
1582
|
"prompt": string;
|
|
1143
1583
|
"rationale": string | null;
|
|
@@ -1149,7 +1589,7 @@ export type BenchmarksUpdateTaskResponse = {
|
|
|
1149
1589
|
};
|
|
1150
1590
|
};
|
|
1151
1591
|
export type BenchmarksUnpublishResponse = {
|
|
1152
|
-
"
|
|
1592
|
+
"study": {
|
|
1153
1593
|
"id": string;
|
|
1154
1594
|
"executiveSummary"?: string | null;
|
|
1155
1595
|
"plannedRunCount": number;
|
|
@@ -1162,7 +1602,7 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1162
1602
|
"skillSlug": string | null;
|
|
1163
1603
|
"skillName": string | null;
|
|
1164
1604
|
"skillPromise": string;
|
|
1165
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
1605
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
1166
1606
|
"subjectName": string | null;
|
|
1167
1607
|
"subjectLabel": string;
|
|
1168
1608
|
"armConfig": {
|
|
@@ -1180,6 +1620,9 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1180
1620
|
};
|
|
1181
1621
|
};
|
|
1182
1622
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
1623
|
+
"subjectBackend": "model" | "harness" | null;
|
|
1624
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
1625
|
+
"importedFrom"?: string;
|
|
1183
1626
|
"baselineLabel": string | null;
|
|
1184
1627
|
"treatmentLabel": string | null;
|
|
1185
1628
|
"promiseMetrics": Array<{
|
|
@@ -1191,19 +1634,48 @@ export type BenchmarksUnpublishResponse = {
|
|
|
1191
1634
|
"harness": string;
|
|
1192
1635
|
"model": string;
|
|
1193
1636
|
"provider"?: string;
|
|
1637
|
+
"params"?: {
|
|
1638
|
+
"temperature"?: number;
|
|
1639
|
+
"maxOutputTokens"?: number;
|
|
1640
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
1641
|
+
"thinkingBudgetTokens"?: number;
|
|
1642
|
+
};
|
|
1194
1643
|
}>;
|
|
1195
|
-
"lenses": Array<"judge" | "evals">;
|
|
1644
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1196
1645
|
"taskCount": number;
|
|
1197
1646
|
"variantsPerTask": number;
|
|
1198
1647
|
"repeats": number;
|
|
1648
|
+
"subset": {
|
|
1649
|
+
"count": number;
|
|
1650
|
+
"seed": string;
|
|
1651
|
+
"of": number;
|
|
1652
|
+
} | null;
|
|
1199
1653
|
"status": string;
|
|
1200
1654
|
"error": string | null;
|
|
1201
|
-
"
|
|
1655
|
+
"preparation"?: {
|
|
1656
|
+
"since": string;
|
|
1657
|
+
"launchedBy": {
|
|
1658
|
+
"userId"?: string;
|
|
1659
|
+
"role"?: string;
|
|
1660
|
+
};
|
|
1661
|
+
"waitingOn": Array<{
|
|
1662
|
+
"imageKey": string;
|
|
1663
|
+
"buildId": string;
|
|
1664
|
+
"stage"?: "image" | "snapshot";
|
|
1665
|
+
}>;
|
|
1666
|
+
} | null;
|
|
1667
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
1668
|
+
"visibilityChangedAt"?: string | null;
|
|
1669
|
+
"visibilityChangedBy"?: string | null;
|
|
1670
|
+
"shareToken"?: string | null;
|
|
1202
1671
|
"lowSampleOverride": boolean;
|
|
1203
1672
|
"publishedAt": string | null;
|
|
1204
1673
|
"verdict": unknown;
|
|
1674
|
+
"definitionVersionId": string | null;
|
|
1675
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
1205
1676
|
"createdAt": string;
|
|
1206
1677
|
"updatedAt": string;
|
|
1678
|
+
"liveUntil": string | null;
|
|
1207
1679
|
};
|
|
1208
1680
|
};
|
|
1209
1681
|
export type ExperimentCatalogResponse = {
|
|
@@ -1227,6 +1699,130 @@ export type ExperimentCatalogResponse = {
|
|
|
1227
1699
|
}>;
|
|
1228
1700
|
}>;
|
|
1229
1701
|
};
|
|
1702
|
+
export type ExperimentImagesBuildsListResponse = {
|
|
1703
|
+
"builds": Array<{
|
|
1704
|
+
"id": string;
|
|
1705
|
+
"organizationId": string;
|
|
1706
|
+
"productKey": string;
|
|
1707
|
+
"requestedByUserId": string | null;
|
|
1708
|
+
"name": string;
|
|
1709
|
+
"source": {
|
|
1710
|
+
"kind": "archive";
|
|
1711
|
+
"sha256": string;
|
|
1712
|
+
} | {
|
|
1713
|
+
"kind": "upload";
|
|
1714
|
+
"fileId": string;
|
|
1715
|
+
} | {
|
|
1716
|
+
"kind": "git";
|
|
1717
|
+
"ref": string;
|
|
1718
|
+
"subdir"?: string;
|
|
1719
|
+
} | {
|
|
1720
|
+
"kind": "inline";
|
|
1721
|
+
"fileCount": number;
|
|
1722
|
+
};
|
|
1723
|
+
"sourceIdentity": string;
|
|
1724
|
+
"dockerfileDir": string;
|
|
1725
|
+
"dockerfileName": string;
|
|
1726
|
+
"buildArgKeys": Array<string>;
|
|
1727
|
+
"platform": string;
|
|
1728
|
+
"trusted": boolean;
|
|
1729
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1730
|
+
"image": string;
|
|
1731
|
+
"digest": string | null;
|
|
1732
|
+
"sizeBytes": number | null;
|
|
1733
|
+
"reason": string | null;
|
|
1734
|
+
"logRef": string | null;
|
|
1735
|
+
"startedAt": string | null;
|
|
1736
|
+
"finishedAt": string | null;
|
|
1737
|
+
"createdAt": string;
|
|
1738
|
+
"updatedAt": string;
|
|
1739
|
+
}>;
|
|
1740
|
+
"nextPageToken"?: string;
|
|
1741
|
+
};
|
|
1742
|
+
export type ExperimentImagesBuildsCreateRequest = {
|
|
1743
|
+
"name": string;
|
|
1744
|
+
"source": {
|
|
1745
|
+
"kind": "archive";
|
|
1746
|
+
"url": string;
|
|
1747
|
+
"sha256": string;
|
|
1748
|
+
} | {
|
|
1749
|
+
"kind": "upload";
|
|
1750
|
+
"fileId": string;
|
|
1751
|
+
} | {
|
|
1752
|
+
"kind": "git";
|
|
1753
|
+
"url": string;
|
|
1754
|
+
"ref": string;
|
|
1755
|
+
"subdir"?: string;
|
|
1756
|
+
} | {
|
|
1757
|
+
"kind": "inline";
|
|
1758
|
+
"dockerfile": string;
|
|
1759
|
+
"files"?: Record<string, unknown>;
|
|
1760
|
+
};
|
|
1761
|
+
"dockerfile"?: {
|
|
1762
|
+
"dir"?: string;
|
|
1763
|
+
"name"?: string;
|
|
1764
|
+
};
|
|
1765
|
+
"buildArgs"?: Record<string, unknown>;
|
|
1766
|
+
"platform"?: string;
|
|
1767
|
+
};
|
|
1768
|
+
export type ExperimentImagesBuildsCreateResponse = {
|
|
1769
|
+
"id": string;
|
|
1770
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1771
|
+
};
|
|
1772
|
+
export type ExperimentImagesBuildsGetResponse = {
|
|
1773
|
+
"build": {
|
|
1774
|
+
"id": string;
|
|
1775
|
+
"organizationId": string;
|
|
1776
|
+
"productKey": string;
|
|
1777
|
+
"requestedByUserId": string | null;
|
|
1778
|
+
"name": string;
|
|
1779
|
+
"source": {
|
|
1780
|
+
"kind": "archive";
|
|
1781
|
+
"sha256": string;
|
|
1782
|
+
} | {
|
|
1783
|
+
"kind": "upload";
|
|
1784
|
+
"fileId": string;
|
|
1785
|
+
} | {
|
|
1786
|
+
"kind": "git";
|
|
1787
|
+
"ref": string;
|
|
1788
|
+
"subdir"?: string;
|
|
1789
|
+
} | {
|
|
1790
|
+
"kind": "inline";
|
|
1791
|
+
"fileCount": number;
|
|
1792
|
+
};
|
|
1793
|
+
"sourceIdentity": string;
|
|
1794
|
+
"dockerfileDir": string;
|
|
1795
|
+
"dockerfileName": string;
|
|
1796
|
+
"buildArgKeys": Array<string>;
|
|
1797
|
+
"platform": string;
|
|
1798
|
+
"trusted": boolean;
|
|
1799
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1800
|
+
"image": string;
|
|
1801
|
+
"digest": string | null;
|
|
1802
|
+
"sizeBytes": number | null;
|
|
1803
|
+
"reason": string | null;
|
|
1804
|
+
"logRef": string | null;
|
|
1805
|
+
"startedAt": string | null;
|
|
1806
|
+
"finishedAt": string | null;
|
|
1807
|
+
"createdAt": string;
|
|
1808
|
+
"updatedAt": string;
|
|
1809
|
+
};
|
|
1810
|
+
};
|
|
1811
|
+
export type ExperimentImagesBuildsCancelResponse = {
|
|
1812
|
+
"id": string;
|
|
1813
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
1814
|
+
};
|
|
1815
|
+
export type ExperimentImagesBuildsStreamResponse = unknown;
|
|
1816
|
+
export type ExperimentImagesRunsCreateRequest = {
|
|
1817
|
+
"buildId"?: string;
|
|
1818
|
+
"image"?: string;
|
|
1819
|
+
"mode": "harness" | "environment";
|
|
1820
|
+
"harness"?: string;
|
|
1821
|
+
"task"?: string;
|
|
1822
|
+
};
|
|
1823
|
+
export type ExperimentImagesRunsCreateResponse = {
|
|
1824
|
+
"runId": string;
|
|
1825
|
+
};
|
|
1230
1826
|
export type ExperimentRunsListResponse = {
|
|
1231
1827
|
"results": Array<{
|
|
1232
1828
|
"id": string;
|
|
@@ -1276,6 +1872,12 @@ export type ExperimentRunsCreateRequest = {
|
|
|
1276
1872
|
"url": string;
|
|
1277
1873
|
"secretId"?: string;
|
|
1278
1874
|
}>;
|
|
1875
|
+
"model_params"?: {
|
|
1876
|
+
"temperature"?: number;
|
|
1877
|
+
"maxOutputTokens"?: number;
|
|
1878
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
1879
|
+
"thinkingBudgetTokens"?: number;
|
|
1880
|
+
};
|
|
1279
1881
|
"wait"?: boolean;
|
|
1280
1882
|
};
|
|
1281
1883
|
export type ExperimentRunsCreateResponse = {
|
|
@@ -1454,7 +2056,7 @@ export type SessionsWakeResponse = {
|
|
|
1454
2056
|
"updatedAt": string;
|
|
1455
2057
|
};
|
|
1456
2058
|
};
|
|
1457
|
-
export type
|
|
2059
|
+
export type ExperimentSkillsListResponse = {
|
|
1458
2060
|
"skills": Array<{
|
|
1459
2061
|
"ref": string;
|
|
1460
2062
|
"source": "builtin" | "custom" | "git";
|
|
@@ -1472,7 +2074,7 @@ export type SkillsListResponse = {
|
|
|
1472
2074
|
"updatedAt"?: string;
|
|
1473
2075
|
}>;
|
|
1474
2076
|
};
|
|
1475
|
-
export type
|
|
2077
|
+
export type ExperimentSkillsCreateRequest = {
|
|
1476
2078
|
"source": "custom";
|
|
1477
2079
|
"name": string;
|
|
1478
2080
|
"body": string;
|
|
@@ -1485,7 +2087,7 @@ export type SkillsCreateRequest = {
|
|
|
1485
2087
|
"subpath"?: string | null;
|
|
1486
2088
|
"visibility"?: "user" | "org";
|
|
1487
2089
|
};
|
|
1488
|
-
export type
|
|
2090
|
+
export type ExperimentSkillsCreateResponse = {
|
|
1489
2091
|
"skill": {
|
|
1490
2092
|
"ref": string;
|
|
1491
2093
|
"source": "builtin" | "custom" | "git";
|
|
@@ -1503,7 +2105,7 @@ export type SkillsCreateResponse = {
|
|
|
1503
2105
|
"updatedAt"?: string;
|
|
1504
2106
|
};
|
|
1505
2107
|
};
|
|
1506
|
-
export type
|
|
2108
|
+
export type ExperimentSkillsUpdateRequest = {
|
|
1507
2109
|
"name"?: string;
|
|
1508
2110
|
"visibility"?: "user" | "org";
|
|
1509
2111
|
"body"?: string;
|
|
@@ -1511,7 +2113,7 @@ export type SkillsUpdateRequest = {
|
|
|
1511
2113
|
"gitRef"?: string;
|
|
1512
2114
|
"subpath"?: string | null;
|
|
1513
2115
|
};
|
|
1514
|
-
export type
|
|
2116
|
+
export type ExperimentSkillsUpdateResponse = {
|
|
1515
2117
|
"skill": {
|
|
1516
2118
|
"ref": string;
|
|
1517
2119
|
"source": "builtin" | "custom" | "git";
|
|
@@ -1529,7 +2131,7 @@ export type SkillsUpdateResponse = {
|
|
|
1529
2131
|
"updatedAt"?: string;
|
|
1530
2132
|
};
|
|
1531
2133
|
};
|
|
1532
|
-
export type
|
|
2134
|
+
export type ExperimentSkillsDeleteResponse = unknown;
|
|
1533
2135
|
export type WebmcpArenaCreateRequest = {
|
|
1534
2136
|
"targetUrl": string;
|
|
1535
2137
|
"intent": string;
|
|
@@ -1541,7 +2143,7 @@ export type WebmcpArenaCreateRequest = {
|
|
|
1541
2143
|
"name"?: string;
|
|
1542
2144
|
};
|
|
1543
2145
|
export type WebmcpArenaCreateResponse = {
|
|
1544
|
-
"
|
|
2146
|
+
"study": {
|
|
1545
2147
|
"id": string;
|
|
1546
2148
|
"executiveSummary"?: string | null;
|
|
1547
2149
|
"plannedRunCount": number;
|
|
@@ -1554,7 +2156,7 @@ export type WebmcpArenaCreateResponse = {
|
|
|
1554
2156
|
"skillSlug": string | null;
|
|
1555
2157
|
"skillName": string | null;
|
|
1556
2158
|
"skillPromise": string;
|
|
1557
|
-
"subjectKind": "skill" | "tools" | "addon" | "custom";
|
|
2159
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
1558
2160
|
"subjectName": string | null;
|
|
1559
2161
|
"subjectLabel": string;
|
|
1560
2162
|
"armConfig": {
|
|
@@ -1572,6 +2174,9 @@ export type WebmcpArenaCreateResponse = {
|
|
|
1572
2174
|
};
|
|
1573
2175
|
};
|
|
1574
2176
|
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
2177
|
+
"subjectBackend": "model" | "harness" | null;
|
|
2178
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
2179
|
+
"importedFrom"?: string;
|
|
1575
2180
|
"baselineLabel": string | null;
|
|
1576
2181
|
"treatmentLabel": string | null;
|
|
1577
2182
|
"promiseMetrics": Array<{
|
|
@@ -1583,19 +2188,48 @@ export type WebmcpArenaCreateResponse = {
|
|
|
1583
2188
|
"harness": string;
|
|
1584
2189
|
"model": string;
|
|
1585
2190
|
"provider"?: string;
|
|
2191
|
+
"params"?: {
|
|
2192
|
+
"temperature"?: number;
|
|
2193
|
+
"maxOutputTokens"?: number;
|
|
2194
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
2195
|
+
"thinkingBudgetTokens"?: number;
|
|
2196
|
+
};
|
|
1586
2197
|
}>;
|
|
1587
|
-
"lenses": Array<"judge" | "evals">;
|
|
2198
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
1588
2199
|
"taskCount": number;
|
|
1589
2200
|
"variantsPerTask": number;
|
|
1590
2201
|
"repeats": number;
|
|
2202
|
+
"subset": {
|
|
2203
|
+
"count": number;
|
|
2204
|
+
"seed": string;
|
|
2205
|
+
"of": number;
|
|
2206
|
+
} | null;
|
|
1591
2207
|
"status": string;
|
|
1592
2208
|
"error": string | null;
|
|
1593
|
-
"
|
|
2209
|
+
"preparation"?: {
|
|
2210
|
+
"since": string;
|
|
2211
|
+
"launchedBy": {
|
|
2212
|
+
"userId"?: string;
|
|
2213
|
+
"role"?: string;
|
|
2214
|
+
};
|
|
2215
|
+
"waitingOn": Array<{
|
|
2216
|
+
"imageKey": string;
|
|
2217
|
+
"buildId": string;
|
|
2218
|
+
"stage"?: "image" | "snapshot";
|
|
2219
|
+
}>;
|
|
2220
|
+
} | null;
|
|
2221
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
2222
|
+
"visibilityChangedAt"?: string | null;
|
|
2223
|
+
"visibilityChangedBy"?: string | null;
|
|
2224
|
+
"shareToken"?: string | null;
|
|
1594
2225
|
"lowSampleOverride": boolean;
|
|
1595
2226
|
"publishedAt": string | null;
|
|
1596
2227
|
"verdict": unknown;
|
|
2228
|
+
"definitionVersionId": string | null;
|
|
2229
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
1597
2230
|
"createdAt": string;
|
|
1598
2231
|
"updatedAt": string;
|
|
2232
|
+
"liveUntil": string | null;
|
|
1599
2233
|
};
|
|
1600
2234
|
"streamUrl": string;
|
|
1601
2235
|
"warnings"?: Array<{
|
|
@@ -1603,6 +2237,20 @@ export type WebmcpArenaCreateResponse = {
|
|
|
1603
2237
|
"kind": string;
|
|
1604
2238
|
}>;
|
|
1605
2239
|
};
|
|
2240
|
+
export type WebmcpRunsBrowserRecordingResponse = {
|
|
2241
|
+
"runId": string;
|
|
2242
|
+
"startedAt": string | null;
|
|
2243
|
+
"durationMs": number;
|
|
2244
|
+
"frames": Array<{
|
|
2245
|
+
"t": number;
|
|
2246
|
+
"data": string;
|
|
2247
|
+
"url"?: string;
|
|
2248
|
+
}>;
|
|
2249
|
+
"truncated"?: boolean;
|
|
2250
|
+
"decimated"?: boolean;
|
|
2251
|
+
"totalFrames"?: number;
|
|
2252
|
+
};
|
|
2253
|
+
export type WebmcpRunsBrowserStreamResponse = unknown;
|
|
1606
2254
|
export type ActivityListResponse = {
|
|
1607
2255
|
"success": true;
|
|
1608
2256
|
"data": Array<{
|
|
@@ -1778,8 +2426,28 @@ export type IntentsListResponse = {
|
|
|
1778
2426
|
};
|
|
1779
2427
|
"lowSample": boolean;
|
|
1780
2428
|
"measurementPolicy": {
|
|
1781
|
-
"cadence": "daily" | "weekly" | "manual";
|
|
2429
|
+
"cadence": "daily" | "weekly" | "monthly" | "manual";
|
|
1782
2430
|
"mcp_server_ids": Array<string>;
|
|
2431
|
+
"agents": Array<{
|
|
2432
|
+
"harness": string;
|
|
2433
|
+
"model": string;
|
|
2434
|
+
"mcp_server_ids": Array<string>;
|
|
2435
|
+
"skill_ids": Array<string>;
|
|
2436
|
+
"browser"?: {
|
|
2437
|
+
"webdriver"?: boolean;
|
|
2438
|
+
"webmcp"?: boolean;
|
|
2439
|
+
"webmcpTargetUrl"?: string;
|
|
2440
|
+
"webmcpSurface"?: "native" | "shim";
|
|
2441
|
+
};
|
|
2442
|
+
}>;
|
|
2443
|
+
"skill_ids": Array<string>;
|
|
2444
|
+
"browser"?: {
|
|
2445
|
+
"webdriver"?: boolean;
|
|
2446
|
+
"webmcp"?: boolean;
|
|
2447
|
+
"webmcpTargetUrl"?: string;
|
|
2448
|
+
"webmcpSurface"?: "native" | "shim";
|
|
2449
|
+
};
|
|
2450
|
+
"competitors": Array<string>;
|
|
1783
2451
|
};
|
|
1784
2452
|
}>;
|
|
1785
2453
|
};
|
|
@@ -2212,6 +2880,12 @@ export type JourneysCreateRequest = {
|
|
|
2212
2880
|
"domains"?: Array<string>;
|
|
2213
2881
|
"competitors"?: Array<string>;
|
|
2214
2882
|
"skillIds"?: Array<string>;
|
|
2883
|
+
"browser"?: {
|
|
2884
|
+
"webdriver"?: boolean;
|
|
2885
|
+
"webmcp"?: boolean;
|
|
2886
|
+
"webmcpTargetUrl"?: string;
|
|
2887
|
+
"webmcpSurface"?: "native" | "shim";
|
|
2888
|
+
};
|
|
2215
2889
|
"schedule"?: unknown;
|
|
2216
2890
|
};
|
|
2217
2891
|
export type JourneysCreateResponse = {
|
|
@@ -2378,6 +3052,12 @@ export type StudioRunsCreateRequest = {
|
|
|
2378
3052
|
"url": string;
|
|
2379
3053
|
"secretId"?: string;
|
|
2380
3054
|
}>;
|
|
3055
|
+
"model_params"?: {
|
|
3056
|
+
"temperature"?: number;
|
|
3057
|
+
"maxOutputTokens"?: number;
|
|
3058
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
3059
|
+
"thinkingBudgetTokens"?: number;
|
|
3060
|
+
};
|
|
2381
3061
|
"wait"?: boolean;
|
|
2382
3062
|
};
|
|
2383
3063
|
export type StudioRunsCreateResponse = {
|
|
@@ -2438,6 +3118,3105 @@ export type StudioRunsRawTraceResponse = unknown;
|
|
|
2438
3118
|
export type StudioRunsSpansResponse = unknown;
|
|
2439
3119
|
export type StudioRunsStreamResponse = unknown;
|
|
2440
3120
|
export type StudioRunsTrajectoryResponse = unknown;
|
|
3121
|
+
export type StudyAddonsListResponse = {
|
|
3122
|
+
"addons": Array<{
|
|
3123
|
+
"id": string;
|
|
3124
|
+
"kind": "capability" | "tool" | "skill";
|
|
3125
|
+
"category"?: string;
|
|
3126
|
+
"displayName": string;
|
|
3127
|
+
"description": string;
|
|
3128
|
+
"configSchema": Record<string, unknown>;
|
|
3129
|
+
"surfaces": Array<"run" | "session">;
|
|
3130
|
+
"harnesses": Array<string>;
|
|
3131
|
+
"multiInstance"?: boolean;
|
|
3132
|
+
}>;
|
|
3133
|
+
};
|
|
3134
|
+
export type StudyBenchmarkDefinitionsListResponse = {
|
|
3135
|
+
"definitions": Array<{
|
|
3136
|
+
"id": string;
|
|
3137
|
+
"slug": string;
|
|
3138
|
+
"name": string;
|
|
3139
|
+
"description": string | null;
|
|
3140
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3141
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3142
|
+
"headlineMetric": string;
|
|
3143
|
+
"methodology": {
|
|
3144
|
+
"variantsAllowed"?: boolean;
|
|
3145
|
+
"recommendedRepeats"?: number;
|
|
3146
|
+
"officialFidelity"?: boolean;
|
|
3147
|
+
"modelParams"?: Array<string>;
|
|
3148
|
+
"maxPlannedRuns"?: number;
|
|
3149
|
+
"sampleFloor"?: number;
|
|
3150
|
+
};
|
|
3151
|
+
"defaultConfig": {
|
|
3152
|
+
"matrix"?: Array<{
|
|
3153
|
+
"harness": string;
|
|
3154
|
+
"model": string;
|
|
3155
|
+
"provider"?: string;
|
|
3156
|
+
"params"?: Record<string, unknown>;
|
|
3157
|
+
}>;
|
|
3158
|
+
"skills"?: Array<string>;
|
|
3159
|
+
"tools"?: "default" | "none" | {
|
|
3160
|
+
"tools": Array<string>;
|
|
3161
|
+
};
|
|
3162
|
+
"addons"?: Record<string, unknown>;
|
|
3163
|
+
"params"?: Record<string, unknown>;
|
|
3164
|
+
} | null;
|
|
3165
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3166
|
+
"license": string | null;
|
|
3167
|
+
"licenseText": string | null;
|
|
3168
|
+
"licenseVerified": boolean;
|
|
3169
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3170
|
+
"redistribution": "allowed" | "reference_only";
|
|
3171
|
+
"disclosure": "open" | "guarded";
|
|
3172
|
+
"disclosureReason": string | null;
|
|
3173
|
+
"access": "open" | "gated" | "private";
|
|
3174
|
+
"termsUrl": string | null;
|
|
3175
|
+
"homepageUrl": string | null;
|
|
3176
|
+
"paperUrl": string | null;
|
|
3177
|
+
"publishedAt": string | null;
|
|
3178
|
+
"currentVersion": number;
|
|
3179
|
+
"currentVersionId": string | null;
|
|
3180
|
+
"currentVersionTaskCount": number | null;
|
|
3181
|
+
"lenses": Array<string>;
|
|
3182
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3183
|
+
"runnability": {
|
|
3184
|
+
"kind": "runnable";
|
|
3185
|
+
} | {
|
|
3186
|
+
"kind": "corpus-missing";
|
|
3187
|
+
"reason": string;
|
|
3188
|
+
} | {
|
|
3189
|
+
"kind": "needs-import";
|
|
3190
|
+
"reason": string;
|
|
3191
|
+
} | {
|
|
3192
|
+
"kind": "needs-token";
|
|
3193
|
+
"reason": string;
|
|
3194
|
+
} | {
|
|
3195
|
+
"kind": "runner-unsupported";
|
|
3196
|
+
"reason": string;
|
|
3197
|
+
};
|
|
3198
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3199
|
+
"taskTextAvailable": boolean;
|
|
3200
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3201
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3202
|
+
"listed": boolean;
|
|
3203
|
+
"visibility": "private" | "org" | "public";
|
|
3204
|
+
"attestation": {
|
|
3205
|
+
"licenseSpdx": string;
|
|
3206
|
+
"redistribution": "allowed" | "reference_only";
|
|
3207
|
+
"sourceUrl": string;
|
|
3208
|
+
"statement": string;
|
|
3209
|
+
"attestedAt": string;
|
|
3210
|
+
} | null;
|
|
3211
|
+
"rejectionReason": string | null;
|
|
3212
|
+
"uploadedBy": {
|
|
3213
|
+
"userId": string;
|
|
3214
|
+
"displayName": string | null;
|
|
3215
|
+
} | null;
|
|
3216
|
+
"notices": Array<{
|
|
3217
|
+
"kind": string;
|
|
3218
|
+
"text": string;
|
|
3219
|
+
}>;
|
|
3220
|
+
}>;
|
|
3221
|
+
"facets": {
|
|
3222
|
+
"families": Array<"static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite">;
|
|
3223
|
+
"sourceKinds": Array<"hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework">;
|
|
3224
|
+
"hubTiers": Array<"featured" | "community" | "mine">;
|
|
3225
|
+
};
|
|
3226
|
+
};
|
|
3227
|
+
export type StudyBenchmarkDefinitionsCreateRequest = {
|
|
3228
|
+
"slug": string;
|
|
3229
|
+
"name": string;
|
|
3230
|
+
"description"?: string | null;
|
|
3231
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3232
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3233
|
+
"sourceConfig"?: Record<string, unknown>;
|
|
3234
|
+
"adapterId"?: string | null;
|
|
3235
|
+
"headlineMetric": string;
|
|
3236
|
+
"gradingSpec"?: {
|
|
3237
|
+
"lenses": Array<{
|
|
3238
|
+
"id": string;
|
|
3239
|
+
"config"?: Record<string, unknown>;
|
|
3240
|
+
}>;
|
|
3241
|
+
"coverage"?: "every-task" | "per-task";
|
|
3242
|
+
};
|
|
3243
|
+
"environmentSpec"?: {
|
|
3244
|
+
"kind": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3245
|
+
};
|
|
3246
|
+
"methodology"?: {
|
|
3247
|
+
"variantsAllowed"?: boolean;
|
|
3248
|
+
"recommendedRepeats"?: number;
|
|
3249
|
+
"officialFidelity"?: boolean;
|
|
3250
|
+
"modelParams"?: Array<string>;
|
|
3251
|
+
"maxPlannedRuns"?: number;
|
|
3252
|
+
"sampleFloor"?: number;
|
|
3253
|
+
};
|
|
3254
|
+
"runnerKind"?: "none" | "image" | "environment" | "framework";
|
|
3255
|
+
"runnerRecipe"?: Record<string, unknown> | null;
|
|
3256
|
+
"license"?: "MIT" | "Apache-2.0" | "BSD-2-Clause" | "BSD-3-Clause" | "CC-BY-4.0" | "CC0-1.0" | "CC-BY-SA-4.0" | "CC-BY-NC-4.0" | "CC-BY-NC-SA-4.0" | "GPL-3.0" | "AGPL-3.0" | "MPL-2.0" | null;
|
|
3257
|
+
"licenseText"?: string | null;
|
|
3258
|
+
"redistribution"?: "allowed" | "reference_only";
|
|
3259
|
+
"access"?: "open" | "gated" | "private";
|
|
3260
|
+
"termsUrl"?: string | null;
|
|
3261
|
+
"homepageUrl"?: string | null;
|
|
3262
|
+
"paperUrl"?: string | null;
|
|
3263
|
+
"defaultConfig"?: {
|
|
3264
|
+
"matrix"?: Array<{
|
|
3265
|
+
"harness": string;
|
|
3266
|
+
"model": string;
|
|
3267
|
+
"provider"?: string;
|
|
3268
|
+
"params"?: Record<string, unknown>;
|
|
3269
|
+
}>;
|
|
3270
|
+
"skills"?: Array<string>;
|
|
3271
|
+
"tools"?: "default" | "none" | {
|
|
3272
|
+
"tools": Array<string>;
|
|
3273
|
+
};
|
|
3274
|
+
"addons"?: Record<string, unknown>;
|
|
3275
|
+
"params"?: Record<string, unknown>;
|
|
3276
|
+
} | null;
|
|
3277
|
+
"visibility"?: "private" | "org" | "public";
|
|
3278
|
+
};
|
|
3279
|
+
export type StudyBenchmarkDefinitionsCreateResponse = {
|
|
3280
|
+
"definition": {
|
|
3281
|
+
"id": string;
|
|
3282
|
+
"slug": string;
|
|
3283
|
+
"name": string;
|
|
3284
|
+
"description": string | null;
|
|
3285
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3286
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3287
|
+
"headlineMetric": string;
|
|
3288
|
+
"methodology": {
|
|
3289
|
+
"variantsAllowed"?: boolean;
|
|
3290
|
+
"recommendedRepeats"?: number;
|
|
3291
|
+
"officialFidelity"?: boolean;
|
|
3292
|
+
"modelParams"?: Array<string>;
|
|
3293
|
+
"maxPlannedRuns"?: number;
|
|
3294
|
+
"sampleFloor"?: number;
|
|
3295
|
+
};
|
|
3296
|
+
"defaultConfig": {
|
|
3297
|
+
"matrix"?: Array<{
|
|
3298
|
+
"harness": string;
|
|
3299
|
+
"model": string;
|
|
3300
|
+
"provider"?: string;
|
|
3301
|
+
"params"?: Record<string, unknown>;
|
|
3302
|
+
}>;
|
|
3303
|
+
"skills"?: Array<string>;
|
|
3304
|
+
"tools"?: "default" | "none" | {
|
|
3305
|
+
"tools": Array<string>;
|
|
3306
|
+
};
|
|
3307
|
+
"addons"?: Record<string, unknown>;
|
|
3308
|
+
"params"?: Record<string, unknown>;
|
|
3309
|
+
} | null;
|
|
3310
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3311
|
+
"license": string | null;
|
|
3312
|
+
"licenseText": string | null;
|
|
3313
|
+
"licenseVerified": boolean;
|
|
3314
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3315
|
+
"redistribution": "allowed" | "reference_only";
|
|
3316
|
+
"disclosure": "open" | "guarded";
|
|
3317
|
+
"disclosureReason": string | null;
|
|
3318
|
+
"access": "open" | "gated" | "private";
|
|
3319
|
+
"termsUrl": string | null;
|
|
3320
|
+
"homepageUrl": string | null;
|
|
3321
|
+
"paperUrl": string | null;
|
|
3322
|
+
"publishedAt": string | null;
|
|
3323
|
+
"currentVersion": number;
|
|
3324
|
+
"currentVersionId": string | null;
|
|
3325
|
+
"currentVersionTaskCount": number | null;
|
|
3326
|
+
"lenses": Array<string>;
|
|
3327
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3328
|
+
"runnability": {
|
|
3329
|
+
"kind": "runnable";
|
|
3330
|
+
} | {
|
|
3331
|
+
"kind": "corpus-missing";
|
|
3332
|
+
"reason": string;
|
|
3333
|
+
} | {
|
|
3334
|
+
"kind": "needs-import";
|
|
3335
|
+
"reason": string;
|
|
3336
|
+
} | {
|
|
3337
|
+
"kind": "needs-token";
|
|
3338
|
+
"reason": string;
|
|
3339
|
+
} | {
|
|
3340
|
+
"kind": "runner-unsupported";
|
|
3341
|
+
"reason": string;
|
|
3342
|
+
};
|
|
3343
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3344
|
+
"taskTextAvailable": boolean;
|
|
3345
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3346
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3347
|
+
"listed": boolean;
|
|
3348
|
+
"visibility": "private" | "org" | "public";
|
|
3349
|
+
"attestation": {
|
|
3350
|
+
"licenseSpdx": string;
|
|
3351
|
+
"redistribution": "allowed" | "reference_only";
|
|
3352
|
+
"sourceUrl": string;
|
|
3353
|
+
"statement": string;
|
|
3354
|
+
"attestedAt": string;
|
|
3355
|
+
} | null;
|
|
3356
|
+
"rejectionReason": string | null;
|
|
3357
|
+
"uploadedBy": {
|
|
3358
|
+
"userId": string;
|
|
3359
|
+
"displayName": string | null;
|
|
3360
|
+
} | null;
|
|
3361
|
+
"notices": Array<{
|
|
3362
|
+
"kind": string;
|
|
3363
|
+
"text": string;
|
|
3364
|
+
}>;
|
|
3365
|
+
};
|
|
3366
|
+
};
|
|
3367
|
+
export type StudyBenchmarkDefinitionsGetResponse = {
|
|
3368
|
+
"definition": {
|
|
3369
|
+
"id": string;
|
|
3370
|
+
"slug": string;
|
|
3371
|
+
"name": string;
|
|
3372
|
+
"description": string | null;
|
|
3373
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3374
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3375
|
+
"headlineMetric": string;
|
|
3376
|
+
"methodology": {
|
|
3377
|
+
"variantsAllowed"?: boolean;
|
|
3378
|
+
"recommendedRepeats"?: number;
|
|
3379
|
+
"officialFidelity"?: boolean;
|
|
3380
|
+
"modelParams"?: Array<string>;
|
|
3381
|
+
"maxPlannedRuns"?: number;
|
|
3382
|
+
"sampleFloor"?: number;
|
|
3383
|
+
};
|
|
3384
|
+
"defaultConfig": {
|
|
3385
|
+
"matrix"?: Array<{
|
|
3386
|
+
"harness": string;
|
|
3387
|
+
"model": string;
|
|
3388
|
+
"provider"?: string;
|
|
3389
|
+
"params"?: Record<string, unknown>;
|
|
3390
|
+
}>;
|
|
3391
|
+
"skills"?: Array<string>;
|
|
3392
|
+
"tools"?: "default" | "none" | {
|
|
3393
|
+
"tools": Array<string>;
|
|
3394
|
+
};
|
|
3395
|
+
"addons"?: Record<string, unknown>;
|
|
3396
|
+
"params"?: Record<string, unknown>;
|
|
3397
|
+
} | null;
|
|
3398
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3399
|
+
"license": string | null;
|
|
3400
|
+
"licenseText": string | null;
|
|
3401
|
+
"licenseVerified": boolean;
|
|
3402
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3403
|
+
"redistribution": "allowed" | "reference_only";
|
|
3404
|
+
"disclosure": "open" | "guarded";
|
|
3405
|
+
"disclosureReason": string | null;
|
|
3406
|
+
"access": "open" | "gated" | "private";
|
|
3407
|
+
"termsUrl": string | null;
|
|
3408
|
+
"homepageUrl": string | null;
|
|
3409
|
+
"paperUrl": string | null;
|
|
3410
|
+
"publishedAt": string | null;
|
|
3411
|
+
"currentVersion": number;
|
|
3412
|
+
"currentVersionId": string | null;
|
|
3413
|
+
"currentVersionTaskCount": number | null;
|
|
3414
|
+
"lenses": Array<string>;
|
|
3415
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3416
|
+
"runnability": {
|
|
3417
|
+
"kind": "runnable";
|
|
3418
|
+
} | {
|
|
3419
|
+
"kind": "corpus-missing";
|
|
3420
|
+
"reason": string;
|
|
3421
|
+
} | {
|
|
3422
|
+
"kind": "needs-import";
|
|
3423
|
+
"reason": string;
|
|
3424
|
+
} | {
|
|
3425
|
+
"kind": "needs-token";
|
|
3426
|
+
"reason": string;
|
|
3427
|
+
} | {
|
|
3428
|
+
"kind": "runner-unsupported";
|
|
3429
|
+
"reason": string;
|
|
3430
|
+
};
|
|
3431
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3432
|
+
"taskTextAvailable": boolean;
|
|
3433
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3434
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3435
|
+
"listed": boolean;
|
|
3436
|
+
"visibility": "private" | "org" | "public";
|
|
3437
|
+
"attestation": {
|
|
3438
|
+
"licenseSpdx": string;
|
|
3439
|
+
"redistribution": "allowed" | "reference_only";
|
|
3440
|
+
"sourceUrl": string;
|
|
3441
|
+
"statement": string;
|
|
3442
|
+
"attestedAt": string;
|
|
3443
|
+
} | null;
|
|
3444
|
+
"rejectionReason": string | null;
|
|
3445
|
+
"uploadedBy": {
|
|
3446
|
+
"userId": string;
|
|
3447
|
+
"displayName": string | null;
|
|
3448
|
+
} | null;
|
|
3449
|
+
"notices": Array<{
|
|
3450
|
+
"kind": string;
|
|
3451
|
+
"text": string;
|
|
3452
|
+
}>;
|
|
3453
|
+
};
|
|
3454
|
+
};
|
|
3455
|
+
export type StudyBenchmarkDefinitionsUpdateRequest = {
|
|
3456
|
+
"name"?: string;
|
|
3457
|
+
"description"?: string | null;
|
|
3458
|
+
"family"?: "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3459
|
+
"sourceKind"?: "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3460
|
+
"sourceConfig"?: Record<string, unknown>;
|
|
3461
|
+
"adapterId"?: string | null;
|
|
3462
|
+
"headlineMetric"?: string;
|
|
3463
|
+
"gradingSpec"?: {
|
|
3464
|
+
"lenses": Array<{
|
|
3465
|
+
"id": string;
|
|
3466
|
+
"config"?: Record<string, unknown>;
|
|
3467
|
+
}>;
|
|
3468
|
+
"coverage"?: "every-task" | "per-task";
|
|
3469
|
+
};
|
|
3470
|
+
"environmentSpec"?: {
|
|
3471
|
+
"kind": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3472
|
+
};
|
|
3473
|
+
"methodology"?: {
|
|
3474
|
+
"variantsAllowed"?: boolean;
|
|
3475
|
+
"recommendedRepeats"?: number;
|
|
3476
|
+
"officialFidelity"?: boolean;
|
|
3477
|
+
"modelParams"?: Array<string>;
|
|
3478
|
+
"maxPlannedRuns"?: number;
|
|
3479
|
+
"sampleFloor"?: number;
|
|
3480
|
+
};
|
|
3481
|
+
"runnerKind"?: "none" | "image" | "environment" | "framework";
|
|
3482
|
+
"runnerRecipe"?: Record<string, unknown> | null;
|
|
3483
|
+
"license"?: "MIT" | "Apache-2.0" | "BSD-2-Clause" | "BSD-3-Clause" | "CC-BY-4.0" | "CC0-1.0" | "CC-BY-SA-4.0" | "CC-BY-NC-4.0" | "CC-BY-NC-SA-4.0" | "GPL-3.0" | "AGPL-3.0" | "MPL-2.0" | null;
|
|
3484
|
+
"licenseText"?: string | null;
|
|
3485
|
+
"redistribution"?: "allowed" | "reference_only";
|
|
3486
|
+
"access"?: "open" | "gated" | "private";
|
|
3487
|
+
"termsUrl"?: string | null;
|
|
3488
|
+
"homepageUrl"?: string | null;
|
|
3489
|
+
"paperUrl"?: string | null;
|
|
3490
|
+
"defaultConfig"?: {
|
|
3491
|
+
"matrix"?: Array<{
|
|
3492
|
+
"harness": string;
|
|
3493
|
+
"model": string;
|
|
3494
|
+
"provider"?: string;
|
|
3495
|
+
"params"?: Record<string, unknown>;
|
|
3496
|
+
}>;
|
|
3497
|
+
"skills"?: Array<string>;
|
|
3498
|
+
"tools"?: "default" | "none" | {
|
|
3499
|
+
"tools": Array<string>;
|
|
3500
|
+
};
|
|
3501
|
+
"addons"?: Record<string, unknown>;
|
|
3502
|
+
"params"?: Record<string, unknown>;
|
|
3503
|
+
} | null;
|
|
3504
|
+
"visibility"?: "private" | "org" | "public";
|
|
3505
|
+
};
|
|
3506
|
+
export type StudyBenchmarkDefinitionsUpdateResponse = {
|
|
3507
|
+
"definition": {
|
|
3508
|
+
"id": string;
|
|
3509
|
+
"slug": string;
|
|
3510
|
+
"name": string;
|
|
3511
|
+
"description": string | null;
|
|
3512
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3513
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3514
|
+
"headlineMetric": string;
|
|
3515
|
+
"methodology": {
|
|
3516
|
+
"variantsAllowed"?: boolean;
|
|
3517
|
+
"recommendedRepeats"?: number;
|
|
3518
|
+
"officialFidelity"?: boolean;
|
|
3519
|
+
"modelParams"?: Array<string>;
|
|
3520
|
+
"maxPlannedRuns"?: number;
|
|
3521
|
+
"sampleFloor"?: number;
|
|
3522
|
+
};
|
|
3523
|
+
"defaultConfig": {
|
|
3524
|
+
"matrix"?: Array<{
|
|
3525
|
+
"harness": string;
|
|
3526
|
+
"model": string;
|
|
3527
|
+
"provider"?: string;
|
|
3528
|
+
"params"?: Record<string, unknown>;
|
|
3529
|
+
}>;
|
|
3530
|
+
"skills"?: Array<string>;
|
|
3531
|
+
"tools"?: "default" | "none" | {
|
|
3532
|
+
"tools": Array<string>;
|
|
3533
|
+
};
|
|
3534
|
+
"addons"?: Record<string, unknown>;
|
|
3535
|
+
"params"?: Record<string, unknown>;
|
|
3536
|
+
} | null;
|
|
3537
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3538
|
+
"license": string | null;
|
|
3539
|
+
"licenseText": string | null;
|
|
3540
|
+
"licenseVerified": boolean;
|
|
3541
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3542
|
+
"redistribution": "allowed" | "reference_only";
|
|
3543
|
+
"disclosure": "open" | "guarded";
|
|
3544
|
+
"disclosureReason": string | null;
|
|
3545
|
+
"access": "open" | "gated" | "private";
|
|
3546
|
+
"termsUrl": string | null;
|
|
3547
|
+
"homepageUrl": string | null;
|
|
3548
|
+
"paperUrl": string | null;
|
|
3549
|
+
"publishedAt": string | null;
|
|
3550
|
+
"currentVersion": number;
|
|
3551
|
+
"currentVersionId": string | null;
|
|
3552
|
+
"currentVersionTaskCount": number | null;
|
|
3553
|
+
"lenses": Array<string>;
|
|
3554
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3555
|
+
"runnability": {
|
|
3556
|
+
"kind": "runnable";
|
|
3557
|
+
} | {
|
|
3558
|
+
"kind": "corpus-missing";
|
|
3559
|
+
"reason": string;
|
|
3560
|
+
} | {
|
|
3561
|
+
"kind": "needs-import";
|
|
3562
|
+
"reason": string;
|
|
3563
|
+
} | {
|
|
3564
|
+
"kind": "needs-token";
|
|
3565
|
+
"reason": string;
|
|
3566
|
+
} | {
|
|
3567
|
+
"kind": "runner-unsupported";
|
|
3568
|
+
"reason": string;
|
|
3569
|
+
};
|
|
3570
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3571
|
+
"taskTextAvailable": boolean;
|
|
3572
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3573
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3574
|
+
"listed": boolean;
|
|
3575
|
+
"visibility": "private" | "org" | "public";
|
|
3576
|
+
"attestation": {
|
|
3577
|
+
"licenseSpdx": string;
|
|
3578
|
+
"redistribution": "allowed" | "reference_only";
|
|
3579
|
+
"sourceUrl": string;
|
|
3580
|
+
"statement": string;
|
|
3581
|
+
"attestedAt": string;
|
|
3582
|
+
} | null;
|
|
3583
|
+
"rejectionReason": string | null;
|
|
3584
|
+
"uploadedBy": {
|
|
3585
|
+
"userId": string;
|
|
3586
|
+
"displayName": string | null;
|
|
3587
|
+
} | null;
|
|
3588
|
+
"notices": Array<{
|
|
3589
|
+
"kind": string;
|
|
3590
|
+
"text": string;
|
|
3591
|
+
}>;
|
|
3592
|
+
};
|
|
3593
|
+
};
|
|
3594
|
+
export type StudyBenchmarkDefinitionsAttachCorpusRequest = {
|
|
3595
|
+
"version"?: number;
|
|
3596
|
+
"limit"?: number;
|
|
3597
|
+
"changelog"?: string;
|
|
3598
|
+
};
|
|
3599
|
+
export type StudyBenchmarkDefinitionsPublishRequest = {
|
|
3600
|
+
"version"?: number;
|
|
3601
|
+
};
|
|
3602
|
+
export type StudyBenchmarkDefinitionsPublishResponse = {
|
|
3603
|
+
"slug": string;
|
|
3604
|
+
"currentVersion": number;
|
|
3605
|
+
"versionId": string;
|
|
3606
|
+
"taskCount": number;
|
|
3607
|
+
};
|
|
3608
|
+
export type StudyBenchmarkDefinitionsSubmitRequest = {
|
|
3609
|
+
"attestation": {
|
|
3610
|
+
"licenseSpdx": "MIT" | "Apache-2.0" | "BSD-2-Clause" | "BSD-3-Clause" | "CC-BY-4.0" | "CC0-1.0" | "CC-BY-SA-4.0" | "CC-BY-NC-4.0" | "CC-BY-NC-SA-4.0" | "GPL-3.0" | "AGPL-3.0" | "MPL-2.0" | "";
|
|
3611
|
+
"redistribution": "allowed" | "reference_only";
|
|
3612
|
+
"sourceUrl": string;
|
|
3613
|
+
"statement": string;
|
|
3614
|
+
};
|
|
3615
|
+
};
|
|
3616
|
+
export type StudyBenchmarkDefinitionsSubmitResponse = {
|
|
3617
|
+
"definition": {
|
|
3618
|
+
"id": string;
|
|
3619
|
+
"slug": string;
|
|
3620
|
+
"name": string;
|
|
3621
|
+
"description": string | null;
|
|
3622
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3623
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3624
|
+
"headlineMetric": string;
|
|
3625
|
+
"methodology": {
|
|
3626
|
+
"variantsAllowed"?: boolean;
|
|
3627
|
+
"recommendedRepeats"?: number;
|
|
3628
|
+
"officialFidelity"?: boolean;
|
|
3629
|
+
"modelParams"?: Array<string>;
|
|
3630
|
+
"maxPlannedRuns"?: number;
|
|
3631
|
+
"sampleFloor"?: number;
|
|
3632
|
+
};
|
|
3633
|
+
"defaultConfig": {
|
|
3634
|
+
"matrix"?: Array<{
|
|
3635
|
+
"harness": string;
|
|
3636
|
+
"model": string;
|
|
3637
|
+
"provider"?: string;
|
|
3638
|
+
"params"?: Record<string, unknown>;
|
|
3639
|
+
}>;
|
|
3640
|
+
"skills"?: Array<string>;
|
|
3641
|
+
"tools"?: "default" | "none" | {
|
|
3642
|
+
"tools": Array<string>;
|
|
3643
|
+
};
|
|
3644
|
+
"addons"?: Record<string, unknown>;
|
|
3645
|
+
"params"?: Record<string, unknown>;
|
|
3646
|
+
} | null;
|
|
3647
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3648
|
+
"license": string | null;
|
|
3649
|
+
"licenseText": string | null;
|
|
3650
|
+
"licenseVerified": boolean;
|
|
3651
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3652
|
+
"redistribution": "allowed" | "reference_only";
|
|
3653
|
+
"disclosure": "open" | "guarded";
|
|
3654
|
+
"disclosureReason": string | null;
|
|
3655
|
+
"access": "open" | "gated" | "private";
|
|
3656
|
+
"termsUrl": string | null;
|
|
3657
|
+
"homepageUrl": string | null;
|
|
3658
|
+
"paperUrl": string | null;
|
|
3659
|
+
"publishedAt": string | null;
|
|
3660
|
+
"currentVersion": number;
|
|
3661
|
+
"currentVersionId": string | null;
|
|
3662
|
+
"currentVersionTaskCount": number | null;
|
|
3663
|
+
"lenses": Array<string>;
|
|
3664
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3665
|
+
"runnability": {
|
|
3666
|
+
"kind": "runnable";
|
|
3667
|
+
} | {
|
|
3668
|
+
"kind": "corpus-missing";
|
|
3669
|
+
"reason": string;
|
|
3670
|
+
} | {
|
|
3671
|
+
"kind": "needs-import";
|
|
3672
|
+
"reason": string;
|
|
3673
|
+
} | {
|
|
3674
|
+
"kind": "needs-token";
|
|
3675
|
+
"reason": string;
|
|
3676
|
+
} | {
|
|
3677
|
+
"kind": "runner-unsupported";
|
|
3678
|
+
"reason": string;
|
|
3679
|
+
};
|
|
3680
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3681
|
+
"taskTextAvailable": boolean;
|
|
3682
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3683
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3684
|
+
"listed": boolean;
|
|
3685
|
+
"visibility": "private" | "org" | "public";
|
|
3686
|
+
"attestation": {
|
|
3687
|
+
"licenseSpdx": string;
|
|
3688
|
+
"redistribution": "allowed" | "reference_only";
|
|
3689
|
+
"sourceUrl": string;
|
|
3690
|
+
"statement": string;
|
|
3691
|
+
"attestedAt": string;
|
|
3692
|
+
} | null;
|
|
3693
|
+
"rejectionReason": string | null;
|
|
3694
|
+
"uploadedBy": {
|
|
3695
|
+
"userId": string;
|
|
3696
|
+
"displayName": string | null;
|
|
3697
|
+
} | null;
|
|
3698
|
+
"notices": Array<{
|
|
3699
|
+
"kind": string;
|
|
3700
|
+
"text": string;
|
|
3701
|
+
}>;
|
|
3702
|
+
};
|
|
3703
|
+
};
|
|
3704
|
+
export type StudyBenchmarkDefinitionsSetVisibilityRequest = {
|
|
3705
|
+
"visibility": "private" | "org" | "public";
|
|
3706
|
+
};
|
|
3707
|
+
export type StudyBenchmarkDefinitionsSetVisibilityResponse = {
|
|
3708
|
+
"definition": {
|
|
3709
|
+
"id": string;
|
|
3710
|
+
"slug": string;
|
|
3711
|
+
"name": string;
|
|
3712
|
+
"description": string | null;
|
|
3713
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3714
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3715
|
+
"headlineMetric": string;
|
|
3716
|
+
"methodology": {
|
|
3717
|
+
"variantsAllowed"?: boolean;
|
|
3718
|
+
"recommendedRepeats"?: number;
|
|
3719
|
+
"officialFidelity"?: boolean;
|
|
3720
|
+
"modelParams"?: Array<string>;
|
|
3721
|
+
"maxPlannedRuns"?: number;
|
|
3722
|
+
"sampleFloor"?: number;
|
|
3723
|
+
};
|
|
3724
|
+
"defaultConfig": {
|
|
3725
|
+
"matrix"?: Array<{
|
|
3726
|
+
"harness": string;
|
|
3727
|
+
"model": string;
|
|
3728
|
+
"provider"?: string;
|
|
3729
|
+
"params"?: Record<string, unknown>;
|
|
3730
|
+
}>;
|
|
3731
|
+
"skills"?: Array<string>;
|
|
3732
|
+
"tools"?: "default" | "none" | {
|
|
3733
|
+
"tools": Array<string>;
|
|
3734
|
+
};
|
|
3735
|
+
"addons"?: Record<string, unknown>;
|
|
3736
|
+
"params"?: Record<string, unknown>;
|
|
3737
|
+
} | null;
|
|
3738
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3739
|
+
"license": string | null;
|
|
3740
|
+
"licenseText": string | null;
|
|
3741
|
+
"licenseVerified": boolean;
|
|
3742
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3743
|
+
"redistribution": "allowed" | "reference_only";
|
|
3744
|
+
"disclosure": "open" | "guarded";
|
|
3745
|
+
"disclosureReason": string | null;
|
|
3746
|
+
"access": "open" | "gated" | "private";
|
|
3747
|
+
"termsUrl": string | null;
|
|
3748
|
+
"homepageUrl": string | null;
|
|
3749
|
+
"paperUrl": string | null;
|
|
3750
|
+
"publishedAt": string | null;
|
|
3751
|
+
"currentVersion": number;
|
|
3752
|
+
"currentVersionId": string | null;
|
|
3753
|
+
"currentVersionTaskCount": number | null;
|
|
3754
|
+
"lenses": Array<string>;
|
|
3755
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3756
|
+
"runnability": {
|
|
3757
|
+
"kind": "runnable";
|
|
3758
|
+
} | {
|
|
3759
|
+
"kind": "corpus-missing";
|
|
3760
|
+
"reason": string;
|
|
3761
|
+
} | {
|
|
3762
|
+
"kind": "needs-import";
|
|
3763
|
+
"reason": string;
|
|
3764
|
+
} | {
|
|
3765
|
+
"kind": "needs-token";
|
|
3766
|
+
"reason": string;
|
|
3767
|
+
} | {
|
|
3768
|
+
"kind": "runner-unsupported";
|
|
3769
|
+
"reason": string;
|
|
3770
|
+
};
|
|
3771
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3772
|
+
"taskTextAvailable": boolean;
|
|
3773
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3774
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3775
|
+
"listed": boolean;
|
|
3776
|
+
"visibility": "private" | "org" | "public";
|
|
3777
|
+
"attestation": {
|
|
3778
|
+
"licenseSpdx": string;
|
|
3779
|
+
"redistribution": "allowed" | "reference_only";
|
|
3780
|
+
"sourceUrl": string;
|
|
3781
|
+
"statement": string;
|
|
3782
|
+
"attestedAt": string;
|
|
3783
|
+
} | null;
|
|
3784
|
+
"rejectionReason": string | null;
|
|
3785
|
+
"uploadedBy": {
|
|
3786
|
+
"userId": string;
|
|
3787
|
+
"displayName": string | null;
|
|
3788
|
+
} | null;
|
|
3789
|
+
"notices": Array<{
|
|
3790
|
+
"kind": string;
|
|
3791
|
+
"text": string;
|
|
3792
|
+
}>;
|
|
3793
|
+
};
|
|
3794
|
+
};
|
|
3795
|
+
export type StudyBenchmarkDefinitionsWithdrawResponse = {
|
|
3796
|
+
"definition": {
|
|
3797
|
+
"id": string;
|
|
3798
|
+
"slug": string;
|
|
3799
|
+
"name": string;
|
|
3800
|
+
"description": string | null;
|
|
3801
|
+
"family": "static_qa" | "code" | "agentic" | "tools" | "web" | "knowledge_work" | "safety" | "suite";
|
|
3802
|
+
"sourceKind": "hf-dataset" | "jsonl" | "git-tasks" | "release-file" | "authored" | "framework";
|
|
3803
|
+
"headlineMetric": string;
|
|
3804
|
+
"methodology": {
|
|
3805
|
+
"variantsAllowed"?: boolean;
|
|
3806
|
+
"recommendedRepeats"?: number;
|
|
3807
|
+
"officialFidelity"?: boolean;
|
|
3808
|
+
"modelParams"?: Array<string>;
|
|
3809
|
+
"maxPlannedRuns"?: number;
|
|
3810
|
+
"sampleFloor"?: number;
|
|
3811
|
+
};
|
|
3812
|
+
"defaultConfig": {
|
|
3813
|
+
"matrix"?: Array<{
|
|
3814
|
+
"harness": string;
|
|
3815
|
+
"model": string;
|
|
3816
|
+
"provider"?: string;
|
|
3817
|
+
"params"?: Record<string, unknown>;
|
|
3818
|
+
}>;
|
|
3819
|
+
"skills"?: Array<string>;
|
|
3820
|
+
"tools"?: "default" | "none" | {
|
|
3821
|
+
"tools": Array<string>;
|
|
3822
|
+
};
|
|
3823
|
+
"addons"?: Record<string, unknown>;
|
|
3824
|
+
"params"?: Record<string, unknown>;
|
|
3825
|
+
} | null;
|
|
3826
|
+
"runnerKind": "none" | "image" | "environment" | "framework";
|
|
3827
|
+
"license": string | null;
|
|
3828
|
+
"licenseText": string | null;
|
|
3829
|
+
"licenseVerified": boolean;
|
|
3830
|
+
"licenseVerifiedByPolicy": boolean;
|
|
3831
|
+
"redistribution": "allowed" | "reference_only";
|
|
3832
|
+
"disclosure": "open" | "guarded";
|
|
3833
|
+
"disclosureReason": string | null;
|
|
3834
|
+
"access": "open" | "gated" | "private";
|
|
3835
|
+
"termsUrl": string | null;
|
|
3836
|
+
"homepageUrl": string | null;
|
|
3837
|
+
"paperUrl": string | null;
|
|
3838
|
+
"publishedAt": string | null;
|
|
3839
|
+
"currentVersion": number;
|
|
3840
|
+
"currentVersionId": string | null;
|
|
3841
|
+
"currentVersionTaskCount": number | null;
|
|
3842
|
+
"lenses": Array<string>;
|
|
3843
|
+
"environment": "none" | "workspace" | "image" | "hosted" | "services" | "stack";
|
|
3844
|
+
"runnability": {
|
|
3845
|
+
"kind": "runnable";
|
|
3846
|
+
} | {
|
|
3847
|
+
"kind": "corpus-missing";
|
|
3848
|
+
"reason": string;
|
|
3849
|
+
} | {
|
|
3850
|
+
"kind": "needs-import";
|
|
3851
|
+
"reason": string;
|
|
3852
|
+
} | {
|
|
3853
|
+
"kind": "needs-token";
|
|
3854
|
+
"reason": string;
|
|
3855
|
+
} | {
|
|
3856
|
+
"kind": "runner-unsupported";
|
|
3857
|
+
"reason": string;
|
|
3858
|
+
};
|
|
3859
|
+
"runnerImage": "not-needed" | "not-built" | "building" | "ready";
|
|
3860
|
+
"taskTextAvailable": boolean;
|
|
3861
|
+
"hubTier": "featured" | "community" | "mine";
|
|
3862
|
+
"hubStatus": "draft" | "submitted" | "verified" | "rejected";
|
|
3863
|
+
"listed": boolean;
|
|
3864
|
+
"visibility": "private" | "org" | "public";
|
|
3865
|
+
"attestation": {
|
|
3866
|
+
"licenseSpdx": string;
|
|
3867
|
+
"redistribution": "allowed" | "reference_only";
|
|
3868
|
+
"sourceUrl": string;
|
|
3869
|
+
"statement": string;
|
|
3870
|
+
"attestedAt": string;
|
|
3871
|
+
} | null;
|
|
3872
|
+
"rejectionReason": string | null;
|
|
3873
|
+
"uploadedBy": {
|
|
3874
|
+
"userId": string;
|
|
3875
|
+
"displayName": string | null;
|
|
3876
|
+
} | null;
|
|
3877
|
+
"notices": Array<{
|
|
3878
|
+
"kind": string;
|
|
3879
|
+
"text": string;
|
|
3880
|
+
}>;
|
|
3881
|
+
};
|
|
3882
|
+
};
|
|
3883
|
+
export type StudyBenchmarkImportsGetResponse = {
|
|
3884
|
+
"job": {
|
|
3885
|
+
"id": string;
|
|
3886
|
+
"definitionId": string;
|
|
3887
|
+
"status": "queued" | "running" | "succeeded" | "failed";
|
|
3888
|
+
"door": "platform" | "organization";
|
|
3889
|
+
"request": Record<string, unknown>;
|
|
3890
|
+
"result": Record<string, unknown> | null;
|
|
3891
|
+
"error": {
|
|
3892
|
+
"code": string;
|
|
3893
|
+
"message": string;
|
|
3894
|
+
"detail"?: unknown;
|
|
3895
|
+
} | null;
|
|
3896
|
+
"executor": string;
|
|
3897
|
+
"worker": string | null;
|
|
3898
|
+
"createdAt": string;
|
|
3899
|
+
"startedAt": string | null;
|
|
3900
|
+
"finishedAt": string | null;
|
|
3901
|
+
};
|
|
3902
|
+
};
|
|
3903
|
+
export type StudyBenchmarkImportsStreamResponse = unknown;
|
|
3904
|
+
export type StudyBenchmarkIntakeAssistRequest = {
|
|
3905
|
+
"strategy": string;
|
|
3906
|
+
"config": Record<string, unknown>;
|
|
3907
|
+
"probe"?: {
|
|
3908
|
+
"strategy": string;
|
|
3909
|
+
"proposals": Array<{
|
|
3910
|
+
"field": "slug" | "name" | "description" | "family" | "sourceKind" | "sourceConfig" | "adapterId" | "headlineMetric" | "gradingSpec" | "environmentSpec" | "methodology" | "runnerKind" | "runnerRecipe" | "license" | "licenseText" | "redistribution" | "access" | "termsUrl" | "homepageUrl" | "paperUrl" | "defaultConfig" | "visibility";
|
|
3911
|
+
"value": unknown;
|
|
3912
|
+
"confidence": "high" | "medium" | "low";
|
|
3913
|
+
"rationale": string;
|
|
3914
|
+
"evidencePath"?: string;
|
|
3915
|
+
}>;
|
|
3916
|
+
"gaps": Array<{
|
|
3917
|
+
"field": "slug" | "name" | "description" | "family" | "sourceKind" | "sourceConfig" | "adapterId" | "headlineMetric" | "gradingSpec" | "environmentSpec" | "methodology" | "runnerKind" | "runnerRecipe" | "license" | "licenseText" | "redistribution" | "access" | "termsUrl" | "homepageUrl" | "paperUrl" | "defaultConfig" | "visibility";
|
|
3918
|
+
"reason": string;
|
|
3919
|
+
}>;
|
|
3920
|
+
"evidence": Array<{
|
|
3921
|
+
"kind": "file" | "metadata";
|
|
3922
|
+
"path": string;
|
|
3923
|
+
"excerpt": string;
|
|
3924
|
+
"bytes": number;
|
|
3925
|
+
}>;
|
|
3926
|
+
"notes": Array<string>;
|
|
3927
|
+
};
|
|
3928
|
+
};
|
|
3929
|
+
export type StudyBenchmarkIntakeAssistResponse = {
|
|
3930
|
+
"strategy": string;
|
|
3931
|
+
"draft": Record<string, unknown>;
|
|
3932
|
+
"fields": Array<{
|
|
3933
|
+
"field": string;
|
|
3934
|
+
"value": unknown;
|
|
3935
|
+
"confidence": "high" | "medium" | "low";
|
|
3936
|
+
"rationale": string;
|
|
3937
|
+
"evidencePath"?: string;
|
|
3938
|
+
"source": "probe" | "agent";
|
|
3939
|
+
}>;
|
|
3940
|
+
"unresolved": Array<{
|
|
3941
|
+
"field": string;
|
|
3942
|
+
"reason": string;
|
|
3943
|
+
}>;
|
|
3944
|
+
"gaps": Array<{
|
|
3945
|
+
"field": string;
|
|
3946
|
+
"reason": string;
|
|
3947
|
+
}>;
|
|
3948
|
+
"evidence": Array<{
|
|
3949
|
+
"kind": "file" | "metadata";
|
|
3950
|
+
"path": string;
|
|
3951
|
+
"excerpt": string;
|
|
3952
|
+
"bytes": number;
|
|
3953
|
+
}>;
|
|
3954
|
+
"notes": Array<string>;
|
|
3955
|
+
"model": string | null;
|
|
3956
|
+
"usage": {
|
|
3957
|
+
"inputTokens": number;
|
|
3958
|
+
"outputTokens": number;
|
|
3959
|
+
} | null;
|
|
3960
|
+
};
|
|
3961
|
+
export type StudyBenchmarkIntakeProbeRequest = {
|
|
3962
|
+
"strategy": string;
|
|
3963
|
+
"config": Record<string, unknown>;
|
|
3964
|
+
};
|
|
3965
|
+
export type StudyBenchmarkIntakeProbeResponse = {
|
|
3966
|
+
"strategy": string;
|
|
3967
|
+
"draft": Record<string, unknown>;
|
|
3968
|
+
"proposals": Array<{
|
|
3969
|
+
"field": string;
|
|
3970
|
+
"value": unknown;
|
|
3971
|
+
"confidence": "high" | "medium" | "low";
|
|
3972
|
+
"rationale": string;
|
|
3973
|
+
"evidencePath"?: string;
|
|
3974
|
+
}>;
|
|
3975
|
+
"gaps": Array<{
|
|
3976
|
+
"field": string;
|
|
3977
|
+
"reason": string;
|
|
3978
|
+
}>;
|
|
3979
|
+
"evidence": Array<{
|
|
3980
|
+
"kind": "file" | "metadata";
|
|
3981
|
+
"path": string;
|
|
3982
|
+
"excerpt": string;
|
|
3983
|
+
"bytes": number;
|
|
3984
|
+
}>;
|
|
3985
|
+
"notes": Array<string>;
|
|
3986
|
+
};
|
|
3987
|
+
export type StudyBenchmarkIntakeStrategiesResponse = {
|
|
3988
|
+
"strategies": Array<{
|
|
3989
|
+
"id": string;
|
|
3990
|
+
"label": string;
|
|
3991
|
+
"summary": string;
|
|
3992
|
+
"fields": Array<{
|
|
3993
|
+
"name": string;
|
|
3994
|
+
"flag": string;
|
|
3995
|
+
"required": boolean;
|
|
3996
|
+
"kind": "string" | "number" | "boolean" | "array" | "enum" | "object";
|
|
3997
|
+
"description"?: string;
|
|
3998
|
+
"options"?: Array<string>;
|
|
3999
|
+
"defaultValue"?: unknown;
|
|
4000
|
+
}>;
|
|
4001
|
+
}>;
|
|
4002
|
+
};
|
|
4003
|
+
export type StudyBenchmarkTermsGetResponse = {
|
|
4004
|
+
"scope": string;
|
|
4005
|
+
"version": string;
|
|
4006
|
+
"text": Array<string>;
|
|
4007
|
+
"acceptance": {
|
|
4008
|
+
"organizationId": string;
|
|
4009
|
+
"scope": string;
|
|
4010
|
+
"acceptedBy": string;
|
|
4011
|
+
"acceptedAt": string;
|
|
4012
|
+
} | null;
|
|
4013
|
+
};
|
|
4014
|
+
export type StudyCatalogResponse = {
|
|
4015
|
+
"harnesses": Array<{
|
|
4016
|
+
"id": string;
|
|
4017
|
+
"name": string;
|
|
4018
|
+
"maker": string;
|
|
4019
|
+
"offerable": boolean;
|
|
4020
|
+
"reason": string | null;
|
|
4021
|
+
"modes": Array<"task" | "session">;
|
|
4022
|
+
"defaultProvider": string | null;
|
|
4023
|
+
"providers": Array<{
|
|
4024
|
+
"id": string;
|
|
4025
|
+
"offerable": boolean;
|
|
4026
|
+
"reason": string | null;
|
|
4027
|
+
"models": Array<{
|
|
4028
|
+
"id": string;
|
|
4029
|
+
"name": string;
|
|
4030
|
+
}>;
|
|
4031
|
+
"defaultModel": string | null;
|
|
4032
|
+
}>;
|
|
4033
|
+
}>;
|
|
4034
|
+
};
|
|
4035
|
+
export type StudyImagesBuildsListResponse = {
|
|
4036
|
+
"builds": Array<{
|
|
4037
|
+
"id": string;
|
|
4038
|
+
"organizationId": string;
|
|
4039
|
+
"productKey": string;
|
|
4040
|
+
"requestedByUserId": string | null;
|
|
4041
|
+
"name": string;
|
|
4042
|
+
"source": {
|
|
4043
|
+
"kind": "archive";
|
|
4044
|
+
"sha256": string;
|
|
4045
|
+
} | {
|
|
4046
|
+
"kind": "upload";
|
|
4047
|
+
"fileId": string;
|
|
4048
|
+
} | {
|
|
4049
|
+
"kind": "git";
|
|
4050
|
+
"ref": string;
|
|
4051
|
+
"subdir"?: string;
|
|
4052
|
+
} | {
|
|
4053
|
+
"kind": "inline";
|
|
4054
|
+
"fileCount": number;
|
|
4055
|
+
};
|
|
4056
|
+
"sourceIdentity": string;
|
|
4057
|
+
"dockerfileDir": string;
|
|
4058
|
+
"dockerfileName": string;
|
|
4059
|
+
"buildArgKeys": Array<string>;
|
|
4060
|
+
"platform": string;
|
|
4061
|
+
"trusted": boolean;
|
|
4062
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4063
|
+
"image": string;
|
|
4064
|
+
"digest": string | null;
|
|
4065
|
+
"sizeBytes": number | null;
|
|
4066
|
+
"reason": string | null;
|
|
4067
|
+
"logRef": string | null;
|
|
4068
|
+
"startedAt": string | null;
|
|
4069
|
+
"finishedAt": string | null;
|
|
4070
|
+
"createdAt": string;
|
|
4071
|
+
"updatedAt": string;
|
|
4072
|
+
}>;
|
|
4073
|
+
"nextPageToken"?: string;
|
|
4074
|
+
};
|
|
4075
|
+
export type StudyImagesBuildsCreateRequest = {
|
|
4076
|
+
"name": string;
|
|
4077
|
+
"source": {
|
|
4078
|
+
"kind": "archive";
|
|
4079
|
+
"url": string;
|
|
4080
|
+
"sha256": string;
|
|
4081
|
+
} | {
|
|
4082
|
+
"kind": "upload";
|
|
4083
|
+
"fileId": string;
|
|
4084
|
+
} | {
|
|
4085
|
+
"kind": "git";
|
|
4086
|
+
"url": string;
|
|
4087
|
+
"ref": string;
|
|
4088
|
+
"subdir"?: string;
|
|
4089
|
+
} | {
|
|
4090
|
+
"kind": "inline";
|
|
4091
|
+
"dockerfile": string;
|
|
4092
|
+
"files"?: Record<string, unknown>;
|
|
4093
|
+
};
|
|
4094
|
+
"dockerfile"?: {
|
|
4095
|
+
"dir"?: string;
|
|
4096
|
+
"name"?: string;
|
|
4097
|
+
};
|
|
4098
|
+
"buildArgs"?: Record<string, unknown>;
|
|
4099
|
+
"platform"?: string;
|
|
4100
|
+
};
|
|
4101
|
+
export type StudyImagesBuildsCreateResponse = {
|
|
4102
|
+
"id": string;
|
|
4103
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4104
|
+
};
|
|
4105
|
+
export type StudyImagesBuildsGetResponse = {
|
|
4106
|
+
"build": {
|
|
4107
|
+
"id": string;
|
|
4108
|
+
"organizationId": string;
|
|
4109
|
+
"productKey": string;
|
|
4110
|
+
"requestedByUserId": string | null;
|
|
4111
|
+
"name": string;
|
|
4112
|
+
"source": {
|
|
4113
|
+
"kind": "archive";
|
|
4114
|
+
"sha256": string;
|
|
4115
|
+
} | {
|
|
4116
|
+
"kind": "upload";
|
|
4117
|
+
"fileId": string;
|
|
4118
|
+
} | {
|
|
4119
|
+
"kind": "git";
|
|
4120
|
+
"ref": string;
|
|
4121
|
+
"subdir"?: string;
|
|
4122
|
+
} | {
|
|
4123
|
+
"kind": "inline";
|
|
4124
|
+
"fileCount": number;
|
|
4125
|
+
};
|
|
4126
|
+
"sourceIdentity": string;
|
|
4127
|
+
"dockerfileDir": string;
|
|
4128
|
+
"dockerfileName": string;
|
|
4129
|
+
"buildArgKeys": Array<string>;
|
|
4130
|
+
"platform": string;
|
|
4131
|
+
"trusted": boolean;
|
|
4132
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4133
|
+
"image": string;
|
|
4134
|
+
"digest": string | null;
|
|
4135
|
+
"sizeBytes": number | null;
|
|
4136
|
+
"reason": string | null;
|
|
4137
|
+
"logRef": string | null;
|
|
4138
|
+
"startedAt": string | null;
|
|
4139
|
+
"finishedAt": string | null;
|
|
4140
|
+
"createdAt": string;
|
|
4141
|
+
"updatedAt": string;
|
|
4142
|
+
};
|
|
4143
|
+
};
|
|
4144
|
+
export type StudyImagesBuildsCancelResponse = {
|
|
4145
|
+
"id": string;
|
|
4146
|
+
"phase": "pending" | "running" | "succeeded" | "failed" | "cancelled";
|
|
4147
|
+
};
|
|
4148
|
+
export type StudyImagesBuildsStreamResponse = unknown;
|
|
4149
|
+
export type StudyImagesRunsCreateRequest = {
|
|
4150
|
+
"buildId"?: string;
|
|
4151
|
+
"image"?: string;
|
|
4152
|
+
"mode": "harness" | "environment";
|
|
4153
|
+
"harness"?: string;
|
|
4154
|
+
"task"?: string;
|
|
4155
|
+
};
|
|
4156
|
+
export type StudyImagesRunsCreateResponse = {
|
|
4157
|
+
"runId": string;
|
|
4158
|
+
};
|
|
4159
|
+
export type StudyRunsListResponse = {
|
|
4160
|
+
"results": Array<{
|
|
4161
|
+
"id": string;
|
|
4162
|
+
"harness": string;
|
|
4163
|
+
"model"?: string;
|
|
4164
|
+
"provider"?: string;
|
|
4165
|
+
"tools"?: "default" | "none" | {
|
|
4166
|
+
"tools": Array<string>;
|
|
4167
|
+
} | null;
|
|
4168
|
+
"intent_text"?: string;
|
|
4169
|
+
"target_domain"?: string;
|
|
4170
|
+
"status": string;
|
|
4171
|
+
"outcome"?: string;
|
|
4172
|
+
"cost_usd"?: number;
|
|
4173
|
+
"num_turns"?: number;
|
|
4174
|
+
"duration_ms"?: number;
|
|
4175
|
+
"total_tokens"?: number;
|
|
4176
|
+
"input_tokens"?: number;
|
|
4177
|
+
"cache_read_tokens"?: number;
|
|
4178
|
+
"cache_creation_tokens"?: number;
|
|
4179
|
+
"output_tokens"?: number;
|
|
4180
|
+
"created_at": string;
|
|
4181
|
+
"finished_at"?: string;
|
|
4182
|
+
"insight"?: {
|
|
4183
|
+
"visibility_score": number;
|
|
4184
|
+
"intent_satisfied": boolean;
|
|
4185
|
+
};
|
|
4186
|
+
}>;
|
|
4187
|
+
"pageToken"?: string;
|
|
4188
|
+
};
|
|
4189
|
+
export type StudyRunsCreateRequest = {
|
|
4190
|
+
"product_key"?: string;
|
|
4191
|
+
"intent": string;
|
|
4192
|
+
"domain"?: string;
|
|
4193
|
+
"harness"?: string;
|
|
4194
|
+
"model"?: string;
|
|
4195
|
+
"mode"?: "task" | "session";
|
|
4196
|
+
"addons"?: Record<string, unknown>;
|
|
4197
|
+
"engine"?: string;
|
|
4198
|
+
"execution_kind"?: "harness" | "direct";
|
|
4199
|
+
"tools"?: "default" | "none" | {
|
|
4200
|
+
"tools": Array<string>;
|
|
4201
|
+
};
|
|
4202
|
+
"requires_tools"?: boolean;
|
|
4203
|
+
"mcp_servers"?: Array<{
|
|
4204
|
+
"name": string;
|
|
4205
|
+
"url": string;
|
|
4206
|
+
"secretId"?: string;
|
|
4207
|
+
}>;
|
|
4208
|
+
"model_params"?: {
|
|
4209
|
+
"temperature"?: number;
|
|
4210
|
+
"maxOutputTokens"?: number;
|
|
4211
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4212
|
+
"thinkingBudgetTokens"?: number;
|
|
4213
|
+
};
|
|
4214
|
+
"wait"?: boolean;
|
|
4215
|
+
};
|
|
4216
|
+
export type StudyRunsCreateResponse = {
|
|
4217
|
+
"runId": string;
|
|
4218
|
+
"status": "succeeded";
|
|
4219
|
+
"answer"?: string;
|
|
4220
|
+
"signals"?: unknown;
|
|
4221
|
+
} | {
|
|
4222
|
+
"runId": string;
|
|
4223
|
+
"status": "error" | "pending" | "queued";
|
|
4224
|
+
"streamUrl": string;
|
|
4225
|
+
"harness"?: string;
|
|
4226
|
+
"model"?: string;
|
|
4227
|
+
"provider"?: string;
|
|
4228
|
+
};
|
|
4229
|
+
export type StudyRunsCancelBatchRequest = {
|
|
4230
|
+
"runIds": Array<string>;
|
|
4231
|
+
};
|
|
4232
|
+
export type StudyRunsCancelBatchResponse = {
|
|
4233
|
+
"requested": number;
|
|
4234
|
+
"canceled": Array<string>;
|
|
4235
|
+
"alreadyTerminal": Array<string>;
|
|
4236
|
+
"skipped": Array<string>;
|
|
4237
|
+
};
|
|
4238
|
+
export type StudyRunsFeedResponse = unknown;
|
|
4239
|
+
export type StudyRunsCancelResponse = {
|
|
4240
|
+
"runId": string;
|
|
4241
|
+
"status": string;
|
|
4242
|
+
"alreadyTerminal": boolean;
|
|
4243
|
+
};
|
|
4244
|
+
export type StudyRunsDirectSignalsResponse = {
|
|
4245
|
+
"brands": Array<{
|
|
4246
|
+
"name": string;
|
|
4247
|
+
"domain"?: string;
|
|
4248
|
+
"status": "recommended" | "mentioned";
|
|
4249
|
+
"position"?: number;
|
|
4250
|
+
"sentiment"?: "positive" | "neutral" | "negative";
|
|
4251
|
+
"reason"?: string;
|
|
4252
|
+
"caveat"?: string;
|
|
4253
|
+
"backed_by"?: Array<string>;
|
|
4254
|
+
}>;
|
|
4255
|
+
"cited_domains": Array<{
|
|
4256
|
+
"host": string;
|
|
4257
|
+
"url"?: string;
|
|
4258
|
+
"rank"?: number;
|
|
4259
|
+
}>;
|
|
4260
|
+
"made_recommendations": boolean;
|
|
4261
|
+
"usage"?: {
|
|
4262
|
+
"input_tokens": number;
|
|
4263
|
+
"output_tokens": number;
|
|
4264
|
+
};
|
|
4265
|
+
"cost_usd"?: number;
|
|
4266
|
+
"execution_mode"?: "live" | "closed";
|
|
4267
|
+
"inconclusive": boolean;
|
|
4268
|
+
};
|
|
4269
|
+
export type StudyRunsInsightResponse = unknown;
|
|
4270
|
+
export type StudyRunsRawTraceResponse = unknown;
|
|
4271
|
+
export type StudyRunsSpansResponse = unknown;
|
|
4272
|
+
export type StudyRunsStreamResponse = unknown;
|
|
4273
|
+
export type StudyRunsTrajectoryResponse = unknown;
|
|
4274
|
+
export type StudySkillsListResponse = {
|
|
4275
|
+
"skills": Array<{
|
|
4276
|
+
"ref": string;
|
|
4277
|
+
"source": "builtin" | "custom" | "git";
|
|
4278
|
+
"id": string;
|
|
4279
|
+
"name": string;
|
|
4280
|
+
"slug": string;
|
|
4281
|
+
"isExample"?: boolean;
|
|
4282
|
+
"description"?: string;
|
|
4283
|
+
"visibility"?: "user" | "org";
|
|
4284
|
+
"authorUserId"?: string;
|
|
4285
|
+
"body"?: string;
|
|
4286
|
+
"repoUrl"?: string;
|
|
4287
|
+
"gitRef"?: string;
|
|
4288
|
+
"subpath"?: string;
|
|
4289
|
+
"updatedAt"?: string;
|
|
4290
|
+
}>;
|
|
4291
|
+
};
|
|
4292
|
+
export type StudySkillsCreateRequest = {
|
|
4293
|
+
"source": "custom";
|
|
4294
|
+
"name": string;
|
|
4295
|
+
"body": string;
|
|
4296
|
+
"visibility"?: "user" | "org";
|
|
4297
|
+
} | {
|
|
4298
|
+
"source": "git";
|
|
4299
|
+
"name": string;
|
|
4300
|
+
"repoUrl": string;
|
|
4301
|
+
"gitRef": string;
|
|
4302
|
+
"subpath"?: string | null;
|
|
4303
|
+
"visibility"?: "user" | "org";
|
|
4304
|
+
};
|
|
4305
|
+
export type StudySkillsCreateResponse = {
|
|
4306
|
+
"skill": {
|
|
4307
|
+
"ref": string;
|
|
4308
|
+
"source": "builtin" | "custom" | "git";
|
|
4309
|
+
"id": string;
|
|
4310
|
+
"name": string;
|
|
4311
|
+
"slug": string;
|
|
4312
|
+
"isExample"?: boolean;
|
|
4313
|
+
"description"?: string;
|
|
4314
|
+
"visibility"?: "user" | "org";
|
|
4315
|
+
"authorUserId"?: string;
|
|
4316
|
+
"body"?: string;
|
|
4317
|
+
"repoUrl"?: string;
|
|
4318
|
+
"gitRef"?: string;
|
|
4319
|
+
"subpath"?: string;
|
|
4320
|
+
"updatedAt"?: string;
|
|
4321
|
+
};
|
|
4322
|
+
};
|
|
4323
|
+
export type StudySkillsUpdateRequest = {
|
|
4324
|
+
"name"?: string;
|
|
4325
|
+
"visibility"?: "user" | "org";
|
|
4326
|
+
"body"?: string;
|
|
4327
|
+
"repoUrl"?: string;
|
|
4328
|
+
"gitRef"?: string;
|
|
4329
|
+
"subpath"?: string | null;
|
|
4330
|
+
};
|
|
4331
|
+
export type StudySkillsUpdateResponse = {
|
|
4332
|
+
"skill": {
|
|
4333
|
+
"ref": string;
|
|
4334
|
+
"source": "builtin" | "custom" | "git";
|
|
4335
|
+
"id": string;
|
|
4336
|
+
"name": string;
|
|
4337
|
+
"slug": string;
|
|
4338
|
+
"isExample"?: boolean;
|
|
4339
|
+
"description"?: string;
|
|
4340
|
+
"visibility"?: "user" | "org";
|
|
4341
|
+
"authorUserId"?: string;
|
|
4342
|
+
"body"?: string;
|
|
4343
|
+
"repoUrl"?: string;
|
|
4344
|
+
"gitRef"?: string;
|
|
4345
|
+
"subpath"?: string;
|
|
4346
|
+
"updatedAt"?: string;
|
|
4347
|
+
};
|
|
4348
|
+
};
|
|
4349
|
+
export type StudySkillsDeleteResponse = unknown;
|
|
4350
|
+
export type StudyListResponse = {
|
|
4351
|
+
"studies": Array<{
|
|
4352
|
+
"id": string;
|
|
4353
|
+
"executiveSummary"?: string | null;
|
|
4354
|
+
"plannedRunCount": number;
|
|
4355
|
+
"organizationId": string;
|
|
4356
|
+
"createdBy": string | null;
|
|
4357
|
+
"name": string;
|
|
4358
|
+
"slug": string;
|
|
4359
|
+
"description": string | null;
|
|
4360
|
+
"skillRef": string | null;
|
|
4361
|
+
"skillSlug": string | null;
|
|
4362
|
+
"skillName": string | null;
|
|
4363
|
+
"skillPromise": string;
|
|
4364
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
4365
|
+
"subjectName": string | null;
|
|
4366
|
+
"subjectLabel": string;
|
|
4367
|
+
"armConfig": {
|
|
4368
|
+
"baseline"?: {
|
|
4369
|
+
"tools"?: "default" | "none" | {
|
|
4370
|
+
"tools": Array<string>;
|
|
4371
|
+
};
|
|
4372
|
+
"addons"?: Record<string, unknown>;
|
|
4373
|
+
};
|
|
4374
|
+
"treatment"?: {
|
|
4375
|
+
"tools"?: "default" | "none" | {
|
|
4376
|
+
"tools": Array<string>;
|
|
4377
|
+
};
|
|
4378
|
+
"addons"?: Record<string, unknown>;
|
|
4379
|
+
};
|
|
4380
|
+
};
|
|
4381
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
4382
|
+
"subjectBackend": "model" | "harness" | null;
|
|
4383
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4384
|
+
"importedFrom"?: string;
|
|
4385
|
+
"baselineLabel": string | null;
|
|
4386
|
+
"treatmentLabel": string | null;
|
|
4387
|
+
"promiseMetrics": Array<{
|
|
4388
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
4389
|
+
"direction": "down" | "up";
|
|
4390
|
+
"label": string;
|
|
4391
|
+
}>;
|
|
4392
|
+
"matrix": Array<{
|
|
4393
|
+
"harness": string;
|
|
4394
|
+
"model": string;
|
|
4395
|
+
"provider"?: string;
|
|
4396
|
+
"params"?: {
|
|
4397
|
+
"temperature"?: number;
|
|
4398
|
+
"maxOutputTokens"?: number;
|
|
4399
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4400
|
+
"thinkingBudgetTokens"?: number;
|
|
4401
|
+
};
|
|
4402
|
+
}>;
|
|
4403
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4404
|
+
"taskCount": number;
|
|
4405
|
+
"variantsPerTask": number;
|
|
4406
|
+
"repeats": number;
|
|
4407
|
+
"subset": {
|
|
4408
|
+
"count": number;
|
|
4409
|
+
"seed": string;
|
|
4410
|
+
"of": number;
|
|
4411
|
+
} | null;
|
|
4412
|
+
"status": string;
|
|
4413
|
+
"error": string | null;
|
|
4414
|
+
"preparation"?: {
|
|
4415
|
+
"since": string;
|
|
4416
|
+
"launchedBy": {
|
|
4417
|
+
"userId"?: string;
|
|
4418
|
+
"role"?: string;
|
|
4419
|
+
};
|
|
4420
|
+
"waitingOn": Array<{
|
|
4421
|
+
"imageKey": string;
|
|
4422
|
+
"buildId": string;
|
|
4423
|
+
"stage"?: "image" | "snapshot";
|
|
4424
|
+
}>;
|
|
4425
|
+
} | null;
|
|
4426
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
4427
|
+
"visibilityChangedAt"?: string | null;
|
|
4428
|
+
"visibilityChangedBy"?: string | null;
|
|
4429
|
+
"shareToken"?: string | null;
|
|
4430
|
+
"lowSampleOverride": boolean;
|
|
4431
|
+
"publishedAt": string | null;
|
|
4432
|
+
"verdict": unknown;
|
|
4433
|
+
"definitionVersionId": string | null;
|
|
4434
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
4435
|
+
"createdAt": string;
|
|
4436
|
+
"updatedAt": string;
|
|
4437
|
+
"liveUntil": string | null;
|
|
4438
|
+
}>;
|
|
4439
|
+
};
|
|
4440
|
+
export type StudyCreateRequest = {
|
|
4441
|
+
"name": string;
|
|
4442
|
+
"description"?: string;
|
|
4443
|
+
"subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
4444
|
+
"skillRef"?: string;
|
|
4445
|
+
"definitionVersionId"?: string;
|
|
4446
|
+
"subjectName"?: string;
|
|
4447
|
+
"armConfig"?: {
|
|
4448
|
+
"baseline"?: {
|
|
4449
|
+
"tools"?: "default" | "none" | {
|
|
4450
|
+
"tools": Array<string>;
|
|
4451
|
+
};
|
|
4452
|
+
"addons"?: Record<string, unknown>;
|
|
4453
|
+
};
|
|
4454
|
+
"treatment"?: {
|
|
4455
|
+
"tools"?: "default" | "none" | {
|
|
4456
|
+
"tools": Array<string>;
|
|
4457
|
+
};
|
|
4458
|
+
"addons"?: Record<string, unknown>;
|
|
4459
|
+
};
|
|
4460
|
+
};
|
|
4461
|
+
"lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4462
|
+
"proposeExpectations"?: boolean;
|
|
4463
|
+
"comparisonMode"?: "ab_vs_none" | "ab_custom" | "single";
|
|
4464
|
+
"subjectBackend"?: "model" | "harness";
|
|
4465
|
+
"baselineLabel"?: string;
|
|
4466
|
+
"treatmentLabel"?: string;
|
|
4467
|
+
"taskCount"?: number;
|
|
4468
|
+
"variantsPerTask"?: number;
|
|
4469
|
+
"repeats"?: number;
|
|
4470
|
+
"tasks"?: Array<{
|
|
4471
|
+
"prompt": string;
|
|
4472
|
+
"rationale"?: string;
|
|
4473
|
+
"category"?: string;
|
|
4474
|
+
"requiresWeb"?: boolean;
|
|
4475
|
+
"rubric"?: Array<string>;
|
|
4476
|
+
"variants"?: Array<string>;
|
|
4477
|
+
"expectations"?: Array<unknown>;
|
|
4478
|
+
}>;
|
|
4479
|
+
"subset"?: {
|
|
4480
|
+
"count": number;
|
|
4481
|
+
"seed"?: string;
|
|
4482
|
+
};
|
|
4483
|
+
"matrix": Array<{
|
|
4484
|
+
"harness": string;
|
|
4485
|
+
"model": string;
|
|
4486
|
+
"provider"?: string;
|
|
4487
|
+
"params"?: {
|
|
4488
|
+
"temperature"?: number;
|
|
4489
|
+
"maxOutputTokens"?: number;
|
|
4490
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4491
|
+
"thinkingBudgetTokens"?: number;
|
|
4492
|
+
};
|
|
4493
|
+
}>;
|
|
4494
|
+
"promise"?: string;
|
|
4495
|
+
};
|
|
4496
|
+
export type StudyCreateResponse = {
|
|
4497
|
+
"study": {
|
|
4498
|
+
"id": string;
|
|
4499
|
+
"executiveSummary"?: string | null;
|
|
4500
|
+
"plannedRunCount": number;
|
|
4501
|
+
"organizationId": string;
|
|
4502
|
+
"createdBy": string | null;
|
|
4503
|
+
"name": string;
|
|
4504
|
+
"slug": string;
|
|
4505
|
+
"description": string | null;
|
|
4506
|
+
"skillRef": string | null;
|
|
4507
|
+
"skillSlug": string | null;
|
|
4508
|
+
"skillName": string | null;
|
|
4509
|
+
"skillPromise": string;
|
|
4510
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
4511
|
+
"subjectName": string | null;
|
|
4512
|
+
"subjectLabel": string;
|
|
4513
|
+
"armConfig": {
|
|
4514
|
+
"baseline"?: {
|
|
4515
|
+
"tools"?: "default" | "none" | {
|
|
4516
|
+
"tools": Array<string>;
|
|
4517
|
+
};
|
|
4518
|
+
"addons"?: Record<string, unknown>;
|
|
4519
|
+
};
|
|
4520
|
+
"treatment"?: {
|
|
4521
|
+
"tools"?: "default" | "none" | {
|
|
4522
|
+
"tools": Array<string>;
|
|
4523
|
+
};
|
|
4524
|
+
"addons"?: Record<string, unknown>;
|
|
4525
|
+
};
|
|
4526
|
+
};
|
|
4527
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
4528
|
+
"subjectBackend": "model" | "harness" | null;
|
|
4529
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4530
|
+
"importedFrom"?: string;
|
|
4531
|
+
"baselineLabel": string | null;
|
|
4532
|
+
"treatmentLabel": string | null;
|
|
4533
|
+
"promiseMetrics": Array<{
|
|
4534
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
4535
|
+
"direction": "down" | "up";
|
|
4536
|
+
"label": string;
|
|
4537
|
+
}>;
|
|
4538
|
+
"matrix": Array<{
|
|
4539
|
+
"harness": string;
|
|
4540
|
+
"model": string;
|
|
4541
|
+
"provider"?: string;
|
|
4542
|
+
"params"?: {
|
|
4543
|
+
"temperature"?: number;
|
|
4544
|
+
"maxOutputTokens"?: number;
|
|
4545
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4546
|
+
"thinkingBudgetTokens"?: number;
|
|
4547
|
+
};
|
|
4548
|
+
}>;
|
|
4549
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4550
|
+
"taskCount": number;
|
|
4551
|
+
"variantsPerTask": number;
|
|
4552
|
+
"repeats": number;
|
|
4553
|
+
"subset": {
|
|
4554
|
+
"count": number;
|
|
4555
|
+
"seed": string;
|
|
4556
|
+
"of": number;
|
|
4557
|
+
} | null;
|
|
4558
|
+
"status": string;
|
|
4559
|
+
"error": string | null;
|
|
4560
|
+
"preparation"?: {
|
|
4561
|
+
"since": string;
|
|
4562
|
+
"launchedBy": {
|
|
4563
|
+
"userId"?: string;
|
|
4564
|
+
"role"?: string;
|
|
4565
|
+
};
|
|
4566
|
+
"waitingOn": Array<{
|
|
4567
|
+
"imageKey": string;
|
|
4568
|
+
"buildId": string;
|
|
4569
|
+
"stage"?: "image" | "snapshot";
|
|
4570
|
+
}>;
|
|
4571
|
+
} | null;
|
|
4572
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
4573
|
+
"visibilityChangedAt"?: string | null;
|
|
4574
|
+
"visibilityChangedBy"?: string | null;
|
|
4575
|
+
"shareToken"?: string | null;
|
|
4576
|
+
"lowSampleOverride": boolean;
|
|
4577
|
+
"publishedAt": string | null;
|
|
4578
|
+
"verdict": unknown;
|
|
4579
|
+
"definitionVersionId": string | null;
|
|
4580
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
4581
|
+
"createdAt": string;
|
|
4582
|
+
"updatedAt": string;
|
|
4583
|
+
"liveUntil": string | null;
|
|
4584
|
+
};
|
|
4585
|
+
"streamUrl": string;
|
|
4586
|
+
"warnings"?: Array<{
|
|
4587
|
+
"harness": string;
|
|
4588
|
+
"kind": string;
|
|
4589
|
+
}>;
|
|
4590
|
+
};
|
|
4591
|
+
export type StudyConvertWebmcpSuiteRequest = {
|
|
4592
|
+
"suite": unknown;
|
|
4593
|
+
"ordered"?: boolean;
|
|
4594
|
+
};
|
|
4595
|
+
export type StudyConvertWebmcpSuiteResponse = {
|
|
4596
|
+
"tasks": Array<{
|
|
4597
|
+
"title": string;
|
|
4598
|
+
"prompt": string;
|
|
4599
|
+
"expectations": Array<unknown>;
|
|
4600
|
+
}>;
|
|
4601
|
+
"warnings": Array<string>;
|
|
4602
|
+
};
|
|
4603
|
+
export type StudyGetResponse = {
|
|
4604
|
+
"study": {
|
|
4605
|
+
"id": string;
|
|
4606
|
+
"executiveSummary"?: string | null;
|
|
4607
|
+
"plannedRunCount": number;
|
|
4608
|
+
"organizationId": string;
|
|
4609
|
+
"createdBy": string | null;
|
|
4610
|
+
"name": string;
|
|
4611
|
+
"slug": string;
|
|
4612
|
+
"description": string | null;
|
|
4613
|
+
"skillRef": string | null;
|
|
4614
|
+
"skillSlug": string | null;
|
|
4615
|
+
"skillName": string | null;
|
|
4616
|
+
"skillPromise": string;
|
|
4617
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
4618
|
+
"subjectName": string | null;
|
|
4619
|
+
"subjectLabel": string;
|
|
4620
|
+
"armConfig": {
|
|
4621
|
+
"baseline"?: {
|
|
4622
|
+
"tools"?: "default" | "none" | {
|
|
4623
|
+
"tools": Array<string>;
|
|
4624
|
+
};
|
|
4625
|
+
"addons"?: Record<string, unknown>;
|
|
4626
|
+
};
|
|
4627
|
+
"treatment"?: {
|
|
4628
|
+
"tools"?: "default" | "none" | {
|
|
4629
|
+
"tools": Array<string>;
|
|
4630
|
+
};
|
|
4631
|
+
"addons"?: Record<string, unknown>;
|
|
4632
|
+
};
|
|
4633
|
+
};
|
|
4634
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
4635
|
+
"subjectBackend": "model" | "harness" | null;
|
|
4636
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
4637
|
+
"importedFrom"?: string;
|
|
4638
|
+
"baselineLabel": string | null;
|
|
4639
|
+
"treatmentLabel": string | null;
|
|
4640
|
+
"promiseMetrics": Array<{
|
|
4641
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
4642
|
+
"direction": "down" | "up";
|
|
4643
|
+
"label": string;
|
|
4644
|
+
}>;
|
|
4645
|
+
"matrix": Array<{
|
|
4646
|
+
"harness": string;
|
|
4647
|
+
"model": string;
|
|
4648
|
+
"provider"?: string;
|
|
4649
|
+
"params"?: {
|
|
4650
|
+
"temperature"?: number;
|
|
4651
|
+
"maxOutputTokens"?: number;
|
|
4652
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
4653
|
+
"thinkingBudgetTokens"?: number;
|
|
4654
|
+
};
|
|
4655
|
+
}>;
|
|
4656
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
4657
|
+
"taskCount": number;
|
|
4658
|
+
"variantsPerTask": number;
|
|
4659
|
+
"repeats": number;
|
|
4660
|
+
"subset": {
|
|
4661
|
+
"count": number;
|
|
4662
|
+
"seed": string;
|
|
4663
|
+
"of": number;
|
|
4664
|
+
} | null;
|
|
4665
|
+
"status": string;
|
|
4666
|
+
"error": string | null;
|
|
4667
|
+
"preparation"?: {
|
|
4668
|
+
"since": string;
|
|
4669
|
+
"launchedBy": {
|
|
4670
|
+
"userId"?: string;
|
|
4671
|
+
"role"?: string;
|
|
4672
|
+
};
|
|
4673
|
+
"waitingOn": Array<{
|
|
4674
|
+
"imageKey": string;
|
|
4675
|
+
"buildId": string;
|
|
4676
|
+
"stage"?: "image" | "snapshot";
|
|
4677
|
+
}>;
|
|
4678
|
+
} | null;
|
|
4679
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
4680
|
+
"visibilityChangedAt"?: string | null;
|
|
4681
|
+
"visibilityChangedBy"?: string | null;
|
|
4682
|
+
"shareToken"?: string | null;
|
|
4683
|
+
"lowSampleOverride": boolean;
|
|
4684
|
+
"publishedAt": string | null;
|
|
4685
|
+
"verdict": unknown;
|
|
4686
|
+
"definitionVersionId": string | null;
|
|
4687
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
4688
|
+
"createdAt": string;
|
|
4689
|
+
"updatedAt": string;
|
|
4690
|
+
"liveUntil": string | null;
|
|
4691
|
+
};
|
|
4692
|
+
"tasks": Array<{
|
|
4693
|
+
"id": string;
|
|
4694
|
+
"studyId": string;
|
|
4695
|
+
"position": number;
|
|
4696
|
+
"prompt": string;
|
|
4697
|
+
"rationale": string | null;
|
|
4698
|
+
"category": string | null;
|
|
4699
|
+
"requiresWeb": boolean;
|
|
4700
|
+
"rubric": Array<string>;
|
|
4701
|
+
"variants"?: Array<string> | null;
|
|
4702
|
+
"expectations"?: Array<unknown> | null;
|
|
4703
|
+
}>;
|
|
4704
|
+
"runs": Array<{
|
|
4705
|
+
"runId": string;
|
|
4706
|
+
"taskId": string;
|
|
4707
|
+
"arm": "baseline" | "skill";
|
|
4708
|
+
"harness": string;
|
|
4709
|
+
"model": string;
|
|
4710
|
+
"provider": string | null;
|
|
4711
|
+
"variantIndex"?: number;
|
|
4712
|
+
"repeatIndex"?: number;
|
|
4713
|
+
"status": string;
|
|
4714
|
+
"outcome": string | null;
|
|
4715
|
+
"outcomeReason": string | null;
|
|
4716
|
+
"totalTokens": number | null;
|
|
4717
|
+
"inputTokens": number | null;
|
|
4718
|
+
"outputTokens": number | null;
|
|
4719
|
+
"cacheReadTokens": number | null;
|
|
4720
|
+
"cacheCreationTokens": number | null;
|
|
4721
|
+
"costUsd": number | null;
|
|
4722
|
+
"numTurns": number | null;
|
|
4723
|
+
"durationMs": number | null;
|
|
4724
|
+
"intentSatisfied": boolean | null;
|
|
4725
|
+
"frictionCount": number | null;
|
|
4726
|
+
"insightSummary": string | null;
|
|
4727
|
+
"qualityScore": number | null;
|
|
4728
|
+
"rubricResults": Array<{
|
|
4729
|
+
"criterion": string;
|
|
4730
|
+
"passed": boolean;
|
|
4731
|
+
"note"?: string;
|
|
4732
|
+
"votes"?: {
|
|
4733
|
+
"passed": number;
|
|
4734
|
+
"judges": number;
|
|
4735
|
+
"byJudge"?: Array<{
|
|
4736
|
+
"judge": string;
|
|
4737
|
+
"passed": boolean;
|
|
4738
|
+
"note"?: string;
|
|
4739
|
+
}>;
|
|
4740
|
+
};
|
|
4741
|
+
}> | null;
|
|
4742
|
+
"gradedAt": string | null;
|
|
4743
|
+
"judgeModel": string | null;
|
|
4744
|
+
"pairVerdict": "skill" | "baseline" | "tie" | "unmeasurable" | null;
|
|
4745
|
+
"pairVotes": Array<{
|
|
4746
|
+
"judge": string;
|
|
4747
|
+
"ordering": "base-first" | "skill-first";
|
|
4748
|
+
"answer": "A" | "B" | "tie";
|
|
4749
|
+
}> | null;
|
|
4750
|
+
"pairGradedAt": string | null;
|
|
4751
|
+
"evalResults": unknown | null;
|
|
4752
|
+
"evalScore": number | null;
|
|
4753
|
+
"evalGradedAt": string | null;
|
|
4754
|
+
"lensGrades"?: {
|
|
4755
|
+
"reference"?: {
|
|
4756
|
+
"pass": boolean | null;
|
|
4757
|
+
"extracted": string | null;
|
|
4758
|
+
"extraction": string;
|
|
4759
|
+
"matcher": string;
|
|
4760
|
+
"votes"?: Array<{
|
|
4761
|
+
"judge": string;
|
|
4762
|
+
"answer": "equivalent" | "different" | "not_attempted" | null;
|
|
4763
|
+
}>;
|
|
4764
|
+
"reason"?: string;
|
|
4765
|
+
"gradedAt": string;
|
|
4766
|
+
};
|
|
4767
|
+
"framework"?: {
|
|
4768
|
+
"framework": string;
|
|
4769
|
+
"primary": {
|
|
4770
|
+
"metricId": string;
|
|
4771
|
+
"provenance": string;
|
|
4772
|
+
"value": number;
|
|
4773
|
+
"scenario": string;
|
|
4774
|
+
"count": number;
|
|
4775
|
+
} | null;
|
|
4776
|
+
"secondary": Array<{
|
|
4777
|
+
"label": string;
|
|
4778
|
+
"name": string;
|
|
4779
|
+
"scenario": string;
|
|
4780
|
+
"value": number | null;
|
|
4781
|
+
"count": number;
|
|
4782
|
+
"split": string | null;
|
|
4783
|
+
}>;
|
|
4784
|
+
"errors": Array<{
|
|
4785
|
+
"scenario": string;
|
|
4786
|
+
"metric": string;
|
|
4787
|
+
"reason": string;
|
|
4788
|
+
}>;
|
|
4789
|
+
"gradedAt": string;
|
|
4790
|
+
};
|
|
4791
|
+
"execution"?: {
|
|
4792
|
+
"pass": boolean | null;
|
|
4793
|
+
"status": "passed" | "failed" | "timeout" | "error" | "unavailable";
|
|
4794
|
+
"language": string;
|
|
4795
|
+
"passedTests": number | null;
|
|
4796
|
+
"failedTests": number | null;
|
|
4797
|
+
"stdout": string;
|
|
4798
|
+
"stdoutTruncated": boolean;
|
|
4799
|
+
"reason"?: string;
|
|
4800
|
+
"gradedAt": string;
|
|
4801
|
+
};
|
|
4802
|
+
} | null;
|
|
4803
|
+
"runCreatedAt": string | null;
|
|
4804
|
+
"startedAt": string | null;
|
|
4805
|
+
"finishedAt": string | null;
|
|
4806
|
+
}>;
|
|
4807
|
+
"aggregate": {
|
|
4808
|
+
"summary": {
|
|
4809
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
4810
|
+
"headline": Array<{
|
|
4811
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
4812
|
+
"label": string;
|
|
4813
|
+
"direction": "down" | "up";
|
|
4814
|
+
"baselineAvg": number | null;
|
|
4815
|
+
"skillAvg": number | null;
|
|
4816
|
+
"deltaPct": number | null;
|
|
4817
|
+
"delta"?: {
|
|
4818
|
+
"unit": "pp";
|
|
4819
|
+
"absolute": number | null;
|
|
4820
|
+
"relativePct": number | null;
|
|
4821
|
+
} | {
|
|
4822
|
+
"unit": "pts";
|
|
4823
|
+
"absolute": number | null;
|
|
4824
|
+
"relativePct"?: number | null;
|
|
4825
|
+
} | {
|
|
4826
|
+
"unit": "%";
|
|
4827
|
+
"relativePct": number | null;
|
|
4828
|
+
};
|
|
4829
|
+
"improved": boolean;
|
|
4830
|
+
}>;
|
|
4831
|
+
"completionRate": {
|
|
4832
|
+
"baseline": number | null;
|
|
4833
|
+
"skill": number | null;
|
|
4834
|
+
};
|
|
4835
|
+
"goalRate": {
|
|
4836
|
+
"baseline": number | null;
|
|
4837
|
+
"skill": number | null;
|
|
4838
|
+
};
|
|
4839
|
+
"pairwise": {
|
|
4840
|
+
"pairs": number;
|
|
4841
|
+
"skillWins": number;
|
|
4842
|
+
"baselineWins": number;
|
|
4843
|
+
"ties": number;
|
|
4844
|
+
"winRate": number | null;
|
|
4845
|
+
};
|
|
4846
|
+
"decision"?: {
|
|
4847
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
4848
|
+
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
4849
|
+
"reason": string;
|
|
4850
|
+
"efficiencyDiscounted": boolean;
|
|
4851
|
+
};
|
|
4852
|
+
"leaderboard"?: Array<unknown>;
|
|
4853
|
+
"concentration": {
|
|
4854
|
+
"perCell": Array<{
|
|
4855
|
+
"cell": string;
|
|
4856
|
+
"goalDeltaPp": number | null;
|
|
4857
|
+
"contributionPct": number | null;
|
|
4858
|
+
}>;
|
|
4859
|
+
"heterogeneity": {
|
|
4860
|
+
"dominantCell": string;
|
|
4861
|
+
"contributionPct": number;
|
|
4862
|
+
} | null;
|
|
4863
|
+
};
|
|
4864
|
+
"matrixShape": {
|
|
4865
|
+
"cells": number;
|
|
4866
|
+
"harnesses": number;
|
|
4867
|
+
"models": number;
|
|
4868
|
+
"crossed": boolean;
|
|
4869
|
+
};
|
|
4870
|
+
"flakiness"?: {
|
|
4871
|
+
"measured": boolean;
|
|
4872
|
+
"conditions": number;
|
|
4873
|
+
"unstable": number;
|
|
4874
|
+
"worst": {
|
|
4875
|
+
"cell": string;
|
|
4876
|
+
"taskId": string;
|
|
4877
|
+
"arm": "baseline" | "skill";
|
|
4878
|
+
"variant"?: number;
|
|
4879
|
+
"passRate": number;
|
|
4880
|
+
} | null;
|
|
4881
|
+
};
|
|
4882
|
+
"arms"?: {
|
|
4883
|
+
"baseline": {
|
|
4884
|
+
"runs": number;
|
|
4885
|
+
"completed": number;
|
|
4886
|
+
"infraFailures": number;
|
|
4887
|
+
"completionRate": number | null;
|
|
4888
|
+
"goalRate": number | null;
|
|
4889
|
+
"goalTally": {
|
|
4890
|
+
"achieved": number;
|
|
4891
|
+
"partial": number;
|
|
4892
|
+
"missed": number;
|
|
4893
|
+
};
|
|
4894
|
+
"avgTotalTokens": number | null;
|
|
4895
|
+
"avgOutputTokens": number | null;
|
|
4896
|
+
"avgCostUsd": number | null;
|
|
4897
|
+
"avgDurationMs": number | null;
|
|
4898
|
+
"avgTurns": number | null;
|
|
4899
|
+
"costCoverage": number | null;
|
|
4900
|
+
"avgQualityScore": number | null;
|
|
4901
|
+
"gradedCoverage": number | null;
|
|
4902
|
+
"medianCostUsd": number | null;
|
|
4903
|
+
"medianTotalTokens": number | null;
|
|
4904
|
+
"medianDurationMs": number | null;
|
|
4905
|
+
"p95CostUsd": number | null;
|
|
4906
|
+
"p95TotalTokens": number | null;
|
|
4907
|
+
"p95DurationMs": number | null;
|
|
4908
|
+
"maxCostUsd": number | null;
|
|
4909
|
+
"maxTotalTokens": number | null;
|
|
4910
|
+
"maxDurationMs": number | null;
|
|
4911
|
+
"evalPassRate"?: number | null;
|
|
4912
|
+
"adoptionRate"?: number | null;
|
|
4913
|
+
};
|
|
4914
|
+
"skill": {
|
|
4915
|
+
"runs": number;
|
|
4916
|
+
"completed": number;
|
|
4917
|
+
"infraFailures": number;
|
|
4918
|
+
"completionRate": number | null;
|
|
4919
|
+
"goalRate": number | null;
|
|
4920
|
+
"goalTally": {
|
|
4921
|
+
"achieved": number;
|
|
4922
|
+
"partial": number;
|
|
4923
|
+
"missed": number;
|
|
4924
|
+
};
|
|
4925
|
+
"avgTotalTokens": number | null;
|
|
4926
|
+
"avgOutputTokens": number | null;
|
|
4927
|
+
"avgCostUsd": number | null;
|
|
4928
|
+
"avgDurationMs": number | null;
|
|
4929
|
+
"avgTurns": number | null;
|
|
4930
|
+
"costCoverage": number | null;
|
|
4931
|
+
"avgQualityScore": number | null;
|
|
4932
|
+
"gradedCoverage": number | null;
|
|
4933
|
+
"medianCostUsd": number | null;
|
|
4934
|
+
"medianTotalTokens": number | null;
|
|
4935
|
+
"medianDurationMs": number | null;
|
|
4936
|
+
"p95CostUsd": number | null;
|
|
4937
|
+
"p95TotalTokens": number | null;
|
|
4938
|
+
"p95DurationMs": number | null;
|
|
4939
|
+
"maxCostUsd": number | null;
|
|
4940
|
+
"maxTotalTokens": number | null;
|
|
4941
|
+
"maxDurationMs": number | null;
|
|
4942
|
+
"evalPassRate"?: number | null;
|
|
4943
|
+
"adoptionRate"?: number | null;
|
|
4944
|
+
};
|
|
4945
|
+
};
|
|
4946
|
+
"totals": {
|
|
4947
|
+
"cells": number;
|
|
4948
|
+
"runs": number;
|
|
4949
|
+
"completed": number;
|
|
4950
|
+
"failed": number;
|
|
4951
|
+
"costUsd": number;
|
|
4952
|
+
"tokens": number;
|
|
4953
|
+
};
|
|
4954
|
+
};
|
|
4955
|
+
"perCell": Array<unknown>;
|
|
4956
|
+
"perHarness": Array<unknown>;
|
|
4957
|
+
"perModel": Array<unknown>;
|
|
4958
|
+
"perTask": Array<unknown>;
|
|
4959
|
+
"webSplit": unknown;
|
|
4960
|
+
"inference": {
|
|
4961
|
+
"columns": Array<{
|
|
4962
|
+
"key": string;
|
|
4963
|
+
"label": string;
|
|
4964
|
+
"family": "rate" | "score" | "magnitude";
|
|
4965
|
+
"betterWhen": "higher" | "lower";
|
|
4966
|
+
}>;
|
|
4967
|
+
"groups": Array<{
|
|
4968
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
4969
|
+
"title": string;
|
|
4970
|
+
"rows": Array<{
|
|
4971
|
+
"key": string;
|
|
4972
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
4973
|
+
"harness": string | null;
|
|
4974
|
+
"model": string | null;
|
|
4975
|
+
"paramsKey"?: string;
|
|
4976
|
+
"runs": number;
|
|
4977
|
+
"cells": Record<string, unknown>;
|
|
4978
|
+
}>;
|
|
4979
|
+
}>;
|
|
4980
|
+
"reference": "baseline" | "field";
|
|
4981
|
+
"draws": number;
|
|
4982
|
+
} | null;
|
|
4983
|
+
"behavior"?: Array<{
|
|
4984
|
+
"harness": string;
|
|
4985
|
+
"model": string;
|
|
4986
|
+
"provider": string | null;
|
|
4987
|
+
"arm": string;
|
|
4988
|
+
"totalRuns": number;
|
|
4989
|
+
"signals": Array<{
|
|
4990
|
+
"id": string;
|
|
4991
|
+
"label": string;
|
|
4992
|
+
"tone": "good" | "bad" | "neutral";
|
|
4993
|
+
"runCount": number;
|
|
4994
|
+
"succeededCount": number;
|
|
4995
|
+
}>;
|
|
4996
|
+
}>;
|
|
4997
|
+
"reference"?: {
|
|
4998
|
+
"cells": Array<{
|
|
4999
|
+
"cell": string;
|
|
5000
|
+
"harness": string;
|
|
5001
|
+
"model": string;
|
|
5002
|
+
"provider": string | null;
|
|
5003
|
+
"paramsKey": string;
|
|
5004
|
+
"label": string;
|
|
5005
|
+
"passRate": {
|
|
5006
|
+
"value": number | null;
|
|
5007
|
+
"ci": Array<unknown> | null;
|
|
5008
|
+
"n": number;
|
|
5009
|
+
};
|
|
5010
|
+
"tasks": number;
|
|
5011
|
+
"graded": number;
|
|
5012
|
+
"unmeasured": number;
|
|
5013
|
+
"matchers": Array<string>;
|
|
5014
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
5015
|
+
}>;
|
|
5016
|
+
"overall": {
|
|
5017
|
+
"value": number | null;
|
|
5018
|
+
"ci": Array<unknown> | null;
|
|
5019
|
+
"n": number;
|
|
5020
|
+
};
|
|
5021
|
+
"graded": number;
|
|
5022
|
+
"unmeasured": number;
|
|
5023
|
+
"modelJudged": boolean;
|
|
5024
|
+
};
|
|
5025
|
+
"execution"?: {
|
|
5026
|
+
"graded": number;
|
|
5027
|
+
"measured": number;
|
|
5028
|
+
"passed": number;
|
|
5029
|
+
"passRate": number | null;
|
|
5030
|
+
"byStatus": {
|
|
5031
|
+
"passed": number;
|
|
5032
|
+
"failed": number;
|
|
5033
|
+
"timeout": number;
|
|
5034
|
+
"error": number;
|
|
5035
|
+
"unavailable": number;
|
|
5036
|
+
};
|
|
5037
|
+
"languages": Array<string>;
|
|
5038
|
+
"unavailableReasons": Array<string>;
|
|
5039
|
+
};
|
|
5040
|
+
};
|
|
5041
|
+
"warnings": Array<{
|
|
5042
|
+
"type": "eval_capability_gap";
|
|
5043
|
+
"harness": string;
|
|
5044
|
+
"kind": string;
|
|
5045
|
+
} | {
|
|
5046
|
+
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
5047
|
+
"detail": string;
|
|
5048
|
+
"taskIds"?: Array<string>;
|
|
5049
|
+
}>;
|
|
5050
|
+
"shareUrl"?: string | null;
|
|
5051
|
+
"canManageSharing"?: boolean;
|
|
5052
|
+
};
|
|
5053
|
+
export type StudyDeleteResponse = unknown;
|
|
5054
|
+
export type StudyRegenerateSummaryResponse = {
|
|
5055
|
+
"status": "queued";
|
|
5056
|
+
};
|
|
5057
|
+
export type StudyGenerateRequest = {
|
|
5058
|
+
"proposeExpectations"?: boolean;
|
|
5059
|
+
};
|
|
5060
|
+
export type StudyGenerateResponse = {
|
|
5061
|
+
"study": {
|
|
5062
|
+
"id": string;
|
|
5063
|
+
"executiveSummary"?: string | null;
|
|
5064
|
+
"plannedRunCount": number;
|
|
5065
|
+
"organizationId": string;
|
|
5066
|
+
"createdBy": string | null;
|
|
5067
|
+
"name": string;
|
|
5068
|
+
"slug": string;
|
|
5069
|
+
"description": string | null;
|
|
5070
|
+
"skillRef": string | null;
|
|
5071
|
+
"skillSlug": string | null;
|
|
5072
|
+
"skillName": string | null;
|
|
5073
|
+
"skillPromise": string;
|
|
5074
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5075
|
+
"subjectName": string | null;
|
|
5076
|
+
"subjectLabel": string;
|
|
5077
|
+
"armConfig": {
|
|
5078
|
+
"baseline"?: {
|
|
5079
|
+
"tools"?: "default" | "none" | {
|
|
5080
|
+
"tools": Array<string>;
|
|
5081
|
+
};
|
|
5082
|
+
"addons"?: Record<string, unknown>;
|
|
5083
|
+
};
|
|
5084
|
+
"treatment"?: {
|
|
5085
|
+
"tools"?: "default" | "none" | {
|
|
5086
|
+
"tools": Array<string>;
|
|
5087
|
+
};
|
|
5088
|
+
"addons"?: Record<string, unknown>;
|
|
5089
|
+
};
|
|
5090
|
+
};
|
|
5091
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5092
|
+
"subjectBackend": "model" | "harness" | null;
|
|
5093
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5094
|
+
"importedFrom"?: string;
|
|
5095
|
+
"baselineLabel": string | null;
|
|
5096
|
+
"treatmentLabel": string | null;
|
|
5097
|
+
"promiseMetrics": Array<{
|
|
5098
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5099
|
+
"direction": "down" | "up";
|
|
5100
|
+
"label": string;
|
|
5101
|
+
}>;
|
|
5102
|
+
"matrix": Array<{
|
|
5103
|
+
"harness": string;
|
|
5104
|
+
"model": string;
|
|
5105
|
+
"provider"?: string;
|
|
5106
|
+
"params"?: {
|
|
5107
|
+
"temperature"?: number;
|
|
5108
|
+
"maxOutputTokens"?: number;
|
|
5109
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5110
|
+
"thinkingBudgetTokens"?: number;
|
|
5111
|
+
};
|
|
5112
|
+
}>;
|
|
5113
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5114
|
+
"taskCount": number;
|
|
5115
|
+
"variantsPerTask": number;
|
|
5116
|
+
"repeats": number;
|
|
5117
|
+
"subset": {
|
|
5118
|
+
"count": number;
|
|
5119
|
+
"seed": string;
|
|
5120
|
+
"of": number;
|
|
5121
|
+
} | null;
|
|
5122
|
+
"status": string;
|
|
5123
|
+
"error": string | null;
|
|
5124
|
+
"preparation"?: {
|
|
5125
|
+
"since": string;
|
|
5126
|
+
"launchedBy": {
|
|
5127
|
+
"userId"?: string;
|
|
5128
|
+
"role"?: string;
|
|
5129
|
+
};
|
|
5130
|
+
"waitingOn": Array<{
|
|
5131
|
+
"imageKey": string;
|
|
5132
|
+
"buildId": string;
|
|
5133
|
+
"stage"?: "image" | "snapshot";
|
|
5134
|
+
}>;
|
|
5135
|
+
} | null;
|
|
5136
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
5137
|
+
"visibilityChangedAt"?: string | null;
|
|
5138
|
+
"visibilityChangedBy"?: string | null;
|
|
5139
|
+
"shareToken"?: string | null;
|
|
5140
|
+
"lowSampleOverride": boolean;
|
|
5141
|
+
"publishedAt": string | null;
|
|
5142
|
+
"verdict": unknown;
|
|
5143
|
+
"definitionVersionId": string | null;
|
|
5144
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
5145
|
+
"createdAt": string;
|
|
5146
|
+
"updatedAt": string;
|
|
5147
|
+
"liveUntil": string | null;
|
|
5148
|
+
};
|
|
5149
|
+
"streamUrl": string;
|
|
5150
|
+
"warnings"?: Array<{
|
|
5151
|
+
"harness": string;
|
|
5152
|
+
"kind": string;
|
|
5153
|
+
}>;
|
|
5154
|
+
};
|
|
5155
|
+
export type StudyLaunchResponse = {
|
|
5156
|
+
"study": {
|
|
5157
|
+
"id": string;
|
|
5158
|
+
"executiveSummary"?: string | null;
|
|
5159
|
+
"plannedRunCount": number;
|
|
5160
|
+
"organizationId": string;
|
|
5161
|
+
"createdBy": string | null;
|
|
5162
|
+
"name": string;
|
|
5163
|
+
"slug": string;
|
|
5164
|
+
"description": string | null;
|
|
5165
|
+
"skillRef": string | null;
|
|
5166
|
+
"skillSlug": string | null;
|
|
5167
|
+
"skillName": string | null;
|
|
5168
|
+
"skillPromise": string;
|
|
5169
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5170
|
+
"subjectName": string | null;
|
|
5171
|
+
"subjectLabel": string;
|
|
5172
|
+
"armConfig": {
|
|
5173
|
+
"baseline"?: {
|
|
5174
|
+
"tools"?: "default" | "none" | {
|
|
5175
|
+
"tools": Array<string>;
|
|
5176
|
+
};
|
|
5177
|
+
"addons"?: Record<string, unknown>;
|
|
5178
|
+
};
|
|
5179
|
+
"treatment"?: {
|
|
5180
|
+
"tools"?: "default" | "none" | {
|
|
5181
|
+
"tools": Array<string>;
|
|
5182
|
+
};
|
|
5183
|
+
"addons"?: Record<string, unknown>;
|
|
5184
|
+
};
|
|
5185
|
+
};
|
|
5186
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5187
|
+
"subjectBackend": "model" | "harness" | null;
|
|
5188
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5189
|
+
"importedFrom"?: string;
|
|
5190
|
+
"baselineLabel": string | null;
|
|
5191
|
+
"treatmentLabel": string | null;
|
|
5192
|
+
"promiseMetrics": Array<{
|
|
5193
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5194
|
+
"direction": "down" | "up";
|
|
5195
|
+
"label": string;
|
|
5196
|
+
}>;
|
|
5197
|
+
"matrix": Array<{
|
|
5198
|
+
"harness": string;
|
|
5199
|
+
"model": string;
|
|
5200
|
+
"provider"?: string;
|
|
5201
|
+
"params"?: {
|
|
5202
|
+
"temperature"?: number;
|
|
5203
|
+
"maxOutputTokens"?: number;
|
|
5204
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5205
|
+
"thinkingBudgetTokens"?: number;
|
|
5206
|
+
};
|
|
5207
|
+
}>;
|
|
5208
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5209
|
+
"taskCount": number;
|
|
5210
|
+
"variantsPerTask": number;
|
|
5211
|
+
"repeats": number;
|
|
5212
|
+
"subset": {
|
|
5213
|
+
"count": number;
|
|
5214
|
+
"seed": string;
|
|
5215
|
+
"of": number;
|
|
5216
|
+
} | null;
|
|
5217
|
+
"status": string;
|
|
5218
|
+
"error": string | null;
|
|
5219
|
+
"preparation"?: {
|
|
5220
|
+
"since": string;
|
|
5221
|
+
"launchedBy": {
|
|
5222
|
+
"userId"?: string;
|
|
5223
|
+
"role"?: string;
|
|
5224
|
+
};
|
|
5225
|
+
"waitingOn": Array<{
|
|
5226
|
+
"imageKey": string;
|
|
5227
|
+
"buildId": string;
|
|
5228
|
+
"stage"?: "image" | "snapshot";
|
|
5229
|
+
}>;
|
|
5230
|
+
} | null;
|
|
5231
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
5232
|
+
"visibilityChangedAt"?: string | null;
|
|
5233
|
+
"visibilityChangedBy"?: string | null;
|
|
5234
|
+
"shareToken"?: string | null;
|
|
5235
|
+
"lowSampleOverride": boolean;
|
|
5236
|
+
"publishedAt": string | null;
|
|
5237
|
+
"verdict": unknown;
|
|
5238
|
+
"definitionVersionId": string | null;
|
|
5239
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
5240
|
+
"createdAt": string;
|
|
5241
|
+
"updatedAt": string;
|
|
5242
|
+
"liveUntil": string | null;
|
|
5243
|
+
};
|
|
5244
|
+
"streamUrl": string;
|
|
5245
|
+
"warnings"?: Array<{
|
|
5246
|
+
"harness": string;
|
|
5247
|
+
"kind": string;
|
|
5248
|
+
}>;
|
|
5249
|
+
};
|
|
5250
|
+
export type StudyUpdateLensesRequest = {
|
|
5251
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5252
|
+
};
|
|
5253
|
+
export type StudyUpdateLensesResponse = {
|
|
5254
|
+
"study": {
|
|
5255
|
+
"id": string;
|
|
5256
|
+
"executiveSummary"?: string | null;
|
|
5257
|
+
"plannedRunCount": number;
|
|
5258
|
+
"organizationId": string;
|
|
5259
|
+
"createdBy": string | null;
|
|
5260
|
+
"name": string;
|
|
5261
|
+
"slug": string;
|
|
5262
|
+
"description": string | null;
|
|
5263
|
+
"skillRef": string | null;
|
|
5264
|
+
"skillSlug": string | null;
|
|
5265
|
+
"skillName": string | null;
|
|
5266
|
+
"skillPromise": string;
|
|
5267
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5268
|
+
"subjectName": string | null;
|
|
5269
|
+
"subjectLabel": string;
|
|
5270
|
+
"armConfig": {
|
|
5271
|
+
"baseline"?: {
|
|
5272
|
+
"tools"?: "default" | "none" | {
|
|
5273
|
+
"tools": Array<string>;
|
|
5274
|
+
};
|
|
5275
|
+
"addons"?: Record<string, unknown>;
|
|
5276
|
+
};
|
|
5277
|
+
"treatment"?: {
|
|
5278
|
+
"tools"?: "default" | "none" | {
|
|
5279
|
+
"tools": Array<string>;
|
|
5280
|
+
};
|
|
5281
|
+
"addons"?: Record<string, unknown>;
|
|
5282
|
+
};
|
|
5283
|
+
};
|
|
5284
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5285
|
+
"subjectBackend": "model" | "harness" | null;
|
|
5286
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5287
|
+
"importedFrom"?: string;
|
|
5288
|
+
"baselineLabel": string | null;
|
|
5289
|
+
"treatmentLabel": string | null;
|
|
5290
|
+
"promiseMetrics": Array<{
|
|
5291
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5292
|
+
"direction": "down" | "up";
|
|
5293
|
+
"label": string;
|
|
5294
|
+
}>;
|
|
5295
|
+
"matrix": Array<{
|
|
5296
|
+
"harness": string;
|
|
5297
|
+
"model": string;
|
|
5298
|
+
"provider"?: string;
|
|
5299
|
+
"params"?: {
|
|
5300
|
+
"temperature"?: number;
|
|
5301
|
+
"maxOutputTokens"?: number;
|
|
5302
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5303
|
+
"thinkingBudgetTokens"?: number;
|
|
5304
|
+
};
|
|
5305
|
+
}>;
|
|
5306
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5307
|
+
"taskCount": number;
|
|
5308
|
+
"variantsPerTask": number;
|
|
5309
|
+
"repeats": number;
|
|
5310
|
+
"subset": {
|
|
5311
|
+
"count": number;
|
|
5312
|
+
"seed": string;
|
|
5313
|
+
"of": number;
|
|
5314
|
+
} | null;
|
|
5315
|
+
"status": string;
|
|
5316
|
+
"error": string | null;
|
|
5317
|
+
"preparation"?: {
|
|
5318
|
+
"since": string;
|
|
5319
|
+
"launchedBy": {
|
|
5320
|
+
"userId"?: string;
|
|
5321
|
+
"role"?: string;
|
|
5322
|
+
};
|
|
5323
|
+
"waitingOn": Array<{
|
|
5324
|
+
"imageKey": string;
|
|
5325
|
+
"buildId": string;
|
|
5326
|
+
"stage"?: "image" | "snapshot";
|
|
5327
|
+
}>;
|
|
5328
|
+
} | null;
|
|
5329
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
5330
|
+
"visibilityChangedAt"?: string | null;
|
|
5331
|
+
"visibilityChangedBy"?: string | null;
|
|
5332
|
+
"shareToken"?: string | null;
|
|
5333
|
+
"lowSampleOverride": boolean;
|
|
5334
|
+
"publishedAt": string | null;
|
|
5335
|
+
"verdict": unknown;
|
|
5336
|
+
"definitionVersionId": string | null;
|
|
5337
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
5338
|
+
"createdAt": string;
|
|
5339
|
+
"updatedAt": string;
|
|
5340
|
+
"liveUntil": string | null;
|
|
5341
|
+
};
|
|
5342
|
+
"warnings": Array<{
|
|
5343
|
+
"type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
|
|
5344
|
+
"detail": string;
|
|
5345
|
+
"taskIds"?: Array<string>;
|
|
5346
|
+
}>;
|
|
5347
|
+
};
|
|
5348
|
+
export type StudyPreviewResponse = {
|
|
5349
|
+
"study": {
|
|
5350
|
+
"executiveSummary"?: string | null;
|
|
5351
|
+
"slug": string;
|
|
5352
|
+
"name": string;
|
|
5353
|
+
"description": string | null;
|
|
5354
|
+
"skillName": string | null;
|
|
5355
|
+
"skillSlug": string | null;
|
|
5356
|
+
"skillPromise": string;
|
|
5357
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5358
|
+
"subjectName": string | null;
|
|
5359
|
+
"subjectLabel": string;
|
|
5360
|
+
"armConfig": {
|
|
5361
|
+
"baseline"?: {
|
|
5362
|
+
"tools"?: "default" | "none" | {
|
|
5363
|
+
"tools": Array<string>;
|
|
5364
|
+
};
|
|
5365
|
+
"addons"?: Array<string>;
|
|
5366
|
+
};
|
|
5367
|
+
"treatment"?: {
|
|
5368
|
+
"tools"?: "default" | "none" | {
|
|
5369
|
+
"tools": Array<string>;
|
|
5370
|
+
};
|
|
5371
|
+
"addons"?: Array<string>;
|
|
5372
|
+
};
|
|
5373
|
+
};
|
|
5374
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5375
|
+
"subjectBackend"?: "model" | "harness";
|
|
5376
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5377
|
+
"baselineLabel": string | null;
|
|
5378
|
+
"treatmentLabel": string | null;
|
|
5379
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5380
|
+
"promiseMetrics": Array<{
|
|
5381
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5382
|
+
"direction": "down" | "up";
|
|
5383
|
+
"label": string;
|
|
5384
|
+
}>;
|
|
5385
|
+
"matrix": Array<{
|
|
5386
|
+
"harness": string;
|
|
5387
|
+
"model": string;
|
|
5388
|
+
"provider"?: string;
|
|
5389
|
+
"params"?: {
|
|
5390
|
+
"temperature"?: number;
|
|
5391
|
+
"maxOutputTokens"?: number;
|
|
5392
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5393
|
+
"thinkingBudgetTokens"?: number;
|
|
5394
|
+
};
|
|
5395
|
+
}>;
|
|
5396
|
+
"taskCount": number;
|
|
5397
|
+
"variantsPerTask": number;
|
|
5398
|
+
"repeats": number;
|
|
5399
|
+
"status": string;
|
|
5400
|
+
"publishedAt": string | null;
|
|
5401
|
+
"createdAt": string;
|
|
5402
|
+
"viewCount"?: number;
|
|
5403
|
+
"boardEligibility"?: {
|
|
5404
|
+
"eligible": boolean;
|
|
5405
|
+
"refusedBy": string | null;
|
|
5406
|
+
"reason": string | null;
|
|
5407
|
+
"failures": Array<{
|
|
5408
|
+
"ruleId": string;
|
|
5409
|
+
"reason": string;
|
|
5410
|
+
}>;
|
|
5411
|
+
"evaluatedAt": string;
|
|
5412
|
+
};
|
|
5413
|
+
"subjectLicenseSpdx"?: string | null;
|
|
5414
|
+
"subjectSourceUrl"?: string | null;
|
|
5415
|
+
"subjectWithheldReason"?: string | null;
|
|
5416
|
+
"subjectNotices"?: Array<{
|
|
5417
|
+
"kind": string;
|
|
5418
|
+
"text": string;
|
|
5419
|
+
}>;
|
|
5420
|
+
};
|
|
5421
|
+
"tasks": Array<{
|
|
5422
|
+
"id": string;
|
|
5423
|
+
"position": number;
|
|
5424
|
+
"prompt"?: string;
|
|
5425
|
+
"rationale"?: string | null;
|
|
5426
|
+
"category": string | null;
|
|
5427
|
+
"requiresWeb": boolean;
|
|
5428
|
+
"rubric"?: Array<string>;
|
|
5429
|
+
"variants"?: Array<string> | null;
|
|
5430
|
+
"expectations"?: Array<unknown> | null;
|
|
5431
|
+
"textWithheld"?: boolean;
|
|
5432
|
+
}>;
|
|
5433
|
+
"runs": Array<{
|
|
5434
|
+
"runId": string;
|
|
5435
|
+
"taskId": string;
|
|
5436
|
+
"arm": "baseline" | "skill";
|
|
5437
|
+
"harness": string;
|
|
5438
|
+
"model": string;
|
|
5439
|
+
"provider": string | null;
|
|
5440
|
+
"variantIndex"?: number;
|
|
5441
|
+
"repeatIndex"?: number;
|
|
5442
|
+
"status": string;
|
|
5443
|
+
"outcome": string | null;
|
|
5444
|
+
"outcomeReason": string | null;
|
|
5445
|
+
"totalTokens": number | null;
|
|
5446
|
+
"inputTokens": number | null;
|
|
5447
|
+
"outputTokens": number | null;
|
|
5448
|
+
"cacheReadTokens": number | null;
|
|
5449
|
+
"cacheCreationTokens": number | null;
|
|
5450
|
+
"costUsd": number | null;
|
|
5451
|
+
"numTurns": number | null;
|
|
5452
|
+
"durationMs": number | null;
|
|
5453
|
+
"intentSatisfied": boolean | null;
|
|
5454
|
+
"frictionCount": number | null;
|
|
5455
|
+
"qualityScore": number | null;
|
|
5456
|
+
"gradedAt": string | null;
|
|
5457
|
+
"judgeModel": string | null;
|
|
5458
|
+
"pairVerdict": "skill" | "baseline" | "tie" | "unmeasurable" | null;
|
|
5459
|
+
"pairGradedAt": string | null;
|
|
5460
|
+
"evalScore": number | null;
|
|
5461
|
+
"evalGradedAt": string | null;
|
|
5462
|
+
"runCreatedAt": string | null;
|
|
5463
|
+
"startedAt": string | null;
|
|
5464
|
+
"finishedAt": string | null;
|
|
5465
|
+
"lensGrades"?: Record<string, unknown>;
|
|
5466
|
+
"rubricPassed": Array<boolean> | null;
|
|
5467
|
+
"evalOutcomes": Array<{
|
|
5468
|
+
"kind": string;
|
|
5469
|
+
"label"?: string;
|
|
5470
|
+
"passed": boolean;
|
|
5471
|
+
"abstained"?: boolean;
|
|
5472
|
+
"decisive"?: boolean;
|
|
5473
|
+
}> | null;
|
|
5474
|
+
"paramsKey"?: string;
|
|
5475
|
+
"textWithheld"?: boolean;
|
|
5476
|
+
}>;
|
|
5477
|
+
"aggregate": {
|
|
5478
|
+
"summary": {
|
|
5479
|
+
"verdict": "delivered" | "partial" | "not_delivered" | "inconclusive" | "measured";
|
|
5480
|
+
"headline": Array<{
|
|
5481
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5482
|
+
"label": string;
|
|
5483
|
+
"direction": "down" | "up";
|
|
5484
|
+
"baselineAvg": number | null;
|
|
5485
|
+
"skillAvg": number | null;
|
|
5486
|
+
"deltaPct": number | null;
|
|
5487
|
+
"delta"?: {
|
|
5488
|
+
"unit": "pp";
|
|
5489
|
+
"absolute": number | null;
|
|
5490
|
+
"relativePct": number | null;
|
|
5491
|
+
} | {
|
|
5492
|
+
"unit": "pts";
|
|
5493
|
+
"absolute": number | null;
|
|
5494
|
+
"relativePct"?: number | null;
|
|
5495
|
+
} | {
|
|
5496
|
+
"unit": "%";
|
|
5497
|
+
"relativePct": number | null;
|
|
5498
|
+
};
|
|
5499
|
+
"improved": boolean;
|
|
5500
|
+
}>;
|
|
5501
|
+
"completionRate": {
|
|
5502
|
+
"baseline": number | null;
|
|
5503
|
+
"skill": number | null;
|
|
5504
|
+
};
|
|
5505
|
+
"goalRate": {
|
|
5506
|
+
"baseline": number | null;
|
|
5507
|
+
"skill": number | null;
|
|
5508
|
+
};
|
|
5509
|
+
"pairwise": {
|
|
5510
|
+
"pairs": number;
|
|
5511
|
+
"skillWins": number;
|
|
5512
|
+
"baselineWins": number;
|
|
5513
|
+
"ties": number;
|
|
5514
|
+
"winRate": number | null;
|
|
5515
|
+
};
|
|
5516
|
+
"decision"?: {
|
|
5517
|
+
"winner": "treatment" | "baseline" | "tie" | "unmeasurable";
|
|
5518
|
+
"basis": "infra" | "goal" | "evals" | "quality" | "efficiency" | "judge" | "insufficient";
|
|
5519
|
+
"reason": string;
|
|
5520
|
+
"efficiencyDiscounted": boolean;
|
|
5521
|
+
};
|
|
5522
|
+
"leaderboard"?: Array<unknown>;
|
|
5523
|
+
"concentration": {
|
|
5524
|
+
"perCell": Array<{
|
|
5525
|
+
"cell": string;
|
|
5526
|
+
"goalDeltaPp": number | null;
|
|
5527
|
+
"contributionPct": number | null;
|
|
5528
|
+
}>;
|
|
5529
|
+
"heterogeneity": {
|
|
5530
|
+
"dominantCell": string;
|
|
5531
|
+
"contributionPct": number;
|
|
5532
|
+
} | null;
|
|
5533
|
+
};
|
|
5534
|
+
"matrixShape": {
|
|
5535
|
+
"cells": number;
|
|
5536
|
+
"harnesses": number;
|
|
5537
|
+
"models": number;
|
|
5538
|
+
"crossed": boolean;
|
|
5539
|
+
};
|
|
5540
|
+
"flakiness"?: {
|
|
5541
|
+
"measured": boolean;
|
|
5542
|
+
"conditions": number;
|
|
5543
|
+
"unstable": number;
|
|
5544
|
+
"worst": {
|
|
5545
|
+
"cell": string;
|
|
5546
|
+
"taskId": string;
|
|
5547
|
+
"arm": "baseline" | "skill";
|
|
5548
|
+
"variant"?: number;
|
|
5549
|
+
"passRate": number;
|
|
5550
|
+
} | null;
|
|
5551
|
+
};
|
|
5552
|
+
"arms"?: {
|
|
5553
|
+
"baseline": {
|
|
5554
|
+
"runs": number;
|
|
5555
|
+
"completed": number;
|
|
5556
|
+
"infraFailures": number;
|
|
5557
|
+
"completionRate": number | null;
|
|
5558
|
+
"goalRate": number | null;
|
|
5559
|
+
"goalTally": {
|
|
5560
|
+
"achieved": number;
|
|
5561
|
+
"partial": number;
|
|
5562
|
+
"missed": number;
|
|
5563
|
+
};
|
|
5564
|
+
"avgTotalTokens": number | null;
|
|
5565
|
+
"avgOutputTokens": number | null;
|
|
5566
|
+
"avgCostUsd": number | null;
|
|
5567
|
+
"avgDurationMs": number | null;
|
|
5568
|
+
"avgTurns": number | null;
|
|
5569
|
+
"costCoverage": number | null;
|
|
5570
|
+
"avgQualityScore": number | null;
|
|
5571
|
+
"gradedCoverage": number | null;
|
|
5572
|
+
"medianCostUsd": number | null;
|
|
5573
|
+
"medianTotalTokens": number | null;
|
|
5574
|
+
"medianDurationMs": number | null;
|
|
5575
|
+
"p95CostUsd": number | null;
|
|
5576
|
+
"p95TotalTokens": number | null;
|
|
5577
|
+
"p95DurationMs": number | null;
|
|
5578
|
+
"maxCostUsd": number | null;
|
|
5579
|
+
"maxTotalTokens": number | null;
|
|
5580
|
+
"maxDurationMs": number | null;
|
|
5581
|
+
"evalPassRate"?: number | null;
|
|
5582
|
+
"adoptionRate"?: number | null;
|
|
5583
|
+
};
|
|
5584
|
+
"skill": {
|
|
5585
|
+
"runs": number;
|
|
5586
|
+
"completed": number;
|
|
5587
|
+
"infraFailures": number;
|
|
5588
|
+
"completionRate": number | null;
|
|
5589
|
+
"goalRate": number | null;
|
|
5590
|
+
"goalTally": {
|
|
5591
|
+
"achieved": number;
|
|
5592
|
+
"partial": number;
|
|
5593
|
+
"missed": number;
|
|
5594
|
+
};
|
|
5595
|
+
"avgTotalTokens": number | null;
|
|
5596
|
+
"avgOutputTokens": number | null;
|
|
5597
|
+
"avgCostUsd": number | null;
|
|
5598
|
+
"avgDurationMs": number | null;
|
|
5599
|
+
"avgTurns": number | null;
|
|
5600
|
+
"costCoverage": number | null;
|
|
5601
|
+
"avgQualityScore": number | null;
|
|
5602
|
+
"gradedCoverage": number | null;
|
|
5603
|
+
"medianCostUsd": number | null;
|
|
5604
|
+
"medianTotalTokens": number | null;
|
|
5605
|
+
"medianDurationMs": number | null;
|
|
5606
|
+
"p95CostUsd": number | null;
|
|
5607
|
+
"p95TotalTokens": number | null;
|
|
5608
|
+
"p95DurationMs": number | null;
|
|
5609
|
+
"maxCostUsd": number | null;
|
|
5610
|
+
"maxTotalTokens": number | null;
|
|
5611
|
+
"maxDurationMs": number | null;
|
|
5612
|
+
"evalPassRate"?: number | null;
|
|
5613
|
+
"adoptionRate"?: number | null;
|
|
5614
|
+
};
|
|
5615
|
+
};
|
|
5616
|
+
"totals": {
|
|
5617
|
+
"cells": number;
|
|
5618
|
+
"runs": number;
|
|
5619
|
+
"completed": number;
|
|
5620
|
+
"failed": number;
|
|
5621
|
+
"costUsd": number;
|
|
5622
|
+
"tokens": number;
|
|
5623
|
+
};
|
|
5624
|
+
};
|
|
5625
|
+
"perCell": Array<unknown>;
|
|
5626
|
+
"perHarness": Array<unknown>;
|
|
5627
|
+
"perModel": Array<unknown>;
|
|
5628
|
+
"perTask": Array<unknown>;
|
|
5629
|
+
"webSplit": unknown;
|
|
5630
|
+
"inference": {
|
|
5631
|
+
"columns": Array<{
|
|
5632
|
+
"key": string;
|
|
5633
|
+
"label": string;
|
|
5634
|
+
"family": "rate" | "score" | "magnitude";
|
|
5635
|
+
"betterWhen": "higher" | "lower";
|
|
5636
|
+
}>;
|
|
5637
|
+
"groups": Array<{
|
|
5638
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
5639
|
+
"title": string;
|
|
5640
|
+
"rows": Array<{
|
|
5641
|
+
"key": string;
|
|
5642
|
+
"kind": "cell" | "harness" | "model" | "field";
|
|
5643
|
+
"harness": string | null;
|
|
5644
|
+
"model": string | null;
|
|
5645
|
+
"paramsKey"?: string;
|
|
5646
|
+
"runs": number;
|
|
5647
|
+
"cells": Record<string, unknown>;
|
|
5648
|
+
}>;
|
|
5649
|
+
}>;
|
|
5650
|
+
"reference": "baseline" | "field";
|
|
5651
|
+
"draws": number;
|
|
5652
|
+
} | null;
|
|
5653
|
+
"behavior"?: Array<{
|
|
5654
|
+
"harness": string;
|
|
5655
|
+
"model": string;
|
|
5656
|
+
"provider": string | null;
|
|
5657
|
+
"arm": string;
|
|
5658
|
+
"totalRuns": number;
|
|
5659
|
+
"signals": Array<{
|
|
5660
|
+
"id": string;
|
|
5661
|
+
"label": string;
|
|
5662
|
+
"tone": "good" | "bad" | "neutral";
|
|
5663
|
+
"runCount": number;
|
|
5664
|
+
"succeededCount": number;
|
|
5665
|
+
}>;
|
|
5666
|
+
}>;
|
|
5667
|
+
"reference"?: {
|
|
5668
|
+
"cells": Array<{
|
|
5669
|
+
"cell": string;
|
|
5670
|
+
"harness": string;
|
|
5671
|
+
"model": string;
|
|
5672
|
+
"provider": string | null;
|
|
5673
|
+
"paramsKey": string;
|
|
5674
|
+
"label": string;
|
|
5675
|
+
"passRate": {
|
|
5676
|
+
"value": number | null;
|
|
5677
|
+
"ci": Array<unknown> | null;
|
|
5678
|
+
"n": number;
|
|
5679
|
+
};
|
|
5680
|
+
"tasks": number;
|
|
5681
|
+
"graded": number;
|
|
5682
|
+
"unmeasured": number;
|
|
5683
|
+
"matchers": Array<string>;
|
|
5684
|
+
"launchProfiles": Array<"as-published" | "harness-default" | "with-setup">;
|
|
5685
|
+
}>;
|
|
5686
|
+
"overall": {
|
|
5687
|
+
"value": number | null;
|
|
5688
|
+
"ci": Array<unknown> | null;
|
|
5689
|
+
"n": number;
|
|
5690
|
+
};
|
|
5691
|
+
"graded": number;
|
|
5692
|
+
"unmeasured": number;
|
|
5693
|
+
"modelJudged": boolean;
|
|
5694
|
+
};
|
|
5695
|
+
"execution"?: {
|
|
5696
|
+
"graded": number;
|
|
5697
|
+
"measured": number;
|
|
5698
|
+
"passed": number;
|
|
5699
|
+
"passRate": number | null;
|
|
5700
|
+
"byStatus": {
|
|
5701
|
+
"passed": number;
|
|
5702
|
+
"failed": number;
|
|
5703
|
+
"timeout": number;
|
|
5704
|
+
"error": number;
|
|
5705
|
+
"unavailable": number;
|
|
5706
|
+
};
|
|
5707
|
+
"languages": Array<string>;
|
|
5708
|
+
"unavailableReasons": Array<string>;
|
|
5709
|
+
};
|
|
5710
|
+
};
|
|
5711
|
+
"textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
|
|
5712
|
+
};
|
|
5713
|
+
export type StudyPublishRequest = {
|
|
5714
|
+
"slug"?: string;
|
|
5715
|
+
"visibility"?: "private" | "org" | "link" | "public";
|
|
5716
|
+
};
|
|
5717
|
+
export type StudyPublishResponse = {
|
|
5718
|
+
"study": {
|
|
5719
|
+
"id": string;
|
|
5720
|
+
"executiveSummary"?: string | null;
|
|
5721
|
+
"plannedRunCount": number;
|
|
5722
|
+
"organizationId": string;
|
|
5723
|
+
"createdBy": string | null;
|
|
5724
|
+
"name": string;
|
|
5725
|
+
"slug": string;
|
|
5726
|
+
"description": string | null;
|
|
5727
|
+
"skillRef": string | null;
|
|
5728
|
+
"skillSlug": string | null;
|
|
5729
|
+
"skillName": string | null;
|
|
5730
|
+
"skillPromise": string;
|
|
5731
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5732
|
+
"subjectName": string | null;
|
|
5733
|
+
"subjectLabel": string;
|
|
5734
|
+
"armConfig": {
|
|
5735
|
+
"baseline"?: {
|
|
5736
|
+
"tools"?: "default" | "none" | {
|
|
5737
|
+
"tools": Array<string>;
|
|
5738
|
+
};
|
|
5739
|
+
"addons"?: Record<string, unknown>;
|
|
5740
|
+
};
|
|
5741
|
+
"treatment"?: {
|
|
5742
|
+
"tools"?: "default" | "none" | {
|
|
5743
|
+
"tools": Array<string>;
|
|
5744
|
+
};
|
|
5745
|
+
"addons"?: Record<string, unknown>;
|
|
5746
|
+
};
|
|
5747
|
+
};
|
|
5748
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5749
|
+
"subjectBackend": "model" | "harness" | null;
|
|
5750
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5751
|
+
"importedFrom"?: string;
|
|
5752
|
+
"baselineLabel": string | null;
|
|
5753
|
+
"treatmentLabel": string | null;
|
|
5754
|
+
"promiseMetrics": Array<{
|
|
5755
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5756
|
+
"direction": "down" | "up";
|
|
5757
|
+
"label": string;
|
|
5758
|
+
}>;
|
|
5759
|
+
"matrix": Array<{
|
|
5760
|
+
"harness": string;
|
|
5761
|
+
"model": string;
|
|
5762
|
+
"provider"?: string;
|
|
5763
|
+
"params"?: {
|
|
5764
|
+
"temperature"?: number;
|
|
5765
|
+
"maxOutputTokens"?: number;
|
|
5766
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5767
|
+
"thinkingBudgetTokens"?: number;
|
|
5768
|
+
};
|
|
5769
|
+
}>;
|
|
5770
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5771
|
+
"taskCount": number;
|
|
5772
|
+
"variantsPerTask": number;
|
|
5773
|
+
"repeats": number;
|
|
5774
|
+
"subset": {
|
|
5775
|
+
"count": number;
|
|
5776
|
+
"seed": string;
|
|
5777
|
+
"of": number;
|
|
5778
|
+
} | null;
|
|
5779
|
+
"status": string;
|
|
5780
|
+
"error": string | null;
|
|
5781
|
+
"preparation"?: {
|
|
5782
|
+
"since": string;
|
|
5783
|
+
"launchedBy": {
|
|
5784
|
+
"userId"?: string;
|
|
5785
|
+
"role"?: string;
|
|
5786
|
+
};
|
|
5787
|
+
"waitingOn": Array<{
|
|
5788
|
+
"imageKey": string;
|
|
5789
|
+
"buildId": string;
|
|
5790
|
+
"stage"?: "image" | "snapshot";
|
|
5791
|
+
}>;
|
|
5792
|
+
} | null;
|
|
5793
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
5794
|
+
"visibilityChangedAt"?: string | null;
|
|
5795
|
+
"visibilityChangedBy"?: string | null;
|
|
5796
|
+
"shareToken"?: string | null;
|
|
5797
|
+
"lowSampleOverride": boolean;
|
|
5798
|
+
"publishedAt": string | null;
|
|
5799
|
+
"verdict": unknown;
|
|
5800
|
+
"definitionVersionId": string | null;
|
|
5801
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
5802
|
+
"createdAt": string;
|
|
5803
|
+
"updatedAt": string;
|
|
5804
|
+
"liveUntil": string | null;
|
|
5805
|
+
};
|
|
5806
|
+
};
|
|
5807
|
+
export type StudyReconcileRunsResponse = {
|
|
5808
|
+
"studyId": string;
|
|
5809
|
+
"laggingRuns": number;
|
|
5810
|
+
"projected": number;
|
|
5811
|
+
"refinedInsights": number;
|
|
5812
|
+
"redrivenLensGrades": Record<string, unknown>;
|
|
5813
|
+
"skipped": boolean;
|
|
5814
|
+
"status": string;
|
|
5815
|
+
};
|
|
5816
|
+
export type StudyRegradeResponse = {
|
|
5817
|
+
"study": {
|
|
5818
|
+
"id": string;
|
|
5819
|
+
"executiveSummary"?: string | null;
|
|
5820
|
+
"plannedRunCount": number;
|
|
5821
|
+
"organizationId": string;
|
|
5822
|
+
"createdBy": string | null;
|
|
5823
|
+
"name": string;
|
|
5824
|
+
"slug": string;
|
|
5825
|
+
"description": string | null;
|
|
5826
|
+
"skillRef": string | null;
|
|
5827
|
+
"skillSlug": string | null;
|
|
5828
|
+
"skillName": string | null;
|
|
5829
|
+
"skillPromise": string;
|
|
5830
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5831
|
+
"subjectName": string | null;
|
|
5832
|
+
"subjectLabel": string;
|
|
5833
|
+
"armConfig": {
|
|
5834
|
+
"baseline"?: {
|
|
5835
|
+
"tools"?: "default" | "none" | {
|
|
5836
|
+
"tools": Array<string>;
|
|
5837
|
+
};
|
|
5838
|
+
"addons"?: Record<string, unknown>;
|
|
5839
|
+
};
|
|
5840
|
+
"treatment"?: {
|
|
5841
|
+
"tools"?: "default" | "none" | {
|
|
5842
|
+
"tools": Array<string>;
|
|
5843
|
+
};
|
|
5844
|
+
"addons"?: Record<string, unknown>;
|
|
5845
|
+
};
|
|
5846
|
+
};
|
|
5847
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5848
|
+
"subjectBackend": "model" | "harness" | null;
|
|
5849
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5850
|
+
"importedFrom"?: string;
|
|
5851
|
+
"baselineLabel": string | null;
|
|
5852
|
+
"treatmentLabel": string | null;
|
|
5853
|
+
"promiseMetrics": Array<{
|
|
5854
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5855
|
+
"direction": "down" | "up";
|
|
5856
|
+
"label": string;
|
|
5857
|
+
}>;
|
|
5858
|
+
"matrix": Array<{
|
|
5859
|
+
"harness": string;
|
|
5860
|
+
"model": string;
|
|
5861
|
+
"provider"?: string;
|
|
5862
|
+
"params"?: {
|
|
5863
|
+
"temperature"?: number;
|
|
5864
|
+
"maxOutputTokens"?: number;
|
|
5865
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5866
|
+
"thinkingBudgetTokens"?: number;
|
|
5867
|
+
};
|
|
5868
|
+
}>;
|
|
5869
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5870
|
+
"taskCount": number;
|
|
5871
|
+
"variantsPerTask": number;
|
|
5872
|
+
"repeats": number;
|
|
5873
|
+
"subset": {
|
|
5874
|
+
"count": number;
|
|
5875
|
+
"seed": string;
|
|
5876
|
+
"of": number;
|
|
5877
|
+
} | null;
|
|
5878
|
+
"status": string;
|
|
5879
|
+
"error": string | null;
|
|
5880
|
+
"preparation"?: {
|
|
5881
|
+
"since": string;
|
|
5882
|
+
"launchedBy": {
|
|
5883
|
+
"userId"?: string;
|
|
5884
|
+
"role"?: string;
|
|
5885
|
+
};
|
|
5886
|
+
"waitingOn": Array<{
|
|
5887
|
+
"imageKey": string;
|
|
5888
|
+
"buildId": string;
|
|
5889
|
+
"stage"?: "image" | "snapshot";
|
|
5890
|
+
}>;
|
|
5891
|
+
} | null;
|
|
5892
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
5893
|
+
"visibilityChangedAt"?: string | null;
|
|
5894
|
+
"visibilityChangedBy"?: string | null;
|
|
5895
|
+
"shareToken"?: string | null;
|
|
5896
|
+
"lowSampleOverride": boolean;
|
|
5897
|
+
"publishedAt": string | null;
|
|
5898
|
+
"verdict": unknown;
|
|
5899
|
+
"definitionVersionId": string | null;
|
|
5900
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
5901
|
+
"createdAt": string;
|
|
5902
|
+
"updatedAt": string;
|
|
5903
|
+
"liveUntil": string | null;
|
|
5904
|
+
};
|
|
5905
|
+
"streamUrl": string;
|
|
5906
|
+
"warnings"?: Array<{
|
|
5907
|
+
"harness": string;
|
|
5908
|
+
"kind": string;
|
|
5909
|
+
}>;
|
|
5910
|
+
};
|
|
5911
|
+
export type StudyListRunAnswersResponse = {
|
|
5912
|
+
"coordinatorRunId": string;
|
|
5913
|
+
"answers": Array<{
|
|
5914
|
+
"runId": string;
|
|
5915
|
+
"createdAt": string;
|
|
5916
|
+
"status": string | null;
|
|
5917
|
+
"outcome": string | null;
|
|
5918
|
+
"totalTokens": number | null;
|
|
5919
|
+
"costUsd": number | null;
|
|
5920
|
+
"startedAt": string | null;
|
|
5921
|
+
"finishedAt": string | null;
|
|
5922
|
+
}>;
|
|
5923
|
+
};
|
|
5924
|
+
export type StudyRunInsightResponse = unknown;
|
|
5925
|
+
export type StudyRunTrajectoryResponse = unknown;
|
|
5926
|
+
export type StudyRotateShareTokenResponse = {
|
|
5927
|
+
"study": {
|
|
5928
|
+
"id": string;
|
|
5929
|
+
"executiveSummary"?: string | null;
|
|
5930
|
+
"plannedRunCount": number;
|
|
5931
|
+
"organizationId": string;
|
|
5932
|
+
"createdBy": string | null;
|
|
5933
|
+
"name": string;
|
|
5934
|
+
"slug": string;
|
|
5935
|
+
"description": string | null;
|
|
5936
|
+
"skillRef": string | null;
|
|
5937
|
+
"skillSlug": string | null;
|
|
5938
|
+
"skillName": string | null;
|
|
5939
|
+
"skillPromise": string;
|
|
5940
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
5941
|
+
"subjectName": string | null;
|
|
5942
|
+
"subjectLabel": string;
|
|
5943
|
+
"armConfig": {
|
|
5944
|
+
"baseline"?: {
|
|
5945
|
+
"tools"?: "default" | "none" | {
|
|
5946
|
+
"tools": Array<string>;
|
|
5947
|
+
};
|
|
5948
|
+
"addons"?: Record<string, unknown>;
|
|
5949
|
+
};
|
|
5950
|
+
"treatment"?: {
|
|
5951
|
+
"tools"?: "default" | "none" | {
|
|
5952
|
+
"tools": Array<string>;
|
|
5953
|
+
};
|
|
5954
|
+
"addons"?: Record<string, unknown>;
|
|
5955
|
+
};
|
|
5956
|
+
};
|
|
5957
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
5958
|
+
"subjectBackend": "model" | "harness" | null;
|
|
5959
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
5960
|
+
"importedFrom"?: string;
|
|
5961
|
+
"baselineLabel": string | null;
|
|
5962
|
+
"treatmentLabel": string | null;
|
|
5963
|
+
"promiseMetrics": Array<{
|
|
5964
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
5965
|
+
"direction": "down" | "up";
|
|
5966
|
+
"label": string;
|
|
5967
|
+
}>;
|
|
5968
|
+
"matrix": Array<{
|
|
5969
|
+
"harness": string;
|
|
5970
|
+
"model": string;
|
|
5971
|
+
"provider"?: string;
|
|
5972
|
+
"params"?: {
|
|
5973
|
+
"temperature"?: number;
|
|
5974
|
+
"maxOutputTokens"?: number;
|
|
5975
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
5976
|
+
"thinkingBudgetTokens"?: number;
|
|
5977
|
+
};
|
|
5978
|
+
}>;
|
|
5979
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
5980
|
+
"taskCount": number;
|
|
5981
|
+
"variantsPerTask": number;
|
|
5982
|
+
"repeats": number;
|
|
5983
|
+
"subset": {
|
|
5984
|
+
"count": number;
|
|
5985
|
+
"seed": string;
|
|
5986
|
+
"of": number;
|
|
5987
|
+
} | null;
|
|
5988
|
+
"status": string;
|
|
5989
|
+
"error": string | null;
|
|
5990
|
+
"preparation"?: {
|
|
5991
|
+
"since": string;
|
|
5992
|
+
"launchedBy": {
|
|
5993
|
+
"userId"?: string;
|
|
5994
|
+
"role"?: string;
|
|
5995
|
+
};
|
|
5996
|
+
"waitingOn": Array<{
|
|
5997
|
+
"imageKey": string;
|
|
5998
|
+
"buildId": string;
|
|
5999
|
+
"stage"?: "image" | "snapshot";
|
|
6000
|
+
}>;
|
|
6001
|
+
} | null;
|
|
6002
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
6003
|
+
"visibilityChangedAt"?: string | null;
|
|
6004
|
+
"visibilityChangedBy"?: string | null;
|
|
6005
|
+
"shareToken"?: string | null;
|
|
6006
|
+
"lowSampleOverride": boolean;
|
|
6007
|
+
"publishedAt": string | null;
|
|
6008
|
+
"verdict": unknown;
|
|
6009
|
+
"definitionVersionId": string | null;
|
|
6010
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
6011
|
+
"createdAt": string;
|
|
6012
|
+
"updatedAt": string;
|
|
6013
|
+
"liveUntil": string | null;
|
|
6014
|
+
};
|
|
6015
|
+
"shareUrl": string | null;
|
|
6016
|
+
};
|
|
6017
|
+
export type StudyStreamResponse = unknown;
|
|
6018
|
+
export type StudyUpdateTaskRequest = {
|
|
6019
|
+
"prompt": string;
|
|
6020
|
+
"expectations"?: Array<unknown>;
|
|
6021
|
+
};
|
|
6022
|
+
export type StudyUpdateTaskResponse = {
|
|
6023
|
+
"task": {
|
|
6024
|
+
"id": string;
|
|
6025
|
+
"studyId": string;
|
|
6026
|
+
"position": number;
|
|
6027
|
+
"prompt": string;
|
|
6028
|
+
"rationale": string | null;
|
|
6029
|
+
"category": string | null;
|
|
6030
|
+
"requiresWeb": boolean;
|
|
6031
|
+
"rubric": Array<string>;
|
|
6032
|
+
"variants"?: Array<string> | null;
|
|
6033
|
+
"expectations"?: Array<unknown> | null;
|
|
6034
|
+
};
|
|
6035
|
+
};
|
|
6036
|
+
export type StudyUnpublishResponse = {
|
|
6037
|
+
"study": {
|
|
6038
|
+
"id": string;
|
|
6039
|
+
"executiveSummary"?: string | null;
|
|
6040
|
+
"plannedRunCount": number;
|
|
6041
|
+
"organizationId": string;
|
|
6042
|
+
"createdBy": string | null;
|
|
6043
|
+
"name": string;
|
|
6044
|
+
"slug": string;
|
|
6045
|
+
"description": string | null;
|
|
6046
|
+
"skillRef": string | null;
|
|
6047
|
+
"skillSlug": string | null;
|
|
6048
|
+
"skillName": string | null;
|
|
6049
|
+
"skillPromise": string;
|
|
6050
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
6051
|
+
"subjectName": string | null;
|
|
6052
|
+
"subjectLabel": string;
|
|
6053
|
+
"armConfig": {
|
|
6054
|
+
"baseline"?: {
|
|
6055
|
+
"tools"?: "default" | "none" | {
|
|
6056
|
+
"tools": Array<string>;
|
|
6057
|
+
};
|
|
6058
|
+
"addons"?: Record<string, unknown>;
|
|
6059
|
+
};
|
|
6060
|
+
"treatment"?: {
|
|
6061
|
+
"tools"?: "default" | "none" | {
|
|
6062
|
+
"tools": Array<string>;
|
|
6063
|
+
};
|
|
6064
|
+
"addons"?: Record<string, unknown>;
|
|
6065
|
+
};
|
|
6066
|
+
};
|
|
6067
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
6068
|
+
"subjectBackend": "model" | "harness" | null;
|
|
6069
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6070
|
+
"importedFrom"?: string;
|
|
6071
|
+
"baselineLabel": string | null;
|
|
6072
|
+
"treatmentLabel": string | null;
|
|
6073
|
+
"promiseMetrics": Array<{
|
|
6074
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6075
|
+
"direction": "down" | "up";
|
|
6076
|
+
"label": string;
|
|
6077
|
+
}>;
|
|
6078
|
+
"matrix": Array<{
|
|
6079
|
+
"harness": string;
|
|
6080
|
+
"model": string;
|
|
6081
|
+
"provider"?: string;
|
|
6082
|
+
"params"?: {
|
|
6083
|
+
"temperature"?: number;
|
|
6084
|
+
"maxOutputTokens"?: number;
|
|
6085
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
6086
|
+
"thinkingBudgetTokens"?: number;
|
|
6087
|
+
};
|
|
6088
|
+
}>;
|
|
6089
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6090
|
+
"taskCount": number;
|
|
6091
|
+
"variantsPerTask": number;
|
|
6092
|
+
"repeats": number;
|
|
6093
|
+
"subset": {
|
|
6094
|
+
"count": number;
|
|
6095
|
+
"seed": string;
|
|
6096
|
+
"of": number;
|
|
6097
|
+
} | null;
|
|
6098
|
+
"status": string;
|
|
6099
|
+
"error": string | null;
|
|
6100
|
+
"preparation"?: {
|
|
6101
|
+
"since": string;
|
|
6102
|
+
"launchedBy": {
|
|
6103
|
+
"userId"?: string;
|
|
6104
|
+
"role"?: string;
|
|
6105
|
+
};
|
|
6106
|
+
"waitingOn": Array<{
|
|
6107
|
+
"imageKey": string;
|
|
6108
|
+
"buildId": string;
|
|
6109
|
+
"stage"?: "image" | "snapshot";
|
|
6110
|
+
}>;
|
|
6111
|
+
} | null;
|
|
6112
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
6113
|
+
"visibilityChangedAt"?: string | null;
|
|
6114
|
+
"visibilityChangedBy"?: string | null;
|
|
6115
|
+
"shareToken"?: string | null;
|
|
6116
|
+
"lowSampleOverride": boolean;
|
|
6117
|
+
"publishedAt": string | null;
|
|
6118
|
+
"verdict": unknown;
|
|
6119
|
+
"definitionVersionId": string | null;
|
|
6120
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
6121
|
+
"createdAt": string;
|
|
6122
|
+
"updatedAt": string;
|
|
6123
|
+
"liveUntil": string | null;
|
|
6124
|
+
};
|
|
6125
|
+
};
|
|
6126
|
+
export type StudySetVisibilityRequest = {
|
|
6127
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
6128
|
+
};
|
|
6129
|
+
export type StudySetVisibilityResponse = {
|
|
6130
|
+
"study": {
|
|
6131
|
+
"id": string;
|
|
6132
|
+
"executiveSummary"?: string | null;
|
|
6133
|
+
"plannedRunCount": number;
|
|
6134
|
+
"organizationId": string;
|
|
6135
|
+
"createdBy": string | null;
|
|
6136
|
+
"name": string;
|
|
6137
|
+
"slug": string;
|
|
6138
|
+
"description": string | null;
|
|
6139
|
+
"skillRef": string | null;
|
|
6140
|
+
"skillSlug": string | null;
|
|
6141
|
+
"skillName": string | null;
|
|
6142
|
+
"skillPromise": string;
|
|
6143
|
+
"subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
|
|
6144
|
+
"subjectName": string | null;
|
|
6145
|
+
"subjectLabel": string;
|
|
6146
|
+
"armConfig": {
|
|
6147
|
+
"baseline"?: {
|
|
6148
|
+
"tools"?: "default" | "none" | {
|
|
6149
|
+
"tools": Array<string>;
|
|
6150
|
+
};
|
|
6151
|
+
"addons"?: Record<string, unknown>;
|
|
6152
|
+
};
|
|
6153
|
+
"treatment"?: {
|
|
6154
|
+
"tools"?: "default" | "none" | {
|
|
6155
|
+
"tools": Array<string>;
|
|
6156
|
+
};
|
|
6157
|
+
"addons"?: Record<string, unknown>;
|
|
6158
|
+
};
|
|
6159
|
+
};
|
|
6160
|
+
"comparisonMode": "ab_vs_none" | "ab_custom" | "single";
|
|
6161
|
+
"subjectBackend": "model" | "harness" | null;
|
|
6162
|
+
"environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
|
|
6163
|
+
"importedFrom"?: string;
|
|
6164
|
+
"baselineLabel": string | null;
|
|
6165
|
+
"treatmentLabel": string | null;
|
|
6166
|
+
"promiseMetrics": Array<{
|
|
6167
|
+
"metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
|
|
6168
|
+
"direction": "down" | "up";
|
|
6169
|
+
"label": string;
|
|
6170
|
+
}>;
|
|
6171
|
+
"matrix": Array<{
|
|
6172
|
+
"harness": string;
|
|
6173
|
+
"model": string;
|
|
6174
|
+
"provider"?: string;
|
|
6175
|
+
"params"?: {
|
|
6176
|
+
"temperature"?: number;
|
|
6177
|
+
"maxOutputTokens"?: number;
|
|
6178
|
+
"reasoningEffort"?: "low" | "medium" | "high";
|
|
6179
|
+
"thinkingBudgetTokens"?: number;
|
|
6180
|
+
};
|
|
6181
|
+
}>;
|
|
6182
|
+
"lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
|
|
6183
|
+
"taskCount": number;
|
|
6184
|
+
"variantsPerTask": number;
|
|
6185
|
+
"repeats": number;
|
|
6186
|
+
"subset": {
|
|
6187
|
+
"count": number;
|
|
6188
|
+
"seed": string;
|
|
6189
|
+
"of": number;
|
|
6190
|
+
} | null;
|
|
6191
|
+
"status": string;
|
|
6192
|
+
"error": string | null;
|
|
6193
|
+
"preparation"?: {
|
|
6194
|
+
"since": string;
|
|
6195
|
+
"launchedBy": {
|
|
6196
|
+
"userId"?: string;
|
|
6197
|
+
"role"?: string;
|
|
6198
|
+
};
|
|
6199
|
+
"waitingOn": Array<{
|
|
6200
|
+
"imageKey": string;
|
|
6201
|
+
"buildId": string;
|
|
6202
|
+
"stage"?: "image" | "snapshot";
|
|
6203
|
+
}>;
|
|
6204
|
+
} | null;
|
|
6205
|
+
"visibility": "private" | "org" | "link" | "public";
|
|
6206
|
+
"visibilityChangedAt"?: string | null;
|
|
6207
|
+
"visibilityChangedBy"?: string | null;
|
|
6208
|
+
"shareToken"?: string | null;
|
|
6209
|
+
"lowSampleOverride": boolean;
|
|
6210
|
+
"publishedAt": string | null;
|
|
6211
|
+
"verdict": unknown;
|
|
6212
|
+
"definitionVersionId": string | null;
|
|
6213
|
+
"runnerKind": "ora-runs" | "batch-job";
|
|
6214
|
+
"createdAt": string;
|
|
6215
|
+
"updatedAt": string;
|
|
6216
|
+
"liveUntil": string | null;
|
|
6217
|
+
};
|
|
6218
|
+
"shareUrl": string | null;
|
|
6219
|
+
};
|
|
2441
6220
|
export type TunnelsListResponse = {
|
|
2442
6221
|
"tunnels": Array<{
|
|
2443
6222
|
"id": string;
|