@roarkanalytics/sdk 3.12.0 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/client.d.mts +5 -2
- package/client.d.mts.map +1 -1
- package/client.d.ts +5 -2
- package/client.d.ts.map +1 -1
- package/client.js +3 -0
- package/client.js.map +1 -1
- package/client.mjs +3 -0
- package/client.mjs.map +1 -1
- package/package.json +1 -1
- package/resources/benchmark.d.mts +547 -0
- package/resources/benchmark.d.mts.map +1 -0
- package/resources/benchmark.d.ts +547 -0
- package/resources/benchmark.d.ts.map +1 -0
- package/resources/benchmark.js +110 -0
- package/resources/benchmark.js.map +1 -0
- package/resources/benchmark.mjs +106 -0
- package/resources/benchmark.mjs.map +1 -0
- package/resources/config.d.mts +6 -6
- package/resources/config.d.mts.map +1 -1
- package/resources/config.d.ts +6 -6
- package/resources/config.d.ts.map +1 -1
- package/resources/customer-flow-edge-case.d.mts +45 -36
- package/resources/customer-flow-edge-case.d.mts.map +1 -1
- package/resources/customer-flow-edge-case.d.ts +45 -36
- package/resources/customer-flow-edge-case.d.ts.map +1 -1
- package/resources/customer-flow.d.mts +189 -144
- package/resources/customer-flow.d.mts.map +1 -1
- package/resources/customer-flow.d.ts +189 -144
- package/resources/customer-flow.d.ts.map +1 -1
- package/resources/index.d.mts +2 -1
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +2 -1
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +3 -1
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +1 -0
- package/resources/index.mjs.map +1 -1
- package/resources/simulation-environment.d.mts +6 -6
- package/resources/simulation-environment.d.mts.map +1 -1
- package/resources/simulation-environment.d.ts +6 -6
- package/resources/simulation-environment.d.ts.map +1 -1
- package/resources/simulation-job.d.mts +8 -6
- package/resources/simulation-job.d.mts.map +1 -1
- package/resources/simulation-job.d.ts +8 -6
- package/resources/simulation-job.d.ts.map +1 -1
- package/resources/simulation-persona.d.mts +24 -18
- package/resources/simulation-persona.d.mts.map +1 -1
- package/resources/simulation-persona.d.ts +24 -18
- package/resources/simulation-persona.d.ts.map +1 -1
- package/resources/simulation-run-plan-job.d.mts +142 -4
- package/resources/simulation-run-plan-job.d.mts.map +1 -1
- package/resources/simulation-run-plan-job.d.ts +142 -4
- package/resources/simulation-run-plan-job.d.ts.map +1 -1
- package/resources/simulation-run-plan-job.js +16 -0
- package/resources/simulation-run-plan-job.js.map +1 -1
- package/resources/simulation-run-plan-job.mjs +16 -0
- package/resources/simulation-run-plan-job.mjs.map +1 -1
- package/resources/simulation-run-plan.d.mts +38 -0
- package/resources/simulation-run-plan.d.mts.map +1 -1
- package/resources/simulation-run-plan.d.ts +38 -0
- package/resources/simulation-run-plan.d.ts.map +1 -1
- package/resources/simulation.d.mts +9 -0
- package/resources/simulation.d.mts.map +1 -1
- package/resources/simulation.d.ts +9 -0
- package/resources/simulation.d.ts.map +1 -1
- package/src/client.ts +33 -0
- package/src/resources/benchmark.ts +740 -0
- package/src/resources/config.ts +18 -6
- package/src/resources/customer-flow-edge-case.ts +117 -36
- package/src/resources/customer-flow.ts +480 -145
- package/src/resources/index.ts +15 -0
- package/src/resources/simulation-environment.ts +30 -6
- package/src/resources/simulation-job.ts +16 -6
- package/src/resources/simulation-persona.ts +48 -18
- package/src/resources/simulation-run-plan-job.ts +206 -3
- package/src/resources/simulation-run-plan.ts +44 -0
- package/src/resources/simulation.ts +10 -0
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
- package/version.mjs +1 -1
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
import { APIResource } from "../core/resource.mjs";
|
|
2
|
+
import { APIPromise } from "../core/api-promise.mjs";
|
|
3
|
+
import { RequestOptions } from "../internal/request-options.mjs";
|
|
4
|
+
export declare class Benchmark extends APIResource {
|
|
5
|
+
/**
|
|
6
|
+
* Returns one ranked page of a suite version’s published targets, each with the
|
|
7
|
+
* metric cells it is judged on. Ranking happens in the database, so paging through
|
|
8
|
+
* the board is consistent. Every default the server applies (suite version,
|
|
9
|
+
* condition, sort metric, sort direction) is echoed on the response, so a page can
|
|
10
|
+
* be cited without guessing how it was ordered.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* const response = await client.benchmark.getLeaderboard({
|
|
15
|
+
* suite: 'x',
|
|
16
|
+
* });
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
getLeaderboard(query: BenchmarkGetLeaderboardParams, options?: RequestOptions): APIPromise<BenchmarkGetLeaderboardResponse>;
|
|
20
|
+
/**
|
|
21
|
+
* Returns a target’s published sweep and every one of its aggregate cells: the
|
|
22
|
+
* overall rollup plus a set per condition. Defaults to the current sweep; pass
|
|
23
|
+
* `publicationId` from the history endpoint to read a superseded one, which is how
|
|
24
|
+
* a regression is compared generation to generation. Numbers only — no transcript
|
|
25
|
+
* and no audio.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```ts
|
|
29
|
+
* const response = await client.benchmark.getTarget(
|
|
30
|
+
* 'targetKey',
|
|
31
|
+
* { suite: 'x' },
|
|
32
|
+
* );
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
getTarget(targetKey: string, query: BenchmarkGetTargetParams, options?: RequestOptions): APIPromise<BenchmarkGetTargetResponse>;
|
|
36
|
+
/**
|
|
37
|
+
* Returns the metrics a suite version published, each with the direction that
|
|
38
|
+
* counts as better. Use it to pick a valid `sortBy` for the leaderboard: a metric
|
|
39
|
+
* key outside this list is rejected rather than silently ranking every target
|
|
40
|
+
* null.
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* ```ts
|
|
44
|
+
* const response = await client.benchmark.listMetrics({
|
|
45
|
+
* suite: 'x',
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
listMetrics(query: BenchmarkListMetricsParams, options?: RequestOptions): APIPromise<BenchmarkListMetricsResponse>;
|
|
50
|
+
/**
|
|
51
|
+
* Returns every benchmark suite with published results, along with the suite
|
|
52
|
+
* version the other endpoints default to. Start here: the suite name is a required
|
|
53
|
+
* parameter everywhere else and there is no other way to discover it.
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* ```ts
|
|
57
|
+
* const response = await client.benchmark.listSuites();
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
listSuites(options?: RequestOptions): APIPromise<BenchmarkListSuitesResponse>;
|
|
61
|
+
/**
|
|
62
|
+
* Returns every sweep published for a target, newest first, including superseded
|
|
63
|
+
* ones. This is the trend read: it answers "did this model regress?" from Roark’s
|
|
64
|
+
* own published record. Metadata only — pass a row’s `publicationId` to GET
|
|
65
|
+
* /v1/benchmark/target/{targetKey} for that generation’s numbers.
|
|
66
|
+
*
|
|
67
|
+
* @example
|
|
68
|
+
* ```ts
|
|
69
|
+
* const response = await client.benchmark.listTargetHistory(
|
|
70
|
+
* 'targetKey',
|
|
71
|
+
* { suite: 'x' },
|
|
72
|
+
* );
|
|
73
|
+
* ```
|
|
74
|
+
*/
|
|
75
|
+
listTargetHistory(targetKey: string, query: BenchmarkListTargetHistoryParams, options?: RequestOptions): APIPromise<BenchmarkListTargetHistoryResponse>;
|
|
76
|
+
/**
|
|
77
|
+
* Returns the individual scored calls behind a target’s aggregate cells, each with
|
|
78
|
+
* the scorer’s rationale — the "why is this number what it is" read. Filter to one
|
|
79
|
+
* cell with `conditionKey` and `metricKey`. Metrics computed without a rationale
|
|
80
|
+
* (latencies, counts) contribute no samples. Returns reasoning only: no transcript
|
|
81
|
+
* and no audio.
|
|
82
|
+
*
|
|
83
|
+
* @example
|
|
84
|
+
* ```ts
|
|
85
|
+
* const response =
|
|
86
|
+
* await client.benchmark.listTargetScoreSamples(
|
|
87
|
+
* 'targetKey',
|
|
88
|
+
* { suite: 'x' },
|
|
89
|
+
* );
|
|
90
|
+
* ```
|
|
91
|
+
*/
|
|
92
|
+
listTargetScoreSamples(targetKey: string, query: BenchmarkListTargetScoreSamplesParams, options?: RequestOptions): APIPromise<BenchmarkListTargetScoreSamplesResponse>;
|
|
93
|
+
}
|
|
94
|
+
export interface BenchmarkGetLeaderboardResponse {
|
|
95
|
+
conditionKey: string;
|
|
96
|
+
data: Array<BenchmarkGetLeaderboardResponse.Data>;
|
|
97
|
+
pagination: BenchmarkGetLeaderboardResponse.Pagination;
|
|
98
|
+
/**
|
|
99
|
+
* The ranking this page was produced under
|
|
100
|
+
*/
|
|
101
|
+
sortedBy: BenchmarkGetLeaderboardResponse.SortedBy;
|
|
102
|
+
suite: string;
|
|
103
|
+
/**
|
|
104
|
+
* The version actually read, after defaulting.
|
|
105
|
+
*/
|
|
106
|
+
suiteVersion: string;
|
|
107
|
+
}
|
|
108
|
+
export declare namespace BenchmarkGetLeaderboardResponse {
|
|
109
|
+
interface Data {
|
|
110
|
+
/**
|
|
111
|
+
* The projected cells for this row, measured under the requested condition.
|
|
112
|
+
*/
|
|
113
|
+
metrics: Array<Data.Metric>;
|
|
114
|
+
/**
|
|
115
|
+
* A model or stack with a published result on a benchmark suite
|
|
116
|
+
*/
|
|
117
|
+
target: Data.Target;
|
|
118
|
+
}
|
|
119
|
+
namespace Data {
|
|
120
|
+
/**
|
|
121
|
+
* One aggregate cell: a metric measured for one target under one condition
|
|
122
|
+
*/
|
|
123
|
+
interface Metric {
|
|
124
|
+
ciHigh: number | null;
|
|
125
|
+
/**
|
|
126
|
+
* Low bound of the 95% CI of the mean (NUMERIC metrics).
|
|
127
|
+
*/
|
|
128
|
+
ciLow: number | null;
|
|
129
|
+
/**
|
|
130
|
+
* The flow variant these numbers were measured under. The empty string is the
|
|
131
|
+
* overall rollup across all conditions.
|
|
132
|
+
*/
|
|
133
|
+
conditionKey: string;
|
|
134
|
+
conditionLabel: string;
|
|
135
|
+
mean: number | null;
|
|
136
|
+
/**
|
|
137
|
+
* Stable metric id, e.g. `response_time`.
|
|
138
|
+
*/
|
|
139
|
+
metricKey: string;
|
|
140
|
+
/**
|
|
141
|
+
* Which statistics are populated on a result. NUMERIC fills p50/p95/mean/ci;
|
|
142
|
+
* BOOLEAN fills passRate and its interval.
|
|
143
|
+
*/
|
|
144
|
+
metricKind: 'NUMERIC' | 'BOOLEAN';
|
|
145
|
+
metricName: string;
|
|
146
|
+
/**
|
|
147
|
+
* Scored samples behind this aggregate.
|
|
148
|
+
*/
|
|
149
|
+
n: number;
|
|
150
|
+
p50: number | null;
|
|
151
|
+
p95: number | null;
|
|
152
|
+
/**
|
|
153
|
+
* Fraction passed, 0..1 (BOOLEAN metrics).
|
|
154
|
+
*/
|
|
155
|
+
passRate: number | null;
|
|
156
|
+
passRateCiHigh: number | null;
|
|
157
|
+
/**
|
|
158
|
+
* Low bound of the 95% Wilson interval on the pass rate.
|
|
159
|
+
*/
|
|
160
|
+
passRateCiLow: number | null;
|
|
161
|
+
/**
|
|
162
|
+
* Display unit for numeric metrics, e.g. `ms`.
|
|
163
|
+
*/
|
|
164
|
+
unitSymbol: string | null;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* A model or stack with a published result on a benchmark suite
|
|
168
|
+
*/
|
|
169
|
+
interface Target {
|
|
170
|
+
/**
|
|
171
|
+
* Whether these are the numbers Roark publishes for this target today.
|
|
172
|
+
*/
|
|
173
|
+
isCurrent: boolean;
|
|
174
|
+
/**
|
|
175
|
+
* Calls the sweep requested per condition.
|
|
176
|
+
*/
|
|
177
|
+
iterations: number;
|
|
178
|
+
/**
|
|
179
|
+
* Id of this one sweep. Changes every time the target is re-published, so it is a
|
|
180
|
+
* render key, not an identity.
|
|
181
|
+
*/
|
|
182
|
+
publicationId: string;
|
|
183
|
+
/**
|
|
184
|
+
* When this sweep was published (ISO-8601), i.e. data freshness.
|
|
185
|
+
*/
|
|
186
|
+
publishedAt: string;
|
|
187
|
+
/**
|
|
188
|
+
* Executed calls that contributed at least one score to these numbers.
|
|
189
|
+
*/
|
|
190
|
+
sampleCallCount: number;
|
|
191
|
+
suite: string;
|
|
192
|
+
suiteVersion: string;
|
|
193
|
+
/**
|
|
194
|
+
* When a later sweep replaced this one. Null means it is the current generation.
|
|
195
|
+
*/
|
|
196
|
+
supersededAt: string | null;
|
|
197
|
+
/**
|
|
198
|
+
* Stable identity of the model/stack across sweeps. Cite this, and pass it to the
|
|
199
|
+
* target endpoints.
|
|
200
|
+
*/
|
|
201
|
+
targetKey: string;
|
|
202
|
+
/**
|
|
203
|
+
* Human-readable name of the model/stack.
|
|
204
|
+
*/
|
|
205
|
+
targetName: string;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
interface Pagination {
|
|
209
|
+
hasMore: boolean;
|
|
210
|
+
limit: number;
|
|
211
|
+
offset: number;
|
|
212
|
+
/**
|
|
213
|
+
* Total matching records, ignoring this page.
|
|
214
|
+
*/
|
|
215
|
+
total: number;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* The ranking this page was produced under
|
|
219
|
+
*/
|
|
220
|
+
interface SortedBy {
|
|
221
|
+
metricKey: string;
|
|
222
|
+
order: 'asc' | 'desc';
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
export interface BenchmarkGetTargetResponse {
|
|
226
|
+
data: BenchmarkGetTargetResponse.Data;
|
|
227
|
+
}
|
|
228
|
+
export declare namespace BenchmarkGetTargetResponse {
|
|
229
|
+
interface Data {
|
|
230
|
+
/**
|
|
231
|
+
* Every cell of this sweep: the overall rollup plus one set per condition.
|
|
232
|
+
*/
|
|
233
|
+
results: Array<Data.Result>;
|
|
234
|
+
/**
|
|
235
|
+
* A model or stack with a published result on a benchmark suite
|
|
236
|
+
*/
|
|
237
|
+
target: Data.Target;
|
|
238
|
+
}
|
|
239
|
+
namespace Data {
|
|
240
|
+
/**
|
|
241
|
+
* One aggregate cell: a metric measured for one target under one condition
|
|
242
|
+
*/
|
|
243
|
+
interface Result {
|
|
244
|
+
ciHigh: number | null;
|
|
245
|
+
/**
|
|
246
|
+
* Low bound of the 95% CI of the mean (NUMERIC metrics).
|
|
247
|
+
*/
|
|
248
|
+
ciLow: number | null;
|
|
249
|
+
/**
|
|
250
|
+
* The flow variant these numbers were measured under. The empty string is the
|
|
251
|
+
* overall rollup across all conditions.
|
|
252
|
+
*/
|
|
253
|
+
conditionKey: string;
|
|
254
|
+
conditionLabel: string;
|
|
255
|
+
mean: number | null;
|
|
256
|
+
/**
|
|
257
|
+
* Stable metric id, e.g. `response_time`.
|
|
258
|
+
*/
|
|
259
|
+
metricKey: string;
|
|
260
|
+
/**
|
|
261
|
+
* Which statistics are populated on a result. NUMERIC fills p50/p95/mean/ci;
|
|
262
|
+
* BOOLEAN fills passRate and its interval.
|
|
263
|
+
*/
|
|
264
|
+
metricKind: 'NUMERIC' | 'BOOLEAN';
|
|
265
|
+
metricName: string;
|
|
266
|
+
/**
|
|
267
|
+
* Scored samples behind this aggregate.
|
|
268
|
+
*/
|
|
269
|
+
n: number;
|
|
270
|
+
p50: number | null;
|
|
271
|
+
p95: number | null;
|
|
272
|
+
/**
|
|
273
|
+
* Fraction passed, 0..1 (BOOLEAN metrics).
|
|
274
|
+
*/
|
|
275
|
+
passRate: number | null;
|
|
276
|
+
passRateCiHigh: number | null;
|
|
277
|
+
/**
|
|
278
|
+
* Low bound of the 95% Wilson interval on the pass rate.
|
|
279
|
+
*/
|
|
280
|
+
passRateCiLow: number | null;
|
|
281
|
+
/**
|
|
282
|
+
* Display unit for numeric metrics, e.g. `ms`.
|
|
283
|
+
*/
|
|
284
|
+
unitSymbol: string | null;
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* A model or stack with a published result on a benchmark suite
|
|
288
|
+
*/
|
|
289
|
+
interface Target {
|
|
290
|
+
/**
|
|
291
|
+
* Whether these are the numbers Roark publishes for this target today.
|
|
292
|
+
*/
|
|
293
|
+
isCurrent: boolean;
|
|
294
|
+
/**
|
|
295
|
+
* Calls the sweep requested per condition.
|
|
296
|
+
*/
|
|
297
|
+
iterations: number;
|
|
298
|
+
/**
|
|
299
|
+
* Id of this one sweep. Changes every time the target is re-published, so it is a
|
|
300
|
+
* render key, not an identity.
|
|
301
|
+
*/
|
|
302
|
+
publicationId: string;
|
|
303
|
+
/**
|
|
304
|
+
* When this sweep was published (ISO-8601), i.e. data freshness.
|
|
305
|
+
*/
|
|
306
|
+
publishedAt: string;
|
|
307
|
+
/**
|
|
308
|
+
* Executed calls that contributed at least one score to these numbers.
|
|
309
|
+
*/
|
|
310
|
+
sampleCallCount: number;
|
|
311
|
+
suite: string;
|
|
312
|
+
suiteVersion: string;
|
|
313
|
+
/**
|
|
314
|
+
* When a later sweep replaced this one. Null means it is the current generation.
|
|
315
|
+
*/
|
|
316
|
+
supersededAt: string | null;
|
|
317
|
+
/**
|
|
318
|
+
* Stable identity of the model/stack across sweeps. Cite this, and pass it to the
|
|
319
|
+
* target endpoints.
|
|
320
|
+
*/
|
|
321
|
+
targetKey: string;
|
|
322
|
+
/**
|
|
323
|
+
* Human-readable name of the model/stack.
|
|
324
|
+
*/
|
|
325
|
+
targetName: string;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
export interface BenchmarkListMetricsResponse {
|
|
330
|
+
data: Array<BenchmarkListMetricsResponse.Data>;
|
|
331
|
+
suite: string;
|
|
332
|
+
suiteVersion: string;
|
|
333
|
+
}
|
|
334
|
+
export declare namespace BenchmarkListMetricsResponse {
|
|
335
|
+
/**
|
|
336
|
+
* A metric a suite version published, and the direction that counts as better
|
|
337
|
+
*/
|
|
338
|
+
interface Data {
|
|
339
|
+
/**
|
|
340
|
+
* Which end of the scale is better. `higher` and `lower` can be ranked; `neutral`
|
|
341
|
+
* is an observational metric that describes a conversation rather than grading it,
|
|
342
|
+
* so sorting on it names no winner.
|
|
343
|
+
*/
|
|
344
|
+
goal: 'higher' | 'lower' | 'neutral';
|
|
345
|
+
metricKey: string;
|
|
346
|
+
/**
|
|
347
|
+
* Which statistics are populated on a result. NUMERIC fills p50/p95/mean/ci;
|
|
348
|
+
* BOOLEAN fills passRate and its interval.
|
|
349
|
+
*/
|
|
350
|
+
metricKind: 'NUMERIC' | 'BOOLEAN';
|
|
351
|
+
metricName: string;
|
|
352
|
+
unitSymbol: string | null;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
export interface BenchmarkListSuitesResponse {
|
|
356
|
+
data: Array<BenchmarkListSuitesResponse.Data>;
|
|
357
|
+
}
|
|
358
|
+
export declare namespace BenchmarkListSuitesResponse {
|
|
359
|
+
/**
|
|
360
|
+
* A benchmark suite with published results
|
|
361
|
+
*/
|
|
362
|
+
interface Data {
|
|
363
|
+
latestPublishedAt: string;
|
|
364
|
+
/**
|
|
365
|
+
* The version every endpoint defaults to for this suite.
|
|
366
|
+
*/
|
|
367
|
+
latestSuiteVersion: string;
|
|
368
|
+
suite: string;
|
|
369
|
+
/**
|
|
370
|
+
* Targets currently published under the latest version.
|
|
371
|
+
*/
|
|
372
|
+
targetCount: number;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
export interface BenchmarkListTargetHistoryResponse {
|
|
376
|
+
data: Array<BenchmarkListTargetHistoryResponse.Data>;
|
|
377
|
+
pagination: BenchmarkListTargetHistoryResponse.Pagination;
|
|
378
|
+
}
|
|
379
|
+
export declare namespace BenchmarkListTargetHistoryResponse {
|
|
380
|
+
/**
|
|
381
|
+
* One published sweep of a target, current or superseded — an entry in its history
|
|
382
|
+
*/
|
|
383
|
+
interface Data {
|
|
384
|
+
/**
|
|
385
|
+
* Whether this is the generation the other endpoints return.
|
|
386
|
+
*/
|
|
387
|
+
isCurrent: boolean;
|
|
388
|
+
iterations: number;
|
|
389
|
+
publicationId: string;
|
|
390
|
+
publishedAt: string;
|
|
391
|
+
sampleCallCount: number;
|
|
392
|
+
suite: string;
|
|
393
|
+
suiteVersion: string;
|
|
394
|
+
/**
|
|
395
|
+
* When a later sweep replaced this generation. Null means it is the current one.
|
|
396
|
+
*/
|
|
397
|
+
supersededAt: string | null;
|
|
398
|
+
targetKey: string;
|
|
399
|
+
targetName: string;
|
|
400
|
+
}
|
|
401
|
+
interface Pagination {
|
|
402
|
+
hasMore: boolean;
|
|
403
|
+
limit: number;
|
|
404
|
+
offset: number;
|
|
405
|
+
/**
|
|
406
|
+
* Total matching records, ignoring this page.
|
|
407
|
+
*/
|
|
408
|
+
total: number;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
export interface BenchmarkListTargetScoreSamplesResponse {
|
|
412
|
+
data: Array<BenchmarkListTargetScoreSamplesResponse.Data>;
|
|
413
|
+
pagination: BenchmarkListTargetScoreSamplesResponse.Pagination;
|
|
414
|
+
/**
|
|
415
|
+
* A model or stack with a published result on a benchmark suite
|
|
416
|
+
*/
|
|
417
|
+
target: BenchmarkListTargetScoreSamplesResponse.Target;
|
|
418
|
+
}
|
|
419
|
+
export declare namespace BenchmarkListTargetScoreSamplesResponse {
|
|
420
|
+
/**
|
|
421
|
+
* One un-aggregated score behind a result cell, with the reasoning that produced
|
|
422
|
+
* it
|
|
423
|
+
*/
|
|
424
|
+
interface Data {
|
|
425
|
+
booleanValue: boolean | null;
|
|
426
|
+
conditionKey: string;
|
|
427
|
+
metricKey: string;
|
|
428
|
+
numericValue: number | null;
|
|
429
|
+
/**
|
|
430
|
+
* The scorer's rationale for this individual score.
|
|
431
|
+
*/
|
|
432
|
+
reasoning: string;
|
|
433
|
+
/**
|
|
434
|
+
* Opaque id of the call this single score came from. Groups samples that share a
|
|
435
|
+
* call; it is not resolvable through this API.
|
|
436
|
+
*/
|
|
437
|
+
sourceCallId: string;
|
|
438
|
+
}
|
|
439
|
+
interface Pagination {
|
|
440
|
+
hasMore: boolean;
|
|
441
|
+
limit: number;
|
|
442
|
+
offset: number;
|
|
443
|
+
/**
|
|
444
|
+
* Total matching records, ignoring this page.
|
|
445
|
+
*/
|
|
446
|
+
total: number;
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* A model or stack with a published result on a benchmark suite
|
|
450
|
+
*/
|
|
451
|
+
interface Target {
|
|
452
|
+
/**
|
|
453
|
+
* Whether these are the numbers Roark publishes for this target today.
|
|
454
|
+
*/
|
|
455
|
+
isCurrent: boolean;
|
|
456
|
+
/**
|
|
457
|
+
* Calls the sweep requested per condition.
|
|
458
|
+
*/
|
|
459
|
+
iterations: number;
|
|
460
|
+
/**
|
|
461
|
+
* Id of this one sweep. Changes every time the target is re-published, so it is a
|
|
462
|
+
* render key, not an identity.
|
|
463
|
+
*/
|
|
464
|
+
publicationId: string;
|
|
465
|
+
/**
|
|
466
|
+
* When this sweep was published (ISO-8601), i.e. data freshness.
|
|
467
|
+
*/
|
|
468
|
+
publishedAt: string;
|
|
469
|
+
/**
|
|
470
|
+
* Executed calls that contributed at least one score to these numbers.
|
|
471
|
+
*/
|
|
472
|
+
sampleCallCount: number;
|
|
473
|
+
suite: string;
|
|
474
|
+
suiteVersion: string;
|
|
475
|
+
/**
|
|
476
|
+
* When a later sweep replaced this one. Null means it is the current generation.
|
|
477
|
+
*/
|
|
478
|
+
supersededAt: string | null;
|
|
479
|
+
/**
|
|
480
|
+
* Stable identity of the model/stack across sweeps. Cite this, and pass it to the
|
|
481
|
+
* target endpoints.
|
|
482
|
+
*/
|
|
483
|
+
targetKey: string;
|
|
484
|
+
/**
|
|
485
|
+
* Human-readable name of the model/stack.
|
|
486
|
+
*/
|
|
487
|
+
targetName: string;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
export interface BenchmarkGetLeaderboardParams {
|
|
491
|
+
suite: string;
|
|
492
|
+
conditionKey?: string;
|
|
493
|
+
/**
|
|
494
|
+
* Maximum number of records to return (default: 20, max: 100)
|
|
495
|
+
*/
|
|
496
|
+
limit?: number;
|
|
497
|
+
/**
|
|
498
|
+
* Comma-separated metric keys to project per row. Defaults to the headline set.
|
|
499
|
+
*/
|
|
500
|
+
metrics?: string;
|
|
501
|
+
/**
|
|
502
|
+
* Pagination offset
|
|
503
|
+
*/
|
|
504
|
+
offset?: number;
|
|
505
|
+
order?: 'asc' | 'desc';
|
|
506
|
+
sortBy?: string;
|
|
507
|
+
suiteVersion?: string;
|
|
508
|
+
}
|
|
509
|
+
export interface BenchmarkGetTargetParams {
|
|
510
|
+
suite: string;
|
|
511
|
+
publicationId?: string;
|
|
512
|
+
suiteVersion?: string;
|
|
513
|
+
}
|
|
514
|
+
export interface BenchmarkListMetricsParams {
|
|
515
|
+
suite: string;
|
|
516
|
+
suiteVersion?: string;
|
|
517
|
+
}
|
|
518
|
+
export interface BenchmarkListTargetHistoryParams {
|
|
519
|
+
suite: string;
|
|
520
|
+
/**
|
|
521
|
+
* Maximum number of records to return (default: 20, max: 100)
|
|
522
|
+
*/
|
|
523
|
+
limit?: number;
|
|
524
|
+
/**
|
|
525
|
+
* Pagination offset
|
|
526
|
+
*/
|
|
527
|
+
offset?: number;
|
|
528
|
+
suiteVersion?: string;
|
|
529
|
+
}
|
|
530
|
+
export interface BenchmarkListTargetScoreSamplesParams {
|
|
531
|
+
suite: string;
|
|
532
|
+
conditionKey?: string;
|
|
533
|
+
/**
|
|
534
|
+
* Maximum number of records to return (default: 20, max: 100)
|
|
535
|
+
*/
|
|
536
|
+
limit?: number;
|
|
537
|
+
metricKey?: string;
|
|
538
|
+
/**
|
|
539
|
+
* Pagination offset
|
|
540
|
+
*/
|
|
541
|
+
offset?: number;
|
|
542
|
+
suiteVersion?: string;
|
|
543
|
+
}
|
|
544
|
+
export declare namespace Benchmark {
|
|
545
|
+
export { type BenchmarkGetLeaderboardResponse as BenchmarkGetLeaderboardResponse, type BenchmarkGetTargetResponse as BenchmarkGetTargetResponse, type BenchmarkListMetricsResponse as BenchmarkListMetricsResponse, type BenchmarkListSuitesResponse as BenchmarkListSuitesResponse, type BenchmarkListTargetHistoryResponse as BenchmarkListTargetHistoryResponse, type BenchmarkListTargetScoreSamplesResponse as BenchmarkListTargetScoreSamplesResponse, type BenchmarkGetLeaderboardParams as BenchmarkGetLeaderboardParams, type BenchmarkGetTargetParams as BenchmarkGetTargetParams, type BenchmarkListMetricsParams as BenchmarkListMetricsParams, type BenchmarkListTargetHistoryParams as BenchmarkListTargetHistoryParams, type BenchmarkListTargetScoreSamplesParams as BenchmarkListTargetScoreSamplesParams, };
|
|
546
|
+
}
|
|
547
|
+
//# sourceMappingURL=benchmark.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmark.d.mts","sourceRoot":"","sources":["../src/resources/benchmark.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,6BAAyB;AAC/C,OAAO,EAAE,UAAU,EAAE,gCAA4B;AACjD,OAAO,EAAE,cAAc,EAAE,wCAAoC;AAG7D,qBAAa,SAAU,SAAQ,WAAW;IACxC;;;;;;;;;;;;;OAaG;IACH,cAAc,CACZ,KAAK,EAAE,6BAA6B,EACpC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,+BAA+B,CAAC;IAI9C;;;;;;;;;;;;;;OAcG;IACH,SAAS,CACP,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,wBAAwB,EAC/B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,0BAA0B,CAAC;IAIzC;;;;;;;;;;;;OAYG;IACH,WAAW,CACT,KAAK,EAAE,0BAA0B,EACjC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,4BAA4B,CAAC;IAI3C;;;;;;;;;OASG;IACH,UAAU,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,2BAA2B,CAAC;IAI7E;;;;;;;;;;;;;OAaG;IACH,iBAAiB,CACf,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,gCAAgC,EACvC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,kCAAkC,CAAC;IAIjD;;;;;;;;;;;;;;;OAeG;IACH,sBAAsB,CACpB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,qCAAqC,EAC5C,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,uCAAuC,CAAC;CAGvD;AAED,MAAM,WAAW,+BAA+B;IAC9C,YAAY,EAAE,MAAM,CAAC;IAErB,IAAI,EAAE,KAAK,CAAC,+BAA+B,CAAC,IAAI,CAAC,CAAC;IAElD,UAAU,EAAE,+BAA+B,CAAC,UAAU,CAAC;IAEvD;;OAEG;IACH,QAAQ,EAAE,+BAA+B,CAAC,QAAQ,CAAC;IAEnD,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,yBAAiB,+BAA+B,CAAC;IAC/C,UAAiB,IAAI;QACnB;;WAEG;QACH,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5B;;WAEG;QACH,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;KACrB;IAED,UAAiB,IAAI,CAAC;QACpB;;WAEG;QACH,UAAiB,MAAM;YACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;YAEtB;;eAEG;YACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;YAErB;;;eAGG;YACH,YAAY,EAAE,MAAM,CAAC;YAErB,cAAc,EAAE,MAAM,CAAC;YAEvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;YAEpB;;eAEG;YACH,SAAS,EAAE,MAAM,CAAC;YAElB;;;eAGG;YACH,UAAU,EAAE,SAAS,GAAG,SAAS,CAAC;YAElC,UAAU,EAAE,MAAM,CAAC;YAEnB;;eAEG;YACH,CAAC,EAAE,MAAM,CAAC;YAEV,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;YAEnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;YAEnB;;eAEG;YACH,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;YAExB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;YAE9B;;eAEG;YACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;YAE7B;;eAEG;YACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;SAC3B;QAED;;WAEG;QACH,UAAiB,MAAM;YACrB;;eAEG;YACH,SAAS,EAAE,OAAO,CAAC;YAEnB;;eAEG;YACH,UAAU,EAAE,MAAM,CAAC;YAEnB;;;eAGG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,WAAW,EAAE,MAAM,CAAC;YAEpB;;eAEG;YACH,eAAe,EAAE,MAAM,CAAC;YAExB,KAAK,EAAE,MAAM,CAAC;YAEd,YAAY,EAAE,MAAM,CAAC;YAErB;;eAEG;YACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;YAE5B;;;eAGG;YACH,SAAS,EAAE,MAAM,CAAC;YAElB;;eAEG;YACH,UAAU,EAAE,MAAM,CAAC;SACpB;KACF;IAED,UAAiB,UAAU;QACzB,OAAO,EAAE,OAAO,CAAC;QAEjB,KAAK,EAAE,MAAM,CAAC;QAEd,MAAM,EAAE,MAAM,CAAC;QAEf;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;KACf;IAED;;OAEG;IACH,UAAiB,QAAQ;QACvB,SAAS,EAAE,MAAM,CAAC;QAElB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;KACvB;CACF;AAED,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,0BAA0B,CAAC,IAAI,CAAC;CACvC;AAED,yBAAiB,0BAA0B,CAAC;IAC1C,UAAiB,IAAI;QACnB;;WAEG;QACH,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5B;;WAEG;QACH,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;KACrB;IAED,UAAiB,IAAI,CAAC;QACpB;;WAEG;QACH,UAAiB,MAAM;YACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;YAEtB;;eAEG;YACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;YAErB;;;eAGG;YACH,YAAY,EAAE,MAAM,CAAC;YAErB,cAAc,EAAE,MAAM,CAAC;YAEvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;YAEpB;;eAEG;YACH,SAAS,EAAE,MAAM,CAAC;YAElB;;;eAGG;YACH,UAAU,EAAE,SAAS,GAAG,SAAS,CAAC;YAElC,UAAU,EAAE,MAAM,CAAC;YAEnB;;eAEG;YACH,CAAC,EAAE,MAAM,CAAC;YAEV,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;YAEnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;YAEnB;;eAEG;YACH,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;YAExB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;YAE9B;;eAEG;YACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;YAE7B;;eAEG;YACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;SAC3B;QAED;;WAEG;QACH,UAAiB,MAAM;YACrB;;eAEG;YACH,SAAS,EAAE,OAAO,CAAC;YAEnB;;eAEG;YACH,UAAU,EAAE,MAAM,CAAC;YAEnB;;;eAGG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,WAAW,EAAE,MAAM,CAAC;YAEpB;;eAEG;YACH,eAAe,EAAE,MAAM,CAAC;YAExB,KAAK,EAAE,MAAM,CAAC;YAEd,YAAY,EAAE,MAAM,CAAC;YAErB;;eAEG;YACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;YAE5B;;;eAGG;YACH,SAAS,EAAE,MAAM,CAAC;YAElB;;eAEG;YACH,UAAU,EAAE,MAAM,CAAC;SACpB;KACF;CACF;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,KAAK,CAAC,4BAA4B,CAAC,IAAI,CAAC,CAAC;IAE/C,KAAK,EAAE,MAAM,CAAC;IAEd,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,yBAAiB,4BAA4B,CAAC;IAC5C;;OAEG;IACH,UAAiB,IAAI;QACnB;;;;WAIG;QACH,IAAI,EAAE,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;QAErC,SAAS,EAAE,MAAM,CAAC;QAElB;;;WAGG;QACH,UAAU,EAAE,SAAS,GAAG,SAAS,CAAC;QAElC,UAAU,EAAE,MAAM,CAAC;QAEnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;KAC3B;CACF;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,KAAK,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;CAC/C;AAED,yBAAiB,2BAA2B,CAAC;IAC3C;;OAEG;IACH,UAAiB,IAAI;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,kBAAkB,EAAE,MAAM,CAAC;QAE3B,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,EAAE,KAAK,CAAC,kCAAkC,CAAC,IAAI,CAAC,CAAC;IAErD,UAAU,EAAE,kCAAkC,CAAC,UAAU,CAAC;CAC3D;AAED,yBAAiB,kCAAkC,CAAC;IAClD;;OAEG;IACH,UAAiB,IAAI;QACnB;;WAEG;QACH,SAAS,EAAE,OAAO,CAAC;QAEnB,UAAU,EAAE,MAAM,CAAC;QAEnB,aAAa,EAAE,MAAM,CAAC;QAEtB,WAAW,EAAE,MAAM,CAAC;QAEpB,eAAe,EAAE,MAAM,CAAC;QAExB,KAAK,EAAE,MAAM,CAAC;QAEd,YAAY,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B,SAAS,EAAE,MAAM,CAAC;QAElB,UAAU,EAAE,MAAM,CAAC;KACpB;IAED,UAAiB,UAAU;QACzB,OAAO,EAAE,OAAO,CAAC;QAEjB,KAAK,EAAE,MAAM,CAAC;QAEd,MAAM,EAAE,MAAM,CAAC;QAEf;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;KACf;CACF;AAED,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,KAAK,CAAC,uCAAuC,CAAC,IAAI,CAAC,CAAC;IAE1D,UAAU,EAAE,uCAAuC,CAAC,UAAU,CAAC;IAE/D;;OAEG;IACH,MAAM,EAAE,uCAAuC,CAAC,MAAM,CAAC;CACxD;AAED,yBAAiB,uCAAuC,CAAC;IACvD;;;OAGG;IACH,UAAiB,IAAI;QACnB,YAAY,EAAE,OAAO,GAAG,IAAI,CAAC;QAE7B,YAAY,EAAE,MAAM,CAAC;QAErB,SAAS,EAAE,MAAM,CAAC;QAElB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;;WAGG;QACH,YAAY,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,UAAU;QACzB,OAAO,EAAE,OAAO,CAAC;QAEjB,KAAK,EAAE,MAAM,CAAC;QAEd,MAAM,EAAE,MAAM,CAAC;QAEf;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;KACf;IAED;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,SAAS,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,UAAU,EAAE,MAAM,CAAC;QAEnB;;;WAGG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAC;QAExB,KAAK,EAAE,MAAM,CAAC;QAEd,YAAY,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;;WAGG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,MAAM,CAAC;KACpB;CACF;AAED,MAAM,WAAW,6BAA6B;IAC5C,KAAK,EAAE,MAAM,CAAC;IAEd,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAEvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IAEd,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IAEd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gCAAgC;IAC/C,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,qCAAqC;IACpD,KAAK,EAAE,MAAM,CAAC;IAEd,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,CAAC,OAAO,WAAW,SAAS,CAAC;IACjC,OAAO,EACL,KAAK,+BAA+B,IAAI,+BAA+B,EACvE,KAAK,0BAA0B,IAAI,0BAA0B,EAC7D,KAAK,4BAA4B,IAAI,4BAA4B,EACjE,KAAK,2BAA2B,IAAI,2BAA2B,EAC/D,KAAK,kCAAkC,IAAI,kCAAkC,EAC7E,KAAK,uCAAuC,IAAI,uCAAuC,EACvF,KAAK,6BAA6B,IAAI,6BAA6B,EACnE,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,0BAA0B,IAAI,0BAA0B,EAC7D,KAAK,gCAAgC,IAAI,gCAAgC,EACzE,KAAK,qCAAqC,IAAI,qCAAqC,GACpF,CAAC;CACH"}
|