@benchsdk/client 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/dist/index.cjs +15 -945
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -756
- package/dist/index.d.ts +7 -756
- package/dist/index.js +15 -929
- package/dist/index.js.map +1 -1
- package/package.json +5 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,760 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
/** API base URL. Defaults to https://platform.computesdk.com/api/v1. */
|
|
7
|
-
baseUrl?: string;
|
|
8
|
-
/** Bearer token. Defaults to process.env.COMPUTESDK_ADMIN_API_KEY, then process.env.COMPUTESDK_API_KEY. */
|
|
9
|
-
apiKey?: string;
|
|
10
|
-
/** Custom fetch implementation, mostly useful for tests. */
|
|
11
|
-
fetch?: typeof fetch;
|
|
12
|
-
}
|
|
13
|
-
interface BenchmarkResource {
|
|
14
|
-
id: string;
|
|
15
|
-
slug: string;
|
|
16
|
-
name: string;
|
|
17
|
-
status?: string;
|
|
18
|
-
config?: JsonObject;
|
|
19
|
-
defaultRunConfig?: JsonObject;
|
|
20
|
-
}
|
|
21
|
-
type BenchmarkRunStatus = 'planned' | 'in_progress' | 'completed' | 'failed';
|
|
22
|
-
type BenchmarkWorkerStatus = 'pending' | 'running' | 'completed' | 'failed';
|
|
23
|
-
interface BenchmarkRun {
|
|
24
|
-
id: string;
|
|
25
|
-
benchmarkId: string;
|
|
26
|
-
name?: string | null;
|
|
27
|
-
status: BenchmarkRunStatus | string;
|
|
28
|
-
/** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */
|
|
29
|
-
runKey?: string | null;
|
|
30
|
-
totalTasks: number;
|
|
31
|
-
/** The run declared no size: `totalTasks` is the sum of what its participants declare. */
|
|
32
|
-
participantSized?: boolean;
|
|
33
|
-
workerCount: number;
|
|
34
|
-
config?: JsonObject;
|
|
35
|
-
createdAt?: string;
|
|
36
|
-
updatedAt?: string;
|
|
37
|
-
}
|
|
38
|
-
interface BenchmarkParticipant {
|
|
39
|
-
id: string;
|
|
40
|
-
benchmarkId: string;
|
|
41
|
-
runId: string;
|
|
42
|
-
slug: string;
|
|
43
|
-
label?: string | null;
|
|
44
|
-
provider?: string | null;
|
|
45
|
-
status: BenchmarkRunStatus | string;
|
|
46
|
-
totalTasks: number;
|
|
47
|
-
workerCount: number;
|
|
48
|
-
config?: JsonObject;
|
|
49
|
-
}
|
|
50
|
-
interface BenchmarkRunWorker {
|
|
51
|
-
id: string;
|
|
52
|
-
benchmarkId: string;
|
|
53
|
-
runId: string;
|
|
54
|
-
participantId: string;
|
|
55
|
-
workerIndex: number;
|
|
56
|
-
workerCount: number;
|
|
57
|
-
taskIndexStart: number;
|
|
58
|
-
taskIndexEnd: number;
|
|
59
|
-
targetConcurrency: number;
|
|
60
|
-
status: BenchmarkWorkerStatus | string;
|
|
61
|
-
progressDone?: number;
|
|
62
|
-
progressInFlight?: number;
|
|
63
|
-
progressErrors?: number;
|
|
64
|
-
progressTotal?: number;
|
|
65
|
-
currentStep?: string | null;
|
|
66
|
-
concurrency?: WorkerConcurrencySample[];
|
|
67
|
-
}
|
|
68
|
-
interface BenchmarkWorkerAttempt {
|
|
69
|
-
id: string;
|
|
70
|
-
benchmarkId: string;
|
|
71
|
-
runId: string;
|
|
72
|
-
participantId: string;
|
|
73
|
-
workerId: string;
|
|
74
|
-
attemptNumber: number;
|
|
75
|
-
status: string;
|
|
76
|
-
}
|
|
77
|
-
interface BenchmarkAssignment {
|
|
78
|
-
benchmarkId: string;
|
|
79
|
-
benchmarkSlug: string;
|
|
80
|
-
runId: string;
|
|
81
|
-
participantId: string;
|
|
82
|
-
participantSlug: string;
|
|
83
|
-
provider?: string | null;
|
|
84
|
-
workerId: string;
|
|
85
|
-
workerIndex: number;
|
|
86
|
-
workerCount: number;
|
|
87
|
-
attemptId: string;
|
|
88
|
-
attemptNumber: number;
|
|
89
|
-
taskRange: {
|
|
90
|
-
start: number;
|
|
91
|
-
end: number;
|
|
92
|
-
count: number;
|
|
93
|
-
};
|
|
94
|
-
targetConcurrency: number;
|
|
95
|
-
config?: JsonObject;
|
|
96
|
-
}
|
|
97
|
-
interface UpsertBenchmarkInput {
|
|
98
|
-
name: string;
|
|
99
|
-
status?: string;
|
|
100
|
-
config?: JsonObject;
|
|
101
|
-
defaultRunConfig?: JsonObject;
|
|
102
|
-
}
|
|
103
|
-
interface UpdateBenchmarkInput {
|
|
104
|
-
name?: string;
|
|
105
|
-
status?: string;
|
|
106
|
-
config?: JsonObject;
|
|
107
|
-
defaultRunConfig?: JsonObject;
|
|
108
|
-
}
|
|
109
|
-
interface CreateRunInput {
|
|
110
|
-
/**
|
|
111
|
-
* Idempotency key for get-or-create: sibling callers passing the same key
|
|
112
|
-
* (per org + benchmark) converge on one run instead of each opening its own.
|
|
113
|
-
*/
|
|
114
|
-
runKey?: string;
|
|
115
|
-
/** Omit to open a participant-sized run: each participant declares its own size when it registers. */
|
|
116
|
-
totalTasks?: number;
|
|
117
|
-
workerCount?: number;
|
|
118
|
-
participants?: string[];
|
|
119
|
-
config?: JsonObject;
|
|
120
|
-
}
|
|
121
|
-
interface UpdateRunInput {
|
|
122
|
-
status?: BenchmarkRunStatus;
|
|
123
|
-
config?: JsonObject;
|
|
124
|
-
}
|
|
125
|
-
interface UpsertParticipantInput {
|
|
126
|
-
label?: string;
|
|
127
|
-
provider?: string;
|
|
128
|
-
status?: string;
|
|
129
|
-
totalTasks?: number;
|
|
130
|
-
workerCount?: number;
|
|
131
|
-
config?: JsonObject;
|
|
132
|
-
}
|
|
133
|
-
type UpdateParticipantInput = UpsertParticipantInput;
|
|
134
|
-
interface UpdateWorkerInput {
|
|
135
|
-
status?: BenchmarkWorkerStatus;
|
|
136
|
-
progressDone?: number;
|
|
137
|
-
progressInFlight?: number;
|
|
138
|
-
progressErrors?: number;
|
|
139
|
-
progressTotal?: number;
|
|
140
|
-
}
|
|
141
|
-
interface ClaimWorkerInput {
|
|
142
|
-
processKind?: string;
|
|
143
|
-
processKey?: string;
|
|
144
|
-
}
|
|
145
|
-
interface PlanWorkersInput {
|
|
146
|
-
workerCount?: number;
|
|
147
|
-
targetConcurrency?: number;
|
|
148
|
-
config?: JsonObject;
|
|
149
|
-
}
|
|
150
|
-
interface TaskResultRecord {
|
|
151
|
-
taskIndex: number;
|
|
152
|
-
status: string;
|
|
153
|
-
startedAt?: string;
|
|
154
|
-
completedAt?: string;
|
|
155
|
-
latencyMs?: number;
|
|
156
|
-
firstCommandMs?: number | null;
|
|
157
|
-
errorCode?: string | null;
|
|
158
|
-
steps?: TaskStepRecord[];
|
|
159
|
-
data?: JsonObject;
|
|
160
|
-
}
|
|
161
|
-
interface TaskStepRecord {
|
|
162
|
-
name: string;
|
|
163
|
-
status: 'success' | 'error';
|
|
164
|
-
startedAt?: string;
|
|
165
|
-
completedAt?: string;
|
|
166
|
-
latencyMs?: number;
|
|
167
|
-
errorCode?: string | null;
|
|
168
|
-
data?: JsonObject;
|
|
169
|
-
/** Number of parallel invocations requested for this step. */
|
|
170
|
-
concurrency?: number;
|
|
171
|
-
/** Per-iteration timeout in milliseconds applied to this step. */
|
|
172
|
-
timeoutMs?: number;
|
|
173
|
-
}
|
|
174
|
-
interface SendTaskResultsInput {
|
|
175
|
-
benchmarkSlug: string;
|
|
176
|
-
runId: string;
|
|
177
|
-
workerId: string;
|
|
178
|
-
attemptId: string;
|
|
179
|
-
sequenceNumber: number;
|
|
180
|
-
isFinal: boolean;
|
|
181
|
-
records: TaskResultRecord[];
|
|
182
|
-
}
|
|
183
|
-
interface TaskResultsResponse {
|
|
184
|
-
accepted?: number;
|
|
185
|
-
eventBatchId?: string;
|
|
186
|
-
queued?: boolean;
|
|
187
|
-
eventBatch?: unknown;
|
|
188
|
-
duplicate?: boolean;
|
|
189
|
-
queueMessageId?: string;
|
|
190
|
-
}
|
|
191
|
-
interface CreateWorkerArtifactInput {
|
|
192
|
-
attemptId: string;
|
|
193
|
-
kind: string;
|
|
194
|
-
contentType?: string;
|
|
195
|
-
name?: string;
|
|
196
|
-
metadata?: JsonObject;
|
|
197
|
-
}
|
|
198
|
-
interface UploadWorkerArtifactInput extends CreateWorkerArtifactInput {
|
|
199
|
-
body: BodyInit;
|
|
200
|
-
}
|
|
201
|
-
interface BenchmarkArtifact {
|
|
202
|
-
id?: string;
|
|
203
|
-
artifactId?: string;
|
|
204
|
-
benchmarkId?: string;
|
|
205
|
-
runId?: string;
|
|
206
|
-
participantId?: string;
|
|
207
|
-
participantSlug?: string;
|
|
208
|
-
workerId?: string;
|
|
209
|
-
attemptId?: string;
|
|
210
|
-
kind: string;
|
|
211
|
-
name?: string | null;
|
|
212
|
-
contentType?: string | null;
|
|
213
|
-
objectKey?: string;
|
|
214
|
-
uploadUrl?: string;
|
|
215
|
-
uploadUrlExpiresAt?: string;
|
|
216
|
-
metadata?: JsonObject;
|
|
217
|
-
createdAt?: string;
|
|
218
|
-
}
|
|
219
|
-
interface CreateWorkerArtifactResponse {
|
|
220
|
-
artifact?: BenchmarkArtifact;
|
|
221
|
-
artifactId?: string;
|
|
222
|
-
uploadUrl?: string;
|
|
223
|
-
uploadUrlExpiresAt?: string;
|
|
224
|
-
objectKey?: string;
|
|
225
|
-
}
|
|
226
|
-
interface BenchmarkResultLatencySummary {
|
|
227
|
-
min: number | null;
|
|
228
|
-
avg: number | null;
|
|
229
|
-
p50: number | null;
|
|
230
|
-
p95: number | null;
|
|
231
|
-
p99: number | null;
|
|
232
|
-
max: number | null;
|
|
233
|
-
}
|
|
234
|
-
interface BenchmarkResultSummary {
|
|
235
|
-
taskCount: number;
|
|
236
|
-
successCount: number;
|
|
237
|
-
errorCount: number;
|
|
238
|
-
otherCount: number;
|
|
239
|
-
latencyCount: number;
|
|
240
|
-
successRate: number;
|
|
241
|
-
latencyMs: BenchmarkResultLatencySummary;
|
|
242
|
-
firstStartedAt: string | null;
|
|
243
|
-
lastCompletedAt: string | null;
|
|
244
|
-
}
|
|
245
|
-
interface BenchmarkParticipantResultSummary extends BenchmarkResultSummary {
|
|
246
|
-
participantSlug: string;
|
|
247
|
-
provider: string | null;
|
|
248
|
-
}
|
|
249
|
-
interface BenchmarkStepResultSummary {
|
|
250
|
-
participantSlug: string;
|
|
251
|
-
provider: string | null;
|
|
252
|
-
stepName: string;
|
|
253
|
-
stepCount: number;
|
|
254
|
-
successCount: number;
|
|
255
|
-
errorCount: number;
|
|
256
|
-
otherCount: number;
|
|
257
|
-
latencyCount: number;
|
|
258
|
-
successRate: number;
|
|
259
|
-
latencyMs: BenchmarkResultLatencySummary;
|
|
260
|
-
}
|
|
261
|
-
interface BenchmarkResultsOverviewInput {
|
|
262
|
-
limit?: number;
|
|
263
|
-
}
|
|
264
|
-
type BenchmarkAnalyticsReadiness = 'ready' | 'complete' | 'partial' | 'pending' | 'unavailable' | 'failed';
|
|
265
|
-
interface BenchmarkRunAnalyticsSummary {
|
|
266
|
-
status: BenchmarkAnalyticsReadiness;
|
|
267
|
-
eventBatches: number;
|
|
268
|
-
persisted: number;
|
|
269
|
-
queued: number;
|
|
270
|
-
failed: number;
|
|
271
|
-
imports: {
|
|
272
|
-
pending: number;
|
|
273
|
-
importing: number;
|
|
274
|
-
imported: number;
|
|
275
|
-
failed: number;
|
|
276
|
-
missing: number;
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
interface BenchmarkResultsOverviewAnalytics {
|
|
280
|
-
status: BenchmarkAnalyticsReadiness;
|
|
281
|
-
query: 'available' | 'unavailable';
|
|
282
|
-
error?: string;
|
|
283
|
-
}
|
|
284
|
-
interface BenchmarkResultsOverviewRun {
|
|
285
|
-
run: BenchmarkRun;
|
|
286
|
-
analytics: BenchmarkRunAnalyticsSummary;
|
|
287
|
-
participants: Array<BenchmarkParticipantResultSummary & {
|
|
288
|
-
runId: string;
|
|
289
|
-
}>;
|
|
290
|
-
}
|
|
291
|
-
interface BenchmarkResultsOverview {
|
|
292
|
-
benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
|
|
293
|
-
generatedAt: string;
|
|
294
|
-
analytics: BenchmarkResultsOverviewAnalytics;
|
|
295
|
-
items: BenchmarkResultsOverviewRun[];
|
|
296
|
-
}
|
|
297
|
-
interface BenchmarkRunResults {
|
|
298
|
-
benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
|
|
299
|
-
run: Pick<BenchmarkRun, 'id' | 'status' | 'totalTasks' | 'workerCount'>;
|
|
300
|
-
generatedAt: string;
|
|
301
|
-
overall: BenchmarkResultSummary;
|
|
302
|
-
participants: BenchmarkParticipantResultSummary[];
|
|
303
|
-
steps: BenchmarkStepResultSummary[];
|
|
304
|
-
}
|
|
305
|
-
interface BenchmarkRunTaskResultsInput {
|
|
306
|
-
bucketSize?: number;
|
|
307
|
-
failureLimit?: number;
|
|
308
|
-
}
|
|
309
|
-
interface BenchmarkTaskBucket {
|
|
310
|
-
participantSlug: string;
|
|
311
|
-
provider: string | null;
|
|
312
|
-
bucketStart: number;
|
|
313
|
-
bucketEnd: number;
|
|
314
|
-
taskIndexMidpoint: number;
|
|
315
|
-
taskCount: number;
|
|
316
|
-
successCount: number;
|
|
317
|
-
errorCount: number;
|
|
318
|
-
latencyMs: Pick<BenchmarkResultLatencySummary, 'p50' | 'p95' | 'max'>;
|
|
319
|
-
}
|
|
320
|
-
interface BenchmarkFailurePoint {
|
|
321
|
-
participantSlug: string;
|
|
322
|
-
provider: string | null;
|
|
323
|
-
taskIndex: number;
|
|
324
|
-
errorCode: string | null;
|
|
325
|
-
}
|
|
326
|
-
interface BenchmarkRunTaskResults {
|
|
327
|
-
run: {
|
|
328
|
-
id: string;
|
|
329
|
-
};
|
|
330
|
-
generatedAt: string;
|
|
331
|
-
bucketSize: number;
|
|
332
|
-
buckets: BenchmarkTaskBucket[];
|
|
333
|
-
failures: BenchmarkFailurePoint[];
|
|
334
|
-
}
|
|
335
|
-
interface BenchmarkRunTimelineInput {
|
|
336
|
-
bucketMs?: number;
|
|
337
|
-
}
|
|
338
|
-
interface BenchmarkEventRateBucket {
|
|
339
|
-
participantSlug: string;
|
|
340
|
-
provider: string | null;
|
|
341
|
-
tMs: number;
|
|
342
|
-
completed: number;
|
|
343
|
-
succeeded: number;
|
|
344
|
-
failed: number;
|
|
345
|
-
}
|
|
346
|
-
interface BenchmarkConcurrencyPoint {
|
|
347
|
-
participantSlug: string;
|
|
348
|
-
provider: string | null;
|
|
349
|
-
workerId: string;
|
|
350
|
-
recordedAt: string;
|
|
351
|
-
tMs: number;
|
|
352
|
-
step: string;
|
|
353
|
-
active: number;
|
|
354
|
-
target: number;
|
|
355
|
-
}
|
|
356
|
-
interface BenchmarkRunTimeline {
|
|
357
|
-
run: {
|
|
358
|
-
id: string;
|
|
359
|
-
};
|
|
360
|
-
generatedAt: string;
|
|
361
|
-
eventRate: {
|
|
362
|
-
bucketMs: number;
|
|
363
|
-
buckets: BenchmarkEventRateBucket[];
|
|
364
|
-
};
|
|
365
|
-
concurrency: {
|
|
366
|
-
firstRecordedAt: string | null;
|
|
367
|
-
heartbeatCount: number;
|
|
368
|
-
points: BenchmarkConcurrencyPoint[];
|
|
369
|
-
};
|
|
370
|
-
}
|
|
371
|
-
interface BenchmarkRunImportsSummary {
|
|
372
|
-
eventBatches: number;
|
|
373
|
-
persisted: number;
|
|
374
|
-
queued: number;
|
|
375
|
-
failed: number;
|
|
376
|
-
imports: {
|
|
377
|
-
pending: number;
|
|
378
|
-
importing: number;
|
|
379
|
-
imported: number;
|
|
380
|
-
failed: number;
|
|
381
|
-
missing: number;
|
|
382
|
-
};
|
|
383
|
-
}
|
|
384
|
-
interface BenchmarkRunImportItem {
|
|
385
|
-
eventBatchId: string;
|
|
386
|
-
batchType: string;
|
|
387
|
-
sequenceNumber: number;
|
|
388
|
-
batchStatus: string;
|
|
389
|
-
eventCount: number;
|
|
390
|
-
objectKey: string | null;
|
|
391
|
-
batchErrorMessage: string | null;
|
|
392
|
-
createdAt: string;
|
|
393
|
-
persistedAt: string | null;
|
|
394
|
-
sink: string | null;
|
|
395
|
-
importStatus: string | null;
|
|
396
|
-
importAttempts: number | null;
|
|
397
|
-
importedAt: string | null;
|
|
398
|
-
failedAt: string | null;
|
|
399
|
-
importErrorMessage: string | null;
|
|
400
|
-
}
|
|
401
|
-
interface BenchmarkRunImports {
|
|
402
|
-
run: {
|
|
403
|
-
id: string;
|
|
404
|
-
};
|
|
405
|
-
generatedAt: string;
|
|
406
|
-
summary: BenchmarkRunImportsSummary;
|
|
407
|
-
items: BenchmarkRunImportItem[];
|
|
408
|
-
}
|
|
409
|
-
interface WorkerConcurrencySample {
|
|
410
|
-
step: string;
|
|
411
|
-
active: number;
|
|
412
|
-
target: number;
|
|
413
|
-
}
|
|
414
|
-
interface WorkerHeartbeatInput {
|
|
415
|
-
attemptId: string;
|
|
416
|
-
progressDone?: number;
|
|
417
|
-
progressInFlight?: number;
|
|
418
|
-
progressErrors?: number;
|
|
419
|
-
progressTotal?: number;
|
|
420
|
-
currentStep?: string | null;
|
|
421
|
-
concurrency?: WorkerConcurrencySample[];
|
|
422
|
-
}
|
|
423
|
-
interface RunProgressConcurrency {
|
|
424
|
-
step: string;
|
|
425
|
-
active: number;
|
|
426
|
-
target: number;
|
|
427
|
-
ready: boolean;
|
|
428
|
-
freshWorkerCount: number;
|
|
429
|
-
}
|
|
430
|
-
type RunProgressStatus = 'planned' | 'in_progress' | 'completed' | 'failed';
|
|
431
|
-
interface RunProgressWorkerCounts {
|
|
432
|
-
pending: number;
|
|
433
|
-
running: number;
|
|
434
|
-
completed: number;
|
|
435
|
-
failed: number;
|
|
436
|
-
stale: number;
|
|
437
|
-
total: number;
|
|
438
|
-
}
|
|
439
|
-
interface RunProgressTaskCounts {
|
|
440
|
-
done: number;
|
|
441
|
-
inFlight: number;
|
|
442
|
-
errors: number;
|
|
443
|
-
total: number;
|
|
444
|
-
completionRatio: number;
|
|
445
|
-
}
|
|
446
|
-
interface RunProgressParticipantCounts {
|
|
447
|
-
planned: number;
|
|
448
|
-
inProgress: number;
|
|
449
|
-
completed: number;
|
|
450
|
-
failed: number;
|
|
451
|
-
total: number;
|
|
452
|
-
}
|
|
453
|
-
interface RunProgressSummary {
|
|
454
|
-
status: RunProgressStatus;
|
|
455
|
-
started: boolean;
|
|
456
|
-
completed: boolean;
|
|
457
|
-
participants: RunProgressParticipantCounts;
|
|
458
|
-
}
|
|
459
|
-
interface RunProgressParticipant {
|
|
460
|
-
id: string;
|
|
461
|
-
slug: string;
|
|
462
|
-
provider?: string | null;
|
|
463
|
-
status: RunProgressStatus;
|
|
464
|
-
totalTasks: number;
|
|
465
|
-
workerCount: number;
|
|
466
|
-
workers: RunProgressWorkerCounts;
|
|
467
|
-
tasks: RunProgressTaskCounts;
|
|
468
|
-
concurrency: RunProgressConcurrency[];
|
|
469
|
-
}
|
|
470
|
-
interface RunProgress {
|
|
471
|
-
run: {
|
|
472
|
-
id: string;
|
|
473
|
-
status: string;
|
|
474
|
-
totalTasks: number;
|
|
475
|
-
workerCount: number;
|
|
476
|
-
};
|
|
477
|
-
summary: RunProgressSummary;
|
|
478
|
-
freshnessWindowSeconds: number;
|
|
479
|
-
generatedAt: string;
|
|
480
|
-
participants: RunProgressParticipant[];
|
|
481
|
-
}
|
|
482
|
-
interface RunWorkerContext {
|
|
483
|
-
assignment: BenchmarkAssignment;
|
|
484
|
-
taskIndex: number;
|
|
485
|
-
step<T>(name: string, fn: () => Promise<T> | T, options?: DefineStepOptions): Promise<T>;
|
|
486
|
-
/**
|
|
487
|
-
* Attaches a JSON measurement to the platform. Called inside a `step`, it
|
|
488
|
-
* lands on that step's `data`; called at task top-level, on the task record's
|
|
489
|
-
* `data`. Repeated calls merge (shallow). Use this for anything you want on
|
|
490
|
-
* the platform — step return values are control flow and are never recorded.
|
|
491
|
-
*/
|
|
492
|
-
measure(data: JsonObject): void;
|
|
493
|
-
/** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
|
|
494
|
-
log(message: string, meta?: JsonObject): void;
|
|
495
|
-
}
|
|
496
|
-
interface WorkerFinishContext {
|
|
497
|
-
assignment: BenchmarkAssignment;
|
|
498
|
-
records: TaskResultRecord[];
|
|
499
|
-
status: 'success' | 'error';
|
|
500
|
-
client: BenchmarkClient;
|
|
501
|
-
uploadArtifact(input: Omit<UploadWorkerArtifactInput, 'attemptId'>): Promise<CreateWorkerArtifactResponse>;
|
|
502
|
-
}
|
|
503
|
-
interface DefineStepOptions {
|
|
504
|
-
/** Report this step as active in heartbeat concurrency samples. Defaults to true. */
|
|
505
|
-
reportConcurrency?: boolean;
|
|
506
|
-
/** Per-worker target for this step. Defaults to worker concurrency/assignment target. */
|
|
507
|
-
concurrency?: number;
|
|
508
|
-
/** Number of parallel invocations the step function should run internally. Used by the runner to record step-level concurrency. */
|
|
509
|
-
stepConcurrency?: number;
|
|
510
|
-
/** Per-invocation timeout in milliseconds for this step. Used by the runner to record step-level timeout metadata. */
|
|
511
|
-
timeoutMs?: number;
|
|
512
|
-
/** Readiness coordination mode. Defaults to internal. */
|
|
513
|
-
readiness?: 'poll' | 'internal';
|
|
514
|
-
/** Poll interval while waiting for readiness. Defaults to 1000ms. */
|
|
515
|
-
readyPollIntervalMs?: number;
|
|
516
|
-
/** Maximum time to wait for readiness. Defaults to no timeout. */
|
|
517
|
-
readyTimeoutMs?: number;
|
|
518
|
-
}
|
|
519
|
-
/**
|
|
520
|
-
* The unit of work a worker runs, once per task index. Steps are declared
|
|
521
|
-
* imperatively via `context.step(...)`; this is the sole task shape the worker
|
|
522
|
-
* engine accepts. Higher-level authoring (`defineTask`) lives in
|
|
523
|
-
* `@benchsdk/runner`, which compiles down to a function of this shape.
|
|
524
|
-
*/
|
|
525
|
-
type TaskFunction = (context: RunWorkerContext) => Promise<JsonObject | void> | JsonObject | void;
|
|
526
|
-
interface RunWorkerResult {
|
|
527
|
-
assignment: BenchmarkAssignment | null;
|
|
528
|
-
records: TaskResultRecord[];
|
|
529
|
-
}
|
|
530
|
-
interface RunWorkerOptions {
|
|
531
|
-
benchmarkSlug: string;
|
|
532
|
-
runId: string;
|
|
533
|
-
participantSlug: string;
|
|
534
|
-
processKind?: string;
|
|
535
|
-
processKey?: string;
|
|
536
|
-
concurrency?: number;
|
|
537
|
-
batchSize?: number;
|
|
538
|
-
flushIntervalMs?: number;
|
|
539
|
-
heartbeatIntervalMs?: number;
|
|
540
|
-
readyPollIntervalMs?: number;
|
|
541
|
-
onResult?: (record: TaskResultRecord) => void;
|
|
542
|
-
/** Runs once after final result flush and before worker completion/failure is reported. */
|
|
543
|
-
onFinish?: (context: WorkerFinishContext) => Promise<void> | void;
|
|
544
|
-
task: TaskFunction;
|
|
545
|
-
}
|
|
546
|
-
interface BenchmarkRunSummaryMetric {
|
|
547
|
-
name: string;
|
|
548
|
-
unit: string;
|
|
549
|
-
median: number;
|
|
550
|
-
p95: number;
|
|
551
|
-
p99: number;
|
|
552
|
-
}
|
|
553
|
-
interface BenchmarkRunSummaryScalar {
|
|
554
|
-
name: string;
|
|
555
|
-
value: number;
|
|
556
|
-
unit: string;
|
|
557
|
-
}
|
|
558
|
-
interface BenchmarkRunSummaryResult {
|
|
559
|
-
provider: string;
|
|
560
|
-
dimensions?: Record<string, unknown>;
|
|
561
|
-
metrics: BenchmarkRunSummaryMetric[];
|
|
562
|
-
scalars?: BenchmarkRunSummaryScalar[];
|
|
563
|
-
compositeScore: number;
|
|
564
|
-
successRate: number;
|
|
565
|
-
scoringVersion?: string | null;
|
|
566
|
-
skipped: boolean;
|
|
567
|
-
skipReason?: string | null;
|
|
568
|
-
}
|
|
569
|
-
interface BenchmarkRunSummaryRunMetadata {
|
|
570
|
-
gitSha?: string;
|
|
571
|
-
gitRef?: string;
|
|
572
|
-
triggeredBy?: string;
|
|
573
|
-
nodeVersion?: string;
|
|
574
|
-
platform?: string;
|
|
575
|
-
arch?: string;
|
|
576
|
-
}
|
|
577
|
-
interface BenchmarkRunSummaryInput {
|
|
578
|
-
run: BenchmarkRunSummaryRunMetadata;
|
|
579
|
-
results: BenchmarkRunSummaryResult[];
|
|
580
|
-
}
|
|
581
|
-
interface BenchmarkClient {
|
|
582
|
-
upsertBenchmark(slug: string, input: UpsertBenchmarkInput): Promise<BenchmarkResource>;
|
|
583
|
-
updateBenchmark(slug: string, input: UpdateBenchmarkInput): Promise<BenchmarkResource>;
|
|
584
|
-
getBenchmark(slug: string): Promise<BenchmarkResource>;
|
|
585
|
-
listBenchmarks(): Promise<BenchmarkResource[]>;
|
|
586
|
-
createRun(benchmarkSlug: string, input: CreateRunInput): Promise<{
|
|
587
|
-
run: BenchmarkRun;
|
|
588
|
-
participants: BenchmarkParticipant[];
|
|
589
|
-
/** The slug of the org the run was attributed to, resolved server-side from the caller's API key. */
|
|
590
|
-
organizationSlug: string;
|
|
591
|
-
}>;
|
|
592
|
-
listRuns(benchmarkSlug: string): Promise<BenchmarkRun[]>;
|
|
593
|
-
getRun(benchmarkSlug: string, runId: string): Promise<BenchmarkRun>;
|
|
594
|
-
updateRun(benchmarkSlug: string, runId: string, input: UpdateRunInput): Promise<BenchmarkRun>;
|
|
595
|
-
upsertParticipant(benchmarkSlug: string, runId: string, participantSlug: string, input?: UpsertParticipantInput): Promise<BenchmarkParticipant>;
|
|
596
|
-
updateParticipant(benchmarkSlug: string, runId: string, participantSlug: string, input: UpdateParticipantInput): Promise<BenchmarkParticipant>;
|
|
597
|
-
listParticipants(benchmarkSlug: string, runId: string): Promise<BenchmarkParticipant[]>;
|
|
598
|
-
getParticipant(benchmarkSlug: string, runId: string, participantSlug: string): Promise<BenchmarkParticipant>;
|
|
599
|
-
listWorkers(benchmarkSlug: string, runId: string, participantSlug: string): Promise<BenchmarkRunWorker[]>;
|
|
600
|
-
planWorkers(benchmarkSlug: string, runId: string, participantSlug: string, input?: PlanWorkersInput): Promise<BenchmarkRunWorker[]>;
|
|
601
|
-
getWorker(benchmarkSlug: string, runId: string, workerId: string): Promise<BenchmarkRunWorker>;
|
|
602
|
-
updateWorker(benchmarkSlug: string, runId: string, workerId: string, input: UpdateWorkerInput): Promise<BenchmarkRunWorker>;
|
|
603
|
-
getRunProgress(benchmarkSlug: string, runId: string): Promise<RunProgress>;
|
|
604
|
-
claimWorker(benchmarkSlug: string, runId: string, participantSlug: string, input?: ClaimWorkerInput): Promise<BenchmarkAssignment | null>;
|
|
605
|
-
releaseWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string): Promise<{
|
|
606
|
-
worker: BenchmarkRunWorker;
|
|
607
|
-
attempt: BenchmarkWorkerAttempt;
|
|
608
|
-
}>;
|
|
609
|
-
sendTaskResults(input: SendTaskResultsInput): Promise<TaskResultsResponse>;
|
|
610
|
-
heartbeatWorker(benchmarkSlug: string, runId: string, workerId: string, input: WorkerHeartbeatInput): Promise<{
|
|
611
|
-
worker: BenchmarkRunWorker;
|
|
612
|
-
attempt: BenchmarkWorkerAttempt;
|
|
613
|
-
}>;
|
|
614
|
-
completeWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string): Promise<{
|
|
615
|
-
worker: BenchmarkRunWorker;
|
|
616
|
-
attempt: BenchmarkWorkerAttempt;
|
|
617
|
-
}>;
|
|
618
|
-
failWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string, error?: unknown): Promise<{
|
|
619
|
-
worker: BenchmarkRunWorker;
|
|
620
|
-
attempt: BenchmarkWorkerAttempt;
|
|
621
|
-
}>;
|
|
622
|
-
createWorkerArtifact(benchmarkSlug: string, runId: string, workerId: string, input: CreateWorkerArtifactInput): Promise<CreateWorkerArtifactResponse>;
|
|
623
|
-
uploadWorkerArtifact(benchmarkSlug: string, runId: string, workerId: string, input: UploadWorkerArtifactInput): Promise<CreateWorkerArtifactResponse>;
|
|
624
|
-
listRunArtifacts(benchmarkSlug: string, runId: string): Promise<BenchmarkArtifact[]>;
|
|
625
|
-
listWorkerArtifacts(benchmarkSlug: string, runId: string, workerId: string): Promise<BenchmarkArtifact[]>;
|
|
626
|
-
getBenchmarkResults(benchmarkSlug: string, input?: BenchmarkResultsOverviewInput): Promise<BenchmarkResultsOverview>;
|
|
627
|
-
getRunResults(benchmarkSlug: string, runId: string): Promise<BenchmarkRunResults>;
|
|
628
|
-
getRunTaskResults(benchmarkSlug: string, runId: string, input?: BenchmarkRunTaskResultsInput): Promise<BenchmarkRunTaskResults>;
|
|
629
|
-
getRunTimeline(benchmarkSlug: string, runId: string, input?: BenchmarkRunTimelineInput): Promise<BenchmarkRunTimeline>;
|
|
630
|
-
getRunImports(benchmarkSlug: string, runId: string): Promise<BenchmarkRunImports>;
|
|
631
|
-
submitRunSummary(benchmarkSlug: string, runId: string, input: BenchmarkRunSummaryInput): Promise<void>;
|
|
1
|
+
import { BenchmarkClient as BenchmarkClient$1, RunWorkerOptions, RunWorkerResult, BenchmarkClientConfig } from '@benchsdk/api';
|
|
2
|
+
export { BenchmarkAnalyticsReadiness, BenchmarkApiError, BenchmarkArtifact, BenchmarkArtifactDownload, BenchmarkAssignment, BenchmarkClientConfig, BenchmarkConcurrencyPoint, BenchmarkEventRateBucket, BenchmarkFailurePoint, BenchmarkLogLevel, BenchmarkLogOptions, BenchmarkParticipant, BenchmarkResource, BenchmarkResultLatencySummary, BenchmarkResultSummary, BenchmarkResultsOverview, BenchmarkResultsOverviewAnalytics, BenchmarkResultsOverviewInput, BenchmarkResultsOverviewRun, BenchmarkRun, BenchmarkRunAnalyticsSummary, BenchmarkRunImportItem, BenchmarkRunImports, BenchmarkRunImportsSummary, BenchmarkRunResults, BenchmarkRunStatus, BenchmarkRunSummaryInput, BenchmarkRunSummaryMetric, BenchmarkRunSummaryResult, BenchmarkRunSummaryRunMetadata, BenchmarkRunSummaryScalar, BenchmarkRunTaskResults, BenchmarkRunTaskResultsInput, BenchmarkRunTimeline, BenchmarkRunTimelineInput, BenchmarkRunWorker, BenchmarkStepOutcome, BenchmarkStepResultSummary, BenchmarkTaskBucket, BenchmarkWorkerAttempt, BenchmarkWorkerStatus, ClaimWorkerInput, CreateRunInput, CreateWorkerArtifactInput, CreateWorkerArtifactResponse, DefineStepOptions, JsonObject, JsonValue, PlanWorkersInput, RunProgress, RunProgressConcurrency, RunProgressParticipant, RunProgressParticipantCounts, RunProgressStatus, RunProgressSummary, RunProgressTaskCounts, RunProgressWorkerCounts, RunWorkerContext, RunWorkerOptions, RunWorkerResult, SendTaskResultsInput, TaskFunction, TaskResultRecord, TaskResultsResponse, TaskStepRecord, UpdateBenchmarkInput, UpdateParticipantInput, UpdateRunInput, UpdateWorkerInput, UploadWorkerArtifactInput, UpsertBenchmarkInput, UpsertParticipantInput, WorkerConcurrencySample, WorkerFinishContext, WorkerHeartbeatInput } from '@benchsdk/api';
|
|
3
|
+
export { BaseParticipant, BenchmarkReporter, BenchmarkReporterArtifactInput, BenchmarkReporterBarrierInput, BenchmarkReporterBarrierResult, BenchmarkReporterConfig, BenchmarkReporterHeartbeatInput, BenchmarkReporterProgress, BenchmarkSystemMetricsCollector, BenchmarkSystemMetricsSample, claimBenchmarkReporter, createSystemMetricsCollector, filterParticipantsByEnv, runWorker, selectParticipants } from '@benchsdk/worker';
|
|
4
|
+
|
|
5
|
+
type BenchmarkClient = BenchmarkClient$1 & {
|
|
632
6
|
runWorker(options: RunWorkerOptions): Promise<RunWorkerResult>;
|
|
633
|
-
}
|
|
7
|
+
};
|
|
634
8
|
|
|
635
|
-
declare class BenchmarkApiError extends Error {
|
|
636
|
-
readonly status: number;
|
|
637
|
-
readonly body: string;
|
|
638
|
-
constructor(message: string, status: number, body: string);
|
|
639
|
-
}
|
|
640
9
|
declare function createBenchmarkClient(config?: BenchmarkClientConfig): BenchmarkClient;
|
|
641
10
|
|
|
642
|
-
|
|
643
|
-
benchmarkSlug: string;
|
|
644
|
-
runId: string;
|
|
645
|
-
participantSlug: string;
|
|
646
|
-
processKind?: string;
|
|
647
|
-
processKey?: string;
|
|
648
|
-
batchSize?: number;
|
|
649
|
-
}
|
|
650
|
-
interface BenchmarkReporterProgress {
|
|
651
|
-
done: number;
|
|
652
|
-
inFlight: number;
|
|
653
|
-
errors: number;
|
|
654
|
-
total?: number;
|
|
655
|
-
}
|
|
656
|
-
interface BenchmarkReporterArtifactInput {
|
|
657
|
-
kind: string;
|
|
658
|
-
name?: string;
|
|
659
|
-
contentType?: string;
|
|
660
|
-
body: BodyInit;
|
|
661
|
-
metadata?: JsonObject;
|
|
662
|
-
}
|
|
663
|
-
interface BenchmarkReporterHeartbeatInput {
|
|
664
|
-
currentStep?: string | null;
|
|
665
|
-
concurrency?: WorkerConcurrencySample[];
|
|
666
|
-
}
|
|
667
|
-
interface BenchmarkReporterBarrierInput {
|
|
668
|
-
step: string;
|
|
669
|
-
timeoutMs?: number;
|
|
670
|
-
pollIntervalMs?: number;
|
|
671
|
-
active?: number;
|
|
672
|
-
target?: number;
|
|
673
|
-
concurrency?: WorkerConcurrencySample[];
|
|
674
|
-
}
|
|
675
|
-
interface BenchmarkReporterBarrierResult {
|
|
676
|
-
active: number | null;
|
|
677
|
-
target: number | null;
|
|
678
|
-
ready: boolean;
|
|
679
|
-
measuredAt: string;
|
|
680
|
-
}
|
|
681
|
-
declare class BenchmarkReporter {
|
|
682
|
-
private readonly client;
|
|
683
|
-
private readonly assignment;
|
|
684
|
-
private readonly cfg;
|
|
685
|
-
private pending;
|
|
686
|
-
private sequenceNumber;
|
|
687
|
-
private flushChain;
|
|
688
|
-
private progress;
|
|
689
|
-
private barrier;
|
|
690
|
-
private constructor();
|
|
691
|
-
static claim(cfg: BenchmarkReporterConfig): Promise<BenchmarkReporter | null>;
|
|
692
|
-
get workerAssignment(): BenchmarkAssignment;
|
|
693
|
-
get taskCount(): number;
|
|
694
|
-
get taskIndexStart(): number;
|
|
695
|
-
setProgress(progress: BenchmarkReporterProgress): void;
|
|
696
|
-
recordResult(record: TaskResultRecord): void;
|
|
697
|
-
heartbeat(input?: BenchmarkReporterHeartbeatInput): Promise<void>;
|
|
698
|
-
waitForStepReady(input: BenchmarkReporterBarrierInput): Promise<BenchmarkReporterBarrierResult>;
|
|
699
|
-
uploadArtifact(input: BenchmarkReporterArtifactInput): Promise<CreateWorkerArtifactResponse | null>;
|
|
700
|
-
flush(isFinal?: boolean): Promise<void>;
|
|
701
|
-
finish(failed?: boolean, error?: unknown): Promise<void>;
|
|
702
|
-
}
|
|
703
|
-
declare function claimBenchmarkReporter(config: BenchmarkReporterConfig): Promise<BenchmarkReporter | null>;
|
|
704
|
-
|
|
705
|
-
interface BenchmarkSystemMetricsSample {
|
|
706
|
-
ts: string;
|
|
707
|
-
uptimeMs: number;
|
|
708
|
-
cpuUserUs: number;
|
|
709
|
-
cpuSystemUs: number;
|
|
710
|
-
memRssMb: number;
|
|
711
|
-
memHeapUsedMb: number;
|
|
712
|
-
memHeapTotalMb: number;
|
|
713
|
-
memExternalMb: number;
|
|
714
|
-
eventLoopP50Ms: number;
|
|
715
|
-
eventLoopP99Ms: number;
|
|
716
|
-
eventLoopMaxMs: number;
|
|
717
|
-
loadavg1m: number;
|
|
718
|
-
loadavg5m: number;
|
|
719
|
-
loadavg15m: number;
|
|
720
|
-
openFds: number | null;
|
|
721
|
-
sockstat: Record<string, number> | null;
|
|
722
|
-
}
|
|
723
|
-
interface BenchmarkSystemMetricsCollector {
|
|
724
|
-
sample(): BenchmarkSystemMetricsSample;
|
|
725
|
-
stop(): void;
|
|
726
|
-
}
|
|
727
|
-
declare function createSystemMetricsCollector(): BenchmarkSystemMetricsCollector;
|
|
728
|
-
|
|
729
|
-
/**
|
|
730
|
-
* Base interface for a benchmark participant — the shared shape across all
|
|
731
|
-
* benchmark categories (sandbox, ai-gateway, browser, storage). Each category
|
|
732
|
-
* extends this with its own provider-specific fields.
|
|
733
|
-
*/
|
|
734
|
-
interface BaseParticipant {
|
|
735
|
-
/** Participant name (e.g. 'e2b', 'daytona', 'openrouter') */
|
|
736
|
-
name: string;
|
|
737
|
-
/** Environment variables that must all be set to run this participant */
|
|
738
|
-
requiredEnvVars: string[];
|
|
739
|
-
}
|
|
740
|
-
/**
|
|
741
|
-
/**
|
|
742
|
-
* Filters `participants` down to those whose `requiredEnvVars` are all set
|
|
743
|
-
* in `process.env`. Returns an object `{ available, skipped }` where `skipped`
|
|
744
|
-
* includes the names and missing vars for logging.
|
|
745
|
-
*/
|
|
746
|
-
declare function filterParticipantsByEnv<T extends BaseParticipant>(participants: T[]): {
|
|
747
|
-
available: T[];
|
|
748
|
-
skipped: {
|
|
749
|
-
name: string;
|
|
750
|
-
missing: string[];
|
|
751
|
-
}[];
|
|
752
|
-
};
|
|
753
|
-
/**
|
|
754
|
-
* Filters `all` down to the requested `names`, exiting with a clear error
|
|
755
|
-
* if any name is unrecognized. Returns `all` unchanged when `names` is
|
|
756
|
-
* undefined (no filter specified).
|
|
757
|
-
*/
|
|
758
|
-
declare function selectParticipants<T extends BaseParticipant>(all: T[], names?: string[]): T[];
|
|
759
|
-
|
|
760
|
-
export { type BaseParticipant, type BenchmarkAnalyticsReadiness, BenchmarkApiError, type BenchmarkArtifact, type BenchmarkAssignment, type BenchmarkClient, type BenchmarkClientConfig, type BenchmarkConcurrencyPoint, type BenchmarkEventRateBucket, type BenchmarkFailurePoint, type BenchmarkParticipant, BenchmarkReporter, type BenchmarkReporterArtifactInput, type BenchmarkReporterBarrierInput, type BenchmarkReporterBarrierResult, type BenchmarkReporterConfig, type BenchmarkReporterHeartbeatInput, type BenchmarkReporterProgress, type BenchmarkResource, type BenchmarkResultLatencySummary, type BenchmarkResultSummary, type BenchmarkResultsOverview, type BenchmarkResultsOverviewAnalytics, type BenchmarkResultsOverviewInput, type BenchmarkResultsOverviewRun, type BenchmarkRun, type BenchmarkRunAnalyticsSummary, type BenchmarkRunImportItem, type BenchmarkRunImports, type BenchmarkRunImportsSummary, type BenchmarkRunResults, type BenchmarkRunStatus, type BenchmarkRunSummaryInput, type BenchmarkRunSummaryMetric, type BenchmarkRunSummaryResult, type BenchmarkRunSummaryRunMetadata, type BenchmarkRunSummaryScalar, type BenchmarkRunTaskResults, type BenchmarkRunTaskResultsInput, type BenchmarkRunTimeline, type BenchmarkRunTimelineInput, type BenchmarkRunWorker, type BenchmarkStepResultSummary, type BenchmarkSystemMetricsCollector, type BenchmarkSystemMetricsSample, type BenchmarkTaskBucket, type BenchmarkWorkerAttempt, type BenchmarkWorkerStatus, type ClaimWorkerInput, type CreateRunInput, type CreateWorkerArtifactInput, type CreateWorkerArtifactResponse, type DefineStepOptions, type JsonObject, type JsonValue, type PlanWorkersInput, type RunProgress, type RunProgressConcurrency, type RunProgressParticipant, type RunProgressParticipantCounts, type RunProgressStatus, type RunProgressSummary, type RunProgressTaskCounts, type RunProgressWorkerCounts, type RunWorkerContext, type RunWorkerOptions, type RunWorkerResult, type SendTaskResultsInput, type TaskFunction, type TaskResultRecord, type TaskResultsResponse, type TaskStepRecord, type UpdateBenchmarkInput, type UpdateParticipantInput, type UpdateRunInput, type UpdateWorkerInput, type UploadWorkerArtifactInput, type UpsertBenchmarkInput, type UpsertParticipantInput, type WorkerConcurrencySample, type WorkerFinishContext, type WorkerHeartbeatInput, claimBenchmarkReporter, createBenchmarkClient, createSystemMetricsCollector, filterParticipantsByEnv, selectParticipants };
|
|
11
|
+
export { type BenchmarkClient, createBenchmarkClient };
|