@benchsdk/client 0.3.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/dist/index.cjs +8 -953
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +14 -753
- package/dist/index.d.ts +14 -753
- package/dist/index.js +8 -937
- package/dist/index.js.map +1 -1
- package/package.json +11 -9
package/dist/index.d.ts
CHANGED
|
@@ -1,760 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
interface BenchmarkClientConfig {
|
|
6
|
-
/** API base URL. Defaults to https://platform.computesdk.com/api/v1. */
|
|
7
|
-
baseUrl?: string;
|
|
8
|
-
/** Bearer token. Defaults to process.env.COMPUTESDK_ADMIN_API_KEY, then process.env.COMPUTESDK_API_KEY. */
|
|
9
|
-
apiKey?: string;
|
|
10
|
-
/** Custom fetch implementation, mostly useful for tests. */
|
|
11
|
-
fetch?: typeof fetch;
|
|
12
|
-
}
|
|
13
|
-
interface BenchmarkResource {
|
|
14
|
-
id: string;
|
|
15
|
-
slug: string;
|
|
16
|
-
name: string;
|
|
17
|
-
status?: string;
|
|
18
|
-
config?: JsonObject;
|
|
19
|
-
defaultRunConfig?: JsonObject;
|
|
20
|
-
}
|
|
21
|
-
type BenchmarkRunStatus = 'planned' | 'in_progress' | 'completed' | 'failed';
|
|
22
|
-
type BenchmarkWorkerStatus = 'pending' | 'running' | 'completed' | 'failed';
|
|
23
|
-
interface BenchmarkRun {
|
|
24
|
-
id: string;
|
|
25
|
-
benchmarkId: string;
|
|
26
|
-
name?: string | null;
|
|
27
|
-
status: BenchmarkRunStatus | string;
|
|
28
|
-
/** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */
|
|
29
|
-
runKey?: string | null;
|
|
30
|
-
totalTasks: number;
|
|
31
|
-
/** The run declared no size: `totalTasks` is the sum of what its participants declare. */
|
|
32
|
-
participantSized?: boolean;
|
|
33
|
-
workerCount: number;
|
|
34
|
-
config?: JsonObject;
|
|
35
|
-
createdAt?: string;
|
|
36
|
-
updatedAt?: string;
|
|
37
|
-
}
|
|
38
|
-
interface BenchmarkParticipant {
|
|
39
|
-
id: string;
|
|
40
|
-
benchmarkId: string;
|
|
41
|
-
runId: string;
|
|
42
|
-
slug: string;
|
|
43
|
-
label?: string | null;
|
|
44
|
-
provider?: string | null;
|
|
45
|
-
status: BenchmarkRunStatus | string;
|
|
46
|
-
totalTasks: number;
|
|
47
|
-
workerCount: number;
|
|
48
|
-
config?: JsonObject;
|
|
49
|
-
}
|
|
50
|
-
interface BenchmarkRunWorker {
|
|
51
|
-
id: string;
|
|
52
|
-
benchmarkId: string;
|
|
53
|
-
runId: string;
|
|
54
|
-
participantId: string;
|
|
55
|
-
workerIndex: number;
|
|
56
|
-
workerCount: number;
|
|
57
|
-
taskIndexStart: number;
|
|
58
|
-
taskIndexEnd: number;
|
|
59
|
-
targetConcurrency: number;
|
|
60
|
-
status: BenchmarkWorkerStatus | string;
|
|
61
|
-
progressDone?: number;
|
|
62
|
-
progressInFlight?: number;
|
|
63
|
-
progressErrors?: number;
|
|
64
|
-
progressTotal?: number;
|
|
65
|
-
currentStep?: string | null;
|
|
66
|
-
concurrency?: WorkerConcurrencySample[];
|
|
67
|
-
}
|
|
68
|
-
interface BenchmarkWorkerAttempt {
|
|
69
|
-
id: string;
|
|
70
|
-
benchmarkId: string;
|
|
71
|
-
runId: string;
|
|
72
|
-
participantId: string;
|
|
73
|
-
workerId: string;
|
|
74
|
-
attemptNumber: number;
|
|
75
|
-
status: string;
|
|
76
|
-
}
|
|
77
|
-
interface BenchmarkAssignment {
|
|
78
|
-
benchmarkId: string;
|
|
79
|
-
benchmarkSlug: string;
|
|
80
|
-
runId: string;
|
|
81
|
-
participantId: string;
|
|
82
|
-
participantSlug: string;
|
|
83
|
-
provider?: string | null;
|
|
84
|
-
workerId: string;
|
|
85
|
-
workerIndex: number;
|
|
86
|
-
workerCount: number;
|
|
87
|
-
attemptId: string;
|
|
88
|
-
attemptNumber: number;
|
|
89
|
-
taskRange: {
|
|
90
|
-
start: number;
|
|
91
|
-
end: number;
|
|
92
|
-
count: number;
|
|
93
|
-
};
|
|
94
|
-
targetConcurrency: number;
|
|
95
|
-
config?: JsonObject;
|
|
96
|
-
}
|
|
97
|
-
interface UpsertBenchmarkInput {
|
|
98
|
-
name: string;
|
|
99
|
-
status?: string;
|
|
100
|
-
config?: JsonObject;
|
|
101
|
-
defaultRunConfig?: JsonObject;
|
|
102
|
-
}
|
|
103
|
-
interface UpdateBenchmarkInput {
|
|
104
|
-
name?: string;
|
|
105
|
-
status?: string;
|
|
106
|
-
config?: JsonObject;
|
|
107
|
-
defaultRunConfig?: JsonObject;
|
|
108
|
-
}
|
|
109
|
-
interface CreateRunInput {
|
|
110
|
-
/**
|
|
111
|
-
* Idempotency key for get-or-create: sibling callers passing the same key
|
|
112
|
-
* (per org + benchmark) converge on one run instead of each opening its own.
|
|
113
|
-
*/
|
|
114
|
-
runKey?: string;
|
|
115
|
-
/** Omit to open a participant-sized run: each participant declares its own size when it registers. */
|
|
116
|
-
totalTasks?: number;
|
|
117
|
-
workerCount?: number;
|
|
118
|
-
participants?: string[];
|
|
119
|
-
config?: JsonObject;
|
|
120
|
-
}
|
|
121
|
-
interface UpdateRunInput {
|
|
122
|
-
status?: BenchmarkRunStatus;
|
|
123
|
-
config?: JsonObject;
|
|
124
|
-
}
|
|
125
|
-
interface UpsertParticipantInput {
|
|
126
|
-
label?: string;
|
|
127
|
-
provider?: string;
|
|
128
|
-
status?: string;
|
|
129
|
-
totalTasks?: number;
|
|
130
|
-
workerCount?: number;
|
|
131
|
-
config?: JsonObject;
|
|
132
|
-
}
|
|
133
|
-
type UpdateParticipantInput = UpsertParticipantInput;
|
|
134
|
-
interface UpdateWorkerInput {
|
|
135
|
-
status?: BenchmarkWorkerStatus;
|
|
136
|
-
progressDone?: number;
|
|
137
|
-
progressInFlight?: number;
|
|
138
|
-
progressErrors?: number;
|
|
139
|
-
progressTotal?: number;
|
|
140
|
-
}
|
|
141
|
-
interface ClaimWorkerInput {
|
|
142
|
-
processKind?: string;
|
|
143
|
-
processKey?: string;
|
|
144
|
-
}
|
|
145
|
-
interface PlanWorkersInput {
|
|
146
|
-
workerCount?: number;
|
|
147
|
-
targetConcurrency?: number;
|
|
148
|
-
config?: JsonObject;
|
|
149
|
-
}
|
|
150
|
-
interface TaskResultRecord {
|
|
151
|
-
taskIndex: number;
|
|
152
|
-
status: string;
|
|
153
|
-
startedAt?: string;
|
|
154
|
-
completedAt?: string;
|
|
155
|
-
latencyMs?: number;
|
|
156
|
-
firstCommandMs?: number | null;
|
|
157
|
-
errorCode?: string | null;
|
|
158
|
-
steps?: TaskStepRecord[];
|
|
159
|
-
data?: JsonObject;
|
|
160
|
-
}
|
|
161
|
-
interface TaskStepRecord {
|
|
162
|
-
name: string;
|
|
163
|
-
status: 'success' | 'error';
|
|
164
|
-
startedAt?: string;
|
|
165
|
-
completedAt?: string;
|
|
166
|
-
latencyMs?: number;
|
|
167
|
-
errorCode?: string | null;
|
|
168
|
-
data?: JsonObject;
|
|
169
|
-
/** Number of parallel invocations requested for this step. */
|
|
170
|
-
concurrency?: number;
|
|
171
|
-
/** Per-iteration timeout in milliseconds applied to this step. */
|
|
172
|
-
timeoutMs?: number;
|
|
173
|
-
}
|
|
174
|
-
interface SendTaskResultsInput {
|
|
175
|
-
benchmarkSlug: string;
|
|
176
|
-
runId: string;
|
|
177
|
-
workerId: string;
|
|
178
|
-
attemptId: string;
|
|
179
|
-
sequenceNumber: number;
|
|
180
|
-
isFinal: boolean;
|
|
181
|
-
records: TaskResultRecord[];
|
|
182
|
-
}
|
|
183
|
-
interface TaskResultsResponse {
|
|
184
|
-
accepted?: number;
|
|
185
|
-
eventBatchId?: string;
|
|
186
|
-
queued?: boolean;
|
|
187
|
-
eventBatch?: unknown;
|
|
188
|
-
duplicate?: boolean;
|
|
189
|
-
queueMessageId?: string;
|
|
190
|
-
}
|
|
191
|
-
interface CreateWorkerArtifactInput {
|
|
192
|
-
attemptId: string;
|
|
193
|
-
kind: string;
|
|
194
|
-
contentType?: string;
|
|
195
|
-
name?: string;
|
|
196
|
-
metadata?: JsonObject;
|
|
197
|
-
}
|
|
198
|
-
interface UploadWorkerArtifactInput extends CreateWorkerArtifactInput {
|
|
199
|
-
body: BodyInit;
|
|
200
|
-
}
|
|
201
|
-
interface BenchmarkArtifact {
|
|
202
|
-
id?: string;
|
|
203
|
-
artifactId?: string;
|
|
204
|
-
benchmarkId?: string;
|
|
205
|
-
runId?: string;
|
|
206
|
-
participantId?: string;
|
|
207
|
-
participantSlug?: string;
|
|
208
|
-
workerId?: string;
|
|
209
|
-
attemptId?: string;
|
|
210
|
-
kind: string;
|
|
211
|
-
name?: string | null;
|
|
212
|
-
contentType?: string | null;
|
|
213
|
-
objectKey?: string;
|
|
214
|
-
uploadUrl?: string;
|
|
215
|
-
uploadUrlExpiresAt?: string;
|
|
216
|
-
metadata?: JsonObject;
|
|
217
|
-
createdAt?: string;
|
|
218
|
-
}
|
|
219
|
-
interface CreateWorkerArtifactResponse {
|
|
220
|
-
artifact?: BenchmarkArtifact;
|
|
221
|
-
artifactId?: string;
|
|
222
|
-
uploadUrl?: string;
|
|
223
|
-
uploadUrlExpiresAt?: string;
|
|
224
|
-
objectKey?: string;
|
|
225
|
-
}
|
|
226
|
-
interface BenchmarkResultLatencySummary {
|
|
227
|
-
min: number | null;
|
|
228
|
-
avg: number | null;
|
|
229
|
-
p50: number | null;
|
|
230
|
-
p95: number | null;
|
|
231
|
-
p99: number | null;
|
|
232
|
-
max: number | null;
|
|
233
|
-
}
|
|
234
|
-
interface BenchmarkResultSummary {
|
|
235
|
-
taskCount: number;
|
|
236
|
-
successCount: number;
|
|
237
|
-
errorCount: number;
|
|
238
|
-
otherCount: number;
|
|
239
|
-
latencyCount: number;
|
|
240
|
-
successRate: number;
|
|
241
|
-
latencyMs: BenchmarkResultLatencySummary;
|
|
242
|
-
firstStartedAt: string | null;
|
|
243
|
-
lastCompletedAt: string | null;
|
|
244
|
-
}
|
|
245
|
-
interface BenchmarkParticipantResultSummary extends BenchmarkResultSummary {
|
|
246
|
-
participantSlug: string;
|
|
247
|
-
provider: string | null;
|
|
248
|
-
}
|
|
249
|
-
interface BenchmarkStepResultSummary {
|
|
250
|
-
participantSlug: string;
|
|
251
|
-
provider: string | null;
|
|
252
|
-
stepName: string;
|
|
253
|
-
stepCount: number;
|
|
254
|
-
successCount: number;
|
|
255
|
-
errorCount: number;
|
|
256
|
-
otherCount: number;
|
|
257
|
-
latencyCount: number;
|
|
258
|
-
successRate: number;
|
|
259
|
-
latencyMs: BenchmarkResultLatencySummary;
|
|
260
|
-
}
|
|
261
|
-
interface BenchmarkResultsOverviewInput {
|
|
262
|
-
limit?: number;
|
|
263
|
-
}
|
|
264
|
-
type BenchmarkAnalyticsReadiness = 'ready' | 'complete' | 'partial' | 'pending' | 'unavailable' | 'failed';
|
|
265
|
-
interface BenchmarkRunAnalyticsSummary {
|
|
266
|
-
status: BenchmarkAnalyticsReadiness;
|
|
267
|
-
eventBatches: number;
|
|
268
|
-
persisted: number;
|
|
269
|
-
queued: number;
|
|
270
|
-
failed: number;
|
|
271
|
-
imports: {
|
|
272
|
-
pending: number;
|
|
273
|
-
importing: number;
|
|
274
|
-
imported: number;
|
|
275
|
-
failed: number;
|
|
276
|
-
missing: number;
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
interface BenchmarkResultsOverviewAnalytics {
|
|
280
|
-
status: BenchmarkAnalyticsReadiness;
|
|
281
|
-
query: 'available' | 'unavailable';
|
|
282
|
-
error?: string;
|
|
283
|
-
}
|
|
284
|
-
interface BenchmarkResultsOverviewRun {
|
|
285
|
-
run: BenchmarkRun;
|
|
286
|
-
analytics: BenchmarkRunAnalyticsSummary;
|
|
287
|
-
participants: Array<BenchmarkParticipantResultSummary & {
|
|
288
|
-
runId: string;
|
|
289
|
-
}>;
|
|
290
|
-
}
|
|
291
|
-
interface BenchmarkResultsOverview {
|
|
292
|
-
benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
|
|
293
|
-
generatedAt: string;
|
|
294
|
-
analytics: BenchmarkResultsOverviewAnalytics;
|
|
295
|
-
items: BenchmarkResultsOverviewRun[];
|
|
296
|
-
}
|
|
297
|
-
interface BenchmarkRunResults {
|
|
298
|
-
benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
|
|
299
|
-
run: Pick<BenchmarkRun, 'id' | 'status' | 'totalTasks' | 'workerCount'>;
|
|
300
|
-
generatedAt: string;
|
|
301
|
-
overall: BenchmarkResultSummary;
|
|
302
|
-
participants: BenchmarkParticipantResultSummary[];
|
|
303
|
-
steps: BenchmarkStepResultSummary[];
|
|
304
|
-
}
|
|
305
|
-
interface BenchmarkRunTaskResultsInput {
|
|
306
|
-
bucketSize?: number;
|
|
307
|
-
failureLimit?: number;
|
|
308
|
-
}
|
|
309
|
-
interface BenchmarkTaskBucket {
|
|
310
|
-
participantSlug: string;
|
|
311
|
-
provider: string | null;
|
|
312
|
-
bucketStart: number;
|
|
313
|
-
bucketEnd: number;
|
|
314
|
-
taskIndexMidpoint: number;
|
|
315
|
-
taskCount: number;
|
|
316
|
-
successCount: number;
|
|
317
|
-
errorCount: number;
|
|
318
|
-
latencyMs: Pick<BenchmarkResultLatencySummary, 'p50' | 'p95' | 'max'>;
|
|
319
|
-
}
|
|
320
|
-
interface BenchmarkFailurePoint {
|
|
321
|
-
participantSlug: string;
|
|
322
|
-
provider: string | null;
|
|
323
|
-
taskIndex: number;
|
|
324
|
-
errorCode: string | null;
|
|
325
|
-
}
|
|
326
|
-
interface BenchmarkRunTaskResults {
|
|
327
|
-
run: {
|
|
328
|
-
id: string;
|
|
329
|
-
};
|
|
330
|
-
generatedAt: string;
|
|
331
|
-
bucketSize: number;
|
|
332
|
-
buckets: BenchmarkTaskBucket[];
|
|
333
|
-
failures: BenchmarkFailurePoint[];
|
|
334
|
-
}
|
|
335
|
-
interface BenchmarkRunTimelineInput {
|
|
336
|
-
bucketMs?: number;
|
|
337
|
-
}
|
|
338
|
-
interface BenchmarkEventRateBucket {
|
|
339
|
-
participantSlug: string;
|
|
340
|
-
provider: string | null;
|
|
341
|
-
tMs: number;
|
|
342
|
-
completed: number;
|
|
343
|
-
succeeded: number;
|
|
344
|
-
failed: number;
|
|
345
|
-
}
|
|
346
|
-
interface BenchmarkConcurrencyPoint {
|
|
347
|
-
participantSlug: string;
|
|
348
|
-
provider: string | null;
|
|
349
|
-
workerId: string;
|
|
350
|
-
recordedAt: string;
|
|
351
|
-
tMs: number;
|
|
352
|
-
step: string;
|
|
353
|
-
active: number;
|
|
354
|
-
target: number;
|
|
355
|
-
}
|
|
356
|
-
interface BenchmarkRunTimeline {
|
|
357
|
-
run: {
|
|
358
|
-
id: string;
|
|
359
|
-
};
|
|
360
|
-
generatedAt: string;
|
|
361
|
-
eventRate: {
|
|
362
|
-
bucketMs: number;
|
|
363
|
-
buckets: BenchmarkEventRateBucket[];
|
|
364
|
-
};
|
|
365
|
-
concurrency: {
|
|
366
|
-
firstRecordedAt: string | null;
|
|
367
|
-
heartbeatCount: number;
|
|
368
|
-
points: BenchmarkConcurrencyPoint[];
|
|
369
|
-
};
|
|
370
|
-
}
|
|
371
|
-
interface BenchmarkRunImportsSummary {
|
|
372
|
-
eventBatches: number;
|
|
373
|
-
persisted: number;
|
|
374
|
-
queued: number;
|
|
375
|
-
failed: number;
|
|
376
|
-
imports: {
|
|
377
|
-
pending: number;
|
|
378
|
-
importing: number;
|
|
379
|
-
imported: number;
|
|
380
|
-
failed: number;
|
|
381
|
-
missing: number;
|
|
382
|
-
};
|
|
383
|
-
}
|
|
384
|
-
interface BenchmarkRunImportItem {
|
|
385
|
-
eventBatchId: string;
|
|
386
|
-
batchType: string;
|
|
387
|
-
sequenceNumber: number;
|
|
388
|
-
batchStatus: string;
|
|
389
|
-
eventCount: number;
|
|
390
|
-
objectKey: string | null;
|
|
391
|
-
batchErrorMessage: string | null;
|
|
392
|
-
createdAt: string;
|
|
393
|
-
persistedAt: string | null;
|
|
394
|
-
sink: string | null;
|
|
395
|
-
importStatus: string | null;
|
|
396
|
-
importAttempts: number | null;
|
|
397
|
-
importedAt: string | null;
|
|
398
|
-
failedAt: string | null;
|
|
399
|
-
importErrorMessage: string | null;
|
|
400
|
-
}
|
|
401
|
-
interface BenchmarkRunImports {
|
|
402
|
-
run: {
|
|
403
|
-
id: string;
|
|
404
|
-
};
|
|
405
|
-
generatedAt: string;
|
|
406
|
-
summary: BenchmarkRunImportsSummary;
|
|
407
|
-
items: BenchmarkRunImportItem[];
|
|
408
|
-
}
|
|
409
|
-
interface WorkerConcurrencySample {
|
|
410
|
-
step: string;
|
|
411
|
-
active: number;
|
|
412
|
-
target: number;
|
|
413
|
-
}
|
|
414
|
-
interface WorkerHeartbeatInput {
|
|
415
|
-
attemptId: string;
|
|
416
|
-
progressDone?: number;
|
|
417
|
-
progressInFlight?: number;
|
|
418
|
-
progressErrors?: number;
|
|
419
|
-
progressTotal?: number;
|
|
420
|
-
currentStep?: string | null;
|
|
421
|
-
concurrency?: WorkerConcurrencySample[];
|
|
422
|
-
}
|
|
423
|
-
interface RunProgressConcurrency {
|
|
424
|
-
step: string;
|
|
425
|
-
active: number;
|
|
426
|
-
target: number;
|
|
427
|
-
ready: boolean;
|
|
428
|
-
freshWorkerCount: number;
|
|
429
|
-
}
|
|
430
|
-
type RunProgressStatus = 'planned' | 'in_progress' | 'completed' | 'failed';
|
|
431
|
-
interface RunProgressWorkerCounts {
|
|
432
|
-
pending: number;
|
|
433
|
-
running: number;
|
|
434
|
-
completed: number;
|
|
435
|
-
failed: number;
|
|
436
|
-
stale: number;
|
|
437
|
-
total: number;
|
|
438
|
-
}
|
|
439
|
-
interface RunProgressTaskCounts {
|
|
440
|
-
done: number;
|
|
441
|
-
inFlight: number;
|
|
442
|
-
errors: number;
|
|
443
|
-
total: number;
|
|
444
|
-
completionRatio: number;
|
|
445
|
-
}
|
|
446
|
-
interface RunProgressParticipantCounts {
|
|
447
|
-
planned: number;
|
|
448
|
-
inProgress: number;
|
|
449
|
-
completed: number;
|
|
450
|
-
failed: number;
|
|
451
|
-
total: number;
|
|
452
|
-
}
|
|
453
|
-
interface RunProgressSummary {
|
|
454
|
-
status: RunProgressStatus;
|
|
455
|
-
started: boolean;
|
|
456
|
-
completed: boolean;
|
|
457
|
-
participants: RunProgressParticipantCounts;
|
|
458
|
-
}
|
|
459
|
-
interface RunProgressParticipant {
|
|
460
|
-
id: string;
|
|
461
|
-
slug: string;
|
|
462
|
-
provider?: string | null;
|
|
463
|
-
status: RunProgressStatus;
|
|
464
|
-
totalTasks: number;
|
|
465
|
-
workerCount: number;
|
|
466
|
-
workers: RunProgressWorkerCounts;
|
|
467
|
-
tasks: RunProgressTaskCounts;
|
|
468
|
-
concurrency: RunProgressConcurrency[];
|
|
469
|
-
}
|
|
470
|
-
interface RunProgress {
|
|
471
|
-
run: {
|
|
472
|
-
id: string;
|
|
473
|
-
status: string;
|
|
474
|
-
totalTasks: number;
|
|
475
|
-
workerCount: number;
|
|
476
|
-
};
|
|
477
|
-
summary: RunProgressSummary;
|
|
478
|
-
freshnessWindowSeconds: number;
|
|
479
|
-
generatedAt: string;
|
|
480
|
-
participants: RunProgressParticipant[];
|
|
481
|
-
}
|
|
482
|
-
interface RunWorkerContext {
|
|
483
|
-
assignment: BenchmarkAssignment;
|
|
484
|
-
taskIndex: number;
|
|
485
|
-
step<T>(name: string, fn: () => Promise<T> | T, options?: DefineStepOptions): Promise<T>;
|
|
486
|
-
/**
|
|
487
|
-
* Attaches a JSON measurement to the platform. Called inside a `step`, it
|
|
488
|
-
* lands on that step's `data`; called at task top-level, on the task record's
|
|
489
|
-
* `data`. Repeated calls merge (shallow). Use this for anything you want on
|
|
490
|
-
* the platform — step return values are control flow and are never recorded.
|
|
491
|
-
*/
|
|
492
|
-
measure(data: JsonObject): void;
|
|
493
|
-
/** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
|
|
494
|
-
log(message: string, meta?: JsonObject): void;
|
|
495
|
-
}
|
|
496
|
-
interface WorkerFinishContext {
|
|
497
|
-
assignment: BenchmarkAssignment;
|
|
498
|
-
records: TaskResultRecord[];
|
|
499
|
-
status: 'success' | 'error';
|
|
500
|
-
client: BenchmarkClient;
|
|
501
|
-
uploadArtifact(input: Omit<UploadWorkerArtifactInput, 'attemptId'>): Promise<CreateWorkerArtifactResponse>;
|
|
502
|
-
}
|
|
503
|
-
interface DefineStepOptions {
|
|
504
|
-
/** Report this step as active in heartbeat concurrency samples. Defaults to true. */
|
|
505
|
-
reportConcurrency?: boolean;
|
|
506
|
-
/** Per-worker target for this step. Defaults to worker concurrency/assignment target. */
|
|
507
|
-
concurrency?: number;
|
|
508
|
-
/** Number of parallel invocations the step function should run internally. Used by the runner to record step-level concurrency. */
|
|
509
|
-
stepConcurrency?: number;
|
|
510
|
-
/** Per-invocation timeout in milliseconds for this step. Used by the runner to record step-level timeout metadata. */
|
|
511
|
-
timeoutMs?: number;
|
|
512
|
-
/** Readiness coordination mode. Defaults to internal. */
|
|
513
|
-
readiness?: 'poll' | 'internal';
|
|
514
|
-
/** Poll interval while waiting for readiness. Defaults to 1000ms. */
|
|
515
|
-
readyPollIntervalMs?: number;
|
|
516
|
-
/** Maximum time to wait for readiness. Defaults to no timeout. */
|
|
517
|
-
readyTimeoutMs?: number;
|
|
518
|
-
}
|
|
519
|
-
/**
|
|
520
|
-
* The unit of work a worker runs, once per task index. Steps are declared
|
|
521
|
-
* imperatively via `context.step(...)`; this is the sole task shape the worker
|
|
522
|
-
* engine accepts. Higher-level authoring (`defineTask`) lives in
|
|
523
|
-
* `@benchsdk/runner`, which compiles down to a function of this shape.
|
|
524
|
-
*/
|
|
525
|
-
type TaskFunction = (context: RunWorkerContext) => Promise<JsonObject | void> | JsonObject | void;
|
|
526
|
-
interface RunWorkerResult {
|
|
527
|
-
assignment: BenchmarkAssignment | null;
|
|
528
|
-
records: TaskResultRecord[];
|
|
529
|
-
}
|
|
530
|
-
interface RunWorkerOptions {
|
|
531
|
-
benchmarkSlug: string;
|
|
532
|
-
runId: string;
|
|
533
|
-
participantSlug: string;
|
|
534
|
-
processKind?: string;
|
|
535
|
-
processKey?: string;
|
|
536
|
-
concurrency?: number;
|
|
537
|
-
batchSize?: number;
|
|
538
|
-
flushIntervalMs?: number;
|
|
539
|
-
heartbeatIntervalMs?: number;
|
|
540
|
-
readyPollIntervalMs?: number;
|
|
541
|
-
onResult?: (record: TaskResultRecord) => void;
|
|
542
|
-
/** Runs once after final result flush and before worker completion/failure is reported. */
|
|
543
|
-
onFinish?: (context: WorkerFinishContext) => Promise<void> | void;
|
|
544
|
-
task: TaskFunction;
|
|
545
|
-
}
|
|
546
|
-
interface BenchmarkRunSummaryMetric {
|
|
547
|
-
name: string;
|
|
548
|
-
unit: string;
|
|
549
|
-
median: number;
|
|
550
|
-
p95: number;
|
|
551
|
-
p99: number;
|
|
552
|
-
}
|
|
553
|
-
interface BenchmarkRunSummaryScalar {
|
|
554
|
-
name: string;
|
|
555
|
-
value: number;
|
|
556
|
-
unit: string;
|
|
557
|
-
}
|
|
558
|
-
interface BenchmarkRunSummaryResult {
|
|
559
|
-
provider: string;
|
|
560
|
-
dimensions?: Record<string, unknown>;
|
|
561
|
-
metrics: BenchmarkRunSummaryMetric[];
|
|
562
|
-
scalars?: BenchmarkRunSummaryScalar[];
|
|
563
|
-
compositeScore: number;
|
|
564
|
-
successRate: number;
|
|
565
|
-
scoringVersion?: string | null;
|
|
566
|
-
skipped: boolean;
|
|
567
|
-
skipReason?: string | null;
|
|
568
|
-
}
|
|
569
|
-
interface BenchmarkRunSummaryRunMetadata {
|
|
570
|
-
gitSha?: string;
|
|
571
|
-
gitRef?: string;
|
|
572
|
-
triggeredBy?: string;
|
|
573
|
-
nodeVersion?: string;
|
|
574
|
-
platform?: string;
|
|
575
|
-
arch?: string;
|
|
576
|
-
}
|
|
577
|
-
interface BenchmarkRunSummaryInput {
|
|
578
|
-
run: BenchmarkRunSummaryRunMetadata;
|
|
579
|
-
results: BenchmarkRunSummaryResult[];
|
|
580
|
-
}
|
|
581
|
-
interface BenchmarkClient {
|
|
582
|
-
upsertBenchmark(slug: string, input: UpsertBenchmarkInput): Promise<BenchmarkResource>;
|
|
583
|
-
updateBenchmark(slug: string, input: UpdateBenchmarkInput): Promise<BenchmarkResource>;
|
|
584
|
-
getBenchmark(slug: string): Promise<BenchmarkResource>;
|
|
585
|
-
listBenchmarks(): Promise<BenchmarkResource[]>;
|
|
586
|
-
createRun(benchmarkSlug: string, input: CreateRunInput): Promise<{
|
|
587
|
-
run: BenchmarkRun;
|
|
588
|
-
participants: BenchmarkParticipant[];
|
|
589
|
-
/** The slug of the org the run was attributed to, resolved server-side from the caller's API key. */
|
|
590
|
-
organizationSlug: string;
|
|
591
|
-
}>;
|
|
592
|
-
listRuns(benchmarkSlug: string): Promise<BenchmarkRun[]>;
|
|
593
|
-
getRun(benchmarkSlug: string, runId: string): Promise<BenchmarkRun>;
|
|
594
|
-
updateRun(benchmarkSlug: string, runId: string, input: UpdateRunInput): Promise<BenchmarkRun>;
|
|
595
|
-
upsertParticipant(benchmarkSlug: string, runId: string, participantSlug: string, input?: UpsertParticipantInput): Promise<BenchmarkParticipant>;
|
|
596
|
-
updateParticipant(benchmarkSlug: string, runId: string, participantSlug: string, input: UpdateParticipantInput): Promise<BenchmarkParticipant>;
|
|
597
|
-
listParticipants(benchmarkSlug: string, runId: string): Promise<BenchmarkParticipant[]>;
|
|
598
|
-
getParticipant(benchmarkSlug: string, runId: string, participantSlug: string): Promise<BenchmarkParticipant>;
|
|
599
|
-
listWorkers(benchmarkSlug: string, runId: string, participantSlug: string): Promise<BenchmarkRunWorker[]>;
|
|
600
|
-
planWorkers(benchmarkSlug: string, runId: string, participantSlug: string, input?: PlanWorkersInput): Promise<BenchmarkRunWorker[]>;
|
|
601
|
-
getWorker(benchmarkSlug: string, runId: string, workerId: string): Promise<BenchmarkRunWorker>;
|
|
602
|
-
updateWorker(benchmarkSlug: string, runId: string, workerId: string, input: UpdateWorkerInput): Promise<BenchmarkRunWorker>;
|
|
603
|
-
getRunProgress(benchmarkSlug: string, runId: string): Promise<RunProgress>;
|
|
604
|
-
claimWorker(benchmarkSlug: string, runId: string, participantSlug: string, input?: ClaimWorkerInput): Promise<BenchmarkAssignment | null>;
|
|
605
|
-
releaseWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string): Promise<{
|
|
606
|
-
worker: BenchmarkRunWorker;
|
|
607
|
-
attempt: BenchmarkWorkerAttempt;
|
|
608
|
-
}>;
|
|
609
|
-
sendTaskResults(input: SendTaskResultsInput): Promise<TaskResultsResponse>;
|
|
610
|
-
heartbeatWorker(benchmarkSlug: string, runId: string, workerId: string, input: WorkerHeartbeatInput): Promise<{
|
|
611
|
-
worker: BenchmarkRunWorker;
|
|
612
|
-
attempt: BenchmarkWorkerAttempt;
|
|
613
|
-
}>;
|
|
614
|
-
completeWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string): Promise<{
|
|
615
|
-
worker: BenchmarkRunWorker;
|
|
616
|
-
attempt: BenchmarkWorkerAttempt;
|
|
617
|
-
}>;
|
|
618
|
-
failWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string, error?: unknown): Promise<{
|
|
619
|
-
worker: BenchmarkRunWorker;
|
|
620
|
-
attempt: BenchmarkWorkerAttempt;
|
|
621
|
-
}>;
|
|
622
|
-
createWorkerArtifact(benchmarkSlug: string, runId: string, workerId: string, input: CreateWorkerArtifactInput): Promise<CreateWorkerArtifactResponse>;
|
|
623
|
-
uploadWorkerArtifact(benchmarkSlug: string, runId: string, workerId: string, input: UploadWorkerArtifactInput): Promise<CreateWorkerArtifactResponse>;
|
|
624
|
-
listRunArtifacts(benchmarkSlug: string, runId: string): Promise<BenchmarkArtifact[]>;
|
|
625
|
-
listWorkerArtifacts(benchmarkSlug: string, runId: string, workerId: string): Promise<BenchmarkArtifact[]>;
|
|
626
|
-
getBenchmarkResults(benchmarkSlug: string, input?: BenchmarkResultsOverviewInput): Promise<BenchmarkResultsOverview>;
|
|
627
|
-
getRunResults(benchmarkSlug: string, runId: string): Promise<BenchmarkRunResults>;
|
|
628
|
-
getRunTaskResults(benchmarkSlug: string, runId: string, input?: BenchmarkRunTaskResultsInput): Promise<BenchmarkRunTaskResults>;
|
|
629
|
-
getRunTimeline(benchmarkSlug: string, runId: string, input?: BenchmarkRunTimelineInput): Promise<BenchmarkRunTimeline>;
|
|
630
|
-
getRunImports(benchmarkSlug: string, runId: string): Promise<BenchmarkRunImports>;
|
|
631
|
-
submitRunSummary(benchmarkSlug: string, runId: string, input: BenchmarkRunSummaryInput): Promise<void>;
|
|
632
|
-
runWorker(options: RunWorkerOptions): Promise<RunWorkerResult>;
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
declare class BenchmarkApiError extends Error {
|
|
636
|
-
readonly status: number;
|
|
637
|
-
readonly body: string;
|
|
638
|
-
constructor(message: string, status: number, body: string);
|
|
639
|
-
}
|
|
640
|
-
declare function createBenchmarkClient(config?: BenchmarkClientConfig): BenchmarkClient;
|
|
641
|
-
|
|
642
|
-
interface BenchmarkReporterConfig extends BenchmarkClientConfig {
|
|
643
|
-
benchmarkSlug: string;
|
|
644
|
-
runId: string;
|
|
645
|
-
participantSlug: string;
|
|
646
|
-
processKind?: string;
|
|
647
|
-
processKey?: string;
|
|
648
|
-
batchSize?: number;
|
|
649
|
-
}
|
|
650
|
-
interface BenchmarkReporterProgress {
|
|
651
|
-
done: number;
|
|
652
|
-
inFlight: number;
|
|
653
|
-
errors: number;
|
|
654
|
-
total?: number;
|
|
655
|
-
}
|
|
656
|
-
interface BenchmarkReporterArtifactInput {
|
|
657
|
-
kind: string;
|
|
658
|
-
name?: string;
|
|
659
|
-
contentType?: string;
|
|
660
|
-
body: BodyInit;
|
|
661
|
-
metadata?: JsonObject;
|
|
662
|
-
}
|
|
663
|
-
interface BenchmarkReporterHeartbeatInput {
|
|
664
|
-
currentStep?: string | null;
|
|
665
|
-
concurrency?: WorkerConcurrencySample[];
|
|
666
|
-
}
|
|
667
|
-
interface BenchmarkReporterBarrierInput {
|
|
668
|
-
step: string;
|
|
669
|
-
timeoutMs?: number;
|
|
670
|
-
pollIntervalMs?: number;
|
|
671
|
-
active?: number;
|
|
672
|
-
target?: number;
|
|
673
|
-
concurrency?: WorkerConcurrencySample[];
|
|
674
|
-
}
|
|
675
|
-
interface BenchmarkReporterBarrierResult {
|
|
676
|
-
active: number | null;
|
|
677
|
-
target: number | null;
|
|
678
|
-
ready: boolean;
|
|
679
|
-
measuredAt: string;
|
|
680
|
-
}
|
|
681
|
-
declare class BenchmarkReporter {
|
|
682
|
-
private readonly client;
|
|
683
|
-
private readonly assignment;
|
|
684
|
-
private readonly cfg;
|
|
685
|
-
private pending;
|
|
686
|
-
private sequenceNumber;
|
|
687
|
-
private flushChain;
|
|
688
|
-
private progress;
|
|
689
|
-
private barrier;
|
|
690
|
-
private constructor();
|
|
691
|
-
static claim(cfg: BenchmarkReporterConfig): Promise<BenchmarkReporter | null>;
|
|
692
|
-
get workerAssignment(): BenchmarkAssignment;
|
|
693
|
-
get taskCount(): number;
|
|
694
|
-
get taskIndexStart(): number;
|
|
695
|
-
setProgress(progress: BenchmarkReporterProgress): void;
|
|
696
|
-
recordResult(record: TaskResultRecord): void;
|
|
697
|
-
heartbeat(input?: BenchmarkReporterHeartbeatInput): Promise<void>;
|
|
698
|
-
waitForStepReady(input: BenchmarkReporterBarrierInput): Promise<BenchmarkReporterBarrierResult>;
|
|
699
|
-
uploadArtifact(input: BenchmarkReporterArtifactInput): Promise<CreateWorkerArtifactResponse | null>;
|
|
700
|
-
flush(isFinal?: boolean): Promise<void>;
|
|
701
|
-
finish(failed?: boolean, error?: unknown): Promise<void>;
|
|
702
|
-
}
|
|
703
|
-
declare function claimBenchmarkReporter(config: BenchmarkReporterConfig): Promise<BenchmarkReporter | null>;
|
|
1
|
+
import { BenchmarkClient as BenchmarkClient$1, RunWorkerOptions, RunWorkerResult, BenchmarkClientConfig } from '@benchsdk/runner';
|
|
2
|
+
export * from '@benchsdk/runner';
|
|
3
|
+
export { BenchmarkClientConfig } from '@benchsdk/runner';
|
|
704
4
|
|
|
705
|
-
interface BenchmarkSystemMetricsSample {
|
|
706
|
-
ts: string;
|
|
707
|
-
uptimeMs: number;
|
|
708
|
-
cpuUserUs: number;
|
|
709
|
-
cpuSystemUs: number;
|
|
710
|
-
memRssMb: number;
|
|
711
|
-
memHeapUsedMb: number;
|
|
712
|
-
memHeapTotalMb: number;
|
|
713
|
-
memExternalMb: number;
|
|
714
|
-
eventLoopP50Ms: number;
|
|
715
|
-
eventLoopP99Ms: number;
|
|
716
|
-
eventLoopMaxMs: number;
|
|
717
|
-
loadavg1m: number;
|
|
718
|
-
loadavg5m: number;
|
|
719
|
-
loadavg15m: number;
|
|
720
|
-
openFds: number | null;
|
|
721
|
-
sockstat: Record<string, number> | null;
|
|
722
|
-
}
|
|
723
|
-
interface BenchmarkSystemMetricsCollector {
|
|
724
|
-
sample(): BenchmarkSystemMetricsSample;
|
|
725
|
-
stop(): void;
|
|
726
|
-
}
|
|
727
|
-
declare function createSystemMetricsCollector(): BenchmarkSystemMetricsCollector;
|
|
728
|
-
|
|
729
|
-
/**
|
|
730
|
-
* Base interface for a benchmark participant — the shared shape across all
|
|
731
|
-
* benchmark categories (sandbox, ai-gateway, browser, storage). Each category
|
|
732
|
-
* extends this with its own provider-specific fields.
|
|
733
|
-
*/
|
|
734
|
-
interface BaseParticipant {
|
|
735
|
-
/** Participant name (e.g. 'e2b', 'daytona', 'openrouter') */
|
|
736
|
-
name: string;
|
|
737
|
-
/** Environment variables that must all be set to run this participant */
|
|
738
|
-
requiredEnvVars: string[];
|
|
739
|
-
}
|
|
740
|
-
/**
|
|
741
5
|
/**
|
|
742
|
-
*
|
|
743
|
-
*
|
|
744
|
-
*
|
|
6
|
+
* @deprecated `@benchsdk/client` is a backwards-compatibility shim. The
|
|
7
|
+
* `runWorker` method is available here for existing callers; new code should
|
|
8
|
+
* import `runWorker` from `@benchsdk/runner` directly.
|
|
745
9
|
*/
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
skipped: {
|
|
749
|
-
name: string;
|
|
750
|
-
missing: string[];
|
|
751
|
-
}[];
|
|
10
|
+
type BenchmarkClient = BenchmarkClient$1 & {
|
|
11
|
+
runWorker(options: RunWorkerOptions): Promise<RunWorkerResult>;
|
|
752
12
|
};
|
|
13
|
+
|
|
753
14
|
/**
|
|
754
|
-
*
|
|
755
|
-
*
|
|
756
|
-
*
|
|
15
|
+
* @deprecated Use `createBenchmarkClient` from `@benchsdk/runner` and call
|
|
16
|
+
* `runWorker(client, options)` instead. This wrapper only exists to preserve the
|
|
17
|
+
* `client.runWorker(...)` spelling used by older `@benchsdk/client` consumers.
|
|
757
18
|
*/
|
|
758
|
-
declare function
|
|
19
|
+
declare function createBenchmarkClient(config?: BenchmarkClientConfig): BenchmarkClient;
|
|
759
20
|
|
|
760
|
-
export { type
|
|
21
|
+
export { type BenchmarkClient, createBenchmarkClient };
|