@benchsdk/client 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +41 -96
- package/dist/index.cjs +125 -140
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +106 -73
- package/dist/index.d.ts +106 -73
- package/dist/index.js +123 -135
- package/dist/index.js.map +1 -1
- package/package.json +14 -15
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ComputeSDK
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -10,77 +10,52 @@ This package talks to the platform-owned benchmark/run/participant/worker API. I
|
|
|
10
10
|
npm install @benchsdk/client
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
> Higher-level benchmark authoring (`defineBenchmarkConfig` / `defineTask` and
|
|
14
|
+
> the local orchestrator) lives in
|
|
15
|
+
> [`@benchsdk/runner`](../benchsdk-runner). This package is REST transport plus
|
|
16
|
+
> the worker engine only.
|
|
17
|
+
|
|
18
|
+
## Run A Worker
|
|
14
19
|
|
|
15
20
|
```ts
|
|
16
|
-
import {
|
|
21
|
+
import { createBenchmarkClient } from '@benchsdk/client';
|
|
17
22
|
import { compute } from 'computesdk';
|
|
18
23
|
|
|
19
|
-
const
|
|
24
|
+
const client = createBenchmarkClient({
|
|
25
|
+
apiKey: process.env.COMPUTESDK_ADMIN_API_KEY,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
// `task` is a raw function; declare named steps imperatively via `step(...)`,
|
|
29
|
+
// so values flow between steps with closures and cleanup runs in a `finally`.
|
|
30
|
+
const { assignment, records } = await client.runWorker({
|
|
20
31
|
benchmarkSlug: 'scale',
|
|
21
32
|
runId: process.env.BENCHMARK_RUN_ID!,
|
|
22
33
|
participantSlug: 'e2b',
|
|
23
34
|
processKind: 'container',
|
|
24
35
|
processKey: process.env.HOSTNAME,
|
|
25
36
|
concurrency: 100,
|
|
26
|
-
task:
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
}),
|
|
42
|
-
defineStep('destroy', async ({ state }) => {
|
|
43
|
-
await (state.sandbox as any).destroy();
|
|
44
|
-
}),
|
|
45
|
-
]),
|
|
37
|
+
task: async ({ assignment, step }) => {
|
|
38
|
+
const sandbox = await step('create', () =>
|
|
39
|
+
compute.sandbox.create({ provider: assignment.provider ?? 'e2b' }),
|
|
40
|
+
);
|
|
41
|
+
try {
|
|
42
|
+
await step('readiness', () => sandbox.runCommand('true'), { readiness: 'internal' });
|
|
43
|
+
await step('exec.first-command', () => sandbox.runCommand('node -v'));
|
|
44
|
+
// A `readiness: 'poll'` step reports active concurrency and waits until
|
|
45
|
+
// the platform reports the participant's step is ready (a barrier).
|
|
46
|
+
await step('pause', () => {}, { readiness: 'poll' });
|
|
47
|
+
return { sandboxId: sandbox.id };
|
|
48
|
+
} finally {
|
|
49
|
+
await step('destroy', () => sandbox.destroy());
|
|
50
|
+
}
|
|
51
|
+
},
|
|
46
52
|
});
|
|
47
|
-
|
|
48
|
-
await worker.run();
|
|
49
53
|
```
|
|
50
54
|
|
|
51
|
-
`
|
|
55
|
+
`client.runWorker(...)` claims the next pending platform assignment for the participant. If no work is available, it returns `{ assignment: null, records: [] }`.
|
|
52
56
|
|
|
53
57
|
Task results are flushed to the platform in batches of 1,000 records by default. Set `batchSize` to tune this per worker; the SDK validates the platform limit of 5,000 records per batch. Workers also flush partial batches every 30 seconds by default via `flushIntervalMs`, and always flush pending records during final completion or shutdown.
|
|
54
58
|
|
|
55
|
-
## Reuse A Bench Definition
|
|
56
|
-
|
|
57
|
-
```ts
|
|
58
|
-
import { defineBench, defineStep, defineTask } from '@benchsdk/client';
|
|
59
|
-
|
|
60
|
-
const lifecycleTask = defineTask('sandbox.lifecycle', [
|
|
61
|
-
defineStep('create', async ({ state }) => {
|
|
62
|
-
state.sandboxId = 'sandbox_123';
|
|
63
|
-
}),
|
|
64
|
-
defineStep('exec.first-command', async ({ state }) => ({
|
|
65
|
-
sandboxId: String(state.sandboxId),
|
|
66
|
-
})),
|
|
67
|
-
]);
|
|
68
|
-
|
|
69
|
-
const bench = defineBench({
|
|
70
|
-
slug: 'scale',
|
|
71
|
-
participantSlug: 'e2b',
|
|
72
|
-
concurrency: 100,
|
|
73
|
-
task: lifecycleTask,
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
const worker = bench.defineWorker({
|
|
77
|
-
runId: process.env.BENCHMARK_RUN_ID!,
|
|
78
|
-
processKey: process.env.HOSTNAME,
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
await worker.run();
|
|
82
|
-
```
|
|
83
|
-
|
|
84
59
|
## Create A Platform Run
|
|
85
60
|
|
|
86
61
|
```ts
|
|
@@ -92,7 +67,6 @@ const client = createBenchmarkClient({
|
|
|
92
67
|
|
|
93
68
|
await client.upsertBenchmark('scale', {
|
|
94
69
|
name: 'Scale',
|
|
95
|
-
kind: 'scale',
|
|
96
70
|
config: { timeoutMs: 120_000 },
|
|
97
71
|
});
|
|
98
72
|
|
|
@@ -110,55 +84,27 @@ await client.planWorkers('scale', run.id, 'modal');
|
|
|
110
84
|
console.log(run.id);
|
|
111
85
|
```
|
|
112
86
|
|
|
113
|
-
Workers must be planned before `
|
|
87
|
+
Workers must be planned before `client.runWorker(...)` can claim assignments.
|
|
114
88
|
|
|
115
89
|
## API
|
|
116
90
|
|
|
117
|
-
###
|
|
91
|
+
### Worker Engine
|
|
118
92
|
|
|
119
93
|
```ts
|
|
120
|
-
|
|
121
|
-
defineTask(name, steps)
|
|
122
|
-
defineWorker(options)
|
|
123
|
-
defineBench(options)
|
|
94
|
+
client.runWorker(options)
|
|
124
95
|
```
|
|
125
96
|
|
|
126
|
-
|
|
97
|
+
The `task` function receives:
|
|
127
98
|
|
|
128
99
|
| Field | Type | Description |
|
|
129
100
|
|-------|------|-------------|
|
|
130
101
|
| `assignment` | `BenchmarkAssignment` | Platform-owned assignment for this worker |
|
|
131
102
|
| `taskIndex` | `number` | Deterministic task index within the benchmark run |
|
|
132
|
-
| `
|
|
133
|
-
|
|
134
|
-
If a step returns a JSON object, it is merged into the task result `data` object. Defined tasks also include `taskName` in `data`.
|
|
135
|
-
|
|
136
|
-
`defineTask(name, steps, options)` supports task cleanup:
|
|
103
|
+
| `step` | `(name, fn, options?) => Promise<R>` | Runs `fn` as a named platform step and records its timing/outcome |
|
|
137
104
|
|
|
138
|
-
|
|
139
|
-
|--------|------|-------------|
|
|
140
|
-
| `cleanup` | `(context) => Promise<void> \| void` | Runs after the task finishes, whether steps succeeded or failed. Use shared `state` to tear down resources created by earlier steps. |
|
|
141
|
-
|
|
142
|
-
```ts
|
|
143
|
-
type SandboxState = {
|
|
144
|
-
sandbox?: Awaited<ReturnType<typeof compute.sandbox.create>>;
|
|
145
|
-
};
|
|
146
|
-
|
|
147
|
-
defineTask<SandboxState>('sandbox.lifecycle', [
|
|
148
|
-
defineStep<SandboxState>('create', async ({ state }) => {
|
|
149
|
-
state.sandbox = await compute.sandbox.create();
|
|
150
|
-
}),
|
|
151
|
-
defineStep<SandboxState>('exec', async ({ state }) => {
|
|
152
|
-
await state.sandbox.runCommand('node -v');
|
|
153
|
-
}),
|
|
154
|
-
], {
|
|
155
|
-
cleanup: async ({ state }) => {
|
|
156
|
-
await state.sandbox?.destroy?.();
|
|
157
|
-
},
|
|
158
|
-
});
|
|
159
|
-
```
|
|
105
|
+
If the task returns a JSON object, it is stored as the task result `data`.
|
|
160
106
|
|
|
161
|
-
`
|
|
107
|
+
`step(name, fn, options)` supports step-level progress coordination via `options`:
|
|
162
108
|
|
|
163
109
|
| Option | Type | Description |
|
|
164
110
|
|--------|------|-------------|
|
|
@@ -202,7 +148,7 @@ client.failWorker(benchmarkSlug, runId, workerId, attemptId, error)
|
|
|
202
148
|
client.runWorker(options)
|
|
203
149
|
```
|
|
204
150
|
|
|
205
|
-
For custom coordinators that do not fit `
|
|
151
|
+
For custom coordinators that do not fit `runWorker`, use the best-effort reporter wrapper:
|
|
206
152
|
|
|
207
153
|
```ts
|
|
208
154
|
const reporter = await BenchmarkReporter.claim({
|
|
@@ -227,10 +173,10 @@ await reporter?.finish(false);
|
|
|
227
173
|
|
|
228
174
|
`BenchmarkReporter` swallows platform telemetry failures for claim, heartbeat, result flushing, artifact upload, and finish calls. Benchmark work can continue even when reporting is temporarily unavailable.
|
|
229
175
|
|
|
230
|
-
For `
|
|
176
|
+
For `runWorker`, use `onFinish` to upload worker-level logs once, after final task results are flushed and before the worker attempt is completed or failed:
|
|
231
177
|
|
|
232
178
|
```ts
|
|
233
|
-
|
|
179
|
+
client.runWorker({
|
|
234
180
|
benchmarkSlug: 'scale',
|
|
235
181
|
runId,
|
|
236
182
|
participantSlug: 'e2b',
|
|
@@ -269,7 +215,7 @@ console.log(participant?.tasks.completionRatio);
|
|
|
269
215
|
console.log(participant?.concurrency.find((item) => item.step === 'pause')?.ready);
|
|
270
216
|
```
|
|
271
217
|
|
|
272
|
-
Most workers should use `
|
|
218
|
+
Most workers should use `client.runWorker(...)`.
|
|
273
219
|
|
|
274
220
|
## Task Result Shape
|
|
275
221
|
|
|
@@ -286,7 +232,6 @@ Most workers should use `defineWorker(...).run()`.
|
|
|
286
232
|
{ "name": "destroy", "status": "success", "startedAt": "...", "completedAt": "...", "latencyMs": 180 }
|
|
287
233
|
],
|
|
288
234
|
"data": {
|
|
289
|
-
"taskName": "sandbox.lifecycle",
|
|
290
235
|
"sandboxId": "..."
|
|
291
236
|
}
|
|
292
237
|
}
|
package/dist/index.cjs
CHANGED
|
@@ -35,11 +35,8 @@ __export(src_exports, {
|
|
|
35
35
|
claimBenchmarkReporter: () => claimBenchmarkReporter,
|
|
36
36
|
createBenchmarkClient: () => createBenchmarkClient,
|
|
37
37
|
createSystemMetricsCollector: () => createSystemMetricsCollector,
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
defineTask: () => defineTask,
|
|
41
|
-
defineWorker: () => defineWorker,
|
|
42
|
-
runBenchmarkWorker: () => runBenchmarkWorker
|
|
38
|
+
filterParticipantsByEnv: () => filterParticipantsByEnv,
|
|
39
|
+
selectParticipants: () => selectParticipants
|
|
43
40
|
});
|
|
44
41
|
module.exports = __toCommonJS(src_exports);
|
|
45
42
|
|
|
@@ -52,6 +49,7 @@ var DEFAULT_READY_POLL_INTERVAL_MS = 1e3;
|
|
|
52
49
|
var MAX_TASK_RESULT_RECORDS = 5e3;
|
|
53
50
|
var MAX_TASK_RECORD_STEPS = 100;
|
|
54
51
|
var MAX_HEARTBEAT_CONCURRENCY_SAMPLES = 20;
|
|
52
|
+
var MAX_WORKER_LOG_LINES = 1e5;
|
|
55
53
|
var BenchmarkApiError = class extends Error {
|
|
56
54
|
constructor(message, status, body) {
|
|
57
55
|
super(message);
|
|
@@ -80,6 +78,9 @@ function getApiKey(input) {
|
|
|
80
78
|
return input ?? (typeof process !== "undefined" ? process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY : void 0);
|
|
81
79
|
}
|
|
82
80
|
function getErrorCode(error) {
|
|
81
|
+
if (error instanceof Error && "code" in error && typeof error.code === "string" && error.code) {
|
|
82
|
+
return error.code;
|
|
83
|
+
}
|
|
83
84
|
if (error instanceof Error && error.name) return error.name;
|
|
84
85
|
return "ERROR";
|
|
85
86
|
}
|
|
@@ -87,6 +88,21 @@ function toJsonObject(value) {
|
|
|
87
88
|
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
88
89
|
return value;
|
|
89
90
|
}
|
|
91
|
+
function mergeMeasures(measures, returned) {
|
|
92
|
+
const merged = { ...measures, ...returned ?? {} };
|
|
93
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
94
|
+
}
|
|
95
|
+
function implicitTaskStep(record, measures) {
|
|
96
|
+
return {
|
|
97
|
+
name: "task",
|
|
98
|
+
status: record.status === "success" ? "success" : "error",
|
|
99
|
+
startedAt: record.startedAt,
|
|
100
|
+
completedAt: record.completedAt,
|
|
101
|
+
latencyMs: record.latencyMs,
|
|
102
|
+
errorCode: record.errorCode ?? null,
|
|
103
|
+
data: Object.keys(measures).length > 0 ? { ...measures } : void 0
|
|
104
|
+
};
|
|
105
|
+
}
|
|
90
106
|
function validateTaskResults(input) {
|
|
91
107
|
if (input.records.length > MAX_TASK_RESULT_RECORDS) {
|
|
92
108
|
throw new Error(`Benchmark task result batches are limited to ${MAX_TASK_RESULT_RECORDS} records.`);
|
|
@@ -132,53 +148,9 @@ function bodySizeBytes(body) {
|
|
|
132
148
|
if (body instanceof URLSearchParams) return new TextEncoder().encode(body.toString()).byteLength;
|
|
133
149
|
return void 0;
|
|
134
150
|
}
|
|
135
|
-
function isDefinedTask(task) {
|
|
136
|
-
return typeof task === "object" && task !== null && Array.isArray(task.steps);
|
|
137
|
-
}
|
|
138
|
-
function mergeJsonObjects(target, source) {
|
|
139
|
-
if (!source) return;
|
|
140
|
-
Object.assign(target, source);
|
|
141
|
-
}
|
|
142
151
|
function sleep(ms) {
|
|
143
152
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
144
153
|
}
|
|
145
|
-
async function runWorkerTask(task, context) {
|
|
146
|
-
if (!isDefinedTask(task)) {
|
|
147
|
-
return task(context);
|
|
148
|
-
}
|
|
149
|
-
const state = {};
|
|
150
|
-
const data = { taskName: task.name };
|
|
151
|
-
try {
|
|
152
|
-
for (const definedStep of task.steps) {
|
|
153
|
-
const stepData = await context.step(
|
|
154
|
-
definedStep.name,
|
|
155
|
-
() => definedStep.fn({
|
|
156
|
-
assignment: context.assignment,
|
|
157
|
-
taskIndex: context.taskIndex,
|
|
158
|
-
state
|
|
159
|
-
}),
|
|
160
|
-
definedStep.options
|
|
161
|
-
);
|
|
162
|
-
mergeJsonObjects(data, stepData);
|
|
163
|
-
}
|
|
164
|
-
} catch (error) {
|
|
165
|
-
try {
|
|
166
|
-
await task.options?.cleanup?.({
|
|
167
|
-
assignment: context.assignment,
|
|
168
|
-
taskIndex: context.taskIndex,
|
|
169
|
-
state
|
|
170
|
-
});
|
|
171
|
-
} catch {
|
|
172
|
-
}
|
|
173
|
-
throw error;
|
|
174
|
-
}
|
|
175
|
-
await task.options?.cleanup?.({
|
|
176
|
-
assignment: context.assignment,
|
|
177
|
-
taskIndex: context.taskIndex,
|
|
178
|
-
state
|
|
179
|
-
});
|
|
180
|
-
return data;
|
|
181
|
-
}
|
|
182
154
|
async function mapPool(items, concurrency, fn) {
|
|
183
155
|
let nextIndex = 0;
|
|
184
156
|
const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
@@ -455,6 +427,13 @@ function createBenchmarkClient(config = {}) {
|
|
|
455
427
|
`/benchmarks/${encodePath(benchmarkSlug)}/runs/${encodePath(runId)}/results/imports`
|
|
456
428
|
);
|
|
457
429
|
},
|
|
430
|
+
async submitRunSummary(benchmarkSlug, runId, input) {
|
|
431
|
+
await request(
|
|
432
|
+
"POST",
|
|
433
|
+
`/benchmarks/${encodePath(benchmarkSlug)}/runs/${encodePath(runId)}/summary`,
|
|
434
|
+
input
|
|
435
|
+
);
|
|
436
|
+
},
|
|
458
437
|
async runWorker(options) {
|
|
459
438
|
if (options.concurrency !== void 0) validatePositiveInteger("concurrency", options.concurrency);
|
|
460
439
|
if (options.batchSize !== void 0) validateBatchSize(options.batchSize);
|
|
@@ -566,6 +545,18 @@ function createBenchmarkClient(config = {}) {
|
|
|
566
545
|
});
|
|
567
546
|
}, options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS);
|
|
568
547
|
resultFlush.unref?.();
|
|
548
|
+
const workerLogLines = [];
|
|
549
|
+
let workerLogTruncated = false;
|
|
550
|
+
function appendWorkerLog(line) {
|
|
551
|
+
if (workerLogLines.length >= MAX_WORKER_LOG_LINES) {
|
|
552
|
+
if (!workerLogTruncated) {
|
|
553
|
+
workerLogTruncated = true;
|
|
554
|
+
workerLogLines.push("... (worker log truncated)");
|
|
555
|
+
}
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
workerLogLines.push(line);
|
|
559
|
+
}
|
|
569
560
|
async function runFinishHook(status) {
|
|
570
561
|
await options.onFinish?.({
|
|
571
562
|
assignment: claimed,
|
|
@@ -580,6 +571,19 @@ function createBenchmarkClient(config = {}) {
|
|
|
580
571
|
}
|
|
581
572
|
});
|
|
582
573
|
}
|
|
574
|
+
async function uploadWorkerLogArtifact() {
|
|
575
|
+
if (workerLogLines.length === 0) return;
|
|
576
|
+
try {
|
|
577
|
+
await client.uploadWorkerArtifact(options.benchmarkSlug, options.runId, claimed.workerId, {
|
|
578
|
+
attemptId: claimed.attemptId,
|
|
579
|
+
kind: "coordinator.log",
|
|
580
|
+
contentType: "text/plain",
|
|
581
|
+
name: "worker.log",
|
|
582
|
+
body: workerLogLines.join("\n") + "\n"
|
|
583
|
+
});
|
|
584
|
+
} catch {
|
|
585
|
+
}
|
|
586
|
+
}
|
|
583
587
|
try {
|
|
584
588
|
await sendHeartbeat().catch(() => {
|
|
585
589
|
});
|
|
@@ -593,6 +597,19 @@ function createBenchmarkClient(config = {}) {
|
|
|
593
597
|
startedAt: startedAtDate.toISOString()
|
|
594
598
|
};
|
|
595
599
|
const steps = [];
|
|
600
|
+
const taskMeasures = {};
|
|
601
|
+
let activeStep = null;
|
|
602
|
+
function measure(data) {
|
|
603
|
+
if (activeStep) {
|
|
604
|
+
activeStep.data = { ...activeStep.data ?? {}, ...data };
|
|
605
|
+
} else {
|
|
606
|
+
Object.assign(taskMeasures, data);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
function log(message, meta) {
|
|
610
|
+
const suffix = meta && Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : "";
|
|
611
|
+
appendWorkerLog(`${(/* @__PURE__ */ new Date()).toISOString()} [task ${taskIndex}] ${message}${suffix}`);
|
|
612
|
+
}
|
|
596
613
|
async function step(name, fn, stepOptions = {}) {
|
|
597
614
|
const stepStartedAtMs = Date.now();
|
|
598
615
|
const stepRecord = {
|
|
@@ -602,6 +619,8 @@ function createBenchmarkClient(config = {}) {
|
|
|
602
619
|
completedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
603
620
|
latencyMs: 0
|
|
604
621
|
};
|
|
622
|
+
if (stepOptions.timeoutMs !== void 0) stepRecord.timeoutMs = stepOptions.timeoutMs;
|
|
623
|
+
if (stepOptions.stepConcurrency !== void 0) stepRecord.concurrency = stepOptions.stepConcurrency;
|
|
605
624
|
const shouldReportConcurrency = stepOptions.reportConcurrency ?? true;
|
|
606
625
|
if (shouldReportConcurrency) {
|
|
607
626
|
const stepConcurrency = stepOptions.concurrency ?? workerConcurrency;
|
|
@@ -610,16 +629,23 @@ function createBenchmarkClient(config = {}) {
|
|
|
610
629
|
activeByStep.set(name, (activeByStep.get(name) ?? 0) + 1);
|
|
611
630
|
requestHeartbeat();
|
|
612
631
|
}
|
|
632
|
+
const previousStep = activeStep;
|
|
633
|
+
activeStep = stepRecord;
|
|
613
634
|
try {
|
|
614
635
|
if (stepOptions.readiness === "poll") {
|
|
615
636
|
await waitForStepReady(name, stepOptions);
|
|
616
637
|
}
|
|
617
|
-
|
|
638
|
+
const value = await fn();
|
|
639
|
+
appendWorkerLog(`${(/* @__PURE__ */ new Date()).toISOString()} [task ${taskIndex}] ${name}`);
|
|
640
|
+
return value;
|
|
618
641
|
} catch (error) {
|
|
619
642
|
stepRecord.status = "error";
|
|
620
643
|
stepRecord.errorCode = getErrorCode(error);
|
|
644
|
+
appendWorkerLog(`${(/* @__PURE__ */ new Date()).toISOString()} [task ${taskIndex}] ${name}`);
|
|
645
|
+
appendWorkerLog(` error: ${error instanceof Error ? error.message : String(error)}`);
|
|
621
646
|
throw error;
|
|
622
647
|
} finally {
|
|
648
|
+
activeStep = previousStep;
|
|
623
649
|
stepRecord.completedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
624
650
|
stepRecord.latencyMs = Date.now() - stepStartedAtMs;
|
|
625
651
|
steps.push(stepRecord);
|
|
@@ -636,16 +662,19 @@ function createBenchmarkClient(config = {}) {
|
|
|
636
662
|
}
|
|
637
663
|
}
|
|
638
664
|
try {
|
|
639
|
-
const data = await
|
|
640
|
-
record.data = toJsonObject(data);
|
|
665
|
+
const data = await options.task({ assignment: claimed, taskIndex, step, measure, log });
|
|
666
|
+
record.data = mergeMeasures(taskMeasures, toJsonObject(data));
|
|
641
667
|
} catch (error) {
|
|
642
668
|
record.status = "error";
|
|
643
669
|
record.errorCode = getErrorCode(error);
|
|
644
|
-
record.data = { errorMessage: error instanceof Error ? error.message : String(error) };
|
|
670
|
+
record.data = mergeMeasures(taskMeasures, { errorMessage: error instanceof Error ? error.message : String(error) });
|
|
645
671
|
} finally {
|
|
646
672
|
record.completedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
647
673
|
record.latencyMs = Date.now() - startedAtMs;
|
|
648
|
-
|
|
674
|
+
if (steps.length === 0) {
|
|
675
|
+
steps.push(implicitTaskStep(record, taskMeasures));
|
|
676
|
+
}
|
|
677
|
+
record.steps = steps;
|
|
649
678
|
doneCount += 1;
|
|
650
679
|
inFlightCount = Math.max(0, inFlightCount - 1);
|
|
651
680
|
if (record.status !== "success") errorCount += 1;
|
|
@@ -677,6 +706,7 @@ function createBenchmarkClient(config = {}) {
|
|
|
677
706
|
});
|
|
678
707
|
throw error;
|
|
679
708
|
} finally {
|
|
709
|
+
await uploadWorkerLogArtifact();
|
|
680
710
|
clearInterval(heartbeat);
|
|
681
711
|
clearInterval(resultFlush);
|
|
682
712
|
}
|
|
@@ -684,84 +714,6 @@ function createBenchmarkClient(config = {}) {
|
|
|
684
714
|
};
|
|
685
715
|
return client;
|
|
686
716
|
}
|
|
687
|
-
async function runBenchmarkWorker(config, options) {
|
|
688
|
-
return createBenchmarkClient(config).runWorker(options);
|
|
689
|
-
}
|
|
690
|
-
function defineStep(name, optionsOrFn, maybeFn) {
|
|
691
|
-
if (name.trim() === "") {
|
|
692
|
-
throw new Error("Benchmark step name must be non-empty.");
|
|
693
|
-
}
|
|
694
|
-
const hasOptions = typeof optionsOrFn !== "function";
|
|
695
|
-
const fn = hasOptions ? maybeFn : optionsOrFn;
|
|
696
|
-
if (!fn) {
|
|
697
|
-
throw new Error("Benchmark step function is required.");
|
|
698
|
-
}
|
|
699
|
-
return { name, options: hasOptions ? optionsOrFn : void 0, fn };
|
|
700
|
-
}
|
|
701
|
-
function defineTask(name, steps, options) {
|
|
702
|
-
if (name.trim() === "") {
|
|
703
|
-
throw new Error("Benchmark task name must be non-empty.");
|
|
704
|
-
}
|
|
705
|
-
if (steps.length === 0) {
|
|
706
|
-
throw new Error("Benchmark task must define at least one step.");
|
|
707
|
-
}
|
|
708
|
-
const names = /* @__PURE__ */ new Set();
|
|
709
|
-
for (const step of steps) {
|
|
710
|
-
if (names.has(step.name)) {
|
|
711
|
-
throw new Error(`Benchmark task step names must be unique. Duplicate step: "${step.name}".`);
|
|
712
|
-
}
|
|
713
|
-
names.add(step.name);
|
|
714
|
-
}
|
|
715
|
-
return { name, steps, options };
|
|
716
|
-
}
|
|
717
|
-
function defineWorker(options) {
|
|
718
|
-
const client = options.client ?? createBenchmarkClient();
|
|
719
|
-
return {
|
|
720
|
-
run(overrides = {}) {
|
|
721
|
-
return client.runWorker({
|
|
722
|
-
benchmarkSlug: options.benchmarkSlug,
|
|
723
|
-
runId: options.runId,
|
|
724
|
-
participantSlug: options.participantSlug,
|
|
725
|
-
processKind: options.processKind,
|
|
726
|
-
processKey: options.processKey,
|
|
727
|
-
concurrency: overrides.concurrency ?? options.concurrency,
|
|
728
|
-
batchSize: overrides.batchSize ?? options.batchSize,
|
|
729
|
-
flushIntervalMs: overrides.flushIntervalMs ?? options.flushIntervalMs,
|
|
730
|
-
heartbeatIntervalMs: overrides.heartbeatIntervalMs ?? options.heartbeatIntervalMs,
|
|
731
|
-
readyPollIntervalMs: overrides.readyPollIntervalMs ?? options.readyPollIntervalMs,
|
|
732
|
-
onFinish: options.onFinish,
|
|
733
|
-
task: options.task
|
|
734
|
-
});
|
|
735
|
-
}
|
|
736
|
-
};
|
|
737
|
-
}
|
|
738
|
-
function defineBench(options) {
|
|
739
|
-
return {
|
|
740
|
-
slug: options.slug,
|
|
741
|
-
task: options.task,
|
|
742
|
-
defineWorker(workerOptions) {
|
|
743
|
-
const participantSlug = workerOptions.participantSlug ?? options.participantSlug;
|
|
744
|
-
if (!participantSlug) {
|
|
745
|
-
throw new Error("Benchmark worker participantSlug is required.");
|
|
746
|
-
}
|
|
747
|
-
return defineWorker({
|
|
748
|
-
benchmarkSlug: options.slug,
|
|
749
|
-
runId: workerOptions.runId,
|
|
750
|
-
participantSlug,
|
|
751
|
-
processKind: workerOptions.processKind,
|
|
752
|
-
processKey: workerOptions.processKey,
|
|
753
|
-
concurrency: workerOptions.concurrency ?? options.concurrency,
|
|
754
|
-
batchSize: workerOptions.batchSize ?? options.batchSize,
|
|
755
|
-
flushIntervalMs: workerOptions.flushIntervalMs ?? options.flushIntervalMs,
|
|
756
|
-
heartbeatIntervalMs: workerOptions.heartbeatIntervalMs ?? options.heartbeatIntervalMs,
|
|
757
|
-
readyPollIntervalMs: workerOptions.readyPollIntervalMs ?? options.readyPollIntervalMs,
|
|
758
|
-
onFinish: workerOptions.onFinish,
|
|
759
|
-
client: workerOptions.client ?? options.client,
|
|
760
|
-
task: workerOptions.task ?? options.task
|
|
761
|
-
});
|
|
762
|
-
}
|
|
763
|
-
};
|
|
764
|
-
}
|
|
765
717
|
|
|
766
718
|
// src/reporter.ts
|
|
767
719
|
var DEFAULT_REPORTER_BATCH_SIZE = 500;
|
|
@@ -794,7 +746,10 @@ var BenchmarkReporter = class _BenchmarkReporter {
|
|
|
794
746
|
processKey: cfg.processKey
|
|
795
747
|
});
|
|
796
748
|
return assignment ? new _BenchmarkReporter(client, cfg, assignment) : null;
|
|
797
|
-
} catch {
|
|
749
|
+
} catch (error) {
|
|
750
|
+
console.warn(
|
|
751
|
+
`[benchsdk] failed to claim worker for ${cfg.benchmarkSlug}/${cfg.participantSlug}: ${error instanceof Error ? error.message : String(error)}`
|
|
752
|
+
);
|
|
798
753
|
return null;
|
|
799
754
|
}
|
|
800
755
|
}
|
|
@@ -869,7 +824,12 @@ var BenchmarkReporter = class _BenchmarkReporter {
|
|
|
869
824
|
contentType: input.contentType,
|
|
870
825
|
metadata: input.metadata,
|
|
871
826
|
body: input.body
|
|
872
|
-
}).catch(() =>
|
|
827
|
+
}).catch((error) => {
|
|
828
|
+
console.warn(
|
|
829
|
+
`[benchsdk] failed to upload ${input.kind} artifact for worker ${this.assignment.workerId}: ${error instanceof Error ? error.message : String(error)}`
|
|
830
|
+
);
|
|
831
|
+
return null;
|
|
832
|
+
});
|
|
873
833
|
}
|
|
874
834
|
flush(isFinal = false) {
|
|
875
835
|
this.flushChain = this.flushChain.then(async () => {
|
|
@@ -885,7 +845,10 @@ var BenchmarkReporter = class _BenchmarkReporter {
|
|
|
885
845
|
isFinal: isFinal && batch.length === this.pending.length,
|
|
886
846
|
records: batch
|
|
887
847
|
});
|
|
888
|
-
} catch {
|
|
848
|
+
} catch (error) {
|
|
849
|
+
console.warn(
|
|
850
|
+
`[benchsdk] dropping ${this.pending.length} unsent task result(s) for worker ${this.assignment.workerId}: ${error instanceof Error ? error.message : String(error)}`
|
|
851
|
+
);
|
|
889
852
|
break;
|
|
890
853
|
}
|
|
891
854
|
this.pending.splice(0, batch.length);
|
|
@@ -987,6 +950,31 @@ function createSystemMetricsCollector() {
|
|
|
987
950
|
}
|
|
988
951
|
};
|
|
989
952
|
}
|
|
953
|
+
|
|
954
|
+
// src/participants.ts
|
|
955
|
+
function filterParticipantsByEnv(participants) {
|
|
956
|
+
const available = [];
|
|
957
|
+
const skipped = [];
|
|
958
|
+
for (const p of participants) {
|
|
959
|
+
const missing = p.requiredEnvVars.filter((v) => !process.env[v]);
|
|
960
|
+
if (missing.length > 0) {
|
|
961
|
+
skipped.push({ name: p.name, missing });
|
|
962
|
+
} else {
|
|
963
|
+
available.push(p);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
return { available, skipped };
|
|
967
|
+
}
|
|
968
|
+
function selectParticipants(all, names) {
|
|
969
|
+
if (!names) return all;
|
|
970
|
+
const unknown = names.filter((n) => !all.some((p) => p.name === n));
|
|
971
|
+
if (unknown.length > 0) {
|
|
972
|
+
console.error(`Unknown participant(s): ${unknown.join(", ")}`);
|
|
973
|
+
console.error(`Available: ${all.map((p) => p.name).join(", ")}`);
|
|
974
|
+
process.exit(1);
|
|
975
|
+
}
|
|
976
|
+
return all.filter((p) => names.includes(p.name));
|
|
977
|
+
}
|
|
990
978
|
// Annotate the CommonJS export names for ESM import in node:
|
|
991
979
|
0 && (module.exports = {
|
|
992
980
|
BenchmarkApiError,
|
|
@@ -994,10 +982,7 @@ function createSystemMetricsCollector() {
|
|
|
994
982
|
claimBenchmarkReporter,
|
|
995
983
|
createBenchmarkClient,
|
|
996
984
|
createSystemMetricsCollector,
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
defineTask,
|
|
1000
|
-
defineWorker,
|
|
1001
|
-
runBenchmarkWorker
|
|
985
|
+
filterParticipantsByEnv,
|
|
986
|
+
selectParticipants
|
|
1002
987
|
});
|
|
1003
988
|
//# sourceMappingURL=index.cjs.map
|