@benchsdk/runner 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -1,10 +1,11 @@
1
- import { TaskResultRecord, JsonObject, BaseParticipant, DefineStepOptions, TaskStepRecord } from '@benchsdk/client';
1
+ import { TaskResultRecord, JsonObject, DefineStepOptions, BenchmarkLogOptions, TaskStepRecord } from '@benchsdk/api';
2
+ import { BaseParticipant } from '@benchsdk/worker';
2
3
 
3
4
  type MetricValue = string | ((record: TaskResultRecord) => number | number[] | undefined);
4
5
  interface MetricScoring {
5
6
  name: string;
6
7
  value?: MetricValue;
7
- unit: string;
8
+ unit?: string;
8
9
  ceiling: number;
9
10
  floor?: number;
10
11
  higherIsBetter?: boolean;
@@ -15,8 +16,43 @@ interface MetricScoring {
15
16
  };
16
17
  trim?: number;
17
18
  }
19
+ interface BenchmarkScoringWeights {
20
+ median: number;
21
+ p95: number;
22
+ p99: number;
23
+ }
24
+ interface BenchmarkScoringMetric {
25
+ key: string;
26
+ label?: string;
27
+ unit?: string;
28
+ ceiling: number;
29
+ floor?: number;
30
+ higherIsBetter?: boolean;
31
+ weights: BenchmarkScoringWeights;
32
+ trim?: number;
33
+ }
34
+ /**
35
+ * Serializable counterpart to a `success` predicate: a record counts as
36
+ * successful only if it succeeded *and* every listed data field equals the
37
+ * given value (e.g. `{ verified: true }`, `{ actionsCompleted: 24 }`). Kept to
38
+ * equality on scalar data fields so the platform can express the same rule as
39
+ * a query predicate rather than replaying benchmark code.
40
+ */
41
+ interface BenchmarkScoringSuccess {
42
+ requireData: Record<string, string | number | boolean>;
43
+ }
44
+ /** Serializable scoring spec declared in a `*.bench.ts` file and uploaded to the platform. */
45
+ interface BenchmarkScoringConfig {
46
+ /** Optional data key to group records by when computing summary rows (e.g. 'file_size'). */
47
+ groupBy?: string;
48
+ /** Extra conditions a record must meet to count as successful. Default: `status === 'success'`. */
49
+ success?: BenchmarkScoringSuccess;
50
+ metrics: BenchmarkScoringMetric[];
51
+ }
18
52
  interface ScoringSpec {
19
53
  dimensions?: Record<string, unknown>;
54
+ /** Optional data key that groups task records into separate summary rows. */
55
+ groupBy?: string;
20
56
  success?: (record: TaskResultRecord) => boolean;
21
57
  metrics: MetricScoring[];
22
58
  }
@@ -42,7 +78,7 @@ interface BenchmarkScoreResult {
42
78
  skipReason?: string;
43
79
  }
44
80
  type LowerIsBetter = (name: string, opts: {
45
- unit: string;
81
+ unit?: string;
46
82
  ceiling: number;
47
83
  value?: MetricValue;
48
84
  weights: {
@@ -53,7 +89,7 @@ type LowerIsBetter = (name: string, opts: {
53
89
  trim?: number;
54
90
  }) => MetricScoring;
55
91
  type HigherIsBetter = (name: string, opts: {
56
- unit: string;
92
+ unit?: string;
57
93
  floor?: number;
58
94
  ceiling: number;
59
95
  value?: MetricValue;
@@ -66,7 +102,21 @@ type HigherIsBetter = (name: string, opts: {
66
102
  }) => MetricScoring;
67
103
  declare const lowerIsBetter: LowerIsBetter;
68
104
  declare const higherIsBetter: HigherIsBetter;
69
- declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec): BenchmarkScoreResult[];
105
+ declare class ScoringSpecError extends Error {
106
+ constructor(message: string);
107
+ }
108
+ declare function validateScoringSpec(spec: ScoringSpec): void;
109
+ declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec, displayMetrics?: {
110
+ key: string;
111
+ unit?: string;
112
+ }[]): BenchmarkScoreResult[];
113
+ /** Builds a runtime {@link ScoringSpec} from a serializable {@link BenchmarkScoringConfig}. */
114
+ declare function scoringConfigToSpec(config: BenchmarkScoringConfig, dimensions?: Record<string, unknown>, display?: {
115
+ metrics?: {
116
+ key: string;
117
+ unit?: string;
118
+ }[];
119
+ }): ScoringSpec;
70
120
 
71
121
  /**
72
122
  * A `*.bench.ts` file is the composition of a **config** and a **task**:
@@ -131,7 +181,7 @@ interface TaskResult {
131
181
  * Pre-measured steps the task timed itself (e.g. socket phases).
132
182
  * Only honored in `groupBy: 'round'` runs, where the runner builds records
133
183
  * manually. In `groupBy: 'participant'` runs the platform worker
134
- * (`client.runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
184
+ * (`runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
135
185
  */
136
186
  steps?: TaskStepRecord[];
137
187
  /** Task-owned overall latency; overrides framework wall-clock (round mode only). */
@@ -167,7 +217,7 @@ interface TaskContext<T extends BaseParticipant = BaseParticipant> {
167
217
  /** Current phase name, when the benchmark declares `phases`. */
168
218
  phase?: string;
169
219
  /**
170
- * Runs `fn` as a named platform step. Mirrors `@benchsdk/client`'s
220
+ * Runs `fn` as a named platform step. Mirrors `@benchsdk/worker`'s
171
221
  * `RunWorkerContext.step`; supports closures and try/finally. A `concurrency`
172
222
  * greater than 1 invokes `fn` that many times in parallel and returns an array.
173
223
  * `timeoutMs` aborts any invocation that exceeds it with a `step_timeout` TaskError.
@@ -180,8 +230,11 @@ interface TaskContext<T extends BaseParticipant = BaseParticipant> {
180
230
  * that step's data; at task top-level it lands on the task record's data.
181
231
  */
182
232
  measure(data: JsonObject): void;
183
- /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
184
- log(message: string, meta?: JsonObject): void;
233
+ /**
234
+ * Appends a line to the worker log, uploaded as an artifact when the worker finishes.
235
+ * `metaOrOptions` can be a metadata JSON object, or `{ level, meta }` to set a log level.
236
+ */
237
+ log(message: string, metaOrOptions?: JsonObject | BenchmarkLogOptions): void;
185
238
  }
186
239
  type BenchmarkTask<T extends BaseParticipant = BaseParticipant> = (ctx: TaskContext<T>) => Promise<TaskResult | void> | TaskResult | void;
187
240
  /**
@@ -203,11 +256,61 @@ interface ParticipantRecords {
203
256
  /** The orchestration knobs a run actually used, after CLI overrides. */
204
257
  interface ResolvedRunConfig {
205
258
  iterations: number;
259
+ /**
260
+ * Iterations each phase runs, when the benchmark declares `phases` and
261
+ * `--iterations` overrode their configured counts. A phase is one arm of a
262
+ * comparison (a file size), so the flag scales every arm equally rather than
263
+ * dividing a total between them.
264
+ */
265
+ phaseIterations?: number;
206
266
  concurrency: number;
207
267
  staggerDelayMs: number;
208
268
  groupBy: GroupBy;
209
269
  providers?: string[];
210
270
  }
271
+ /** Display metadata for a single custom metric a benchmark reports via `ctx.measure`. */
272
+ interface BenchmarkMetricDisplay {
273
+ /** Stable metric key, matching the key in `ctx.measure` or `data`. */
274
+ key: string;
275
+ /** Human-readable label shown in the platform UI. */
276
+ label: string;
277
+ /** Optional unit shown after the value (e.g. `Mbps`, `/s`, `ms`). */
278
+ unit?: string;
279
+ /** Number of decimal places when formatting numeric values. Defaults to the display format. */
280
+ decimals?: number;
281
+ /** Whether higher or lower values rank better. */
282
+ direction?: 'higher-better' | 'lower-better';
283
+ /** Optional ordering hint for metric lists. */
284
+ order?: number;
285
+ }
286
+ /** Display metadata for a single task lifecycle step. */
287
+ interface BenchmarkStepDisplay {
288
+ /** Stable step name, matching the string passed to `ctx.step`. */
289
+ key: string;
290
+ /** Human-readable label shown in the platform UI. */
291
+ label: string;
292
+ /** Optional ordering hint for step lists. */
293
+ order?: number;
294
+ }
295
+ /** Display defaults for the benchmark overview page. */
296
+ interface BenchmarkOverviewDisplay {
297
+ /** Metric key to rank participants by by default (falls back to overall task latency). */
298
+ defaultMetric?: string;
299
+ /** Default overview layout. */
300
+ defaultLayout?: 'ranking' | 'cards' | 'chart' | 'leaderboard';
301
+ }
302
+ /**
303
+ * Optional platform display manifest. A `*.bench.ts` file owns not only how the
304
+ * benchmark runs, but how it should be rendered, without a platform code change.
305
+ */
306
+ interface BenchmarkDisplayConfig {
307
+ /** Metric catalog — labels, units, and ranking direction for `ctx.measure` keys. */
308
+ metrics?: BenchmarkMetricDisplay[];
309
+ /** Step catalog — human labels for lifecycle steps reported via `ctx.step`. */
310
+ steps?: BenchmarkStepDisplay[];
311
+ /** Overview defaults. */
312
+ overview?: BenchmarkOverviewDisplay;
313
+ }
211
314
  /**
212
315
  * Result of a benchmark run, passed to `config.onComplete`. Exposes the raw
213
316
  * per-participant records so completion hooks can write legacy local results.
@@ -270,6 +373,12 @@ interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
270
373
  defaultProviders?: string[];
271
374
  /** The participants this benchmark can run against. `--provider` selects a subset by name. */
272
375
  participants: T[];
376
+ /**
377
+ * Static run-level dimensions copied into the submitted summary (e.g.
378
+ * `{ file_size: '10MB' }`). Useful for distinguishing runs of the same
379
+ * benchmark that differ by an external parameter.
380
+ */
381
+ dimensions?: Record<string, unknown>;
273
382
  /**
274
383
  * Run-level scoring hook, called once with `lowerIsBetter` and `higherIsBetter`
275
384
  * primitives after the outcome is assembled but before `onComplete`. Use it to
@@ -282,6 +391,23 @@ interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
282
391
  * writers). This is the run-level counterpart to per-step `ctx.measure`.
283
392
  */
284
393
  onComplete?: (outcome: BenchmarkRunOutcome) => void | Promise<void>;
394
+ /**
395
+ * Serializable scoring spec uploaded to the platform. When provided without
396
+ * `onScore`, the runner computes the run summary from this spec automatically.
397
+ * The platform can recompute `compositeScore` from the same spec at read time.
398
+ */
399
+ scoring?: BenchmarkScoringConfig;
400
+ /**
401
+ * Custom CLI flags this benchmark reads from `process.argv` (e.g. `--file-size`).
402
+ * Declaring them lets the runner distinguish intentional pass-through flags
403
+ * from typos and report unknown flags accurately.
404
+ */
405
+ customCliFlags?: readonly string[];
406
+ /**
407
+ * Optional display manifest. Lets the bench author configure metric labels,
408
+ * step labels, and overview defaults without editing the platform.
409
+ */
410
+ display?: BenchmarkDisplayConfig;
285
411
  }
286
412
  /** Validates `config` at file-evaluation time so mistakes surface immediately. */
287
413
  declare function defineBenchmarkConfig<T extends BaseParticipant = BaseParticipant>(config: BenchmarkConfig<T>): BenchmarkConfig<T>;
@@ -336,11 +462,15 @@ interface CliArgs {
336
462
  noIngest?: boolean;
337
463
  }
338
464
  /**
339
- * Parses the orchestration flags this runner understands, ignoring anything
340
- * else. Supports both `--flag value` and `--flag=value`; `--provider` accepts
465
+ * Parses the orchestration flags this runner understands, rejecting unknown
466
+ * flags. Supports both `--flag value` and `--flag=value`; `--provider` accepts
341
467
  * a comma-separated list and may be repeated.
468
+ *
469
+ * `allowedCustomFlags` lists pass-through flags the benchmark file reads from
470
+ * `process.argv` itself; the runner validates and skips them without choking on
471
+ * their values.
342
472
  */
343
- declare function parseCliArgs(argv: string[]): CliArgs;
473
+ declare function parseCliArgs(argv: string[], allowedCustomFlags?: readonly string[]): CliArgs;
344
474
  /** Merges CLI overrides over config defaults, filling in knob fallbacks. */
345
475
  declare function mergeConfig<T extends BaseParticipant>(config: BenchmarkConfig<T>, args: CliArgs): ResolvedRunConfig;
346
476
  /**
@@ -360,7 +490,9 @@ declare function runBenchmark<T extends BaseParticipant>(fileConfig: BenchmarkCo
360
490
  * exit. Does not call `process.exit`.
361
491
  */
362
492
  declare function runBenchmarkFile(argv: string[]): Promise<void>;
363
- /** Executable entry: runs the file and maps outcomes to process exit codes. */
493
+ /** Executable entry: dispatches to benchmark execution or platform data commands. */
364
494
  declare function run(argv: string[]): Promise<void>;
365
495
 
366
- export { type BenchmarkConfig, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type ResolvedRunConfig, type ScoringSpec, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, score };
496
+ declare const BENCHSDK_RUNNER_VERSION: string;
497
+
498
+ export { BENCHSDK_RUNNER_VERSION, type BenchmarkConfig, type BenchmarkDisplayConfig, type BenchmarkMetricDisplay, type BenchmarkOverviewDisplay, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkScoringConfig, type BenchmarkScoringMetric, type BenchmarkScoringSuccess, type BenchmarkScoringWeights, type BenchmarkStepDisplay, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type ResolvedRunConfig, type ScoringSpec, ScoringSpecError, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, score, scoringConfigToSpec, validateScoringSpec };
package/dist/index.d.ts CHANGED
@@ -1,10 +1,11 @@
1
- import { TaskResultRecord, JsonObject, BaseParticipant, DefineStepOptions, TaskStepRecord } from '@benchsdk/client';
1
+ import { TaskResultRecord, JsonObject, DefineStepOptions, BenchmarkLogOptions, TaskStepRecord } from '@benchsdk/api';
2
+ import { BaseParticipant } from '@benchsdk/worker';
2
3
 
3
4
  type MetricValue = string | ((record: TaskResultRecord) => number | number[] | undefined);
4
5
  interface MetricScoring {
5
6
  name: string;
6
7
  value?: MetricValue;
7
- unit: string;
8
+ unit?: string;
8
9
  ceiling: number;
9
10
  floor?: number;
10
11
  higherIsBetter?: boolean;
@@ -15,8 +16,43 @@ interface MetricScoring {
15
16
  };
16
17
  trim?: number;
17
18
  }
19
+ interface BenchmarkScoringWeights {
20
+ median: number;
21
+ p95: number;
22
+ p99: number;
23
+ }
24
+ interface BenchmarkScoringMetric {
25
+ key: string;
26
+ label?: string;
27
+ unit?: string;
28
+ ceiling: number;
29
+ floor?: number;
30
+ higherIsBetter?: boolean;
31
+ weights: BenchmarkScoringWeights;
32
+ trim?: number;
33
+ }
34
+ /**
35
+ * Serializable counterpart to a `success` predicate: a record counts as
36
+ * successful only if it succeeded *and* every listed data field equals the
37
+ * given value (e.g. `{ verified: true }`, `{ actionsCompleted: 24 }`). Kept to
38
+ * equality on scalar data fields so the platform can express the same rule as
39
+ * a query predicate rather than replaying benchmark code.
40
+ */
41
+ interface BenchmarkScoringSuccess {
42
+ requireData: Record<string, string | number | boolean>;
43
+ }
44
+ /** Serializable scoring spec declared in a `*.bench.ts` file and uploaded to the platform. */
45
+ interface BenchmarkScoringConfig {
46
+ /** Optional data key to group records by when computing summary rows (e.g. 'file_size'). */
47
+ groupBy?: string;
48
+ /** Extra conditions a record must meet to count as successful. Default: `status === 'success'`. */
49
+ success?: BenchmarkScoringSuccess;
50
+ metrics: BenchmarkScoringMetric[];
51
+ }
18
52
  interface ScoringSpec {
19
53
  dimensions?: Record<string, unknown>;
54
+ /** Optional data key that groups task records into separate summary rows. */
55
+ groupBy?: string;
20
56
  success?: (record: TaskResultRecord) => boolean;
21
57
  metrics: MetricScoring[];
22
58
  }
@@ -42,7 +78,7 @@ interface BenchmarkScoreResult {
42
78
  skipReason?: string;
43
79
  }
44
80
  type LowerIsBetter = (name: string, opts: {
45
- unit: string;
81
+ unit?: string;
46
82
  ceiling: number;
47
83
  value?: MetricValue;
48
84
  weights: {
@@ -53,7 +89,7 @@ type LowerIsBetter = (name: string, opts: {
53
89
  trim?: number;
54
90
  }) => MetricScoring;
55
91
  type HigherIsBetter = (name: string, opts: {
56
- unit: string;
92
+ unit?: string;
57
93
  floor?: number;
58
94
  ceiling: number;
59
95
  value?: MetricValue;
@@ -66,7 +102,21 @@ type HigherIsBetter = (name: string, opts: {
66
102
  }) => MetricScoring;
67
103
  declare const lowerIsBetter: LowerIsBetter;
68
104
  declare const higherIsBetter: HigherIsBetter;
69
- declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec): BenchmarkScoreResult[];
105
+ declare class ScoringSpecError extends Error {
106
+ constructor(message: string);
107
+ }
108
+ declare function validateScoringSpec(spec: ScoringSpec): void;
109
+ declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec, displayMetrics?: {
110
+ key: string;
111
+ unit?: string;
112
+ }[]): BenchmarkScoreResult[];
113
+ /** Builds a runtime {@link ScoringSpec} from a serializable {@link BenchmarkScoringConfig}. */
114
+ declare function scoringConfigToSpec(config: BenchmarkScoringConfig, dimensions?: Record<string, unknown>, display?: {
115
+ metrics?: {
116
+ key: string;
117
+ unit?: string;
118
+ }[];
119
+ }): ScoringSpec;
70
120
 
71
121
  /**
72
122
  * A `*.bench.ts` file is the composition of a **config** and a **task**:
@@ -131,7 +181,7 @@ interface TaskResult {
131
181
  * Pre-measured steps the task timed itself (e.g. socket phases).
132
182
  * Only honored in `groupBy: 'round'` runs, where the runner builds records
133
183
  * manually. In `groupBy: 'participant'` runs the platform worker
134
- * (`client.runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
184
+ * (`runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
135
185
  */
136
186
  steps?: TaskStepRecord[];
137
187
  /** Task-owned overall latency; overrides framework wall-clock (round mode only). */
@@ -167,7 +217,7 @@ interface TaskContext<T extends BaseParticipant = BaseParticipant> {
167
217
  /** Current phase name, when the benchmark declares `phases`. */
168
218
  phase?: string;
169
219
  /**
170
- * Runs `fn` as a named platform step. Mirrors `@benchsdk/client`'s
220
+ * Runs `fn` as a named platform step. Mirrors `@benchsdk/worker`'s
171
221
  * `RunWorkerContext.step`; supports closures and try/finally. A `concurrency`
172
222
  * greater than 1 invokes `fn` that many times in parallel and returns an array.
173
223
  * `timeoutMs` aborts any invocation that exceeds it with a `step_timeout` TaskError.
@@ -180,8 +230,11 @@ interface TaskContext<T extends BaseParticipant = BaseParticipant> {
180
230
  * that step's data; at task top-level it lands on the task record's data.
181
231
  */
182
232
  measure(data: JsonObject): void;
183
- /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
184
- log(message: string, meta?: JsonObject): void;
233
+ /**
234
+ * Appends a line to the worker log, uploaded as an artifact when the worker finishes.
235
+ * `metaOrOptions` can be a metadata JSON object, or `{ level, meta }` to set a log level.
236
+ */
237
+ log(message: string, metaOrOptions?: JsonObject | BenchmarkLogOptions): void;
185
238
  }
186
239
  type BenchmarkTask<T extends BaseParticipant = BaseParticipant> = (ctx: TaskContext<T>) => Promise<TaskResult | void> | TaskResult | void;
187
240
  /**
@@ -203,11 +256,61 @@ interface ParticipantRecords {
203
256
  /** The orchestration knobs a run actually used, after CLI overrides. */
204
257
  interface ResolvedRunConfig {
205
258
  iterations: number;
259
+ /**
260
+ * Iterations each phase runs, when the benchmark declares `phases` and
261
+ * `--iterations` overrode their configured counts. A phase is one arm of a
262
+ * comparison (a file size), so the flag scales every arm equally rather than
263
+ * dividing a total between them.
264
+ */
265
+ phaseIterations?: number;
206
266
  concurrency: number;
207
267
  staggerDelayMs: number;
208
268
  groupBy: GroupBy;
209
269
  providers?: string[];
210
270
  }
271
+ /** Display metadata for a single custom metric a benchmark reports via `ctx.measure`. */
272
+ interface BenchmarkMetricDisplay {
273
+ /** Stable metric key, matching the key in `ctx.measure` or `data`. */
274
+ key: string;
275
+ /** Human-readable label shown in the platform UI. */
276
+ label: string;
277
+ /** Optional unit shown after the value (e.g. `Mbps`, `/s`, `ms`). */
278
+ unit?: string;
279
+ /** Number of decimal places when formatting numeric values. Defaults to the display format. */
280
+ decimals?: number;
281
+ /** Whether higher or lower values rank better. */
282
+ direction?: 'higher-better' | 'lower-better';
283
+ /** Optional ordering hint for metric lists. */
284
+ order?: number;
285
+ }
286
+ /** Display metadata for a single task lifecycle step. */
287
+ interface BenchmarkStepDisplay {
288
+ /** Stable step name, matching the string passed to `ctx.step`. */
289
+ key: string;
290
+ /** Human-readable label shown in the platform UI. */
291
+ label: string;
292
+ /** Optional ordering hint for step lists. */
293
+ order?: number;
294
+ }
295
+ /** Display defaults for the benchmark overview page. */
296
+ interface BenchmarkOverviewDisplay {
297
+ /** Metric key to rank participants by by default (falls back to overall task latency). */
298
+ defaultMetric?: string;
299
+ /** Default overview layout. */
300
+ defaultLayout?: 'ranking' | 'cards' | 'chart' | 'leaderboard';
301
+ }
302
+ /**
303
+ * Optional platform display manifest. A `*.bench.ts` file owns not only how the
304
+ * benchmark runs, but how it should be rendered, without a platform code change.
305
+ */
306
+ interface BenchmarkDisplayConfig {
307
+ /** Metric catalog — labels, units, and ranking direction for `ctx.measure` keys. */
308
+ metrics?: BenchmarkMetricDisplay[];
309
+ /** Step catalog — human labels for lifecycle steps reported via `ctx.step`. */
310
+ steps?: BenchmarkStepDisplay[];
311
+ /** Overview defaults. */
312
+ overview?: BenchmarkOverviewDisplay;
313
+ }
211
314
  /**
212
315
  * Result of a benchmark run, passed to `config.onComplete`. Exposes the raw
213
316
  * per-participant records so completion hooks can write legacy local results.
@@ -270,6 +373,12 @@ interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
270
373
  defaultProviders?: string[];
271
374
  /** The participants this benchmark can run against. `--provider` selects a subset by name. */
272
375
  participants: T[];
376
+ /**
377
+ * Static run-level dimensions copied into the submitted summary (e.g.
378
+ * `{ file_size: '10MB' }`). Useful for distinguishing runs of the same
379
+ * benchmark that differ by an external parameter.
380
+ */
381
+ dimensions?: Record<string, unknown>;
273
382
  /**
274
383
  * Run-level scoring hook, called once with `lowerIsBetter` and `higherIsBetter`
275
384
  * primitives after the outcome is assembled but before `onComplete`. Use it to
@@ -282,6 +391,23 @@ interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
282
391
  * writers). This is the run-level counterpart to per-step `ctx.measure`.
283
392
  */
284
393
  onComplete?: (outcome: BenchmarkRunOutcome) => void | Promise<void>;
394
+ /**
395
+ * Serializable scoring spec uploaded to the platform. When provided without
396
+ * `onScore`, the runner computes the run summary from this spec automatically.
397
+ * The platform can recompute `compositeScore` from the same spec at read time.
398
+ */
399
+ scoring?: BenchmarkScoringConfig;
400
+ /**
401
+ * Custom CLI flags this benchmark reads from `process.argv` (e.g. `--file-size`).
402
+ * Declaring them lets the runner distinguish intentional pass-through flags
403
+ * from typos and report unknown flags accurately.
404
+ */
405
+ customCliFlags?: readonly string[];
406
+ /**
407
+ * Optional display manifest. Lets the bench author configure metric labels,
408
+ * step labels, and overview defaults without editing the platform.
409
+ */
410
+ display?: BenchmarkDisplayConfig;
285
411
  }
286
412
  /** Validates `config` at file-evaluation time so mistakes surface immediately. */
287
413
  declare function defineBenchmarkConfig<T extends BaseParticipant = BaseParticipant>(config: BenchmarkConfig<T>): BenchmarkConfig<T>;
@@ -336,11 +462,15 @@ interface CliArgs {
336
462
  noIngest?: boolean;
337
463
  }
338
464
  /**
339
- * Parses the orchestration flags this runner understands, ignoring anything
340
- * else. Supports both `--flag value` and `--flag=value`; `--provider` accepts
465
+ * Parses the orchestration flags this runner understands, rejecting unknown
466
+ * flags. Supports both `--flag value` and `--flag=value`; `--provider` accepts
341
467
  * a comma-separated list and may be repeated.
468
+ *
469
+ * `allowedCustomFlags` lists pass-through flags the benchmark file reads from
470
+ * `process.argv` itself; the runner validates and skips them without choking on
471
+ * their values.
342
472
  */
343
- declare function parseCliArgs(argv: string[]): CliArgs;
473
+ declare function parseCliArgs(argv: string[], allowedCustomFlags?: readonly string[]): CliArgs;
344
474
  /** Merges CLI overrides over config defaults, filling in knob fallbacks. */
345
475
  declare function mergeConfig<T extends BaseParticipant>(config: BenchmarkConfig<T>, args: CliArgs): ResolvedRunConfig;
346
476
  /**
@@ -360,7 +490,9 @@ declare function runBenchmark<T extends BaseParticipant>(fileConfig: BenchmarkCo
360
490
  * exit. Does not call `process.exit`.
361
491
  */
362
492
  declare function runBenchmarkFile(argv: string[]): Promise<void>;
363
- /** Executable entry: runs the file and maps outcomes to process exit codes. */
493
+ /** Executable entry: dispatches to benchmark execution or platform data commands. */
364
494
  declare function run(argv: string[]): Promise<void>;
365
495
 
366
- export { type BenchmarkConfig, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type ResolvedRunConfig, type ScoringSpec, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, score };
496
+ declare const BENCHSDK_RUNNER_VERSION: string;
497
+
498
+ export { BENCHSDK_RUNNER_VERSION, type BenchmarkConfig, type BenchmarkDisplayConfig, type BenchmarkMetricDisplay, type BenchmarkOverviewDisplay, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkScoringConfig, type BenchmarkScoringMetric, type BenchmarkScoringSuccess, type BenchmarkScoringWeights, type BenchmarkStepDisplay, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type ResolvedRunConfig, type ScoringSpec, ScoringSpecError, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, score, scoringConfigToSpec, validateScoringSpec };