@benchsdk/runner 0.2.0 → 0.5.2

package/dist/index.d.cts

@@ -1,10 +1,14 @@
-import { TaskResultRecord, JsonObject, BaseParticipant, DefineStepOptions, TaskStepRecord } from '@benchsdk/client';
+import { TaskResultRecord, JsonObject, DefineStepOptions, BenchmarkLogOptions, TaskStepRecord } from '@benchsdk/api';
+export { BenchmarkAnalyticsReadiness, BenchmarkApiError, BenchmarkArtifact, BenchmarkArtifactDownload, BenchmarkAssignment, BenchmarkClient, BenchmarkClientConfig, BenchmarkConcurrencyPoint, BenchmarkEventRateBucket, BenchmarkFailurePoint, BenchmarkLogLevel, BenchmarkLogOptions, BenchmarkParticipant, BenchmarkResource, BenchmarkResultLatencySummary, BenchmarkResultSummary, BenchmarkResultsOverview, BenchmarkResultsOverviewAnalytics, BenchmarkResultsOverviewInput, BenchmarkResultsOverviewRun, BenchmarkRun, BenchmarkRunAnalyticsSummary, BenchmarkRunImportItem, BenchmarkRunImports, BenchmarkRunImportsSummary, BenchmarkRunResults, BenchmarkRunStatus, BenchmarkRunSummaryInput, BenchmarkRunSummaryMetric, BenchmarkRunSummaryResult, BenchmarkRunSummaryRunMetadata, BenchmarkRunSummaryScalar, BenchmarkRunTaskResults, BenchmarkRunTaskResultsInput, BenchmarkRunTimeline, BenchmarkRunTimelineInput, BenchmarkRunWorker, BenchmarkStepOutcome, BenchmarkStepResultSummary, BenchmarkTaskBucket, BenchmarkWorkerAttempt, BenchmarkWorkerStatus, ClaimWorkerInput, CreateRunInput, CreateWorkerArtifactInput, CreateWorkerArtifactResponse, DefineStepOptions, JsonObject, JsonValue, PlanWorkersInput, RunProgress, RunProgressConcurrency, RunProgressParticipant, RunProgressParticipantCounts, RunProgressStatus, RunProgressSummary, RunProgressTaskCounts, RunProgressWorkerCounts, RunWorkerContext, RunWorkerOptions, RunWorkerResult, SendTaskResultsInput, TaskFunction, TaskResultRecord, TaskResultsResponse, TaskStepRecord, UpdateBenchmarkInput, UpdateParticipantInput, UpdateRunInput, UpdateWorkerInput, UploadWorkerArtifactInput, UpsertBenchmarkInput, UpsertParticipantInput, WorkerConcurrencySample, WorkerFinishContext, WorkerHeartbeatInput, createBenchmarkClient } from '@benchsdk/api';
+import { BaseParticipant } from '@benchsdk/worker';
+export { BaseParticipant, BenchmarkReporter, BenchmarkReporterArtifactInput, BenchmarkReporterBarrierInput, BenchmarkReporterBarrierResult, BenchmarkReporterConfig, BenchmarkReporterHeartbeatInput, BenchmarkReporterProgress, BenchmarkSystemMetricsCollector, BenchmarkSystemMetricsSample, claimBenchmarkReporter, createSystemMetricsCollector, filterParticipantsByEnv, runWorker, selectParticipants } from '@benchsdk/worker';
+export { AuthError, CliAuth, createApiClient, resolveAuth } from '@benchsdk/cli';
 
 type MetricValue = string | ((record: TaskResultRecord) => number | number[] | undefined);
 interface MetricScoring {
     name: string;
     value?: MetricValue;
-    unit: string;
+    unit?: string;
     ceiling: number;
     floor?: number;
     higherIsBetter?: boolean;
@@ -15,8 +19,43 @@ interface MetricScoring {
     };
     trim?: number;
 }
+interface BenchmarkScoringWeights {
+    median: number;
+    p95: number;
+    p99: number;
+}
+interface BenchmarkScoringMetric {
+    key: string;
+    label?: string;
+    unit?: string;
+    ceiling: number;
+    floor?: number;
+    higherIsBetter?: boolean;
+    weights: BenchmarkScoringWeights;
+    trim?: number;
+}
+/**
+ * Serializable counterpart to a `success` predicate: a record counts as
+ * successful only if it succeeded *and* every listed data field equals the
+ * given value (e.g. `{ verified: true }`, `{ actionsCompleted: 24 }`). Kept to
+ * equality on scalar data fields so the platform can express the same rule as
+ * a query predicate rather than replaying benchmark code.
+ */
+interface BenchmarkScoringSuccess {
+    requireData: Record<string, string | number | boolean>;
+}
+/** Serializable scoring spec declared in a `*.bench.ts` file and uploaded to the platform. */
+interface BenchmarkScoringConfig {
+    /** Optional data key to group records by when computing summary rows (e.g. 'file_size'). */
+    groupBy?: string;
+    /** Extra conditions a record must meet to count as successful. Default: `status === 'success'`. */
+    success?: BenchmarkScoringSuccess;
+    metrics: BenchmarkScoringMetric[];
+}
 interface ScoringSpec {
     dimensions?: Record<string, unknown>;
+    /** Optional data key that groups task records into separate summary rows. */
+    groupBy?: string;
     success?: (record: TaskResultRecord) => boolean;
     metrics: MetricScoring[];
 }
@@ -42,7 +81,7 @@ interface BenchmarkScoreResult {
     skipReason?: string;
 }
 type LowerIsBetter = (name: string, opts: {
-    unit: string;
+    unit?: string;
     ceiling: number;
     value?: MetricValue;
     weights: {
@@ -53,7 +92,7 @@ type LowerIsBetter = (name: string, opts: {
     trim?: number;
 }) => MetricScoring;
 type HigherIsBetter = (name: string, opts: {
-    unit: string;
+    unit?: string;
     floor?: number;
     ceiling: number;
     value?: MetricValue;
@@ -66,7 +105,21 @@ type HigherIsBetter = (name: string, opts: {
 }) => MetricScoring;
 declare const lowerIsBetter: LowerIsBetter;
 declare const higherIsBetter: HigherIsBetter;
-declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec): BenchmarkScoreResult[];
+declare class ScoringSpecError extends Error {
+    constructor(message: string);
+}
+declare function validateScoringSpec(spec: ScoringSpec): void;
+declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec, displayMetrics?: {
+    key: string;
+    unit?: string;
+}[]): BenchmarkScoreResult[];
+/** Builds a runtime {@link ScoringSpec} from a serializable {@link BenchmarkScoringConfig}. */
+declare function scoringConfigToSpec(config: BenchmarkScoringConfig, dimensions?: Record<string, unknown>, display?: {
+    metrics?: {
+        key: string;
+        unit?: string;
+    }[];
+}): ScoringSpec;
 
 /**
  * A `*.bench.ts` file is the composition of a **config** and a **task**:
@@ -131,7 +184,7 @@ interface TaskResult {
      * Pre-measured steps the task timed itself (e.g. socket phases).
      * Only honored in `groupBy: 'round'` runs, where the runner builds records
      * manually. In `groupBy: 'participant'` runs the platform worker
-     * (`client.runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
+     * (`runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
      */
     steps?: TaskStepRecord[];
     /** Task-owned overall latency; overrides framework wall-clock (round mode only). */
@@ -142,8 +195,19 @@ interface TaskStepOptions extends Omit<DefineStepOptions, 'concurrency' | 'stepC
     /** Per-iteration timeout in milliseconds. If an invocation exceeds this, it is aborted and a `step_timeout` TaskError is thrown. */
     timeoutMs?: number;
     /** Number of times to invoke `fn` in parallel. Defaults to 1. When greater than 1, the step returns an array of results. */
+    parallelInvocations?: number;
+    /** @deprecated use `parallelInvocations` instead. */
     concurrency?: number;
 }
+interface TaskErrorOptions {
+    code?: string;
+    data?: JsonObject;
+    steps?: TaskStepRecord[];
+    /** The step name this error was thrown from, when known. */
+    step?: string;
+    /** The timeout that was exceeded, when this is a timeout error. */
+    timeoutMs?: number;
+}
 /**
  * Throw this from a task to record a failure while preserving domain data and
  * any pre-measured steps (a plain thrown Error loses them).
@@ -152,11 +216,10 @@ declare class TaskError extends Error {
     readonly code?: string;
     readonly data?: JsonObject;
     readonly steps?: TaskStepRecord[];
-    constructor(message: string, opts?: {
-        code?: string;
-        data?: JsonObject;
-        steps?: TaskStepRecord[];
-    });
+    readonly step?: string;
+    readonly timeoutMs?: number;
+    constructor(message: string, opts?: TaskErrorOptions);
+    toString(): string;
 }
 /** Context handed to a benchmark `task` for a single iteration. */
 interface TaskContext<T extends BaseParticipant = BaseParticipant> {
@@ -167,12 +230,13 @@ interface TaskContext<T extends BaseParticipant = BaseParticipant> {
     /** Current phase name, when the benchmark declares `phases`. */
     phase?: string;
     /**
-     * Runs `fn` as a named platform step. Mirrors `@benchsdk/client`'s
-     * `RunWorkerContext.step`; supports closures and try/finally. A `concurrency`
+     * Runs `fn` as a named platform step. Mirrors `@benchsdk/worker`'s
+     * `RunWorkerContext.step`; supports closures and try/finally. `parallelInvocations`
      * greater than 1 invokes `fn` that many times in parallel and returns an array.
      * `timeoutMs` aborts any invocation that exceeds it with a `step_timeout` TaskError.
      */
     step<R, C extends number = 1>(name: string, fn: () => Promise<R> | R, options?: TaskStepOptions & {
+        parallelInvocations?: C;
         concurrency?: C;
     }): Promise<C extends 1 ? R : R[]>;
     /**
@@ -180,8 +244,11 @@ interface TaskContext<T extends BaseParticipant = BaseParticipant> {
      * that step's data; at task top-level it lands on the task record's data.
      */
     measure(data: JsonObject): void;
-    /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
-    log(message: string, meta?: JsonObject): void;
+    /**
+     * Appends a line to the worker log, uploaded as an artifact when the worker finishes.
+     * `metaOrOptions` can be a metadata JSON object, or `{ level, meta }` to set a log level.
+     */
+    log(message: string, metaOrOptions?: JsonObject | BenchmarkLogOptions): void;
 }
 type BenchmarkTask<T extends BaseParticipant = BaseParticipant> = (ctx: TaskContext<T>) => Promise<TaskResult | void> | TaskResult | void;
 /**
@@ -203,6 +270,13 @@ interface ParticipantRecords {
 /** The orchestration knobs a run actually used, after CLI overrides. */
 interface ResolvedRunConfig {
     iterations: number;
+    /**
+     * Iterations each phase runs, when the benchmark declares `phases` and
+     * `--iterations` overrode their configured counts. A phase is one arm of a
+     * comparison (a file size), so the flag scales every arm equally rather than
+     * dividing a total between them.
+     */
+    phaseIterations?: number;
     concurrency: number;
     staggerDelayMs: number;
     groupBy: GroupBy;
@@ -270,6 +344,12 @@ interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
     defaultProviders?: string[];
     /** The participants this benchmark can run against. `--provider` selects a subset by name. */
     participants: T[];
+    /**
+     * Static run-level dimensions copied into the submitted summary (e.g.
+     * `{ file_size: '10MB' }`). Useful for distinguishing runs of the same
+     * benchmark that differ by an external parameter.
+     */
+    dimensions?: Record<string, unknown>;
     /**
      * Run-level scoring hook, called once with `lowerIsBetter` and `higherIsBetter`
      * primitives after the outcome is assembled but before `onComplete`. Use it to
@@ -282,6 +362,75 @@ interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
      * writers). This is the run-level counterpart to per-step `ctx.measure`.
      */
     onComplete?: (outcome: BenchmarkRunOutcome) => void | Promise<void>;
+    /**
+     * Serializable scoring spec uploaded to the platform. When provided without
+     * `onScore`, the runner computes the run summary from this spec automatically.
+     * The platform can recompute `compositeScore` from the same spec at read time.
+     */
+    scoring?: BenchmarkScoringConfig;
+    /**
+     * Custom CLI flags this benchmark reads from `process.argv` (e.g. `--file-size`).
+     * Declaring them lets the runner distinguish intentional pass-through flags
+     * from typos and report unknown flags accurately.
+     */
+    customCliFlags?: readonly string[];
+    /**
+     * Optional display manifest. Lets the bench author configure metric labels,
+     * step labels, and overview defaults without editing the platform.
+     */
+    display?: BenchmarkDisplayConfig;
+}
+/** Display metadata for a single custom metric a benchmark reports via `ctx.measure`. */
+interface BenchmarkMetricDisplay {
+    /** Stable metric key, matching the key in `ctx.measure` or `data`. */
+    key: string;
+    /** Human-readable label shown in the platform UI. */
+    label: string;
+    /** Optional unit shown after the value (e.g. `Mbps`, `/s`, `ms`). */
+    unit?: string;
+    /** Number of decimal places when formatting numeric values. Defaults to the display format. */
+    decimals?: number;
+    /** Whether higher or lower values rank better. */
+    direction?: 'higher-better' | 'lower-better';
+    /** Optional ordering hint for metric lists. */
+    order?: number;
+}
+/** Display metadata for a single task lifecycle step. */
+interface BenchmarkStepDisplay {
+    /** Stable step name, matching the string passed to `ctx.step`. */
+    key: string;
+    /** Human-readable label shown in the platform UI. */
+    label: string;
+    /** Optional ordering hint for step lists. */
+    order?: number;
+}
+/** Display defaults for the benchmark overview page. */
+interface BenchmarkOverviewDisplay {
+    /** Metric key to rank participants by by default (falls back to overall task latency). */
+    defaultMetric?: string;
+    /** Default overview layout. */
+    defaultLayout?: 'ranking' | 'cards' | 'chart' | 'leaderboard';
+}
+/**
+ * Optional platform display manifest. A `*.bench.ts` file owns not only how the
+ * benchmark runs, but how it should be rendered, without a platform code change.
+ */
+interface BenchmarkDisplayConfig {
+    /** Metric catalog — labels, units, and ranking direction for `ctx.measure` keys. */
+    metrics?: BenchmarkMetricDisplay[];
+    /** Step catalog — human labels for lifecycle steps reported via `ctx.step`. */
+    steps?: BenchmarkStepDisplay[];
+    /** Overview defaults. */
+    overview?: BenchmarkOverviewDisplay;
+}
+interface BenchmarkConfigErrorItem {
+    field: string;
+    message: string;
+}
+declare class BenchmarkConfigError extends Error {
+    readonly issues: BenchmarkConfigErrorItem[];
+    constructor(issues: BenchmarkConfigErrorItem[]);
+    private static formatIssues;
 }
 /** Validates `config` at file-evaluation time so mistakes surface immediately. */
 declare function defineBenchmarkConfig<T extends BaseParticipant = BaseParticipant>(config: BenchmarkConfig<T>): BenchmarkConfig<T>;
@@ -297,6 +446,16 @@ declare function defineBenchmarkConfig<T extends BaseParticipant = BaseParticipa
  *   });
  */
 declare function defineTask<T extends BaseParticipant = BaseParticipant>(task: BenchmarkTask<T>): BenchmarkTask<T>;
+/**
+ * Validates a `BenchmarkConfig` without throwing, returning a list of
+ * `{ field, message }` issues. Returns an empty array when the config is valid.
+ */
+declare function validateBenchmarkConfig<T extends BaseParticipant = BaseParticipant>(config: BenchmarkConfig<T>): BenchmarkConfigErrorItem[];
+/**
+ * Typed helper for `config.onComplete` callbacks. The body receives the full
+ * `BenchmarkRunOutcome` and can be sync or async.
+ */
+declare function defineOnComplete(onComplete: (outcome: BenchmarkRunOutcome) => void | Promise<void>): (outcome: BenchmarkRunOutcome) => void | Promise<void>;
 
 /**
  * Thrown when every selected participant was env-gated out, i.e. none of their
@@ -336,13 +495,22 @@ interface CliArgs {
     noIngest?: boolean;
 }
 /**
- * Parses the orchestration flags this runner understands, ignoring anything
- * else. Supports both `--flag value` and `--flag=value`; `--provider` accepts
+ * Parses the orchestration flags this runner understands, rejecting unknown
+ * flags. Supports both `--flag value` and `--flag=value`; `--provider` accepts
  * a comma-separated list and may be repeated.
+ *
+ * `allowedCustomFlags` lists pass-through flags the benchmark file reads from
+ * `process.argv` itself; the runner validates and skips them without choking on
+ * their values.
  */
-declare function parseCliArgs(argv: string[]): CliArgs;
+declare function parseCliArgs(argv: string[], allowedCustomFlags?: readonly string[]): CliArgs;
 /** Merges CLI overrides over config defaults, filling in knob fallbacks. */
 declare function mergeConfig<T extends BaseParticipant>(config: BenchmarkConfig<T>, args: CliArgs): ResolvedRunConfig;
+interface PlatformConfig {
+    baseUrl?: string;
+    apiKey?: string;
+}
+type RunBenchmarkOptions = PlatformConfig;
 /**
  * Runs `config`'s `task` against its participants. Selects participants by
  * `--provider` (if given), env-gates them, then drives them per the resolved
@@ -352,7 +520,26 @@ declare function mergeConfig<T extends BaseParticipant>(config: BenchmarkConfig<
  * CI job per provider) get-or-create one shared run and each registers only its
  * own participants.
  */
-declare function runBenchmark<T extends BaseParticipant>(fileConfig: BenchmarkConfig<T>, task: BenchmarkTask<T>, argv?: string[]): Promise<BenchmarkRunOutcome>;
+declare function runBenchmark<T extends BaseParticipant>(fileConfig: BenchmarkConfig<T>, task: BenchmarkTask<T>, argv?: string[], options?: RunBenchmarkOptions): Promise<BenchmarkRunOutcome>;
+interface RunBenchmarkWorkerOptions<T extends BaseParticipant = BaseParticipant> {
+    benchmarkSlug: string;
+    benchmarkName?: string;
+    runKey?: string;
+    participant: T;
+    task: BenchmarkTask<T>;
+    iterations?: number;
+    concurrency?: number;
+    staggerDelayMs?: number;
+    groupBy?: GroupBy;
+    noIngest?: boolean;
+}
+/**
+ * One-shot helper: run a single participant's worker for a benchmark without
+ * creating a `*.bench.ts` file. This is a convenience wrapper around
+ * `runBenchmark` that builds a minimal `BenchmarkConfig` from the supplied
+ * options.
+ */
+declare function runBenchmarkWorker<T extends BaseParticipant>(options: RunBenchmarkWorkerOptions<T>): Promise<BenchmarkRunOutcome>;
 
 /**
  * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets
@@ -360,7 +547,9 @@ declare function runBenchmark<T extends BaseParticipant>(fileConfig: BenchmarkCo
  * exit. Does not call `process.exit`.
  */
 declare function runBenchmarkFile(argv: string[]): Promise<void>;
-/** Executable entry: runs the file and maps outcomes to process exit codes. */
+/** Executable entry: dispatches to benchmark execution or platform data commands. */
 declare function run(argv: string[]): Promise<void>;
 
-export { type BenchmarkConfig, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type ResolvedRunConfig, type ScoringSpec, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, score };
+declare const BENCHSDK_RUNNER_VERSION: string;
+
+export { BENCHSDK_RUNNER_VERSION, type BenchmarkConfig, BenchmarkConfigError, type BenchmarkConfigErrorItem, type BenchmarkDisplayConfig, type BenchmarkMetricDisplay, type BenchmarkOverviewDisplay, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkScoringConfig, type BenchmarkScoringMetric, type BenchmarkScoringSuccess, type BenchmarkScoringWeights, type BenchmarkStepDisplay, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type PlatformConfig, type ResolvedRunConfig, type RunBenchmarkOptions, type RunBenchmarkWorkerOptions, type ScoringSpec, ScoringSpecError, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineOnComplete, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, runBenchmarkWorker, score, scoringConfigToSpec, validateBenchmarkConfig, validateScoringSpec };