@benchsdk/runner 0.2.0

package/dist/index.d.ts

@@ -0,0 +1,366 @@
+import { TaskResultRecord, JsonObject, BaseParticipant, DefineStepOptions, TaskStepRecord } from '@benchsdk/client';
+
+type MetricValue = string | ((record: TaskResultRecord) => number | number[] | undefined);
+interface MetricScoring {
+    name: string;
+    value?: MetricValue;
+    unit: string;
+    ceiling: number;
+    floor?: number;
+    higherIsBetter?: boolean;
+    weights: {
+        median: number;
+        p95: number;
+        p99: number;
+    };
+    trim?: number;
+}
+interface ScoringSpec {
+    dimensions?: Record<string, unknown>;
+    success?: (record: TaskResultRecord) => boolean;
+    metrics: MetricScoring[];
+}
+interface BenchmarkScoreResult {
+    provider: string;
+    dimensions: JsonObject;
+    metrics: {
+        name: string;
+        unit: string;
+        median: number;
+        p95: number;
+        p99: number;
+    }[];
+    scalars?: {
+        name: string;
+        value: number;
+        unit: string;
+    }[];
+    compositeScore: number;
+    successRate: number;
+    scoringVersion?: string;
+    skipped: boolean;
+    skipReason?: string;
+}
+type LowerIsBetter = (name: string, opts: {
+    unit: string;
+    ceiling: number;
+    value?: MetricValue;
+    weights: {
+        median: number;
+        p95: number;
+        p99: number;
+    };
+    trim?: number;
+}) => MetricScoring;
+type HigherIsBetter = (name: string, opts: {
+    unit: string;
+    floor?: number;
+    ceiling: number;
+    value?: MetricValue;
+    weights: {
+        median: number;
+        p95: number;
+        p99: number;
+    };
+    trim?: number;
+}) => MetricScoring;
+declare const lowerIsBetter: LowerIsBetter;
+declare const higherIsBetter: HigherIsBetter;
+declare function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec): BenchmarkScoreResult[];
+
+/**
+ * A `*.bench.ts` file is the composition of a **config** and a **task**:
+ *
+ *   export const config = defineBenchmarkConfig({ benchmarkSlug, participants, ... });
+ *   export const task = defineTask(async (ctx) => { await ctx.step('work', () => ...); });
+ *
+ * `defineBenchmarkConfig` holds the orchestration knobs (including the
+ * participants and an optional `onComplete` hook); `defineTask` holds the
+ * workload. The `bench run <file>` binary imports the module, reads those two
+ * exports, and drives the run. There is no "mode": all orchestration shapes
+ * emerge from the knobs.
+ *
+ *   iterations       total tasks to run (default 1)
+ *   concurrency      max tasks in flight at once — 1 = sequential, N = burst (default 1)
+ *   staggerDelayMs   delay each task's start by taskIndex * staggerDelayMs (default 0)
+ *
+ * Common shapes:
+ *   sequential  { iterations: N, concurrency: 1 }
+ *   burst       { iterations: N, concurrency: N }
+ *   staggered   { iterations: N, concurrency: N, staggerDelayMs: 200 }
+ *
+ * A benchmark can name these variants up front via `shapes`, so one file backs
+ * several platform benchmarks (`bench run <file> --shape burst`) without
+ * restating each one's slug/name in scripts and CI.
+ *
+ * A task is comprised of steps, declared via `ctx.step` inside a task function
+ * — it supports closures, conditionals and try/finally, so values (a created
+ * sandbox, say) flow naturally between steps. A task that declares no steps is
+ * recorded as a single implicit `task` step. Measurements reach the platform
+ * via `ctx.measure(...)`; step return values are control flow and never
+ * recorded.
+ */
+
+/** How tasks are ordered across participants. */
+type GroupBy = 'participant' | 'round';
+/**
+ * A named variant of a benchmark, selected with `--shape <name>`. A shape
+ * carries only the parts that make it a distinct *benchmark* — its platform
+ * identity plus any stable distinguishing knob (e.g. staggered's delay). The
+ * scale knobs that vary per environment (`--iterations`, `--concurrency`) stay
+ * on the invocation, so a shape never sets a value only to have the CLI
+ * override it.
+ */
+interface BenchmarkShape {
+    /** Platform slug this shape reports under (e.g. 'sandbox-tti'). */
+    slug: string;
+    /** Display name shown on the platform; defaults to the slug. */
+    name?: string;
+    /** Default stagger delay (ms) for this shape; overridable with `--stagger-delay-ms`. */
+    staggerDelayMs?: number;
+}
+/**
+ * What a task returns: whatever it measured itself. This replaces the
+ * assumption that the framework owns all timing. A plain data payload is
+ * written explicitly as `{ data: {...} }`.
+ */
+interface TaskResult {
+    /** Free-form domain payload attached to the record (tokens, receipts, ...). */
+    data?: JsonObject;
+    /**
+     * Pre-measured steps the task timed itself (e.g. socket phases).
+     * Only honored in `groupBy: 'round'` runs, where the runner builds records
+     * manually. In `groupBy: 'participant'` runs the platform worker
+     * (`client.runWorker`) owns steps, so `steps` and `latencyMs` are ignored.
+     */
+    steps?: TaskStepRecord[];
+    /** Task-owned overall latency; overrides framework wall-clock (round mode only). */
+    latencyMs?: number;
+}
+/** Options for a single `ctx.step` invocation. */
+interface TaskStepOptions extends Omit<DefineStepOptions, 'concurrency' | 'stepConcurrency'> {
+    /** Per-iteration timeout in milliseconds. If an invocation exceeds this, it is aborted and a `step_timeout` TaskError is thrown. */
+    timeoutMs?: number;
+    /** Number of times to invoke `fn` in parallel. Defaults to 1. When greater than 1, the step returns an array of results. */
+    concurrency?: number;
+}
+/**
+ * Throw this from a task to record a failure while preserving domain data and
+ * any pre-measured steps (a plain thrown Error loses them).
+ */
+declare class TaskError extends Error {
+    readonly code?: string;
+    readonly data?: JsonObject;
+    readonly steps?: TaskStepRecord[];
+    constructor(message: string, opts?: {
+        code?: string;
+        data?: JsonObject;
+        steps?: TaskStepRecord[];
+    });
+}
+/** Context handed to a benchmark `task` for a single iteration. */
+interface TaskContext<T extends BaseParticipant = BaseParticipant> {
+    /** The participant this task is running for. */
+    participant: T;
+    /** Zero-based global task ordinal (matches the platform record's taskIndex). */
+    taskIndex: number;
+    /** Current phase name, when the benchmark declares `phases`. */
+    phase?: string;
+    /**
+     * Runs `fn` as a named platform step. Mirrors `@benchsdk/client`'s
+     * `RunWorkerContext.step`; supports closures and try/finally. A `concurrency`
+     * greater than 1 invokes `fn` that many times in parallel and returns an array.
+     * `timeoutMs` aborts any invocation that exceeds it with a `step_timeout` TaskError.
+     */
+    step<R, C extends number = 1>(name: string, fn: () => Promise<R> | R, options?: TaskStepOptions & {
+        concurrency?: C;
+    }): Promise<C extends 1 ? R : R[]>;
+    /**
+     * Attaches a JSON measurement to the platform. Inside a `step` it lands on
+     * that step's data; at task top-level it lands on the task record's data.
+     */
+    measure(data: JsonObject): void;
+    /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
+    log(message: string, meta?: JsonObject): void;
+}
+type BenchmarkTask<T extends BaseParticipant = BaseParticipant> = (ctx: TaskContext<T>) => Promise<TaskResult | void> | TaskResult | void;
+/**
+ * A named run segment with its own iteration count. Phases run in order; each
+ * record is tagged with the phase name via `data.phase`, and `ctx.phase` lets
+ * the task branch on identity instead of index arithmetic.
+ */
+interface Phase {
+    /** Phase name, tagged onto every record produced in this phase. */
+    name: string;
+    /** Iterations to run in this phase. */
+    iterations: number;
+}
+/** One participant's collected task records from a run. */
+interface ParticipantRecords {
+    participant: string;
+    records: TaskResultRecord[];
+}
+/** The orchestration knobs a run actually used, after CLI overrides. */
+interface ResolvedRunConfig {
+    iterations: number;
+    concurrency: number;
+    staggerDelayMs: number;
+    groupBy: GroupBy;
+    providers?: string[];
+}
+/**
+ * Result of a benchmark run, passed to `config.onComplete`. Exposes the raw
+ * per-participant records so completion hooks can write legacy local results.
+ */
+interface BenchmarkRunOutcome {
+    runId: string;
+    /** Link to this run on the platform dashboard. */
+    dashboardUrl: string;
+    participants: ParticipantRecords[];
+    config: ResolvedRunConfig;
+}
+/**
+ * Orchestration config for a benchmark. Holds identity, the knobs, the
+ * participants, and the optional completion hook — the workload lives in a
+ * separate `defineTask`. `bench run <file>` reads the `config` and `task`
+ * exports from the module and drives the run.
+ */
+interface BenchmarkConfig<T extends BaseParticipant = BaseParticipant> {
+    /**
+     * Stable platform slug for this benchmark (e.g. 'sandbox-tti-local').
+     * Selectable per run with `--shape` (or overridable with `--benchmark`), so
+     * one entrypoint can report under several benchmarks.
+     */
+    benchmarkSlug: string;
+    /** Human-readable name shown on the platform. Overridable with `--name`. */
+    benchmarkName: string;
+    /**
+     * Named variants of this benchmark, selected with `--shape <name>`. Each
+     * shape swaps in its own platform identity (and optional stable knob) while
+     * reusing the same task and participants, so one bench file can back several
+     * platform benchmarks without duplicating the slug/name triple across
+     * package scripts and CI.
+     */
+    shapes?: Record<string, BenchmarkShape>;
+    /**
+     * Total tasks to run per participant. Default: 1. Mutually exclusive with
+     * `phases` — when `phases` is set, total iterations = sum of phase iterations.
+     */
+    iterations?: number;
+    /**
+     * Named run segments (e.g. cold/warm). Runs in order; each record is tagged
+     * with the phase name via `data.phase`. Mutually exclusive with `iterations`.
+     */
+    phases?: Phase[];
+    /** Max tasks in flight at once. 1 = sequential, N = burst. Default: 1. */
+    concurrency?: number;
+    /** Delay each task's start by `taskIndex * staggerDelayMs`. Default: 0. */
+    staggerDelayMs?: number;
+    /**
+     * Task ordering across participants. Default: 'participant' (run each
+     * participant's tasks to completion, then the next). 'round' takes turns:
+     * every participant runs its Nth task before anyone runs their (N+1)th, so
+     * all participants' Nth tasks happen back-to-back under the same conditions.
+     */
+    groupBy?: GroupBy;
+    /**
+     * Default participant names to run when `--provider` is not passed. Omit to
+     * run all env-available participants. `--provider` always overrides this.
+     */
+    defaultProviders?: string[];
+    /** The participants this benchmark can run against. `--provider` selects a subset by name. */
+    participants: T[];
+    /**
+     * Run-level scoring hook, called once with `lowerIsBetter` and `higherIsBetter`
+     * primitives after the outcome is assembled but before `onComplete`. Use it to
+     * define how the run should be scored and reported to the platform.
+     */
+    onScore?: (lowerIsBetter: LowerIsBetter, higherIsBetter: HigherIsBetter) => ScoringSpec | Promise<ScoringSpec>;
+    /**
+     * Run-level completion hook, called once with the full outcome after every
+     * participant finishes. Use it for aggregate output (legacy JSON/SVG
+     * writers). This is the run-level counterpart to per-step `ctx.measure`.
+     */
+    onComplete?: (outcome: BenchmarkRunOutcome) => void | Promise<void>;
+}
+/** Validates `config` at file-evaluation time so mistakes surface immediately. */
+declare function defineBenchmarkConfig<T extends BaseParticipant = BaseParticipant>(config: BenchmarkConfig<T>): BenchmarkConfig<T>;
+/**
+ * Declares the workload for a benchmark: a function invoked once per iteration.
+ * Steps are named via `ctx.step`, which supports closures and try/finally so
+ * values flow naturally between steps.
+ *
+ *   export const task = defineTask(async (ctx) => {
+ *     const sandbox = await ctx.step('create', () => provider.create());
+ *     try { await ctx.step('exec', () => sandbox.run('node -v')); }
+ *     finally { await ctx.step('destroy', () => sandbox.destroy()); }
+ *   });
+ */
+declare function defineTask<T extends BaseParticipant = BaseParticipant>(task: BenchmarkTask<T>): BenchmarkTask<T>;
+
+/**
+ * Thrown when every selected participant was env-gated out, i.e. none of their
+ * `requiredEnvVars` are set. This is a "nothing to do" outcome rather than a
+ * failure — a benchmark job for a provider whose credentials aren't provisioned
+ * should skip, not go red — so callers are expected to exit 0 on it.
+ */
+declare class NoAvailableParticipantsError extends Error {
+    readonly skipped: {
+        name: string;
+        missing: string[];
+    }[];
+    constructor(skipped: {
+        name: string;
+        missing: string[];
+    }[]);
+}
+
+interface CliArgs {
+    /** Which platform benchmark to report as (`--benchmark`, aka the benchmark slug). */
+    benchmark?: string;
+    name?: string;
+    /** Named variant from the bench file's `shapes` (`--shape`), swapping in its identity. */
+    shape?: string;
+    /**
+     * Idempotency key (`--run-key`): sibling processes passing the same key share
+     * one run (get-or-created), instead of each opening its own.
+     */
+    runKey?: string;
+    iterations?: number;
+    concurrency?: number;
+    staggerDelayMs?: number;
+    groupBy?: GroupBy;
+    /** Participant names from `--provider a,b` (repeatable). */
+    providers?: string[];
+    /** When true, run locally and do not ingest/report to the platform. */
+    noIngest?: boolean;
+}
+/**
+ * Parses the orchestration flags this runner understands, ignoring anything
+ * else. Supports both `--flag value` and `--flag=value`; `--provider` accepts
+ * a comma-separated list and may be repeated.
+ */
+declare function parseCliArgs(argv: string[]): CliArgs;
+/** Merges CLI overrides over config defaults, filling in knob fallbacks. */
+declare function mergeConfig<T extends BaseParticipant>(config: BenchmarkConfig<T>, args: CliArgs): ResolvedRunConfig;
+/**
+ * Runs `config`'s `task` against its participants. Selects participants by
+ * `--provider` (if given), env-gates them, then drives them per the resolved
+ * `groupBy`. `--shape` swaps in a declared variant's identity; `--benchmark`/
+ * `--name` retarget the run at a different platform benchmark, so one entrypoint
+ * can report under several slugs. With `--run-key`, sibling processes (e.g. one
+ * CI job per provider) get-or-create one shared run and each registers only its
+ * own participants.
+ */
+declare function runBenchmark<T extends BaseParticipant>(fileConfig: BenchmarkConfig<T>, task: BenchmarkTask<T>, argv?: string[]): Promise<BenchmarkRunOutcome>;
+
+/**
+ * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets
+ * `NoAvailableParticipantsError` propagate so the caller can map it to a clean
+ * exit. Does not call `process.exit`.
+ */
+declare function runBenchmarkFile(argv: string[]): Promise<void>;
+/** Executable entry: runs the file and maps outcomes to process exit codes. */
+declare function run(argv: string[]): Promise<void>;
+
+export { type BenchmarkConfig, type BenchmarkRunOutcome, type BenchmarkScoreResult, type BenchmarkTask, type CliArgs, type GroupBy, type MetricScoring, NoAvailableParticipantsError, type ParticipantRecords, type Phase, type ResolvedRunConfig, type ScoringSpec, type TaskContext, TaskError, type TaskResult, type TaskStepOptions, defineBenchmarkConfig, defineTask, higherIsBetter, lowerIsBetter, mergeConfig, parseCliArgs, run, runBenchmark, runBenchmarkFile, score };