@benchsdk/client 0.2.1 → 0.3.0

package/dist/index.d.cts

@@ -14,7 +14,6 @@ interface BenchmarkResource {
     id: string;
     slug: string;
     name: string;
-    kind?: string | null;
     status?: string;
     config?: JsonObject;
     defaultRunConfig?: JsonObject;
@@ -26,7 +25,11 @@ interface BenchmarkRun {
     benchmarkId: string;
     name?: string | null;
     status: BenchmarkRunStatus | string;
+    /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */
+    runKey?: string | null;
     totalTasks: number;
+    /** The run declared no size: `totalTasks` is the sum of what its participants declare. */
+    participantSized?: boolean;
     workerCount: number;
     config?: JsonObject;
     createdAt?: string;
@@ -93,27 +96,29 @@ interface BenchmarkAssignment {
 }
 interface UpsertBenchmarkInput {
     name: string;
-    kind?: string;
     status?: string;
     config?: JsonObject;
     defaultRunConfig?: JsonObject;
 }
 interface UpdateBenchmarkInput {
     name?: string;
-    kind?: string;
     status?: string;
     config?: JsonObject;
     defaultRunConfig?: JsonObject;
 }
 interface CreateRunInput {
-    name?: string;
-    totalTasks: number;
-    workerCount: number;
+    /**
+     * Idempotency key for get-or-create: sibling callers passing the same key
+     * (per org + benchmark) converge on one run instead of each opening its own.
+     */
+    runKey?: string;
+    /** Omit to open a participant-sized run: each participant declares its own size when it registers. */
+    totalTasks?: number;
+    workerCount?: number;
     participants?: string[];
     config?: JsonObject;
 }
 interface UpdateRunInput {
-    name?: string;
     status?: BenchmarkRunStatus;
     config?: JsonObject;
 }
@@ -161,6 +166,10 @@ interface TaskStepRecord {
     latencyMs?: number;
     errorCode?: string | null;
     data?: JsonObject;
+    /** Number of parallel invocations requested for this step. */
+    concurrency?: number;
+    /** Per-iteration timeout in milliseconds applied to this step. */
+    timeoutMs?: number;
 }
 interface SendTaskResultsInput {
     benchmarkSlug: string;
@@ -280,13 +289,13 @@ interface BenchmarkResultsOverviewRun {
     }>;
 }
 interface BenchmarkResultsOverview {
-    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name' | 'kind'>;
+    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
     generatedAt: string;
     analytics: BenchmarkResultsOverviewAnalytics;
     items: BenchmarkResultsOverviewRun[];
 }
 interface BenchmarkRunResults {
-    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name' | 'kind'>;
+    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
     run: Pick<BenchmarkRun, 'id' | 'status' | 'totalTasks' | 'workerCount'>;
     generatedAt: string;
     overall: BenchmarkResultSummary;
@@ -474,6 +483,15 @@ interface RunWorkerContext {
     assignment: BenchmarkAssignment;
     taskIndex: number;
     step<T>(name: string, fn: () => Promise<T> | T, options?: DefineStepOptions): Promise<T>;
+    /**
+     * Attaches a JSON measurement to the platform. Called inside a `step`, it
+     * lands on that step's `data`; called at task top-level, on the task record's
+     * `data`. Repeated calls merge (shallow). Use this for anything you want on
+     * the platform — step return values are control flow and are never recorded.
+     */
+    measure(data: JsonObject): void;
+    /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
+    log(message: string, meta?: JsonObject): void;
 }
 interface WorkerFinishContext {
     assignment: BenchmarkAssignment;
@@ -482,17 +500,15 @@ interface WorkerFinishContext {
     client: BenchmarkClient;
     uploadArtifact(input: Omit<UploadWorkerArtifactInput, 'attemptId'>): Promise<CreateWorkerArtifactResponse>;
 }
-interface StepContext<TState extends Record<string, unknown> = Record<string, unknown>> {
-    assignment: BenchmarkAssignment;
-    taskIndex: number;
-    state: TState;
-}
-type CleanupContext<TState extends Record<string, unknown> = Record<string, unknown>> = StepContext<TState>;
 interface DefineStepOptions {
     /** Report this step as active in heartbeat concurrency samples. Defaults to true. */
     reportConcurrency?: boolean;
     /** Per-worker target for this step. Defaults to worker concurrency/assignment target. */
     concurrency?: number;
+    /** Number of parallel invocations the step function should run internally. Used by the runner to record step-level concurrency. */
+    stepConcurrency?: number;
+    /** Per-invocation timeout in milliseconds for this step. Used by the runner to record step-level timeout metadata. */
+    timeoutMs?: number;
     /** Readiness coordination mode. Defaults to internal. */
     readiness?: 'poll' | 'internal';
     /** Poll interval while waiting for readiness. Defaults to 1000ms. */
@@ -500,25 +516,13 @@ interface DefineStepOptions {
     /** Maximum time to wait for readiness. Defaults to no timeout. */
     readyTimeoutMs?: number;
 }
-interface DefinedStep<TState extends Record<string, unknown> = Record<string, unknown>> {
-    name: string;
-    options?: DefineStepOptions;
-    fn: (context: StepContext<TState>) => Promise<JsonObject | void> | JsonObject | void;
-}
-interface DefineTaskOptions<TState extends Record<string, unknown> = Record<string, unknown>> {
-    /**
-     * Runs after the task finishes, whether it succeeded or failed.
-     * Use this to tear down resources stored in task state.
-     */
-    cleanup?: (context: CleanupContext<TState>) => Promise<void> | void;
-}
-interface DefinedTask<TState extends Record<string, unknown> = Record<string, unknown>> {
-    name: string;
-    steps: DefinedStep<TState>[];
-    options?: DefineTaskOptions<TState>;
-}
+/**
+ * The unit of work a worker runs, once per task index. Steps are declared
+ * imperatively via `context.step(...)`; this is the sole task shape the worker
+ * engine accepts. Higher-level authoring (`defineTask`) lives in
+ * `@benchsdk/runner`, which compiles down to a function of this shape.
+ */
 type TaskFunction = (context: RunWorkerContext) => Promise<JsonObject | void> | JsonObject | void;
-type WorkerTask = DefinedTask | TaskFunction;
 interface RunWorkerResult {
     assignment: BenchmarkAssignment | null;
     records: TaskResultRecord[];
@@ -537,42 +541,42 @@ interface RunWorkerOptions {
     onResult?: (record: TaskResultRecord) => void;
     /** Runs once after final result flush and before worker completion/failure is reported. */
     onFinish?: (context: WorkerFinishContext) => Promise<void> | void;
-    task: WorkerTask;
+    task: TaskFunction;
 }
-interface WorkerDefaults {
-    concurrency?: number;
-    batchSize?: number;
-    flushIntervalMs?: number;
-    heartbeatIntervalMs?: number;
-    readyPollIntervalMs?: number;
-}
-interface DefineWorkerOptions extends WorkerDefaults {
-    benchmarkSlug: string;
-    runId: string;
-    participantSlug: string;
-    processKind?: string;
-    processKey?: string;
-    client?: BenchmarkClient;
-    onFinish?: RunWorkerOptions['onFinish'];
-    task: WorkerTask;
-}
-interface BenchmarkWorker {
-    run(overrides?: Partial<WorkerDefaults>): Promise<RunWorkerResult>;
-}
-interface DefineBenchOptions extends WorkerDefaults {
-    slug: string;
-    participantSlug?: string;
-    client?: BenchmarkClient;
-    task: WorkerTask;
+interface BenchmarkRunSummaryMetric {
+    name: string;
+    unit: string;
+    median: number;
+    p95: number;
+    p99: number;
 }
-interface BenchDefinition {
-    slug: string;
-    task: WorkerTask;
-    defineWorker(options: Omit<DefineWorkerOptions, 'benchmarkSlug' | 'client' | 'task' | 'participantSlug'> & {
-        client?: BenchmarkClient;
-        participantSlug?: string;
-        task?: WorkerTask;
-    }): BenchmarkWorker;
+interface BenchmarkRunSummaryScalar {
+    name: string;
+    value: number;
+    unit: string;
+}
+interface BenchmarkRunSummaryResult {
+    provider: string;
+    dimensions?: Record<string, unknown>;
+    metrics: BenchmarkRunSummaryMetric[];
+    scalars?: BenchmarkRunSummaryScalar[];
+    compositeScore: number;
+    successRate: number;
+    scoringVersion?: string | null;
+    skipped: boolean;
+    skipReason?: string | null;
+}
+interface BenchmarkRunSummaryRunMetadata {
+    gitSha?: string;
+    gitRef?: string;
+    triggeredBy?: string;
+    nodeVersion?: string;
+    platform?: string;
+    arch?: string;
+}
+interface BenchmarkRunSummaryInput {
+    run: BenchmarkRunSummaryRunMetadata;
+    results: BenchmarkRunSummaryResult[];
 }
 interface BenchmarkClient {
     upsertBenchmark(slug: string, input: UpsertBenchmarkInput): Promise<BenchmarkResource>;
@@ -582,6 +586,8 @@ interface BenchmarkClient {
     createRun(benchmarkSlug: string, input: CreateRunInput): Promise<{
         run: BenchmarkRun;
         participants: BenchmarkParticipant[];
+        /** The slug of the org the run was attributed to, resolved server-side from the caller's API key. */
+        organizationSlug: string;
     }>;
     listRuns(benchmarkSlug: string): Promise<BenchmarkRun[]>;
     getRun(benchmarkSlug: string, runId: string): Promise<BenchmarkRun>;
@@ -622,6 +628,7 @@ interface BenchmarkClient {
     getRunTaskResults(benchmarkSlug: string, runId: string, input?: BenchmarkRunTaskResultsInput): Promise<BenchmarkRunTaskResults>;
     getRunTimeline(benchmarkSlug: string, runId: string, input?: BenchmarkRunTimelineInput): Promise<BenchmarkRunTimeline>;
     getRunImports(benchmarkSlug: string, runId: string): Promise<BenchmarkRunImports>;
+    submitRunSummary(benchmarkSlug: string, runId: string, input: BenchmarkRunSummaryInput): Promise<void>;
     runWorker(options: RunWorkerOptions): Promise<RunWorkerResult>;
 }
 
@@ -631,11 +638,6 @@ declare class BenchmarkApiError extends Error {
     constructor(message: string, status: number, body: string);
 }
 declare function createBenchmarkClient(config?: BenchmarkClientConfig): BenchmarkClient;
-declare function runBenchmarkWorker(config: BenchmarkClientConfig, options: RunWorkerOptions): Promise<RunWorkerResult>;
-declare function defineStep<TState extends Record<string, unknown> = Record<string, unknown>>(name: string, optionsOrFn: DefineStepOptions | DefinedStep<TState>['fn'], maybeFn?: DefinedStep<TState>['fn']): DefinedStep<TState>;
-declare function defineTask<TState extends Record<string, unknown> = Record<string, unknown>>(name: string, steps: DefinedStep<TState>[], options?: DefineTaskOptions<TState>): DefinedTask<TState>;
-declare function defineWorker(options: DefineWorkerOptions): BenchmarkWorker;
-declare function defineBench(options: DefineBenchOptions): BenchDefinition;
 
 interface BenchmarkReporterConfig extends BenchmarkClientConfig {
     benchmarkSlug: string;
@@ -724,4 +726,35 @@ interface BenchmarkSystemMetricsCollector {
 }
 declare function createSystemMetricsCollector(): BenchmarkSystemMetricsCollector;
 
-export { type BenchDefinition, type BenchmarkAnalyticsReadiness, BenchmarkApiError, type BenchmarkArtifact, type BenchmarkAssignment, type BenchmarkClient, type BenchmarkClientConfig, type BenchmarkConcurrencyPoint, type BenchmarkEventRateBucket, type BenchmarkFailurePoint, type BenchmarkParticipant, BenchmarkReporter, type BenchmarkReporterArtifactInput, type BenchmarkReporterBarrierInput, type BenchmarkReporterBarrierResult, type BenchmarkReporterConfig, type BenchmarkReporterHeartbeatInput, type BenchmarkReporterProgress, type BenchmarkResource, type BenchmarkResultLatencySummary, type BenchmarkResultSummary, type BenchmarkResultsOverview, type BenchmarkResultsOverviewAnalytics, type BenchmarkResultsOverviewInput, type BenchmarkResultsOverviewRun, type BenchmarkRun, type BenchmarkRunAnalyticsSummary, type BenchmarkRunImportItem, type BenchmarkRunImports, type BenchmarkRunImportsSummary, type BenchmarkRunResults, type BenchmarkRunStatus, type BenchmarkRunTaskResults, type BenchmarkRunTaskResultsInput, type BenchmarkRunTimeline, type BenchmarkRunTimelineInput, type BenchmarkRunWorker, type BenchmarkStepResultSummary, type BenchmarkSystemMetricsCollector, type BenchmarkSystemMetricsSample, type BenchmarkTaskBucket, type BenchmarkWorker, type BenchmarkWorkerAttempt, type BenchmarkWorkerStatus, type ClaimWorkerInput, type CreateRunInput, type CreateWorkerArtifactInput, type CreateWorkerArtifactResponse, type DefineBenchOptions, type DefineStepOptions, type DefineWorkerOptions, type DefinedStep, type DefinedTask, type JsonObject, type JsonValue, type PlanWorkersInput, type RunProgress, type RunProgressConcurrency, type RunProgressParticipant, type RunProgressParticipantCounts, type RunProgressStatus, type RunProgressSummary, type RunProgressTaskCounts, type RunProgressWorkerCounts, type RunWorkerContext, type RunWorkerOptions, type RunWorkerResult, type SendTaskResultsInput, type TaskFunction, type TaskResultRecord, type TaskResultsResponse, type TaskStepRecord, type UpdateBenchmarkInput, type UpdateParticipantInput, type UpdateRunInput, type UpdateWorkerInput, type UploadWorkerArtifactInput, type UpsertBenchmarkInput, type UpsertParticipantInput, type WorkerConcurrencySample, type WorkerFinishContext, type WorkerHeartbeatInput, type WorkerTask, claimBenchmarkReporter, createBenchmarkClient, createSystemMetricsCollector, defineBench, defineStep, defineTask, defineWorker, runBenchmarkWorker };
+/**
+ * Base interface for a benchmark participant — the shared shape across all
+ * benchmark categories (sandbox, ai-gateway, browser, storage). Each category
+ * extends this with its own provider-specific fields.
+ */
+interface BaseParticipant {
+    /** Participant name (e.g. 'e2b', 'daytona', 'openrouter') */
+    name: string;
+    /** Environment variables that must all be set to run this participant */
+    requiredEnvVars: string[];
+}
+/**
+/**
+ * Filters `participants` down to those whose `requiredEnvVars` are all set
+ * in `process.env`. Returns an object `{ available, skipped }` where `skipped`
+ * includes the names and missing vars for logging.
+ */
+declare function filterParticipantsByEnv<T extends BaseParticipant>(participants: T[]): {
+    available: T[];
+    skipped: {
+        name: string;
+        missing: string[];
+    }[];
+};
+/**
+ * Filters `all` down to the requested `names`, exiting with a clear error
+ * if any name is unrecognized. Returns `all` unchanged when `names` is
+ * undefined (no filter specified).
+ */
+declare function selectParticipants<T extends BaseParticipant>(all: T[], names?: string[]): T[];
+
+export { type BaseParticipant, type BenchmarkAnalyticsReadiness, BenchmarkApiError, type BenchmarkArtifact, type BenchmarkAssignment, type BenchmarkClient, type BenchmarkClientConfig, type BenchmarkConcurrencyPoint, type BenchmarkEventRateBucket, type BenchmarkFailurePoint, type BenchmarkParticipant, BenchmarkReporter, type BenchmarkReporterArtifactInput, type BenchmarkReporterBarrierInput, type BenchmarkReporterBarrierResult, type BenchmarkReporterConfig, type BenchmarkReporterHeartbeatInput, type BenchmarkReporterProgress, type BenchmarkResource, type BenchmarkResultLatencySummary, type BenchmarkResultSummary, type BenchmarkResultsOverview, type BenchmarkResultsOverviewAnalytics, type BenchmarkResultsOverviewInput, type BenchmarkResultsOverviewRun, type BenchmarkRun, type BenchmarkRunAnalyticsSummary, type BenchmarkRunImportItem, type BenchmarkRunImports, type BenchmarkRunImportsSummary, type BenchmarkRunResults, type BenchmarkRunStatus, type BenchmarkRunSummaryInput, type BenchmarkRunSummaryMetric, type BenchmarkRunSummaryResult, type BenchmarkRunSummaryRunMetadata, type BenchmarkRunSummaryScalar, type BenchmarkRunTaskResults, type BenchmarkRunTaskResultsInput, type BenchmarkRunTimeline, type BenchmarkRunTimelineInput, type BenchmarkRunWorker, type BenchmarkStepResultSummary, type BenchmarkSystemMetricsCollector, type BenchmarkSystemMetricsSample, type BenchmarkTaskBucket, type BenchmarkWorkerAttempt, type BenchmarkWorkerStatus, type ClaimWorkerInput, type CreateRunInput, type CreateWorkerArtifactInput, type CreateWorkerArtifactResponse, type DefineStepOptions, type JsonObject, type JsonValue, type PlanWorkersInput, type RunProgress, type RunProgressConcurrency, type RunProgressParticipant, type RunProgressParticipantCounts, type RunProgressStatus, type RunProgressSummary, type RunProgressTaskCounts, type RunProgressWorkerCounts, type RunWorkerContext, type RunWorkerOptions, type RunWorkerResult, type SendTaskResultsInput, type TaskFunction, type TaskResultRecord, type TaskResultsResponse, type TaskStepRecord, type UpdateBenchmarkInput, type UpdateParticipantInput, type UpdateRunInput, type UpdateWorkerInput, type UploadWorkerArtifactInput, type UpsertBenchmarkInput, type UpsertParticipantInput, type WorkerConcurrencySample, type WorkerFinishContext, type WorkerHeartbeatInput, claimBenchmarkReporter, createBenchmarkClient, createSystemMetricsCollector, filterParticipantsByEnv, selectParticipants };

package/dist/index.d.ts

@@ -14,7 +14,6 @@ interface BenchmarkResource {
     id: string;
     slug: string;
     name: string;
-    kind?: string | null;
     status?: string;
     config?: JsonObject;
     defaultRunConfig?: JsonObject;
@@ -26,7 +25,11 @@ interface BenchmarkRun {
     benchmarkId: string;
     name?: string | null;
     status: BenchmarkRunStatus | string;
+    /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */
+    runKey?: string | null;
     totalTasks: number;
+    /** The run declared no size: `totalTasks` is the sum of what its participants declare. */
+    participantSized?: boolean;
     workerCount: number;
     config?: JsonObject;
     createdAt?: string;
@@ -93,27 +96,29 @@ interface BenchmarkAssignment {
 }
 interface UpsertBenchmarkInput {
     name: string;
-    kind?: string;
     status?: string;
     config?: JsonObject;
     defaultRunConfig?: JsonObject;
 }
 interface UpdateBenchmarkInput {
     name?: string;
-    kind?: string;
     status?: string;
     config?: JsonObject;
     defaultRunConfig?: JsonObject;
 }
 interface CreateRunInput {
-    name?: string;
-    totalTasks: number;
-    workerCount: number;
+    /**
+     * Idempotency key for get-or-create: sibling callers passing the same key
+     * (per org + benchmark) converge on one run instead of each opening its own.
+     */
+    runKey?: string;
+    /** Omit to open a participant-sized run: each participant declares its own size when it registers. */
+    totalTasks?: number;
+    workerCount?: number;
     participants?: string[];
     config?: JsonObject;
 }
 interface UpdateRunInput {
-    name?: string;
     status?: BenchmarkRunStatus;
     config?: JsonObject;
 }
@@ -161,6 +166,10 @@ interface TaskStepRecord {
     latencyMs?: number;
     errorCode?: string | null;
     data?: JsonObject;
+    /** Number of parallel invocations requested for this step. */
+    concurrency?: number;
+    /** Per-iteration timeout in milliseconds applied to this step. */
+    timeoutMs?: number;
 }
 interface SendTaskResultsInput {
     benchmarkSlug: string;
@@ -280,13 +289,13 @@ interface BenchmarkResultsOverviewRun {
     }>;
 }
 interface BenchmarkResultsOverview {
-    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name' | 'kind'>;
+    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
     generatedAt: string;
     analytics: BenchmarkResultsOverviewAnalytics;
     items: BenchmarkResultsOverviewRun[];
 }
 interface BenchmarkRunResults {
-    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name' | 'kind'>;
+    benchmark: Pick<BenchmarkResource, 'id' | 'slug' | 'name'>;
     run: Pick<BenchmarkRun, 'id' | 'status' | 'totalTasks' | 'workerCount'>;
     generatedAt: string;
     overall: BenchmarkResultSummary;
@@ -474,6 +483,15 @@ interface RunWorkerContext {
     assignment: BenchmarkAssignment;
     taskIndex: number;
     step<T>(name: string, fn: () => Promise<T> | T, options?: DefineStepOptions): Promise<T>;
+    /**
+     * Attaches a JSON measurement to the platform. Called inside a `step`, it
+     * lands on that step's `data`; called at task top-level, on the task record's
+     * `data`. Repeated calls merge (shallow). Use this for anything you want on
+     * the platform — step return values are control flow and are never recorded.
+     */
+    measure(data: JsonObject): void;
+    /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */
+    log(message: string, meta?: JsonObject): void;
 }
 interface WorkerFinishContext {
     assignment: BenchmarkAssignment;
@@ -482,17 +500,15 @@ interface WorkerFinishContext {
     client: BenchmarkClient;
     uploadArtifact(input: Omit<UploadWorkerArtifactInput, 'attemptId'>): Promise<CreateWorkerArtifactResponse>;
 }
-interface StepContext<TState extends Record<string, unknown> = Record<string, unknown>> {
-    assignment: BenchmarkAssignment;
-    taskIndex: number;
-    state: TState;
-}
-type CleanupContext<TState extends Record<string, unknown> = Record<string, unknown>> = StepContext<TState>;
 interface DefineStepOptions {
     /** Report this step as active in heartbeat concurrency samples. Defaults to true. */
     reportConcurrency?: boolean;
     /** Per-worker target for this step. Defaults to worker concurrency/assignment target. */
     concurrency?: number;
+    /** Number of parallel invocations the step function should run internally. Used by the runner to record step-level concurrency. */
+    stepConcurrency?: number;
+    /** Per-invocation timeout in milliseconds for this step. Used by the runner to record step-level timeout metadata. */
+    timeoutMs?: number;
     /** Readiness coordination mode. Defaults to internal. */
     readiness?: 'poll' | 'internal';
     /** Poll interval while waiting for readiness. Defaults to 1000ms. */
@@ -500,25 +516,13 @@ interface DefineStepOptions {
     /** Maximum time to wait for readiness. Defaults to no timeout. */
     readyTimeoutMs?: number;
 }
-interface DefinedStep<TState extends Record<string, unknown> = Record<string, unknown>> {
-    name: string;
-    options?: DefineStepOptions;
-    fn: (context: StepContext<TState>) => Promise<JsonObject | void> | JsonObject | void;
-}
-interface DefineTaskOptions<TState extends Record<string, unknown> = Record<string, unknown>> {
-    /**
-     * Runs after the task finishes, whether it succeeded or failed.
-     * Use this to tear down resources stored in task state.
-     */
-    cleanup?: (context: CleanupContext<TState>) => Promise<void> | void;
-}
-interface DefinedTask<TState extends Record<string, unknown> = Record<string, unknown>> {
-    name: string;
-    steps: DefinedStep<TState>[];
-    options?: DefineTaskOptions<TState>;
-}
+/**
+ * The unit of work a worker runs, once per task index. Steps are declared
+ * imperatively via `context.step(...)`; this is the sole task shape the worker
+ * engine accepts. Higher-level authoring (`defineTask`) lives in
+ * `@benchsdk/runner`, which compiles down to a function of this shape.
+ */
 type TaskFunction = (context: RunWorkerContext) => Promise<JsonObject | void> | JsonObject | void;
-type WorkerTask = DefinedTask | TaskFunction;
 interface RunWorkerResult {
     assignment: BenchmarkAssignment | null;
     records: TaskResultRecord[];
@@ -537,42 +541,42 @@ interface RunWorkerOptions {
     onResult?: (record: TaskResultRecord) => void;
     /** Runs once after final result flush and before worker completion/failure is reported. */
     onFinish?: (context: WorkerFinishContext) => Promise<void> | void;
-    task: WorkerTask;
+    task: TaskFunction;
 }
-interface WorkerDefaults {
-    concurrency?: number;
-    batchSize?: number;
-    flushIntervalMs?: number;
-    heartbeatIntervalMs?: number;
-    readyPollIntervalMs?: number;
-}
-interface DefineWorkerOptions extends WorkerDefaults {
-    benchmarkSlug: string;
-    runId: string;
-    participantSlug: string;
-    processKind?: string;
-    processKey?: string;
-    client?: BenchmarkClient;
-    onFinish?: RunWorkerOptions['onFinish'];
-    task: WorkerTask;
-}
-interface BenchmarkWorker {
-    run(overrides?: Partial<WorkerDefaults>): Promise<RunWorkerResult>;
-}
-interface DefineBenchOptions extends WorkerDefaults {
-    slug: string;
-    participantSlug?: string;
-    client?: BenchmarkClient;
-    task: WorkerTask;
+interface BenchmarkRunSummaryMetric {
+    name: string;
+    unit: string;
+    median: number;
+    p95: number;
+    p99: number;
 }
-interface BenchDefinition {
-    slug: string;
-    task: WorkerTask;
-    defineWorker(options: Omit<DefineWorkerOptions, 'benchmarkSlug' | 'client' | 'task' | 'participantSlug'> & {
-        client?: BenchmarkClient;
-        participantSlug?: string;
-        task?: WorkerTask;
-    }): BenchmarkWorker;
+interface BenchmarkRunSummaryScalar {
+    name: string;
+    value: number;
+    unit: string;
+}
+interface BenchmarkRunSummaryResult {
+    provider: string;
+    dimensions?: Record<string, unknown>;
+    metrics: BenchmarkRunSummaryMetric[];
+    scalars?: BenchmarkRunSummaryScalar[];
+    compositeScore: number;
+    successRate: number;
+    scoringVersion?: string | null;
+    skipped: boolean;
+    skipReason?: string | null;
+}
+interface BenchmarkRunSummaryRunMetadata {
+    gitSha?: string;
+    gitRef?: string;
+    triggeredBy?: string;
+    nodeVersion?: string;
+    platform?: string;
+    arch?: string;
+}
+interface BenchmarkRunSummaryInput {
+    run: BenchmarkRunSummaryRunMetadata;
+    results: BenchmarkRunSummaryResult[];
 }
 interface BenchmarkClient {
     upsertBenchmark(slug: string, input: UpsertBenchmarkInput): Promise<BenchmarkResource>;
@@ -582,6 +586,8 @@ interface BenchmarkClient {
     createRun(benchmarkSlug: string, input: CreateRunInput): Promise<{
         run: BenchmarkRun;
         participants: BenchmarkParticipant[];
+        /** The slug of the org the run was attributed to, resolved server-side from the caller's API key. */
+        organizationSlug: string;
     }>;
     listRuns(benchmarkSlug: string): Promise<BenchmarkRun[]>;
     getRun(benchmarkSlug: string, runId: string): Promise<BenchmarkRun>;
@@ -622,6 +628,7 @@ interface BenchmarkClient {
     getRunTaskResults(benchmarkSlug: string, runId: string, input?: BenchmarkRunTaskResultsInput): Promise<BenchmarkRunTaskResults>;
     getRunTimeline(benchmarkSlug: string, runId: string, input?: BenchmarkRunTimelineInput): Promise<BenchmarkRunTimeline>;
     getRunImports(benchmarkSlug: string, runId: string): Promise<BenchmarkRunImports>;
+    submitRunSummary(benchmarkSlug: string, runId: string, input: BenchmarkRunSummaryInput): Promise<void>;
     runWorker(options: RunWorkerOptions): Promise<RunWorkerResult>;
 }
 
@@ -631,11 +638,6 @@ declare class BenchmarkApiError extends Error {
     constructor(message: string, status: number, body: string);
 }
 declare function createBenchmarkClient(config?: BenchmarkClientConfig): BenchmarkClient;
-declare function runBenchmarkWorker(config: BenchmarkClientConfig, options: RunWorkerOptions): Promise<RunWorkerResult>;
-declare function defineStep<TState extends Record<string, unknown> = Record<string, unknown>>(name: string, optionsOrFn: DefineStepOptions | DefinedStep<TState>['fn'], maybeFn?: DefinedStep<TState>['fn']): DefinedStep<TState>;
-declare function defineTask<TState extends Record<string, unknown> = Record<string, unknown>>(name: string, steps: DefinedStep<TState>[], options?: DefineTaskOptions<TState>): DefinedTask<TState>;
-declare function defineWorker(options: DefineWorkerOptions): BenchmarkWorker;
-declare function defineBench(options: DefineBenchOptions): BenchDefinition;
 
 interface BenchmarkReporterConfig extends BenchmarkClientConfig {
     benchmarkSlug: string;
@@ -724,4 +726,35 @@ interface BenchmarkSystemMetricsCollector {
 }
 declare function createSystemMetricsCollector(): BenchmarkSystemMetricsCollector;
 
-export { type BenchDefinition, type BenchmarkAnalyticsReadiness, BenchmarkApiError, type BenchmarkArtifact, type BenchmarkAssignment, type BenchmarkClient, type BenchmarkClientConfig, type BenchmarkConcurrencyPoint, type BenchmarkEventRateBucket, type BenchmarkFailurePoint, type BenchmarkParticipant, BenchmarkReporter, type BenchmarkReporterArtifactInput, type BenchmarkReporterBarrierInput, type BenchmarkReporterBarrierResult, type BenchmarkReporterConfig, type BenchmarkReporterHeartbeatInput, type BenchmarkReporterProgress, type BenchmarkResource, type BenchmarkResultLatencySummary, type BenchmarkResultSummary, type BenchmarkResultsOverview, type BenchmarkResultsOverviewAnalytics, type BenchmarkResultsOverviewInput, type BenchmarkResultsOverviewRun, type BenchmarkRun, type BenchmarkRunAnalyticsSummary, type BenchmarkRunImportItem, type BenchmarkRunImports, type BenchmarkRunImportsSummary, type BenchmarkRunResults, type BenchmarkRunStatus, type BenchmarkRunTaskResults, type BenchmarkRunTaskResultsInput, type BenchmarkRunTimeline, type BenchmarkRunTimelineInput, type BenchmarkRunWorker, type BenchmarkStepResultSummary, type BenchmarkSystemMetricsCollector, type BenchmarkSystemMetricsSample, type BenchmarkTaskBucket, type BenchmarkWorker, type BenchmarkWorkerAttempt, type BenchmarkWorkerStatus, type ClaimWorkerInput, type CreateRunInput, type CreateWorkerArtifactInput, type CreateWorkerArtifactResponse, type DefineBenchOptions, type DefineStepOptions, type DefineWorkerOptions, type DefinedStep, type DefinedTask, type JsonObject, type JsonValue, type PlanWorkersInput, type RunProgress, type RunProgressConcurrency, type RunProgressParticipant, type RunProgressParticipantCounts, type RunProgressStatus, type RunProgressSummary, type RunProgressTaskCounts, type RunProgressWorkerCounts, type RunWorkerContext, type RunWorkerOptions, type RunWorkerResult, type SendTaskResultsInput, type TaskFunction, type TaskResultRecord, type TaskResultsResponse, type TaskStepRecord, type UpdateBenchmarkInput, type UpdateParticipantInput, type UpdateRunInput, type UpdateWorkerInput, type UploadWorkerArtifactInput, type UpsertBenchmarkInput, type UpsertParticipantInput, type WorkerConcurrencySample, type WorkerFinishContext, type WorkerHeartbeatInput, type WorkerTask, claimBenchmarkReporter, createBenchmarkClient, createSystemMetricsCollector, defineBench, defineStep, defineTask, defineWorker, runBenchmarkWorker };
+/**
+ * Base interface for a benchmark participant — the shared shape across all
+ * benchmark categories (sandbox, ai-gateway, browser, storage). Each category
+ * extends this with its own provider-specific fields.
+ */
+interface BaseParticipant {
+    /** Participant name (e.g. 'e2b', 'daytona', 'openrouter') */
+    name: string;
+    /** Environment variables that must all be set to run this participant */
+    requiredEnvVars: string[];
+}
+/**
+/**
+ * Filters `participants` down to those whose `requiredEnvVars` are all set
+ * in `process.env`. Returns an object `{ available, skipped }` where `skipped`
+ * includes the names and missing vars for logging.
+ */
+declare function filterParticipantsByEnv<T extends BaseParticipant>(participants: T[]): {
+    available: T[];
+    skipped: {
+        name: string;
+        missing: string[];
+    }[];
+};
+/**
+ * Filters `all` down to the requested `names`, exiting with a clear error
+ * if any name is unrecognized. Returns `all` unchanged when `names` is
+ * undefined (no filter specified).
+ */
+declare function selectParticipants<T extends BaseParticipant>(all: T[], names?: string[]): T[];
+
+export { type BaseParticipant, type BenchmarkAnalyticsReadiness, BenchmarkApiError, type BenchmarkArtifact, type BenchmarkAssignment, type BenchmarkClient, type BenchmarkClientConfig, type BenchmarkConcurrencyPoint, type BenchmarkEventRateBucket, type BenchmarkFailurePoint, type BenchmarkParticipant, BenchmarkReporter, type BenchmarkReporterArtifactInput, type BenchmarkReporterBarrierInput, type BenchmarkReporterBarrierResult, type BenchmarkReporterConfig, type BenchmarkReporterHeartbeatInput, type BenchmarkReporterProgress, type BenchmarkResource, type BenchmarkResultLatencySummary, type BenchmarkResultSummary, type BenchmarkResultsOverview, type BenchmarkResultsOverviewAnalytics, type BenchmarkResultsOverviewInput, type BenchmarkResultsOverviewRun, type BenchmarkRun, type BenchmarkRunAnalyticsSummary, type BenchmarkRunImportItem, type BenchmarkRunImports, type BenchmarkRunImportsSummary, type BenchmarkRunResults, type BenchmarkRunStatus, type BenchmarkRunSummaryInput, type BenchmarkRunSummaryMetric, type BenchmarkRunSummaryResult, type BenchmarkRunSummaryRunMetadata, type BenchmarkRunSummaryScalar, type BenchmarkRunTaskResults, type BenchmarkRunTaskResultsInput, type BenchmarkRunTimeline, type BenchmarkRunTimelineInput, type BenchmarkRunWorker, type BenchmarkStepResultSummary, type BenchmarkSystemMetricsCollector, type BenchmarkSystemMetricsSample, type BenchmarkTaskBucket, type BenchmarkWorkerAttempt, type BenchmarkWorkerStatus, type ClaimWorkerInput, type CreateRunInput, type CreateWorkerArtifactInput, type CreateWorkerArtifactResponse, type DefineStepOptions, type JsonObject, type JsonValue, type PlanWorkersInput, type RunProgress, type RunProgressConcurrency, type RunProgressParticipant, type RunProgressParticipantCounts, type RunProgressStatus, type RunProgressSummary, type RunProgressTaskCounts, type RunProgressWorkerCounts, type RunWorkerContext, type RunWorkerOptions, type RunWorkerResult, type SendTaskResultsInput, type TaskFunction, type TaskResultRecord, type TaskResultsResponse, type TaskStepRecord, type UpdateBenchmarkInput, type UpdateParticipantInput, type UpdateRunInput, type UpdateWorkerInput, type UploadWorkerArtifactInput, type UpsertBenchmarkInput, type UpsertParticipantInput, type WorkerConcurrencySample, type WorkerFinishContext, type WorkerHeartbeatInput, claimBenchmarkReporter, createBenchmarkClient, createSystemMetricsCollector, filterParticipantsByEnv, selectParticipants };