@benchsdk/runner 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ComputeSDK
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,74 @@
+# @benchsdk/runner
+
+Benchmark framework for authoring `*.bench.ts` files that report to the benchmarks platform via [`@benchsdk/client`](../benchsdk).
+
+## What it provides
+
+- **`defineBenchmarkConfig`** / **`defineTask`** — A `*.bench.ts` file exports exactly two things: a **config** (`defineBenchmarkConfig`, the orchestration knobs + `participants` + an optional `onComplete` hook) and a **task** (`defineTask`, the workload for one iteration). There is no "mode": the orchestration shape (sequential / staggered / burst) emerges from the `iterations`, `concurrency`, and `staggerDelayMs` knobs, and `groupBy` (`'participant'` | `'round'`) selects the ordering across participants.
+- **`bench run <file>`** — The CLI entrypoint. It imports the module, reads its `config` and `task`, applies CLI overrides, and drives the run. Benchmark files declare; they never call the runner themselves.
+- **`TaskError`** / **`NoAvailableParticipantsError`** — Structured errors: throw `TaskError` from a task to attach a code / data / pre-measured steps; `bench run` treats `NoAvailableParticipantsError` (every participant env-gated out) as a clean no-op exit.
+
+## Install
+
+```sh
+pnpm add @benchsdk/runner @benchsdk/client
+```
+
+## Usage
+
+A benchmark file exports a `config` and a `task` — nothing else:
+
+```ts
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import { providers } from './providers.js';
+import { writeLegacyResults } from './legacy-results.js';
+
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'sandbox-tti-local',
+  benchmarkName: 'Sandbox TTI (local)',
+  iterations: 100,       // total tasks per participant
+  concurrency: 1,        // 1 = sequential, N = burst, N + staggerDelayMs = staggered
+  participants: providers,
+  // Aggregate post-run work (the one thing a single task can't see) lives here.
+  onComplete: (outcome) => writeLegacyResults(outcome.participants),
+});
+
+export const task = defineTask(async ({ participant, step, measure, log }) => {
+  log(`creating sandbox on ${participant.name}`);
+  // Named steps via `ctx.step`: values flow between steps with closures and
+  // cleanup runs in a `finally`. Each step is a first-class platform record
+  // with its own timing/status.
+  const sandbox = await step('create', () => participant.createCompute().sandbox.create());
+  try {
+    const t0 = performance.now();
+    await step('exec', () => sandbox.runCommand('node -v'));
+    measure({ ttiMs: performance.now() - t0 });   // metrics → the platform
+  } finally {
+    await step('destroy', () => sandbox.destroy());
+  }
+});
+```
+
+Run it with the CLI (flags override the config knobs):
+
+```sh
+bench run benchmarks/sandbox/sandbox-tti.bench.ts --iterations 100 --concurrency 20 --provider e2b,modal
+```
+
+To load a TypeScript benchmark without a build step, run the CLI under a TS loader:
+
+```sh
+tsx node_modules/@benchsdk/runner/dist/bin.js run sandbox-tti.bench.ts
+```
+
+### Context channels
+
+Inside a task the context exposes three separate channels:
+
+- **`step(name, fn, options?)`** — returns `fn`'s value to your code (thread live objects between steps); records the step's timing/status on the platform. Return values are never auto-recorded as data.
+- **`measure(data)`** — explicit metric channel. Called inside a `step()` it merges into that step's data; called at task top-level it merges into the task record. A task with no explicit steps is recorded as one implicit `'task'` step carrying its measurements.
+- **`log(message, meta?)`** — human-readable narration to the run timeline.
+
+## License
+
+MIT

package/dist/bin.cjs

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+"use strict";
+
+// src/bin.ts
+var import_runner = require("@benchsdk/runner");
+(0, import_runner.run)(process.argv.slice(2));
+//# sourceMappingURL=bin.cjs.map

package/dist/bin.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\n// Imports from the package entry (not a relative path) so the bin and any\n// benchmark module it loads share a single `@benchsdk/runner` instance — this\n// keeps `instanceof` checks (TaskError / NoAvailableParticipantsError) valid\n// across the bin and the dynamically imported `*.bench.ts`.\nimport { run } from '@benchsdk/runner';\n\nrun(process.argv.slice(2));\n"],"mappings":";;;;AAKA,oBAAoB;AAAA,IAEpB,mBAAI,QAAQ,KAAK,MAAM,CAAC,CAAC;","names":[]}

package/dist/bin.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+// src/bin.ts
+import { run } from "@benchsdk/runner";
+run(process.argv.slice(2));
+//# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\n// Imports from the package entry (not a relative path) so the bin and any\n// benchmark module it loads share a single `@benchsdk/runner` instance — this\n// keeps `instanceof` checks (TaskError / NoAvailableParticipantsError) valid\n// across the bin and the dynamically imported `*.bench.ts`.\nimport { run } from '@benchsdk/runner';\n\nrun(process.argv.slice(2));\n"],"mappings":";;;AAKA,SAAS,WAAW;AAEpB,IAAI,QAAQ,KAAK,MAAM,CAAC,CAAC;","names":[]}