@benchsdk/runner 0.2.0

package/dist/index.cjs

@@ -0,0 +1,944 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  NoAvailableParticipantsError: () => NoAvailableParticipantsError,
+  TaskError: () => TaskError,
+  defineBenchmarkConfig: () => defineBenchmarkConfig,
+  defineTask: () => defineTask,
+  higherIsBetter: () => higherIsBetter,
+  lowerIsBetter: () => lowerIsBetter,
+  mergeConfig: () => mergeConfig,
+  parseCliArgs: () => parseCliArgs,
+  run: () => run,
+  runBenchmark: () => runBenchmark,
+  runBenchmarkFile: () => runBenchmarkFile,
+  score: () => score
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/bench-config.ts
+var TaskError = class extends Error {
+  code;
+  data;
+  steps;
+  constructor(message, opts) {
+    super(message);
+    this.name = "TaskError";
+    this.code = opts?.code;
+    this.data = opts?.data;
+    this.steps = opts?.steps;
+  }
+};
+function assertPositiveInt(value, field) {
+  if (value === void 0) return;
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`${field} must be an integer >= 1 (got ${value})`);
+  }
+}
+function defineBenchmarkConfig(config) {
+  if (!config.benchmarkSlug || typeof config.benchmarkSlug !== "string") {
+    throw new Error("benchmarkSlug is required");
+  }
+  if (!config.benchmarkName || typeof config.benchmarkName !== "string") {
+    throw new Error("benchmarkName is required");
+  }
+  if (config.phases !== void 0) {
+    if (config.iterations !== void 0) {
+      throw new Error("phases and iterations are mutually exclusive");
+    }
+    if (!Array.isArray(config.phases) || config.phases.length === 0) {
+      throw new Error("phases must be a non-empty array");
+    }
+    const seen = /* @__PURE__ */ new Set();
+    for (const phase of config.phases) {
+      if (!phase.name || typeof phase.name !== "string") {
+        throw new Error("each phase requires a non-empty name");
+      }
+      if (seen.has(phase.name)) {
+        throw new Error(`duplicate phase name: ${phase.name}`);
+      }
+      seen.add(phase.name);
+      assertPositiveInt(phase.iterations, `phase '${phase.name}' iterations`);
+    }
+  }
+  assertPositiveInt(config.iterations, "iterations");
+  assertPositiveInt(config.concurrency, "concurrency");
+  if (config.staggerDelayMs !== void 0 && (!Number.isFinite(config.staggerDelayMs) || config.staggerDelayMs < 0)) {
+    throw new Error(`staggerDelayMs must be a number >= 0 (got ${config.staggerDelayMs})`);
+  }
+  if (config.groupBy !== void 0 && config.groupBy !== "participant" && config.groupBy !== "round") {
+    throw new Error(`groupBy must be 'participant' or 'round' (got ${config.groupBy})`);
+  }
+  if (config.shapes !== void 0) {
+    for (const [shapeName, shape] of Object.entries(config.shapes)) {
+      if (!shape.slug || !/^[a-z0-9][a-z0-9-]*$/.test(shape.slug)) {
+        throw new Error(`shape '${shapeName}' needs a lowercase slug (got ${JSON.stringify(shape.slug)})`);
+      }
+      if (shape.name !== void 0 && (typeof shape.name !== "string" || shape.name.trim() === "")) {
+        throw new Error(`shape '${shapeName}' name must be a non-empty string`);
+      }
+      if (shape.staggerDelayMs !== void 0 && (!Number.isFinite(shape.staggerDelayMs) || shape.staggerDelayMs < 0)) {
+        throw new Error(`shape '${shapeName}' staggerDelayMs must be a number >= 0 (got ${shape.staggerDelayMs})`);
+      }
+    }
+  }
+  return config;
+}
+function defineTask(task) {
+  if (typeof task !== "function") {
+    throw new Error("defineTask requires a task function.");
+  }
+  return task;
+}
+
+// src/no-available-participants.ts
+var NoAvailableParticipantsError = class extends Error {
+  skipped;
+  constructor(skipped) {
+    super(
+      `No participants have their required env vars set \u2014 nothing to run${skipped.length > 0 ? ` (skipped: ${skipped.map((s) => s.name).join(", ")})` : ""}.`
+    );
+    this.name = "NoAvailableParticipantsError";
+    this.skipped = skipped;
+  }
+};
+
+// src/runner.ts
+var import_node_child_process = require("child_process");
+var import_node_os = __toESM(require("os"), 1);
+var import_client = require("@benchsdk/client");
+
+// src/scoring.ts
+function isFiniteNumber(v) {
+  return typeof v === "number" && Number.isFinite(v);
+}
+function percentile(sorted, p) {
+  if (sorted.length === 0) return 0;
+  const idx = Math.max(0, Math.ceil(p / 100 * sorted.length) - 1);
+  return sorted[Math.min(idx, sorted.length - 1)];
+}
+function computeStats(values, trimPercent = 0.05) {
+  if (values.length === 0) return { median: 0, p95: 0, p99: 0 };
+  const sorted = [...values].sort((a, b) => a - b);
+  const trimCount = Math.floor(sorted.length * trimPercent);
+  const trimmed = trimCount > 0 && sorted.length - 2 * trimCount > 0 ? sorted.slice(trimCount, sorted.length - trimCount) : sorted;
+  const mid = Math.floor(trimmed.length / 2);
+  const median = trimmed.length % 2 === 0 ? (trimmed[mid - 1] + trimmed[mid]) / 2 : trimmed[mid];
+  return { median, p95: percentile(trimmed, 95), p99: percentile(trimmed, 99) };
+}
+function toJsonObject(value) {
+  return JSON.parse(JSON.stringify(value ?? {}));
+}
+function collectSamples(metric, records) {
+  const samples = [];
+  for (const record of records) {
+    const raw = typeof metric.value === "function" ? metric.value(record) : record.data?.[metric.value ?? metric.name];
+    if (Array.isArray(raw)) {
+      for (const item of raw) {
+        if (isFiniteNumber(item)) samples.push(item);
+      }
+    } else if (isFiniteNumber(raw)) {
+      samples.push(raw);
+    }
+  }
+  return samples;
+}
+function scoreStat(stat, metric) {
+  if (metric.higherIsBetter) {
+    const floor = metric.floor ?? 0;
+    if (stat <= floor) return 0;
+    if (stat >= metric.ceiling) return 100;
+    return (stat - floor) / (metric.ceiling - floor) * 100;
+  }
+  return Math.max(0, 100 * (1 - stat / metric.ceiling));
+}
+var lowerIsBetter = (name, opts) => ({
+  name,
+  ...opts,
+  higherIsBetter: false
+});
+var higherIsBetter = (name, opts) => ({
+  name,
+  ...opts,
+  higherIsBetter: true
+});
+function score(outcome, spec) {
+  const successFilter = spec.success ?? ((r) => r.status === "success");
+  const dimensions = toJsonObject(spec.dimensions ?? {});
+  const results = [];
+  for (const { participant, records } of outcome.participants) {
+    const passing = records.filter(successFilter);
+    const successRate = records.length === 0 ? 0 : passing.length / records.length;
+    const skipped = records.length === 0;
+    let metricScoresSum = 0;
+    const metrics = [];
+    for (const metric of spec.metrics) {
+      const samples = collectSamples(metric, passing);
+      if (samples.length === 0) {
+        continue;
+      }
+      const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05);
+      const metricScore = metric.weights.median * scoreStat(median, metric) + metric.weights.p95 * scoreStat(p95, metric) + metric.weights.p99 * scoreStat(p99, metric);
+      metricScoresSum += metricScore;
+      metrics.push({ name: metric.name, unit: metric.unit, median, p95, p99 });
+    }
+    const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100;
+    results.push({
+      provider: participant,
+      dimensions,
+      metrics,
+      compositeScore,
+      successRate,
+      skipped
+    });
+  }
+  return results;
+}
+
+// src/log-buffer.ts
+var LogBuffer = class {
+  lines = [];
+  step(taskIndex, stepName, outcome) {
+    const header = `[task ${taskIndex}] ${stepName}`;
+    this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} ${header}`);
+    if (outcome.stdout?.trim()) {
+      this.lines.push(indent(outcome.stdout));
+    }
+    if (outcome.stderr?.trim()) {
+      this.lines.push(indent(outcome.stderr, "stderr: "));
+    }
+    if (outcome.error) {
+      this.lines.push(indent(outcome.error, "error: "));
+    }
+  }
+  /** Appends a free-form narration line (backs the task context's `log`). */
+  line(message, meta) {
+    const suffix = meta && Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : "";
+    this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} ${message}${suffix}`);
+  }
+  isEmpty() {
+    return this.lines.length === 0;
+  }
+  toText() {
+    return this.lines.join("\n") + "\n";
+  }
+};
+function indent(text, prefix = "") {
+  return text.trimEnd().split("\n").map((line) => `  ${prefix}${line}`).join("\n");
+}
+
+// src/runner.ts
+var DEFAULT_PLATFORM_URL = "https://platform.computesdk.com";
+function isEnvNoIngest() {
+  const v = process.env.BENCHSDK_NO_INGEST;
+  return v === "1" || v?.toLowerCase() === "true";
+}
+function sleep(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
+function getErrorCode(error) {
+  return error instanceof Error && error.name ? error.name : "ERROR";
+}
+function withTimeout(promise, ms, name) {
+  return new Promise((resolve2, reject) => {
+    const timer = setTimeout(
+      () => reject(new TaskError(`Step "${name}" timed out after ${ms}ms`, { code: "step_timeout" })),
+      ms
+    );
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve2(value);
+      },
+      (error) => {
+        clearTimeout(timer);
+        reject(error);
+      }
+    );
+  });
+}
+async function runStepInvocations(name, fn, options) {
+  const requestedConcurrency = options?.concurrency;
+  if (requestedConcurrency !== void 0 && (!Number.isInteger(requestedConcurrency) || requestedConcurrency < 1)) {
+    throw new Error(`step "${name}" concurrency must be an integer >= 1 (got ${requestedConcurrency})`);
+  }
+  const timeoutMs = options?.timeoutMs;
+  if (timeoutMs !== void 0 && (!Number.isFinite(timeoutMs) || timeoutMs < 0)) {
+    throw new Error(`step "${name}" timeoutMs must be a number >= 0 (got ${timeoutMs})`);
+  }
+  const count = requestedConcurrency ?? 1;
+  const invocations = Array.from({ length: count }, () => {
+    const promise = Promise.resolve().then(() => fn());
+    if (timeoutMs === void 0) return promise;
+    return withTimeout(promise, timeoutMs, name);
+  });
+  if (count === 1) {
+    return invocations[0];
+  }
+  const outcomes = await Promise.allSettled(invocations);
+  const results = [];
+  let firstError;
+  for (const outcome of outcomes) {
+    if (outcome.status === "fulfilled") {
+      results.push(outcome.value);
+    } else if (firstError === void 0) {
+      firstError = outcome.reason;
+    }
+  }
+  if (firstError !== void 0) throw firstError;
+  return results;
+}
+async function runStepWithClient(clientStep, name, fn, options) {
+  const { concurrency: runnerConcurrency, timeoutMs, ...clientOptions } = options ?? {};
+  const clientStepOptions = {
+    ...clientOptions,
+    timeoutMs,
+    stepConcurrency: runnerConcurrency
+  };
+  const result = await clientStep(name, () => runStepInvocations(name, fn, options), clientStepOptions);
+  return result;
+}
+function parseCliArgs(argv) {
+  const args = {};
+  const readValue = (raw, i) => {
+    const eq = raw.indexOf("=");
+    if (eq !== -1) return { value: raw.slice(eq + 1), nextIndex: i };
+    return { value: argv[i + 1] ?? "", nextIndex: i + 1 };
+  };
+  const intFlag = (raw, flag) => {
+    if (raw.trim() === "") throw new Error(`${flag} expects a value`);
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n < 1) throw new Error(`${flag} expects an integer >= 1 (got "${raw}")`);
+    return n;
+  };
+  const nonNegFlag = (raw, flag) => {
+    if (raw.trim() === "") throw new Error(`${flag} expects a value`);
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n < 0) throw new Error(`${flag} expects a number >= 0 (got "${raw}")`);
+    return n;
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    const name = arg.includes("=") ? arg.slice(0, arg.indexOf("=")) : arg;
+    switch (name) {
+      // `--slug` is the pre-`--benchmark` spelling, kept working for existing scripts.
+      case "--slug":
+      case "--benchmark": {
+        const { value, nextIndex } = readValue(arg, i);
+        if (!/^[a-z0-9][a-z0-9-]*$/.test(value)) {
+          throw new Error(`${name} expects a lowercase benchmark slug (got "${value}")`);
+        }
+        args.benchmark = value;
+        i = nextIndex;
+        break;
+      }
+      case "--name": {
+        const { value, nextIndex } = readValue(arg, i);
+        if (value.trim() === "") throw new Error("--name expects a value");
+        args.name = value;
+        i = nextIndex;
+        break;
+      }
+      case "--shape": {
+        const { value, nextIndex } = readValue(arg, i);
+        if (value.trim() === "") throw new Error("--shape expects a value");
+        args.shape = value;
+        i = nextIndex;
+        break;
+      }
+      case "--run-key": {
+        const { value, nextIndex } = readValue(arg, i);
+        if (value.trim() === "") throw new Error("--run-key expects a value");
+        args.runKey = value;
+        i = nextIndex;
+        break;
+      }
+      case "--iterations": {
+        const { value, nextIndex } = readValue(arg, i);
+        args.iterations = intFlag(value, "--iterations");
+        i = nextIndex;
+        break;
+      }
+      case "--concurrency": {
+        const { value, nextIndex } = readValue(arg, i);
+        args.concurrency = intFlag(value, "--concurrency");
+        i = nextIndex;
+        break;
+      }
+      case "--stagger-delay-ms": {
+        const { value, nextIndex } = readValue(arg, i);
+        args.staggerDelayMs = nonNegFlag(value, "--stagger-delay-ms");
+        i = nextIndex;
+        break;
+      }
+      case "--group-by": {
+        const { value, nextIndex } = readValue(arg, i);
+        if (value !== "participant" && value !== "round") {
+          throw new Error(`--group-by expects 'participant' or 'round' (got "${value}")`);
+        }
+        args.groupBy = value;
+        i = nextIndex;
+        break;
+      }
+      case "--provider": {
+        const { value, nextIndex } = readValue(arg, i);
+        const names = value.split(",").map((s) => s.trim()).filter(Boolean);
+        args.providers = [...args.providers ?? [], ...names];
+        i = nextIndex;
+        break;
+      }
+      case "--no-ingest":
+      case "--dry-run":
+        args.noIngest = true;
+        break;
+      default:
+        break;
+    }
+  }
+  if (!args.noIngest && isEnvNoIngest()) {
+    args.noIngest = true;
+  }
+  return args;
+}
+function mergeConfig(config, args) {
+  const phaseTotal = config.phases?.reduce((sum, p) => sum + p.iterations, 0);
+  if (phaseTotal !== void 0 && args.iterations !== void 0) {
+    console.warn("--iterations is ignored because this benchmark declares phases.");
+  }
+  const resolved = {
+    iterations: phaseTotal ?? args.iterations ?? config.iterations ?? 1,
+    concurrency: args.concurrency ?? config.concurrency ?? 1,
+    staggerDelayMs: args.staggerDelayMs ?? config.staggerDelayMs ?? 0,
+    groupBy: args.groupBy ?? config.groupBy ?? "participant",
+    providers: args.providers ?? config.defaultProviders
+  };
+  if (!Number.isInteger(resolved.iterations) || resolved.iterations < 1) {
+    throw new Error(`iterations must be an integer >= 1 (got ${resolved.iterations})`);
+  }
+  if (!Number.isInteger(resolved.concurrency) || resolved.concurrency < 1) {
+    throw new Error(`concurrency must be an integer >= 1 (got ${resolved.concurrency})`);
+  }
+  return resolved;
+}
+function buildSchedule(config, iterations, task) {
+  if (config.phases?.length) {
+    return config.phases.flatMap(
+      (phase) => Array.from({ length: phase.iterations }, () => ({ phase: phase.name, task }))
+    );
+  }
+  return Array.from({ length: iterations }, () => ({ phase: void 0, task }));
+}
+function defaultOnResult(record, meta) {
+  const n = record.taskIndex + 1;
+  if (record.status === "success") {
+    const data = record.data && Object.keys(record.data).length > 0 ? ` ${JSON.stringify(record.data)}` : "";
+    console.log(`  [${meta.participant}] Task ${n}/${meta.iterations}: success${data}`);
+  } else {
+    console.log(`  [${meta.participant}] Task ${n}/${meta.iterations}: FAILED \u2014 ${record.errorCode ?? "unknown error"}`);
+  }
+}
+function resolvePlatform() {
+  const root = (process.env.BENCHMARKS_PLATFORM_URL || DEFAULT_PLATFORM_URL).replace(/\/+$/, "");
+  const apiKey = process.env.BENCHMARKS_PLATFORM_API_KEY;
+  if (!apiKey) {
+    throw new Error(
+      "BENCHMARKS_PLATFORM_API_KEY is required. Create an org-scoped API key in your organization settings on the platform and set it in your .env."
+    );
+  }
+  return {
+    baseUrl: `${root}/api/v1`,
+    apiKey
+  };
+}
+function resolveShape(config, shapeName) {
+  if (!shapeName) return void 0;
+  const shape = config.shapes?.[shapeName];
+  if (!shape) {
+    const known = Object.keys(config.shapes ?? {});
+    throw new Error(
+      known.length > 0 ? `Unknown --shape "${shapeName}". Known shapes: ${known.join(", ")}.` : `Unknown --shape "${shapeName}": this benchmark declares no shapes.`
+    );
+  }
+  return shape;
+}
+function applyShape(config, shape) {
+  if (!shape) return config;
+  return {
+    ...config,
+    benchmarkSlug: shape.slug,
+    benchmarkName: shape.name ?? shape.slug,
+    ...shape.staggerDelayMs !== void 0 ? { staggerDelayMs: shape.staggerDelayMs } : {}
+  };
+}
+function applyIdentityOverrides(fileConfig, args) {
+  return {
+    ...fileConfig,
+    ...args.benchmark ? { benchmarkSlug: args.benchmark } : {},
+    ...args.name ? { benchmarkName: args.name } : {}
+  };
+}
+function dashboardUrlFor(baseUrl, organizationSlug, benchmarkSlug, runId) {
+  return `${baseUrl.replace(/\/api\/v1\/?$/, "")}/${organizationSlug}/benchmarks/${benchmarkSlug}/runs/${runId}`;
+}
+function resolveParticipants(config, resolved) {
+  const { available, skipped } = (0, import_client.filterParticipantsByEnv)((0, import_client.selectParticipants)(config.participants, resolved.providers));
+  for (const s of skipped) {
+    console.log(`Skipping ${s.name}: missing ${s.missing.join(", ")}`);
+  }
+  if (available.length === 0) throw new NoAvailableParticipantsError(skipped);
+  return available;
+}
+async function runBenchmark(fileConfig, task, argv = []) {
+  const args = parseCliArgs(argv);
+  const noIngest = args.noIngest ?? isEnvNoIngest();
+  const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape));
+  const config = applyIdentityOverrides(shaped, args);
+  const resolved = mergeConfig(config, args);
+  const available = resolveParticipants(config, resolved);
+  let baseUrl = "";
+  let apiKey = "";
+  let client = null;
+  if (!noIngest) {
+    ({ baseUrl, apiKey } = resolvePlatform());
+    client = (0, import_client.createBenchmarkClient)({ baseUrl, apiKey });
+  }
+  const schedule = buildSchedule(config, resolved.iterations, task);
+  const totalTasks = schedule.length;
+  const concurrencyLabel = resolved.groupBy === "round" ? "n/a (round mode)" : String(resolved.concurrency);
+  console.log(`${config.benchmarkName} (self-contained)`);
+  console.log(`Date: ${(/* @__PURE__ */ new Date()).toISOString()}`);
+  if (noIngest) {
+    console.log("Dry run: no platform ingest or reporting.\n");
+  }
+  console.log(
+    `Knobs: iterations=${totalTasks}, concurrency=${concurrencyLabel}, staggerDelayMs=${resolved.staggerDelayMs}, groupBy=${resolved.groupBy}
+`
+  );
+  const identityIsOurs = args.shape !== void 0 || args.name !== void 0 || !args.benchmark || args.benchmark === fileConfig.benchmarkSlug;
+  let runId;
+  let dashboardUrl;
+  if (noIngest) {
+    runId = "no-ingest";
+    dashboardUrl = "";
+  } else {
+    if (identityIsOurs) {
+      await client.upsertBenchmark(config.benchmarkSlug, {
+        name: config.benchmarkName
+      });
+    }
+    if (args.runKey) {
+      const { run: run2, organizationSlug } = await client.createRun(config.benchmarkSlug, {
+        runKey: args.runKey
+      });
+      runId = run2.id;
+      dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run2.id);
+      for (const participant of available) {
+        await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks });
+      }
+      console.log(`Shared run (key "${args.runKey}"): ${run2.name} (${runId})`);
+      console.log(`View at: ${dashboardUrl}
+`);
+    } else {
+      const { run: run2, organizationSlug } = await client.createRun(config.benchmarkSlug, {
+        totalTasks,
+        workerCount: 1,
+        participants: available.map((p) => p.name)
+      });
+      runId = run2.id;
+      dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run2.id);
+      console.log(`Run created: ${run2.name} (${runId})`);
+      console.log(`View at: ${dashboardUrl}
+`);
+    }
+  }
+  const onResult = defaultOnResult;
+  let participantRecords;
+  if (resolved.groupBy === "round") {
+    participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult, noIngest);
+  } else {
+    participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult);
+  }
+  console.log(`All done. ${noIngest ? "No platform run created." : `View at: ${dashboardUrl}`}`);
+  const outcome = {
+    runId,
+    dashboardUrl,
+    participants: participantRecords,
+    config: resolved
+  };
+  if (client && config.onScore) {
+    try {
+      const spec = await config.onScore(lowerIsBetter, higherIsBetter);
+      const scored = score(outcome, spec);
+      const run2 = {
+        gitSha: process.env.GITHUB_SHA ?? getGitSha(),
+        gitRef: process.env.GITHUB_REF_NAME ?? process.env.GITHUB_REF ?? getGitRef(),
+        triggeredBy: process.env.GITHUB_EVENT_NAME ?? "manual",
+        nodeVersion: process.version,
+        platform: import_node_os.default.platform(),
+        arch: import_node_os.default.arch()
+      };
+      await client.submitRunSummary(config.benchmarkSlug, runId, { run: run2, results: scored });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.warn(`[benchsdk-runner] failed to submit run summary: ${message}`);
+    }
+  }
+  if (config.onComplete) await config.onComplete(outcome);
+  return outcome;
+}
+function getGitSha() {
+  if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA;
+  try {
+    return (0, import_node_child_process.execSync)("git rev-parse HEAD", { encoding: "utf8", stdio: "pipe" }).trim();
+  } catch {
+    return void 0;
+  }
+}
+function getGitRef() {
+  if (process.env.GITHUB_REF_NAME) return process.env.GITHUB_REF_NAME;
+  if (process.env.GITHUB_REF) return process.env.GITHUB_REF;
+  try {
+    return (0, import_node_child_process.execSync)("git rev-parse --abbrev-ref HEAD", { encoding: "utf8", stdio: "pipe" }).trim();
+  } catch {
+    return void 0;
+  }
+}
+async function runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult) {
+  const participantRecords = [];
+  for (const participant of available) {
+    console.log(`${"=".repeat(70)}`);
+    console.log(`  Participant: ${participant.name}`);
+    console.log("=".repeat(70));
+    if (!client) {
+      const records = [];
+      let rampStartMs2;
+      let nextIndex = 0;
+      const logBuffer = new LogBuffer();
+      const runSlot = async (scheduleIndex) => {
+        if (resolved.staggerDelayMs > 0) {
+          rampStartMs2 ??= Date.now();
+          const waitMs = rampStartMs2 + scheduleIndex * resolved.staggerDelayMs - Date.now();
+          if (waitMs > 0) await sleep(waitMs);
+        }
+        const slot = schedule[scheduleIndex];
+        const record = await runTaskRecord(slot.task, participant, scheduleIndex, scheduleIndex, slot.phase, logBuffer);
+        onResult(record, { iterations: schedule.length, participant: participant.name });
+        records.push(record);
+      };
+      const worker = async () => {
+        while (nextIndex < schedule.length) {
+          const index = nextIndex++;
+          await runSlot(index);
+        }
+      };
+      await Promise.all(Array.from({ length: resolved.concurrency }, () => worker()));
+      records.sort((a, b) => a.taskIndex - b.taskIndex);
+      const ok2 = records.filter((r) => r.status === "success").length;
+      console.log(`  Done: ${ok2}/${records.length} succeeded.
+`);
+      participantRecords.push({ participant: participant.name, records });
+      continue;
+    }
+    let rampStartMs;
+    await client.planWorkers(config.benchmarkSlug, runId, participant.name);
+    const result = await client.runWorker({
+      benchmarkSlug: config.benchmarkSlug,
+      runId,
+      participantSlug: participant.name,
+      concurrency: resolved.concurrency,
+      task: async (ctx) => {
+        const scheduleIndex = ctx.taskIndex - ctx.assignment.taskRange.start;
+        if (resolved.staggerDelayMs > 0) {
+          rampStartMs ??= Date.now();
+          const waitMs = rampStartMs + scheduleIndex * resolved.staggerDelayMs - Date.now();
+          if (waitMs > 0) await sleep(waitMs);
+        }
+        const slot = schedule[scheduleIndex];
+        const taskResult = await slot.task({
+          participant,
+          taskIndex: scheduleIndex,
+          phase: slot.phase,
+          step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options),
+          measure: ctx.measure,
+          log: ctx.log
+        });
+        if (slot.phase) ctx.measure({ phase: slot.phase });
+        return taskResult?.data;
+      },
+      onResult: (record) => onResult(record, { iterations: schedule.length, participant: participant.name })
+    });
+    if (!result.assignment) {
+      console.error(`  No pending worker to claim for run ${runId} \u2014 it may already be fully claimed.`);
+      participantRecords.push({ participant: participant.name, records: result.records ?? [] });
+      continue;
+    }
+    const ok = result.records.filter((r) => r.status === "success").length;
+    console.log(`  Done: ${ok}/${result.records.length} succeeded.
+`);
+    participantRecords.push({ participant: participant.name, records: result.records });
+  }
+  return participantRecords;
+}
+async function runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult, noIngest = false) {
+  const reporters = /* @__PURE__ */ new Map();
+  const logBuffers = /* @__PURE__ */ new Map();
+  const failed = /* @__PURE__ */ new Map();
+  const recordsByParticipant = /* @__PURE__ */ new Map();
+  for (const participant of available) {
+    logBuffers.set(participant.name, new LogBuffer());
+    failed.set(participant.name, false);
+    if (noIngest || !client) {
+      reporters.set(participant.name, null);
+      continue;
+    }
+    await client.planWorkers(config.benchmarkSlug, runId, participant.name, {
+      workerCount: 1,
+      targetConcurrency: schedule.length
+    });
+    let reporter = null;
+    try {
+      reporter = await import_client.BenchmarkReporter.claim({
+        baseUrl,
+        apiKey,
+        benchmarkSlug: config.benchmarkSlug,
+        runId,
+        participantSlug: participant.name,
+        processKind: "process",
+        processKey: process.env.HOSTNAME ?? "local"
+      });
+    } catch (error) {
+      console.warn(`  ${participant.name}: reporter claim failed (${error instanceof Error ? error.message : String(error)}) \u2014 running without platform reporting.`);
+    }
+    if (!reporter) {
+      console.warn(`  ${participant.name}: could not claim a platform worker \u2014 running without platform reporting.`);
+    }
+    reporters.set(participant.name, reporter);
+  }
+  console.log(`Interleaving ${available.length} participant(s), ${schedule.length} round(s) each.
+`);
+  for (let i = 0; i < schedule.length; i++) {
+    const slot = schedule[i];
+    if (resolved.staggerDelayMs > 0 && i > 0) {
+      await sleep(resolved.staggerDelayMs);
+    }
+    for (const participant of available) {
+      const reporter = reporters.get(participant.name) ?? null;
+      const logBuffer = logBuffers.get(participant.name);
+      const record = await runTaskRecord(
+        slot.task,
+        participant,
+        i,
+        (reporter?.taskIndexStart ?? 0) + i,
+        slot.phase,
+        logBuffer
+      );
+      if (record.status !== "success") failed.set(participant.name, true);
+      onResult(record, { iterations: schedule.length, participant: participant.name });
+      reporter?.recordResult(record);
+      if (!recordsByParticipant.has(participant.name)) {
+        recordsByParticipant.set(participant.name, []);
+      }
+      const participantRecords = recordsByParticipant.get(participant.name);
+      participantRecords.push(record);
+      if (reporter) {
+        reporter.setProgress({
+          done: participantRecords.length,
+          inFlight: 0,
+          errors: participantRecords.filter((item) => item.status !== "success").length,
+          total: schedule.length
+        });
+        await reporter.heartbeat();
+      }
+    }
+  }
+  for (const participant of available) {
+    const reporter = reporters.get(participant.name) ?? null;
+    const logBuffer = logBuffers.get(participant.name);
+    if (reporter && !logBuffer.isEmpty()) {
+      await reporter.uploadArtifact({ kind: "coordinator.log", contentType: "text/plain", name: "worker.log", body: logBuffer.toText() }).catch(() => {
+      });
+    }
+    await reporter?.finish(failed.get(participant.name) ?? false);
+    console.log(`  ${participant.name}: done${failed.get(participant.name) ? " (with errors)" : ""}.`);
+  }
+  return available.map((p) => ({ participant: p.name, records: recordsByParticipant.get(p.name) ?? [] }));
+}
+function mergeData(data, phase) {
+  const merged = { ...data ?? {}, ...phase ? { phase } : {} };
+  return Object.keys(merged).length > 0 ? merged : void 0;
+}
+async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase, logBuffer) {
+  const startedAtMs = Date.now();
+  const record = {
+    taskIndex,
+    status: "success",
+    startedAt: new Date(startedAtMs).toISOString()
+  };
+  const frameworkSteps = [];
+  const taskMeasures = {};
+  let activeStep = null;
+  const ctx = {
+    participant,
+    taskIndex: scheduleIndex,
+    phase,
+    async step(name, fn, options) {
+      const stepStartedAtMs = Date.now();
+      const stepRecord = {
+        name,
+        status: "success",
+        startedAt: new Date(stepStartedAtMs).toISOString(),
+        completedAt: new Date(stepStartedAtMs).toISOString(),
+        latencyMs: 0
+      };
+      if (options?.concurrency !== void 0) stepRecord.concurrency = options.concurrency;
+      if (options?.timeoutMs !== void 0) stepRecord.timeoutMs = options.timeoutMs;
+      const previousStep = activeStep;
+      activeStep = stepRecord;
+      try {
+        const result2 = await runStepInvocations(name, fn, options);
+        logBuffer.step(taskIndex, name, {});
+        return result2;
+      } catch (error) {
+        stepRecord.status = "error";
+        stepRecord.errorCode = error instanceof TaskError ? error.code ?? error.name : getErrorCode(error);
+        logBuffer.step(taskIndex, name, { error: error instanceof Error ? error.message : String(error) });
+        throw error;
+      } finally {
+        activeStep = previousStep;
+        stepRecord.completedAt = (/* @__PURE__ */ new Date()).toISOString();
+        stepRecord.latencyMs = Date.now() - stepStartedAtMs;
+        frameworkSteps.push(stepRecord);
+      }
+    },
+    measure(data) {
+      if (activeStep) {
+        activeStep.data = { ...activeStep.data ?? {}, ...data };
+      } else {
+        Object.assign(taskMeasures, data);
+      }
+    },
+    log(message, meta) {
+      logBuffer.line(`[task ${taskIndex}] ${message}`, meta);
+    }
+  };
+  let result = void 0;
+  try {
+    result = await task(ctx);
+    record.data = mergeData({ ...taskMeasures, ...result?.data ?? {} }, phase);
+  } catch (error) {
+    record.status = "error";
+    if (error instanceof TaskError) {
+      record.errorCode = error.code ?? error.name;
+      record.data = mergeData({ ...taskMeasures, ...error.data ?? {} }, phase);
+      if (error.steps?.length) frameworkSteps.push(...error.steps);
+    } else {
+      record.errorCode = getErrorCode(error);
+      record.data = mergeData(
+        { ...taskMeasures, errorMessage: error instanceof Error ? error.message : String(error) },
+        phase
+      );
+    }
+  } finally {
+    const endMs = Date.now();
+    record.completedAt = new Date(endMs).toISOString();
+    record.latencyMs = record.status === "success" && result && typeof result.latencyMs === "number" ? result.latencyMs : endMs - startedAtMs;
+    const taskSteps = record.status === "success" && result?.steps ? result.steps : [];
+    const allSteps = [...frameworkSteps, ...taskSteps];
+    if (allSteps.length === 0) {
+      allSteps.push({
+        name: "task",
+        status: record.status === "success" ? "success" : "error",
+        startedAt: record.startedAt,
+        completedAt: record.completedAt,
+        latencyMs: record.latencyMs,
+        errorCode: record.errorCode ?? null,
+        data: Object.keys(taskMeasures).length > 0 ? { ...taskMeasures } : void 0
+      });
+    }
+    record.steps = allSteps;
+  }
+  return record;
+}
+
+// src/cli.ts
+var import_node_path = require("path");
+var import_node_url = require("url");
+var USAGE = 'Usage:\n  bench run <file.bench.ts> [--shape name] [--provider a,b] [--run-key key]\n      [--benchmark slug] [--name "My benchmark"]\n      [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]\n      [--no-ingest | --dry-run]';
+function isBenchmarkConfig(value) {
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value;
+  return typeof candidate.benchmarkSlug === "string" && Array.isArray(candidate.participants);
+}
+async function runBenchmarkFile(argv) {
+  const [command, ...rest] = argv;
+  const [file, ...flags] = rest;
+  if (command !== "run" || !file || file.startsWith("-")) throw new Error(USAGE);
+  const mod = await import((0, import_node_url.pathToFileURL)((0, import_node_path.resolve)(process.cwd(), file)).href);
+  const config = mod.config;
+  const task = mod.task ?? mod.default;
+  if (!isBenchmarkConfig(config)) {
+    throw new Error(`${file} must export a \`config\` created with defineBenchmarkConfig (with participants).`);
+  }
+  if (typeof task !== "function") {
+    throw new Error(`${file} must export a \`task\` created with defineTask.`);
+  }
+  await runBenchmark(config, task, flags);
+}
+async function run(argv) {
+  try {
+    await runBenchmarkFile(argv);
+    process.exit(0);
+  } catch (err) {
+    if (err instanceof NoAvailableParticipantsError) {
+      console.log(err.message);
+      process.exit(0);
+    }
+    console.error("Benchmark failed:", err instanceof Error ? err.message : err);
+    process.exit(1);
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  NoAvailableParticipantsError,
+  TaskError,
+  defineBenchmarkConfig,
+  defineTask,
+  higherIsBetter,
+  lowerIsBetter,
+  mergeConfig,
+  parseCliArgs,
+  run,
+  runBenchmark,
+  runBenchmarkFile,
+  score
+});
+//# sourceMappingURL=index.cjs.map