@benchsdk/runner 0.2.0 → 0.5.2

package/dist/index.cjs

@@ -30,18 +30,37 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
+  AuthError: () => import_cli4.AuthError,
+  BENCHSDK_RUNNER_VERSION: () => BENCHSDK_RUNNER_VERSION,
+  BenchmarkApiError: () => import_api3.BenchmarkApiError,
+  BenchmarkConfigError: () => BenchmarkConfigError,
+  BenchmarkReporter: () => import_worker3.BenchmarkReporter,
   NoAvailableParticipantsError: () => NoAvailableParticipantsError,
+  ScoringSpecError: () => ScoringSpecError,
   TaskError: () => TaskError,
+  claimBenchmarkReporter: () => import_worker3.claimBenchmarkReporter,
+  createApiClient: () => import_cli4.createApiClient,
+  createBenchmarkClient: () => import_api3.createBenchmarkClient,
+  createSystemMetricsCollector: () => import_worker3.createSystemMetricsCollector,
   defineBenchmarkConfig: () => defineBenchmarkConfig,
+  defineOnComplete: () => defineOnComplete,
   defineTask: () => defineTask,
+  filterParticipantsByEnv: () => import_worker3.filterParticipantsByEnv,
   higherIsBetter: () => higherIsBetter,
   lowerIsBetter: () => lowerIsBetter,
   mergeConfig: () => mergeConfig,
   parseCliArgs: () => parseCliArgs,
+  resolveAuth: () => import_cli4.resolveAuth,
   run: () => run,
   runBenchmark: () => runBenchmark,
   runBenchmarkFile: () => runBenchmarkFile,
-  score: () => score
+  runBenchmarkWorker: () => runBenchmarkWorker,
+  runWorker: () => import_worker3.runWorker,
+  score: () => score,
+  scoringConfigToSpec: () => scoringConfigToSpec,
+  selectParticipants: () => import_worker3.selectParticipants,
+  validateBenchmarkConfig: () => validateBenchmarkConfig,
+  validateScoringSpec: () => validateScoringSpec
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -50,67 +69,231 @@ var TaskError = class extends Error {
   code;
   data;
   steps;
+  step;
+  timeoutMs;
   constructor(message, opts) {
     super(message);
     this.name = "TaskError";
     this.code = opts?.code;
     this.data = opts?.data;
     this.steps = opts?.steps;
+    this.step = opts?.step;
+    this.timeoutMs = opts?.timeoutMs;
+  }
+  toString() {
+    let s = `[${this.name}${this.code ? ` (${this.code})` : ""}] ${this.message}`;
+    if (this.step) {
+      s += `
+  step: ${this.step}`;
+    }
+    if (this.timeoutMs !== void 0) {
+      s += `
+  timeoutMs: ${this.timeoutMs}`;
+    }
+    if (this.data && Object.keys(this.data).length > 0) {
+      s += `
+  data: ${JSON.stringify(this.data, null, 2)}`;
+    }
+    return s;
+  }
+};
+var BenchmarkConfigError = class _BenchmarkConfigError extends Error {
+  issues;
+  constructor(issues) {
+    super(_BenchmarkConfigError.formatIssues(issues));
+    this.name = "BenchmarkConfigError";
+    this.issues = issues;
+  }
+  static formatIssues(issues) {
+    const lines = issues.map((i) => `  - ${i.field}: ${i.message}`);
+    return `Invalid benchmark config:
+${lines.join("\n")}
+
+Fix the fields above and try again.`;
   }
 };
-function assertPositiveInt(value, field) {
-  if (value === void 0) return;
-  if (!Number.isInteger(value) || value < 1) {
-    throw new Error(`${field} must be an integer >= 1 (got ${value})`);
+function assertFiniteNumber(value, field) {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${field} must be a finite number (got ${value})`);
   }
+  return value;
 }
-function defineBenchmarkConfig(config) {
-  if (!config.benchmarkSlug || typeof config.benchmarkSlug !== "string") {
-    throw new Error("benchmarkSlug is required");
+function assertNonEmptyString(value, field) {
+  if (typeof value !== "string" || value.trim() === "") {
+    throw new Error(`${field} must be a non-empty string`);
   }
-  if (!config.benchmarkName || typeof config.benchmarkName !== "string") {
-    throw new Error("benchmarkName is required");
+  return value;
+}
+function assertOnlyAllowedKeys(value, allowed, field) {
+  for (const key of Object.keys(value)) {
+    if (!allowed.includes(key)) {
+      throw new Error(`${field} contains unexpected key: '${key}'`);
+    }
   }
-  if (config.phases !== void 0) {
-    if (config.iterations !== void 0) {
-      throw new Error("phases and iterations are mutually exclusive");
+}
+function validateBenchmarkScoringConfig(scoring, display) {
+  if (!Array.isArray(scoring.metrics) || scoring.metrics.length === 0) {
+    throw new Error("scoring.metrics must be a non-empty array");
+  }
+  if (scoring.success !== void 0) {
+    const requireData = scoring.success.requireData;
+    if (requireData === null || typeof requireData !== "object" || Array.isArray(requireData)) {
+      throw new Error("scoring.success.requireData must be a plain object");
     }
-    if (!Array.isArray(config.phases) || config.phases.length === 0) {
-      throw new Error("phases must be a non-empty array");
+    if (Object.keys(requireData).length === 0) {
+      throw new Error("scoring.success.requireData must declare at least one data field");
     }
-    const seen = /* @__PURE__ */ new Set();
-    for (const phase of config.phases) {
-      if (!phase.name || typeof phase.name !== "string") {
-        throw new Error("each phase requires a non-empty name");
+    for (const [key, value] of Object.entries(requireData)) {
+      const type = typeof value;
+      if (type !== "string" && type !== "number" && type !== "boolean") {
+        throw new Error(
+          `scoring.success.requireData.${key} must be a string, number, or boolean (got ${type})`
+        );
       }
-      if (seen.has(phase.name)) {
-        throw new Error(`duplicate phase name: ${phase.name}`);
+    }
+  }
+  const seen = /* @__PURE__ */ new Set();
+  let totalWeight = 0;
+  for (let i = 0; i < scoring.metrics.length; i++) {
+    const metric = scoring.metrics[i];
+    if (metric === null || typeof metric !== "object" || Array.isArray(metric)) {
+      throw new Error(`scoring.metrics[${i}] must be an object`);
+    }
+    const key = metric.key;
+    if (typeof key !== "string" || key.trim() === "") {
+      throw new Error(`scoring.metrics[${i}].key must be a non-empty string`);
+    }
+    if (seen.has(key)) {
+      throw new Error(`duplicate scoring metric key: ${key}`);
+    }
+    seen.add(key);
+    const displayMetrics = Array.isArray(display?.metrics) ? display.metrics : void 0;
+    const displayMetric = displayMetrics?.find((m) => m?.key === key);
+    if (metric.unit !== void 0) {
+      if (typeof metric.unit !== "string") {
+        throw new Error(`scoring.metrics[${i}].unit must be a string`);
       }
-      seen.add(phase.name);
-      assertPositiveInt(phase.iterations, `phase '${phase.name}' iterations`);
+      if (displayMetric?.unit !== void 0 && metric.unit !== displayMetric.unit) {
+        throw new Error(
+          `scoring.metrics[${i}].unit '${metric.unit}' conflicts with display.metrics[${i}].unit '${displayMetric.unit}' for key '${key}'`
+        );
+      }
+    }
+    assertFiniteNumber(metric.ceiling, `scoring.metrics[${i}].ceiling`);
+    if (metric.floor !== void 0) {
+      assertFiniteNumber(metric.floor, `scoring.metrics[${i}].floor`);
+    }
+    if (metric.weights === null || typeof metric.weights !== "object" || Array.isArray(metric.weights)) {
+      throw new Error(`scoring.metrics[${i}].weights must be an object`);
+    }
+    const median = assertFiniteNumber(metric.weights.median, `scoring.metrics[${i}].weights.median`);
+    const p95 = assertFiniteNumber(metric.weights.p95, `scoring.metrics[${i}].weights.p95`);
+    const p99 = assertFiniteNumber(metric.weights.p99, `scoring.metrics[${i}].weights.p99`);
+    if (median < 0 || p95 < 0 || p99 < 0) {
+      throw new Error(`scoring.metrics[${i}].weights must be non-negative`);
+    }
+    totalWeight += median + p95 + p99;
+    if (metric.trim !== void 0) {
+      assertFiniteNumber(metric.trim, `scoring.metrics[${i}].trim`);
     }
   }
-  assertPositiveInt(config.iterations, "iterations");
-  assertPositiveInt(config.concurrency, "concurrency");
-  if (config.staggerDelayMs !== void 0 && (!Number.isFinite(config.staggerDelayMs) || config.staggerDelayMs < 0)) {
-    throw new Error(`staggerDelayMs must be a number >= 0 (got ${config.staggerDelayMs})`);
+  if (Math.abs(totalWeight - 1) > 0.01) {
+    throw new Error(`scoring metric weights must sum to 1.0 (got ${totalWeight.toFixed(3)})`);
   }
-  if (config.groupBy !== void 0 && config.groupBy !== "participant" && config.groupBy !== "round") {
-    throw new Error(`groupBy must be 'participant' or 'round' (got ${config.groupBy})`);
+}
+function validateBenchmarkDisplayConfig(display) {
+  if (typeof display !== "object" || display === null || Array.isArray(display)) {
+    throw new Error("display must be an object");
   }
-  if (config.shapes !== void 0) {
-    for (const [shapeName, shape] of Object.entries(config.shapes)) {
-      if (!shape.slug || !/^[a-z0-9][a-z0-9-]*$/.test(shape.slug)) {
-        throw new Error(`shape '${shapeName}' needs a lowercase slug (got ${JSON.stringify(shape.slug)})`);
+  assertOnlyAllowedKeys(display, ["metrics", "steps", "overview"], "display");
+  const displayMetricKeys = /* @__PURE__ */ new Set();
+  if (display.metrics !== void 0) {
+    if (!Array.isArray(display.metrics)) {
+      throw new Error("display.metrics must be an array");
+    }
+    for (let i = 0; i < display.metrics.length; i++) {
+      const metric = display.metrics[i];
+      if (metric === null || typeof metric !== "object" || Array.isArray(metric)) {
+        throw new Error(`display.metrics[${i}] must be an object`);
+      }
+      assertOnlyAllowedKeys(metric, ["key", "label", "unit", "direction", "decimals", "order"], `display.metrics[${i}]`);
+      const key = assertNonEmptyString(metric.key, `display.metrics[${i}].key`);
+      if (displayMetricKeys.has(key)) {
+        throw new Error(`duplicate display metric key: ${key}`);
       }
-      if (shape.name !== void 0 && (typeof shape.name !== "string" || shape.name.trim() === "")) {
-        throw new Error(`shape '${shapeName}' name must be a non-empty string`);
+      displayMetricKeys.add(key);
+      assertNonEmptyString(metric.label, `display.metrics[${i}].label`);
+      if (metric.unit !== void 0 && typeof metric.unit !== "string") {
+        throw new Error(`display.metrics[${i}].unit must be a string`);
       }
-      if (shape.staggerDelayMs !== void 0 && (!Number.isFinite(shape.staggerDelayMs) || shape.staggerDelayMs < 0)) {
-        throw new Error(`shape '${shapeName}' staggerDelayMs must be a number >= 0 (got ${shape.staggerDelayMs})`);
+      if (metric.direction !== void 0 && metric.direction !== "higher-better" && metric.direction !== "lower-better") {
+        throw new Error(`display.metrics[${i}].direction must be 'higher-better' or 'lower-better'`);
+      }
+      if (metric.decimals !== void 0 && (!Number.isInteger(metric.decimals) || metric.decimals < 0)) {
+        throw new Error(`display.metrics[${i}].decimals must be a non-negative integer`);
+      }
+      if (metric.order !== void 0 && (!Number.isInteger(metric.order) || metric.order < 0)) {
+        throw new Error(`display.metrics[${i}].order must be a non-negative integer`);
       }
     }
   }
+  if (display.steps !== void 0) {
+    if (!Array.isArray(display.steps)) {
+      throw new Error("display.steps must be an array");
+    }
+    const seenStepKeys = /* @__PURE__ */ new Set();
+    for (let i = 0; i < display.steps.length; i++) {
+      const step = display.steps[i];
+      if (step === null || typeof step !== "object" || Array.isArray(step)) {
+        throw new Error(`display.steps[${i}] must be an object`);
+      }
+      assertOnlyAllowedKeys(step, ["key", "label", "order"], `display.steps[${i}]`);
+      const key = assertNonEmptyString(step.key, `display.steps[${i}].key`);
+      if (seenStepKeys.has(key)) {
+        throw new Error(`duplicate display step key: ${key}`);
+      }
+      seenStepKeys.add(key);
+      assertNonEmptyString(step.label, `display.steps[${i}].label`);
+      if (step.order !== void 0 && (!Number.isInteger(step.order) || step.order < 0)) {
+        throw new Error(`display.steps[${i}].order must be a non-negative integer`);
+      }
+    }
+  }
+  if (display.overview !== void 0) {
+    if (typeof display.overview !== "object" || display.overview === null || Array.isArray(display.overview)) {
+      throw new Error("display.overview must be an object");
+    }
+    assertOnlyAllowedKeys(display.overview, ["defaultMetric", "defaultLayout"], "display.overview");
+    const { defaultMetric, defaultLayout } = display.overview;
+    if (defaultMetric !== void 0) {
+      const metric = assertNonEmptyString(defaultMetric, "display.overview.defaultMetric");
+      const validDefaultMetrics = new Set(displayMetricKeys);
+      validDefaultMetrics.add("compositeScore");
+      validDefaultMetrics.add("task");
+      if (display.metrics !== void 0 && !validDefaultMetrics.has(metric)) {
+        throw new Error(`display.overview.defaultMetric '${metric}' is not declared in display.metrics and is not a known default (compositeScore, task)`);
+      }
+    }
+    if (defaultLayout !== void 0 && !["ranking", "cards", "chart", "leaderboard"].includes(defaultLayout)) {
+      throw new Error("display.overview.defaultLayout must be 'ranking', 'cards', 'chart', or 'leaderboard'");
+    }
+  }
+}
+function defineBenchmarkConfig(config) {
+  const issues = validateBenchmarkConfig(config);
+  if (config.scoring !== void 0) {
+    try {
+      validateBenchmarkScoringConfig(config.scoring, config.display);
+    } catch (error) {
+      issues.push({
+        field: "scoring",
+        message: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+  if (issues.length > 0) {
+    throw new BenchmarkConfigError(issues);
+  }
   return config;
 }
 function defineTask(task) {
@@ -119,6 +302,136 @@ function defineTask(task) {
   }
   return task;
 }
+function validateBenchmarkConfig(config) {
+  const issues = [];
+  if (!config.benchmarkSlug || typeof config.benchmarkSlug !== "string") {
+    issues.push({ field: "benchmarkSlug", message: "is required" });
+  }
+  if (!config.benchmarkName || typeof config.benchmarkName !== "string") {
+    issues.push({ field: "benchmarkName", message: "is required" });
+  }
+  if (!Array.isArray(config.participants) || config.participants.length === 0) {
+    issues.push({ field: "participants", message: "must be a non-empty array" });
+  } else {
+    const seenParticipants = /* @__PURE__ */ new Set();
+    for (let i = 0; i < config.participants.length; i++) {
+      const p = config.participants[i];
+      if (p === null || typeof p !== "object" || Array.isArray(p)) {
+        issues.push({ field: `participants[${i}]`, message: "must be an object" });
+        continue;
+      }
+      const participant = p;
+      if (typeof participant.name !== "string" || participant.name.trim() === "") {
+        issues.push({ field: `participants[${i}].name`, message: "must be a non-empty string" });
+      } else if (seenParticipants.has(participant.name)) {
+        issues.push({ field: `participants[${i}].name`, message: `duplicate participant name: ${participant.name}` });
+      } else {
+        seenParticipants.add(participant.name);
+      }
+      if (participant.requiredEnvVars !== void 0 && (!Array.isArray(participant.requiredEnvVars) || !participant.requiredEnvVars.every((v) => typeof v === "string"))) {
+        issues.push({ field: `participants[${i}].requiredEnvVars`, message: "must be an array of strings" });
+      }
+    }
+  }
+  if (config.phases !== void 0) {
+    if (config.iterations !== void 0) {
+      issues.push({ field: "iterations", message: "phases and iterations are mutually exclusive" });
+    }
+    if (!Array.isArray(config.phases) || config.phases.length === 0) {
+      issues.push({ field: "phases", message: "must be a non-empty array" });
+    } else {
+      const seen = /* @__PURE__ */ new Set();
+      for (let i = 0; i < config.phases.length; i++) {
+        const phase = config.phases[i];
+        if (phase === null || typeof phase !== "object" || Array.isArray(phase)) {
+          issues.push({ field: `phases[${i}]`, message: "must be an object" });
+          continue;
+        }
+        const phaseObj = phase;
+        if (typeof phaseObj.name !== "string" || phaseObj.name.trim() === "") {
+          issues.push({ field: `phases[${i}]`, message: "must have a non-empty string name" });
+        } else {
+          const name = phaseObj.name;
+          if (seen.has(name)) {
+            issues.push({ field: `phases['${name}']`, message: `duplicate phase name: ${name}` });
+          }
+          seen.add(name);
+          const iterations = phaseObj.iterations;
+          if (typeof iterations !== "number" || !Number.isInteger(iterations) || iterations < 1) {
+            issues.push({ field: `phases['${name}'].iterations`, message: `must be an integer >= 1 (got ${iterations})` });
+          }
+        }
+      }
+    }
+  }
+  if (config.iterations !== void 0 && (!Number.isInteger(config.iterations) || config.iterations < 1)) {
+    issues.push({ field: "iterations", message: `must be an integer >= 1 (got ${config.iterations})` });
+  }
+  if (config.concurrency !== void 0 && (!Number.isInteger(config.concurrency) || config.concurrency < 1)) {
+    issues.push({ field: "concurrency", message: `must be an integer >= 1 (got ${config.concurrency})` });
+  }
+  if (config.staggerDelayMs !== void 0 && (!Number.isFinite(config.staggerDelayMs) || config.staggerDelayMs < 0)) {
+    issues.push({ field: "staggerDelayMs", message: `must be a number >= 0 (got ${config.staggerDelayMs})` });
+  }
+  if (config.groupBy !== void 0 && config.groupBy !== "participant" && config.groupBy !== "round") {
+    issues.push({ field: "groupBy", message: `must be 'participant' or 'round' (got ${config.groupBy})` });
+  }
+  if (config.shapes !== void 0) {
+    if (typeof config.shapes !== "object" || config.shapes === null || Array.isArray(config.shapes)) {
+      issues.push({ field: "shapes", message: "must be a plain object" });
+    } else {
+      for (const [shapeName, shape] of Object.entries(config.shapes)) {
+        if (shape === null || typeof shape !== "object" || Array.isArray(shape)) {
+          issues.push({ field: `shapes['${shapeName}']`, message: "must be an object" });
+          continue;
+        }
+        const shapeObj = shape;
+        const slug = shapeObj.slug;
+        if (typeof slug !== "string" || slug === "" || !/^[a-z0-9][a-z0-9-]*$/.test(slug)) {
+          issues.push({ field: `shapes['${shapeName}'].slug`, message: `needs a lowercase slug (got ${JSON.stringify(slug)})` });
+        }
+        const name = shapeObj.name;
+        if (name !== void 0 && (typeof name !== "string" || name.trim() === "")) {
+          issues.push({ field: `shapes['${shapeName}'].name`, message: "must be a non-empty string" });
+        }
+        const staggerDelayMs = shapeObj.staggerDelayMs;
+        if (staggerDelayMs !== void 0 && (typeof staggerDelayMs !== "number" || !Number.isFinite(staggerDelayMs) || staggerDelayMs < 0)) {
+          issues.push({ field: `shapes['${shapeName}'].staggerDelayMs`, message: `must be a number >= 0 (got ${staggerDelayMs})` });
+        }
+      }
+    }
+  }
+  if (config.dimensions !== void 0) {
+    if (config.dimensions === null || typeof config.dimensions !== "object" || Array.isArray(config.dimensions)) {
+      issues.push({ field: "dimensions", message: "must be a plain object" });
+    }
+  }
+  if (config.customCliFlags !== void 0) {
+    if (!Array.isArray(config.customCliFlags) || !config.customCliFlags.every((f) => typeof f === "string" && f.startsWith("--"))) {
+      issues.push({ field: "customCliFlags", message: 'must be an array of strings starting with "--"' });
+    }
+  }
+  if (config.display !== void 0) {
+    try {
+      validateBenchmarkDisplayConfig(config.display);
+    } catch (error) {
+      issues.push({
+        field: "display",
+        message: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+  if (config.display?.overview?.defaultMetric === "compositeScore" && config.scoring === void 0 && config.onScore === void 0) {
+    issues.push({
+      field: "display.overview.defaultMetric",
+      message: "cannot be 'compositeScore' without config.scoring or config.onScore"
+    });
+  }
+  return issues;
+}
+function defineOnComplete(onComplete) {
+  return onComplete;
+}
 
 // src/no-available-participants.ts
 var NoAvailableParticipantsError = class extends Error {
@@ -135,7 +448,9 @@ var NoAvailableParticipantsError = class extends Error {
 // src/runner.ts
 var import_node_child_process = require("child_process");
 var import_node_os = __toESM(require("os"), 1);
-var import_client = require("@benchsdk/client");
+var import_api = require("@benchsdk/api");
+var import_cli = require("@benchsdk/cli");
+var import_worker = require("@benchsdk/worker");
 
 // src/scoring.ts
 function isFiniteNumber(v) {
@@ -191,40 +506,121 @@ var higherIsBetter = (name, opts) => ({
   ...opts,
   higherIsBetter: true
 });
-function score(outcome, spec) {
+var WEIGHT_SUM_TOLERANCE = 0.01;
+var ScoringSpecError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ScoringSpecError";
+  }
+};
+function validateScoringSpec(spec) {
+  const totalWeight = spec.metrics.reduce(
+    (sum, m) => sum + m.weights.median + m.weights.p95 + m.weights.p99,
+    0
+  );
+  if (Math.abs(totalWeight - 1) > WEIGHT_SUM_TOLERANCE) {
+    const breakdown = spec.metrics.map((m) => `${m.name}=${(m.weights.median + m.weights.p95 + m.weights.p99).toFixed(3)}`).join(", ");
+    throw new ScoringSpecError(
+      `Scoring spec weights sum to ${totalWeight.toFixed(3)}, expected 1.0 (\xB1${WEIGHT_SUM_TOLERANCE}). Each metric's weights.median + weights.p95 + weights.p99, summed across every declared metric, must total 1.0 for compositeScore to stay a meaningful 0-100 scale. Per-metric totals: ${breakdown || "(no metrics declared)"}`
+    );
+  }
+}
+function groupRecordsByKey(records, key) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const raw = record.data?.[key];
+    const value = raw === void 0 ? void 0 : raw;
+    const mapKey = value === void 0 ? "__undefined__" : JSON.stringify(value);
+    let group = groups.get(mapKey);
+    if (!group) {
+      group = { value, records: [] };
+      groups.set(mapKey, group);
+    }
+    group.records.push(record);
+  }
+  return Array.from(groups.values());
+}
+function scoreGroup(records, spec, baseDimensions, groupKey, groupValue, provider, unitByMetric) {
   const successFilter = spec.success ?? ((r) => r.status === "success");
-  const dimensions = toJsonObject(spec.dimensions ?? {});
+  const passing = records.filter(successFilter);
+  const successRate = records.length === 0 ? 0 : passing.length / records.length;
+  const skipped = records.length === 0;
+  let metricScoresSum = 0;
+  const metrics = [];
+  for (const metric of spec.metrics) {
+    const samples = collectSamples(metric, passing);
+    if (samples.length === 0) {
+      continue;
+    }
+    const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05);
+    const metricScore = metric.weights.median * scoreStat(median, metric) + metric.weights.p95 * scoreStat(p95, metric) + metric.weights.p99 * scoreStat(p99, metric);
+    metricScoresSum += metricScore;
+    const unitKey = typeof metric.value === "string" ? metric.value : metric.name;
+    const unit = metric.unit ?? unitByMetric.get(unitKey) ?? "";
+    metrics.push({ name: metric.name, unit, median, p95, p99 });
+  }
+  const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100;
+  const dimensions = toJsonObject({
+    ...baseDimensions,
+    ...groupKey !== void 0 && groupValue !== void 0 ? { [groupKey]: groupValue } : {}
+  });
+  return {
+    provider,
+    dimensions,
+    metrics,
+    compositeScore,
+    successRate,
+    skipped
+  };
+}
+function score(outcome, spec, displayMetrics) {
+  validateScoringSpec(spec);
+  const baseDimensions = toJsonObject(spec.dimensions ?? {});
+  const unitByMetric = new Map(displayMetrics?.map((m) => [m.key, m.unit]));
   const results = [];
   for (const { participant, records } of outcome.participants) {
-    const passing = records.filter(successFilter);
-    const successRate = records.length === 0 ? 0 : passing.length / records.length;
-    const skipped = records.length === 0;
-    let metricScoresSum = 0;
-    const metrics = [];
-    for (const metric of spec.metrics) {
-      const samples = collectSamples(metric, passing);
-      if (samples.length === 0) {
-        continue;
-      }
-      const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05);
-      const metricScore = metric.weights.median * scoreStat(median, metric) + metric.weights.p95 * scoreStat(p95, metric) + metric.weights.p99 * scoreStat(p99, metric);
-      metricScoresSum += metricScore;
-      metrics.push({ name: metric.name, unit: metric.unit, median, p95, p99 });
-    }
-    const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100;
-    results.push({
-      provider: participant,
-      dimensions,
-      metrics,
-      compositeScore,
-      successRate,
-      skipped
-    });
+    const groups = spec.groupBy && records.length > 0 ? [{ value: void 0, records }, ...groupRecordsByKey(records, spec.groupBy)] : [{ value: void 0, records }];
+    for (const group of groups) {
+      results.push(scoreGroup(group.records, spec, baseDimensions, spec.groupBy, group.value, participant, unitByMetric));
+    }
   }
   return results;
 }
+function scoringConfigToSpec(config, dimensions, display) {
+  const success = config.success;
+  const unitByMetric = new Map(display?.metrics?.map((m) => [m.key, m.unit]));
+  return {
+    ...dimensions ? { dimensions: toJsonObject(dimensions) } : {},
+    ...config.groupBy ? { groupBy: config.groupBy } : {},
+    ...success ? {
+      success: (record) => record.status === "success" && Object.entries(success.requireData).every(([key, value]) => record.data?.[key] === value)
+    } : {},
+    metrics: config.metrics.map((metric) => ({
+      name: metric.key,
+      value: metric.key,
+      unit: (display?.metrics === void 0 ? metric.unit : unitByMetric.get(metric.key) ?? metric.unit) ?? "",
+      ceiling: metric.ceiling,
+      floor: metric.floor,
+      higherIsBetter: metric.higherIsBetter,
+      weights: metric.weights,
+      trim: metric.trim
+    }))
+  };
+}
 
 // src/log-buffer.ts
+var LOG_LEVEL_ORDER = ["debug", "info", "warn", "error"];
+function isLogOptions(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const o = value;
+  const keys = Object.keys(o);
+  if (keys.length === 0) return false;
+  if (!keys.every((k) => k === "level" || k === "meta")) return false;
+  if (o.level !== void 0 && (typeof o.level !== "string" || !LOG_LEVEL_ORDER.includes(o.level))) {
+    return false;
+  }
+  return true;
+}
 var LogBuffer = class {
   lines = [];
   step(taskIndex, stepName, outcome) {
@@ -241,9 +637,15 @@ var LogBuffer = class {
     }
   }
   /** Appends a free-form narration line (backs the task context's `log`). */
-  line(message, meta) {
-    const suffix = meta && Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : "";
-    this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} ${message}${suffix}`);
+  line(message, metaOrOptions) {
+    const opts = isLogOptions(metaOrOptions) ? { level: metaOrOptions.level ?? "info", meta: metaOrOptions.meta } : { level: "info", meta: metaOrOptions };
+    const suffix = opts.meta && Object.keys(opts.meta).length > 0 ? ` ${JSON.stringify(opts.meta)}` : "";
+    const taskMatch = message.match(/^(\[task \d+\])\s*(.*)$/);
+    if (taskMatch) {
+      this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} ${taskMatch[1]} [${opts.level}] ${taskMatch[2]}${suffix}`);
+    } else {
+      this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} [${opts.level}] ${message}${suffix}`);
+    }
   }
   isEmpty() {
     return this.lines.length === 0;
@@ -257,7 +659,6 @@ function indent(text, prefix = "") {
 }
 
 // src/runner.ts
-var DEFAULT_PLATFORM_URL = "https://platform.computesdk.com";
 function isEnvNoIngest() {
   const v = process.env.BENCHSDK_NO_INGEST;
   return v === "1" || v?.toLowerCase() === "true";
@@ -265,15 +666,68 @@ function isEnvNoIngest() {
 function sleep(ms) {
   return new Promise((resolve2) => setTimeout(resolve2, ms));
 }
+function runWithConcurrency(fns, limit) {
+  if (limit >= fns.length || fns.length === 0) {
+    return Promise.all(fns.map((fn) => fn()));
+  }
+  const results = new Array(fns.length);
+  let running = 0;
+  let completed = 0;
+  let nextIndex = 0;
+  return new Promise((resolve2, reject) => {
+    const runNext = () => {
+      if (completed === fns.length) {
+        resolve2(results);
+        return;
+      }
+      while (running < limit && nextIndex < fns.length) {
+        const index = nextIndex++;
+        running++;
+        fns[index]().then(
+          (value) => {
+            results[index] = value;
+            running--;
+            completed++;
+            runNext();
+          },
+          (error) => reject(error)
+        );
+      }
+    };
+    runNext();
+  });
+}
 function getErrorCode(error) {
-  return error instanceof Error && error.name ? error.name : "ERROR";
+  if (error instanceof Error && "code" in error && typeof error.code === "string" && error.code) {
+    return error.code;
+  }
+  if (error instanceof Error && error.name) return error.name;
+  return "ERROR";
+}
+function isTaskError(error) {
+  return error instanceof Error && (error instanceof TaskError || error.name === "TaskError");
 }
-function withTimeout(promise, ms, name) {
+var STEP_OUTCOME_KEYS = /* @__PURE__ */ new Set(["stdout", "stderr", "error", "exitCode", "code", "signal", "pid"]);
+function isStepOutcome(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const o = value;
+  const keys = Object.keys(o);
+  if (keys.length === 0) return false;
+  if (!keys.every((k) => STEP_OUTCOME_KEYS.has(k))) return false;
+  return typeof o.stdout === "string" || typeof o.stderr === "string" || typeof o.error === "string";
+}
+function withTimeout(promise, { stepName, timeoutMs, participantSlug }) {
   return new Promise((resolve2, reject) => {
-    const timer = setTimeout(
-      () => reject(new TaskError(`Step "${name}" timed out after ${ms}ms`, { code: "step_timeout" })),
-      ms
-    );
+    const timer = setTimeout(() => {
+      const participant = participantSlug ? ` for participant "${participantSlug}"` : "";
+      reject(
+        new TaskError(`Step "${stepName}" timed out after ${timeoutMs}ms${participant}`, {
+          code: "step_timeout",
+          step: stepName,
+          timeoutMs
+        })
+      );
+    }, timeoutMs);
     promise.then(
       (value) => {
         clearTimeout(timer);
@@ -286,20 +740,23 @@ function withTimeout(promise, ms, name) {
     );
   });
 }
-async function runStepInvocations(name, fn, options) {
-  const requestedConcurrency = options?.concurrency;
-  if (requestedConcurrency !== void 0 && (!Number.isInteger(requestedConcurrency) || requestedConcurrency < 1)) {
-    throw new Error(`step "${name}" concurrency must be an integer >= 1 (got ${requestedConcurrency})`);
+async function runStepInvocations(name, fn, options, participantSlug) {
+  if (options?.concurrency !== void 0) {
+    console.warn(`[benchsdk] step "${name}" option "concurrency" is deprecated; use "parallelInvocations"`);
+  }
+  const requestedParallelism = options?.parallelInvocations ?? options?.concurrency;
+  if (requestedParallelism !== void 0 && (!Number.isInteger(requestedParallelism) || requestedParallelism < 1)) {
+    throw new Error(`step "${name}" parallelInvocations must be an integer >= 1 (got ${requestedParallelism})`);
   }
   const timeoutMs = options?.timeoutMs;
   if (timeoutMs !== void 0 && (!Number.isFinite(timeoutMs) || timeoutMs < 0)) {
     throw new Error(`step "${name}" timeoutMs must be a number >= 0 (got ${timeoutMs})`);
   }
-  const count = requestedConcurrency ?? 1;
+  const count = requestedParallelism ?? 1;
   const invocations = Array.from({ length: count }, () => {
     const promise = Promise.resolve().then(() => fn());
     if (timeoutMs === void 0) return promise;
-    return withTimeout(promise, timeoutMs, name);
+    return withTimeout(promise, { stepName: name, timeoutMs, participantSlug });
   });
   if (count === 1) {
     return invocations[0];
@@ -317,18 +774,20 @@ async function runStepInvocations(name, fn, options) {
   if (firstError !== void 0) throw firstError;
   return results;
 }
-async function runStepWithClient(clientStep, name, fn, options) {
-  const { concurrency: runnerConcurrency, timeoutMs, ...clientOptions } = options ?? {};
+async function runStepWithClient(clientStep, name, fn, options, participantSlug) {
+  const { parallelInvocations: runnerParallelism, concurrency: deprecatedConcurrency, timeoutMs, ...clientOptions } = options ?? {};
   const clientStepOptions = {
     ...clientOptions,
     timeoutMs,
-    stepConcurrency: runnerConcurrency
+    stepConcurrency: runnerParallelism ?? deprecatedConcurrency
   };
-  const result = await clientStep(name, () => runStepInvocations(name, fn, options), clientStepOptions);
+  const result = await clientStep(name, () => runStepInvocations(name, fn, options, participantSlug), clientStepOptions);
   return result;
 }
-function parseCliArgs(argv) {
+function parseCliArgs(argv, allowedCustomFlags) {
   const args = {};
+  const unknown = [];
+  const allowed = new Set(allowedCustomFlags ?? []);
   const readValue = (raw, i) => {
     const eq = raw.indexOf("=");
     if (eq !== -1) return { value: raw.slice(eq + 1), nextIndex: i };
@@ -420,22 +879,46 @@ function parseCliArgs(argv) {
       case "--dry-run":
         args.noIngest = true;
         break;
-      default:
+      default: {
+        if (allowed.has(name)) {
+          if (!arg.includes("=")) {
+            const next = argv[i + 1];
+            if (next && !next.startsWith("-")) {
+              i++;
+            }
+          }
+        } else {
+          unknown.push(name);
+          if (!arg.includes("=")) {
+            const next = argv[i + 1];
+            if (next && !next.startsWith("-")) {
+              i++;
+            }
+          }
+        }
         break;
+      }
     }
   }
+  if (unknown.length > 0) {
+    throw new Error(`Unknown flag(s): ${unknown.join(", ")}`);
+  }
   if (!args.noIngest && isEnvNoIngest()) {
     args.noIngest = true;
   }
   return args;
 }
 function mergeConfig(config, args) {
-  const phaseTotal = config.phases?.reduce((sum, p) => sum + p.iterations, 0);
-  if (phaseTotal !== void 0 && args.iterations !== void 0) {
-    console.warn("--iterations is ignored because this benchmark declares phases.");
+  const phases = config.phases;
+  const unevenPhases = phases !== void 0 && phases.some((p) => p.iterations !== phases[0].iterations);
+  if (unevenPhases && args.iterations !== void 0) {
+    console.warn("--iterations is ignored because this benchmark sizes its phases individually.");
   }
+  const phaseIterations = phases !== void 0 && !unevenPhases ? args.iterations : void 0;
+  const phaseTotal = phases !== void 0 ? phaseIterations !== void 0 ? phaseIterations * phases.length : phases.reduce((sum, p) => sum + p.iterations, 0) : void 0;
   const resolved = {
     iterations: phaseTotal ?? args.iterations ?? config.iterations ?? 1,
+    phaseIterations,
     concurrency: args.concurrency ?? config.concurrency ?? 1,
     staggerDelayMs: args.staggerDelayMs ?? config.staggerDelayMs ?? 0,
     groupBy: args.groupBy ?? config.groupBy ?? "participant",
@@ -449,13 +932,16 @@ function mergeConfig(config, args) {
   }
   return resolved;
 }
-function buildSchedule(config, iterations, task) {
+function buildSchedule(config, resolved, task) {
   if (config.phases?.length) {
     return config.phases.flatMap(
-      (phase) => Array.from({ length: phase.iterations }, () => ({ phase: phase.name, task }))
+      (phase) => Array.from({ length: resolved.phaseIterations ?? phase.iterations }, () => ({
+        phase: phase.name,
+        task
+      }))
     );
   }
-  return Array.from({ length: iterations }, () => ({ phase: void 0, task }));
+  return Array.from({ length: resolved.iterations }, () => ({ phase: void 0, task }));
 }
 function defaultOnResult(record, meta) {
   const n = record.taskIndex + 1;
@@ -463,21 +949,10 @@ function defaultOnResult(record, meta) {
     const data = record.data && Object.keys(record.data).length > 0 ? ` ${JSON.stringify(record.data)}` : "";
     console.log(`  [${meta.participant}] Task ${n}/${meta.iterations}: success${data}`);
   } else {
-    console.log(`  [${meta.participant}] Task ${n}/${meta.iterations}: FAILED \u2014 ${record.errorCode ?? "unknown error"}`);
-  }
-}
-function resolvePlatform() {
-  const root = (process.env.BENCHMARKS_PLATFORM_URL || DEFAULT_PLATFORM_URL).replace(/\/+$/, "");
-  const apiKey = process.env.BENCHMARKS_PLATFORM_API_KEY;
-  if (!apiKey) {
-    throw new Error(
-      "BENCHMARKS_PLATFORM_API_KEY is required. Create an org-scoped API key in your organization settings on the platform and set it in your .env."
-    );
+    const detail = record.data?.errorMessage ?? record.data?.error;
+    const suffix = typeof detail === "string" && detail.length > 0 ? `: ${detail}` : "";
+    console.log(`  [${meta.participant}] Task ${n}/${meta.iterations}: FAILED \u2014 ${record.errorCode ?? "unknown error"}${suffix}`);
   }
-  return {
-    baseUrl: `${root}/api/v1`,
-    apiKey
-  };
 }
 function resolveShape(config, shapeName) {
   if (!shapeName) return void 0;
@@ -510,30 +985,81 @@ function dashboardUrlFor(baseUrl, organizationSlug, benchmarkSlug, runId) {
   return `${baseUrl.replace(/\/api\/v1\/?$/, "")}/${organizationSlug}/benchmarks/${benchmarkSlug}/runs/${runId}`;
 }
 function resolveParticipants(config, resolved) {
-  const { available, skipped } = (0, import_client.filterParticipantsByEnv)((0, import_client.selectParticipants)(config.participants, resolved.providers));
+  const { available, skipped } = (0, import_worker.filterParticipantsByEnv)((0, import_worker.selectParticipants)(config.participants, resolved.providers));
   for (const s of skipped) {
     console.log(`Skipping ${s.name}: missing ${s.missing.join(", ")}`);
   }
   if (available.length === 0) throw new NoAvailableParticipantsError(skipped);
   return available;
 }
-async function runBenchmark(fileConfig, task, argv = []) {
-  const args = parseCliArgs(argv);
+function resolveTriggerSource(env = process.env) {
+  const source = env.BENCH_TRIGGER_SOURCE?.trim();
+  if (source) return source;
+  return env.GITHUB_EVENT_NAME?.trim() || "manual";
+}
+function triggerToJson(env = process.env) {
+  const requestedBy = env.BENCH_TRIGGER_REQUESTED_BY?.trim();
+  const requestId = env.BENCH_TRIGGER_REQUEST_ID?.trim();
+  const event = env.GITHUB_EVENT_NAME?.trim();
+  return {
+    source: resolveTriggerSource(env),
+    ...event ? { event } : {},
+    ...requestedBy ? { requestedBy } : {},
+    ...requestId ? { requestId } : {}
+  };
+}
+function runConfigToJson(config, resolved, participants, env = process.env) {
+  const phases = config.phases?.map((phase) => ({
+    name: phase.name,
+    iterations: resolved.phaseIterations ?? phase.iterations
+  }));
+  const runConfig = {
+    benchmarkSlug: config.benchmarkSlug,
+    benchmarkName: config.benchmarkName,
+    ...resolved.phaseIterations !== void 0 ? { phaseIterations: resolved.phaseIterations } : {},
+    ...phases ? { phases } : {},
+    ...!config.phases ? { iterations: resolved.iterations } : {},
+    concurrency: resolved.concurrency,
+    staggerDelayMs: resolved.staggerDelayMs,
+    groupBy: resolved.groupBy,
+    ...config.dimensions ? { dimensions: config.dimensions } : {},
+    ...config.scoring ? { scoring: config.scoring } : {},
+    participants,
+    trigger: triggerToJson(env)
+  };
+  return JSON.parse(JSON.stringify(runConfig));
+}
+async function runBenchmark(fileConfig, task, argv = [], options = {}) {
+  const args = parseCliArgs(argv, fileConfig.customCliFlags);
   const noIngest = args.noIngest ?? isEnvNoIngest();
   const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape));
   const config = applyIdentityOverrides(shaped, args);
   const resolved = mergeConfig(config, args);
-  const available = resolveParticipants(config, resolved);
   let baseUrl = "";
   let apiKey = "";
+  let token;
+  let orgSlug;
+  let orgId;
   let client = null;
-  if (!noIngest) {
-    ({ baseUrl, apiKey } = resolvePlatform());
-    client = (0, import_client.createBenchmarkClient)({ baseUrl, apiKey });
+  const auth = noIngest ? await (0, import_cli.resolveAuth)({ baseUrl: options.baseUrl, apiKey: options.apiKey }).catch(() => null) : await (0, import_cli.resolveAuth)({ baseUrl: options.baseUrl, apiKey: options.apiKey });
+  if (auth) {
+    baseUrl = auth.apiBaseUrl;
+    apiKey = auth.apiKey ?? "";
+    token = auth.token;
+    orgSlug = auth.orgSlug;
+    orgId = auth.orgId;
+    client = (0, import_api.createBenchmarkClient)({
+      baseUrl: auth.apiBaseUrl,
+      apiKey: auth.apiKey,
+      token: auth.token,
+      orgSlug: auth.orgSlug,
+      orgId: auth.orgId
+    });
   }
-  const schedule = buildSchedule(config, resolved.iterations, task);
+  const available = resolveParticipants(config, resolved);
+  const schedule = buildSchedule(config, resolved, task);
   const totalTasks = schedule.length;
-  const concurrencyLabel = resolved.groupBy === "round" ? "n/a (round mode)" : String(resolved.concurrency);
+  const concurrencyLabel = String(resolved.concurrency);
   console.log(`${config.benchmarkName} (self-contained)`);
   console.log(`Date: ${(/* @__PURE__ */ new Date()).toISOString()}`);
   if (noIngest) {
@@ -550,14 +1076,32 @@ async function runBenchmark(fileConfig, task, argv = []) {
     runId = "no-ingest";
     dashboardUrl = "";
   } else {
-    if (identityIsOurs) {
-      await client.upsertBenchmark(config.benchmarkSlug, {
-        name: config.benchmarkName
-      });
+    {
+      const benchmarkConfig = {
+        ...config.scoring ? { scoring: config.scoring } : {},
+        ...config.display ? { display: config.display } : {}
+      };
+      let initializeTarget = identityIsOurs;
+      if (!initializeTarget) {
+        try {
+          await client.getBenchmark(config.benchmarkSlug);
+        } catch (error) {
+          if (!(error instanceof import_api.BenchmarkApiError && error.status === 404)) throw error;
+          initializeTarget = true;
+        }
+      }
+      const upsertInput = {};
+      if (initializeTarget) {
+        upsertInput.name = config.benchmarkName;
+        if (Object.keys(benchmarkConfig).length > 0) upsertInput.config = benchmarkConfig;
+      }
+      await client.upsertBenchmark(config.benchmarkSlug, upsertInput);
     }
+    const runConfig = client ? runConfigToJson(config, resolved, available.map((p) => p.name)) : {};
     if (args.runKey) {
       const { run: run2, organizationSlug } = await client.createRun(config.benchmarkSlug, {
-        runKey: args.runKey
+        runKey: args.runKey,
+        config: runConfig
       });
       runId = run2.id;
       dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run2.id);
@@ -571,7 +1115,8 @@ async function runBenchmark(fileConfig, task, argv = []) {
       const { run: run2, organizationSlug } = await client.createRun(config.benchmarkSlug, {
         totalTasks,
         workerCount: 1,
-        participants: available.map((p) => p.name)
+        participants: available.map((p) => p.name),
+        config: runConfig
       });
       runId = run2.id;
       dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run2.id);
@@ -583,9 +1128,9 @@ async function runBenchmark(fileConfig, task, argv = []) {
   const onResult = defaultOnResult;
   let participantRecords;
   if (resolved.groupBy === "round") {
-    participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult, noIngest);
+    participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, token, orgSlug, orgId, onResult, noIngest);
   } else {
-    participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult);
+    participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult, noIngest);
   }
   console.log(`All done. ${noIngest ? "No platform run created." : `View at: ${dashboardUrl}`}`);
   const outcome = {
@@ -594,20 +1139,25 @@ async function runBenchmark(fileConfig, task, argv = []) {
     participants: participantRecords,
     config: resolved
   };
-  if (client && config.onScore) {
+  if (!noIngest && client && (config.onScore || config.scoring)) {
     try {
-      const spec = await config.onScore(lowerIsBetter, higherIsBetter);
-      const scored = score(outcome, spec);
+      const spec = config.onScore ? await config.onScore(lowerIsBetter, higherIsBetter) : scoringConfigToSpec(config.scoring, config.dimensions, config.display);
+      const scored = score(outcome, spec, config.display?.metrics);
       const run2 = {
         gitSha: process.env.GITHUB_SHA ?? getGitSha(),
         gitRef: process.env.GITHUB_REF_NAME ?? process.env.GITHUB_REF ?? getGitRef(),
-        triggeredBy: process.env.GITHUB_EVENT_NAME ?? "manual",
+        triggeredBy: resolveTriggerSource(),
         nodeVersion: process.version,
         platform: import_node_os.default.platform(),
         arch: import_node_os.default.arch()
       };
-      await client.submitRunSummary(config.benchmarkSlug, runId, { run: run2, results: scored });
+      await client.submitRunSummary(config.benchmarkSlug, runId, {
+        run: run2,
+        results: scored,
+        ...config.scoring ? { scoring: config.scoring } : {}
+      });
     } catch (err) {
+      if (err instanceof ScoringSpecError) throw err;
       const message = err instanceof Error ? err.message : String(err);
       console.warn(`[benchsdk-runner] failed to submit run summary: ${message}`);
     }
@@ -615,6 +1165,22 @@ async function runBenchmark(fileConfig, task, argv = []) {
   if (config.onComplete) await config.onComplete(outcome);
   return outcome;
 }
+async function runBenchmarkWorker(options) {
+  const config = defineBenchmarkConfig({
+    benchmarkSlug: options.benchmarkSlug,
+    benchmarkName: options.benchmarkName ?? options.benchmarkSlug,
+    participants: [options.participant],
+    iterations: options.iterations ?? 1,
+    concurrency: options.concurrency ?? 1,
+    staggerDelayMs: options.staggerDelayMs ?? 0,
+    groupBy: options.groupBy ?? "participant",
+    defaultProviders: [options.participant.name]
+  });
+  const argv = [];
+  if (options.runKey) argv.push("--run-key", options.runKey);
+  if (options.noIngest) argv.push("--dry-run");
+  return runBenchmark(config, options.task, argv);
+}
 function getGitSha() {
   if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA;
   try {
@@ -632,13 +1198,13 @@ function getGitRef() {
     return void 0;
   }
 }
-async function runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult) {
+async function runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult, noIngest) {
   const participantRecords = [];
   for (const participant of available) {
     console.log(`${"=".repeat(70)}`);
     console.log(`  Participant: ${participant.name}`);
     console.log("=".repeat(70));
-    if (!client) {
+    if (noIngest || !client) {
       const records = [];
       let rampStartMs2;
       let nextIndex = 0;
@@ -670,7 +1236,7 @@ async function runGroupedByParticipant(config, schedule, available, resolved, cl
     }
     let rampStartMs;
     await client.planWorkers(config.benchmarkSlug, runId, participant.name);
-    const result = await client.runWorker({
+    const result = await (0, import_worker.runWorker)(client, {
       benchmarkSlug: config.benchmarkSlug,
       runId,
       participantSlug: participant.name,
@@ -683,16 +1249,21 @@ async function runGroupedByParticipant(config, schedule, available, resolved, cl
           if (waitMs > 0) await sleep(waitMs);
         }
         const slot = schedule[scheduleIndex];
-        const taskResult = await slot.task({
-          participant,
-          taskIndex: scheduleIndex,
-          phase: slot.phase,
-          step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options),
-          measure: ctx.measure,
-          log: ctx.log
-        });
         if (slot.phase) ctx.measure({ phase: slot.phase });
-        return taskResult?.data;
+        try {
+          const taskResult = await slot.task({
+            participant,
+            taskIndex: scheduleIndex,
+            phase: slot.phase,
+            step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options, participant.name),
+            measure: ctx.measure,
+            log: ctx.log
+          });
+          return taskResult?.data;
+        } catch (error) {
+          if (isTaskError(error) && error.data) ctx.measure(error.data);
+          throw error;
+        }
       },
       onResult: (record) => onResult(record, { iterations: schedule.length, participant: participant.name })
     });
@@ -708,14 +1279,17 @@ async function runGroupedByParticipant(config, schedule, available, resolved, cl
   }
   return participantRecords;
 }
-async function runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult, noIngest = false) {
+async function runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, token, orgSlug, orgId, onResult, noIngest = false) {
   const reporters = /* @__PURE__ */ new Map();
   const logBuffers = /* @__PURE__ */ new Map();
   const failed = /* @__PURE__ */ new Map();
   const recordsByParticipant = /* @__PURE__ */ new Map();
+  let metricsCollector;
+  const metricsSamples = [];
   for (const participant of available) {
     logBuffers.set(participant.name, new LogBuffer());
     failed.set(participant.name, false);
+    recordsByParticipant.set(participant.name, []);
     if (noIngest || !client) {
       reporters.set(participant.name, null);
       continue;
@@ -726,9 +1300,12 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
     });
     let reporter = null;
     try {
-      reporter = await import_client.BenchmarkReporter.claim({
+      reporter = await import_worker.BenchmarkReporter.claim({
         baseUrl,
         apiKey,
+        token,
+        orgSlug,
+        orgId,
         benchmarkSlug: config.benchmarkSlug,
         runId,
         participantSlug: participant.name,
@@ -742,6 +1319,10 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
       console.warn(`  ${participant.name}: could not claim a platform worker \u2014 running without platform reporting.`);
     }
     reporters.set(participant.name, reporter);
+    if (reporter && !metricsCollector) {
+      metricsCollector = (0, import_worker.createSystemMetricsCollector)();
+      metricsSamples.push(await metricsCollector.sample());
+    }
   }
   console.log(`Interleaving ${available.length} participant(s), ${schedule.length} round(s) each.
 `);
@@ -750,7 +1331,7 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
     if (resolved.staggerDelayMs > 0 && i > 0) {
       await sleep(resolved.staggerDelayMs);
     }
-    for (const participant of available) {
+    const roundFns = available.map((participant) => async () => {
       const reporter = reporters.get(participant.name) ?? null;
       const logBuffer = logBuffers.get(participant.name);
       const record = await runTaskRecord(
@@ -764,9 +1345,6 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
       if (record.status !== "success") failed.set(participant.name, true);
       onResult(record, { iterations: schedule.length, participant: participant.name });
       reporter?.recordResult(record);
-      if (!recordsByParticipant.has(participant.name)) {
-        recordsByParticipant.set(participant.name, []);
-      }
       const participantRecords = recordsByParticipant.get(participant.name);
       participantRecords.push(record);
       if (reporter) {
@@ -778,7 +1356,22 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
         });
         await reporter.heartbeat();
       }
-    }
+    });
+    await runWithConcurrency(roundFns, resolved.concurrency);
+    if (metricsCollector) metricsSamples.push(await metricsCollector.sample());
+  }
+  if (metricsCollector) metricsSamples.push(await metricsCollector.sample());
+  metricsCollector?.stop();
+  const metricsReporter = available.map((p) => reporters.get(p.name)).find((r) => Boolean(r));
+  if (metricsReporter && metricsSamples.length > 0) {
+    await metricsReporter.uploadArtifact({
+      kind: "system-metrics",
+      contentType: "application/x-ndjson",
+      name: "metrics.jsonl",
+      metadata: { scope: "shared-process", participants: available.map((p) => p.name) },
+      body: metricsSamples.map((sample) => JSON.stringify(sample)).join("\n") + "\n"
+    }).catch(() => {
+    });
   }
   for (const participant of available) {
     const reporter = reporters.get(participant.name) ?? null;
@@ -819,17 +1412,19 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
         completedAt: new Date(stepStartedAtMs).toISOString(),
         latencyMs: 0
       };
-      if (options?.concurrency !== void 0) stepRecord.concurrency = options.concurrency;
+      const requestedParallelism = options?.parallelInvocations ?? options?.concurrency;
+      if (requestedParallelism !== void 0) stepRecord.concurrency = requestedParallelism;
       if (options?.timeoutMs !== void 0) stepRecord.timeoutMs = options.timeoutMs;
       const previousStep = activeStep;
       activeStep = stepRecord;
       try {
-        const result2 = await runStepInvocations(name, fn, options);
-        logBuffer.step(taskIndex, name, {});
+        const result2 = await runStepInvocations(name, fn, options, participant.name);
+        const outcome = options?.captureOutput !== false && !Array.isArray(result2) && isStepOutcome(result2) ? result2 : {};
+        logBuffer.step(taskIndex, name, outcome);
         return result2;
       } catch (error) {
         stepRecord.status = "error";
-        stepRecord.errorCode = error instanceof TaskError ? error.code ?? error.name : getErrorCode(error);
+        stepRecord.errorCode = isTaskError(error) ? error.code ?? error.name : getErrorCode(error);
         logBuffer.step(taskIndex, name, { error: error instanceof Error ? error.message : String(error) });
         throw error;
       } finally {
@@ -846,8 +1441,8 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
         Object.assign(taskMeasures, data);
       }
     },
-    log(message, meta) {
-      logBuffer.line(`[task ${taskIndex}] ${message}`, meta);
+    log(message, metaOrOptions) {
+      logBuffer.line(`[task ${taskIndex}] ${message}`, metaOrOptions);
     }
   };
   let result = void 0;
@@ -856,7 +1451,7 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
     record.data = mergeData({ ...taskMeasures, ...result?.data ?? {} }, phase);
   } catch (error) {
     record.status = "error";
-    if (error instanceof TaskError) {
+    if (isTaskError(error)) {
       record.errorCode = error.code ?? error.name;
       record.data = mergeData({ ...taskMeasures, ...error.data ?? {} }, phase);
       if (error.steps?.length) frameworkSteps.push(...error.steps);
@@ -892,16 +1487,145 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
 // src/cli.ts
 var import_node_path = require("path");
 var import_node_url = require("url");
-var USAGE = 'Usage:\n  bench run <file.bench.ts> [--shape name] [--provider a,b] [--run-key key]\n      [--benchmark slug] [--name "My benchmark"]\n      [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]\n      [--no-ingest | --dry-run]';
+var import_cli2 = require("@benchsdk/cli");
+var import_api2 = require("@benchsdk/api");
+var import_worker2 = require("@benchsdk/worker");
+var USAGE = 'Usage:\n  bench run <file.bench.ts> [--shape name] [--provider a,b] [--run-key key]\n      [--benchmark slug] [--name "My benchmark"]\n      [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]\n      [--no-ingest | --dry-run] [--check]\n  bench check <file.bench.ts> [--base-url <url>] [--api-key <key>]';
 function isBenchmarkConfig(value) {
   if (typeof value !== "object" || value === null) return false;
   const candidate = value;
   return typeof candidate.benchmarkSlug === "string" && Array.isArray(candidate.participants);
 }
+function shiftFlag(argv, name) {
+  const prefix = `--${name}`;
+  const prefixEq = `${prefix}=`;
+  const result = [];
+  let value;
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === prefix) {
+      const next = argv[++i];
+      if (!next || next.startsWith("--")) throw new Error(USAGE);
+      value = next;
+      continue;
+    }
+    if (arg.startsWith(prefixEq)) {
+      const eqValue = arg.slice(prefixEq.length);
+      if (!eqValue) throw new Error(USAGE);
+      value = eqValue;
+      continue;
+    }
+    result.push(arg);
+  }
+  return { value, argv: result };
+}
+function shiftPlatformFlags(flags) {
+  const { value: baseUrl, argv: withoutBaseUrl } = shiftFlag(flags, "base-url");
+  const { value: apiKey, argv: rest } = shiftFlag(withoutBaseUrl, "api-key");
+  return { baseUrl, apiKey, flags: rest };
+}
+async function runCheck(argv) {
+  const [command, ...rest] = argv;
+  const [file, ...flags] = rest;
+  if (command !== "check" || !file || file.startsWith("-")) throw new Error(USAGE);
+  const mod = await import((0, import_node_url.pathToFileURL)((0, import_node_path.resolve)(process.cwd(), file)).href);
+  const config = mod.config;
+  const task = mod.task ?? mod.default;
+  if (!isBenchmarkConfig(config)) {
+    throw new Error(`${file} must export a \`config\` created with defineBenchmarkConfig (with participants).`);
+  }
+  if (typeof task !== "function") {
+    throw new Error(`${file} must export a \`task\` created with defineTask.`);
+  }
+  const cfg = config;
+  const configIssues = validateBenchmarkConfig(cfg);
+  if (configIssues.length > 0) {
+    throw new BenchmarkConfigError(configIssues);
+  }
+  const { baseUrl, apiKey, flags: runnerFlags } = shiftPlatformFlags(flags);
+  const parsed = parseCliArgs(runnerFlags, cfg.customCliFlags ?? []);
+  resolveShape(cfg, parsed.shape);
+  const dryRun = parsed.noIngest ?? false;
+  let client;
+  let apiOk = dryRun;
+  let auth = null;
+  if (!dryRun) {
+    try {
+      auth = await (0, import_cli2.resolveAuth)({ baseUrl, apiKey });
+      client = (0, import_api2.createBenchmarkClient)({
+        baseUrl: auth.apiBaseUrl,
+        apiKey: auth.apiKey,
+        token: auth.token,
+        orgSlug: auth.orgSlug,
+        orgId: auth.orgId
+      });
+      await client.listBenchmarks({ limit: 1 });
+      apiOk = true;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.warn(`[benchsdk] API connectivity check failed: ${message}`);
+    }
+  }
+  const effectiveProviderNames = parsed.providers ?? cfg.defaultProviders;
+  let selected;
+  try {
+    selected = (0, import_worker2.selectParticipants)(cfg.participants, effectiveProviderNames);
+  } catch (err) {
+    throw new Error(`Participant selection failed: ${err instanceof Error ? err.message : err}`);
+  }
+  const { available, skipped } = (0, import_worker2.filterParticipantsByEnv)(selected);
+  let scoringOk = true;
+  if (cfg.onScore) {
+    try {
+      const spec = await cfg.onScore(lowerIsBetter, higherIsBetter);
+      validateScoringSpec(spec);
+    } catch (err) {
+      scoringOk = false;
+      const message = err instanceof Error ? err.message : String(err);
+      console.warn(`[benchsdk] Scoring validation failed: ${message}`);
+    }
+  } else if (cfg.scoring) {
+    try {
+      const spec = scoringConfigToSpec(cfg.scoring, cfg.dimensions);
+      validateScoringSpec(spec);
+    } catch (err) {
+      scoringOk = false;
+      const message = err instanceof Error ? err.message : String(err);
+      console.warn(`[benchsdk] Scoring validation failed: ${message}`);
+    }
+  }
+  const missingPlatformAuth = dryRun ? [] : auth ? [] : [["BENCHMARKS_PLATFORM_API_KEY or BENCHMARKS_PLATFORM_TOKEN", void 0]];
+  for (const [name] of missingPlatformAuth) {
+    console.warn(`[benchsdk] ${name} is not set`);
+  }
+  const report = {
+    file,
+    benchmarkSlug: cfg.benchmarkSlug,
+    apiOk,
+    envOk: dryRun || missingPlatformAuth.length === 0,
+    participants: {
+      requested: selected.map((p) => p.name),
+      available: available.map((p) => p.name),
+      skipped: skipped.map((s) => ({ name: s.name, missing: s.missing }))
+    },
+    scoringOk: cfg.scoring || cfg.onScore ? scoringOk : void 0
+  };
+  console.log(JSON.stringify(report, null, 2));
+  const authFailure = !dryRun && missingPlatformAuth.length > 0;
+  if (!apiOk || available.length === 0 || scoringOk === false || authFailure) {
+    throw new Error("Benchmark check failed. See warnings above for details.");
+  }
+}
 async function runBenchmarkFile(argv) {
   const [command, ...rest] = argv;
   const [file, ...flags] = rest;
   if (command !== "run" || !file || file.startsWith("-")) throw new Error(USAGE);
+  const check = flags.includes("--check") || flags.includes("--validate");
+  if (check) {
+    const checkFlags = flags.filter((f) => f !== "--check" && f !== "--validate");
+    return runCheck(["check", file, ...checkFlags]);
+  }
+  const { baseUrl, apiKey, flags: runnerFlags } = shiftPlatformFlags(flags);
   const mod = await import((0, import_node_url.pathToFileURL)((0, import_node_path.resolve)(process.cwd(), file)).href);
   const config = mod.config;
   const task = mod.task ?? mod.default;
@@ -911,34 +1635,78 @@ async function runBenchmarkFile(argv) {
   if (typeof task !== "function") {
     throw new Error(`${file} must export a \`task\` created with defineTask.`);
   }
-  await runBenchmark(config, task, flags);
+  const envFlags = [];
+  if (process.env.BENCHMARK_SLUG) {
+    envFlags.push("--benchmark", process.env.BENCHMARK_SLUG);
+  }
+  if (process.env.BENCHMARK_NAME) {
+    envFlags.push("--name", process.env.BENCHMARK_NAME);
+  }
+  await runBenchmark(
+    config,
+    task,
+    [...envFlags, ...runnerFlags],
+    { baseUrl, apiKey }
+  );
 }
 async function run(argv) {
+  const [command, ...rest] = argv;
   try {
-    await runBenchmarkFile(argv);
+    if (command === "run") {
+      await runBenchmarkFile(argv);
+    } else if (command === "check") {
+      await runCheck(argv);
+    } else {
+      return (0, import_cli2.run)(argv);
+    }
     process.exit(0);
   } catch (err) {
     if (err instanceof NoAvailableParticipantsError) {
       console.log(err.message);
       process.exit(0);
     }
-    console.error("Benchmark failed:", err instanceof Error ? err.message : err);
+    console.error("Benchmark failed:", String(err));
     process.exit(1);
   }
 }
+
+// src/index.ts
+var import_cli4 = require("@benchsdk/cli");
+var import_worker3 = require("@benchsdk/worker");
+var import_api3 = require("@benchsdk/api");
+var BENCHSDK_RUNNER_VERSION = "0.5.2";
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  AuthError,
+  BENCHSDK_RUNNER_VERSION,
+  BenchmarkApiError,
+  BenchmarkConfigError,
+  BenchmarkReporter,
   NoAvailableParticipantsError,
+  ScoringSpecError,
   TaskError,
+  claimBenchmarkReporter,
+  createApiClient,
+  createBenchmarkClient,
+  createSystemMetricsCollector,
   defineBenchmarkConfig,
+  defineOnComplete,
   defineTask,
+  filterParticipantsByEnv,
   higherIsBetter,
   lowerIsBetter,
   mergeConfig,
   parseCliArgs,
+  resolveAuth,
   run,
   runBenchmark,
   runBenchmarkFile,
-  score
+  runBenchmarkWorker,
+  runWorker,
+  score,
+  scoringConfigToSpec,
+  selectParticipants,
+  validateBenchmarkConfig,
+  validateScoringSpec
 });
 //# sourceMappingURL=index.cjs.map