@benchsdk/runner 0.2.0 → 0.3.0

package/dist/index.js

@@ -11,12 +11,101 @@ var TaskError = class extends Error {
     this.steps = opts?.steps;
   }
 };
+function assertNonEmptyString(value, field) {
+  if (typeof value !== "string" || value.trim() === "") {
+    throw new Error(`${field} must be a non-empty string`);
+  }
+  return value;
+}
+function assertOnlyAllowedKeys(value, allowed, field) {
+  for (const key of Object.keys(value)) {
+    if (!allowed.includes(key)) {
+      throw new Error(`${field} contains unexpected key: '${key}'`);
+    }
+  }
+}
 function assertPositiveInt(value, field) {
   if (value === void 0) return;
   if (!Number.isInteger(value) || value < 1) {
     throw new Error(`${field} must be an integer >= 1 (got ${value})`);
   }
 }
+function assertFiniteNumber(value, field) {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${field} must be a finite number (got ${value})`);
+  }
+  return value;
+}
+function validateBenchmarkScoringConfig(scoring, display) {
+  if (!Array.isArray(scoring.metrics) || scoring.metrics.length === 0) {
+    throw new Error("scoring.metrics must be a non-empty array");
+  }
+  if (scoring.success !== void 0) {
+    const requireData = scoring.success.requireData;
+    if (requireData === null || typeof requireData !== "object" || Array.isArray(requireData)) {
+      throw new Error("scoring.success.requireData must be a plain object");
+    }
+    if (Object.keys(requireData).length === 0) {
+      throw new Error("scoring.success.requireData must declare at least one data field");
+    }
+    for (const [key, value] of Object.entries(requireData)) {
+      const type = typeof value;
+      if (type !== "string" && type !== "number" && type !== "boolean") {
+        throw new Error(
+          `scoring.success.requireData.${key} must be a string, number, or boolean (got ${type})`
+        );
+      }
+    }
+  }
+  const seen = /* @__PURE__ */ new Set();
+  let totalWeight = 0;
+  for (let i = 0; i < scoring.metrics.length; i++) {
+    const metric = scoring.metrics[i];
+    if (metric === null || typeof metric !== "object" || Array.isArray(metric)) {
+      throw new Error(`scoring.metrics[${i}] must be an object`);
+    }
+    const key = metric.key;
+    if (typeof key !== "string" || key.trim() === "") {
+      throw new Error(`scoring.metrics[${i}].key must be a non-empty string`);
+    }
+    if (seen.has(key)) {
+      throw new Error(`duplicate scoring metric key: ${key}`);
+    }
+    seen.add(key);
+    const displayMetrics = Array.isArray(display?.metrics) ? display.metrics : void 0;
+    const displayMetric = displayMetrics?.find((m) => m?.key === key);
+    if (metric.unit !== void 0) {
+      if (typeof metric.unit !== "string") {
+        throw new Error(`scoring.metrics[${i}].unit must be a string`);
+      }
+      if (displayMetric?.unit !== void 0 && metric.unit !== displayMetric.unit) {
+        throw new Error(
+          `scoring.metrics[${i}].unit '${metric.unit}' conflicts with display.metrics[${i}].unit '${displayMetric.unit}' for key '${key}'`
+        );
+      }
+    }
+    assertFiniteNumber(metric.ceiling, `scoring.metrics[${i}].ceiling`);
+    if (metric.floor !== void 0) {
+      assertFiniteNumber(metric.floor, `scoring.metrics[${i}].floor`);
+    }
+    if (metric.weights === null || typeof metric.weights !== "object" || Array.isArray(metric.weights)) {
+      throw new Error(`scoring.metrics[${i}].weights must be an object`);
+    }
+    const median = assertFiniteNumber(metric.weights.median, `scoring.metrics[${i}].weights.median`);
+    const p95 = assertFiniteNumber(metric.weights.p95, `scoring.metrics[${i}].weights.p95`);
+    const p99 = assertFiniteNumber(metric.weights.p99, `scoring.metrics[${i}].weights.p99`);
+    if (median < 0 || p95 < 0 || p99 < 0) {
+      throw new Error(`scoring.metrics[${i}].weights must be non-negative`);
+    }
+    totalWeight += median + p95 + p99;
+    if (metric.trim !== void 0) {
+      assertFiniteNumber(metric.trim, `scoring.metrics[${i}].trim`);
+    }
+  }
+  if (Math.abs(totalWeight - 1) > 0.01) {
+    throw new Error(`scoring metric weights must sum to 1.0 (got ${totalWeight.toFixed(3)})`);
+  }
+}
 function defineBenchmarkConfig(config) {
   if (!config.benchmarkSlug || typeof config.benchmarkSlug !== "string") {
     throw new Error("benchmarkSlug is required");
@@ -64,6 +153,100 @@ function defineBenchmarkConfig(config) {
       }
     }
   }
+  if (config.dimensions !== void 0) {
+    if (config.dimensions === null || typeof config.dimensions !== "object" || Array.isArray(config.dimensions)) {
+      throw new Error("dimensions must be a plain object");
+    }
+  }
+  if (config.scoring !== void 0) {
+    validateBenchmarkScoringConfig(config.scoring, config.display);
+  }
+  if (config.customCliFlags !== void 0) {
+    if (!Array.isArray(config.customCliFlags) || !config.customCliFlags.every((f) => typeof f === "string" && f.startsWith("--"))) {
+      throw new Error('customCliFlags must be an array of strings starting with "--"');
+    }
+  }
+  if (config.display !== void 0) {
+    if (typeof config.display !== "object" || config.display === null || Array.isArray(config.display)) {
+      throw new Error("display must be an object");
+    }
+    assertOnlyAllowedKeys(config.display, ["metrics", "steps", "overview"], "display");
+    const displayMetricKeys = /* @__PURE__ */ new Set();
+    if (config.display.metrics !== void 0) {
+      if (!Array.isArray(config.display.metrics)) {
+        throw new Error("display.metrics must be an array");
+      }
+      for (let i = 0; i < config.display.metrics.length; i++) {
+        const metric = config.display.metrics[i];
+        if (metric === null || typeof metric !== "object" || Array.isArray(metric)) {
+          throw new Error(`display.metrics[${i}] must be an object`);
+        }
+        assertOnlyAllowedKeys(metric, ["key", "label", "unit", "direction", "decimals", "order"], `display.metrics[${i}]`);
+        const key = assertNonEmptyString(metric.key, `display.metrics[${i}].key`);
+        if (displayMetricKeys.has(key)) {
+          throw new Error(`duplicate display metric key: ${key}`);
+        }
+        displayMetricKeys.add(key);
+        assertNonEmptyString(metric.label, `display.metrics[${i}].label`);
+        if (metric.unit !== void 0 && typeof metric.unit !== "string") {
+          throw new Error(`display.metrics[${i}].unit must be a string`);
+        }
+        if (metric.direction !== void 0 && metric.direction !== "higher-better" && metric.direction !== "lower-better") {
+          throw new Error(`display.metrics[${i}].direction must be 'higher-better' or 'lower-better'`);
+        }
+        if (metric.decimals !== void 0 && (!Number.isInteger(metric.decimals) || metric.decimals < 0)) {
+          throw new Error(`display.metrics[${i}].decimals must be a non-negative integer`);
+        }
+        if (metric.order !== void 0 && (!Number.isInteger(metric.order) || metric.order < 0)) {
+          throw new Error(`display.metrics[${i}].order must be a non-negative integer`);
+        }
+      }
+    }
+    if (config.display.steps !== void 0) {
+      if (!Array.isArray(config.display.steps)) {
+        throw new Error("display.steps must be an array");
+      }
+      const seenStepKeys = /* @__PURE__ */ new Set();
+      for (let i = 0; i < config.display.steps.length; i++) {
+        const step = config.display.steps[i];
+        if (step === null || typeof step !== "object" || Array.isArray(step)) {
+          throw new Error(`display.steps[${i}] must be an object`);
+        }
+        assertOnlyAllowedKeys(step, ["key", "label", "order"], `display.steps[${i}]`);
+        const key = assertNonEmptyString(step.key, `display.steps[${i}].key`);
+        if (seenStepKeys.has(key)) {
+          throw new Error(`duplicate display step key: ${key}`);
+        }
+        seenStepKeys.add(key);
+        assertNonEmptyString(step.label, `display.steps[${i}].label`);
+        if (step.order !== void 0 && (!Number.isInteger(step.order) || step.order < 0)) {
+          throw new Error(`display.steps[${i}].order must be a non-negative integer`);
+        }
+      }
+    }
+    if (config.display.overview !== void 0) {
+      if (typeof config.display.overview !== "object" || config.display.overview === null || Array.isArray(config.display.overview)) {
+        throw new Error("display.overview must be an object");
+      }
+      assertOnlyAllowedKeys(config.display.overview, ["defaultMetric", "defaultLayout"], "display.overview");
+      const { defaultMetric, defaultLayout } = config.display.overview;
+      if (defaultMetric !== void 0) {
+        const metric = assertNonEmptyString(defaultMetric, "display.overview.defaultMetric");
+        const validDefaultMetrics = new Set(displayMetricKeys);
+        validDefaultMetrics.add("compositeScore");
+        validDefaultMetrics.add("task");
+        if (config.display.metrics !== void 0 && !validDefaultMetrics.has(metric)) {
+          throw new Error(`display.overview.defaultMetric '${metric}' is not declared in display.metrics and is not a known default (compositeScore, task)`);
+        }
+      }
+      if (defaultLayout !== void 0 && !["ranking", "cards", "chart", "leaderboard"].includes(defaultLayout)) {
+        throw new Error("display.overview.defaultLayout must be 'ranking', 'cards', 'chart', or 'leaderboard'");
+      }
+    }
+  }
+  if (config.display?.overview?.defaultMetric === "compositeScore" && config.scoring === void 0 && config.onScore === void 0) {
+    throw new Error("display.overview.defaultMetric cannot be 'compositeScore' without config.scoring or config.onScore");
+  }
   return config;
 }
 function defineTask(task) {
@@ -88,12 +271,15 @@ var NoAvailableParticipantsError = class extends Error {
 // src/runner.ts
 import { execSync } from "child_process";
 import os from "os";
+import { createBenchmarkClient } from "@benchsdk/api";
+import { resolveAuth } from "@benchsdk/cli";
 import {
   BenchmarkReporter,
-  createBenchmarkClient,
+  createSystemMetricsCollector,
   filterParticipantsByEnv,
+  runWorker,
   selectParticipants
-} from "@benchsdk/client";
+} from "@benchsdk/worker";
 
 // src/scoring.ts
 function isFiniteNumber(v) {
@@ -149,40 +335,121 @@ var higherIsBetter = (name, opts) => ({
   ...opts,
   higherIsBetter: true
 });
-function score(outcome, spec) {
+var WEIGHT_SUM_TOLERANCE = 0.01;
+var ScoringSpecError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ScoringSpecError";
+  }
+};
+function validateScoringSpec(spec) {
+  const totalWeight = spec.metrics.reduce(
+    (sum, m) => sum + m.weights.median + m.weights.p95 + m.weights.p99,
+    0
+  );
+  if (Math.abs(totalWeight - 1) > WEIGHT_SUM_TOLERANCE) {
+    const breakdown = spec.metrics.map((m) => `${m.name}=${(m.weights.median + m.weights.p95 + m.weights.p99).toFixed(3)}`).join(", ");
+    throw new ScoringSpecError(
+      `Scoring spec weights sum to ${totalWeight.toFixed(3)}, expected 1.0 (\xB1${WEIGHT_SUM_TOLERANCE}). Each metric's weights.median + weights.p95 + weights.p99, summed across every declared metric, must total 1.0 for compositeScore to stay a meaningful 0-100 scale. Per-metric totals: ${breakdown || "(no metrics declared)"}`
+    );
+  }
+}
+function groupRecordsByKey(records, key) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const raw = record.data?.[key];
+    const value = raw === void 0 ? void 0 : raw;
+    const mapKey = value === void 0 ? "__undefined__" : JSON.stringify(value);
+    let group = groups.get(mapKey);
+    if (!group) {
+      group = { value, records: [] };
+      groups.set(mapKey, group);
+    }
+    group.records.push(record);
+  }
+  return Array.from(groups.values());
+}
+function scoreGroup(records, spec, baseDimensions, groupKey, groupValue, provider, unitByMetric) {
   const successFilter = spec.success ?? ((r) => r.status === "success");
-  const dimensions = toJsonObject(spec.dimensions ?? {});
+  const passing = records.filter(successFilter);
+  const successRate = records.length === 0 ? 0 : passing.length / records.length;
+  const skipped = records.length === 0;
+  let metricScoresSum = 0;
+  const metrics = [];
+  for (const metric of spec.metrics) {
+    const samples = collectSamples(metric, passing);
+    if (samples.length === 0) {
+      continue;
+    }
+    const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05);
+    const metricScore = metric.weights.median * scoreStat(median, metric) + metric.weights.p95 * scoreStat(p95, metric) + metric.weights.p99 * scoreStat(p99, metric);
+    metricScoresSum += metricScore;
+    const unitKey = typeof metric.value === "string" ? metric.value : metric.name;
+    const unit = metric.unit ?? unitByMetric.get(unitKey) ?? "";
+    metrics.push({ name: metric.name, unit, median, p95, p99 });
+  }
+  const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100;
+  const dimensions = toJsonObject({
+    ...baseDimensions,
+    ...groupKey !== void 0 && groupValue !== void 0 ? { [groupKey]: groupValue } : {}
+  });
+  return {
+    provider,
+    dimensions,
+    metrics,
+    compositeScore,
+    successRate,
+    skipped
+  };
+}
+function score(outcome, spec, displayMetrics) {
+  validateScoringSpec(spec);
+  const baseDimensions = toJsonObject(spec.dimensions ?? {});
+  const unitByMetric = new Map(displayMetrics?.map((m) => [m.key, m.unit]));
   const results = [];
   for (const { participant, records } of outcome.participants) {
-    const passing = records.filter(successFilter);
-    const successRate = records.length === 0 ? 0 : passing.length / records.length;
-    const skipped = records.length === 0;
-    let metricScoresSum = 0;
-    const metrics = [];
-    for (const metric of spec.metrics) {
-      const samples = collectSamples(metric, passing);
-      if (samples.length === 0) {
-        continue;
-      }
-      const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05);
-      const metricScore = metric.weights.median * scoreStat(median, metric) + metric.weights.p95 * scoreStat(p95, metric) + metric.weights.p99 * scoreStat(p99, metric);
-      metricScoresSum += metricScore;
-      metrics.push({ name: metric.name, unit: metric.unit, median, p95, p99 });
-    }
-    const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100;
-    results.push({
-      provider: participant,
-      dimensions,
-      metrics,
-      compositeScore,
-      successRate,
-      skipped
-    });
+    const groups = spec.groupBy && records.length > 0 ? [{ value: void 0, records }, ...groupRecordsByKey(records, spec.groupBy)] : [{ value: void 0, records }];
+    for (const group of groups) {
+      results.push(scoreGroup(group.records, spec, baseDimensions, spec.groupBy, group.value, participant, unitByMetric));
+    }
   }
   return results;
 }
+function scoringConfigToSpec(config, dimensions, display) {
+  const success = config.success;
+  const unitByMetric = new Map(display?.metrics?.map((m) => [m.key, m.unit]));
+  return {
+    ...dimensions ? { dimensions: toJsonObject(dimensions) } : {},
+    ...config.groupBy ? { groupBy: config.groupBy } : {},
+    ...success ? {
+      success: (record) => record.status === "success" && Object.entries(success.requireData).every(([key, value]) => record.data?.[key] === value)
+    } : {},
+    metrics: config.metrics.map((metric) => ({
+      name: metric.key,
+      value: metric.key,
+      unit: (display?.metrics === void 0 ? metric.unit : unitByMetric.get(metric.key) ?? metric.unit) ?? "",
+      ceiling: metric.ceiling,
+      floor: metric.floor,
+      higherIsBetter: metric.higherIsBetter,
+      weights: metric.weights,
+      trim: metric.trim
+    }))
+  };
+}
 
 // src/log-buffer.ts
+var LOG_LEVEL_ORDER = ["debug", "info", "warn", "error"];
+function isLogOptions(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const o = value;
+  const keys = Object.keys(o);
+  if (keys.length === 0) return false;
+  if (!keys.every((k) => k === "level" || k === "meta")) return false;
+  if (o.level !== void 0 && (typeof o.level !== "string" || !LOG_LEVEL_ORDER.includes(o.level))) {
+    return false;
+  }
+  return true;
+}
 var LogBuffer = class {
   lines = [];
   step(taskIndex, stepName, outcome) {
@@ -199,9 +466,15 @@ var LogBuffer = class {
     }
   }
   /** Appends a free-form narration line (backs the task context's `log`). */
-  line(message, meta) {
-    const suffix = meta && Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : "";
-    this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} ${message}${suffix}`);
+  line(message, metaOrOptions) {
+    const opts = isLogOptions(metaOrOptions) ? { level: metaOrOptions.level ?? "info", meta: metaOrOptions.meta } : { level: "info", meta: metaOrOptions };
+    const suffix = opts.meta && Object.keys(opts.meta).length > 0 ? ` ${JSON.stringify(opts.meta)}` : "";
+    const taskMatch = message.match(/^(\[task \d+\])\s*(.*)$/);
+    if (taskMatch) {
+      this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} ${taskMatch[1]} [${opts.level}] ${taskMatch[2]}${suffix}`);
+    } else {
+      this.lines.push(`${(/* @__PURE__ */ new Date()).toISOString()} [${opts.level}] ${message}${suffix}`);
+    }
   }
   isEmpty() {
     return this.lines.length === 0;
@@ -215,7 +488,6 @@ function indent(text, prefix = "") {
 }
 
 // src/runner.ts
-var DEFAULT_PLATFORM_URL = "https://platform.computesdk.com";
 function isEnvNoIngest() {
   const v = process.env.BENCHSDK_NO_INGEST;
   return v === "1" || v?.toLowerCase() === "true";
@@ -223,8 +495,55 @@ function isEnvNoIngest() {
 function sleep(ms) {
   return new Promise((resolve2) => setTimeout(resolve2, ms));
 }
+function runWithConcurrency(fns, limit) {
+  if (limit >= fns.length || fns.length === 0) {
+    return Promise.all(fns.map((fn) => fn()));
+  }
+  const results = new Array(fns.length);
+  let running = 0;
+  let completed = 0;
+  let nextIndex = 0;
+  return new Promise((resolve2, reject) => {
+    const runNext = () => {
+      if (completed === fns.length) {
+        resolve2(results);
+        return;
+      }
+      while (running < limit && nextIndex < fns.length) {
+        const index = nextIndex++;
+        running++;
+        fns[index]().then(
+          (value) => {
+            results[index] = value;
+            running--;
+            completed++;
+            runNext();
+          },
+          (error) => reject(error)
+        );
+      }
+    };
+    runNext();
+  });
+}
 function getErrorCode(error) {
-  return error instanceof Error && error.name ? error.name : "ERROR";
+  if (error instanceof Error && "code" in error && typeof error.code === "string" && error.code) {
+    return error.code;
+  }
+  if (error instanceof Error && error.name) return error.name;
+  return "ERROR";
+}
+function isTaskError(error) {
+  return error instanceof Error && (error instanceof TaskError || error.name === "TaskError");
+}
+var STEP_OUTCOME_KEYS = /* @__PURE__ */ new Set(["stdout", "stderr", "error", "exitCode", "code", "signal", "pid"]);
+function isStepOutcome(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const o = value;
+  const keys = Object.keys(o);
+  if (keys.length === 0) return false;
+  if (!keys.every((k) => STEP_OUTCOME_KEYS.has(k))) return false;
+  return typeof o.stdout === "string" || typeof o.stderr === "string" || typeof o.error === "string";
 }
 function withTimeout(promise, ms, name) {
   return new Promise((resolve2, reject) => {
@@ -285,8 +604,10 @@ async function runStepWithClient(clientStep, name, fn, options) {
   const result = await clientStep(name, () => runStepInvocations(name, fn, options), clientStepOptions);
   return result;
 }
-function parseCliArgs(argv) {
+function parseCliArgs(argv, allowedCustomFlags) {
   const args = {};
+  const unknown = [];
+  const allowed = new Set(allowedCustomFlags ?? []);
   const readValue = (raw, i) => {
     const eq = raw.indexOf("=");
     if (eq !== -1) return { value: raw.slice(eq + 1), nextIndex: i };
@@ -378,22 +699,46 @@ function parseCliArgs(argv) {
       case "--dry-run":
         args.noIngest = true;
         break;
-      default:
+      default: {
+        if (allowed.has(name)) {
+          if (!arg.includes("=")) {
+            const next = argv[i + 1];
+            if (next && !next.startsWith("-")) {
+              i++;
+            }
+          }
+        } else {
+          unknown.push(name);
+          if (!arg.includes("=")) {
+            const next = argv[i + 1];
+            if (next && !next.startsWith("-")) {
+              i++;
+            }
+          }
+        }
         break;
+      }
     }
   }
+  if (unknown.length > 0) {
+    throw new Error(`Unknown flag(s): ${unknown.join(", ")}`);
+  }
   if (!args.noIngest && isEnvNoIngest()) {
     args.noIngest = true;
   }
   return args;
 }
 function mergeConfig(config, args) {
-  const phaseTotal = config.phases?.reduce((sum, p) => sum + p.iterations, 0);
-  if (phaseTotal !== void 0 && args.iterations !== void 0) {
-    console.warn("--iterations is ignored because this benchmark declares phases.");
+  const phases = config.phases;
+  const unevenPhases = phases !== void 0 && phases.some((p) => p.iterations !== phases[0].iterations);
+  if (unevenPhases && args.iterations !== void 0) {
+    console.warn("--iterations is ignored because this benchmark sizes its phases individually.");
   }
+  const phaseIterations = phases !== void 0 && !unevenPhases ? args.iterations : void 0;
+  const phaseTotal = phases !== void 0 ? phaseIterations !== void 0 ? phaseIterations * phases.length : phases.reduce((sum, p) => sum + p.iterations, 0) : void 0;
   const resolved = {
     iterations: phaseTotal ?? args.iterations ?? config.iterations ?? 1,
+    phaseIterations,
     concurrency: args.concurrency ?? config.concurrency ?? 1,
     staggerDelayMs: args.staggerDelayMs ?? config.staggerDelayMs ?? 0,
     groupBy: args.groupBy ?? config.groupBy ?? "participant",
@@ -407,13 +752,16 @@ function mergeConfig(config, args) {
   }
   return resolved;
 }
-function buildSchedule(config, iterations, task) {
+function buildSchedule(config, resolved, task) {
   if (config.phases?.length) {
     return config.phases.flatMap(
-      (phase) => Array.from({ length: phase.iterations }, () => ({ phase: phase.name, task }))
+      (phase) => Array.from({ length: resolved.phaseIterations ?? phase.iterations }, () => ({
+        phase: phase.name,
+        task
+      }))
     );
   }
-  return Array.from({ length: iterations }, () => ({ phase: void 0, task }));
+  return Array.from({ length: resolved.iterations }, () => ({ phase: void 0, task }));
 }
 function defaultOnResult(record, meta) {
   const n = record.taskIndex + 1;
@@ -424,19 +772,6 @@ function defaultOnResult(record, meta) {
     console.log(`  [${meta.participant}] Task ${n}/${meta.iterations}: FAILED \u2014 ${record.errorCode ?? "unknown error"}`);
   }
 }
-function resolvePlatform() {
-  const root = (process.env.BENCHMARKS_PLATFORM_URL || DEFAULT_PLATFORM_URL).replace(/\/+$/, "");
-  const apiKey = process.env.BENCHMARKS_PLATFORM_API_KEY;
-  if (!apiKey) {
-    throw new Error(
-      "BENCHMARKS_PLATFORM_API_KEY is required. Create an org-scoped API key in your organization settings on the platform and set it in your .env."
-    );
-  }
-  return {
-    baseUrl: `${root}/api/v1`,
-    apiKey
-  };
-}
 function resolveShape(config, shapeName) {
   if (!shapeName) return void 0;
   const shape = config.shapes?.[shapeName];
@@ -475,23 +810,44 @@ function resolveParticipants(config, resolved) {
   if (available.length === 0) throw new NoAvailableParticipantsError(skipped);
   return available;
 }
+function runConfigToJson(config, resolved, participants) {
+  const phases = config.phases?.map((phase) => ({
+    name: phase.name,
+    iterations: resolved.phaseIterations ?? phase.iterations
+  }));
+  const runConfig = {
+    benchmarkSlug: config.benchmarkSlug,
+    benchmarkName: config.benchmarkName,
+    ...resolved.phaseIterations !== void 0 ? { phaseIterations: resolved.phaseIterations } : {},
+    ...phases ? { phases } : {},
+    ...!config.phases ? { iterations: resolved.iterations } : {},
+    concurrency: resolved.concurrency,
+    staggerDelayMs: resolved.staggerDelayMs,
+    groupBy: resolved.groupBy,
+    ...config.dimensions ? { dimensions: config.dimensions } : {},
+    ...config.scoring ? { scoring: config.scoring } : {},
+    participants
+  };
+  return JSON.parse(JSON.stringify(runConfig));
+}
 async function runBenchmark(fileConfig, task, argv = []) {
-  const args = parseCliArgs(argv);
+  const args = parseCliArgs(argv, fileConfig.customCliFlags);
   const noIngest = args.noIngest ?? isEnvNoIngest();
   const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape));
   const config = applyIdentityOverrides(shaped, args);
   const resolved = mergeConfig(config, args);
+  const auth = await resolveAuth();
+  const client = createBenchmarkClient({
+    baseUrl: auth.apiBaseUrl,
+    apiKey: auth.apiKey,
+    token: auth.token,
+    orgSlug: auth.orgSlug,
+    orgId: auth.orgId
+  });
   const available = resolveParticipants(config, resolved);
-  let baseUrl = "";
-  let apiKey = "";
-  let client = null;
-  if (!noIngest) {
-    ({ baseUrl, apiKey } = resolvePlatform());
-    client = createBenchmarkClient({ baseUrl, apiKey });
-  }
-  const schedule = buildSchedule(config, resolved.iterations, task);
+  const schedule = buildSchedule(config, resolved, task);
   const totalTasks = schedule.length;
-  const concurrencyLabel = resolved.groupBy === "round" ? "n/a (round mode)" : String(resolved.concurrency);
+  const concurrencyLabel = String(resolved.concurrency);
   console.log(`${config.benchmarkName} (self-contained)`);
   console.log(`Date: ${(/* @__PURE__ */ new Date()).toISOString()}`);
   if (noIngest) {
@@ -509,16 +865,23 @@ async function runBenchmark(fileConfig, task, argv = []) {
     dashboardUrl = "";
   } else {
     if (identityIsOurs) {
+      const benchmarkConfig = {
+        ...config.scoring ? { scoring: config.scoring } : {},
+        ...config.display ? { display: config.display } : {}
+      };
       await client.upsertBenchmark(config.benchmarkSlug, {
-        name: config.benchmarkName
+        name: config.benchmarkName,
+        ...Object.keys(benchmarkConfig).length > 0 ? { config: benchmarkConfig } : {}
       });
     }
+    const runConfig = client ? runConfigToJson(config, resolved, available.map((p) => p.name)) : {};
     if (args.runKey) {
       const { run: run2, organizationSlug } = await client.createRun(config.benchmarkSlug, {
-        runKey: args.runKey
+        runKey: args.runKey,
+        config: runConfig
       });
       runId = run2.id;
-      dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run2.id);
+      dashboardUrl = dashboardUrlFor(auth.apiBaseUrl, organizationSlug, config.benchmarkSlug, run2.id);
       for (const participant of available) {
         await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks });
       }
@@ -529,10 +892,11 @@ async function runBenchmark(fileConfig, task, argv = []) {
       const { run: run2, organizationSlug } = await client.createRun(config.benchmarkSlug, {
         totalTasks,
         workerCount: 1,
-        participants: available.map((p) => p.name)
+        participants: available.map((p) => p.name),
+        config: runConfig
       });
       runId = run2.id;
-      dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run2.id);
+      dashboardUrl = dashboardUrlFor(auth.apiBaseUrl, organizationSlug, config.benchmarkSlug, run2.id);
       console.log(`Run created: ${run2.name} (${runId})`);
       console.log(`View at: ${dashboardUrl}
 `);
@@ -541,9 +905,9 @@ async function runBenchmark(fileConfig, task, argv = []) {
   const onResult = defaultOnResult;
   let participantRecords;
   if (resolved.groupBy === "round") {
-    participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult, noIngest);
+    participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, auth.apiBaseUrl, auth.apiKey, auth.token, auth.orgSlug, auth.orgId, onResult, noIngest);
   } else {
-    participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult);
+    participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult, noIngest);
   }
   console.log(`All done. ${noIngest ? "No platform run created." : `View at: ${dashboardUrl}`}`);
   const outcome = {
@@ -552,10 +916,10 @@ async function runBenchmark(fileConfig, task, argv = []) {
     participants: participantRecords,
     config: resolved
   };
-  if (client && config.onScore) {
+  if (!noIngest && (config.onScore || config.scoring)) {
     try {
-      const spec = await config.onScore(lowerIsBetter, higherIsBetter);
-      const scored = score(outcome, spec);
+      const spec = config.onScore ? await config.onScore(lowerIsBetter, higherIsBetter) : scoringConfigToSpec(config.scoring, config.dimensions, config.display);
+      const scored = score(outcome, spec, config.display?.metrics);
       const run2 = {
         gitSha: process.env.GITHUB_SHA ?? getGitSha(),
         gitRef: process.env.GITHUB_REF_NAME ?? process.env.GITHUB_REF ?? getGitRef(),
@@ -564,8 +928,13 @@ async function runBenchmark(fileConfig, task, argv = []) {
         platform: os.platform(),
         arch: os.arch()
       };
-      await client.submitRunSummary(config.benchmarkSlug, runId, { run: run2, results: scored });
+      await client.submitRunSummary(config.benchmarkSlug, runId, {
+        run: run2,
+        results: scored,
+        ...config.scoring ? { scoring: config.scoring } : {}
+      });
     } catch (err) {
+      if (err instanceof ScoringSpecError) throw err;
       const message = err instanceof Error ? err.message : String(err);
       console.warn(`[benchsdk-runner] failed to submit run summary: ${message}`);
     }
@@ -590,13 +959,13 @@ function getGitRef() {
     return void 0;
   }
 }
-async function runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult) {
+async function runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult, noIngest) {
   const participantRecords = [];
   for (const participant of available) {
     console.log(`${"=".repeat(70)}`);
     console.log(`  Participant: ${participant.name}`);
     console.log("=".repeat(70));
-    if (!client) {
+    if (noIngest || !client) {
       const records = [];
       let rampStartMs2;
       let nextIndex = 0;
@@ -628,7 +997,7 @@ async function runGroupedByParticipant(config, schedule, available, resolved, cl
     }
     let rampStartMs;
     await client.planWorkers(config.benchmarkSlug, runId, participant.name);
-    const result = await client.runWorker({
+    const result = await runWorker(client, {
       benchmarkSlug: config.benchmarkSlug,
       runId,
       participantSlug: participant.name,
@@ -641,16 +1010,21 @@ async function runGroupedByParticipant(config, schedule, available, resolved, cl
           if (waitMs > 0) await sleep(waitMs);
         }
         const slot = schedule[scheduleIndex];
-        const taskResult = await slot.task({
-          participant,
-          taskIndex: scheduleIndex,
-          phase: slot.phase,
-          step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options),
-          measure: ctx.measure,
-          log: ctx.log
-        });
         if (slot.phase) ctx.measure({ phase: slot.phase });
-        return taskResult?.data;
+        try {
+          const taskResult = await slot.task({
+            participant,
+            taskIndex: scheduleIndex,
+            phase: slot.phase,
+            step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options),
+            measure: ctx.measure,
+            log: ctx.log
+          });
+          return taskResult?.data;
+        } catch (error) {
+          if (isTaskError(error) && error.data) ctx.measure(error.data);
+          throw error;
+        }
       },
       onResult: (record) => onResult(record, { iterations: schedule.length, participant: participant.name })
     });
@@ -666,14 +1040,17 @@ async function runGroupedByParticipant(config, schedule, available, resolved, cl
   }
   return participantRecords;
 }
-async function runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult, noIngest = false) {
+async function runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, token, orgSlug, orgId, onResult, noIngest = false) {
   const reporters = /* @__PURE__ */ new Map();
   const logBuffers = /* @__PURE__ */ new Map();
   const failed = /* @__PURE__ */ new Map();
   const recordsByParticipant = /* @__PURE__ */ new Map();
+  let metricsCollector;
+  const metricsSamples = [];
   for (const participant of available) {
     logBuffers.set(participant.name, new LogBuffer());
     failed.set(participant.name, false);
+    recordsByParticipant.set(participant.name, []);
     if (noIngest || !client) {
       reporters.set(participant.name, null);
       continue;
@@ -687,6 +1064,9 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
       reporter = await BenchmarkReporter.claim({
         baseUrl,
         apiKey,
+        token,
+        orgSlug,
+        orgId,
         benchmarkSlug: config.benchmarkSlug,
         runId,
         participantSlug: participant.name,
@@ -700,6 +1080,10 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
       console.warn(`  ${participant.name}: could not claim a platform worker \u2014 running without platform reporting.`);
     }
     reporters.set(participant.name, reporter);
+    if (reporter && !metricsCollector) {
+      metricsCollector = createSystemMetricsCollector();
+      metricsSamples.push(await metricsCollector.sample());
+    }
   }
   console.log(`Interleaving ${available.length} participant(s), ${schedule.length} round(s) each.
 `);
@@ -708,7 +1092,7 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
     if (resolved.staggerDelayMs > 0 && i > 0) {
       await sleep(resolved.staggerDelayMs);
     }
-    for (const participant of available) {
+    const roundFns = available.map((participant) => async () => {
       const reporter = reporters.get(participant.name) ?? null;
       const logBuffer = logBuffers.get(participant.name);
       const record = await runTaskRecord(
@@ -722,9 +1106,6 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
       if (record.status !== "success") failed.set(participant.name, true);
       onResult(record, { iterations: schedule.length, participant: participant.name });
       reporter?.recordResult(record);
-      if (!recordsByParticipant.has(participant.name)) {
-        recordsByParticipant.set(participant.name, []);
-      }
       const participantRecords = recordsByParticipant.get(participant.name);
       participantRecords.push(record);
       if (reporter) {
@@ -736,7 +1117,22 @@ async function runGroupedByRound(config, schedule, available, resolved, client,
         });
         await reporter.heartbeat();
       }
-    }
+    });
+    await runWithConcurrency(roundFns, resolved.concurrency);
+    if (metricsCollector) metricsSamples.push(await metricsCollector.sample());
+  }
+  if (metricsCollector) metricsSamples.push(await metricsCollector.sample());
+  metricsCollector?.stop();
+  const metricsReporter = available.map((p) => reporters.get(p.name)).find((r) => Boolean(r));
+  if (metricsReporter && metricsSamples.length > 0) {
+    await metricsReporter.uploadArtifact({
+      kind: "system-metrics",
+      contentType: "application/x-ndjson",
+      name: "metrics.jsonl",
+      metadata: { scope: "shared-process", participants: available.map((p) => p.name) },
+      body: metricsSamples.map((sample) => JSON.stringify(sample)).join("\n") + "\n"
+    }).catch(() => {
+    });
   }
   for (const participant of available) {
     const reporter = reporters.get(participant.name) ?? null;
@@ -783,11 +1179,12 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
       activeStep = stepRecord;
       try {
         const result2 = await runStepInvocations(name, fn, options);
-        logBuffer.step(taskIndex, name, {});
+        const outcome = options?.captureOutput !== false && !Array.isArray(result2) && isStepOutcome(result2) ? result2 : {};
+        logBuffer.step(taskIndex, name, outcome);
         return result2;
       } catch (error) {
         stepRecord.status = "error";
-        stepRecord.errorCode = error instanceof TaskError ? error.code ?? error.name : getErrorCode(error);
+        stepRecord.errorCode = isTaskError(error) ? error.code ?? error.name : getErrorCode(error);
         logBuffer.step(taskIndex, name, { error: error instanceof Error ? error.message : String(error) });
         throw error;
       } finally {
@@ -804,8 +1201,8 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
         Object.assign(taskMeasures, data);
       }
     },
-    log(message, meta) {
-      logBuffer.line(`[task ${taskIndex}] ${message}`, meta);
+    log(message, metaOrOptions) {
+      logBuffer.line(`[task ${taskIndex}] ${message}`, metaOrOptions);
     }
   };
   let result = void 0;
@@ -814,7 +1211,7 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
     record.data = mergeData({ ...taskMeasures, ...result?.data ?? {} }, phase);
   } catch (error) {
     record.status = "error";
-    if (error instanceof TaskError) {
+    if (isTaskError(error)) {
       record.errorCode = error.code ?? error.name;
       record.data = mergeData({ ...taskMeasures, ...error.data ?? {} }, phase);
       if (error.steps?.length) frameworkSteps.push(...error.steps);
@@ -850,6 +1247,7 @@ async function runTaskRecord(task, participant, scheduleIndex, taskIndex, phase,
 // src/cli.ts
 import { resolve } from "path";
 import { pathToFileURL } from "url";
+import { run as runPlatformCli } from "@benchsdk/cli";
 var USAGE = 'Usage:\n  bench run <file.bench.ts> [--shape name] [--provider a,b] [--run-key key]\n      [--benchmark slug] [--name "My benchmark"]\n      [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]\n      [--no-ingest | --dry-run]';
 function isBenchmarkConfig(value) {
   if (typeof value !== "object" || value === null) return false;
@@ -872,6 +1270,9 @@ async function runBenchmarkFile(argv) {
   await runBenchmark(config, task, flags);
 }
 async function run(argv) {
+  if (argv[0] !== "run") {
+    return runPlatformCli(argv);
+  }
   try {
     await runBenchmarkFile(argv);
     process.exit(0);
@@ -884,8 +1285,13 @@ async function run(argv) {
     process.exit(1);
   }
 }
+
+// src/index.ts
+var BENCHSDK_RUNNER_VERSION = "0.3.0";
 export {
+  BENCHSDK_RUNNER_VERSION,
   NoAvailableParticipantsError,
+  ScoringSpecError,
   TaskError,
   defineBenchmarkConfig,
   defineTask,
@@ -896,6 +1302,8 @@ export {
   run,
   runBenchmark,
   runBenchmarkFile,
-  score
+  score,
+  scoringConfigToSpec,
+  validateScoringSpec
 };
 //# sourceMappingURL=index.js.map