badgr-cli 1.0.46 → 1.0.48

package/src/commands/batch.js

@@ -0,0 +1,620 @@
+/**
+ * badgr batch — generic containerized batch-job runner.
+ *
+ *   badgr batch run <workload.yml>
+ *   badgr batch status <run_id>
+ *   badgr batch logs <run_id>
+ *   badgr batch artifacts <run_id>
+ *   badgr batch receipt <run_id>
+ *   badgr batch compare <run_a> <run_b>
+ *
+ * This is the generic bridge from `badgr run` into non-vLLM GPU workloads
+ * (CV batch, video batch, scientific batch, Monte Carlo, sim, physical-AI
+ * eval) — a thin wrapper over the same /v1/run + monitorBatchJob primitives
+ * `badgr train`/`transcribe`/`embed` already use, not a new execution path.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, statSync, readdirSync } from 'fs';
+import { join, resolve } from 'path';
+import os from 'os';
+
+import { requireApiKey, CONFIG_DIR } from '../config.js';
+import { parseWorkloadYaml } from '../workloadSpec.js';
+import { addReceipt, updateReceipt, generateReceiptId, loadStore } from '../store.js';
+import { normalizeTier, callWithFallback } from '../fallback.js';
+import { monitorBatchJob, fmtRuntime } from '../batch.js';
+import { uploadBlob } from '../api.js';
+import { logsCommand } from './logs.js';
+
+// Server-side watchdog / batch.js reason strings, mapped to the vocabulary
+// the batch spec asks for. The raw reason is always shown too.
+const FAILURE_REASON_LABELS = {
+  max_runtime:          'runtime_limit_exceeded',
+  'max-runtime':         'runtime_limit_exceeded',
+  max_cost:             'cost_cap_exceeded',
+  'max-cost':            'cost_cap_exceeded',
+  provisioning_timeout: 'provisioning_timeout',
+  idle_timeout:         'idle_timeout',
+  infrastructure:       'infrastructure_failure',
+  interrupted:          'interrupted_by_user',
+};
+
+export function displayFailureReason(raw) {
+  if (!raw) return null;
+  return FAILURE_REASON_LABELS[raw] ?? raw;
+}
+
+function artifactsDir(runId) {
+  return join(CONFIG_DIR, 'batch-artifacts', runId);
+}
+
+function findBatchRun(runId) {
+  const { receipts } = loadStore();
+  return receipts.find(r => r.receiptId === runId || r.deploymentId === runId) ?? null;
+}
+
+async function apiRequest(config, path, opts = {}) {
+  const { callApi } = await import('../api.js');
+  return callApi(path, { apiKey: config.apiKey, baseUrl: config.baseUrl, ...opts });
+}
+
+// ---------------------------------------------------------------------------
+// inputs archive (tar.gz, member paths = container path minus leading '/')
+// ---------------------------------------------------------------------------
+
+async function _buildInputsArchive(inputs, chalk) {
+  const { default: archiver } = await import('archiver');
+  const { createWriteStream } = await import('fs');
+  const tmpFile = join(os.tmpdir(), `badgr-batch-inputs-${Date.now()}.tar.gz`);
+
+  await new Promise((res, rej) => {
+    const output = createWriteStream(tmpFile);
+    const archive = archiver('tar', { gzip: true, gzipOptions: { level: 6 } });
+    output.on('close', res);
+    archive.on('error', rej);
+    archive.pipe(output);
+    for (const { localPath, containerPath } of inputs) {
+      if (!existsSync(localPath)) {
+        rej(new Error(`Input not found: ${localPath} (declared as ${containerPath})`));
+        return;
+      }
+      const memberName = containerPath.replace(/^\/+/, '');
+      const stat = statSync(localPath);
+      if (stat.isDirectory()) {
+        archive.directory(localPath, memberName);
+      } else {
+        archive.file(localPath, { name: memberName });
+      }
+    }
+    archive.finalize();
+  });
+
+  return tmpFile;
+}
+
+async function _uploadInputsArchive(config, inputs, chalk) {
+  process.stdout.write(chalk.dim('  Packing inputs...'));
+  const tmpFile = await _buildInputsArchive(inputs, chalk);
+  const sizeMb = (statSync(tmpFile).size / 1024 / 1024).toFixed(1);
+  process.stdout.write(chalk.dim(` ${sizeMb} MB\n`));
+
+  process.stdout.write(chalk.dim('  Uploading inputs...'));
+  const fileData = readFileSync(tmpFile);
+  let uploadResp;
+  try {
+    uploadResp = await uploadBlob(config, { data: fileData, filename: 'inputs.tar.gz', contentType: 'application/gzip' });
+  } catch (err) {
+    throw new Error(
+      `Inputs upload failed: ${err.message}\n` +
+      `  No GPU has been provisioned yet — nothing was billed. Fix the upload (check network/API key) and re-run.`,
+    );
+  } finally {
+    unlinkSync(tmpFile);
+  }
+  process.stdout.write(chalk.dim(' done\n'));
+  return uploadResp.code_uri;
+}
+
+// ---------------------------------------------------------------------------
+// output artifacts: fetch metadata → download tar.gz → extract
+// ---------------------------------------------------------------------------
+
+async function _fetchArtifactMetadata(config, runId) {
+  try {
+    return await apiRequest(config, `/deployments/${runId}/artifacts`);
+  } catch (err) {
+    if (err.httpStatus === 404) {
+      throw new Error(
+        `No artifact was ever uploaded for run ${runId}. This means either: the workload had no ` +
+        `\`outputs:\` declared, the command failed before producing any of them, or the runner ` +
+        `could not reach Badgr to upload (check \`badgr batch logs ${runId}\` for an upload error).`,
+      );
+    }
+    if (err.httpStatus === 410) {
+      throw new Error(`Artifact for run ${runId} has expired (48h retention) and can no longer be downloaded.`);
+    }
+    throw err;
+  }
+}
+
+async function downloadAndExtractArtifacts(config, runId) {
+  const meta = await _fetchArtifactMetadata(config, runId);
+  const destDir = artifactsDir(runId);
+  mkdirSync(destDir, { recursive: true });
+
+  // Prefer the CLI's configured HTTPS base URL over meta.download_url.
+  // Behind the gateway, request.base_url is often http://…, and fetch() following
+  // the http→https redirect drops the Authorization header → 401.
+  const base = String(config.baseUrl || '').replace(/\/$/, '');
+  const downloadUrl = base
+    ? `${base}/deployments/${runId}/artifacts/download`
+    : meta.download_url;
+
+  const res = await fetch(downloadUrl, {
+    headers: { Authorization: `Bearer ${config.apiKey}` },
+  });
+  if (!res.ok) {
+    throw new Error(
+      `Artifact download failed for run ${runId}: ${res.status} ${res.statusText}. ` +
+      `The upload succeeded (metadata exists) but the file itself couldn't be fetched — retry, ` +
+      `or check backend storage (BADGR_STORAGE_DIR) if this persists.`,
+    );
+  }
+  const buf = Buffer.from(await res.arrayBuffer());
+  const tmpFile = join(os.tmpdir(), `badgr-batch-artifacts-${runId}.tar.gz`);
+  writeFileSync(tmpFile, buf);
+  try {
+    const tar = await import('tar');
+    await tar.x({ file: tmpFile, cwd: destDir });
+  } finally {
+    unlinkSync(tmpFile);
+  }
+  return destDir;
+}
+
+export function missingOutputPaths(destDir, declaredOutputs) {
+  return declaredOutputs.filter(p => !existsSync(join(destDir, p)));
+}
+
+export function readMetricFile(destDir, metricFilePath) {
+  const full = join(destDir, metricFilePath);
+  if (!existsSync(full)) return null;
+  try {
+    return JSON.parse(readFileSync(full, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// badgr batch run <workload.yml>
+// ---------------------------------------------------------------------------
+
+async function runSubcommand(config, args, chalk) {
+  const flags = {};
+  const positional = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--tier')   { flags.tier = args[++i]; continue; }
+    if (args[i] === '--region') { flags.region = args[++i]; continue; }
+    if (args[i] === '--gpu')    { flags.gpu = args[++i]; continue; }
+    positional.push(args[i]);
+  }
+  const yamlPath = positional[0];
+  if (!yamlPath) {
+    console.error(chalk.red('\n  Usage: badgr batch run <workload.yml>\n'));
+    process.exitCode = 1;
+    return;
+  }
+
+  requireApiKey(config);
+
+  let spec;
+  try {
+    spec = parseWorkloadYaml(resolve(yamlPath));
+  } catch (err) {
+    console.error(chalk.red(`\n  ✗ ${err.message}\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  console.log(chalk.bold(`\n📦 Batch: ${spec.name}\n`));
+  console.log(`  ${chalk.bold('Image:')}        ${spec.image}`);
+  console.log(`  ${chalk.bold('Command:')}      ${spec.command.join(' ')}`);
+  console.log(`  ${chalk.bold('Max cost:')}     $${spec.maxCost.toFixed(2)}`);
+  console.log(`  ${chalk.bold('Max runtime:')}  ${spec.maxRuntimeMinutes}min`);
+  if (spec.inputs.length)  console.log(`  ${chalk.bold('Inputs:')}       ${spec.inputs.length} path(s)`);
+  if (spec.outputs.length) console.log(`  ${chalk.bold('Outputs:')}      ${spec.outputs.join(', ')}`);
+  console.log();
+
+  let inputsArchiveUrl = null;
+  if (spec.inputs.length > 0) {
+    try {
+      inputsArchiveUrl = await _uploadInputsArchive(config, spec.inputs, chalk);
+    } catch (err) {
+      console.error(chalk.red(`\n  ✗ ${err.message}\n`));
+      process.exitCode = 1;
+      return;
+    }
+  }
+
+  const effectiveTier = normalizeTier(flags.tier);
+  const envObj = { ...spec.env };
+  if (inputsArchiveUrl) envObj.BADGR_INPUTS_ARCHIVE_URL = inputsArchiveUrl;
+  if (spec.outputs.length > 0) envObj.BADGR_OUTPUT_PATHS_JSON = JSON.stringify(spec.outputs);
+
+  function buildBody(tierOverride) {
+    return {
+      image: spec.image,
+      command: spec.command,
+      custom_image: true, // run spec.image directly — it must embed the badgr-runner entrypoint contract
+      gpu: flags.gpu ? flags.gpu.toUpperCase().replace(/-/g, '_') : 'AUTO',
+      gpu_count: 1,
+      ...(flags.region ? { region: flags.region.toUpperCase() } : {}),
+      tier: tierOverride || effectiveTier,
+      env: envObj,
+      name: spec.name,
+      max_cost_usd: spec.maxCost,
+      max_runtime_seconds: spec.maxRuntimeMinutes * 60,
+    };
+  }
+
+  process.stdout.write(chalk.dim('  Finding suitable capacity...\n'));
+
+  let dep;
+  try {
+    dep = await callWithFallback(
+      '/run',
+      { apiKey: config.apiKey, baseUrl: config.baseUrl },
+      buildBody,
+      effectiveTier,
+      chalk,
+      { thing: 'batch workload', cmd: 'badgr batch run' },
+    );
+  } catch (err) {
+    if (err.isPaymentRequired) {
+      console.error(chalk.yellow(err.message));
+      process.exitCode = 1;
+      return;
+    }
+    console.error(err.message);
+    process.exitCode = 1;
+    return;
+  }
+
+  const rcptId = dep.receipt_id || generateReceiptId();
+  addReceipt({
+    receiptId: rcptId,
+    action: 'badgr batch run',
+    deploymentId: dep.deployment_id,
+    workloadName: spec.name,
+    workloadImage: spec.image,
+    outputsDeclared: spec.outputs,
+    successMetric: spec.successMetric,
+    gpu: dep.gpu_type,
+    providerRoute: dep.provider ?? null,
+    tier: dep.tier ?? null,
+    maxCost: spec.maxCost,
+    maxRuntime: spec.maxRuntimeMinutes,
+    status: dep.status,
+    createdAt: new Date().toISOString(),
+  });
+
+  const rate = dep.cost_per_hour || 0;
+  console.log(chalk.dim('  Capacity found.\n'));
+  console.log(chalk.bold(`  Run ID:  ${chalk.cyan(dep.deployment_id)}`));
+  console.log(`  ${chalk.bold('GPU:')}      ${dep.gpu_type} × ${dep.gpu_count}`);
+  if (rate > 0) console.log(`  ${chalk.bold('Rate:')}     $${rate.toFixed(2)}/hr`);
+  console.log(`  ${chalk.bold('Receipt:')}  ${chalk.dim(rcptId)}`);
+
+  const result = await monitorBatchJob(config, dep.deployment_id, rcptId, {
+    chalk,
+    maxRuntimeMs: spec.maxRuntimeMinutes * 60 * 1000,
+    maxCost: spec.maxCost,
+    ratePerHour: rate,
+  });
+
+  console.log();
+  const finalCost = rate * (result.runtimeMs / 3_600_000);
+  console.log(`  ${chalk.bold('Runtime:')}  ${fmtRuntime(result.runtimeMs)}`);
+  if (rate > 0) console.log(`  ${chalk.bold('Cost:')}     $${finalCost.toFixed(4)}`);
+
+  let dep2;
+  try {
+    dep2 = await apiRequest(config, `/deployments/${dep.deployment_id}`);
+  } catch {
+    dep2 = null;
+  }
+  const failureReason = displayFailureReason(dep2?.failure_reason);
+  if (failureReason) console.log(`  ${chalk.bold('Failure reason:')} ${chalk.red(failureReason)}`);
+  console.log(`  ${chalk.bold('Teardown:')} ${dep2?.teardown_ok === 'ok' ? chalk.green('ok') : dep2?.teardown_ok === 'failed' ? chalk.red('failed') : chalk.dim('n/a')}`);
+  console.log(`  ${chalk.bold('Receipt:')}  ${chalk.dim(rcptId)}`);
+
+  // Artifacts — best-effort; a job that never produced output paths (or
+  // that failed before the runner could upload) simply has none.
+  let missing = [];
+  if (spec.outputs.length > 0) {
+    try {
+      const destDir = await downloadAndExtractArtifacts(config, dep.deployment_id);
+      missing = missingOutputPaths(destDir, spec.outputs);
+      console.log(`  ${chalk.bold('Artifacts:')} ${destDir}`);
+      if (missing.length > 0) {
+        console.log(chalk.yellow(`    Missing declared output paths: ${missing.join(', ')}`));
+      }
+      if (spec.successMetric) {
+        const metrics = readMetricFile(destDir, spec.successMetric.file);
+        if (metrics && spec.successMetric.key in metrics) {
+          const value = metrics[spec.successMetric.key];
+          console.log(`  ${chalk.bold(spec.successMetric.key + ':')} ${value}`);
+          if (Array.isArray(metrics.scenarios)) {
+            const failed = metrics.scenarios.filter(s => s.passed === false).map(s => s.id);
+            if (failed.length > 0) console.log(`  ${chalk.bold('Failed scenarios:')} ${failed.join(', ')}`);
+          }
+        }
+      }
+    } catch (err) {
+      console.log(chalk.dim(`  Artifacts not available: ${err.message}`));
+    }
+  }
+
+  updateReceipt(rcptId, {
+    status: dep2?.status ?? result.status,
+    failureReason: dep2?.failure_reason ?? null,
+    teardownOk: dep2?.teardown_ok ?? null,
+    runtimeSeconds: dep2?.runtime_seconds ?? Math.round(result.runtimeMs / 1000),
+    finalCost: dep2?.accrued_cost_usd ?? finalCost,
+    provider: dep2?.provider ?? null,
+    missingOutputs: missing,
+  });
+
+  const incomplete = spec.outputs.length > 0 && missing.length > 0;
+  const succeeded = ['succeeded', 'completed'].includes(dep2?.status ?? result.status) && !incomplete;
+  if (succeeded) {
+    console.log(chalk.green('\n  ✓ Batch run complete\n'));
+  } else if (incomplete) {
+    console.error(chalk.red(`\n  ✗ Batch run incomplete — missing declared outputs\n`));
+    process.exitCode = 1;
+  } else {
+    console.error(chalk.red('\n  ✗ Batch run failed\n'));
+    process.exitCode = 1;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// badgr batch status <run_id>
+// ---------------------------------------------------------------------------
+
+async function statusSubcommand(config, args, chalk) {
+  const runId = args[0];
+  if (!runId) {
+    console.error(chalk.red('\n  Usage: badgr batch status <run_id>\n'));
+    process.exitCode = 1;
+    return;
+  }
+  requireApiKey(config);
+
+  let dep;
+  try {
+    dep = await apiRequest(config, `/deployments/${runId}`);
+  } catch (err) {
+    console.error(chalk.red(`\n  ✗ Could not fetch status: ${err.message}\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  const local = findBatchRun(runId);
+  console.log(chalk.bold(`\n📦 Batch run: ${runId}\n`));
+  if (local?.workloadName) console.log(`  ${chalk.bold('Workload:')} ${local.workloadName}`);
+  console.log(`  ${chalk.bold('Status:')}   ${dep.status}`);
+  if (dep.provider) console.log(`  ${chalk.bold('Provider:')} ${dep.provider}`);
+  if (dep.runtime_seconds != null) console.log(`  ${chalk.bold('Runtime:')}  ${fmtRuntime(dep.runtime_seconds * 1000)}`);
+  if (dep.accrued_cost_usd != null) console.log(`  ${chalk.bold('Cost:')}     $${dep.accrued_cost_usd.toFixed(4)}`);
+  const failureReason = displayFailureReason(dep.failure_reason);
+  if (failureReason) console.log(`  ${chalk.bold('Failure:')}  ${chalk.red(failureReason)}`);
+  console.log(`  ${chalk.bold('Teardown:')} ${dep.teardown_ok === 'ok' ? 'ok' : dep.teardown_ok === 'failed' ? 'failed' : 'n/a'}`);
+  console.log();
+}
+
+// ---------------------------------------------------------------------------
+// badgr batch artifacts <run_id>
+// ---------------------------------------------------------------------------
+
+async function artifactsSubcommand(config, args, chalk) {
+  const runId = args[0];
+  if (!runId) {
+    console.error(chalk.red('\n  Usage: badgr batch artifacts <run_id>\n'));
+    process.exitCode = 1;
+    return;
+  }
+  requireApiKey(config);
+
+  let destDir;
+  try {
+    destDir = await downloadAndExtractArtifacts(config, runId);
+  } catch (err) {
+    console.error(chalk.red(`\n  ✗ Could not fetch artifacts: ${err.message}\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  const local = findBatchRun(runId);
+  console.log(chalk.bold(`\n📦 Artifacts: ${runId}\n`));
+  console.log(`  ${chalk.bold('Extracted to:')} ${destDir}\n`);
+
+  function walk(dir, prefix = '') {
+    for (const entry of readdirSync(dir)) {
+      const full = join(dir, entry);
+      const rel = prefix ? `${prefix}/${entry}` : entry;
+      if (statSync(full).isDirectory()) {
+        walk(full, rel);
+      } else {
+        console.log(`  ${chalk.dim(rel)}`);
+      }
+    }
+  }
+  walk(destDir);
+
+  if (local?.outputsDeclared?.length) {
+    const missing = missingOutputPaths(destDir, local.outputsDeclared);
+    if (missing.length > 0) {
+      console.log(chalk.yellow(`\n  Missing declared output paths: ${missing.join(', ')}`));
+    }
+  }
+  console.log();
+}
+
+// ---------------------------------------------------------------------------
+// badgr batch receipt <run_id>
+// ---------------------------------------------------------------------------
+
+async function receiptSubcommand(config, args, chalk) {
+  const runId = args[0];
+  if (!runId) {
+    console.error(chalk.red('\n  Usage: badgr batch receipt <run_id>\n'));
+    process.exitCode = 1;
+    return;
+  }
+  requireApiKey(config);
+
+  const local = findBatchRun(runId);
+  let dep = null;
+  try {
+    dep = await apiRequest(config, `/deployments/${runId}`);
+  } catch {
+    dep = null;
+  }
+
+  if (!local && !dep) {
+    console.error(chalk.red(`\n  Receipt not found: ${runId}\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  console.log(chalk.bold(`\n🧾 Batch receipt: ${runId}\n`));
+  console.log(`  ${chalk.bold('Workload:')}  ${local?.workloadName ?? '—'}`);
+  console.log(`  ${chalk.bold('Image:')}     ${local?.workloadImage ?? '—'}`);
+  console.log(`  ${chalk.bold('Status:')}    ${dep?.status ?? local?.status ?? '—'}`);
+  console.log(`  ${chalk.bold('Provider:')}  ${dep?.provider ?? local?.provider ?? '—'}`);
+  const runtimeS = dep?.runtime_seconds ?? local?.runtimeSeconds;
+  if (runtimeS != null) console.log(`  ${chalk.bold('Runtime:')}   ${fmtRuntime(runtimeS * 1000)}`);
+  const cost = dep?.accrued_cost_usd ?? local?.finalCost;
+  if (cost != null) console.log(`  ${chalk.bold('Cost:')}      $${cost.toFixed(4)}`);
+  const failureReason = displayFailureReason(dep?.failure_reason ?? local?.failureReason);
+  if (failureReason) console.log(`  ${chalk.bold('Failure:')}   ${chalk.red(failureReason)}`);
+  const teardownOk = dep?.teardown_ok ?? local?.teardownOk;
+  console.log(`  ${chalk.bold('Teardown:')}  ${teardownOk === 'ok' ? 'ok' : teardownOk === 'failed' ? 'failed' : 'n/a'}`);
+  console.log(`  ${chalk.bold('Artifacts:')} ${existsSync(artifactsDir(runId)) ? artifactsDir(runId) : chalk.dim('not fetched — run `badgr batch artifacts ' + runId + '`')}`);
+  if (local?.missingOutputs?.length) {
+    console.log(chalk.yellow(`  Missing outputs: ${local.missingOutputs.join(', ')}`));
+  }
+  console.log();
+}
+
+// ---------------------------------------------------------------------------
+// badgr batch compare <run_a> <run_b>
+// ---------------------------------------------------------------------------
+
+export async function resolveMetric(config, ref, flags, chalk) {
+  // Direct path to a metrics.json file — for easy testing without a live run.
+  if (ref.endsWith('.json') && existsSync(resolve(ref))) {
+    const metrics = JSON.parse(readFileSync(resolve(ref), 'utf8'));
+    const key = flags.key ?? 'pass_rate';
+    const higherIsBetter = flags.higherIsBetter ?? true;
+    return { ref, key, higherIsBetter, value: metrics[key] };
+  }
+
+  // Run ID — use the declared success_metric from the local receipt.
+  const local = findBatchRun(ref);
+  if (!local?.successMetric) {
+    throw new Error(`No success_metric known for run ${ref} (run \`badgr batch run\` with success_metric set, or pass a metrics.json path directly)`);
+  }
+  const destDir = existsSync(artifactsDir(ref)) ? artifactsDir(ref) : await downloadAndExtractArtifacts(config, ref);
+  const metrics = readMetricFile(destDir, local.successMetric.file);
+  if (!metrics) throw new Error(`Could not read ${local.successMetric.file} for run ${ref}`);
+  return {
+    ref,
+    key: local.successMetric.key,
+    higherIsBetter: local.successMetric.higherIsBetter,
+    value: metrics[local.successMetric.key],
+  };
+}
+
+async function compareSubcommand(config, args, chalk) {
+  const flags = {};
+  const positional = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--key') { flags.key = args[++i]; continue; }
+    if (args[i] === '--higher-is-better') { flags.higherIsBetter = args[++i] !== 'false'; continue; }
+    positional.push(args[i]);
+  }
+  const [runA, runB] = positional;
+  if (!runA || !runB) {
+    console.error(chalk.red('\n  Usage: badgr batch compare <run_a> <run_b>\n'));
+    process.exitCode = 1;
+    return;
+  }
+
+  let a, b;
+  try {
+    a = await resolveMetric(config, runA, flags, chalk);
+    b = await resolveMetric(config, runB, flags, chalk);
+  } catch (err) {
+    console.error(chalk.red(`\n  ✗ ${err.message}\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  if (typeof a.value !== 'number' || typeof b.value !== 'number') {
+    console.error(chalk.red(`\n  ✗ Metric "${a.key}" is not numeric on one or both runs\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  const delta = b.value - a.value;
+  const higherIsBetter = a.higherIsBetter;
+  const improved = higherIsBetter ? delta > 0 : delta < 0;
+  const regressed = higherIsBetter ? delta < 0 : delta > 0;
+
+  console.log(chalk.bold(`\n📊 Compare: ${runA} vs ${runB}\n`));
+  console.log(`  ${chalk.bold('Metric:')}  ${a.key}`);
+  console.log(`  ${chalk.bold(runA + ':')}  ${a.value}`);
+  console.log(`  ${chalk.bold(runB + ':')}  ${b.value}`);
+  console.log(`  ${chalk.bold('Delta:')}   ${delta >= 0 ? '+' : ''}${delta}`);
+
+  if (regressed) {
+    console.log(chalk.red(`\n  ✗ Regression — ${runB} is worse than ${runA} on ${a.key}\n`));
+    process.exitCode = 1;
+  } else if (improved) {
+    console.log(chalk.green(`\n  ✓ Improvement — ${runB} is better than ${runA} on ${a.key}\n`));
+  } else {
+    console.log(chalk.dim(`\n  No change on ${a.key}\n`));
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Router
+// ---------------------------------------------------------------------------
+
+export async function batchCommand(config, args, chalk) {
+  const sub = args[0];
+  const rest = args.slice(1);
+
+  switch (sub) {
+    case 'run':       return runSubcommand(config, rest, chalk);
+    case 'status':     return statusSubcommand(config, rest, chalk);
+    case 'logs':       return logsCommand(config, rest, chalk);
+    case 'artifacts':  return artifactsSubcommand(config, rest, chalk);
+    case 'receipt':    return receiptSubcommand(config, rest, chalk);
+    case 'compare':    return compareSubcommand(config, rest, chalk);
+    default:
+      console.error(chalk.red(`Unknown batch subcommand: ${sub || '(none)'}\n`));
+      console.log([
+        'Usage:',
+        '  badgr batch run <workload.yml>',
+        '  badgr batch status <run_id>',
+        '  badgr batch logs <run_id>',
+        '  badgr batch artifacts <run_id>',
+        '  badgr batch receipt <run_id>',
+        '  badgr batch compare <run_a> <run_b>',
+      ].join('\n'));
+      process.exitCode = 1;
+  }
+}

package/src/commands/rerun.js

@@ -0,0 +1,75 @@
+import { requireApiKey } from '../config.js';
+import { findDeployment, addDeployment, addReceipt, generateReceiptId } from '../store.js';
+import { rerunDeployment } from '../api.js';
+
+/**
+ * badgr rerun <deployment-id|name>
+ *
+ * Replays a previous job or endpoint with its exact original spec — same
+ * image, command, env vars, GPU, and cost/runtime caps. Works for one-off
+ * jobs (unlike `badgr restart`, which is endpoint-only) and never tears
+ * down the source deployment. Always creates a new deployment_id.
+ */
+export async function rerunCommand(config, args, chalk) {
+  const idOrName = args.find(a => !a.startsWith('--'));
+
+  if (!idOrName) {
+    console.error(chalk.red('Usage: badgr rerun <deployment-id|name>'));
+    return;
+  }
+
+  requireApiKey(config);
+
+  const localDep     = findDeployment(idOrName);
+  const deploymentId = localDep?.id ?? idOrName;
+
+  process.stdout.write(chalk.dim(`  Replaying ${deploymentId}...`));
+
+  let dep;
+  try {
+    dep = await rerunDeployment(config, deploymentId);
+  } catch (err) {
+    process.stdout.write('\n');
+    console.error(chalk.red(`\n  ✗ Could not rerun deployment: ${err.message}\n`));
+    process.exitCode = 1;
+    return;
+  }
+
+  process.stdout.write('\n');
+
+  addDeployment({
+    id:          dep.deployment_id,
+    name:        dep.name,
+    type:        dep.workload_type,
+    model:       dep.model,
+    gpu:         dep.gpu_type,
+    count:       dep.gpu_count,
+    status:      dep.status,
+    endpointUrl: dep.endpoint_url || dep.openai_base_url,
+    receiptId:   dep.receipt_id,
+    createdAt:   new Date().toISOString(),
+    costPerHour: dep.cost_per_hour || 0,
+    providerRoute: dep.provider ?? null,
+    tier:          dep.tier ?? null,
+  });
+
+  const rcptId = dep.receipt_id || generateReceiptId();
+  addReceipt({
+    receiptId:    rcptId,
+    action:       'badgr rerun',
+    deploymentId: dep.deployment_id,
+    gpu:          dep.gpu_type,
+    status:       dep.status,
+    rerunOf:      deploymentId,
+    createdAt:    new Date().toISOString(),
+  });
+
+  const endpointUrl = dep.endpoint_url || dep.openai_base_url;
+  console.log(chalk.green('\n  ✓ Rerun submitted\n'));
+  console.log(`  ${chalk.bold('Replayed from:')}  ${chalk.dim(deploymentId)}`);
+  console.log(`  ${chalk.bold('New deployment:')} ${chalk.cyan(dep.deployment_id)}`);
+  if (endpointUrl) console.log(`  ${chalk.bold('Base URL:')}        ${chalk.cyan(endpointUrl)}`);
+  console.log(`  ${chalk.bold('Status:')}          badgr status ${dep.deployment_id}`);
+  console.log(`  ${chalk.bold('Logs:')}            badgr logs ${dep.deployment_id}`);
+  console.log();
+}