badgr-cli 1.0.8 → 1.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "badgr-cli",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "Badgr — run or serve GPU workloads from one command",
   "type": "module",
   "bin": {

package/src/badgr.js

@@ -26,12 +26,17 @@ ${chalk.bold('CORE COMMANDS')}
   ${chalk.cyan('badgr capacity [--gpu <type>]')}       Check live GPU availability and alternatives
 
 ${chalk.bold('badgr run OPTIONS')}
-  --gpu <type>        GPU: RTX_4090, A100, L40S, H100  (default: RTX_4090)
-  --image <image>     Docker image (default: python:3.11-slim)
-  --count <n>         GPU count (default: 1)
-  --region US|EU|AU   Region preference (default: US)
-  --max-price <$/hr>  Hard spend cap per GPU-hour
-  --name <name>       Job name (auto-generated if omitted)
+  --gpu <type>              GPU: RTX_4090, A100, L40S, H100  (default: RTX_4090)
+  --image <image>           Docker image (default: python:3.11-slim)
+  --count <n>               GPU count (default: 1)
+  --region US|EU|AU         Region preference (default: US)
+  --max-price <$/hr>        Hard spend cap per GPU-hour
+  --name <name>             Job name (auto-generated if omitted)
+  --fallback closest|cheapest  If requested GPU is unavailable, interactively pick the
+                            closest (default) or cheapest alternative
+  --no-fallback             Exit immediately when requested GPU is unavailable
+  --max-runtime <minutes>   Auto-teardown job after N minutes (prevents runaway billing)
+  --max-cost <$>            Auto-teardown when total spend reaches this amount
 
 ${chalk.bold('badgr serve OPTIONS')}
   --gpu <type>        GPU: L40S, A100, H100, RTX_4090  (default: L40S)
@@ -43,6 +48,9 @@ ${chalk.bold('badgr serve OPTIONS')}
 ${chalk.bold('EXAMPLES')}
   badgr login
   badgr run python train.py --gpu A100
+  badgr run python train.py --gpu A100 --fallback closest
+  badgr run python train.py --gpu A100 --fallback cheapest
+  badgr run python train.py --gpu A100 --no-fallback
   badgr run --image my/image:latest --gpu L40S
   badgr serve meta-llama/Llama-3.1-8B-Instruct --gpu L40S
   badgr serve mistralai/Mistral-7B-v0.1 --gpu RTX_4090

package/src/commands/receipts.js

@@ -5,19 +5,34 @@ function fmtMs(ms) {
   return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`;
 }
 
+function fmtRuntime(s) {
+  if (s < 60) return `${s}s`;
+  return `${Math.floor(s / 60)}m ${s % 60}s`;
+}
+
 function printReceipt(r, chalk) {
   const id     = r.receiptId ?? r.request_id ?? r.id ?? '—';
   const ts     = r.createdAt ?? r.created_at ?? '—';
-  const cost   = r.route?.ratePerHour ?? r.cost_usd ?? 0;
-  const prov   = r.route?.provider ?? r.provider ?? r.model_provider ?? '—';
-  const lat    = r.latencyMs ?? r.latency_ms;
   const action = r.action ?? r.endpoint ?? '—';
+  const lat    = r.latencyMs ?? r.latency_ms;
 
   console.log(`  ${chalk.cyan(id)}`);
   console.log(`    ${chalk.bold('action:')}   ${action}`);
+
+  // GPU job fields (from badgr run / badgr down)
+  if (r.gpu)            console.log(`    ${chalk.bold('gpu:')}      ${r.gpu}`);
+  if (r.runtimeSeconds) console.log(`    ${chalk.bold('runtime:')}  ${fmtRuntime(r.runtimeSeconds)}`);
+  if (r.finalCost)      console.log(`    ${chalk.bold('cost:')}     $${r.finalCost.toFixed(4)}`);
+  else if (r.cost_usd)  console.log(`    ${chalk.bold('cost:')}     $${r.cost_usd.toFixed(4)}`);
+  if (r.exitCode !== undefined && r.exitCode !== null) {
+    console.log(`    ${chalk.bold('exit:')}     ${r.exitCode !== 0 ? chalk.red(r.exitCode) : chalk.green(r.exitCode)}`);
+  }
+
+  // API inference receipt fields
   if (lat !== undefined) console.log(`    ${chalk.bold('latency:')}  ${fmtMs(lat)}`);
-  if (cost) console.log(`    ${chalk.bold('cost:')}     $${typeof cost === 'number' ? cost.toFixed(4) : cost}/hr`);
   if (r.retries !== undefined) console.log(`    ${chalk.bold('retries:')}  ${r.retries}`);
+  if (r.route?.ratePerHour) console.log(`    ${chalk.bold('rate:')}     $${r.route.ratePerHour.toFixed(4)}/hr`);
+
   if (r.status) console.log(`    ${chalk.bold('status:')}   ${r.status}`);
   console.log(`    ${chalk.dim(ts)}`);
   console.log();

package/src/commands/run.js

@@ -1,6 +1,7 @@
 import { requireApiKey } from '../config.js';
-import { callApi } from '../api.js';
-import { addReceipt, generateReceiptId } from '../store.js';
+import { callApi, terminateDeployment } from '../api.js';
+import { addReceipt, updateReceipt, generateReceiptId } from '../store.js';
+import { rankAlternatives, promptFallback } from '../fallback.js';
 
 /**
  * badgr run python train.py --gpu A100           # attached (default)
@@ -20,60 +21,137 @@ export function parseRunArgs(args) {
     if (args[i] === '--region')         { flags.region = args[++i]; i++; continue; }
     if (args[i] === '--max-price')      { flags.maxPrice = parseFloat(args[++i]); i++; continue; }
     if (args[i] === '--name')           { flags.name = args[++i]; i++; continue; }
-    if (args[i] === '--detach')         { flags.detach = true; i++; continue; }
+    if (args[i] === '--detach')          { flags.detach = true; i++; continue; }
+    if (args[i] === '--fallback')        { flags.fallback = args[++i]; i++; continue; }
+    if (args[i] === '--no-fallback')     { flags.noFallback = true; i++; continue; }
+    if (args[i] === '--max-runtime')     { flags.maxRuntime = parseFloat(args[++i]); i++; continue; }
+    if (args[i] === '--max-cost')        { flags.maxCost = parseFloat(args[++i]); i++; continue; }
     positional.push(args[i++]);
   }
   return { flags, positional };
 }
 
-// Poll /v1/deployments/:id until terminal status, streaming new log lines.
-async function attachToJob(config, depId, chalk) {
+function fmtRuntime(ms) {
+  const s = Math.round(ms / 1000);
+  if (s < 60) return `${s}s`;
+  const m = Math.floor(s / 60);
+  return `${m}m ${s % 60}s`;
+}
+
+// Polls above this count without a response while 'running' → heartbeat lost.
+const HEARTBEAT_WARN_POLLS  = 3;   // ~12s  → warn
+const HEARTBEAT_KILL_POLLS  = 15;  // ~60s  → treat as infrastructure failure
+
+/**
+ * Classify a failure into 'customer_code' or 'infrastructure'.
+ * customer_code: process ran and exited non-zero.
+ * infrastructure: provider issue — job never completed normally.
+ */
+export function classifyFailure(finalStatus, exitCode) {
+  if (finalStatus === 'failed' && (exitCode === null || exitCode === undefined)) return 'infrastructure';
+  if (exitCode !== null && exitCode !== undefined && exitCode !== 0) return 'customer_code';
+  return null;
+}
+
+/**
+ * Poll until terminal status, streaming new log lines.
+ * Returns { status, exitCode, runtimeMs, failureType }.
+ * Calls onTeardown(reason) when SIGINT, maxRuntime, maxCost, or heartbeat loss hits.
+ */
+async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost = null, ratePerHour = 0, onTeardown }) {
   const TERMINAL = new Set(['stopped', 'failed', 'completed']);
   const POLL_MS  = 4000;
-  let seenLines  = 0;
-  let lastStatus = '';
+  let seenLines       = 0;
+  let lastStatus      = '';
+  let consecutiveErrs = 0;
+  const startMs       = Date.now();
 
-  while (true) {
-    await new Promise(r => setTimeout(r, POLL_MS));
+  let tearing = false;
+  const sigintHandler = () => {
+    if (tearing) return;
+    tearing = true;
+    onTeardown('interrupted');
+  };
+  process.once('SIGINT', sigintHandler);
 
-    // Fetch status
-    let dep;
-    try {
-      dep = await callApi(`/deployments/${depId}`, {
-        apiKey: config.apiKey,
-        baseUrl: config.baseUrl,
-      });
-    } catch {
-      // transient network error — keep trying
-      continue;
-    }
+  try {
+    while (true) {
+      await new Promise(r => setTimeout(r, POLL_MS));
+
+      const elapsedMs  = Date.now() - startMs;
+      const spentSoFar = ratePerHour * (elapsedMs / 3_600_000);
 
-    const status = dep.status;
-    if (status !== lastStatus) {
-      if (status === 'running' && lastStatus === 'provisioning') {
-        console.log(chalk.dim('  [running]'));
+      // Hard spend cap
+      if (maxCost !== null && spentSoFar >= maxCost) {
+        tearing = true;
+        onTeardown('max-cost');
+        return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
       }
-      lastStatus = status;
-    }
 
-    // Stream any new log lines
-    try {
-      const logData = await callApi(`/deployments/${depId}/logs`, {
-        apiKey: config.apiKey,
-        baseUrl: config.baseUrl,
-      });
-      const lines = logData?.logs ?? [];
-      for (let i = seenLines; i < lines.length; i++) {
-        console.log(`  ${chalk.dim(lines[i])}`);
+      // Hard runtime cap
+      if (maxRuntimeMs !== null && elapsedMs >= maxRuntimeMs) {
+        tearing = true;
+        onTeardown('max-runtime');
+        return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
+      }
+
+      let dep;
+      try {
+        dep = await callApi(`/deployments/${depId}`, {
+          apiKey: config.apiKey,
+          baseUrl: config.baseUrl,
+        });
+        consecutiveErrs = 0;
+      } catch {
+        consecutiveErrs++;
+        if (lastStatus === 'running') {
+          const lostSec = Math.round((consecutiveErrs * POLL_MS) / 1000);
+          if (consecutiveErrs === HEARTBEAT_WARN_POLLS) {
+            console.log(chalk.yellow(`\n  ⚠ No heartbeat for ${lostSec}s — Vast machine may be unresponsive`));
+          } else if (consecutiveErrs >= HEARTBEAT_KILL_POLLS) {
+            tearing = true;
+            onTeardown('heartbeat-lost');
+            return { status: 'failed', exitCode: null, runtimeMs: elapsedMs, failureType: 'infrastructure' };
+          }
+        }
+        continue;
       }
-      seenLines = lines.length;
-    } catch {
-      // logs not ready yet
-    }
 
-    if (TERMINAL.has(status)) {
-      return status;
+      const status = dep.status;
+      if (status !== lastStatus) {
+        if (status === 'running' && lastStatus === 'provisioning') {
+          console.log(chalk.dim('  [running]'));
+        }
+        lastStatus = status;
+      }
+
+      try {
+        const logData = await callApi(`/deployments/${depId}/logs`, {
+          apiKey: config.apiKey,
+          baseUrl: config.baseUrl,
+        });
+        const lines = logData?.logs ?? [];
+        for (let i = seenLines; i < lines.length; i++) {
+          console.log(`  ${chalk.dim(lines[i])}`);
+        }
+        seenLines = lines.length;
+      } catch {
+        // logs not ready yet
+      }
+
+      if (TERMINAL.has(status)) {
+        process.removeListener('SIGINT', sigintHandler);
+        const exitCode = dep.exit_code ?? null;
+        return {
+          status,
+          exitCode,
+          runtimeMs: Date.now() - startMs,
+          failureType: classifyFailure(status, exitCode),
+        };
+      }
     }
+  } finally {
+    process.removeListener('SIGINT', sigintHandler);
   }
 }
 
@@ -88,19 +166,49 @@ export async function runCommand(config, args, chalk) {
 
   requireApiKey(config);
 
-  const command = positional.length > 0 ? positional : undefined;
-  const gpu     = flags.gpu || 'RTX_4090';
-  const image   = flags.image || (command ? 'python:3.11-slim' : undefined);
-  const detach  = flags.detach || false;
+  const command        = positional.length > 0 ? positional : undefined;
+  const gpu            = flags.gpu || 'RTX_4090';
+  const image          = flags.image || (command ? 'python:3.11-slim' : undefined);
+  const detach         = flags.detach || false;
+  const fallbackMode   = flags.noFallback ? 'none' : (flags.fallback || 'closest');
+  const maxRuntimeMs   = flags.maxRuntime ? flags.maxRuntime * 60 * 1000 : null;
+  const maxCost        = flags.maxCost ?? null;
 
   console.log(chalk.bold('\n⚡ Running GPU job\n'));
   if (command) console.log(`  ${chalk.bold('Command:')}  ${command.join(' ')}`);
   if (image)   console.log(`  ${chalk.bold('Image:')}    ${image}`);
   console.log(`  ${chalk.bold('GPU:')}      ${gpu}`);
+  if (flags.maxRuntime) console.log(`  ${chalk.bold('Max runtime:')} ${flags.maxRuntime}min`);
+  if (maxCost)          console.log(`  ${chalk.bold('Max cost:')}    $${maxCost.toFixed(2)}`);
+  if (flags.maxPrice)   console.log(`  ${chalk.bold('Max price:')}   $${flags.maxPrice.toFixed(2)}/hr`);
   if (detach)  console.log(`  ${chalk.dim('(detached — returns immediately)')}`);
   console.log();
+
+  // Safety guidance for jobs without a spend cap
+  if (!detach && !flags.maxRuntime && !maxCost) {
+    console.log(chalk.dim('  Tip: add --max-runtime 60 or --max-cost 5.00 to cap spend automatically'));
+  }
+  if (flags.maxRuntime > 60) {
+    console.log(chalk.yellow('  Note: jobs over 60min should checkpoint progress to recover from failures'));
+  }
+
   console.log(chalk.dim('  Finding best available GPU capacity...'));
-  console.log(chalk.dim(`  API: ${config.baseUrl}`));
+  if (process.env.BADGR_DEBUG === '1' || process.env.BADGR_DEBUG === 'true') {
+    console.log(chalk.dim(`  API: ${config.baseUrl}`));
+  }
+
+  function buildBody(gpuOverride, regionOverride) {
+    const effectiveRegion = regionOverride ?? (flags.region ? flags.region.toUpperCase() : undefined);
+    return {
+      command,
+      image,
+      gpu: (gpuOverride || gpu).toUpperCase().replace('-', '_'),
+      gpu_count: flags.count || 1,
+      ...(effectiveRegion ? { region: effectiveRegion } : {}),
+      max_price_per_hour: flags.maxPrice,
+      name: flags.name,
+    };
+  }
 
   let dep;
   try {
@@ -108,57 +216,58 @@ export async function runCommand(config, args, chalk) {
       method: 'POST',
       apiKey: config.apiKey,
       baseUrl: config.baseUrl,
-      body: {
-        command,
-        image,
-        gpu: gpu.toUpperCase().replace('-', '_'),
-        gpu_count: flags.count || 1,
-        ...(flags.region ? { region: flags.region.toUpperCase() } : {}),
-        max_price_per_hour: flags.maxPrice,
-        name: flags.name,
-      },
+      body: buildBody(),
     });
   } catch (err) {
     const d = err.errorData;
     if (d?.code === 'NO_CAPACITY_MATCH') {
-      const f = d.filters ?? {};
-      const scope = f.region && f.region !== 'any' ? `in ${f.region}` : 'globally';
-      console.error(chalk.red(`\n  ✗ No ${f.gpu || gpu} found under ${f.max_price || '$10/hr'} ${scope}.\n`));
-
-      const sameGpuElsewhere = Array.isArray(d.same_gpu_other_regions) ? d.same_gpu_other_regions : [];
-      const alternatives = Array.isArray(d.alternatives) ? d.alternatives : [];
-      const cmdPrefix = command ? command.join(' ') : flags.image ? `--image ${flags.image}` : '...';
-
-      if (sameGpuElsewhere.length > 0) {
-        // Requested GPU exists, but not in the specified region.
-        console.error(chalk.dim(`  ${f.gpu} is available in other regions:\n`));
-        console.error(chalk.dim('  Try:'));
-        for (const a of sameGpuElsewhere) {
-          console.error(chalk.cyan(`    badgr run ${cmdPrefix} --gpu ${a.gpu} --region ${a.region}`));
-          console.error(chalk.dim(`      from $${a.price.toFixed(2)}/hr`));
-        }
-      } else if (alternatives.length > 0) {
-        // Global search failed — show what is actually available.
-        console.error(chalk.dim('  Available right now:\n'));
-        for (const a of alternatives) {
-          console.error(chalk.dim(`    • ${a.gpu} in ${a.region} from $${a.price.toFixed(2)}/hr`));
-        }
-        console.error(chalk.dim('\n  Try:'));
-        for (const a of alternatives) {
-          console.error(chalk.cyan(`    badgr run ${cmdPrefix} --gpu ${a.gpu}`));
-        }
-      } else {
-        console.error(chalk.dim('  No alternative GPU capacity found right now.'));
+      if (fallbackMode === 'none') {
+        console.error(chalk.red(`\n  ✗ ${gpu} isn't available right now.\n`));
+        console.error(chalk.dim('  Pass --fallback closest to auto-select an alternative.'));
+        process.exit(1);
+      }
+
+      // Merge same-GPU-other-regions and cross-GPU alternatives into one ranked list.
+      const pool = [
+        ...(Array.isArray(d.same_gpu_other_regions) ? d.same_gpu_other_regions : []),
+        ...(Array.isArray(d.alternatives)            ? d.alternatives            : []),
+      ];
+
+      if (pool.length === 0) {
+        console.error(chalk.red(`\n  ✗ ${gpu} isn't available right now and no alternatives found.\n`));
         console.error(chalk.dim('  Try increasing --max-price or contact support for a manual quote.'));
+        process.exit(1);
+      }
+
+      const ranked = rankAlternatives(gpu, pool, fallbackMode);
+      const chosen = await promptFallback(gpu, ranked, chalk);
+
+      if (!chosen) {
+        console.log(chalk.dim('\n  Cancelled.\n'));
+        process.exit(0);
+      }
+
+      console.log(chalk.dim(`\n  Running on ${chosen.gpu} in ${chosen.region}...\n`));
+
+      try {
+        dep = await callApi('/run', {
+          method: 'POST',
+          apiKey: config.apiKey,
+          baseUrl: config.baseUrl,
+          body: buildBody(chosen.gpu, chosen.region),
+        });
+      } catch (err2) {
+        console.error(chalk.red(`\n  ✗ Job failed to start on ${chosen.gpu}: ${err2.message}`));
+        console.error(chalk.dim(`\n  Debug: BADGR_DEBUG=1 badgr run … — shows full request/response\n`));
+        process.exit(1);
       }
-      console.error('');
     } else {
       console.error(chalk.red(`\n  ✗ Job failed to start: ${err.message}`));
       console.error(chalk.dim(`\n  Debug: BADGR_DEBUG=1 badgr run … — shows full request/response`));
       console.error(chalk.dim(`  Config:  badgr config`));
       console.error(chalk.dim(`  Docs:    badgr --help\n`));
+      process.exit(1);
     }
-    process.exit(1);
   }
 
   const rcptId = dep.receipt_id || generateReceiptId();
@@ -183,18 +292,73 @@ export async function runCommand(config, args, chalk) {
     return;
   }
 
-  // ── Attached mode: stream logs until job completes ───────────────────────
-  console.log(chalk.dim('\n  ── Attaching (Ctrl+C to detach) ──────────────────────────────\n'));
+  // ── Attached mode: stream logs until job completes ────────────────────────
+  const ratePerHour = dep.cost_per_hour || 0;
+  console.log(chalk.dim('\n  ── Attaching (Ctrl+C tears down job) ─────────────────────────\n'));
+
+  // Teardown helper — called on SIGINT, max-runtime, max-cost, or heartbeat loss.
+  async function teardown(reason) {
+    const labels = {
+      'max-runtime':    chalk.yellow('\n  ⏱ Max runtime reached — stopping job...'),
+      'max-cost':       chalk.yellow('\n  💰 Spend cap reached — stopping job...'),
+      'heartbeat-lost': chalk.red('\n  ✗ Heartbeat lost — infrastructure failure, stopping job...'),
+      'interrupted':    chalk.yellow('\n  Stopping job...'),
+    };
+    console.log(labels[reason] ?? chalk.yellow('\n  Stopping job...'));
+    try {
+      await terminateDeployment(config, dep.deployment_id);
+    } catch {
+      // best-effort
+    }
+    const runtimeMs = Date.now() - attachStart;
+    const finalCost = ratePerHour * (runtimeMs / 3_600_000);
+    const failureType = reason === 'heartbeat-lost' ? 'infrastructure' : null;
+    updateReceipt(rcptId, { status: reason, runtimeSeconds: Math.round(runtimeMs / 1000), finalCost, failureType });
+    console.log(chalk.dim(`  Runtime: ${fmtRuntime(runtimeMs)}  •  Est. cost: $${finalCost.toFixed(4)}`));
+    console.log(chalk.dim('  Job stopped. Billing ended.\n'));
+    process.exit(reason === 'interrupted' ? 0 : 1);
+  }
 
-  const finalStatus = await attachToJob(config, dep.deployment_id, chalk);
+  const attachStart = Date.now();
+  const { status: finalStatus, exitCode, runtimeMs, failureType } = await attachToJob(config, dep.deployment_id, {
+    chalk,
+    maxRuntimeMs,
+    maxCost,
+    ratePerHour,
+    onTeardown: teardown,
+  });
 
   console.log();
 
-  if (finalStatus === 'failed') {
-    console.error(chalk.red(`\n  ✗ Job failed (${dep.deployment_id})\n`));
+  const finalCost = ratePerHour * (runtimeMs / 3_600_000);
+  updateReceipt(rcptId, {
+    status: finalStatus,
+    exitCode,
+    runtimeSeconds: Math.round(runtimeMs / 1000),
+    finalCost,
+    failureType,
+  });
+
+  // ── Job summary ──────────────────────────────────────────────────────────
+  console.log(`  ${chalk.bold('Runtime:')}  ${fmtRuntime(runtimeMs)}`);
+  if (ratePerHour > 0) {
+    console.log(`  ${chalk.bold('Cost:')}     $${finalCost.toFixed(4)}  (${chalk.dim(`$${ratePerHour.toFixed(2)}/hr`)})`);
+  }
+  if (exitCode !== null && exitCode !== undefined) {
+    console.log(`  ${chalk.bold('Exit code:')} ${exitCode !== 0 ? chalk.red(exitCode) : chalk.green(exitCode)}`);
+  }
+
+  if (finalStatus === 'failed' || (exitCode !== null && exitCode !== 0)) {
+    const label = failureType === 'infrastructure'
+      ? chalk.red(`\n  ✗ Infrastructure failure (${dep.deployment_id}) — not your code\n`)
+      : chalk.red(`\n  ✗ Job failed (${dep.deployment_id})\n`);
+    console.error(label);
+    if (failureType === 'infrastructure') {
+      console.error(chalk.dim('  This is a provider-side failure. Contact support with your receipt ID.'));
+    }
     console.error(chalk.dim(`  Logs: badgr logs ${dep.deployment_id}\n`));
-    process.exit(1);
+    process.exit(exitCode ?? 1);
   } else {
-    console.log(chalk.green(`\n  ✓ Job complete (${finalStatus})\n`));
+    console.log(chalk.green(`\n  ✓ Job complete\n`));
   }
 }

package/src/commands/serve.js

@@ -1,6 +1,7 @@
 import { requireApiKey } from '../config.js';
 import { callApi } from '../api.js';
-import { addDeployment, addReceipt, generateReceiptId } from '../store.js';
+import { addDeployment, addReceipt, updateReceipt, generateReceiptId } from '../store.js';
+import { rankAlternatives, promptFallback } from '../fallback.js';
 
 /**
  * badgr serve meta-llama/Llama-3.1-8B-Instruct --gpu L40S
@@ -69,7 +70,11 @@ export async function serveCommand(config, args, chalk) {
   console.log(`  ${chalk.bold('GPU:')}    ${gpu}`);
   console.log();
   console.log(chalk.dim('  Finding best available GPU capacity...'));
-  console.log(chalk.dim(`  API: ${config.baseUrl}`));
+  if (process.env.BADGR_DEBUG === '1' || process.env.BADGR_DEBUG === 'true') {
+    console.log(chalk.dim(`  API: ${config.baseUrl}`));
+  }
+
+  const effectiveRegion = flags.region ? flags.region.toUpperCase() : undefined;
 
   let dep;
   try {
@@ -81,17 +86,73 @@ export async function serveCommand(config, args, chalk) {
         model,
         gpu: gpu.toUpperCase().replace('-', '_'),
         gpu_count: flags.count || 1,
-        region: flags.region || 'US',
+        ...(effectiveRegion ? { region: effectiveRegion } : {}),
         max_price_per_hour: flags.maxPrice,
         name: flags.name,
       },
     });
   } catch (err) {
-    console.error(chalk.red(`\n  ✗ Serve failed: ${err.message}`));
-    console.error(chalk.dim(`\n  Debug: BADGR_DEBUG=1 badgr serve … — shows full request/response`));
-    console.error(chalk.dim(`  Config:  badgr config`));
-    console.error(chalk.dim(`  Docs:    badgr --help\n`));
-    return;
+    const d = err.errorData;
+    if (d?.code === 'NO_CAPACITY_MATCH') {
+      const pool = [
+        ...(Array.isArray(d.same_gpu_other_regions) ? d.same_gpu_other_regions : []),
+        ...(Array.isArray(d.alternatives)            ? d.alternatives            : []),
+      ];
+
+      if (pool.length === 0) {
+        console.error(chalk.red(`\n  ✗ ${gpu} isn't available right now and no alternatives found.\n`));
+        console.error(chalk.dim('  Try increasing --max-price or contact support for a manual quote.'));
+        process.exit(1);
+      }
+
+      const ranked = rankAlternatives(gpu, pool, 'closest');
+      const chosen = await promptFallback(gpu, ranked, chalk);
+
+      if (!chosen) {
+        console.log(chalk.dim('\n  Cancelled.\n'));
+        process.exit(0);
+      }
+
+      console.log(chalk.dim(`\n  Serving on ${chosen.gpu} in ${chosen.region}...\n`));
+
+      try {
+        dep = await callApi('/serve', {
+          method: 'POST',
+          apiKey: config.apiKey,
+          baseUrl: config.baseUrl,
+          body: {
+            model,
+            gpu: chosen.gpu.toUpperCase().replace('-', '_'),
+            gpu_count: flags.count || 1,
+            region: chosen.region,
+            max_price_per_hour: flags.maxPrice,
+            name: flags.name,
+          },
+        });
+      } catch (err2) {
+        console.error(chalk.red(`\n  ✗ Serve failed on ${chosen.gpu}: ${err2.message}`));
+        console.error(chalk.dim(`\n  Debug: BADGR_DEBUG=1 badgr serve … — shows full request/response\n`));
+        process.exit(1);
+      }
+    } else {
+      // Write a failure receipt so every serve attempt is auditable.
+      const failRcptId = generateReceiptId();
+      addReceipt({
+        receiptId:   failRcptId,
+        action:      'badgr serve',
+        model,
+        gpu,
+        status:      'failed',
+        failureType: 'infrastructure',
+        createdAt:   new Date().toISOString(),
+      });
+      console.error(chalk.red(`\n  ✗ Serve failed: ${err.message}`));
+      console.error(chalk.dim(`\n  Receipt: ${failRcptId}`));
+      console.error(chalk.dim(`  Debug: BADGR_DEBUG=1 badgr serve … — shows full request/response`));
+      console.error(chalk.dim(`  Config:  badgr config`));
+      console.error(chalk.dim(`  Docs:    badgr --help\n`));
+      return;
+    }
   }
 
   // Mirror to local store so badgr down/status/receipts work offline
@@ -127,21 +188,27 @@ export async function serveCommand(config, args, chalk) {
   let endpointReady = false;
 
   if (flags.noWait) {
-    console.log(chalk.yellow('\n⏳ Endpoint provisioning (skipped health check — use --no-wait)\n'));
+    console.log(chalk.yellow('\n⏳ Skipped health check (--no-wait)\n'));
   } else {
     console.log(chalk.dim('\n  Health-checking endpoint (up to 5 min)...'));
     endpointReady = await waitForEndpoint(endpointUrl, 5 * 60 * 1000, chalk);
     process.stdout.write('\n');
+    if (!endpointReady) {
+      updateReceipt(rcptId, { status: 'health_check_timeout' });
+    }
   }
 
   // ── Print result ──────────────────────────────────────────────────────────
   if (endpointReady) {
     console.log(chalk.green('✓ Endpoint ready\n'));
   } else {
-    console.log(chalk.yellow('⏳ Endpoint starting\n'));
-    console.log(chalk.dim(`  Not yet responding. Check status:`));
+    console.log(chalk.yellow('⏳ Endpoint still starting (health check timed out)\n'));
+    console.log(chalk.dim('  The deployment was provisioned but did not respond within 5 minutes.'));
+    console.log(chalk.dim('  It may still be loading the model. Check:'));
     console.log(chalk.dim(`    badgr status`));
+    console.log(chalk.dim(`    badgr logs ${dep.deployment_id}`));
     console.log(chalk.dim(`    curl ${endpointUrl}/models`));
+    console.log(chalk.dim(`  If it never starts, run: badgr down ${dep.deployment_id}`));
     console.log();
   }
 
@@ -150,6 +217,7 @@ export async function serveCommand(config, args, chalk) {
   console.log(`  ${chalk.bold('GPU:')}        ${dep.gpu_type} × ${dep.gpu_count}`);
   console.log(`  ${chalk.bold('Endpoint:')}   ${chalk.cyan(endpointUrl)}`);
   if (dep.cost_per_hour > 0) console.log(`  ${chalk.bold('Rate:')}       $${dep.cost_per_hour.toFixed(2)}/hr`);
+  console.log(`  ${chalk.bold('Logs:')}       ${chalk.dim(`badgr logs ${dep.deployment_id}`)}`);
   console.log(`\n  ${chalk.bold('Receipt:')}    ${chalk.dim(rcptId)}`);
 
   if (endpointReady) {

package/src/fallback.js

@@ -0,0 +1,123 @@
+import readline from 'readline';
+
+// Static GPU metadata for local ranking. Routing still happens server-side.
+const GPU_META = {
+  RTX_3080: { vramGb: 10,  family: 'consumer',   tags: ['inference', 'dev'],             desc: 'Dev, light inference' },
+  RTX_4080: { vramGb: 16,  family: 'consumer',   tags: ['inference', 'dev'],             desc: 'Inference and dev workloads' },
+  RTX_4090: { vramGb: 24,  family: 'consumer',   tags: ['inference', 'training', 'dev'], desc: 'Inference, training, dev' },
+  L40S:     { vramGb: 48,  family: 'datacenter', tags: ['inference', 'training'],         desc: 'Inference, vLLM, batch jobs' },
+  A6000:    { vramGb: 48,  family: 'datacenter', tags: ['training', 'inference'],         desc: 'Training, large models, inference' },
+  A100:     { vramGb: 40,  family: 'datacenter', tags: ['training', 'inference'],         desc: 'Large-scale training and inference' },
+  H100:     { vramGb: 80,  family: 'datacenter', tags: ['training', 'large-model'],       desc: 'Large model training, best throughput' },
+};
+
+function altScore(alt, reqMeta) {
+  const meta = GPU_META[alt.gpu] ?? { vramGb: 0, family: 'unknown', tags: [] };
+  let s = 0;
+
+  // 1. GPU class/family similarity
+  if (meta.family === reqMeta.family) s += 200;
+
+  // 2. VRAM — prefer same or more, penalize less
+  const vramDiff = meta.vramGb - reqMeta.vramGb;
+  if (vramDiff >= 0) {
+    s += 80 - Math.min(vramDiff, 40) * 0.5;
+  } else {
+    s += 80 + vramDiff * 4;
+  }
+
+  // 3. Workload tag overlap
+  s += reqMeta.tags.filter(t => meta.tags.includes(t)).length * 20;
+
+  // 4. Availability confidence — not in alternatives payload, treated equal
+
+  // 5. Price — lower wins
+  s -= alt.price * 10;
+
+  return s;
+}
+
+/**
+ * Rank a list of { gpu, region, price } alternatives.
+ * mode 'closest'  → family / VRAM / tag / price (default)
+ * mode 'cheapest' → price ascending only
+ */
+export function rankAlternatives(requestedGpu, alternatives, mode = 'closest') {
+  if (!alternatives || alternatives.length === 0) return [];
+  if (mode === 'cheapest') return [...alternatives].sort((a, b) => a.price - b.price);
+
+  const reqMeta = GPU_META[requestedGpu] ?? { vramGb: 0, family: 'unknown', tags: [] };
+  return [...alternatives].sort((a, b) => altScore(b, reqMeta) - altScore(a, reqMeta));
+}
+
+/**
+ * One-line human-readable diff between requested and chosen GPU.
+ * e.g. "less VRAM (24GB vs 40GB), different GPU class than A100"
+ */
+export function diffDescription(requestedGpu, altGpu) {
+  const req = GPU_META[requestedGpu];
+  const alt = GPU_META[altGpu];
+  if (!req || !alt) return '';
+
+  const parts = [];
+  const vramDiff = alt.vramGb - req.vramGb;
+  if (vramDiff < -4)  parts.push(`less VRAM (${alt.vramGb}GB vs ${req.vramGb}GB)`);
+  else if (vramDiff > 4) parts.push(`more VRAM (${alt.vramGb}GB vs ${req.vramGb}GB)`);
+
+  if (alt.family !== req.family) parts.push('different GPU class');
+
+  if (parts.length === 0) return `similar specs to ${requestedGpu}`;
+  return parts.join(', ') + ` than ${requestedGpu}`;
+}
+
+function ask(prompt) {
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise(resolve => rl.question(prompt, ans => { rl.close(); resolve(ans.trim()); }));
+}
+
+/**
+ * Show the interactive fallback prompt and return the chosen alternative or null.
+ * Non-TTY environments auto-select the top-ranked option.
+ */
+export async function promptFallback(requestedGpu, ranked, chalk) {
+  if (ranked.length === 0) return null;
+
+  const top     = ranked[0];
+  const others  = ranked.slice(1, 4); // cap at 3 extras for readability
+  const topMeta = GPU_META[top.gpu];
+
+  console.log(chalk.yellow(`\n  ${requestedGpu} isn't available right now.\n`));
+  console.log(chalk.bold('  Closest match:'));
+  console.log(`    ${chalk.cyan(top.gpu)} in ${top.region}`);
+  console.log(`    ${chalk.green('$' + top.price.toFixed(2) + '/hr')} estimated`);
+  if (topMeta?.desc) console.log(`    ${topMeta.desc}`);
+  console.log();
+
+  if (!process.stdin.isTTY) {
+    console.log(chalk.dim(`  Auto-selecting ${top.gpu} (non-interactive).`));
+    return top;
+  }
+
+  console.log(`  Press ${chalk.bold('Enter')} to run on ${chalk.cyan(top.gpu)}`);
+  if (others.length > 0) {
+    console.log('  or type:');
+    for (const [i, alt] of others.entries()) {
+      const meta = GPU_META[alt.gpu];
+      const hint = meta ? meta.desc.split(',')[0].toLowerCase() : '';
+      console.log(`    ${chalk.bold(String(i + 1))} = ${alt.gpu}${hint ? ', ' + hint : ''}`);
+    }
+  }
+  console.log(`    ${chalk.bold('q')} = cancel`);
+  console.log();
+
+  const answer = await ask('  > ');
+
+  if (answer === '') return top;
+  if (answer.toLowerCase() === 'q') return null;
+
+  const idx = parseInt(answer, 10);
+  if (!isNaN(idx) && idx >= 1 && idx <= others.length) return others[idx - 1];
+
+  console.log(chalk.dim(`  Unrecognised input — using ${top.gpu}.`));
+  return top;
+}

package/src/store.js

@@ -83,6 +83,15 @@ export function addReceipt(receipt, storeFile = STORE_FILE) {
   return receipt;
 }
 
+export function updateReceipt(receiptId, updates, storeFile = STORE_FILE) {
+  const store = loadStore(storeFile);
+  const idx = store.receipts.findIndex(r => r.receiptId === receiptId);
+  if (idx === -1) return null;
+  store.receipts[idx] = { ...store.receipts[idx], ...updates };
+  saveStore(store, storeFile);
+  return store.receipts[idx];
+}
+
 export function listReceipts(limit = 20, storeFile = STORE_FILE) {
   return loadStore(storeFile).receipts.slice(0, limit);
 }

package/tests/commands.test.js

@@ -1,6 +1,7 @@
-import { describe, it, expect } from 'vitest';
-import { parseRunArgs } from '../src/commands/run.js';
+import { describe, it, expect, vi } from 'vitest';
+import { parseRunArgs, classifyFailure } from '../src/commands/run.js';
 import { parseServeArgs } from '../src/commands/serve.js';
+import { rankAlternatives, diffDescription, promptFallback } from '../src/fallback.js';
 
 describe('parseRunArgs', () => {
   it('parses a plain command', () => {
@@ -43,6 +44,112 @@ describe('parseRunArgs', () => {
     expect(positional).toEqual([]);
     expect(flags.gpu).toBeUndefined();
   });
+
+  it('parses --fallback closest', () => {
+    const { flags } = parseRunArgs(['python', 'train.py', '--gpu', 'A100', '--fallback', 'closest']);
+    expect(flags.fallback).toBe('closest');
+    expect(flags.noFallback).toBeUndefined();
+  });
+
+  it('parses --fallback cheapest', () => {
+    const { flags } = parseRunArgs(['python', 'train.py', '--fallback', 'cheapest']);
+    expect(flags.fallback).toBe('cheapest');
+  });
+
+  it('parses --no-fallback', () => {
+    const { flags } = parseRunArgs(['python', 'train.py', '--gpu', 'A100', '--no-fallback']);
+    expect(flags.noFallback).toBe(true);
+    expect(flags.fallback).toBeUndefined();
+  });
+
+  it('parses --max-runtime as a float (minutes)', () => {
+    const { flags } = parseRunArgs(['python', 'train.py', '--max-runtime', '30']);
+    expect(flags.maxRuntime).toBe(30);
+  });
+
+  it('parses --max-cost as a float (dollars)', () => {
+    const { flags } = parseRunArgs(['python', 'train.py', '--max-cost', '5.00']);
+    expect(flags.maxCost).toBe(5.0);
+  });
+});
+
+describe('classifyFailure', () => {
+  it('returns infrastructure when status is failed and no exit code', () => {
+    expect(classifyFailure('failed', null)).toBe('infrastructure');
+    expect(classifyFailure('failed', undefined)).toBe('infrastructure');
+  });
+
+  it('returns customer_code when exit code is non-zero', () => {
+    expect(classifyFailure('failed', 1)).toBe('customer_code');
+    expect(classifyFailure('completed', 2)).toBe('customer_code');
+  });
+
+  it('returns null for successful jobs', () => {
+    expect(classifyFailure('completed', 0)).toBeNull();
+    expect(classifyFailure('completed', null)).toBeNull();
+  });
+
+  it('returns null for stopped jobs with no exit code', () => {
+    expect(classifyFailure('stopped', null)).toBeNull();
+  });
+});
+
+describe('rankAlternatives', () => {
+  const pool = [
+    { gpu: 'RTX_4090', region: 'US', price: 0.72 },
+    { gpu: 'L40S',     region: 'US', price: 1.25 },
+    { gpu: 'H100',     region: 'EU', price: 2.80 },
+    { gpu: 'A6000',    region: 'US', price: 1.20 },
+  ];
+
+  it('cheapest mode sorts by price ascending', () => {
+    const ranked = rankAlternatives('A100', pool, 'cheapest');
+    expect(ranked[0].price).toBe(0.72);
+    expect(ranked[ranked.length - 1].price).toBe(2.80);
+  });
+
+  it('closest mode prefers same GPU family over cheaper consumer GPU', () => {
+    const ranked = rankAlternatives('A100', pool, 'closest');
+    const topGpu = ranked[0].gpu;
+    // H100, A6000, and L40S are all datacenter — any of them should beat RTX_4090
+    expect(['H100', 'A6000', 'L40S']).toContain(topGpu);
+    const rtxPos = ranked.findIndex(a => a.gpu === 'RTX_4090');
+    expect(rtxPos).toBeGreaterThan(0);
+  });
+
+  it('returns empty array for empty pool', () => {
+    expect(rankAlternatives('A100', [])).toEqual([]);
+  });
+
+  it('returns empty array for null pool', () => {
+    expect(rankAlternatives('A100', null)).toEqual([]);
+  });
+});
+
+describe('diffDescription', () => {
+  it('reports more VRAM when alt has significantly more', () => {
+    const desc = diffDescription('RTX_4090', 'L40S'); // 24GB vs 48GB
+    expect(desc).toContain('more VRAM');
+  });
+
+  it('reports less VRAM when alt has significantly less', () => {
+    const desc = diffDescription('A100', 'RTX_4090'); // 40GB vs 24GB
+    expect(desc).toContain('less VRAM');
+  });
+
+  it('reports different GPU class when family differs', () => {
+    const desc = diffDescription('A100', 'RTX_4090');
+    expect(desc).toContain('different GPU class');
+  });
+
+  it('returns similar specs for same-class same VRAM GPUs', () => {
+    const desc = diffDescription('L40S', 'A6000'); // both 48GB, both datacenter
+    expect(desc).toContain('similar specs');
+  });
+
+  it('returns empty string for unknown GPUs', () => {
+    expect(diffDescription('UNKNOWN', 'ALSO_UNKNOWN')).toBe('');
+  });
 });
 
 describe('parseServeArgs', () => {
@@ -79,3 +186,26 @@ describe('parseServeArgs', () => {
     expect(model).toBeNull();
   });
 });
+
+describe('promptFallback output', () => {
+  it('does not print a Difference line', async () => {
+    const lines = [];
+    const chalk = { yellow: s => s, bold: s => s, cyan: s => s, green: s => s, dim: s => s };
+    const origLog = console.log;
+    console.log = (...args) => lines.push(args.join(' '));
+    // force non-TTY so it auto-selects without asking
+    const origIsTTY = process.stdin.isTTY;
+    process.stdin.isTTY = false;
+
+    const pool = [{ gpu: 'L40S', region: 'US', price: 1.25 }];
+    await promptFallback('A100', pool, chalk);
+
+    console.log = origLog;
+    process.stdin.isTTY = origIsTTY;
+
+    const joined = lines.join('\n');
+    expect(joined).not.toContain('Difference');
+    expect(joined).toContain('L40S');
+    expect(joined).toContain('1.25');
+  });
+});

package/tests/store.test.js

@@ -5,7 +5,7 @@ import { rmSync, existsSync } from 'fs';
 import {
   loadStore, saveStore,
   addDeployment, updateDeployment, removeDeployment, findDeployment, listDeployments,
-  addReceipt, listReceipts,
+  addReceipt, updateReceipt, listReceipts,
   generateDeploymentId, generateReceiptId,
 } from '../src/store.js';
 
@@ -124,3 +124,43 @@ describe('addReceipt / listReceipts', () => {
     expect(listReceipts(3, file)).toHaveLength(3);
   });
 });
+
+describe('updateReceipt', () => {
+  it('merges updates into an existing receipt', () => {
+    addReceipt({ receiptId: 'r-upd', action: 'badgr run', status: 'running' }, file);
+    updateReceipt('r-upd', { status: 'completed', finalCost: 0.012, runtimeSeconds: 42 }, file);
+    const [r] = listReceipts(1, file);
+    expect(r.status).toBe('completed');
+    expect(r.finalCost).toBe(0.012);
+    expect(r.runtimeSeconds).toBe(42);
+    expect(r.action).toBe('badgr run'); // preserved
+  });
+
+  it('returns null for unknown receipt id', () => {
+    expect(updateReceipt('no-such-id', { status: 'done' }, file)).toBeNull();
+  });
+
+  it('records a serve failure receipt with failureType=infrastructure', () => {
+    addReceipt({
+      receiptId:   'r-serve-fail',
+      action:      'badgr serve',
+      model:       'meta-llama/Llama-3.1-8B-Instruct',
+      gpu:         'L40S',
+      status:      'failed',
+      failureType: 'infrastructure',
+      createdAt:   new Date().toISOString(),
+    }, file);
+    const [r] = listReceipts(1, file);
+    expect(r.failureType).toBe('infrastructure');
+    expect(r.action).toBe('badgr serve');
+    expect(r.status).toBe('failed');
+  });
+
+  it('updates a serve receipt to health_check_timeout', () => {
+    addReceipt({ receiptId: 'r-hc', action: 'badgr serve', status: 'provisioning' }, file);
+    updateReceipt('r-hc', { status: 'health_check_timeout' }, file);
+    const receipts = listReceipts(10, file);
+    const r = receipts.find(x => x.receiptId === 'r-hc');
+    expect(r.status).toBe('health_check_timeout');
+  });
+});