badgr-cli 1.0.17 → 1.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "badgr-cli",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "description": "Badgr, run or serve GPU workloads from one command",
   "type": "module",
   "bin": {

package/src/commands/run.js

@@ -46,18 +46,111 @@ export function classifyFailure(finalStatus, exitCode) {
   return null;
 }
 
+// Lines the log stream never needs to print — we surface them in the status bar instead.
+const LOG_META_RE = /^\[dep-[^\]]+\] (status|gpu|region|endpoint|cost|receipt|provider_status|uptime)=/;
+
+// Extract structured values from provider status lines so we can show them nicely.
+function parseProviderLine(line) {
+  const gpuUtil  = line.match(/\bgpu_util=([\d.]+)%/);
+  const cpuUtil  = line.match(/\bcpu_util=([\d.]+)%/);
+  const ssh      = line.match(/\bssh=(\S+)/);
+  const provSt   = line.match(/\bprovider_status=(\S+)/);
+  return {
+    gpuUtil:  gpuUtil  ? parseFloat(gpuUtil[1])  : null,
+    cpuUtil:  cpuUtil  ? parseFloat(cpuUtil[1])  : null,
+    ssh:      ssh      ? ssh[1]                  : null,
+    providerStatus: provSt ? provSt[1]           : null,
+  };
+}
+
+function renderStatusBar(chalk, { elapsedMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost }) {
+  const spent = ratePerHour * (elapsedMs / 3_600_000);
+  const parts = [`⏱ ${fmtRuntime(elapsedMs)}`];
+  if (ratePerHour > 0) parts.push(`$${spent.toFixed(4)} spent`);
+  if (gpuUtil !== null) parts.push(`GPU ${gpuUtil.toFixed(0)}%`);
+  if (cpuUtil !== null) parts.push(`CPU ${cpuUtil.toFixed(0)}%`);
+  if (maxRuntimeMs) {
+    const left = Math.max(0, maxRuntimeMs - elapsedMs);
+    parts.push(`${fmtRuntime(left)} left`);
+  }
+  if (maxCost && ratePerHour > 0) {
+    const budgetLeft = Math.max(0, maxCost - spent);
+    parts.push(`$${budgetLeft.toFixed(4)} budget left`);
+  }
+  parts.push('Ctrl+C to stop');
+  return chalk.dim('  ' + parts.join('  •  '));
+}
+
+// Wait for status to leave 'starting' (smoke check running on backend).
+// Returns the final dep object with status 'running' or 'failed'.
+async function waitForRunning(config, depId, chalk) {
+  const POLL_MS = 3000;
+  const TIMEOUT_MS = 120_000; // 2 min max for smoke check
+  const startMs = Date.now();
+  let dots = 0;
+
+  process.stdout.write(chalk.dim('  Waiting for container to start'));
+  const ticker = setInterval(() => {
+    process.stdout.write('.');
+    dots++;
+  }, 1000);
+
+  try {
+    while (Date.now() - startMs < TIMEOUT_MS) {
+      await new Promise(r => setTimeout(r, POLL_MS));
+      const dep = await callApi(`/deployments/${depId}`, {
+        apiKey: config.apiKey,
+        baseUrl: config.baseUrl,
+      });
+      if (dep.status !== 'starting') {
+        clearInterval(ticker);
+        process.stdout.write('\n');
+        return dep;
+      }
+    }
+  } finally {
+    clearInterval(ticker);
+    if (dots > 0) process.stdout.write('\n');
+  }
+
+  // Timed out waiting — return whatever we have
+  return await callApi(`/deployments/${depId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
+}
+
 async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost = null, ratePerHour = 0, onTeardown }) {
   const TERMINAL = new Set(['stopped', 'failed', 'completed']);
   const POLL_MS  = 4000;
-  let seenLines       = 0;
+  let seenContent     = new Set();  // track by content, not index, to avoid reprinting stale lines
   let lastStatus      = '';
   let consecutiveErrs = 0;
+  let gpuUtil         = null;
+  let cpuUtil         = null;
+  let sshShown        = false;
+  let statusBarActive = false;
   const startMs       = Date.now();
 
+  // Ticker: update status bar in place every second between polls.
+  let tickerInterval = null;
+  const startTicker = () => {
+    if (tickerInterval) return;
+    statusBarActive = true;
+    tickerInterval = setInterval(() => {
+      const bar = renderStatusBar(chalk, {
+        elapsedMs: Date.now() - startMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost,
+      });
+      process.stdout.write(`\r${bar}   `);
+    }, 1000);
+  };
+  const stopTicker = () => {
+    if (tickerInterval) { clearInterval(tickerInterval); tickerInterval = null; }
+    if (statusBarActive) { process.stdout.write('\r\x1b[2K'); statusBarActive = false; } // clear line
+  };
+
   let tearing = false;
   const sigintHandler = () => {
     if (tearing) return;
     tearing = true;
+    stopTicker();
     onTeardown('interrupted');
   };
   process.once('SIGINT', sigintHandler);
@@ -71,13 +164,15 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
 
       if (maxCost !== null && spentSoFar >= maxCost) {
         tearing = true;
-        onTeardown('max-cost');
+        stopTicker();
+        await onTeardown('max-cost');
         return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
       }
 
       if (maxRuntimeMs !== null && elapsedMs >= maxRuntimeMs) {
         tearing = true;
-        onTeardown('max-runtime');
+        stopTicker();
+        await onTeardown('max-runtime');
         return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
       }
 
@@ -93,10 +188,13 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
         if (lastStatus === 'running') {
           const lostSec = Math.round((consecutiveErrs * POLL_MS) / 1000);
           if (consecutiveErrs === HEARTBEAT_WARN_POLLS) {
+            stopTicker();
             console.log(chalk.yellow(`\n  ⚠ No heartbeat for ${lostSec}s — cloud machine may be unresponsive`));
+            startTicker();
           } else if (consecutiveErrs >= HEARTBEAT_KILL_POLLS) {
             tearing = true;
-            onTeardown('heartbeat-lost');
+            stopTicker();
+            await onTeardown('heartbeat-lost');
             return { status: 'failed', exitCode: null, runtimeMs: elapsedMs, failureType: 'infrastructure' };
           }
         }
@@ -106,6 +204,7 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
       const status = dep.status;
       if (status !== lastStatus) {
         if (status === 'running' && lastStatus === 'provisioning') {
+          stopTicker();
           console.log(chalk.dim('  [running]'));
         }
         lastStatus = status;
@@ -117,15 +216,42 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
           baseUrl: config.baseUrl,
         });
         const lines = logData?.logs ?? [];
-        for (let i = seenLines; i < lines.length; i++) {
-          console.log(`  ${chalk.dim(lines[i])}`);
+
+        for (const line of lines) {
+          // Extract structured provider values (gpu_util, ssh, etc.) from any line.
+          const parsed = parseProviderLine(line);
+          if (parsed.gpuUtil  !== null) gpuUtil = parsed.gpuUtil;
+          if (parsed.cpuUtil  !== null) cpuUtil = parsed.cpuUtil;
+
+          // Show SSH address once, prominently.
+          if (parsed.ssh && !sshShown) {
+            stopTicker();
+            console.log(`  ${chalk.bold('SSH:')}      ${chalk.cyan(parsed.ssh)}`);
+            sshShown = true;
+            startTicker();
+          }
+
+          // Skip lines we've already printed and pure metadata lines.
+          if (seenContent.has(line)) continue;
+          seenContent.add(line);
+          if (LOG_META_RE.test(line)) continue;
+          // Skip provider util lines — they're shown in the status bar instead.
+          if (/\b(gpu_util|cpu_util|provider_status|uptime)=/.test(line)) continue;
+
+          const isErrorLine = /^error\b/i.test(line) || /Error response from daemon/i.test(line);
+          stopTicker();
+          console.log(`  ${isErrorLine ? chalk.red(line) : chalk.dim(line)}`);
+          startTicker();
         }
-        seenLines = lines.length;
       } catch {
         // logs not ready yet
       }
 
+      // Start the ticker once the job is confirmed running.
+      if (status === 'running') startTicker();
+
       if (TERMINAL.has(status)) {
+        stopTicker();
         process.removeListener('SIGINT', sigintHandler);
         const exitCode = dep.exit_code ?? null;
         return {
@@ -137,6 +263,7 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
       }
     }
   } finally {
+    stopTicker();
     process.removeListener('SIGINT', sigintHandler);
   }
 }
@@ -370,8 +497,22 @@ export async function runCommand(config, args, chalk) {
     return;
   }
 
+  // If the backend is still running the smoke check, wait for it to finish.
+  if (dep.status === 'starting') {
+    console.log();
+    dep = await waitForRunning(config, dep.deployment_id, chalk);
+  }
+
+  if (dep.status === 'failed') {
+    console.error(chalk.red('\n  ✗ Container failed to start (infrastructure error).\n'));
+    console.error(chalk.dim('  The backend retried automatically. All attempts failed.'));
+    console.error(chalk.dim(`  Contact support with receipt ID: ${rcptId}`));
+    console.log();
+    process.exit(1);
+  }
+
   const ratePerHour = dep.cost_per_hour || 0;
-  console.log(chalk.dim('\n  ── Streaming logs (Ctrl+C to stop job) ─────────────────────────\n'));
+  console.log(chalk.dim('\n  ── Live status (Ctrl+C to stop) ─────────────────────────────────\n'));
 
   async function teardown(reason) {
     const labels = {
@@ -433,7 +574,7 @@ export async function runCommand(config, args, chalk) {
     }
     console.log();
     process.exit(exitCode ?? 1);
-  } else {
+  } else if (finalStatus === 'completed' && (exitCode === 0 || exitCode === null)) {
     console.log(chalk.green(`\n  ✓ Complete\n`));
   }
 }