badgr-cli 1.0.44 → 1.0.46

package/src/detect.js

@@ -0,0 +1,362 @@
+/**
+ * Workload detection — infer how to run an arbitrary GPU project from its
+ * files, without asking the user to describe their infrastructure.
+ *
+ * Domain-agnostic by design: no code path here is specific to any one
+ * framework (ComfyUI, vLLM, RunPod, etc.) or provider. Detection works off
+ * generic, corroborating file/content signals — entry-point filenames,
+ * dependency manifests, port/health conventions, model-reference patterns —
+ * so the same engine covers custom scripts, endpoints, batch jobs, and
+ * training repos alike. Framework-specific signals are just additional
+ * evidence fed into the same generic scoring, never a special-cased branch.
+ *
+ * Pure and local: reads files under the target directory only, makes no
+ * network calls, and never reveals provider/routing details — that's the
+ * caller's job (see commands/detect.js and commands/run.js).
+ */
+import fs from 'fs';
+import path from 'path';
+
+// Mirrors _ZIP_EXCLUDES in commands/run.js — directories we never walk into.
+const SKIP_DIRS = new Set([
+  '.git', 'node_modules', '__pycache__', '.venv', 'venv', 'env',
+  'dist', 'build', '.next', '.nuxt', 'coverage', '.pytest_cache',
+  '.mypy_cache', '.ruff_cache', '.DS_Store', '.idea', '.vscode',
+]);
+
+const MAX_READ_BYTES = 200_000; // don't slurp huge files for signal-scanning
+const MAX_WALK_ENTRIES = 5000;  // safety bound on very large trees
+
+function safeReadText(absPath) {
+  try {
+    const stat = fs.statSync(absPath);
+    if (!stat.isFile() || stat.size > 5_000_000) return '';
+    const fd = fs.openSync(absPath, 'r');
+    const len = Math.min(stat.size, MAX_READ_BYTES);
+    const buf = Buffer.alloc(len);
+    fs.readSync(fd, buf, 0, len, 0);
+    fs.closeSync(fd);
+    return buf.toString('utf8');
+  } catch {
+    return '';
+  }
+}
+
+function walk(dir, files = [], depth = 0) {
+  if (files.length >= MAX_WALK_ENTRIES || depth > 8) return files;
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return files;
+  }
+  for (const entry of entries) {
+    if (files.length >= MAX_WALK_ENTRIES) break;
+    if (entry.name.startsWith('.') && !['.env.example'].includes(entry.name)) {
+      if (entry.isDirectory()) continue;
+    }
+    const abs = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.has(entry.name)) continue;
+      walk(abs, files, depth + 1);
+    } else {
+      files.push(abs);
+    }
+  }
+  return files;
+}
+
+// ── Signal catalogs (generic — ordered by how strongly they imply an entry point) ──
+
+const ENTRYPOINT_CANDIDATES = [
+  'main.py', 'app.py', 'server.py', 'run.py', 'predict.py', 'inference.py',
+  'handler.py', 'rp_handler.py', 'worker.py', 'train.py', 'finetune.py',
+  'generate.py', 'transcribe.py', 'index.js', 'server.js', 'main.js',
+];
+
+const REQUIREMENTS_FILES = [
+  'requirements.txt', 'pyproject.toml', 'environment.yml', 'environment.yaml',
+  'Pipfile', 'package.json', 'setup.py', 'poetry.lock',
+];
+
+const MODEL_DIR_NAMES = ['models', 'checkpoints', 'ckpt', 'weights', 'model'];
+const INPUT_DIR_NAMES = ['input', 'inputs', 'data', 'dataset', 'datasets'];
+const OUTPUT_DIR_NAMES = ['output', 'outputs', 'out', 'results', 'generated'];
+const CHECKPOINT_DIR_NAMES = ['checkpoints', 'checkpoint', 'ckpt', 'ckpts'];
+
+const HF_REPO_RE = /\b[a-zA-Z0-9][\w.-]{1,60}\/[a-zA-Z0-9][\w.-]{1,80}\b/g;
+const WEIGHT_FILE_RE = /[\w.-]+\.(safetensors|ckpt|gguf|bin|pt|pth|onnx)\b/gi;
+const PARAM_SIZE_RE = /\b(\d{1,3})\s*[bB]\b/g;
+
+// ── File-content sniffers ────────────────────────────────────────────────
+
+function detectPorts(files) {
+  const ports = new Set();
+  for (const f of files) {
+    const base = path.basename(f);
+    const text = base === 'Dockerfile' || /\.(py|js|ts|toml|ya?ml)$/.test(base)
+      ? safeReadText(f) : '';
+    if (!text) continue;
+    for (const m of text.matchAll(/\bEXPOSE\s+(\d{2,5})/gi)) ports.add(Number(m[1]));
+    for (const m of text.matchAll(/\bport\s*[=:]\s*(\d{2,5})\b/gi)) ports.add(Number(m[1]));
+    for (const m of text.matchAll(/--port[= ](\d{2,5})/g)) ports.add(Number(m[1]));
+  }
+  return [...ports];
+}
+
+function detectModelRefs(files) {
+  const refs = new Set();
+  for (const f of files) {
+    const base = path.basename(f);
+    if (!/\.(py|json|ya?ml|toml|txt|md)$/.test(base)) continue;
+    const text = safeReadText(f);
+    if (!text) continue;
+    for (const m of text.matchAll(WEIGHT_FILE_RE)) refs.add(m[0]);
+    for (const m of text.matchAll(/\b(base_model|model_name|model_id|MODEL_NAME|--model)\s*[:=]\s*["']?([\w./-]+)["']?/g)) {
+      refs.add(m[2]);
+    }
+    // Hugging Face "org/name" style refs — only keep ones that look plausible
+    // (avoid matching file paths like src/foo by requiring at least one dot-free segment
+    // and no leading './').
+    for (const m of text.matchAll(HF_REPO_RE)) {
+      const val = m[0];
+      if (val.startsWith('.') || val.includes('://') || /\.(py|js|ts|json|ya?ml|txt|md|toml)$/.test(val)) continue;
+      if (/^(https?|import|from|src|lib|utils|tests?|scripts?|docs?)\//.test(val)) continue;
+      if (val.split('/').length !== 2) continue;
+      refs.add(val);
+    }
+  }
+  return [...refs].slice(0, 20);
+}
+
+function listDirsMatching(rootFiles, rootDir, names) {
+  const found = new Set();
+  for (const name of names) {
+    const abs = path.join(rootDir, name);
+    if (fs.existsSync(abs) && fs.statSync(abs).isDirectory()) found.add(name + '/');
+  }
+  return [...found];
+}
+
+function estimateVramFromModelRefs(refs, workloadType) {
+  let maxB = 0;
+  for (const ref of refs) {
+    for (const m of ref.matchAll(PARAM_SIZE_RE)) {
+      const n = parseInt(m[1], 10);
+      if (n > maxB && n <= 700) maxB = n;
+    }
+  }
+  if (maxB > 0) {
+    if (maxB <= 3) return '8+ GB';
+    if (maxB <= 8) return '16+ GB';
+    if (maxB <= 14) return '24+ GB';
+    if (maxB <= 34) return '40+ GB';
+    return '80+ GB';
+  }
+  const defaults = {
+    training: '40+ GB',
+    image_gen: '16+ GB',
+    endpoint: '24+ GB',
+    batch: '16+ GB',
+    custom: '16+ GB',
+  };
+  return defaults[workloadType] || '16+ GB';
+}
+
+function estimateRuntimeMinutes(workloadType) {
+  if (workloadType === 'endpoint') return null; // runs until stopped, not a fixed job
+  const defaults = {
+    training: 120,
+    image_gen: 20,
+    batch: 30,
+    custom: 30,
+  };
+  return defaults[workloadType] ?? 30;
+}
+
+// ── Entry-point + workload-type inference ────────────────────────────────
+
+function pickEntrypoint(rootDir, allFiles) {
+  const relTop = allFiles
+    .filter(f => path.dirname(f) === rootDir)
+    .map(f => path.basename(f));
+
+  for (const candidate of ENTRYPOINT_CANDIDATES) {
+    if (relTop.includes(candidate)) {
+      return { file: candidate, confidenceBoost: 2 };
+    }
+  }
+  // Fall back to *any* single top-level .py/.js file if there's exactly one —
+  // common for "plain script" projects with no conventional entry-point name.
+  const topScripts = relTop.filter(f => /\.(py|js)$/.test(f));
+  if (topScripts.length === 1) {
+    return { file: topScripts[0], confidenceBoost: 1 };
+  }
+  return null;
+}
+
+function classifyWorkload({ entrypointText, hasWorkflowJson, hasDockerfile, hasTrainSignal, hasServeSignal, hasHandlerSignal, ports }) {
+  const signals = [];
+  let type = 'custom';
+  let score = 0;
+
+  if (hasHandlerSignal) {
+    signals.push('serverless-style handler function detected');
+    type = 'endpoint';
+    score += 2;
+  }
+  if (hasServeSignal || ports.length > 0) {
+    signals.push(ports.length > 0 ? `listens on port ${ports[0]}` : 'starts an HTTP server');
+    type = 'endpoint';
+    score += 2;
+  }
+  if (hasTrainSignal) {
+    signals.push('training/fine-tuning loop detected');
+    type = 'training';
+    score += 2;
+  }
+  if (hasWorkflowJson) {
+    signals.push('image-generation workflow file detected');
+    if (type === 'custom') type = 'image_gen';
+    score += 2;
+  }
+  if (hasDockerfile) {
+    signals.push('Dockerfile present');
+    score += 1;
+  }
+  if (/for\s+\w+\s+in\s+.*(os\.listdir|glob\(|Path\(.*\)\.iterdir)/.test(entrypointText || '')) {
+    signals.push('iterates over an input directory');
+    if (type === 'custom') type = 'batch';
+    score += 1;
+  }
+
+  return { type, score, signals };
+}
+
+/**
+ * Detect the shape of a GPU workload from a local project directory.
+ * Returns a plain object describing what was found — never throws for a
+ * missing/empty directory, so callers can always render a report.
+ */
+export function detectWorkload(dirPath) {
+  const rootDir = path.resolve(dirPath);
+  if (!fs.existsSync(rootDir) || !fs.statSync(rootDir).isDirectory()) {
+    return {
+      rootDir,
+      workloadType: 'custom',
+      confidence: 'low',
+      command: null,
+      image: null,
+      dockerfile: null,
+      requirements: [],
+      models: [],
+      inputs: [],
+      outputs: [],
+      checkpoints: [],
+      ports: [],
+      healthPath: null,
+      vram: '16+ GB',
+      runtimeEstimateMinutes: 30,
+      cacheableAssets: [],
+      signals: [],
+      error: 'directory not found',
+    };
+  }
+
+  const allFiles = walk(rootDir);
+
+  const dockerfilePath = allFiles.find(f => path.basename(f) === 'Dockerfile' && path.dirname(f) === rootDir);
+  const dockerfileText = dockerfilePath ? safeReadText(dockerfilePath) : '';
+
+  const requirements = REQUIREMENTS_FILES.filter(name =>
+    allFiles.some(f => path.basename(f) === name && path.dirname(f) === rootDir));
+
+  const workflowJson = allFiles.find(f =>
+    /workflow.*\.json$/i.test(path.basename(f)) && path.dirname(f) === rootDir);
+
+  const entry = pickEntrypoint(rootDir, allFiles);
+  const entrypointText = entry ? safeReadText(path.join(rootDir, entry.file)) : '';
+  const combinedText = entrypointText + '\n' + dockerfileText;
+
+  const hasServeSignal = /(uvicorn|fastapi|flask|app\.run\(|vllm\s+serve|VLLM|litserve|gradio|\.listen\()/i.test(combinedText);
+  const hasHandlerSignal = /(runpod\.serverless\.start|def\s+handler\s*\(|exports\.handler\s*=)/i.test(combinedText);
+  const hasTrainSignal = /(\.backward\(\)|optimizer\.step\(|Trainer\(|axolotl|lora|qlora|peft|accelerate|kohya)/i.test(combinedText) ||
+    (entry && /train|finetune/i.test(entry.file));
+  const ports = detectPorts([dockerfilePath, entry ? path.join(rootDir, entry.file) : null].filter(Boolean));
+
+  const { type: workloadType, score, signals } = classifyWorkload({
+    entrypointText: combinedText,
+    hasWorkflowJson: Boolean(workflowJson),
+    hasDockerfile: Boolean(dockerfilePath),
+    hasTrainSignal,
+    hasServeSignal,
+    hasHandlerSignal,
+    ports,
+  });
+
+  const models = detectModelRefs([
+    ...(entry ? [path.join(rootDir, entry.file)] : []),
+    ...(workflowJson ? [workflowJson] : []),
+    ...allFiles.filter(f => /\.ya?ml$/i.test(f) && path.dirname(f) === rootDir),
+  ]);
+
+  const inputs = listDirsMatching(allFiles, rootDir, INPUT_DIR_NAMES);
+  const outputs = listDirsMatching(allFiles, rootDir, OUTPUT_DIR_NAMES);
+  const checkpoints = listDirsMatching(allFiles, rootDir, CHECKPOINT_DIR_NAMES);
+  const modelDirs = listDirsMatching(allFiles, rootDir, MODEL_DIR_NAMES);
+
+  let command = null;
+  if (entry) {
+    if (/\.py$/.test(entry.file)) command = `python ${entry.file}`;
+    else if (/\.js$/.test(entry.file)) command = `node ${entry.file}`;
+  } else if (workflowJson) {
+    command = `python main.py --workflow ${path.basename(workflowJson)}`;
+  }
+
+  let confidenceScore = score + (entry ? entry.confidenceBoost : 0) + (requirements.length > 0 ? 1 : 0);
+  let confidence = 'low';
+  if (confidenceScore >= 4) confidence = 'high';
+  else if (confidenceScore >= 2) confidence = 'medium';
+
+  if (entry) signals.push(`entry point: ${entry.file}`);
+  if (requirements.length) signals.push(`dependency manifest: ${requirements.join(', ')}`);
+  if (!entry && !workflowJson) signals.push('no recognizable entry point — command must be supplied');
+
+  const healthPath = hasHandlerSignal ? null : (hasServeSignal ? '/health' : null);
+
+  const cacheableAssets = [
+    ...modelDirs,
+    ...(requirements.length ? requirements : []),
+  ];
+
+  return {
+    rootDir,
+    workloadType,
+    confidence,
+    command,
+    image: dockerfilePath ? null : undefined, // Dockerfile present → image is built from it, not picked
+    dockerfile: dockerfilePath ? path.relative(rootDir, dockerfilePath) : null,
+    requirements,
+    models,
+    inputs,
+    outputs,
+    checkpoints,
+    ports,
+    healthPath,
+    vram: estimateVramFromModelRefs(models, workloadType),
+    runtimeEstimateMinutes: estimateRuntimeMinutes(workloadType),
+    cacheableAssets,
+    signals,
+  };
+}
+
+/** Human label for a workload type — used by both `detect` and `run` reports. */
+export function workloadTypeLabel(type) {
+  return {
+    endpoint: 'endpoint',
+    training: 'training',
+    image_gen: 'batch (image generation)',
+    batch: 'batch',
+    custom: 'custom GPU job',
+  }[type] || 'custom GPU job';
+}

package/src/progress.js

@@ -40,3 +40,163 @@ export function renderLiveBlock(chalk, { stageLine, elapsedSec, statusWord, spen
     chalk.dim(`        Stop billing: badgr down ${id}`),
   ];
 }
+
+// ---------------------------------------------------------------------------
+// GET /v1/jobs/{id} polling — shared by every command that submits through
+// the productized job API (comfy.batch, train.lora, custom.run, model.serve,
+// image.generate) instead of the raw /deployments API. Surfaces the shared
+// progress contract (stage/health/progress_current/progress_total/...) and
+// makes backend failures and poll failures visible instead of swallowing
+// them, per the job-reliability spec.
+// ---------------------------------------------------------------------------
+
+export const TERMINAL_JOB_STATUSES = new Set(['completed', 'failed', 'canceled']);
+
+// After this many consecutive poll failures, warn the user we've lost contact.
+const POLL_FAIL_WARN_THRESHOLD = 3;
+// After this many, stop polling and report clearly instead of hanging forever.
+const POLL_FAIL_GIVEUP_THRESHOLD = 8;
+
+/**
+ * Poll GET /jobs/{id} until it reaches a terminal status (or maxMs elapses).
+ * `callApi` is injected (rather than imported) to avoid a circular import
+ * with api.js and to keep this module easy to unit test.
+ */
+export async function pollJobUntilTerminal(callApi, config, jobId, { chalk, maxMs, pollMs = 15000 } = {}) {
+  const startMs = Date.now();
+  let consecutiveFailures = 0;
+  let lastDetail = null;
+  let warned = false;
+  let announcedRetry = false;
+
+  while (Date.now() - startMs < maxMs) {
+    await new Promise(r => setTimeout(r, pollMs));
+
+    let detail;
+    try {
+      detail = await callApi(`/jobs/${jobId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
+    } catch (err) {
+      consecutiveFailures++;
+      if (consecutiveFailures >= POLL_FAIL_WARN_THRESHOLD && !warned) {
+        const lostSec = Math.round((consecutiveFailures * pollMs) / 1000);
+        process.stdout.write(
+          `\n${chalk.yellow(`  ⚠ Lost contact with Badgr for ~${lostSec}s (${err.message}). ` +
+            `Last known status: ${lastDetail?.status || 'unknown'}`)}\n`
+        );
+        warned = true;
+      }
+      if (consecutiveFailures >= POLL_FAIL_GIVEUP_THRESHOLD) {
+        return { outcome: 'polling_failed', detail: lastDetail, jobId };
+      }
+      continue;
+    }
+    consecutiveFailures = 0;
+    warned = false;
+    lastDetail = detail;
+
+    // One-time announcement of the automatic retry-on-a-different-route —
+    // printed once per retry (not every poll tick) so it reads as an event,
+    // not a repeated status line. See job_progress.py's retrying_different_route
+    // stage — this is what "one safe retry" looks like from the CLI.
+    if (detail.stage === 'retrying_different_route' && !announcedRetry) {
+      announcedRetry = true;
+      const teardown = detail.teardown_status === 'ok' ? 'succeeded'
+        : detail.teardown_status === 'failed' ? 'failed — run `badgr status` to check'
+        : 'pending';
+      process.stdout.write(
+        `\n${chalk.yellow(`      ${detail.progress_message || 'Retrying on a different route...'}`)}\n` +
+        `      ${chalk.dim(`Previous attempt teardown: ${teardown}`)}\n` +
+        `      ${chalk.dim('Billing: stopped')}\n`
+      );
+    } else if (detail.stage !== 'retrying_different_route') {
+      announcedRetry = false;
+    }
+
+    const elapsedSec = detail.elapsed_seconds ?? Math.floor((Date.now() - startMs) / 1000);
+    const label = detail.stage_label || detail.status;
+    const counter = (detail.progress_current != null && detail.progress_total != null)
+      ? `  ${detail.progress_current}/${detail.progress_total}${detail.progress_unit ? ' ' + detail.progress_unit : ''}`
+      : '';
+    const healthWord = detail.health || detail.status;
+    const latest = detail.progress_message ? `  Latest: ${detail.progress_message}` : '';
+    process.stdout.write(
+      `\r\x1b[2K  [${label}]${counter}  ${elapsedSec}s elapsed  Status: ${healthWord}${latest}`
+    );
+
+    if (TERMINAL_JOB_STATUSES.has(detail.status)) {
+      process.stdout.write('\n');
+      return { outcome: detail.status, detail, jobId };
+    }
+  }
+
+  return { outcome: 'timed_out', detail: lastDetail, jobId };
+}
+
+function _formatElapsed(seconds) {
+  if (seconds == null) return 'unknown';
+  const m = Math.floor(seconds / 60);
+  const s = Math.floor(seconds % 60);
+  return `${m}m ${s}s`;
+}
+
+/**
+ * Prints the `Class:`/`Next:` lines for a failed GpuDeployment (the raw
+ * /deployments/{id} path used by `badgr run`/`badgr serve`/`badgr comfyui run`)
+ * using the exact same failure_class/next_action fields — computed by the
+ * same backend/job_progress.py classify_failure()/next_step_for() functions —
+ * that renderJobClosingBlock below already prints for the productized
+ * /v1/jobs path (comfy.batch, train.lora, custom.run, model.serve). One
+ * failure-class vocabulary shown the same way regardless of which API path
+ * a command happens to use.
+ */
+export function printFailureClass(chalk, dep) {
+  if (!dep?.failure_class) return;
+  console.error(chalk.dim(`  Class: ${dep.failure_class}`));
+  if (dep.next_action) console.error(chalk.dim(`  Next: ${dep.next_action}`));
+}
+
+/**
+ * The standard closing block shown by every job-family CLI command
+ * (comfy.batch, train.lora, custom.run, model.serve) once GET /jobs/{id}
+ * reaches a terminal status — same shape regardless of job type, so a user
+ * always sees runtime/teardown/billing/logs/receipt in the same place.
+ */
+export function renderJobClosingBlock(chalk, detail, rcptId) {
+  const elapsed = _formatElapsed(detail.elapsed_seconds);
+  const teardown = detail.teardown_status === 'ok' ? 'succeeded'
+    : detail.teardown_status === 'failed' ? 'failed — run `badgr status` to check'
+    : detail.status === 'completed' || detail.status === 'failed' || detail.status === 'canceled' ? 'not required'
+    : 'pending';
+  const billing = detail.billing_status === 'stopped' ? 'stopped' : 'running';
+
+  const lines = [];
+  if (detail.status === 'completed') {
+    lines.push(chalk.green('\n  Complete\n'));
+    lines.push(`  Runtime: ${elapsed}`);
+    if (detail.charged_usd != null) lines.push(`  Estimated cost: ~$${detail.charged_usd.toFixed(2)}`);
+  } else if (detail.status === 'failed') {
+    const capped = detail.failure_class === 'runtime_cap_reached' || detail.failure_class === 'spend_cap_reached';
+    const headline = detail.error?.message || detail.progress_message || 'Job failed';
+    lines.push(chalk[capped ? 'yellow' : 'red'](`\n  ${capped ? 'Stopped — limit reached' : 'Failed'} — ${headline}\n`));
+    if (detail.failure_class) lines.push(`  Class: ${detail.failure_class}`);
+    if (detail.stage) lines.push(`  Stage: ${detail.stage}`);
+    lines.push(`  Runtime: ${elapsed}`);
+    if (detail.progress_current != null && detail.progress_total != null) {
+      lines.push(`  Progress: ${detail.progress_current}/${detail.progress_total}${detail.progress_unit ? ' ' + detail.progress_unit : ''}`);
+    }
+  } else if (detail.status === 'canceled') {
+    lines.push(chalk.yellow('\n  Cancelled by user\n'));
+    lines.push(`  Runtime: ${elapsed}`);
+  } else {
+    lines.push(chalk.yellow(`\n  Still running — detached (status: ${detail.status})\n`));
+  }
+
+  lines.push(`  Teardown: ${teardown}`);
+  lines.push(`  Billing: ${billing}`);
+  lines.push(`  Logs: badgr logs ${detail.job_id}`);
+  lines.push(`  Job ID: ${detail.job_id}`);
+  if (rcptId) lines.push(`  Receipt: badgr receipts ${rcptId}`);
+  if (detail.status === 'failed' && detail.next_action) lines.push(`  Next: ${detail.next_action}`);
+
+  return lines.join('\n') + '\n';
+}

package/src/store.js

@@ -95,3 +95,14 @@ export function updateReceipt(receiptId, updates, storeFile = STORE_FILE) {
 export function listReceipts(limit = 20, storeFile = STORE_FILE) {
   return loadStore(storeFile).receipts.slice(0, limit);
 }
+
+// Local receipt IDs (rcpt-...) are minted client-side by job-family commands
+// (badgr comfyui batch, badgr train lora, ...) and are never registered with
+// the backend — they exist only to point back at the job_id that made them.
+// `badgr receipts <id>` needs this lookup so it can fetch the *job's* status
+// instead of hitting the backend's unrelated /v1/receipts (LLM inference
+// receipt ledger), which has never heard of a locally-minted rcpt- id.
+export function findReceipt(idOrJobId, storeFile = STORE_FILE) {
+  const { receipts } = loadStore(storeFile);
+  return receipts.find(r => r.receiptId === idOrJobId || r.job_id === idOrJobId) ?? null;
+}