badgr-cli 1.0.44 → 1.0.45

package/src/progress.js

@@ -40,3 +40,163 @@ export function renderLiveBlock(chalk, { stageLine, elapsedSec, statusWord, spen
     chalk.dim(`        Stop billing: badgr down ${id}`),
   ];
 }
+
+// ---------------------------------------------------------------------------
+// GET /v1/jobs/{id} polling — shared by every command that submits through
+// the productized job API (comfy.batch, train.lora, custom.run, model.serve,
+// image.generate) instead of the raw /deployments API. Surfaces the shared
+// progress contract (stage/health/progress_current/progress_total/...) and
+// makes backend failures and poll failures visible instead of swallowing
+// them, per the job-reliability spec.
+// ---------------------------------------------------------------------------
+
+export const TERMINAL_JOB_STATUSES = new Set(['completed', 'failed', 'canceled']);
+
+// After this many consecutive poll failures, warn the user we've lost contact.
+const POLL_FAIL_WARN_THRESHOLD = 3;
+// After this many, stop polling and report clearly instead of hanging forever.
+const POLL_FAIL_GIVEUP_THRESHOLD = 8;
+
+/**
+ * Poll GET /jobs/{id} until it reaches a terminal status (or maxMs elapses).
+ * `callApi` is injected (rather than imported) to avoid a circular import
+ * with api.js and to keep this module easy to unit test.
+ */
+export async function pollJobUntilTerminal(callApi, config, jobId, { chalk, maxMs, pollMs = 15000 } = {}) {
+  const startMs = Date.now();
+  let consecutiveFailures = 0;
+  let lastDetail = null;
+  let warned = false;
+  let announcedRetry = false;
+
+  while (Date.now() - startMs < maxMs) {
+    await new Promise(r => setTimeout(r, pollMs));
+
+    let detail;
+    try {
+      detail = await callApi(`/jobs/${jobId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
+    } catch (err) {
+      consecutiveFailures++;
+      if (consecutiveFailures >= POLL_FAIL_WARN_THRESHOLD && !warned) {
+        const lostSec = Math.round((consecutiveFailures * pollMs) / 1000);
+        process.stdout.write(
+          `\n${chalk.yellow(`  ⚠ Lost contact with Badgr for ~${lostSec}s (${err.message}). ` +
+            `Last known status: ${lastDetail?.status || 'unknown'}`)}\n`
+        );
+        warned = true;
+      }
+      if (consecutiveFailures >= POLL_FAIL_GIVEUP_THRESHOLD) {
+        return { outcome: 'polling_failed', detail: lastDetail, jobId };
+      }
+      continue;
+    }
+    consecutiveFailures = 0;
+    warned = false;
+    lastDetail = detail;
+
+    // One-time announcement of the automatic retry-on-a-different-route —
+    // printed once per retry (not every poll tick) so it reads as an event,
+    // not a repeated status line. See job_progress.py's retrying_different_route
+    // stage — this is what "one safe retry" looks like from the CLI.
+    if (detail.stage === 'retrying_different_route' && !announcedRetry) {
+      announcedRetry = true;
+      const teardown = detail.teardown_status === 'ok' ? 'succeeded'
+        : detail.teardown_status === 'failed' ? 'failed — run `badgr status` to check'
+        : 'pending';
+      process.stdout.write(
+        `\n${chalk.yellow(`      ${detail.progress_message || 'Retrying on a different route...'}`)}\n` +
+        `      ${chalk.dim(`Previous attempt teardown: ${teardown}`)}\n` +
+        `      ${chalk.dim('Billing: stopped')}\n`
+      );
+    } else if (detail.stage !== 'retrying_different_route') {
+      announcedRetry = false;
+    }
+
+    const elapsedSec = detail.elapsed_seconds ?? Math.floor((Date.now() - startMs) / 1000);
+    const label = detail.stage_label || detail.status;
+    const counter = (detail.progress_current != null && detail.progress_total != null)
+      ? `  ${detail.progress_current}/${detail.progress_total}${detail.progress_unit ? ' ' + detail.progress_unit : ''}`
+      : '';
+    const healthWord = detail.health || detail.status;
+    const latest = detail.progress_message ? `  Latest: ${detail.progress_message}` : '';
+    process.stdout.write(
+      `\r\x1b[2K  [${label}]${counter}  ${elapsedSec}s elapsed  Status: ${healthWord}${latest}`
+    );
+
+    if (TERMINAL_JOB_STATUSES.has(detail.status)) {
+      process.stdout.write('\n');
+      return { outcome: detail.status, detail, jobId };
+    }
+  }
+
+  return { outcome: 'timed_out', detail: lastDetail, jobId };
+}
+
+function _formatElapsed(seconds) {
+  if (seconds == null) return 'unknown';
+  const m = Math.floor(seconds / 60);
+  const s = Math.floor(seconds % 60);
+  return `${m}m ${s}s`;
+}
+
+/**
+ * Prints the `Class:`/`Next:` lines for a failed GpuDeployment (the raw
+ * /deployments/{id} path used by `badgr run`/`badgr serve`/`badgr comfyui run`)
+ * using the exact same failure_class/next_action fields — computed by the
+ * same backend/job_progress.py classify_failure()/next_step_for() functions —
+ * that renderJobClosingBlock below already prints for the productized
+ * /v1/jobs path (comfy.batch, train.lora, custom.run, model.serve). One
+ * failure-class vocabulary shown the same way regardless of which API path
+ * a command happens to use.
+ */
+export function printFailureClass(chalk, dep) {
+  if (!dep?.failure_class) return;
+  console.error(chalk.dim(`  Class: ${dep.failure_class}`));
+  if (dep.next_action) console.error(chalk.dim(`  Next: ${dep.next_action}`));
+}
+
+/**
+ * The standard closing block shown by every job-family CLI command
+ * (comfy.batch, train.lora, custom.run, model.serve) once GET /jobs/{id}
+ * reaches a terminal status — same shape regardless of job type, so a user
+ * always sees runtime/teardown/billing/logs/receipt in the same place.
+ */
+export function renderJobClosingBlock(chalk, detail, rcptId) {
+  const elapsed = _formatElapsed(detail.elapsed_seconds);
+  const teardown = detail.teardown_status === 'ok' ? 'succeeded'
+    : detail.teardown_status === 'failed' ? 'failed — run `badgr status` to check'
+    : detail.status === 'completed' || detail.status === 'failed' || detail.status === 'canceled' ? 'not required'
+    : 'pending';
+  const billing = detail.billing_status === 'stopped' ? 'stopped' : 'running';
+
+  const lines = [];
+  if (detail.status === 'completed') {
+    lines.push(chalk.green('\n  Complete\n'));
+    lines.push(`  Runtime: ${elapsed}`);
+    if (detail.charged_usd != null) lines.push(`  Estimated cost: ~$${detail.charged_usd.toFixed(2)}`);
+  } else if (detail.status === 'failed') {
+    const capped = detail.failure_class === 'runtime_cap_reached' || detail.failure_class === 'spend_cap_reached';
+    const headline = detail.error?.message || detail.progress_message || 'Job failed';
+    lines.push(chalk[capped ? 'yellow' : 'red'](`\n  ${capped ? 'Stopped — limit reached' : 'Failed'} — ${headline}\n`));
+    if (detail.failure_class) lines.push(`  Class: ${detail.failure_class}`);
+    if (detail.stage) lines.push(`  Stage: ${detail.stage}`);
+    lines.push(`  Runtime: ${elapsed}`);
+    if (detail.progress_current != null && detail.progress_total != null) {
+      lines.push(`  Progress: ${detail.progress_current}/${detail.progress_total}${detail.progress_unit ? ' ' + detail.progress_unit : ''}`);
+    }
+  } else if (detail.status === 'canceled') {
+    lines.push(chalk.yellow('\n  Cancelled by user\n'));
+    lines.push(`  Runtime: ${elapsed}`);
+  } else {
+    lines.push(chalk.yellow(`\n  Still running — detached (status: ${detail.status})\n`));
+  }
+
+  lines.push(`  Teardown: ${teardown}`);
+  lines.push(`  Billing: ${billing}`);
+  lines.push(`  Logs: badgr logs ${detail.job_id}`);
+  lines.push(`  Job ID: ${detail.job_id}`);
+  if (rcptId) lines.push(`  Receipt: badgr receipts ${rcptId}`);
+  if (detail.status === 'failed' && detail.next_action) lines.push(`  Next: ${detail.next_action}`);
+
+  return lines.join('\n') + '\n';
+}

package/src/store.js

@@ -95,3 +95,14 @@ export function updateReceipt(receiptId, updates, storeFile = STORE_FILE) {
 export function listReceipts(limit = 20, storeFile = STORE_FILE) {
   return loadStore(storeFile).receipts.slice(0, limit);
 }
+
+// Local receipt IDs (rcpt-...) are minted client-side by job-family commands
+// (badgr comfyui batch, badgr train lora, ...) and are never registered with
+// the backend — they exist only to point back at the job_id that made them.
+// `badgr receipts <id>` needs this lookup so it can fetch the *job's* status
+// instead of hitting the backend's unrelated /v1/receipts (LLM inference
+// receipt ledger), which has never heard of a locally-minted rcpt- id.
+export function findReceipt(idOrJobId, storeFile = STORE_FILE) {
+  const { receipts } = loadStore(storeFile);
+  return receipts.find(r => r.receiptId === idOrJobId || r.job_id === idOrJobId) ?? null;
+}

package/tests/detect.test.js

@@ -0,0 +1,191 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { detectWorkload, workloadTypeLabel } from '../src/detect.js';
+
+let tmpRoot;
+
+function makeProject(files) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'badgr-detect-'));
+  for (const [rel, content] of Object.entries(files)) {
+    const abs = path.join(dir, rel);
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, content);
+  }
+  return dir;
+}
+
+beforeEach(() => { tmpRoot = []; });
+afterEach(() => {
+  for (const dir of tmpRoot) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+function project(files) {
+  const dir = makeProject(files);
+  tmpRoot.push(dir);
+  return dir;
+}
+
+describe('detectWorkload — missing / empty directory', () => {
+  it('reports an error without throwing for a missing directory', () => {
+    const report = detectWorkload('/nonexistent/badgr-detect-fixture');
+    expect(report.error).toBeTruthy();
+    expect(report.confidence).toBe('low');
+  });
+
+  it('is low confidence with no command for an empty directory', () => {
+    const dir = project({});
+    const report = detectWorkload(dir);
+    expect(report.command).toBeNull();
+    expect(report.confidence).toBe('low');
+  });
+});
+
+describe('detectWorkload — plain Python CUDA script', () => {
+  it('finds the entry point and requirements, classifies as custom', () => {
+    const dir = project({
+      'train.py': `
+import torch
+model = torch.nn.Linear(10, 10).cuda()
+for step in range(100):
+    optimizer.step()
+`,
+      'requirements.txt': 'torch\nnumpy\n',
+      'output/.gitkeep': '',
+    });
+    const report = detectWorkload(dir);
+    expect(report.command).toBe('python train.py');
+    expect(report.requirements).toContain('requirements.txt');
+    expect(report.outputs).toContain('output/');
+    expect(report.workloadType).toBe('training');
+    expect(['medium', 'high']).toContain(report.confidence);
+  });
+});
+
+describe('detectWorkload — vLLM endpoint', () => {
+  it('classifies as endpoint and detects the port', () => {
+    const dir = project({
+      'server.py': `
+from vllm import LLM
+import uvicorn
+app = None
+
+if __name__ == "__main__":
+    uvicorn.run(app, port=8000)
+`,
+      'requirements.txt': 'vllm\nfastapi\nuvicorn\n',
+    });
+    const report = detectWorkload(dir);
+    expect(report.command).toBe('python server.py');
+    expect(report.workloadType).toBe('endpoint');
+    expect(report.ports).toContain(8000);
+    expect(report.healthPath).toBe('/health');
+  });
+});
+
+describe('detectWorkload — LoRA training repo', () => {
+  it('classifies as training and estimates high VRAM for a large model', () => {
+    const dir = project({
+      'train.py': `
+from peft import LoraConfig
+base_model = "meta-llama/Llama-3.1-70B-Instruct"
+trainer = Trainer(model=model)
+trainer.train()
+`,
+      'config.yaml': 'base_model: meta-llama/Llama-3.1-70B-Instruct\nlora_r: 16\n',
+      'requirements.txt': 'peft\ntransformers\naccelerate\n',
+      'checkpoints/.gitkeep': '',
+    });
+    const report = detectWorkload(dir);
+    expect(report.workloadType).toBe('training');
+    expect(report.checkpoints).toContain('checkpoints/');
+    expect(report.models.some(m => m.includes('Llama-3.1-70B'))).toBe(true);
+    expect(report.vram).toBe('80+ GB');
+  });
+});
+
+describe('detectWorkload — ComfyUI-style project', () => {
+  it('classifies as image_gen from a workflow json without special-casing ComfyUI by name', () => {
+    const dir = project({
+      'workflow_api.json': JSON.stringify({ '1': { class_type: 'CheckpointLoaderSimple' } }),
+      'requirements.txt': 'torch\npillow\n',
+    });
+    const report = detectWorkload(dir);
+    expect(report.workloadType).toBe('image_gen');
+    expect(workloadTypeLabel(report.workloadType)).toMatch(/image generation/);
+  });
+});
+
+describe('detectWorkload — custom Docker image', () => {
+  it('detects the Dockerfile and exposed port instead of guessing a base image', () => {
+    const dir = project({
+      Dockerfile: `
+FROM python:3.11-slim
+COPY . /app
+EXPOSE 7860
+CMD ["python", "app.py"]
+`,
+      'app.py': `
+import gradio as gr
+demo = gr.Interface()
+demo.launch(server_port=7860)
+`,
+    });
+    const report = detectWorkload(dir);
+    expect(report.dockerfile).toBe('Dockerfile');
+    expect(report.image).toBeNull();
+    expect(report.ports).toContain(7860);
+    expect(report.workloadType).toBe('endpoint');
+  });
+});
+
+describe('detectWorkload — RunPod-style serverless handler job', () => {
+  it('classifies a handler-based job as an endpoint without naming any provider', () => {
+    const dir = project({
+      'rp_handler.py': `
+import runpod
+
+def handler(event):
+    return {"output": "ok"}
+
+runpod.serverless.start({"handler": handler})
+`,
+      'requirements.txt': 'runpod\n',
+    });
+    const report = detectWorkload(dir);
+    expect(report.command).toBe('python rp_handler.py');
+    expect(report.workloadType).toBe('endpoint');
+    // No provider name should leak into any user-visible field.
+    const serialized = JSON.stringify(report).toLowerCase();
+    expect(serialized).not.toContain('runpod.com');
+  });
+});
+
+describe('detectWorkload — batch image generation over an input directory', () => {
+  it('classifies as batch when the script iterates an input directory', () => {
+    const dir = project({
+      'generate.py': `
+import os
+for fname in os.listdir("input"):
+    process(fname)
+`,
+      'input/.gitkeep': '',
+      'output/.gitkeep': '',
+    });
+    const report = detectWorkload(dir);
+    expect(report.workloadType).toBe('batch');
+    expect(report.inputs).toContain('input/');
+    expect(report.outputs).toContain('output/');
+  });
+});
+
+describe('workloadTypeLabel', () => {
+  it('never returns a provider-specific label', () => {
+    for (const t of ['endpoint', 'training', 'image_gen', 'batch', 'custom', 'unknown']) {
+      expect(workloadTypeLabel(t)).not.toMatch(/runpod|vast|hyperstack/i);
+    }
+  });
+});

package/tests/job-progress-poll.test.js

@@ -0,0 +1,136 @@
+import { describe, it, expect, vi } from 'vitest';
+import { pollJobUntilTerminal, renderJobClosingBlock } from '../src/progress.js';
+
+const chalk = new Proxy({}, { get: () => (s) => s });
+const config = { apiKey: 'k', baseUrl: 'https://api.test' };
+
+function silenceStdout() {
+  return vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+}
+
+describe('pollJobUntilTerminal', () => {
+  it('returns immediately once the job reaches a terminal status', async () => {
+    const writeSpy = silenceStdout();
+    const detail = { job_id: 'job_1', status: 'completed', stage_label: 'Complete', health: 'completed' };
+    const callApi = vi.fn().mockResolvedValue(detail);
+
+    const result = await pollJobUntilTerminal(callApi, config, 'job_1', { chalk, maxMs: 60_000, pollMs: 1 });
+
+    expect(result.outcome).toBe('completed');
+    expect(result.detail).toBe(detail);
+    writeSpy.mockRestore();
+  });
+
+  it('surfaces progress fields (stage/current/total) while polling', async () => {
+    const writeSpy = silenceStdout();
+    const callApi = vi.fn()
+      .mockResolvedValueOnce({ job_id: 'job_2', status: 'running', stage_label: 'Generating images', progress_current: 3, progress_total: 10, progress_unit: 'images', health: 'progressing' })
+      .mockResolvedValueOnce({ job_id: 'job_2', status: 'completed', stage_label: 'Complete', health: 'completed' });
+
+    const result = await pollJobUntilTerminal(callApi, config, 'job_2', { chalk, maxMs: 60_000, pollMs: 1 });
+
+    expect(result.outcome).toBe('completed');
+    const output = writeSpy.mock.calls.map(c => c[0]).join('');
+    expect(output).toContain('3/10 images');
+    writeSpy.mockRestore();
+  });
+
+  it('does not swallow repeated poll failures silently — gives up and reports polling_failed', async () => {
+    const writeSpy = silenceStdout();
+    const callApi = vi.fn().mockRejectedValue(new Error('network down'));
+
+    const result = await pollJobUntilTerminal(callApi, config, 'job_3', { chalk, maxMs: 60_000, pollMs: 1 });
+
+    expect(result.outcome).toBe('polling_failed');
+    // Must have warned the user at least once instead of looping silently.
+    const output = writeSpy.mock.calls.map(c => c[0]).join('');
+    expect(output).toMatch(/Lost contact/i);
+    writeSpy.mockRestore();
+  });
+
+  it('announces a retry-on-a-different-route once, with teardown/billing status', async () => {
+    const writeSpy = silenceStdout();
+    const callApi = vi.fn()
+      .mockResolvedValueOnce({
+        job_id: 'job_retry', status: 'running', stage: 'retrying_different_route',
+        stage_label: 'Retrying on a different route', health: 'progressing',
+        progress_message: 'Retrying on a different route... attempt 2/2',
+        teardown_status: 'ok',
+      })
+      .mockResolvedValueOnce({
+        job_id: 'job_retry', status: 'completed', stage_label: 'Complete', health: 'completed',
+      });
+
+    const result = await pollJobUntilTerminal(callApi, config, 'job_retry', { chalk, maxMs: 60_000, pollMs: 1 });
+
+    expect(result.outcome).toBe('completed');
+    const output = writeSpy.mock.calls.map(c => c[0]).join('');
+    expect(output).toContain('Retrying on a different route... attempt 2/2');
+    expect(output).toContain('Previous attempt teardown: succeeded');
+    expect(output).toContain('Billing: stopped');
+    writeSpy.mockRestore();
+  });
+
+  it('recovers from transient poll failures once the API responds again', async () => {
+    const writeSpy = silenceStdout();
+    const callApi = vi.fn()
+      .mockRejectedValueOnce(new Error('timeout'))
+      .mockRejectedValueOnce(new Error('timeout'))
+      .mockResolvedValueOnce({ job_id: 'job_4', status: 'completed', stage_label: 'Complete', health: 'completed' });
+
+    const result = await pollJobUntilTerminal(callApi, config, 'job_4', { chalk, maxMs: 60_000, pollMs: 1 });
+
+    expect(result.outcome).toBe('completed');
+    writeSpy.mockRestore();
+  });
+});
+
+describe('renderJobClosingBlock', () => {
+  it('shows teardown/billing/logs/receipt for a completed job', () => {
+    const block = renderJobClosingBlock(chalk, {
+      job_id: 'job_5', status: 'completed', elapsed_seconds: 252,
+      charged_usd: 0.08, teardown_status: 'ok', billing_status: 'stopped',
+    }, 'rcpt-abc');
+
+    expect(block).toContain('Complete');
+    expect(block).toContain('Teardown: succeeded');
+    expect(block).toContain('Billing: stopped');
+    expect(block).toContain('badgr logs job_5');
+    expect(block).toContain('badgr receipts rcpt-abc');
+  });
+
+  it('shows failure class, stage, and next action for a failed job', () => {
+    const block = renderJobClosingBlock(chalk, {
+      job_id: 'job_6', status: 'failed', elapsed_seconds: 1112,
+      error: { code: 'comfy_health_failed', message: 'ComfyUI did not start within 10 minutes.' },
+      failure_class: 'health_check_failed', stage: 'comfy_waiting_for_health',
+      teardown_status: 'ok', billing_status: 'stopped',
+      next_action: 'Run `badgr logs <job_id>` or retry with a longer max-runtime.',
+    });
+
+    expect(block).toContain('Failed');
+    expect(block).toContain('Class: health_check_failed');
+    expect(block).toContain('Stage: comfy_waiting_for_health');
+    expect(block).toContain('Next:');
+  });
+
+  it('labels a runtime/spend cap failure as "Stopped — limit reached", not a crash', () => {
+    const block = renderJobClosingBlock(chalk, {
+      job_id: 'job_7', status: 'failed', elapsed_seconds: 545,
+      error: { code: 'max_runtime_reached', message: 'Job reached max runtime and was stopped. Billing ended.' },
+      failure_class: 'runtime_cap_reached',
+      teardown_status: 'ok', billing_status: 'stopped',
+    });
+
+    expect(block).toContain('Stopped — limit reached');
+  });
+
+  it('shows cancelled block distinctly from failed', () => {
+    const block = renderJobClosingBlock(chalk, {
+      job_id: 'job_8', status: 'canceled', elapsed_seconds: 164,
+      teardown_status: 'ok', billing_status: 'stopped',
+    });
+
+    expect(block).toContain('Cancelled by user');
+  });
+});

package/tests/productized-runners.test.js

@@ -131,6 +131,13 @@ describe('BLESSED_COMFY_WORKFLOWS catalog', () => {
     expect(wf.gpu_type).toBe('RTX_4090');
     expect(wf.output_type).toBe('images');
   });
+
+  it('has flux-basic with gpu_type and output_type', () => {
+    const wf = BLESSED_COMFY_WORKFLOWS['flux-basic'];
+    expect(wf).toBeDefined();
+    expect(wf.gpu_type).toBe('RTX_4090');
+    expect(wf.output_type).toBe('images');
+  });
 });
 
 // ── parseServeArgs alias recognition ─────────────────────────────────────────

package/tests/run-lifecycle.test.js

@@ -15,7 +15,7 @@
  *   - Receipt explains state   → status, exitCode, failureType, runtimeSeconds, finalCost
  */
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { runCommand } from '../src/commands/run.js';
+import { runCommand, parseRunArgs } from '../src/commands/run.js';
 
 // ── Module mocks ──────────────────────────────────────────────────────────────
 
@@ -222,6 +222,26 @@ describe('container fails to start', () => {
 
     expect(process.exitCode).toBe(1);
   });
+
+  it('prints the backend failure_class/next_action using the same vocabulary as the job API', async () => {
+    api.callApi
+      .mockResolvedValueOnce(makeDep({ status: 'provisioning' }))
+      .mockResolvedValueOnce({
+        status: 'failed',
+        failure_class: 'container_start_failed',
+        next_action: 'Check the image and command, then retry.',
+      });
+    const logs = [];
+    const origError = console.error;
+    console.error = (...args) => { logs.push(args.join(' ')); origError(...args); };
+    const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
+    await vi.advanceTimersByTimeAsync(4000);
+    await p;
+    console.error = origError;
+
+    expect(logs.some(l => l.includes('Class: container_start_failed'))).toBe(true);
+    expect(logs.some(l => l.includes('Next: Check the image and command, then retry.'))).toBe(true);
+  });
 });
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -543,3 +563,93 @@ describe('--dry-run', () => {
     expect(api.callApi).not.toHaveBeenCalled();
   });
 });
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 13. Progress-loss handling — --output / --checkpoint / --retry-safe / --resume-cmd
+// ─────────────────────────────────────────────────────────────────────────────
+
+describe('--output / --checkpoint / --retry-safe / --resume-cmd', () => {
+  it('parseRunArgs parses all four flags', () => {
+    const { flags } = parseRunArgs([
+      '--output', './outputs', '--checkpoint', './checkpoints', '--retry-safe',
+      '--resume-cmd', 'python train.py --resume ./checkpoints/latest',
+      '--max-cost', '5', '--', 'python', 'train.py',
+    ]);
+    expect(flags.output).toBe('./outputs');
+    expect(flags.checkpoint).toBe('./checkpoints');
+    expect(flags.retrySafe).toBe(true);
+    expect(flags.resumeCmd).toBe('python train.py --resume ./checkpoints/latest');
+  });
+
+  it('wires --output/--checkpoint/--retry-safe into the job env as BADGR_* vars', async () => {
+    api.callApi
+      .mockResolvedValueOnce(makeDep())
+      .mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
+      .mockResolvedValueOnce({ logs: [] });
+    const p = runCommand(config, [
+      '--output', './outputs', '--checkpoint', './checkpoints', '--retry-safe',
+      '--max-cost', '5', '--', 'python', 'train.py',
+    ], chalk);
+    await vi.advanceTimersByTimeAsync(5000);
+    await p;
+
+    const postRunBody = api.callApi.mock.calls[0][1].body;
+    expect(postRunBody.env).toEqual(expect.objectContaining({
+      BADGR_OUTPUT_DIR: './outputs',
+      BADGR_CHECKPOINT_DIR: './checkpoints',
+      BADGR_RETRY_SAFE: '1',
+    }));
+  });
+
+  it('explicit --env overrides the BADGR_* convention var for the same key', async () => {
+    api.callApi
+      .mockResolvedValueOnce(makeDep())
+      .mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
+      .mockResolvedValueOnce({ logs: [] });
+    const p = runCommand(config, [
+      '--output', './outputs', '--env', 'BADGR_OUTPUT_DIR=./custom-outputs',
+      '--max-cost', '5', '--', 'python', 'train.py',
+    ], chalk);
+    await vi.advanceTimersByTimeAsync(5000);
+    await p;
+
+    const postRunBody = api.callApi.mock.calls[0][1].body;
+    expect(postRunBody.env.BADGR_OUTPUT_DIR).toBe('./custom-outputs');
+  });
+
+  it('prints the --resume-cmd hint when the job fails', async () => {
+    api.callApi
+      .mockResolvedValueOnce(makeDep())
+      .mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
+      .mockResolvedValueOnce({ logs: [] });
+    const logs = [];
+    const captureChalk = { bold: s => s, dim: s => s, red: s => s, yellow: s => s, green: s => s, cyan: s => s };
+    const origLog = console.log;
+    console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
+    const p = runCommand(config, [
+      '--resume-cmd', 'python train.py --resume ./checkpoints/latest',
+      '--max-cost', '5', '--', 'python', 'train.py',
+    ], captureChalk);
+    await vi.advanceTimersByTimeAsync(5000);
+    await p;
+    console.log = origLog;
+
+    expect(logs.some(l => l.includes('python train.py --resume ./checkpoints/latest'))).toBe(true);
+  });
+
+  it('does not print a Resume line when --resume-cmd was not given', async () => {
+    api.callApi
+      .mockResolvedValueOnce(makeDep())
+      .mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
+      .mockResolvedValueOnce({ logs: [] });
+    const logs = [];
+    const origLog = console.log;
+    console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
+    const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
+    await vi.advanceTimersByTimeAsync(5000);
+    await p;
+    console.log = origLog;
+
+    expect(logs.some(l => l.includes('Resume:'))).toBe(false);
+  });
+});