badgr-cli 1.1.1 → 1.1.2

package/tests/batch.test.js

@@ -1,641 +0,0 @@
-/**
- * badgr batch — /run body shape, missing-artifact-path detection, and
- * `badgr batch compare` regression logic against literal metrics.json
- * fixtures (satisfies test scenario 5 without needing a live run).
- */
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'fs';
-import { tmpdir } from 'os';
-import { join } from 'path';
-
-vi.mock('../src/store.js', () => ({
-  selectedComputeFromDeployment: (dep) => ({
-    gpu: dep.gpu_type ?? null,
-    gpuCount: dep.gpu_count ?? null,
-    vcpus: dep.selected_vcpus ?? null,
-    ramGb: dep.selected_ram_gb ?? null,
-    vramGb: dep.selected_vram_gb ?? null,
-  }),
-  addReceipt: vi.fn(),
-  updateReceipt: vi.fn(),
-  generateReceiptId: vi.fn(() => 'rcpt-batch-001'),
-  loadStore: vi.fn(() => ({ receipts: [], deployments: [] })),
-}));
-
-vi.mock('../src/api.js', () => ({
-  callApi: vi.fn(),
-  uploadBlob: vi.fn(),
-}));
-
-vi.mock('../src/fallback.js', async (importOriginal) => {
-  const actual = await importOriginal();
-  return { ...actual, callWithFallback: vi.fn() };
-});
-
-vi.mock('../src/batch.js', () => ({
-  monitorBatchJob: vi.fn(() => Promise.resolve({ status: 'succeeded', exitCode: 0, runtimeMs: 42000, reason: 'complete' })),
-  fmtRuntime: (ms) => `${Math.round(ms / 1000)}s`,
-}));
-
-vi.mock('archiver', () => ({
-  default: () => {
-    let out;
-    return {
-      on: () => {},
-      pipe: (o) => { out = o; },
-      directory: () => {},
-      file: () => {},
-      finalize: () => { out.end(); },
-    };
-  },
-}));
-
-import {
-  batchCommand,
-  displayFailureReason,
-  missingOutputPaths,
-  readMetricFile,
-  nextArchiveTmpFile,
-} from '../src/commands/batch.js';
-import * as store from '../src/store.js';
-import * as api from '../src/api.js';
-import * as fallback from '../src/fallback.js';
-
-const chalk = {
-  bold: s => s, dim: s => s, red: s => s, yellow: s => s, green: s => s, cyan: s => s,
-};
-
-const config = { apiKey: 'sk-test', baseUrl: 'https://api.test/v1' };
-
-let dir;
-beforeEach(() => {
-  dir = mkdtempSync(join(tmpdir(), 'badgr-batch-cmd-test-'));
-  process.exitCode = undefined;
-  vi.spyOn(console, 'log').mockImplementation(() => {});
-  vi.spyOn(console, 'error').mockImplementation(() => {});
-  vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
-  vi.clearAllMocks();
-  store.loadStore.mockReturnValue({ receipts: [], deployments: [] });
-});
-
-afterEach(() => {
-  rmSync(dir, { recursive: true, force: true });
-  vi.restoreAllMocks();
-});
-
-// ---------------------------------------------------------------------------
-// displayFailureReason
-// ---------------------------------------------------------------------------
-
-describe('displayFailureReason', () => {
-  it('maps max_runtime to runtime_limit_exceeded', () => {
-    expect(displayFailureReason('max_runtime')).toBe('runtime_limit_exceeded');
-  });
-  it('maps max_cost to cost_cap_exceeded', () => {
-    expect(displayFailureReason('max_cost')).toBe('cost_cap_exceeded');
-  });
-  it('passes through unknown reasons unchanged', () => {
-    expect(displayFailureReason('something_else')).toBe('something_else');
-  });
-  it('returns null for no reason', () => {
-    expect(displayFailureReason(null)).toBeNull();
-    expect(displayFailureReason(undefined)).toBeNull();
-  });
-});
-
-// ---------------------------------------------------------------------------
-// missingOutputPaths / readMetricFile
-// ---------------------------------------------------------------------------
-
-describe('nextArchiveTmpFile', () => {
-  it('never returns the same path twice, even called back-to-back in the same millisecond', () => {
-    // Regression test: the original implementation used only Date.now() for
-    // the temp filename, which collides when --fan-out builds multiple
-    // archives in the same millisecond (routine once concurrency > 1),
-    // corrupting one task's archive out from under another and silently
-    // dropping tasks. A monotonic counter fixes it — assert directly rather
-    // than through the full CLI/filesystem, since the guarantee is about the
-    // string, not the I/O around it.
-    const paths = new Set();
-    for (let i = 0; i < 500; i++) paths.add(nextArchiveTmpFile());
-    expect(paths.size).toBe(500);
-  });
-
-  it('always includes the badgr-batch-inputs prefix and .tar.gz suffix', () => {
-    const p = nextArchiveTmpFile();
-    expect(p).toMatch(/badgr-batch-inputs-\d+-\d+\.tar\.gz$/);
-  });
-});
-
-describe('missingOutputPaths', () => {
-  it('reports declared paths that do not exist under the extraction dir', () => {
-    mkdirSync(join(dir, 'outputs'), { recursive: true });
-    writeFileSync(join(dir, 'outputs', 'metrics.json'), '{}');
-    const missing = missingOutputPaths(dir, ['/outputs/metrics.json', '/outputs/videos', '/outputs/logs']);
-    expect(missing).toEqual(['/outputs/videos', '/outputs/logs']);
-  });
-
-  it('reports nothing missing when all declared paths exist', () => {
-    mkdirSync(join(dir, 'outputs', 'videos'), { recursive: true });
-    writeFileSync(join(dir, 'outputs', 'metrics.json'), '{}');
-    const missing = missingOutputPaths(dir, ['/outputs/metrics.json', '/outputs/videos']);
-    expect(missing).toEqual([]);
-  });
-});
-
-describe('readMetricFile', () => {
-  it('reads and parses a metrics JSON file', () => {
-    mkdirSync(join(dir, 'outputs'), { recursive: true });
-    writeFileSync(join(dir, 'outputs', 'metrics.json'), JSON.stringify({ pass_rate: 0.9 }));
-    expect(readMetricFile(dir, '/outputs/metrics.json')).toEqual({ pass_rate: 0.9 });
-  });
-
-  it('returns null when the file does not exist', () => {
-    expect(readMetricFile(dir, '/outputs/metrics.json')).toBeNull();
-  });
-
-  it('returns null on malformed JSON', () => {
-    mkdirSync(join(dir, 'outputs'), { recursive: true });
-    writeFileSync(join(dir, 'outputs', 'metrics.json'), 'not json');
-    expect(readMetricFile(dir, '/outputs/metrics.json')).toBeNull();
-  });
-});
-
-// ---------------------------------------------------------------------------
-// badgr batch run — /run body shape (image/command/env/caps)
-// ---------------------------------------------------------------------------
-
-describe('badgr batch run — /run request shape', () => {
-  it('builds the /run body from workload.yml with inputs/outputs env wiring', async () => {
-    writeFileSync(join(dir, 'policy.py'), 'def act(obs): return 0');
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    const yamlPath = join(dir, 'workload.yml');
-    writeFileSync(yamlPath, `
-name: toy-robot-eval
-image: badgr/physical-ai-eval-lite:latest
-command: [python, run_eval.py, --policy, /inputs/policy.py, --scenarios, /inputs/scenarios]
-inputs:
-  - ./policy.py:/inputs/policy.py
-  - ./scenarios:/inputs/scenarios
-outputs:
-  - /outputs/metrics.json
-  - /outputs/videos
-  - /outputs/logs
-max_cost: 20
-max_runtime_minutes: 60
-success_metric:
-  file: /outputs/metrics.json
-  key: pass_rate
-  higher_is_better: true
-`);
-
-    let capturedBody;
-    fallback.callWithFallback.mockImplementation((endpoint, callOpts, buildBody) => {
-      capturedBody = buildBody();
-      return Promise.resolve({
-        deployment_id: 'dep-abc123', receipt_id: 'rcpt-1', gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.5, provider: 'runpod', tier: '1', status: 'running',
-      });
-    });
-
-    api.callApi.mockImplementation((path) => {
-      if (path.endsWith('/artifacts')) return Promise.reject(new Error('no artifacts yet'));
-      if (path.startsWith('/deployments/')) {
-        return Promise.resolve({
-          status: 'succeeded', failure_reason: null, teardown_ok: 'ok',
-          runtime_seconds: 42, accrued_cost_usd: 0.01, provider: 'runpod',
-        });
-      }
-      return Promise.resolve({});
-    });
-
-    api.uploadBlob.mockResolvedValue({ code_uri: 'https://cdn.example.com/inputs.tar.gz', upload_id: 'up_1' });
-
-    await batchCommand(config, ['run', yamlPath], chalk);
-
-    expect(capturedBody.image).toBe('badgr/physical-ai-eval-lite:latest');
-    expect(capturedBody.command).toEqual(['python', 'run_eval.py', '--policy', '/inputs/policy.py', '--scenarios', '/inputs/scenarios']);
-    expect(capturedBody.max_cost_usd).toBe(20);
-    expect(capturedBody.max_runtime_seconds).toBe(3600);
-    expect(capturedBody.custom_image).toBe(true);
-    expect(capturedBody.env.BADGR_INPUTS_ARCHIVE_URL).toBe('https://cdn.example.com/inputs.tar.gz');
-    expect(JSON.parse(capturedBody.env.BADGR_OUTPUT_PATHS_JSON)).toEqual([
-      '/outputs/metrics.json', '/outputs/videos', '/outputs/logs',
-    ]);
-    expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({ workloadShape: 'container-batch' }));
-  });
-
-  it('rejects a workload.yml missing required fields before touching the network', async () => {
-    const yamlPath = join(dir, 'bad.yml');
-    writeFileSync(yamlPath, 'image: busybox\n');
-    await batchCommand(config, ['run', yamlPath], chalk);
-    expect(process.exitCode).toBe(1);
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-  });
-
-  it('--dry-run never submits — regression: this flag was previously silently ignored, submitting real GPU jobs anyway', async () => {
-    // Found live: `badgr batch run workload.yml --dry-run` (and the
-    // --fan-out form below) provisioned real GPU deployments despite
-    // --dry-run, because the flag was never parsed or checked anywhere in
-    // this file — every real submission this test guards against actually
-    // happened once, on real infrastructure, before this fix.
-    const yamlPath = join(dir, 'workload.yml');
-    writeFileSync(yamlPath, 'name: t\nimage: busybox\ncommand: [echo, hi]\nmax_cost: 1\nmax_runtime_minutes: 5\n');
-
-    await batchCommand(config, ['run', yamlPath, '--dry-run'], chalk);
-
-    expect(process.exitCode).toBeUndefined();
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-    expect(api.uploadBlob).not.toHaveBeenCalled();
-    expect(store.addReceipt).not.toHaveBeenCalled();
-  });
-
-  it('builds a body with no BADGR_* env vars when inputs/outputs are omitted', async () => {
-    const yamlPath = join(dir, 'workload.yml');
-    writeFileSync(yamlPath, `
-name: simple-job
-image: busybox
-command: [echo, hi]
-max_cost: 1
-max_runtime_minutes: 5
-`);
-
-    let capturedBody;
-    fallback.callWithFallback.mockImplementation((endpoint, callOpts, buildBody) => {
-      capturedBody = buildBody();
-      return Promise.resolve({
-        deployment_id: 'dep-xyz', receipt_id: 'rcpt-2', gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.1, provider: 'runpod', tier: '1', status: 'running',
-      });
-    });
-    api.callApi.mockImplementation((path) => {
-      if (path.startsWith('/deployments/')) {
-        return Promise.resolve({ status: 'succeeded', teardown_ok: 'ok' });
-      }
-      return Promise.resolve({});
-    });
-
-    await batchCommand(config, ['run', yamlPath], chalk);
-
-    expect(capturedBody.env.BADGR_INPUTS_ARCHIVE_URL).toBeUndefined();
-    expect(capturedBody.env.BADGR_OUTPUT_PATHS_JSON).toBeUndefined();
-  });
-});
-
-// ---------------------------------------------------------------------------
-// badgr batch run --fan-out <dir> — "run one program across many inputs"
-// ---------------------------------------------------------------------------
-
-describe('badgr batch run --fan-out', () => {
-  function writeFanOutYaml(dirPath, extra = '') {
-    const yamlPath = join(dirPath, 'workload.yml');
-    writeFileSync(yamlPath, `
-name: screen
-image: badgr/screen:latest
-command: [python, screen.py, --input, /inputs/scenario.json]
-inputs:
-  - ./scenarios/PLACEHOLDER.json:/inputs/scenario.json
-outputs:
-  - /outputs/result.json
-max_cost: 1
-max_runtime_minutes: 10
-${extra}
-`);
-    return yamlPath;
-  }
-
-  it('submits one deployment per file in the fan-out directory, in parallel', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 's1.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 'PLACEHOLDER.json'), '{}'); // satisfies inputs: parse; overridden per task
-    const yamlPath = writeFanOutYaml(dir);
-
-    let callCount = 0;
-    const capturedBodies = [];
-    fallback.callWithFallback.mockImplementation((endpoint, callOpts, buildBody) => {
-      callCount += 1;
-      capturedBodies.push(buildBody());
-      return Promise.resolve({
-        deployment_id: `dep-${callCount}`, receipt_id: `rcpt-${callCount}`, gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.2, provider: 'runpod', tier: '1', status: 'running',
-      });
-    });
-    api.callApi.mockImplementation((path) => {
-      if (path.startsWith('/deployments/')) {
-        return Promise.resolve({ status: 'succeeded', teardown_ok: 'ok', runtime_seconds: 3, accrued_cost_usd: 0.01 });
-      }
-      return Promise.resolve({});
-    });
-    api.uploadBlob.mockResolvedValue({ code_uri: 'https://cdn.example.com/inputs.tar.gz', upload_id: 'up_1' });
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios')], chalk);
-
-    // 3 files in scenarios/ (PLACEHOLDER.json, s0.json, s1.json), one deployment each.
-    expect(callCount).toBe(3);
-    expect(store.addReceipt).toHaveBeenCalledTimes(3);
-    const names = capturedBodies.map(b => b.name);
-    expect(names.every(n => n.startsWith('screen-'))).toBe(true);
-    expect(new Set(names).size).toBe(3); // distinct per-file names
-    expect(capturedBodies[0].env.BADGR_FANOUT_INPUT).toBeDefined();
-    expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({ workloadShape: 'fan-out' }));
-    expect(process.exitCode).toBeUndefined();
-  });
-
-  it('rejects --fan-out when workload.yml declares more than one input', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    const yamlPath = join(dir, 'workload.yml');
-    writeFileSync(yamlPath, `
-name: multi-input
-image: busybox
-command: [echo, hi]
-inputs:
-  - ./a.txt:/inputs/a.txt
-  - ./b.txt:/inputs/b.txt
-max_cost: 1
-max_runtime_minutes: 5
-`);
-    writeFileSync(join(dir, 'a.txt'), 'x');
-    writeFileSync(join(dir, 'b.txt'), 'x');
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios')], chalk);
-    expect(process.exitCode).toBe(1);
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-  });
-
-  it('rejects a --fan-out path that does not exist', async () => {
-    const yamlPath = writeFanOutYaml(dir);
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'does-not-exist')], chalk);
-    expect(process.exitCode).toBe(1);
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-  });
-
-  it('--dry-run with --fan-out never submits any task — same regression as the non-fan-out case', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 's1.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 'PLACEHOLDER.json'), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios'), '--dry-run'], chalk);
-
-    expect(process.exitCode).toBeUndefined();
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-    expect(api.uploadBlob).not.toHaveBeenCalled();
-    expect(store.addReceipt).not.toHaveBeenCalled();
-  });
-
-  it('rejects an empty --fan-out directory', async () => {
-    mkdirSync(join(dir, 'empty'));
-    const yamlPath = writeFanOutYaml(dir);
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'empty')], chalk);
-    expect(process.exitCode).toBe(1);
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-  });
-
-  it('reports a non-zero exit code when at least one fan-out task fails', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 's1.json'), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    const { monitorBatchJob } = await import('../src/batch.js');
-    let monitorCall = 0;
-    monitorBatchJob.mockImplementation(() => {
-      monitorCall += 1;
-      return Promise.resolve(
-        monitorCall === 1
-          ? { status: 'failed', exitCode: 1, runtimeMs: 100, reason: 'infrastructure' }
-          : { status: 'succeeded', exitCode: 0, runtimeMs: 100, reason: 'complete' },
-      );
-    });
-
-    let depCall = 0;
-    fallback.callWithFallback.mockImplementation((endpoint, callOpts, buildBody) => {
-      depCall += 1;
-      return Promise.resolve({
-        deployment_id: `dep-${depCall}`, receipt_id: `rcpt-${depCall}`, gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.2, provider: 'runpod', tier: '1', status: 'running',
-      });
-    });
-    api.callApi.mockImplementation((path) => {
-      if (path.startsWith('/deployments/dep-1')) return Promise.resolve({ status: 'failed', teardown_ok: 'ok' });
-      if (path.startsWith('/deployments/')) return Promise.resolve({ status: 'succeeded', teardown_ok: 'ok' });
-      return Promise.resolve({});
-    });
-    api.uploadBlob.mockResolvedValue({ code_uri: 'https://cdn.example.com/inputs.tar.gz', upload_id: 'up_1' });
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios')], chalk);
-    expect(process.exitCode).toBe(1);
-  });
-
-  it('prints a ready-to-copy --only rerun command listing the failed inputs', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 's1.json'), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    const { monitorBatchJob } = await import('../src/batch.js');
-    monitorBatchJob
-      .mockResolvedValueOnce({ status: 'failed', exitCode: 1, runtimeMs: 100, reason: 'infrastructure' })
-      .mockResolvedValueOnce({ status: 'succeeded', exitCode: 0, runtimeMs: 100, reason: 'complete' });
-    let depCall = 0;
-    fallback.callWithFallback.mockImplementation(() => {
-      depCall += 1;
-      return Promise.resolve({
-        deployment_id: `dep-${depCall}`, receipt_id: `rcpt-${depCall}`, gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.2, provider: 'runpod', tier: '1', status: 'running',
-      });
-    });
-    api.callApi.mockImplementation((path) => {
-      if (path.startsWith('/deployments/dep-1')) return Promise.resolve({ status: 'failed', teardown_ok: 'ok' });
-      if (path.startsWith('/deployments/')) return Promise.resolve({ status: 'succeeded', teardown_ok: 'ok' });
-      return Promise.resolve({});
-    });
-    api.uploadBlob.mockResolvedValue({ code_uri: 'https://cdn.example.com/inputs.tar.gz', upload_id: 'up_1' });
-
-    const logs = [];
-    const origLog = console.log;
-    console.log = (...args) => { logs.push(args.join(' ')); };
-    // --max-concurrency 1 makes submit/monitor order deterministic (s0 fully
-    // completes before s1 starts), so the first mocked monitor result (failed)
-    // is guaranteed to land on s0 rather than depending on scheduling order.
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios'), '--max-concurrency', '1'], chalk);
-    console.log = origLog;
-
-    expect(process.exitCode).toBe(1);
-    expect(logs.some(l => l.includes('--only s0.json'))).toBe(true);
-  });
-
-  it('--only restricts the fan-out to the named files', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 's1.json'), '{}');
-    writeFileSync(join(dir, 'scenarios', 's2.json'), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    let callCount = 0;
-    fallback.callWithFallback.mockImplementation(() => {
-      callCount += 1;
-      return Promise.resolve({
-        deployment_id: `dep-${callCount}`, receipt_id: `rcpt-${callCount}`, gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.2, provider: 'runpod', tier: '1', status: 'running',
-      });
-    });
-    api.callApi.mockResolvedValue({ status: 'succeeded', teardown_ok: 'ok', runtime_seconds: 1, accrued_cost_usd: 0.01 });
-    api.uploadBlob.mockResolvedValue({ code_uri: 'https://cdn.example.com/inputs.tar.gz', upload_id: 'up_1' });
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios'), '--only', 's0.json,s2.json'], chalk);
-
-    expect(callCount).toBe(2); // not 3 — s1.json excluded
-    expect(process.exitCode).toBeUndefined();
-  });
-
-  it('--only rejects a filename that does not exist in the fan-out directory', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios'), '--only', 'does-not-exist.json'], chalk);
-
-    expect(process.exitCode).toBe(1);
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-  });
-
-  it('rejects a non-positive --max-concurrency before touching the network', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    writeFileSync(join(dir, 'scenarios', 's0.json'), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios'), '--max-concurrency', '0'], chalk);
-
-    expect(process.exitCode).toBe(1);
-    expect(fallback.callWithFallback).not.toHaveBeenCalled();
-  });
-
-  it('never exceeds --max-concurrency in-flight submissions', async () => {
-    mkdirSync(join(dir, 'scenarios'));
-    for (let i = 0; i < 6; i++) writeFileSync(join(dir, 'scenarios', `s${i}.json`), '{}');
-    const yamlPath = writeFanOutYaml(dir);
-
-    let inFlight = 0;
-    let maxInFlight = 0;
-    let depCall = 0;
-    fallback.callWithFallback.mockImplementation(async () => {
-      depCall += 1;
-      inFlight++;
-      maxInFlight = Math.max(maxInFlight, inFlight);
-      await new Promise(r => setTimeout(r, 5));
-      return {
-        deployment_id: `dep-${depCall}`, receipt_id: `rcpt-${depCall}`, gpu_type: 'RTX_4090',
-        gpu_count: 1, cost_per_hour: 0.2, provider: 'runpod', tier: '1', status: 'running',
-      };
-    });
-    const { monitorBatchJob } = await import('../src/batch.js');
-    monitorBatchJob.mockImplementation(async () => {
-      await new Promise(r => setTimeout(r, 5));
-      inFlight--;
-      return { status: 'succeeded', exitCode: 0, runtimeMs: 5, reason: 'complete' };
-    });
-    api.callApi.mockResolvedValue({ status: 'succeeded', teardown_ok: 'ok', runtime_seconds: 1, accrued_cost_usd: 0.01 });
-    api.uploadBlob.mockResolvedValue({ code_uri: 'https://cdn.example.com/inputs.tar.gz', upload_id: 'up_1' });
-
-    await batchCommand(config, ['run', yamlPath, '--fan-out', join(dir, 'scenarios'), '--max-concurrency', '2'], chalk);
-
-    expect(maxInFlight).toBeLessThanOrEqual(2);
-    expect(depCall).toBe(6);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// badgr batch artifacts — clear, actionable failure messages
-// ---------------------------------------------------------------------------
-
-describe('badgr batch artifacts — error clarity', () => {
-  it('explains a 404 (no artifact ever uploaded) with likely causes and a next step', async () => {
-    const err = new Error('GET /deployments/dep-nofiles/artifacts → HTTP 404: artifact_not_found');
-    err.httpStatus = 404;
-    api.callApi.mockImplementation((path) => {
-      if (path.endsWith('/artifacts')) return Promise.reject(err);
-      return Promise.resolve({});
-    });
-
-    await batchCommand(config, ['artifacts', 'dep-nofiles'], chalk);
-
-    expect(process.exitCode).toBe(1);
-    const logged = console.error.mock.calls.flat().join('\n');
-    expect(logged).toMatch(/No artifact was ever uploaded for run dep-nofiles/);
-    expect(logged).toMatch(/badgr batch logs dep-nofiles/);
-  });
-
-  it('explains a 410 (expired, 48h retention) plainly', async () => {
-    const err = new Error('GET /deployments/dep-old/artifacts → HTTP 410: artifact_expired');
-    err.httpStatus = 410;
-    api.callApi.mockImplementation((path) => {
-      if (path.endsWith('/artifacts')) return Promise.reject(err);
-      return Promise.resolve({});
-    });
-
-    await batchCommand(config, ['artifacts', 'dep-old'], chalk);
-
-    expect(process.exitCode).toBe(1);
-    const logged = console.error.mock.calls.flat().join('\n');
-    expect(logged).toMatch(/expired \(48h retention\)/);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// badgr batch compare — direct metrics.json files (test scenario 5)
-// ---------------------------------------------------------------------------
-
-describe('badgr batch compare', () => {
-  it('reports a regression with delta when run B is worse (higher_is_better)', async () => {
-    const a = join(dir, 'a.json');
-    const b = join(dir, 'b.json');
-    writeFileSync(a, JSON.stringify({ pass_rate: 0.92 }));
-    writeFileSync(b, JSON.stringify({ pass_rate: 0.87 }));
-
-    await batchCommand(config, ['compare', a, b], chalk);
-
-    expect(process.exitCode).toBe(1);
-    const logged = console.log.mock.calls.flat().join('\n');
-    expect(logged).toMatch(/Regression/);
-    expect(logged).toContain('0.92');
-    expect(logged).toContain('0.87');
-  });
-
-  it('reports an improvement when run B is better', async () => {
-    const a = join(dir, 'a.json');
-    const b = join(dir, 'b.json');
-    writeFileSync(a, JSON.stringify({ pass_rate: 0.80 }));
-    writeFileSync(b, JSON.stringify({ pass_rate: 0.95 }));
-
-    await batchCommand(config, ['compare', a, b], chalk);
-
-    expect(process.exitCode).toBeUndefined();
-    const logged = console.log.mock.calls.flat().join('\n');
-    expect(logged).toMatch(/Improvement/);
-  });
-
-  it('respects --key and --higher-is-better overrides for raw files', async () => {
-    const a = join(dir, 'a.json');
-    const b = join(dir, 'b.json');
-    writeFileSync(a, JSON.stringify({ error_rate: 0.05 }));
-    writeFileSync(b, JSON.stringify({ error_rate: 0.10 }));
-
-    // error_rate: lower is better, so b (higher) is a regression.
-    await batchCommand(config, ['compare', a, b, '--key', 'error_rate', '--higher-is-better', 'false'], chalk);
-
-    expect(process.exitCode).toBe(1);
-    const logged = console.log.mock.calls.flat().join('\n');
-    expect(logged).toMatch(/Regression/);
-  });
-
-  it('errors out when a run_id has no known success_metric and no local YAML', async () => {
-    store.loadStore.mockReturnValueOnce({ receipts: [], deployments: [] });
-    await batchCommand(config, ['compare', 'dep-unknown-a', 'dep-unknown-b'], chalk);
-    expect(process.exitCode).toBe(1);
-  });
-});