badgr-cli 1.0.46 → 1.0.47

package/tests/workload-spec.test.js

@@ -0,0 +1,180 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, writeFileSync, rmSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import {
+  parseWorkloadYaml,
+  parseInputEntry,
+  validateWorkloadSpec,
+  WorkloadSpecError,
+} from '../src/workloadSpec.js';
+
+let dir;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'badgr-batch-test-'));
+});
+
+afterEach(() => {
+  rmSync(dir, { recursive: true, force: true });
+});
+
+function writeYaml(name, content) {
+  const path = join(dir, name);
+  writeFileSync(path, content);
+  return path;
+}
+
+describe('parseInputEntry', () => {
+  it('splits local:container on the last colon', () => {
+    const { localPath, containerPath } = parseInputEntry('./policy.py:/inputs/policy.py');
+    expect(localPath).toBe('./policy.py');
+    expect(containerPath).toBe('/inputs/policy.py');
+  });
+
+  it('rejects a non-absolute container path', () => {
+    expect(() => parseInputEntry('./policy.py:inputs/policy.py')).toThrow(WorkloadSpecError);
+  });
+
+  it('rejects an entry with no colon', () => {
+    expect(() => parseInputEntry('./policy.py')).toThrow(WorkloadSpecError);
+  });
+});
+
+describe('validateWorkloadSpec', () => {
+  it('requires name, image, command, max_cost, max_runtime_minutes', () => {
+    const errors = validateWorkloadSpec({});
+    expect(errors).toContain('name is required');
+    expect(errors).toContain('image is required');
+    expect(errors).toContain('command is required and must be a non-empty list');
+    expect(errors).toContain('max_cost is required and must be a positive number');
+    expect(errors).toContain('max_runtime_minutes is required and must be a positive number');
+  });
+
+  it('passes on a minimal valid spec', () => {
+    const errors = validateWorkloadSpec({
+      name: 'toy', image: 'busybox', command: ['echo', 'hi'],
+      max_cost: 5, max_runtime_minutes: 10,
+    });
+    expect(errors).toEqual([]);
+  });
+
+  it('flags a malformed inputs entry', () => {
+    const errors = validateWorkloadSpec({
+      name: 'toy', image: 'busybox', command: ['echo'],
+      max_cost: 5, max_runtime_minutes: 10,
+      inputs: ['./policy.py'],
+    });
+    expect(errors.some(e => e.includes('inputs entry'))).toBe(true);
+  });
+
+  it('flags outputs that are not absolute paths', () => {
+    const errors = validateWorkloadSpec({
+      name: 'toy', image: 'busybox', command: ['echo'],
+      max_cost: 5, max_runtime_minutes: 10,
+      outputs: ['outputs/metrics.json'],
+    });
+    expect(errors).toContain('outputs must be a list of absolute container paths');
+  });
+
+  it('requires success_metric.file and key when success_metric is set', () => {
+    const errors = validateWorkloadSpec({
+      name: 'toy', image: 'busybox', command: ['echo'],
+      max_cost: 5, max_runtime_minutes: 10,
+      success_metric: {},
+    });
+    expect(errors.some(e => e.includes('success_metric.file'))).toBe(true);
+    expect(errors.some(e => e.includes('success_metric.key'))).toBe(true);
+  });
+});
+
+describe('parseWorkloadYaml', () => {
+  it('parses the minimal toy-robot-eval example from the spec', () => {
+    const path = writeYaml('workload.yml', `
+name: toy-robot-eval
+image: badgr/physical-ai-eval-lite:latest
+
+command:
+  - python
+  - run_eval.py
+  - --policy
+  - /inputs/policy.py
+  - --scenarios
+  - /inputs/scenarios
+
+inputs:
+  - ./policy.py:/inputs/policy.py
+  - ./scenarios:/inputs/scenarios
+
+outputs:
+  - /outputs/metrics.json
+  - /outputs/videos
+  - /outputs/logs
+
+max_cost: 20
+max_runtime_minutes: 60
+
+success_metric:
+  file: /outputs/metrics.json
+  key: pass_rate
+  higher_is_better: true
+`);
+    const spec = parseWorkloadYaml(path);
+    expect(spec.name).toBe('toy-robot-eval');
+    expect(spec.image).toBe('badgr/physical-ai-eval-lite:latest');
+    expect(spec.command).toEqual(['python', 'run_eval.py', '--policy', '/inputs/policy.py', '--scenarios', '/inputs/scenarios']);
+    expect(spec.inputs).toHaveLength(2);
+    expect(spec.inputs[0].containerPath).toBe('/inputs/policy.py');
+    expect(spec.inputs[0].localPath).toBe(join(dir, 'policy.py'));
+    expect(spec.outputs).toEqual(['/outputs/metrics.json', '/outputs/videos', '/outputs/logs']);
+    expect(spec.maxCost).toBe(20);
+    expect(spec.maxRuntimeMinutes).toBe(60);
+    expect(spec.successMetric).toEqual({ file: '/outputs/metrics.json', key: 'pass_rate', higherIsBetter: true });
+  });
+
+  it('defaults higherIsBetter to true when omitted', () => {
+    const path = writeYaml('workload.yml', `
+name: toy
+image: busybox
+command: [echo, hi]
+max_cost: 1
+max_runtime_minutes: 1
+success_metric:
+  file: /outputs/metrics.json
+  key: score
+`);
+    const spec = parseWorkloadYaml(path);
+    expect(spec.successMetric.higherIsBetter).toBe(true);
+  });
+
+  it('defaults inputs/outputs/env to empty', () => {
+    const path = writeYaml('workload.yml', `
+name: toy
+image: busybox
+command: [echo, hi]
+max_cost: 1
+max_runtime_minutes: 1
+`);
+    const spec = parseWorkloadYaml(path);
+    expect(spec.inputs).toEqual([]);
+    expect(spec.outputs).toEqual([]);
+    expect(spec.env).toEqual({});
+    expect(spec.successMetric).toBeNull();
+  });
+
+  it('throws WorkloadSpecError with all problems on an invalid file', () => {
+    const path = writeYaml('workload.yml', `
+image: busybox
+`);
+    expect(() => parseWorkloadYaml(path)).toThrow(WorkloadSpecError);
+  });
+
+  it('throws WorkloadSpecError on malformed YAML', () => {
+    const path = writeYaml('workload.yml', '{ not: valid: yaml: [');
+    expect(() => parseWorkloadYaml(path)).toThrow(WorkloadSpecError);
+  });
+
+  it('throws WorkloadSpecError when the file does not exist', () => {
+    expect(() => parseWorkloadYaml(join(dir, 'nope.yml'))).toThrow(WorkloadSpecError);
+  });
+});