badgr-cli 1.0.48 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/package.json +1 -1
- package/src/api.js +16 -2
- package/src/artifactDownload.js +55 -0
- package/src/badgr.js +104 -0
- package/src/batch.js +22 -4
- package/src/browser.js +23 -0
- package/src/commands/artifacts.js +75 -0
- package/src/commands/batch.js +221 -28
- package/src/commands/billing.js +1 -12
- package/src/commands/capacity.js +9 -4
- package/src/commands/comfyui.js +3 -3
- package/src/commands/connect.js +83 -0
- package/src/commands/doctor.js +127 -0
- package/src/commands/down.js +29 -6
- package/src/commands/launch.js +431 -0
- package/src/commands/pull.js +137 -0
- package/src/commands/run.js +253 -37
- package/src/commands/sbatch.js +232 -0
- package/src/commands/serve.js +3 -3
- package/src/commands/status.js +12 -4
- package/src/commands/task.js +25 -0
- package/src/commands/test-run.js +4 -2
- package/src/credentials.js +65 -0
- package/src/fallback.js +7 -2
- package/src/fanout.js +70 -0
- package/src/gpuDoctor/diskInfo.js +42 -0
- package/src/gpuDoctor/doctor.js +451 -0
- package/src/gpuDoctor/gpuInfo.js +70 -0
- package/src/gpuDoctor/healthCheck.js +63 -0
- package/src/gpuDoctor/logClassifier.js +138 -0
- package/src/gpuDoctor/modelFit.js +107 -0
- package/src/gpuDoctor/probeCache.js +38 -0
- package/src/gpuDoctor/redact.js +29 -0
- package/src/gpuDoctor/torchInfo.js +61 -0
- package/src/gpuDoctor/workflowDoctor.js +96 -0
- package/src/onboarding.js +124 -0
- package/src/slurm.js +193 -0
- package/src/spec.js +59 -2
- package/src/store.js +16 -0
- package/tests/agent-images.test.js +17 -0
- package/tests/artifactDownload.test.js +113 -0
- package/tests/artifacts.test.js +168 -0
- package/tests/batch.test.js +312 -0
- package/tests/browser.test.js +51 -0
- package/tests/capacity.test.js +68 -0
- package/tests/commands.test.js +44 -0
- package/tests/connect.test.js +83 -0
- package/tests/down.test.js +23 -1
- package/tests/fallback-timeout.test.js +41 -0
- package/tests/fanout.test.js +124 -0
- package/tests/gpu-doctor-classifiers.test.js +402 -0
- package/tests/gpu-doctor-doctor.test.js +304 -0
- package/tests/gpu-doctor-probe-cache.test.js +110 -0
- package/tests/gpu-doctor-probes.test.js +257 -0
- package/tests/launch-command-argv.test.js +93 -0
- package/tests/launch-readiness.test.js +1 -0
- package/tests/launch.test.js +440 -0
- package/tests/onboarding.test.js +134 -0
- package/tests/pull.test.js +266 -0
- package/tests/run-lifecycle.test.js +405 -6
- package/tests/sbatch.test.js +190 -0
- package/tests/secrets.test.js +16 -0
- package/tests/slurm.test.js +77 -0
- package/tests/spec.test.js +59 -1
- package/tests/status.test.js +73 -0
- package/tests/task.test.js +109 -0
- package/tests/template.test.js +7 -0
|
@@ -15,16 +15,39 @@
|
|
|
15
15
|
* - Receipt explains state → status, exitCode, failureType, runtimeSeconds, finalCost
|
|
16
16
|
*/
|
|
17
17
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
18
|
+
import { writeFileSync } from 'fs';
|
|
18
19
|
import { runCommand, parseRunArgs } from '../src/commands/run.js';
|
|
20
|
+
import { launchCommand } from '../src/commands/launch.js';
|
|
19
21
|
|
|
20
22
|
// ── Module mocks ──────────────────────────────────────────────────────────────
|
|
21
23
|
|
|
22
24
|
vi.mock('../src/api.js', () => ({
|
|
23
25
|
callApi: vi.fn(),
|
|
24
|
-
terminateDeployment: vi.fn().mockResolvedValue({}),
|
|
26
|
+
terminateDeployment: vi.fn().mockResolvedValue({ teardown_ok: 'ok' }),
|
|
27
|
+
uploadBlob: vi.fn().mockResolvedValue({ code_uri: 'blob://project.zip' }),
|
|
28
|
+
quoteRun: vi.fn(),
|
|
29
|
+
}));
|
|
30
|
+
|
|
31
|
+
vi.mock('archiver', () => ({
|
|
32
|
+
default: () => {
|
|
33
|
+
let output;
|
|
34
|
+
return {
|
|
35
|
+
on: vi.fn(),
|
|
36
|
+
pipe: vi.fn(o => { output = o; }),
|
|
37
|
+
glob: vi.fn(),
|
|
38
|
+
finalize: vi.fn(() => { if (output?.path) writeFileSync(output.path, 'zip'); output?.emit?.('close'); }),
|
|
39
|
+
};
|
|
40
|
+
},
|
|
25
41
|
}));
|
|
26
42
|
|
|
27
43
|
vi.mock('../src/store.js', () => ({
|
|
44
|
+
selectedComputeFromDeployment: (dep) => ({
|
|
45
|
+
gpu: dep.gpu_type ?? null,
|
|
46
|
+
gpuCount: dep.gpu_count ?? null,
|
|
47
|
+
vcpus: dep.selected_vcpus ?? null,
|
|
48
|
+
ramGb: dep.selected_ram_gb ?? null,
|
|
49
|
+
vramGb: dep.selected_vram_gb ?? null,
|
|
50
|
+
}),
|
|
28
51
|
addDeployment: vi.fn(),
|
|
29
52
|
addReceipt: vi.fn(),
|
|
30
53
|
updateReceipt: vi.fn(),
|
|
@@ -75,7 +98,7 @@ function setupSuccessfulRun(depOverrides = {}, exitCode = 0) {
|
|
|
75
98
|
.mockResolvedValueOnce(makeDep(depOverrides)) // POST /run
|
|
76
99
|
.mockResolvedValueOnce({ status: 'completed', exit_code: exitCode }) // GET /deployments/…
|
|
77
100
|
.mockResolvedValueOnce({ logs: ['step 1', 'step 2'] }); // GET /deployments/…/logs
|
|
78
|
-
api.terminateDeployment.mockResolvedValue({});
|
|
101
|
+
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
79
102
|
}
|
|
80
103
|
|
|
81
104
|
// ── Setup / teardown ─────────────────────────────────────────────────────────
|
|
@@ -104,7 +127,8 @@ beforeEach(() => {
|
|
|
104
127
|
store.listDeployments.mockReturnValue([]);
|
|
105
128
|
store.listReceipts.mockReturnValue([]);
|
|
106
129
|
store.findDeployment.mockReturnValue(null);
|
|
107
|
-
api.terminateDeployment.mockResolvedValue({});
|
|
130
|
+
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
131
|
+
api.uploadBlob.mockResolvedValue({ code_uri: 'blob://project.zip' });
|
|
108
132
|
});
|
|
109
133
|
|
|
110
134
|
afterEach(() => {
|
|
@@ -158,6 +182,37 @@ describe('GPU / provider matrix', () => {
|
|
|
158
182
|
});
|
|
159
183
|
});
|
|
160
184
|
|
|
185
|
+
describe('CPU launch — price quoted before provisioning', () => {
|
|
186
|
+
it('shows the quoted Badgr rate before the deployment is created, and does not repeat it after', async () => {
|
|
187
|
+
api.quoteRun.mockResolvedValueOnce({ vm_class: 'small', vcpus: 2, memory_gb: 4, region: 'US', rate_per_hour: 0.13 });
|
|
188
|
+
api.callApi
|
|
189
|
+
.mockResolvedValueOnce(makeDep({ gpu_type: 'CPU', cost_per_hour: 0.13 })) // POST /run
|
|
190
|
+
.mockResolvedValueOnce({});
|
|
191
|
+
|
|
192
|
+
await launchCommand(config, ['cline', '--max-cost', '1', 'Fix the checkout bug'], chalk);
|
|
193
|
+
|
|
194
|
+
expect(api.quoteRun).toHaveBeenCalledWith(config, expect.objectContaining({ compute: 'cpu', agent: 'cline' }));
|
|
195
|
+
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
196
|
+
expect(logged).toContain('Badgr rate: $0.13/hour');
|
|
197
|
+
// The quoted rate is shown once, pre-provisioning — not repeated as the
|
|
198
|
+
// post-creation fallback line too.
|
|
199
|
+
expect(logged.match(/Badgr rate:/g)?.length).toBe(1);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it('still launches, falling back to the post-creation rate, when the quote call fails', async () => {
|
|
203
|
+
api.quoteRun.mockRejectedValueOnce(new Error('network error'));
|
|
204
|
+
api.callApi
|
|
205
|
+
.mockResolvedValueOnce(makeDep({ gpu_type: 'CPU', cost_per_hour: 0.13 })) // POST /run
|
|
206
|
+
.mockResolvedValueOnce({});
|
|
207
|
+
|
|
208
|
+
await launchCommand(config, ['cline', '--max-cost', '1', 'Fix the checkout bug'], chalk);
|
|
209
|
+
|
|
210
|
+
expect(process.exitCode).toBeFalsy();
|
|
211
|
+
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
212
|
+
expect(logged).toContain('Badgr rate: $0.13/hour');
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
|
|
161
216
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
162
217
|
// 2. Job exits non-zero
|
|
163
218
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -179,6 +234,51 @@ describe('non-zero exit code', () => {
|
|
|
179
234
|
expect(process.exitCode).toBe(1);
|
|
180
235
|
});
|
|
181
236
|
|
|
237
|
+
it('reports "Teardown: succeeded" on a customer-code failure when the backend already confirmed teardown', async () => {
|
|
238
|
+
// The job-runner's own /complete webhook (complete_job_atomic) already
|
|
239
|
+
// attempts + confirms teardown on any exit code, before the CLI ever
|
|
240
|
+
// observes the failure via polling — teardown_ok in the poll response
|
|
241
|
+
// reflects that already-confirmed result.
|
|
242
|
+
api.callApi
|
|
243
|
+
.mockResolvedValueOnce(makeDep())
|
|
244
|
+
.mockResolvedValueOnce({ status: 'failed', exit_code: 1, teardown_ok: 'ok' })
|
|
245
|
+
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
246
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
247
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
248
|
+
await p;
|
|
249
|
+
|
|
250
|
+
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
251
|
+
expect(logged).toContain('succeeded');
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it('reports "Teardown: failed" on a customer-code failure when the backend never confirmed deletion', async () => {
|
|
255
|
+
api.callApi
|
|
256
|
+
.mockResolvedValueOnce(makeDep())
|
|
257
|
+
.mockResolvedValueOnce({ status: 'failed', exit_code: 1, teardown_ok: 'failed' })
|
|
258
|
+
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
259
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
260
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
261
|
+
await p;
|
|
262
|
+
|
|
263
|
+
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
264
|
+
expect(logged).not.toContain('Teardown: succeeded');
|
|
265
|
+
expect(logged).toContain('failed');
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
it('writes an initial receipt for a job that ultimately fails', async () => {
|
|
269
|
+
api.callApi
|
|
270
|
+
.mockResolvedValueOnce(makeDep())
|
|
271
|
+
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
272
|
+
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
273
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
274
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
275
|
+
await p;
|
|
276
|
+
|
|
277
|
+
// A receipt must exist from the start of the job, not only once it fails —
|
|
278
|
+
// updateReceipt above mutates the same receipt addReceipt created here.
|
|
279
|
+
expect(store.addReceipt).toHaveBeenCalled();
|
|
280
|
+
});
|
|
281
|
+
|
|
182
282
|
it('distinguishes customer_code from infrastructure when exit code is null', async () => {
|
|
183
283
|
api.callApi
|
|
184
284
|
.mockResolvedValueOnce(makeDep())
|
|
@@ -255,7 +355,7 @@ describe('max-runtime cap', () => {
|
|
|
255
355
|
.mockResolvedValueOnce(makeDep({ cost_per_hour: 2.50 })) // POST /run
|
|
256
356
|
.mockResolvedValueOnce({ status: 'running', exit_code: null }) // poll 1
|
|
257
357
|
.mockResolvedValueOnce({ logs: [] }); // logs 1
|
|
258
|
-
api.terminateDeployment.mockResolvedValue({});
|
|
358
|
+
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
259
359
|
|
|
260
360
|
const p = runCommand(config, ['python', 'train.py', '--max-runtime', '0.05', '--max-cost', '5'], chalk);
|
|
261
361
|
// attachToJob POLL_MS = 4000, maxRuntime = 0.05 min = 3000ms → cap fires after 4000ms (first elapsedMs check)
|
|
@@ -282,7 +382,7 @@ describe('max-cost cap', () => {
|
|
|
282
382
|
.mockResolvedValueOnce(makeDep({ cost_per_hour: 3600 })) // POST /run
|
|
283
383
|
.mockResolvedValueOnce({ status: 'running' }) // poll 1 (never reached — cap fires first)
|
|
284
384
|
.mockResolvedValueOnce({ logs: [] });
|
|
285
|
-
api.terminateDeployment.mockResolvedValue({});
|
|
385
|
+
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
286
386
|
|
|
287
387
|
const p = runCommand(config, ['python', 'train.py', '--max-cost', '0.001'], chalk);
|
|
288
388
|
await vi.advanceTimersByTimeAsync(5000);
|
|
@@ -312,7 +412,7 @@ describe('heartbeat lost', () => {
|
|
|
312
412
|
.mockResolvedValueOnce({ status: 'running' }) // first dep poll → sets lastStatus = 'running'
|
|
313
413
|
.mockResolvedValueOnce({ logs: [] }) // first logs poll
|
|
314
414
|
.mockRejectedValue(new Error('ETIMEDOUT')); // all subsequent dep/logs polls fail
|
|
315
|
-
api.terminateDeployment.mockResolvedValue({});
|
|
415
|
+
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
316
416
|
|
|
317
417
|
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
318
418
|
// Advance past 1 successful poll (4000ms) + 15 failed polls (60000ms) = 64000ms
|
|
@@ -496,6 +596,20 @@ describe('billing lifecycle', () => {
|
|
|
496
596
|
expect(typeof updateCall[1].runtimeSeconds).toBe('number');
|
|
497
597
|
expect(typeof updateCall[1].finalCost).toBe('number');
|
|
498
598
|
});
|
|
599
|
+
|
|
600
|
+
it('reports "Teardown: failed" — never "succeeded" — when the backend accepted the DELETE but did not confirm deletion', async () => {
|
|
601
|
+
// A 200 response with teardown_ok: 'failed' must never be reported as a
|
|
602
|
+
// successful teardown — the caller only knows deletion was *requested*.
|
|
603
|
+
setupSuccessfulRun();
|
|
604
|
+
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'failed' });
|
|
605
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
606
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
607
|
+
await p;
|
|
608
|
+
|
|
609
|
+
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
610
|
+
expect(logged).not.toContain('Teardown: succeeded');
|
|
611
|
+
expect(logged).toContain('failed');
|
|
612
|
+
});
|
|
499
613
|
});
|
|
500
614
|
|
|
501
615
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -653,3 +767,288 @@ describe('--output / --checkpoint / --retry-safe / --resume-cmd', () => {
|
|
|
653
767
|
expect(logs.some(l => l.includes('Resume:'))).toBe(false);
|
|
654
768
|
});
|
|
655
769
|
});
|
|
770
|
+
|
|
771
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
772
|
+
// 13b. --cpu / --memory / --gpu-memory — CPU/RAM/VRAM resource matching
|
|
773
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
774
|
+
|
|
775
|
+
describe('--cpu / --memory / --gpu-memory', () => {
|
|
776
|
+
it('wires --cpu and --memory into the /run request body as cpu/memory_gb', async () => {
|
|
777
|
+
setupSuccessfulRun();
|
|
778
|
+
const p = runCommand(config, [
|
|
779
|
+
'--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--', 'python', 'train.py',
|
|
780
|
+
], chalk);
|
|
781
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
782
|
+
await p;
|
|
783
|
+
|
|
784
|
+
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
785
|
+
expect(postRunBody.cpu).toBe(16);
|
|
786
|
+
expect(postRunBody.memory_gb).toBe(64);
|
|
787
|
+
});
|
|
788
|
+
|
|
789
|
+
it('--gpu-memory sets min_vram the same way --min-vram does', async () => {
|
|
790
|
+
setupSuccessfulRun();
|
|
791
|
+
const p = runCommand(config, [
|
|
792
|
+
'--gpu-memory', '24GB', '--max-cost', '5', '--', 'python', 'train.py',
|
|
793
|
+
], chalk);
|
|
794
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
795
|
+
await p;
|
|
796
|
+
|
|
797
|
+
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
798
|
+
expect(postRunBody.min_vram).toBe(24);
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
it('omits cpu/memory_gb/min_vram from the body when none were requested', async () => {
|
|
802
|
+
setupSuccessfulRun();
|
|
803
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
804
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
805
|
+
await p;
|
|
806
|
+
|
|
807
|
+
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
808
|
+
expect(postRunBody.cpu).toBeUndefined();
|
|
809
|
+
expect(postRunBody.memory_gb).toBeUndefined();
|
|
810
|
+
expect(postRunBody.min_vram).toBeUndefined();
|
|
811
|
+
});
|
|
812
|
+
|
|
813
|
+
it('--dry-run shows CPU/Memory/Min VRAM without calling the API', async () => {
|
|
814
|
+
const output = [];
|
|
815
|
+
const dryChalk = { bold: (s) => s, dim: (s) => s, cyan: (s) => s, red: (s) => s, yellow: (s) => s, green: (s) => s };
|
|
816
|
+
const origLog = console.log;
|
|
817
|
+
console.log = (...args) => output.push(args.join(' '));
|
|
818
|
+
await runCommand(config, [
|
|
819
|
+
'--dry-run', '--cpu', '16', '--memory', '64GB', '--gpu-memory', '24GB',
|
|
820
|
+
'--image', 'node:20', '--max-cost', '1', '--', 'node', '-e', "console.log('dry')",
|
|
821
|
+
], dryChalk);
|
|
822
|
+
console.log = origLog;
|
|
823
|
+
expect(api.callApi).not.toHaveBeenCalled();
|
|
824
|
+
expect(output.some(l => /CPU:.*16 cores/.test(l))).toBe(true);
|
|
825
|
+
expect(output.some(l => /Memory:.*64 GB/.test(l))).toBe(true);
|
|
826
|
+
expect(output.some(l => /Min VRAM:.*24 GB/.test(l))).toBe(true);
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
it('rejects an unparseable --memory value before touching the API', async () => {
|
|
830
|
+
await runCommand(config, ['python', 'train.py', '--memory', 'not-a-size', '--max-cost', '5'], chalk);
|
|
831
|
+
expect(api.callApi).not.toHaveBeenCalled();
|
|
832
|
+
expect(process.exitCode).toBe(1);
|
|
833
|
+
});
|
|
834
|
+
|
|
835
|
+
it('rejects an unparseable --gpu-memory value before touching the API', async () => {
|
|
836
|
+
await runCommand(config, ['python', 'train.py', '--gpu-memory', 'huge', '--max-cost', '5'], chalk);
|
|
837
|
+
expect(api.callApi).not.toHaveBeenCalled();
|
|
838
|
+
expect(process.exitCode).toBe(1);
|
|
839
|
+
});
|
|
840
|
+
|
|
841
|
+
it('rejects a non-integer --cpu before touching the API', async () => {
|
|
842
|
+
await runCommand(config, ['python', 'train.py', '--cpu', 'sixteen', '--max-cost', '5'], chalk);
|
|
843
|
+
expect(api.callApi).not.toHaveBeenCalled();
|
|
844
|
+
expect(process.exitCode).toBe(1);
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
it('rejects --no-gpu combined with --gpu', async () => {
|
|
848
|
+
await runCommand(config, ['python', 'sim.py', '--no-gpu', '--gpu', 'A100', '--max-cost', '5'], chalk);
|
|
849
|
+
expect(api.callApi).not.toHaveBeenCalled();
|
|
850
|
+
expect(process.exitCode).toBe(1);
|
|
851
|
+
});
|
|
852
|
+
|
|
853
|
+
it('--no-gpu sends no_gpu:true and no gpu-specific fields beyond the default', async () => {
|
|
854
|
+
setupSuccessfulRun();
|
|
855
|
+
const p = runCommand(config, [
|
|
856
|
+
'--no-gpu', '--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--', 'python', 'sim.py',
|
|
857
|
+
], chalk);
|
|
858
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
859
|
+
await p;
|
|
860
|
+
|
|
861
|
+
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
862
|
+
expect(postRunBody.no_gpu).toBe(true);
|
|
863
|
+
expect(postRunBody.cpu).toBe(16);
|
|
864
|
+
expect(postRunBody.memory_gb).toBe(64);
|
|
865
|
+
});
|
|
866
|
+
|
|
867
|
+
it('shows what was actually provisioned when a resource floor was requested', async () => {
|
|
868
|
+
api.callApi
|
|
869
|
+
.mockResolvedValueOnce(makeDep({ gpu_type: 'A100', gpu_count: 1, selected_vcpus: 16, selected_ram_gb: 64, selected_vram_gb: 40 }))
|
|
870
|
+
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
871
|
+
.mockResolvedValueOnce({ logs: [] });
|
|
872
|
+
const logs = [];
|
|
873
|
+
const origLog = console.log;
|
|
874
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
875
|
+
const p = runCommand(config, [
|
|
876
|
+
'--gpu-memory', '24GB', '--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--', 'python', 'train.py',
|
|
877
|
+
], chalk);
|
|
878
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
879
|
+
await p;
|
|
880
|
+
console.log = origLog;
|
|
881
|
+
|
|
882
|
+
expect(logs.some(l => l.includes('Provisioned:') && l.includes('A100') && l.includes('40GB VRAM') && l.includes('16 vCPU') && l.includes('64GB RAM'))).toBe(true);
|
|
883
|
+
});
|
|
884
|
+
|
|
885
|
+
it('does not print a Provisioned line when no resource floor was requested', async () => {
|
|
886
|
+
setupSuccessfulRun();
|
|
887
|
+
const logs = [];
|
|
888
|
+
const origLog = console.log;
|
|
889
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
890
|
+
const p = runCommand(config, ['python', 'train.py', '--gpu', 'A100', '--max-cost', '5'], chalk);
|
|
891
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
892
|
+
await p;
|
|
893
|
+
console.log = origLog;
|
|
894
|
+
|
|
895
|
+
expect(logs.some(l => l.includes('Provisioned:'))).toBe(false);
|
|
896
|
+
});
|
|
897
|
+
|
|
898
|
+
it('records workload shape and requested/selected compute on the receipt', async () => {
|
|
899
|
+
api.callApi
|
|
900
|
+
.mockResolvedValueOnce(makeDep({ gpu_type: 'A100', gpu_count: 1, selected_vcpus: 16, selected_ram_gb: 64, selected_vram_gb: 40 }))
|
|
901
|
+
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
902
|
+
.mockResolvedValueOnce({ logs: [] });
|
|
903
|
+
const p = runCommand(config, [
|
|
904
|
+
'--gpu-memory', '24GB', '--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--image', 'node:20', '--', 'node', 'train.js',
|
|
905
|
+
], chalk);
|
|
906
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
907
|
+
await p;
|
|
908
|
+
|
|
909
|
+
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({
|
|
910
|
+
workloadShape: 'container',
|
|
911
|
+
computeRequested: expect.objectContaining({ minVram: 24, cpu: 16, memoryGb: 64, noGpu: false }),
|
|
912
|
+
computeSelected: expect.objectContaining({ gpu: 'A100', vcpus: 16, ramGb: 64, vramGb: 40 }),
|
|
913
|
+
}));
|
|
914
|
+
});
|
|
915
|
+
|
|
916
|
+
it('tags a local-path run as workloadShape "project"', async () => {
|
|
917
|
+
setupSuccessfulRun();
|
|
918
|
+
const p = runCommand(config, ['.', '--cmd', 'python train.py', '--max-cost', '5'], chalk);
|
|
919
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
920
|
+
await p;
|
|
921
|
+
|
|
922
|
+
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({ workloadShape: 'project' }));
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
describe('badgr launch (CPU VM)', () => {
|
|
927
|
+
it('maps `badgr launch . -- <command>` to a detached CPU project run', async () => {
|
|
928
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
929
|
+
const p = launchCommand(config, ['.', '--max-cost', '5', '--', 'claude', '-p', 'Fix the failing tests'], chalk);
|
|
930
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
931
|
+
await p;
|
|
932
|
+
|
|
933
|
+
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
934
|
+
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
935
|
+
expect(body.compute).toBe('cpu');
|
|
936
|
+
expect(body.gpu).toBe('CPU');
|
|
937
|
+
expect(body.cmd).toBe('claude -p Fix the failing tests');
|
|
938
|
+
expect(body.code_uri).toBeTruthy();
|
|
939
|
+
expect(body.agent).toBeUndefined();
|
|
940
|
+
expect(body.task).toBeUndefined();
|
|
941
|
+
});
|
|
942
|
+
|
|
943
|
+
it('defaults to detached (returns without waiting on the job)', async () => {
|
|
944
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
945
|
+
const logs = [];
|
|
946
|
+
const origLog = console.log;
|
|
947
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
948
|
+
await launchCommand(config, ['.', '--max-cost', '5', '--', 'python', 'narrgo.py'], chalk);
|
|
949
|
+
console.log = origLog;
|
|
950
|
+
|
|
951
|
+
expect(logs.some(l => l.includes('Detached'))).toBe(true);
|
|
952
|
+
});
|
|
953
|
+
|
|
954
|
+
it('accepts the quoted --cmd form as equivalent to -- passthrough', async () => {
|
|
955
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
956
|
+
const p = launchCommand(config, ['.', '--max-cost', '5', '--cmd', 'codex exec "Write tests"'], chalk);
|
|
957
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
958
|
+
await p;
|
|
959
|
+
|
|
960
|
+
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
961
|
+
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
962
|
+
expect(body.cmd).toBe('codex exec "Write tests"');
|
|
963
|
+
});
|
|
964
|
+
|
|
965
|
+
it('sends declared --artifacts paths as output_paths in the /run request body', async () => {
|
|
966
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
967
|
+
const p = launchCommand(config, [
|
|
968
|
+
'.', '--max-cost', '5',
|
|
969
|
+
'--artifacts', 'playwright-report', '--artifacts', 'test-results',
|
|
970
|
+
'--', 'npx', 'playwright', 'test',
|
|
971
|
+
], chalk);
|
|
972
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
973
|
+
await p;
|
|
974
|
+
|
|
975
|
+
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
976
|
+
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
977
|
+
expect(body.output_paths).toEqual(['playwright-report', 'test-results']);
|
|
978
|
+
});
|
|
979
|
+
|
|
980
|
+
it('does not send output_paths when --artifacts is not passed', async () => {
|
|
981
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
982
|
+
const p = launchCommand(config, ['.', '--max-cost', '5', '--', 'npm', 'test'], chalk);
|
|
983
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
984
|
+
await p;
|
|
985
|
+
|
|
986
|
+
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
987
|
+
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
988
|
+
expect(body.output_paths).toBeUndefined();
|
|
989
|
+
});
|
|
990
|
+
|
|
991
|
+
it('--no-detach streams the job to completion instead of returning immediately', async () => {
|
|
992
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
993
|
+
const logs = [];
|
|
994
|
+
const origLog = console.log;
|
|
995
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
996
|
+
const p = launchCommand(config, ['.', '--max-cost', '5', '--no-detach', '--', 'npm', 'test'], chalk);
|
|
997
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
998
|
+
await p;
|
|
999
|
+
console.log = origLog;
|
|
1000
|
+
|
|
1001
|
+
expect(logs.some(l => l.includes('Detached'))).toBe(false);
|
|
1002
|
+
expect(logs.some(l => l.includes('Complete'))).toBe(true);
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
it('rejects --gpu end to end without ever calling the API', async () => {
|
|
1006
|
+
await launchCommand(config, ['.', '--gpu', 'A100', '--max-cost', '5', '--', 'npm', 'test'], chalk);
|
|
1007
|
+
expect(process.exitCode).toBe(1);
|
|
1008
|
+
expect(api.callApi).not.toHaveBeenCalled();
|
|
1009
|
+
process.exitCode = undefined;
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
it('prints a secret-shaped --env warning before submitting the run', async () => {
|
|
1013
|
+
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
1014
|
+
const logs = [];
|
|
1015
|
+
const origLog = console.log;
|
|
1016
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
1017
|
+
const p = launchCommand(config, [
|
|
1018
|
+
'.', '--max-cost', '5', '--env', 'ANTHROPIC_API_KEY=sk-ant-fake',
|
|
1019
|
+
'--', 'claude', '-p', 'fix',
|
|
1020
|
+
], chalk);
|
|
1021
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
1022
|
+
await p;
|
|
1023
|
+
console.log = origLog;
|
|
1024
|
+
|
|
1025
|
+
const joinedLogs = logs.join('\n');
|
|
1026
|
+
expect(joinedLogs).toContain('ANTHROPIC_API_KEY');
|
|
1027
|
+
expect(joinedLogs).toContain('shell history');
|
|
1028
|
+
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
1029
|
+
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
1030
|
+
expect(body.env.ANTHROPIC_API_KEY).toBe('sk-ant-fake');
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
it('a failed CPU launch command still updates the receipt with a usable status', async () => {
|
|
1034
|
+
api.callApi
|
|
1035
|
+
.mockResolvedValueOnce({
|
|
1036
|
+
deployment_id: 'dep-launch-001', status: 'running', gpu_type: 'CPU',
|
|
1037
|
+
cost_per_hour: 0.10, receipt_id: 'rcpt-launch-001', tier: '1',
|
|
1038
|
+
})
|
|
1039
|
+
.mockResolvedValueOnce({ status: 'completed', exit_code: 1 })
|
|
1040
|
+
.mockResolvedValueOnce({ logs: ['error: test failed'] });
|
|
1041
|
+
|
|
1042
|
+
const p = launchCommand(config, ['.', '--max-cost', '5', '--no-detach', '--', 'npm', 'test'], chalk);
|
|
1043
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
1044
|
+
await p;
|
|
1045
|
+
|
|
1046
|
+
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
1047
|
+
status: 'completed',
|
|
1048
|
+
exitCode: 1,
|
|
1049
|
+
failureType: 'customer_code',
|
|
1050
|
+
}));
|
|
1051
|
+
expect(process.exitCode).toBe(1);
|
|
1052
|
+
process.exitCode = undefined;
|
|
1053
|
+
});
|
|
1054
|
+
});
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* badgr sbatch — /run body translation from #SBATCH directives, dry-run
|
|
3
|
+
* behavior, and job-array fan-out (one deployment per task index).
|
|
4
|
+
*/
|
|
5
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
6
|
+
import { mkdtempSync, writeFileSync, rmSync } from 'fs';
|
|
7
|
+
import { tmpdir } from 'os';
|
|
8
|
+
import { join } from 'path';
|
|
9
|
+
|
|
10
|
+
vi.mock('../src/store.js', () => ({
|
|
11
|
+
selectedComputeFromDeployment: (dep) => ({
|
|
12
|
+
gpu: dep.gpu_type ?? null,
|
|
13
|
+
gpuCount: dep.gpu_count ?? null,
|
|
14
|
+
vcpus: dep.selected_vcpus ?? null,
|
|
15
|
+
ramGb: dep.selected_ram_gb ?? null,
|
|
16
|
+
vramGb: dep.selected_vram_gb ?? null,
|
|
17
|
+
}),
|
|
18
|
+
addReceipt: vi.fn(),
|
|
19
|
+
updateReceipt: vi.fn(),
|
|
20
|
+
generateReceiptId: vi.fn(() => 'rcpt-sbatch-001'),
|
|
21
|
+
loadStore: vi.fn(() => ({ receipts: [], deployments: [] })),
|
|
22
|
+
}));
|
|
23
|
+
|
|
24
|
+
vi.mock('../src/api.js', () => ({
|
|
25
|
+
callApi: vi.fn(),
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
vi.mock('../src/fallback.js', async (importOriginal) => {
|
|
29
|
+
const actual = await importOriginal();
|
|
30
|
+
return { ...actual, callWithFallback: vi.fn() };
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
vi.mock('../src/batch.js', () => ({
|
|
34
|
+
monitorBatchJob: vi.fn(() => Promise.resolve({ status: 'succeeded', exitCode: 0, runtimeMs: 5000, reason: 'complete' })),
|
|
35
|
+
fmtRuntime: (ms) => `${Math.round(ms / 1000)}s`,
|
|
36
|
+
}));
|
|
37
|
+
|
|
38
|
+
import { sbatchCommand, buildRunBody } from '../src/commands/sbatch.js';
|
|
39
|
+
import * as store from '../src/store.js';
|
|
40
|
+
import * as api from '../src/api.js';
|
|
41
|
+
import * as fallback from '../src/fallback.js';
|
|
42
|
+
|
|
43
|
+
const chalk = {
|
|
44
|
+
bold: s => s, dim: s => s, red: s => s, yellow: s => s, green: s => s, cyan: s => s,
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const config = { apiKey: 'sk-test', baseUrl: 'https://api.test/v1' };
|
|
48
|
+
|
|
49
|
+
let dir;
|
|
50
|
+
beforeEach(() => {
|
|
51
|
+
dir = mkdtempSync(join(tmpdir(), 'badgr-sbatch-cmd-test-'));
|
|
52
|
+
process.exitCode = undefined;
|
|
53
|
+
vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
54
|
+
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
55
|
+
vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
|
56
|
+
vi.clearAllMocks();
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterEach(() => {
|
|
60
|
+
rmSync(dir, { recursive: true, force: true });
|
|
61
|
+
vi.restoreAllMocks();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe('buildRunBody', () => {
|
|
65
|
+
it('translates a GPU job into the /run body shape', () => {
|
|
66
|
+
const job = {
|
|
67
|
+
name: 'train', command: 'python train.py', cpus: 32, memGb: 128,
|
|
68
|
+
gpuCount: 2, gpuType: 'a100', env: { FOO: 'bar' },
|
|
69
|
+
};
|
|
70
|
+
const body = buildRunBody(job, {
|
|
71
|
+
image: 'python:3.11-slim', tier: '1', region: null,
|
|
72
|
+
maxCostUsd: 5, maxRuntimeMinutes: 60,
|
|
73
|
+
});
|
|
74
|
+
expect(body.command).toEqual(['bash', '-lc', 'python train.py']);
|
|
75
|
+
expect(body.gpu).toBe('A100');
|
|
76
|
+
expect(body.gpu_count).toBe(2);
|
|
77
|
+
expect(body.cpu).toBe(32);
|
|
78
|
+
expect(body.memory_gb).toBe(128);
|
|
79
|
+
expect(body.env).toEqual({ FOO: 'bar' });
|
|
80
|
+
expect(body.max_cost_usd).toBe(5);
|
|
81
|
+
expect(body.max_runtime_seconds).toBe(3600);
|
|
82
|
+
expect(body.name).toBe('train');
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('requests no GPU when the script has no gres/gpus directive', () => {
|
|
86
|
+
const job = { name: 'cpu-job', command: 'echo hi', cpus: null, memGb: null, gpuCount: 0, gpuType: null, env: {} };
|
|
87
|
+
const body = buildRunBody(job, { image: 'python:3.11-slim', tier: '1', region: null, maxCostUsd: 1, maxRuntimeMinutes: 10 });
|
|
88
|
+
expect(body.gpu).toBe('NONE');
|
|
89
|
+
// Backend's RunBody requires gpu_count >= 1; CPU-only routing goes through
|
|
90
|
+
// the separate no_gpu flag instead of gpu_count: 0 (which the real
|
|
91
|
+
// Pydantic-validated backend rejects with a 422 before provisioning).
|
|
92
|
+
expect(body.gpu_count).toBe(1);
|
|
93
|
+
expect(body.no_gpu).toBe(true);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it('suffixes the name and injects SLURM_ARRAY_TASK_ID for array tasks', () => {
|
|
97
|
+
const job = { name: 'screen', command: 'python screen.py', cpus: null, memGb: null, gpuCount: 1, gpuType: null, env: {} };
|
|
98
|
+
const body = buildRunBody(job, { image: 'python:3.11-slim', tier: '1', region: null, maxCostUsd: 1, maxRuntimeMinutes: 10, taskId: 7, arrayJobId: 'rcpt-x' });
|
|
99
|
+
expect(body.name).toBe('screen-7');
|
|
100
|
+
expect(body.env.SLURM_ARRAY_TASK_ID).toBe('7');
|
|
101
|
+
expect(body.env.SLURM_ARRAY_JOB_ID).toBe('rcpt-x');
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
describe('sbatchCommand', () => {
|
|
106
|
+
function writeScript(name, contents) {
|
|
107
|
+
const p = join(dir, name);
|
|
108
|
+
writeFileSync(p, contents);
|
|
109
|
+
return p;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
it('dry-run parses and prints the plan without calling the network', async () => {
|
|
113
|
+
const scriptPath = writeScript('job.slurm', `#!/bin/bash\n#SBATCH --job-name=t\n#SBATCH --gres=gpu:1\npython run.py\n`);
|
|
114
|
+
await sbatchCommand(config, [scriptPath, '--dry-run'], chalk);
|
|
115
|
+
expect(fallback.callWithFallback).not.toHaveBeenCalled();
|
|
116
|
+
expect(process.exitCode).toBeUndefined();
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('submits a single job and reports success', async () => {
|
|
120
|
+
const scriptPath = writeScript('job.slurm', `#!/bin/bash\n#SBATCH --job-name=t\n#SBATCH --gres=gpu:1\npython run.py\n`);
|
|
121
|
+
|
|
122
|
+
fallback.callWithFallback.mockResolvedValue({
|
|
123
|
+
deployment_id: 'dep-1', receipt_id: 'rcpt-1', gpu_type: 'RTX_4090',
|
|
124
|
+
gpu_count: 1, cost_per_hour: 0.5, provider: 'runpod', tier: '1', status: 'running',
|
|
125
|
+
});
|
|
126
|
+
api.callApi.mockResolvedValue({
|
|
127
|
+
status: 'succeeded', failure_reason: null, teardown_ok: 'ok',
|
|
128
|
+
runtime_seconds: 5, accrued_cost_usd: 0.01, provider: 'runpod',
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
await sbatchCommand(config, [scriptPath], chalk);
|
|
132
|
+
|
|
133
|
+
expect(fallback.callWithFallback).toHaveBeenCalledTimes(1);
|
|
134
|
+
expect(store.addReceipt).toHaveBeenCalledTimes(1);
|
|
135
|
+
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({
|
|
136
|
+
workloadShape: 'slurm',
|
|
137
|
+
computeSelected: expect.objectContaining({ gpu: 'RTX_4090', gpuCount: 1 }),
|
|
138
|
+
}));
|
|
139
|
+
expect(process.exitCode).toBeUndefined();
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('fans out array jobs into one deployment per task', async () => {
|
|
143
|
+
const scriptPath = writeScript('array.slurm', `#!/bin/bash\n#SBATCH --job-name=screen\n#SBATCH --array=1-3\n#SBATCH --gpus=1\npython screen.py\n`);
|
|
144
|
+
|
|
145
|
+
let callCount = 0;
|
|
146
|
+
fallback.callWithFallback.mockImplementation(() => {
|
|
147
|
+
callCount += 1;
|
|
148
|
+
return Promise.resolve({
|
|
149
|
+
deployment_id: `dep-${callCount}`, receipt_id: `rcpt-${callCount}`, gpu_type: 'RTX_4090',
|
|
150
|
+
gpu_count: 1, cost_per_hour: 0.5, provider: 'runpod', tier: '1', status: 'running',
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
api.callApi.mockResolvedValue({
|
|
154
|
+
status: 'succeeded', failure_reason: null, teardown_ok: 'ok',
|
|
155
|
+
runtime_seconds: 5, accrued_cost_usd: 0.01, provider: 'runpod',
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
await sbatchCommand(config, [scriptPath], chalk);
|
|
159
|
+
|
|
160
|
+
expect(fallback.callWithFallback).toHaveBeenCalledTimes(3);
|
|
161
|
+
expect(store.addReceipt).toHaveBeenCalledTimes(3);
|
|
162
|
+
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({ workloadShape: 'slurm-array' }));
|
|
163
|
+
expect(process.exitCode).toBeUndefined();
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it('reports failure exit code when a task does not succeed', async () => {
|
|
167
|
+
const scriptPath = writeScript('job.slurm', `#!/bin/bash\n#SBATCH --gres=gpu:1\npython run.py\n`);
|
|
168
|
+
const { monitorBatchJob } = await import('../src/batch.js');
|
|
169
|
+
monitorBatchJob.mockResolvedValueOnce({ status: 'failed', exitCode: 1, runtimeMs: 1000, reason: 'infrastructure' });
|
|
170
|
+
|
|
171
|
+
fallback.callWithFallback.mockResolvedValue({
|
|
172
|
+
deployment_id: 'dep-1', receipt_id: 'rcpt-1', gpu_type: 'RTX_4090',
|
|
173
|
+
gpu_count: 1, cost_per_hour: 0.5, provider: 'runpod', tier: '1', status: 'running',
|
|
174
|
+
});
|
|
175
|
+
api.callApi.mockResolvedValue({
|
|
176
|
+
status: 'failed', failure_reason: 'infrastructure', teardown_ok: 'ok',
|
|
177
|
+
runtime_seconds: 1, accrued_cost_usd: 0.01, provider: 'runpod',
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
await sbatchCommand(config, [scriptPath], chalk);
|
|
181
|
+
expect(process.exitCode).toBe(1);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it('errors before touching the network when the script has no command', async () => {
|
|
185
|
+
const scriptPath = writeScript('empty.slurm', `#!/bin/bash\n#SBATCH --job-name=empty\n`);
|
|
186
|
+
await sbatchCommand(config, [scriptPath], chalk);
|
|
187
|
+
expect(process.exitCode).toBe(1);
|
|
188
|
+
expect(fallback.callWithFallback).not.toHaveBeenCalled();
|
|
189
|
+
});
|
|
190
|
+
});
|