badgr-cli 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +207 -0
- package/README.md +13 -6
- package/package.json +44 -2
- package/src/api.js +16 -0
- package/src/badgr.js +26 -10
- package/src/commands/batch.js +11 -0
- package/src/commands/billing.js +3 -3
- package/src/commands/comfyui.js +31 -15
- package/src/commands/connect.js +4 -1
- package/src/commands/diagnose.js +493 -0
- package/src/commands/embed.js +13 -10
- package/src/commands/job.js +246 -0
- package/src/commands/launch.js +152 -16
- package/src/commands/login.js +75 -20
- package/src/commands/run.js +74 -16
- package/src/commands/sbatch.js +6 -1
- package/src/commands/serve.js +44 -30
- package/src/commands/train.js +8 -12
- package/src/commands/transcribe.js +13 -10
- package/src/credentials.js +33 -0
- package/src/envFlag.js +10 -0
- package/src/fallback.js +13 -2
- package/src/onboarding.js +8 -1
- package/src/progress.js +48 -0
- package/src/commands/task.js +0 -25
- package/tests/agent-images.test.js +0 -17
- package/tests/api.test.js +0 -168
- package/tests/artifactDownload.test.js +0 -113
- package/tests/artifacts.test.js +0 -168
- package/tests/batch.test.js +0 -641
- package/tests/browser.test.js +0 -51
- package/tests/capacity.test.js +0 -68
- package/tests/commands.test.js +0 -417
- package/tests/config.test.js +0 -96
- package/tests/connect.test.js +0 -83
- package/tests/detect.test.js +0 -191
- package/tests/down.test.js +0 -150
- package/tests/errors.test.js +0 -130
- package/tests/fallback-timeout.test.js +0 -41
- package/tests/fanout.test.js +0 -124
- package/tests/gpu-doctor-classifiers.test.js +0 -402
- package/tests/gpu-doctor-doctor.test.js +0 -304
- package/tests/gpu-doctor-probe-cache.test.js +0 -110
- package/tests/gpu-doctor-probes.test.js +0 -257
- package/tests/heartbeat.test.js +0 -70
- package/tests/job-progress-poll.test.js +0 -136
- package/tests/launch-command-argv.test.js +0 -93
- package/tests/launch-readiness.test.js +0 -403
- package/tests/launch.test.js +0 -440
- package/tests/onboarding.test.js +0 -134
- package/tests/productized-dry-run.test.js +0 -141
- package/tests/productized-runners.test.js +0 -237
- package/tests/pull.test.js +0 -266
- package/tests/rerun.test.js +0 -94
- package/tests/restart.test.js +0 -88
- package/tests/router.test.js +0 -98
- package/tests/run-lifecycle.test.js +0 -1054
- package/tests/sbatch.test.js +0 -190
- package/tests/secrets.test.js +0 -16
- package/tests/serve-apps.test.js +0 -189
- package/tests/serve-lifecycle.test.js +0 -931
- package/tests/slurm.test.js +0 -77
- package/tests/spec.test.js +0 -201
- package/tests/status.test.js +0 -73
- package/tests/store.test.js +0 -187
- package/tests/task.test.js +0 -109
- package/tests/template.test.js +0 -556
- package/tests/train-lora-dataset.test.js +0 -176
- package/tests/upload.test.js +0 -79
- package/tests/workload-rerun.test.js +0 -56
- package/tests/workload-spec.test.js +0 -180
- package/tests/workload-templates.test.js +0 -865
- package/tests/workload-workspace-paths.test.js +0 -46
|
@@ -1,1054 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* badgr run — end-to-end lifecycle tests (mocked API)
|
|
3
|
-
*
|
|
4
|
-
* Covers the launch-readiness matrix:
|
|
5
|
-
* - Successful run on two GPU types (A100, L40S) via two providers (runpod, vastai)
|
|
6
|
-
* - Job exits non-zero → receipt failureType = customer_code
|
|
7
|
-
* - Job exceeds max runtime → teardown called, receipt reason = max-runtime
|
|
8
|
-
* - Job exceeds max cost → teardown called, receipt reason = max-cost
|
|
9
|
-
* - Container fails before running (infrastructure)
|
|
10
|
-
* - Heartbeat lost mid-run → teardown called, receipt reason = heartbeat-lost
|
|
11
|
-
* - Log streaming → log lines appear in console output
|
|
12
|
-
* - Tier-1 unavailable → auto-expand to tier-2, receipt records tier
|
|
13
|
-
* - No capacity at all → CapacityError displayed, exitCode = 1
|
|
14
|
-
* - Billing stops → terminateDeployment called on every terminal path
|
|
15
|
-
* - Receipt explains state → status, exitCode, failureType, runtimeSeconds, finalCost
|
|
16
|
-
*/
|
|
17
|
-
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
18
|
-
import { writeFileSync } from 'fs';
|
|
19
|
-
import { runCommand, parseRunArgs } from '../src/commands/run.js';
|
|
20
|
-
import { launchCommand } from '../src/commands/launch.js';
|
|
21
|
-
|
|
22
|
-
// ── Module mocks ──────────────────────────────────────────────────────────────
|
|
23
|
-
|
|
24
|
-
vi.mock('../src/api.js', () => ({
|
|
25
|
-
callApi: vi.fn(),
|
|
26
|
-
terminateDeployment: vi.fn().mockResolvedValue({ teardown_ok: 'ok' }),
|
|
27
|
-
uploadBlob: vi.fn().mockResolvedValue({ code_uri: 'blob://project.zip' }),
|
|
28
|
-
quoteRun: vi.fn(),
|
|
29
|
-
}));
|
|
30
|
-
|
|
31
|
-
vi.mock('archiver', () => ({
|
|
32
|
-
default: () => {
|
|
33
|
-
let output;
|
|
34
|
-
return {
|
|
35
|
-
on: vi.fn(),
|
|
36
|
-
pipe: vi.fn(o => { output = o; }),
|
|
37
|
-
glob: vi.fn(),
|
|
38
|
-
finalize: vi.fn(() => { if (output?.path) writeFileSync(output.path, 'zip'); output?.emit?.('close'); }),
|
|
39
|
-
};
|
|
40
|
-
},
|
|
41
|
-
}));
|
|
42
|
-
|
|
43
|
-
vi.mock('../src/store.js', () => ({
|
|
44
|
-
selectedComputeFromDeployment: (dep) => ({
|
|
45
|
-
gpu: dep.gpu_type ?? null,
|
|
46
|
-
gpuCount: dep.gpu_count ?? null,
|
|
47
|
-
vcpus: dep.selected_vcpus ?? null,
|
|
48
|
-
ramGb: dep.selected_ram_gb ?? null,
|
|
49
|
-
vramGb: dep.selected_vram_gb ?? null,
|
|
50
|
-
}),
|
|
51
|
-
addDeployment: vi.fn(),
|
|
52
|
-
addReceipt: vi.fn(),
|
|
53
|
-
updateReceipt: vi.fn(),
|
|
54
|
-
generateReceiptId: vi.fn(() => 'rcpt-launch-001'),
|
|
55
|
-
generateDeploymentId: vi.fn(() => 'dep-launch-001'),
|
|
56
|
-
listDeployments: vi.fn(() => []),
|
|
57
|
-
listReceipts: vi.fn(() => []),
|
|
58
|
-
findDeployment: vi.fn(() => null),
|
|
59
|
-
updateDeployment: vi.fn(),
|
|
60
|
-
removeDeployment: vi.fn(),
|
|
61
|
-
}));
|
|
62
|
-
|
|
63
|
-
// Imported after mock declarations so we get the mocked versions
|
|
64
|
-
import * as api from '../src/api.js';
|
|
65
|
-
import * as store from '../src/store.js';
|
|
66
|
-
|
|
67
|
-
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
68
|
-
|
|
69
|
-
const config = { apiKey: 'sk-test', baseUrl: 'https://api.test/v1' };
|
|
70
|
-
|
|
71
|
-
// Chalk no-op (the real chalk would add ANSI codes, which makes assertions noisy)
|
|
72
|
-
const chalk = {
|
|
73
|
-
bold: s => s,
|
|
74
|
-
dim: s => s,
|
|
75
|
-
red: s => s,
|
|
76
|
-
yellow: s => s,
|
|
77
|
-
green: s => s,
|
|
78
|
-
cyan: s => s,
|
|
79
|
-
};
|
|
80
|
-
|
|
81
|
-
function makeDep(overrides = {}) {
|
|
82
|
-
return {
|
|
83
|
-
deployment_id: 'dep-launch-001',
|
|
84
|
-
status: 'running',
|
|
85
|
-
gpu_type: 'A100',
|
|
86
|
-
gpu_count: 1,
|
|
87
|
-
cost_per_hour: 2.50,
|
|
88
|
-
provider: 'runpod',
|
|
89
|
-
receipt_id: 'rcpt-launch-001',
|
|
90
|
-
tier: '1',
|
|
91
|
-
...overrides,
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// Standard happy-path sequence: POST /run → GET /deployments → GET /logs → (no DELETE needed for success)
|
|
96
|
-
function setupSuccessfulRun(depOverrides = {}, exitCode = 0) {
|
|
97
|
-
api.callApi
|
|
98
|
-
.mockResolvedValueOnce(makeDep(depOverrides)) // POST /run
|
|
99
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: exitCode }) // GET /deployments/…
|
|
100
|
-
.mockResolvedValueOnce({ logs: ['step 1', 'step 2'] }); // GET /deployments/…/logs
|
|
101
|
-
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// ── Setup / teardown ─────────────────────────────────────────────────────────
|
|
105
|
-
|
|
106
|
-
// Suppress MaxListenersExceededWarning — each runCommand call adds a SIGINT listener
|
|
107
|
-
// and we run many tests in sequence.
|
|
108
|
-
process.setMaxListeners(50);
|
|
109
|
-
|
|
110
|
-
beforeEach(() => {
|
|
111
|
-
// Explicitly list what to fake — vitest 4.x also fakes queueMicrotask and
|
|
112
|
-
// nextTick by default which deadlocks await-inside-while-loop patterns.
|
|
113
|
-
vi.useFakeTimers({
|
|
114
|
-
toFake: ['setTimeout', 'setInterval', 'clearTimeout', 'clearInterval', 'Date', 'performance', 'setImmediate', 'clearImmediate'],
|
|
115
|
-
});
|
|
116
|
-
process.exitCode = undefined;
|
|
117
|
-
vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
118
|
-
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
119
|
-
vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
|
120
|
-
// resetAllMocks clears both call history AND the mockResolvedValueOnce queue
|
|
121
|
-
// (clearAllMocks only clears history in vitest 4.x, leaving stale queued values
|
|
122
|
-
// that can corrupt subsequent tests).
|
|
123
|
-
vi.resetAllMocks();
|
|
124
|
-
// Restore default mock return values cleared by resetAllMocks
|
|
125
|
-
store.generateReceiptId.mockReturnValue('rcpt-launch-001');
|
|
126
|
-
store.generateDeploymentId.mockReturnValue('dep-launch-001');
|
|
127
|
-
store.listDeployments.mockReturnValue([]);
|
|
128
|
-
store.listReceipts.mockReturnValue([]);
|
|
129
|
-
store.findDeployment.mockReturnValue(null);
|
|
130
|
-
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
131
|
-
api.uploadBlob.mockResolvedValue({ code_uri: 'blob://project.zip' });
|
|
132
|
-
});
|
|
133
|
-
|
|
134
|
-
afterEach(() => {
|
|
135
|
-
vi.useRealTimers();
|
|
136
|
-
vi.restoreAllMocks();
|
|
137
|
-
process.exitCode = undefined;
|
|
138
|
-
});
|
|
139
|
-
|
|
140
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
141
|
-
// 1. GPU / provider matrix
|
|
142
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
143
|
-
|
|
144
|
-
describe('GPU / provider matrix', () => {
|
|
145
|
-
it('A100 via RunPod (tier-1) completes successfully', async () => {
|
|
146
|
-
setupSuccessfulRun({ gpu_type: 'A100', provider: 'runpod', tier: '1' });
|
|
147
|
-
const p = runCommand(config, ['python', 'train.py', '--gpu', 'A100', '--max-cost', '5'], chalk);
|
|
148
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
149
|
-
await p;
|
|
150
|
-
|
|
151
|
-
expect(api.callApi).toHaveBeenCalledWith('/run', expect.objectContaining({ method: 'POST' }));
|
|
152
|
-
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({
|
|
153
|
-
gpu: 'A100',
|
|
154
|
-
providerRoute: 'runpod',
|
|
155
|
-
tier: '1',
|
|
156
|
-
}));
|
|
157
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
158
|
-
status: 'completed',
|
|
159
|
-
exitCode: 0,
|
|
160
|
-
}));
|
|
161
|
-
expect(process.exitCode).toBeFalsy();
|
|
162
|
-
});
|
|
163
|
-
|
|
164
|
-
it('L40S via Vast.ai (tier-1) completes successfully', async () => {
|
|
165
|
-
api.callApi
|
|
166
|
-
.mockResolvedValueOnce(makeDep({ gpu_type: 'L40S', provider: 'vastai', tier: '1', cost_per_hour: 1.80 }))
|
|
167
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
168
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
169
|
-
const p = runCommand(config, ['python', 'train.py', '--gpu', 'L40S', '--max-cost', '5'], chalk);
|
|
170
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
171
|
-
await p;
|
|
172
|
-
|
|
173
|
-
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({
|
|
174
|
-
gpu: 'L40S',
|
|
175
|
-
providerRoute: 'vastai',
|
|
176
|
-
}));
|
|
177
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
178
|
-
status: 'completed',
|
|
179
|
-
exitCode: 0,
|
|
180
|
-
}));
|
|
181
|
-
expect(process.exitCode).toBeFalsy();
|
|
182
|
-
});
|
|
183
|
-
});
|
|
184
|
-
|
|
185
|
-
describe('CPU launch — price quoted before provisioning', () => {
|
|
186
|
-
it('shows the quoted Badgr rate before the deployment is created, and does not repeat it after', async () => {
|
|
187
|
-
api.quoteRun.mockResolvedValueOnce({ vm_class: 'small', vcpus: 2, memory_gb: 4, region: 'US', rate_per_hour: 0.13 });
|
|
188
|
-
api.callApi
|
|
189
|
-
.mockResolvedValueOnce(makeDep({ gpu_type: 'CPU', cost_per_hour: 0.13 })) // POST /run
|
|
190
|
-
.mockResolvedValueOnce({});
|
|
191
|
-
|
|
192
|
-
await launchCommand(config, ['cline', '--max-cost', '1', 'Fix the checkout bug'], chalk);
|
|
193
|
-
|
|
194
|
-
expect(api.quoteRun).toHaveBeenCalledWith(config, expect.objectContaining({ compute: 'cpu', agent: 'cline' }));
|
|
195
|
-
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
196
|
-
expect(logged).toContain('Badgr rate: $0.13/hour');
|
|
197
|
-
// The quoted rate is shown once, pre-provisioning — not repeated as the
|
|
198
|
-
// post-creation fallback line too.
|
|
199
|
-
expect(logged.match(/Badgr rate:/g)?.length).toBe(1);
|
|
200
|
-
});
|
|
201
|
-
|
|
202
|
-
it('still launches, falling back to the post-creation rate, when the quote call fails', async () => {
|
|
203
|
-
api.quoteRun.mockRejectedValueOnce(new Error('network error'));
|
|
204
|
-
api.callApi
|
|
205
|
-
.mockResolvedValueOnce(makeDep({ gpu_type: 'CPU', cost_per_hour: 0.13 })) // POST /run
|
|
206
|
-
.mockResolvedValueOnce({});
|
|
207
|
-
|
|
208
|
-
await launchCommand(config, ['cline', '--max-cost', '1', 'Fix the checkout bug'], chalk);
|
|
209
|
-
|
|
210
|
-
expect(process.exitCode).toBeFalsy();
|
|
211
|
-
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
212
|
-
expect(logged).toContain('Badgr rate: $0.13/hour');
|
|
213
|
-
});
|
|
214
|
-
});
|
|
215
|
-
|
|
216
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
217
|
-
// 2. Job exits non-zero
|
|
218
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
219
|
-
|
|
220
|
-
describe('non-zero exit code', () => {
|
|
221
|
-
it('records customer_code failure and sets exitCode', async () => {
|
|
222
|
-
api.callApi
|
|
223
|
-
.mockResolvedValueOnce(makeDep())
|
|
224
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
225
|
-
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
226
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
227
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
228
|
-
await p;
|
|
229
|
-
|
|
230
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
231
|
-
exitCode: 1,
|
|
232
|
-
failureType: 'customer_code',
|
|
233
|
-
}));
|
|
234
|
-
expect(process.exitCode).toBe(1);
|
|
235
|
-
});
|
|
236
|
-
|
|
237
|
-
it('reports "Teardown: succeeded" on a customer-code failure when the backend already confirmed teardown', async () => {
|
|
238
|
-
// The job-runner's own /complete webhook (complete_job_atomic) already
|
|
239
|
-
// attempts + confirms teardown on any exit code, before the CLI ever
|
|
240
|
-
// observes the failure via polling — teardown_ok in the poll response
|
|
241
|
-
// reflects that already-confirmed result.
|
|
242
|
-
api.callApi
|
|
243
|
-
.mockResolvedValueOnce(makeDep())
|
|
244
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: 1, teardown_ok: 'ok' })
|
|
245
|
-
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
246
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
247
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
248
|
-
await p;
|
|
249
|
-
|
|
250
|
-
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
251
|
-
expect(logged).toContain('succeeded');
|
|
252
|
-
});
|
|
253
|
-
|
|
254
|
-
it('reports "Teardown: failed" on a customer-code failure when the backend never confirmed deletion', async () => {
|
|
255
|
-
api.callApi
|
|
256
|
-
.mockResolvedValueOnce(makeDep())
|
|
257
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: 1, teardown_ok: 'failed' })
|
|
258
|
-
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
259
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
260
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
261
|
-
await p;
|
|
262
|
-
|
|
263
|
-
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
264
|
-
expect(logged).not.toContain('Teardown: succeeded');
|
|
265
|
-
expect(logged).toContain('failed');
|
|
266
|
-
});
|
|
267
|
-
|
|
268
|
-
it('writes an initial receipt for a job that ultimately fails', async () => {
|
|
269
|
-
api.callApi
|
|
270
|
-
.mockResolvedValueOnce(makeDep())
|
|
271
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
272
|
-
.mockResolvedValueOnce({ logs: ['Traceback (most recent call last)'] });
|
|
273
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
274
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
275
|
-
await p;
|
|
276
|
-
|
|
277
|
-
// A receipt must exist from the start of the job, not only once it fails —
|
|
278
|
-
// updateReceipt above mutates the same receipt addReceipt created here.
|
|
279
|
-
expect(store.addReceipt).toHaveBeenCalled();
|
|
280
|
-
});
|
|
281
|
-
|
|
282
|
-
it('distinguishes customer_code from infrastructure when exit code is null', async () => {
|
|
283
|
-
api.callApi
|
|
284
|
-
.mockResolvedValueOnce(makeDep())
|
|
285
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: null })
|
|
286
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
287
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
288
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
289
|
-
await p;
|
|
290
|
-
|
|
291
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
292
|
-
failureType: 'infrastructure',
|
|
293
|
-
}));
|
|
294
|
-
});
|
|
295
|
-
});
|
|
296
|
-
|
|
297
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
298
|
-
// 3. Container fails to start (infrastructure failure before running)
|
|
299
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
300
|
-
|
|
301
|
-
describe('container fails to start', () => {
|
|
302
|
-
it('reports infrastructure error when dep arrives already failed', async () => {
|
|
303
|
-
api.callApi
|
|
304
|
-
.mockResolvedValueOnce(makeDep({ status: 'failed' })) // POST /run returns already-failed
|
|
305
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
306
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
307
|
-
await vi.advanceTimersByTimeAsync(1000);
|
|
308
|
-
await p;
|
|
309
|
-
|
|
310
|
-
// runCommand exits with process.exitCode = 1 when dep.status === 'failed' after provision
|
|
311
|
-
expect(process.exitCode).toBe(1);
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
it('reports infrastructure error when waitForRunning resolves to failed', async () => {
|
|
315
|
-
api.callApi
|
|
316
|
-
.mockResolvedValueOnce(makeDep({ status: 'provisioning' })) // POST /run → provisioning
|
|
317
|
-
.mockResolvedValueOnce({ status: 'failed' }); // poll → failed
|
|
318
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
319
|
-
// advance past POLL_MS (3000ms) in waitForRunning
|
|
320
|
-
await vi.advanceTimersByTimeAsync(4000);
|
|
321
|
-
await p;
|
|
322
|
-
|
|
323
|
-
expect(process.exitCode).toBe(1);
|
|
324
|
-
});
|
|
325
|
-
|
|
326
|
-
it('prints the backend failure_class/next_action using the same vocabulary as the job API', async () => {
|
|
327
|
-
api.callApi
|
|
328
|
-
.mockResolvedValueOnce(makeDep({ status: 'provisioning' }))
|
|
329
|
-
.mockResolvedValueOnce({
|
|
330
|
-
status: 'failed',
|
|
331
|
-
failure_class: 'container_start_failed',
|
|
332
|
-
next_action: 'Check the image and command, then retry.',
|
|
333
|
-
});
|
|
334
|
-
const logs = [];
|
|
335
|
-
const origError = console.error;
|
|
336
|
-
console.error = (...args) => { logs.push(args.join(' ')); origError(...args); };
|
|
337
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
338
|
-
await vi.advanceTimersByTimeAsync(4000);
|
|
339
|
-
await p;
|
|
340
|
-
console.error = origError;
|
|
341
|
-
|
|
342
|
-
expect(logs.some(l => l.includes('Class: container_start_failed'))).toBe(true);
|
|
343
|
-
expect(logs.some(l => l.includes('Next: Check the image and command, then retry.'))).toBe(true);
|
|
344
|
-
});
|
|
345
|
-
});
|
|
346
|
-
|
|
347
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
348
|
-
// 4. Max-runtime cap
|
|
349
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
350
|
-
|
|
351
|
-
describe('max-runtime cap', () => {
|
|
352
|
-
it('stops job and records max-runtime when runtime exceeded', async () => {
|
|
353
|
-
// max-runtime 0.05 min = 3 seconds; ratePerHour = 2.50
|
|
354
|
-
api.callApi
|
|
355
|
-
.mockResolvedValueOnce(makeDep({ cost_per_hour: 2.50 })) // POST /run
|
|
356
|
-
.mockResolvedValueOnce({ status: 'running', exit_code: null }) // poll 1
|
|
357
|
-
.mockResolvedValueOnce({ logs: [] }); // logs 1
|
|
358
|
-
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
359
|
-
|
|
360
|
-
const p = runCommand(config, ['python', 'train.py', '--max-runtime', '0.05', '--max-cost', '5'], chalk);
|
|
361
|
-
// attachToJob POLL_MS = 4000, maxRuntime = 0.05 min = 3000ms → cap fires after 4000ms (first elapsedMs check)
|
|
362
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
363
|
-
await p;
|
|
364
|
-
|
|
365
|
-
expect(api.terminateDeployment).toHaveBeenCalledWith(config, 'dep-launch-001');
|
|
366
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
367
|
-
status: 'max-runtime',
|
|
368
|
-
teardownStatus: 'terminated',
|
|
369
|
-
}));
|
|
370
|
-
expect(process.exitCode).toBe(1);
|
|
371
|
-
});
|
|
372
|
-
});
|
|
373
|
-
|
|
374
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
375
|
-
// 5. Max-cost cap
|
|
376
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
377
|
-
|
|
378
|
-
describe('max-cost cap', () => {
|
|
379
|
-
it('stops job and records max-cost when budget exceeded', async () => {
|
|
380
|
-
// ratePerHour = 3600 ($1/s), maxCost = 0.001 → cap fires after 3.6ms, but poll fires at 4000ms
|
|
381
|
-
api.callApi
|
|
382
|
-
.mockResolvedValueOnce(makeDep({ cost_per_hour: 3600 })) // POST /run
|
|
383
|
-
.mockResolvedValueOnce({ status: 'running' }) // poll 1 (never reached — cap fires first)
|
|
384
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
385
|
-
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
386
|
-
|
|
387
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '0.001'], chalk);
|
|
388
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
389
|
-
await p;
|
|
390
|
-
|
|
391
|
-
expect(api.terminateDeployment).toHaveBeenCalledWith(config, 'dep-launch-001');
|
|
392
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
393
|
-
status: 'max-cost',
|
|
394
|
-
teardownStatus: 'terminated',
|
|
395
|
-
}));
|
|
396
|
-
expect(process.exitCode).toBe(1);
|
|
397
|
-
});
|
|
398
|
-
});
|
|
399
|
-
|
|
400
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
401
|
-
// 6. Heartbeat lost
|
|
402
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
403
|
-
|
|
404
|
-
describe('heartbeat lost', () => {
|
|
405
|
-
it('stops job and records heartbeat-lost after HEARTBEAT_KILL_POLLS consecutive errors', async () => {
|
|
406
|
-
// HEARTBEAT_KILL_POLLS = 15; POLL_MS = 4000ms
|
|
407
|
-
// First dep poll must succeed and return 'running' (to set lastStatus = 'running').
|
|
408
|
-
// After that, 15 consecutive failures trigger the kill.
|
|
409
|
-
// Total timer advances: (1 success + 15 failures) × 4000ms = 64000ms
|
|
410
|
-
api.callApi
|
|
411
|
-
.mockResolvedValueOnce(makeDep({ status: 'running' })) // POST /run
|
|
412
|
-
.mockResolvedValueOnce({ status: 'running' }) // first dep poll → sets lastStatus = 'running'
|
|
413
|
-
.mockResolvedValueOnce({ logs: [] }) // first logs poll
|
|
414
|
-
.mockRejectedValue(new Error('ETIMEDOUT')); // all subsequent dep/logs polls fail
|
|
415
|
-
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'ok' });
|
|
416
|
-
|
|
417
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
418
|
-
// Advance past 1 successful poll (4000ms) + 15 failed polls (60000ms) = 64000ms
|
|
419
|
-
await vi.advanceTimersByTimeAsync(66000);
|
|
420
|
-
await p;
|
|
421
|
-
|
|
422
|
-
expect(api.terminateDeployment).toHaveBeenCalledWith(config, 'dep-launch-001');
|
|
423
|
-
|
|
424
|
-
// teardown() writes first: billing stopped, status = 'heartbeat-lost'
|
|
425
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
426
|
-
status: 'heartbeat-lost',
|
|
427
|
-
teardownStatus: 'terminated',
|
|
428
|
-
}));
|
|
429
|
-
// runCommand writes second: final summary with failureType
|
|
430
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
431
|
-
failureType: 'infrastructure',
|
|
432
|
-
}));
|
|
433
|
-
expect(process.exitCode).toBeTruthy();
|
|
434
|
-
}, 30000 /* 30s real timeout — fake timers advance instantly but need sufficient budget */);
|
|
435
|
-
});
|
|
436
|
-
|
|
437
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
438
|
-
// 7. Log streaming
|
|
439
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
440
|
-
|
|
441
|
-
describe('log streaming', () => {
|
|
442
|
-
it('prints user-visible log lines and suppresses meta-lines', async () => {
|
|
443
|
-
const logLines = [];
|
|
444
|
-
console.log.mockImplementation((...args) => logLines.push(args.join(' ')));
|
|
445
|
-
|
|
446
|
-
api.callApi
|
|
447
|
-
.mockResolvedValueOnce(makeDep())
|
|
448
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
449
|
-
.mockResolvedValueOnce({ logs: [
|
|
450
|
-
'Epoch 1/3: loss=0.42', // should appear
|
|
451
|
-
'[dep-abc] status=running', // meta → should be filtered
|
|
452
|
-
'[dep-abc] gpu_util=72%', // meta → should be filtered
|
|
453
|
-
'Epoch 2/3: loss=0.31', // should appear
|
|
454
|
-
]});
|
|
455
|
-
|
|
456
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
457
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
458
|
-
await p;
|
|
459
|
-
|
|
460
|
-
const joined = logLines.join('\n');
|
|
461
|
-
expect(joined).toContain('Epoch 1/3: loss=0.42');
|
|
462
|
-
expect(joined).toContain('Epoch 2/3: loss=0.31');
|
|
463
|
-
// Meta-lines are filtered by the LOG_META_RE in run.js
|
|
464
|
-
expect(joined).not.toContain('[dep-abc] status=running');
|
|
465
|
-
});
|
|
466
|
-
});
|
|
467
|
-
|
|
468
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
469
|
-
// 8. Routing: tier-1 unavailable → auto-expand to tier-2
|
|
470
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
471
|
-
|
|
472
|
-
describe('routing fallback', () => {
|
|
473
|
-
it('expands to tier-2 when tier-1 returns NO_CAPACITY_MATCH', async () => {
|
|
474
|
-
const capacityErr = Object.assign(new Error('no capacity'), {
|
|
475
|
-
errorData: { code: 'NO_CAPACITY_MATCH' },
|
|
476
|
-
httpStatus: 503,
|
|
477
|
-
});
|
|
478
|
-
api.callApi
|
|
479
|
-
.mockRejectedValueOnce(capacityErr) // tier-1 attempt
|
|
480
|
-
.mockResolvedValueOnce(makeDep({ provider: 'vastai', tier: '2' })) // tier-2 succeeds
|
|
481
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
482
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
483
|
-
|
|
484
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
485
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
486
|
-
await p;
|
|
487
|
-
|
|
488
|
-
// Should have been called twice: once for tier-1, once for tier-2
|
|
489
|
-
const postCalls = api.callApi.mock.calls.filter(c => c[1]?.method === 'POST');
|
|
490
|
-
expect(postCalls.length).toBeGreaterThanOrEqual(2);
|
|
491
|
-
|
|
492
|
-
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({
|
|
493
|
-
providerRoute: 'vastai',
|
|
494
|
-
tier: '2',
|
|
495
|
-
}));
|
|
496
|
-
expect(process.exitCode).toBeFalsy();
|
|
497
|
-
});
|
|
498
|
-
|
|
499
|
-
it('reports CapacityError when both tiers fail', async () => {
|
|
500
|
-
const capacityErr = Object.assign(new Error('no capacity'), {
|
|
501
|
-
errorData: { code: 'NO_CAPACITY_MATCH' },
|
|
502
|
-
httpStatus: 503,
|
|
503
|
-
});
|
|
504
|
-
api.callApi.mockRejectedValue(capacityErr);
|
|
505
|
-
|
|
506
|
-
const logs = [];
|
|
507
|
-
console.error.mockImplementation(msg => logs.push(msg));
|
|
508
|
-
|
|
509
|
-
await runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
510
|
-
|
|
511
|
-
expect(process.exitCode).toBe(1);
|
|
512
|
-
const combined = logs.join('\n');
|
|
513
|
-
expect(combined).toMatch(/No suitable GPU capacity|capacity/i);
|
|
514
|
-
});
|
|
515
|
-
|
|
516
|
-
it('does not fall back to tier-2 when --no-fallback is set', async () => {
|
|
517
|
-
const capacityErr = Object.assign(new Error('no capacity'), {
|
|
518
|
-
errorData: { code: 'NO_CAPACITY_MATCH' },
|
|
519
|
-
httpStatus: 503,
|
|
520
|
-
});
|
|
521
|
-
api.callApi.mockRejectedValueOnce(capacityErr);
|
|
522
|
-
|
|
523
|
-
await runCommand(config, ['python', 'train.py', '--no-fallback', '--max-cost', '5'], chalk);
|
|
524
|
-
|
|
525
|
-
// Only one POST /run call was made (no tier-2 expansion)
|
|
526
|
-
const postCalls = api.callApi.mock.calls.filter(c => c[1]?.method === 'POST');
|
|
527
|
-
expect(postCalls.length).toBe(1);
|
|
528
|
-
expect(process.exitCode).toBe(1);
|
|
529
|
-
});
|
|
530
|
-
|
|
531
|
-
it('PROVISIONING_FAILED error is surfaced clearly without showing internal debug info by default', async () => {
|
|
532
|
-
const provErr = Object.assign(new Error('provisioning failed'), {
|
|
533
|
-
errorData: { code: 'PROVISIONING_FAILED', debug_error: 'CUDA driver mismatch' },
|
|
534
|
-
httpStatus: 503,
|
|
535
|
-
});
|
|
536
|
-
api.callApi.mockRejectedValue(provErr);
|
|
537
|
-
|
|
538
|
-
const errLines = [];
|
|
539
|
-
console.error.mockImplementation(msg => errLines.push(msg));
|
|
540
|
-
|
|
541
|
-
await runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
542
|
-
|
|
543
|
-
expect(process.exitCode).toBe(1);
|
|
544
|
-
const combined = errLines.join('\n');
|
|
545
|
-
expect(combined).not.toContain('CUDA driver mismatch'); // debug info hidden by default
|
|
546
|
-
expect(combined).toMatch(/try again|failed to start/i);
|
|
547
|
-
});
|
|
548
|
-
});
|
|
549
|
-
|
|
550
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
551
|
-
// 9. Payment required
|
|
552
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
553
|
-
|
|
554
|
-
describe('payment required', () => {
|
|
555
|
-
it('shows billing message and rerun hint, exits 1', async () => {
|
|
556
|
-
const payErr = Object.assign(new Error('Payment required'), {
|
|
557
|
-
isPaymentRequired: true,
|
|
558
|
-
httpStatus: 402,
|
|
559
|
-
});
|
|
560
|
-
api.callApi.mockRejectedValueOnce(payErr);
|
|
561
|
-
|
|
562
|
-
const errLines = [];
|
|
563
|
-
console.error.mockImplementation(msg => errLines.push(msg));
|
|
564
|
-
|
|
565
|
-
await runCommand(config, ['python', 'train.py', '--gpu', 'A100', '--max-cost', '5'], chalk);
|
|
566
|
-
|
|
567
|
-
expect(process.exitCode).toBe(1);
|
|
568
|
-
const combined = errLines.join('\n');
|
|
569
|
-
expect(combined).toContain('Payment required');
|
|
570
|
-
});
|
|
571
|
-
});
|
|
572
|
-
|
|
573
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
574
|
-
// 10. Billing stops — terminateDeployment called on every terminal path
|
|
575
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
576
|
-
|
|
577
|
-
describe('billing lifecycle', () => {
|
|
578
|
-
it('terminateDeployment is called after successful completion', async () => {
|
|
579
|
-
setupSuccessfulRun();
|
|
580
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
581
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
582
|
-
await p;
|
|
583
|
-
expect(api.terminateDeployment).toHaveBeenCalledWith(config, 'dep-launch-001');
|
|
584
|
-
});
|
|
585
|
-
|
|
586
|
-
it('receipt records runtime and finalCost on completion', async () => {
|
|
587
|
-
setupSuccessfulRun({ cost_per_hour: 1.80 });
|
|
588
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
589
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
590
|
-
await p;
|
|
591
|
-
|
|
592
|
-
const updateCall = store.updateReceipt.mock.calls.find(
|
|
593
|
-
c => c[1]?.runtimeSeconds !== undefined,
|
|
594
|
-
);
|
|
595
|
-
expect(updateCall).toBeTruthy();
|
|
596
|
-
expect(typeof updateCall[1].runtimeSeconds).toBe('number');
|
|
597
|
-
expect(typeof updateCall[1].finalCost).toBe('number');
|
|
598
|
-
});
|
|
599
|
-
|
|
600
|
-
it('reports "Teardown: failed" — never "succeeded" — when the backend accepted the DELETE but did not confirm deletion', async () => {
|
|
601
|
-
// A 200 response with teardown_ok: 'failed' must never be reported as a
|
|
602
|
-
// successful teardown — the caller only knows deletion was *requested*.
|
|
603
|
-
setupSuccessfulRun();
|
|
604
|
-
api.terminateDeployment.mockResolvedValue({ teardown_ok: 'failed' });
|
|
605
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
606
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
607
|
-
await p;
|
|
608
|
-
|
|
609
|
-
const logged = console.log.mock.calls.map(c => c.join(' ')).join('\n');
|
|
610
|
-
expect(logged).not.toContain('Teardown: succeeded');
|
|
611
|
-
expect(logged).toContain('failed');
|
|
612
|
-
});
|
|
613
|
-
});
|
|
614
|
-
|
|
615
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
616
|
-
// 11. Input validation before provisioning
|
|
617
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
618
|
-
|
|
619
|
-
describe('input validation', () => {
|
|
620
|
-
it('rejects --count 0 before any API call', async () => {
|
|
621
|
-
await runCommand(config, ['python', 'train.py', '--count', '0'], chalk);
|
|
622
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
623
|
-
expect(process.exitCode).toBe(1);
|
|
624
|
-
});
|
|
625
|
-
|
|
626
|
-
it('rejects --max-cost 0 before any API call', async () => {
|
|
627
|
-
await runCommand(config, ['python', 'train.py', '--max-cost', '0'], chalk);
|
|
628
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
629
|
-
expect(process.exitCode).toBe(1);
|
|
630
|
-
});
|
|
631
|
-
|
|
632
|
-
it('rejects unknown --region before any API call', async () => {
|
|
633
|
-
await runCommand(config, ['python', 'train.py', '--region', 'MARS'], chalk);
|
|
634
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
635
|
-
expect(process.exitCode).toBe(1);
|
|
636
|
-
});
|
|
637
|
-
|
|
638
|
-
it('rejects empty command with no --image', async () => {
|
|
639
|
-
await runCommand(config, [], chalk);
|
|
640
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
641
|
-
});
|
|
642
|
-
|
|
643
|
-
it('rejects -- with nothing after and no --image', async () => {
|
|
644
|
-
await runCommand(config, ['--gpu', 'RTX_4090', '--max-cost', '1', '--'], chalk);
|
|
645
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
646
|
-
expect(process.exitCode).toBe(1);
|
|
647
|
-
});
|
|
648
|
-
});
|
|
649
|
-
|
|
650
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
651
|
-
// 12. --dry-run: shows config, never provisions
|
|
652
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
653
|
-
|
|
654
|
-
describe('--dry-run', () => {
|
|
655
|
-
it('prints dry-run summary and never calls the API', async () => {
|
|
656
|
-
const output = [];
|
|
657
|
-
const dryChalk = { bold: (s) => s, dim: (s) => s, cyan: (s) => s, red: (s) => s, yellow: (s) => s, green: (s) => s };
|
|
658
|
-
const origLog = console.log;
|
|
659
|
-
console.log = (...args) => output.push(args.join(' '));
|
|
660
|
-
await runCommand(config, [
|
|
661
|
-
'--dry-run', '--gpu', 'RTX_4090', '--image', 'node:20', '--max-cost', '1', '--',
|
|
662
|
-
'node', '-e', "console.log('dry')",
|
|
663
|
-
], dryChalk);
|
|
664
|
-
console.log = origLog;
|
|
665
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
666
|
-
expect(output.some(l => /dry run/i.test(l))).toBe(true);
|
|
667
|
-
expect(output.some(l => /node -e/.test(l) || /console\.log/.test(l))).toBe(true);
|
|
668
|
-
expect(output.some(l => /RTX_4090/.test(l))).toBe(true);
|
|
669
|
-
});
|
|
670
|
-
|
|
671
|
-
it('--dry-run works without --max-cost', async () => {
|
|
672
|
-
const dryChalk = { bold: (s) => s, dim: (s) => s, cyan: (s) => s, red: (s) => s, yellow: (s) => s, green: (s) => s };
|
|
673
|
-
await runCommand(config, [
|
|
674
|
-
'--dry-run', '--gpu', 'A100', '--image', 'node:20', '--',
|
|
675
|
-
'node', 'script.js',
|
|
676
|
-
], dryChalk);
|
|
677
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
678
|
-
});
|
|
679
|
-
});
|
|
680
|
-
|
|
681
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
682
|
-
// 13. Progress-loss handling — --output / --checkpoint / --retry-safe / --resume-cmd
|
|
683
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
684
|
-
|
|
685
|
-
describe('--output / --checkpoint / --retry-safe / --resume-cmd', () => {
|
|
686
|
-
it('parseRunArgs parses all four flags', () => {
|
|
687
|
-
const { flags } = parseRunArgs([
|
|
688
|
-
'--output', './outputs', '--checkpoint', './checkpoints', '--retry-safe',
|
|
689
|
-
'--resume-cmd', 'python train.py --resume ./checkpoints/latest',
|
|
690
|
-
'--max-cost', '5', '--', 'python', 'train.py',
|
|
691
|
-
]);
|
|
692
|
-
expect(flags.output).toBe('./outputs');
|
|
693
|
-
expect(flags.checkpoint).toBe('./checkpoints');
|
|
694
|
-
expect(flags.retrySafe).toBe(true);
|
|
695
|
-
expect(flags.resumeCmd).toBe('python train.py --resume ./checkpoints/latest');
|
|
696
|
-
});
|
|
697
|
-
|
|
698
|
-
it('wires --output/--checkpoint/--retry-safe into the job env as BADGR_* vars', async () => {
|
|
699
|
-
api.callApi
|
|
700
|
-
.mockResolvedValueOnce(makeDep())
|
|
701
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
702
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
703
|
-
const p = runCommand(config, [
|
|
704
|
-
'--output', './outputs', '--checkpoint', './checkpoints', '--retry-safe',
|
|
705
|
-
'--max-cost', '5', '--', 'python', 'train.py',
|
|
706
|
-
], chalk);
|
|
707
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
708
|
-
await p;
|
|
709
|
-
|
|
710
|
-
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
711
|
-
expect(postRunBody.env).toEqual(expect.objectContaining({
|
|
712
|
-
BADGR_OUTPUT_DIR: './outputs',
|
|
713
|
-
BADGR_CHECKPOINT_DIR: './checkpoints',
|
|
714
|
-
BADGR_RETRY_SAFE: '1',
|
|
715
|
-
}));
|
|
716
|
-
});
|
|
717
|
-
|
|
718
|
-
it('explicit --env overrides the BADGR_* convention var for the same key', async () => {
|
|
719
|
-
api.callApi
|
|
720
|
-
.mockResolvedValueOnce(makeDep())
|
|
721
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
722
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
723
|
-
const p = runCommand(config, [
|
|
724
|
-
'--output', './outputs', '--env', 'BADGR_OUTPUT_DIR=./custom-outputs',
|
|
725
|
-
'--max-cost', '5', '--', 'python', 'train.py',
|
|
726
|
-
], chalk);
|
|
727
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
728
|
-
await p;
|
|
729
|
-
|
|
730
|
-
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
731
|
-
expect(postRunBody.env.BADGR_OUTPUT_DIR).toBe('./custom-outputs');
|
|
732
|
-
});
|
|
733
|
-
|
|
734
|
-
it('prints the --resume-cmd hint when the job fails', async () => {
|
|
735
|
-
api.callApi
|
|
736
|
-
.mockResolvedValueOnce(makeDep())
|
|
737
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
738
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
739
|
-
const logs = [];
|
|
740
|
-
const captureChalk = { bold: s => s, dim: s => s, red: s => s, yellow: s => s, green: s => s, cyan: s => s };
|
|
741
|
-
const origLog = console.log;
|
|
742
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
743
|
-
const p = runCommand(config, [
|
|
744
|
-
'--resume-cmd', 'python train.py --resume ./checkpoints/latest',
|
|
745
|
-
'--max-cost', '5', '--', 'python', 'train.py',
|
|
746
|
-
], captureChalk);
|
|
747
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
748
|
-
await p;
|
|
749
|
-
console.log = origLog;
|
|
750
|
-
|
|
751
|
-
expect(logs.some(l => l.includes('python train.py --resume ./checkpoints/latest'))).toBe(true);
|
|
752
|
-
});
|
|
753
|
-
|
|
754
|
-
it('does not print a Resume line when --resume-cmd was not given', async () => {
|
|
755
|
-
api.callApi
|
|
756
|
-
.mockResolvedValueOnce(makeDep())
|
|
757
|
-
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
758
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
759
|
-
const logs = [];
|
|
760
|
-
const origLog = console.log;
|
|
761
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
762
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
763
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
764
|
-
await p;
|
|
765
|
-
console.log = origLog;
|
|
766
|
-
|
|
767
|
-
expect(logs.some(l => l.includes('Resume:'))).toBe(false);
|
|
768
|
-
});
|
|
769
|
-
});
|
|
770
|
-
|
|
771
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
772
|
-
// 13b. --cpu / --memory / --gpu-memory — CPU/RAM/VRAM resource matching
|
|
773
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
774
|
-
|
|
775
|
-
describe('--cpu / --memory / --gpu-memory', () => {
|
|
776
|
-
it('wires --cpu and --memory into the /run request body as cpu/memory_gb', async () => {
|
|
777
|
-
setupSuccessfulRun();
|
|
778
|
-
const p = runCommand(config, [
|
|
779
|
-
'--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--', 'python', 'train.py',
|
|
780
|
-
], chalk);
|
|
781
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
782
|
-
await p;
|
|
783
|
-
|
|
784
|
-
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
785
|
-
expect(postRunBody.cpu).toBe(16);
|
|
786
|
-
expect(postRunBody.memory_gb).toBe(64);
|
|
787
|
-
});
|
|
788
|
-
|
|
789
|
-
it('--gpu-memory sets min_vram the same way --min-vram does', async () => {
|
|
790
|
-
setupSuccessfulRun();
|
|
791
|
-
const p = runCommand(config, [
|
|
792
|
-
'--gpu-memory', '24GB', '--max-cost', '5', '--', 'python', 'train.py',
|
|
793
|
-
], chalk);
|
|
794
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
795
|
-
await p;
|
|
796
|
-
|
|
797
|
-
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
798
|
-
expect(postRunBody.min_vram).toBe(24);
|
|
799
|
-
});
|
|
800
|
-
|
|
801
|
-
it('omits cpu/memory_gb/min_vram from the body when none were requested', async () => {
|
|
802
|
-
setupSuccessfulRun();
|
|
803
|
-
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
804
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
805
|
-
await p;
|
|
806
|
-
|
|
807
|
-
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
808
|
-
expect(postRunBody.cpu).toBeUndefined();
|
|
809
|
-
expect(postRunBody.memory_gb).toBeUndefined();
|
|
810
|
-
expect(postRunBody.min_vram).toBeUndefined();
|
|
811
|
-
});
|
|
812
|
-
|
|
813
|
-
it('--dry-run shows CPU/Memory/Min VRAM without calling the API', async () => {
|
|
814
|
-
const output = [];
|
|
815
|
-
const dryChalk = { bold: (s) => s, dim: (s) => s, cyan: (s) => s, red: (s) => s, yellow: (s) => s, green: (s) => s };
|
|
816
|
-
const origLog = console.log;
|
|
817
|
-
console.log = (...args) => output.push(args.join(' '));
|
|
818
|
-
await runCommand(config, [
|
|
819
|
-
'--dry-run', '--cpu', '16', '--memory', '64GB', '--gpu-memory', '24GB',
|
|
820
|
-
'--image', 'node:20', '--max-cost', '1', '--', 'node', '-e', "console.log('dry')",
|
|
821
|
-
], dryChalk);
|
|
822
|
-
console.log = origLog;
|
|
823
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
824
|
-
expect(output.some(l => /CPU:.*16 cores/.test(l))).toBe(true);
|
|
825
|
-
expect(output.some(l => /Memory:.*64 GB/.test(l))).toBe(true);
|
|
826
|
-
expect(output.some(l => /Min VRAM:.*24 GB/.test(l))).toBe(true);
|
|
827
|
-
});
|
|
828
|
-
|
|
829
|
-
it('rejects an unparseable --memory value before touching the API', async () => {
|
|
830
|
-
await runCommand(config, ['python', 'train.py', '--memory', 'not-a-size', '--max-cost', '5'], chalk);
|
|
831
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
832
|
-
expect(process.exitCode).toBe(1);
|
|
833
|
-
});
|
|
834
|
-
|
|
835
|
-
it('rejects an unparseable --gpu-memory value before touching the API', async () => {
|
|
836
|
-
await runCommand(config, ['python', 'train.py', '--gpu-memory', 'huge', '--max-cost', '5'], chalk);
|
|
837
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
838
|
-
expect(process.exitCode).toBe(1);
|
|
839
|
-
});
|
|
840
|
-
|
|
841
|
-
it('rejects a non-integer --cpu before touching the API', async () => {
|
|
842
|
-
await runCommand(config, ['python', 'train.py', '--cpu', 'sixteen', '--max-cost', '5'], chalk);
|
|
843
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
844
|
-
expect(process.exitCode).toBe(1);
|
|
845
|
-
});
|
|
846
|
-
|
|
847
|
-
it('rejects --no-gpu combined with --gpu', async () => {
|
|
848
|
-
await runCommand(config, ['python', 'sim.py', '--no-gpu', '--gpu', 'A100', '--max-cost', '5'], chalk);
|
|
849
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
850
|
-
expect(process.exitCode).toBe(1);
|
|
851
|
-
});
|
|
852
|
-
|
|
853
|
-
it('--no-gpu sends no_gpu:true and no gpu-specific fields beyond the default', async () => {
|
|
854
|
-
setupSuccessfulRun();
|
|
855
|
-
const p = runCommand(config, [
|
|
856
|
-
'--no-gpu', '--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--', 'python', 'sim.py',
|
|
857
|
-
], chalk);
|
|
858
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
859
|
-
await p;
|
|
860
|
-
|
|
861
|
-
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
862
|
-
expect(postRunBody.no_gpu).toBe(true);
|
|
863
|
-
expect(postRunBody.cpu).toBe(16);
|
|
864
|
-
expect(postRunBody.memory_gb).toBe(64);
|
|
865
|
-
});
|
|
866
|
-
|
|
867
|
-
it('shows what was actually provisioned when a resource floor was requested', async () => {
|
|
868
|
-
api.callApi
|
|
869
|
-
.mockResolvedValueOnce(makeDep({ gpu_type: 'A100', gpu_count: 1, selected_vcpus: 16, selected_ram_gb: 64, selected_vram_gb: 40 }))
|
|
870
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
871
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
872
|
-
const logs = [];
|
|
873
|
-
const origLog = console.log;
|
|
874
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
875
|
-
const p = runCommand(config, [
|
|
876
|
-
'--gpu-memory', '24GB', '--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--', 'python', 'train.py',
|
|
877
|
-
], chalk);
|
|
878
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
879
|
-
await p;
|
|
880
|
-
console.log = origLog;
|
|
881
|
-
|
|
882
|
-
expect(logs.some(l => l.includes('Provisioned:') && l.includes('A100') && l.includes('40GB VRAM') && l.includes('16 vCPU') && l.includes('64GB RAM'))).toBe(true);
|
|
883
|
-
});
|
|
884
|
-
|
|
885
|
-
it('does not print a Provisioned line when no resource floor was requested', async () => {
|
|
886
|
-
setupSuccessfulRun();
|
|
887
|
-
const logs = [];
|
|
888
|
-
const origLog = console.log;
|
|
889
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
890
|
-
const p = runCommand(config, ['python', 'train.py', '--gpu', 'A100', '--max-cost', '5'], chalk);
|
|
891
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
892
|
-
await p;
|
|
893
|
-
console.log = origLog;
|
|
894
|
-
|
|
895
|
-
expect(logs.some(l => l.includes('Provisioned:'))).toBe(false);
|
|
896
|
-
});
|
|
897
|
-
|
|
898
|
-
it('records workload shape and requested/selected compute on the receipt', async () => {
|
|
899
|
-
api.callApi
|
|
900
|
-
.mockResolvedValueOnce(makeDep({ gpu_type: 'A100', gpu_count: 1, selected_vcpus: 16, selected_ram_gb: 64, selected_vram_gb: 40 }))
|
|
901
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
902
|
-
.mockResolvedValueOnce({ logs: [] });
|
|
903
|
-
const p = runCommand(config, [
|
|
904
|
-
'--gpu-memory', '24GB', '--cpu', '16', '--memory', '64GB', '--max-cost', '5', '--image', 'node:20', '--', 'node', 'train.js',
|
|
905
|
-
], chalk);
|
|
906
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
907
|
-
await p;
|
|
908
|
-
|
|
909
|
-
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({
|
|
910
|
-
workloadShape: 'container',
|
|
911
|
-
computeRequested: expect.objectContaining({ minVram: 24, cpu: 16, memoryGb: 64, noGpu: false }),
|
|
912
|
-
computeSelected: expect.objectContaining({ gpu: 'A100', vcpus: 16, ramGb: 64, vramGb: 40 }),
|
|
913
|
-
}));
|
|
914
|
-
});
|
|
915
|
-
|
|
916
|
-
it('tags a local-path run as workloadShape "project"', async () => {
|
|
917
|
-
setupSuccessfulRun();
|
|
918
|
-
const p = runCommand(config, ['.', '--cmd', 'python train.py', '--max-cost', '5'], chalk);
|
|
919
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
920
|
-
await p;
|
|
921
|
-
|
|
922
|
-
expect(store.addReceipt).toHaveBeenCalledWith(expect.objectContaining({ workloadShape: 'project' }));
|
|
923
|
-
});
|
|
924
|
-
});
|
|
925
|
-
|
|
926
|
-
describe('badgr launch (CPU VM)', () => {
|
|
927
|
-
it('maps `badgr launch . -- <command>` to a detached CPU project run', async () => {
|
|
928
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
929
|
-
const p = launchCommand(config, ['.', '--max-cost', '5', '--', 'claude', '-p', 'Fix the failing tests'], chalk);
|
|
930
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
931
|
-
await p;
|
|
932
|
-
|
|
933
|
-
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
934
|
-
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
935
|
-
expect(body.compute).toBe('cpu');
|
|
936
|
-
expect(body.gpu).toBe('CPU');
|
|
937
|
-
expect(body.cmd).toBe('claude -p Fix the failing tests');
|
|
938
|
-
expect(body.code_uri).toBeTruthy();
|
|
939
|
-
expect(body.agent).toBeUndefined();
|
|
940
|
-
expect(body.task).toBeUndefined();
|
|
941
|
-
});
|
|
942
|
-
|
|
943
|
-
it('defaults to detached (returns without waiting on the job)', async () => {
|
|
944
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
945
|
-
const logs = [];
|
|
946
|
-
const origLog = console.log;
|
|
947
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
948
|
-
await launchCommand(config, ['.', '--max-cost', '5', '--', 'python', 'narrgo.py'], chalk);
|
|
949
|
-
console.log = origLog;
|
|
950
|
-
|
|
951
|
-
expect(logs.some(l => l.includes('Detached'))).toBe(true);
|
|
952
|
-
});
|
|
953
|
-
|
|
954
|
-
it('accepts the quoted --cmd form as equivalent to -- passthrough', async () => {
|
|
955
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
956
|
-
const p = launchCommand(config, ['.', '--max-cost', '5', '--cmd', 'codex exec "Write tests"'], chalk);
|
|
957
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
958
|
-
await p;
|
|
959
|
-
|
|
960
|
-
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
961
|
-
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
962
|
-
expect(body.cmd).toBe('codex exec "Write tests"');
|
|
963
|
-
});
|
|
964
|
-
|
|
965
|
-
it('sends declared --artifacts paths as output_paths in the /run request body', async () => {
|
|
966
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
967
|
-
const p = launchCommand(config, [
|
|
968
|
-
'.', '--max-cost', '5',
|
|
969
|
-
'--artifacts', 'playwright-report', '--artifacts', 'test-results',
|
|
970
|
-
'--', 'npx', 'playwright', 'test',
|
|
971
|
-
], chalk);
|
|
972
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
973
|
-
await p;
|
|
974
|
-
|
|
975
|
-
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
976
|
-
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
977
|
-
expect(body.output_paths).toEqual(['playwright-report', 'test-results']);
|
|
978
|
-
});
|
|
979
|
-
|
|
980
|
-
it('does not send output_paths when --artifacts is not passed', async () => {
|
|
981
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
982
|
-
const p = launchCommand(config, ['.', '--max-cost', '5', '--', 'npm', 'test'], chalk);
|
|
983
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
984
|
-
await p;
|
|
985
|
-
|
|
986
|
-
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
987
|
-
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
988
|
-
expect(body.output_paths).toBeUndefined();
|
|
989
|
-
});
|
|
990
|
-
|
|
991
|
-
it('--no-detach streams the job to completion instead of returning immediately', async () => {
|
|
992
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
993
|
-
const logs = [];
|
|
994
|
-
const origLog = console.log;
|
|
995
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
996
|
-
const p = launchCommand(config, ['.', '--max-cost', '5', '--no-detach', '--', 'npm', 'test'], chalk);
|
|
997
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
998
|
-
await p;
|
|
999
|
-
console.log = origLog;
|
|
1000
|
-
|
|
1001
|
-
expect(logs.some(l => l.includes('Detached'))).toBe(false);
|
|
1002
|
-
expect(logs.some(l => l.includes('Complete'))).toBe(true);
|
|
1003
|
-
});
|
|
1004
|
-
|
|
1005
|
-
it('rejects --gpu end to end without ever calling the API', async () => {
|
|
1006
|
-
await launchCommand(config, ['.', '--gpu', 'A100', '--max-cost', '5', '--', 'npm', 'test'], chalk);
|
|
1007
|
-
expect(process.exitCode).toBe(1);
|
|
1008
|
-
expect(api.callApi).not.toHaveBeenCalled();
|
|
1009
|
-
process.exitCode = undefined;
|
|
1010
|
-
});
|
|
1011
|
-
|
|
1012
|
-
it('prints a secret-shaped --env warning before submitting the run', async () => {
|
|
1013
|
-
setupSuccessfulRun({ gpu_type: 'CPU', cost_per_hour: 0.10 });
|
|
1014
|
-
const logs = [];
|
|
1015
|
-
const origLog = console.log;
|
|
1016
|
-
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
1017
|
-
const p = launchCommand(config, [
|
|
1018
|
-
'.', '--max-cost', '5', '--env', 'ANTHROPIC_API_KEY=sk-ant-fake',
|
|
1019
|
-
'--', 'claude', '-p', 'fix',
|
|
1020
|
-
], chalk);
|
|
1021
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
1022
|
-
await p;
|
|
1023
|
-
console.log = origLog;
|
|
1024
|
-
|
|
1025
|
-
const joinedLogs = logs.join('\n');
|
|
1026
|
-
expect(joinedLogs).toContain('ANTHROPIC_API_KEY');
|
|
1027
|
-
expect(joinedLogs).toContain('shell history');
|
|
1028
|
-
const [, opts] = api.callApi.mock.calls.find(([path]) => path === '/run');
|
|
1029
|
-
const body = typeof opts.body === 'string' ? JSON.parse(opts.body) : opts.body;
|
|
1030
|
-
expect(body.env.ANTHROPIC_API_KEY).toBe('sk-ant-fake');
|
|
1031
|
-
});
|
|
1032
|
-
|
|
1033
|
-
it('a failed CPU launch command still updates the receipt with a usable status', async () => {
|
|
1034
|
-
api.callApi
|
|
1035
|
-
.mockResolvedValueOnce({
|
|
1036
|
-
deployment_id: 'dep-launch-001', status: 'running', gpu_type: 'CPU',
|
|
1037
|
-
cost_per_hour: 0.10, receipt_id: 'rcpt-launch-001', tier: '1',
|
|
1038
|
-
})
|
|
1039
|
-
.mockResolvedValueOnce({ status: 'completed', exit_code: 1 })
|
|
1040
|
-
.mockResolvedValueOnce({ logs: ['error: test failed'] });
|
|
1041
|
-
|
|
1042
|
-
const p = launchCommand(config, ['.', '--max-cost', '5', '--no-detach', '--', 'npm', 'test'], chalk);
|
|
1043
|
-
await vi.advanceTimersByTimeAsync(5000);
|
|
1044
|
-
await p;
|
|
1045
|
-
|
|
1046
|
-
expect(store.updateReceipt).toHaveBeenCalledWith('rcpt-launch-001', expect.objectContaining({
|
|
1047
|
-
status: 'completed',
|
|
1048
|
-
exitCode: 1,
|
|
1049
|
-
failureType: 'customer_code',
|
|
1050
|
-
}));
|
|
1051
|
-
expect(process.exitCode).toBe(1);
|
|
1052
|
-
process.exitCode = undefined;
|
|
1053
|
-
});
|
|
1054
|
-
});
|