badgr-cli 1.0.30 → 1.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_IT_WORKS.md +4 -4
- package/package.json +4 -12
- package/src/api.js +10 -87
- package/src/badgr.js +37 -67
- package/src/commands/down.js +23 -26
- package/src/commands/login.js +7 -23
- package/src/commands/logs.js +5 -57
- package/src/commands/models.js +6 -25
- package/src/commands/receipts.js +4 -19
- package/src/commands/run.js +90 -458
- package/src/commands/serve.js +60 -132
- package/src/commands/status.js +48 -35
- package/src/commands/up.js +26 -32
- package/src/config.js +4 -49
- package/src/router.js +73 -16
- package/src/store.js +1 -10
- package/tests/commands.test.js +2 -183
- package/tests/config.test.js +1 -24
- package/tests/router.test.js +68 -9
- package/tests/store.test.js +1 -41
- package/src/commands/billing.js +0 -93
- package/src/commands/capacity.js +0 -111
- package/src/commands/test-run.js +0 -240
- package/src/fallback.js +0 -170
package/src/commands/run.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { requireApiKey } from '../config.js';
|
|
2
|
-
import { callApi
|
|
3
|
-
import { addReceipt,
|
|
4
|
-
import { normalizeTier, callWithFallback, HIGH_RATE_THRESHOLD } from '../fallback.js';
|
|
2
|
+
import { callApi } from '../api.js';
|
|
3
|
+
import { addReceipt, generateReceiptId } from '../store.js';
|
|
5
4
|
|
|
6
5
|
/**
|
|
7
|
-
* badgr run python train.py
|
|
8
|
-
* badgr run python train.py --gpu A100 # specific GPU
|
|
6
|
+
* badgr run python train.py --gpu A100 # attached (default)
|
|
9
7
|
* badgr run --image my/image:latest --gpu L40S --detach
|
|
8
|
+
*
|
|
9
|
+
* Attaches by default: polls status + streams logs until the job finishes,
|
|
10
|
+
* then exits with the job's exit code. Pass --detach to return immediately.
|
|
10
11
|
*/
|
|
11
12
|
export function parseRunArgs(args) {
|
|
12
13
|
const flags = {};
|
|
@@ -17,292 +18,62 @@ export function parseRunArgs(args) {
|
|
|
17
18
|
if (args[i] === '--image') { flags.image = args[++i]; i++; continue; }
|
|
18
19
|
if (args[i] === '--count') { flags.count = parseInt(args[++i], 10); i++; continue; }
|
|
19
20
|
if (args[i] === '--region') { flags.region = args[++i]; i++; continue; }
|
|
20
|
-
if (args[i] === '--tier') { flags.tier = args[++i]; i++; continue; }
|
|
21
21
|
if (args[i] === '--max-price') { flags.maxPrice = parseFloat(args[++i]); i++; continue; }
|
|
22
22
|
if (args[i] === '--name') { flags.name = args[++i]; i++; continue; }
|
|
23
|
-
if (args[i] === '--detach')
|
|
24
|
-
if (args[i] === '--fallback') { flags.fallback = args[++i]; i++; continue; }
|
|
25
|
-
if (args[i] === '--no-fallback') { flags.noFallback = true; i++; continue; }
|
|
26
|
-
if (args[i] === '--strict-capacity') { flags.noFallback = true; i++; continue; }
|
|
27
|
-
if (args[i] === '--no-expanded-search') { flags.noFallback = true; i++; continue; }
|
|
28
|
-
if (args[i] === '--max-runtime') { flags.maxRuntime = parseFloat(args[++i]); i++; continue; }
|
|
29
|
-
if (args[i] === '--max-cost') { flags.maxCost = parseFloat(args[++i]); i++; continue; }
|
|
23
|
+
if (args[i] === '--detach') { flags.detach = true; i++; continue; }
|
|
30
24
|
positional.push(args[i++]);
|
|
31
25
|
}
|
|
32
26
|
return { flags, positional };
|
|
33
27
|
}
|
|
34
28
|
|
|
35
|
-
//
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
general: { label: 'GPU job', vram: '16+ GB', gpus: ['RTX 4090', 'RTX 3090', 'A6000', 'L40S'] },
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
function inferProfileFromCommand(cmdStr) {
|
|
45
|
-
const s = cmdStr.toLowerCase();
|
|
46
|
-
if (/print\s*\(|['"]hello/.test(s) && s.length < 100) return 'smoke_test';
|
|
47
|
-
if (/lora|qlora|finetune|fine[_-]tun|peft/.test(s)) return 'lora_finetune';
|
|
48
|
-
if (/diffusion|stable.?diff|sdxl|sd.?xl|comfyui|a1111|invoke|kohya/.test(s)) return 'image_gen';
|
|
49
|
-
if (/vllm|tgi|text.generation.inference/.test(s)) return 'inference_small';
|
|
50
|
-
if (/\btrain\.py\b/.test(s)) return 'lora_finetune';
|
|
51
|
-
return 'general';
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function fmtRuntime(ms) {
|
|
55
|
-
const s = Math.round(ms / 1000);
|
|
56
|
-
if (s < 60) return `${s}s`;
|
|
57
|
-
const m = Math.floor(s / 60);
|
|
58
|
-
return `${m}m ${s % 60}s`;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
const HEARTBEAT_WARN_POLLS = 3;
|
|
62
|
-
const HEARTBEAT_KILL_POLLS = 15;
|
|
63
|
-
|
|
64
|
-
export function classifyFailure(finalStatus, exitCode) {
|
|
65
|
-
if (finalStatus === 'failed' && (exitCode === null || exitCode === undefined)) return 'infrastructure';
|
|
66
|
-
if (exitCode !== null && exitCode !== undefined && exitCode !== 0) return 'customer_code';
|
|
67
|
-
return null;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// Lines the log stream never needs to print — we surface them in the status bar instead.
|
|
71
|
-
const LOG_META_RE = /^\[dep-[^\]]+\] (status|gpu|region|endpoint|cost|receipt|provider_status|uptime)=/;
|
|
72
|
-
|
|
73
|
-
// Extract structured values from provider status lines so we can show them nicely.
|
|
74
|
-
function parseProviderLine(line) {
|
|
75
|
-
const gpuUtil = line.match(/\bgpu_util=([\d.]+)%/);
|
|
76
|
-
const cpuUtil = line.match(/\bcpu_util=([\d.]+)%/);
|
|
77
|
-
const ssh = line.match(/\bssh=(\S+)/);
|
|
78
|
-
const provSt = line.match(/\bprovider_status=(\S+)/);
|
|
79
|
-
return {
|
|
80
|
-
gpuUtil: gpuUtil ? parseFloat(gpuUtil[1]) : null,
|
|
81
|
-
cpuUtil: cpuUtil ? parseFloat(cpuUtil[1]) : null,
|
|
82
|
-
ssh: ssh ? ssh[1] : null,
|
|
83
|
-
providerStatus: provSt ? provSt[1] : null,
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function renderStatusBar(chalk, { elapsedMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost }) {
|
|
88
|
-
const spent = ratePerHour * (elapsedMs / 3_600_000);
|
|
89
|
-
const parts = [`⏱ ${fmtRuntime(elapsedMs)}`];
|
|
90
|
-
if (ratePerHour > 0) parts.push(`$${spent.toFixed(4)} spent`);
|
|
91
|
-
if (gpuUtil !== null) parts.push(`GPU ${gpuUtil.toFixed(0)}%`);
|
|
92
|
-
if (cpuUtil !== null) parts.push(`CPU ${cpuUtil.toFixed(0)}%`);
|
|
93
|
-
if (maxRuntimeMs) {
|
|
94
|
-
const left = Math.max(0, maxRuntimeMs - elapsedMs);
|
|
95
|
-
parts.push(`${fmtRuntime(left)} left`);
|
|
96
|
-
}
|
|
97
|
-
if (maxCost && ratePerHour > 0) {
|
|
98
|
-
const budgetLeft = Math.max(0, maxCost - spent);
|
|
99
|
-
parts.push(`$${budgetLeft.toFixed(4)} budget left`);
|
|
100
|
-
}
|
|
101
|
-
parts.push('Ctrl+C to stop');
|
|
102
|
-
return chalk.dim(' ' + parts.join(' • '));
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
// Wait for status to leave 'starting'/'queued'/'provisioning'.
|
|
106
|
-
// The runtime limit does NOT start until this returns.
|
|
107
|
-
// Returns the final dep object with status 'running' or 'failed'.
|
|
108
|
-
async function waitForRunning(config, depId, chalk) {
|
|
109
|
-
const POLL_MS = 3000;
|
|
110
|
-
const TIMEOUT_MS = 5 * 60 * 1000; // 5 min startup grace (image pull + container init)
|
|
111
|
-
const startMs = Date.now();
|
|
112
|
-
const PHASES = [
|
|
113
|
-
{ afterMs: 0, label: ' Starting container' },
|
|
114
|
-
{ afterMs: 15000, label: ' Pulling image' },
|
|
115
|
-
{ afterMs: 60000, label: ' Starting container' },
|
|
116
|
-
{ afterMs: 180000, label: ' Running command' },
|
|
117
|
-
];
|
|
29
|
+
// Poll /v1/deployments/:id until terminal status, streaming new log lines.
|
|
30
|
+
async function attachToJob(config, depId, chalk) {
|
|
31
|
+
const TERMINAL = new Set(['stopped', 'failed', 'completed']);
|
|
32
|
+
const POLL_MS = 4000;
|
|
33
|
+
let seenLines = 0;
|
|
34
|
+
let lastStatus = '';
|
|
118
35
|
|
|
119
|
-
|
|
120
|
-
|
|
36
|
+
while (true) {
|
|
37
|
+
await new Promise(r => setTimeout(r, POLL_MS));
|
|
121
38
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
for (let i = 0; i < PHASES.length; i++) {
|
|
127
|
-
if (elapsed >= PHASES[i].afterMs) phaseIdx = i;
|
|
128
|
-
}
|
|
129
|
-
if (phaseIdx !== lastPhaseIdx) {
|
|
130
|
-
process.stdout.write('\r\x1b[2K');
|
|
131
|
-
process.stdout.write(chalk.dim(PHASES[phaseIdx].label));
|
|
132
|
-
lastPhaseIdx = phaseIdx;
|
|
133
|
-
}
|
|
134
|
-
process.stdout.write('.');
|
|
135
|
-
}, 1000);
|
|
136
|
-
|
|
137
|
-
try {
|
|
138
|
-
while (Date.now() - startMs < TIMEOUT_MS) {
|
|
139
|
-
await new Promise(r => setTimeout(r, POLL_MS));
|
|
140
|
-
const dep = await callApi(`/deployments/${depId}`, {
|
|
39
|
+
// Fetch status
|
|
40
|
+
let dep;
|
|
41
|
+
try {
|
|
42
|
+
dep = await callApi(`/deployments/${depId}`, {
|
|
141
43
|
apiKey: config.apiKey,
|
|
142
44
|
baseUrl: config.baseUrl,
|
|
143
45
|
});
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
46
|
+
} catch {
|
|
47
|
+
// transient network error — keep trying
|
|
48
|
+
continue;
|
|
148
49
|
}
|
|
149
|
-
} finally {
|
|
150
|
-
clearInterval(ticker);
|
|
151
|
-
process.stdout.write('\n');
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
// Timed out — return last known state
|
|
155
|
-
return await callApi(`/deployments/${depId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost = null, ratePerHour = 0, onTeardown }) {
|
|
159
|
-
const TERMINAL = new Set(['stopped', 'failed', 'completed', 'succeeded']);
|
|
160
|
-
const POLL_MS = 4000;
|
|
161
|
-
let seenContent = new Set(); // track by content, not index, to avoid reprinting stale lines
|
|
162
|
-
let lastStatus = '';
|
|
163
|
-
let consecutiveErrs = 0;
|
|
164
|
-
let gpuUtil = null;
|
|
165
|
-
let cpuUtil = null;
|
|
166
|
-
let sshShown = false;
|
|
167
|
-
let statusBarActive = false;
|
|
168
|
-
const startMs = Date.now();
|
|
169
|
-
|
|
170
|
-
// Ticker: update status bar in place every second between polls.
|
|
171
|
-
let tickerInterval = null;
|
|
172
|
-
const startTicker = () => {
|
|
173
|
-
if (tickerInterval) return;
|
|
174
|
-
statusBarActive = true;
|
|
175
|
-
tickerInterval = setInterval(() => {
|
|
176
|
-
const bar = renderStatusBar(chalk, {
|
|
177
|
-
elapsedMs: Date.now() - startMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost,
|
|
178
|
-
});
|
|
179
|
-
process.stdout.write(`\r${bar} `);
|
|
180
|
-
}, 1000);
|
|
181
|
-
};
|
|
182
|
-
const stopTicker = () => {
|
|
183
|
-
if (tickerInterval) { clearInterval(tickerInterval); tickerInterval = null; }
|
|
184
|
-
if (statusBarActive) { process.stdout.write('\r\x1b[2K'); statusBarActive = false; } // clear line
|
|
185
|
-
};
|
|
186
|
-
|
|
187
|
-
let tearing = false;
|
|
188
|
-
const sigintHandler = () => {
|
|
189
|
-
if (tearing) return;
|
|
190
|
-
tearing = true;
|
|
191
|
-
stopTicker();
|
|
192
|
-
onTeardown('interrupted');
|
|
193
|
-
};
|
|
194
|
-
process.once('SIGINT', sigintHandler);
|
|
195
|
-
|
|
196
|
-
try {
|
|
197
|
-
while (true) {
|
|
198
|
-
await new Promise(r => setTimeout(r, POLL_MS));
|
|
199
|
-
|
|
200
|
-
const elapsedMs = Date.now() - startMs;
|
|
201
|
-
const spentSoFar = ratePerHour * (elapsedMs / 3_600_000);
|
|
202
50
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
|
|
51
|
+
const status = dep.status;
|
|
52
|
+
if (status !== lastStatus) {
|
|
53
|
+
if (status === 'running' && lastStatus === 'provisioning') {
|
|
54
|
+
console.log(chalk.dim(' [running]'));
|
|
208
55
|
}
|
|
56
|
+
lastStatus = status;
|
|
57
|
+
}
|
|
209
58
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
let
|
|
218
|
-
|
|
219
|
-
dep = await callApi(`/deployments/${depId}`, {
|
|
220
|
-
apiKey: config.apiKey,
|
|
221
|
-
baseUrl: config.baseUrl,
|
|
222
|
-
});
|
|
223
|
-
consecutiveErrs = 0;
|
|
224
|
-
} catch {
|
|
225
|
-
consecutiveErrs++;
|
|
226
|
-
if (lastStatus === 'running') {
|
|
227
|
-
const lostSec = Math.round((consecutiveErrs * POLL_MS) / 1000);
|
|
228
|
-
if (consecutiveErrs === HEARTBEAT_WARN_POLLS) {
|
|
229
|
-
stopTicker();
|
|
230
|
-
console.log(chalk.yellow(`\n ⚠ No heartbeat for ${lostSec}s — cloud machine may be unresponsive`));
|
|
231
|
-
startTicker();
|
|
232
|
-
} else if (consecutiveErrs >= HEARTBEAT_KILL_POLLS) {
|
|
233
|
-
tearing = true;
|
|
234
|
-
stopTicker();
|
|
235
|
-
await onTeardown('heartbeat-lost');
|
|
236
|
-
return { status: 'failed', exitCode: null, runtimeMs: elapsedMs, failureType: 'infrastructure' };
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
continue;
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const status = dep.status;
|
|
243
|
-
if (status !== lastStatus) {
|
|
244
|
-
if (status === 'running' && lastStatus === 'provisioning') {
|
|
245
|
-
stopTicker();
|
|
246
|
-
console.log(chalk.dim(' [running]'));
|
|
247
|
-
}
|
|
248
|
-
lastStatus = status;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
try {
|
|
252
|
-
const logData = await callApi(`/deployments/${depId}/logs`, {
|
|
253
|
-
apiKey: config.apiKey,
|
|
254
|
-
baseUrl: config.baseUrl,
|
|
255
|
-
});
|
|
256
|
-
const lines = logData?.logs ?? [];
|
|
257
|
-
|
|
258
|
-
for (const line of lines) {
|
|
259
|
-
// Extract structured provider values (gpu_util, ssh, etc.) from any line.
|
|
260
|
-
const parsed = parseProviderLine(line);
|
|
261
|
-
if (parsed.gpuUtil !== null) gpuUtil = parsed.gpuUtil;
|
|
262
|
-
if (parsed.cpuUtil !== null) cpuUtil = parsed.cpuUtil;
|
|
263
|
-
|
|
264
|
-
// Show SSH address once, prominently.
|
|
265
|
-
if (parsed.ssh && !sshShown) {
|
|
266
|
-
stopTicker();
|
|
267
|
-
console.log(` ${chalk.bold('SSH:')} ${chalk.cyan(parsed.ssh)}`);
|
|
268
|
-
sshShown = true;
|
|
269
|
-
startTicker();
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
// Skip lines we've already printed and pure metadata lines.
|
|
273
|
-
if (seenContent.has(line)) continue;
|
|
274
|
-
seenContent.add(line);
|
|
275
|
-
if (LOG_META_RE.test(line)) continue;
|
|
276
|
-
// Skip provider util lines — they're shown in the status bar instead.
|
|
277
|
-
if (/\b(gpu_util|cpu_util|provider_status|uptime)=/.test(line)) continue;
|
|
278
|
-
|
|
279
|
-
const isErrorLine = /^error\b/i.test(line) || /Error response from daemon/i.test(line);
|
|
280
|
-
stopTicker();
|
|
281
|
-
console.log(` ${isErrorLine ? chalk.red(line) : chalk.dim(line)}`);
|
|
282
|
-
startTicker();
|
|
283
|
-
}
|
|
284
|
-
} catch {
|
|
285
|
-
// logs not ready yet
|
|
59
|
+
// Stream any new log lines
|
|
60
|
+
try {
|
|
61
|
+
const logData = await callApi(`/deployments/${depId}/logs`, {
|
|
62
|
+
apiKey: config.apiKey,
|
|
63
|
+
baseUrl: config.baseUrl,
|
|
64
|
+
});
|
|
65
|
+
const lines = logData?.logs ?? [];
|
|
66
|
+
for (let i = seenLines; i < lines.length; i++) {
|
|
67
|
+
console.log(` ${chalk.dim(lines[i])}`);
|
|
286
68
|
}
|
|
69
|
+
seenLines = lines.length;
|
|
70
|
+
} catch {
|
|
71
|
+
// logs not ready yet
|
|
72
|
+
}
|
|
287
73
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
if (TERMINAL.has(status)) {
|
|
292
|
-
stopTicker();
|
|
293
|
-
process.removeListener('SIGINT', sigintHandler);
|
|
294
|
-
const exitCode = dep.exit_code ?? null;
|
|
295
|
-
return {
|
|
296
|
-
status,
|
|
297
|
-
exitCode,
|
|
298
|
-
runtimeMs: Date.now() - startMs,
|
|
299
|
-
failureType: classifyFailure(status, exitCode),
|
|
300
|
-
};
|
|
301
|
-
}
|
|
74
|
+
if (TERMINAL.has(status)) {
|
|
75
|
+
return status;
|
|
302
76
|
}
|
|
303
|
-
} finally {
|
|
304
|
-
stopTicker();
|
|
305
|
-
process.removeListener('SIGINT', sigintHandler);
|
|
306
77
|
}
|
|
307
78
|
}
|
|
308
79
|
|
|
@@ -310,220 +81,81 @@ export async function runCommand(config, args, chalk) {
|
|
|
310
81
|
const { flags, positional } = parseRunArgs(args);
|
|
311
82
|
|
|
312
83
|
if (positional.length === 0 && !flags.image) {
|
|
313
|
-
console.error(chalk.red('Usage: badgr run <command...>'));
|
|
314
|
-
console.error(chalk.red(' badgr run --image my/image:latest'));
|
|
84
|
+
console.error(chalk.red('Usage: badgr run <command...> --gpu <type>'));
|
|
85
|
+
console.error(chalk.red(' badgr run --image my/image:latest --gpu A100'));
|
|
315
86
|
return;
|
|
316
87
|
}
|
|
317
88
|
|
|
318
89
|
requireApiKey(config);
|
|
319
90
|
|
|
320
|
-
const command
|
|
321
|
-
|
|
322
|
-
const
|
|
323
|
-
const
|
|
324
|
-
const inferredImage = isSmoke ? 'python:3.11-alpine' : 'python:3.11-slim';
|
|
325
|
-
const image = flags.image || (command ? inferredImage : undefined);
|
|
326
|
-
const detach = flags.detach || false;
|
|
327
|
-
const fallbackMode = flags.noFallback ? 'none' : (flags.fallback || 'closest');
|
|
328
|
-
const maxRuntimeMs = flags.maxRuntime ? flags.maxRuntime * 60 * 1000 : null;
|
|
329
|
-
const maxCost = flags.maxCost ?? null;
|
|
330
|
-
|
|
331
|
-
const effectiveTier = normalizeTier(flags.tier);
|
|
332
|
-
|
|
333
|
-
// ── Auto GPU selection (no --gpu specified) ────────────────────────────────
|
|
334
|
-
let gpu = flags.gpu ? flags.gpu.toUpperCase().replace('-', '_') : 'auto';
|
|
335
|
-
|
|
336
|
-
if (gpu === 'auto') {
|
|
337
|
-
console.log(chalk.bold('\n⚡ Running GPU job\n'));
|
|
338
|
-
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
339
|
-
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
340
|
-
// Show workload estimate so the user knows what Badgr inferred.
|
|
341
|
-
if (command) {
|
|
342
|
-
const profKey = inferProfileFromCommand(cmdStr);
|
|
343
|
-
const prof = _PROFILES[profKey];
|
|
344
|
-
console.log();
|
|
345
|
-
console.log(` ${chalk.bold('Estimated workload:')} ${prof.label}`);
|
|
346
|
-
console.log(` ${chalk.bold('Estimated minimum VRAM:')} ${prof.vram}`);
|
|
347
|
-
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost.toFixed(2)}`);
|
|
348
|
-
if (flags.maxRuntime) console.log(` ${chalk.bold('Max runtime:')} ${flags.maxRuntime}min`);
|
|
349
|
-
}
|
|
350
|
-
console.log();
|
|
351
|
-
} else {
|
|
352
|
-
// Specific GPU requested — show header
|
|
353
|
-
console.log(chalk.bold('\n⚡ Running GPU job\n'));
|
|
354
|
-
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
355
|
-
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
356
|
-
console.log(` ${chalk.bold('GPU:')} ${gpu}`);
|
|
357
|
-
if (flags.maxRuntime) console.log(` ${chalk.bold('Max runtime:')} ${flags.maxRuntime}min`);
|
|
358
|
-
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost.toFixed(2)}`);
|
|
359
|
-
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice.toFixed(2)}/hr`);
|
|
360
|
-
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
361
|
-
console.log();
|
|
91
|
+
const command = positional.length > 0 ? positional : undefined;
|
|
92
|
+
const gpu = flags.gpu || 'RTX_4090';
|
|
93
|
+
const image = flags.image || (command ? 'python:3.11-slim' : undefined);
|
|
94
|
+
const detach = flags.detach || false;
|
|
362
95
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
console.log(
|
|
369
|
-
|
|
370
|
-
console.log(chalk.dim(` API: ${config.baseUrl}`));
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
function buildBody(gpuOverride, tierOverride) {
|
|
374
|
-
const effectiveRegion = flags.region ? flags.region.toUpperCase() : undefined;
|
|
375
|
-
return {
|
|
376
|
-
command,
|
|
377
|
-
image,
|
|
378
|
-
gpu: (gpuOverride || gpu),
|
|
379
|
-
gpu_count: flags.count || 1,
|
|
380
|
-
...(effectiveRegion ? { region: effectiveRegion } : {}),
|
|
381
|
-
max_price_per_hour: flags.maxPrice,
|
|
382
|
-
name: flags.name,
|
|
383
|
-
tier: tierOverride || effectiveTier,
|
|
384
|
-
};
|
|
385
|
-
}
|
|
96
|
+
console.log(chalk.bold('\n⚡ Running GPU job\n'));
|
|
97
|
+
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
98
|
+
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
99
|
+
console.log(` ${chalk.bold('GPU:')} ${gpu}`);
|
|
100
|
+
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
101
|
+
console.log();
|
|
102
|
+
console.log(chalk.dim(' Finding best available GPU capacity...'));
|
|
386
103
|
|
|
387
104
|
let dep;
|
|
388
105
|
try {
|
|
389
|
-
dep = await
|
|
390
|
-
'
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
106
|
+
dep = await callApi('/run', {
|
|
107
|
+
method: 'POST',
|
|
108
|
+
apiKey: config.apiKey,
|
|
109
|
+
baseUrl: config.baseUrl,
|
|
110
|
+
body: {
|
|
111
|
+
command,
|
|
112
|
+
image,
|
|
113
|
+
gpu: gpu.toUpperCase().replace('-', '_'),
|
|
114
|
+
gpu_count: flags.count || 1,
|
|
115
|
+
region: flags.region || 'US',
|
|
116
|
+
max_price_per_hour: flags.maxPrice,
|
|
117
|
+
name: flags.name,
|
|
118
|
+
},
|
|
119
|
+
});
|
|
397
120
|
} catch (err) {
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
const rerun = ['badgr run', ...args].join(' ');
|
|
401
|
-
console.error(chalk.dim(`After payment, rerun:\n ${rerun}\n`));
|
|
402
|
-
process.exit(1);
|
|
403
|
-
}
|
|
404
|
-
throw err;
|
|
121
|
+
console.error(chalk.red(`\n ✗ Job failed to start: ${err.message}\n`));
|
|
122
|
+
process.exit(1);
|
|
405
123
|
}
|
|
406
124
|
|
|
407
125
|
const rcptId = dep.receipt_id || generateReceiptId();
|
|
408
126
|
addReceipt({
|
|
409
|
-
receiptId:
|
|
410
|
-
action:
|
|
411
|
-
deploymentId:
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
maxRuntime: flags.maxRuntime ?? null,
|
|
417
|
-
status: dep.status,
|
|
418
|
-
createdAt: new Date().toISOString(),
|
|
127
|
+
receiptId: rcptId,
|
|
128
|
+
action: 'badgr run',
|
|
129
|
+
deploymentId: dep.deployment_id,
|
|
130
|
+
provider: dep.provider,
|
|
131
|
+
gpu: dep.gpu_type,
|
|
132
|
+
status: dep.status,
|
|
133
|
+
createdAt: new Date().toISOString(),
|
|
419
134
|
});
|
|
420
135
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
console.log(chalk.dim(' Capacity found.\n'));
|
|
424
|
-
console.log(chalk.bold(` Job ID: ${chalk.cyan(dep.deployment_id)}`));
|
|
136
|
+
console.log(chalk.bold(`\n Job ID: ${chalk.cyan(dep.deployment_id)}`));
|
|
425
137
|
console.log(` ${chalk.bold('GPU:')} ${dep.gpu_type} × ${dep.gpu_count}`);
|
|
426
|
-
if (
|
|
427
|
-
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost.toFixed(2)}`);
|
|
138
|
+
if (dep.cost_per_hour > 0) console.log(` ${chalk.bold('Rate:')} $${dep.cost_per_hour.toFixed(2)}/hr`);
|
|
428
139
|
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
429
140
|
|
|
430
|
-
if (rate > HIGH_RATE_THRESHOLD && !maxCost) {
|
|
431
|
-
console.log(chalk.yellow(`\n Selected capacity rate: $${rate.toFixed(2)}/hr`));
|
|
432
|
-
console.log(chalk.dim(' Tip: use --max-cost to enforce a hard ceiling.'));
|
|
433
|
-
}
|
|
434
|
-
|
|
435
141
|
if (detach) {
|
|
436
142
|
console.log(`\n ${chalk.bold('Logs:')} ${dep.logs_url || `badgr logs ${dep.deployment_id}`}`);
|
|
437
143
|
console.log(chalk.dim(`\n Detached. Track progress: badgr logs ${dep.deployment_id}\n`));
|
|
438
144
|
return;
|
|
439
145
|
}
|
|
440
146
|
|
|
441
|
-
//
|
|
442
|
-
|
|
443
|
-
const STARTUP_STATES = new Set(['queued', 'provisioning', 'starting']);
|
|
444
|
-
if (STARTUP_STATES.has(dep.status)) {
|
|
445
|
-
console.log();
|
|
446
|
-
dep = await waitForRunning(config, dep.deployment_id, chalk);
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
if (dep.status === 'failed') {
|
|
450
|
-
console.error(chalk.red('\n ✗ Container failed to start (infrastructure error).\n'));
|
|
451
|
-
console.error(chalk.dim(' The backend retried automatically. All attempts failed.'));
|
|
452
|
-
console.error(chalk.dim(` Contact support with receipt ID: ${rcptId}`));
|
|
453
|
-
console.log();
|
|
454
|
-
process.exit(1);
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
const ratePerHour = dep.cost_per_hour || 0;
|
|
458
|
-
console.log(chalk.dim('\n ── Running command (Ctrl+C to stop) ────────────────────────────\n'));
|
|
459
|
-
|
|
460
|
-
async function teardown(reason) {
|
|
461
|
-
const labels = {
|
|
462
|
-
'max-runtime': chalk.yellow('\n ⏱ Max runtime reached — stopping job...'),
|
|
463
|
-
'max-cost': chalk.yellow('\n 💰 Spend cap reached — stopping job...'),
|
|
464
|
-
'heartbeat-lost': chalk.red('\n ✗ No response from machine — stopping job...'),
|
|
465
|
-
'interrupted': chalk.yellow('\n Stopping job...'),
|
|
466
|
-
};
|
|
467
|
-
console.log(labels[reason] ?? chalk.yellow('\n Stopping job...'));
|
|
468
|
-
try {
|
|
469
|
-
await terminateDeployment(config, dep.deployment_id);
|
|
470
|
-
} catch {
|
|
471
|
-
// best-effort
|
|
472
|
-
}
|
|
473
|
-
const runtimeMs = Date.now() - attachStart;
|
|
474
|
-
const finalCost = ratePerHour * (runtimeMs / 3_600_000);
|
|
475
|
-
updateReceipt(rcptId, { status: reason, teardownStatus: 'terminated', runtimeSeconds: Math.round(runtimeMs / 1000), finalCost });
|
|
476
|
-
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Est. cost: $${finalCost.toFixed(4)}`));
|
|
477
|
-
console.log(chalk.dim(' Job stopped. Billing ended.\n'));
|
|
478
|
-
process.exit(reason === 'interrupted' ? 0 : 1);
|
|
479
|
-
}
|
|
147
|
+
// ── Attached mode: stream logs until job completes ───────────────────────
|
|
148
|
+
console.log(chalk.dim('\n ── Attaching (Ctrl+C to detach) ──────────────────────────────\n'));
|
|
480
149
|
|
|
481
|
-
const
|
|
482
|
-
const { status: finalStatus, exitCode, runtimeMs, failureType } = await attachToJob(config, dep.deployment_id, {
|
|
483
|
-
chalk,
|
|
484
|
-
maxRuntimeMs,
|
|
485
|
-
maxCost,
|
|
486
|
-
ratePerHour,
|
|
487
|
-
onTeardown: teardown,
|
|
488
|
-
});
|
|
150
|
+
const finalStatus = await attachToJob(config, dep.deployment_id, chalk);
|
|
489
151
|
|
|
490
152
|
console.log();
|
|
491
153
|
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
failureType,
|
|
499
|
-
teardownStatus: (finalStatus === 'completed' || finalStatus === 'succeeded') ? 'terminated' : 'failed',
|
|
500
|
-
});
|
|
501
|
-
|
|
502
|
-
console.log(` ${chalk.bold('Runtime:')} ${fmtRuntime(runtimeMs)}`);
|
|
503
|
-
if (ratePerHour > 0) {
|
|
504
|
-
console.log(` ${chalk.bold('Cost:')} $${finalCost.toFixed(4)} (${chalk.dim(`$${ratePerHour.toFixed(2)}/hr`)})`);
|
|
505
|
-
}
|
|
506
|
-
if (exitCode !== null && exitCode !== undefined) {
|
|
507
|
-
console.log(` ${chalk.bold('Exit code:')} ${exitCode !== 0 ? chalk.red(exitCode) : chalk.green(exitCode)}`);
|
|
508
|
-
}
|
|
509
|
-
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
510
|
-
|
|
511
|
-
if (finalStatus === 'failed' || (exitCode !== null && exitCode !== 0)) {
|
|
512
|
-
if (failureType === 'infrastructure') {
|
|
513
|
-
console.error(chalk.red(`\n ✗ Machine failure — this is not your code.\n`));
|
|
514
|
-
console.error(chalk.dim(' Contact support with your receipt ID for a refund.'));
|
|
515
|
-
} else {
|
|
516
|
-
console.error(chalk.red(`\n ✗ Job failed (exit ${exitCode ?? 'unknown'})\n`));
|
|
517
|
-
console.error(chalk.dim(` Check logs: badgr logs ${dep.deployment_id}`));
|
|
518
|
-
}
|
|
519
|
-
console.log();
|
|
520
|
-
process.exit(exitCode ?? 1);
|
|
521
|
-
} else if ((finalStatus === 'completed' || finalStatus === 'succeeded') && (exitCode === 0 || exitCode === null)) {
|
|
522
|
-
try {
|
|
523
|
-
await terminateDeployment(config, dep.deployment_id);
|
|
524
|
-
} catch { /* already stopped */ }
|
|
525
|
-
console.log(chalk.green(`\n ✓ Complete`));
|
|
526
|
-
console.log(chalk.dim(` Billing ended`));
|
|
527
|
-
console.log();
|
|
154
|
+
if (finalStatus === 'failed') {
|
|
155
|
+
console.error(chalk.red(`\n ✗ Job failed (${dep.deployment_id})\n`));
|
|
156
|
+
console.error(chalk.dim(` Logs: badgr logs ${dep.deployment_id}\n`));
|
|
157
|
+
process.exit(1);
|
|
158
|
+
} else {
|
|
159
|
+
console.log(chalk.green(`\n ✓ Job complete (${finalStatus})\n`));
|
|
528
160
|
}
|
|
529
161
|
}
|