badgr-cli 1.0.29 → 1.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_IT_WORKS.md +4 -4
- package/package.json +4 -12
- package/src/api.js +10 -87
- package/src/badgr.js +37 -67
- package/src/commands/down.js +23 -26
- package/src/commands/login.js +7 -23
- package/src/commands/logs.js +5 -57
- package/src/commands/models.js +6 -25
- package/src/commands/receipts.js +4 -19
- package/src/commands/run.js +84 -489
- package/src/commands/serve.js +51 -146
- package/src/commands/status.js +48 -35
- package/src/commands/up.js +26 -32
- package/src/config.js +4 -49
- package/src/router.js +73 -16
- package/src/store.js +1 -10
- package/tests/commands.test.js +2 -183
- package/tests/config.test.js +1 -24
- package/tests/router.test.js +68 -9
- package/tests/store.test.js +1 -41
- package/src/commands/billing.js +0 -93
- package/src/commands/capacity.js +0 -111
- package/src/commands/test-run.js +0 -240
- package/src/fallback.js +0 -95
package/src/commands/run.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import readline from 'readline';
|
|
2
1
|
import { requireApiKey } from '../config.js';
|
|
3
|
-
import { callApi
|
|
4
|
-
import { addReceipt,
|
|
2
|
+
import { callApi } from '../api.js';
|
|
3
|
+
import { addReceipt, generateReceiptId } from '../store.js';
|
|
5
4
|
|
|
6
5
|
/**
|
|
7
|
-
* badgr run python train.py
|
|
8
|
-
* badgr run python train.py --gpu A100 # specific GPU
|
|
6
|
+
* badgr run python train.py --gpu A100 # attached (default)
|
|
9
7
|
* badgr run --image my/image:latest --gpu L40S --detach
|
|
8
|
+
*
|
|
9
|
+
* Attaches by default: polls status + streams logs until the job finishes,
|
|
10
|
+
* then exits with the job's exit code. Pass --detach to return immediately.
|
|
10
11
|
*/
|
|
11
12
|
export function parseRunArgs(args) {
|
|
12
13
|
const flags = {};
|
|
@@ -17,350 +18,88 @@ export function parseRunArgs(args) {
|
|
|
17
18
|
if (args[i] === '--image') { flags.image = args[++i]; i++; continue; }
|
|
18
19
|
if (args[i] === '--count') { flags.count = parseInt(args[++i], 10); i++; continue; }
|
|
19
20
|
if (args[i] === '--region') { flags.region = args[++i]; i++; continue; }
|
|
20
|
-
if (args[i] === '--tier') { flags.tier = args[++i]; i++; continue; }
|
|
21
21
|
if (args[i] === '--max-price') { flags.maxPrice = parseFloat(args[++i]); i++; continue; }
|
|
22
22
|
if (args[i] === '--name') { flags.name = args[++i]; i++; continue; }
|
|
23
|
-
if (args[i] === '--detach')
|
|
24
|
-
if (args[i] === '--fallback') { flags.fallback = args[++i]; i++; continue; }
|
|
25
|
-
if (args[i] === '--no-fallback') { flags.noFallback = true; i++; continue; }
|
|
26
|
-
if (args[i] === '--max-runtime') { flags.maxRuntime = parseFloat(args[++i]); i++; continue; }
|
|
27
|
-
if (args[i] === '--max-cost') { flags.maxCost = parseFloat(args[++i]); i++; continue; }
|
|
23
|
+
if (args[i] === '--detach') { flags.detach = true; i++; continue; }
|
|
28
24
|
positional.push(args[i++]);
|
|
29
25
|
}
|
|
30
26
|
return { flags, positional };
|
|
31
27
|
}
|
|
32
28
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const HEARTBEAT_WARN_POLLS = 3;
|
|
41
|
-
const HEARTBEAT_KILL_POLLS = 15;
|
|
42
|
-
|
|
43
|
-
export function classifyFailure(finalStatus, exitCode) {
|
|
44
|
-
if (finalStatus === 'failed' && (exitCode === null || exitCode === undefined)) return 'infrastructure';
|
|
45
|
-
if (exitCode !== null && exitCode !== undefined && exitCode !== 0) return 'customer_code';
|
|
46
|
-
return null;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Lines the log stream never needs to print — we surface them in the status bar instead.
|
|
50
|
-
const LOG_META_RE = /^\[dep-[^\]]+\] (status|gpu|region|endpoint|cost|receipt|provider_status|uptime)=/;
|
|
51
|
-
|
|
52
|
-
// Extract structured values from provider status lines so we can show them nicely.
|
|
53
|
-
function parseProviderLine(line) {
|
|
54
|
-
const gpuUtil = line.match(/\bgpu_util=([\d.]+)%/);
|
|
55
|
-
const cpuUtil = line.match(/\bcpu_util=([\d.]+)%/);
|
|
56
|
-
const ssh = line.match(/\bssh=(\S+)/);
|
|
57
|
-
const provSt = line.match(/\bprovider_status=(\S+)/);
|
|
58
|
-
return {
|
|
59
|
-
gpuUtil: gpuUtil ? parseFloat(gpuUtil[1]) : null,
|
|
60
|
-
cpuUtil: cpuUtil ? parseFloat(cpuUtil[1]) : null,
|
|
61
|
-
ssh: ssh ? ssh[1] : null,
|
|
62
|
-
providerStatus: provSt ? provSt[1] : null,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
function renderStatusBar(chalk, { elapsedMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost }) {
|
|
67
|
-
const spent = ratePerHour * (elapsedMs / 3_600_000);
|
|
68
|
-
const parts = [`⏱ ${fmtRuntime(elapsedMs)}`];
|
|
69
|
-
if (ratePerHour > 0) parts.push(`$${spent.toFixed(4)} spent`);
|
|
70
|
-
if (gpuUtil !== null) parts.push(`GPU ${gpuUtil.toFixed(0)}%`);
|
|
71
|
-
if (cpuUtil !== null) parts.push(`CPU ${cpuUtil.toFixed(0)}%`);
|
|
72
|
-
if (maxRuntimeMs) {
|
|
73
|
-
const left = Math.max(0, maxRuntimeMs - elapsedMs);
|
|
74
|
-
parts.push(`${fmtRuntime(left)} left`);
|
|
75
|
-
}
|
|
76
|
-
if (maxCost && ratePerHour > 0) {
|
|
77
|
-
const budgetLeft = Math.max(0, maxCost - spent);
|
|
78
|
-
parts.push(`$${budgetLeft.toFixed(4)} budget left`);
|
|
79
|
-
}
|
|
80
|
-
parts.push('Ctrl+C to stop');
|
|
81
|
-
return chalk.dim(' ' + parts.join(' • '));
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// Wait for status to leave 'starting'/'queued'/'provisioning'.
|
|
85
|
-
// This covers image pull time — runtime limit does NOT start until this returns.
|
|
86
|
-
// Returns the final dep object with status 'running' or 'failed'.
|
|
87
|
-
async function waitForRunning(config, depId, chalk) {
|
|
88
|
-
const POLL_MS = 3000;
|
|
89
|
-
const TIMEOUT_MS = 5 * 60 * 1000; // 5 min startup grace (image pull + container init)
|
|
90
|
-
const startMs = Date.now();
|
|
91
|
-
const PHASES = [
|
|
92
|
-
{ afterMs: 0, label: ' Starting container' },
|
|
93
|
-
{ afterMs: 15000, label: ' Pulling image' },
|
|
94
|
-
{ afterMs: 60000, label: ' Starting container' },
|
|
95
|
-
{ afterMs: 180000, label: ' Running command' },
|
|
96
|
-
];
|
|
97
|
-
|
|
98
|
-
let lastPhaseIdx = -1;
|
|
99
|
-
const STARTUP_STATES = new Set(['queued', 'provisioning', 'starting']);
|
|
29
|
+
// Poll /v1/deployments/:id until terminal status, streaming new log lines.
|
|
30
|
+
async function attachToJob(config, depId, chalk) {
|
|
31
|
+
const TERMINAL = new Set(['stopped', 'failed', 'completed']);
|
|
32
|
+
const POLL_MS = 4000;
|
|
33
|
+
let seenLines = 0;
|
|
34
|
+
let lastStatus = '';
|
|
100
35
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
// Find the latest phase whose afterMs has been passed
|
|
104
|
-
let phaseIdx = 0;
|
|
105
|
-
for (let i = 0; i < PHASES.length; i++) {
|
|
106
|
-
if (elapsed >= PHASES[i].afterMs) phaseIdx = i;
|
|
107
|
-
}
|
|
108
|
-
if (phaseIdx !== lastPhaseIdx) {
|
|
109
|
-
process.stdout.write('\r\x1b[2K');
|
|
110
|
-
process.stdout.write(chalk.dim(PHASES[phaseIdx].label));
|
|
111
|
-
lastPhaseIdx = phaseIdx;
|
|
112
|
-
}
|
|
113
|
-
process.stdout.write('.');
|
|
114
|
-
}, 1000);
|
|
36
|
+
while (true) {
|
|
37
|
+
await new Promise(r => setTimeout(r, POLL_MS));
|
|
115
38
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
39
|
+
// Fetch status
|
|
40
|
+
let dep;
|
|
41
|
+
try {
|
|
42
|
+
dep = await callApi(`/deployments/${depId}`, {
|
|
120
43
|
apiKey: config.apiKey,
|
|
121
44
|
baseUrl: config.baseUrl,
|
|
122
45
|
});
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
46
|
+
} catch {
|
|
47
|
+
// transient network error — keep trying
|
|
48
|
+
continue;
|
|
127
49
|
}
|
|
128
|
-
} finally {
|
|
129
|
-
clearInterval(ticker);
|
|
130
|
-
process.stdout.write('\n');
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// Timed out — return last known state
|
|
134
|
-
return await callApi(`/deployments/${depId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost = null, ratePerHour = 0, onTeardown }) {
|
|
138
|
-
const TERMINAL = new Set(['stopped', 'failed', 'completed', 'succeeded']);
|
|
139
|
-
const POLL_MS = 4000;
|
|
140
|
-
let seenContent = new Set(); // track by content, not index, to avoid reprinting stale lines
|
|
141
|
-
let lastStatus = '';
|
|
142
|
-
let consecutiveErrs = 0;
|
|
143
|
-
let gpuUtil = null;
|
|
144
|
-
let cpuUtil = null;
|
|
145
|
-
let sshShown = false;
|
|
146
|
-
let statusBarActive = false;
|
|
147
|
-
const startMs = Date.now();
|
|
148
|
-
|
|
149
|
-
// Ticker: update status bar in place every second between polls.
|
|
150
|
-
let tickerInterval = null;
|
|
151
|
-
const startTicker = () => {
|
|
152
|
-
if (tickerInterval) return;
|
|
153
|
-
statusBarActive = true;
|
|
154
|
-
tickerInterval = setInterval(() => {
|
|
155
|
-
const bar = renderStatusBar(chalk, {
|
|
156
|
-
elapsedMs: Date.now() - startMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost,
|
|
157
|
-
});
|
|
158
|
-
process.stdout.write(`\r${bar} `);
|
|
159
|
-
}, 1000);
|
|
160
|
-
};
|
|
161
|
-
const stopTicker = () => {
|
|
162
|
-
if (tickerInterval) { clearInterval(tickerInterval); tickerInterval = null; }
|
|
163
|
-
if (statusBarActive) { process.stdout.write('\r\x1b[2K'); statusBarActive = false; } // clear line
|
|
164
|
-
};
|
|
165
|
-
|
|
166
|
-
let tearing = false;
|
|
167
|
-
const sigintHandler = () => {
|
|
168
|
-
if (tearing) return;
|
|
169
|
-
tearing = true;
|
|
170
|
-
stopTicker();
|
|
171
|
-
onTeardown('interrupted');
|
|
172
|
-
};
|
|
173
|
-
process.once('SIGINT', sigintHandler);
|
|
174
|
-
|
|
175
|
-
try {
|
|
176
|
-
while (true) {
|
|
177
|
-
await new Promise(r => setTimeout(r, POLL_MS));
|
|
178
|
-
|
|
179
|
-
const elapsedMs = Date.now() - startMs;
|
|
180
|
-
const spentSoFar = ratePerHour * (elapsedMs / 3_600_000);
|
|
181
|
-
|
|
182
|
-
if (maxCost !== null && spentSoFar >= maxCost) {
|
|
183
|
-
tearing = true;
|
|
184
|
-
stopTicker();
|
|
185
|
-
await onTeardown('max-cost');
|
|
186
|
-
return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
if (maxRuntimeMs !== null && elapsedMs >= maxRuntimeMs) {
|
|
190
|
-
tearing = true;
|
|
191
|
-
stopTicker();
|
|
192
|
-
await onTeardown('max-runtime');
|
|
193
|
-
return { status: 'timeout', exitCode: null, runtimeMs: elapsedMs, failureType: null };
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
let dep;
|
|
197
|
-
try {
|
|
198
|
-
dep = await callApi(`/deployments/${depId}`, {
|
|
199
|
-
apiKey: config.apiKey,
|
|
200
|
-
baseUrl: config.baseUrl,
|
|
201
|
-
});
|
|
202
|
-
consecutiveErrs = 0;
|
|
203
|
-
} catch {
|
|
204
|
-
consecutiveErrs++;
|
|
205
|
-
if (lastStatus === 'running') {
|
|
206
|
-
const lostSec = Math.round((consecutiveErrs * POLL_MS) / 1000);
|
|
207
|
-
if (consecutiveErrs === HEARTBEAT_WARN_POLLS) {
|
|
208
|
-
stopTicker();
|
|
209
|
-
console.log(chalk.yellow(`\n ⚠ No heartbeat for ${lostSec}s — cloud machine may be unresponsive`));
|
|
210
|
-
startTicker();
|
|
211
|
-
} else if (consecutiveErrs >= HEARTBEAT_KILL_POLLS) {
|
|
212
|
-
tearing = true;
|
|
213
|
-
stopTicker();
|
|
214
|
-
await onTeardown('heartbeat-lost');
|
|
215
|
-
return { status: 'failed', exitCode: null, runtimeMs: elapsedMs, failureType: 'infrastructure' };
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
continue;
|
|
219
|
-
}
|
|
220
50
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
console.log(chalk.dim(' [running]'));
|
|
226
|
-
}
|
|
227
|
-
lastStatus = status;
|
|
51
|
+
const status = dep.status;
|
|
52
|
+
if (status !== lastStatus) {
|
|
53
|
+
if (status === 'running' && lastStatus === 'provisioning') {
|
|
54
|
+
console.log(chalk.dim(' [running]'));
|
|
228
55
|
}
|
|
56
|
+
lastStatus = status;
|
|
57
|
+
}
|
|
229
58
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const parsed = parseProviderLine(line);
|
|
240
|
-
if (parsed.gpuUtil !== null) gpuUtil = parsed.gpuUtil;
|
|
241
|
-
if (parsed.cpuUtil !== null) cpuUtil = parsed.cpuUtil;
|
|
242
|
-
|
|
243
|
-
// Show SSH address once, prominently.
|
|
244
|
-
if (parsed.ssh && !sshShown) {
|
|
245
|
-
stopTicker();
|
|
246
|
-
console.log(` ${chalk.bold('SSH:')} ${chalk.cyan(parsed.ssh)}`);
|
|
247
|
-
sshShown = true;
|
|
248
|
-
startTicker();
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
// Skip lines we've already printed and pure metadata lines.
|
|
252
|
-
if (seenContent.has(line)) continue;
|
|
253
|
-
seenContent.add(line);
|
|
254
|
-
if (LOG_META_RE.test(line)) continue;
|
|
255
|
-
// Skip provider util lines — they're shown in the status bar instead.
|
|
256
|
-
if (/\b(gpu_util|cpu_util|provider_status|uptime)=/.test(line)) continue;
|
|
257
|
-
|
|
258
|
-
const isErrorLine = /^error\b/i.test(line) || /Error response from daemon/i.test(line);
|
|
259
|
-
stopTicker();
|
|
260
|
-
console.log(` ${isErrorLine ? chalk.red(line) : chalk.dim(line)}`);
|
|
261
|
-
startTicker();
|
|
262
|
-
}
|
|
263
|
-
} catch {
|
|
264
|
-
// logs not ready yet
|
|
59
|
+
// Stream any new log lines
|
|
60
|
+
try {
|
|
61
|
+
const logData = await callApi(`/deployments/${depId}/logs`, {
|
|
62
|
+
apiKey: config.apiKey,
|
|
63
|
+
baseUrl: config.baseUrl,
|
|
64
|
+
});
|
|
65
|
+
const lines = logData?.logs ?? [];
|
|
66
|
+
for (let i = seenLines; i < lines.length; i++) {
|
|
67
|
+
console.log(` ${chalk.dim(lines[i])}`);
|
|
265
68
|
}
|
|
69
|
+
seenLines = lines.length;
|
|
70
|
+
} catch {
|
|
71
|
+
// logs not ready yet
|
|
72
|
+
}
|
|
266
73
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
if (TERMINAL.has(status)) {
|
|
271
|
-
stopTicker();
|
|
272
|
-
process.removeListener('SIGINT', sigintHandler);
|
|
273
|
-
const exitCode = dep.exit_code ?? null;
|
|
274
|
-
return {
|
|
275
|
-
status,
|
|
276
|
-
exitCode,
|
|
277
|
-
runtimeMs: Date.now() - startMs,
|
|
278
|
-
failureType: classifyFailure(status, exitCode),
|
|
279
|
-
};
|
|
280
|
-
}
|
|
74
|
+
if (TERMINAL.has(status)) {
|
|
75
|
+
return status;
|
|
281
76
|
}
|
|
282
|
-
} finally {
|
|
283
|
-
stopTicker();
|
|
284
|
-
process.removeListener('SIGINT', sigintHandler);
|
|
285
77
|
}
|
|
286
78
|
}
|
|
287
79
|
|
|
288
|
-
function askConfirm(prompt) {
|
|
289
|
-
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
290
|
-
return new Promise(resolve => rl.question(prompt, ans => { rl.close(); resolve(ans.trim()); }));
|
|
291
|
-
}
|
|
292
|
-
|
|
293
80
|
export async function runCommand(config, args, chalk) {
|
|
294
81
|
const { flags, positional } = parseRunArgs(args);
|
|
295
82
|
|
|
296
83
|
if (positional.length === 0 && !flags.image) {
|
|
297
|
-
console.error(chalk.red('Usage: badgr run <command...>'));
|
|
298
|
-
console.error(chalk.red(' badgr run --image my/image:latest'));
|
|
84
|
+
console.error(chalk.red('Usage: badgr run <command...> --gpu <type>'));
|
|
85
|
+
console.error(chalk.red(' badgr run --image my/image:latest --gpu A100'));
|
|
299
86
|
return;
|
|
300
87
|
}
|
|
301
88
|
|
|
302
89
|
requireApiKey(config);
|
|
303
90
|
|
|
304
|
-
const command
|
|
305
|
-
|
|
306
|
-
const
|
|
307
|
-
const
|
|
308
|
-
const inferredImage = isSmoke ? 'python:3.11-alpine' : 'python:3.11-slim';
|
|
309
|
-
const image = flags.image || (command ? inferredImage : undefined);
|
|
310
|
-
const detach = flags.detach || false;
|
|
311
|
-
const fallbackMode = flags.noFallback ? 'none' : (flags.fallback || 'closest');
|
|
312
|
-
const maxRuntimeMs = flags.maxRuntime ? flags.maxRuntime * 60 * 1000 : null;
|
|
313
|
-
const maxCost = flags.maxCost ?? null;
|
|
314
|
-
|
|
315
|
-
// Tier 1 = managed routing (default). Tier 2 = marketplace routing, opt-in via --tier 2.
|
|
316
|
-
const effectiveTier = (flags.tier === '2' || flags.tier === 'tier2' || flags.tier === 'tier-2')
|
|
317
|
-
? '2'
|
|
318
|
-
: (flags.tier || '1');
|
|
319
|
-
|
|
320
|
-
// ── Auto GPU selection (no --gpu specified) ────────────────────────────────
|
|
321
|
-
let gpu = flags.gpu ? flags.gpu.toUpperCase().replace('-', '_') : 'auto';
|
|
322
|
-
|
|
323
|
-
if (gpu === 'auto') {
|
|
324
|
-
console.log(chalk.bold('\n⚡ Running GPU job\n'));
|
|
325
|
-
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
326
|
-
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
327
|
-
if (effectiveTier === '2') console.log(` ${chalk.dim('(tier 2 — marketplace routing)')}`)
|
|
328
|
-
console.log();
|
|
329
|
-
} else {
|
|
330
|
-
// Specific GPU requested — show header
|
|
331
|
-
console.log(chalk.bold('\n⚡ Running GPU job\n'));
|
|
332
|
-
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
333
|
-
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
334
|
-
console.log(` ${chalk.bold('GPU:')} ${gpu}`);
|
|
335
|
-
if (flags.maxRuntime) console.log(` ${chalk.bold('Max runtime:')} ${flags.maxRuntime}min`);
|
|
336
|
-
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost.toFixed(2)}`);
|
|
337
|
-
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice.toFixed(2)}/hr`);
|
|
338
|
-
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
339
|
-
console.log();
|
|
340
|
-
|
|
341
|
-
if (!detach && !flags.maxRuntime && !maxCost) {
|
|
342
|
-
console.log(chalk.dim(' Tip: add --max-runtime 60 or --max-cost 5.00 to cap spend automatically'));
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
console.log(chalk.dim(' Finding reliable capacity...'));
|
|
347
|
-
if (process.env.BADGR_DEBUG === '1' || process.env.BADGR_DEBUG === 'true') {
|
|
348
|
-
console.log(chalk.dim(` API: ${config.baseUrl}`));
|
|
349
|
-
}
|
|
91
|
+
const command = positional.length > 0 ? positional : undefined;
|
|
92
|
+
const gpu = flags.gpu || 'RTX_4090';
|
|
93
|
+
const image = flags.image || (command ? 'python:3.11-slim' : undefined);
|
|
94
|
+
const detach = flags.detach || false;
|
|
350
95
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
...(effectiveRegion ? { region: effectiveRegion } : {}),
|
|
359
|
-
max_price_per_hour: flags.maxPrice,
|
|
360
|
-
name: flags.name,
|
|
361
|
-
tier: effectiveTier,
|
|
362
|
-
};
|
|
363
|
-
}
|
|
96
|
+
console.log(chalk.bold('\n⚡ Running GPU job\n'));
|
|
97
|
+
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
98
|
+
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
99
|
+
console.log(` ${chalk.bold('GPU:')} ${gpu}`);
|
|
100
|
+
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
101
|
+
console.log();
|
|
102
|
+
console.log(chalk.dim(' Finding best available GPU capacity...'));
|
|
364
103
|
|
|
365
104
|
let dep;
|
|
366
105
|
try {
|
|
@@ -368,104 +107,34 @@ export async function runCommand(config, args, chalk) {
|
|
|
368
107
|
method: 'POST',
|
|
369
108
|
apiKey: config.apiKey,
|
|
370
109
|
baseUrl: config.baseUrl,
|
|
371
|
-
body:
|
|
110
|
+
body: {
|
|
111
|
+
command,
|
|
112
|
+
image,
|
|
113
|
+
gpu: gpu.toUpperCase().replace('-', '_'),
|
|
114
|
+
gpu_count: flags.count || 1,
|
|
115
|
+
region: flags.region || 'US',
|
|
116
|
+
max_price_per_hour: flags.maxPrice,
|
|
117
|
+
name: flags.name,
|
|
118
|
+
},
|
|
372
119
|
});
|
|
373
120
|
} catch (err) {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
if (effectiveTier === '2') {
|
|
377
|
-
console.error(chalk.red('\n ✗ No GPU capacity available right now on any provider.\n'));
|
|
378
|
-
console.error(chalk.dim(' Try `badgr capacity` to see what\'s available, or try again shortly.'));
|
|
379
|
-
process.exit(1);
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
// Tier 1 out of capacity — offer tier 2 marketplace routing.
|
|
383
|
-
if (process.stdin.isTTY) {
|
|
384
|
-
const answer = await askConfirm(
|
|
385
|
-
`\n No tier 1 capacity available. Try tier 2 marketplace routing? [${chalk.bold('Enter')}/${chalk.bold('q')}]: `
|
|
386
|
-
);
|
|
387
|
-
if (answer.toLowerCase() === 'q') {
|
|
388
|
-
console.log(chalk.dim('\n Cancelled.\n'));
|
|
389
|
-
process.exit(0);
|
|
390
|
-
}
|
|
391
|
-
} else {
|
|
392
|
-
console.log(chalk.dim('\n No tier 1 capacity — trying tier 2 marketplace routing...\n'));
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
console.log(chalk.dim(' Searching tier 2 capacity...'));
|
|
396
|
-
try {
|
|
397
|
-
dep = await callApi('/run', {
|
|
398
|
-
method: 'POST',
|
|
399
|
-
apiKey: config.apiKey,
|
|
400
|
-
baseUrl: config.baseUrl,
|
|
401
|
-
body: { ...buildBody(), tier: '2' },
|
|
402
|
-
});
|
|
403
|
-
} catch (err2) {
|
|
404
|
-
const d2 = err2.errorData;
|
|
405
|
-
if (d2?.code === 'NO_CAPACITY_MATCH') {
|
|
406
|
-
console.error(chalk.red('\n ✗ No GPU capacity available right now on any provider.\n'));
|
|
407
|
-
console.error(chalk.dim(' Try `badgr capacity` to see what\'s available, or try again shortly.'));
|
|
408
|
-
} else if (d2?.code === 'PROVISIONING_FAILED' || d2?.code === 'PROVIDER_ADAPTER_ERROR') {
|
|
409
|
-
console.error(chalk.red(`\n ✗ Budget provider found capacity but could not start the machine. Please try again.\n`));
|
|
410
|
-
// (error detail kept below)
|
|
411
|
-
if (process.env.BADGR_DEBUG === '1' || process.env.BADGR_DEBUG === 'true') {
|
|
412
|
-
if (d2?.debug_error) console.error(chalk.dim(` Provider detail: ${d2.debug_error}`));
|
|
413
|
-
} else {
|
|
414
|
-
console.error(chalk.dim(` Run BADGR_DEBUG=1 badgr run … for a full trace.\n`));
|
|
415
|
-
}
|
|
416
|
-
} else {
|
|
417
|
-
console.error(chalk.red(`\n ✗ Could not start job on tier 2: ${err2.message}\n`));
|
|
418
|
-
console.error(chalk.dim(` Run BADGR_DEBUG=1 badgr run … for a full trace.`));
|
|
419
|
-
}
|
|
420
|
-
process.exit(1);
|
|
421
|
-
}
|
|
422
|
-
} else if (d?.code === 'PROVISIONING_FAILED' || d?.code === 'PROVIDER_ADAPTER_ERROR') {
|
|
423
|
-
if (d?.low_cost_provider_failed) {
|
|
424
|
-
console.error(chalk.red(`\n ✗ Tier 2 unavailable. Tier 1 also unavailable. Try again shortly.\n`));
|
|
425
|
-
} else {
|
|
426
|
-
console.error(chalk.red(`\n ✗ Badgr found ${gpu} capacity but could not start the machine. Please try again.\n`));
|
|
427
|
-
}
|
|
428
|
-
if (process.env.BADGR_DEBUG === '1' || process.env.BADGR_DEBUG === 'true') {
|
|
429
|
-
if (d?.debug_error) console.error(chalk.dim(` Provider detail: ${d.debug_error}`));
|
|
430
|
-
} else {
|
|
431
|
-
console.error(chalk.dim(` Run BADGR_DEBUG=1 badgr run … for a full trace.\n`));
|
|
432
|
-
}
|
|
433
|
-
process.exit(1);
|
|
434
|
-
} else if (err.isPaymentRequired) {
|
|
435
|
-
console.error(chalk.yellow(err.message));
|
|
436
|
-
const rerun = ['badgr run', ...args].join(' ');
|
|
437
|
-
console.error(chalk.dim(`After payment, rerun:\n ${rerun}\n`));
|
|
438
|
-
process.exit(1);
|
|
439
|
-
} else {
|
|
440
|
-
console.error(chalk.red(`\n ✗ Could not start job: ${err.message}\n`));
|
|
441
|
-
console.error(chalk.dim(` Run BADGR_DEBUG=1 badgr run … for a full trace.`));
|
|
442
|
-
console.error(chalk.dim(` Check config: badgr config\n`));
|
|
443
|
-
process.exit(1);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
if (dep.provider_fallback_note === 'tier2_failed_using_tier1') {
|
|
448
|
-
console.log(chalk.yellow(' ℹ Tier 2 unavailable. Running on Tier 1 instead.\n'));
|
|
121
|
+
console.error(chalk.red(`\n ✗ Job failed to start: ${err.message}\n`));
|
|
122
|
+
process.exit(1);
|
|
449
123
|
}
|
|
450
124
|
|
|
451
125
|
const rcptId = dep.receipt_id || generateReceiptId();
|
|
452
126
|
addReceipt({
|
|
453
|
-
receiptId:
|
|
454
|
-
action:
|
|
455
|
-
deploymentId:
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
status: dep.status,
|
|
461
|
-
createdAt: new Date().toISOString(),
|
|
127
|
+
receiptId: rcptId,
|
|
128
|
+
action: 'badgr run',
|
|
129
|
+
deploymentId: dep.deployment_id,
|
|
130
|
+
provider: dep.provider,
|
|
131
|
+
gpu: dep.gpu_type,
|
|
132
|
+
status: dep.status,
|
|
133
|
+
createdAt: new Date().toISOString(),
|
|
462
134
|
});
|
|
463
135
|
|
|
464
|
-
console.log();
|
|
465
|
-
console.log(chalk.bold(` Job ID: ${chalk.cyan(dep.deployment_id)}`));
|
|
136
|
+
console.log(chalk.bold(`\n Job ID: ${chalk.cyan(dep.deployment_id)}`));
|
|
466
137
|
console.log(` ${chalk.bold('GPU:')} ${dep.gpu_type} × ${dep.gpu_count}`);
|
|
467
|
-
if (dep.workload_desc) console.log(` ${chalk.bold('Workload:')} ${dep.workload_desc}`);
|
|
468
|
-
if (dep.tier) console.log(` ${chalk.bold('Tier:')} ${dep.tier}`);
|
|
469
138
|
if (dep.cost_per_hour > 0) console.log(` ${chalk.bold('Rate:')} $${dep.cost_per_hour.toFixed(2)}/hr`);
|
|
470
139
|
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
471
140
|
|
|
@@ -475,92 +144,18 @@ export async function runCommand(config, args, chalk) {
|
|
|
475
144
|
return;
|
|
476
145
|
}
|
|
477
146
|
|
|
478
|
-
//
|
|
479
|
-
|
|
480
|
-
const STARTUP_STATES = new Set(['queued', 'provisioning', 'starting']);
|
|
481
|
-
if (STARTUP_STATES.has(dep.status)) {
|
|
482
|
-
console.log();
|
|
483
|
-
dep = await waitForRunning(config, dep.deployment_id, chalk);
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
if (dep.status === 'failed') {
|
|
487
|
-
console.error(chalk.red('\n ✗ Container failed to start (infrastructure error).\n'));
|
|
488
|
-
console.error(chalk.dim(' The backend retried automatically. All attempts failed.'));
|
|
489
|
-
console.error(chalk.dim(` Contact support with receipt ID: ${rcptId}`));
|
|
490
|
-
console.log();
|
|
491
|
-
process.exit(1);
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
const ratePerHour = dep.cost_per_hour || 0;
|
|
495
|
-
console.log(chalk.dim('\n ── Running command (Ctrl+C to stop) ────────────────────────────\n'));
|
|
496
|
-
|
|
497
|
-
async function teardown(reason) {
|
|
498
|
-
const labels = {
|
|
499
|
-
'max-runtime': chalk.yellow('\n ⏱ Max runtime reached — stopping job...'),
|
|
500
|
-
'max-cost': chalk.yellow('\n 💰 Spend cap reached — stopping job...'),
|
|
501
|
-
'heartbeat-lost': chalk.red('\n ✗ No response from machine — stopping job...'),
|
|
502
|
-
'interrupted': chalk.yellow('\n Stopping job...'),
|
|
503
|
-
};
|
|
504
|
-
console.log(labels[reason] ?? chalk.yellow('\n Stopping job...'));
|
|
505
|
-
try {
|
|
506
|
-
await terminateDeployment(config, dep.deployment_id);
|
|
507
|
-
} catch {
|
|
508
|
-
// best-effort
|
|
509
|
-
}
|
|
510
|
-
const runtimeMs = Date.now() - attachStart;
|
|
511
|
-
const finalCost = ratePerHour * (runtimeMs / 3_600_000);
|
|
512
|
-
updateReceipt(rcptId, { status: reason, teardownStatus: 'terminated', runtimeSeconds: Math.round(runtimeMs / 1000), finalCost });
|
|
513
|
-
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Est. cost: $${finalCost.toFixed(4)}`));
|
|
514
|
-
console.log(chalk.dim(' Job stopped. Billing ended.\n'));
|
|
515
|
-
process.exit(reason === 'interrupted' ? 0 : 1);
|
|
516
|
-
}
|
|
147
|
+
// ── Attached mode: stream logs until job completes ───────────────────────
|
|
148
|
+
console.log(chalk.dim('\n ── Attaching (Ctrl+C to detach) ──────────────────────────────\n'));
|
|
517
149
|
|
|
518
|
-
const
|
|
519
|
-
const { status: finalStatus, exitCode, runtimeMs, failureType } = await attachToJob(config, dep.deployment_id, {
|
|
520
|
-
chalk,
|
|
521
|
-
maxRuntimeMs,
|
|
522
|
-
maxCost,
|
|
523
|
-
ratePerHour,
|
|
524
|
-
onTeardown: teardown,
|
|
525
|
-
});
|
|
150
|
+
const finalStatus = await attachToJob(config, dep.deployment_id, chalk);
|
|
526
151
|
|
|
527
152
|
console.log();
|
|
528
153
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
failureType,
|
|
536
|
-
teardownStatus: (finalStatus === 'completed' || finalStatus === 'succeeded') ? 'terminated' : 'failed',
|
|
537
|
-
});
|
|
538
|
-
|
|
539
|
-
console.log(` ${chalk.bold('Runtime:')} ${fmtRuntime(runtimeMs)}`);
|
|
540
|
-
if (ratePerHour > 0) {
|
|
541
|
-
console.log(` ${chalk.bold('Cost:')} $${finalCost.toFixed(4)} (${chalk.dim(`$${ratePerHour.toFixed(2)}/hr`)})`);
|
|
542
|
-
}
|
|
543
|
-
if (exitCode !== null && exitCode !== undefined) {
|
|
544
|
-
console.log(` ${chalk.bold('Exit code:')} ${exitCode !== 0 ? chalk.red(exitCode) : chalk.green(exitCode)}`);
|
|
545
|
-
}
|
|
546
|
-
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
547
|
-
|
|
548
|
-
if (finalStatus === 'failed' || (exitCode !== null && exitCode !== 0)) {
|
|
549
|
-
if (failureType === 'infrastructure') {
|
|
550
|
-
console.error(chalk.red(`\n ✗ Machine failure — this is not your code.\n`));
|
|
551
|
-
console.error(chalk.dim(' Contact support with your receipt ID for a refund.'));
|
|
552
|
-
} else {
|
|
553
|
-
console.error(chalk.red(`\n ✗ Job failed (exit ${exitCode ?? 'unknown'})\n`));
|
|
554
|
-
console.error(chalk.dim(` Check logs: badgr logs ${dep.deployment_id}`));
|
|
555
|
-
}
|
|
556
|
-
console.log();
|
|
557
|
-
process.exit(exitCode ?? 1);
|
|
558
|
-
} else if ((finalStatus === 'completed' || finalStatus === 'succeeded') && (exitCode === 0 || exitCode === null)) {
|
|
559
|
-
try {
|
|
560
|
-
await terminateDeployment(config, dep.deployment_id);
|
|
561
|
-
} catch { /* already stopped */ }
|
|
562
|
-
console.log(chalk.green(`\n ✓ Complete`));
|
|
563
|
-
console.log(chalk.dim(` Billing ended`));
|
|
564
|
-
console.log();
|
|
154
|
+
if (finalStatus === 'failed') {
|
|
155
|
+
console.error(chalk.red(`\n ✗ Job failed (${dep.deployment_id})\n`));
|
|
156
|
+
console.error(chalk.dim(` Logs: badgr logs ${dep.deployment_id}\n`));
|
|
157
|
+
process.exit(1);
|
|
158
|
+
} else {
|
|
159
|
+
console.log(chalk.green(`\n ✓ Job complete (${finalStatus})\n`));
|
|
565
160
|
}
|
|
566
161
|
}
|