badgr-cli 1.0.44 → 1.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -240
- package/package.json +1 -1
- package/src/badgr.js +12 -0
- package/src/catalog.js +31 -0
- package/src/commands/comfyui.js +70 -56
- package/src/commands/detect.js +58 -0
- package/src/commands/receipts.js +39 -2
- package/src/commands/run.js +78 -3
- package/src/commands/serve.js +116 -5
- package/src/commands/train.js +22 -27
- package/src/detect.js +362 -0
- package/src/progress.js +160 -0
- package/src/store.js +11 -0
- package/tests/detect.test.js +191 -0
- package/tests/job-progress-poll.test.js +136 -0
- package/tests/productized-runners.test.js +7 -0
- package/tests/run-lifecycle.test.js +111 -1
- package/tests/serve-apps.test.js +189 -0
- package/tests/serve-lifecycle.test.js +21 -0
- package/tests/store.test.js +22 -1
- package/tests/template.test.js +4 -4
- package/tests/workload-templates.test.js +22 -0
package/src/catalog.js
CHANGED
|
@@ -112,6 +112,32 @@ export const TEMPLATES = [
|
|
|
112
112
|
'OpenAI client: OpenAI(base_url="<endpoint>/v1", api_key="<key>")',
|
|
113
113
|
],
|
|
114
114
|
},
|
|
115
|
+
{
|
|
116
|
+
name: 'openwebui',
|
|
117
|
+
title: 'Open WebUI',
|
|
118
|
+
description: 'Self-hosted ChatGPT-style web UI for any OpenAI-compatible endpoint',
|
|
119
|
+
type: 'endpoint',
|
|
120
|
+
image: process.env.OPENWEBUI_IMAGE || 'ghcr.io/open-webui/open-webui:main',
|
|
121
|
+
gpu: 'RTX_3080',
|
|
122
|
+
gpu_count: 1,
|
|
123
|
+
port: 8080,
|
|
124
|
+
health_path: '/health',
|
|
125
|
+
min_vram_gb: 8,
|
|
126
|
+
env: {
|
|
127
|
+
OPENAI_API_BASE_URL: '<set-automatically-by-badgr-serve-openwebui>',
|
|
128
|
+
OPENAI_API_KEY: '<your-badgr-api-key>',
|
|
129
|
+
WEBUI_AUTH: 'False',
|
|
130
|
+
},
|
|
131
|
+
notes: [
|
|
132
|
+
'Preferred: `badgr serve openwebui --model qwen-7b` auto-launches (or reuses) a vLLM',
|
|
133
|
+
'endpoint and connects Open WebUI to it — no manual env wiring needed.',
|
|
134
|
+
'Open WebUI is a separate served app from the model — it needs a model endpoint',
|
|
135
|
+
'behind it (usually vLLM), it is not itself powered by vLLM.',
|
|
136
|
+
'Or set OPENAI_API_BASE_URL / OPENAI_API_KEY manually via --env to point at any',
|
|
137
|
+
'OpenAI-compatible server, including one you already have running.',
|
|
138
|
+
'Chat history and settings persist only for the life of this deployment.',
|
|
139
|
+
],
|
|
140
|
+
},
|
|
115
141
|
{
|
|
116
142
|
name: 'llama-cpp',
|
|
117
143
|
title: 'llama.cpp Server',
|
|
@@ -493,6 +519,11 @@ export const BLESSED_COMFY_WORKFLOWS = {
|
|
|
493
519
|
gpu_type: 'RTX_4090',
|
|
494
520
|
output_type: 'images',
|
|
495
521
|
},
|
|
522
|
+
'flux-basic': {
|
|
523
|
+
description: 'FLUX.1-schnell text-to-image',
|
|
524
|
+
gpu_type: 'RTX_4090',
|
|
525
|
+
output_type: 'images',
|
|
526
|
+
},
|
|
496
527
|
};
|
|
497
528
|
|
|
498
529
|
/**
|
package/src/commands/comfyui.js
CHANGED
|
@@ -9,9 +9,10 @@ import { readFileSync, existsSync } from 'fs';
|
|
|
9
9
|
import { requireApiKey } from '../config.js';
|
|
10
10
|
import { callApi, listDeployments } from '../api.js';
|
|
11
11
|
import { addDeployment, addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
12
|
-
import { normalizeTier, callWithFallback
|
|
12
|
+
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
13
13
|
import { formatCliError } from '../errors.js';
|
|
14
14
|
import { BLESSED_COMFY_WORKFLOWS } from '../catalog.js';
|
|
15
|
+
import { pollJobUntilTerminal, renderJobClosingBlock, stage as _stage, stageDone as _stageDone, writeBlock as _writeBlock, clearBlock as _clearBlock, renderLiveBlock as _renderLiveBlock, printFailureClass as _printFailureClass } from '../progress.js';
|
|
15
16
|
|
|
16
17
|
const COMFYUI_IMAGE = process.env.COMFYUI_IMAGE || 'yanwk/comfyui-boot:cu126-megapak';
|
|
17
18
|
const HEALTH_PATH = '/system_stats';
|
|
@@ -54,16 +55,14 @@ function parseEnvFlag(envList) {
|
|
|
54
55
|
return obj;
|
|
55
56
|
}
|
|
56
57
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
async function waitForComfyUI(endpointUrl, depId, config, chalk) {
|
|
58
|
+
// Live block while ComfyUI comes up — same generic "Checking health" stage
|
|
59
|
+
// line and status block as every other launch path (see serve.js's
|
|
60
|
+
// waitForEndpoint), so ComfyUI doesn't read as a different UX than vLLM.
|
|
61
|
+
async function waitForComfyUI(endpointUrl, depId, config, chalk, costCtx = {}) {
|
|
62
|
+
const { costPerHour = 0, stageLine = '' } = costCtx;
|
|
65
63
|
const startMs = Date.now();
|
|
66
64
|
const deadline = startMs + WAIT_TIMEOUT_MS;
|
|
65
|
+
let blockLines = 0;
|
|
67
66
|
|
|
68
67
|
while (Date.now() < deadline) {
|
|
69
68
|
try {
|
|
@@ -71,22 +70,32 @@ async function waitForComfyUI(endpointUrl, depId, config, chalk) {
|
|
|
71
70
|
apiKey: config.apiKey, baseUrl: config.baseUrl, timeoutMs: 10_000,
|
|
72
71
|
});
|
|
73
72
|
if (['failed', 'terminated', 'error', 'stopped'].includes(dep.status)) {
|
|
74
|
-
|
|
75
|
-
return {
|
|
73
|
+
if (blockLines > 0) { _clearBlock(blockLines); blockLines = 0; }
|
|
74
|
+
return {
|
|
75
|
+
ready: false, timedOut: false, depFailed: true,
|
|
76
|
+
failReason: dep.error || dep.status,
|
|
77
|
+
failureClass: dep.failure_class ?? null,
|
|
78
|
+
nextAction: dep.next_action ?? null,
|
|
79
|
+
};
|
|
76
80
|
}
|
|
77
81
|
} catch {}
|
|
78
82
|
|
|
79
83
|
try {
|
|
80
84
|
const res = await fetch(`${endpointUrl}${HEALTH_PATH}`, { signal: AbortSignal.timeout(8000) });
|
|
81
|
-
if (res.ok) {
|
|
85
|
+
if (res.ok) {
|
|
86
|
+
if (blockLines > 0) { _clearBlock(blockLines); blockLines = 0; }
|
|
87
|
+
return { ready: true, timedOut: false, depFailed: false };
|
|
88
|
+
}
|
|
82
89
|
} catch {}
|
|
83
90
|
|
|
84
|
-
const
|
|
85
|
-
|
|
91
|
+
const elapsedSec = Math.round((Date.now() - startMs) / 1000);
|
|
92
|
+
const spend = costPerHour * (elapsedSec / 3600);
|
|
93
|
+
blockLines = _writeBlock(blockLines, _renderLiveBlock(chalk, {
|
|
94
|
+
stageLine, elapsedSec, statusWord: 'starting', spend, id: depId,
|
|
95
|
+
}));
|
|
86
96
|
await new Promise(r => setTimeout(r, 8000));
|
|
87
97
|
}
|
|
88
98
|
|
|
89
|
-
process.stdout.write('\n');
|
|
90
99
|
return { ready: false, timedOut: true, depFailed: false };
|
|
91
100
|
}
|
|
92
101
|
|
|
@@ -142,7 +151,7 @@ export async function comfyBatchCommand(config, args, chalk) {
|
|
|
142
151
|
if (!flags.workflow) {
|
|
143
152
|
console.error(chalk.red('\n Usage: badgr comfyui batch --workflow sdxl-basic --prompts prompts.txt --max-cost 10\n'));
|
|
144
153
|
console.error(chalk.dim(' Runs a batch of prompts through a blessed ComfyUI workflow and returns image URLs.\n'));
|
|
145
|
-
console.error(chalk.dim(
|
|
154
|
+
console.error(chalk.dim(` Blessed workflows: ${Object.keys(BLESSED_COMFY_WORKFLOWS).join(', ')}\n`));
|
|
146
155
|
process.exitCode = 1;
|
|
147
156
|
return;
|
|
148
157
|
}
|
|
@@ -232,36 +241,30 @@ export async function comfyBatchCommand(config, args, chalk) {
|
|
|
232
241
|
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
233
242
|
console.log(chalk.dim('\n Polling for completion…\n'));
|
|
234
243
|
|
|
235
|
-
const startMs = Date.now();
|
|
236
244
|
const maxMs = maxRuntime * 60 * 1000;
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
console.log(`\n ${chalk.bold(
|
|
255
|
-
|
|
256
|
-
}
|
|
257
|
-
if (detail.status === 'failed') {
|
|
258
|
-
console.error(chalk.red(`\n\n ✗ Batch failed: ${detail.error_code || ''} — ${detail.error_message || ''}\n`));
|
|
259
|
-
process.exitCode = 1;
|
|
260
|
-
return;
|
|
245
|
+
const { outcome, detail } = await pollJobUntilTerminal(callApi, config, job.job_id, { chalk, maxMs });
|
|
246
|
+
|
|
247
|
+
if (outcome === 'polling_failed') {
|
|
248
|
+
console.error(chalk.red(`\n\n ✗ Lost contact with Badgr — could not confirm job status.\n`));
|
|
249
|
+
console.error(chalk.dim(` badgr status\n badgr logs ${job.job_id}\n`));
|
|
250
|
+
process.exitCode = 1;
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
if (outcome === 'timed_out') {
|
|
254
|
+
console.error(chalk.yellow('\n\n Batch still running — detached. Check status:\n'));
|
|
255
|
+
console.error(chalk.dim(` badgr status\n`));
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (outcome === 'completed') {
|
|
260
|
+
const out = detail.output || {};
|
|
261
|
+
if (out.image_urls && out.image_urls.length > 0) {
|
|
262
|
+
console.log(`\n ${chalk.bold(`Images (${out.image_urls.length}):`)}`);
|
|
263
|
+
out.image_urls.forEach((url, i) => console.log(` ${i + 1}. ${url}`));
|
|
261
264
|
}
|
|
262
265
|
}
|
|
263
|
-
console.
|
|
264
|
-
|
|
266
|
+
console.log(renderJobClosingBlock(chalk, detail, rcptId));
|
|
267
|
+
if (outcome === 'failed') process.exitCode = 1;
|
|
265
268
|
}
|
|
266
269
|
|
|
267
270
|
export async function comfyuiCommand(config, args, chalk) {
|
|
@@ -322,7 +325,10 @@ export async function comfyuiCommand(config, args, chalk) {
|
|
|
322
325
|
const effectiveTier = normalizeTier(flags.tier);
|
|
323
326
|
const envObj = { ...parseEnvFlag(flags.env), COMFYUI_WORKFLOW_B64: workflowB64 };
|
|
324
327
|
|
|
325
|
-
|
|
328
|
+
const STAGE_TOTAL = 4;
|
|
329
|
+
let stageN = 1;
|
|
330
|
+
|
|
331
|
+
console.log(chalk.bold('\n⚡ Running ComfyUI\n'));
|
|
326
332
|
console.log(` ${chalk.bold('Workflow:')} ${workflow} (${nodeCount} nodes)`);
|
|
327
333
|
console.log(` ${chalk.bold('Image:')} ${COMFYUI_IMAGE}`);
|
|
328
334
|
console.log(` ${chalk.bold('GPU:')} ${gpu === 'AUTO' ? chalk.dim('auto (16+ GB VRAM)') : gpu}`);
|
|
@@ -332,7 +338,6 @@ export async function comfyuiCommand(config, args, chalk) {
|
|
|
332
338
|
console.log(chalk.yellow(' ⚠ Persistent — billing until: badgr down <id>'));
|
|
333
339
|
}
|
|
334
340
|
console.log();
|
|
335
|
-
process.stdout.write(chalk.dim(' Finding suitable capacity...\n'));
|
|
336
341
|
|
|
337
342
|
// Duplicate check
|
|
338
343
|
if (config.apiKey) {
|
|
@@ -412,6 +417,11 @@ export async function comfyuiCommand(config, args, chalk) {
|
|
|
412
417
|
createdAt: new Date().toISOString(),
|
|
413
418
|
});
|
|
414
419
|
|
|
420
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Finding a working route...')));
|
|
421
|
+
stageN++;
|
|
422
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Starting runtime...')));
|
|
423
|
+
stageN++;
|
|
424
|
+
|
|
415
425
|
const endpointUrl = dep.endpoint_url || dep.openai_base_url;
|
|
416
426
|
if (!endpointUrl) {
|
|
417
427
|
console.error(chalk.red(`\n ✗ No endpoint URL returned. Run: badgr status ${dep.deployment_id}\n`));
|
|
@@ -420,25 +430,36 @@ export async function comfyuiCommand(config, args, chalk) {
|
|
|
420
430
|
return;
|
|
421
431
|
}
|
|
422
432
|
|
|
433
|
+
const healthStageN = stageN;
|
|
434
|
+
const readyStageN = stageN + 1;
|
|
435
|
+
|
|
423
436
|
let endpointReady = false;
|
|
424
437
|
if (flags.noWait) {
|
|
425
|
-
console.log(chalk.
|
|
438
|
+
console.log(chalk.dim(_stage(healthStageN, STAGE_TOTAL, 'Checking health...')) + chalk.yellow(' (skipped — --no-wait)'));
|
|
426
439
|
} else {
|
|
427
|
-
const
|
|
428
|
-
|
|
440
|
+
const stageLine = _stage(healthStageN, STAGE_TOTAL, 'Checking health...');
|
|
441
|
+
const result = await waitForComfyUI(endpointUrl, dep.deployment_id, config, chalk, {
|
|
442
|
+
costPerHour: dep.cost_per_hour || 0, stageLine,
|
|
443
|
+
});
|
|
429
444
|
|
|
430
445
|
if (result.depFailed) {
|
|
431
446
|
console.error(formatCliError('HEALTH_CHECK_DEPLOY_FAILED', {
|
|
432
447
|
deploymentId: dep.deployment_id,
|
|
433
448
|
failReason: result.failReason,
|
|
434
449
|
}, chalk));
|
|
450
|
+
_printFailureClass(chalk, { failure_class: result.failureClass, next_action: result.nextAction });
|
|
435
451
|
updateReceipt(rcptId, { status: 'failed', failReason: result.failReason });
|
|
436
452
|
process.exitCode = 1;
|
|
437
453
|
return;
|
|
438
454
|
}
|
|
439
455
|
|
|
440
456
|
endpointReady = result.ready;
|
|
441
|
-
if (
|
|
457
|
+
if (endpointReady) {
|
|
458
|
+
console.log(chalk.dim(_stage(healthStageN, STAGE_TOTAL, 'Checking health...')));
|
|
459
|
+
console.log(chalk.green(_stage(readyStageN, STAGE_TOTAL, 'Ready')));
|
|
460
|
+
} else {
|
|
461
|
+
updateReceipt(rcptId, { status: 'health_check_timeout' });
|
|
462
|
+
}
|
|
442
463
|
}
|
|
443
464
|
|
|
444
465
|
if (flags.checkNodes && endpointReady) {
|
|
@@ -460,18 +481,11 @@ export async function comfyuiCommand(config, args, chalk) {
|
|
|
460
481
|
console.log();
|
|
461
482
|
}
|
|
462
483
|
|
|
463
|
-
const rate = dep.cost_per_hour || 0;
|
|
464
484
|
console.log(` ${chalk.bold('URL:')} ${chalk.cyan(endpointUrl)}`);
|
|
465
|
-
console.log(` ${chalk.bold('GPU:')} ${dep.gpu_type} × ${dep.gpu_count}`);
|
|
466
|
-
if (rate > 0) console.log(` ${chalk.bold('Rate:')} $${rate.toFixed(2)}/hr`);
|
|
467
485
|
if (flags.maxCost) console.log(` ${chalk.bold('Max cost:')} $${flags.maxCost.toFixed(2)}`);
|
|
468
486
|
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
469
487
|
console.log(` ${chalk.bold('Logs:')} ${chalk.dim(`badgr logs ${dep.deployment_id}`)}`);
|
|
470
488
|
console.log(` ${chalk.bold('Stop billing:')} ${chalk.cyan(`badgr down ${dep.deployment_id}`)}`);
|
|
471
489
|
console.log();
|
|
472
|
-
|
|
473
|
-
if (rate > HIGH_RATE_THRESHOLD && !flags.maxCost) {
|
|
474
|
-
console.log(chalk.yellow(` Rate: $${rate.toFixed(2)}/hr — use --max-cost to cap total spend.\n`));
|
|
475
|
-
}
|
|
476
490
|
console.log(chalk.dim(' Billing continues until you run: ') + chalk.cyan(`badgr down ${dep.deployment_id}`));
|
|
477
491
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { detectWorkload, workloadTypeLabel } from '../detect.js';
|
|
3
|
+
|
|
4
|
+
function fmtList(items, empty = 'none found') {
|
|
5
|
+
return items && items.length ? items.join(', ') : empty;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function suggestedCommandLine(report, { save } = {}) {
|
|
9
|
+
const parts = ['badgr run .'];
|
|
10
|
+
if (report.command) parts.push(`--cmd "${report.command}"`);
|
|
11
|
+
if (report.outputs?.length) parts.push(`--output ${report.outputs[0]}`);
|
|
12
|
+
if (report.checkpoints?.length) parts.push(`--checkpoint ${report.checkpoints[0]}`);
|
|
13
|
+
parts.push('--max-cost 5');
|
|
14
|
+
if (save) parts.push(`--save ${save}`);
|
|
15
|
+
return parts.join(' ');
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function renderDetectReport(report, chalk) {
|
|
19
|
+
const lines = [];
|
|
20
|
+
lines.push('');
|
|
21
|
+
lines.push(chalk.bold(`Workload: ${workloadTypeLabel(report.workloadType)}`));
|
|
22
|
+
const confColor = report.confidence === 'high' ? chalk.green : report.confidence === 'medium' ? chalk.yellow : chalk.red;
|
|
23
|
+
lines.push(`Confidence: ${confColor(report.confidence)}`);
|
|
24
|
+
lines.push('');
|
|
25
|
+
lines.push(`Detected command: ${report.command ? chalk.cyan(report.command) : chalk.dim('(none — pass --cmd)')}`);
|
|
26
|
+
if (report.dockerfile) lines.push(`Detected image: Dockerfile (${report.dockerfile})`);
|
|
27
|
+
if (report.requirements.length) lines.push(`Detected requirements: ${fmtList(report.requirements)}`);
|
|
28
|
+
lines.push(`Detected inputs: ${fmtList(report.inputs)}`);
|
|
29
|
+
lines.push(`Detected outputs: ${fmtList(report.outputs)}`);
|
|
30
|
+
if (report.checkpoints.length) lines.push(`Detected checkpoints: ${fmtList(report.checkpoints)}`);
|
|
31
|
+
lines.push(`Detected models/assets: ${fmtList(report.models)}`);
|
|
32
|
+
if (report.ports.length) lines.push(`Detected ports: ${report.ports.join(', ')}`);
|
|
33
|
+
lines.push(`Estimated VRAM: ${report.vram}`);
|
|
34
|
+
lines.push(`Estimated runtime: ${report.runtimeEstimateMinutes != null ? `${report.runtimeEstimateMinutes} min` : 'until stopped (endpoint)'}`);
|
|
35
|
+
if (report.signals.length) {
|
|
36
|
+
lines.push('');
|
|
37
|
+
lines.push(chalk.dim(`Why: ${report.signals.join('; ')}`));
|
|
38
|
+
}
|
|
39
|
+
lines.push('');
|
|
40
|
+
lines.push(`Suggested Badgr command:`);
|
|
41
|
+
lines.push(` ${chalk.cyan(suggestedCommandLine(report))}`);
|
|
42
|
+
lines.push('');
|
|
43
|
+
return lines.join('\n');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function detectCommand(config, args, chalk) {
|
|
47
|
+
const target = args.find(a => !a.startsWith('--')) || '.';
|
|
48
|
+
const report = detectWorkload(target);
|
|
49
|
+
|
|
50
|
+
if (report.error) {
|
|
51
|
+
console.error(chalk.red(`\n ✗ ${report.error}: ${path.resolve(target)}\n`));
|
|
52
|
+
process.exitCode = 1;
|
|
53
|
+
return report;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
console.log(renderDetectReport(report, chalk));
|
|
57
|
+
return report;
|
|
58
|
+
}
|
package/src/commands/receipts.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { listReceipts as localReceipts } from '../store.js';
|
|
2
|
-
import { listReceipts as apiReceipts, getReceipt as apiGetReceipt } from '../api.js';
|
|
1
|
+
import { listReceipts as localReceipts, findReceipt } from '../store.js';
|
|
2
|
+
import { listReceipts as apiReceipts, getReceipt as apiGetReceipt, getJobStatus } from '../api.js';
|
|
3
3
|
|
|
4
4
|
function fmtMs(ms) {
|
|
5
5
|
return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`;
|
|
@@ -54,6 +54,43 @@ export async function receiptsCommand(config, args, chalk) {
|
|
|
54
54
|
console.log(chalk.dim(' Run `badgr login` to look up receipts from the API.\n'));
|
|
55
55
|
return;
|
|
56
56
|
}
|
|
57
|
+
|
|
58
|
+
// Job-family receipts (badgr comfyui batch / train lora / run / serve)
|
|
59
|
+
// mint a *local* receipt id that points at a job_id — the backend's
|
|
60
|
+
// /v1/receipts endpoint has never heard of it (that's the separate LLM
|
|
61
|
+
// inference receipt ledger), so look it up via the job's own status
|
|
62
|
+
// instead of 404ing against the wrong system.
|
|
63
|
+
const local = findReceipt(firstArg);
|
|
64
|
+
if (local?.job_id) {
|
|
65
|
+
try {
|
|
66
|
+
const job = await getJobStatus(config, local.job_id);
|
|
67
|
+
if (job.status === 'queued' || job.status === 'provisioning' || job.status === 'running' || job.status === 'tearing_down') {
|
|
68
|
+
console.log(chalk.yellow(' Receipt is still finalizing. Try again shortly.\n'));
|
|
69
|
+
console.log(` ${chalk.bold('Job ID:')} ${local.job_id} (status: ${job.status})\n`);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
printReceipt({
|
|
73
|
+
receiptId: firstArg,
|
|
74
|
+
action: local.type,
|
|
75
|
+
status: job.status,
|
|
76
|
+
created_at: new Date(local.started_at || Date.now()).toISOString(),
|
|
77
|
+
runtimeSeconds: job.elapsed_seconds,
|
|
78
|
+
finalCost: job.charged_usd ?? undefined,
|
|
79
|
+
failure_reason: job.failure_class,
|
|
80
|
+
}, chalk);
|
|
81
|
+
console.log(` ${chalk.bold('Teardown:')} ${job.teardown_status === 'ok' ? 'succeeded' : job.teardown_status === 'failed' ? 'failed' : 'n/a'}`);
|
|
82
|
+
console.log(` ${chalk.bold('Billing:')} ${job.billing_status === 'stopped' ? 'stopped' : 'running'}\n`);
|
|
83
|
+
} catch (err) {
|
|
84
|
+
// The job finished (we have a local record of a terminal receipt),
|
|
85
|
+
// but the backend can't confirm it right now — don't silently show
|
|
86
|
+
// nothing, since billing may or may not actually be settled.
|
|
87
|
+
console.log(chalk.yellow(' Billing stopped, but receipt finalization failed.\n'));
|
|
88
|
+
console.log(` ${chalk.bold('Job ID:')} ${local.job_id}`);
|
|
89
|
+
console.log(chalk.dim(' Contact support with this ID.\n'));
|
|
90
|
+
}
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
|
|
57
94
|
try {
|
|
58
95
|
const r = await apiGetReceipt(config, firstArg);
|
|
59
96
|
printReceipt(r, chalk);
|
package/src/commands/run.js
CHANGED
|
@@ -8,7 +8,8 @@ import { addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
|
8
8
|
import { normalizeTier, callWithFallback, HIGH_RATE_THRESHOLD } from '../fallback.js';
|
|
9
9
|
import { formatCliError } from '../errors.js';
|
|
10
10
|
import { TEMPLATE_MAP, buildTemplateFlags, parseTemplateOverrides } from '../catalog.js';
|
|
11
|
-
import { stage as _stage, stageDone as _stageDone, writeBlock as _writeBlock, clearBlock as _clearBlock, renderLiveBlock as _renderLiveBlock } from '../progress.js';
|
|
11
|
+
import { stage as _stage, stageDone as _stageDone, writeBlock as _writeBlock, clearBlock as _clearBlock, renderLiveBlock as _renderLiveBlock, printFailureClass as _printFailureClass } from '../progress.js';
|
|
12
|
+
import { detectWorkload, workloadTypeLabel } from '../detect.js';
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Flow 1 — local project (primary):
|
|
@@ -56,6 +57,10 @@ export function parseRunArgs(args) {
|
|
|
56
57
|
if (flagArgs[i] === '--save') { flags.save = flagArgs[++i]; i++; continue; }
|
|
57
58
|
if (flagArgs[i] === '--workspace') { flags.workspace = flagArgs[++i]; i++; continue; }
|
|
58
59
|
if (flagArgs[i] === '--cmd') { flags.cmd = flagArgs[++i]; i++; continue; }
|
|
60
|
+
if (flagArgs[i] === '--output') { flags.output = flagArgs[++i]; i++; continue; }
|
|
61
|
+
if (flagArgs[i] === '--checkpoint') { flags.checkpoint = flagArgs[++i]; i++; continue; }
|
|
62
|
+
if (flagArgs[i] === '--retry-safe') { flags.retrySafe = true; i++; continue; }
|
|
63
|
+
if (flagArgs[i] === '--resume-cmd') { flags.resumeCmd = flagArgs[++i]; i++; continue; }
|
|
59
64
|
if (flagArgs[i] === '--env') {
|
|
60
65
|
const kv = flagArgs[++i]; i++;
|
|
61
66
|
if (!flags.env) flags.env = [];
|
|
@@ -339,6 +344,8 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
339
344
|
exitCode,
|
|
340
345
|
runtimeMs: Date.now() - startMs,
|
|
341
346
|
failureType: classifyFailure(status, exitCode),
|
|
347
|
+
failureClass: dep.failure_class ?? null,
|
|
348
|
+
nextAction: dep.next_action ?? null,
|
|
342
349
|
};
|
|
343
350
|
}
|
|
344
351
|
}
|
|
@@ -357,6 +364,7 @@ const _KNOWN_RUN_FLAGS = new Set([
|
|
|
357
364
|
'--detach', '--fallback', '--no-fallback', '--strict-capacity',
|
|
358
365
|
'--no-expanded-search', '--max-runtime', '--max-cost', '--min-vram', '--env',
|
|
359
366
|
'--dry-run', '--cmd', '--save', '--workspace',
|
|
367
|
+
'--output', '--checkpoint', '--retry-safe', '--resume-cmd',
|
|
360
368
|
]);
|
|
361
369
|
|
|
362
370
|
// Directories and files always excluded from project zip uploads.
|
|
@@ -497,10 +505,43 @@ export async function runCommand(config, args, chalk) {
|
|
|
497
505
|
const isGitHubUrl = firstArg && _isGitHubUrl(firstArg);
|
|
498
506
|
const isCodeSource = isLocalPath || isGitHubUrl;
|
|
499
507
|
|
|
508
|
+
// Local paths can be inspected on disk, so a missing --cmd doesn't have to be
|
|
509
|
+
// an error — try to infer it (and output/checkpoint conventions) the same way
|
|
510
|
+
// `badgr detect .` would, before ever provisioning anything.
|
|
511
|
+
let detectionReport = null;
|
|
512
|
+
if (isLocalPath && !flags.cmd) {
|
|
513
|
+
detectionReport = detectWorkload(firstArg);
|
|
514
|
+
if (detectionReport.command && detectionReport.confidence !== 'low') {
|
|
515
|
+
flags.cmd = detectionReport.command;
|
|
516
|
+
console.log(chalk.dim(`\n Detected command (${detectionReport.confidence} confidence): ${detectionReport.command}`));
|
|
517
|
+
console.log(chalk.dim(` Workload: ${workloadTypeLabel(detectionReport.workloadType)} • badgr detect . for the full report • --cmd to override`));
|
|
518
|
+
}
|
|
519
|
+
if (!flags.output && detectionReport.outputs?.length) flags.output = detectionReport.outputs[0];
|
|
520
|
+
if (!flags.checkpoint && detectionReport.checkpoints?.length) flags.checkpoint = detectionReport.checkpoints[0];
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// Detection couldn't confidently name a command — ask for just that one
|
|
524
|
+
// missing essential (interactively, if we have a terminal to ask on).
|
|
525
|
+
if (isLocalPath && !flags.cmd && process.stdin.isTTY && process.stdout.isTTY) {
|
|
526
|
+
console.log(chalk.dim(`\n Badgr couldn't confidently detect a command to run in ${path.resolve(firstArg)}.`));
|
|
527
|
+
try {
|
|
528
|
+
const { input } = await import('@inquirer/prompts');
|
|
529
|
+
const answer = await input({
|
|
530
|
+
message: 'Which command should run?',
|
|
531
|
+
default: detectionReport?.command || undefined,
|
|
532
|
+
validate: v => v.trim() ? true : 'A command is required',
|
|
533
|
+
});
|
|
534
|
+
flags.cmd = answer.trim();
|
|
535
|
+
} catch {
|
|
536
|
+
// Ctrl+C or a non-interactive stdin that lied about isTTY — fall through to the hard error below.
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
500
540
|
if (isCodeSource && !flags.cmd) {
|
|
501
541
|
console.error(chalk.red(`\n ✗ --cmd is required when running from a ${isLocalPath ? 'local path' : 'GitHub URL'}.\n`));
|
|
502
542
|
if (isLocalPath) {
|
|
503
543
|
console.error(chalk.dim(' Example: badgr run . --cmd "python train.py" --max-cost 5\n'));
|
|
544
|
+
console.error(chalk.dim(' Or check what Badgr detects first: badgr detect .\n'));
|
|
504
545
|
} else {
|
|
505
546
|
console.error(chalk.dim(' Example: badgr run https://github.com/user/repo --cmd "python train.py" --max-cost 5\n'));
|
|
506
547
|
}
|
|
@@ -550,6 +591,20 @@ export async function runCommand(config, args, chalk) {
|
|
|
550
591
|
return;
|
|
551
592
|
}
|
|
552
593
|
|
|
594
|
+
if (!flags.maxCost && !flags.dryRun && isLocalPath && process.stdin.isTTY && process.stdout.isTTY) {
|
|
595
|
+
try {
|
|
596
|
+
const { input } = await import('@inquirer/prompts');
|
|
597
|
+
const answer = await input({
|
|
598
|
+
message: 'What max cost ($) should cap this run?',
|
|
599
|
+
default: '5',
|
|
600
|
+
validate: v => (Number.isFinite(parseFloat(v)) && parseFloat(v) > 0) ? true : 'Enter a number greater than 0',
|
|
601
|
+
});
|
|
602
|
+
flags.maxCost = parseFloat(answer);
|
|
603
|
+
} catch {
|
|
604
|
+
// fall through to the hard error below
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
553
608
|
if (!flags.maxCost && !flags.dryRun) {
|
|
554
609
|
console.error(chalk.red('\n ✗ --max-cost is required for run workloads.\n'));
|
|
555
610
|
console.error(chalk.dim(' Example: badgr run --gpu RTX_4090 --image node:20 --max-cost 5 -- node script.js\n'));
|
|
@@ -578,7 +633,16 @@ export async function runCommand(config, args, chalk) {
|
|
|
578
633
|
const maxRuntimeMs = effectiveMaxRuntime * 60 * 1000;
|
|
579
634
|
|
|
580
635
|
const maxCost = flags.maxCost ?? null;
|
|
581
|
-
|
|
636
|
+
// BADGR_OUTPUT_DIR / BADGR_CHECKPOINT_DIR / BADGR_RETRY_SAFE are a
|
|
637
|
+
// convention for custom.run containers to write recoverable state outside
|
|
638
|
+
// the pod — Badgr doesn't enforce what the container does with them, it
|
|
639
|
+
// just wires the directories through and surfaces --resume-cmd on failure.
|
|
640
|
+
// Explicit --env always wins if the user passes the same key directly.
|
|
641
|
+
const conventionEnv = {};
|
|
642
|
+
if (flags.output) conventionEnv.BADGR_OUTPUT_DIR = flags.output;
|
|
643
|
+
if (flags.checkpoint) conventionEnv.BADGR_CHECKPOINT_DIR = flags.checkpoint;
|
|
644
|
+
if (flags.retrySafe) conventionEnv.BADGR_RETRY_SAFE = '1';
|
|
645
|
+
const envObj = { ...conventionEnv, ...parseEnvFlag(flags.env) };
|
|
582
646
|
const effectiveTier = normalizeTier(flags.tier);
|
|
583
647
|
|
|
584
648
|
const gpu = flags.gpu ? flags.gpu.toUpperCase().replace('-', '_') : undefined;
|
|
@@ -595,6 +659,10 @@ export async function runCommand(config, args, chalk) {
|
|
|
595
659
|
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost}`);
|
|
596
660
|
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice}/hr`);
|
|
597
661
|
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${flags.env.join(', ')}`);
|
|
662
|
+
if (flags.output) console.log(` ${chalk.bold('Output:')} ${flags.output}`);
|
|
663
|
+
if (flags.checkpoint) console.log(` ${chalk.bold('Checkpoint:')} ${flags.checkpoint}`);
|
|
664
|
+
if (flags.retrySafe) console.log(` ${chalk.bold('Retry-safe:')} enabled`);
|
|
665
|
+
if (flags.resumeCmd) console.log(` ${chalk.bold('Resume cmd:')} ${flags.resumeCmd}`);
|
|
598
666
|
console.log(chalk.dim('\n Remove --dry-run to provision.\n'));
|
|
599
667
|
return;
|
|
600
668
|
}
|
|
@@ -616,6 +684,9 @@ export async function runCommand(config, args, chalk) {
|
|
|
616
684
|
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice.toFixed(2)}/hr`);
|
|
617
685
|
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
618
686
|
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${flags.env.join(', ')}`);
|
|
687
|
+
if (flags.output) console.log(` ${chalk.bold('Output:')} ${flags.output}`);
|
|
688
|
+
if (flags.checkpoint) console.log(` ${chalk.bold('Checkpoint:')} ${flags.checkpoint}`);
|
|
689
|
+
if (flags.retrySafe) console.log(` ${chalk.bold('Retry-safe:')} enabled`);
|
|
619
690
|
console.log();
|
|
620
691
|
|
|
621
692
|
|
|
@@ -775,6 +846,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
775
846
|
console.log(chalk.dim(_stage(STAGE_TOTAL, STAGE_TOTAL, stageLabels[reason] ?? 'Stopped')));
|
|
776
847
|
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
777
848
|
_printFinalInfo(chalk, { exitCode: null, teardownOk, jobId: dep.deployment_id, rcptId });
|
|
849
|
+
if (flags.resumeCmd) console.log(` ${chalk.bold('Resume:')} ${flags.resumeCmd}`);
|
|
778
850
|
console.log();
|
|
779
851
|
}
|
|
780
852
|
|
|
@@ -805,6 +877,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
805
877
|
if (dep.status === 'failed') {
|
|
806
878
|
process.removeListener('SIGINT', handleShutdown);
|
|
807
879
|
console.error(formatCliError('JOB_INFRASTRUCTURE_FAILURE', { receiptId: rcptId }, chalk));
|
|
880
|
+
_printFailureClass(chalk, dep);
|
|
808
881
|
process.exitCode = 1;
|
|
809
882
|
return;
|
|
810
883
|
}
|
|
@@ -813,7 +886,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
813
886
|
console.log(chalk.dim(`\n [${stageN}/${STAGE_TOTAL}] Running command (Ctrl+C to stop)`));
|
|
814
887
|
|
|
815
888
|
attachStart = Date.now();
|
|
816
|
-
const { status: finalStatus, exitCode, runtimeMs, failureType } = await attachToJob(config, dep.deployment_id, {
|
|
889
|
+
const { status: finalStatus, exitCode, runtimeMs, failureType, failureClass, nextAction } = await attachToJob(config, dep.deployment_id, {
|
|
817
890
|
chalk,
|
|
818
891
|
maxRuntimeMs,
|
|
819
892
|
maxCost,
|
|
@@ -853,11 +926,13 @@ export async function runCommand(config, args, chalk) {
|
|
|
853
926
|
} else {
|
|
854
927
|
console.error(formatCliError('JOB_FAILED', { exitCode, deploymentId: dep.deployment_id }, chalk));
|
|
855
928
|
}
|
|
929
|
+
_printFailureClass(chalk, { failure_class: failureClass, next_action: nextAction });
|
|
856
930
|
console.log(chalk.dim(_stage(STAGE_TOTAL, STAGE_TOTAL, 'Failed')));
|
|
857
931
|
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
858
932
|
// The container already reached a terminal state on the provider side by the
|
|
859
933
|
// time we observe it here, so billing is already stopped — no extra teardown call needed.
|
|
860
934
|
_printFinalInfo(chalk, { exitCode, teardownOk: true, jobId: dep.deployment_id, rcptId });
|
|
935
|
+
if (flags.resumeCmd) console.log(` ${chalk.bold('Resume:')} ${flags.resumeCmd}`);
|
|
861
936
|
process.exitCode = exitCode ?? 1;
|
|
862
937
|
return;
|
|
863
938
|
}
|