badgr-cli 1.0.47 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/package.json +1 -1
- package/src/api.js +16 -2
- package/src/artifactDownload.js +55 -0
- package/src/badgr.js +104 -0
- package/src/batch.js +22 -4
- package/src/browser.js +23 -0
- package/src/commands/artifacts.js +75 -0
- package/src/commands/batch.js +221 -20
- package/src/commands/billing.js +1 -12
- package/src/commands/capacity.js +9 -4
- package/src/commands/comfyui.js +3 -3
- package/src/commands/connect.js +83 -0
- package/src/commands/doctor.js +127 -0
- package/src/commands/down.js +29 -6
- package/src/commands/launch.js +431 -0
- package/src/commands/pull.js +137 -0
- package/src/commands/run.js +253 -37
- package/src/commands/sbatch.js +232 -0
- package/src/commands/serve.js +3 -3
- package/src/commands/status.js +12 -4
- package/src/commands/task.js +25 -0
- package/src/commands/test-run.js +4 -2
- package/src/credentials.js +65 -0
- package/src/fallback.js +7 -2
- package/src/fanout.js +70 -0
- package/src/gpuDoctor/diskInfo.js +42 -0
- package/src/gpuDoctor/doctor.js +451 -0
- package/src/gpuDoctor/gpuInfo.js +70 -0
- package/src/gpuDoctor/healthCheck.js +63 -0
- package/src/gpuDoctor/logClassifier.js +138 -0
- package/src/gpuDoctor/modelFit.js +107 -0
- package/src/gpuDoctor/probeCache.js +38 -0
- package/src/gpuDoctor/redact.js +29 -0
- package/src/gpuDoctor/torchInfo.js +61 -0
- package/src/gpuDoctor/workflowDoctor.js +96 -0
- package/src/onboarding.js +124 -0
- package/src/slurm.js +193 -0
- package/src/spec.js +59 -2
- package/src/store.js +16 -0
- package/tests/agent-images.test.js +17 -0
- package/tests/artifactDownload.test.js +113 -0
- package/tests/artifacts.test.js +168 -0
- package/tests/batch.test.js +312 -0
- package/tests/browser.test.js +51 -0
- package/tests/capacity.test.js +68 -0
- package/tests/commands.test.js +44 -0
- package/tests/connect.test.js +83 -0
- package/tests/down.test.js +23 -1
- package/tests/fallback-timeout.test.js +41 -0
- package/tests/fanout.test.js +124 -0
- package/tests/gpu-doctor-classifiers.test.js +402 -0
- package/tests/gpu-doctor-doctor.test.js +304 -0
- package/tests/gpu-doctor-probe-cache.test.js +110 -0
- package/tests/gpu-doctor-probes.test.js +257 -0
- package/tests/launch-command-argv.test.js +93 -0
- package/tests/launch-readiness.test.js +1 -0
- package/tests/launch.test.js +440 -0
- package/tests/onboarding.test.js +134 -0
- package/tests/pull.test.js +266 -0
- package/tests/run-lifecycle.test.js +405 -6
- package/tests/sbatch.test.js +190 -0
- package/tests/secrets.test.js +16 -0
- package/tests/slurm.test.js +77 -0
- package/tests/spec.test.js +59 -1
- package/tests/status.test.js +73 -0
- package/tests/task.test.js +109 -0
- package/tests/template.test.js +7 -0
package/src/commands/batch.js
CHANGED
|
@@ -13,16 +13,18 @@
|
|
|
13
13
|
* eval) — a thin wrapper over the same /v1/run + monitorBatchJob primitives
|
|
14
14
|
* `badgr train`/`transcribe`/`embed` already use, not a new execution path.
|
|
15
15
|
*/
|
|
16
|
-
import { existsSync,
|
|
16
|
+
import { existsSync, readFileSync, unlinkSync, statSync, readdirSync } from 'fs';
|
|
17
17
|
import { join, resolve } from 'path';
|
|
18
18
|
import os from 'os';
|
|
19
19
|
|
|
20
20
|
import { requireApiKey, CONFIG_DIR } from '../config.js';
|
|
21
21
|
import { parseWorkloadYaml } from '../workloadSpec.js';
|
|
22
|
-
import { addReceipt, updateReceipt, generateReceiptId, loadStore } from '../store.js';
|
|
22
|
+
import { addReceipt, updateReceipt, generateReceiptId, loadStore, selectedComputeFromDeployment } from '../store.js';
|
|
23
23
|
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
24
24
|
import { monitorBatchJob, fmtRuntime } from '../batch.js';
|
|
25
|
+
import { runFanOut, DEFAULT_CONCURRENCY } from '../fanout.js';
|
|
25
26
|
import { uploadBlob } from '../api.js';
|
|
27
|
+
import { downloadAndExtractArtifact } from '../artifactDownload.js';
|
|
26
28
|
import { logsCommand } from './logs.js';
|
|
27
29
|
|
|
28
30
|
// Server-side watchdog / batch.js reason strings, mapped to the vocabulary
|
|
@@ -61,10 +63,21 @@ async function apiRequest(config, path, opts = {}) {
|
|
|
61
63
|
// inputs archive (tar.gz, member paths = container path minus leading '/')
|
|
62
64
|
// ---------------------------------------------------------------------------
|
|
63
65
|
|
|
66
|
+
let _archiveTmpFileCounter = 0;
|
|
67
|
+
|
|
68
|
+
// Date.now() alone collides when --fan-out builds multiple archives
|
|
69
|
+
// concurrently (millisecond resolution, sub-millisecond call spacing) — a
|
|
70
|
+
// monotonic counter makes every call's tmp path unique regardless of
|
|
71
|
+
// timing. Exported as a pure function so the uniqueness guarantee is
|
|
72
|
+
// directly unit-testable without exercising the real filesystem/archiver.
|
|
73
|
+
export function nextArchiveTmpFile() {
|
|
74
|
+
return join(os.tmpdir(), `badgr-batch-inputs-${Date.now()}-${++_archiveTmpFileCounter}.tar.gz`);
|
|
75
|
+
}
|
|
76
|
+
|
|
64
77
|
async function _buildInputsArchive(inputs, chalk) {
|
|
65
78
|
const { default: archiver } = await import('archiver');
|
|
66
79
|
const { createWriteStream } = await import('fs');
|
|
67
|
-
const tmpFile =
|
|
80
|
+
const tmpFile = nextArchiveTmpFile();
|
|
68
81
|
|
|
69
82
|
await new Promise((res, rej) => {
|
|
70
83
|
const output = createWriteStream(tmpFile);
|
|
@@ -137,29 +150,17 @@ async function _fetchArtifactMetadata(config, runId) {
|
|
|
137
150
|
}
|
|
138
151
|
|
|
139
152
|
async function downloadAndExtractArtifacts(config, runId) {
|
|
140
|
-
|
|
153
|
+
await _fetchArtifactMetadata(config, runId); // 404/410 raise their own clear messages before we even try downloading
|
|
141
154
|
const destDir = artifactsDir(runId);
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
headers: { Authorization: `Bearer ${config.apiKey}` },
|
|
146
|
-
});
|
|
147
|
-
if (!res.ok) {
|
|
155
|
+
try {
|
|
156
|
+
await downloadAndExtractArtifact(config, runId, destDir);
|
|
157
|
+
} catch (err) {
|
|
148
158
|
throw new Error(
|
|
149
|
-
`Artifact download failed for run ${runId}: ${
|
|
159
|
+
`Artifact download failed for run ${runId}: ${err.httpStatus ?? ''} ${err.statusText ?? err.message}. ` +
|
|
150
160
|
`The upload succeeded (metadata exists) but the file itself couldn't be fetched — retry, ` +
|
|
151
161
|
`or check backend storage (BADGR_STORAGE_DIR) if this persists.`,
|
|
152
162
|
);
|
|
153
163
|
}
|
|
154
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
155
|
-
const tmpFile = join(os.tmpdir(), `badgr-batch-artifacts-${runId}.tar.gz`);
|
|
156
|
-
writeFileSync(tmpFile, buf);
|
|
157
|
-
try {
|
|
158
|
-
const tar = await import('tar');
|
|
159
|
-
await tar.x({ file: tmpFile, cwd: destDir });
|
|
160
|
-
} finally {
|
|
161
|
-
unlinkSync(tmpFile);
|
|
162
|
-
}
|
|
163
164
|
return destDir;
|
|
164
165
|
}
|
|
165
166
|
|
|
@@ -181,6 +182,191 @@ export function readMetricFile(destDir, metricFilePath) {
|
|
|
181
182
|
// badgr batch run <workload.yml>
|
|
182
183
|
// ---------------------------------------------------------------------------
|
|
183
184
|
|
|
185
|
+
// ---------------------------------------------------------------------------
|
|
186
|
+
// badgr batch run <workload.yml> --fan-out <dir>
|
|
187
|
+
//
|
|
188
|
+
// "Run one program across many inputs" — the single non-trivial pattern
|
|
189
|
+
// that covers batch inference, rendering frames, Monte Carlo sweeps, robotics
|
|
190
|
+
// scenarios, and dozens of other superficially different verticals. One
|
|
191
|
+
// deployment per file in <dir>, run in parallel, rolled up into one summary.
|
|
192
|
+
// Shares its submit/monitor/summarize loop with `badgr sbatch --array` via
|
|
193
|
+
// ../fanout.js — same "N independent tasks" shape, two entry points.
|
|
194
|
+
// ---------------------------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
function listFanOutFiles(dir) {
|
|
197
|
+
return readdirSync(dir, { withFileTypes: true })
|
|
198
|
+
.filter(e => e.isFile())
|
|
199
|
+
.map(e => e.name)
|
|
200
|
+
.sort();
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
async function runFanOutSubcommand(config, spec, flags, chalk, yamlPath) {
|
|
204
|
+
if (spec.inputs.length !== 1) {
|
|
205
|
+
console.error(chalk.red(
|
|
206
|
+
`\n ✗ --fan-out requires workload.yml to declare exactly one inputs: entry ` +
|
|
207
|
+
`(the file that varies per task) — found ${spec.inputs.length}.\n`,
|
|
208
|
+
));
|
|
209
|
+
process.exitCode = 1;
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const fanOutDir = resolve(flags.fanOut);
|
|
213
|
+
if (!existsSync(fanOutDir) || !statSync(fanOutDir).isDirectory()) {
|
|
214
|
+
console.error(chalk.red(`\n ✗ --fan-out path is not a directory: ${fanOutDir}\n`));
|
|
215
|
+
process.exitCode = 1;
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (flags.maxConcurrency != null && (!Number.isInteger(flags.maxConcurrency) || flags.maxConcurrency < 1)) {
|
|
219
|
+
console.error(chalk.red(`\n ✗ --max-concurrency must be a positive integer, got: ${flags.maxConcurrency}\n`));
|
|
220
|
+
process.exitCode = 1;
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
let files = listFanOutFiles(fanOutDir);
|
|
224
|
+
if (files.length === 0) {
|
|
225
|
+
console.error(chalk.red(`\n ✗ --fan-out directory has no files: ${fanOutDir}\n`));
|
|
226
|
+
process.exitCode = 1;
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (flags.only?.length) {
|
|
230
|
+
const onlySet = new Set(flags.only);
|
|
231
|
+
const missing = flags.only.filter(f => !files.includes(f));
|
|
232
|
+
if (missing.length) {
|
|
233
|
+
console.error(chalk.red(`\n ✗ --only names file(s) not found in ${fanOutDir}: ${missing.join(', ')}\n`));
|
|
234
|
+
process.exitCode = 1;
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
files = files.filter(f => onlySet.has(f));
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const [{ containerPath }] = spec.inputs;
|
|
241
|
+
const effectiveTier = normalizeTier(flags.tier);
|
|
242
|
+
const concurrency = flags.maxConcurrency ?? DEFAULT_CONCURRENCY;
|
|
243
|
+
|
|
244
|
+
console.log(chalk.bold(`\n📦 Batch fan-out: ${spec.name}\n`));
|
|
245
|
+
console.log(` ${chalk.bold('Image:')} ${spec.image}`);
|
|
246
|
+
console.log(` ${chalk.bold('Command:')} ${spec.command.join(' ')}`);
|
|
247
|
+
console.log(` ${chalk.bold('Inputs dir:')} ${fanOutDir} (${files.length} task(s), max ${Math.min(concurrency, files.length)} concurrent)`);
|
|
248
|
+
console.log(` ${chalk.bold('Max cost:')} $${spec.maxCost.toFixed(2)} per task`);
|
|
249
|
+
console.log(` ${chalk.bold('Max runtime:')} ${spec.maxRuntimeMinutes}min per task`);
|
|
250
|
+
console.log();
|
|
251
|
+
|
|
252
|
+
if (flags.dryRun) {
|
|
253
|
+
console.log(chalk.bold(`⚡ Dry run — no GPU will be provisioned (${files.length} task(s) would be submitted)\n`));
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const { results, allSucceeded, submittedCount, failedSubmitCount } = await runFanOut({
|
|
258
|
+
tasks: files,
|
|
259
|
+
concurrency,
|
|
260
|
+
onSubmitError: (file, err) => {
|
|
261
|
+
if (err.isPaymentRequired) console.error(chalk.yellow(err.message));
|
|
262
|
+
else console.error(chalk.red(` ✗ ${file} failed to submit: ${err.message}`));
|
|
263
|
+
},
|
|
264
|
+
submitTask: async (file) => {
|
|
265
|
+
process.stdout.write(chalk.dim(` Submitting ${file}...\n`));
|
|
266
|
+
const localPath = join(fanOutDir, file);
|
|
267
|
+
const inputsArchiveUrl = await _uploadInputsArchive(config, [{ localPath, containerPath }], chalk);
|
|
268
|
+
|
|
269
|
+
const envObj = { ...spec.env, BADGR_INPUTS_ARCHIVE_URL: inputsArchiveUrl, BADGR_FANOUT_INPUT: file };
|
|
270
|
+
if (spec.outputs.length > 0) envObj.BADGR_OUTPUT_PATHS_JSON = JSON.stringify(spec.outputs);
|
|
271
|
+
|
|
272
|
+
const body = {
|
|
273
|
+
image: spec.image,
|
|
274
|
+
command: spec.command,
|
|
275
|
+
custom_image: true,
|
|
276
|
+
gpu: flags.gpu ? flags.gpu.toUpperCase().replace(/-/g, '_') : 'AUTO',
|
|
277
|
+
gpu_count: 1,
|
|
278
|
+
...(flags.region ? { region: flags.region.toUpperCase() } : {}),
|
|
279
|
+
tier: effectiveTier,
|
|
280
|
+
env: envObj,
|
|
281
|
+
name: `${spec.name}-${file.replace(/[^\w.-]/g, '_')}`,
|
|
282
|
+
max_cost_usd: spec.maxCost,
|
|
283
|
+
max_runtime_seconds: spec.maxRuntimeMinutes * 60,
|
|
284
|
+
};
|
|
285
|
+
const dep = await callWithFallback(
|
|
286
|
+
'/run',
|
|
287
|
+
{ apiKey: config.apiKey, baseUrl: config.baseUrl },
|
|
288
|
+
() => body,
|
|
289
|
+
effectiveTier,
|
|
290
|
+
chalk,
|
|
291
|
+
{ thing: `fan-out task (${file})`, cmd: 'badgr batch run --fan-out' },
|
|
292
|
+
);
|
|
293
|
+
const rcptId = dep.receipt_id || generateReceiptId();
|
|
294
|
+
addReceipt({
|
|
295
|
+
receiptId: rcptId,
|
|
296
|
+
action: 'badgr batch run --fan-out',
|
|
297
|
+
deploymentId: dep.deployment_id,
|
|
298
|
+
workloadName: body.name,
|
|
299
|
+
workloadImage: spec.image,
|
|
300
|
+
outputsDeclared: spec.outputs,
|
|
301
|
+
gpu: dep.gpu_type,
|
|
302
|
+
providerRoute: dep.provider ?? null,
|
|
303
|
+
tier: dep.tier ?? null,
|
|
304
|
+
maxCost: spec.maxCost,
|
|
305
|
+
maxRuntime: spec.maxRuntimeMinutes,
|
|
306
|
+
status: dep.status,
|
|
307
|
+
createdAt: new Date().toISOString(),
|
|
308
|
+
fanOutInput: file,
|
|
309
|
+
workloadShape: 'fan-out',
|
|
310
|
+
computeSelected: selectedComputeFromDeployment(dep),
|
|
311
|
+
});
|
|
312
|
+
console.log(` ${chalk.bold('Run ID:')} ${chalk.cyan(dep.deployment_id)} ${chalk.dim(file)}`);
|
|
313
|
+
return { deploymentId: dep.deployment_id, receiptId: rcptId, ratePerHour: dep.cost_per_hour || 0 };
|
|
314
|
+
},
|
|
315
|
+
monitorTask: async ({ deploymentId, receiptId, ratePerHour }) => {
|
|
316
|
+
const result = await monitorBatchJob(config, deploymentId, receiptId, {
|
|
317
|
+
chalk,
|
|
318
|
+
maxRuntimeMs: spec.maxRuntimeMinutes * 60 * 1000,
|
|
319
|
+
maxCost: spec.maxCost,
|
|
320
|
+
ratePerHour,
|
|
321
|
+
});
|
|
322
|
+
let dep2 = null;
|
|
323
|
+
try {
|
|
324
|
+
dep2 = await apiRequest(config, `/deployments/${deploymentId}`);
|
|
325
|
+
} catch { /* best-effort */ }
|
|
326
|
+
updateReceipt(receiptId, {
|
|
327
|
+
status: dep2?.status ?? result.status,
|
|
328
|
+
failureReason: dep2?.failure_reason ?? null,
|
|
329
|
+
teardownOk: dep2?.teardown_ok ?? null,
|
|
330
|
+
runtimeSeconds: dep2?.runtime_seconds ?? Math.round(result.runtimeMs / 1000),
|
|
331
|
+
finalCost: dep2?.accrued_cost_usd ?? (ratePerHour * (result.runtimeMs / 3_600_000)),
|
|
332
|
+
});
|
|
333
|
+
const status = dep2?.status ?? result.status;
|
|
334
|
+
return { status, succeeded: ['succeeded', 'completed'].includes(status), runtimeMs: result.runtimeMs };
|
|
335
|
+
},
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
if (submittedCount === 0) {
|
|
339
|
+
console.error(chalk.red('\n ✗ No fan-out tasks were submitted\n'));
|
|
340
|
+
process.exitCode = 1;
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
console.log(chalk.bold(`\n📦 Fan-out summary: ${spec.name}\n`));
|
|
345
|
+
for (const r of results) {
|
|
346
|
+
if (r.error) {
|
|
347
|
+
console.log(` ${chalk.red('✗')} ${r.task} ${chalk.dim('never submitted')}`);
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
const icon = r.succeeded ? chalk.green('✓') : chalk.red('✗');
|
|
351
|
+
console.log(` ${icon} ${r.task} ${r.deploymentId} ${r.status} ${fmtRuntime(r.runtimeMs)}`);
|
|
352
|
+
}
|
|
353
|
+
if (failedSubmitCount) console.log(chalk.red(` ${failedSubmitCount} task(s) never submitted`));
|
|
354
|
+
|
|
355
|
+
console.log();
|
|
356
|
+
if (allSucceeded) {
|
|
357
|
+
console.log(chalk.green(` ✓ Fan-out complete (${files.length}/${files.length} task(s) succeeded)\n`));
|
|
358
|
+
} else {
|
|
359
|
+
const okCount = results.filter(r => r.succeeded).length;
|
|
360
|
+
const failedFiles = results.filter(r => r.error || !r.succeeded).map(r => r.task);
|
|
361
|
+
console.error(chalk.red(` ✗ Fan-out incomplete (${okCount}/${files.length} task(s) succeeded)\n`));
|
|
362
|
+
if (failedFiles.length) {
|
|
363
|
+
console.log(chalk.dim(' Rerun only the failed task(s):'));
|
|
364
|
+
console.log(chalk.cyan(` badgr batch run ${yamlPath} --fan-out ${fanOutDir} --only ${failedFiles.join(',')}\n`));
|
|
365
|
+
}
|
|
366
|
+
process.exitCode = 1;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
184
370
|
async function runSubcommand(config, args, chalk) {
|
|
185
371
|
const flags = {};
|
|
186
372
|
const positional = [];
|
|
@@ -188,6 +374,10 @@ async function runSubcommand(config, args, chalk) {
|
|
|
188
374
|
if (args[i] === '--tier') { flags.tier = args[++i]; continue; }
|
|
189
375
|
if (args[i] === '--region') { flags.region = args[++i]; continue; }
|
|
190
376
|
if (args[i] === '--gpu') { flags.gpu = args[++i]; continue; }
|
|
377
|
+
if (args[i] === '--fan-out') { flags.fanOut = args[++i]; continue; }
|
|
378
|
+
if (args[i] === '--max-concurrency') { flags.maxConcurrency = parseInt(args[++i], 10); continue; }
|
|
379
|
+
if (args[i] === '--only') { flags.only = args[++i].split(',').map(s => s.trim()).filter(Boolean); continue; }
|
|
380
|
+
if (args[i] === '--dry-run') { flags.dryRun = true; continue; }
|
|
191
381
|
positional.push(args[i]);
|
|
192
382
|
}
|
|
193
383
|
const yamlPath = positional[0];
|
|
@@ -208,6 +398,10 @@ async function runSubcommand(config, args, chalk) {
|
|
|
208
398
|
return;
|
|
209
399
|
}
|
|
210
400
|
|
|
401
|
+
if (flags.fanOut) {
|
|
402
|
+
return runFanOutSubcommand(config, spec, flags, chalk, yamlPath);
|
|
403
|
+
}
|
|
404
|
+
|
|
211
405
|
console.log(chalk.bold(`\n📦 Batch: ${spec.name}\n`));
|
|
212
406
|
console.log(` ${chalk.bold('Image:')} ${spec.image}`);
|
|
213
407
|
console.log(` ${chalk.bold('Command:')} ${spec.command.join(' ')}`);
|
|
@@ -217,6 +411,11 @@ async function runSubcommand(config, args, chalk) {
|
|
|
217
411
|
if (spec.outputs.length) console.log(` ${chalk.bold('Outputs:')} ${spec.outputs.join(', ')}`);
|
|
218
412
|
console.log();
|
|
219
413
|
|
|
414
|
+
if (flags.dryRun) {
|
|
415
|
+
console.log(chalk.bold('⚡ Dry run — no GPU will be provisioned\n'));
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
418
|
+
|
|
220
419
|
let inputsArchiveUrl = null;
|
|
221
420
|
if (spec.inputs.length > 0) {
|
|
222
421
|
try {
|
|
@@ -288,6 +487,8 @@ async function runSubcommand(config, args, chalk) {
|
|
|
288
487
|
maxRuntime: spec.maxRuntimeMinutes,
|
|
289
488
|
status: dep.status,
|
|
290
489
|
createdAt: new Date().toISOString(),
|
|
490
|
+
workloadShape: 'container-batch',
|
|
491
|
+
computeSelected: selectedComputeFromDeployment(dep),
|
|
291
492
|
});
|
|
292
493
|
|
|
293
494
|
const rate = dep.cost_per_hour || 0;
|
package/src/commands/billing.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { execSync } from 'child_process';
|
|
2
1
|
import { requireApiKey } from '../config.js';
|
|
3
2
|
import { callApi } from '../api.js';
|
|
3
|
+
import { openBrowser } from '../browser.js';
|
|
4
4
|
|
|
5
5
|
const BILLING_HELP = `
|
|
6
6
|
badgr billing — manage your AI Badgr balance
|
|
@@ -16,17 +16,6 @@ EXAMPLES
|
|
|
16
16
|
badgr billing add 50
|
|
17
17
|
`;
|
|
18
18
|
|
|
19
|
-
function openBrowser(url) {
|
|
20
|
-
const platform = process.platform;
|
|
21
|
-
try {
|
|
22
|
-
if (platform === 'darwin') execSync(`open "${url}"`);
|
|
23
|
-
else if (platform === 'win32') execSync(`start "" "${url}"`);
|
|
24
|
-
else execSync(`xdg-open "${url}"`);
|
|
25
|
-
} catch {
|
|
26
|
-
// Silently ignore — we print the URL anyway
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
19
|
async function billingStatus(config, chalk) {
|
|
31
20
|
requireApiKey(config);
|
|
32
21
|
try {
|
package/src/commands/capacity.js
CHANGED
|
@@ -21,7 +21,9 @@ export async function capacityCommand(config, args, chalk) {
|
|
|
21
21
|
const flags = parseCapacityArgs(args);
|
|
22
22
|
requireApiKey(config);
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
// No --max-price: no cap. Availability browsing shouldn't silently hide
|
|
25
|
+
// real capacity behind an unrequested price ceiling.
|
|
26
|
+
const maxPrice = flags.maxPrice;
|
|
25
27
|
|
|
26
28
|
// No --gpu: show cheapest runnable GPU across all types
|
|
27
29
|
if (!flags.gpu) {
|
|
@@ -30,7 +32,8 @@ export async function capacityCommand(config, args, chalk) {
|
|
|
30
32
|
|
|
31
33
|
let data;
|
|
32
34
|
try {
|
|
33
|
-
const params = new URLSearchParams(
|
|
35
|
+
const params = new URLSearchParams();
|
|
36
|
+
if (maxPrice !== undefined) params.set('max_price', String(maxPrice));
|
|
34
37
|
if (flags.region) params.set('region', flags.region.toUpperCase());
|
|
35
38
|
data = await callApi(`/capacity/auto?${params}`, {
|
|
36
39
|
apiKey: config.apiKey,
|
|
@@ -60,7 +63,8 @@ export async function capacityCommand(config, args, chalk) {
|
|
|
60
63
|
|
|
61
64
|
// --gpu specified: show availability for that type
|
|
62
65
|
const gpu = flags.gpu.toUpperCase().replace('-', '_');
|
|
63
|
-
const params = new URLSearchParams({ gpu
|
|
66
|
+
const params = new URLSearchParams({ gpu });
|
|
67
|
+
if (maxPrice !== undefined) params.set('max_price', String(maxPrice));
|
|
64
68
|
if (flags.region) params.set('region', flags.region.toUpperCase());
|
|
65
69
|
|
|
66
70
|
console.log(chalk.bold(`\nCapacity: ${gpu}\n`));
|
|
@@ -90,7 +94,8 @@ export async function capacityCommand(config, args, chalk) {
|
|
|
90
94
|
console.log();
|
|
91
95
|
} else {
|
|
92
96
|
const regionLabel = flags.region ? ` in ${flags.region.toUpperCase()}` : '';
|
|
93
|
-
|
|
97
|
+
const priceLabel = maxPrice !== undefined ? ` under $${maxPrice.toFixed(2)}/hr` : '';
|
|
98
|
+
console.log(chalk.dim(`\n No ${gpu} available right now${priceLabel}${regionLabel}.\n`));
|
|
94
99
|
|
|
95
100
|
const alternatives = data.alternatives ?? [];
|
|
96
101
|
if (alternatives.length > 0) {
|
package/src/commands/comfyui.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* can queue it at startup.
|
|
7
7
|
*/
|
|
8
8
|
import { readFileSync, existsSync } from 'fs';
|
|
9
|
-
import {
|
|
9
|
+
import { ensureBadgrReady } from '../onboarding.js';
|
|
10
10
|
import { callApi, listDeployments } from '../api.js';
|
|
11
11
|
import { addDeployment, addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
12
12
|
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
@@ -192,7 +192,7 @@ export async function comfyBatchCommand(config, args, chalk) {
|
|
|
192
192
|
return;
|
|
193
193
|
}
|
|
194
194
|
|
|
195
|
-
|
|
195
|
+
config = await ensureBadgrReady(config, chalk);
|
|
196
196
|
|
|
197
197
|
if (prompts.length === 0) {
|
|
198
198
|
console.error(chalk.red('\n ✗ No prompts provided. Use --prompts file.txt or --prompt "text"\n'));
|
|
@@ -282,7 +282,7 @@ export async function comfyuiCommand(config, args, chalk) {
|
|
|
282
282
|
return;
|
|
283
283
|
}
|
|
284
284
|
|
|
285
|
-
|
|
285
|
+
config = await ensureBadgrReady(config, chalk);
|
|
286
286
|
|
|
287
287
|
if (!flags.maxCost && !flags.persistent) {
|
|
288
288
|
console.error(chalk.red('\n ✗ ComfyUI endpoints bill continuously. Specify a spending limit:\n'));
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import {
|
|
2
|
+
KNOWN_PROVIDERS,
|
|
3
|
+
getCredential,
|
|
4
|
+
setCredential,
|
|
5
|
+
removeCredential,
|
|
6
|
+
listCredentials,
|
|
7
|
+
CREDENTIALS_FILE,
|
|
8
|
+
} from '../credentials.js';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* badgr connect <provider> [--key <value>] [--remove]
|
|
12
|
+
* badgr connect (lists what's already connected)
|
|
13
|
+
*
|
|
14
|
+
* One-time credential storage so `badgr launch claude "..."` doesn't need
|
|
15
|
+
* an explicit --env ANTHROPIC_API_KEY=... on every invocation. Stored in
|
|
16
|
+
* ~/.badgr/credentials.json, chmod 600 — a local file, not the real OS
|
|
17
|
+
* keychain (see credentials.js for why).
|
|
18
|
+
*/
|
|
19
|
+
export async function connectCommand(args, chalk, deps = {}) {
|
|
20
|
+
const provider = args[0];
|
|
21
|
+
|
|
22
|
+
if (!provider || provider === 'list' || provider === 'ls') {
|
|
23
|
+
const connected = listCredentials();
|
|
24
|
+
if (connected.length === 0) {
|
|
25
|
+
console.log(chalk.dim('\n No providers connected yet.'));
|
|
26
|
+
console.log(chalk.dim(` Run: badgr connect anthropic\n`));
|
|
27
|
+
} else {
|
|
28
|
+
console.log(chalk.bold('\nConnected providers:'));
|
|
29
|
+
for (const p of connected) console.log(` ✓ ${p}`);
|
|
30
|
+
console.log();
|
|
31
|
+
}
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (!KNOWN_PROVIDERS.includes(provider)) {
|
|
36
|
+
console.error(chalk.red(`\n ✗ Unknown provider: ${provider}`));
|
|
37
|
+
console.error(chalk.dim(` Supported: ${KNOWN_PROVIDERS.join(', ')}\n`));
|
|
38
|
+
process.exitCode = 1;
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const flags = {};
|
|
43
|
+
for (let i = 1; i < args.length; i++) {
|
|
44
|
+
if (args[i] === '--key') { flags.key = args[++i]; continue; }
|
|
45
|
+
if (args[i] === '--remove') { flags.remove = true; continue; }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (flags.remove) {
|
|
49
|
+
const removed = removeCredential(provider);
|
|
50
|
+
console.log(removed
|
|
51
|
+
? chalk.green(`\n ✓ Removed stored credential for ${provider}\n`)
|
|
52
|
+
: chalk.dim(`\n No credential stored for ${provider}\n`));
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let key = flags.key;
|
|
57
|
+
if (!key) {
|
|
58
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
59
|
+
console.error(chalk.red(`\n ✗ --key <value> is required in non-interactive mode.\n`));
|
|
60
|
+
console.error(chalk.dim(` Example: badgr connect ${provider} --key sk-...\n`));
|
|
61
|
+
process.exitCode = 1;
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const { password } = deps.prompts || await import('@inquirer/prompts');
|
|
65
|
+
key = await password({
|
|
66
|
+
message: `Enter your ${provider} API key:`,
|
|
67
|
+
validate: v => v.trim() ? true : 'API key is required',
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
key = key.trim();
|
|
72
|
+
if (!key) {
|
|
73
|
+
console.error(chalk.red('\n ✗ API key is required.\n'));
|
|
74
|
+
process.exitCode = 1;
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const alreadyConnected = Boolean(getCredential(provider));
|
|
79
|
+
setCredential(provider, key);
|
|
80
|
+
console.log(chalk.green(`\n ✓ ${alreadyConnected ? 'Updated' : 'Connected'} ${provider}`));
|
|
81
|
+
console.log(chalk.dim(` Stored in ${CREDENTIALS_FILE} (owner-only file permissions)`));
|
|
82
|
+
console.log(chalk.dim(` Used automatically by: badgr launch ${provider === 'anthropic' ? 'claude' : provider === 'openai' ? 'codex' : provider} "<task>"\n`));
|
|
83
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { runGpuDoctor } from '../gpuDoctor/doctor.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* badgr doctor [--model <id>] [--serve] [--workflow <path>] [--logs <path>]
|
|
5
|
+
* [--url <url>] [--gpu <name>] [--vram-gb <n>] [--json]
|
|
6
|
+
*
|
|
7
|
+
* Read-only diagnosis. Never provisions, restarts, or mutates anything.
|
|
8
|
+
*/
|
|
9
|
+
export function parseDoctorArgs(args) {
|
|
10
|
+
const flags = {};
|
|
11
|
+
let i = 0;
|
|
12
|
+
while (i < args.length) {
|
|
13
|
+
const a = args[i];
|
|
14
|
+
if (a === '--model') { flags.model = args[++i]; i++; continue; }
|
|
15
|
+
if (a === '--serve') { flags.serve = true; i++; continue; }
|
|
16
|
+
if (a === '--workflow') { flags.workflowPath = args[++i]; i++; continue; }
|
|
17
|
+
if (a === '--logs') { flags.logsPath = args[++i]; i++; continue; }
|
|
18
|
+
if (a === '--url') { flags.url = args[++i]; i++; continue; }
|
|
19
|
+
if (a === '--gpu') { flags.gpu = args[++i]; i++; continue; }
|
|
20
|
+
if (a === '--gpu-count') { flags.gpuCount = parseInt(args[++i], 10); i++; continue; }
|
|
21
|
+
if (a === '--vram-gb') { flags.vramGb = parseFloat(args[++i]); i++; continue; }
|
|
22
|
+
if (a === '--context-len') { flags.contextLen = parseInt(args[++i], 10); i++; continue; }
|
|
23
|
+
if (a === '--json') { flags.json = true; i++; continue; }
|
|
24
|
+
if (a === '--no-cache') { flags.noCache = true; i++; continue; }
|
|
25
|
+
i++;
|
|
26
|
+
}
|
|
27
|
+
return flags;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function printText(report, chalk) {
|
|
31
|
+
console.log();
|
|
32
|
+
console.log(`${chalk.bold('Verdict:')} ${report.verdict}`);
|
|
33
|
+
console.log();
|
|
34
|
+
console.log(chalk.bold('Likely cause:'));
|
|
35
|
+
console.log(report.likelyCause);
|
|
36
|
+
|
|
37
|
+
if (report.evidence.length) {
|
|
38
|
+
console.log();
|
|
39
|
+
console.log(chalk.bold('Evidence:'));
|
|
40
|
+
for (const line of report.evidence) console.log(`- ${line}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (report.suggestedFixes.length) {
|
|
44
|
+
console.log();
|
|
45
|
+
console.log(chalk.bold('Suggested fix:'));
|
|
46
|
+
for (const fix of report.suggestedFixes) console.log(`- ${fix}`);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (report.badgrCommand) {
|
|
50
|
+
console.log();
|
|
51
|
+
console.log(chalk.bold('Badgr route:'));
|
|
52
|
+
console.log(chalk.cyan(report.badgrCommand));
|
|
53
|
+
}
|
|
54
|
+
console.log();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function printJson(report) {
|
|
58
|
+
console.log(JSON.stringify({
|
|
59
|
+
verdict: report.verdictSlug,
|
|
60
|
+
verdict_label: report.verdict,
|
|
61
|
+
category: report.category,
|
|
62
|
+
likely_cause: report.likelyCause,
|
|
63
|
+
evidence: report.evidence,
|
|
64
|
+
suggested_fixes: report.suggestedFixes,
|
|
65
|
+
badgr_command: report.badgrCommand,
|
|
66
|
+
}, null, 2));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const DOCTOR_HELP = `
|
|
70
|
+
Diagnose why a GPU workload is likely failing — read-only, no login needed.
|
|
71
|
+
|
|
72
|
+
Usage:
|
|
73
|
+
badgr doctor
|
|
74
|
+
badgr doctor --model <id> [--serve]
|
|
75
|
+
badgr doctor --logs <path>
|
|
76
|
+
badgr doctor --workflow <path>
|
|
77
|
+
badgr doctor --url <url>
|
|
78
|
+
badgr doctor --json
|
|
79
|
+
|
|
80
|
+
Flags:
|
|
81
|
+
--model <id> Model to evaluate against available VRAM, e.g. Qwen/Qwen2.5-7B-Instruct
|
|
82
|
+
--serve Mark the model check as a serving workload (only with --model)
|
|
83
|
+
--workflow <path> Path to a ComfyUI workflow JSON to diagnose
|
|
84
|
+
--logs <path> Path to a vLLM/GPU workload log file to classify
|
|
85
|
+
--url <url> Endpoint URL to health-check, e.g. http://localhost:8000/v1/models
|
|
86
|
+
--gpu <name> Named GPU to size against, e.g. RTX4090, A100, L40S
|
|
87
|
+
--gpu-count <n> Number of GPUs for a tensor-parallel estimate (default: 1)
|
|
88
|
+
--vram-gb <n> Override available VRAM in GB directly
|
|
89
|
+
--context-len <n> Context length used in the model VRAM estimate (default: 8192)
|
|
90
|
+
--json Machine-readable JSON output
|
|
91
|
+
--no-cache Force fresh nvidia-smi/torch probes instead of a cached result (<=10s old)
|
|
92
|
+
|
|
93
|
+
Priority when multiple sources are given: --logs > --url > --workflow > --model.
|
|
94
|
+
Never mutates the machine: no restarts, no driver/CUDA changes, no installs, no uploads.
|
|
95
|
+
|
|
96
|
+
Full reference: docs/gpu-doctor.md`;
|
|
97
|
+
|
|
98
|
+
export async function doctorCommand(config, args, chalk) {
|
|
99
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
100
|
+
console.log(DOCTOR_HELP);
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const flags = parseDoctorArgs(args);
|
|
105
|
+
|
|
106
|
+
if (flags.serve && !flags.model) {
|
|
107
|
+
console.log(chalk.yellow('\n Note: --serve only applies together with --model; ignoring --serve.\n'));
|
|
108
|
+
flags.serve = false;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let report;
|
|
112
|
+
try {
|
|
113
|
+
report = await runGpuDoctor(flags, { noCache: flags.noCache });
|
|
114
|
+
} catch (err) {
|
|
115
|
+
console.error(chalk.red(`\n ✗ ${err.message}\n`));
|
|
116
|
+
process.exitCode = 1;
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (flags.json) {
|
|
121
|
+
printJson(report);
|
|
122
|
+
} else {
|
|
123
|
+
printText(report, chalk);
|
|
124
|
+
}
|
|
125
|
+
// Diagnosis outcomes (including "likely fail") are never tool errors —
|
|
126
|
+
// exit 0 unless we hit an actual runtime problem above.
|
|
127
|
+
}
|