badgr-cli 1.0.48 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/package.json +1 -1
- package/src/api.js +16 -2
- package/src/artifactDownload.js +55 -0
- package/src/badgr.js +104 -0
- package/src/batch.js +22 -4
- package/src/browser.js +23 -0
- package/src/commands/artifacts.js +75 -0
- package/src/commands/batch.js +221 -28
- package/src/commands/billing.js +1 -12
- package/src/commands/capacity.js +9 -4
- package/src/commands/comfyui.js +3 -3
- package/src/commands/connect.js +83 -0
- package/src/commands/doctor.js +127 -0
- package/src/commands/down.js +29 -6
- package/src/commands/launch.js +431 -0
- package/src/commands/pull.js +137 -0
- package/src/commands/run.js +253 -37
- package/src/commands/sbatch.js +232 -0
- package/src/commands/serve.js +3 -3
- package/src/commands/status.js +12 -4
- package/src/commands/task.js +25 -0
- package/src/commands/test-run.js +4 -2
- package/src/credentials.js +65 -0
- package/src/fallback.js +7 -2
- package/src/fanout.js +70 -0
- package/src/gpuDoctor/diskInfo.js +42 -0
- package/src/gpuDoctor/doctor.js +451 -0
- package/src/gpuDoctor/gpuInfo.js +70 -0
- package/src/gpuDoctor/healthCheck.js +63 -0
- package/src/gpuDoctor/logClassifier.js +138 -0
- package/src/gpuDoctor/modelFit.js +107 -0
- package/src/gpuDoctor/probeCache.js +38 -0
- package/src/gpuDoctor/redact.js +29 -0
- package/src/gpuDoctor/torchInfo.js +61 -0
- package/src/gpuDoctor/workflowDoctor.js +96 -0
- package/src/onboarding.js +124 -0
- package/src/slurm.js +193 -0
- package/src/spec.js +59 -2
- package/src/store.js +16 -0
- package/tests/agent-images.test.js +17 -0
- package/tests/artifactDownload.test.js +113 -0
- package/tests/artifacts.test.js +168 -0
- package/tests/batch.test.js +312 -0
- package/tests/browser.test.js +51 -0
- package/tests/capacity.test.js +68 -0
- package/tests/commands.test.js +44 -0
- package/tests/connect.test.js +83 -0
- package/tests/down.test.js +23 -1
- package/tests/fallback-timeout.test.js +41 -0
- package/tests/fanout.test.js +124 -0
- package/tests/gpu-doctor-classifiers.test.js +402 -0
- package/tests/gpu-doctor-doctor.test.js +304 -0
- package/tests/gpu-doctor-probe-cache.test.js +110 -0
- package/tests/gpu-doctor-probes.test.js +257 -0
- package/tests/launch-command-argv.test.js +93 -0
- package/tests/launch-readiness.test.js +1 -0
- package/tests/launch.test.js +440 -0
- package/tests/onboarding.test.js +134 -0
- package/tests/pull.test.js +266 -0
- package/tests/run-lifecycle.test.js +405 -6
- package/tests/sbatch.test.js +190 -0
- package/tests/secrets.test.js +16 -0
- package/tests/slurm.test.js +77 -0
- package/tests/spec.test.js +59 -1
- package/tests/status.test.js +73 -0
- package/tests/task.test.js +109 -0
- package/tests/template.test.js +7 -0
package/src/commands/run.js
CHANGED
|
@@ -3,13 +3,21 @@ import path from 'path';
|
|
|
3
3
|
import os from 'os';
|
|
4
4
|
import { createWriteStream } from 'fs';
|
|
5
5
|
import { requireApiKey } from '../config.js';
|
|
6
|
-
import { callApi, terminateDeployment, uploadBlob } from '../api.js';
|
|
7
|
-
import { addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
6
|
+
import { callApi, terminateDeployment, uploadBlob, quoteRun } from '../api.js';
|
|
7
|
+
import { addReceipt, updateReceipt, generateReceiptId, selectedComputeFromDeployment } from '../store.js';
|
|
8
8
|
import { normalizeTier, callWithFallback, HIGH_RATE_THRESHOLD } from '../fallback.js';
|
|
9
9
|
import { formatCliError } from '../errors.js';
|
|
10
10
|
import { TEMPLATE_MAP, buildTemplateFlags, parseTemplateOverrides } from '../catalog.js';
|
|
11
11
|
import { stage as _stage, stageDone as _stageDone, writeBlock as _writeBlock, clearBlock as _clearBlock, renderLiveBlock as _renderLiveBlock, printFailureClass as _printFailureClass } from '../progress.js';
|
|
12
12
|
import { detectWorkload, workloadTypeLabel } from '../detect.js';
|
|
13
|
+
import { ensureBadgrReady } from '../onboarding.js';
|
|
14
|
+
import { VM_CLASSES, parseGbSize } from '../spec.js';
|
|
15
|
+
|
|
16
|
+
function vmClassLine(sizeKey) {
|
|
17
|
+
const vmClass = VM_CLASSES[sizeKey];
|
|
18
|
+
if (!vmClass) return sizeKey;
|
|
19
|
+
return `${sizeKey} (${vmClass.vcpu} vCPU, ${vmClass.ramGb} GB RAM)`;
|
|
20
|
+
}
|
|
13
21
|
|
|
14
22
|
/**
|
|
15
23
|
* Flow 1 — local project (primary):
|
|
@@ -25,7 +33,12 @@ import { detectWorkload, workloadTypeLabel } from '../detect.js';
|
|
|
25
33
|
* Legacy / direct command:
|
|
26
34
|
* badgr run python train.py --gpu A100
|
|
27
35
|
* badgr run --image my/image:latest --gpu L40S --detach
|
|
36
|
+
*
|
|
37
|
+
* CPU launches (see commands/launch.js) reuse this same flow via
|
|
38
|
+
* runCommand(config, args, chalk, { isLaunch: true }) — a source plus a
|
|
39
|
+
* `--cmd`/`-- <command>` command, provisioned on a CPU VM instead of a GPU.
|
|
28
40
|
*/
|
|
41
|
+
|
|
29
42
|
export function parseRunArgs(args) {
|
|
30
43
|
const flags = {};
|
|
31
44
|
const positional = [];
|
|
@@ -46,6 +59,7 @@ export function parseRunArgs(args) {
|
|
|
46
59
|
if (flagArgs[i] === '--max-price') { flags.maxPrice = parseFloat(flagArgs[++i]); i++; continue; }
|
|
47
60
|
if (flagArgs[i] === '--name') { flags.name = flagArgs[++i]; i++; continue; }
|
|
48
61
|
if (flagArgs[i] === '--detach') { flags.detach = true; i++; continue; }
|
|
62
|
+
if (flagArgs[i] === '--no-detach') { flags.noDetach = true; i++; continue; }
|
|
49
63
|
if (flagArgs[i] === '--fallback') { flags.fallback = flagArgs[++i]; i++; continue; }
|
|
50
64
|
if (flagArgs[i] === '--no-fallback') { flags.noFallback = true; i++; continue; }
|
|
51
65
|
if (flagArgs[i] === '--strict-capacity') { flags.noFallback = true; i++; continue; }
|
|
@@ -53,6 +67,10 @@ export function parseRunArgs(args) {
|
|
|
53
67
|
if (flagArgs[i] === '--max-runtime') { flags.maxRuntime = parseFloat(flagArgs[++i]); i++; continue; }
|
|
54
68
|
if (flagArgs[i] === '--max-cost') { flags.maxCost = parseFloat(flagArgs[++i]); i++; continue; }
|
|
55
69
|
if (flagArgs[i] === '--min-vram') { flags.minVram = parseFloat(flagArgs[++i]); i++; continue; }
|
|
70
|
+
if (flagArgs[i] === '--gpu-memory') { flags.gpuMemoryRaw = flagArgs[++i]; flags.minVram = parseGbSize(flags.gpuMemoryRaw); i++; continue; }
|
|
71
|
+
if (flagArgs[i] === '--cpu') { flags.cpuRaw = flagArgs[++i]; flags.cpu = parseInt(flags.cpuRaw, 10); i++; continue; }
|
|
72
|
+
if (flagArgs[i] === '--memory') { flags.memoryRaw = flagArgs[++i]; flags.memory = parseGbSize(flags.memoryRaw); i++; continue; }
|
|
73
|
+
if (flagArgs[i] === '--no-gpu') { flags.noGpu = true; i++; continue; }
|
|
56
74
|
if (flagArgs[i] === '--dry-run') { flags.dryRun = true; i++; continue; }
|
|
57
75
|
if (flagArgs[i] === '--save') { flags.save = flagArgs[++i]; i++; continue; }
|
|
58
76
|
if (flagArgs[i] === '--workspace') { flags.workspace = flagArgs[++i]; i++; continue; }
|
|
@@ -61,12 +79,20 @@ export function parseRunArgs(args) {
|
|
|
61
79
|
if (flagArgs[i] === '--checkpoint') { flags.checkpoint = flagArgs[++i]; i++; continue; }
|
|
62
80
|
if (flagArgs[i] === '--retry-safe') { flags.retrySafe = true; i++; continue; }
|
|
63
81
|
if (flagArgs[i] === '--resume-cmd') { flags.resumeCmd = flagArgs[++i]; i++; continue; }
|
|
82
|
+
if (flagArgs[i] === '--agent-name') { flags.agentName = flagArgs[++i]; i++; continue; }
|
|
83
|
+
if (flagArgs[i] === '--size') { flags.size = flagArgs[++i]; i++; continue; }
|
|
64
84
|
if (flagArgs[i] === '--env') {
|
|
65
85
|
const kv = flagArgs[++i]; i++;
|
|
66
86
|
if (!flags.env) flags.env = [];
|
|
67
87
|
flags.env.push(kv);
|
|
68
88
|
continue;
|
|
69
89
|
}
|
|
90
|
+
if (flagArgs[i] === '--artifacts') {
|
|
91
|
+
const p = flagArgs[++i]; i++;
|
|
92
|
+
if (!flags.artifacts) flags.artifacts = [];
|
|
93
|
+
flags.artifacts.push(p);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
70
96
|
positional.push(flagArgs[i++]);
|
|
71
97
|
}
|
|
72
98
|
return { flags, positional, commandArgv };
|
|
@@ -81,6 +107,33 @@ function parseEnvFlag(envList) {
|
|
|
81
107
|
return obj;
|
|
82
108
|
}
|
|
83
109
|
|
|
110
|
+
// Heuristic for --env keys that look like secrets — used to warn (not
|
|
111
|
+
// block) since there is no dashboard --profile injection path yet and
|
|
112
|
+
// --env is currently the only way to get a provider key into a launch VM.
|
|
113
|
+
const _SECRET_ENV_KEY_RE = /(API_KEY|SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CREDENTIAL)/i;
|
|
114
|
+
|
|
115
|
+
function warnAboutSecretEnvFlags(envList, chalk) {
|
|
116
|
+
const secretKeys = (envList || [])
|
|
117
|
+
.map(kv => kv.slice(0, kv.indexOf('=')))
|
|
118
|
+
.filter(key => key && _SECRET_ENV_KEY_RE.test(key));
|
|
119
|
+
if (secretKeys.length === 0) return;
|
|
120
|
+
console.log(chalk.yellow(`\n ⚠ --env ${secretKeys.join(', ')} looks like a secret passed on the command line.`));
|
|
121
|
+
console.log(chalk.dim(' It may be saved in your shell history and is visible to anyone who can run `ps`/`history` on this machine.'));
|
|
122
|
+
console.log(chalk.dim(' Dashboard-managed secret profiles (--profile <name>) are planned but not built yet — see docs/badgr-cloud-phase1-2-checklist.md.'));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Never print a credential value to the terminal — dry-run banner, live
|
|
126
|
+
// banner, and anywhere else --env gets echoed back. Applies to both
|
|
127
|
+
// user-typed --env and any credential badgr connect injected automatically.
|
|
128
|
+
function redactEnvForDisplay(envList) {
|
|
129
|
+
return (envList || []).map(kv => {
|
|
130
|
+
const idx = kv.indexOf('=');
|
|
131
|
+
if (idx <= 0) return kv;
|
|
132
|
+
const key = kv.slice(0, idx);
|
|
133
|
+
return _SECRET_ENV_KEY_RE.test(key) ? `${key}=<redacted>` : kv;
|
|
134
|
+
}).join(', ');
|
|
135
|
+
}
|
|
136
|
+
|
|
84
137
|
// Mirror of backend workload_profile.py — kept in sync for pre-flight display.
|
|
85
138
|
const _PROFILES = {
|
|
86
139
|
smoke_test: { label: 'smoke test', vram: '4 GB', gpus: ['RTX 3080', 'RTX 3090', 'RTX 4090'] },
|
|
@@ -346,6 +399,10 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
346
399
|
failureType: classifyFailure(status, exitCode),
|
|
347
400
|
failureClass: dep.failure_class ?? null,
|
|
348
401
|
nextAction: dep.next_action ?? null,
|
|
402
|
+
// Reflects whatever the backend's own teardown attempt (the job
|
|
403
|
+
// runner's /complete webhook, which fires on any exit code) has
|
|
404
|
+
// already confirmed — not a guess made by this CLI.
|
|
405
|
+
teardownOk: dep.teardown_ok === 'ok',
|
|
349
406
|
};
|
|
350
407
|
}
|
|
351
408
|
}
|
|
@@ -361,10 +418,11 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
361
418
|
// Known badgr run flags — used to detect broken shell line continuation.
|
|
362
419
|
const _KNOWN_RUN_FLAGS = new Set([
|
|
363
420
|
'--gpu', '--image', '--count', '--region', '--tier', '--max-price', '--name',
|
|
364
|
-
'--detach', '--fallback', '--no-fallback', '--strict-capacity',
|
|
365
|
-
'--no-expanded-search', '--max-runtime', '--max-cost', '--min-vram', '--
|
|
421
|
+
'--detach', '--no-detach', '--fallback', '--no-fallback', '--strict-capacity',
|
|
422
|
+
'--no-expanded-search', '--max-runtime', '--max-cost', '--min-vram', '--gpu-memory',
|
|
423
|
+
'--cpu', '--memory', '--no-gpu', '--env',
|
|
366
424
|
'--dry-run', '--cmd', '--save', '--workspace',
|
|
367
|
-
'--output', '--checkpoint', '--retry-safe', '--resume-cmd',
|
|
425
|
+
'--output', '--checkpoint', '--retry-safe', '--resume-cmd', '--artifacts',
|
|
368
426
|
]);
|
|
369
427
|
|
|
370
428
|
// Directories and files always excluded from project zip uploads.
|
|
@@ -378,14 +436,14 @@ function _shouldExclude(name) {
|
|
|
378
436
|
return _ZIP_EXCLUDES.has(name) || name.startsWith('.') && name !== '.gitignore';
|
|
379
437
|
}
|
|
380
438
|
|
|
381
|
-
function _isLocalPath(arg) {
|
|
439
|
+
export function _isLocalPath(arg) {
|
|
382
440
|
// '.' or './' or relative/absolute paths that exist on disk
|
|
383
441
|
if (arg === '.') return true;
|
|
384
442
|
if (arg.startsWith('./') || arg.startsWith('../') || arg.startsWith('/')) return true;
|
|
385
443
|
return false;
|
|
386
444
|
}
|
|
387
445
|
|
|
388
|
-
function _isGitHubUrl(arg) {
|
|
446
|
+
export function _isGitHubUrl(arg) {
|
|
389
447
|
return /^https?:\/\/(www\.)?github\.com\//.test(arg);
|
|
390
448
|
}
|
|
391
449
|
|
|
@@ -440,9 +498,20 @@ export async function _uploadCodeZip(config, dirPath, chalk) {
|
|
|
440
498
|
return uploadResp.code_uri;
|
|
441
499
|
}
|
|
442
500
|
|
|
443
|
-
export async function runCommand(config, args, chalk) {
|
|
501
|
+
export async function runCommand(config, args, chalk, opts = {}) {
|
|
502
|
+
const isLaunch = Boolean(opts.isLaunch);
|
|
503
|
+
const maxCostIsDefault = Boolean(opts.maxCostIsDefault);
|
|
504
|
+
// Real argv array for a code-source job's command — set only by internal
|
|
505
|
+
// callers (badgr launch's agent workloads), never parsed from a shell
|
|
506
|
+
// string. Passed to the backend as command_argv, which the worker always
|
|
507
|
+
// prefers over the legacy cmd string + shlex.split() path (see
|
|
508
|
+
// images/badgr-job-runner/entrypoint.py). flags.cmd stays the
|
|
509
|
+
// human-readable display string only.
|
|
510
|
+
const cmdArgv = Array.isArray(opts.cmdArgv) ? opts.cmdArgv : undefined;
|
|
511
|
+
const cmdName = isLaunch ? 'badgr launch' : 'badgr run';
|
|
512
|
+
|
|
444
513
|
// `badgr run template <name> [flags]` — expand template defaults then re-dispatch
|
|
445
|
-
if (args[0] === 'template') {
|
|
514
|
+
if (!isLaunch && args[0] === 'template') {
|
|
446
515
|
const name = args[1];
|
|
447
516
|
const t = name && TEMPLATE_MAP[name];
|
|
448
517
|
if (!t) {
|
|
@@ -475,7 +544,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
475
544
|
console.error(chalk.dim(' This usually means a line continuation has a trailing space.'));
|
|
476
545
|
console.error(chalk.dim(' Use a single line, or end each continued line with \\ and no space after:'));
|
|
477
546
|
console.error(chalk.dim(''));
|
|
478
|
-
console.error(chalk.dim(
|
|
547
|
+
console.error(chalk.dim(` ${cmdName} python script.py \\`));
|
|
479
548
|
console.error(chalk.dim(' --gpu RTX_4090 --max-cost 10 --max-runtime 60'));
|
|
480
549
|
console.error(chalk.dim(''));
|
|
481
550
|
process.exitCode = 1;
|
|
@@ -489,10 +558,13 @@ export async function runCommand(config, args, chalk) {
|
|
|
489
558
|
const isGitHubUrl = firstArg && _isGitHubUrl(firstArg);
|
|
490
559
|
const isCodeSource = isLocalPath || isGitHubUrl;
|
|
491
560
|
|
|
561
|
+
if (isLaunch) flags.detach = flags.noDetach ? false : (flags.detach ?? true);
|
|
562
|
+
|
|
492
563
|
// Local paths can be inspected on disk, so a missing --cmd doesn't have to be
|
|
493
564
|
// an error — try to infer it (and output/checkpoint conventions) the same way
|
|
494
565
|
// `badgr detect .` would, before ever provisioning anything.
|
|
495
566
|
let detectionReport = null;
|
|
567
|
+
|
|
496
568
|
if (isLocalPath && !flags.cmd) {
|
|
497
569
|
detectionReport = detectWorkload(firstArg);
|
|
498
570
|
if (detectionReport.command && detectionReport.confidence !== 'low') {
|
|
@@ -522,19 +594,27 @@ export async function runCommand(config, args, chalk) {
|
|
|
522
594
|
}
|
|
523
595
|
|
|
524
596
|
if (isCodeSource && !flags.cmd) {
|
|
525
|
-
|
|
597
|
+
const example = isLaunch ? '-- claude -p "Fix the checkout bug"' : '--cmd "python train.py" --max-cost 5';
|
|
598
|
+
console.error(chalk.red(`\n ✗ ${isLaunch ? 'A command (`-- <command>` or --cmd) is' : '--cmd is'} required when running from a ${isLocalPath ? 'local path' : 'GitHub URL'}.\n`));
|
|
526
599
|
if (isLocalPath) {
|
|
527
|
-
console.error(chalk.dim(
|
|
528
|
-
console.error(chalk.dim(' Or check what Badgr detects first: badgr detect .\n'));
|
|
600
|
+
console.error(chalk.dim(` Example: ${cmdName} . ${example}\n`));
|
|
601
|
+
if (!isLaunch) console.error(chalk.dim(' Or check what Badgr detects first: badgr detect .\n'));
|
|
529
602
|
} else {
|
|
530
|
-
console.error(chalk.dim(
|
|
603
|
+
console.error(chalk.dim(` Example: ${cmdName} https://github.com/user/repo ${example}\n`));
|
|
531
604
|
}
|
|
532
605
|
process.exitCode = 1;
|
|
533
606
|
return;
|
|
534
607
|
}
|
|
535
608
|
|
|
609
|
+
if (isLaunch && !isCodeSource) {
|
|
610
|
+
console.error(chalk.red(`\n ✗ ${cmdName} requires a source: a local path ('.') or a GitHub URL.\n`));
|
|
611
|
+
console.error(chalk.dim(' Example: badgr launch . --max-cost 1 -- claude -p "Fix the checkout bug"\n'));
|
|
612
|
+
process.exitCode = 1;
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
|
|
536
616
|
// Legacy direct-command mode: positional args form the command
|
|
537
|
-
const isDirectCommand = !isCodeSource && (positional.length > 0 || commandArgv !== null || flags.image);
|
|
617
|
+
const isDirectCommand = !isLaunch && !isCodeSource && (positional.length > 0 || commandArgv !== null || flags.image);
|
|
538
618
|
|
|
539
619
|
if (!isCodeSource && positional.length === 0 && commandArgv === null && !flags.image) {
|
|
540
620
|
console.error(chalk.red('\nUsage:'));
|
|
@@ -551,7 +631,17 @@ export async function runCommand(config, args, chalk) {
|
|
|
551
631
|
process.exitCode = 1;
|
|
552
632
|
return;
|
|
553
633
|
}
|
|
554
|
-
|
|
634
|
+
// Just-in-time login + funding — the first meaningful action should be
|
|
635
|
+
// `badgr run`/`badgr launch`, not a separate `badgr login` step. A dry run
|
|
636
|
+
// previews the plan only and never provisions or spends anything, so it
|
|
637
|
+
// still needs a stored key (to keep its existing fail-fast behavior in
|
|
638
|
+
// non-interactive contexts like tests/CI) but doesn't trigger the
|
|
639
|
+
// interactive browser flow.
|
|
640
|
+
if (flags.dryRun) {
|
|
641
|
+
requireApiKey(config);
|
|
642
|
+
} else {
|
|
643
|
+
config = await ensureBadgrReady(config, chalk);
|
|
644
|
+
}
|
|
555
645
|
|
|
556
646
|
// ── Validate flags early ───────────────────────────────────────────────────
|
|
557
647
|
if (flags.count !== undefined && (!Number.isFinite(flags.count) || flags.count < 1)) {
|
|
@@ -574,6 +664,26 @@ export async function runCommand(config, args, chalk) {
|
|
|
574
664
|
process.exitCode = 1;
|
|
575
665
|
return;
|
|
576
666
|
}
|
|
667
|
+
if (flags.cpuRaw !== undefined && (!Number.isFinite(flags.cpu) || flags.cpu < 1 || !Number.isInteger(flags.cpu))) {
|
|
668
|
+
console.error(chalk.red(` ✗ --cpu must be a positive whole number of cores, got: ${flags.cpuRaw}`));
|
|
669
|
+
process.exitCode = 1;
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
if (flags.memoryRaw !== undefined && (flags.memory == null || !Number.isFinite(flags.memory) || flags.memory <= 0)) {
|
|
673
|
+
console.error(chalk.red(` ✗ --memory could not be parsed as a size, got: ${flags.memoryRaw} (try "64GB" or "65536MB")`));
|
|
674
|
+
process.exitCode = 1;
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
if (flags.gpuMemoryRaw !== undefined && (flags.minVram == null || !Number.isFinite(flags.minVram) || flags.minVram <= 0)) {
|
|
678
|
+
console.error(chalk.red(` ✗ --gpu-memory could not be parsed as a size, got: ${flags.gpuMemoryRaw} (try "24GB" or "24576MB")`));
|
|
679
|
+
process.exitCode = 1;
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
682
|
+
if (flags.noGpu && (flags.gpu || flags.minVram)) {
|
|
683
|
+
console.error(chalk.red(' ✗ --no-gpu conflicts with --gpu/--gpu-memory/--min-vram — pick one'));
|
|
684
|
+
process.exitCode = 1;
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
577
687
|
|
|
578
688
|
if (!flags.maxCost && !flags.dryRun && isLocalPath && process.stdin.isTTY && process.stdout.isTTY) {
|
|
579
689
|
try {
|
|
@@ -627,50 +737,91 @@ export async function runCommand(config, args, chalk) {
|
|
|
627
737
|
if (flags.checkpoint) conventionEnv.BADGR_CHECKPOINT_DIR = flags.checkpoint;
|
|
628
738
|
if (flags.retrySafe) conventionEnv.BADGR_RETRY_SAFE = '1';
|
|
629
739
|
const envObj = { ...conventionEnv, ...parseEnvFlag(flags.env) };
|
|
740
|
+
if (isLaunch) warnAboutSecretEnvFlags(flags.env, chalk);
|
|
630
741
|
const effectiveTier = normalizeTier(flags.tier);
|
|
631
742
|
|
|
632
|
-
const gpu = flags.gpu ? flags.gpu.toUpperCase().replace('-', '_') : undefined;
|
|
743
|
+
const gpu = isLaunch ? 'CPU' : (flags.gpu ? flags.gpu.toUpperCase().replace('-', '_') : undefined);
|
|
744
|
+
|
|
745
|
+
// Detect VM → show price → launch VM: for CPU launches, ask the backend
|
|
746
|
+
// what this would cost *before* calling /run, so the customer sees the
|
|
747
|
+
// rate before anything is provisioned or billed — not just after. This is
|
|
748
|
+
// a best-effort preview (short timeout, never fatal): if the quote call
|
|
749
|
+
// fails for any reason, the launch still proceeds and the rate falls back
|
|
750
|
+
// to being shown once the deployment is actually created (see below).
|
|
751
|
+
let quotedRate = null;
|
|
752
|
+
if (isLaunch) {
|
|
753
|
+
try {
|
|
754
|
+
const quote = await quoteRun(config, {
|
|
755
|
+
compute: 'cpu',
|
|
756
|
+
...(flags.agentName ? { agent: flags.agentName } : {}),
|
|
757
|
+
...(flags.size ? { size: flags.size } : {}),
|
|
758
|
+
...(flags.region ? { region: flags.region.toUpperCase() } : {}),
|
|
759
|
+
});
|
|
760
|
+
if (quote && typeof quote.rate_per_hour === 'number') quotedRate = quote.rate_per_hour;
|
|
761
|
+
} catch {
|
|
762
|
+
// Non-fatal — the launch itself still shows the confirmed rate once
|
|
763
|
+
// the deployment is created.
|
|
764
|
+
}
|
|
765
|
+
}
|
|
633
766
|
|
|
634
767
|
if (flags.dryRun) {
|
|
635
|
-
console.log(chalk.bold(
|
|
768
|
+
console.log(chalk.bold(`\n⚡ Dry run — no ${isLaunch ? 'VM' : 'GPU'} will be provisioned\n`));
|
|
636
769
|
if (isLocalPath) console.log(` ${chalk.bold('Source:')} ${path.resolve(firstArg)} (local project)`);
|
|
637
770
|
if (isGitHubUrl) console.log(` ${chalk.bold('Source:')} ${firstArg} (GitHub)`);
|
|
771
|
+
if (flags.agentName) console.log(` ${chalk.bold('Workload:')} ${flags.agentName}`);
|
|
638
772
|
if (flags.cmd) console.log(` ${chalk.bold('Command:')} ${flags.cmd}`);
|
|
639
773
|
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
640
774
|
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
641
|
-
console.log(` ${chalk.bold('GPU:')}
|
|
775
|
+
console.log(` ${chalk.bold(isLaunch || flags.noGpu ? 'Compute:' : 'GPU:')} ${isLaunch || flags.noGpu ? 'CPU VM (no GPU)' : (gpu || chalk.dim('auto'))}`);
|
|
776
|
+
if (isLaunch && flags.size) console.log(` ${chalk.bold('VM class:')} ${vmClassLine(flags.size)}`);
|
|
777
|
+
if (isLaunch && quotedRate != null) console.log(` ${chalk.bold('Badgr rate:')} $${quotedRate.toFixed(2)}/hour`);
|
|
778
|
+
if (flags.minVram) console.log(` ${chalk.bold('Min VRAM:')} ${flags.minVram} GB`);
|
|
779
|
+
if (flags.cpu) console.log(` ${chalk.bold('CPU:')} ${flags.cpu} cores`);
|
|
780
|
+
if (flags.memory) console.log(` ${chalk.bold('Memory:')} ${flags.memory} GB`);
|
|
642
781
|
console.log(` ${chalk.bold('Max runtime:')} ${effectiveMaxRuntime}min`);
|
|
643
|
-
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost}`);
|
|
782
|
+
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost}${maxCostIsDefault ? chalk.dim(' (default — use --max-cost N to override)') : ''}`);
|
|
644
783
|
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice}/hr`);
|
|
645
|
-
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${flags.env
|
|
784
|
+
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${redactEnvForDisplay(flags.env)}`);
|
|
646
785
|
if (flags.output) console.log(` ${chalk.bold('Output:')} ${flags.output}`);
|
|
647
786
|
if (flags.checkpoint) console.log(` ${chalk.bold('Checkpoint:')} ${flags.checkpoint}`);
|
|
648
787
|
if (flags.retrySafe) console.log(` ${chalk.bold('Retry-safe:')} enabled`);
|
|
649
788
|
if (flags.resumeCmd) console.log(` ${chalk.bold('Resume cmd:')} ${flags.resumeCmd}`);
|
|
789
|
+
if (flags.artifacts?.length) console.log(` ${chalk.bold('Artifacts:')} ${flags.artifacts.join(', ')}`);
|
|
650
790
|
console.log(chalk.dim('\n Remove --dry-run to provision.\n'));
|
|
651
791
|
return;
|
|
652
792
|
}
|
|
653
793
|
|
|
654
|
-
console.log(chalk.bold(
|
|
794
|
+
console.log(chalk.bold(`\n⚡ ${isLaunch ? 'Launching command on CPU VM' : 'Running GPU job'}\n`));
|
|
655
795
|
if (isLocalPath) console.log(` ${chalk.bold('Source:')} ${path.resolve(firstArg)}`);
|
|
656
796
|
if (isGitHubUrl) console.log(` ${chalk.bold('Source:')} ${firstArg}`);
|
|
797
|
+
if (flags.agentName) console.log(` ${chalk.bold('Workload:')} ${flags.agentName}`);
|
|
657
798
|
if (flags.cmd) console.log(` ${chalk.bold('Command:')} ${flags.cmd}`);
|
|
658
799
|
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
659
800
|
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
660
|
-
if (
|
|
801
|
+
if (isLaunch) console.log(` ${chalk.bold('Compute:')} CPU VM`);
|
|
802
|
+
else if (flags.noGpu) console.log(` ${chalk.bold('Compute:')} CPU VM (no GPU)`);
|
|
803
|
+
else if (flags.gpu) console.log(` ${chalk.bold('GPU:')} ${gpu}`);
|
|
804
|
+
if (isLaunch && flags.size) console.log(` ${chalk.bold('VM class:')} ${vmClassLine(flags.size)}`);
|
|
805
|
+
if (isLaunch && quotedRate != null) console.log(` ${chalk.bold('Badgr rate:')} $${quotedRate.toFixed(2)}/hour`);
|
|
661
806
|
if (flags.minVram) console.log(` ${chalk.bold('Min VRAM:')} ${flags.minVram} GB`);
|
|
807
|
+
if (flags.cpu) console.log(` ${chalk.bold('CPU:')} ${flags.cpu} cores`);
|
|
808
|
+
if (flags.memory) console.log(` ${chalk.bold('Memory:')} ${flags.memory} GB`);
|
|
662
809
|
const runtimeLabel = isDefaultRuntime
|
|
663
810
|
? `${effectiveMaxRuntime} min ${chalk.dim('(default — use --max-runtime N to override)')}`
|
|
664
811
|
: `${effectiveMaxRuntime} min`;
|
|
665
|
-
|
|
812
|
+
const maxCostLabel = maxCost
|
|
813
|
+
? `$${maxCost.toFixed(2)}${maxCostIsDefault ? chalk.dim(' (default — use --max-cost N to override)') : ''}`
|
|
814
|
+
: chalk.dim('none');
|
|
815
|
+
console.log(` ${chalk.bold('Max cost:')} ${maxCostLabel}`);
|
|
666
816
|
console.log(` ${chalk.bold('Max runtime:')} ${runtimeLabel}`);
|
|
667
817
|
console.log(` ${chalk.bold('Auto-stop:')} ${maxCost ? 'enabled' : chalk.yellow('disabled — stop manually with badgr down')}`);
|
|
668
818
|
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice.toFixed(2)}/hr`);
|
|
669
819
|
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
670
|
-
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${flags.env
|
|
820
|
+
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${redactEnvForDisplay(flags.env)}`);
|
|
671
821
|
if (flags.output) console.log(` ${chalk.bold('Output:')} ${flags.output}`);
|
|
672
822
|
if (flags.checkpoint) console.log(` ${chalk.bold('Checkpoint:')} ${flags.checkpoint}`);
|
|
673
823
|
if (flags.retrySafe) console.log(` ${chalk.bold('Retry-safe:')} enabled`);
|
|
824
|
+
if (flags.artifacts?.length) console.log(` ${chalk.bold('Artifacts:')} ${flags.artifacts.join(', ')}`);
|
|
674
825
|
console.log();
|
|
675
826
|
|
|
676
827
|
|
|
@@ -719,7 +870,11 @@ export async function runCommand(config, args, chalk) {
|
|
|
719
870
|
...(command ? { command } : {}),
|
|
720
871
|
...(image ? { image } : {}),
|
|
721
872
|
gpu: gpuOverride || gpu || 'auto',
|
|
873
|
+
...(isLaunch ? { compute: 'cpu' } : {}),
|
|
874
|
+
...(flags.noGpu ? { no_gpu: true } : {}),
|
|
722
875
|
...(flags.minVram ? { min_vram: flags.minVram } : {}),
|
|
876
|
+
...(flags.cpu ? { cpu: flags.cpu } : {}),
|
|
877
|
+
...(flags.memory ? { memory_gb: flags.memory } : {}),
|
|
723
878
|
gpu_count: flags.count || 1,
|
|
724
879
|
...(effectiveRegion ? { region: effectiveRegion } : {}),
|
|
725
880
|
max_price_per_hour: flags.maxPrice,
|
|
@@ -733,6 +888,10 @@ export async function runCommand(config, args, chalk) {
|
|
|
733
888
|
...(codeUri ? { code_uri: codeUri } : {}),
|
|
734
889
|
...(isGitHubUrl ? { github_url: firstArg } : {}),
|
|
735
890
|
...(flags.cmd ? { cmd: flags.cmd } : {}),
|
|
891
|
+
...(cmdArgv ? { command_argv: cmdArgv } : {}),
|
|
892
|
+
...(flags.artifacts?.length ? { output_paths: flags.artifacts } : {}),
|
|
893
|
+
...(flags.agentName ? { agent: flags.agentName } : {}),
|
|
894
|
+
...(flags.size ? { size: flags.size } : {}),
|
|
736
895
|
};
|
|
737
896
|
}
|
|
738
897
|
|
|
@@ -744,13 +903,13 @@ export async function runCommand(config, args, chalk) {
|
|
|
744
903
|
(tierOverride) => buildBody(undefined, tierOverride),
|
|
745
904
|
effectiveTier,
|
|
746
905
|
chalk,
|
|
747
|
-
{ thing: 'job', cmd:
|
|
906
|
+
{ thing: 'job', cmd: cmdName },
|
|
748
907
|
{ allowTier2Fallback: !flags.noFallback },
|
|
749
908
|
);
|
|
750
909
|
} catch (err) {
|
|
751
910
|
if (err.isPaymentRequired) {
|
|
752
911
|
console.error(chalk.yellow(err.message));
|
|
753
|
-
const rerun = [
|
|
912
|
+
const rerun = [cmdName, ...args].join(' ');
|
|
754
913
|
console.error(chalk.dim(`After payment, rerun:\n ${rerun}\n`));
|
|
755
914
|
process.exitCode = 1;
|
|
756
915
|
return;
|
|
@@ -761,10 +920,20 @@ export async function runCommand(config, args, chalk) {
|
|
|
761
920
|
return;
|
|
762
921
|
}
|
|
763
922
|
|
|
923
|
+
// Minimum product-learning data: what shape of workload this was, what
|
|
924
|
+
// was requested vs. what was actually provisioned. No analytics system,
|
|
925
|
+
// no dashboard — just enough on each receipt that repeat-usage and
|
|
926
|
+
// resource-fit questions are answerable later from stored receipts.
|
|
927
|
+
const workloadShape = isLaunch ? `agent:${flags.agentName || 'custom'}`
|
|
928
|
+
: isLocalPath ? 'project'
|
|
929
|
+
: isGitHubUrl ? 'git-repo'
|
|
930
|
+
: image ? 'container'
|
|
931
|
+
: 'command';
|
|
932
|
+
|
|
764
933
|
const rcptId = dep.receipt_id || generateReceiptId();
|
|
765
934
|
addReceipt({
|
|
766
935
|
receiptId: rcptId,
|
|
767
|
-
action:
|
|
936
|
+
action: cmdName,
|
|
768
937
|
deploymentId: dep.deployment_id,
|
|
769
938
|
gpu: dep.gpu_type,
|
|
770
939
|
providerRoute: dep.provider ?? null,
|
|
@@ -773,15 +942,47 @@ export async function runCommand(config, args, chalk) {
|
|
|
773
942
|
maxRuntime: flags.maxRuntime ?? null,
|
|
774
943
|
status: dep.status,
|
|
775
944
|
createdAt: new Date().toISOString(),
|
|
945
|
+
workloadShape,
|
|
946
|
+
computeRequested: {
|
|
947
|
+
gpu: flags.gpu ?? null,
|
|
948
|
+
gpuCount: flags.count ?? null,
|
|
949
|
+
minVram: flags.minVram ?? null,
|
|
950
|
+
cpu: flags.cpu ?? null,
|
|
951
|
+
memoryGb: flags.memory ?? null,
|
|
952
|
+
noGpu: flags.noGpu ?? false,
|
|
953
|
+
},
|
|
954
|
+
computeSelected: selectedComputeFromDeployment(dep),
|
|
776
955
|
});
|
|
777
956
|
|
|
778
957
|
const rate = dep.cost_per_hour || 0;
|
|
779
958
|
|
|
959
|
+
// Show what was actually provisioned, not just what was requested — only
|
|
960
|
+
// when the caller asked for a resource floor and the provider reported
|
|
961
|
+
// enough to say something concrete (best-effort; not every provider
|
|
962
|
+
// exposes vcpu/ram/vram per-offer, see deployment_service.py's
|
|
963
|
+
// chosen_instance.extra enrichment).
|
|
964
|
+
if (flags.cpu || flags.memory || flags.minVram) {
|
|
965
|
+
const parts = [];
|
|
966
|
+
if (dep.gpu_type && dep.gpu_type !== 'CPU') parts.push(`${dep.gpu_count || 1}× ${dep.gpu_type}`);
|
|
967
|
+
if (dep.selected_vram_gb != null) parts.push(`${dep.selected_vram_gb}GB VRAM`);
|
|
968
|
+
if (dep.selected_vcpus != null) parts.push(`${dep.selected_vcpus} vCPU`);
|
|
969
|
+
if (dep.selected_ram_gb != null) parts.push(`${dep.selected_ram_gb}GB RAM`);
|
|
970
|
+
if (parts.length) console.log(` ${chalk.bold('Provisioned:')} ${parts.join(', ')}`);
|
|
971
|
+
}
|
|
972
|
+
|
|
780
973
|
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Finding a working route...')));
|
|
781
974
|
stageN++;
|
|
782
975
|
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Starting runtime...')));
|
|
783
976
|
stageN++;
|
|
784
977
|
|
|
978
|
+
// The rate was already shown pre-provisioning via the quote call above,
|
|
979
|
+
// in the normal case — this is only a fallback for when that quote call
|
|
980
|
+
// failed (network hiccup, older backend without the endpoint, etc.), so
|
|
981
|
+
// the customer still sees the rate somewhere rather than never at all.
|
|
982
|
+
if (isLaunch && quotedRate == null && rate > 0) {
|
|
983
|
+
console.log(chalk.dim(`\n Badgr rate: $${rate.toFixed(2)}/hour`));
|
|
984
|
+
}
|
|
985
|
+
|
|
785
986
|
if (rate > HIGH_RATE_THRESHOLD && !maxCost) {
|
|
786
987
|
console.log(chalk.yellow(`\n Selected capacity rate: $${rate.toFixed(2)}/hr`));
|
|
787
988
|
console.log(chalk.dim(' Tip: use --max-cost to enforce a hard ceiling.'));
|
|
@@ -810,9 +1011,12 @@ export async function runCommand(config, args, chalk) {
|
|
|
810
1011
|
'interrupted': 'Stopped',
|
|
811
1012
|
};
|
|
812
1013
|
|
|
813
|
-
let teardownOk =
|
|
1014
|
+
let teardownOk = false;
|
|
814
1015
|
try {
|
|
815
|
-
|
|
1016
|
+
// A 200 response only means deletion was requested, not confirmed —
|
|
1017
|
+
// see the identical note on the success-path teardown below.
|
|
1018
|
+
const result = await terminateDeployment(config, dep.deployment_id);
|
|
1019
|
+
teardownOk = result?.teardown_ok === 'ok';
|
|
816
1020
|
} catch {
|
|
817
1021
|
// terminateDeployment retries 3×; best-effort if all fail
|
|
818
1022
|
teardownOk = false;
|
|
@@ -870,7 +1074,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
870
1074
|
console.log(chalk.dim(`\n [${stageN}/${STAGE_TOTAL}] Running command (Ctrl+C to stop)`));
|
|
871
1075
|
|
|
872
1076
|
attachStart = Date.now();
|
|
873
|
-
const { status: finalStatus, exitCode, runtimeMs, failureType, failureClass, nextAction } = await attachToJob(config, dep.deployment_id, {
|
|
1077
|
+
const { status: finalStatus, exitCode, runtimeMs, failureType, failureClass, nextAction, teardownOk: confirmedTeardownOk } = await attachToJob(config, dep.deployment_id, {
|
|
874
1078
|
chalk,
|
|
875
1079
|
maxRuntimeMs,
|
|
876
1080
|
maxCost,
|
|
@@ -899,7 +1103,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
899
1103
|
runtimeSeconds: Math.round(runtimeMs / 1000),
|
|
900
1104
|
finalCost,
|
|
901
1105
|
failureType,
|
|
902
|
-
teardownStatus:
|
|
1106
|
+
teardownStatus: confirmedTeardownOk ? 'terminated' : 'failed',
|
|
903
1107
|
});
|
|
904
1108
|
|
|
905
1109
|
console.log();
|
|
@@ -913,18 +1117,27 @@ export async function runCommand(config, args, chalk) {
|
|
|
913
1117
|
_printFailureClass(chalk, { failure_class: failureClass, next_action: nextAction });
|
|
914
1118
|
console.log(chalk.dim(_stage(STAGE_TOTAL, STAGE_TOTAL, 'Failed')));
|
|
915
1119
|
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
916
|
-
// The
|
|
917
|
-
//
|
|
918
|
-
|
|
1120
|
+
// The backend's own /complete webhook already attempted teardown (on any
|
|
1121
|
+
// exit code) by the time we observe this — no extra teardown call from
|
|
1122
|
+
// here is needed. But whether it was *confirmed* is a real result from
|
|
1123
|
+
// that attempt, not something to assume — see attachToJob's teardownOk.
|
|
1124
|
+
_printFinalInfo(chalk, { exitCode, teardownOk: confirmedTeardownOk, jobId: dep.deployment_id, rcptId });
|
|
919
1125
|
if (flags.resumeCmd) console.log(` ${chalk.bold('Resume:')} ${flags.resumeCmd}`);
|
|
920
1126
|
process.exitCode = exitCode ?? 1;
|
|
921
1127
|
return;
|
|
922
1128
|
}
|
|
923
1129
|
|
|
924
1130
|
if ((finalStatus === 'completed' || finalStatus === 'succeeded') && (exitCode === 0 || exitCode === null)) {
|
|
925
|
-
|
|
1131
|
+
// A 200 response only means the backend requested deletion, not that it
|
|
1132
|
+
// confirmed the underlying resource is gone — /v1/deployments/{id} never
|
|
1133
|
+
// raises just because teardown failed, it returns 200 with
|
|
1134
|
+
// teardown_ok: "failed" in the body. Checking only for a thrown
|
|
1135
|
+
// exception here would report "Teardown: succeeded" while the resource
|
|
1136
|
+
// may still exist and still be billing.
|
|
1137
|
+
let teardownOk = false;
|
|
926
1138
|
try {
|
|
927
|
-
await terminateDeployment(config, dep.deployment_id);
|
|
1139
|
+
const result = await terminateDeployment(config, dep.deployment_id);
|
|
1140
|
+
teardownOk = result?.teardown_ok === 'ok';
|
|
928
1141
|
} catch { teardownOk = false; /* already stopped, or best-effort */ }
|
|
929
1142
|
console.log(chalk.green(_stage(STAGE_TOTAL, STAGE_TOTAL, 'Complete')));
|
|
930
1143
|
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
@@ -943,11 +1156,14 @@ export async function runCommand(config, args, chalk) {
|
|
|
943
1156
|
...(command ? { command } : {}),
|
|
944
1157
|
...(image ? { image } : {}),
|
|
945
1158
|
...(flags.cmd ? { cmd: flags.cmd } : {}),
|
|
1159
|
+
...(cmdArgv ? { command_argv: cmdArgv } : {}),
|
|
946
1160
|
// For local-path workloads, code_uri is a snapshot — user can re-upload on next run.
|
|
947
1161
|
// GitHub URL is stable and stored directly.
|
|
948
1162
|
...(isGitHubUrl ? { github_url: firstArg } : {}),
|
|
949
1163
|
gpu: gpu || 'auto',
|
|
950
1164
|
...(flags.minVram ? { min_vram: flags.minVram } : {}),
|
|
1165
|
+
...(flags.cpu ? { cpu: flags.cpu } : {}),
|
|
1166
|
+
...(flags.memory ? { memory_gb: flags.memory } : {}),
|
|
951
1167
|
gpu_count: flags.count || 1,
|
|
952
1168
|
...(flags.maxPrice ? { max_price_per_hour: flags.maxPrice } : {}),
|
|
953
1169
|
...(Object.keys(envObj).length > 0 ? { env: envObj } : {}),
|