badgr-cli 1.0.48 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/package.json +1 -1
- package/src/api.js +16 -2
- package/src/artifactDownload.js +55 -0
- package/src/badgr.js +104 -0
- package/src/batch.js +22 -4
- package/src/browser.js +23 -0
- package/src/commands/artifacts.js +75 -0
- package/src/commands/batch.js +221 -28
- package/src/commands/billing.js +1 -12
- package/src/commands/capacity.js +9 -4
- package/src/commands/comfyui.js +3 -3
- package/src/commands/connect.js +83 -0
- package/src/commands/doctor.js +127 -0
- package/src/commands/down.js +29 -6
- package/src/commands/launch.js +431 -0
- package/src/commands/pull.js +137 -0
- package/src/commands/run.js +253 -37
- package/src/commands/sbatch.js +232 -0
- package/src/commands/serve.js +3 -3
- package/src/commands/status.js +12 -4
- package/src/commands/task.js +25 -0
- package/src/commands/test-run.js +4 -2
- package/src/credentials.js +65 -0
- package/src/fallback.js +7 -2
- package/src/fanout.js +70 -0
- package/src/gpuDoctor/diskInfo.js +42 -0
- package/src/gpuDoctor/doctor.js +451 -0
- package/src/gpuDoctor/gpuInfo.js +70 -0
- package/src/gpuDoctor/healthCheck.js +63 -0
- package/src/gpuDoctor/logClassifier.js +138 -0
- package/src/gpuDoctor/modelFit.js +107 -0
- package/src/gpuDoctor/probeCache.js +38 -0
- package/src/gpuDoctor/redact.js +29 -0
- package/src/gpuDoctor/torchInfo.js +61 -0
- package/src/gpuDoctor/workflowDoctor.js +96 -0
- package/src/onboarding.js +124 -0
- package/src/slurm.js +193 -0
- package/src/spec.js +59 -2
- package/src/store.js +16 -0
- package/tests/agent-images.test.js +17 -0
- package/tests/artifactDownload.test.js +113 -0
- package/tests/artifacts.test.js +168 -0
- package/tests/batch.test.js +312 -0
- package/tests/browser.test.js +51 -0
- package/tests/capacity.test.js +68 -0
- package/tests/commands.test.js +44 -0
- package/tests/connect.test.js +83 -0
- package/tests/down.test.js +23 -1
- package/tests/fallback-timeout.test.js +41 -0
- package/tests/fanout.test.js +124 -0
- package/tests/gpu-doctor-classifiers.test.js +402 -0
- package/tests/gpu-doctor-doctor.test.js +304 -0
- package/tests/gpu-doctor-probe-cache.test.js +110 -0
- package/tests/gpu-doctor-probes.test.js +257 -0
- package/tests/launch-command-argv.test.js +93 -0
- package/tests/launch-readiness.test.js +1 -0
- package/tests/launch.test.js +440 -0
- package/tests/onboarding.test.js +134 -0
- package/tests/pull.test.js +266 -0
- package/tests/run-lifecycle.test.js +405 -6
- package/tests/sbatch.test.js +190 -0
- package/tests/secrets.test.js +16 -0
- package/tests/slurm.test.js +77 -0
- package/tests/spec.test.js +59 -1
- package/tests/status.test.js +73 -0
- package/tests/task.test.js +109 -0
- package/tests/template.test.js +7 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* badgr sbatch <job.slurm> — run an existing Slurm batch script without
|
|
3
|
+
* asking HPC users to redesign their workflow.
|
|
4
|
+
*
|
|
5
|
+
* badgr sbatch job.slurm
|
|
6
|
+
* badgr sbatch job.slurm --dry-run
|
|
7
|
+
* badgr sbatch job.slurm --image myorg/hpc:latest
|
|
8
|
+
*
|
|
9
|
+
* Translates the common 20% of #SBATCH directives (cpus-per-task, mem,
|
|
10
|
+
* gres=gpu:N / gpus, array, time, export, job-name) into the same /run
|
|
11
|
+
* request shape `badgr batch run` submits — this is another entry point
|
|
12
|
+
* into the existing execution path, not a new one. Job arrays (--array)
|
|
13
|
+
* fan out into one deployment per task index, each with
|
|
14
|
+
* SLURM_ARRAY_TASK_ID set, mirroring how a real Slurm array behaves.
|
|
15
|
+
*/
|
|
16
|
+
import { resolve } from 'path';
|
|
17
|
+
import { requireApiKey } from '../config.js';
|
|
18
|
+
import { loadSlurmScript, envForArrayTask, SlurmParseError } from '../slurm.js';
|
|
19
|
+
import { addReceipt, updateReceipt, generateReceiptId, selectedComputeFromDeployment } from '../store.js';
|
|
20
|
+
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
21
|
+
import { normalizeGpuType } from '../spec.js';
|
|
22
|
+
import { monitorBatchJob, fmtRuntime } from '../batch.js';
|
|
23
|
+
import { runFanOut, DEFAULT_CONCURRENCY } from '../fanout.js';
|
|
24
|
+
import { callApi } from '../api.js';
|
|
25
|
+
|
|
26
|
+
const DEFAULT_IMAGE = 'python:3.11-slim';
|
|
27
|
+
const DEFAULT_MAX_RUNTIME_MIN = 60;
|
|
28
|
+
const DEFAULT_MAX_COST = 2;
|
|
29
|
+
|
|
30
|
+
function parseSbatchArgs(args) {
|
|
31
|
+
const flags = {};
|
|
32
|
+
const positional = [];
|
|
33
|
+
for (let i = 0; i < args.length; i++) {
|
|
34
|
+
if (args[i] === '--image') { flags.image = args[++i]; continue; }
|
|
35
|
+
if (args[i] === '--tier') { flags.tier = args[++i]; continue; }
|
|
36
|
+
if (args[i] === '--region') { flags.region = args[++i]; continue; }
|
|
37
|
+
if (args[i] === '--max-cost') { flags.maxCost = parseFloat(args[++i]); continue; }
|
|
38
|
+
if (args[i] === '--max-runtime') { flags.maxRuntime = parseFloat(args[++i]); continue; }
|
|
39
|
+
if (args[i] === '--max-concurrency') { flags.maxConcurrency = parseInt(args[++i], 10); continue; }
|
|
40
|
+
if (args[i] === '--dry-run') { flags.dryRun = true; continue; }
|
|
41
|
+
positional.push(args[i]);
|
|
42
|
+
}
|
|
43
|
+
return { flags, positional };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Translate a parsed Slurm job (or one array task of it) into a /run body. */
|
|
47
|
+
export function buildRunBody(job, { image, tier, region, maxCostUsd, maxRuntimeMinutes, taskId = null, arrayJobId = null }) {
|
|
48
|
+
const env = envForArrayTask(job.env, taskId, arrayJobId);
|
|
49
|
+
return {
|
|
50
|
+
image,
|
|
51
|
+
command: ['bash', '-lc', job.command],
|
|
52
|
+
custom_image: image !== DEFAULT_IMAGE,
|
|
53
|
+
gpu: job.gpuCount > 0 ? (job.gpuType ? normalizeGpuType(job.gpuType) : 'AUTO') : 'NONE',
|
|
54
|
+
gpu_count: job.gpuCount > 0 ? job.gpuCount : 1,
|
|
55
|
+
...(job.gpuCount > 0 ? {} : { no_gpu: true }),
|
|
56
|
+
...(job.cpus ? { cpu: job.cpus } : {}),
|
|
57
|
+
...(job.memGb ? { memory_gb: job.memGb } : {}),
|
|
58
|
+
...(region ? { region: region.toUpperCase() } : {}),
|
|
59
|
+
tier,
|
|
60
|
+
env,
|
|
61
|
+
name: taskId != null ? `${job.name}-${taskId}` : job.name,
|
|
62
|
+
max_cost_usd: maxCostUsd,
|
|
63
|
+
max_runtime_seconds: maxRuntimeMinutes * 60,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function printPlan(job, chalk) {
|
|
68
|
+
console.log(chalk.bold(`\n📜 Slurm job: ${job.name}\n`));
|
|
69
|
+
console.log(` ${chalk.bold('Command:')} ${job.command.split('\n')[0]}${job.command.includes('\n') ? ' ...' : ''}`);
|
|
70
|
+
if (job.cpus) console.log(` ${chalk.bold('CPUs:')} ${job.cpus}`);
|
|
71
|
+
if (job.memGb) console.log(` ${chalk.bold('Memory:')} ${job.memGb.toFixed(1)} GB`);
|
|
72
|
+
console.log(` ${chalk.bold('GPUs:')} ${job.gpuCount > 0 ? job.gpuCount + (job.gpuType ? ` × ${job.gpuType}` : '') : 'none'}`);
|
|
73
|
+
if (job.timeMinutes) console.log(` ${chalk.bold('Time limit:')} ${job.timeMinutes}min`);
|
|
74
|
+
if (job.arrayIndices.length) console.log(` ${chalk.bold('Array:')} ${job.arrayIndices.length} task(s) [${job.arrayIndices[0]}..${job.arrayIndices[job.arrayIndices.length - 1]}]`);
|
|
75
|
+
if (Object.keys(job.env).length) console.log(` ${chalk.bold('Env:')} ${Object.keys(job.env).join(', ')}`);
|
|
76
|
+
if (job.ignoredDirectives.length) {
|
|
77
|
+
console.log(chalk.yellow(` Ignored directives (no Badgr equivalent): ${job.ignoredDirectives.join(', ')}`));
|
|
78
|
+
}
|
|
79
|
+
console.log();
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function submitOne(config, job, opts, chalk, { taskId = null, arrayJobId = null } = {}) {
|
|
83
|
+
const body = buildRunBody(job, { ...opts, taskId, arrayJobId });
|
|
84
|
+
const effectiveTier = opts.tier;
|
|
85
|
+
const dep = await callWithFallback(
|
|
86
|
+
'/run',
|
|
87
|
+
{ apiKey: config.apiKey, baseUrl: config.baseUrl },
|
|
88
|
+
() => body,
|
|
89
|
+
effectiveTier,
|
|
90
|
+
chalk,
|
|
91
|
+
{ thing: 'Slurm job', cmd: 'badgr sbatch' },
|
|
92
|
+
);
|
|
93
|
+
const rcptId = dep.receipt_id || generateReceiptId();
|
|
94
|
+
addReceipt({
|
|
95
|
+
receiptId: rcptId,
|
|
96
|
+
action: 'badgr sbatch',
|
|
97
|
+
deploymentId: dep.deployment_id,
|
|
98
|
+
workloadName: body.name,
|
|
99
|
+
workloadImage: body.image,
|
|
100
|
+
gpu: dep.gpu_type,
|
|
101
|
+
providerRoute: dep.provider ?? null,
|
|
102
|
+
tier: dep.tier ?? null,
|
|
103
|
+
maxCost: opts.maxCostUsd,
|
|
104
|
+
maxRuntime: opts.maxRuntimeMinutes,
|
|
105
|
+
status: dep.status,
|
|
106
|
+
createdAt: new Date().toISOString(),
|
|
107
|
+
slurmArrayTaskId: taskId,
|
|
108
|
+
workloadShape: taskId != null ? 'slurm-array' : 'slurm',
|
|
109
|
+
computeRequested: { cpu: job.cpus ?? null, memoryGb: job.memGb ?? null, gpuCount: job.gpuCount, gpuType: job.gpuType ?? null },
|
|
110
|
+
computeSelected: selectedComputeFromDeployment(dep),
|
|
111
|
+
});
|
|
112
|
+
return { dep, rcptId };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export async function sbatchCommand(config, args, chalk) {
|
|
116
|
+
const { flags, positional } = parseSbatchArgs(args);
|
|
117
|
+
const scriptPath = positional[0];
|
|
118
|
+
if (!scriptPath) {
|
|
119
|
+
console.error(chalk.red('\n Usage: badgr sbatch <job.slurm> [--image <image>] [--dry-run]\n'));
|
|
120
|
+
process.exitCode = 1;
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
let job;
|
|
125
|
+
try {
|
|
126
|
+
job = loadSlurmScript(resolve(scriptPath));
|
|
127
|
+
} catch (err) {
|
|
128
|
+
if (err instanceof SlurmParseError) {
|
|
129
|
+
console.error(chalk.red(`\n ✗ ${err.message}\n`));
|
|
130
|
+
process.exitCode = 1;
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
throw err;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (flags.maxConcurrency != null && (!Number.isInteger(flags.maxConcurrency) || flags.maxConcurrency < 1)) {
|
|
137
|
+
console.error(chalk.red(`\n ✗ --max-concurrency must be a positive integer, got: ${flags.maxConcurrency}\n`));
|
|
138
|
+
process.exitCode = 1;
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
printPlan(job, chalk);
|
|
143
|
+
|
|
144
|
+
const opts = {
|
|
145
|
+
image: flags.image || DEFAULT_IMAGE,
|
|
146
|
+
tier: normalizeTier(flags.tier),
|
|
147
|
+
region: flags.region,
|
|
148
|
+
maxCostUsd: flags.maxCost ?? DEFAULT_MAX_COST,
|
|
149
|
+
maxRuntimeMinutes: flags.maxRuntime ?? job.timeMinutes ?? DEFAULT_MAX_RUNTIME_MIN,
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const isArray = job.arrayIndices.length > 0;
|
|
153
|
+
const taskIds = isArray ? job.arrayIndices : [null];
|
|
154
|
+
const concurrency = flags.maxConcurrency ?? DEFAULT_CONCURRENCY;
|
|
155
|
+
|
|
156
|
+
if (flags.dryRun) {
|
|
157
|
+
console.log(chalk.dim(` Dry run — no GPU provisioned. Would submit ${taskIds.length} job(s) (max ${Math.min(concurrency, taskIds.length)} concurrent) with:`));
|
|
158
|
+
console.log(chalk.dim(` image=${opts.image} tier=${opts.tier} max_cost=$${opts.maxCostUsd} max_runtime=${opts.maxRuntimeMinutes}min\n`));
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
requireApiKey(config);
|
|
163
|
+
|
|
164
|
+
const arrayJobId = isArray ? generateReceiptId() : null;
|
|
165
|
+
|
|
166
|
+
if (isArray) {
|
|
167
|
+
console.log(chalk.dim(` Submitting ${taskIds.length} array task(s), up to ${Math.min(concurrency, taskIds.length)} at a time...\n`));
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const { results, allSucceeded, submittedCount, failedSubmitCount } = await runFanOut({
|
|
171
|
+
tasks: taskIds,
|
|
172
|
+
concurrency,
|
|
173
|
+
onSubmitError: (taskId, err) => {
|
|
174
|
+
if (err.isPaymentRequired) console.error(chalk.yellow(err.message));
|
|
175
|
+
else console.error(chalk.red(` ✗ Task ${taskId ?? ''} failed to submit: ${err.message}`));
|
|
176
|
+
},
|
|
177
|
+
submitTask: async (taskId) => {
|
|
178
|
+
process.stdout.write(chalk.dim(taskId != null ? ` Submitting array task ${taskId}...\n` : ' Submitting job...\n'));
|
|
179
|
+
const { dep, rcptId } = await submitOne(config, job, opts, chalk, { taskId, arrayJobId });
|
|
180
|
+
console.log(` ${chalk.bold('Run ID:')} ${chalk.cyan(dep.deployment_id)}${taskId != null ? chalk.dim(` (task ${taskId})`) : ''}`);
|
|
181
|
+
return { deploymentId: dep.deployment_id, receiptId: rcptId, ratePerHour: dep.cost_per_hour || 0 };
|
|
182
|
+
},
|
|
183
|
+
monitorTask: async ({ deploymentId, receiptId, ratePerHour }) => {
|
|
184
|
+
const result = await monitorBatchJob(config, deploymentId, receiptId, {
|
|
185
|
+
chalk,
|
|
186
|
+
maxRuntimeMs: opts.maxRuntimeMinutes * 60 * 1000,
|
|
187
|
+
maxCost: opts.maxCostUsd,
|
|
188
|
+
ratePerHour,
|
|
189
|
+
});
|
|
190
|
+
let dep2 = null;
|
|
191
|
+
try {
|
|
192
|
+
dep2 = await callApi(`/deployments/${deploymentId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
|
|
193
|
+
} catch { /* best-effort */ }
|
|
194
|
+
updateReceipt(receiptId, {
|
|
195
|
+
status: dep2?.status ?? result.status,
|
|
196
|
+
failureReason: dep2?.failure_reason ?? null,
|
|
197
|
+
teardownOk: dep2?.teardown_ok ?? null,
|
|
198
|
+
runtimeSeconds: dep2?.runtime_seconds ?? Math.round(result.runtimeMs / 1000),
|
|
199
|
+
finalCost: dep2?.accrued_cost_usd ?? (ratePerHour * (result.runtimeMs / 3_600_000)),
|
|
200
|
+
});
|
|
201
|
+
const status = dep2?.status ?? result.status;
|
|
202
|
+
return { status, succeeded: ['succeeded', 'completed'].includes(status), runtimeMs: result.runtimeMs };
|
|
203
|
+
},
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
if (submittedCount === 0) {
|
|
207
|
+
console.error(chalk.red('\n ✗ No tasks were submitted\n'));
|
|
208
|
+
process.exitCode = 1;
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
console.log(chalk.bold(`\n📜 Slurm job summary: ${job.name}\n`));
|
|
213
|
+
for (const r of results) {
|
|
214
|
+
const label = r.task != null ? `task ${r.task}` : 'job';
|
|
215
|
+
if (r.error) {
|
|
216
|
+
console.log(` ${chalk.red('✗')} ${label} ${chalk.dim('never submitted')}`);
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
const icon = r.succeeded ? chalk.green('✓') : chalk.red('✗');
|
|
220
|
+
console.log(` ${icon} ${label} ${r.deploymentId} ${r.status} ${fmtRuntime(r.runtimeMs)}`);
|
|
221
|
+
}
|
|
222
|
+
if (failedSubmitCount) console.log(chalk.red(` ${failedSubmitCount} task(s) never submitted`));
|
|
223
|
+
|
|
224
|
+
console.log();
|
|
225
|
+
if (allSucceeded) {
|
|
226
|
+
console.log(chalk.green(` ✓ Slurm job complete (${taskIds.length}/${taskIds.length} task(s) succeeded)\n`));
|
|
227
|
+
} else {
|
|
228
|
+
const okCount = results.filter(r => r.succeeded).length;
|
|
229
|
+
console.error(chalk.red(` ✗ Slurm job incomplete (${okCount}/${taskIds.length} task(s) succeeded)\n`));
|
|
230
|
+
process.exitCode = 1;
|
|
231
|
+
}
|
|
232
|
+
}
|
package/src/commands/serve.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ensureBadgrReady } from '../onboarding.js';
|
|
2
2
|
import { callApi, listDeployments } from '../api.js';
|
|
3
3
|
import { addDeployment, addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
4
4
|
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
@@ -254,7 +254,7 @@ async function findRunningVllmEndpoint(config, modelId) {
|
|
|
254
254
|
* no model is ever touched in that mode.
|
|
255
255
|
*/
|
|
256
256
|
async function serveOpenWebUICommand(config, args, chalk) {
|
|
257
|
-
|
|
257
|
+
config = await ensureBadgrReady(config, chalk);
|
|
258
258
|
const { value: connect, rest: afterConnect } = extractFlag(args, '--connect');
|
|
259
259
|
const { value: model, rest } = extractFlag(afterConnect, '--model');
|
|
260
260
|
|
|
@@ -396,7 +396,7 @@ export async function serveCommand(config, args, chalk) {
|
|
|
396
396
|
return;
|
|
397
397
|
}
|
|
398
398
|
|
|
399
|
-
|
|
399
|
+
config = await ensureBadgrReady(config, chalk);
|
|
400
400
|
|
|
401
401
|
// ── Validate flags early ───────────────────────────────────────────────────
|
|
402
402
|
if (flags.count !== undefined && (!Number.isFinite(flags.count) || flags.count < 1)) {
|
package/src/commands/status.js
CHANGED
|
@@ -36,7 +36,15 @@ export async function statusCommand(config, args, chalk) {
|
|
|
36
36
|
}));
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
// Filter OUT known-terminal statuses rather than filtering IN a fixed
|
|
40
|
+
// active list — the backend has more active states than just "running"/
|
|
41
|
+
// "provisioning" (also "starting", "queued"), and a status this command
|
|
42
|
+
// doesn't recognize yet must still show as active rather than being
|
|
43
|
+
// silently hidden (a live run found deployments stuck in "starting" that
|
|
44
|
+
// this filter used to drop entirely, reporting "Nothing running" while
|
|
45
|
+
// real billable resources kept running).
|
|
46
|
+
const TERMINAL_STATUSES = new Set(['completed', 'succeeded', 'success', 'failed', 'stopped']);
|
|
47
|
+
const running = deployments.filter(d => !TERMINAL_STATUSES.has(d.status));
|
|
40
48
|
|
|
41
49
|
if (running.length === 0) {
|
|
42
50
|
console.log(chalk.dim('\n Nothing running.\n'));
|
|
@@ -51,9 +59,9 @@ export async function statusCommand(config, args, chalk) {
|
|
|
51
59
|
const type = d.workload_type === 'endpoint' ? 'endpoint' : 'job';
|
|
52
60
|
const gpu = d.gpu_type || '—';
|
|
53
61
|
const rate = d.cost_per_hour > 0 ? chalk.yellow(`$${d.cost_per_hour.toFixed(2)}/hr`) : '';
|
|
54
|
-
const badge = d.status === '
|
|
55
|
-
? chalk.
|
|
56
|
-
: chalk.
|
|
62
|
+
const badge = d.status === 'running'
|
|
63
|
+
? chalk.green('● running')
|
|
64
|
+
: chalk.yellow('● starting');
|
|
57
65
|
|
|
58
66
|
console.log(` ${badge} ${chalk.bold(id)} ${type} ${gpu} ${rate}`);
|
|
59
67
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { launchCommand } from './launch.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* badgr task "<description>" -- <command>
|
|
5
|
+
*
|
|
6
|
+
* The MVP version of "assign a task" — a thin label wrapper over
|
|
7
|
+
* `badgr launch . -- <command>`. It does not queue, schedule, or track
|
|
8
|
+
* anything beyond what `badgr launch` already does; the description is
|
|
9
|
+
* printed for the human and otherwise discarded (the receipt/status page
|
|
10
|
+
* still key off the deployment ID, same as any other launch).
|
|
11
|
+
*/
|
|
12
|
+
export async function taskCommand(config, args, chalk) {
|
|
13
|
+
const description = args[0];
|
|
14
|
+
if (!description || description.startsWith('-')) {
|
|
15
|
+
console.error(chalk.red('\nUsage: badgr task "<description>" [badgr launch flags...] -- <command>\n'));
|
|
16
|
+
console.error(chalk.dim(' The description must come first, before any flags.'));
|
|
17
|
+
console.error(chalk.dim(' Example: badgr task "Run the Chromium tests and tell me what failed" --max-cost 1 -- npm run test:chromium'));
|
|
18
|
+
console.error(chalk.dim(' Example: badgr task "Fix the failing checkout test" --max-cost 1 -- claude -p "Fix the failing checkout test"\n'));
|
|
19
|
+
process.exitCode = 1;
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
console.log(chalk.dim(` Task: ${description}`));
|
|
24
|
+
return launchCommand(config, ['.', ...args.slice(1)], chalk);
|
|
25
|
+
}
|
package/src/commands/test-run.js
CHANGED
|
@@ -207,8 +207,10 @@ export async function testCommand(config, args, chalk) {
|
|
|
207
207
|
process.stdout.write(chalk.dim(' Stopping deployment...'));
|
|
208
208
|
let stopped = false;
|
|
209
209
|
try {
|
|
210
|
-
|
|
211
|
-
|
|
210
|
+
// A 200 response only means deletion was requested, not confirmed —
|
|
211
|
+
// teardown_ok reflects whether the provider resource is actually gone.
|
|
212
|
+
const result = await terminateDeployment(config, depId);
|
|
213
|
+
stopped = result?.teardown_ok === 'ok';
|
|
212
214
|
} catch { /* best-effort */ }
|
|
213
215
|
process.stdout.write('\n');
|
|
214
216
|
step(chalk, stopped, 'Billing stopped');
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync, chmodSync } from 'fs';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { CONFIG_DIR } from './config.js';
|
|
4
|
+
|
|
5
|
+
export const CREDENTIALS_FILE = join(CONFIG_DIR, 'credentials.json');
|
|
6
|
+
|
|
7
|
+
// Env var each provider's credential is exposed under inside the launch VM.
|
|
8
|
+
export const PROVIDER_ENV_KEYS = {
|
|
9
|
+
anthropic: 'ANTHROPIC_API_KEY',
|
|
10
|
+
openai: 'OPENAI_API_KEY',
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export const KNOWN_PROVIDERS = Object.keys(PROVIDER_ENV_KEYS);
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Credential storage for `badgr connect <provider>`. This is a local file
|
|
17
|
+
* under ~/.badgr with owner-only permissions (chmod 600) — not the
|
|
18
|
+
* platform's real OS keychain (macOS Keychain / libsecret / Windows
|
|
19
|
+
* Credential Manager), since that needs a native module this CLI doesn't
|
|
20
|
+
* depend on yet. Documented as such; do not claim OS-keychain storage.
|
|
21
|
+
*/
|
|
22
|
+
function readStore() {
|
|
23
|
+
if (!existsSync(CREDENTIALS_FILE)) return {};
|
|
24
|
+
try {
|
|
25
|
+
return JSON.parse(readFileSync(CREDENTIALS_FILE, 'utf8'));
|
|
26
|
+
} catch {
|
|
27
|
+
return {};
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function writeStore(store) {
|
|
32
|
+
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
33
|
+
// mode: 0o600 on writeFileSync itself sets permissions atomically at
|
|
34
|
+
// creation — passing it here (not just the chmodSync below) closes a
|
|
35
|
+
// brief window where a freshly-created file would otherwise sit at the
|
|
36
|
+
// default (world-readable) mode until the chmod call caught up.
|
|
37
|
+
writeFileSync(CREDENTIALS_FILE, JSON.stringify(store, null, 2), { mode: 0o600 });
|
|
38
|
+
try {
|
|
39
|
+
chmodSync(CREDENTIALS_FILE, 0o600);
|
|
40
|
+
} catch {
|
|
41
|
+
// best-effort — not all filesystems support chmod (e.g. some Windows setups)
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function getCredential(provider) {
|
|
46
|
+
return readStore()[provider] ?? null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function setCredential(provider, value) {
|
|
50
|
+
const store = readStore();
|
|
51
|
+
store[provider] = value;
|
|
52
|
+
writeStore(store);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function removeCredential(provider) {
|
|
56
|
+
const store = readStore();
|
|
57
|
+
if (!(provider in store)) return false;
|
|
58
|
+
delete store[provider];
|
|
59
|
+
writeStore(store);
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function listCredentials() {
|
|
64
|
+
return Object.keys(readStore());
|
|
65
|
+
}
|
package/src/fallback.js
CHANGED
|
@@ -44,9 +44,14 @@ export async function callWithFallback(endpoint, callOpts, buildBody, effectiveT
|
|
|
44
44
|
const cmd = labels?.cmd ?? 'badgr run';
|
|
45
45
|
const allowTier2Fallback = opts.allowTier2Fallback !== false; // default true
|
|
46
46
|
|
|
47
|
-
//
|
|
47
|
+
// 220s: comfortably above backend's BADGR_PROVISION_TIMEOUT_SECONDS (default
|
|
48
|
+
// 200s, itself set above deployment_service.py's 180s routing-search
|
|
49
|
+
// deadline). Found live: the previous 130s value fired before the server's
|
|
50
|
+
// own (then-120s) deadline could, so the CLI reported "failed to submit"
|
|
51
|
+
// for jobs that had actually been created and were already billing —
|
|
52
|
+
// aborting client-side never cancels the server's in-flight work.
|
|
48
53
|
async function attempt(body) {
|
|
49
|
-
return callApi(endpoint, { method: 'POST', ...callOpts, body, timeoutMs:
|
|
54
|
+
return callApi(endpoint, { method: 'POST', ...callOpts, body, timeoutMs: 220_000 });
|
|
50
55
|
}
|
|
51
56
|
|
|
52
57
|
// Map server error codes → catalog keys and extract context for templates.
|
package/src/fanout.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared fan-out primitive: submit N independent /run tasks — at most
|
|
3
|
+
* `concurrency` in flight (submitted-but-not-yet-terminal) at once — monitor
|
|
4
|
+
* each to a terminal state, and roll the results up into one summary.
|
|
5
|
+
*
|
|
6
|
+
* "Run the same program many times with different inputs" is structurally
|
|
7
|
+
* identical whether the N tasks come from a Slurm `#SBATCH --array` range
|
|
8
|
+
* (see commands/sbatch.js) or a directory of input files (see `badgr batch
|
|
9
|
+
* run --fan-out`, commands/batch.js) — both funnel through this module so
|
|
10
|
+
* the submit/monitor/summarize logic, and the concurrency cap that bounds
|
|
11
|
+
* how much compute a single command can accidentally provision, exist
|
|
12
|
+
* exactly once.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const DEFAULT_CONCURRENCY = 5;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {Array<any>} tasks - opaque per-task descriptors, passed straight
|
|
19
|
+
* through to submitTask/monitorTask/onSubmitError.
|
|
20
|
+
* @param {(task) => Promise<{deploymentId, receiptId, ratePerHour}>} submitTask
|
|
21
|
+
* Submits one task. Rejecting marks it as a submit failure (never monitored).
|
|
22
|
+
* @param {(submitted: {task, deploymentId, receiptId, ratePerHour}) => Promise<{status, succeeded, runtimeMs}>} monitorTask
|
|
23
|
+
* Waits for one submitted task to reach a terminal state.
|
|
24
|
+
* @param {(task, err) => void} [onSubmitError] - called synchronously as each submit fails.
|
|
25
|
+
* @param {number} [concurrency] - max tasks with an active deployment (submitted,
|
|
26
|
+
* not yet terminal) at once. Defaults to 5 — fan-out must never submit an
|
|
27
|
+
* unbounded number of deployments from one command by default.
|
|
28
|
+
* @returns {Promise<{results: Array, allSucceeded: boolean, submittedCount: number, failedSubmitCount: number}>}
|
|
29
|
+
* `results` has one entry per input task, in the original task order, each
|
|
30
|
+
* either `{ task, error }` (never submitted) or
|
|
31
|
+
* `{ task, deploymentId, receiptId, status, succeeded, runtimeMs }`.
|
|
32
|
+
*/
|
|
33
|
+
export async function runFanOut({ tasks, submitTask, monitorTask, onSubmitError = () => {}, concurrency = DEFAULT_CONCURRENCY }) {
|
|
34
|
+
const effectiveConcurrency = Math.max(1, concurrency || DEFAULT_CONCURRENCY);
|
|
35
|
+
const results = new Array(tasks.length);
|
|
36
|
+
let nextIndex = 0;
|
|
37
|
+
|
|
38
|
+
async function worker() {
|
|
39
|
+
while (nextIndex < tasks.length) {
|
|
40
|
+
const i = nextIndex++;
|
|
41
|
+
const task = tasks[i];
|
|
42
|
+
let submitted;
|
|
43
|
+
try {
|
|
44
|
+
submitted = await submitTask(task);
|
|
45
|
+
} catch (err) {
|
|
46
|
+
onSubmitError(task, err);
|
|
47
|
+
results[i] = { task, error: err };
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const monitored = await monitorTask({ task, ...submitted });
|
|
51
|
+
results[i] = { task, ...submitted, ...monitored };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const workerCount = Math.min(effectiveConcurrency, tasks.length);
|
|
56
|
+
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
57
|
+
|
|
58
|
+
const submittedCount = results.filter(r => !r.error).length;
|
|
59
|
+
const failedSubmitCount = results.filter(r => r.error).length;
|
|
60
|
+
const allSucceeded = failedSubmitCount === 0 && results.every(r => r.succeeded);
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
results,
|
|
64
|
+
allSucceeded,
|
|
65
|
+
submittedCount,
|
|
66
|
+
failedSubmitCount,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export { DEFAULT_CONCURRENCY };
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
function round1(n) {
|
|
6
|
+
return Math.round(n * 10) / 10;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Read-only disk/cache probe. Never creates or deletes anything.
|
|
11
|
+
*/
|
|
12
|
+
export function detectDisk(fsImpl = fs) {
|
|
13
|
+
const cachePath = process.env.HF_HOME || process.env.HUGGINGFACE_HUB_CACHE || path.join(os.homedir(), '.cache', 'huggingface');
|
|
14
|
+
const result = {
|
|
15
|
+
cachePath,
|
|
16
|
+
cacheSource: process.env.HF_HOME ? 'HF_HOME' : process.env.HUGGINGFACE_HUB_CACHE ? 'HUGGINGFACE_HUB_CACHE' : 'default (~/.cache/huggingface)',
|
|
17
|
+
cacheExists: false,
|
|
18
|
+
writable: false,
|
|
19
|
+
freeGb: null,
|
|
20
|
+
totalGb: null,
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
result.cacheExists = fsImpl.existsSync(cachePath);
|
|
24
|
+
const probePath = result.cacheExists ? cachePath : path.dirname(cachePath);
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
const stat = fsImpl.statfsSync(probePath);
|
|
28
|
+
result.freeGb = round1((stat.bavail * stat.bsize) / 1e9);
|
|
29
|
+
result.totalGb = round1((stat.blocks * stat.bsize) / 1e9);
|
|
30
|
+
} catch {
|
|
31
|
+
// statfs unsupported on this platform — leave free/total unknown
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
try {
|
|
35
|
+
fsImpl.accessSync(fsImpl.existsSync(probePath) ? probePath : os.tmpdir(), fs.constants.W_OK);
|
|
36
|
+
result.writable = true;
|
|
37
|
+
} catch {
|
|
38
|
+
result.writable = false;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return result;
|
|
42
|
+
}
|