badgr-cli 1.0.48 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +38 -0
  2. package/package.json +1 -1
  3. package/src/api.js +16 -2
  4. package/src/artifactDownload.js +55 -0
  5. package/src/badgr.js +104 -0
  6. package/src/batch.js +22 -4
  7. package/src/browser.js +23 -0
  8. package/src/commands/artifacts.js +75 -0
  9. package/src/commands/batch.js +221 -28
  10. package/src/commands/billing.js +1 -12
  11. package/src/commands/capacity.js +9 -4
  12. package/src/commands/comfyui.js +3 -3
  13. package/src/commands/connect.js +83 -0
  14. package/src/commands/doctor.js +127 -0
  15. package/src/commands/down.js +29 -6
  16. package/src/commands/launch.js +431 -0
  17. package/src/commands/pull.js +137 -0
  18. package/src/commands/run.js +253 -37
  19. package/src/commands/sbatch.js +232 -0
  20. package/src/commands/serve.js +3 -3
  21. package/src/commands/status.js +12 -4
  22. package/src/commands/task.js +25 -0
  23. package/src/commands/test-run.js +4 -2
  24. package/src/credentials.js +65 -0
  25. package/src/fallback.js +7 -2
  26. package/src/fanout.js +70 -0
  27. package/src/gpuDoctor/diskInfo.js +42 -0
  28. package/src/gpuDoctor/doctor.js +451 -0
  29. package/src/gpuDoctor/gpuInfo.js +70 -0
  30. package/src/gpuDoctor/healthCheck.js +63 -0
  31. package/src/gpuDoctor/logClassifier.js +138 -0
  32. package/src/gpuDoctor/modelFit.js +107 -0
  33. package/src/gpuDoctor/probeCache.js +38 -0
  34. package/src/gpuDoctor/redact.js +29 -0
  35. package/src/gpuDoctor/torchInfo.js +61 -0
  36. package/src/gpuDoctor/workflowDoctor.js +96 -0
  37. package/src/onboarding.js +124 -0
  38. package/src/slurm.js +193 -0
  39. package/src/spec.js +59 -2
  40. package/src/store.js +16 -0
  41. package/tests/agent-images.test.js +17 -0
  42. package/tests/artifactDownload.test.js +113 -0
  43. package/tests/artifacts.test.js +168 -0
  44. package/tests/batch.test.js +312 -0
  45. package/tests/browser.test.js +51 -0
  46. package/tests/capacity.test.js +68 -0
  47. package/tests/commands.test.js +44 -0
  48. package/tests/connect.test.js +83 -0
  49. package/tests/down.test.js +23 -1
  50. package/tests/fallback-timeout.test.js +41 -0
  51. package/tests/fanout.test.js +124 -0
  52. package/tests/gpu-doctor-classifiers.test.js +402 -0
  53. package/tests/gpu-doctor-doctor.test.js +304 -0
  54. package/tests/gpu-doctor-probe-cache.test.js +110 -0
  55. package/tests/gpu-doctor-probes.test.js +257 -0
  56. package/tests/launch-command-argv.test.js +93 -0
  57. package/tests/launch-readiness.test.js +1 -0
  58. package/tests/launch.test.js +440 -0
  59. package/tests/onboarding.test.js +134 -0
  60. package/tests/pull.test.js +266 -0
  61. package/tests/run-lifecycle.test.js +405 -6
  62. package/tests/sbatch.test.js +190 -0
  63. package/tests/secrets.test.js +16 -0
  64. package/tests/slurm.test.js +77 -0
  65. package/tests/spec.test.js +59 -1
  66. package/tests/status.test.js +73 -0
  67. package/tests/task.test.js +109 -0
  68. package/tests/template.test.js +7 -0
@@ -0,0 +1,232 @@
1
+ /**
2
+ * badgr sbatch <job.slurm> — run an existing Slurm batch script without
3
+ * asking HPC users to redesign their workflow.
4
+ *
5
+ * badgr sbatch job.slurm
6
+ * badgr sbatch job.slurm --dry-run
7
+ * badgr sbatch job.slurm --image myorg/hpc:latest
8
+ *
9
+ * Translates the common 20% of #SBATCH directives (cpus-per-task, mem,
10
+ * gres=gpu:N / gpus, array, time, export, job-name) into the same /run
11
+ * request shape `badgr batch run` submits — this is another entry point
12
+ * into the existing execution path, not a new one. Job arrays (--array)
13
+ * fan out into one deployment per task index, each with
14
+ * SLURM_ARRAY_TASK_ID set, mirroring how a real Slurm array behaves.
15
+ */
16
+ import { resolve } from 'path';
17
+ import { requireApiKey } from '../config.js';
18
+ import { loadSlurmScript, envForArrayTask, SlurmParseError } from '../slurm.js';
19
+ import { addReceipt, updateReceipt, generateReceiptId, selectedComputeFromDeployment } from '../store.js';
20
+ import { normalizeTier, callWithFallback } from '../fallback.js';
21
+ import { normalizeGpuType } from '../spec.js';
22
+ import { monitorBatchJob, fmtRuntime } from '../batch.js';
23
+ import { runFanOut, DEFAULT_CONCURRENCY } from '../fanout.js';
24
+ import { callApi } from '../api.js';
25
+
26
+ const DEFAULT_IMAGE = 'python:3.11-slim';
27
+ const DEFAULT_MAX_RUNTIME_MIN = 60;
28
+ const DEFAULT_MAX_COST = 2;
29
+
30
+ function parseSbatchArgs(args) {
31
+ const flags = {};
32
+ const positional = [];
33
+ for (let i = 0; i < args.length; i++) {
34
+ if (args[i] === '--image') { flags.image = args[++i]; continue; }
35
+ if (args[i] === '--tier') { flags.tier = args[++i]; continue; }
36
+ if (args[i] === '--region') { flags.region = args[++i]; continue; }
37
+ if (args[i] === '--max-cost') { flags.maxCost = parseFloat(args[++i]); continue; }
38
+ if (args[i] === '--max-runtime') { flags.maxRuntime = parseFloat(args[++i]); continue; }
39
+ if (args[i] === '--max-concurrency') { flags.maxConcurrency = parseInt(args[++i], 10); continue; }
40
+ if (args[i] === '--dry-run') { flags.dryRun = true; continue; }
41
+ positional.push(args[i]);
42
+ }
43
+ return { flags, positional };
44
+ }
45
+
46
+ /** Translate a parsed Slurm job (or one array task of it) into a /run body. */
47
+ export function buildRunBody(job, { image, tier, region, maxCostUsd, maxRuntimeMinutes, taskId = null, arrayJobId = null }) {
48
+ const env = envForArrayTask(job.env, taskId, arrayJobId);
49
+ return {
50
+ image,
51
+ command: ['bash', '-lc', job.command],
52
+ custom_image: image !== DEFAULT_IMAGE,
53
+ gpu: job.gpuCount > 0 ? (job.gpuType ? normalizeGpuType(job.gpuType) : 'AUTO') : 'NONE',
54
+ gpu_count: job.gpuCount > 0 ? job.gpuCount : 1,
55
+ ...(job.gpuCount > 0 ? {} : { no_gpu: true }),
56
+ ...(job.cpus ? { cpu: job.cpus } : {}),
57
+ ...(job.memGb ? { memory_gb: job.memGb } : {}),
58
+ ...(region ? { region: region.toUpperCase() } : {}),
59
+ tier,
60
+ env,
61
+ name: taskId != null ? `${job.name}-${taskId}` : job.name,
62
+ max_cost_usd: maxCostUsd,
63
+ max_runtime_seconds: maxRuntimeMinutes * 60,
64
+ };
65
+ }
66
+
67
+ function printPlan(job, chalk) {
68
+ console.log(chalk.bold(`\n📜 Slurm job: ${job.name}\n`));
69
+ console.log(` ${chalk.bold('Command:')} ${job.command.split('\n')[0]}${job.command.includes('\n') ? ' ...' : ''}`);
70
+ if (job.cpus) console.log(` ${chalk.bold('CPUs:')} ${job.cpus}`);
71
+ if (job.memGb) console.log(` ${chalk.bold('Memory:')} ${job.memGb.toFixed(1)} GB`);
72
+ console.log(` ${chalk.bold('GPUs:')} ${job.gpuCount > 0 ? job.gpuCount + (job.gpuType ? ` × ${job.gpuType}` : '') : 'none'}`);
73
+ if (job.timeMinutes) console.log(` ${chalk.bold('Time limit:')} ${job.timeMinutes}min`);
74
+ if (job.arrayIndices.length) console.log(` ${chalk.bold('Array:')} ${job.arrayIndices.length} task(s) [${job.arrayIndices[0]}..${job.arrayIndices[job.arrayIndices.length - 1]}]`);
75
+ if (Object.keys(job.env).length) console.log(` ${chalk.bold('Env:')} ${Object.keys(job.env).join(', ')}`);
76
+ if (job.ignoredDirectives.length) {
77
+ console.log(chalk.yellow(` Ignored directives (no Badgr equivalent): ${job.ignoredDirectives.join(', ')}`));
78
+ }
79
+ console.log();
80
+ }
81
+
82
+ async function submitOne(config, job, opts, chalk, { taskId = null, arrayJobId = null } = {}) {
83
+ const body = buildRunBody(job, { ...opts, taskId, arrayJobId });
84
+ const effectiveTier = opts.tier;
85
+ const dep = await callWithFallback(
86
+ '/run',
87
+ { apiKey: config.apiKey, baseUrl: config.baseUrl },
88
+ () => body,
89
+ effectiveTier,
90
+ chalk,
91
+ { thing: 'Slurm job', cmd: 'badgr sbatch' },
92
+ );
93
+ const rcptId = dep.receipt_id || generateReceiptId();
94
+ addReceipt({
95
+ receiptId: rcptId,
96
+ action: 'badgr sbatch',
97
+ deploymentId: dep.deployment_id,
98
+ workloadName: body.name,
99
+ workloadImage: body.image,
100
+ gpu: dep.gpu_type,
101
+ providerRoute: dep.provider ?? null,
102
+ tier: dep.tier ?? null,
103
+ maxCost: opts.maxCostUsd,
104
+ maxRuntime: opts.maxRuntimeMinutes,
105
+ status: dep.status,
106
+ createdAt: new Date().toISOString(),
107
+ slurmArrayTaskId: taskId,
108
+ workloadShape: taskId != null ? 'slurm-array' : 'slurm',
109
+ computeRequested: { cpu: job.cpus ?? null, memoryGb: job.memGb ?? null, gpuCount: job.gpuCount, gpuType: job.gpuType ?? null },
110
+ computeSelected: selectedComputeFromDeployment(dep),
111
+ });
112
+ return { dep, rcptId };
113
+ }
114
+
115
+ export async function sbatchCommand(config, args, chalk) {
116
+ const { flags, positional } = parseSbatchArgs(args);
117
+ const scriptPath = positional[0];
118
+ if (!scriptPath) {
119
+ console.error(chalk.red('\n Usage: badgr sbatch <job.slurm> [--image <image>] [--dry-run]\n'));
120
+ process.exitCode = 1;
121
+ return;
122
+ }
123
+
124
+ let job;
125
+ try {
126
+ job = loadSlurmScript(resolve(scriptPath));
127
+ } catch (err) {
128
+ if (err instanceof SlurmParseError) {
129
+ console.error(chalk.red(`\n ✗ ${err.message}\n`));
130
+ process.exitCode = 1;
131
+ return;
132
+ }
133
+ throw err;
134
+ }
135
+
136
+ if (flags.maxConcurrency != null && (!Number.isInteger(flags.maxConcurrency) || flags.maxConcurrency < 1)) {
137
+ console.error(chalk.red(`\n ✗ --max-concurrency must be a positive integer, got: ${flags.maxConcurrency}\n`));
138
+ process.exitCode = 1;
139
+ return;
140
+ }
141
+
142
+ printPlan(job, chalk);
143
+
144
+ const opts = {
145
+ image: flags.image || DEFAULT_IMAGE,
146
+ tier: normalizeTier(flags.tier),
147
+ region: flags.region,
148
+ maxCostUsd: flags.maxCost ?? DEFAULT_MAX_COST,
149
+ maxRuntimeMinutes: flags.maxRuntime ?? job.timeMinutes ?? DEFAULT_MAX_RUNTIME_MIN,
150
+ };
151
+
152
+ const isArray = job.arrayIndices.length > 0;
153
+ const taskIds = isArray ? job.arrayIndices : [null];
154
+ const concurrency = flags.maxConcurrency ?? DEFAULT_CONCURRENCY;
155
+
156
+ if (flags.dryRun) {
157
+ console.log(chalk.dim(` Dry run — no GPU provisioned. Would submit ${taskIds.length} job(s) (max ${Math.min(concurrency, taskIds.length)} concurrent) with:`));
158
+ console.log(chalk.dim(` image=${opts.image} tier=${opts.tier} max_cost=$${opts.maxCostUsd} max_runtime=${opts.maxRuntimeMinutes}min\n`));
159
+ return;
160
+ }
161
+
162
+ requireApiKey(config);
163
+
164
+ const arrayJobId = isArray ? generateReceiptId() : null;
165
+
166
+ if (isArray) {
167
+ console.log(chalk.dim(` Submitting ${taskIds.length} array task(s), up to ${Math.min(concurrency, taskIds.length)} at a time...\n`));
168
+ }
169
+
170
+ const { results, allSucceeded, submittedCount, failedSubmitCount } = await runFanOut({
171
+ tasks: taskIds,
172
+ concurrency,
173
+ onSubmitError: (taskId, err) => {
174
+ if (err.isPaymentRequired) console.error(chalk.yellow(err.message));
175
+ else console.error(chalk.red(` ✗ Task ${taskId ?? ''} failed to submit: ${err.message}`));
176
+ },
177
+ submitTask: async (taskId) => {
178
+ process.stdout.write(chalk.dim(taskId != null ? ` Submitting array task ${taskId}...\n` : ' Submitting job...\n'));
179
+ const { dep, rcptId } = await submitOne(config, job, opts, chalk, { taskId, arrayJobId });
180
+ console.log(` ${chalk.bold('Run ID:')} ${chalk.cyan(dep.deployment_id)}${taskId != null ? chalk.dim(` (task ${taskId})`) : ''}`);
181
+ return { deploymentId: dep.deployment_id, receiptId: rcptId, ratePerHour: dep.cost_per_hour || 0 };
182
+ },
183
+ monitorTask: async ({ deploymentId, receiptId, ratePerHour }) => {
184
+ const result = await monitorBatchJob(config, deploymentId, receiptId, {
185
+ chalk,
186
+ maxRuntimeMs: opts.maxRuntimeMinutes * 60 * 1000,
187
+ maxCost: opts.maxCostUsd,
188
+ ratePerHour,
189
+ });
190
+ let dep2 = null;
191
+ try {
192
+ dep2 = await callApi(`/deployments/${deploymentId}`, { apiKey: config.apiKey, baseUrl: config.baseUrl });
193
+ } catch { /* best-effort */ }
194
+ updateReceipt(receiptId, {
195
+ status: dep2?.status ?? result.status,
196
+ failureReason: dep2?.failure_reason ?? null,
197
+ teardownOk: dep2?.teardown_ok ?? null,
198
+ runtimeSeconds: dep2?.runtime_seconds ?? Math.round(result.runtimeMs / 1000),
199
+ finalCost: dep2?.accrued_cost_usd ?? (ratePerHour * (result.runtimeMs / 3_600_000)),
200
+ });
201
+ const status = dep2?.status ?? result.status;
202
+ return { status, succeeded: ['succeeded', 'completed'].includes(status), runtimeMs: result.runtimeMs };
203
+ },
204
+ });
205
+
206
+ if (submittedCount === 0) {
207
+ console.error(chalk.red('\n ✗ No tasks were submitted\n'));
208
+ process.exitCode = 1;
209
+ return;
210
+ }
211
+
212
+ console.log(chalk.bold(`\n📜 Slurm job summary: ${job.name}\n`));
213
+ for (const r of results) {
214
+ const label = r.task != null ? `task ${r.task}` : 'job';
215
+ if (r.error) {
216
+ console.log(` ${chalk.red('✗')} ${label} ${chalk.dim('never submitted')}`);
217
+ continue;
218
+ }
219
+ const icon = r.succeeded ? chalk.green('✓') : chalk.red('✗');
220
+ console.log(` ${icon} ${label} ${r.deploymentId} ${r.status} ${fmtRuntime(r.runtimeMs)}`);
221
+ }
222
+ if (failedSubmitCount) console.log(chalk.red(` ${failedSubmitCount} task(s) never submitted`));
223
+
224
+ console.log();
225
+ if (allSucceeded) {
226
+ console.log(chalk.green(` ✓ Slurm job complete (${taskIds.length}/${taskIds.length} task(s) succeeded)\n`));
227
+ } else {
228
+ const okCount = results.filter(r => r.succeeded).length;
229
+ console.error(chalk.red(` ✗ Slurm job incomplete (${okCount}/${taskIds.length} task(s) succeeded)\n`));
230
+ process.exitCode = 1;
231
+ }
232
+ }
@@ -1,4 +1,4 @@
1
- import { requireApiKey } from '../config.js';
1
+ import { ensureBadgrReady } from '../onboarding.js';
2
2
  import { callApi, listDeployments } from '../api.js';
3
3
  import { addDeployment, addReceipt, updateReceipt, generateReceiptId } from '../store.js';
4
4
  import { normalizeTier, callWithFallback } from '../fallback.js';
@@ -254,7 +254,7 @@ async function findRunningVllmEndpoint(config, modelId) {
254
254
  * no model is ever touched in that mode.
255
255
  */
256
256
  async function serveOpenWebUICommand(config, args, chalk) {
257
- requireApiKey(config);
257
+ config = await ensureBadgrReady(config, chalk);
258
258
  const { value: connect, rest: afterConnect } = extractFlag(args, '--connect');
259
259
  const { value: model, rest } = extractFlag(afterConnect, '--model');
260
260
 
@@ -396,7 +396,7 @@ export async function serveCommand(config, args, chalk) {
396
396
  return;
397
397
  }
398
398
 
399
- requireApiKey(config);
399
+ config = await ensureBadgrReady(config, chalk);
400
400
 
401
401
  // ── Validate flags early ───────────────────────────────────────────────────
402
402
  if (flags.count !== undefined && (!Number.isFinite(flags.count) || flags.count < 1)) {
@@ -36,7 +36,15 @@ export async function statusCommand(config, args, chalk) {
36
36
  }));
37
37
  }
38
38
 
39
- const running = deployments.filter(d => d.status === 'running' || d.status === 'provisioning');
39
+ // Filter OUT known-terminal statuses rather than filtering IN a fixed
40
+ // active list — the backend has more active states than just "running"/
41
+ // "provisioning" (also "starting", "queued"), and a status this command
42
+ // doesn't recognize yet must still show as active rather than being
43
+ // silently hidden (a live run found deployments stuck in "starting" that
44
+ // this filter used to drop entirely, reporting "Nothing running" while
45
+ // real billable resources kept running).
46
+ const TERMINAL_STATUSES = new Set(['completed', 'succeeded', 'success', 'failed', 'stopped']);
47
+ const running = deployments.filter(d => !TERMINAL_STATUSES.has(d.status));
40
48
 
41
49
  if (running.length === 0) {
42
50
  console.log(chalk.dim('\n Nothing running.\n'));
@@ -51,9 +59,9 @@ export async function statusCommand(config, args, chalk) {
51
59
  const type = d.workload_type === 'endpoint' ? 'endpoint' : 'job';
52
60
  const gpu = d.gpu_type || '—';
53
61
  const rate = d.cost_per_hour > 0 ? chalk.yellow(`$${d.cost_per_hour.toFixed(2)}/hr`) : '';
54
- const badge = d.status === 'provisioning'
55
- ? chalk.yellow('● starting')
56
- : chalk.green('● running');
62
+ const badge = d.status === 'running'
63
+ ? chalk.green('● running')
64
+ : chalk.yellow('● starting');
57
65
 
58
66
  console.log(` ${badge} ${chalk.bold(id)} ${type} ${gpu} ${rate}`);
59
67
 
@@ -0,0 +1,25 @@
1
+ import { launchCommand } from './launch.js';
2
+
3
+ /**
4
+ * badgr task "<description>" -- <command>
5
+ *
6
+ * The MVP version of "assign a task" — a thin label wrapper over
7
+ * `badgr launch . -- <command>`. It does not queue, schedule, or track
8
+ * anything beyond what `badgr launch` already does; the description is
9
+ * printed for the human and otherwise discarded (the receipt/status page
10
+ * still key off the deployment ID, same as any other launch).
11
+ */
12
+ export async function taskCommand(config, args, chalk) {
13
+ const description = args[0];
14
+ if (!description || description.startsWith('-')) {
15
+ console.error(chalk.red('\nUsage: badgr task "<description>" [badgr launch flags...] -- <command>\n'));
16
+ console.error(chalk.dim(' The description must come first, before any flags.'));
17
+ console.error(chalk.dim(' Example: badgr task "Run the Chromium tests and tell me what failed" --max-cost 1 -- npm run test:chromium'));
18
+ console.error(chalk.dim(' Example: badgr task "Fix the failing checkout test" --max-cost 1 -- claude -p "Fix the failing checkout test"\n'));
19
+ process.exitCode = 1;
20
+ return;
21
+ }
22
+
23
+ console.log(chalk.dim(` Task: ${description}`));
24
+ return launchCommand(config, ['.', ...args.slice(1)], chalk);
25
+ }
@@ -207,8 +207,10 @@ export async function testCommand(config, args, chalk) {
207
207
  process.stdout.write(chalk.dim(' Stopping deployment...'));
208
208
  let stopped = false;
209
209
  try {
210
- await terminateDeployment(config, depId);
211
- stopped = true;
210
+ // A 200 response only means deletion was requested, not confirmed —
211
+ // teardown_ok reflects whether the provider resource is actually gone.
212
+ const result = await terminateDeployment(config, depId);
213
+ stopped = result?.teardown_ok === 'ok';
212
214
  } catch { /* best-effort */ }
213
215
  process.stdout.write('\n');
214
216
  step(chalk, stopped, 'Billing stopped');
@@ -0,0 +1,65 @@
1
+ import { readFileSync, writeFileSync, mkdirSync, existsSync, chmodSync } from 'fs';
2
+ import { join } from 'path';
3
+ import { CONFIG_DIR } from './config.js';
4
+
5
+ export const CREDENTIALS_FILE = join(CONFIG_DIR, 'credentials.json');
6
+
7
+ // Env var each provider's credential is exposed under inside the launch VM.
8
+ export const PROVIDER_ENV_KEYS = {
9
+ anthropic: 'ANTHROPIC_API_KEY',
10
+ openai: 'OPENAI_API_KEY',
11
+ };
12
+
13
+ export const KNOWN_PROVIDERS = Object.keys(PROVIDER_ENV_KEYS);
14
+
15
+ /**
16
+ * Credential storage for `badgr connect <provider>`. This is a local file
17
+ * under ~/.badgr with owner-only permissions (chmod 600) — not the
18
+ * platform's real OS keychain (macOS Keychain / libsecret / Windows
19
+ * Credential Manager), since that needs a native module this CLI doesn't
20
+ * depend on yet. Documented as such; do not claim OS-keychain storage.
21
+ */
22
+ function readStore() {
23
+ if (!existsSync(CREDENTIALS_FILE)) return {};
24
+ try {
25
+ return JSON.parse(readFileSync(CREDENTIALS_FILE, 'utf8'));
26
+ } catch {
27
+ return {};
28
+ }
29
+ }
30
+
31
+ function writeStore(store) {
32
+ mkdirSync(CONFIG_DIR, { recursive: true });
33
+ // mode: 0o600 on writeFileSync itself sets permissions atomically at
34
+ // creation — passing it here (not just the chmodSync below) closes a
35
+ // brief window where a freshly-created file would otherwise sit at the
36
+ // default (world-readable) mode until the chmod call caught up.
37
+ writeFileSync(CREDENTIALS_FILE, JSON.stringify(store, null, 2), { mode: 0o600 });
38
+ try {
39
+ chmodSync(CREDENTIALS_FILE, 0o600);
40
+ } catch {
41
+ // best-effort — not all filesystems support chmod (e.g. some Windows setups)
42
+ }
43
+ }
44
+
45
+ export function getCredential(provider) {
46
+ return readStore()[provider] ?? null;
47
+ }
48
+
49
+ export function setCredential(provider, value) {
50
+ const store = readStore();
51
+ store[provider] = value;
52
+ writeStore(store);
53
+ }
54
+
55
+ export function removeCredential(provider) {
56
+ const store = readStore();
57
+ if (!(provider in store)) return false;
58
+ delete store[provider];
59
+ writeStore(store);
60
+ return true;
61
+ }
62
+
63
+ export function listCredentials() {
64
+ return Object.keys(readStore());
65
+ }
package/src/fallback.js CHANGED
@@ -44,9 +44,14 @@ export async function callWithFallback(endpoint, callOpts, buildBody, effectiveT
44
44
  const cmd = labels?.cmd ?? 'badgr run';
45
45
  const allowTier2Fallback = opts.allowTier2Fallback !== false; // default true
46
46
 
47
- // 130s: backend provisioning runs up to BADGR_PROVISION_TIMEOUT_SECONDS (default 120s).
47
+ // 220s: comfortably above backend's BADGR_PROVISION_TIMEOUT_SECONDS (default
48
+ // 200s, itself set above deployment_service.py's 180s routing-search
49
+ // deadline). Found live: the previous 130s value fired before the server's
50
+ // own (then-120s) deadline could, so the CLI reported "failed to submit"
51
+ // for jobs that had actually been created and were already billing —
52
+ // aborting client-side never cancels the server's in-flight work.
48
53
  async function attempt(body) {
49
- return callApi(endpoint, { method: 'POST', ...callOpts, body, timeoutMs: 130_000 });
54
+ return callApi(endpoint, { method: 'POST', ...callOpts, body, timeoutMs: 220_000 });
50
55
  }
51
56
 
52
57
  // Map server error codes → catalog keys and extract context for templates.
package/src/fanout.js ADDED
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Shared fan-out primitive: submit N independent /run tasks — at most
3
+ * `concurrency` in flight (submitted-but-not-yet-terminal) at once — monitor
4
+ * each to a terminal state, and roll the results up into one summary.
5
+ *
6
+ * "Run the same program many times with different inputs" is structurally
7
+ * identical whether the N tasks come from a Slurm `#SBATCH --array` range
8
+ * (see commands/sbatch.js) or a directory of input files (see `badgr batch
9
+ * run --fan-out`, commands/batch.js) — both funnel through this module so
10
+ * the submit/monitor/summarize logic, and the concurrency cap that bounds
11
+ * how much compute a single command can accidentally provision, exist
12
+ * exactly once.
13
+ */
14
+
15
+ const DEFAULT_CONCURRENCY = 5;
16
+
17
+ /**
18
+ * @param {Array<any>} tasks - opaque per-task descriptors, passed straight
19
+ * through to submitTask/monitorTask/onSubmitError.
20
+ * @param {(task) => Promise<{deploymentId, receiptId, ratePerHour}>} submitTask
21
+ * Submits one task. Rejecting marks it as a submit failure (never monitored).
22
+ * @param {(submitted: {task, deploymentId, receiptId, ratePerHour}) => Promise<{status, succeeded, runtimeMs}>} monitorTask
23
+ * Waits for one submitted task to reach a terminal state.
24
+ * @param {(task, err) => void} [onSubmitError] - called synchronously as each submit fails.
25
+ * @param {number} [concurrency] - max tasks with an active deployment (submitted,
26
+ * not yet terminal) at once. Defaults to 5 — fan-out must never submit an
27
+ * unbounded number of deployments from one command by default.
28
+ * @returns {Promise<{results: Array, allSucceeded: boolean, submittedCount: number, failedSubmitCount: number}>}
29
+ * `results` has one entry per input task, in the original task order, each
30
+ * either `{ task, error }` (never submitted) or
31
+ * `{ task, deploymentId, receiptId, status, succeeded, runtimeMs }`.
32
+ */
33
+ export async function runFanOut({ tasks, submitTask, monitorTask, onSubmitError = () => {}, concurrency = DEFAULT_CONCURRENCY }) {
34
+ const effectiveConcurrency = Math.max(1, concurrency || DEFAULT_CONCURRENCY);
35
+ const results = new Array(tasks.length);
36
+ let nextIndex = 0;
37
+
38
+ async function worker() {
39
+ while (nextIndex < tasks.length) {
40
+ const i = nextIndex++;
41
+ const task = tasks[i];
42
+ let submitted;
43
+ try {
44
+ submitted = await submitTask(task);
45
+ } catch (err) {
46
+ onSubmitError(task, err);
47
+ results[i] = { task, error: err };
48
+ continue;
49
+ }
50
+ const monitored = await monitorTask({ task, ...submitted });
51
+ results[i] = { task, ...submitted, ...monitored };
52
+ }
53
+ }
54
+
55
+ const workerCount = Math.min(effectiveConcurrency, tasks.length);
56
+ await Promise.all(Array.from({ length: workerCount }, () => worker()));
57
+
58
+ const submittedCount = results.filter(r => !r.error).length;
59
+ const failedSubmitCount = results.filter(r => r.error).length;
60
+ const allSucceeded = failedSubmitCount === 0 && results.every(r => r.succeeded);
61
+
62
+ return {
63
+ results,
64
+ allSucceeded,
65
+ submittedCount,
66
+ failedSubmitCount,
67
+ };
68
+ }
69
+
70
+ export { DEFAULT_CONCURRENCY };
@@ -0,0 +1,42 @@
1
+ import fs from 'fs';
2
+ import os from 'os';
3
+ import path from 'path';
4
+
5
+ function round1(n) {
6
+ return Math.round(n * 10) / 10;
7
+ }
8
+
9
+ /**
10
+ * Read-only disk/cache probe. Never creates or deletes anything.
11
+ */
12
+ export function detectDisk(fsImpl = fs) {
13
+ const cachePath = process.env.HF_HOME || process.env.HUGGINGFACE_HUB_CACHE || path.join(os.homedir(), '.cache', 'huggingface');
14
+ const result = {
15
+ cachePath,
16
+ cacheSource: process.env.HF_HOME ? 'HF_HOME' : process.env.HUGGINGFACE_HUB_CACHE ? 'HUGGINGFACE_HUB_CACHE' : 'default (~/.cache/huggingface)',
17
+ cacheExists: false,
18
+ writable: false,
19
+ freeGb: null,
20
+ totalGb: null,
21
+ };
22
+
23
+ result.cacheExists = fsImpl.existsSync(cachePath);
24
+ const probePath = result.cacheExists ? cachePath : path.dirname(cachePath);
25
+
26
+ try {
27
+ const stat = fsImpl.statfsSync(probePath);
28
+ result.freeGb = round1((stat.bavail * stat.bsize) / 1e9);
29
+ result.totalGb = round1((stat.blocks * stat.bsize) / 1e9);
30
+ } catch {
31
+ // statfs unsupported on this platform — leave free/total unknown
32
+ }
33
+
34
+ try {
35
+ fsImpl.accessSync(fsImpl.existsSync(probePath) ? probePath : os.tmpdir(), fs.constants.W_OK);
36
+ result.writable = true;
37
+ } catch {
38
+ result.writable = false;
39
+ }
40
+
41
+ return result;
42
+ }