badgr-cli 1.1.4 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -14
- package/package.json +2 -2
- package/src/api.js +28 -0
- package/src/badgr.js +1 -1
- package/src/commands/artifacts.js +5 -3
- package/src/commands/batch.js +6 -5
- package/src/commands/billing.js +3 -3
- package/src/commands/capacity.js +2 -2
- package/src/commands/deploy.js +2 -2
- package/src/commands/diagnose.js +276 -87
- package/src/commands/down.js +69 -14
- package/src/commands/embed.js +2 -2
- package/src/commands/heartbeat.js +3 -3
- package/src/commands/job.js +14 -3
- package/src/commands/login.js +4 -1
- package/src/commands/logs.js +35 -18
- package/src/commands/pull.js +4 -3
- package/src/commands/rerun.js +5 -3
- package/src/commands/restart.js +5 -3
- package/src/commands/run.js +5 -23
- package/src/commands/sbatch.js +2 -1
- package/src/commands/serve.js +145 -24
- package/src/commands/shell.js +2 -2
- package/src/commands/status.js +57 -13
- package/src/commands/test-run.js +2 -2
- package/src/commands/train.js +3 -3
- package/src/commands/transcribe.js +2 -2
- package/src/commands/up.js +2 -1
- package/src/config.js +19 -0
- package/src/deploymentLog.js +45 -0
- package/src/errors.js +7 -3
- package/src/fallback.js +51 -31
- package/src/onboarding.js +63 -4
- package/src/progress.js +5 -2
package/src/commands/down.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ensureLoggedInReady, withReauthRetry } from '../onboarding.js';
|
|
2
2
|
import { findDeployment, removeDeployment, addReceipt, generateReceiptId } from '../store.js';
|
|
3
|
-
import { terminateDeployment, listDeployments } from '../api.js';
|
|
3
|
+
import { terminateDeployment, listDeployments, adminTerminateDeployment } from '../api.js';
|
|
4
4
|
|
|
5
5
|
function formatRuntime(minutes) {
|
|
6
6
|
if (minutes < 60) return `${minutes}m`;
|
|
@@ -12,8 +12,9 @@ function formatRuntime(minutes) {
|
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
export async function downCommand(config, args, chalk) {
|
|
15
|
-
const hasAll
|
|
16
|
-
const hasYes
|
|
15
|
+
const hasAll = args.includes('--all');
|
|
16
|
+
const hasYes = args.includes('--yes') || args.includes('-y');
|
|
17
|
+
const isAdmin = args.includes('--admin');
|
|
17
18
|
|
|
18
19
|
if (hasAll) {
|
|
19
20
|
return _downAll(config, chalk, hasYes);
|
|
@@ -23,12 +24,43 @@ export async function downCommand(config, args, chalk) {
|
|
|
23
24
|
|
|
24
25
|
if (!idOrName) {
|
|
25
26
|
console.error(chalk.red('Usage: badgr down <deployment-id|name>'));
|
|
26
|
-
console.error(chalk.dim(' badgr down --all
|
|
27
|
-
console.error(chalk.dim(' badgr down --all --yes
|
|
27
|
+
console.error(chalk.dim(' badgr down --all stop everything in your own account'));
|
|
28
|
+
console.error(chalk.dim(' badgr down --all --yes stop everything without confirmation'));
|
|
29
|
+
console.error(chalk.dim(' badgr down <id> --admin admin-only: stop any account\'s deployment'));
|
|
28
30
|
return;
|
|
29
31
|
}
|
|
30
32
|
|
|
31
|
-
|
|
33
|
+
config = await ensureLoggedInReady(config, chalk);
|
|
34
|
+
|
|
35
|
+
// --admin bypasses per-user ownership (michaelhireitem@gmail.com only --
|
|
36
|
+
// see deployment_routes.py's DELETE /v1/admin/deployments/{id}) so a
|
|
37
|
+
// deployment found via `badgr status --admin` (owned by a different
|
|
38
|
+
// account) can actually be stopped. Local name lookup only applies to the
|
|
39
|
+
// caller's own deployments, so it's skipped here -- an admin shutdown
|
|
40
|
+
// always targets a real deployment_id from `badgr status --admin`.
|
|
41
|
+
if (isAdmin) {
|
|
42
|
+
process.stdout.write(chalk.dim(` [admin] Stopping ${idOrName}...`));
|
|
43
|
+
try {
|
|
44
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => adminTerminateDeployment(cfg, idOrName));
|
|
45
|
+
config = refreshed.config;
|
|
46
|
+
const dep = refreshed.result;
|
|
47
|
+
process.stdout.write('\n');
|
|
48
|
+
const teardownConfirmed = dep.teardown_ok === 'ok';
|
|
49
|
+
if (teardownConfirmed) {
|
|
50
|
+
console.log(chalk.green('\n✓ Stopped'));
|
|
51
|
+
console.log(chalk.green(` Billing ended for ${dep.user_email || dep.user_id || 'owner'}\n`));
|
|
52
|
+
} else {
|
|
53
|
+
console.log(chalk.yellow('\n⚠ Stop requested but not confirmed'));
|
|
54
|
+
console.log(chalk.yellow(` Resource may still be billing — retry \`badgr down ${idOrName} --admin\`\n`));
|
|
55
|
+
process.exitCode = 1;
|
|
56
|
+
}
|
|
57
|
+
} catch (err) {
|
|
58
|
+
process.stdout.write('\n');
|
|
59
|
+
console.error(chalk.red(`\n ✗ Could not stop deployment: ${err.message}\n`));
|
|
60
|
+
process.exitCode = 1;
|
|
61
|
+
}
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
32
64
|
|
|
33
65
|
const localDep = findDeployment(idOrName);
|
|
34
66
|
const deploymentId = localDep?.id ?? idOrName;
|
|
@@ -37,21 +69,29 @@ export async function downCommand(config, args, chalk) {
|
|
|
37
69
|
|
|
38
70
|
let dep;
|
|
39
71
|
try {
|
|
40
|
-
|
|
72
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => terminateDeployment(cfg, deploymentId));
|
|
73
|
+
config = refreshed.config;
|
|
74
|
+
dep = refreshed.result;
|
|
41
75
|
} catch (err) {
|
|
42
76
|
process.stdout.write('\n');
|
|
43
77
|
console.error(chalk.red(`\n ✗ Could not stop deployment: ${err.message}\n`));
|
|
78
|
+
process.exitCode = 1;
|
|
44
79
|
return;
|
|
45
80
|
}
|
|
46
81
|
|
|
47
82
|
process.stdout.write('\n');
|
|
48
83
|
|
|
49
|
-
//
|
|
84
|
+
// The backend owns billing truth. Its accrued value includes billable
|
|
85
|
+
// provisioning/startup and failed fallback attempts, while started_at
|
|
86
|
+
// alone cannot reconstruct those intervals.
|
|
50
87
|
const stoppedAt = dep.stopped_at ?? (Date.now() / 1000);
|
|
51
88
|
const startedAt = dep.started_at ?? stoppedAt;
|
|
52
89
|
const runtimeMin = Math.round((stoppedAt - startedAt) / 60);
|
|
53
90
|
const runtimeHr = (stoppedAt - startedAt) / 3600;
|
|
54
|
-
const
|
|
91
|
+
const timestampCost = (dep.cost_per_hour || 0) * runtimeHr;
|
|
92
|
+
const finalCost = dep.accrued_cost_usd == null
|
|
93
|
+
? timestampCost
|
|
94
|
+
: Number(dep.accrued_cost_usd);
|
|
55
95
|
|
|
56
96
|
// dep.teardown_ok reflects whether the provider resource was actually
|
|
57
97
|
// confirmed gone (not just that the DELETE call didn't throw) — a 200
|
|
@@ -79,6 +119,7 @@ export async function downCommand(config, args, chalk) {
|
|
|
79
119
|
} else {
|
|
80
120
|
console.log(chalk.yellow('\n⚠ Stop requested but not confirmed'));
|
|
81
121
|
console.log(chalk.yellow(` Resource may still be billing — check \`badgr status\` and retry \`badgr down ${deploymentId}\`\n`));
|
|
122
|
+
process.exitCode = 1;
|
|
82
123
|
}
|
|
83
124
|
console.log(` ${chalk.bold('Runtime:')} ${formatRuntime(runtimeMin)}`);
|
|
84
125
|
if (finalCost > 0) console.log(` ${chalk.bold('Final cost:')} $${finalCost.toFixed(4)}`);
|
|
@@ -86,13 +127,16 @@ export async function downCommand(config, args, chalk) {
|
|
|
86
127
|
}
|
|
87
128
|
|
|
88
129
|
async function _downAll(config, chalk, skipConfirm) {
|
|
89
|
-
|
|
130
|
+
config = await ensureLoggedInReady(config, chalk);
|
|
90
131
|
|
|
91
132
|
let result;
|
|
92
133
|
try {
|
|
93
|
-
|
|
134
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => listDeployments(cfg));
|
|
135
|
+
config = refreshed.config;
|
|
136
|
+
result = refreshed.result;
|
|
94
137
|
} catch (err) {
|
|
95
138
|
console.error(chalk.red(`\n ✗ Could not fetch deployments: ${err.message}\n`));
|
|
139
|
+
process.exitCode = 1;
|
|
96
140
|
return;
|
|
97
141
|
}
|
|
98
142
|
|
|
@@ -129,12 +173,15 @@ async function _downAll(config, chalk, skipConfirm) {
|
|
|
129
173
|
|
|
130
174
|
let stoppedCount = 0;
|
|
131
175
|
let unconfirmedCount = 0;
|
|
176
|
+
let errorCount = 0;
|
|
132
177
|
let totalCost = 0;
|
|
133
178
|
|
|
134
179
|
for (const dep of active) {
|
|
135
180
|
process.stdout.write(chalk.dim(` Stopping ${dep.deployment_id}...`));
|
|
136
181
|
try {
|
|
137
|
-
const
|
|
182
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => terminateDeployment(cfg, dep.deployment_id));
|
|
183
|
+
config = refreshed.config;
|
|
184
|
+
const stopped = refreshed.result;
|
|
138
185
|
const stoppedAt = stopped.stopped_at ?? (Date.now() / 1000);
|
|
139
186
|
const startedAt = stopped.started_at ?? stoppedAt;
|
|
140
187
|
const cost = (stopped.cost_per_hour || 0) * (stoppedAt - startedAt) / 3600;
|
|
@@ -163,6 +210,7 @@ async function _downAll(config, chalk, skipConfirm) {
|
|
|
163
210
|
});
|
|
164
211
|
} catch (err) {
|
|
165
212
|
process.stdout.write(chalk.red(` ✗ ${err.message}\n`));
|
|
213
|
+
errorCount++;
|
|
166
214
|
}
|
|
167
215
|
}
|
|
168
216
|
|
|
@@ -170,6 +218,13 @@ async function _downAll(config, chalk, skipConfirm) {
|
|
|
170
218
|
if (unconfirmedCount > 0) {
|
|
171
219
|
console.log(chalk.yellow(` ${unconfirmedCount} deployment(s) not confirmed stopped — may still be billing, check \`badgr status\`\n`));
|
|
172
220
|
}
|
|
221
|
+
if (errorCount > 0) {
|
|
222
|
+
console.log(chalk.red(` ${errorCount} deployment(s) failed to stop — check \`badgr status\` and retry\n`));
|
|
223
|
+
}
|
|
173
224
|
if (totalCost > 0) console.log(` Estimated total: $${totalCost.toFixed(4)}`);
|
|
174
|
-
|
|
225
|
+
if (unconfirmedCount > 0 || errorCount > 0) {
|
|
226
|
+
process.exitCode = 1;
|
|
227
|
+
} else {
|
|
228
|
+
console.log(chalk.green(' Billing ended\n'));
|
|
229
|
+
}
|
|
175
230
|
}
|
package/src/commands/embed.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* badgr embed BAAI/bge-large-en-v1.5 s3://bucket/corpus.jsonl
|
|
11
11
|
*/
|
|
12
12
|
import { readFileSync, existsSync, statSync } from 'fs';
|
|
13
|
-
import {
|
|
13
|
+
import { ensureBadgrReady } from '../onboarding.js';
|
|
14
14
|
import { addReceipt, generateReceiptId } from '../store.js';
|
|
15
15
|
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
16
16
|
import { monitorBatchJob, fmtRuntime } from '../batch.js';
|
|
@@ -90,7 +90,7 @@ export async function embedCommand(config, args, chalk) {
|
|
|
90
90
|
return;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
|
|
93
|
+
config = await ensureBadgrReady(config, chalk);
|
|
94
94
|
|
|
95
95
|
const resolved = resolveEmbedInput(input);
|
|
96
96
|
if (resolved.error) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ensureLoggedInReady, withReauthRetry } from '../onboarding.js';
|
|
2
2
|
import { findDeployment } from '../store.js';
|
|
3
3
|
import { heartbeatDeployment } from '../api.js';
|
|
4
4
|
|
|
@@ -20,13 +20,13 @@ export async function heartbeatCommand(config, args, chalk) {
|
|
|
20
20
|
return;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
config = await ensureLoggedInReady(config, chalk);
|
|
24
24
|
|
|
25
25
|
const localDep = findDeployment(idOrName);
|
|
26
26
|
const deploymentId = localDep?.id ?? idOrName;
|
|
27
27
|
|
|
28
28
|
try {
|
|
29
|
-
const result = await
|
|
29
|
+
const { result } = await withReauthRetry(config, chalk, cfg => heartbeatDeployment(cfg, deploymentId));
|
|
30
30
|
console.log(chalk.green(` ✓ Heartbeat recorded for ${deploymentId}`));
|
|
31
31
|
if (result.last_activity_at) {
|
|
32
32
|
console.log(chalk.dim(` last_activity_at: ${new Date(result.last_activity_at * 1000).toISOString()}`));
|
package/src/commands/job.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { callApi } from '../api.js';
|
|
2
2
|
import { requireApiKey } from '../config.js';
|
|
3
|
+
import { ensureBadgrReady } from '../onboarding.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* badgr job <agent> "<instruction>" --check "npm test"
|
|
@@ -115,9 +116,19 @@ export async function jobCommand(config, args, chalk) {
|
|
|
115
116
|
export async function runJob(config, opts, chalk) {
|
|
116
117
|
const { agent: agentName, instruction: instructionText, check } = opts;
|
|
117
118
|
|
|
118
|
-
//
|
|
119
|
-
|
|
120
|
-
|
|
119
|
+
// Just-in-time login — same flow badgr run/serve/comfyui trigger: if not
|
|
120
|
+
// logged in, opens a browser login link and blocks right here until it
|
|
121
|
+
// completes, then this function keeps running with the now-populated
|
|
122
|
+
// config — no separate `badgr login` step, and the job's own progress/
|
|
123
|
+
// logs print in this same terminal once auth resolves. A dry run only
|
|
124
|
+
// previews the plan and never provisions/spends, so it keeps the old
|
|
125
|
+
// fail-fast behavior instead of opening a browser (matches run.js).
|
|
126
|
+
if (opts.dryRun) {
|
|
127
|
+
requireApiKey(config, chalk);
|
|
128
|
+
} else {
|
|
129
|
+
config = await ensureBadgrReady(config, chalk);
|
|
130
|
+
}
|
|
131
|
+
const apiKey = config.apiKey;
|
|
121
132
|
|
|
122
133
|
const provider = opts.provider ?? null;
|
|
123
134
|
const model = opts.model ?? null;
|
package/src/commands/login.js
CHANGED
|
@@ -51,7 +51,10 @@ export async function loginCommand(chalk, saveConfigFn, args = []) {
|
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
// Keep the configured API origin. Local development needs the browser
|
|
55
|
+
// to complete the session on localhost, not against the production
|
|
56
|
+
// backend's unrelated CLI-session store.
|
|
57
|
+
const config = await ensureLoggedIn({ ...DEFAULTS, baseUrl: existing.baseUrl }, chalk);
|
|
55
58
|
console.log(chalk.dim(` Run ${chalk.cyan('badgr run python train.py')} to launch your first job.\n`));
|
|
56
59
|
return config;
|
|
57
60
|
}
|
package/src/commands/logs.js
CHANGED
|
@@ -1,18 +1,16 @@
|
|
|
1
1
|
import { findDeployment, listDeployments } from '../store.js';
|
|
2
|
-
import {
|
|
2
|
+
import { ensureLoggedInReady, withReauthRetry } from '../onboarding.js';
|
|
3
3
|
import { getDeploymentLogs, callApi } from '../api.js';
|
|
4
|
+
import { isMetaLogLine, isErrorLogLine } from '../deploymentLog.js';
|
|
4
5
|
|
|
5
6
|
const TERMINAL_STATUSES = new Set(['succeeded', 'completed', 'failed', 'stopped', 'terminated']);
|
|
6
7
|
const FOLLOW_POLL_MS = 3000;
|
|
7
8
|
|
|
8
|
-
// Lines that carry structured metadata shown elsewhere (status bar in `badgr run`).
|
|
9
|
-
const LOG_META_RE = /^\[dep-[^\]]+\] (status|gpu|region|endpoint|cost|receipt|provider_status|uptime)=/;
|
|
10
|
-
|
|
11
9
|
export async function logsCommand(config, args, chalk) {
|
|
12
10
|
const idOrName = args.find(a => !a.startsWith('--'));
|
|
13
11
|
const follow = args.includes('--follow') || args.includes('-f');
|
|
14
12
|
|
|
15
|
-
|
|
13
|
+
config = await ensureLoggedInReady(config, chalk);
|
|
16
14
|
|
|
17
15
|
if (!idOrName) {
|
|
18
16
|
const deps = listDeployments();
|
|
@@ -35,14 +33,17 @@ export async function logsCommand(config, args, chalk) {
|
|
|
35
33
|
// Fetch and print initial batch of logs.
|
|
36
34
|
const seen = new Set();
|
|
37
35
|
try {
|
|
38
|
-
const data = await
|
|
36
|
+
const { config: refreshed, result: data } = await withReauthRetry(
|
|
37
|
+
config, chalk, cfg => getDeploymentLogs(cfg, deploymentId),
|
|
38
|
+
);
|
|
39
|
+
config = refreshed;
|
|
39
40
|
const lines = data?.logs ?? [];
|
|
40
41
|
if (lines.length === 0 && !follow) {
|
|
41
42
|
console.log(chalk.dim(' No log lines available yet.\n'));
|
|
42
43
|
} else {
|
|
43
44
|
for (const line of lines) {
|
|
44
45
|
seen.add(line);
|
|
45
|
-
if (!
|
|
46
|
+
if (!isMetaLogLine(line)) console.log(` ${chalk.dim(line)}`);
|
|
46
47
|
}
|
|
47
48
|
}
|
|
48
49
|
} catch (err) {
|
|
@@ -68,26 +69,42 @@ export async function logsCommand(config, args, chalk) {
|
|
|
68
69
|
|
|
69
70
|
let status = null;
|
|
70
71
|
try {
|
|
71
|
-
const dep = await
|
|
72
|
-
|
|
73
|
-
baseUrl:
|
|
74
|
-
|
|
72
|
+
const { config: refreshed, result: dep } = await withReauthRetry(
|
|
73
|
+
config, chalk,
|
|
74
|
+
cfg => callApi(`/deployments/${deploymentId}`, { apiKey: cfg.apiKey, baseUrl: cfg.baseUrl }),
|
|
75
|
+
);
|
|
76
|
+
config = refreshed;
|
|
75
77
|
status = dep?.status ?? null;
|
|
76
|
-
} catch {
|
|
77
|
-
//
|
|
78
|
+
} catch (err) {
|
|
79
|
+
// withReauthRetry already attempted exactly one bounded reauth for a
|
|
80
|
+
// 401/403 -- if it still failed, don't loop forever pretending to
|
|
81
|
+
// follow: fail visibly instead of looking silently hung.
|
|
82
|
+
if (err.httpStatus === 401 || err.httpStatus === 403) {
|
|
83
|
+
console.error(chalk.red(`\n ✗ AUTH_FAILED: ${err.message}\n`));
|
|
84
|
+
process.exitCode = 1;
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
// network blip -- keep following
|
|
78
88
|
}
|
|
79
89
|
|
|
80
90
|
try {
|
|
81
|
-
const data = await
|
|
91
|
+
const { config: refreshed, result: data } = await withReauthRetry(
|
|
92
|
+
config, chalk, cfg => getDeploymentLogs(cfg, deploymentId),
|
|
93
|
+
);
|
|
94
|
+
config = refreshed;
|
|
82
95
|
for (const line of (data?.logs ?? [])) {
|
|
83
96
|
if (seen.has(line)) continue;
|
|
84
97
|
seen.add(line);
|
|
85
|
-
if (!
|
|
86
|
-
|
|
87
|
-
console.log(` ${isErr ? chalk.red(line) : chalk.dim(line)}`);
|
|
98
|
+
if (!isMetaLogLine(line)) {
|
|
99
|
+
console.log(` ${isErrorLogLine(line) ? chalk.red(line) : chalk.dim(line)}`);
|
|
88
100
|
}
|
|
89
101
|
}
|
|
90
|
-
} catch {
|
|
102
|
+
} catch (err) {
|
|
103
|
+
if (err.httpStatus === 401 || err.httpStatus === 403) {
|
|
104
|
+
console.error(chalk.red(`\n ✗ AUTH_FAILED: ${err.message}\n`));
|
|
105
|
+
process.exitCode = 1;
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
91
108
|
// logs endpoint temporarily unavailable
|
|
92
109
|
}
|
|
93
110
|
|
package/src/commands/pull.js
CHANGED
|
@@ -2,7 +2,7 @@ import fs from 'fs';
|
|
|
2
2
|
import os from 'os';
|
|
3
3
|
import path from 'path';
|
|
4
4
|
import { spawnSync } from 'child_process';
|
|
5
|
-
import {
|
|
5
|
+
import { ensureLoggedInReady, withReauthRetry } from '../onboarding.js';
|
|
6
6
|
import { downloadAndExtractArtifact } from '../artifactDownload.js';
|
|
7
7
|
|
|
8
8
|
function runGit(args, options = {}) {
|
|
@@ -67,7 +67,7 @@ function findPatchFile(dir) {
|
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
export async function pullCommand(config, args, chalk) {
|
|
70
|
-
|
|
70
|
+
config = await ensureLoggedInReady(config, chalk);
|
|
71
71
|
const { deploymentId, flags } = parsePullArgs(args);
|
|
72
72
|
if (!deploymentId) {
|
|
73
73
|
console.error(chalk.red('\n Usage: badgr pull <deployment-id> [--diff-only|--branch|--yes]\n'));
|
|
@@ -84,7 +84,8 @@ export async function pullCommand(config, args, chalk) {
|
|
|
84
84
|
|
|
85
85
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), `badgr-pull-${deploymentId}-`));
|
|
86
86
|
try {
|
|
87
|
-
await
|
|
87
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => downloadAndExtractArtifact(cfg, deploymentId, tmp));
|
|
88
|
+
config = refreshed.config;
|
|
88
89
|
const patchFile = findPatchFile(tmp);
|
|
89
90
|
if (!patchFile) {
|
|
90
91
|
// Not every run changes code — a test/eval command with --artifacts
|
package/src/commands/rerun.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ensureBadgrReady, withReauthRetry } from '../onboarding.js';
|
|
2
2
|
import { findDeployment, addDeployment, addReceipt, generateReceiptId } from '../store.js';
|
|
3
3
|
import { rerunDeployment } from '../api.js';
|
|
4
4
|
|
|
@@ -18,7 +18,7 @@ export async function rerunCommand(config, args, chalk) {
|
|
|
18
18
|
return;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
config = await ensureBadgrReady(config, chalk);
|
|
22
22
|
|
|
23
23
|
const localDep = findDeployment(idOrName);
|
|
24
24
|
const deploymentId = localDep?.id ?? idOrName;
|
|
@@ -27,7 +27,9 @@ export async function rerunCommand(config, args, chalk) {
|
|
|
27
27
|
|
|
28
28
|
let dep;
|
|
29
29
|
try {
|
|
30
|
-
|
|
30
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => rerunDeployment(cfg, deploymentId));
|
|
31
|
+
config = refreshed.config;
|
|
32
|
+
dep = refreshed.result;
|
|
31
33
|
} catch (err) {
|
|
32
34
|
process.stdout.write('\n');
|
|
33
35
|
console.error(chalk.red(`\n ✗ Could not rerun deployment: ${err.message}\n`));
|
package/src/commands/restart.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ensureBadgrReady, withReauthRetry } from '../onboarding.js';
|
|
2
2
|
import { findDeployment, removeDeployment, addDeployment, addReceipt, generateReceiptId } from '../store.js';
|
|
3
3
|
import { restartDeployment } from '../api.js';
|
|
4
4
|
|
|
@@ -17,7 +17,7 @@ export async function restartCommand(config, args, chalk) {
|
|
|
17
17
|
return;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
config = await ensureBadgrReady(config, chalk);
|
|
21
21
|
|
|
22
22
|
const localDep = findDeployment(idOrName);
|
|
23
23
|
const deploymentId = localDep?.id ?? idOrName;
|
|
@@ -26,7 +26,9 @@ export async function restartCommand(config, args, chalk) {
|
|
|
26
26
|
|
|
27
27
|
let dep;
|
|
28
28
|
try {
|
|
29
|
-
|
|
29
|
+
const refreshed = await withReauthRetry(config, chalk, cfg => restartDeployment(cfg, deploymentId));
|
|
30
|
+
config = refreshed.config;
|
|
31
|
+
dep = refreshed.result;
|
|
30
32
|
} catch (err) {
|
|
31
33
|
process.stdout.write('\n');
|
|
32
34
|
console.error(chalk.red(`\n ✗ Could not restart deployment: ${err.message}\n`));
|
package/src/commands/run.js
CHANGED
|
@@ -13,6 +13,7 @@ import { detectWorkload, workloadTypeLabel } from '../detect.js';
|
|
|
13
13
|
import { ensureBadgrReady } from '../onboarding.js';
|
|
14
14
|
import { VM_CLASSES, parseGbSize } from '../spec.js';
|
|
15
15
|
import { parseEnvFlag } from '../envFlag.js';
|
|
16
|
+
import { isMetaLogLine, isErrorLogLine, parseProviderStatusLine } from '../deploymentLog.js';
|
|
16
17
|
|
|
17
18
|
function vmClassLine(sizeKey) {
|
|
18
19
|
const vmClass = VM_CLASSES[sizeKey];
|
|
@@ -177,23 +178,6 @@ export function classifyFailure(finalStatus, exitCode) {
|
|
|
177
178
|
return null;
|
|
178
179
|
}
|
|
179
180
|
|
|
180
|
-
// Lines the log stream never needs to print — we surface them in the status bar instead.
|
|
181
|
-
const LOG_META_RE = /^\[dep-[^\]]+\] (status|gpu|region|cost|receipt|provider_status|uptime)=/;
|
|
182
|
-
|
|
183
|
-
// Extract structured values from provider status lines so we can show them nicely.
|
|
184
|
-
function parseProviderLine(line) {
|
|
185
|
-
const gpuUtil = line.match(/\bgpu_util=([\d.]+)%/);
|
|
186
|
-
const cpuUtil = line.match(/\bcpu_util=([\d.]+)%/);
|
|
187
|
-
const ssh = line.match(/\bssh=(\S+)/);
|
|
188
|
-
const provSt = line.match(/\bprovider_status=(\S+)/);
|
|
189
|
-
return {
|
|
190
|
-
gpuUtil: gpuUtil ? parseFloat(gpuUtil[1]) : null,
|
|
191
|
-
cpuUtil: cpuUtil ? parseFloat(cpuUtil[1]) : null,
|
|
192
|
-
ssh: ssh ? ssh[1] : null,
|
|
193
|
-
providerStatus: provSt ? provSt[1] : null,
|
|
194
|
-
};
|
|
195
|
-
}
|
|
196
|
-
|
|
197
181
|
// Wait for status to leave 'starting'/'queued'/'provisioning'.
|
|
198
182
|
// Returns the dep once it leaves startup states (or the last known state on timeout).
|
|
199
183
|
async function waitForRunning(config, depId, chalk) {
|
|
@@ -354,7 +338,7 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
354
338
|
const lines = logData?.logs ?? [];
|
|
355
339
|
|
|
356
340
|
for (const line of lines) {
|
|
357
|
-
const parsed =
|
|
341
|
+
const parsed = parseProviderStatusLine(line);
|
|
358
342
|
if (parsed.gpuUtil !== null) gpuUtil = parsed.gpuUtil;
|
|
359
343
|
if (parsed.cpuUtil !== null) cpuUtil = parsed.cpuUtil;
|
|
360
344
|
|
|
@@ -367,12 +351,10 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
367
351
|
|
|
368
352
|
if (seenContent.has(line)) continue;
|
|
369
353
|
seenContent.add(line);
|
|
370
|
-
if (
|
|
371
|
-
if (/\b(gpu_util|cpu_util|provider_status|uptime)=/.test(line)) continue;
|
|
354
|
+
if (isMetaLogLine(line)) continue;
|
|
372
355
|
|
|
373
|
-
const isErrorLine = /^error\b/i.test(line) || /Error response from daemon/i.test(line);
|
|
374
356
|
stopTicker();
|
|
375
|
-
console.log(` ${
|
|
357
|
+
console.log(` ${isErrorLogLine(line) ? chalk.red(line) : chalk.dim(line)}`);
|
|
376
358
|
startTicker();
|
|
377
359
|
}
|
|
378
360
|
} catch {
|
|
@@ -1048,7 +1030,7 @@ export async function runCommand(config, args, chalk, opts = {}) {
|
|
|
1048
1030
|
|
|
1049
1031
|
if (detach) {
|
|
1050
1032
|
console.log(`\n ${chalk.bold('Logs:')} ${dep.logs_url || `badgr logs ${dep.deployment_id}`}`);
|
|
1051
|
-
console.log(chalk.dim(`\n Detached.
|
|
1033
|
+
console.log(chalk.dim(`\n Detached. Watch it from your terminal: badgr logs ${dep.deployment_id} --follow\n`));
|
|
1052
1034
|
return;
|
|
1053
1035
|
}
|
|
1054
1036
|
|
package/src/commands/sbatch.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { resolve } from 'path';
|
|
17
17
|
import { requireApiKey } from '../config.js';
|
|
18
|
+
import { ensureBadgrReady } from '../onboarding.js';
|
|
18
19
|
import { loadSlurmScript, envForArrayTask, SlurmParseError } from '../slurm.js';
|
|
19
20
|
import { addReceipt, updateReceipt, generateReceiptId, selectedComputeFromDeployment } from '../store.js';
|
|
20
21
|
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
@@ -164,7 +165,7 @@ export async function sbatchCommand(config, args, chalk) {
|
|
|
164
165
|
return;
|
|
165
166
|
}
|
|
166
167
|
|
|
167
|
-
|
|
168
|
+
config = await ensureBadgrReady(config, chalk);
|
|
168
169
|
|
|
169
170
|
const arrayJobId = isArray ? generateReceiptId() : null;
|
|
170
171
|
|