badgr-cli 1.0.43 → 1.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/package.json +1 -1
- package/src/catalog.js +15 -0
- package/src/commands/run.js +73 -66
- package/src/commands/serve.js +166 -114
- package/src/progress.js +42 -0
- package/tests/serve-lifecycle.test.js +95 -2
package/README.md
CHANGED
|
@@ -72,7 +72,19 @@ badgr down <deployment-id>
|
|
|
72
72
|
| `badgr transcribe <audio>` | Whisper transcription, print transcript |
|
|
73
73
|
| `badgr embed <model> <input>` | Text embeddings, output JSONL |
|
|
74
74
|
|
|
75
|
-
`badgr serve --list-aliases` lists the
|
|
75
|
+
`badgr serve --list-aliases` lists the tested vLLM model routes (`qwen-7b`, `llama-8b`, `qwen-coder-7b`) that can be used in place of a full model ID.
|
|
76
|
+
|
|
77
|
+
### Model support levels
|
|
78
|
+
|
|
79
|
+
`badgr serve qwen-7b` is the happy path — a tested route with no extra setup. `badgr serve` also accepts any other model ID or a custom container, with the CLI honest (but brief) about what that means:
|
|
80
|
+
|
|
81
|
+
| Level | What it means |
|
|
82
|
+
|-------|---------------|
|
|
83
|
+
| **Tested route** (`badgr serve qwen-7b`) | One of the aliases above — tested and officially supported. No extra caveats printed. |
|
|
84
|
+
| **Best-effort Hugging Face model** (`badgr serve <org>/<model>`) | Any other Hugging Face model ID. Badgr will try a compatible route — not a guarantee every model works. |
|
|
85
|
+
| **Custom container** (`badgr serve --image ...`) | You own the server behavior; Badgr manages runtime, logs, spend caps, teardown, and the receipt. |
|
|
86
|
+
|
|
87
|
+
Gated Hugging Face models (e.g. Llama, Gemma) may need `--env HF_TOKEN=$HF_TOKEN`. Badgr doesn't warn about this up front — it only prints the hint if the deployment actually fails to start, so tested and working launches stay short.
|
|
76
88
|
|
|
77
89
|
---
|
|
78
90
|
|
package/package.json
CHANGED
package/src/catalog.js
CHANGED
|
@@ -471,6 +471,21 @@ export const BLESSED_VLLM_MODELS = {
|
|
|
471
471
|
},
|
|
472
472
|
};
|
|
473
473
|
|
|
474
|
+
// Hugging Face org/model prefixes that are known to gate access behind a license
|
|
475
|
+
// click-through. Used only to print a helpful HF_TOKEN hint — not exhaustive.
|
|
476
|
+
const GATED_MODEL_PREFIXES = [
|
|
477
|
+
'meta-llama/',
|
|
478
|
+
'google/gemma',
|
|
479
|
+
'mistralai/Mistral-Large',
|
|
480
|
+
'mistralai/Mixtral-8x22B',
|
|
481
|
+
];
|
|
482
|
+
|
|
483
|
+
/** Best-effort heuristic: is this HF model ID likely to require HF_TOKEN? */
|
|
484
|
+
export function isLikelyGatedModel(modelId) {
|
|
485
|
+
if (!modelId) return false;
|
|
486
|
+
return GATED_MODEL_PREFIXES.some(prefix => modelId.startsWith(prefix));
|
|
487
|
+
}
|
|
488
|
+
|
|
474
489
|
/** Blessed ComfyUI workflows accepted by `POST /v1/jobs` comfy.batch. */
|
|
475
490
|
export const BLESSED_COMFY_WORKFLOWS = {
|
|
476
491
|
'sdxl-basic': {
|
package/src/commands/run.js
CHANGED
|
@@ -8,6 +8,7 @@ import { addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
|
8
8
|
import { normalizeTier, callWithFallback, HIGH_RATE_THRESHOLD } from '../fallback.js';
|
|
9
9
|
import { formatCliError } from '../errors.js';
|
|
10
10
|
import { TEMPLATE_MAP, buildTemplateFlags, parseTemplateOverrides } from '../catalog.js';
|
|
11
|
+
import { stage as _stage, stageDone as _stageDone, writeBlock as _writeBlock, clearBlock as _clearBlock, renderLiveBlock as _renderLiveBlock } from '../progress.js';
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* Flow 1 — local project (primary):
|
|
@@ -105,6 +106,20 @@ const HEARTBEAT_WARN_POLLS = 3;
|
|
|
105
106
|
const HEARTBEAT_KILL_POLLS = 15;
|
|
106
107
|
const STARTUP_STATES = new Set(['queued', 'provisioning', 'starting']);
|
|
107
108
|
|
|
109
|
+
// Shared closing block for every terminal path (success, failure, cap, heartbeat loss) —
|
|
110
|
+
// always states exit code (when known), whether teardown/billing succeeded, and how to
|
|
111
|
+
// pull the receipt, so the user is never left guessing what happened.
|
|
112
|
+
function _printFinalInfo(chalk, { exitCode = undefined, teardownOk, jobId, rcptId, logsAvailable = true }) {
|
|
113
|
+
if (exitCode !== null && exitCode !== undefined) {
|
|
114
|
+
console.log(` ${chalk.bold('Exit code:')} ${exitCode !== 0 ? chalk.red(exitCode) : chalk.green(exitCode)}`);
|
|
115
|
+
}
|
|
116
|
+
console.log(` ${chalk.bold('Teardown:')} ${teardownOk ? chalk.green('succeeded') : chalk.red(`failed — run: badgr down ${jobId}`)}`);
|
|
117
|
+
console.log(` ${chalk.bold('Billing:')} ${teardownOk ? 'stopped' : 'unconfirmed — check receipt'}`);
|
|
118
|
+
if (logsAvailable) console.log(` ${chalk.bold('Logs:')} badgr logs ${jobId}`);
|
|
119
|
+
console.log(` ${chalk.bold('Job ID:')} ${jobId}`);
|
|
120
|
+
console.log(` ${chalk.bold('Receipt:')} badgr receipts ${rcptId}`);
|
|
121
|
+
}
|
|
122
|
+
|
|
108
123
|
export function classifyFailure(finalStatus, exitCode) {
|
|
109
124
|
if (finalStatus === 'failed' && (exitCode === null || exitCode === undefined)) return 'infrastructure';
|
|
110
125
|
if (exitCode !== null && exitCode !== undefined && exitCode !== 0) return 'customer_code';
|
|
@@ -128,24 +143,6 @@ function parseProviderLine(line) {
|
|
|
128
143
|
};
|
|
129
144
|
}
|
|
130
145
|
|
|
131
|
-
function renderStatusBar(chalk, { elapsedMs, ratePerHour, gpuUtil, cpuUtil, maxRuntimeMs, maxCost }) {
|
|
132
|
-
const spent = ratePerHour * (elapsedMs / 3_600_000);
|
|
133
|
-
const parts = [`⏱ ${fmtRuntime(elapsedMs)}`];
|
|
134
|
-
if (ratePerHour > 0) parts.push(`$${spent.toFixed(4)} spent`);
|
|
135
|
-
if (gpuUtil !== null) parts.push(`GPU ${gpuUtil.toFixed(0)}%`);
|
|
136
|
-
if (cpuUtil !== null) parts.push(`CPU ${cpuUtil.toFixed(0)}%`);
|
|
137
|
-
if (maxRuntimeMs) {
|
|
138
|
-
const left = Math.max(0, maxRuntimeMs - elapsedMs);
|
|
139
|
-
parts.push(`${fmtRuntime(left)} left`);
|
|
140
|
-
}
|
|
141
|
-
if (maxCost && ratePerHour > 0) {
|
|
142
|
-
const budgetLeft = Math.max(0, maxCost - spent);
|
|
143
|
-
parts.push(`$${budgetLeft.toFixed(4)} budget left`);
|
|
144
|
-
}
|
|
145
|
-
parts.push('Ctrl+C to stop');
|
|
146
|
-
return chalk.dim(' ' + parts.join(' • '));
|
|
147
|
-
}
|
|
148
|
-
|
|
149
146
|
// Wait for status to leave 'starting'/'queued'/'provisioning'.
|
|
150
147
|
// Returns the dep once it leaves startup states (or the last known state on timeout).
|
|
151
148
|
async function waitForRunning(config, depId, chalk) {
|
|
@@ -200,7 +197,7 @@ async function waitForRunning(config, depId, chalk) {
|
|
|
200
197
|
});
|
|
201
198
|
}
|
|
202
199
|
|
|
203
|
-
async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost = null, ratePerHour = 0, onTeardown, isShuttingDown }) {
|
|
200
|
+
async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost = null, ratePerHour = 0, onTeardown, isShuttingDown, stageLine }) {
|
|
204
201
|
const TERMINAL = new Set(['stopped', 'failed', 'completed', 'succeeded']);
|
|
205
202
|
const POLL_MS = 4000;
|
|
206
203
|
let seenContent = new Set();
|
|
@@ -209,7 +206,8 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
209
206
|
let gpuUtil = null;
|
|
210
207
|
let cpuUtil = null;
|
|
211
208
|
let sshShown = false;
|
|
212
|
-
let
|
|
209
|
+
let statusWord = 'running';
|
|
210
|
+
let blockLines = 0;
|
|
213
211
|
const startMs = Date.now();
|
|
214
212
|
|
|
215
213
|
// tearing: guards against double-teardown for cap/heartbeat paths within this function.
|
|
@@ -218,17 +216,17 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
218
216
|
let tickerInterval = null;
|
|
219
217
|
const startTicker = () => {
|
|
220
218
|
if (tickerInterval) return;
|
|
221
|
-
statusBarActive = true;
|
|
222
219
|
tickerInterval = setInterval(() => {
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
220
|
+
const elapsedSec = Math.round((Date.now() - startMs) / 1000);
|
|
221
|
+
const spend = ratePerHour * (elapsedSec / 3600);
|
|
222
|
+
blockLines = _writeBlock(blockLines, _renderLiveBlock(chalk, {
|
|
223
|
+
stageLine, elapsedSec, statusWord, spend, id: depId,
|
|
224
|
+
}));
|
|
227
225
|
}, 1000);
|
|
228
226
|
};
|
|
229
227
|
const stopTicker = () => {
|
|
230
228
|
if (tickerInterval) { clearInterval(tickerInterval); tickerInterval = null; }
|
|
231
|
-
if (
|
|
229
|
+
if (blockLines > 0) { _clearBlock(blockLines); blockLines = 0; }
|
|
232
230
|
};
|
|
233
231
|
|
|
234
232
|
try {
|
|
@@ -271,6 +269,7 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
271
269
|
consecutiveErrs++;
|
|
272
270
|
if (lastStatus === 'running') {
|
|
273
271
|
const lostSec = Math.round((consecutiveErrs * POLL_MS) / 1000);
|
|
272
|
+
if (consecutiveErrs >= HEARTBEAT_WARN_POLLS) statusWord = 'stuck';
|
|
274
273
|
if (consecutiveErrs === HEARTBEAT_WARN_POLLS) {
|
|
275
274
|
stopTicker();
|
|
276
275
|
console.log(chalk.yellow(`\n ⚠ No heartbeat for ${lostSec}s — cloud machine may be unresponsive`));
|
|
@@ -293,6 +292,7 @@ async function attachToJob(config, depId, { chalk, maxRuntimeMs = null, maxCost
|
|
|
293
292
|
}
|
|
294
293
|
lastStatus = status;
|
|
295
294
|
}
|
|
295
|
+
if (status === 'running') statusWord = seenContent.size === 0 ? 'no logs yet' : 'running';
|
|
296
296
|
|
|
297
297
|
try {
|
|
298
298
|
const logData = await callApi(`/deployments/${depId}/logs`, {
|
|
@@ -605,14 +605,14 @@ export async function runCommand(config, args, chalk) {
|
|
|
605
605
|
if (flags.cmd) console.log(` ${chalk.bold('Command:')} ${flags.cmd}`);
|
|
606
606
|
if (command) console.log(` ${chalk.bold('Command:')} ${command.join(' ')}`);
|
|
607
607
|
if (image) console.log(` ${chalk.bold('Image:')} ${image}`);
|
|
608
|
-
if (gpu)
|
|
609
|
-
else console.log(` ${chalk.bold('GPU:')} ${chalk.dim('auto')}`);
|
|
608
|
+
if (flags.gpu) console.log(` ${chalk.bold('GPU:')} ${gpu}`);
|
|
610
609
|
if (flags.minVram) console.log(` ${chalk.bold('Min VRAM:')} ${flags.minVram} GB`);
|
|
611
610
|
const runtimeLabel = isDefaultRuntime
|
|
612
|
-
? `${effectiveMaxRuntime}min ${chalk.dim('(default — use --max-runtime N to override)')}`
|
|
613
|
-
: `${effectiveMaxRuntime}min`;
|
|
611
|
+
? `${effectiveMaxRuntime} min ${chalk.dim('(default — use --max-runtime N to override)')}`
|
|
612
|
+
: `${effectiveMaxRuntime} min`;
|
|
613
|
+
console.log(` ${chalk.bold('Max cost:')} ${maxCost ? `$${maxCost.toFixed(2)}` : chalk.dim('none')}`);
|
|
614
614
|
console.log(` ${chalk.bold('Max runtime:')} ${runtimeLabel}`);
|
|
615
|
-
|
|
615
|
+
console.log(` ${chalk.bold('Auto-stop:')} ${maxCost ? 'enabled' : chalk.yellow('disabled — stop manually with badgr down')}`);
|
|
616
616
|
if (flags.maxPrice) console.log(` ${chalk.bold('Max price:')} $${flags.maxPrice.toFixed(2)}/hr`);
|
|
617
617
|
if (detach) console.log(` ${chalk.dim('(detached — returns immediately)')}`);
|
|
618
618
|
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${flags.env.join(', ')}`);
|
|
@@ -636,6 +636,10 @@ export async function runCommand(config, args, chalk) {
|
|
|
636
636
|
}
|
|
637
637
|
}
|
|
638
638
|
|
|
639
|
+
// Total stages: local-path runs get a "Preparing upload" stage the others don't.
|
|
640
|
+
const STAGE_TOTAL = isLocalPath ? 5 : 4;
|
|
641
|
+
let stageN = 1;
|
|
642
|
+
|
|
639
643
|
// ── Upload local project zip (Flow 1) ─────────────────────────────────────
|
|
640
644
|
let codeUri = null;
|
|
641
645
|
if (isLocalPath) {
|
|
@@ -646,9 +650,10 @@ export async function runCommand(config, args, chalk) {
|
|
|
646
650
|
process.exitCode = 1;
|
|
647
651
|
return;
|
|
648
652
|
}
|
|
653
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Preparing upload...')));
|
|
654
|
+
stageN++;
|
|
649
655
|
}
|
|
650
656
|
|
|
651
|
-
console.log(chalk.dim(' Finding suitable capacity...'));
|
|
652
657
|
if (process.env.BADGR_DEBUG === '1' || process.env.BADGR_DEBUG === 'true') {
|
|
653
658
|
console.log(chalk.dim(` API: ${config.baseUrl}`));
|
|
654
659
|
}
|
|
@@ -717,12 +722,10 @@ export async function runCommand(config, args, chalk) {
|
|
|
717
722
|
|
|
718
723
|
const rate = dep.cost_per_hour || 0;
|
|
719
724
|
|
|
720
|
-
console.log(chalk.dim('
|
|
721
|
-
|
|
722
|
-
console.log(
|
|
723
|
-
|
|
724
|
-
if (maxCost) console.log(` ${chalk.bold('Max cost:')} $${maxCost.toFixed(2)}`);
|
|
725
|
-
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
725
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Finding a working route...')));
|
|
726
|
+
stageN++;
|
|
727
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Starting runtime...')));
|
|
728
|
+
stageN++;
|
|
726
729
|
|
|
727
730
|
if (rate > HIGH_RATE_THRESHOLD && !maxCost) {
|
|
728
731
|
console.log(chalk.yellow(`\n Selected capacity rate: $${rate.toFixed(2)}/hr`));
|
|
@@ -745,30 +748,34 @@ export async function runCommand(config, args, chalk) {
|
|
|
745
748
|
if (teardownCalled) return;
|
|
746
749
|
teardownCalled = true;
|
|
747
750
|
|
|
748
|
-
const
|
|
749
|
-
'max-runtime':
|
|
750
|
-
'max-cost':
|
|
751
|
-
'heartbeat-lost':
|
|
752
|
-
'interrupted':
|
|
751
|
+
const stageLabels = {
|
|
752
|
+
'max-runtime': 'Stopped — runtime cap reached',
|
|
753
|
+
'max-cost': 'Stopped — spend cap reached',
|
|
754
|
+
'heartbeat-lost': 'Failed — lost connection to machine',
|
|
755
|
+
'interrupted': 'Stopped',
|
|
753
756
|
};
|
|
754
|
-
console.log(labels[reason] ?? chalk.yellow('\n Stopping job...'));
|
|
755
757
|
|
|
758
|
+
let teardownOk = true;
|
|
756
759
|
try {
|
|
757
760
|
await terminateDeployment(config, dep.deployment_id);
|
|
758
761
|
} catch {
|
|
759
762
|
// terminateDeployment retries 3×; best-effort if all fail
|
|
763
|
+
teardownOk = false;
|
|
760
764
|
}
|
|
761
765
|
|
|
762
766
|
const runtimeMs = Date.now() - attachStart;
|
|
763
767
|
const finalCost = ratePerHour * (runtimeMs / 3_600_000);
|
|
764
768
|
updateReceipt(rcptId, {
|
|
765
769
|
status: reason,
|
|
766
|
-
teardownStatus: 'terminated',
|
|
770
|
+
teardownStatus: teardownOk ? 'terminated' : 'failed',
|
|
767
771
|
runtimeSeconds: Math.round(runtimeMs / 1000),
|
|
768
772
|
finalCost,
|
|
769
773
|
});
|
|
770
|
-
console.log(
|
|
771
|
-
console.log(chalk.dim(
|
|
774
|
+
console.log();
|
|
775
|
+
console.log(chalk.dim(_stage(STAGE_TOTAL, STAGE_TOTAL, stageLabels[reason] ?? 'Stopped')));
|
|
776
|
+
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
777
|
+
_printFinalInfo(chalk, { exitCode: null, teardownOk, jobId: dep.deployment_id, rcptId });
|
|
778
|
+
console.log();
|
|
772
779
|
}
|
|
773
780
|
|
|
774
781
|
// ── SIGINT handler — installed immediately after we have a deployment ID ───
|
|
@@ -802,7 +809,8 @@ export async function runCommand(config, args, chalk) {
|
|
|
802
809
|
return;
|
|
803
810
|
}
|
|
804
811
|
|
|
805
|
-
|
|
812
|
+
const runStageLine = _stage(stageN, STAGE_TOTAL, 'Running command...');
|
|
813
|
+
console.log(chalk.dim(`\n [${stageN}/${STAGE_TOTAL}] Running command (Ctrl+C to stop)`));
|
|
806
814
|
|
|
807
815
|
attachStart = Date.now();
|
|
808
816
|
const { status: finalStatus, exitCode, runtimeMs, failureType } = await attachToJob(config, dep.deployment_id, {
|
|
@@ -812,6 +820,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
812
820
|
ratePerHour,
|
|
813
821
|
onTeardown: teardown,
|
|
814
822
|
isShuttingDown: () => shuttingDown,
|
|
823
|
+
stageLine: runStageLine,
|
|
815
824
|
});
|
|
816
825
|
|
|
817
826
|
// Remove SIGINT handler — job is done (or SIGINT was handled)
|
|
@@ -820,7 +829,11 @@ export async function runCommand(config, args, chalk) {
|
|
|
820
829
|
// 'interrupted' = SIGINT handler is managing teardown + exit — don't duplicate
|
|
821
830
|
if (finalStatus === 'interrupted') return;
|
|
822
831
|
|
|
823
|
-
|
|
832
|
+
// 'capped' = max-runtime or max-cost path; teardown() already printed the full summary
|
|
833
|
+
if (finalStatus === 'capped') {
|
|
834
|
+
process.exitCode = 1;
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
824
837
|
|
|
825
838
|
const finalCost = ratePerHour * (runtimeMs / 3_600_000);
|
|
826
839
|
updateReceipt(rcptId, {
|
|
@@ -832,20 +845,7 @@ export async function runCommand(config, args, chalk) {
|
|
|
832
845
|
teardownStatus: (finalStatus === 'completed' || finalStatus === 'succeeded') ? 'terminated' : 'failed',
|
|
833
846
|
});
|
|
834
847
|
|
|
835
|
-
console.log(
|
|
836
|
-
if (ratePerHour > 0) {
|
|
837
|
-
console.log(` ${chalk.bold('Cost:')} $${finalCost.toFixed(4)} (${chalk.dim(`$${ratePerHour.toFixed(2)}/hr`)})`);
|
|
838
|
-
}
|
|
839
|
-
if (exitCode !== null && exitCode !== undefined) {
|
|
840
|
-
console.log(` ${chalk.bold('Exit code:')} ${exitCode !== 0 ? chalk.red(exitCode) : chalk.green(exitCode)}`);
|
|
841
|
-
}
|
|
842
|
-
console.log(` ${chalk.bold('Receipt:')} ${chalk.dim(rcptId)}`);
|
|
843
|
-
|
|
844
|
-
// 'capped' = max-runtime or max-cost path; teardown message already printed
|
|
845
|
-
if (finalStatus === 'capped') {
|
|
846
|
-
process.exitCode = 1;
|
|
847
|
-
return;
|
|
848
|
-
}
|
|
848
|
+
console.log();
|
|
849
849
|
|
|
850
850
|
if (finalStatus === 'failed' || (exitCode !== null && exitCode !== 0)) {
|
|
851
851
|
if (failureType === 'infrastructure') {
|
|
@@ -853,16 +853,23 @@ export async function runCommand(config, args, chalk) {
|
|
|
853
853
|
} else {
|
|
854
854
|
console.error(formatCliError('JOB_FAILED', { exitCode, deploymentId: dep.deployment_id }, chalk));
|
|
855
855
|
}
|
|
856
|
+
console.log(chalk.dim(_stage(STAGE_TOTAL, STAGE_TOTAL, 'Failed')));
|
|
857
|
+
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
858
|
+
// The container already reached a terminal state on the provider side by the
|
|
859
|
+
// time we observe it here, so billing is already stopped — no extra teardown call needed.
|
|
860
|
+
_printFinalInfo(chalk, { exitCode, teardownOk: true, jobId: dep.deployment_id, rcptId });
|
|
856
861
|
process.exitCode = exitCode ?? 1;
|
|
857
862
|
return;
|
|
858
863
|
}
|
|
859
864
|
|
|
860
865
|
if ((finalStatus === 'completed' || finalStatus === 'succeeded') && (exitCode === 0 || exitCode === null)) {
|
|
866
|
+
let teardownOk = true;
|
|
861
867
|
try {
|
|
862
868
|
await terminateDeployment(config, dep.deployment_id);
|
|
863
|
-
} catch { /* already stopped */ }
|
|
864
|
-
console.log(chalk.green(
|
|
865
|
-
console.log(chalk.dim(`
|
|
869
|
+
} catch { teardownOk = false; /* already stopped, or best-effort */ }
|
|
870
|
+
console.log(chalk.green(_stage(STAGE_TOTAL, STAGE_TOTAL, 'Complete')));
|
|
871
|
+
console.log(chalk.dim(` Runtime: ${fmtRuntime(runtimeMs)} • Estimated cost: ~$${finalCost.toFixed(2)}`));
|
|
872
|
+
_printFinalInfo(chalk, { exitCode, teardownOk, jobId: dep.deployment_id, rcptId });
|
|
866
873
|
|
|
867
874
|
if (flags.save && config.apiKey) {
|
|
868
875
|
try {
|
package/src/commands/serve.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { requireApiKey } from '../config.js';
|
|
2
2
|
import { callApi, listDeployments } from '../api.js';
|
|
3
3
|
import { addDeployment, addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
4
|
-
import { normalizeTier, callWithFallback
|
|
4
|
+
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
5
5
|
import { formatCliError } from '../errors.js';
|
|
6
|
-
import { TEMPLATE_MAP, buildTemplateFlags, parseTemplateOverrides, BLESSED_VLLM_MODELS } from '../catalog.js';
|
|
6
|
+
import { TEMPLATE_MAP, buildTemplateFlags, parseTemplateOverrides, BLESSED_VLLM_MODELS, isLikelyGatedModel } from '../catalog.js';
|
|
7
|
+
import { stage as _stage, stageDone as _stageDone, writeBlock as _writeBlock, clearBlock as _clearBlock, renderLiveBlock as _renderLiveBlock } from '../progress.js';
|
|
7
8
|
|
|
8
9
|
const LLAMA_CPP_IMAGE = 'michaelmanleyx/llama-cpp:server-cuda';
|
|
9
10
|
|
|
@@ -64,18 +65,36 @@ function parseEnvFlag(envList) {
|
|
|
64
65
|
return obj;
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
function envObjHasHfToken(envList) {
|
|
69
|
+
return (envList || []).some(kv => kv.startsWith('HF_TOKEN='));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Extract a parameter-count-in-billions hint from a model/file name, e.g.
|
|
73
|
+
// "Qwen2.5-0.5B-Instruct" → 0.5, "Llama-3.1-8B-Instruct" → 8, "Mixtral-8x7B" → 56.
|
|
74
|
+
// Splits into delimiter-bounded segments first so a version number like "2.5"
|
|
75
|
+
// in "Qwen2.5-0.5B" is never mistaken for the param count — only a segment that
|
|
76
|
+
// IS entirely "<digits>b" or "<digits>x<digits>b" counts as a size hint.
|
|
77
|
+
function _extractParamsB(name) {
|
|
78
|
+
const s = name.toLowerCase();
|
|
79
|
+
const segments = s.split(/[^a-z0-9.]+/).filter(Boolean);
|
|
80
|
+
for (const seg of segments) {
|
|
81
|
+
const moe = seg.match(/^(\d+)x(\d+)b$/);
|
|
82
|
+
if (moe) return parseInt(moe[1], 10) * parseInt(moe[2], 10);
|
|
83
|
+
}
|
|
84
|
+
for (const seg of segments) {
|
|
85
|
+
const m = seg.match(/^(\d+(?:\.\d+)?)b$/);
|
|
86
|
+
if (m) return parseFloat(m[1]);
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
|
|
67
91
|
// Mirror of backend workload_profile.py infer_profile_from_model — for pre-flight display.
|
|
92
|
+
// Sizing is only asserted when a param-count hint is found in the name; unknown
|
|
93
|
+
// sizing falls back to the 7B–8B/24GB+ default rather than guessing small or large.
|
|
68
94
|
function _inferServeProfile(modelName) {
|
|
69
|
-
const
|
|
70
|
-
const moe = s.match(/(\d+)x(\d+)b/);
|
|
71
|
-
let paramsB;
|
|
72
|
-
if (moe) {
|
|
73
|
-
paramsB = parseInt(moe[1]) * parseInt(moe[2]);
|
|
74
|
-
} else {
|
|
75
|
-
const m = s.match(/(\d+)b/);
|
|
76
|
-
paramsB = m ? parseInt(m[1]) : null;
|
|
77
|
-
}
|
|
95
|
+
const paramsB = _extractParamsB(modelName);
|
|
78
96
|
if (paramsB === null) return { label: 'inference (7B–8B model)', vram: '24+ GB', gpus: ['RTX 4090', 'A6000', 'L40S'] };
|
|
97
|
+
if (paramsB <= 3) return { label: 'inference (≤3B model)', vram: '8+ GB', gpus: ['RTX 3090', 'RTX 4090', 'L4'] };
|
|
79
98
|
if (paramsB <= 9) return { label: 'inference (7B–8B model)', vram: '24+ GB', gpus: ['RTX 4090', 'A6000', 'L40S'] };
|
|
80
99
|
if (paramsB <= 35) return { label: 'inference (30B–34B model)', vram: '40+ GB', gpus: ['A6000', 'L40S', 'A100'] };
|
|
81
100
|
return { label: 'inference (70B+ model)', vram: '80+ GB', gpus: ['H100', 'A100'] };
|
|
@@ -84,32 +103,16 @@ function _inferServeProfile(modelName) {
|
|
|
84
103
|
// Mirror of backend workload_profile.py infer_profile_from_gguf.
|
|
85
104
|
// Accepts the --hf-file filename; looks for param-count hints like "35B" or "8x7B".
|
|
86
105
|
function _inferGgufProfile(ggufPath) {
|
|
87
|
-
const
|
|
88
|
-
const moe = s.match(/(\d+)x(\d+)b/);
|
|
89
|
-
let paramsB;
|
|
90
|
-
if (moe) {
|
|
91
|
-
paramsB = parseInt(moe[1]) * parseInt(moe[2]);
|
|
92
|
-
} else {
|
|
93
|
-
const m = s.match(/(\d+)b/);
|
|
94
|
-
paramsB = m ? parseInt(m[1]) : null;
|
|
95
|
-
}
|
|
106
|
+
const paramsB = _extractParamsB(ggufPath);
|
|
96
107
|
if (paramsB === null || paramsB <= 9) return { label: 'GGUF inference (≤9B, llama.cpp)', vram: '8+ GB', gpus: ['RTX 4090', 'RTX 3090', 'A6000'] };
|
|
97
108
|
if (paramsB <= 35) return { label: 'GGUF inference (10B–35B, llama.cpp)', vram: '24+ GB', gpus: ['RTX 4090', 'A6000', 'L40S'] };
|
|
98
109
|
return { label: 'GGUF inference (36B+, llama.cpp)', vram: '48+ GB', gpus: ['A6000', 'L40S', 'A100'] };
|
|
99
110
|
}
|
|
100
111
|
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
case 'port_unreachable': return 'Waiting for container to start…';
|
|
106
|
-
case 'http_404': return 'Model loading — waiting for the API to come up…';
|
|
107
|
-
case 'http_error': return `Waiting for ${healthPath || 'endpoint'} (non-200 response)…`;
|
|
108
|
-
default: break;
|
|
109
|
-
}
|
|
110
|
-
if (elapsedSec < 30) return 'Starting container…';
|
|
111
|
-
return `Waiting for ${healthPath || 'endpoint'}…`;
|
|
112
|
-
}
|
|
112
|
+
// If the readiness reason hasn't changed for this long, the status word
|
|
113
|
+
// flips from "starting" to "stuck" so a silent stall never looks identical
|
|
114
|
+
// to normal progress.
|
|
115
|
+
const SERVE_STUCK_THRESHOLD_MS = 90_000;
|
|
113
116
|
|
|
114
117
|
function _detectHealthPath(image) {
|
|
115
118
|
if (!image) return null;
|
|
@@ -129,10 +132,15 @@ function _detectHealthPath(image) {
|
|
|
129
132
|
// vLLM cold start (model download + load) often exceeds 5 min on first boot.
|
|
130
133
|
const VLLM_SERVE_WAIT_MS = 15 * 60 * 1000;
|
|
131
134
|
|
|
132
|
-
async function waitForEndpoint(deploymentId, config, timeoutMs = VLLM_SERVE_WAIT_MS, chalk, healthPath = '/models') {
|
|
135
|
+
async function waitForEndpoint(deploymentId, config, timeoutMs = VLLM_SERVE_WAIT_MS, chalk, healthPath = '/models', costCtx = {}) {
|
|
136
|
+
const { costPerHour = 0, maxCost = null, stageLine = '' } = costCtx;
|
|
133
137
|
const startMs = Date.now();
|
|
134
138
|
const deadline = startMs + timeoutMs;
|
|
135
139
|
|
|
140
|
+
let lastReason = null;
|
|
141
|
+
let reasonSinceMs = startMs;
|
|
142
|
+
let blockLines = 0;
|
|
143
|
+
|
|
136
144
|
while (Date.now() < deadline) {
|
|
137
145
|
try {
|
|
138
146
|
const dep = await callApi(`/deployments/${deploymentId}`, {
|
|
@@ -141,24 +149,30 @@ async function waitForEndpoint(deploymentId, config, timeoutMs = VLLM_SERVE_WAIT
|
|
|
141
149
|
timeoutMs: 10_000,
|
|
142
150
|
});
|
|
143
151
|
if (['failed', 'terminated', 'error', 'stopped'].includes(dep.status)) {
|
|
144
|
-
|
|
152
|
+
if (blockLines > 0) { _clearBlock(blockLines); blockLines = 0; }
|
|
145
153
|
return { ready: false, timedOut: false, depFailed: true, failReason: dep.fix_hint || dep.error || dep.status };
|
|
146
154
|
}
|
|
147
155
|
if (dep.endpoint_ready) {
|
|
148
|
-
|
|
156
|
+
if (blockLines > 0) { _clearBlock(blockLines); blockLines = 0; }
|
|
149
157
|
return { ready: true, timedOut: false, depFailed: false };
|
|
150
158
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
159
|
+
|
|
160
|
+
const now = Date.now();
|
|
161
|
+
const elapsed = Math.round((now - startMs) / 1000);
|
|
162
|
+
const reason = dep.readiness_reason || 'starting';
|
|
163
|
+
if (reason !== lastReason) { lastReason = reason; reasonSinceMs = now; }
|
|
164
|
+
const statusWord = (now - reasonSinceMs) >= SERVE_STUCK_THRESHOLD_MS ? 'stuck' : 'still starting';
|
|
165
|
+
const spend = costPerHour * (elapsed / 3600);
|
|
166
|
+
|
|
167
|
+
blockLines = _writeBlock(blockLines, _renderLiveBlock(chalk, {
|
|
168
|
+
stageLine, elapsedSec: elapsed, statusWord, spend, id: deploymentId,
|
|
169
|
+
}));
|
|
155
170
|
} catch {
|
|
156
171
|
// status check failed (transient network/API issue) — retry next tick
|
|
157
172
|
}
|
|
158
173
|
await new Promise(r => setTimeout(r, 8000));
|
|
159
174
|
}
|
|
160
175
|
|
|
161
|
-
process.stdout.write('\n');
|
|
162
176
|
return { ready: false, timedOut: true, depFailed: false };
|
|
163
177
|
}
|
|
164
178
|
|
|
@@ -217,13 +231,14 @@ export async function serveCommand(config, args, chalk) {
|
|
|
217
231
|
}
|
|
218
232
|
|
|
219
233
|
if (args.includes('--list-aliases')) {
|
|
220
|
-
console.log(chalk.bold('\
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
console.log(` ${chalk.cyan(alias.padEnd(16))} ${spec.model_id}`);
|
|
224
|
-
console.log(` ${''.padEnd(16)} ${chalk.dim(spec.description)}`);
|
|
234
|
+
console.log(chalk.bold('\nTested model routes:\n'));
|
|
235
|
+
for (const alias of Object.keys(BLESSED_VLLM_MODELS)) {
|
|
236
|
+
console.log(` - ${chalk.cyan(alias)}`);
|
|
225
237
|
}
|
|
226
238
|
console.log();
|
|
239
|
+
console.log(chalk.dim('You can also try a Hugging Face model ID:'));
|
|
240
|
+
console.log(chalk.dim(' badgr serve Qwen/Qwen2.5-7B-Instruct --max-cost 10'));
|
|
241
|
+
console.log();
|
|
227
242
|
return;
|
|
228
243
|
}
|
|
229
244
|
|
|
@@ -314,50 +329,60 @@ export async function serveCommand(config, args, chalk) {
|
|
|
314
329
|
|
|
315
330
|
const effectiveTier = normalizeTier(flags.tier);
|
|
316
331
|
|
|
332
|
+
// Header fields + trailing note (shown after Max cost) per serve mode.
|
|
333
|
+
// Built as [label, value] pairs so every mode renders through one aligned
|
|
334
|
+
// printer instead of four hand-spaced copies.
|
|
335
|
+
let title;
|
|
336
|
+
const headerLines = [];
|
|
337
|
+
let trailingNote = null;
|
|
338
|
+
let sizeProfile = null; // { label, vram } — only shown when GPU sizing was inferred, not chosen
|
|
339
|
+
|
|
317
340
|
if (isLlamaCpp) {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
if (
|
|
324
|
-
if (gpu === 'AUTO') {
|
|
325
|
-
const prof = _inferGgufProfile(flags.hfFile);
|
|
326
|
-
console.log();
|
|
327
|
-
console.log(` ${chalk.bold('Estimated workload:')} ${prof.label}`);
|
|
328
|
-
console.log(` ${chalk.bold('Estimated minimum VRAM:')} ${prof.vram}`);
|
|
329
|
-
}
|
|
341
|
+
title = flags.hfRepo;
|
|
342
|
+
headerLines.push(['Route', 'best-effort Hugging Face model']);
|
|
343
|
+
headerLines.push(['Mode', 'OpenAI-compatible endpoint (llama.cpp)']);
|
|
344
|
+
headerLines.push(['File', flags.hfFile]);
|
|
345
|
+
trailingNote = 'Badgr will try a compatible route.';
|
|
346
|
+
if (gpu === 'AUTO') sizeProfile = _inferGgufProfile(flags.hfFile);
|
|
330
347
|
} else if (customImage) {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
348
|
+
title = 'custom container';
|
|
349
|
+
headerLines.push(['Mode', 'custom server']);
|
|
350
|
+
trailingNote = 'Badgr manages runtime, logs, caps, teardown, and receipts.\n Your container owns the app behavior.';
|
|
351
|
+
} else if (vllmAlias) {
|
|
352
|
+
title = model;
|
|
353
|
+
headerLines.push(['Route', 'tested']);
|
|
354
|
+
headerLines.push(['Mode', 'OpenAI-compatible endpoint']);
|
|
336
355
|
} else {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
}
|
|
343
|
-
console.log(` ${chalk.bold('GPU:')} ${gpuLabel}`);
|
|
344
|
-
if (flags.task) console.log(` ${chalk.bold('Task:')} ${flags.task}`);
|
|
345
|
-
if (flags.env?.length) console.log(` ${chalk.bold('Env:')} ${flags.env.join(', ')}`);
|
|
346
|
-
|
|
347
|
-
if (gpu === 'AUTO') {
|
|
348
|
-
const prof = _inferServeProfile(effectiveModel);
|
|
349
|
-
console.log();
|
|
350
|
-
console.log(` ${chalk.bold('Estimated workload:')} ${prof.label}`);
|
|
351
|
-
console.log(` ${chalk.bold('Estimated minimum VRAM:')} ${prof.vram}`);
|
|
352
|
-
}
|
|
356
|
+
title = effectiveModel;
|
|
357
|
+
headerLines.push(['Route', 'best-effort Hugging Face model']);
|
|
358
|
+
headerLines.push(['Mode', 'OpenAI-compatible endpoint']);
|
|
359
|
+
trailingNote = 'Badgr will try a compatible route.';
|
|
360
|
+
if (gpu === 'AUTO') sizeProfile = _inferServeProfile(effectiveModel);
|
|
353
361
|
}
|
|
354
|
-
if (flags.
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
362
|
+
if (flags.gpu) headerLines.push(['GPU', gpuLabel]);
|
|
363
|
+
if (flags.task) headerLines.push(['Task', flags.task]);
|
|
364
|
+
if (flags.env?.length) headerLines.push(['Env', flags.env.join(', ')]);
|
|
365
|
+
|
|
366
|
+
console.log(chalk.bold(`\n⚡ Serving ${title}\n`));
|
|
367
|
+
const labelWidth = Math.max(...headerLines.map(([label]) => label.length)) + 1;
|
|
368
|
+
for (const [label, value] of headerLines) {
|
|
369
|
+
console.log(` ${chalk.bold(`${label}:`.padEnd(labelWidth + 1))}${value}`);
|
|
370
|
+
}
|
|
371
|
+
if (sizeProfile) {
|
|
372
|
+
console.log();
|
|
373
|
+
console.log(` ${chalk.bold('Estimated workload:')} ${sizeProfile.label}`);
|
|
374
|
+
console.log(` ${chalk.bold('Estimated minimum VRAM:')} ${sizeProfile.vram}`);
|
|
375
|
+
}
|
|
376
|
+
console.log(` ${chalk.bold('Max cost:')} ${flags.maxCost ? `$${flags.maxCost.toFixed(2)}` : chalk.dim('none')}`);
|
|
377
|
+
console.log(` ${chalk.bold('Auto-stop:')} ${flags.maxCost ? 'enabled' : chalk.yellow('disabled — stop manually with badgr down')}`);
|
|
378
|
+
if (trailingNote) {
|
|
379
|
+
console.log();
|
|
380
|
+
console.log(chalk.dim(` ${trailingNote}`));
|
|
358
381
|
}
|
|
359
382
|
console.log();
|
|
360
|
-
|
|
383
|
+
|
|
384
|
+
const STAGE_TOTAL = 5;
|
|
385
|
+
let stageN = 1;
|
|
361
386
|
|
|
362
387
|
// ── Duplicate check ────────────────────────────────────────────────────────
|
|
363
388
|
if (config.apiKey) {
|
|
@@ -469,6 +494,11 @@ export async function serveCommand(config, args, chalk) {
|
|
|
469
494
|
createdAt: new Date().toISOString(),
|
|
470
495
|
});
|
|
471
496
|
|
|
497
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Finding a working route...')));
|
|
498
|
+
stageN++;
|
|
499
|
+
console.log(chalk.dim(_stageDone(stageN, STAGE_TOTAL, 'Starting runtime...')));
|
|
500
|
+
stageN++;
|
|
501
|
+
|
|
472
502
|
// ── Fix 7: never fall back to config.baseUrl for endpoint health check ─────
|
|
473
503
|
const endpointUrl = dep.endpoint_url || dep.openai_base_url;
|
|
474
504
|
if (!endpointUrl) {
|
|
@@ -499,18 +529,47 @@ export async function serveCommand(config, args, chalk) {
|
|
|
499
529
|
resolvedHealthPath = _detectHealthPath(customImage); // '/system_stats' for comfyui, null otherwise
|
|
500
530
|
}
|
|
501
531
|
|
|
532
|
+
// Gated-model guidance is only shown when it's actually needed — on failure —
|
|
533
|
+
// not up front, so common launches stay short and uncluttered.
|
|
534
|
+
const gatedModelId = isLlamaCpp ? flags.hfRepo : effectiveModel;
|
|
535
|
+
const gatedHintNeeded = isLikelyGatedModel(gatedModelId) && !envObjHasHfToken(flags.env);
|
|
536
|
+
|
|
537
|
+
// Shared reporting for "deployment failed to start" — hit from both the
|
|
538
|
+
// pre-poll status check and the waitForEndpoint poll loop below.
|
|
539
|
+
function reportDeployFailure(failReason) {
|
|
540
|
+
console.error(formatCliError('HEALTH_CHECK_DEPLOY_FAILED', {
|
|
541
|
+
deploymentId: dep.deployment_id,
|
|
542
|
+
failReason,
|
|
543
|
+
}, chalk));
|
|
544
|
+
if (gatedHintNeeded) {
|
|
545
|
+
const rerun = args.join(' ');
|
|
546
|
+
const retryCmd = rerun.includes('HF_TOKEN=') ? rerun : `${rerun} --env HF_TOKEN=$HF_TOKEN`;
|
|
547
|
+
console.error();
|
|
548
|
+
console.error(chalk.yellow(' This model may require Hugging Face access.'));
|
|
549
|
+
console.error(chalk.dim(' Retry:'));
|
|
550
|
+
console.error(chalk.dim(` badgr serve ${retryCmd}`));
|
|
551
|
+
}
|
|
552
|
+
updateReceipt(rcptId, { status: 'failed', failReason });
|
|
553
|
+
process.exitCode = 1;
|
|
554
|
+
}
|
|
555
|
+
|
|
502
556
|
// ── Health check ──────────────────────────────────────────────────────────
|
|
557
|
+
const loadingStageN = stageN; // "Loading model..."
|
|
558
|
+
const healthStageN = stageN + 1; // "Checking endpoint health..."
|
|
559
|
+
const readyStageN = stageN + 2; // "Ready"
|
|
560
|
+
|
|
503
561
|
let endpointReady = false;
|
|
504
562
|
if (flags.noWait) {
|
|
505
|
-
console.log(chalk.yellow('
|
|
563
|
+
console.log(chalk.dim(_stage(loadingStageN, STAGE_TOTAL, 'Loading model...')) + chalk.yellow(' (skipped — --no-wait)'));
|
|
564
|
+
console.log(chalk.dim(_stage(healthStageN, STAGE_TOTAL, 'Checking endpoint health...')) + chalk.yellow(' (skipped — --no-wait)'));
|
|
565
|
+
console.log(chalk.yellow(_stage(readyStageN, STAGE_TOTAL, 'Not confirmed ready — check badgr logs') + `\n`));
|
|
506
566
|
} else if (resolvedHealthPath === null) {
|
|
567
|
+
console.log(chalk.dim(_stage(loadingStageN, STAGE_TOTAL, 'Loading model...')));
|
|
507
568
|
console.log(chalk.yellow(
|
|
508
|
-
|
|
569
|
+
_stage(healthStageN, STAGE_TOTAL, 'Checking endpoint health...') +
|
|
570
|
+
' (skipped — custom image; add --health-path /your-readiness-path to enable)\n'
|
|
509
571
|
));
|
|
510
572
|
} else {
|
|
511
|
-
if (resolvedHealthPath !== '/models') {
|
|
512
|
-
process.stdout.write(chalk.dim(` Checking ${resolvedHealthPath} for readiness…\n`));
|
|
513
|
-
}
|
|
514
573
|
// Check deployment status once before starting the 5-min wait
|
|
515
574
|
try {
|
|
516
575
|
const latest = await callApi(`/deployments/${dep.deployment_id}`, {
|
|
@@ -519,33 +578,31 @@ export async function serveCommand(config, args, chalk) {
|
|
|
519
578
|
timeoutMs: 10_000,
|
|
520
579
|
});
|
|
521
580
|
if (['failed', 'terminated', 'error'].includes(latest.status)) {
|
|
522
|
-
|
|
523
|
-
deploymentId: dep.deployment_id,
|
|
524
|
-
failReason: latest.error || latest.status,
|
|
525
|
-
}, chalk));
|
|
526
|
-
updateReceipt(rcptId, { status: 'failed', failReason: latest.error || latest.status });
|
|
527
|
-
process.exitCode = 1;
|
|
581
|
+
reportDeployFailure(latest.error || latest.status);
|
|
528
582
|
return;
|
|
529
583
|
}
|
|
530
584
|
} catch {
|
|
531
585
|
// status check failed — proceed with endpoint poll anyway
|
|
532
586
|
}
|
|
533
587
|
|
|
534
|
-
const
|
|
535
|
-
|
|
588
|
+
const loadingStageLine = _stage(loadingStageN, STAGE_TOTAL, 'Loading model...');
|
|
589
|
+
const healthResult = await waitForEndpoint(
|
|
590
|
+
dep.deployment_id, config, VLLM_SERVE_WAIT_MS, chalk, resolvedHealthPath,
|
|
591
|
+
{ costPerHour: dep.cost_per_hour || 0, maxCost: flags.maxCost || null, stageLine: loadingStageLine },
|
|
592
|
+
);
|
|
536
593
|
|
|
537
594
|
if (healthResult.depFailed) {
|
|
538
|
-
|
|
539
|
-
deploymentId: dep.deployment_id,
|
|
540
|
-
failReason: healthResult.failReason,
|
|
541
|
-
}, chalk));
|
|
542
|
-
updateReceipt(rcptId, { status: 'failed', failReason: healthResult.failReason });
|
|
543
|
-
process.exitCode = 1;
|
|
595
|
+
reportDeployFailure(healthResult.failReason);
|
|
544
596
|
return;
|
|
545
597
|
}
|
|
546
598
|
|
|
547
599
|
endpointReady = healthResult.ready;
|
|
548
|
-
if (
|
|
600
|
+
if (endpointReady) {
|
|
601
|
+
console.log(chalk.dim(_stage(healthStageN, STAGE_TOTAL, 'Checking endpoint health...')));
|
|
602
|
+
console.log(chalk.green(_stage(readyStageN, STAGE_TOTAL, 'Ready')));
|
|
603
|
+
} else {
|
|
604
|
+
updateReceipt(rcptId, { status: 'health_check_timeout' });
|
|
605
|
+
}
|
|
549
606
|
}
|
|
550
607
|
|
|
551
608
|
// ── Custom-node validation (ComfyUI) ─────────────────────────────────────
|
|
@@ -570,8 +627,6 @@ export async function serveCommand(config, args, chalk) {
|
|
|
570
627
|
console.log();
|
|
571
628
|
}
|
|
572
629
|
|
|
573
|
-
const serveRate = dep.cost_per_hour || 0;
|
|
574
|
-
|
|
575
630
|
console.log(` ${chalk.bold('Base URL:')} ${chalk.cyan(endpointUrl)}`);
|
|
576
631
|
if (isLlamaCpp) {
|
|
577
632
|
console.log(` ${chalk.bold('HF Repo:')} ${flags.hfRepo}`);
|
|
@@ -579,23 +634,20 @@ export async function serveCommand(config, args, chalk) {
|
|
|
579
634
|
}
|
|
580
635
|
else if (dep.model || effectiveModel) console.log(` ${chalk.bold('Model:')} ${dep.model || effectiveModel}`);
|
|
581
636
|
if (customImage) console.log(` ${chalk.bold('Image:')} ${customImage}`);
|
|
582
|
-
console.log(` ${chalk.bold('GPU:')} ${dep.gpu_type} × ${dep.gpu_count}`);
|
|
583
|
-
if (serveRate > 0) console.log(` ${chalk.bold('Rate:')} $${serveRate.toFixed(2)}/hr`);
|
|
584
637
|
if (flags.maxCost) console.log(` ${chalk.bold('Max cost:')} $${flags.maxCost.toFixed(2)}`);
|
|
585
|
-
console.log(` ${chalk.bold('
|
|
586
|
-
console.log(` ${chalk.bold('Logs:')} ${chalk.dim(`badgr logs ${dep.deployment_id}`)}`);
|
|
638
|
+
console.log(` ${chalk.bold('Logs:')} badgr logs ${dep.deployment_id}`);
|
|
587
639
|
console.log(` ${chalk.bold('Stop billing:')} ${chalk.cyan(`badgr down ${dep.deployment_id}`)}`);
|
|
640
|
+
console.log(` ${chalk.bold('Receipt:')} badgr receipts ${rcptId}`);
|
|
588
641
|
console.log();
|
|
589
|
-
|
|
590
|
-
if (serveRate > HIGH_RATE_THRESHOLD && !flags.maxCost) {
|
|
591
|
-
console.log(chalk.yellow(` Selected capacity rate: $${serveRate.toFixed(2)}/hr`));
|
|
592
|
-
console.log(chalk.dim(' Tip: use --max-cost to enforce a hard ceiling.\n'));
|
|
593
|
-
}
|
|
594
642
|
console.log(chalk.dim(' Billing continues until you run: ') + chalk.cyan(`badgr down ${dep.deployment_id}`));
|
|
595
643
|
|
|
596
644
|
if (endpointReady && !customImage) {
|
|
597
645
|
const keySnip = config.apiKey?.slice(0, 4) || 'sk-...';
|
|
598
646
|
const sdkModel = isLlamaCpp ? 'default' : (dep.model || effectiveModel);
|
|
647
|
+
console.log(` ${chalk.bold('Test with curl:')}`);
|
|
648
|
+
console.log(chalk.dim(` curl ${endpointUrl}/chat/completions \\`));
|
|
649
|
+
console.log(chalk.dim(` -H "Authorization: Bearer ${keySnip}..." -H "Content-Type: application/json" \\`));
|
|
650
|
+
console.log(chalk.dim(` -d '{"model":"${sdkModel}","messages":[{"role":"user","content":"Hello"}]}'`));
|
|
599
651
|
console.log(` ${chalk.bold('Use with OpenAI SDK:')}`);
|
|
600
652
|
console.log(chalk.dim(` from openai import OpenAI`));
|
|
601
653
|
console.log(chalk.dim(` client = OpenAI(base_url="${endpointUrl}", api_key="${keySnip}...")`));
|
package/src/progress.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Shared staged-progress rendering used by `badgr serve` and `badgr run` —
|
|
2
|
+
// numbered stage lines plus an in-place-redrawing status block, so neither
|
|
3
|
+
// command spams repeated identical loading lines.
|
|
4
|
+
|
|
5
|
+
export function stage(n, total, label) {
|
|
6
|
+
return ` [${n}/${total}] ${label}`;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function stageDone(n, total, label) {
|
|
10
|
+
return ` [${n}/${total}] ${label} done`;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Redraws a fixed-height status block in place instead of appending new
|
|
14
|
+
// lines each tick — satisfies "no repeated identical loading lines" while
|
|
15
|
+
// still showing live elapsed time. Returns the new line count to pass back
|
|
16
|
+
// in as prevLineCount on the next call.
|
|
17
|
+
export function writeBlock(prevLineCount, lines) {
|
|
18
|
+
if (prevLineCount > 0) process.stdout.write(`\x1b[${prevLineCount}A`);
|
|
19
|
+
for (const line of lines) process.stdout.write('\x1b[2K' + line + '\n');
|
|
20
|
+
return lines.length;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function clearBlock(lineCount) {
|
|
24
|
+
if (lineCount <= 0) return;
|
|
25
|
+
process.stdout.write(`\x1b[${lineCount}A`);
|
|
26
|
+
for (let i = 0; i < lineCount; i++) process.stdout.write('\x1b[2K\n');
|
|
27
|
+
process.stdout.write(`\x1b[${lineCount}A`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// The live block shown while a stage is in progress — same shape for both
|
|
31
|
+
// `badgr serve` (deployment) and `badgr run` (job): elapsed time, a status
|
|
32
|
+
// word, an *estimated* spend (never presented as confirmed while billing is
|
|
33
|
+
// still live), and the exact commands to inspect logs or stop billing.
|
|
34
|
+
export function renderLiveBlock(chalk, { stageLine, elapsedSec, statusWord, spend, id }) {
|
|
35
|
+
return [
|
|
36
|
+
chalk.dim(`${stageLine} ${elapsedSec}s elapsed`),
|
|
37
|
+
chalk.dim(` Status: ${statusWord}`),
|
|
38
|
+
chalk.dim(` Estimated spend: ~$${spend.toFixed(2)}`),
|
|
39
|
+
chalk.dim(` Logs: badgr logs ${id}`),
|
|
40
|
+
chalk.dim(` Stop billing: badgr down ${id}`),
|
|
41
|
+
];
|
|
42
|
+
}
|
|
@@ -510,13 +510,106 @@ describe('input validation for serve', () => {
|
|
|
510
510
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
511
511
|
|
|
512
512
|
describe('--list-aliases', () => {
|
|
513
|
-
it('prints
|
|
513
|
+
it('prints tested model routes without requiring a model or API call', async () => {
|
|
514
514
|
await serveCommand({ apiKey: null, baseUrl: 'https://api.test/v1' }, ['--list-aliases'], chalk);
|
|
515
515
|
expect(api.callApi).not.toHaveBeenCalled();
|
|
516
516
|
expect(process.exitCode).toBeFalsy();
|
|
517
517
|
const logged = console.log.mock.calls.flat().join('\n');
|
|
518
|
+
expect(logged).toContain('Tested model routes:');
|
|
518
519
|
expect(logged).toContain('qwen-7b');
|
|
519
|
-
expect(logged).toContain('
|
|
520
|
+
expect(logged).toContain('You can also try a Hugging Face model ID:');
|
|
521
|
+
});
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
525
|
+
// Model support-level messaging — blessed route vs best-effort HF model vs
|
|
526
|
+
// custom container, plus gated-model HF_TOKEN guidance shown only on failure.
|
|
527
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
528
|
+
|
|
529
|
+
describe('model support-level messaging', () => {
|
|
530
|
+
it('labels a full Hugging Face model ID as best-effort, not tested', async () => {
|
|
531
|
+
api.callApi
|
|
532
|
+
.mockResolvedValueOnce(makeServeDep({ model: 'Qwen/Qwen2.5-7B-Instruct' }))
|
|
533
|
+
.mockResolvedValueOnce({ status: 'running' })
|
|
534
|
+
.mockResolvedValueOnce(readyStatus());
|
|
535
|
+
|
|
536
|
+
const p = serveCommand(config, ['Qwen/Qwen2.5-7B-Instruct', '--max-cost', '10'], chalk);
|
|
537
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
538
|
+
await p;
|
|
539
|
+
|
|
540
|
+
const logged = console.log.mock.calls.flat().join('\n');
|
|
541
|
+
expect(logged).toContain('Route: best-effort Hugging Face model');
|
|
542
|
+
expect(logged).not.toContain('Route: tested');
|
|
543
|
+
});
|
|
544
|
+
|
|
545
|
+
it('labels a blessed alias as a tested route without extra caveats', async () => {
|
|
546
|
+
api.callApi
|
|
547
|
+
.mockResolvedValueOnce(makeServeDep({ model: 'Qwen/Qwen2.5-7B-Instruct' }))
|
|
548
|
+
.mockResolvedValueOnce({ status: 'running' })
|
|
549
|
+
.mockResolvedValueOnce(readyStatus());
|
|
550
|
+
|
|
551
|
+
const p = serveCommand(config, ['qwen-7b', '--max-cost', '10'], chalk);
|
|
552
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
553
|
+
await p;
|
|
554
|
+
|
|
555
|
+
const logged = console.log.mock.calls.flat().join('\n');
|
|
556
|
+
expect(logged).toContain('Route: tested');
|
|
557
|
+
expect(logged).not.toContain('best-effort');
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
it('does not show HF_TOKEN guidance up front for a gated model that launches fine', async () => {
|
|
561
|
+
api.callApi
|
|
562
|
+
.mockResolvedValueOnce(makeServeDep({ model: 'meta-llama/Llama-3.1-8B-Instruct' }))
|
|
563
|
+
.mockResolvedValueOnce({ status: 'running' })
|
|
564
|
+
.mockResolvedValueOnce(readyStatus());
|
|
565
|
+
|
|
566
|
+
const p = serveCommand(config, ['meta-llama/Llama-3.1-8B-Instruct', '--max-cost', '10'], chalk);
|
|
567
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
568
|
+
await p;
|
|
569
|
+
|
|
570
|
+
const logged = console.log.mock.calls.flat().join('\n');
|
|
571
|
+
expect(logged).not.toContain('may require Hugging Face access');
|
|
572
|
+
});
|
|
573
|
+
|
|
574
|
+
it('shows HF_TOKEN guidance only once a gated model actually fails to start', async () => {
|
|
575
|
+
api.callApi
|
|
576
|
+
.mockResolvedValueOnce(makeServeDep({ model: 'meta-llama/Llama-3.1-8B-Instruct' }))
|
|
577
|
+
.mockResolvedValueOnce({ status: 'failed', error: 'gated repo — 401' });
|
|
578
|
+
|
|
579
|
+
const p = serveCommand(config, ['meta-llama/Llama-3.1-8B-Instruct', '--max-cost', '10'], chalk);
|
|
580
|
+
await p;
|
|
581
|
+
|
|
582
|
+
const logged = console.error.mock.calls.flat().join('\n');
|
|
583
|
+
expect(logged).toContain('may require Hugging Face access');
|
|
584
|
+
expect(logged).toContain('--env HF_TOKEN=$HF_TOKEN');
|
|
585
|
+
expect(process.exitCode).toBe(1);
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
it('omits HF_TOKEN guidance on failure when HF_TOKEN is already provided', async () => {
|
|
589
|
+
api.callApi
|
|
590
|
+
.mockResolvedValueOnce(makeServeDep({ model: 'meta-llama/Llama-3.1-8B-Instruct' }))
|
|
591
|
+
.mockResolvedValueOnce({ status: 'failed', error: 'crashed' });
|
|
592
|
+
|
|
593
|
+
const p = serveCommand(config, [
|
|
594
|
+
'meta-llama/Llama-3.1-8B-Instruct', '--max-cost', '10', '--env', 'HF_TOKEN=hf_abc123',
|
|
595
|
+
], chalk);
|
|
596
|
+
await p;
|
|
597
|
+
|
|
598
|
+
const logged = console.error.mock.calls.flat().join('\n');
|
|
599
|
+
expect(logged).not.toContain('may require Hugging Face access');
|
|
600
|
+
});
|
|
601
|
+
|
|
602
|
+
it('labels a custom container as "custom server", not a Hugging Face model', async () => {
|
|
603
|
+
api.callApi.mockResolvedValueOnce(makeServeDep({ model: undefined, image: 'my/custom-server:latest' }));
|
|
604
|
+
|
|
605
|
+
const p = serveCommand(config, [
|
|
606
|
+
'--image', 'my/custom-server:latest', '--max-cost', '10', '--no-wait',
|
|
607
|
+
], chalk);
|
|
608
|
+
await p;
|
|
609
|
+
|
|
610
|
+
const logged = console.log.mock.calls.flat().join('\n');
|
|
611
|
+
expect(logged).toContain('custom server');
|
|
612
|
+
expect(logged).toContain('Your container owns the app behavior.');
|
|
520
613
|
});
|
|
521
614
|
});
|
|
522
615
|
|