@infernetprotocol/cli 0.1.47 → 0.1.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +32 -28
- package/lib/vllm.js +53 -0
- package/package.json +7 -7
package/commands/start.js
CHANGED
|
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
|
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
49
|
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
50
|
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
51
|
-
import { startVllmServe, vllmInstalled, waitForVllmModel,
|
|
51
|
+
import { startVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
|
|
52
52
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
53
53
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
54
54
|
import { pullLatestBinary } from './upgrade.js';
|
|
@@ -629,50 +629,54 @@ async function runDaemon(args, ctx) {
|
|
|
629
629
|
let result;
|
|
630
630
|
if (command === 'model_install') {
|
|
631
631
|
const modelName = String(args?.model);
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
632
|
+
// The node decides the backend by HARDWARE: prefer vLLM (GPU),
|
|
633
|
+
// fall back to Ollama (CPU/Mac). A catalog model carries an
|
|
634
|
+
// `hf` repo (ungated, vLLM-servable); a bare `hf:` name is
|
|
635
|
+
// vLLM-only; everything else is an Ollama tag.
|
|
636
|
+
const hfRepo = args?.hf
|
|
637
|
+
? String(args.hf)
|
|
638
|
+
: (modelName.startsWith('hf:') ? modelName.slice(3) : null);
|
|
639
|
+
|
|
640
|
+
if (vllmInstalled() && hfRepo) {
|
|
641
|
+
// GPU path — serve the HF weights on vLLM. Keep the friendly
|
|
642
|
+
// name (e.g. "qwen2.5:7b" or "hf:org/repo") as the served
|
|
643
|
+
// model name so chat requests + served_models match, but the
|
|
644
|
+
// engine underneath is vLLM.
|
|
642
645
|
const token = process.env.HF_TOKEN || (await resolveHfToken().catch(() => null));
|
|
643
|
-
const localPath = await downloadHfModel(
|
|
644
|
-
// Serve the repo id, not a computed cache path — vLLM
|
|
645
|
-
// resolves it straight from the HF cache we just filled
|
|
646
|
-
// (the snapshot dir is a commit hash, not "main"). Serve
|
|
647
|
-
// under the hf: pull name so it matches chat requests and
|
|
648
|
-
// gets advertised in served_models → shows in /chat.
|
|
649
|
-
const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
|
|
646
|
+
const localPath = await downloadHfModel(hfRepo, token);
|
|
650
647
|
// Don't report "completed" on spawn — wait until vLLM has
|
|
651
648
|
// mapped the weights and /v1/models actually lists the model.
|
|
652
|
-
// If it never comes up (OOM, unsupported arch,
|
|
653
|
-
// fail the command WITH the log
|
|
654
|
-
|
|
655
|
-
process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
|
|
649
|
+
// If it never comes up (OOM, unsupported arch, gated repo),
|
|
650
|
+
// fail the command WITH the log root cause instead of lying.
|
|
651
|
+
const serve = await startVllmServe({ source: hfRepo, servedName: modelName, token });
|
|
652
|
+
process.stdout.write(`[${new Date().toISOString()}] ${modelName} (${hfRepo}) downloaded; waiting for vLLM to load weights…\n`);
|
|
656
653
|
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
|
|
657
654
|
if (!up.serving) {
|
|
658
|
-
const tail = await
|
|
655
|
+
const tail = await extractVllmError();
|
|
659
656
|
throw new Error(
|
|
660
|
-
`vLLM failed to serve ${modelName}: ${up.reason}.` +
|
|
661
|
-
(tail ? `\n--- vllm.log (
|
|
657
|
+
`vLLM failed to serve ${modelName} (${hfRepo}): ${up.reason}.` +
|
|
658
|
+
(tail ? `\n--- vllm.log (root cause) ---\n${tail}` : ' (no vllm.log output)')
|
|
662
659
|
);
|
|
663
660
|
}
|
|
664
|
-
// Serving for real — bust the specs cache so the very next
|
|
665
|
-
// heartbeat advertises the new model in served_models
|
|
666
|
-
// (otherwise it's stale until the TTL lapses).
|
|
667
661
|
invalidateSpecsCache();
|
|
668
662
|
result = {
|
|
669
663
|
model: modelName,
|
|
670
664
|
backend: 'vllm',
|
|
665
|
+
source: hfRepo,
|
|
671
666
|
localPath,
|
|
672
667
|
vllm: { pid: serve.pid, port: serve.port },
|
|
673
668
|
note: `Serving on vLLM :${serve.port} and advertised in served_models.`,
|
|
674
669
|
};
|
|
670
|
+
} else if (modelName.startsWith('hf:')) {
|
|
671
|
+
// hf: model requested but this node has no vLLM (CPU/Mac).
|
|
672
|
+
throw new Error(
|
|
673
|
+
`${modelName} needs vLLM, which isn't installed on this node. ` +
|
|
674
|
+
`Re-run the installer on an NVIDIA host (it installs vLLM), or: pip install vllm. ` +
|
|
675
|
+
`Alternatively push an Ollama model — those run on CPU.`
|
|
676
|
+
);
|
|
675
677
|
} else {
|
|
678
|
+
// CPU/Mac path (or a GPU box without a vLLM-servable repo) —
|
|
679
|
+
// Ollama serves the GGUF tag.
|
|
676
680
|
const fits = await checkModelFits(modelName);
|
|
677
681
|
if (fits && !fits.ok) {
|
|
678
682
|
throw new Error(formatFitFailure(modelName, fits));
|
package/lib/vllm.js
CHANGED
|
@@ -96,6 +96,43 @@ export async function readVllmLogTail(lines = 40) {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
// The lines that actually explain a vLLM EngineCore failure. vLLM prints the
|
|
100
|
+
// root cause (OOM, unsupported arch, dtype/quant mismatch) ABOVE the API-server
|
|
101
|
+
// traceback, then says "See root cause above" — so a plain tail misses it.
|
|
102
|
+
const VLLM_ERROR_PATTERNS = [
|
|
103
|
+
/OutOfMemory|CUDA out of memory|No available memory for the cache|KV cache/i,
|
|
104
|
+
/max_model_len|max seq len|model.?s max/i,
|
|
105
|
+
/not supported|unsupported|no.*implementation|unrecognized|unknown (model|architecture)/i,
|
|
106
|
+
/ValueError|RuntimeError|AssertionError|ImportError|ModuleNotFoundError/,
|
|
107
|
+
/\bERROR\b/,
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Pull the MEANINGFUL error window out of the vLLM log rather than the last
|
|
112
|
+
* few useless traceback frames. Finds the first genuine error line and returns
|
|
113
|
+
* a window around it; falls back to the plain tail if nothing matches.
|
|
114
|
+
*/
|
|
115
|
+
export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
|
|
116
|
+
let txt;
|
|
117
|
+
try {
|
|
118
|
+
txt = await fs.readFile(logPath(), 'utf8');
|
|
119
|
+
} catch {
|
|
120
|
+
return '';
|
|
121
|
+
}
|
|
122
|
+
const lines = txt.split('\n');
|
|
123
|
+
// Ignore the generic wrapper line — it points elsewhere for the real cause.
|
|
124
|
+
const isReal = (l) =>
|
|
125
|
+
!/Engine core initialization failed/i.test(l) &&
|
|
126
|
+
VLLM_ERROR_PATTERNS.some((re) => re.test(l));
|
|
127
|
+
const firstErr = lines.findIndex(isReal);
|
|
128
|
+
if (firstErr === -1) {
|
|
129
|
+
return lines.slice(-fallbackTail).join('\n').trim();
|
|
130
|
+
}
|
|
131
|
+
const start = Math.max(0, firstErr - 2);
|
|
132
|
+
const end = Math.min(lines.length, firstErr + context);
|
|
133
|
+
return lines.slice(start, end).join('\n').trim();
|
|
134
|
+
}
|
|
135
|
+
|
|
99
136
|
/**
|
|
100
137
|
* Block until vLLM is actually serving `servedName` (weights mapped, :8000
|
|
101
138
|
* answering /v1/models) — NOT just spawned. Returns as soon as the model
|
|
@@ -151,6 +188,22 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
|
|
|
151
188
|
|
|
152
189
|
const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
|
|
153
190
|
if (servedName) args.push('--served-model-name', servedName);
|
|
191
|
+
|
|
192
|
+
// Sane defaults so a model that fits BY WEIGHT doesn't blow up on KV-cache
|
|
193
|
+
// allocation. Many finetunes inherit a huge native context (Qwen3.5 →
|
|
194
|
+
// 256K); vLLM pre-allocates KV cache for the FULL context, which OOMs the
|
|
195
|
+
// EngineCore even when the weights fit easily. Cap the served context and
|
|
196
|
+
// leave GPU headroom. Both overridable via env; only applied when the
|
|
197
|
+
// caller didn't already pass the flag in extraArgs.
|
|
198
|
+
const has = (flag) => extraArgs.includes(flag);
|
|
199
|
+
if (!has('--max-model-len')) {
|
|
200
|
+
const maxLen = process.env.VLLM_MAX_MODEL_LEN || '16384';
|
|
201
|
+
args.push('--max-model-len', String(maxLen));
|
|
202
|
+
}
|
|
203
|
+
if (!has('--gpu-memory-utilization')) {
|
|
204
|
+
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
|
|
205
|
+
args.push('--gpu-memory-utilization', String(util));
|
|
206
|
+
}
|
|
154
207
|
args.push(...extraArgs);
|
|
155
208
|
|
|
156
209
|
const env = { ...process.env };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.49",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,12 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
38
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
33
|
+
"@infernetprotocol/auth": "0.1.49",
|
|
34
|
+
"@infernetprotocol/config": "0.1.49",
|
|
35
|
+
"@infernetprotocol/engine": "0.1.49",
|
|
36
|
+
"@infernetprotocol/gpu": "0.1.49",
|
|
37
|
+
"@infernetprotocol/deploy-providers": "0.1.49",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.49"
|
|
39
39
|
},
|
|
40
40
|
"repository": {
|
|
41
41
|
"type": "git",
|