@infernetprotocol/cli 0.1.47 → 0.1.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +3 -3
- package/lib/vllm.js +53 -0
- package/package.json +7 -7
package/commands/start.js
CHANGED
|
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
|
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
49
|
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
50
|
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
51
|
-
import { startVllmServe, vllmInstalled, waitForVllmModel,
|
|
51
|
+
import { startVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
|
|
52
52
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
53
53
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
54
54
|
import { pullLatestBinary } from './upgrade.js';
|
|
@@ -655,10 +655,10 @@ async function runDaemon(args, ctx) {
|
|
|
655
655
|
process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
|
|
656
656
|
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
|
|
657
657
|
if (!up.serving) {
|
|
658
|
-
const tail = await
|
|
658
|
+
const tail = await extractVllmError();
|
|
659
659
|
throw new Error(
|
|
660
660
|
`vLLM failed to serve ${modelName}: ${up.reason}.` +
|
|
661
|
-
(tail ? `\n--- vllm.log (
|
|
661
|
+
(tail ? `\n--- vllm.log (root cause) ---\n${tail}` : ' (no vllm.log output)')
|
|
662
662
|
);
|
|
663
663
|
}
|
|
664
664
|
// Serving for real — bust the specs cache so the very next
|
package/lib/vllm.js
CHANGED
|
@@ -96,6 +96,43 @@ export async function readVllmLogTail(lines = 40) {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
// The lines that actually explain a vLLM EngineCore failure. vLLM prints the
|
|
100
|
+
// root cause (OOM, unsupported arch, dtype/quant mismatch) ABOVE the API-server
|
|
101
|
+
// traceback, then says "See root cause above" — so a plain tail misses it.
|
|
102
|
+
const VLLM_ERROR_PATTERNS = [
|
|
103
|
+
/OutOfMemory|CUDA out of memory|No available memory for the cache|KV cache/i,
|
|
104
|
+
/max_model_len|max seq len|model.?s max/i,
|
|
105
|
+
/not supported|unsupported|no.*implementation|unrecognized|unknown (model|architecture)/i,
|
|
106
|
+
/ValueError|RuntimeError|AssertionError|ImportError|ModuleNotFoundError/,
|
|
107
|
+
/\bERROR\b/,
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Pull the MEANINGFUL error window out of the vLLM log rather than the last
|
|
112
|
+
* few useless traceback frames. Finds the first genuine error line and returns
|
|
113
|
+
* a window around it; falls back to the plain tail if nothing matches.
|
|
114
|
+
*/
|
|
115
|
+
export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
|
|
116
|
+
let txt;
|
|
117
|
+
try {
|
|
118
|
+
txt = await fs.readFile(logPath(), 'utf8');
|
|
119
|
+
} catch {
|
|
120
|
+
return '';
|
|
121
|
+
}
|
|
122
|
+
const lines = txt.split('\n');
|
|
123
|
+
// Ignore the generic wrapper line — it points elsewhere for the real cause.
|
|
124
|
+
const isReal = (l) =>
|
|
125
|
+
!/Engine core initialization failed/i.test(l) &&
|
|
126
|
+
VLLM_ERROR_PATTERNS.some((re) => re.test(l));
|
|
127
|
+
const firstErr = lines.findIndex(isReal);
|
|
128
|
+
if (firstErr === -1) {
|
|
129
|
+
return lines.slice(-fallbackTail).join('\n').trim();
|
|
130
|
+
}
|
|
131
|
+
const start = Math.max(0, firstErr - 2);
|
|
132
|
+
const end = Math.min(lines.length, firstErr + context);
|
|
133
|
+
return lines.slice(start, end).join('\n').trim();
|
|
134
|
+
}
|
|
135
|
+
|
|
99
136
|
/**
|
|
100
137
|
* Block until vLLM is actually serving `servedName` (weights mapped, :8000
|
|
101
138
|
* answering /v1/models) — NOT just spawned. Returns as soon as the model
|
|
@@ -151,6 +188,22 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
|
|
|
151
188
|
|
|
152
189
|
const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
|
|
153
190
|
if (servedName) args.push('--served-model-name', servedName);
|
|
191
|
+
|
|
192
|
+
// Sane defaults so a model that fits BY WEIGHT doesn't blow up on KV-cache
|
|
193
|
+
// allocation. Many finetunes inherit a huge native context (Qwen3.5 →
|
|
194
|
+
// 256K); vLLM pre-allocates KV cache for the FULL context, which OOMs the
|
|
195
|
+
// EngineCore even when the weights fit easily. Cap the served context and
|
|
196
|
+
// leave GPU headroom. Both overridable via env; only applied when the
|
|
197
|
+
// caller didn't already pass the flag in extraArgs.
|
|
198
|
+
const has = (flag) => extraArgs.includes(flag);
|
|
199
|
+
if (!has('--max-model-len')) {
|
|
200
|
+
const maxLen = process.env.VLLM_MAX_MODEL_LEN || '16384';
|
|
201
|
+
args.push('--max-model-len', String(maxLen));
|
|
202
|
+
}
|
|
203
|
+
if (!has('--gpu-memory-utilization')) {
|
|
204
|
+
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
|
|
205
|
+
args.push('--gpu-memory-utilization', String(util));
|
|
206
|
+
}
|
|
154
207
|
args.push(...extraArgs);
|
|
155
208
|
|
|
156
209
|
const env = { ...process.env };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.48",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,12 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/gpu": "0.1.
|
|
38
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
33
|
+
"@infernetprotocol/auth": "0.1.48",
|
|
34
|
+
"@infernetprotocol/config": "0.1.48",
|
|
35
|
+
"@infernetprotocol/engine": "0.1.48",
|
|
36
|
+
"@infernetprotocol/deploy-providers": "0.1.48",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.48",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.48"
|
|
39
39
|
},
|
|
40
40
|
"repository": {
|
|
41
41
|
"type": "git",
|