@infernetprotocol/cli 0.1.49 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +4 -1
- package/lib/vllm.js +45 -5
- package/package.json +7 -7
package/commands/start.js
CHANGED
|
@@ -650,7 +650,10 @@ async function runDaemon(args, ctx) {
|
|
|
650
650
|
// fail the command WITH the log root cause instead of lying.
|
|
651
651
|
const serve = await startVllmServe({ source: hfRepo, servedName: modelName, token });
|
|
652
652
|
process.stdout.write(`[${new Date().toISOString()}] ${modelName} (${hfRepo}) downloaded; waiting for vLLM to load weights…\n`);
|
|
653
|
-
|
|
653
|
+
// Slow (virtualized) GPUs can take 10+ min to bring the
|
|
654
|
+
// engine up; match vLLM's raised VLLM_ENGINE_READY_TIMEOUT_S.
|
|
655
|
+
const vllmWaitMs = Number(process.env.VLLM_ENGINE_READY_TIMEOUT_S || 1800) * 1000;
|
|
656
|
+
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: vllmWaitMs });
|
|
654
657
|
if (!up.serving) {
|
|
655
658
|
const tail = await extractVllmError();
|
|
656
659
|
throw new Error(
|
package/lib/vllm.js
CHANGED
|
@@ -112,7 +112,7 @@ const VLLM_ERROR_PATTERNS = [
|
|
|
112
112
|
* few useless traceback frames. Finds the first genuine error line and returns
|
|
113
113
|
* a window around it; falls back to the plain tail if nothing matches.
|
|
114
114
|
*/
|
|
115
|
-
export async function extractVllmError({
|
|
115
|
+
export async function extractVllmError({ headLines = 16, tailLines = 30, fallbackTail = 60 } = {}) {
|
|
116
116
|
let txt;
|
|
117
117
|
try {
|
|
118
118
|
txt = await fs.readFile(logPath(), 'utf8');
|
|
@@ -128,9 +128,20 @@ export async function extractVllmError({ context = 24, fallbackTail = 60 } = {})
|
|
|
128
128
|
if (firstErr === -1) {
|
|
129
129
|
return lines.slice(-fallbackTail).join('\n').trim();
|
|
130
130
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
// The error region runs from the first error line to the end of the log.
|
|
132
|
+
// vLLM's ACTUAL exception (RuntimeError/OutOfMemory/ValueError) is the LAST
|
|
133
|
+
// line of the traceback, so we must show the tail — not just the head where
|
|
134
|
+
// the "EngineCore failed to start" banner sits. For a long region, show the
|
|
135
|
+
// banner + the final exception with the middle elided.
|
|
136
|
+
const region = lines.slice(Math.max(0, firstErr - 2));
|
|
137
|
+
if (region.length <= headLines + tailLines) {
|
|
138
|
+
return region.join('\n').trim();
|
|
139
|
+
}
|
|
140
|
+
return [
|
|
141
|
+
...region.slice(0, headLines),
|
|
142
|
+
` … (${region.length - headLines - tailLines} lines elided) …`,
|
|
143
|
+
...region.slice(-tailLines),
|
|
144
|
+
].join('\n').trim();
|
|
134
145
|
}
|
|
135
146
|
|
|
136
147
|
/**
|
|
@@ -201,13 +212,42 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
|
|
|
201
212
|
args.push('--max-model-len', String(maxLen));
|
|
202
213
|
}
|
|
203
214
|
if (!has('--gpu-memory-utilization')) {
|
|
204
|
-
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.
|
|
215
|
+
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.90';
|
|
205
216
|
args.push('--gpu-memory-utilization', String(util));
|
|
206
217
|
}
|
|
218
|
+
// --enforce-eager: skip torch.compile + CUDA-graph capture. On normal GPUs
|
|
219
|
+
// that costs a little throughput, but on network-attached/virtualized GPUs
|
|
220
|
+
// (ThunderCompute, vGPU) compile + graph capture are pathologically slow and
|
|
221
|
+
// blow past vLLM's engine-ready timeout — so default it ON for reliable
|
|
222
|
+
// startup. Set VLLM_ENFORCE_EAGER=0 to re-enable compilation on fast GPUs.
|
|
223
|
+
const enforceEager = !/^(0|false|no)$/i.test(process.env.VLLM_ENFORCE_EAGER || '');
|
|
224
|
+
if (enforceEager && !has('--enforce-eager')) {
|
|
225
|
+
args.push('--enforce-eager');
|
|
226
|
+
}
|
|
207
227
|
args.push(...extraArgs);
|
|
208
228
|
|
|
209
229
|
const env = { ...process.env };
|
|
210
230
|
if (token) env.HF_TOKEN = token;
|
|
231
|
+
// Put the venv's bin on PATH so vLLM can find tools it shells out to at
|
|
232
|
+
// runtime — notably `ninja`, which FlashInfer needs to JIT-compile its
|
|
233
|
+
// sampling kernel during profile_run. Without this vLLM dies with
|
|
234
|
+
// "FileNotFoundError: 'ninja'" even though it's pip-installed in the venv.
|
|
235
|
+
const venvBin = path.dirname(bin);
|
|
236
|
+
env.PATH = `${venvBin}${path.delimiter}${env.PATH || ''}`;
|
|
237
|
+
// Give the engine plenty of time to come up on slow (virtualized) GPUs —
|
|
238
|
+
// vLLM's own default is 600s, which ThunderCompute exceeds. Overridable.
|
|
239
|
+
if (!env.VLLM_ENGINE_READY_TIMEOUT_S) {
|
|
240
|
+
env.VLLM_ENGINE_READY_TIMEOUT_S = process.env.VLLM_ENGINE_READY_TIMEOUT_S || '1800';
|
|
241
|
+
}
|
|
242
|
+
// Default OFF the V2 model runner. Its GPU worker maps pinned host memory to
|
|
243
|
+
// a device pointer (cudaHostGetDevicePointer / UVA), which FAILS on
|
|
244
|
+
// network-attached & virtualized GPUs (ThunderCompute, many vGPU/cloud
|
|
245
|
+
// providers) with "invalid argument" — killing EngineCore at init_device.
|
|
246
|
+
// The stable V1 runner has no such requirement and works everywhere. On
|
|
247
|
+
// bare-metal you can re-enable it with VLLM_USE_V2_MODEL_RUNNER=1.
|
|
248
|
+
if (env.VLLM_USE_V2_MODEL_RUNNER == null || env.VLLM_USE_V2_MODEL_RUNNER === '') {
|
|
249
|
+
env.VLLM_USE_V2_MODEL_RUNNER = '0';
|
|
250
|
+
}
|
|
211
251
|
|
|
212
252
|
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
213
253
|
const out = fss.openSync(logPath(), 'a');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.51",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,12 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
38
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
33
|
+
"@infernetprotocol/config": "0.1.51",
|
|
34
|
+
"@infernetprotocol/deploy-providers": "0.1.51",
|
|
35
|
+
"@infernetprotocol/auth": "0.1.51",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.51",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.51",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.51"
|
|
39
39
|
},
|
|
40
40
|
"repository": {
|
|
41
41
|
"type": "git",
|