@infernetprotocol/cli 0.1.50 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +4 -1
- package/lib/vllm.js +30 -1
- package/package.json +7 -7
package/commands/start.js
CHANGED
|
@@ -650,7 +650,10 @@ async function runDaemon(args, ctx) {
|
|
|
650
650
|
// fail the command WITH the log root cause instead of lying.
|
|
651
651
|
const serve = await startVllmServe({ source: hfRepo, servedName: modelName, token });
|
|
652
652
|
process.stdout.write(`[${new Date().toISOString()}] ${modelName} (${hfRepo}) downloaded; waiting for vLLM to load weights…\n`);
|
|
653
|
-
|
|
653
|
+
// Slow (virtualized) GPUs can take 10+ min to bring the
|
|
654
|
+
// engine up; match vLLM's raised VLLM_ENGINE_READY_TIMEOUT_S.
|
|
655
|
+
const vllmWaitMs = Number(process.env.VLLM_ENGINE_READY_TIMEOUT_S || 1800) * 1000;
|
|
656
|
+
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: vllmWaitMs });
|
|
654
657
|
if (!up.serving) {
|
|
655
658
|
const tail = await extractVllmError();
|
|
656
659
|
throw new Error(
|
package/lib/vllm.js
CHANGED
|
@@ -212,13 +212,42 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
|
|
|
212
212
|
args.push('--max-model-len', String(maxLen));
|
|
213
213
|
}
|
|
214
214
|
if (!has('--gpu-memory-utilization')) {
|
|
215
|
-
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.
|
|
215
|
+
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.90';
|
|
216
216
|
args.push('--gpu-memory-utilization', String(util));
|
|
217
217
|
}
|
|
218
|
+
// --enforce-eager: skip torch.compile + CUDA-graph capture. On normal GPUs
|
|
219
|
+
// that costs a little throughput, but on network-attached/virtualized GPUs
|
|
220
|
+
// (ThunderCompute, vGPU) compile + graph capture are pathologically slow and
|
|
221
|
+
// blow past vLLM's engine-ready timeout — so default it ON for reliable
|
|
222
|
+
// startup. Set VLLM_ENFORCE_EAGER=0 to re-enable compilation on fast GPUs.
|
|
223
|
+
const enforceEager = !/^(0|false|no)$/i.test(process.env.VLLM_ENFORCE_EAGER || '');
|
|
224
|
+
if (enforceEager && !has('--enforce-eager')) {
|
|
225
|
+
args.push('--enforce-eager');
|
|
226
|
+
}
|
|
218
227
|
args.push(...extraArgs);
|
|
219
228
|
|
|
220
229
|
const env = { ...process.env };
|
|
221
230
|
if (token) env.HF_TOKEN = token;
|
|
231
|
+
// Put the venv's bin on PATH so vLLM can find tools it shells out to at
|
|
232
|
+
// runtime — notably `ninja`, which FlashInfer needs to JIT-compile its
|
|
233
|
+
// sampling kernel during profile_run. Without this vLLM dies with
|
|
234
|
+
// "FileNotFoundError: 'ninja'" even though it's pip-installed in the venv.
|
|
235
|
+
const venvBin = path.dirname(bin);
|
|
236
|
+
env.PATH = `${venvBin}${path.delimiter}${env.PATH || ''}`;
|
|
237
|
+
// Give the engine plenty of time to come up on slow (virtualized) GPUs —
|
|
238
|
+
// vLLM's own default is 600s, which ThunderCompute exceeds. Overridable.
|
|
239
|
+
if (!env.VLLM_ENGINE_READY_TIMEOUT_S) {
|
|
240
|
+
env.VLLM_ENGINE_READY_TIMEOUT_S = process.env.VLLM_ENGINE_READY_TIMEOUT_S || '1800';
|
|
241
|
+
}
|
|
242
|
+
// Default OFF the V2 model runner. Its GPU worker maps pinned host memory to
|
|
243
|
+
// a device pointer (cudaHostGetDevicePointer / UVA), which FAILS on
|
|
244
|
+
// network-attached & virtualized GPUs (ThunderCompute, many vGPU/cloud
|
|
245
|
+
// providers) with "invalid argument" — killing EngineCore at init_device.
|
|
246
|
+
// The stable V1 runner has no such requirement and works everywhere. On
|
|
247
|
+
// bare-metal you can re-enable it with VLLM_USE_V2_MODEL_RUNNER=1.
|
|
248
|
+
if (env.VLLM_USE_V2_MODEL_RUNNER == null || env.VLLM_USE_V2_MODEL_RUNNER === '') {
|
|
249
|
+
env.VLLM_USE_V2_MODEL_RUNNER = '0';
|
|
250
|
+
}
|
|
222
251
|
|
|
223
252
|
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
224
253
|
const out = fss.openSync(logPath(), 'a');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.51",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,12 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
38
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
33
|
+
"@infernetprotocol/config": "0.1.51",
|
|
34
|
+
"@infernetprotocol/deploy-providers": "0.1.51",
|
|
35
|
+
"@infernetprotocol/auth": "0.1.51",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.51",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.51",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.51"
|
|
39
39
|
},
|
|
40
40
|
"repository": {
|
|
41
41
|
"type": "git",
|