@infernetprotocol/cli 0.1.49 → 0.1.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/start.js CHANGED
@@ -650,7 +650,10 @@ async function runDaemon(args, ctx) {
650
650
  // fail the command WITH the log root cause instead of lying.
651
651
  const serve = await startVllmServe({ source: hfRepo, servedName: modelName, token });
652
652
  process.stdout.write(`[${new Date().toISOString()}] ${modelName} (${hfRepo}) downloaded; waiting for vLLM to load weights…\n`);
653
- const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
653
+ // Slow (virtualized) GPUs can take 10+ min to bring the
654
+ // engine up; match vLLM's raised VLLM_ENGINE_READY_TIMEOUT_S.
655
+ const vllmWaitMs = Number(process.env.VLLM_ENGINE_READY_TIMEOUT_S || 1800) * 1000;
656
+ const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: vllmWaitMs });
654
657
  if (!up.serving) {
655
658
  const tail = await extractVllmError();
656
659
  throw new Error(
package/lib/vllm.js CHANGED
@@ -112,7 +112,7 @@ const VLLM_ERROR_PATTERNS = [
112
112
  * few useless traceback frames. Finds the first genuine error line and returns
113
113
  * a window around it; falls back to the plain tail if nothing matches.
114
114
  */
115
- export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
115
+ export async function extractVllmError({ headLines = 16, tailLines = 30, fallbackTail = 60 } = {}) {
116
116
  let txt;
117
117
  try {
118
118
  txt = await fs.readFile(logPath(), 'utf8');
@@ -128,9 +128,20 @@ export async function extractVllmError({ context = 24, fallbackTail = 60 } = {})
128
128
  if (firstErr === -1) {
129
129
  return lines.slice(-fallbackTail).join('\n').trim();
130
130
  }
131
- const start = Math.max(0, firstErr - 2);
132
- const end = Math.min(lines.length, firstErr + context);
133
- return lines.slice(start, end).join('\n').trim();
131
+ // The error region runs from the first error line to the end of the log.
132
+ // vLLM's ACTUAL exception (RuntimeError/OutOfMemory/ValueError) is the LAST
133
+ // line of the traceback, so we must show the tail — not just the head where
134
+ // the "EngineCore failed to start" banner sits. For a long region, show the
135
+ // banner + the final exception with the middle elided.
136
+ const region = lines.slice(Math.max(0, firstErr - 2));
137
+ if (region.length <= headLines + tailLines) {
138
+ return region.join('\n').trim();
139
+ }
140
+ return [
141
+ ...region.slice(0, headLines),
142
+ ` … (${region.length - headLines - tailLines} lines elided) …`,
143
+ ...region.slice(-tailLines),
144
+ ].join('\n').trim();
134
145
  }
135
146
 
136
147
  /**
@@ -201,13 +212,42 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
201
212
  args.push('--max-model-len', String(maxLen));
202
213
  }
203
214
  if (!has('--gpu-memory-utilization')) {
204
- const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
215
+ const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.90';
205
216
  args.push('--gpu-memory-utilization', String(util));
206
217
  }
218
+ // --enforce-eager: skip torch.compile + CUDA-graph capture. On normal GPUs
219
+ // that costs a little throughput, but on network-attached/virtualized GPUs
220
+ // (ThunderCompute, vGPU) compile + graph capture are pathologically slow and
221
+ // blow past vLLM's engine-ready timeout — so default it ON for reliable
222
+ // startup. Set VLLM_ENFORCE_EAGER=0 to re-enable compilation on fast GPUs.
223
+ const enforceEager = !/^(0|false|no)$/i.test(process.env.VLLM_ENFORCE_EAGER || '');
224
+ if (enforceEager && !has('--enforce-eager')) {
225
+ args.push('--enforce-eager');
226
+ }
207
227
  args.push(...extraArgs);
208
228
 
209
229
  const env = { ...process.env };
210
230
  if (token) env.HF_TOKEN = token;
231
+ // Put the venv's bin on PATH so vLLM can find tools it shells out to at
232
+ // runtime — notably `ninja`, which FlashInfer needs to JIT-compile its
233
+ // sampling kernel during profile_run. Without this vLLM dies with
234
+ // "FileNotFoundError: 'ninja'" even though it's pip-installed in the venv.
235
+ const venvBin = path.dirname(bin);
236
+ env.PATH = `${venvBin}${path.delimiter}${env.PATH || ''}`;
237
+ // Give the engine plenty of time to come up on slow (virtualized) GPUs —
238
+ // vLLM's own default is 600s, which ThunderCompute exceeds. Overridable.
239
+ if (!env.VLLM_ENGINE_READY_TIMEOUT_S) {
240
+ env.VLLM_ENGINE_READY_TIMEOUT_S = process.env.VLLM_ENGINE_READY_TIMEOUT_S || '1800';
241
+ }
242
+ // Default OFF the V2 model runner. Its GPU worker maps pinned host memory to
243
+ // a device pointer (cudaHostGetDevicePointer / UVA), which FAILS on
244
+ // network-attached & virtualized GPUs (ThunderCompute, many vGPU/cloud
245
+ // providers) with "invalid argument" — killing EngineCore at init_device.
246
+ // The stable V1 runner has no such requirement and works everywhere. On
247
+ // bare-metal you can re-enable it with VLLM_USE_V2_MODEL_RUNNER=1.
248
+ if (env.VLLM_USE_V2_MODEL_RUNNER == null || env.VLLM_USE_V2_MODEL_RUNNER === '') {
249
+ env.VLLM_USE_V2_MODEL_RUNNER = '0';
250
+ }
211
251
 
212
252
  await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
213
253
  const out = fss.openSync(logPath(), 'a');
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.49",
3
+ "version": "0.1.51",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,12 +30,12 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/auth": "0.1.49",
34
- "@infernetprotocol/config": "0.1.49",
35
- "@infernetprotocol/engine": "0.1.49",
36
- "@infernetprotocol/gpu": "0.1.49",
37
- "@infernetprotocol/deploy-providers": "0.1.49",
38
- "@infernetprotocol/rpc-adapter": "0.1.49"
33
+ "@infernetprotocol/config": "0.1.51",
34
+ "@infernetprotocol/deploy-providers": "0.1.51",
35
+ "@infernetprotocol/auth": "0.1.51",
36
+ "@infernetprotocol/engine": "0.1.51",
37
+ "@infernetprotocol/gpu": "0.1.51",
38
+ "@infernetprotocol/rpc-adapter": "0.1.51"
39
39
  },
40
40
  "repository": {
41
41
  "type": "git",