@infernetprotocol/cli 0.1.47 → 0.1.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/start.js CHANGED
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
48
48
  import { gatherCoarseSpecs } from './register.js';
49
49
  import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
50
50
  import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
51
- import { startVllmServe, vllmInstalled, waitForVllmModel, readVllmLogTail } from '../lib/vllm.js';
51
+ import { startVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
52
52
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
53
53
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
54
54
  import { pullLatestBinary } from './upgrade.js';
@@ -655,10 +655,10 @@ async function runDaemon(args, ctx) {
655
655
  process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
656
656
  const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
657
657
  if (!up.serving) {
658
- const tail = await readVllmLogTail(40);
658
+ const tail = await extractVllmError();
659
659
  throw new Error(
660
660
  `vLLM failed to serve ${modelName}: ${up.reason}.` +
661
- (tail ? `\n--- vllm.log (last 40 lines) ---\n${tail}` : ' (no vllm.log output)')
661
+ (tail ? `\n--- vllm.log (root cause) ---\n${tail}` : ' (no vllm.log output)')
662
662
  );
663
663
  }
664
664
  // Serving for real — bust the specs cache so the very next
package/lib/vllm.js CHANGED
@@ -96,6 +96,43 @@ export async function readVllmLogTail(lines = 40) {
96
96
  }
97
97
  }
98
98
 
99
+ // The lines that actually explain a vLLM EngineCore failure. vLLM prints the
100
+ // root cause (OOM, unsupported arch, dtype/quant mismatch) ABOVE the API-server
101
+ // traceback, then says "See root cause above" — so a plain tail misses it.
102
+ const VLLM_ERROR_PATTERNS = [
103
+ /OutOfMemory|CUDA out of memory|No available memory for the cache|KV cache/i,
104
+ /max_model_len|max seq len|model.?s max/i,
105
+ /not supported|unsupported|no.*implementation|unrecognized|unknown (model|architecture)/i,
106
+ /ValueError|RuntimeError|AssertionError|ImportError|ModuleNotFoundError/,
107
+ /\bERROR\b/,
108
+ ];
109
+
110
+ /**
111
+ * Pull the MEANINGFUL error window out of the vLLM log rather than the last
112
+ * few useless traceback frames. Finds the first genuine error line and returns
113
+ * a window around it; falls back to the plain tail if nothing matches.
114
+ */
115
+ export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
116
+ let txt;
117
+ try {
118
+ txt = await fs.readFile(logPath(), 'utf8');
119
+ } catch {
120
+ return '';
121
+ }
122
+ const lines = txt.split('\n');
123
+ // Ignore the generic wrapper line — it points elsewhere for the real cause.
124
+ const isReal = (l) =>
125
+ !/Engine core initialization failed/i.test(l) &&
126
+ VLLM_ERROR_PATTERNS.some((re) => re.test(l));
127
+ const firstErr = lines.findIndex(isReal);
128
+ if (firstErr === -1) {
129
+ return lines.slice(-fallbackTail).join('\n').trim();
130
+ }
131
+ const start = Math.max(0, firstErr - 2);
132
+ const end = Math.min(lines.length, firstErr + context);
133
+ return lines.slice(start, end).join('\n').trim();
134
+ }
135
+
99
136
  /**
100
137
  * Block until vLLM is actually serving `servedName` (weights mapped, :8000
101
138
  * answering /v1/models) — NOT just spawned. Returns as soon as the model
@@ -151,6 +188,22 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
151
188
 
152
189
  const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
153
190
  if (servedName) args.push('--served-model-name', servedName);
191
+
192
+ // Sane defaults so a model that fits BY WEIGHT doesn't blow up on KV-cache
193
+ // allocation. Many finetunes inherit a huge native context (Qwen3.5 →
194
+ // 256K); vLLM pre-allocates KV cache for the FULL context, which OOMs the
195
+ // EngineCore even when the weights fit easily. Cap the served context and
196
+ // leave GPU headroom. Both overridable via env; only applied when the
197
+ // caller didn't already pass the flag in extraArgs.
198
+ const has = (flag) => extraArgs.includes(flag);
199
+ if (!has('--max-model-len')) {
200
+ const maxLen = process.env.VLLM_MAX_MODEL_LEN || '16384';
201
+ args.push('--max-model-len', String(maxLen));
202
+ }
203
+ if (!has('--gpu-memory-utilization')) {
204
+ const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
205
+ args.push('--gpu-memory-utilization', String(util));
206
+ }
154
207
  args.push(...extraArgs);
155
208
 
156
209
  const env = { ...process.env };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.47",
3
+ "version": "0.1.48",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,12 +30,12 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/config": "0.1.47",
34
- "@infernetprotocol/auth": "0.1.47",
35
- "@infernetprotocol/deploy-providers": "0.1.47",
36
- "@infernetprotocol/engine": "0.1.47",
37
- "@infernetprotocol/gpu": "0.1.47",
38
- "@infernetprotocol/rpc-adapter": "0.1.47"
33
+ "@infernetprotocol/auth": "0.1.48",
34
+ "@infernetprotocol/config": "0.1.48",
35
+ "@infernetprotocol/engine": "0.1.48",
36
+ "@infernetprotocol/deploy-providers": "0.1.48",
37
+ "@infernetprotocol/gpu": "0.1.48",
38
+ "@infernetprotocol/rpc-adapter": "0.1.48"
39
39
  },
40
40
  "repository": {
41
41
  "type": "git",