@infernetprotocol/cli 0.1.47 → 0.1.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/start.js CHANGED
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
48
48
  import { gatherCoarseSpecs } from './register.js';
49
49
  import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
50
50
  import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
51
- import { startVllmServe, vllmInstalled, waitForVllmModel, readVllmLogTail } from '../lib/vllm.js';
51
+ import { startVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
52
52
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
53
53
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
54
54
  import { pullLatestBinary } from './upgrade.js';
@@ -629,50 +629,54 @@ async function runDaemon(args, ctx) {
629
629
  let result;
630
630
  if (command === 'model_install') {
631
631
  const modelName = String(args?.model);
632
- if (modelName.startsWith('hf:')) {
633
- // HuggingFace model → download the weights, then TURN ON
634
- // vLLM to serve them (Ollama can't — it 400s on an hf: repo).
635
- if (!vllmInstalled()) {
636
- throw new Error(
637
- `${modelName} needs vLLM, which isn't installed on this node. ` +
638
- `Re-run the installer on this NVIDIA host (it installs vLLM), or: pip install vllm.`
639
- );
640
- }
641
- const repoId = modelName.slice(3);
632
+ // The node decides the backend by HARDWARE: prefer vLLM (GPU),
633
+ // fall back to Ollama (CPU/Mac). A catalog model carries an
634
+ // `hf` repo (ungated, vLLM-servable); a bare `hf:` name is
635
+ // vLLM-only; everything else is an Ollama tag.
636
+ const hfRepo = args?.hf
637
+ ? String(args.hf)
638
+ : (modelName.startsWith('hf:') ? modelName.slice(3) : null);
639
+
640
+ if (vllmInstalled() && hfRepo) {
641
+ // GPU path — serve the HF weights on vLLM. Keep the friendly
642
+ // name (e.g. "qwen2.5:7b" or "hf:org/repo") as the served
643
+ // model name so chat requests + served_models match, but the
644
+ // engine underneath is vLLM.
642
645
  const token = process.env.HF_TOKEN || (await resolveHfToken().catch(() => null));
643
- const localPath = await downloadHfModel(repoId, token);
644
- // Serve the repo id, not a computed cache path — vLLM
645
- // resolves it straight from the HF cache we just filled
646
- // (the snapshot dir is a commit hash, not "main"). Serve
647
- // under the hf: pull name so it matches chat requests and
648
- // gets advertised in served_models → shows in /chat.
649
- const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
646
+ const localPath = await downloadHfModel(hfRepo, token);
650
647
  // Don't report "completed" on spawn — wait until vLLM has
651
648
  // mapped the weights and /v1/models actually lists the model.
652
- // If it never comes up (OOM, unsupported arch, bad dtype),
653
- // fail the command WITH the log tail so the operator sees why
654
- // instead of a lying "completed".
655
- process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
649
+ // If it never comes up (OOM, unsupported arch, gated repo),
650
+ // fail the command WITH the log root cause instead of lying.
651
+ const serve = await startVllmServe({ source: hfRepo, servedName: modelName, token });
652
+ process.stdout.write(`[${new Date().toISOString()}] ${modelName} (${hfRepo}) downloaded; waiting for vLLM to load weights…\n`);
656
653
  const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
657
654
  if (!up.serving) {
658
- const tail = await readVllmLogTail(40);
655
+ const tail = await extractVllmError();
659
656
  throw new Error(
660
- `vLLM failed to serve ${modelName}: ${up.reason}.` +
661
- (tail ? `\n--- vllm.log (last 40 lines) ---\n${tail}` : ' (no vllm.log output)')
657
+ `vLLM failed to serve ${modelName} (${hfRepo}): ${up.reason}.` +
658
+ (tail ? `\n--- vllm.log (root cause) ---\n${tail}` : ' (no vllm.log output)')
662
659
  );
663
660
  }
664
- // Serving for real — bust the specs cache so the very next
665
- // heartbeat advertises the new model in served_models
666
- // (otherwise it's stale until the TTL lapses).
667
661
  invalidateSpecsCache();
668
662
  result = {
669
663
  model: modelName,
670
664
  backend: 'vllm',
665
+ source: hfRepo,
671
666
  localPath,
672
667
  vllm: { pid: serve.pid, port: serve.port },
673
668
  note: `Serving on vLLM :${serve.port} and advertised in served_models.`,
674
669
  };
670
+ } else if (modelName.startsWith('hf:')) {
671
+ // hf: model requested but this node has no vLLM (CPU/Mac).
672
+ throw new Error(
673
+ `${modelName} needs vLLM, which isn't installed on this node. ` +
674
+ `Re-run the installer on an NVIDIA host (it installs vLLM), or: pip install vllm. ` +
675
+ `Alternatively push an Ollama model — those run on CPU.`
676
+ );
675
677
  } else {
678
+ // CPU/Mac path (or a GPU box without a vLLM-servable repo) —
679
+ // Ollama serves the GGUF tag.
676
680
  const fits = await checkModelFits(modelName);
677
681
  if (fits && !fits.ok) {
678
682
  throw new Error(formatFitFailure(modelName, fits));
package/lib/vllm.js CHANGED
@@ -96,6 +96,43 @@ export async function readVllmLogTail(lines = 40) {
96
96
  }
97
97
  }
98
98
 
99
+ // The lines that actually explain a vLLM EngineCore failure. vLLM prints the
100
+ // root cause (OOM, unsupported arch, dtype/quant mismatch) ABOVE the API-server
101
+ // traceback, then says "See root cause above" — so a plain tail misses it.
102
+ const VLLM_ERROR_PATTERNS = [
103
+ /OutOfMemory|CUDA out of memory|No available memory for the cache|KV cache/i,
104
+ /max_model_len|max seq len|model.?s max/i,
105
+ /not supported|unsupported|no.*implementation|unrecognized|unknown (model|architecture)/i,
106
+ /ValueError|RuntimeError|AssertionError|ImportError|ModuleNotFoundError/,
107
+ /\bERROR\b/,
108
+ ];
109
+
110
+ /**
111
+ * Pull the MEANINGFUL error window out of the vLLM log rather than the last
112
+ * few useless traceback frames. Finds the first genuine error line and returns
113
+ * a window around it; falls back to the plain tail if nothing matches.
114
+ */
115
+ export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
116
+ let txt;
117
+ try {
118
+ txt = await fs.readFile(logPath(), 'utf8');
119
+ } catch {
120
+ return '';
121
+ }
122
+ const lines = txt.split('\n');
123
+ // Ignore the generic wrapper line — it points elsewhere for the real cause.
124
+ const isReal = (l) =>
125
+ !/Engine core initialization failed/i.test(l) &&
126
+ VLLM_ERROR_PATTERNS.some((re) => re.test(l));
127
+ const firstErr = lines.findIndex(isReal);
128
+ if (firstErr === -1) {
129
+ return lines.slice(-fallbackTail).join('\n').trim();
130
+ }
131
+ const start = Math.max(0, firstErr - 2);
132
+ const end = Math.min(lines.length, firstErr + context);
133
+ return lines.slice(start, end).join('\n').trim();
134
+ }
135
+
99
136
  /**
100
137
  * Block until vLLM is actually serving `servedName` (weights mapped, :8000
101
138
  * answering /v1/models) — NOT just spawned. Returns as soon as the model
@@ -151,6 +188,22 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
151
188
 
152
189
  const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
153
190
  if (servedName) args.push('--served-model-name', servedName);
191
+
192
+ // Sane defaults so a model that fits BY WEIGHT doesn't blow up on KV-cache
193
+ // allocation. Many finetunes inherit a huge native context (Qwen3.5 →
194
+ // 256K); vLLM pre-allocates KV cache for the FULL context, which OOMs the
195
+ // EngineCore even when the weights fit easily. Cap the served context and
196
+ // leave GPU headroom. Both overridable via env; only applied when the
197
+ // caller didn't already pass the flag in extraArgs.
198
+ const has = (flag) => extraArgs.includes(flag);
199
+ if (!has('--max-model-len')) {
200
+ const maxLen = process.env.VLLM_MAX_MODEL_LEN || '16384';
201
+ args.push('--max-model-len', String(maxLen));
202
+ }
203
+ if (!has('--gpu-memory-utilization')) {
204
+ const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
205
+ args.push('--gpu-memory-utilization', String(util));
206
+ }
154
207
  args.push(...extraArgs);
155
208
 
156
209
  const env = { ...process.env };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.47",
3
+ "version": "0.1.49",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,12 +30,12 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/config": "0.1.47",
34
- "@infernetprotocol/auth": "0.1.47",
35
- "@infernetprotocol/deploy-providers": "0.1.47",
36
- "@infernetprotocol/engine": "0.1.47",
37
- "@infernetprotocol/gpu": "0.1.47",
38
- "@infernetprotocol/rpc-adapter": "0.1.47"
33
+ "@infernetprotocol/auth": "0.1.49",
34
+ "@infernetprotocol/config": "0.1.49",
35
+ "@infernetprotocol/engine": "0.1.49",
36
+ "@infernetprotocol/gpu": "0.1.49",
37
+ "@infernetprotocol/deploy-providers": "0.1.49",
38
+ "@infernetprotocol/rpc-adapter": "0.1.49"
39
39
  },
40
40
  "repository": {
41
41
  "type": "git",