@infernetprotocol/cli 0.1.46 → 0.1.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/start.js CHANGED
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
48
48
  import { gatherCoarseSpecs } from './register.js';
49
49
  import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
50
50
  import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
51
- import { startVllmServe, vllmInstalled } from '../lib/vllm.js';
51
+ import { startVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
52
52
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
53
53
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
54
54
  import { pullLatestBinary } from './upgrade.js';
@@ -234,6 +234,10 @@ async function runDaemon(args, ctx) {
234
234
  const SPECS_TTL_MS = 5 * 60 * 1000;
235
235
  let cachedSpecs = null;
236
236
  let cachedSpecsAt = 0;
237
+ // Force the next freshSpecs() to re-detect (served_models etc.) instead of
238
+ // returning the cached copy. Called after any capability change (model
239
+ // install/remove) so the change is advertised on the very next heartbeat.
240
+ const invalidateSpecsCache = () => { cachedSpecs = null; cachedSpecsAt = 0; };
237
241
 
238
242
  // Rolling benchmark — last N completed chat jobs' (tokens, duration)
239
243
  // pairs. Used to compute tokens_per_second_avg for the heartbeat,
@@ -643,12 +647,30 @@ async function runDaemon(args, ctx) {
643
647
  // under the hf: pull name so it matches chat requests and
644
648
  // gets advertised in served_models → shows in /chat.
645
649
  const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
650
+ // Don't report "completed" on spawn — wait until vLLM has
651
+ // mapped the weights and /v1/models actually lists the model.
652
+ // If it never comes up (OOM, unsupported arch, bad dtype),
653
+ // fail the command WITH the log tail so the operator sees why
654
+ // instead of a lying "completed".
655
+ process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
656
+ const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
657
+ if (!up.serving) {
658
+ const tail = await extractVllmError();
659
+ throw new Error(
660
+ `vLLM failed to serve ${modelName}: ${up.reason}.` +
661
+ (tail ? `\n--- vllm.log (root cause) ---\n${tail}` : ' (no vllm.log output)')
662
+ );
663
+ }
664
+ // Serving for real — bust the specs cache so the very next
665
+ // heartbeat advertises the new model in served_models
666
+ // (otherwise it's stale until the TTL lapses).
667
+ invalidateSpecsCache();
646
668
  result = {
647
669
  model: modelName,
648
670
  backend: 'vllm',
649
671
  localPath,
650
672
  vllm: { pid: serve.pid, port: serve.port },
651
- note: `Serving on vLLM :${serve.port} (first load can take a minute while weights map to GPU).`,
673
+ note: `Serving on vLLM :${serve.port} and advertised in served_models.`,
652
674
  };
653
675
  } else {
654
676
  const fits = await checkModelFits(modelName);
@@ -656,9 +678,11 @@ async function runDaemon(args, ctx) {
656
678
  throw new Error(formatFitFailure(modelName, fits));
657
679
  }
658
680
  result = await ollamaPullWithProgress(modelName, id);
681
+ invalidateSpecsCache();
659
682
  }
660
683
  } else if (command === 'model_remove') {
661
684
  result = await ollamaSpawn(['rm', String(args?.model)]);
685
+ invalidateSpecsCache();
662
686
  } else if (command === 'train_shard') {
663
687
  result = await runTrainShard(args, id);
664
688
  } else {
@@ -666,6 +690,11 @@ async function runDaemon(args, ctx) {
666
690
  }
667
691
  await client.completeCommand(id, { status: 'completed', result });
668
692
  process.stdout.write(`[${new Date().toISOString()}] completed command ${id}\n`);
693
+ // Push the capability change immediately — don't wait up to 30s for
694
+ // the next scheduled heartbeat. freshSpecs() re-detects because the
695
+ // cache was invalidated above, so served_models reflects the change
696
+ // and the model shows in "Installed on node" + /chat right away.
697
+ heartbeat().catch(() => { /* next scheduled beat will retry */ });
669
698
  } catch (err) {
670
699
  const msg = err?.message ?? String(err);
671
700
  process.stderr.write(`command ${id} failed: ${msg}\n`);
package/lib/vllm.js CHANGED
@@ -84,6 +84,76 @@ export async function detectVllmModels(host = VLLM_HOST, timeoutMs = 1500) {
84
84
  }
85
85
  }
86
86
 
87
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
88
+
89
+ /** Last N lines of the vLLM serve log — surfaced on startup failure. */
90
+ export async function readVllmLogTail(lines = 40) {
91
+ try {
92
+ const txt = await fs.readFile(logPath(), 'utf8');
93
+ return txt.split('\n').slice(-lines).join('\n').trim();
94
+ } catch {
95
+ return '';
96
+ }
97
+ }
98
+
99
+ // The lines that actually explain a vLLM EngineCore failure. vLLM prints the
100
+ // root cause (OOM, unsupported arch, dtype/quant mismatch) ABOVE the API-server
101
+ // traceback, then says "See root cause above" — so a plain tail misses it.
102
+ const VLLM_ERROR_PATTERNS = [
103
+ /OutOfMemory|CUDA out of memory|No available memory for the cache|KV cache/i,
104
+ /max_model_len|max seq len|model.?s max/i,
105
+ /not supported|unsupported|no.*implementation|unrecognized|unknown (model|architecture)/i,
106
+ /ValueError|RuntimeError|AssertionError|ImportError|ModuleNotFoundError/,
107
+ /\bERROR\b/,
108
+ ];
109
+
110
+ /**
111
+ * Pull the MEANINGFUL error window out of the vLLM log rather than the last
112
+ * few useless traceback frames. Finds the first genuine error line and returns
113
+ * a window around it; falls back to the plain tail if nothing matches.
114
+ */
115
+ export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
116
+ let txt;
117
+ try {
118
+ txt = await fs.readFile(logPath(), 'utf8');
119
+ } catch {
120
+ return '';
121
+ }
122
+ const lines = txt.split('\n');
123
+ // Ignore the generic wrapper line — it points elsewhere for the real cause.
124
+ const isReal = (l) =>
125
+ !/Engine core initialization failed/i.test(l) &&
126
+ VLLM_ERROR_PATTERNS.some((re) => re.test(l));
127
+ const firstErr = lines.findIndex(isReal);
128
+ if (firstErr === -1) {
129
+ return lines.slice(-fallbackTail).join('\n').trim();
130
+ }
131
+ const start = Math.max(0, firstErr - 2);
132
+ const end = Math.min(lines.length, firstErr + context);
133
+ return lines.slice(start, end).join('\n').trim();
134
+ }
135
+
136
+ /**
137
+ * Block until vLLM is actually serving `servedName` (weights mapped, :8000
138
+ * answering /v1/models) — NOT just spawned. Returns as soon as the model
139
+ * appears, bails early if the process dies, and times out otherwise.
140
+ * A 9B model maps to GPU in ~1-2 min, so the default budget is generous.
141
+ */
142
+ export async function waitForVllmModel(servedName, { pid = null, timeoutMs = 300_000, intervalMs = 3000 } = {}) {
143
+ const deadline = Date.now() + timeoutMs;
144
+ while (Date.now() < deadline) {
145
+ if (pid && !isPidAlive(pid)) {
146
+ return { serving: false, reason: 'vLLM process exited during startup (check the log tail below)' };
147
+ }
148
+ const models = await detectVllmModels();
149
+ if (servedName ? models.includes(servedName) : models.length > 0) {
150
+ return { serving: true, models };
151
+ }
152
+ await sleep(intervalMs);
153
+ }
154
+ return { serving: false, reason: `timed out after ${Math.round(timeoutMs / 1000)}s waiting for /v1/models to list ${servedName}` };
155
+ }
156
+
87
157
  /** True when the tracked serve pid is alive AND /v1/models answers. */
88
158
  export async function isVllmServing() {
89
159
  const st = await readVllmState();
@@ -118,6 +188,22 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
118
188
 
119
189
  const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
120
190
  if (servedName) args.push('--served-model-name', servedName);
191
+
192
+ // Sane defaults so a model that fits BY WEIGHT doesn't blow up on KV-cache
193
+ // allocation. Many finetunes inherit a huge native context (Qwen3.5 →
194
+ // 256K); vLLM pre-allocates KV cache for the FULL context, which OOMs the
195
+ // EngineCore even when the weights fit easily. Cap the served context and
196
+ // leave GPU headroom. Both overridable via env; only applied when the
197
+ // caller didn't already pass the flag in extraArgs.
198
+ const has = (flag) => extraArgs.includes(flag);
199
+ if (!has('--max-model-len')) {
200
+ const maxLen = process.env.VLLM_MAX_MODEL_LEN || '16384';
201
+ args.push('--max-model-len', String(maxLen));
202
+ }
203
+ if (!has('--gpu-memory-utilization')) {
204
+ const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
205
+ args.push('--gpu-memory-utilization', String(util));
206
+ }
121
207
  args.push(...extraArgs);
122
208
 
123
209
  const env = { ...process.env };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.46",
3
+ "version": "0.1.48",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,12 +30,12 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/auth": "0.1.46",
34
- "@infernetprotocol/config": "0.1.46",
35
- "@infernetprotocol/engine": "0.1.46",
36
- "@infernetprotocol/gpu": "0.1.46",
37
- "@infernetprotocol/rpc-adapter": "0.1.46",
38
- "@infernetprotocol/deploy-providers": "0.1.46"
33
+ "@infernetprotocol/auth": "0.1.48",
34
+ "@infernetprotocol/config": "0.1.48",
35
+ "@infernetprotocol/engine": "0.1.48",
36
+ "@infernetprotocol/deploy-providers": "0.1.48",
37
+ "@infernetprotocol/gpu": "0.1.48",
38
+ "@infernetprotocol/rpc-adapter": "0.1.48"
39
39
  },
40
40
  "repository": {
41
41
  "type": "git",