@infernetprotocol/cli 0.1.46 → 0.1.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +31 -2
- package/lib/vllm.js +86 -0
- package/package.json +7 -7
package/commands/start.js
CHANGED
|
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
|
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
49
|
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
50
|
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
51
|
-
import { startVllmServe, vllmInstalled } from '../lib/vllm.js';
|
|
51
|
+
import { startVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
|
|
52
52
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
53
53
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
54
54
|
import { pullLatestBinary } from './upgrade.js';
|
|
@@ -234,6 +234,10 @@ async function runDaemon(args, ctx) {
|
|
|
234
234
|
const SPECS_TTL_MS = 5 * 60 * 1000;
|
|
235
235
|
let cachedSpecs = null;
|
|
236
236
|
let cachedSpecsAt = 0;
|
|
237
|
+
// Force the next freshSpecs() to re-detect (served_models etc.) instead of
|
|
238
|
+
// returning the cached copy. Called after any capability change (model
|
|
239
|
+
// install/remove) so the change is advertised on the very next heartbeat.
|
|
240
|
+
const invalidateSpecsCache = () => { cachedSpecs = null; cachedSpecsAt = 0; };
|
|
237
241
|
|
|
238
242
|
// Rolling benchmark — last N completed chat jobs' (tokens, duration)
|
|
239
243
|
// pairs. Used to compute tokens_per_second_avg for the heartbeat,
|
|
@@ -643,12 +647,30 @@ async function runDaemon(args, ctx) {
|
|
|
643
647
|
// under the hf: pull name so it matches chat requests and
|
|
644
648
|
// gets advertised in served_models → shows in /chat.
|
|
645
649
|
const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
|
|
650
|
+
// Don't report "completed" on spawn — wait until vLLM has
|
|
651
|
+
// mapped the weights and /v1/models actually lists the model.
|
|
652
|
+
// If it never comes up (OOM, unsupported arch, bad dtype),
|
|
653
|
+
// fail the command WITH the log tail so the operator sees why
|
|
654
|
+
// instead of a lying "completed".
|
|
655
|
+
process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
|
|
656
|
+
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
|
|
657
|
+
if (!up.serving) {
|
|
658
|
+
const tail = await extractVllmError();
|
|
659
|
+
throw new Error(
|
|
660
|
+
`vLLM failed to serve ${modelName}: ${up.reason}.` +
|
|
661
|
+
(tail ? `\n--- vllm.log (root cause) ---\n${tail}` : ' (no vllm.log output)')
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
// Serving for real — bust the specs cache so the very next
|
|
665
|
+
// heartbeat advertises the new model in served_models
|
|
666
|
+
// (otherwise it's stale until the TTL lapses).
|
|
667
|
+
invalidateSpecsCache();
|
|
646
668
|
result = {
|
|
647
669
|
model: modelName,
|
|
648
670
|
backend: 'vllm',
|
|
649
671
|
localPath,
|
|
650
672
|
vllm: { pid: serve.pid, port: serve.port },
|
|
651
|
-
note: `Serving on vLLM :${serve.port}
|
|
673
|
+
note: `Serving on vLLM :${serve.port} and advertised in served_models.`,
|
|
652
674
|
};
|
|
653
675
|
} else {
|
|
654
676
|
const fits = await checkModelFits(modelName);
|
|
@@ -656,9 +678,11 @@ async function runDaemon(args, ctx) {
|
|
|
656
678
|
throw new Error(formatFitFailure(modelName, fits));
|
|
657
679
|
}
|
|
658
680
|
result = await ollamaPullWithProgress(modelName, id);
|
|
681
|
+
invalidateSpecsCache();
|
|
659
682
|
}
|
|
660
683
|
} else if (command === 'model_remove') {
|
|
661
684
|
result = await ollamaSpawn(['rm', String(args?.model)]);
|
|
685
|
+
invalidateSpecsCache();
|
|
662
686
|
} else if (command === 'train_shard') {
|
|
663
687
|
result = await runTrainShard(args, id);
|
|
664
688
|
} else {
|
|
@@ -666,6 +690,11 @@ async function runDaemon(args, ctx) {
|
|
|
666
690
|
}
|
|
667
691
|
await client.completeCommand(id, { status: 'completed', result });
|
|
668
692
|
process.stdout.write(`[${new Date().toISOString()}] completed command ${id}\n`);
|
|
693
|
+
// Push the capability change immediately — don't wait up to 30s for
|
|
694
|
+
// the next scheduled heartbeat. freshSpecs() re-detects because the
|
|
695
|
+
// cache was invalidated above, so served_models reflects the change
|
|
696
|
+
// and the model shows in "Installed on node" + /chat right away.
|
|
697
|
+
heartbeat().catch(() => { /* next scheduled beat will retry */ });
|
|
669
698
|
} catch (err) {
|
|
670
699
|
const msg = err?.message ?? String(err);
|
|
671
700
|
process.stderr.write(`command ${id} failed: ${msg}\n`);
|
package/lib/vllm.js
CHANGED
|
@@ -84,6 +84,76 @@ export async function detectVllmModels(host = VLLM_HOST, timeoutMs = 1500) {
|
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
88
|
+
|
|
89
|
+
/** Last N lines of the vLLM serve log — surfaced on startup failure. */
|
|
90
|
+
export async function readVllmLogTail(lines = 40) {
|
|
91
|
+
try {
|
|
92
|
+
const txt = await fs.readFile(logPath(), 'utf8');
|
|
93
|
+
return txt.split('\n').slice(-lines).join('\n').trim();
|
|
94
|
+
} catch {
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// The lines that actually explain a vLLM EngineCore failure. vLLM prints the
|
|
100
|
+
// root cause (OOM, unsupported arch, dtype/quant mismatch) ABOVE the API-server
|
|
101
|
+
// traceback, then says "See root cause above" — so a plain tail misses it.
|
|
102
|
+
const VLLM_ERROR_PATTERNS = [
|
|
103
|
+
/OutOfMemory|CUDA out of memory|No available memory for the cache|KV cache/i,
|
|
104
|
+
/max_model_len|max seq len|model.?s max/i,
|
|
105
|
+
/not supported|unsupported|no.*implementation|unrecognized|unknown (model|architecture)/i,
|
|
106
|
+
/ValueError|RuntimeError|AssertionError|ImportError|ModuleNotFoundError/,
|
|
107
|
+
/\bERROR\b/,
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Pull the MEANINGFUL error window out of the vLLM log rather than the last
|
|
112
|
+
* few useless traceback frames. Finds the first genuine error line and returns
|
|
113
|
+
* a window around it; falls back to the plain tail if nothing matches.
|
|
114
|
+
*/
|
|
115
|
+
export async function extractVllmError({ context = 24, fallbackTail = 60 } = {}) {
|
|
116
|
+
let txt;
|
|
117
|
+
try {
|
|
118
|
+
txt = await fs.readFile(logPath(), 'utf8');
|
|
119
|
+
} catch {
|
|
120
|
+
return '';
|
|
121
|
+
}
|
|
122
|
+
const lines = txt.split('\n');
|
|
123
|
+
// Ignore the generic wrapper line — it points elsewhere for the real cause.
|
|
124
|
+
const isReal = (l) =>
|
|
125
|
+
!/Engine core initialization failed/i.test(l) &&
|
|
126
|
+
VLLM_ERROR_PATTERNS.some((re) => re.test(l));
|
|
127
|
+
const firstErr = lines.findIndex(isReal);
|
|
128
|
+
if (firstErr === -1) {
|
|
129
|
+
return lines.slice(-fallbackTail).join('\n').trim();
|
|
130
|
+
}
|
|
131
|
+
const start = Math.max(0, firstErr - 2);
|
|
132
|
+
const end = Math.min(lines.length, firstErr + context);
|
|
133
|
+
return lines.slice(start, end).join('\n').trim();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Block until vLLM is actually serving `servedName` (weights mapped, :8000
|
|
138
|
+
* answering /v1/models) — NOT just spawned. Returns as soon as the model
|
|
139
|
+
* appears, bails early if the process dies, and times out otherwise.
|
|
140
|
+
* A 9B model maps to GPU in ~1-2 min, so the default budget is generous.
|
|
141
|
+
*/
|
|
142
|
+
export async function waitForVllmModel(servedName, { pid = null, timeoutMs = 300_000, intervalMs = 3000 } = {}) {
|
|
143
|
+
const deadline = Date.now() + timeoutMs;
|
|
144
|
+
while (Date.now() < deadline) {
|
|
145
|
+
if (pid && !isPidAlive(pid)) {
|
|
146
|
+
return { serving: false, reason: 'vLLM process exited during startup (check the log tail below)' };
|
|
147
|
+
}
|
|
148
|
+
const models = await detectVllmModels();
|
|
149
|
+
if (servedName ? models.includes(servedName) : models.length > 0) {
|
|
150
|
+
return { serving: true, models };
|
|
151
|
+
}
|
|
152
|
+
await sleep(intervalMs);
|
|
153
|
+
}
|
|
154
|
+
return { serving: false, reason: `timed out after ${Math.round(timeoutMs / 1000)}s waiting for /v1/models to list ${servedName}` };
|
|
155
|
+
}
|
|
156
|
+
|
|
87
157
|
/** True when the tracked serve pid is alive AND /v1/models answers. */
|
|
88
158
|
export async function isVllmServing() {
|
|
89
159
|
const st = await readVllmState();
|
|
@@ -118,6 +188,22 @@ export async function startVllmServe({ source, servedName, port = VLLM_PORT, tok
|
|
|
118
188
|
|
|
119
189
|
const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
|
|
120
190
|
if (servedName) args.push('--served-model-name', servedName);
|
|
191
|
+
|
|
192
|
+
// Sane defaults so a model that fits BY WEIGHT doesn't blow up on KV-cache
|
|
193
|
+
// allocation. Many finetunes inherit a huge native context (Qwen3.5 →
|
|
194
|
+
// 256K); vLLM pre-allocates KV cache for the FULL context, which OOMs the
|
|
195
|
+
// EngineCore even when the weights fit easily. Cap the served context and
|
|
196
|
+
// leave GPU headroom. Both overridable via env; only applied when the
|
|
197
|
+
// caller didn't already pass the flag in extraArgs.
|
|
198
|
+
const has = (flag) => extraArgs.includes(flag);
|
|
199
|
+
if (!has('--max-model-len')) {
|
|
200
|
+
const maxLen = process.env.VLLM_MAX_MODEL_LEN || '16384';
|
|
201
|
+
args.push('--max-model-len', String(maxLen));
|
|
202
|
+
}
|
|
203
|
+
if (!has('--gpu-memory-utilization')) {
|
|
204
|
+
const util = process.env.VLLM_GPU_MEMORY_UTILIZATION || '0.92';
|
|
205
|
+
args.push('--gpu-memory-utilization', String(util));
|
|
206
|
+
}
|
|
121
207
|
args.push(...extraArgs);
|
|
122
208
|
|
|
123
209
|
const env = { ...process.env };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.48",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,12 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/config": "0.1.
|
|
35
|
-
"@infernetprotocol/engine": "0.1.
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
38
|
-
"@infernetprotocol/
|
|
33
|
+
"@infernetprotocol/auth": "0.1.48",
|
|
34
|
+
"@infernetprotocol/config": "0.1.48",
|
|
35
|
+
"@infernetprotocol/engine": "0.1.48",
|
|
36
|
+
"@infernetprotocol/deploy-providers": "0.1.48",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.48",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.48"
|
|
39
39
|
},
|
|
40
40
|
"repository": {
|
|
41
41
|
"type": "git",
|