@infernetprotocol/cli 0.1.46 → 0.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +31 -2
- package/lib/vllm.js +33 -0
- package/package.json +7 -7
package/commands/start.js
CHANGED
|
@@ -48,7 +48,7 @@ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executo
|
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
49
|
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
50
|
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
51
|
-
import { startVllmServe, vllmInstalled } from '../lib/vllm.js';
|
|
51
|
+
import { startVllmServe, vllmInstalled, waitForVllmModel, readVllmLogTail } from '../lib/vllm.js';
|
|
52
52
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
53
53
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
54
54
|
import { pullLatestBinary } from './upgrade.js';
|
|
@@ -234,6 +234,10 @@ async function runDaemon(args, ctx) {
|
|
|
234
234
|
const SPECS_TTL_MS = 5 * 60 * 1000;
|
|
235
235
|
let cachedSpecs = null;
|
|
236
236
|
let cachedSpecsAt = 0;
|
|
237
|
+
// Force the next freshSpecs() to re-detect (served_models etc.) instead of
|
|
238
|
+
// returning the cached copy. Called after any capability change (model
|
|
239
|
+
// install/remove) so the change is advertised on the very next heartbeat.
|
|
240
|
+
const invalidateSpecsCache = () => { cachedSpecs = null; cachedSpecsAt = 0; };
|
|
237
241
|
|
|
238
242
|
// Rolling benchmark — last N completed chat jobs' (tokens, duration)
|
|
239
243
|
// pairs. Used to compute tokens_per_second_avg for the heartbeat,
|
|
@@ -643,12 +647,30 @@ async function runDaemon(args, ctx) {
|
|
|
643
647
|
// under the hf: pull name so it matches chat requests and
|
|
644
648
|
// gets advertised in served_models → shows in /chat.
|
|
645
649
|
const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
|
|
650
|
+
// Don't report "completed" on spawn — wait until vLLM has
|
|
651
|
+
// mapped the weights and /v1/models actually lists the model.
|
|
652
|
+
// If it never comes up (OOM, unsupported arch, bad dtype),
|
|
653
|
+
// fail the command WITH the log tail so the operator sees why
|
|
654
|
+
// instead of a lying "completed".
|
|
655
|
+
process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
|
|
656
|
+
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
|
|
657
|
+
if (!up.serving) {
|
|
658
|
+
const tail = await readVllmLogTail(40);
|
|
659
|
+
throw new Error(
|
|
660
|
+
`vLLM failed to serve ${modelName}: ${up.reason}.` +
|
|
661
|
+
(tail ? `\n--- vllm.log (last 40 lines) ---\n${tail}` : ' (no vllm.log output)')
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
// Serving for real — bust the specs cache so the very next
|
|
665
|
+
// heartbeat advertises the new model in served_models
|
|
666
|
+
// (otherwise it's stale until the TTL lapses).
|
|
667
|
+
invalidateSpecsCache();
|
|
646
668
|
result = {
|
|
647
669
|
model: modelName,
|
|
648
670
|
backend: 'vllm',
|
|
649
671
|
localPath,
|
|
650
672
|
vllm: { pid: serve.pid, port: serve.port },
|
|
651
|
-
note: `Serving on vLLM :${serve.port}
|
|
673
|
+
note: `Serving on vLLM :${serve.port} and advertised in served_models.`,
|
|
652
674
|
};
|
|
653
675
|
} else {
|
|
654
676
|
const fits = await checkModelFits(modelName);
|
|
@@ -656,9 +678,11 @@ async function runDaemon(args, ctx) {
|
|
|
656
678
|
throw new Error(formatFitFailure(modelName, fits));
|
|
657
679
|
}
|
|
658
680
|
result = await ollamaPullWithProgress(modelName, id);
|
|
681
|
+
invalidateSpecsCache();
|
|
659
682
|
}
|
|
660
683
|
} else if (command === 'model_remove') {
|
|
661
684
|
result = await ollamaSpawn(['rm', String(args?.model)]);
|
|
685
|
+
invalidateSpecsCache();
|
|
662
686
|
} else if (command === 'train_shard') {
|
|
663
687
|
result = await runTrainShard(args, id);
|
|
664
688
|
} else {
|
|
@@ -666,6 +690,11 @@ async function runDaemon(args, ctx) {
|
|
|
666
690
|
}
|
|
667
691
|
await client.completeCommand(id, { status: 'completed', result });
|
|
668
692
|
process.stdout.write(`[${new Date().toISOString()}] completed command ${id}\n`);
|
|
693
|
+
// Push the capability change immediately — don't wait up to 30s for
|
|
694
|
+
// the next scheduled heartbeat. freshSpecs() re-detects because the
|
|
695
|
+
// cache was invalidated above, so served_models reflects the change
|
|
696
|
+
// and the model shows in "Installed on node" + /chat right away.
|
|
697
|
+
heartbeat().catch(() => { /* next scheduled beat will retry */ });
|
|
669
698
|
} catch (err) {
|
|
670
699
|
const msg = err?.message ?? String(err);
|
|
671
700
|
process.stderr.write(`command ${id} failed: ${msg}\n`);
|
package/lib/vllm.js
CHANGED
|
@@ -84,6 +84,39 @@ export async function detectVllmModels(host = VLLM_HOST, timeoutMs = 1500) {
|
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
88
|
+
|
|
89
|
+
/** Last N lines of the vLLM serve log — surfaced on startup failure. */
|
|
90
|
+
export async function readVllmLogTail(lines = 40) {
|
|
91
|
+
try {
|
|
92
|
+
const txt = await fs.readFile(logPath(), 'utf8');
|
|
93
|
+
return txt.split('\n').slice(-lines).join('\n').trim();
|
|
94
|
+
} catch {
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Block until vLLM is actually serving `servedName` (weights mapped, :8000
|
|
101
|
+
* answering /v1/models) — NOT just spawned. Returns as soon as the model
|
|
102
|
+
* appears, bails early if the process dies, and times out otherwise.
|
|
103
|
+
* A 9B model maps to GPU in ~1-2 min, so the default budget is generous.
|
|
104
|
+
*/
|
|
105
|
+
export async function waitForVllmModel(servedName, { pid = null, timeoutMs = 300_000, intervalMs = 3000 } = {}) {
|
|
106
|
+
const deadline = Date.now() + timeoutMs;
|
|
107
|
+
while (Date.now() < deadline) {
|
|
108
|
+
if (pid && !isPidAlive(pid)) {
|
|
109
|
+
return { serving: false, reason: 'vLLM process exited during startup (check the log tail below)' };
|
|
110
|
+
}
|
|
111
|
+
const models = await detectVllmModels();
|
|
112
|
+
if (servedName ? models.includes(servedName) : models.length > 0) {
|
|
113
|
+
return { serving: true, models };
|
|
114
|
+
}
|
|
115
|
+
await sleep(intervalMs);
|
|
116
|
+
}
|
|
117
|
+
return { serving: false, reason: `timed out after ${Math.round(timeoutMs / 1000)}s waiting for /v1/models to list ${servedName}` };
|
|
118
|
+
}
|
|
119
|
+
|
|
87
120
|
/** True when the tracked serve pid is alive AND /v1/models answers. */
|
|
88
121
|
export async function isVllmServing() {
|
|
89
122
|
const st = await readVllmState();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.47",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,12 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
38
|
-
"@infernetprotocol/
|
|
33
|
+
"@infernetprotocol/config": "0.1.47",
|
|
34
|
+
"@infernetprotocol/auth": "0.1.47",
|
|
35
|
+
"@infernetprotocol/deploy-providers": "0.1.47",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.47",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.47",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.47"
|
|
39
39
|
},
|
|
40
40
|
"repository": {
|
|
41
41
|
"type": "git",
|