@infernetprotocol/cli 0.1.45 → 0.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/register.js +9 -1
- package/commands/rpc.js +8 -9
- package/commands/setup.js +25 -15
- package/commands/start.js +84 -77
- package/commands/uncensored.js +18 -0
- package/lib/chat-executor.js +71 -29
- package/lib/hf-model.js +23 -47
- package/lib/prompt.js +19 -3
- package/lib/recommender.js +5 -1
- package/lib/vllm.js +173 -0
- package/package.json +7 -8
- package/lib/discovery-bridge.js +0 -168
package/commands/register.js
CHANGED
|
@@ -193,7 +193,15 @@ export async function gatherCoarseSpecs() {
|
|
|
193
193
|
|
|
194
194
|
// Advertise all pulled models — operator explicitly installed them so we
|
|
195
195
|
// trust they know their hardware. The fit check is informational only.
|
|
196
|
-
|
|
196
|
+
// Also fold in any model vLLM is currently serving (:8000/v1/models),
|
|
197
|
+
// advertised under its `hf:` pull name via --served-model-name, so vLLM
|
|
198
|
+
// models show up in /chat alongside Ollama ones.
|
|
199
|
+
let vllmModels = [];
|
|
200
|
+
try {
|
|
201
|
+
const { detectVllmModels } = await import('../lib/vllm.js');
|
|
202
|
+
vllmModels = await detectVllmModels();
|
|
203
|
+
} catch { /* vLLM not installed / not serving — Ollama-only */ }
|
|
204
|
+
const served_models = [...new Set([...pulledModels.map((m) => m.name), ...vllmModels])];
|
|
197
205
|
|
|
198
206
|
return {
|
|
199
207
|
cpu: summarizeCpu(),
|
package/commands/rpc.js
CHANGED
|
@@ -4,15 +4,14 @@
|
|
|
4
4
|
*
|
|
5
5
|
* infernet rpc census --model <id>
|
|
6
6
|
* Asks THIS daemon's HTTP /v1/rpc/census endpoint for the set
|
|
7
|
-
* of peers it sees on
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* the DHT is actually finding anyone.
|
|
7
|
+
* of peers it sees on rpc:<model>. The answer is a second
|
|
8
|
+
* opinion to the control-plane census — useful when the
|
|
9
|
+
* providers table looks stale.
|
|
11
10
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* The peer discovery substrate moved out of infernet — it's now
|
|
12
|
+
* c0mpute's libp2p Kad-DHT + gossipsub. Until /v1/rpc/census is
|
|
13
|
+
* wired through the c0mpute capability registry, this returns an
|
|
14
|
+
* empty set with a `note`.
|
|
16
15
|
*/
|
|
17
16
|
|
|
18
17
|
const HELP = `infernet rpc — federated-inference diagnostics
|
|
@@ -23,7 +22,7 @@ Usage:
|
|
|
23
22
|
|
|
24
23
|
Flags:
|
|
25
24
|
--model <id> Required. Canonical model id (e.g. qwen2.5:72b).
|
|
26
|
-
--kind <ns> Topic namespace (default: rpc
|
|
25
|
+
--kind <ns> Topic namespace (default: rpc).
|
|
27
26
|
--daemon <url> Daemon HTTP base. Defaults to http://127.0.0.1:8080,
|
|
28
27
|
matching the local daemon's healthz port. Set this
|
|
29
28
|
to query a peer's daemon directly.
|
package/commands/setup.js
CHANGED
|
@@ -46,7 +46,7 @@ Flags:
|
|
|
46
46
|
--confirm, --yes Auto-confirm every prompt. Use for unattended runs.
|
|
47
47
|
--model <name> Pre-select model to pull (e.g. qwen2.5:7b).
|
|
48
48
|
--skip-pull Skip the model-pull step.
|
|
49
|
-
--backend <kind> Pin engine backend (ollama | mojo | stub). Default:
|
|
49
|
+
--backend <kind> Pin engine backend (vllm | ollama | mojo | stub). Default: auto (prefers vLLM, falls back to Ollama).
|
|
50
50
|
--host <url> Override Ollama host. Default: http://localhost:11434.
|
|
51
51
|
--port <n> P2P port to open in the firewall. Default: ${DEFAULT_P2P_PORT}.
|
|
52
52
|
--no-firewall Skip the firewall step entirely.
|
|
@@ -284,8 +284,9 @@ async function isRegistered(config) {
|
|
|
284
284
|
*
|
|
285
285
|
* The control-plane-mediated chat path doesn't actually need inbound
|
|
286
286
|
* (daemon polls/posts outbound), so unreachable is a soft warning.
|
|
287
|
-
* But
|
|
288
|
-
*
|
|
287
|
+
* But the direct P2P chat transport DOES need inbound, so we surface
|
|
288
|
+
* this clearly. (Peer discovery for the chat transport runs through
|
|
289
|
+
* c0mpute now — install the c0mpute plugin to enable it.)
|
|
289
290
|
*/
|
|
290
291
|
async function reportReachability(row, config, port) {
|
|
291
292
|
const address = row?.address ?? null;
|
|
@@ -480,7 +481,15 @@ export default async function setup(args) {
|
|
|
480
481
|
const skipIdentity = args.has("skip-identity");
|
|
481
482
|
const skipRegister = args.has("skip-register");
|
|
482
483
|
const skipDaemon = args.has("skip-daemon") || args.has("no-start");
|
|
483
|
-
|
|
484
|
+
// Default backend is "auto": we DON'T pin it in config, so the engine's
|
|
485
|
+
// auto-select governs at serve time — and that precedence prefers
|
|
486
|
+
// vLLM > llama.cpp > Ollama. This is what makes vLLM the network default
|
|
487
|
+
// wherever it's installed, while boxes without vLLM still fall back to
|
|
488
|
+
// Ollama. Pass --backend ollama|vllm|stub to force one.
|
|
489
|
+
const backend = String(args.get("backend") ?? "auto");
|
|
490
|
+
// Still pull an Ollama starter model on "auto" so a fresh box has
|
|
491
|
+
// *something* to serve before an hf:/vLLM model is pushed.
|
|
492
|
+
const wantsOllamaStep = backend === "ollama" || backend === "auto";
|
|
484
493
|
const host = String(args.get("host") ?? process.env.OLLAMA_HOST ?? "http://localhost:11434");
|
|
485
494
|
const preselectedModel = args.get("model") ? String(args.get("model")) : null;
|
|
486
495
|
|
|
@@ -500,7 +509,7 @@ export default async function setup(args) {
|
|
|
500
509
|
// Track the actual maximum so the printed counter never goes [7/6].
|
|
501
510
|
const isProviderRole = (existing?.node?.role ?? "provider") === "provider";
|
|
502
511
|
let total = 2; // hardware + node.js
|
|
503
|
-
if (
|
|
512
|
+
if (wantsOllamaStep) total += 2; // ollama + model
|
|
504
513
|
if (!skipFirewall && process.platform === "linux") total += 1;
|
|
505
514
|
if (process.platform === "linux") total += 1; // logrotate
|
|
506
515
|
total += 1; // config
|
|
@@ -599,7 +608,7 @@ export default async function setup(args) {
|
|
|
599
608
|
let ollamaState = { running: false, models: [] };
|
|
600
609
|
let chosenModel = null;
|
|
601
610
|
|
|
602
|
-
if (
|
|
611
|
+
if (wantsOllamaStep) {
|
|
603
612
|
n += 1;
|
|
604
613
|
step(n, total, `Ollama (${host})`);
|
|
605
614
|
ollamaState = await detectOllama(host);
|
|
@@ -750,18 +759,19 @@ export default async function setup(args) {
|
|
|
750
759
|
|
|
751
760
|
n += 1;
|
|
752
761
|
step(n, total, "Config");
|
|
753
|
-
const
|
|
754
|
-
...existing,
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
backend,
|
|
758
|
-
...(host ? { ollamaHost: host } : {}),
|
|
759
|
-
...(chosenModel ? { model: chosenModel } : {})
|
|
760
|
-
}
|
|
762
|
+
const engineCfg = {
|
|
763
|
+
...(existing.engine ?? {}),
|
|
764
|
+
...(host ? { ollamaHost: host } : {}),
|
|
765
|
+
...(chosenModel ? { model: chosenModel } : {})
|
|
761
766
|
};
|
|
767
|
+
// "auto" → don't pin a backend; let the engine auto-select (vLLM > Ollama)
|
|
768
|
+
// at serve time. Any explicit --backend value is honored.
|
|
769
|
+
if (backend === "auto") delete engineCfg.backend;
|
|
770
|
+
else engineCfg.backend = backend;
|
|
771
|
+
const merged = { ...existing, engine: engineCfg };
|
|
762
772
|
await saveConfig(merged);
|
|
763
773
|
ok(`saved to ${getConfigPath()}`);
|
|
764
|
-
process.stdout.write(` engine.backend: ${merged.engine.backend}\n`);
|
|
774
|
+
process.stdout.write(` engine.backend: ${merged.engine.backend ?? "auto (vLLM > Ollama)"}\n`);
|
|
765
775
|
if (merged.engine.ollamaHost) {
|
|
766
776
|
process.stdout.write(` engine.ollamaHost: ${merged.engine.ollamaHost}\n`);
|
|
767
777
|
}
|
package/commands/start.js
CHANGED
|
@@ -47,13 +47,14 @@ import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/netwo
|
|
|
47
47
|
import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
49
|
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
|
+
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
51
|
+
import { startVllmServe, vllmInstalled, waitForVllmModel, readVllmLogTail } from '../lib/vllm.js';
|
|
50
52
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
51
53
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
52
54
|
import { pullLatestBinary } from './upgrade.js';
|
|
53
55
|
import { nodeTokenLogin } from './login.js';
|
|
54
56
|
import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from '../lib/version.js';
|
|
55
57
|
import { readInferenceState, readInferenceStateSync, isPidAlive } from '../lib/inference/state.js';
|
|
56
|
-
import { startDiscoveryBridge } from '../lib/discovery-bridge.js';
|
|
57
58
|
|
|
58
59
|
function readInferenceStateSyncSafe() {
|
|
59
60
|
try { return readInferenceStateSync(); } catch { return {}; }
|
|
@@ -130,7 +131,6 @@ async function spawnAndReturn(args) {
|
|
|
130
131
|
if (args.has('no-p2p')) passthrough.push('--no-p2p');
|
|
131
132
|
if (args.has('no-advertise')) passthrough.push('--no-advertise');
|
|
132
133
|
if (args.has('once')) passthrough.push('--once');
|
|
133
|
-
if (args.has('no-dht')) passthrough.push('--no-dht');
|
|
134
134
|
|
|
135
135
|
const { pid, logPath } = spawnDetachedDaemon(passthrough);
|
|
136
136
|
process.stdout.write(`infernet daemon started (pid ${pid})\n`);
|
|
@@ -154,11 +154,6 @@ async function runDaemon(args, ctx) {
|
|
|
154
154
|
|
|
155
155
|
const p2pDisabled = args.has('no-p2p');
|
|
156
156
|
const noAdvertise = args.has('no-advertise') || node.address === null;
|
|
157
|
-
// IPIP-0032: DHT discovery is on by default. Operators who want
|
|
158
|
-
// to opt out (private-only deployments, restricted networks, or
|
|
159
|
-
// an emergency kill switch) pass --no-dht or set
|
|
160
|
-
// INFERNET_DISABLE_DHT=1.
|
|
161
|
-
const enableDht = !(args.has('no-dht') || process.env.INFERNET_DISABLE_DHT === '1');
|
|
162
157
|
// Bind port (what we listen on, locally) and advertised port (what we
|
|
163
158
|
// tell the control plane to dial). Same value 99% of the time, but
|
|
164
159
|
// hosting platforms that NAT the container (RunPod, anything with
|
|
@@ -229,7 +224,6 @@ async function runDaemon(args, ctx) {
|
|
|
229
224
|
let ipcServer = null;
|
|
230
225
|
let p2pServer = null;
|
|
231
226
|
let healthServer = null;
|
|
232
|
-
let dhtBridge = null;
|
|
233
227
|
let p2pConnections = 0;
|
|
234
228
|
let p2pLastConnectionAt = null;
|
|
235
229
|
|
|
@@ -240,6 +234,10 @@ async function runDaemon(args, ctx) {
|
|
|
240
234
|
const SPECS_TTL_MS = 5 * 60 * 1000;
|
|
241
235
|
let cachedSpecs = null;
|
|
242
236
|
let cachedSpecsAt = 0;
|
|
237
|
+
// Force the next freshSpecs() to re-detect (served_models etc.) instead of
|
|
238
|
+
// returning the cached copy. Called after any capability change (model
|
|
239
|
+
// install/remove) so the change is advertised on the very next heartbeat.
|
|
240
|
+
const invalidateSpecsCache = () => { cachedSpecs = null; cachedSpecsAt = 0; };
|
|
243
241
|
|
|
244
242
|
// Rolling benchmark — last N completed chat jobs' (tokens, duration)
|
|
245
243
|
// pairs. Used to compute tokens_per_second_avg for the heartbeat,
|
|
@@ -631,13 +629,60 @@ async function runDaemon(args, ctx) {
|
|
|
631
629
|
let result;
|
|
632
630
|
if (command === 'model_install') {
|
|
633
631
|
const modelName = String(args?.model);
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
632
|
+
if (modelName.startsWith('hf:')) {
|
|
633
|
+
// HuggingFace model → download the weights, then TURN ON
|
|
634
|
+
// vLLM to serve them (Ollama can't — it 400s on an hf: repo).
|
|
635
|
+
if (!vllmInstalled()) {
|
|
636
|
+
throw new Error(
|
|
637
|
+
`${modelName} needs vLLM, which isn't installed on this node. ` +
|
|
638
|
+
`Re-run the installer on this NVIDIA host (it installs vLLM), or: pip install vllm.`
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
const repoId = modelName.slice(3);
|
|
642
|
+
const token = process.env.HF_TOKEN || (await resolveHfToken().catch(() => null));
|
|
643
|
+
const localPath = await downloadHfModel(repoId, token);
|
|
644
|
+
// Serve the repo id, not a computed cache path — vLLM
|
|
645
|
+
// resolves it straight from the HF cache we just filled
|
|
646
|
+
// (the snapshot dir is a commit hash, not "main"). Serve
|
|
647
|
+
// under the hf: pull name so it matches chat requests and
|
|
648
|
+
// gets advertised in served_models → shows in /chat.
|
|
649
|
+
const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
|
|
650
|
+
// Don't report "completed" on spawn — wait until vLLM has
|
|
651
|
+
// mapped the weights and /v1/models actually lists the model.
|
|
652
|
+
// If it never comes up (OOM, unsupported arch, bad dtype),
|
|
653
|
+
// fail the command WITH the log tail so the operator sees why
|
|
654
|
+
// instead of a lying "completed".
|
|
655
|
+
process.stdout.write(`[${new Date().toISOString()}] ${modelName} downloaded; waiting for vLLM to load weights…\n`);
|
|
656
|
+
const up = await waitForVllmModel(modelName, { pid: serve.pid, timeoutMs: 300_000 });
|
|
657
|
+
if (!up.serving) {
|
|
658
|
+
const tail = await readVllmLogTail(40);
|
|
659
|
+
throw new Error(
|
|
660
|
+
`vLLM failed to serve ${modelName}: ${up.reason}.` +
|
|
661
|
+
(tail ? `\n--- vllm.log (last 40 lines) ---\n${tail}` : ' (no vllm.log output)')
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
// Serving for real — bust the specs cache so the very next
|
|
665
|
+
// heartbeat advertises the new model in served_models
|
|
666
|
+
// (otherwise it's stale until the TTL lapses).
|
|
667
|
+
invalidateSpecsCache();
|
|
668
|
+
result = {
|
|
669
|
+
model: modelName,
|
|
670
|
+
backend: 'vllm',
|
|
671
|
+
localPath,
|
|
672
|
+
vllm: { pid: serve.pid, port: serve.port },
|
|
673
|
+
note: `Serving on vLLM :${serve.port} and advertised in served_models.`,
|
|
674
|
+
};
|
|
675
|
+
} else {
|
|
676
|
+
const fits = await checkModelFits(modelName);
|
|
677
|
+
if (fits && !fits.ok) {
|
|
678
|
+
throw new Error(formatFitFailure(modelName, fits));
|
|
679
|
+
}
|
|
680
|
+
result = await ollamaPullWithProgress(modelName, id);
|
|
681
|
+
invalidateSpecsCache();
|
|
637
682
|
}
|
|
638
|
-
result = await ollamaPullWithProgress(modelName, id);
|
|
639
683
|
} else if (command === 'model_remove') {
|
|
640
684
|
result = await ollamaSpawn(['rm', String(args?.model)]);
|
|
685
|
+
invalidateSpecsCache();
|
|
641
686
|
} else if (command === 'train_shard') {
|
|
642
687
|
result = await runTrainShard(args, id);
|
|
643
688
|
} else {
|
|
@@ -645,6 +690,11 @@ async function runDaemon(args, ctx) {
|
|
|
645
690
|
}
|
|
646
691
|
await client.completeCommand(id, { status: 'completed', result });
|
|
647
692
|
process.stdout.write(`[${new Date().toISOString()}] completed command ${id}\n`);
|
|
693
|
+
// Push the capability change immediately — don't wait up to 30s for
|
|
694
|
+
// the next scheduled heartbeat. freshSpecs() re-detects because the
|
|
695
|
+
// cache was invalidated above, so served_models reflects the change
|
|
696
|
+
// and the model shows in "Installed on node" + /chat right away.
|
|
697
|
+
heartbeat().catch(() => { /* next scheduled beat will retry */ });
|
|
648
698
|
} catch (err) {
|
|
649
699
|
const msg = err?.message ?? String(err);
|
|
650
700
|
process.stderr.write(`command ${id} failed: ${msg}\n`);
|
|
@@ -1002,18 +1052,17 @@ async function runDaemon(args, ctx) {
|
|
|
1002
1052
|
}
|
|
1003
1053
|
|
|
1004
1054
|
/**
|
|
1005
|
-
*
|
|
1006
|
-
*
|
|
1007
|
-
* its Hyperswarm DHT view (verified handshakes, currently
|
|
1008
|
-
* connected). Useful as a second opinion when the control
|
|
1009
|
-
* plane's heartbeat-based census looks stale, and as a smoke
|
|
1010
|
-
* test for "is the DHT actually finding anyone."
|
|
1055
|
+
* Daemon-side peer census for the rpc-adapter (IPIP-0033 — llama.cpp
|
|
1056
|
+
* RPC peer assignment).
|
|
1011
1057
|
*
|
|
1012
1058
|
* GET /v1/rpc/census?model=<id>[&kind=rpc|model|class]
|
|
1013
|
-
* response: { kind, value, count, peers
|
|
1059
|
+
* response: { kind, value, count, peers, dht, note? }
|
|
1014
1060
|
*
|
|
1015
|
-
*
|
|
1016
|
-
*
|
|
1061
|
+
* Peer discovery used to come from the daemon's own Hyperswarm DHT
|
|
1062
|
+
* (IPIP-0032). That layer has been moved to c0mpute. Until the
|
|
1063
|
+
* c0mpute-backed query is wired in, this endpoint always returns
|
|
1064
|
+
* `count: 0` with a note — the chat client falls back to
|
|
1065
|
+
* control-plane routing.
|
|
1017
1066
|
*/
|
|
1018
1067
|
async function handleRpcCensus(req, res) {
|
|
1019
1068
|
if (req.method !== 'GET') return false;
|
|
@@ -1028,30 +1077,17 @@ async function runDaemon(args, ctx) {
|
|
|
1028
1077
|
return true;
|
|
1029
1078
|
}
|
|
1030
1079
|
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
};
|
|
1036
|
-
if (!dhtBridge?.node?.peersOnTopic) {
|
|
1037
|
-
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1038
|
-
res.end(JSON.stringify(empty));
|
|
1039
|
-
return true;
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
const peers = dhtBridge.node.peersOnTopic(kind, model);
|
|
1080
|
+
// Peer discovery has moved to c0mpute (libp2p Kad-DHT +
|
|
1081
|
+
// gossipsub capability ads). Until the c0mpute-backed query
|
|
1082
|
+
// is wired in, return an empty list with a note so the chat
|
|
1083
|
+
// client falls back to control-plane routing.
|
|
1043
1084
|
const body = {
|
|
1044
1085
|
kind,
|
|
1045
1086
|
value: model,
|
|
1046
|
-
count:
|
|
1047
|
-
peers:
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
first_seen: new Date(p.firstSeen).toISOString(),
|
|
1051
|
-
last_seen: new Date(p.lastSeen).toISOString()
|
|
1052
|
-
})),
|
|
1053
|
-
dht: true,
|
|
1054
|
-
self: { pubkey: dhtBridge.node.pubkey, topics: dhtBridge.node.topics }
|
|
1087
|
+
count: 0,
|
|
1088
|
+
peers: [],
|
|
1089
|
+
dht: false,
|
|
1090
|
+
note: 'peer discovery moved to c0mpute; query c0mpute capability registry instead'
|
|
1055
1091
|
};
|
|
1056
1092
|
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1057
1093
|
res.end(JSON.stringify(body));
|
|
@@ -1280,7 +1316,6 @@ async function runDaemon(args, ctx) {
|
|
|
1280
1316
|
if (ipcServer) { try { ipcServer.close(); } catch {} }
|
|
1281
1317
|
if (p2pServer) { try { p2pServer.close(); } catch {} }
|
|
1282
1318
|
if (healthServer) { try { healthServer.close(); } catch {} }
|
|
1283
|
-
if (dhtBridge) { try { await dhtBridge.stop(); } catch {} }
|
|
1284
1319
|
try { await shutdownEngine(); } catch {}
|
|
1285
1320
|
removeSocketFile();
|
|
1286
1321
|
await removePidFile();
|
|
@@ -1298,40 +1333,12 @@ async function runDaemon(args, ctx) {
|
|
|
1298
1333
|
process.stderr.write(`Failed to bind IPC socket at ${socketPath}: ${err?.message ?? err}\n`);
|
|
1299
1334
|
}
|
|
1300
1335
|
|
|
1301
|
-
//
|
|
1302
|
-
//
|
|
1303
|
-
//
|
|
1304
|
-
//
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
const dhtAddress = noAdvertise || !advertisedAddress
|
|
1308
|
-
? null
|
|
1309
|
-
: `${advertisedAddress}:${advertisedPort}`;
|
|
1310
|
-
dhtBridge = await startDiscoveryBridge({
|
|
1311
|
-
config,
|
|
1312
|
-
advertise: !noAdvertise,
|
|
1313
|
-
address: dhtAddress,
|
|
1314
|
-
getState: () => ({
|
|
1315
|
-
servedModels: cachedSpecs?.served_models ?? [],
|
|
1316
|
-
inferenceState: readInferenceStateSyncSafe()
|
|
1317
|
-
}),
|
|
1318
|
-
onPeer: ({ peer }) => {
|
|
1319
|
-
process.stdout.write(
|
|
1320
|
-
`[dht] peer ${peer.pubkey.slice(0, 8)}… topics=[${peer.topics.join(',')}]\n`
|
|
1321
|
-
);
|
|
1322
|
-
},
|
|
1323
|
-
onHandshakeFailed: ({ error }) => {
|
|
1324
|
-
process.stderr.write(`[dht] handshake-failed: ${error?.message ?? error}\n`);
|
|
1325
|
-
}
|
|
1326
|
-
});
|
|
1327
|
-
process.stdout.write(
|
|
1328
|
-
`[dht] joined ${dhtBridge.currentTopics.length} topic(s): ${dhtBridge.currentTopics.join(', ')}\n`
|
|
1329
|
-
);
|
|
1330
|
-
} catch (err) {
|
|
1331
|
-
process.stderr.write(`[dht] startup failed: ${err?.message ?? err}\n`);
|
|
1332
|
-
process.stderr.write(`[dht] continuing without DHT discovery — heartbeats still active\n`);
|
|
1333
|
-
}
|
|
1334
|
-
}
|
|
1336
|
+
// Peer discovery (formerly Hyperswarm DHT, IPIP-0032) has moved
|
|
1337
|
+
// to c0mpute's libp2p Kad-DHT + gossipsub. The infernet daemon no
|
|
1338
|
+
// longer maintains its own discovery bridge — workers advertise
|
|
1339
|
+
// capabilities via the c0mpute supervisor and the chat client
|
|
1340
|
+
// routes through the control plane until c0mpute-backed query
|
|
1341
|
+
// is wired in.
|
|
1335
1342
|
|
|
1336
1343
|
// /healthz — bind a tiny HTTP server on $PORT (default 8080) so
|
|
1337
1344
|
// Docker / Runpod / Kubernetes HEALTHCHECK probes can reach the
|
package/commands/uncensored.js
CHANGED
|
@@ -54,6 +54,13 @@ const HF_MODELS = [
|
|
|
54
54
|
minVramGb: 8,
|
|
55
55
|
description: 'Hermes 3 8B (Nous Research) — uncensored, instruction-tuned, Apache 2.0',
|
|
56
56
|
},
|
|
57
|
+
{
|
|
58
|
+
repoId: 'DavidAU/Qwen3.5-9B-Claude-4.6-HighIQ-THINKING-HERETIC-UNCENSORED',
|
|
59
|
+
sizeGb: 18.8,
|
|
60
|
+
sizeGb4bit: 6.0,
|
|
61
|
+
minVramGb: 24,
|
|
62
|
+
description: 'Qwen3.5 9B Heretic (DavidAU) — uncensored, thinking + vision, bf16 via vLLM',
|
|
63
|
+
},
|
|
57
64
|
{
|
|
58
65
|
repoId: 'cognitivecomputations/dolphin-2.9.4-llama3.1-70b',
|
|
59
66
|
sizeGb: 140.0,
|
|
@@ -61,6 +68,17 @@ const HF_MODELS = [
|
|
|
61
68
|
minVramGb: 40,
|
|
62
69
|
description: 'Dolphin 2.9.4 70B (Eric Hartford) — uncensored, Llama 3.1 base',
|
|
63
70
|
},
|
|
71
|
+
{
|
|
72
|
+
// 754B MoE, FP8 weights (~755 GB) — flagship uncensored model. Only
|
|
73
|
+
// auto-selected on a cluster that can actually hold it (minVramGb set
|
|
74
|
+
// so it never picks on a single GPU). Gated on HF: set HF_TOKEN to pull.
|
|
75
|
+
repoId: 'zandenAI/GLM-5.2-FP8-Uncensored',
|
|
76
|
+
sizeGb: 755.0,
|
|
77
|
+
sizeGb4bit: 755.0,
|
|
78
|
+
minVramGb: 768,
|
|
79
|
+
gated: true,
|
|
80
|
+
description: 'GLM-5.2 754B FP8 (abliterated) — uncensored flagship; needs a multi-GPU cluster (~768 GB VRAM) + HF token',
|
|
81
|
+
},
|
|
64
82
|
];
|
|
65
83
|
|
|
66
84
|
// Ollama models, ordered by VRAM requirement ascending
|
package/lib/chat-executor.js
CHANGED
|
@@ -93,44 +93,86 @@ class EventBuffer {
|
|
|
93
93
|
// /chat endpoint sometimes doesn't pass a model name). Without this
|
|
94
94
|
// fallback, every model-unspecified job died with
|
|
95
95
|
// "ollama backend: no model — set INFERNET_ENGINE_MODEL or pass model in the job"
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
106
|
-
// Optional override for the per-request num_thread cap.
|
|
107
|
-
// Default (50% of cores) is computed inside the Ollama
|
|
108
|
-
// backend; only forward when the operator explicitly set
|
|
109
|
-
// an override in config.
|
|
110
|
-
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
111
|
-
opts.numThread = eng.ollama_num_thread;
|
|
112
|
-
}
|
|
113
|
-
return createEngine(opts);
|
|
114
|
-
})().catch((err) => {
|
|
96
|
+
// One engine per backend, cached. We route PER MODEL: if the requested model
|
|
97
|
+
// is being served by vLLM (:8000), use the vLLM engine; otherwise the default
|
|
98
|
+
// (config backend or auto-select → Ollama). This is what makes vLLM the actual
|
|
99
|
+
// serving path for hf: models while Ollama keeps handling its own tags.
|
|
100
|
+
const engineCache = new Map(); // cacheKey -> Promise<engine>
|
|
101
|
+
|
|
102
|
+
function buildEngine(cacheKey, opts) {
|
|
103
|
+
if (!engineCache.has(cacheKey)) {
|
|
104
|
+
const p = (async () => createEngine(opts))().catch((err) => {
|
|
115
105
|
// Reset so a transient failure doesn't permanently poison the
|
|
116
106
|
// daemon — next job will retry initialization.
|
|
117
|
-
|
|
107
|
+
engineCache.delete(cacheKey);
|
|
118
108
|
throw err;
|
|
119
109
|
});
|
|
110
|
+
engineCache.set(cacheKey, p);
|
|
120
111
|
}
|
|
121
|
-
return
|
|
112
|
+
return engineCache.get(cacheKey);
|
|
122
113
|
}
|
|
123
114
|
|
|
124
|
-
|
|
125
|
-
|
|
115
|
+
async function defaultEngine() {
|
|
116
|
+
const config = (await loadConfig()) ?? {};
|
|
117
|
+
const eng = config.engine ?? {};
|
|
118
|
+
const opts = {};
|
|
119
|
+
if (eng.backend) opts.backend = eng.backend;
|
|
120
|
+
if (eng.model) opts.defaultModel = eng.model;
|
|
121
|
+
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
122
|
+
// Optional override for the per-request num_thread cap. Default (50% of
|
|
123
|
+
// cores) is computed inside the Ollama backend; only forward when set.
|
|
124
|
+
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
125
|
+
opts.numThread = eng.ollama_num_thread;
|
|
126
|
+
}
|
|
127
|
+
return buildEngine(`default:${eng.backend ?? 'auto'}`, opts);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Best-effort: does Ollama (at host) currently serve this exact tag? */
|
|
131
|
+
async function ollamaHasModel(modelName, host) {
|
|
126
132
|
try {
|
|
127
|
-
const
|
|
128
|
-
|
|
133
|
+
const ctrl = new AbortController();
|
|
134
|
+
const t = setTimeout(() => ctrl.abort(), 800);
|
|
135
|
+
const r = await fetch(`${host}/api/tags`, { signal: ctrl.signal });
|
|
136
|
+
clearTimeout(t);
|
|
137
|
+
if (!r.ok) return false;
|
|
138
|
+
const data = await r.json();
|
|
139
|
+
return (data.models ?? []).some((m) => m.name === modelName || m.model === modelName);
|
|
129
140
|
} catch {
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function getEngineForModel(modelName) {
|
|
146
|
+
// vLLM is the default serving path. Route PER MODEL so both engines coexist:
|
|
147
|
+
// 1. vLLM is serving this model → vLLM (preferred, high throughput)
|
|
148
|
+
// 2. Ollama has this GGUF tag → Ollama (its library keeps working even
|
|
149
|
+
// while vLLM is up serving another model)
|
|
150
|
+
// 3. otherwise → default (auto-select, which prefers vLLM)
|
|
151
|
+
if (modelName) {
|
|
152
|
+
try {
|
|
153
|
+
const { detectVllmModels, VLLM_HOST } = await import('./vllm.js');
|
|
154
|
+
const vModels = await detectVllmModels();
|
|
155
|
+
if (vModels.includes(modelName)) {
|
|
156
|
+
return buildEngine('vllm', { backend: 'vllm', host: VLLM_HOST, defaultModel: modelName });
|
|
157
|
+
}
|
|
158
|
+
} catch {
|
|
159
|
+
// vLLM module/probe failed — try Ollama, then the default.
|
|
160
|
+
}
|
|
161
|
+
const config = (await loadConfig().catch(() => null)) ?? {};
|
|
162
|
+
const ollamaHost = config.engine?.ollamaHost ?? process.env.OLLAMA_HOST ?? 'http://localhost:11434';
|
|
163
|
+
if (await ollamaHasModel(modelName, ollamaHost)) {
|
|
164
|
+
return buildEngine('ollama', { backend: 'ollama', host: ollamaHost, defaultModel: modelName });
|
|
165
|
+
}
|
|
133
166
|
}
|
|
167
|
+
return defaultEngine();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export async function shutdownEngine() {
|
|
171
|
+
const engines = [...engineCache.values()];
|
|
172
|
+
engineCache.clear();
|
|
173
|
+
await Promise.all(engines.map(async (p) => {
|
|
174
|
+
try { (await p).shutdown(); } catch { /* best-effort */ }
|
|
175
|
+
}));
|
|
134
176
|
}
|
|
135
177
|
|
|
136
178
|
/**
|
|
@@ -167,7 +209,7 @@ export async function executeChatJob({ client, job, node }) {
|
|
|
167
209
|
}
|
|
168
210
|
}
|
|
169
211
|
|
|
170
|
-
const engine = await
|
|
212
|
+
const engine = await getEngineForModel(job.model_name);
|
|
171
213
|
const buffer = new EventBuffer(client, job.id);
|
|
172
214
|
|
|
173
215
|
const t0 = Date.now();
|
package/lib/hf-model.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { spawn } from 'node:child_process';
|
|
10
|
+
import { existsSync } from 'node:fs';
|
|
10
11
|
import { loadConfig } from './config.js';
|
|
11
12
|
|
|
12
13
|
const HF_API = 'https://huggingface.co/api';
|
|
@@ -123,68 +124,43 @@ export async function fetchHfModelInfo(repoId, token) {
|
|
|
123
124
|
};
|
|
124
125
|
}
|
|
125
126
|
|
|
126
|
-
/**
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
p.on('error', () => resolve(false));
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/** Check whether Python huggingface_hub is importable. */
|
|
136
|
-
async function hfHubAvailable() {
|
|
137
|
-
return new Promise(resolve => {
|
|
138
|
-
const p = spawn('python3', ['-c', 'import huggingface_hub'], { stdio: 'ignore' });
|
|
139
|
-
p.on('exit', code => resolve(code === 0));
|
|
140
|
-
p.on('error', () => resolve(false));
|
|
141
|
-
});
|
|
127
|
+
/** The `hf` CLI the installer puts in the vLLM venv. One place, no fallbacks. */
|
|
128
|
+
function hfCli() {
|
|
129
|
+
const home = process.env.HOME || process.env.USERPROFILE || '';
|
|
130
|
+
const base = process.env.INFERNET_HOME || `${home}/.infernet`;
|
|
131
|
+
return `${base}/vllm-venv/bin/hf`;
|
|
142
132
|
}
|
|
143
133
|
|
|
144
134
|
/**
|
|
145
|
-
* Download a model
|
|
146
|
-
*
|
|
147
|
-
* Returns the local cache path.
|
|
135
|
+
* Download a HuggingFace model via `hf download`. Streams progress and returns
|
|
136
|
+
* the local cache path; surfaces the CLI's stderr on failure.
|
|
148
137
|
*/
|
|
149
138
|
export async function downloadHfModel(repoId, token, { localDir = null } = {}) {
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (!hasHub) {
|
|
139
|
+
const cli = hfCli();
|
|
140
|
+
if (!existsSync(cli)) {
|
|
154
141
|
throw new Error(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
' (or: pip3 install -U huggingface_hub)'
|
|
142
|
+
`HuggingFace CLI not found at ${cli}. Re-run the installer on this NVIDIA ` +
|
|
143
|
+
`host — it installs vLLM + huggingface_hub.`
|
|
158
144
|
);
|
|
159
145
|
}
|
|
160
146
|
|
|
161
|
-
// Build the download command
|
|
162
147
|
const args = ['download', repoId];
|
|
163
148
|
if (token) args.push('--token', token);
|
|
164
149
|
if (localDir) args.push('--local-dir', localDir);
|
|
165
150
|
|
|
166
|
-
|
|
167
|
-
const cmd = hasCli ? 'huggingface-cli' : 'python3';
|
|
168
|
-
const finalArgs = hasCli ? args : [
|
|
169
|
-
'-c',
|
|
170
|
-
`from huggingface_hub import snapshot_download; ` +
|
|
171
|
-
`p = snapshot_download(${JSON.stringify(repoId)}` +
|
|
172
|
-
`${token ? `, token=${JSON.stringify(token)}` : ''}` +
|
|
173
|
-
`${localDir ? `, local_dir=${JSON.stringify(localDir)}` : ''}` +
|
|
174
|
-
`); print(p)`
|
|
175
|
-
];
|
|
176
|
-
|
|
177
|
-
process.stdout.write(`\nDownloading ${repoId} from HuggingFace...\n`);
|
|
178
|
-
if (!hasCli) {
|
|
179
|
-
process.stdout.write('(tip: pip install huggingface_hub for a nicer progress bar)\n\n');
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
const child = spawn(cmd, finalArgs, { stdio: ['ignore', 'inherit', 'inherit'] });
|
|
183
|
-
let localPath = localDir;
|
|
151
|
+
process.stdout.write(`\nDownloading ${repoId} from HuggingFace…\n`);
|
|
184
152
|
|
|
153
|
+
return new Promise((resolve, reject) => {
|
|
154
|
+
const child = spawn(cli, args, { stdio: ['ignore', 'inherit', 'pipe'] });
|
|
155
|
+
let errTail = '';
|
|
156
|
+
child.stderr.on('data', (d) => {
|
|
157
|
+
const s = d.toString();
|
|
158
|
+
process.stderr.write(s);
|
|
159
|
+
errTail = (errTail + s).slice(-1200);
|
|
160
|
+
});
|
|
185
161
|
child.on('exit', code => {
|
|
186
|
-
if (code === 0) resolve(
|
|
187
|
-
else reject(new Error(`
|
|
162
|
+
if (code === 0) resolve(localDir ?? defaultCachePath(repoId));
|
|
163
|
+
else reject(new Error(`hf download exited ${code}${errTail.trim() ? `:\n${errTail.trim()}` : ''}`));
|
|
188
164
|
});
|
|
189
165
|
child.on('error', reject);
|
|
190
166
|
});
|
package/lib/prompt.js
CHANGED
|
@@ -17,15 +17,31 @@ import { stdin, stdout } from 'node:process';
|
|
|
17
17
|
export async function question(prompt, options = {}) {
|
|
18
18
|
const { default: defaultValue, secret = false } = options;
|
|
19
19
|
|
|
20
|
+
const suffix = defaultValue ? ` [${secret ? '****' : defaultValue}]` : '';
|
|
21
|
+
const full = `${prompt}${suffix}: `;
|
|
22
|
+
|
|
23
|
+
// Non-interactive stdin (piped `curl | sh` install, `--yes` automation,
|
|
24
|
+
// CI, nohup): there's no terminal to read an answer from. Creating a
|
|
25
|
+
// readline with terminal:true and awaiting rl.question() here NEVER
|
|
26
|
+
// resolves at EOF — the await hangs, the event loop drains, and Node
|
|
27
|
+
// exits 0 mid-function, so callers like `init` silently skip saveConfig
|
|
28
|
+
// and write no config at all. Echo the prompt + chosen default and
|
|
29
|
+
// return it immediately instead.
|
|
30
|
+
if (!stdin.isTTY) {
|
|
31
|
+
stdout.write(
|
|
32
|
+
defaultValue !== undefined
|
|
33
|
+
? `${full}${secret ? '****' : defaultValue} (default, non-interactive)\n`
|
|
34
|
+
: `${full}(non-interactive)\n`
|
|
35
|
+
);
|
|
36
|
+
return defaultValue !== undefined ? defaultValue : '';
|
|
37
|
+
}
|
|
38
|
+
|
|
20
39
|
const rl = readline.createInterface({
|
|
21
40
|
input: stdin,
|
|
22
41
|
output: stdout,
|
|
23
42
|
terminal: true
|
|
24
43
|
});
|
|
25
44
|
|
|
26
|
-
const suffix = defaultValue ? ` [${secret ? '****' : defaultValue}]` : '';
|
|
27
|
-
const full = `${prompt}${suffix}: `;
|
|
28
|
-
|
|
29
45
|
try {
|
|
30
46
|
let answer;
|
|
31
47
|
if (secret) {
|
package/lib/recommender.js
CHANGED
|
@@ -41,6 +41,7 @@ export const CATALOG = [
|
|
|
41
41
|
{ id: "qwen2.5-coder-14b",name:"Qwen2.5 Coder 14B", paramsB: 14, vramMin: 12, ramMin: 32, pullName: "qwen2.5-coder:14b", backend: "ollama", quality: 9, speed: 5, useCases: ["coding","agents"] },
|
|
42
42
|
|
|
43
43
|
// ── 32B (24GB+ VRAM, RTX 4090 / A100 sweet spot) ──
|
|
44
|
+
{ id: "qwen3.5-9b-heretic", name: "Qwen3.5 9B Heretic (uncensored)", paramsB: 9, vramMin: 24, ramMin: 32, pullName: "hf:DavidAU/Qwen3.5-9B-Claude-4.6-HighIQ-THINKING-HERETIC-UNCENSORED", backend: "vllm", quality: 8, speed: 6, useCases: ["chat","uncensored","vision","agents","study"], uncensored: true },
|
|
44
45
|
{ id: "qwen2.5-32b", name: "Qwen2.5 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5:32b", backend: "ollama", quality: 9, speed: 4, useCases: ["chat","coding","study","agents"], notes: "Single-80GB-GPU sweet spot." },
|
|
45
46
|
{ id: "qwen2.5-coder-32b",name:"Qwen2.5 Coder 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5-coder:32b", backend: "ollama", quality: 10, speed: 4, useCases: ["coding","agents"] },
|
|
46
47
|
|
|
@@ -49,7 +50,10 @@ export const CATALOG = [
|
|
|
49
50
|
{ id: "qwen2.5-72b", name: "Qwen2.5 72B", paramsB: 72, vramMin: 40, ramMin: 64, pullName: "qwen2.5:72b", backend: "ollama", quality: 10, speed: 3, useCases: ["chat","coding","study","agents"] },
|
|
50
51
|
|
|
51
52
|
// ── MoE (high VRAM, vLLM only) ──
|
|
52
|
-
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true }
|
|
53
|
+
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true },
|
|
54
|
+
|
|
55
|
+
// ── flagship / cluster-tier (multi-GPU or cloud; won't fit a single card) ──
|
|
56
|
+
{ id: "glm-5.2-uncensored", name: "GLM-5.2 754B FP8 (uncensored)", paramsB: 754, vramMin: 768, ramMin: 768, pullName: "hf:zandenAI/GLM-5.2-FP8-Uncensored", backend: "vllm", quality: 10, speed: 2, useCases: ["chat","uncensored","cybersecurity","agents"], uncensored: true, gated: true }
|
|
53
57
|
];
|
|
54
58
|
|
|
55
59
|
/**
|
package/lib/vllm.js
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vLLM serve manager — turns ON the vLLM backend the installer already put on
|
|
3
|
+
* the box (venv → `$INFERNET_BIN/vllm`) but that nothing ever started.
|
|
4
|
+
*
|
|
5
|
+
* vLLM serves ONE model per GPU process on an OpenAI-compatible API (:8000),
|
|
6
|
+
* so we run a single model at a time; installing a new `hf:` model replaces
|
|
7
|
+
* the current one. We track the process in a state file and advertise the
|
|
8
|
+
* served model (by its `hf:` pull name via `--served-model-name`) so it lands
|
|
9
|
+
* in `served_models` → shows in /chat, and the engine auto-selects vLLM over
|
|
10
|
+
* Ollama (per createEngine's precedence) once /v1/models answers.
|
|
11
|
+
*
|
|
12
|
+
* Everything here is best-effort + feature-detected: if vLLM isn't installed
|
|
13
|
+
* or a serve fails, callers fall back to Ollama — the working path is never
|
|
14
|
+
* broken.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { spawn } from 'node:child_process';
|
|
18
|
+
import fs from 'node:fs/promises';
|
|
19
|
+
import fss from 'node:fs';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import { getConfigDir } from './config.js';
|
|
22
|
+
import { isPidAlive } from './inference/state.js';
|
|
23
|
+
|
|
24
|
+
export const VLLM_PORT = Number(process.env.VLLM_PORT) || 8000;
|
|
25
|
+
export const VLLM_HOST = process.env.VLLM_HOST || `http://localhost:${VLLM_PORT}`;
|
|
26
|
+
|
|
27
|
+
function statePath() {
|
|
28
|
+
return path.join(getConfigDir(), 'vllm.json');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function logPath() {
|
|
32
|
+
return path.join(getConfigDir(), 'vllm.log');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Locate the vllm binary the installer symlinked (or a venv/PATH copy). */
|
|
36
|
+
export function resolveVllmBin() {
|
|
37
|
+
const home = process.env.HOME || process.env.USERPROFILE || '';
|
|
38
|
+
const candidates = [
|
|
39
|
+
process.env.VLLM_BIN,
|
|
40
|
+
path.join(home, '.local/bin/vllm'),
|
|
41
|
+
path.join(home, '.infernet/vllm-venv/bin/vllm'),
|
|
42
|
+
].filter(Boolean);
|
|
43
|
+
for (const c of candidates) {
|
|
44
|
+
try {
|
|
45
|
+
if (fss.existsSync(c) && (fss.statSync(c).mode & 0o111)) return c;
|
|
46
|
+
} catch { /* keep looking */ }
|
|
47
|
+
}
|
|
48
|
+
return null; // not installed
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function vllmInstalled() {
|
|
52
|
+
return resolveVllmBin() !== null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function readVllmState() {
|
|
56
|
+
try {
|
|
57
|
+
return JSON.parse(await fs.readFile(statePath(), 'utf8'));
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function writeVllmState(state) {
|
|
64
|
+
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
65
|
+
await fs.writeFile(statePath(), JSON.stringify(state, null, 2) + '\n', { mode: 0o600 });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function clearVllmState() {
|
|
69
|
+
try { await fs.rm(statePath()); } catch { /* already gone */ }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** GET /v1/models — the model(s) vLLM is currently serving (empty if down). */
|
|
73
|
+
export async function detectVllmModels(host = VLLM_HOST, timeoutMs = 1500) {
|
|
74
|
+
try {
|
|
75
|
+
const ctrl = new AbortController();
|
|
76
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
77
|
+
const res = await fetch(new URL('/v1/models', host), { signal: ctrl.signal });
|
|
78
|
+
clearTimeout(t);
|
|
79
|
+
if (!res.ok) return [];
|
|
80
|
+
const body = await res.json();
|
|
81
|
+
return (body?.data ?? []).map((m) => m.id).filter((id) => typeof id === 'string');
|
|
82
|
+
} catch {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
88
|
+
|
|
89
|
+
/** Last N lines of the vLLM serve log — surfaced on startup failure. */
|
|
90
|
+
export async function readVllmLogTail(lines = 40) {
|
|
91
|
+
try {
|
|
92
|
+
const txt = await fs.readFile(logPath(), 'utf8');
|
|
93
|
+
return txt.split('\n').slice(-lines).join('\n').trim();
|
|
94
|
+
} catch {
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Block until vLLM is actually serving `servedName` (weights mapped, :8000
|
|
101
|
+
* answering /v1/models) — NOT just spawned. Returns as soon as the model
|
|
102
|
+
* appears, bails early if the process dies, and times out otherwise.
|
|
103
|
+
* A 9B model maps to GPU in ~1-2 min, so the default budget is generous.
|
|
104
|
+
*/
|
|
105
|
+
export async function waitForVllmModel(servedName, { pid = null, timeoutMs = 300_000, intervalMs = 3000 } = {}) {
|
|
106
|
+
const deadline = Date.now() + timeoutMs;
|
|
107
|
+
while (Date.now() < deadline) {
|
|
108
|
+
if (pid && !isPidAlive(pid)) {
|
|
109
|
+
return { serving: false, reason: 'vLLM process exited during startup (check the log tail below)' };
|
|
110
|
+
}
|
|
111
|
+
const models = await detectVllmModels();
|
|
112
|
+
if (servedName ? models.includes(servedName) : models.length > 0) {
|
|
113
|
+
return { serving: true, models };
|
|
114
|
+
}
|
|
115
|
+
await sleep(intervalMs);
|
|
116
|
+
}
|
|
117
|
+
return { serving: false, reason: `timed out after ${Math.round(timeoutMs / 1000)}s waiting for /v1/models to list ${servedName}` };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** True when the tracked serve pid is alive AND /v1/models answers. */
|
|
121
|
+
export async function isVllmServing() {
|
|
122
|
+
const st = await readVllmState();
|
|
123
|
+
if (st?.pid && !isPidAlive(st.pid)) return false;
|
|
124
|
+
return (await detectVllmModels()).length > 0;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Stop the tracked vLLM serve process (best-effort). */
|
|
128
|
+
export async function stopVllmServe() {
|
|
129
|
+
const st = await readVllmState();
|
|
130
|
+
if (st?.pid && isPidAlive(st.pid)) {
|
|
131
|
+
try { process.kill(st.pid, 'SIGTERM'); } catch { /* already dead */ }
|
|
132
|
+
}
|
|
133
|
+
await clearVllmState();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Start `vllm serve` for a model. `servedName` is what clients/heartbeat use
|
|
138
|
+
* (the `hf:org/repo` pull name) so the advertised model matches chat requests.
|
|
139
|
+
* `source` is the local weights dir (preferred) or the repo id vLLM fetches.
|
|
140
|
+
* Only one model runs at a time — replaces any current serve.
|
|
141
|
+
*/
|
|
142
|
+
export async function startVllmServe({ source, servedName, port = VLLM_PORT, token = null, extraArgs = [] }) {
|
|
143
|
+
const bin = resolveVllmBin();
|
|
144
|
+
if (!bin) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
'vLLM is not installed on this node (no vllm binary in ~/.local/bin or vllm-venv). ' +
|
|
147
|
+
'Re-run the installer on an NVIDIA host, or: pip install vllm.'
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
await stopVllmServe().catch(() => {});
|
|
151
|
+
|
|
152
|
+
const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
|
|
153
|
+
if (servedName) args.push('--served-model-name', servedName);
|
|
154
|
+
args.push(...extraArgs);
|
|
155
|
+
|
|
156
|
+
const env = { ...process.env };
|
|
157
|
+
if (token) env.HF_TOKEN = token;
|
|
158
|
+
|
|
159
|
+
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
160
|
+
const out = fss.openSync(logPath(), 'a');
|
|
161
|
+
const child = spawn(bin, args, { detached: true, stdio: ['ignore', out, out], env });
|
|
162
|
+
child.unref();
|
|
163
|
+
|
|
164
|
+
await writeVllmState({
|
|
165
|
+
pid: child.pid,
|
|
166
|
+
model: servedName ?? source,
|
|
167
|
+
source,
|
|
168
|
+
port,
|
|
169
|
+
startedAt: new Date().toISOString(),
|
|
170
|
+
log: logPath(),
|
|
171
|
+
});
|
|
172
|
+
return { pid: child.pid, port, model: servedName ?? source, log: logPath() };
|
|
173
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.47",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,13 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
36
|
-
"@infernetprotocol/engine": "0.1.
|
|
37
|
-
"@infernetprotocol/gpu": "0.1.
|
|
38
|
-
"@infernetprotocol/
|
|
39
|
-
"@infernetprotocol/rpc-adapter": "0.1.45"
|
|
33
|
+
"@infernetprotocol/config": "0.1.47",
|
|
34
|
+
"@infernetprotocol/auth": "0.1.47",
|
|
35
|
+
"@infernetprotocol/deploy-providers": "0.1.47",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.47",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.47",
|
|
38
|
+
"@infernetprotocol/rpc-adapter": "0.1.47"
|
|
40
39
|
},
|
|
41
40
|
"repository": {
|
|
42
41
|
"type": "git",
|
package/lib/discovery-bridge.js
DELETED
|
@@ -1,168 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* IPIP-0032 Phase 3 — bridge between the long-running daemon and the
|
|
3
|
-
* Hyperswarm DiscoveryNode in @infernetprotocol/discovery.
|
|
4
|
-
*
|
|
5
|
-
* Owns one DiscoveryNode for the daemon's lifetime. The set of joined
|
|
6
|
-
* topics is derived from:
|
|
7
|
-
* - the daemon's pubkey → node:<pubkey>
|
|
8
|
-
* - specs.served_models → model:<id> (single-node serving)
|
|
9
|
-
* - state.rpc_slice.models → rpc:<id> (IPIP-0033 slice)
|
|
10
|
-
* - state.rpc_primary.models → rpc:<id> (IPIP-0033 primary)
|
|
11
|
-
* - active rpc role / fleet-class → class:B or class:B5
|
|
12
|
-
*
|
|
13
|
-
* Recomputed periodically so daemons that come online and start
|
|
14
|
-
* serving a new model picked up the new topic without restart.
|
|
15
|
-
*
|
|
16
|
-
* Discovery is gated behind `--enable-dht` per the IPIP rollout —
|
|
17
|
-
* this module is only constructed when the flag is present, so
|
|
18
|
-
* Hyperswarm is never imported on legacy daemons.
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Pure topic-set builder — exported for tests. Returns a deduped
|
|
23
|
-
* array of `{ kind, value }` ready to hand to DiscoveryNode.setTopics.
|
|
24
|
-
*
|
|
25
|
-
* @param {Object} args
|
|
26
|
-
* @param {string} args.pubkey 64-char hex
|
|
27
|
-
* @param {string[]} [args.servedModels] specs.served_models
|
|
28
|
-
* @param {Object} [args.inferenceState] shape from lib/inference/state.js
|
|
29
|
-
* @param {'provider'|'aggregator'|'client'} [args.role]
|
|
30
|
-
*/
|
|
31
|
-
export function computeTopics({ pubkey, servedModels = [], inferenceState = {}, role }) {
|
|
32
|
-
const topics = [];
|
|
33
|
-
|
|
34
|
-
if (typeof pubkey === 'string' && /^[0-9a-fA-F]{64}$/.test(pubkey)) {
|
|
35
|
-
topics.push({ kind: 'node', value: pubkey.toLowerCase() });
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
for (const m of servedModels) {
|
|
39
|
-
if (typeof m === 'string' && m.length > 0) {
|
|
40
|
-
topics.push({ kind: 'model', value: m });
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const slice = inferenceState?.rpc_slice;
|
|
45
|
-
const primary = inferenceState?.rpc_primary;
|
|
46
|
-
const sliceModels = Array.isArray(slice?.models) ? slice.models : [];
|
|
47
|
-
const primaryModels = Array.isArray(primary?.models) ? primary.models : [];
|
|
48
|
-
for (const m of sliceModels) {
|
|
49
|
-
if (typeof m === 'string' && m.length > 0) topics.push({ kind: 'rpc', value: m });
|
|
50
|
-
}
|
|
51
|
-
for (const m of primaryModels) {
|
|
52
|
-
if (typeof m === 'string' && m.length > 0) topics.push({ kind: 'rpc', value: m });
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
// Workload class (per IPIP-0010): nodes running federated inference
|
|
56
|
-
// are class:B5; otherwise default providers are class:B. Aggregators
|
|
57
|
-
// and clients don't get a class topic — they're not job-takers.
|
|
58
|
-
const hasRpc = sliceModels.length > 0 || primaryModels.length > 0;
|
|
59
|
-
if (hasRpc) {
|
|
60
|
-
topics.push({ kind: 'class', value: 'B5' });
|
|
61
|
-
} else if (role === 'provider') {
|
|
62
|
-
topics.push({ kind: 'class', value: 'B' });
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
// Dedupe — primary + slice for the same model collapse to one rpc:
|
|
66
|
-
// entry; node: is unique by construction.
|
|
67
|
-
const seen = new Set();
|
|
68
|
-
return topics.filter((t) => {
|
|
69
|
-
const k = `${t.kind}:${t.value}`;
|
|
70
|
-
if (seen.has(k)) return false;
|
|
71
|
-
seen.add(k);
|
|
72
|
-
return true;
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Serialize the topic set into a stable string so we only call
|
|
78
|
-
* setTopics() when the set actually changes (saves a DHT churn).
|
|
79
|
-
*/
|
|
80
|
-
export function topicSetSignature(topics) {
|
|
81
|
-
return topics
|
|
82
|
-
.map((t) => `${t.kind}:${t.value}`)
|
|
83
|
-
.sort()
|
|
84
|
-
.join(',');
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Spin up the DiscoveryNode and resync its topic set on every
|
|
89
|
-
* `intervalMs` tick. Returns `{ stop }` for clean shutdown.
|
|
90
|
-
*
|
|
91
|
-
* The DiscoveryNode itself is loaded via dynamic import so legacy
|
|
92
|
-
* daemons that don't pass --enable-dht never touch the hyperswarm
|
|
93
|
-
* dep tree.
|
|
94
|
-
*/
|
|
95
|
-
export async function startDiscoveryBridge({
|
|
96
|
-
config,
|
|
97
|
-
getState,
|
|
98
|
-
advertise = true,
|
|
99
|
-
address = null,
|
|
100
|
-
intervalMs = 30_000,
|
|
101
|
-
onPeer = null,
|
|
102
|
-
onHandshakeFailed = null
|
|
103
|
-
}) {
|
|
104
|
-
const { createDiscoveryNode } = await import('@infernetprotocol/discovery');
|
|
105
|
-
const pubkey = config?.node?.publicKey;
|
|
106
|
-
const privateKey = config?.node?.privateKey;
|
|
107
|
-
const role = config?.node?.role;
|
|
108
|
-
if (!pubkey || !privateKey) {
|
|
109
|
-
throw new Error('discovery-bridge: config.node.{publicKey,privateKey} required');
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
const initialState = getState();
|
|
113
|
-
const initialTopics = computeTopics({
|
|
114
|
-
pubkey,
|
|
115
|
-
servedModels: initialState?.servedModels ?? [],
|
|
116
|
-
inferenceState: initialState?.inferenceState ?? {},
|
|
117
|
-
role
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
const node = await createDiscoveryNode({
|
|
121
|
-
privateKey,
|
|
122
|
-
publicKey: pubkey,
|
|
123
|
-
advertise,
|
|
124
|
-
address,
|
|
125
|
-
topics: initialTopics
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
let lastSig = topicSetSignature(initialTopics);
|
|
129
|
-
|
|
130
|
-
if (typeof onPeer === 'function') {
|
|
131
|
-
node.on('peer', onPeer);
|
|
132
|
-
}
|
|
133
|
-
if (typeof onHandshakeFailed === 'function') {
|
|
134
|
-
node.on('handshake-failed', onHandshakeFailed);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
let stopped = false;
|
|
138
|
-
const interval = setInterval(() => {
|
|
139
|
-
if (stopped) return;
|
|
140
|
-
try {
|
|
141
|
-
const s = getState();
|
|
142
|
-
const next = computeTopics({
|
|
143
|
-
pubkey,
|
|
144
|
-
servedModels: s?.servedModels ?? [],
|
|
145
|
-
inferenceState: s?.inferenceState ?? {},
|
|
146
|
-
role
|
|
147
|
-
});
|
|
148
|
-
const sig = topicSetSignature(next);
|
|
149
|
-
if (sig !== lastSig) {
|
|
150
|
-
node.setTopics(next);
|
|
151
|
-
lastSig = sig;
|
|
152
|
-
}
|
|
153
|
-
} catch {
|
|
154
|
-
// Diff failed — leave previous topics in place.
|
|
155
|
-
}
|
|
156
|
-
}, intervalMs);
|
|
157
|
-
if (typeof interval.unref === 'function') interval.unref();
|
|
158
|
-
|
|
159
|
-
return {
|
|
160
|
-
node,
|
|
161
|
-
get currentTopics() { return node.topics; },
|
|
162
|
-
stop: async () => {
|
|
163
|
-
stopped = true;
|
|
164
|
-
clearInterval(interval);
|
|
165
|
-
try { await node.destroy(); } catch { /* ignore */ }
|
|
166
|
-
}
|
|
167
|
-
};
|
|
168
|
-
}
|