@infernetprotocol/cli 0.1.45 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/register.js +9 -1
- package/commands/rpc.js +8 -9
- package/commands/setup.js +25 -15
- package/commands/start.js +55 -77
- package/commands/uncensored.js +18 -0
- package/lib/chat-executor.js +71 -29
- package/lib/hf-model.js +23 -47
- package/lib/prompt.js +19 -3
- package/lib/recommender.js +5 -1
- package/lib/vllm.js +140 -0
- package/package.json +7 -8
- package/lib/discovery-bridge.js +0 -168
package/commands/register.js
CHANGED
|
@@ -193,7 +193,15 @@ export async function gatherCoarseSpecs() {
|
|
|
193
193
|
|
|
194
194
|
// Advertise all pulled models — operator explicitly installed them so we
|
|
195
195
|
// trust they know their hardware. The fit check is informational only.
|
|
196
|
-
|
|
196
|
+
// Also fold in any model vLLM is currently serving (:8000/v1/models),
|
|
197
|
+
// advertised under its `hf:` pull name via --served-model-name, so vLLM
|
|
198
|
+
// models show up in /chat alongside Ollama ones.
|
|
199
|
+
let vllmModels = [];
|
|
200
|
+
try {
|
|
201
|
+
const { detectVllmModels } = await import('../lib/vllm.js');
|
|
202
|
+
vllmModels = await detectVllmModels();
|
|
203
|
+
} catch { /* vLLM not installed / not serving — Ollama-only */ }
|
|
204
|
+
const served_models = [...new Set([...pulledModels.map((m) => m.name), ...vllmModels])];
|
|
197
205
|
|
|
198
206
|
return {
|
|
199
207
|
cpu: summarizeCpu(),
|
package/commands/rpc.js
CHANGED
|
@@ -4,15 +4,14 @@
|
|
|
4
4
|
*
|
|
5
5
|
* infernet rpc census --model <id>
|
|
6
6
|
* Asks THIS daemon's HTTP /v1/rpc/census endpoint for the set
|
|
7
|
-
* of peers it sees on
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* the DHT is actually finding anyone.
|
|
7
|
+
* of peers it sees on rpc:<model>. The answer is a second
|
|
8
|
+
* opinion to the control-plane census — useful when the
|
|
9
|
+
* providers table looks stale.
|
|
11
10
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* The peer discovery substrate moved out of infernet — it's now
|
|
12
|
+
* c0mpute's libp2p Kad-DHT + gossipsub. Until /v1/rpc/census is
|
|
13
|
+
* wired through the c0mpute capability registry, this returns an
|
|
14
|
+
* empty set with a `note`.
|
|
16
15
|
*/
|
|
17
16
|
|
|
18
17
|
const HELP = `infernet rpc — federated-inference diagnostics
|
|
@@ -23,7 +22,7 @@ Usage:
|
|
|
23
22
|
|
|
24
23
|
Flags:
|
|
25
24
|
--model <id> Required. Canonical model id (e.g. qwen2.5:72b).
|
|
26
|
-
--kind <ns> Topic namespace (default: rpc
|
|
25
|
+
--kind <ns> Topic namespace (default: rpc).
|
|
27
26
|
--daemon <url> Daemon HTTP base. Defaults to http://127.0.0.1:8080,
|
|
28
27
|
matching the local daemon's healthz port. Set this
|
|
29
28
|
to query a peer's daemon directly.
|
package/commands/setup.js
CHANGED
|
@@ -46,7 +46,7 @@ Flags:
|
|
|
46
46
|
--confirm, --yes Auto-confirm every prompt. Use for unattended runs.
|
|
47
47
|
--model <name> Pre-select model to pull (e.g. qwen2.5:7b).
|
|
48
48
|
--skip-pull Skip the model-pull step.
|
|
49
|
-
--backend <kind> Pin engine backend (ollama | mojo | stub). Default:
|
|
49
|
+
--backend <kind> Pin engine backend (vllm | ollama | mojo | stub). Default: auto (prefers vLLM, falls back to Ollama).
|
|
50
50
|
--host <url> Override Ollama host. Default: http://localhost:11434.
|
|
51
51
|
--port <n> P2P port to open in the firewall. Default: ${DEFAULT_P2P_PORT}.
|
|
52
52
|
--no-firewall Skip the firewall step entirely.
|
|
@@ -284,8 +284,9 @@ async function isRegistered(config) {
|
|
|
284
284
|
*
|
|
285
285
|
* The control-plane-mediated chat path doesn't actually need inbound
|
|
286
286
|
* (daemon polls/posts outbound), so unreachable is a soft warning.
|
|
287
|
-
* But
|
|
288
|
-
*
|
|
287
|
+
* But the direct P2P chat transport DOES need inbound, so we surface
|
|
288
|
+
* this clearly. (Peer discovery for the chat transport runs through
|
|
289
|
+
* c0mpute now — install the c0mpute plugin to enable it.)
|
|
289
290
|
*/
|
|
290
291
|
async function reportReachability(row, config, port) {
|
|
291
292
|
const address = row?.address ?? null;
|
|
@@ -480,7 +481,15 @@ export default async function setup(args) {
|
|
|
480
481
|
const skipIdentity = args.has("skip-identity");
|
|
481
482
|
const skipRegister = args.has("skip-register");
|
|
482
483
|
const skipDaemon = args.has("skip-daemon") || args.has("no-start");
|
|
483
|
-
|
|
484
|
+
// Default backend is "auto": we DON'T pin it in config, so the engine's
|
|
485
|
+
// auto-select governs at serve time — and that precedence prefers
|
|
486
|
+
// vLLM > llama.cpp > Ollama. This is what makes vLLM the network default
|
|
487
|
+
// wherever it's installed, while boxes without vLLM still fall back to
|
|
488
|
+
// Ollama. Pass --backend ollama|vllm|stub to force one.
|
|
489
|
+
const backend = String(args.get("backend") ?? "auto");
|
|
490
|
+
// Still pull an Ollama starter model on "auto" so a fresh box has
|
|
491
|
+
// *something* to serve before an hf:/vLLM model is pushed.
|
|
492
|
+
const wantsOllamaStep = backend === "ollama" || backend === "auto";
|
|
484
493
|
const host = String(args.get("host") ?? process.env.OLLAMA_HOST ?? "http://localhost:11434");
|
|
485
494
|
const preselectedModel = args.get("model") ? String(args.get("model")) : null;
|
|
486
495
|
|
|
@@ -500,7 +509,7 @@ export default async function setup(args) {
|
|
|
500
509
|
// Track the actual maximum so the printed counter never goes [7/6].
|
|
501
510
|
const isProviderRole = (existing?.node?.role ?? "provider") === "provider";
|
|
502
511
|
let total = 2; // hardware + node.js
|
|
503
|
-
if (
|
|
512
|
+
if (wantsOllamaStep) total += 2; // ollama + model
|
|
504
513
|
if (!skipFirewall && process.platform === "linux") total += 1;
|
|
505
514
|
if (process.platform === "linux") total += 1; // logrotate
|
|
506
515
|
total += 1; // config
|
|
@@ -599,7 +608,7 @@ export default async function setup(args) {
|
|
|
599
608
|
let ollamaState = { running: false, models: [] };
|
|
600
609
|
let chosenModel = null;
|
|
601
610
|
|
|
602
|
-
if (
|
|
611
|
+
if (wantsOllamaStep) {
|
|
603
612
|
n += 1;
|
|
604
613
|
step(n, total, `Ollama (${host})`);
|
|
605
614
|
ollamaState = await detectOllama(host);
|
|
@@ -750,18 +759,19 @@ export default async function setup(args) {
|
|
|
750
759
|
|
|
751
760
|
n += 1;
|
|
752
761
|
step(n, total, "Config");
|
|
753
|
-
const
|
|
754
|
-
...existing,
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
backend,
|
|
758
|
-
...(host ? { ollamaHost: host } : {}),
|
|
759
|
-
...(chosenModel ? { model: chosenModel } : {})
|
|
760
|
-
}
|
|
762
|
+
const engineCfg = {
|
|
763
|
+
...(existing.engine ?? {}),
|
|
764
|
+
...(host ? { ollamaHost: host } : {}),
|
|
765
|
+
...(chosenModel ? { model: chosenModel } : {})
|
|
761
766
|
};
|
|
767
|
+
// "auto" → don't pin a backend; let the engine auto-select (vLLM > Ollama)
|
|
768
|
+
// at serve time. Any explicit --backend value is honored.
|
|
769
|
+
if (backend === "auto") delete engineCfg.backend;
|
|
770
|
+
else engineCfg.backend = backend;
|
|
771
|
+
const merged = { ...existing, engine: engineCfg };
|
|
762
772
|
await saveConfig(merged);
|
|
763
773
|
ok(`saved to ${getConfigPath()}`);
|
|
764
|
-
process.stdout.write(` engine.backend: ${merged.engine.backend}\n`);
|
|
774
|
+
process.stdout.write(` engine.backend: ${merged.engine.backend ?? "auto (vLLM > Ollama)"}\n`);
|
|
765
775
|
if (merged.engine.ollamaHost) {
|
|
766
776
|
process.stdout.write(` engine.ollamaHost: ${merged.engine.ollamaHost}\n`);
|
|
767
777
|
}
|
package/commands/start.js
CHANGED
|
@@ -47,13 +47,14 @@ import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/netwo
|
|
|
47
47
|
import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
49
|
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
|
+
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
51
|
+
import { startVllmServe, vllmInstalled } from '../lib/vllm.js';
|
|
50
52
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
51
53
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
52
54
|
import { pullLatestBinary } from './upgrade.js';
|
|
53
55
|
import { nodeTokenLogin } from './login.js';
|
|
54
56
|
import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from '../lib/version.js';
|
|
55
57
|
import { readInferenceState, readInferenceStateSync, isPidAlive } from '../lib/inference/state.js';
|
|
56
|
-
import { startDiscoveryBridge } from '../lib/discovery-bridge.js';
|
|
57
58
|
|
|
58
59
|
function readInferenceStateSyncSafe() {
|
|
59
60
|
try { return readInferenceStateSync(); } catch { return {}; }
|
|
@@ -130,7 +131,6 @@ async function spawnAndReturn(args) {
|
|
|
130
131
|
if (args.has('no-p2p')) passthrough.push('--no-p2p');
|
|
131
132
|
if (args.has('no-advertise')) passthrough.push('--no-advertise');
|
|
132
133
|
if (args.has('once')) passthrough.push('--once');
|
|
133
|
-
if (args.has('no-dht')) passthrough.push('--no-dht');
|
|
134
134
|
|
|
135
135
|
const { pid, logPath } = spawnDetachedDaemon(passthrough);
|
|
136
136
|
process.stdout.write(`infernet daemon started (pid ${pid})\n`);
|
|
@@ -154,11 +154,6 @@ async function runDaemon(args, ctx) {
|
|
|
154
154
|
|
|
155
155
|
const p2pDisabled = args.has('no-p2p');
|
|
156
156
|
const noAdvertise = args.has('no-advertise') || node.address === null;
|
|
157
|
-
// IPIP-0032: DHT discovery is on by default. Operators who want
|
|
158
|
-
// to opt out (private-only deployments, restricted networks, or
|
|
159
|
-
// an emergency kill switch) pass --no-dht or set
|
|
160
|
-
// INFERNET_DISABLE_DHT=1.
|
|
161
|
-
const enableDht = !(args.has('no-dht') || process.env.INFERNET_DISABLE_DHT === '1');
|
|
162
157
|
// Bind port (what we listen on, locally) and advertised port (what we
|
|
163
158
|
// tell the control plane to dial). Same value 99% of the time, but
|
|
164
159
|
// hosting platforms that NAT the container (RunPod, anything with
|
|
@@ -229,7 +224,6 @@ async function runDaemon(args, ctx) {
|
|
|
229
224
|
let ipcServer = null;
|
|
230
225
|
let p2pServer = null;
|
|
231
226
|
let healthServer = null;
|
|
232
|
-
let dhtBridge = null;
|
|
233
227
|
let p2pConnections = 0;
|
|
234
228
|
let p2pLastConnectionAt = null;
|
|
235
229
|
|
|
@@ -631,11 +625,38 @@ async function runDaemon(args, ctx) {
|
|
|
631
625
|
let result;
|
|
632
626
|
if (command === 'model_install') {
|
|
633
627
|
const modelName = String(args?.model);
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
628
|
+
if (modelName.startsWith('hf:')) {
|
|
629
|
+
// HuggingFace model → download the weights, then TURN ON
|
|
630
|
+
// vLLM to serve them (Ollama can't — it 400s on an hf: repo).
|
|
631
|
+
if (!vllmInstalled()) {
|
|
632
|
+
throw new Error(
|
|
633
|
+
`${modelName} needs vLLM, which isn't installed on this node. ` +
|
|
634
|
+
`Re-run the installer on this NVIDIA host (it installs vLLM), or: pip install vllm.`
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
const repoId = modelName.slice(3);
|
|
638
|
+
const token = process.env.HF_TOKEN || (await resolveHfToken().catch(() => null));
|
|
639
|
+
const localPath = await downloadHfModel(repoId, token);
|
|
640
|
+
// Serve the repo id, not a computed cache path — vLLM
|
|
641
|
+
// resolves it straight from the HF cache we just filled
|
|
642
|
+
// (the snapshot dir is a commit hash, not "main"). Serve
|
|
643
|
+
// under the hf: pull name so it matches chat requests and
|
|
644
|
+
// gets advertised in served_models → shows in /chat.
|
|
645
|
+
const serve = await startVllmServe({ source: repoId, servedName: modelName, token });
|
|
646
|
+
result = {
|
|
647
|
+
model: modelName,
|
|
648
|
+
backend: 'vllm',
|
|
649
|
+
localPath,
|
|
650
|
+
vllm: { pid: serve.pid, port: serve.port },
|
|
651
|
+
note: `Serving on vLLM :${serve.port} (first load can take a minute while weights map to GPU).`,
|
|
652
|
+
};
|
|
653
|
+
} else {
|
|
654
|
+
const fits = await checkModelFits(modelName);
|
|
655
|
+
if (fits && !fits.ok) {
|
|
656
|
+
throw new Error(formatFitFailure(modelName, fits));
|
|
657
|
+
}
|
|
658
|
+
result = await ollamaPullWithProgress(modelName, id);
|
|
637
659
|
}
|
|
638
|
-
result = await ollamaPullWithProgress(modelName, id);
|
|
639
660
|
} else if (command === 'model_remove') {
|
|
640
661
|
result = await ollamaSpawn(['rm', String(args?.model)]);
|
|
641
662
|
} else if (command === 'train_shard') {
|
|
@@ -1002,18 +1023,17 @@ async function runDaemon(args, ctx) {
|
|
|
1002
1023
|
}
|
|
1003
1024
|
|
|
1004
1025
|
/**
|
|
1005
|
-
*
|
|
1006
|
-
*
|
|
1007
|
-
* its Hyperswarm DHT view (verified handshakes, currently
|
|
1008
|
-
* connected). Useful as a second opinion when the control
|
|
1009
|
-
* plane's heartbeat-based census looks stale, and as a smoke
|
|
1010
|
-
* test for "is the DHT actually finding anyone."
|
|
1026
|
+
* Daemon-side peer census for the rpc-adapter (IPIP-0033 — llama.cpp
|
|
1027
|
+
* RPC peer assignment).
|
|
1011
1028
|
*
|
|
1012
1029
|
* GET /v1/rpc/census?model=<id>[&kind=rpc|model|class]
|
|
1013
|
-
* response: { kind, value, count, peers
|
|
1030
|
+
* response: { kind, value, count, peers, dht, note? }
|
|
1014
1031
|
*
|
|
1015
|
-
*
|
|
1016
|
-
*
|
|
1032
|
+
* Peer discovery used to come from the daemon's own Hyperswarm DHT
|
|
1033
|
+
* (IPIP-0032). That layer has been moved to c0mpute. Until the
|
|
1034
|
+
* c0mpute-backed query is wired in, this endpoint always returns
|
|
1035
|
+
* `count: 0` with a note — the chat client falls back to
|
|
1036
|
+
* control-plane routing.
|
|
1017
1037
|
*/
|
|
1018
1038
|
async function handleRpcCensus(req, res) {
|
|
1019
1039
|
if (req.method !== 'GET') return false;
|
|
@@ -1028,30 +1048,17 @@ async function runDaemon(args, ctx) {
|
|
|
1028
1048
|
return true;
|
|
1029
1049
|
}
|
|
1030
1050
|
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
};
|
|
1036
|
-
if (!dhtBridge?.node?.peersOnTopic) {
|
|
1037
|
-
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1038
|
-
res.end(JSON.stringify(empty));
|
|
1039
|
-
return true;
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
const peers = dhtBridge.node.peersOnTopic(kind, model);
|
|
1051
|
+
// Peer discovery has moved to c0mpute (libp2p Kad-DHT +
|
|
1052
|
+
// gossipsub capability ads). Until the c0mpute-backed query
|
|
1053
|
+
// is wired in, return an empty list with a note so the chat
|
|
1054
|
+
// client falls back to control-plane routing.
|
|
1043
1055
|
const body = {
|
|
1044
1056
|
kind,
|
|
1045
1057
|
value: model,
|
|
1046
|
-
count:
|
|
1047
|
-
peers:
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
first_seen: new Date(p.firstSeen).toISOString(),
|
|
1051
|
-
last_seen: new Date(p.lastSeen).toISOString()
|
|
1052
|
-
})),
|
|
1053
|
-
dht: true,
|
|
1054
|
-
self: { pubkey: dhtBridge.node.pubkey, topics: dhtBridge.node.topics }
|
|
1058
|
+
count: 0,
|
|
1059
|
+
peers: [],
|
|
1060
|
+
dht: false,
|
|
1061
|
+
note: 'peer discovery moved to c0mpute; query c0mpute capability registry instead'
|
|
1055
1062
|
};
|
|
1056
1063
|
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1057
1064
|
res.end(JSON.stringify(body));
|
|
@@ -1280,7 +1287,6 @@ async function runDaemon(args, ctx) {
|
|
|
1280
1287
|
if (ipcServer) { try { ipcServer.close(); } catch {} }
|
|
1281
1288
|
if (p2pServer) { try { p2pServer.close(); } catch {} }
|
|
1282
1289
|
if (healthServer) { try { healthServer.close(); } catch {} }
|
|
1283
|
-
if (dhtBridge) { try { await dhtBridge.stop(); } catch {} }
|
|
1284
1290
|
try { await shutdownEngine(); } catch {}
|
|
1285
1291
|
removeSocketFile();
|
|
1286
1292
|
await removePidFile();
|
|
@@ -1298,40 +1304,12 @@ async function runDaemon(args, ctx) {
|
|
|
1298
1304
|
process.stderr.write(`Failed to bind IPC socket at ${socketPath}: ${err?.message ?? err}\n`);
|
|
1299
1305
|
}
|
|
1300
1306
|
|
|
1301
|
-
//
|
|
1302
|
-
//
|
|
1303
|
-
//
|
|
1304
|
-
//
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
const dhtAddress = noAdvertise || !advertisedAddress
|
|
1308
|
-
? null
|
|
1309
|
-
: `${advertisedAddress}:${advertisedPort}`;
|
|
1310
|
-
dhtBridge = await startDiscoveryBridge({
|
|
1311
|
-
config,
|
|
1312
|
-
advertise: !noAdvertise,
|
|
1313
|
-
address: dhtAddress,
|
|
1314
|
-
getState: () => ({
|
|
1315
|
-
servedModels: cachedSpecs?.served_models ?? [],
|
|
1316
|
-
inferenceState: readInferenceStateSyncSafe()
|
|
1317
|
-
}),
|
|
1318
|
-
onPeer: ({ peer }) => {
|
|
1319
|
-
process.stdout.write(
|
|
1320
|
-
`[dht] peer ${peer.pubkey.slice(0, 8)}… topics=[${peer.topics.join(',')}]\n`
|
|
1321
|
-
);
|
|
1322
|
-
},
|
|
1323
|
-
onHandshakeFailed: ({ error }) => {
|
|
1324
|
-
process.stderr.write(`[dht] handshake-failed: ${error?.message ?? error}\n`);
|
|
1325
|
-
}
|
|
1326
|
-
});
|
|
1327
|
-
process.stdout.write(
|
|
1328
|
-
`[dht] joined ${dhtBridge.currentTopics.length} topic(s): ${dhtBridge.currentTopics.join(', ')}\n`
|
|
1329
|
-
);
|
|
1330
|
-
} catch (err) {
|
|
1331
|
-
process.stderr.write(`[dht] startup failed: ${err?.message ?? err}\n`);
|
|
1332
|
-
process.stderr.write(`[dht] continuing without DHT discovery — heartbeats still active\n`);
|
|
1333
|
-
}
|
|
1334
|
-
}
|
|
1307
|
+
// Peer discovery (formerly Hyperswarm DHT, IPIP-0032) has moved
|
|
1308
|
+
// to c0mpute's libp2p Kad-DHT + gossipsub. The infernet daemon no
|
|
1309
|
+
// longer maintains its own discovery bridge — workers advertise
|
|
1310
|
+
// capabilities via the c0mpute supervisor and the chat client
|
|
1311
|
+
// routes through the control plane until c0mpute-backed query
|
|
1312
|
+
// is wired in.
|
|
1335
1313
|
|
|
1336
1314
|
// /healthz — bind a tiny HTTP server on $PORT (default 8080) so
|
|
1337
1315
|
// Docker / Runpod / Kubernetes HEALTHCHECK probes can reach the
|
package/commands/uncensored.js
CHANGED
|
@@ -54,6 +54,13 @@ const HF_MODELS = [
|
|
|
54
54
|
minVramGb: 8,
|
|
55
55
|
description: 'Hermes 3 8B (Nous Research) — uncensored, instruction-tuned, Apache 2.0',
|
|
56
56
|
},
|
|
57
|
+
{
|
|
58
|
+
repoId: 'DavidAU/Qwen3.5-9B-Claude-4.6-HighIQ-THINKING-HERETIC-UNCENSORED',
|
|
59
|
+
sizeGb: 18.8,
|
|
60
|
+
sizeGb4bit: 6.0,
|
|
61
|
+
minVramGb: 24,
|
|
62
|
+
description: 'Qwen3.5 9B Heretic (DavidAU) — uncensored, thinking + vision, bf16 via vLLM',
|
|
63
|
+
},
|
|
57
64
|
{
|
|
58
65
|
repoId: 'cognitivecomputations/dolphin-2.9.4-llama3.1-70b',
|
|
59
66
|
sizeGb: 140.0,
|
|
@@ -61,6 +68,17 @@ const HF_MODELS = [
|
|
|
61
68
|
minVramGb: 40,
|
|
62
69
|
description: 'Dolphin 2.9.4 70B (Eric Hartford) — uncensored, Llama 3.1 base',
|
|
63
70
|
},
|
|
71
|
+
{
|
|
72
|
+
// 754B MoE, FP8 weights (~755 GB) — flagship uncensored model. Only
|
|
73
|
+
// auto-selected on a cluster that can actually hold it (minVramGb set
|
|
74
|
+
// so it never picks on a single GPU). Gated on HF: set HF_TOKEN to pull.
|
|
75
|
+
repoId: 'zandenAI/GLM-5.2-FP8-Uncensored',
|
|
76
|
+
sizeGb: 755.0,
|
|
77
|
+
sizeGb4bit: 755.0,
|
|
78
|
+
minVramGb: 768,
|
|
79
|
+
gated: true,
|
|
80
|
+
description: 'GLM-5.2 754B FP8 (abliterated) — uncensored flagship; needs a multi-GPU cluster (~768 GB VRAM) + HF token',
|
|
81
|
+
},
|
|
64
82
|
];
|
|
65
83
|
|
|
66
84
|
// Ollama models, ordered by VRAM requirement ascending
|
package/lib/chat-executor.js
CHANGED
|
@@ -93,44 +93,86 @@ class EventBuffer {
|
|
|
93
93
|
// /chat endpoint sometimes doesn't pass a model name). Without this
|
|
94
94
|
// fallback, every model-unspecified job died with
|
|
95
95
|
// "ollama backend: no model — set INFERNET_ENGINE_MODEL or pass model in the job"
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
106
|
-
// Optional override for the per-request num_thread cap.
|
|
107
|
-
// Default (50% of cores) is computed inside the Ollama
|
|
108
|
-
// backend; only forward when the operator explicitly set
|
|
109
|
-
// an override in config.
|
|
110
|
-
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
111
|
-
opts.numThread = eng.ollama_num_thread;
|
|
112
|
-
}
|
|
113
|
-
return createEngine(opts);
|
|
114
|
-
})().catch((err) => {
|
|
96
|
+
// One engine per backend, cached. We route PER MODEL: if the requested model
|
|
97
|
+
// is being served by vLLM (:8000), use the vLLM engine; otherwise the default
|
|
98
|
+
// (config backend or auto-select → Ollama). This is what makes vLLM the actual
|
|
99
|
+
// serving path for hf: models while Ollama keeps handling its own tags.
|
|
100
|
+
const engineCache = new Map(); // cacheKey -> Promise<engine>
|
|
101
|
+
|
|
102
|
+
function buildEngine(cacheKey, opts) {
|
|
103
|
+
if (!engineCache.has(cacheKey)) {
|
|
104
|
+
const p = (async () => createEngine(opts))().catch((err) => {
|
|
115
105
|
// Reset so a transient failure doesn't permanently poison the
|
|
116
106
|
// daemon — next job will retry initialization.
|
|
117
|
-
|
|
107
|
+
engineCache.delete(cacheKey);
|
|
118
108
|
throw err;
|
|
119
109
|
});
|
|
110
|
+
engineCache.set(cacheKey, p);
|
|
120
111
|
}
|
|
121
|
-
return
|
|
112
|
+
return engineCache.get(cacheKey);
|
|
122
113
|
}
|
|
123
114
|
|
|
124
|
-
|
|
125
|
-
|
|
115
|
+
async function defaultEngine() {
|
|
116
|
+
const config = (await loadConfig()) ?? {};
|
|
117
|
+
const eng = config.engine ?? {};
|
|
118
|
+
const opts = {};
|
|
119
|
+
if (eng.backend) opts.backend = eng.backend;
|
|
120
|
+
if (eng.model) opts.defaultModel = eng.model;
|
|
121
|
+
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
122
|
+
// Optional override for the per-request num_thread cap. Default (50% of
|
|
123
|
+
// cores) is computed inside the Ollama backend; only forward when set.
|
|
124
|
+
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
125
|
+
opts.numThread = eng.ollama_num_thread;
|
|
126
|
+
}
|
|
127
|
+
return buildEngine(`default:${eng.backend ?? 'auto'}`, opts);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Best-effort: does Ollama (at host) currently serve this exact tag? */
|
|
131
|
+
async function ollamaHasModel(modelName, host) {
|
|
126
132
|
try {
|
|
127
|
-
const
|
|
128
|
-
|
|
133
|
+
const ctrl = new AbortController();
|
|
134
|
+
const t = setTimeout(() => ctrl.abort(), 800);
|
|
135
|
+
const r = await fetch(`${host}/api/tags`, { signal: ctrl.signal });
|
|
136
|
+
clearTimeout(t);
|
|
137
|
+
if (!r.ok) return false;
|
|
138
|
+
const data = await r.json();
|
|
139
|
+
return (data.models ?? []).some((m) => m.name === modelName || m.model === modelName);
|
|
129
140
|
} catch {
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function getEngineForModel(modelName) {
|
|
146
|
+
// vLLM is the default serving path. Route PER MODEL so both engines coexist:
|
|
147
|
+
// 1. vLLM is serving this model → vLLM (preferred, high throughput)
|
|
148
|
+
// 2. Ollama has this GGUF tag → Ollama (its library keeps working even
|
|
149
|
+
// while vLLM is up serving another model)
|
|
150
|
+
// 3. otherwise → default (auto-select, which prefers vLLM)
|
|
151
|
+
if (modelName) {
|
|
152
|
+
try {
|
|
153
|
+
const { detectVllmModels, VLLM_HOST } = await import('./vllm.js');
|
|
154
|
+
const vModels = await detectVllmModels();
|
|
155
|
+
if (vModels.includes(modelName)) {
|
|
156
|
+
return buildEngine('vllm', { backend: 'vllm', host: VLLM_HOST, defaultModel: modelName });
|
|
157
|
+
}
|
|
158
|
+
} catch {
|
|
159
|
+
// vLLM module/probe failed — try Ollama, then the default.
|
|
160
|
+
}
|
|
161
|
+
const config = (await loadConfig().catch(() => null)) ?? {};
|
|
162
|
+
const ollamaHost = config.engine?.ollamaHost ?? process.env.OLLAMA_HOST ?? 'http://localhost:11434';
|
|
163
|
+
if (await ollamaHasModel(modelName, ollamaHost)) {
|
|
164
|
+
return buildEngine('ollama', { backend: 'ollama', host: ollamaHost, defaultModel: modelName });
|
|
165
|
+
}
|
|
133
166
|
}
|
|
167
|
+
return defaultEngine();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export async function shutdownEngine() {
|
|
171
|
+
const engines = [...engineCache.values()];
|
|
172
|
+
engineCache.clear();
|
|
173
|
+
await Promise.all(engines.map(async (p) => {
|
|
174
|
+
try { (await p).shutdown(); } catch { /* best-effort */ }
|
|
175
|
+
}));
|
|
134
176
|
}
|
|
135
177
|
|
|
136
178
|
/**
|
|
@@ -167,7 +209,7 @@ export async function executeChatJob({ client, job, node }) {
|
|
|
167
209
|
}
|
|
168
210
|
}
|
|
169
211
|
|
|
170
|
-
const engine = await
|
|
212
|
+
const engine = await getEngineForModel(job.model_name);
|
|
171
213
|
const buffer = new EventBuffer(client, job.id);
|
|
172
214
|
|
|
173
215
|
const t0 = Date.now();
|
package/lib/hf-model.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { spawn } from 'node:child_process';
|
|
10
|
+
import { existsSync } from 'node:fs';
|
|
10
11
|
import { loadConfig } from './config.js';
|
|
11
12
|
|
|
12
13
|
const HF_API = 'https://huggingface.co/api';
|
|
@@ -123,68 +124,43 @@ export async function fetchHfModelInfo(repoId, token) {
|
|
|
123
124
|
};
|
|
124
125
|
}
|
|
125
126
|
|
|
126
|
-
/**
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
p.on('error', () => resolve(false));
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/** Check whether Python huggingface_hub is importable. */
|
|
136
|
-
async function hfHubAvailable() {
|
|
137
|
-
return new Promise(resolve => {
|
|
138
|
-
const p = spawn('python3', ['-c', 'import huggingface_hub'], { stdio: 'ignore' });
|
|
139
|
-
p.on('exit', code => resolve(code === 0));
|
|
140
|
-
p.on('error', () => resolve(false));
|
|
141
|
-
});
|
|
127
|
+
/** The `hf` CLI the installer puts in the vLLM venv. One place, no fallbacks. */
|
|
128
|
+
function hfCli() {
|
|
129
|
+
const home = process.env.HOME || process.env.USERPROFILE || '';
|
|
130
|
+
const base = process.env.INFERNET_HOME || `${home}/.infernet`;
|
|
131
|
+
return `${base}/vllm-venv/bin/hf`;
|
|
142
132
|
}
|
|
143
133
|
|
|
144
134
|
/**
|
|
145
|
-
* Download a model
|
|
146
|
-
*
|
|
147
|
-
* Returns the local cache path.
|
|
135
|
+
* Download a HuggingFace model via `hf download`. Streams progress and returns
|
|
136
|
+
* the local cache path; surfaces the CLI's stderr on failure.
|
|
148
137
|
*/
|
|
149
138
|
export async function downloadHfModel(repoId, token, { localDir = null } = {}) {
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (!hasHub) {
|
|
139
|
+
const cli = hfCli();
|
|
140
|
+
if (!existsSync(cli)) {
|
|
154
141
|
throw new Error(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
' (or: pip3 install -U huggingface_hub)'
|
|
142
|
+
`HuggingFace CLI not found at ${cli}. Re-run the installer on this NVIDIA ` +
|
|
143
|
+
`host — it installs vLLM + huggingface_hub.`
|
|
158
144
|
);
|
|
159
145
|
}
|
|
160
146
|
|
|
161
|
-
// Build the download command
|
|
162
147
|
const args = ['download', repoId];
|
|
163
148
|
if (token) args.push('--token', token);
|
|
164
149
|
if (localDir) args.push('--local-dir', localDir);
|
|
165
150
|
|
|
166
|
-
|
|
167
|
-
const cmd = hasCli ? 'huggingface-cli' : 'python3';
|
|
168
|
-
const finalArgs = hasCli ? args : [
|
|
169
|
-
'-c',
|
|
170
|
-
`from huggingface_hub import snapshot_download; ` +
|
|
171
|
-
`p = snapshot_download(${JSON.stringify(repoId)}` +
|
|
172
|
-
`${token ? `, token=${JSON.stringify(token)}` : ''}` +
|
|
173
|
-
`${localDir ? `, local_dir=${JSON.stringify(localDir)}` : ''}` +
|
|
174
|
-
`); print(p)`
|
|
175
|
-
];
|
|
176
|
-
|
|
177
|
-
process.stdout.write(`\nDownloading ${repoId} from HuggingFace...\n`);
|
|
178
|
-
if (!hasCli) {
|
|
179
|
-
process.stdout.write('(tip: pip install huggingface_hub for a nicer progress bar)\n\n');
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
const child = spawn(cmd, finalArgs, { stdio: ['ignore', 'inherit', 'inherit'] });
|
|
183
|
-
let localPath = localDir;
|
|
151
|
+
process.stdout.write(`\nDownloading ${repoId} from HuggingFace…\n`);
|
|
184
152
|
|
|
153
|
+
return new Promise((resolve, reject) => {
|
|
154
|
+
const child = spawn(cli, args, { stdio: ['ignore', 'inherit', 'pipe'] });
|
|
155
|
+
let errTail = '';
|
|
156
|
+
child.stderr.on('data', (d) => {
|
|
157
|
+
const s = d.toString();
|
|
158
|
+
process.stderr.write(s);
|
|
159
|
+
errTail = (errTail + s).slice(-1200);
|
|
160
|
+
});
|
|
185
161
|
child.on('exit', code => {
|
|
186
|
-
if (code === 0) resolve(
|
|
187
|
-
else reject(new Error(`
|
|
162
|
+
if (code === 0) resolve(localDir ?? defaultCachePath(repoId));
|
|
163
|
+
else reject(new Error(`hf download exited ${code}${errTail.trim() ? `:\n${errTail.trim()}` : ''}`));
|
|
188
164
|
});
|
|
189
165
|
child.on('error', reject);
|
|
190
166
|
});
|
package/lib/prompt.js
CHANGED
|
@@ -17,15 +17,31 @@ import { stdin, stdout } from 'node:process';
|
|
|
17
17
|
export async function question(prompt, options = {}) {
|
|
18
18
|
const { default: defaultValue, secret = false } = options;
|
|
19
19
|
|
|
20
|
+
const suffix = defaultValue ? ` [${secret ? '****' : defaultValue}]` : '';
|
|
21
|
+
const full = `${prompt}${suffix}: `;
|
|
22
|
+
|
|
23
|
+
// Non-interactive stdin (piped `curl | sh` install, `--yes` automation,
|
|
24
|
+
// CI, nohup): there's no terminal to read an answer from. Creating a
|
|
25
|
+
// readline with terminal:true and awaiting rl.question() here NEVER
|
|
26
|
+
// resolves at EOF — the await hangs, the event loop drains, and Node
|
|
27
|
+
// exits 0 mid-function, so callers like `init` silently skip saveConfig
|
|
28
|
+
// and write no config at all. Echo the prompt + chosen default and
|
|
29
|
+
// return it immediately instead.
|
|
30
|
+
if (!stdin.isTTY) {
|
|
31
|
+
stdout.write(
|
|
32
|
+
defaultValue !== undefined
|
|
33
|
+
? `${full}${secret ? '****' : defaultValue} (default, non-interactive)\n`
|
|
34
|
+
: `${full}(non-interactive)\n`
|
|
35
|
+
);
|
|
36
|
+
return defaultValue !== undefined ? defaultValue : '';
|
|
37
|
+
}
|
|
38
|
+
|
|
20
39
|
const rl = readline.createInterface({
|
|
21
40
|
input: stdin,
|
|
22
41
|
output: stdout,
|
|
23
42
|
terminal: true
|
|
24
43
|
});
|
|
25
44
|
|
|
26
|
-
const suffix = defaultValue ? ` [${secret ? '****' : defaultValue}]` : '';
|
|
27
|
-
const full = `${prompt}${suffix}: `;
|
|
28
|
-
|
|
29
45
|
try {
|
|
30
46
|
let answer;
|
|
31
47
|
if (secret) {
|
package/lib/recommender.js
CHANGED
|
@@ -41,6 +41,7 @@ export const CATALOG = [
|
|
|
41
41
|
{ id: "qwen2.5-coder-14b",name:"Qwen2.5 Coder 14B", paramsB: 14, vramMin: 12, ramMin: 32, pullName: "qwen2.5-coder:14b", backend: "ollama", quality: 9, speed: 5, useCases: ["coding","agents"] },
|
|
42
42
|
|
|
43
43
|
// ── 32B (24GB+ VRAM, RTX 4090 / A100 sweet spot) ──
|
|
44
|
+
{ id: "qwen3.5-9b-heretic", name: "Qwen3.5 9B Heretic (uncensored)", paramsB: 9, vramMin: 24, ramMin: 32, pullName: "hf:DavidAU/Qwen3.5-9B-Claude-4.6-HighIQ-THINKING-HERETIC-UNCENSORED", backend: "vllm", quality: 8, speed: 6, useCases: ["chat","uncensored","vision","agents","study"], uncensored: true },
|
|
44
45
|
{ id: "qwen2.5-32b", name: "Qwen2.5 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5:32b", backend: "ollama", quality: 9, speed: 4, useCases: ["chat","coding","study","agents"], notes: "Single-80GB-GPU sweet spot." },
|
|
45
46
|
{ id: "qwen2.5-coder-32b",name:"Qwen2.5 Coder 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5-coder:32b", backend: "ollama", quality: 10, speed: 4, useCases: ["coding","agents"] },
|
|
46
47
|
|
|
@@ -49,7 +50,10 @@ export const CATALOG = [
|
|
|
49
50
|
{ id: "qwen2.5-72b", name: "Qwen2.5 72B", paramsB: 72, vramMin: 40, ramMin: 64, pullName: "qwen2.5:72b", backend: "ollama", quality: 10, speed: 3, useCases: ["chat","coding","study","agents"] },
|
|
50
51
|
|
|
51
52
|
// ── MoE (high VRAM, vLLM only) ──
|
|
52
|
-
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true }
|
|
53
|
+
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true },
|
|
54
|
+
|
|
55
|
+
// ── flagship / cluster-tier (multi-GPU or cloud; won't fit a single card) ──
|
|
56
|
+
{ id: "glm-5.2-uncensored", name: "GLM-5.2 754B FP8 (uncensored)", paramsB: 754, vramMin: 768, ramMin: 768, pullName: "hf:zandenAI/GLM-5.2-FP8-Uncensored", backend: "vllm", quality: 10, speed: 2, useCases: ["chat","uncensored","cybersecurity","agents"], uncensored: true, gated: true }
|
|
53
57
|
];
|
|
54
58
|
|
|
55
59
|
/**
|
package/lib/vllm.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vLLM serve manager — turns ON the vLLM backend the installer already put on
|
|
3
|
+
* the box (venv → `$INFERNET_BIN/vllm`) but that nothing ever started.
|
|
4
|
+
*
|
|
5
|
+
* vLLM serves ONE model per GPU process on an OpenAI-compatible API (:8000),
|
|
6
|
+
* so we run a single model at a time; installing a new `hf:` model replaces
|
|
7
|
+
* the current one. We track the process in a state file and advertise the
|
|
8
|
+
* served model (by its `hf:` pull name via `--served-model-name`) so it lands
|
|
9
|
+
* in `served_models` → shows in /chat, and the engine auto-selects vLLM over
|
|
10
|
+
* Ollama (per createEngine's precedence) once /v1/models answers.
|
|
11
|
+
*
|
|
12
|
+
* Everything here is best-effort + feature-detected: if vLLM isn't installed
|
|
13
|
+
* or a serve fails, callers fall back to Ollama — the working path is never
|
|
14
|
+
* broken.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { spawn } from 'node:child_process';
|
|
18
|
+
import fs from 'node:fs/promises';
|
|
19
|
+
import fss from 'node:fs';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import { getConfigDir } from './config.js';
|
|
22
|
+
import { isPidAlive } from './inference/state.js';
|
|
23
|
+
|
|
24
|
+
export const VLLM_PORT = Number(process.env.VLLM_PORT) || 8000;
|
|
25
|
+
export const VLLM_HOST = process.env.VLLM_HOST || `http://localhost:${VLLM_PORT}`;
|
|
26
|
+
|
|
27
|
+
function statePath() {
|
|
28
|
+
return path.join(getConfigDir(), 'vllm.json');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function logPath() {
|
|
32
|
+
return path.join(getConfigDir(), 'vllm.log');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Locate the vllm binary the installer symlinked (or a venv/PATH copy). */
|
|
36
|
+
export function resolveVllmBin() {
|
|
37
|
+
const home = process.env.HOME || process.env.USERPROFILE || '';
|
|
38
|
+
const candidates = [
|
|
39
|
+
process.env.VLLM_BIN,
|
|
40
|
+
path.join(home, '.local/bin/vllm'),
|
|
41
|
+
path.join(home, '.infernet/vllm-venv/bin/vllm'),
|
|
42
|
+
].filter(Boolean);
|
|
43
|
+
for (const c of candidates) {
|
|
44
|
+
try {
|
|
45
|
+
if (fss.existsSync(c) && (fss.statSync(c).mode & 0o111)) return c;
|
|
46
|
+
} catch { /* keep looking */ }
|
|
47
|
+
}
|
|
48
|
+
return null; // not installed
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function vllmInstalled() {
|
|
52
|
+
return resolveVllmBin() !== null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function readVllmState() {
|
|
56
|
+
try {
|
|
57
|
+
return JSON.parse(await fs.readFile(statePath(), 'utf8'));
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function writeVllmState(state) {
|
|
64
|
+
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
65
|
+
await fs.writeFile(statePath(), JSON.stringify(state, null, 2) + '\n', { mode: 0o600 });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function clearVllmState() {
|
|
69
|
+
try { await fs.rm(statePath()); } catch { /* already gone */ }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** GET /v1/models — the model(s) vLLM is currently serving (empty if down). */
|
|
73
|
+
export async function detectVllmModels(host = VLLM_HOST, timeoutMs = 1500) {
|
|
74
|
+
try {
|
|
75
|
+
const ctrl = new AbortController();
|
|
76
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
77
|
+
const res = await fetch(new URL('/v1/models', host), { signal: ctrl.signal });
|
|
78
|
+
clearTimeout(t);
|
|
79
|
+
if (!res.ok) return [];
|
|
80
|
+
const body = await res.json();
|
|
81
|
+
return (body?.data ?? []).map((m) => m.id).filter((id) => typeof id === 'string');
|
|
82
|
+
} catch {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** True when the tracked serve pid is alive AND /v1/models answers. */
|
|
88
|
+
export async function isVllmServing() {
|
|
89
|
+
const st = await readVllmState();
|
|
90
|
+
if (st?.pid && !isPidAlive(st.pid)) return false;
|
|
91
|
+
return (await detectVllmModels()).length > 0;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Stop the tracked vLLM serve process (best-effort). */
|
|
95
|
+
export async function stopVllmServe() {
|
|
96
|
+
const st = await readVllmState();
|
|
97
|
+
if (st?.pid && isPidAlive(st.pid)) {
|
|
98
|
+
try { process.kill(st.pid, 'SIGTERM'); } catch { /* already dead */ }
|
|
99
|
+
}
|
|
100
|
+
await clearVllmState();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Start `vllm serve` for a model. `servedName` is what clients/heartbeat use
|
|
105
|
+
* (the `hf:org/repo` pull name) so the advertised model matches chat requests.
|
|
106
|
+
* `source` is the local weights dir (preferred) or the repo id vLLM fetches.
|
|
107
|
+
* Only one model runs at a time — replaces any current serve.
|
|
108
|
+
*/
|
|
109
|
+
export async function startVllmServe({ source, servedName, port = VLLM_PORT, token = null, extraArgs = [] }) {
|
|
110
|
+
const bin = resolveVllmBin();
|
|
111
|
+
if (!bin) {
|
|
112
|
+
throw new Error(
|
|
113
|
+
'vLLM is not installed on this node (no vllm binary in ~/.local/bin or vllm-venv). ' +
|
|
114
|
+
'Re-run the installer on an NVIDIA host, or: pip install vllm.'
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
await stopVllmServe().catch(() => {});
|
|
118
|
+
|
|
119
|
+
const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
|
|
120
|
+
if (servedName) args.push('--served-model-name', servedName);
|
|
121
|
+
args.push(...extraArgs);
|
|
122
|
+
|
|
123
|
+
const env = { ...process.env };
|
|
124
|
+
if (token) env.HF_TOKEN = token;
|
|
125
|
+
|
|
126
|
+
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
127
|
+
const out = fss.openSync(logPath(), 'a');
|
|
128
|
+
const child = spawn(bin, args, { detached: true, stdio: ['ignore', out, out], env });
|
|
129
|
+
child.unref();
|
|
130
|
+
|
|
131
|
+
await writeVllmState({
|
|
132
|
+
pid: child.pid,
|
|
133
|
+
model: servedName ?? source,
|
|
134
|
+
source,
|
|
135
|
+
port,
|
|
136
|
+
startedAt: new Date().toISOString(),
|
|
137
|
+
log: logPath(),
|
|
138
|
+
});
|
|
139
|
+
return { pid: child.pid, port, model: servedName ?? source, log: logPath() };
|
|
140
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.46",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,13 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/config": "0.1.
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
38
|
-
"@infernetprotocol/
|
|
39
|
-
"@infernetprotocol/rpc-adapter": "0.1.45"
|
|
33
|
+
"@infernetprotocol/auth": "0.1.46",
|
|
34
|
+
"@infernetprotocol/config": "0.1.46",
|
|
35
|
+
"@infernetprotocol/engine": "0.1.46",
|
|
36
|
+
"@infernetprotocol/gpu": "0.1.46",
|
|
37
|
+
"@infernetprotocol/rpc-adapter": "0.1.46",
|
|
38
|
+
"@infernetprotocol/deploy-providers": "0.1.46"
|
|
40
39
|
},
|
|
41
40
|
"repository": {
|
|
42
41
|
"type": "git",
|
package/lib/discovery-bridge.js
DELETED
|
@@ -1,168 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* IPIP-0032 Phase 3 — bridge between the long-running daemon and the
|
|
3
|
-
* Hyperswarm DiscoveryNode in @infernetprotocol/discovery.
|
|
4
|
-
*
|
|
5
|
-
* Owns one DiscoveryNode for the daemon's lifetime. The set of joined
|
|
6
|
-
* topics is derived from:
|
|
7
|
-
* - the daemon's pubkey → node:<pubkey>
|
|
8
|
-
* - specs.served_models → model:<id> (single-node serving)
|
|
9
|
-
* - state.rpc_slice.models → rpc:<id> (IPIP-0033 slice)
|
|
10
|
-
* - state.rpc_primary.models → rpc:<id> (IPIP-0033 primary)
|
|
11
|
-
* - active rpc role / fleet-class → class:B or class:B5
|
|
12
|
-
*
|
|
13
|
-
* Recomputed periodically so daemons that come online and start
|
|
14
|
-
* serving a new model picked up the new topic without restart.
|
|
15
|
-
*
|
|
16
|
-
* Discovery is gated behind `--enable-dht` per the IPIP rollout —
|
|
17
|
-
* this module is only constructed when the flag is present, so
|
|
18
|
-
* Hyperswarm is never imported on legacy daemons.
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Pure topic-set builder — exported for tests. Returns a deduped
|
|
23
|
-
* array of `{ kind, value }` ready to hand to DiscoveryNode.setTopics.
|
|
24
|
-
*
|
|
25
|
-
* @param {Object} args
|
|
26
|
-
* @param {string} args.pubkey 64-char hex
|
|
27
|
-
* @param {string[]} [args.servedModels] specs.served_models
|
|
28
|
-
* @param {Object} [args.inferenceState] shape from lib/inference/state.js
|
|
29
|
-
* @param {'provider'|'aggregator'|'client'} [args.role]
|
|
30
|
-
*/
|
|
31
|
-
export function computeTopics({ pubkey, servedModels = [], inferenceState = {}, role }) {
|
|
32
|
-
const topics = [];
|
|
33
|
-
|
|
34
|
-
if (typeof pubkey === 'string' && /^[0-9a-fA-F]{64}$/.test(pubkey)) {
|
|
35
|
-
topics.push({ kind: 'node', value: pubkey.toLowerCase() });
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
for (const m of servedModels) {
|
|
39
|
-
if (typeof m === 'string' && m.length > 0) {
|
|
40
|
-
topics.push({ kind: 'model', value: m });
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const slice = inferenceState?.rpc_slice;
|
|
45
|
-
const primary = inferenceState?.rpc_primary;
|
|
46
|
-
const sliceModels = Array.isArray(slice?.models) ? slice.models : [];
|
|
47
|
-
const primaryModels = Array.isArray(primary?.models) ? primary.models : [];
|
|
48
|
-
for (const m of sliceModels) {
|
|
49
|
-
if (typeof m === 'string' && m.length > 0) topics.push({ kind: 'rpc', value: m });
|
|
50
|
-
}
|
|
51
|
-
for (const m of primaryModels) {
|
|
52
|
-
if (typeof m === 'string' && m.length > 0) topics.push({ kind: 'rpc', value: m });
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
// Workload class (per IPIP-0010): nodes running federated inference
|
|
56
|
-
// are class:B5; otherwise default providers are class:B. Aggregators
|
|
57
|
-
// and clients don't get a class topic — they're not job-takers.
|
|
58
|
-
const hasRpc = sliceModels.length > 0 || primaryModels.length > 0;
|
|
59
|
-
if (hasRpc) {
|
|
60
|
-
topics.push({ kind: 'class', value: 'B5' });
|
|
61
|
-
} else if (role === 'provider') {
|
|
62
|
-
topics.push({ kind: 'class', value: 'B' });
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
// Dedupe — primary + slice for the same model collapse to one rpc:
|
|
66
|
-
// entry; node: is unique by construction.
|
|
67
|
-
const seen = new Set();
|
|
68
|
-
return topics.filter((t) => {
|
|
69
|
-
const k = `${t.kind}:${t.value}`;
|
|
70
|
-
if (seen.has(k)) return false;
|
|
71
|
-
seen.add(k);
|
|
72
|
-
return true;
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Serialize the topic set into a stable string so we only call
|
|
78
|
-
* setTopics() when the set actually changes (saves a DHT churn).
|
|
79
|
-
*/
|
|
80
|
-
export function topicSetSignature(topics) {
|
|
81
|
-
return topics
|
|
82
|
-
.map((t) => `${t.kind}:${t.value}`)
|
|
83
|
-
.sort()
|
|
84
|
-
.join(',');
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Spin up the DiscoveryNode and resync its topic set on every
|
|
89
|
-
* `intervalMs` tick. Returns `{ stop }` for clean shutdown.
|
|
90
|
-
*
|
|
91
|
-
* The DiscoveryNode itself is loaded via dynamic import so legacy
|
|
92
|
-
* daemons that don't pass --enable-dht never touch the hyperswarm
|
|
93
|
-
* dep tree.
|
|
94
|
-
*/
|
|
95
|
-
export async function startDiscoveryBridge({
|
|
96
|
-
config,
|
|
97
|
-
getState,
|
|
98
|
-
advertise = true,
|
|
99
|
-
address = null,
|
|
100
|
-
intervalMs = 30_000,
|
|
101
|
-
onPeer = null,
|
|
102
|
-
onHandshakeFailed = null
|
|
103
|
-
}) {
|
|
104
|
-
const { createDiscoveryNode } = await import('@infernetprotocol/discovery');
|
|
105
|
-
const pubkey = config?.node?.publicKey;
|
|
106
|
-
const privateKey = config?.node?.privateKey;
|
|
107
|
-
const role = config?.node?.role;
|
|
108
|
-
if (!pubkey || !privateKey) {
|
|
109
|
-
throw new Error('discovery-bridge: config.node.{publicKey,privateKey} required');
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
const initialState = getState();
|
|
113
|
-
const initialTopics = computeTopics({
|
|
114
|
-
pubkey,
|
|
115
|
-
servedModels: initialState?.servedModels ?? [],
|
|
116
|
-
inferenceState: initialState?.inferenceState ?? {},
|
|
117
|
-
role
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
const node = await createDiscoveryNode({
|
|
121
|
-
privateKey,
|
|
122
|
-
publicKey: pubkey,
|
|
123
|
-
advertise,
|
|
124
|
-
address,
|
|
125
|
-
topics: initialTopics
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
let lastSig = topicSetSignature(initialTopics);
|
|
129
|
-
|
|
130
|
-
if (typeof onPeer === 'function') {
|
|
131
|
-
node.on('peer', onPeer);
|
|
132
|
-
}
|
|
133
|
-
if (typeof onHandshakeFailed === 'function') {
|
|
134
|
-
node.on('handshake-failed', onHandshakeFailed);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
let stopped = false;
|
|
138
|
-
const interval = setInterval(() => {
|
|
139
|
-
if (stopped) return;
|
|
140
|
-
try {
|
|
141
|
-
const s = getState();
|
|
142
|
-
const next = computeTopics({
|
|
143
|
-
pubkey,
|
|
144
|
-
servedModels: s?.servedModels ?? [],
|
|
145
|
-
inferenceState: s?.inferenceState ?? {},
|
|
146
|
-
role
|
|
147
|
-
});
|
|
148
|
-
const sig = topicSetSignature(next);
|
|
149
|
-
if (sig !== lastSig) {
|
|
150
|
-
node.setTopics(next);
|
|
151
|
-
lastSig = sig;
|
|
152
|
-
}
|
|
153
|
-
} catch {
|
|
154
|
-
// Diff failed — leave previous topics in place.
|
|
155
|
-
}
|
|
156
|
-
}, intervalMs);
|
|
157
|
-
if (typeof interval.unref === 'function') interval.unref();
|
|
158
|
-
|
|
159
|
-
return {
|
|
160
|
-
node,
|
|
161
|
-
get currentTopics() { return node.topics; },
|
|
162
|
-
stop: async () => {
|
|
163
|
-
stopped = true;
|
|
164
|
-
clearInterval(interval);
|
|
165
|
-
try { await node.destroy(); } catch { /* ignore */ }
|
|
166
|
-
}
|
|
167
|
-
};
|
|
168
|
-
}
|