@infernetprotocol/cli 0.1.41 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/inference.js +196 -179
- package/commands/register.js +10 -39
- package/commands/rpc.js +133 -0
- package/commands/setup.js +25 -15
- package/commands/start.js +281 -48
- package/commands/train.js +69 -17
- package/commands/uncensored.js +18 -0
- package/index.js +3 -2
- package/lib/chat-executor.js +71 -29
- package/lib/data-crawler.js +118 -9
- package/lib/hf-model.js +23 -47
- package/lib/inference/state.js +81 -0
- package/lib/prompt.js +19 -3
- package/lib/recommender.js +5 -1
- package/lib/vllm.js +140 -0
- package/package.json +7 -6
- package/lib/inference/petals-client.py.js +0 -115
- package/lib/inference/petals-engine.js +0 -75
package/lib/vllm.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vLLM serve manager — turns ON the vLLM backend the installer already put on
|
|
3
|
+
* the box (venv → `$INFERNET_BIN/vllm`) but that nothing ever started.
|
|
4
|
+
*
|
|
5
|
+
* vLLM serves ONE model per GPU process on an OpenAI-compatible API (:8000),
|
|
6
|
+
* so we run a single model at a time; installing a new `hf:` model replaces
|
|
7
|
+
* the current one. We track the process in a state file and advertise the
|
|
8
|
+
* served model (by its `hf:` pull name via `--served-model-name`) so it lands
|
|
9
|
+
* in `served_models` → shows in /chat, and the engine auto-selects vLLM over
|
|
10
|
+
* Ollama (per createEngine's precedence) once /v1/models answers.
|
|
11
|
+
*
|
|
12
|
+
* Everything here is best-effort + feature-detected: if vLLM isn't installed
|
|
13
|
+
* or a serve fails, callers fall back to Ollama — the working path is never
|
|
14
|
+
* broken.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { spawn } from 'node:child_process';
|
|
18
|
+
import fs from 'node:fs/promises';
|
|
19
|
+
import fss from 'node:fs';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import { getConfigDir } from './config.js';
|
|
22
|
+
import { isPidAlive } from './inference/state.js';
|
|
23
|
+
|
|
24
|
+
export const VLLM_PORT = Number(process.env.VLLM_PORT) || 8000;
|
|
25
|
+
export const VLLM_HOST = process.env.VLLM_HOST || `http://localhost:${VLLM_PORT}`;
|
|
26
|
+
|
|
27
|
+
function statePath() {
|
|
28
|
+
return path.join(getConfigDir(), 'vllm.json');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function logPath() {
|
|
32
|
+
return path.join(getConfigDir(), 'vllm.log');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Locate the vllm binary the installer symlinked (or a venv/PATH copy). */
|
|
36
|
+
export function resolveVllmBin() {
|
|
37
|
+
const home = process.env.HOME || process.env.USERPROFILE || '';
|
|
38
|
+
const candidates = [
|
|
39
|
+
process.env.VLLM_BIN,
|
|
40
|
+
path.join(home, '.local/bin/vllm'),
|
|
41
|
+
path.join(home, '.infernet/vllm-venv/bin/vllm'),
|
|
42
|
+
].filter(Boolean);
|
|
43
|
+
for (const c of candidates) {
|
|
44
|
+
try {
|
|
45
|
+
if (fss.existsSync(c) && (fss.statSync(c).mode & 0o111)) return c;
|
|
46
|
+
} catch { /* keep looking */ }
|
|
47
|
+
}
|
|
48
|
+
return null; // not installed
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function vllmInstalled() {
|
|
52
|
+
return resolveVllmBin() !== null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function readVllmState() {
|
|
56
|
+
try {
|
|
57
|
+
return JSON.parse(await fs.readFile(statePath(), 'utf8'));
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function writeVllmState(state) {
|
|
64
|
+
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
65
|
+
await fs.writeFile(statePath(), JSON.stringify(state, null, 2) + '\n', { mode: 0o600 });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function clearVllmState() {
|
|
69
|
+
try { await fs.rm(statePath()); } catch { /* already gone */ }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** GET /v1/models — the model(s) vLLM is currently serving (empty if down). */
|
|
73
|
+
export async function detectVllmModels(host = VLLM_HOST, timeoutMs = 1500) {
|
|
74
|
+
try {
|
|
75
|
+
const ctrl = new AbortController();
|
|
76
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
77
|
+
const res = await fetch(new URL('/v1/models', host), { signal: ctrl.signal });
|
|
78
|
+
clearTimeout(t);
|
|
79
|
+
if (!res.ok) return [];
|
|
80
|
+
const body = await res.json();
|
|
81
|
+
return (body?.data ?? []).map((m) => m.id).filter((id) => typeof id === 'string');
|
|
82
|
+
} catch {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** True when the tracked serve pid is alive AND /v1/models answers. */
|
|
88
|
+
export async function isVllmServing() {
|
|
89
|
+
const st = await readVllmState();
|
|
90
|
+
if (st?.pid && !isPidAlive(st.pid)) return false;
|
|
91
|
+
return (await detectVllmModels()).length > 0;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Stop the tracked vLLM serve process (best-effort). */
|
|
95
|
+
export async function stopVllmServe() {
|
|
96
|
+
const st = await readVllmState();
|
|
97
|
+
if (st?.pid && isPidAlive(st.pid)) {
|
|
98
|
+
try { process.kill(st.pid, 'SIGTERM'); } catch { /* already dead */ }
|
|
99
|
+
}
|
|
100
|
+
await clearVllmState();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Start `vllm serve` for a model. `servedName` is what clients/heartbeat use
|
|
105
|
+
* (the `hf:org/repo` pull name) so the advertised model matches chat requests.
|
|
106
|
+
* `source` is the local weights dir (preferred) or the repo id vLLM fetches.
|
|
107
|
+
* Only one model runs at a time — replaces any current serve.
|
|
108
|
+
*/
|
|
109
|
+
export async function startVllmServe({ source, servedName, port = VLLM_PORT, token = null, extraArgs = [] }) {
|
|
110
|
+
const bin = resolveVllmBin();
|
|
111
|
+
if (!bin) {
|
|
112
|
+
throw new Error(
|
|
113
|
+
'vLLM is not installed on this node (no vllm binary in ~/.local/bin or vllm-venv). ' +
|
|
114
|
+
'Re-run the installer on an NVIDIA host, or: pip install vllm.'
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
await stopVllmServe().catch(() => {});
|
|
118
|
+
|
|
119
|
+
const args = ['serve', source, '--host', '0.0.0.0', '--port', String(port)];
|
|
120
|
+
if (servedName) args.push('--served-model-name', servedName);
|
|
121
|
+
args.push(...extraArgs);
|
|
122
|
+
|
|
123
|
+
const env = { ...process.env };
|
|
124
|
+
if (token) env.HF_TOKEN = token;
|
|
125
|
+
|
|
126
|
+
await fs.mkdir(getConfigDir(), { recursive: true, mode: 0o700 });
|
|
127
|
+
const out = fss.openSync(logPath(), 'a');
|
|
128
|
+
const child = spawn(bin, args, { detached: true, stdio: ['ignore', out, out], env });
|
|
129
|
+
child.unref();
|
|
130
|
+
|
|
131
|
+
await writeVllmState({
|
|
132
|
+
pid: child.pid,
|
|
133
|
+
model: servedName ?? source,
|
|
134
|
+
source,
|
|
135
|
+
port,
|
|
136
|
+
startedAt: new Date().toISOString(),
|
|
137
|
+
log: logPath(),
|
|
138
|
+
});
|
|
139
|
+
return { pid: child.pid, port, model: servedName ?? source, log: logPath() };
|
|
140
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.46",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,11 +30,12 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/config": "0.1.
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
33
|
+
"@infernetprotocol/auth": "0.1.46",
|
|
34
|
+
"@infernetprotocol/config": "0.1.46",
|
|
35
|
+
"@infernetprotocol/engine": "0.1.46",
|
|
36
|
+
"@infernetprotocol/gpu": "0.1.46",
|
|
37
|
+
"@infernetprotocol/rpc-adapter": "0.1.46",
|
|
38
|
+
"@infernetprotocol/deploy-providers": "0.1.46"
|
|
38
39
|
},
|
|
39
40
|
"repository": {
|
|
40
41
|
"type": "git",
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Bundled Python petals client. Spawned by the daemon when it accepts
|
|
3
|
-
* an /v1/petals/inference POST. Fans out across the public Petals
|
|
4
|
-
* swarm using AutoDistributedModelForCausalLM and streams JSON lines
|
|
5
|
-
* (one per token) to stdout. The daemon converts those to SSE frames.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
export const PETALS_CLIENT_PY = `#!/usr/bin/env python3
|
|
9
|
-
"""Petals client — fans out across the swarm + streams tokens.
|
|
10
|
-
|
|
11
|
-
Reads: JSON request from stdin: { model, messages, max_tokens, temperature }
|
|
12
|
-
Writes: JSON lines to stdout: { event: 'token' | 'done' | 'error', ... }
|
|
13
|
-
|
|
14
|
-
Each output line is a single JSON object terminated with \\n so the
|
|
15
|
-
parent process can parse them line-by-line.
|
|
16
|
-
"""
|
|
17
|
-
import json
|
|
18
|
-
import sys
|
|
19
|
-
import time
|
|
20
|
-
|
|
21
|
-
def emit(obj):
|
|
22
|
-
sys.stdout.write(json.dumps(obj) + "\\n")
|
|
23
|
-
sys.stdout.flush()
|
|
24
|
-
|
|
25
|
-
def main():
|
|
26
|
-
try:
|
|
27
|
-
req = json.loads(sys.stdin.read())
|
|
28
|
-
except Exception as e:
|
|
29
|
-
emit({"event": "error", "message": f"bad request: {e}"})
|
|
30
|
-
sys.exit(2)
|
|
31
|
-
|
|
32
|
-
model = req.get("model")
|
|
33
|
-
messages = req.get("messages", [])
|
|
34
|
-
max_tokens = int(req.get("max_tokens") or 256)
|
|
35
|
-
temperature = float(req.get("temperature") or 0.7)
|
|
36
|
-
if not model:
|
|
37
|
-
emit({"event": "error", "message": "model required"})
|
|
38
|
-
sys.exit(2)
|
|
39
|
-
|
|
40
|
-
try:
|
|
41
|
-
import torch
|
|
42
|
-
from transformers import AutoTokenizer
|
|
43
|
-
from petals import AutoDistributedModelForCausalLM
|
|
44
|
-
except ImportError as e:
|
|
45
|
-
emit({"event": "error", "message": f"petals not installed: {e}. Install: pip install -U petals"})
|
|
46
|
-
sys.exit(2)
|
|
47
|
-
|
|
48
|
-
emit({"event": "loading", "model": model})
|
|
49
|
-
tokenizer = AutoTokenizer.from_pretrained(model)
|
|
50
|
-
model_obj = AutoDistributedModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
|
|
51
|
-
|
|
52
|
-
# Compose a single prompt from messages using the tokenizer's chat template.
|
|
53
|
-
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
54
|
-
input_ids = tokenizer.encode(prompt, return_tensors="pt")
|
|
55
|
-
|
|
56
|
-
emit({"event": "ready", "input_tokens": int(input_ids.shape[-1])})
|
|
57
|
-
|
|
58
|
-
# Petals streams via inference_session — emit one JSON line per generated token.
|
|
59
|
-
full_text = ""
|
|
60
|
-
t0 = time.time()
|
|
61
|
-
with model_obj.inference_session(max_length=int(input_ids.shape[-1]) + max_tokens) as session:
|
|
62
|
-
# Prefill
|
|
63
|
-
outputs = model_obj.generate(
|
|
64
|
-
input_ids,
|
|
65
|
-
max_new_tokens=1,
|
|
66
|
-
do_sample=temperature > 0,
|
|
67
|
-
temperature=max(temperature, 1e-3),
|
|
68
|
-
session=session,
|
|
69
|
-
)
|
|
70
|
-
new_id = int(outputs[0, -1])
|
|
71
|
-
token_text = tokenizer.decode([new_id], skip_special_tokens=True)
|
|
72
|
-
full_text += token_text
|
|
73
|
-
emit({"event": "token", "text": token_text})
|
|
74
|
-
|
|
75
|
-
# IPIP-0031 per-layer attribution: after the first generate the
|
|
76
|
-
# session has resolved which DHT peers serve which transformer
|
|
77
|
-
# blocks. Emit that mapping so the control plane can split the
|
|
78
|
-
# CPR receipt across layer-contributing operators by block share.
|
|
79
|
-
try:
|
|
80
|
-
chosen = []
|
|
81
|
-
for s in getattr(session, "chosen_servers", []) or []:
|
|
82
|
-
chosen.append({
|
|
83
|
-
"peer_id": str(getattr(s, "peer_id", "")),
|
|
84
|
-
"start_block": int(getattr(s, "start", getattr(s, "start_block", 0))),
|
|
85
|
-
"end_block": int(getattr(s, "end", getattr(s, "end_block", 0))),
|
|
86
|
-
})
|
|
87
|
-
if chosen:
|
|
88
|
-
emit({"event": "routing", "peers": chosen})
|
|
89
|
-
except Exception as e:
|
|
90
|
-
emit({"event": "log", "warn": f"chosen_servers extract failed: {e}"})
|
|
91
|
-
|
|
92
|
-
for _ in range(max_tokens - 1):
|
|
93
|
-
outputs = model_obj.generate(
|
|
94
|
-
outputs,
|
|
95
|
-
max_new_tokens=1,
|
|
96
|
-
do_sample=temperature > 0,
|
|
97
|
-
temperature=max(temperature, 1e-3),
|
|
98
|
-
session=session,
|
|
99
|
-
)
|
|
100
|
-
new_id = int(outputs[0, -1])
|
|
101
|
-
if new_id == tokenizer.eos_token_id:
|
|
102
|
-
break
|
|
103
|
-
token_text = tokenizer.decode([new_id], skip_special_tokens=True)
|
|
104
|
-
full_text += token_text
|
|
105
|
-
emit({"event": "token", "text": token_text})
|
|
106
|
-
|
|
107
|
-
emit({"event": "done", "text": full_text, "elapsed_s": round(time.time() - t0, 2)})
|
|
108
|
-
|
|
109
|
-
if __name__ == "__main__":
|
|
110
|
-
try:
|
|
111
|
-
main()
|
|
112
|
-
except Exception as e:
|
|
113
|
-
emit({"event": "error", "message": str(e)})
|
|
114
|
-
sys.exit(1)
|
|
115
|
-
`;
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Petals engine adapter — DISTRIBUTED inference across the volunteer
|
|
3
|
-
* swarm (IPIP-0031, follow-up PR).
|
|
4
|
-
*
|
|
5
|
-
* Petals (https://github.com/bigscience-workshop/petals) splits a
|
|
6
|
-
* Llama/Mixtral-style transformer by layer across DHT-discovered nodes.
|
|
7
|
-
* Each node holds a few transformer blocks; activations flow through.
|
|
8
|
-
* Designed for residential bandwidth — exactly the Infernet model.
|
|
9
|
-
*
|
|
10
|
-
* STATUS: scaffolding only. The flow:
|
|
11
|
-
*
|
|
12
|
-
* serve() — operator runs `infernet inference serve --backend petals
|
|
13
|
-
* --model meta-llama/Llama-3.1-70B`. We:
|
|
14
|
-
* 1. spawn `python -m petals.cli.run_server <model>`
|
|
15
|
-
* 2. register the operator as a Petals server in our
|
|
16
|
-
* own registry: row in providers.specs.petals_models
|
|
17
|
-
* 3. heartbeat that the server is alive
|
|
18
|
-
*
|
|
19
|
-
* client() — control plane's chat router, when input_spec.distributed
|
|
20
|
-
* is true, runs a Petals CLIENT (Python subprocess or
|
|
21
|
-
* a thin TS wrapper) that fans out across the swarm.
|
|
22
|
-
* Tokens stream back to the SSE handler exactly like
|
|
23
|
-
* Ollama's stream.
|
|
24
|
-
*
|
|
25
|
-
* The SSE handler in apps/web/app/api/chat/stream/[jobId]/route.js
|
|
26
|
-
* already has the typed events plumbing — when distributed===true,
|
|
27
|
-
* it would call into routePetalsRequest below instead of subscribing
|
|
28
|
-
* to job_events.
|
|
29
|
-
*
|
|
30
|
-
* To-do for the follow-up PR:
|
|
31
|
-
* - bundle a thin Python venv with Petals at infernet setup time
|
|
32
|
-
* - add `infernet inference serve --backend petals` as a top-level command
|
|
33
|
-
* - register petals_models on heartbeat so the dashboard can show
|
|
34
|
-
* "this model is served by 8 nodes via Petals · ~12 tok/s" badges
|
|
35
|
-
* - wire the SSE handler's distributed branch through routePetalsRequest
|
|
36
|
-
* - per-token receipts to operators contributing layers (CPR / IPIP-0007)
|
|
37
|
-
*/
|
|
38
|
-
|
|
39
|
-
import { spawn } from "node:child_process";
|
|
40
|
-
|
|
41
|
-
const DEFAULT_DHT_PREFIX = "/petals/v3-public";
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Start a Petals server contributing this node's VRAM to the swarm.
|
|
45
|
-
* Returns a child handle the daemon can monitor + restart.
|
|
46
|
-
*/
|
|
47
|
-
export function startPetalsServer({ model, numBlocks, port = 31330, prefix = DEFAULT_DHT_PREFIX }) {
|
|
48
|
-
const args = ["-m", "petals.cli.run_server", model];
|
|
49
|
-
if (numBlocks) args.push("--num_blocks", String(numBlocks));
|
|
50
|
-
if (port) args.push("--port", String(port));
|
|
51
|
-
if (prefix) args.push("--initial_peers", prefix);
|
|
52
|
-
const child = spawn("python3", args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
53
|
-
return child;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Run a single inference through a Petals client and stream tokens.
|
|
58
|
-
* Returns an async iterable of { type: 'token' | 'done' | 'error', data }.
|
|
59
|
-
*
|
|
60
|
-
* Currently a stub — wires up the subprocess but doesn't translate the
|
|
61
|
-
* Python output into typed events. Follow-up PR: pip install
|
|
62
|
-
* `petals` in our control-plane container and call from a Python
|
|
63
|
-
* worker, OR use a JS-native bridge once one exists.
|
|
64
|
-
*/
|
|
65
|
-
export async function* routePetalsRequest({ model, messages, maxTokens = 256, temperature = 0.7 }) {
|
|
66
|
-
yield {
|
|
67
|
-
type: "error",
|
|
68
|
-
data: {
|
|
69
|
-
message:
|
|
70
|
-
`Distributed inference via Petals is in scaffolding (IPIP-0031). ` +
|
|
71
|
-
`For now, this request needs to fall back to a single-node provider. ` +
|
|
72
|
-
`Track progress at https://github.com/infernetprotocol/infernet-protocol/issues`
|
|
73
|
-
}
|
|
74
|
-
};
|
|
75
|
-
}
|