@infernetprotocol/cli 0.1.37 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/register.js +24 -1
- package/commands/start.js +81 -0
- package/lib/inference/petals-client.py.js +98 -0
- package/package.json +6 -6
package/commands/register.js
CHANGED
|
@@ -204,10 +204,33 @@ export async function gatherCoarseSpecs() {
|
|
|
204
204
|
model: typeof g.model === 'string' ? g.model.slice(0, 64) : null
|
|
205
205
|
})),
|
|
206
206
|
interconnects: summarizeInterconnects(interconnects),
|
|
207
|
-
served_models
|
|
207
|
+
served_models,
|
|
208
|
+
// IPIP-0031: models this node is contributing to a Petals swarm.
|
|
209
|
+
// Empty array if the operator hasn't run `infernet inference serve`.
|
|
210
|
+
// Read from local state file written by inference.js.
|
|
211
|
+
petals_models: await readPetalsModels()
|
|
208
212
|
};
|
|
209
213
|
}
|
|
210
214
|
|
|
215
|
+
async function readPetalsModels() {
|
|
216
|
+
try {
|
|
217
|
+
const fsp = await import('node:fs/promises');
|
|
218
|
+
const path = await import('node:path');
|
|
219
|
+
const statePath = path.join(process.env.HOME ?? '/tmp', '.infernet', 'inference', 'state.json');
|
|
220
|
+
const raw = await fsp.readFile(statePath, 'utf8');
|
|
221
|
+
const state = JSON.parse(raw);
|
|
222
|
+
// Only advertise if the server is still alive.
|
|
223
|
+
if (state.pid) {
|
|
224
|
+
try { process.kill(state.pid, 0); } // 0 = check liveness
|
|
225
|
+
catch { return []; } // pid stale
|
|
226
|
+
} else { return []; }
|
|
227
|
+
if (state.backend === 'petals' && typeof state.model === 'string') {
|
|
228
|
+
return [state.model];
|
|
229
|
+
}
|
|
230
|
+
return [];
|
|
231
|
+
} catch { return []; }
|
|
232
|
+
}
|
|
233
|
+
|
|
211
234
|
export default async function register(args, ctx) {
|
|
212
235
|
if (args.has('help') || args.has('h')) {
|
|
213
236
|
process.stdout.write(HELP);
|
package/commands/start.js
CHANGED
|
@@ -776,6 +776,78 @@ async function runDaemon(args, ctx) {
|
|
|
776
776
|
};
|
|
777
777
|
}
|
|
778
778
|
|
|
779
|
+
/**
|
|
780
|
+
* IPIP-0031: distributed inference proxy. Accepts a chat request,
|
|
781
|
+
* spawns a Python Petals client subprocess, streams its JSON-line
|
|
782
|
+
* output back as SSE.
|
|
783
|
+
*
|
|
784
|
+
* POST /v1/petals/inference
|
|
785
|
+
* body: { model, messages, max_tokens?, temperature? }
|
|
786
|
+
* response: text/event-stream with token / done / error frames
|
|
787
|
+
*/
|
|
788
|
+
async function handlePetalsInference(req, res) {
|
|
789
|
+
if (req.method !== 'POST' || req.url !== '/v1/petals/inference') return false;
|
|
790
|
+
|
|
791
|
+
const fsp = await import('node:fs/promises');
|
|
792
|
+
const path = await import('node:path');
|
|
793
|
+
const os = await import('node:os');
|
|
794
|
+
const { PETALS_CLIENT_PY } = await import('../lib/inference/petals-client.py.js');
|
|
795
|
+
|
|
796
|
+
// Read JSON body
|
|
797
|
+
let body = '';
|
|
798
|
+
await new Promise((resolve, reject) => {
|
|
799
|
+
req.on('data', (chunk) => { body += chunk.toString(); });
|
|
800
|
+
req.on('end', resolve);
|
|
801
|
+
req.on('error', reject);
|
|
802
|
+
});
|
|
803
|
+
let payload;
|
|
804
|
+
try { payload = JSON.parse(body); } catch {
|
|
805
|
+
res.writeHead(400, { 'content-type': 'text/plain' });
|
|
806
|
+
res.end('bad json\n');
|
|
807
|
+
return true;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
// Drop the bundled Python client to a temp file and spawn it.
|
|
811
|
+
const tmp = path.join(os.tmpdir(), `petals-${process.pid}-${Date.now()}.py`);
|
|
812
|
+
await fsp.writeFile(tmp, PETALS_CLIENT_PY);
|
|
813
|
+
|
|
814
|
+
res.writeHead(200, {
|
|
815
|
+
'content-type': 'text/event-stream',
|
|
816
|
+
'cache-control': 'no-cache',
|
|
817
|
+
'x-accel-buffering': 'no'
|
|
818
|
+
});
|
|
819
|
+
|
|
820
|
+
const child = spawn('python3', [tmp], { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
821
|
+
child.stdin.write(JSON.stringify(payload));
|
|
822
|
+
child.stdin.end();
|
|
823
|
+
|
|
824
|
+
let stdoutBuf = '';
|
|
825
|
+
child.stdout.on('data', (chunk) => {
|
|
826
|
+
stdoutBuf += chunk.toString();
|
|
827
|
+
let nl;
|
|
828
|
+
while ((nl = stdoutBuf.indexOf('\n')) >= 0) {
|
|
829
|
+
const line = stdoutBuf.slice(0, nl).trim();
|
|
830
|
+
stdoutBuf = stdoutBuf.slice(nl + 1);
|
|
831
|
+
if (!line) continue;
|
|
832
|
+
try {
|
|
833
|
+
const ev = JSON.parse(line);
|
|
834
|
+
res.write(`event: ${ev.event ?? 'data'}\ndata: ${JSON.stringify(ev)}\n\n`);
|
|
835
|
+
} catch {
|
|
836
|
+
res.write(`event: log\ndata: ${JSON.stringify({ raw: line })}\n\n`);
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
});
|
|
840
|
+
child.stderr.on('data', (chunk) => {
|
|
841
|
+
res.write(`event: log\ndata: ${JSON.stringify({ stderr: chunk.toString() })}\n\n`);
|
|
842
|
+
});
|
|
843
|
+
child.on('exit', async (code) => {
|
|
844
|
+
res.write(`event: exit\ndata: ${JSON.stringify({ code })}\n\n`);
|
|
845
|
+
res.end();
|
|
846
|
+
try { await fsp.unlink(tmp); } catch { /* ignore */ }
|
|
847
|
+
});
|
|
848
|
+
return true;
|
|
849
|
+
}
|
|
850
|
+
|
|
779
851
|
/**
|
|
780
852
|
* IPIP-0030: shard-hosting routes layered on top of /healthz.
|
|
781
853
|
* Returns true if the request matched and was handled.
|
|
@@ -1063,6 +1135,15 @@ async function runDaemon(args, ctx) {
|
|
|
1063
1135
|
return;
|
|
1064
1136
|
}
|
|
1065
1137
|
|
|
1138
|
+
try {
|
|
1139
|
+
const petalsHandled = await handlePetalsInference(req, res);
|
|
1140
|
+
if (petalsHandled) return;
|
|
1141
|
+
} catch (err) {
|
|
1142
|
+
res.writeHead(500, { 'content-type': 'text/plain' });
|
|
1143
|
+
res.end(`petals http error: ${err?.message ?? err}\n`);
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1066
1147
|
res.writeHead(404, { 'content-type': 'text/plain' });
|
|
1067
1148
|
res.end('not found\n');
|
|
1068
1149
|
});
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bundled Python petals client. Spawned by the daemon when it accepts
|
|
3
|
+
* an /v1/petals/inference POST. Fans out across the public Petals
|
|
4
|
+
* swarm using AutoDistributedModelForCausalLM and streams JSON lines
|
|
5
|
+
* (one per token) to stdout. The daemon converts those to SSE frames.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export const PETALS_CLIENT_PY = `#!/usr/bin/env python3
|
|
9
|
+
"""Petals client — fans out across the swarm + streams tokens.
|
|
10
|
+
|
|
11
|
+
Reads: JSON request from stdin: { model, messages, max_tokens, temperature }
|
|
12
|
+
Writes: JSON lines to stdout: { event: 'token' | 'done' | 'error', ... }
|
|
13
|
+
|
|
14
|
+
Each output line is a single JSON object terminated with \\n so the
|
|
15
|
+
parent process can parse them line-by-line.
|
|
16
|
+
"""
|
|
17
|
+
import json
|
|
18
|
+
import sys
|
|
19
|
+
import time
|
|
20
|
+
|
|
21
|
+
def emit(obj):
|
|
22
|
+
sys.stdout.write(json.dumps(obj) + "\\n")
|
|
23
|
+
sys.stdout.flush()
|
|
24
|
+
|
|
25
|
+
def main():
|
|
26
|
+
try:
|
|
27
|
+
req = json.loads(sys.stdin.read())
|
|
28
|
+
except Exception as e:
|
|
29
|
+
emit({"event": "error", "message": f"bad request: {e}"})
|
|
30
|
+
sys.exit(2)
|
|
31
|
+
|
|
32
|
+
model = req.get("model")
|
|
33
|
+
messages = req.get("messages", [])
|
|
34
|
+
max_tokens = int(req.get("max_tokens") or 256)
|
|
35
|
+
temperature = float(req.get("temperature") or 0.7)
|
|
36
|
+
if not model:
|
|
37
|
+
emit({"event": "error", "message": "model required"})
|
|
38
|
+
sys.exit(2)
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
import torch
|
|
42
|
+
from transformers import AutoTokenizer
|
|
43
|
+
from petals import AutoDistributedModelForCausalLM
|
|
44
|
+
except ImportError as e:
|
|
45
|
+
emit({"event": "error", "message": f"petals not installed: {e}. Install: pip install -U petals"})
|
|
46
|
+
sys.exit(2)
|
|
47
|
+
|
|
48
|
+
emit({"event": "loading", "model": model})
|
|
49
|
+
tokenizer = AutoTokenizer.from_pretrained(model)
|
|
50
|
+
model_obj = AutoDistributedModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
|
|
51
|
+
|
|
52
|
+
# Compose a single prompt from messages using the tokenizer's chat template.
|
|
53
|
+
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
54
|
+
input_ids = tokenizer.encode(prompt, return_tensors="pt")
|
|
55
|
+
|
|
56
|
+
emit({"event": "ready", "input_tokens": int(input_ids.shape[-1])})
|
|
57
|
+
|
|
58
|
+
# Petals streams via inference_session — emit one JSON line per generated token.
|
|
59
|
+
full_text = ""
|
|
60
|
+
t0 = time.time()
|
|
61
|
+
with model_obj.inference_session(max_length=int(input_ids.shape[-1]) + max_tokens) as session:
|
|
62
|
+
# Prefill
|
|
63
|
+
outputs = model_obj.generate(
|
|
64
|
+
input_ids,
|
|
65
|
+
max_new_tokens=1,
|
|
66
|
+
do_sample=temperature > 0,
|
|
67
|
+
temperature=max(temperature, 1e-3),
|
|
68
|
+
session=session,
|
|
69
|
+
)
|
|
70
|
+
new_id = int(outputs[0, -1])
|
|
71
|
+
token_text = tokenizer.decode([new_id], skip_special_tokens=True)
|
|
72
|
+
full_text += token_text
|
|
73
|
+
emit({"event": "token", "text": token_text})
|
|
74
|
+
|
|
75
|
+
for _ in range(max_tokens - 1):
|
|
76
|
+
outputs = model_obj.generate(
|
|
77
|
+
outputs,
|
|
78
|
+
max_new_tokens=1,
|
|
79
|
+
do_sample=temperature > 0,
|
|
80
|
+
temperature=max(temperature, 1e-3),
|
|
81
|
+
session=session,
|
|
82
|
+
)
|
|
83
|
+
new_id = int(outputs[0, -1])
|
|
84
|
+
if new_id == tokenizer.eos_token_id:
|
|
85
|
+
break
|
|
86
|
+
token_text = tokenizer.decode([new_id], skip_special_tokens=True)
|
|
87
|
+
full_text += token_text
|
|
88
|
+
emit({"event": "token", "text": token_text})
|
|
89
|
+
|
|
90
|
+
emit({"event": "done", "text": full_text, "elapsed_s": round(time.time() - t0, 2)})
|
|
91
|
+
|
|
92
|
+
if __name__ == "__main__":
|
|
93
|
+
try:
|
|
94
|
+
main()
|
|
95
|
+
except Exception as e:
|
|
96
|
+
emit({"event": "error", "message": str(e)})
|
|
97
|
+
sys.exit(1)
|
|
98
|
+
`;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.38",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,11 +30,11 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
36
|
-
"@infernetprotocol/
|
|
37
|
-
"@infernetprotocol/
|
|
33
|
+
"@infernetprotocol/auth": "0.1.38",
|
|
34
|
+
"@infernetprotocol/config": "0.1.38",
|
|
35
|
+
"@infernetprotocol/deploy-providers": "0.1.38",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.38",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.38"
|
|
38
38
|
},
|
|
39
39
|
"repository": {
|
|
40
40
|
"type": "git",
|