@infernetprotocol/cli 0.1.36 → 0.1.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,207 @@
1
+ /**
2
+ * `infernet inference` — distributed inference serving (IPIP-0031).
3
+ *
4
+ * infernet inference serve --backend petals --model <hf-id>
5
+ * Starts a Petals server contributing this node's VRAM to the
6
+ * public swarm for the given model. Re-registers with the control
7
+ * plane so the dashboard learns this node is petals-serving X.
8
+ *
9
+ * infernet inference status
10
+ * Shows whether a Petals server is running locally + which blocks
11
+ * it's holding for which model.
12
+ *
13
+ * infernet inference stop
14
+ * Stops the locally-running Petals server.
15
+ *
16
+ * Petals (https://github.com/bigscience-workshop/petals) splits a
17
+ * Llama-style transformer by layer across DHT-discovered volunteer
18
+ * nodes. Each node holds N transformer blocks; activations flow through.
19
+ * Bandwidth-tolerant — designed for residential connections.
20
+ */
21
+
22
+ import fs from "node:fs";
23
+ import fsp from "node:fs/promises";
24
+ import path from "node:path";
25
+ import { spawn } from "node:child_process";
26
+ import { loadConfig } from "../lib/config.js";
27
+ import { startPetalsServer } from "../lib/inference/petals-engine.js";
28
+
29
+ const HELP = `infernet inference — distributed inference serving (Petals swarm)
30
+
31
+ Usage:
32
+ infernet inference serve --model <hf-id> [flags]
33
+ infernet inference status
34
+ infernet inference stop
35
+
36
+ Flags (serve):
37
+ --model <hf-id> Required. e.g. meta-llama/Llama-3.1-70B-Instruct
38
+ --backend <name> petals (default; only supported backend today)
39
+ --num-blocks <n> How many transformer blocks to host (auto if omitted)
40
+ --port <n> Petals server port (default: 31330)
41
+ --dht-prefix <str> DHT initial peers (default: /petals/v3-public)
42
+ --help
43
+
44
+ Examples:
45
+ # Contribute to a Llama 3.1 70B swarm — node holds whatever blocks fit
46
+ infernet inference serve --model meta-llama/Llama-3.1-70B-Instruct
47
+
48
+ # Same but only host 6 blocks (smaller VRAM footprint)
49
+ infernet inference serve --model meta-llama/Llama-3.1-70B-Instruct --num-blocks 6
50
+
51
+ What this means for the network:
52
+ When you serve a model via Petals, the control plane registers your
53
+ node as part of the swarm for that model. Inference requests with
54
+ the "Distribute across all nodes" checkbox set on /chat will route
55
+ through a Petals client that fans out across all swarm participants
56
+ (you + everyone else serving the same model). Per-token receipts
57
+ pay you for the blocks you contributed.
58
+
59
+ Prerequisites:
60
+ pip install -U petals # Python 3.10+
61
+ Inbound port \${port} reachable (cloudflared works for residential setups)
62
+ `;
63
+
64
+ const STATE_DIR = path.join(process.env.HOME ?? "/tmp", ".infernet", "inference");
65
+
66
+ async function ensureStateDir() {
67
+ await fsp.mkdir(STATE_DIR, { recursive: true });
68
+ }
69
+
70
+ async function readState() {
71
+ try {
72
+ const raw = await fsp.readFile(path.join(STATE_DIR, "state.json"), "utf8");
73
+ return JSON.parse(raw);
74
+ } catch { return null; }
75
+ }
76
+
77
+ async function writeState(s) {
78
+ await ensureStateDir();
79
+ await fsp.writeFile(path.join(STATE_DIR, "state.json"), JSON.stringify(s, null, 2));
80
+ }
81
+
82
+ async function pythonHasPetals() {
83
+ return new Promise((resolve) => {
84
+ const p = spawn("python3", ["-c", "import petals"], { stdio: "ignore" });
85
+ p.on("exit", (code) => resolve(code === 0));
86
+ p.on("error", () => resolve(false));
87
+ });
88
+ }
89
+
90
+ async function cmdServe(args) {
91
+ const model = args.get("model");
92
+ if (!model) {
93
+ process.stderr.write("error: --model <hf-id> is required\n");
94
+ return 2;
95
+ }
96
+ const backend = args.get("backend") ?? "petals";
97
+ if (backend !== "petals") {
98
+ process.stderr.write(`error: only --backend petals is supported today (got ${backend})\n`);
99
+ return 2;
100
+ }
101
+ const numBlocks = args.get("num-blocks") ? Number.parseInt(args.get("num-blocks"), 10) : null;
102
+ const port = Number.parseInt(args.get("port") ?? "31330", 10);
103
+ const prefix = args.get("dht-prefix") ?? "/petals/v3-public";
104
+
105
+ if (!(await pythonHasPetals())) {
106
+ process.stderr.write(
107
+ "error: petals isn't installed. Run:\n pip install -U petals\n" +
108
+ " python3 -c 'import petals' # verify\n"
109
+ );
110
+ return 1;
111
+ }
112
+
113
+ process.stdout.write(`Starting Petals server for ${model} on :${port} (${numBlocks ?? "auto"} blocks)…\n`);
114
+ process.stdout.write("Initial DHT peers: " + prefix + "\n");
115
+
116
+ const child = startPetalsServer({ model, numBlocks, port, prefix });
117
+ if (!child?.pid) {
118
+ process.stderr.write("error: failed to spawn petals server\n");
119
+ return 1;
120
+ }
121
+
122
+ await writeState({
123
+ backend,
124
+ model,
125
+ port,
126
+ prefix,
127
+ num_blocks: numBlocks,
128
+ pid: child.pid,
129
+ started_at: new Date().toISOString()
130
+ });
131
+
132
+ // Pipe child output so the operator sees Petals' own startup logs
133
+ // (which blocks it grabbed, DHT peers, etc).
134
+ child.stdout?.pipe(process.stdout);
135
+ child.stderr?.pipe(process.stderr);
136
+
137
+ process.stdout.write(
138
+ `\n✓ Petals server pid=${child.pid} started.\n` +
139
+ ` Re-register your node so the control plane learns about this:\n` +
140
+ ` infernet register\n\n` +
141
+ ` Stop with: infernet inference stop\n`
142
+ );
143
+
144
+ // Stay attached so the user sees logs; Ctrl-C will SIGTERM the child.
145
+ process.on("SIGINT", () => { try { child.kill("SIGTERM"); } catch { /* ignore */ } });
146
+ return new Promise((resolve) => {
147
+ child.on("exit", async (code) => {
148
+ await writeState({ backend, model, port, prefix, num_blocks: numBlocks, pid: null, exited_with: code, exited_at: new Date().toISOString() });
149
+ resolve(code === 0 ? 0 : 1);
150
+ });
151
+ });
152
+ }
153
+
154
+ async function cmdStatus() {
155
+ const state = await readState();
156
+ if (!state) {
157
+ process.stdout.write("(no inference server running locally)\n");
158
+ return 0;
159
+ }
160
+ process.stdout.write(`backend: ${state.backend}\n`);
161
+ process.stdout.write(`model: ${state.model}\n`);
162
+ process.stdout.write(`port: ${state.port}\n`);
163
+ process.stdout.write(`pid: ${state.pid ?? "(exited)"}\n`);
164
+ if (state.pid) {
165
+ const alive = (() => { try { process.kill(state.pid, 0); return true; } catch { return false; } })();
166
+ process.stdout.write(`alive: ${alive ? "yes" : "no — pid stale"}\n`);
167
+ }
168
+ if (state.exited_at) {
169
+ process.stdout.write(`exited: ${state.exited_at} (code ${state.exited_with})\n`);
170
+ }
171
+ return 0;
172
+ }
173
+
174
+ async function cmdStop() {
175
+ const state = await readState();
176
+ if (!state?.pid) {
177
+ process.stdout.write("(nothing to stop)\n");
178
+ return 0;
179
+ }
180
+ try {
181
+ process.kill(state.pid, "SIGTERM");
182
+ process.stdout.write(`SIGTERM sent to pid ${state.pid}\n`);
183
+ } catch (err) {
184
+ process.stderr.write(`could not kill pid ${state.pid}: ${err?.message ?? err}\n`);
185
+ return 1;
186
+ }
187
+ return 0;
188
+ }
189
+
190
+ export default async function inference(args) {
191
+ if (args.has("help") || args.has("h")) {
192
+ process.stdout.write(HELP);
193
+ return 0;
194
+ }
195
+ const sub = args.positional?.[0];
196
+ switch (sub) {
197
+ case "serve": case "start": return cmdServe(args);
198
+ case "status": return cmdStatus();
199
+ case "stop": return cmdStop();
200
+ default:
201
+ process.stderr.write(sub ? `unknown subcommand: ${sub}\n\n` : "error: missing subcommand\n\n");
202
+ process.stderr.write(HELP);
203
+ return 2;
204
+ }
205
+ }
206
+
207
+ export { HELP };
@@ -204,10 +204,33 @@ export async function gatherCoarseSpecs() {
204
204
  model: typeof g.model === 'string' ? g.model.slice(0, 64) : null
205
205
  })),
206
206
  interconnects: summarizeInterconnects(interconnects),
207
- served_models
207
+ served_models,
208
+ // IPIP-0031: models this node is contributing to a Petals swarm.
209
+ // Empty array if the operator hasn't run `infernet inference serve`.
210
+ // Read from local state file written by inference.js.
211
+ petals_models: await readPetalsModels()
208
212
  };
209
213
  }
210
214
 
215
+ async function readPetalsModels() {
216
+ try {
217
+ const fsp = await import('node:fs/promises');
218
+ const path = await import('node:path');
219
+ const statePath = path.join(process.env.HOME ?? '/tmp', '.infernet', 'inference', 'state.json');
220
+ const raw = await fsp.readFile(statePath, 'utf8');
221
+ const state = JSON.parse(raw);
222
+ // Only advertise if the server is still alive.
223
+ if (state.pid) {
224
+ try { process.kill(state.pid, 0); } // 0 = check liveness
225
+ catch { return []; } // pid stale
226
+ } else { return []; }
227
+ if (state.backend === 'petals' && typeof state.model === 'string') {
228
+ return [state.model];
229
+ }
230
+ return [];
231
+ } catch { return []; }
232
+ }
233
+
211
234
  export default async function register(args, ctx) {
212
235
  if (args.has('help') || args.has('h')) {
213
236
  process.stdout.write(HELP);
package/commands/start.js CHANGED
@@ -776,6 +776,78 @@ async function runDaemon(args, ctx) {
776
776
  };
777
777
  }
778
778
 
779
+ /**
780
+ * IPIP-0031: distributed inference proxy. Accepts a chat request,
781
+ * spawns a Python Petals client subprocess, streams its JSON-line
782
+ * output back as SSE.
783
+ *
784
+ * POST /v1/petals/inference
785
+ * body: { model, messages, max_tokens?, temperature? }
786
+ * response: text/event-stream with token / done / error frames
787
+ */
788
+ async function handlePetalsInference(req, res) {
789
+ if (req.method !== 'POST' || req.url !== '/v1/petals/inference') return false;
790
+
791
+ const fsp = await import('node:fs/promises');
792
+ const path = await import('node:path');
793
+ const os = await import('node:os');
794
+ const { PETALS_CLIENT_PY } = await import('../lib/inference/petals-client.py.js');
795
+
796
+ // Read JSON body
797
+ let body = '';
798
+ await new Promise((resolve, reject) => {
799
+ req.on('data', (chunk) => { body += chunk.toString(); });
800
+ req.on('end', resolve);
801
+ req.on('error', reject);
802
+ });
803
+ let payload;
804
+ try { payload = JSON.parse(body); } catch {
805
+ res.writeHead(400, { 'content-type': 'text/plain' });
806
+ res.end('bad json\n');
807
+ return true;
808
+ }
809
+
810
+ // Drop the bundled Python client to a temp file and spawn it.
811
+ const tmp = path.join(os.tmpdir(), `petals-${process.pid}-${Date.now()}.py`);
812
+ await fsp.writeFile(tmp, PETALS_CLIENT_PY);
813
+
814
+ res.writeHead(200, {
815
+ 'content-type': 'text/event-stream',
816
+ 'cache-control': 'no-cache',
817
+ 'x-accel-buffering': 'no'
818
+ });
819
+
820
+ const child = spawn('python3', [tmp], { stdio: ['pipe', 'pipe', 'pipe'] });
821
+ child.stdin.write(JSON.stringify(payload));
822
+ child.stdin.end();
823
+
824
+ let stdoutBuf = '';
825
+ child.stdout.on('data', (chunk) => {
826
+ stdoutBuf += chunk.toString();
827
+ let nl;
828
+ while ((nl = stdoutBuf.indexOf('\n')) >= 0) {
829
+ const line = stdoutBuf.slice(0, nl).trim();
830
+ stdoutBuf = stdoutBuf.slice(nl + 1);
831
+ if (!line) continue;
832
+ try {
833
+ const ev = JSON.parse(line);
834
+ res.write(`event: ${ev.event ?? 'data'}\ndata: ${JSON.stringify(ev)}\n\n`);
835
+ } catch {
836
+ res.write(`event: log\ndata: ${JSON.stringify({ raw: line })}\n\n`);
837
+ }
838
+ }
839
+ });
840
+ child.stderr.on('data', (chunk) => {
841
+ res.write(`event: log\ndata: ${JSON.stringify({ stderr: chunk.toString() })}\n\n`);
842
+ });
843
+ child.on('exit', async (code) => {
844
+ res.write(`event: exit\ndata: ${JSON.stringify({ code })}\n\n`);
845
+ res.end();
846
+ try { await fsp.unlink(tmp); } catch { /* ignore */ }
847
+ });
848
+ return true;
849
+ }
850
+
779
851
  /**
780
852
  * IPIP-0030: shard-hosting routes layered on top of /healthz.
781
853
  * Returns true if the request matched and was handled.
@@ -1063,6 +1135,15 @@ async function runDaemon(args, ctx) {
1063
1135
  return;
1064
1136
  }
1065
1137
 
1138
+ try {
1139
+ const petalsHandled = await handlePetalsInference(req, res);
1140
+ if (petalsHandled) return;
1141
+ } catch (err) {
1142
+ res.writeHead(500, { 'content-type': 'text/plain' });
1143
+ res.end(`petals http error: ${err?.message ?? err}\n`);
1144
+ return;
1145
+ }
1146
+
1066
1147
  res.writeHead(404, { 'content-type': 'text/plain' });
1067
1148
  res.end('not found\n');
1068
1149
  });
package/index.js CHANGED
@@ -45,6 +45,7 @@ import deploy from './commands/deploy.js';
45
45
  import consoleCmd from './commands/console.js';
46
46
  import train from './commands/train.js';
47
47
  import publish from './commands/publish.js';
48
+ import inference from './commands/inference.js';
48
49
  import uncensored from './commands/uncensored.js';
49
50
 
50
51
  function parseArgs(argv) {
@@ -107,11 +108,11 @@ const COMMANDS = {
107
108
  // aliases
108
109
  uninstall: remove,
109
110
  start, status, stop, restart, stats, logs,
110
- payout, payments, gpu, firewall, chat, setup, model, train, publish, uncensored, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help
111
+ payout, payments, gpu, firewall, chat, setup, model, train, publish, inference, uncensored, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help
111
112
  };
112
113
 
113
114
  // Commands that can run without a loaded config.
114
- const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'publish', 'uncensored', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall']);
115
+ const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'publish', 'inference', 'uncensored', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall']);
115
116
  // Commands that need a config but not a control-plane client (none today
116
117
  // — kept as a future escape hatch).
117
118
  const NO_CLIENT = new Set();
@@ -0,0 +1,98 @@
1
+ /**
2
+ * Bundled Python petals client. Spawned by the daemon when it accepts
3
+ * an /v1/petals/inference POST. Fans out across the public Petals
4
+ * swarm using AutoDistributedModelForCausalLM and streams JSON lines
5
+ * (one per token) to stdout. The daemon converts those to SSE frames.
6
+ */
7
+
8
+ export const PETALS_CLIENT_PY = `#!/usr/bin/env python3
9
+ """Petals client — fans out across the swarm + streams tokens.
10
+
11
+ Reads: JSON request from stdin: { model, messages, max_tokens, temperature }
12
+ Writes: JSON lines to stdout: { event: 'token' | 'done' | 'error', ... }
13
+
14
+ Each output line is a single JSON object terminated with \\n so the
15
+ parent process can parse them line-by-line.
16
+ """
17
+ import json
18
+ import sys
19
+ import time
20
+
21
+ def emit(obj):
22
+ sys.stdout.write(json.dumps(obj) + "\\n")
23
+ sys.stdout.flush()
24
+
25
+ def main():
26
+ try:
27
+ req = json.loads(sys.stdin.read())
28
+ except Exception as e:
29
+ emit({"event": "error", "message": f"bad request: {e}"})
30
+ sys.exit(2)
31
+
32
+ model = req.get("model")
33
+ messages = req.get("messages", [])
34
+ max_tokens = int(req.get("max_tokens") or 256)
35
+ temperature = float(req.get("temperature") or 0.7)
36
+ if not model:
37
+ emit({"event": "error", "message": "model required"})
38
+ sys.exit(2)
39
+
40
+ try:
41
+ import torch
42
+ from transformers import AutoTokenizer
43
+ from petals import AutoDistributedModelForCausalLM
44
+ except ImportError as e:
45
+ emit({"event": "error", "message": f"petals not installed: {e}. Install: pip install -U petals"})
46
+ sys.exit(2)
47
+
48
+ emit({"event": "loading", "model": model})
49
+ tokenizer = AutoTokenizer.from_pretrained(model)
50
+ model_obj = AutoDistributedModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
51
+
52
+ # Compose a single prompt from messages using the tokenizer's chat template.
53
+ prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
54
+ input_ids = tokenizer.encode(prompt, return_tensors="pt")
55
+
56
+ emit({"event": "ready", "input_tokens": int(input_ids.shape[-1])})
57
+
58
+ # Petals streams via inference_session — emit one JSON line per generated token.
59
+ full_text = ""
60
+ t0 = time.time()
61
+ with model_obj.inference_session(max_length=int(input_ids.shape[-1]) + max_tokens) as session:
62
+ # Prefill
63
+ outputs = model_obj.generate(
64
+ input_ids,
65
+ max_new_tokens=1,
66
+ do_sample=temperature > 0,
67
+ temperature=max(temperature, 1e-3),
68
+ session=session,
69
+ )
70
+ new_id = int(outputs[0, -1])
71
+ token_text = tokenizer.decode([new_id], skip_special_tokens=True)
72
+ full_text += token_text
73
+ emit({"event": "token", "text": token_text})
74
+
75
+ for _ in range(max_tokens - 1):
76
+ outputs = model_obj.generate(
77
+ outputs,
78
+ max_new_tokens=1,
79
+ do_sample=temperature > 0,
80
+ temperature=max(temperature, 1e-3),
81
+ session=session,
82
+ )
83
+ new_id = int(outputs[0, -1])
84
+ if new_id == tokenizer.eos_token_id:
85
+ break
86
+ token_text = tokenizer.decode([new_id], skip_special_tokens=True)
87
+ full_text += token_text
88
+ emit({"event": "token", "text": token_text})
89
+
90
+ emit({"event": "done", "text": full_text, "elapsed_s": round(time.time() - t0, 2)})
91
+
92
+ if __name__ == "__main__":
93
+ try:
94
+ main()
95
+ except Exception as e:
96
+ emit({"event": "error", "message": str(e)})
97
+ sys.exit(1)
98
+ `;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.36",
3
+ "version": "0.1.38",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,11 +30,11 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/auth": "0.1.36",
34
- "@infernetprotocol/config": "0.1.36",
35
- "@infernetprotocol/deploy-providers": "0.1.36",
36
- "@infernetprotocol/engine": "0.1.36",
37
- "@infernetprotocol/gpu": "0.1.36"
33
+ "@infernetprotocol/auth": "0.1.38",
34
+ "@infernetprotocol/config": "0.1.38",
35
+ "@infernetprotocol/deploy-providers": "0.1.38",
36
+ "@infernetprotocol/engine": "0.1.38",
37
+ "@infernetprotocol/gpu": "0.1.38"
38
38
  },
39
39
  "repository": {
40
40
  "type": "git",