@infernetprotocol/cli 0.1.41 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/inference.js +196 -179
- package/commands/register.js +10 -39
- package/commands/rpc.js +133 -0
- package/commands/setup.js +25 -15
- package/commands/start.js +281 -48
- package/commands/train.js +69 -17
- package/commands/uncensored.js +18 -0
- package/index.js +3 -2
- package/lib/chat-executor.js +71 -29
- package/lib/data-crawler.js +118 -9
- package/lib/hf-model.js +23 -47
- package/lib/inference/state.js +81 -0
- package/lib/prompt.js +19 -3
- package/lib/recommender.js +5 -1
- package/lib/vllm.js +140 -0
- package/package.json +7 -6
- package/lib/inference/petals-client.py.js +0 -115
- package/lib/inference/petals-engine.js +0 -75
package/commands/inference.js
CHANGED
|
@@ -1,253 +1,270 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `infernet inference` —
|
|
2
|
+
* `infernet inference` — federated inference serving via llama.cpp RPC.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* Starts a Petals server contributing this node's VRAM to the
|
|
6
|
-
* public swarm for the given model. Re-registers with the control
|
|
7
|
-
* plane so the dashboard learns this node is petals-serving X.
|
|
4
|
+
* IPIP-0033 is the active and only path. Two roles:
|
|
8
5
|
*
|
|
9
|
-
* infernet inference
|
|
10
|
-
*
|
|
11
|
-
*
|
|
6
|
+
* slice — `infernet inference serve --backend rpc --model <id>`
|
|
7
|
+
* Spawns llama.cpp's `rpc-server`, advertises specs.rpc.{models,
|
|
8
|
+
* host, port} on heartbeat, lets primaries dial in via
|
|
9
|
+
* /v1/rpc/inference.
|
|
12
10
|
*
|
|
13
|
-
* infernet inference
|
|
14
|
-
*
|
|
11
|
+
* primary — `infernet inference primary --model <id> --gguf <path>`
|
|
12
|
+
* Records the model→GGUF mapping. The daemon's
|
|
13
|
+
* /v1/rpc/inference handler spawns llama-server with the
|
|
14
|
+
* control plane's --rpc slice list per request.
|
|
15
15
|
*
|
|
16
|
-
* Petals (
|
|
17
|
-
*
|
|
18
|
-
* nodes. Each node holds N transformer blocks; activations flow through.
|
|
19
|
-
* Bandwidth-tolerant — designed for residential connections.
|
|
16
|
+
* The Petals path (IPIP-0031, Replaced) was removed. Daemons that
|
|
17
|
+
* relied on it have been migrated.
|
|
20
18
|
*/
|
|
21
19
|
|
|
22
|
-
import
|
|
23
|
-
import
|
|
24
|
-
import
|
|
25
|
-
import { spawn } from
|
|
26
|
-
import {
|
|
27
|
-
|
|
20
|
+
import fsp from 'node:fs/promises';
|
|
21
|
+
import path from 'node:path';
|
|
22
|
+
import os from 'node:os';
|
|
23
|
+
import { spawn } from 'node:child_process';
|
|
24
|
+
import {
|
|
25
|
+
readInferenceState,
|
|
26
|
+
patchInferenceState,
|
|
27
|
+
isPidAlive
|
|
28
|
+
} from '../lib/inference/state.js';
|
|
28
29
|
|
|
29
|
-
const HELP = `infernet inference —
|
|
30
|
+
const HELP = `infernet inference — federated inference serving (IPIP-0033)
|
|
30
31
|
|
|
31
32
|
Usage:
|
|
32
|
-
infernet inference serve --model <
|
|
33
|
+
infernet inference serve --backend rpc --model <id> [flags]
|
|
34
|
+
infernet inference primary --model <id> --gguf <path>
|
|
33
35
|
infernet inference status
|
|
34
36
|
infernet inference stop
|
|
35
|
-
infernet inference list
|
|
36
37
|
|
|
37
|
-
Flags (serve):
|
|
38
|
-
--model <
|
|
39
|
-
--
|
|
40
|
-
--
|
|
41
|
-
--
|
|
42
|
-
--
|
|
43
|
-
--
|
|
38
|
+
Flags (serve --backend rpc):
|
|
39
|
+
--model <id> Required. Canonical model id (e.g. qwen2.5:72b).
|
|
40
|
+
--port <n> rpc-server port (default: 50052).
|
|
41
|
+
--host <addr> Bind interface (default: 0.0.0.0).
|
|
42
|
+
--binary <path> rpc-server binary (default: rpc-server in PATH).
|
|
43
|
+
--vram-gb <n> VRAM offered to RPC clients (advertised in heartbeat).
|
|
44
|
+
--ram-gb <n> RAM offered to RPC clients.
|
|
45
|
+
--max-concurrent <n> Max simultaneous RPC clients (default: 1).
|
|
44
46
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
Flags (primary):
|
|
48
|
+
--model <id> Canonical model id.
|
|
49
|
+
--gguf <path> Absolute path to the GGUF file the primary holds.
|
|
48
50
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
+
Examples:
|
|
52
|
+
# Slice an RPC server for any primary hosting Qwen 2.5 72B
|
|
53
|
+
infernet inference serve --backend rpc --model qwen2.5:72b --port 50052
|
|
51
54
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
node as part of the swarm for that model. Inference requests with
|
|
55
|
-
the "Distribute across all nodes" checkbox set on /chat will route
|
|
56
|
-
through a Petals client that fans out across all swarm participants
|
|
57
|
-
(you + everyone else serving the same model). Per-token receipts
|
|
58
|
-
pay you for the blocks you contributed.
|
|
55
|
+
# Register this node as a primary holding the GGUF locally
|
|
56
|
+
infernet inference primary --model qwen2.5:72b --gguf ~/models/qwen-72b.gguf
|
|
59
57
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
58
|
+
What happens next:
|
|
59
|
+
Re-register so the control plane sees specs.rpc / specs.rpc_primary:
|
|
60
|
+
infernet register
|
|
61
|
+
Or just keep the daemon running — heartbeats pick up state changes
|
|
62
|
+
on the next tick.
|
|
63
63
|
`;
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
async function ensureStateDir() {
|
|
68
|
-
await fsp.mkdir(STATE_DIR, { recursive: true });
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
async function readState() {
|
|
72
|
-
try {
|
|
73
|
-
const raw = await fsp.readFile(path.join(STATE_DIR, "state.json"), "utf8");
|
|
74
|
-
return JSON.parse(raw);
|
|
75
|
-
} catch { return null; }
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async function writeState(s) {
|
|
79
|
-
await ensureStateDir();
|
|
80
|
-
await fsp.writeFile(path.join(STATE_DIR, "state.json"), JSON.stringify(s, null, 2));
|
|
81
|
-
}
|
|
65
|
+
// ---- IPIP-0033: serve --backend rpc -----------------------------------------
|
|
82
66
|
|
|
83
|
-
async function
|
|
84
|
-
|
|
85
|
-
const p = spawn("python3", ["-c", "import petals"], { stdio: "ignore" });
|
|
86
|
-
p.on("exit", (code) => resolve(code === 0));
|
|
87
|
-
p.on("error", () => resolve(false));
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
async function cmdServe(args) {
|
|
92
|
-
const model = args.get("model");
|
|
67
|
+
async function cmdServeRpc(args) {
|
|
68
|
+
const model = args.get('model');
|
|
93
69
|
if (!model) {
|
|
94
|
-
process.stderr.write(
|
|
70
|
+
process.stderr.write('error: --model <id> is required\n');
|
|
95
71
|
return 2;
|
|
96
72
|
}
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
73
|
+
const port = Number.parseInt(args.get('port') ?? '50052', 10);
|
|
74
|
+
const host = args.get('host') ?? '0.0.0.0';
|
|
75
|
+
const binary = args.get('binary') ?? 'rpc-server';
|
|
76
|
+
const vramGb = args.get('vram-gb') ? Number(args.get('vram-gb')) : null;
|
|
77
|
+
const ramGb = args.get('ram-gb') ? Number(args.get('ram-gb')) : null;
|
|
78
|
+
const maxConcurrent = args.get('max-concurrent')
|
|
79
|
+
? Number.parseInt(args.get('max-concurrent'), 10)
|
|
80
|
+
: 1;
|
|
105
81
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
)
|
|
82
|
+
process.stdout.write(`Starting llama.cpp rpc-server for ${model} on ${host}:${port}…\n`);
|
|
83
|
+
|
|
84
|
+
let child;
|
|
85
|
+
try {
|
|
86
|
+
child = spawn(binary, ['-H', host, '-p', String(port)], {
|
|
87
|
+
stdio: ['ignore', 'inherit', 'inherit']
|
|
88
|
+
});
|
|
89
|
+
} catch (err) {
|
|
90
|
+
process.stderr.write(`error: failed to spawn ${binary}: ${err?.message ?? err}\n`);
|
|
111
91
|
return 1;
|
|
112
92
|
}
|
|
113
|
-
|
|
114
|
-
process.stdout.write(`Starting Petals server for ${model} on :${port} (${numBlocks ?? "auto"} blocks)…\n`);
|
|
115
|
-
process.stdout.write("Initial DHT peers: " + prefix + "\n");
|
|
116
|
-
|
|
117
|
-
const child = startPetalsServer({ model, numBlocks, port, prefix });
|
|
118
93
|
if (!child?.pid) {
|
|
119
|
-
process.stderr.write(
|
|
94
|
+
process.stderr.write(`error: ${binary} did not start (binary missing? install llama.cpp)\n`);
|
|
120
95
|
return 1;
|
|
121
96
|
}
|
|
122
97
|
|
|
123
|
-
await
|
|
124
|
-
|
|
125
|
-
model
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
// receipts to layer-contributing operators.
|
|
139
|
-
const tapForPeerId = (stream, sink) => {
|
|
140
|
-
let buf = "";
|
|
141
|
-
stream.on("data", async (chunk) => {
|
|
142
|
-
const txt = chunk.toString();
|
|
143
|
-
sink.write(txt);
|
|
144
|
-
buf += txt;
|
|
145
|
-
const m = buf.match(/peer ID[:\s]+([1-9A-HJ-NP-Za-km-z]{40,80})/i);
|
|
146
|
-
if (m) {
|
|
147
|
-
const state = (await readState()) ?? {};
|
|
148
|
-
if (!state.peer_id) {
|
|
149
|
-
state.peer_id = m[1];
|
|
150
|
-
await writeState(state);
|
|
151
|
-
process.stdout.write(`[infernet] captured petals peer id: ${m[1]}\n`);
|
|
152
|
-
}
|
|
153
|
-
buf = "";
|
|
98
|
+
await patchInferenceState((s) => {
|
|
99
|
+
const prev = s.rpc_slice ?? {};
|
|
100
|
+
const models = new Set([...(prev.models ?? []), model]);
|
|
101
|
+
return {
|
|
102
|
+
...s,
|
|
103
|
+
rpc_slice: {
|
|
104
|
+
models: [...models],
|
|
105
|
+
host,
|
|
106
|
+
port,
|
|
107
|
+
pid: child.pid,
|
|
108
|
+
binary,
|
|
109
|
+
vram_gb: vramGb ?? prev.vram_gb ?? null,
|
|
110
|
+
ram_gb: ramGb ?? prev.ram_gb ?? null,
|
|
111
|
+
max_concurrent: maxConcurrent,
|
|
112
|
+
started_at: new Date().toISOString()
|
|
154
113
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
};
|
|
158
|
-
if (child.stdout) tapForPeerId(child.stdout, process.stdout);
|
|
159
|
-
if (child.stderr) tapForPeerId(child.stderr, process.stderr);
|
|
114
|
+
};
|
|
115
|
+
});
|
|
160
116
|
|
|
161
117
|
process.stdout.write(
|
|
162
|
-
`\n✓
|
|
163
|
-
`
|
|
164
|
-
` infernet register
|
|
118
|
+
`\n✓ rpc-server pid=${child.pid} started.\n` +
|
|
119
|
+
` Heartbeat will advertise this slot as specs.rpc — re-run\n` +
|
|
120
|
+
` \`infernet register\` if you don't want to wait for the next tick.\n\n` +
|
|
165
121
|
` Stop with: infernet inference stop\n`
|
|
166
122
|
);
|
|
167
123
|
|
|
168
|
-
|
|
169
|
-
|
|
124
|
+
process.on('SIGINT', () => { try { child.kill('SIGTERM'); } catch { /* ignore */ } });
|
|
125
|
+
|
|
170
126
|
return new Promise((resolve) => {
|
|
171
|
-
child.on(
|
|
172
|
-
await
|
|
127
|
+
child.on('exit', async (code) => {
|
|
128
|
+
await patchInferenceState((s) => {
|
|
129
|
+
if (s.rpc_slice?.pid === child.pid) {
|
|
130
|
+
return {
|
|
131
|
+
...s,
|
|
132
|
+
rpc_slice: {
|
|
133
|
+
...s.rpc_slice,
|
|
134
|
+
pid: null,
|
|
135
|
+
exited_at: new Date().toISOString(),
|
|
136
|
+
exited_with: code
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
return s;
|
|
141
|
+
});
|
|
173
142
|
resolve(code === 0 ? 0 : 1);
|
|
174
143
|
});
|
|
175
144
|
});
|
|
176
145
|
}
|
|
177
146
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
147
|
+
// ---- IPIP-0033: primary -----------------------------------------------------
|
|
148
|
+
|
|
149
|
+
async function cmdPrimary(args) {
|
|
150
|
+
const model = args.get('model');
|
|
151
|
+
const ggufPath = args.get('gguf');
|
|
152
|
+
if (!model || !ggufPath) {
|
|
153
|
+
process.stderr.write('error: --model <id> and --gguf <path> are both required\n');
|
|
154
|
+
return 2;
|
|
183
155
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
process.stdout.write(`alive: ${alive ? "yes" : "no — pid stale"}\n`);
|
|
156
|
+
const absPath = path.isAbsolute(ggufPath) ? ggufPath : path.resolve(process.cwd(), ggufPath);
|
|
157
|
+
try {
|
|
158
|
+
await fsp.access(absPath, fsp.constants.R_OK);
|
|
159
|
+
} catch {
|
|
160
|
+
process.stderr.write(`error: gguf not readable at ${absPath}\n`);
|
|
161
|
+
return 1;
|
|
191
162
|
}
|
|
192
|
-
|
|
193
|
-
|
|
163
|
+
|
|
164
|
+
await patchInferenceState((s) => {
|
|
165
|
+
const prev = s.rpc_primary ?? {};
|
|
166
|
+
const models = new Set([...(prev.models ?? []), model]);
|
|
167
|
+
const ggufPaths = { ...(prev.gguf_paths ?? {}), [model]: absPath };
|
|
168
|
+
return {
|
|
169
|
+
...s,
|
|
170
|
+
rpc_primary: {
|
|
171
|
+
models: [...models],
|
|
172
|
+
gguf_paths: ggufPaths,
|
|
173
|
+
updated_at: new Date().toISOString()
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
// Convenience symlink at ~/.infernet/models/<id>.gguf so the
|
|
179
|
+
// /v1/rpc/inference handler can resolve either the absolute path
|
|
180
|
+
// from state OR this symlink, whichever exists.
|
|
181
|
+
try {
|
|
182
|
+
const modelsDir = path.join(os.homedir(), '.infernet', 'models');
|
|
183
|
+
await fsp.mkdir(modelsDir, { recursive: true });
|
|
184
|
+
const link = path.join(modelsDir, `${model}.gguf`);
|
|
185
|
+
try { await fsp.unlink(link); } catch { /* ignore */ }
|
|
186
|
+
await fsp.symlink(absPath, link);
|
|
187
|
+
} catch (err) {
|
|
188
|
+
process.stderr.write(`warn: could not create symlink (${err?.message ?? err})\n`);
|
|
194
189
|
}
|
|
190
|
+
|
|
191
|
+
process.stdout.write(
|
|
192
|
+
`✓ Registered as RPC primary for ${model}.\n` +
|
|
193
|
+
` GGUF: ${absPath}\n` +
|
|
194
|
+
` Heartbeat will advertise specs.rpc_primary.\n` +
|
|
195
|
+
` Run \`infernet register\` to push the change immediately.\n`
|
|
196
|
+
);
|
|
195
197
|
return 0;
|
|
196
198
|
}
|
|
197
199
|
|
|
198
|
-
async function
|
|
199
|
-
const
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
200
|
+
async function cmdServe(args) {
|
|
201
|
+
const backend = (args.get('backend') ?? 'rpc').toLowerCase();
|
|
202
|
+
if (backend === 'rpc') return cmdServeRpc(args);
|
|
203
|
+
process.stderr.write(
|
|
204
|
+
`error: --backend ${backend} is not supported. Use --backend rpc (IPIP-0033).\n` +
|
|
205
|
+
` The legacy Petals backend was removed.\n`
|
|
206
|
+
);
|
|
207
|
+
return 2;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// ---- status / stop ----------------------------------------------------------
|
|
211
|
+
|
|
212
|
+
async function cmdStatus() {
|
|
213
|
+
const state = await readInferenceState();
|
|
214
|
+
if (!state || Object.keys(state).length === 0) {
|
|
215
|
+
process.stdout.write('(no inference roles configured)\n');
|
|
210
216
|
return 0;
|
|
211
217
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
process.stdout.write(` ${
|
|
218
|
+
|
|
219
|
+
if (state.rpc_slice) {
|
|
220
|
+
const s = state.rpc_slice;
|
|
221
|
+
process.stdout.write(`rpc-slice:\n`);
|
|
222
|
+
process.stdout.write(` models: ${(s.models ?? []).join(', ')}\n`);
|
|
223
|
+
process.stdout.write(` endpoint: ${s.host}:${s.port}\n`);
|
|
224
|
+
process.stdout.write(` pid: ${s.pid ?? '(exited)'}\n`);
|
|
225
|
+
if (s.pid) process.stdout.write(` alive: ${isPidAlive(s.pid) ? 'yes' : 'no — pid stale'}\n`);
|
|
226
|
+
if (s.exited_at) process.stdout.write(` exited: ${s.exited_at} (code ${s.exited_with})\n`);
|
|
227
|
+
}
|
|
228
|
+
if (state.rpc_primary) {
|
|
229
|
+
const p = state.rpc_primary;
|
|
230
|
+
process.stdout.write(`rpc-primary:\n`);
|
|
231
|
+
process.stdout.write(` models: ${(p.models ?? []).join(', ')}\n`);
|
|
232
|
+
for (const [m, gguf] of Object.entries(p.gguf_paths ?? {})) {
|
|
233
|
+
process.stdout.write(` ${m} → ${gguf}\n`);
|
|
234
|
+
}
|
|
217
235
|
}
|
|
218
|
-
process.stdout.write(`\nQuery as a user: tick \"Distribute across all nodes\" on /chat and pick a model above.\n\n`);
|
|
219
236
|
return 0;
|
|
220
237
|
}
|
|
221
238
|
|
|
222
239
|
async function cmdStop() {
|
|
223
|
-
const state = await
|
|
224
|
-
if (!state?.pid) {
|
|
225
|
-
process.stdout.write(
|
|
240
|
+
const state = await readInferenceState();
|
|
241
|
+
if (!state.rpc_slice?.pid) {
|
|
242
|
+
process.stdout.write('(nothing to stop)\n');
|
|
226
243
|
return 0;
|
|
227
244
|
}
|
|
228
245
|
try {
|
|
229
|
-
process.kill(state.pid,
|
|
230
|
-
process.stdout.write(`SIGTERM
|
|
246
|
+
process.kill(state.rpc_slice.pid, 'SIGTERM');
|
|
247
|
+
process.stdout.write(`SIGTERM → rpc-slice pid ${state.rpc_slice.pid}\n`);
|
|
231
248
|
} catch (err) {
|
|
232
|
-
process.stderr.write(`could not kill pid ${state.pid}: ${err?.message ?? err}\n`);
|
|
249
|
+
process.stderr.write(`could not kill rpc-slice pid ${state.rpc_slice.pid}: ${err?.message ?? err}\n`);
|
|
233
250
|
return 1;
|
|
234
251
|
}
|
|
235
252
|
return 0;
|
|
236
253
|
}
|
|
237
254
|
|
|
238
255
|
export default async function inference(args) {
|
|
239
|
-
if (args.has(
|
|
256
|
+
if (args.has('help') || args.has('h')) {
|
|
240
257
|
process.stdout.write(HELP);
|
|
241
258
|
return 0;
|
|
242
259
|
}
|
|
243
260
|
const sub = args.positional?.[0];
|
|
244
261
|
switch (sub) {
|
|
245
|
-
case
|
|
246
|
-
case
|
|
247
|
-
case
|
|
248
|
-
case
|
|
262
|
+
case 'serve': case 'start': return cmdServe(args);
|
|
263
|
+
case 'primary': return cmdPrimary(args);
|
|
264
|
+
case 'status': return cmdStatus();
|
|
265
|
+
case 'stop': return cmdStop();
|
|
249
266
|
default:
|
|
250
|
-
process.stderr.write(sub ? `unknown subcommand: ${sub}\n\n` :
|
|
267
|
+
process.stderr.write(sub ? `unknown subcommand: ${sub}\n\n` : 'error: missing subcommand\n\n');
|
|
251
268
|
process.stderr.write(HELP);
|
|
252
269
|
return 2;
|
|
253
270
|
}
|
package/commands/register.js
CHANGED
|
@@ -193,7 +193,15 @@ export async function gatherCoarseSpecs() {
|
|
|
193
193
|
|
|
194
194
|
// Advertise all pulled models — operator explicitly installed them so we
|
|
195
195
|
// trust they know their hardware. The fit check is informational only.
|
|
196
|
-
|
|
196
|
+
// Also fold in any model vLLM is currently serving (:8000/v1/models),
|
|
197
|
+
// advertised under its `hf:` pull name via --served-model-name, so vLLM
|
|
198
|
+
// models show up in /chat alongside Ollama ones.
|
|
199
|
+
let vllmModels = [];
|
|
200
|
+
try {
|
|
201
|
+
const { detectVllmModels } = await import('../lib/vllm.js');
|
|
202
|
+
vllmModels = await detectVllmModels();
|
|
203
|
+
} catch { /* vLLM not installed / not serving — Ollama-only */ }
|
|
204
|
+
const served_models = [...new Set([...pulledModels.map((m) => m.name), ...vllmModels])];
|
|
197
205
|
|
|
198
206
|
return {
|
|
199
207
|
cpu: summarizeCpu(),
|
|
@@ -204,47 +212,10 @@ export async function gatherCoarseSpecs() {
|
|
|
204
212
|
model: typeof g.model === 'string' ? g.model.slice(0, 64) : null
|
|
205
213
|
})),
|
|
206
214
|
interconnects: summarizeInterconnects(interconnects),
|
|
207
|
-
served_models
|
|
208
|
-
// IPIP-0031: models this node is contributing to a Petals swarm.
|
|
209
|
-
// Empty array if the operator hasn't run `infernet inference serve`.
|
|
210
|
-
// Read from local state file written by inference.js.
|
|
211
|
-
petals_models: await readPetalsModels(),
|
|
212
|
-
// IPIP-0031 §per-layer-attribution: libp2p peer ID Petals
|
|
213
|
-
// assigned this server. The control plane uses this to map
|
|
214
|
-
// chosen_servers from a client's routing event back to the
|
|
215
|
-
// corresponding provider rows for CPR receipt distribution.
|
|
216
|
-
petals_peer_id: await readPetalsPeerId()
|
|
215
|
+
served_models
|
|
217
216
|
};
|
|
218
217
|
}
|
|
219
218
|
|
|
220
|
-
async function readPetalsModels() {
|
|
221
|
-
const state = await readPetalsState();
|
|
222
|
-
if (!state) return [];
|
|
223
|
-
if (state.backend === 'petals' && typeof state.model === 'string') {
|
|
224
|
-
return [state.model];
|
|
225
|
-
}
|
|
226
|
-
return [];
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
async function readPetalsPeerId() {
|
|
230
|
-
const state = await readPetalsState();
|
|
231
|
-
return (state && typeof state.peer_id === 'string') ? state.peer_id : null;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
async function readPetalsState() {
|
|
235
|
-
try {
|
|
236
|
-
const fsp = await import('node:fs/promises');
|
|
237
|
-
const path = await import('node:path');
|
|
238
|
-
const statePath = path.join(process.env.HOME ?? '/tmp', '.infernet', 'inference', 'state.json');
|
|
239
|
-
const raw = await fsp.readFile(statePath, 'utf8');
|
|
240
|
-
const state = JSON.parse(raw);
|
|
241
|
-
if (!state.pid) return null;
|
|
242
|
-
try { process.kill(state.pid, 0); }
|
|
243
|
-
catch { return null; }
|
|
244
|
-
return state;
|
|
245
|
-
} catch { return null; }
|
|
246
|
-
}
|
|
247
|
-
|
|
248
219
|
export default async function register(args, ctx) {
|
|
249
220
|
if (args.has('help') || args.has('h')) {
|
|
250
221
|
process.stdout.write(HELP);
|
package/commands/rpc.js
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `infernet rpc` — operator-side diagnostics for IPIP-0033 federated
|
|
3
|
+
* inference. Today only one subcommand:
|
|
4
|
+
*
|
|
5
|
+
* infernet rpc census --model <id>
|
|
6
|
+
* Asks THIS daemon's HTTP /v1/rpc/census endpoint for the set
|
|
7
|
+
* of peers it sees on rpc:<model>. The answer is a second
|
|
8
|
+
* opinion to the control-plane census — useful when the
|
|
9
|
+
* providers table looks stale.
|
|
10
|
+
*
|
|
11
|
+
* The peer discovery substrate moved out of infernet — it's now
|
|
12
|
+
* c0mpute's libp2p Kad-DHT + gossipsub. Until /v1/rpc/census is
|
|
13
|
+
* wired through the c0mpute capability registry, this returns an
|
|
14
|
+
* empty set with a `note`.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const HELP = `infernet rpc — federated-inference diagnostics
|
|
18
|
+
|
|
19
|
+
Usage:
|
|
20
|
+
infernet rpc census --model <id> [--kind rpc|model|class]
|
|
21
|
+
[--daemon http://host:port]
|
|
22
|
+
|
|
23
|
+
Flags:
|
|
24
|
+
--model <id> Required. Canonical model id (e.g. qwen2.5:72b).
|
|
25
|
+
--kind <ns> Topic namespace (default: rpc).
|
|
26
|
+
--daemon <url> Daemon HTTP base. Defaults to http://127.0.0.1:8080,
|
|
27
|
+
matching the local daemon's healthz port. Set this
|
|
28
|
+
to query a peer's daemon directly.
|
|
29
|
+
|
|
30
|
+
Examples:
|
|
31
|
+
# What does my daemon see on rpc:qwen2.5:72b right now?
|
|
32
|
+
infernet rpc census --model qwen2.5:72b
|
|
33
|
+
|
|
34
|
+
# Same query against a remote daemon's HTTP surface.
|
|
35
|
+
infernet rpc census --model qwen2.5:72b --daemon http://10.0.0.7:8080
|
|
36
|
+
`;
|
|
37
|
+
|
|
38
|
+
const DEFAULT_DAEMON = 'http://127.0.0.1:8080';
|
|
39
|
+
|
|
40
|
+
async function cmdCensus(args) {
|
|
41
|
+
const model = args.get('model');
|
|
42
|
+
if (!model) {
|
|
43
|
+
process.stderr.write('error: --model <id> is required\n');
|
|
44
|
+
return 2;
|
|
45
|
+
}
|
|
46
|
+
const kind = args.get('kind') ?? 'rpc';
|
|
47
|
+
const daemon = (args.get('daemon') ?? DEFAULT_DAEMON).replace(/\/+$/, '');
|
|
48
|
+
const qs = new URLSearchParams({ model, kind }).toString();
|
|
49
|
+
const url = `${daemon}/v1/rpc/census?${qs}`;
|
|
50
|
+
|
|
51
|
+
let res;
|
|
52
|
+
try {
|
|
53
|
+
res = await fetch(url);
|
|
54
|
+
} catch (err) {
|
|
55
|
+
process.stderr.write(
|
|
56
|
+
`error: could not reach daemon at ${daemon}: ${err?.message ?? err}\n` +
|
|
57
|
+
` is the daemon running? (\`infernet status\` to check)\n`
|
|
58
|
+
);
|
|
59
|
+
return 1;
|
|
60
|
+
}
|
|
61
|
+
if (!res.ok) {
|
|
62
|
+
process.stderr.write(`error: HTTP ${res.status} from ${url}\n`);
|
|
63
|
+
try {
|
|
64
|
+
const text = await res.text();
|
|
65
|
+
if (text) process.stderr.write(` body: ${text}\n`);
|
|
66
|
+
} catch { /* ignore */ }
|
|
67
|
+
return 1;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const body = await res.json();
|
|
71
|
+
if (body?.dht === false) {
|
|
72
|
+
process.stdout.write(
|
|
73
|
+
`(this daemon has DHT discovery disabled — start it without --no-dht ` +
|
|
74
|
+
`or unset INFERNET_DISABLE_DHT to enable.)\n`
|
|
75
|
+
);
|
|
76
|
+
return 0;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const peers = Array.isArray(body?.peers) ? body.peers : [];
|
|
80
|
+
process.stdout.write(`\n${kind}:${model} — ${body.count ?? peers.length} peer(s) live in this daemon's DHT view\n`);
|
|
81
|
+
if (peers.length === 0) {
|
|
82
|
+
process.stdout.write(
|
|
83
|
+
` No one else on the topic. Check \`infernet status\` to confirm\n` +
|
|
84
|
+
` the daemon's DHT is up, and ask another operator to run\n` +
|
|
85
|
+
` \`infernet inference serve --backend rpc --model ${model}\`.\n\n`
|
|
86
|
+
);
|
|
87
|
+
return 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
process.stdout.write(`\n ${'PUBKEY'.padEnd(18)} ${'ADDRESS'.padEnd(24)} LAST SEEN\n`);
|
|
91
|
+
process.stdout.write(` ${'──────'.padEnd(18)} ${'───────'.padEnd(24)} ─────────\n`);
|
|
92
|
+
for (const p of peers) {
|
|
93
|
+
const pk = (p.pubkey ?? '').slice(0, 12) + '…';
|
|
94
|
+
const addr = p.address ?? '(private)';
|
|
95
|
+
const last = p.last_seen ? relativeAgo(p.last_seen) : '—';
|
|
96
|
+
process.stdout.write(` ${pk.padEnd(18)} ${String(addr).padEnd(24)} ${last}\n`);
|
|
97
|
+
}
|
|
98
|
+
if (body?.self?.pubkey) {
|
|
99
|
+
process.stdout.write(`\n (this daemon: ${body.self.pubkey.slice(0, 12)}… — topics: ${body.self.topics?.join(', ') ?? 'none'})\n\n`);
|
|
100
|
+
} else {
|
|
101
|
+
process.stdout.write('\n');
|
|
102
|
+
}
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function relativeAgo(iso) {
|
|
107
|
+
const ms = Date.now() - new Date(iso).getTime();
|
|
108
|
+
if (!Number.isFinite(ms) || ms < 0) return iso;
|
|
109
|
+
const s = Math.floor(ms / 1000);
|
|
110
|
+
if (s < 5) return 'just now';
|
|
111
|
+
if (s < 60) return `${s}s ago`;
|
|
112
|
+
const m = Math.floor(s / 60);
|
|
113
|
+
if (m < 60) return `${m}m ago`;
|
|
114
|
+
const h = Math.floor(m / 60);
|
|
115
|
+
return `${h}h ago`;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export default async function rpc(args) {
|
|
119
|
+
if (args.has('help') || args.has('h')) {
|
|
120
|
+
process.stdout.write(HELP);
|
|
121
|
+
return 0;
|
|
122
|
+
}
|
|
123
|
+
const sub = args.positional?.[0];
|
|
124
|
+
switch (sub) {
|
|
125
|
+
case 'census': return cmdCensus(args);
|
|
126
|
+
default:
|
|
127
|
+
process.stderr.write(sub ? `unknown subcommand: ${sub}\n\n` : 'error: missing subcommand\n\n');
|
|
128
|
+
process.stderr.write(HELP);
|
|
129
|
+
return 2;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export { HELP };
|