@infernetprotocol/cli 0.1.41 → 0.1.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/inference.js +196 -179
- package/commands/register.js +1 -38
- package/commands/rpc.js +134 -0
- package/commands/start.js +299 -44
- package/commands/train.js +69 -17
- package/index.js +3 -2
- package/lib/data-crawler.js +118 -9
- package/lib/discovery-bridge.js +168 -0
- package/lib/inference/state.js +81 -0
- package/package.json +8 -6
- package/lib/inference/petals-client.py.js +0 -115
- package/lib/inference/petals-engine.js +0 -75
package/lib/data-crawler.js
CHANGED
|
@@ -2,21 +2,28 @@
|
|
|
2
2
|
* Search-driven training-data collector.
|
|
3
3
|
*
|
|
4
4
|
* Pipeline:
|
|
5
|
-
* 1. Hit
|
|
5
|
+
* 1. Hit infernetprotocol.com/api/v1/search with a query (Nostr-signed).
|
|
6
|
+
* The control plane proxies to the real provider, holds the upstream
|
|
7
|
+
* key, and enforces a per-pubkey daily quota.
|
|
6
8
|
* 2. For each URL: fetch HTML, strip nav/script/style, extract main text.
|
|
7
9
|
* 3. For each page: chunk into Q/A pairs (heading-based) and emit JSONL.
|
|
8
10
|
*
|
|
9
11
|
* The output JSONL uses the chatml `messages` shape, ready to feed into
|
|
10
12
|
* `infernet train run` or any HF/Unsloth/Axolotl pipeline.
|
|
11
13
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
+
* Self-host escape hatch: set INFERNET_DIRECT_VALUESERP=1 with a
|
|
15
|
+
* VALUESERP_API_KEY in env (or `integrations.valueserp.api_key` in config)
|
|
16
|
+
* to bypass the control plane and call valueserp directly.
|
|
14
17
|
*/
|
|
15
18
|
|
|
16
19
|
import fs from "node:fs/promises";
|
|
17
20
|
import path from "node:path";
|
|
21
|
+
import { signRequest, AUTH_HEADER } from "@infernetprotocol/auth";
|
|
18
22
|
import { loadConfig } from "./config.js";
|
|
19
23
|
|
|
24
|
+
const SEARCH_PATH = "/api/v1/search";
|
|
25
|
+
|
|
26
|
+
/** Resolve a direct VALUESERP_API_KEY for the escape hatch. */
|
|
20
27
|
export async function resolveValueSerpKey() {
|
|
21
28
|
if (process.env.VALUESERP_API_KEY) return process.env.VALUESERP_API_KEY;
|
|
22
29
|
const cfg = await loadConfig().catch(() => null);
|
|
@@ -26,12 +33,88 @@ export async function resolveValueSerpKey() {
|
|
|
26
33
|
}
|
|
27
34
|
|
|
28
35
|
/**
|
|
29
|
-
* Search the
|
|
36
|
+
* Search via the control plane (default path). Auth is the node's existing
|
|
37
|
+
* Nostr keypair — operators don't need a separate API key.
|
|
38
|
+
*
|
|
30
39
|
* @param {object} opts
|
|
31
40
|
* @param {string} opts.query
|
|
32
|
-
* @param {string} opts.
|
|
41
|
+
* @param {string} opts.controlPlaneUrl
|
|
42
|
+
* @param {string} opts.publicKey - 64-hex Nostr pubkey
|
|
43
|
+
* @param {string} opts.privateKey - 64-hex Nostr privkey
|
|
33
44
|
* @param {number} [opts.num=20]
|
|
34
45
|
* @param {string} [opts.location]
|
|
46
|
+
* @param {string[]} [opts.domains]
|
|
47
|
+
* @param {number} [opts.timeoutMs=25000]
|
|
48
|
+
*/
|
|
49
|
+
export async function infernetSearch({
|
|
50
|
+
query,
|
|
51
|
+
controlPlaneUrl,
|
|
52
|
+
publicKey,
|
|
53
|
+
privateKey,
|
|
54
|
+
num = 20,
|
|
55
|
+
location,
|
|
56
|
+
domains,
|
|
57
|
+
timeoutMs = 25_000
|
|
58
|
+
} = {}) {
|
|
59
|
+
if (!query) throw new Error("search: query required");
|
|
60
|
+
if (!controlPlaneUrl) throw new Error("search: controlPlaneUrl required");
|
|
61
|
+
if (!publicKey || !privateKey) throw new Error("search: Nostr keypair required (run `infernet init`)");
|
|
62
|
+
|
|
63
|
+
const base = controlPlaneUrl.replace(/\/+$/, "");
|
|
64
|
+
const bodyObj = { query, num };
|
|
65
|
+
if (location) bodyObj.location = location;
|
|
66
|
+
if (domains?.length) bodyObj.domains = domains;
|
|
67
|
+
const bodyText = JSON.stringify(bodyObj);
|
|
68
|
+
|
|
69
|
+
const { header } = signRequest({
|
|
70
|
+
method: "POST",
|
|
71
|
+
path: SEARCH_PATH,
|
|
72
|
+
body: bodyText,
|
|
73
|
+
publicKey,
|
|
74
|
+
privateKey
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const ctrl = new AbortController();
|
|
78
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
79
|
+
let res;
|
|
80
|
+
try {
|
|
81
|
+
res = await fetch(base + SEARCH_PATH, {
|
|
82
|
+
method: "POST",
|
|
83
|
+
headers: {
|
|
84
|
+
"content-type": "application/json",
|
|
85
|
+
[AUTH_HEADER]: header
|
|
86
|
+
},
|
|
87
|
+
body: bodyText,
|
|
88
|
+
signal: ctrl.signal
|
|
89
|
+
});
|
|
90
|
+
} finally {
|
|
91
|
+
clearTimeout(t);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const text = await res.text();
|
|
95
|
+
let payload = null;
|
|
96
|
+
if (text) {
|
|
97
|
+
try { payload = JSON.parse(text); } catch { payload = { raw: text }; }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (!res.ok) {
|
|
101
|
+
const msg = payload?.error ?? `HTTP ${res.status}`;
|
|
102
|
+
const err = new Error(`search failed: ${msg}`);
|
|
103
|
+
err.status = res.status;
|
|
104
|
+
err.body = payload;
|
|
105
|
+
throw err;
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
results: payload?.data ?? [],
|
|
109
|
+
remaining: payload?.remaining,
|
|
110
|
+
limit: payload?.limit,
|
|
111
|
+
resetAt: payload?.reset_at
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Direct ValueSerp call — only used when INFERNET_DIRECT_VALUESERP=1.
|
|
117
|
+
* Kept for self-hosters who don't want to depend on the control plane.
|
|
35
118
|
*/
|
|
36
119
|
export async function valueSerpSearch({ query, apiKey, num = 20, location } = {}) {
|
|
37
120
|
if (!query) throw new Error("valueserp: query required");
|
|
@@ -142,11 +225,15 @@ export function paragraphsToChatml({ url, title, text, query, minChars = 200, ma
|
|
|
142
225
|
* Top-level helper: search + crawl + chunk → write training JSONL.
|
|
143
226
|
* Used by the `infernet train data` CLI surface.
|
|
144
227
|
*
|
|
145
|
-
*
|
|
228
|
+
* Pass `search: { mode: "control-plane", controlPlaneUrl, publicKey, privateKey }`
|
|
229
|
+
* to route through the platform proxy (default), or
|
|
230
|
+
* `search: { mode: "direct", apiKey }` to call the upstream provider yourself.
|
|
231
|
+
*
|
|
232
|
+
* @returns {{ urls: number, examples: number, outPath: string, quota?: object }}
|
|
146
233
|
*/
|
|
147
234
|
export async function buildTrainingDataset({
|
|
148
235
|
query,
|
|
149
|
-
|
|
236
|
+
search,
|
|
150
237
|
outPath,
|
|
151
238
|
num = 20,
|
|
152
239
|
minChars = 200,
|
|
@@ -154,7 +241,29 @@ export async function buildTrainingDataset({
|
|
|
154
241
|
domains = null, // optional whitelist (e.g. ["svelte.dev", "github.com"])
|
|
155
242
|
onProgress = () => {}
|
|
156
243
|
}) {
|
|
157
|
-
|
|
244
|
+
if (!search?.mode) throw new Error("buildTrainingDataset: search.mode is required");
|
|
245
|
+
|
|
246
|
+
let results;
|
|
247
|
+
let quota = null;
|
|
248
|
+
if (search.mode === "direct") {
|
|
249
|
+
results = await valueSerpSearch({ query, apiKey: search.apiKey, num });
|
|
250
|
+
} else if (search.mode === "control-plane") {
|
|
251
|
+
// Server applies the domain filter, but we still pass it so the
|
|
252
|
+
// server can short-circuit irrelevant results before billing them.
|
|
253
|
+
const out = await infernetSearch({
|
|
254
|
+
query,
|
|
255
|
+
num,
|
|
256
|
+
domains,
|
|
257
|
+
controlPlaneUrl: search.controlPlaneUrl,
|
|
258
|
+
publicKey: search.publicKey,
|
|
259
|
+
privateKey: search.privateKey
|
|
260
|
+
});
|
|
261
|
+
results = out.results;
|
|
262
|
+
quota = { remaining: out.remaining, limit: out.limit, resetAt: out.resetAt };
|
|
263
|
+
} else {
|
|
264
|
+
throw new Error(`buildTrainingDataset: unknown search.mode "${search.mode}"`);
|
|
265
|
+
}
|
|
266
|
+
|
|
158
267
|
const filtered = domains
|
|
159
268
|
? results.filter((r) => domains.some((d) => r.domain?.includes(d)))
|
|
160
269
|
: results;
|
|
@@ -179,5 +288,5 @@ export async function buildTrainingDataset({
|
|
|
179
288
|
}
|
|
180
289
|
}
|
|
181
290
|
await out.close();
|
|
182
|
-
return { urls: urlsUsed, examples: totalExamples, outPath };
|
|
291
|
+
return { urls: urlsUsed, examples: totalExamples, outPath, quota };
|
|
183
292
|
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IPIP-0032 Phase 3 — bridge between the long-running daemon and the
|
|
3
|
+
* Hyperswarm DiscoveryNode in @infernetprotocol/discovery.
|
|
4
|
+
*
|
|
5
|
+
* Owns one DiscoveryNode for the daemon's lifetime. The set of joined
|
|
6
|
+
* topics is derived from:
|
|
7
|
+
* - the daemon's pubkey → node:<pubkey>
|
|
8
|
+
* - specs.served_models → model:<id> (single-node serving)
|
|
9
|
+
* - state.rpc_slice.models → rpc:<id> (IPIP-0033 slice)
|
|
10
|
+
* - state.rpc_primary.models → rpc:<id> (IPIP-0033 primary)
|
|
11
|
+
* - active rpc role / fleet-class → class:B or class:B5
|
|
12
|
+
*
|
|
13
|
+
* Recomputed periodically so daemons that come online and start
|
|
14
|
+
* serving a new model picked up the new topic without restart.
|
|
15
|
+
*
|
|
16
|
+
* Discovery is gated behind `--enable-dht` per the IPIP rollout —
|
|
17
|
+
* this module is only constructed when the flag is present, so
|
|
18
|
+
* Hyperswarm is never imported on legacy daemons.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Pure topic-set builder — exported for tests. Returns a deduped
|
|
23
|
+
* array of `{ kind, value }` ready to hand to DiscoveryNode.setTopics.
|
|
24
|
+
*
|
|
25
|
+
* @param {Object} args
|
|
26
|
+
* @param {string} args.pubkey 64-char hex
|
|
27
|
+
* @param {string[]} [args.servedModels] specs.served_models
|
|
28
|
+
* @param {Object} [args.inferenceState] shape from lib/inference/state.js
|
|
29
|
+
* @param {'provider'|'aggregator'|'client'} [args.role]
|
|
30
|
+
*/
|
|
31
|
+
export function computeTopics({ pubkey, servedModels = [], inferenceState = {}, role }) {
|
|
32
|
+
const topics = [];
|
|
33
|
+
|
|
34
|
+
if (typeof pubkey === 'string' && /^[0-9a-fA-F]{64}$/.test(pubkey)) {
|
|
35
|
+
topics.push({ kind: 'node', value: pubkey.toLowerCase() });
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
for (const m of servedModels) {
|
|
39
|
+
if (typeof m === 'string' && m.length > 0) {
|
|
40
|
+
topics.push({ kind: 'model', value: m });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const slice = inferenceState?.rpc_slice;
|
|
45
|
+
const primary = inferenceState?.rpc_primary;
|
|
46
|
+
const sliceModels = Array.isArray(slice?.models) ? slice.models : [];
|
|
47
|
+
const primaryModels = Array.isArray(primary?.models) ? primary.models : [];
|
|
48
|
+
for (const m of sliceModels) {
|
|
49
|
+
if (typeof m === 'string' && m.length > 0) topics.push({ kind: 'rpc', value: m });
|
|
50
|
+
}
|
|
51
|
+
for (const m of primaryModels) {
|
|
52
|
+
if (typeof m === 'string' && m.length > 0) topics.push({ kind: 'rpc', value: m });
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Workload class (per IPIP-0010): nodes running federated inference
|
|
56
|
+
// are class:B5; otherwise default providers are class:B. Aggregators
|
|
57
|
+
// and clients don't get a class topic — they're not job-takers.
|
|
58
|
+
const hasRpc = sliceModels.length > 0 || primaryModels.length > 0;
|
|
59
|
+
if (hasRpc) {
|
|
60
|
+
topics.push({ kind: 'class', value: 'B5' });
|
|
61
|
+
} else if (role === 'provider') {
|
|
62
|
+
topics.push({ kind: 'class', value: 'B' });
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Dedupe — primary + slice for the same model collapse to one rpc:
|
|
66
|
+
// entry; node: is unique by construction.
|
|
67
|
+
const seen = new Set();
|
|
68
|
+
return topics.filter((t) => {
|
|
69
|
+
const k = `${t.kind}:${t.value}`;
|
|
70
|
+
if (seen.has(k)) return false;
|
|
71
|
+
seen.add(k);
|
|
72
|
+
return true;
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Serialize the topic set into a stable string so we only call
|
|
78
|
+
* setTopics() when the set actually changes (saves a DHT churn).
|
|
79
|
+
*/
|
|
80
|
+
export function topicSetSignature(topics) {
|
|
81
|
+
return topics
|
|
82
|
+
.map((t) => `${t.kind}:${t.value}`)
|
|
83
|
+
.sort()
|
|
84
|
+
.join(',');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Spin up the DiscoveryNode and resync its topic set on every
|
|
89
|
+
* `intervalMs` tick. Returns `{ stop }` for clean shutdown.
|
|
90
|
+
*
|
|
91
|
+
* The DiscoveryNode itself is loaded via dynamic import so legacy
|
|
92
|
+
* daemons that don't pass --enable-dht never touch the hyperswarm
|
|
93
|
+
* dep tree.
|
|
94
|
+
*/
|
|
95
|
+
export async function startDiscoveryBridge({
|
|
96
|
+
config,
|
|
97
|
+
getState,
|
|
98
|
+
advertise = true,
|
|
99
|
+
address = null,
|
|
100
|
+
intervalMs = 30_000,
|
|
101
|
+
onPeer = null,
|
|
102
|
+
onHandshakeFailed = null
|
|
103
|
+
}) {
|
|
104
|
+
const { createDiscoveryNode } = await import('@infernetprotocol/discovery');
|
|
105
|
+
const pubkey = config?.node?.publicKey;
|
|
106
|
+
const privateKey = config?.node?.privateKey;
|
|
107
|
+
const role = config?.node?.role;
|
|
108
|
+
if (!pubkey || !privateKey) {
|
|
109
|
+
throw new Error('discovery-bridge: config.node.{publicKey,privateKey} required');
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const initialState = getState();
|
|
113
|
+
const initialTopics = computeTopics({
|
|
114
|
+
pubkey,
|
|
115
|
+
servedModels: initialState?.servedModels ?? [],
|
|
116
|
+
inferenceState: initialState?.inferenceState ?? {},
|
|
117
|
+
role
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
const node = await createDiscoveryNode({
|
|
121
|
+
privateKey,
|
|
122
|
+
publicKey: pubkey,
|
|
123
|
+
advertise,
|
|
124
|
+
address,
|
|
125
|
+
topics: initialTopics
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
let lastSig = topicSetSignature(initialTopics);
|
|
129
|
+
|
|
130
|
+
if (typeof onPeer === 'function') {
|
|
131
|
+
node.on('peer', onPeer);
|
|
132
|
+
}
|
|
133
|
+
if (typeof onHandshakeFailed === 'function') {
|
|
134
|
+
node.on('handshake-failed', onHandshakeFailed);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
let stopped = false;
|
|
138
|
+
const interval = setInterval(() => {
|
|
139
|
+
if (stopped) return;
|
|
140
|
+
try {
|
|
141
|
+
const s = getState();
|
|
142
|
+
const next = computeTopics({
|
|
143
|
+
pubkey,
|
|
144
|
+
servedModels: s?.servedModels ?? [],
|
|
145
|
+
inferenceState: s?.inferenceState ?? {},
|
|
146
|
+
role
|
|
147
|
+
});
|
|
148
|
+
const sig = topicSetSignature(next);
|
|
149
|
+
if (sig !== lastSig) {
|
|
150
|
+
node.setTopics(next);
|
|
151
|
+
lastSig = sig;
|
|
152
|
+
}
|
|
153
|
+
} catch {
|
|
154
|
+
// Diff failed — leave previous topics in place.
|
|
155
|
+
}
|
|
156
|
+
}, intervalMs);
|
|
157
|
+
if (typeof interval.unref === 'function') interval.unref();
|
|
158
|
+
|
|
159
|
+
return {
|
|
160
|
+
node,
|
|
161
|
+
get currentTopics() { return node.topics; },
|
|
162
|
+
stop: async () => {
|
|
163
|
+
stopped = true;
|
|
164
|
+
clearInterval(interval);
|
|
165
|
+
try { await node.destroy(); } catch { /* ignore */ }
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared `~/.infernet/inference/state.json` reader/writer. The file
|
|
3
|
+
* is the single source of truth between `infernet inference …`
|
|
4
|
+
* subcommands and the long-running daemon (which reads it on every
|
|
5
|
+
* heartbeat to assemble specs.rpc / specs.rpc_primary).
|
|
6
|
+
*
|
|
7
|
+
* Shape (all fields optional — the daemon assembles only what's
|
|
8
|
+
* present):
|
|
9
|
+
*
|
|
10
|
+
* {
|
|
11
|
+
* "rpc_slice": { models: [...], host, port, pid, started_at,
|
|
12
|
+
* binary, gpu, vram_gb, ram_gb, max_concurrent },
|
|
13
|
+
* "rpc_primary": { models: [...], gguf_paths: { <model_id>: <path> },
|
|
14
|
+
* version }
|
|
15
|
+
* }
|
|
16
|
+
*
|
|
17
|
+
* Atomic writes via temp-file rename so concurrent subcommands +
|
|
18
|
+
* daemon heartbeats can't see a torn file. Reads tolerate a missing
|
|
19
|
+
* file (returns {}).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import fs from 'node:fs';
|
|
23
|
+
import fsp from 'node:fs/promises';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
|
|
26
|
+
const STATE_DIR = path.join(process.env.HOME ?? '/tmp', '.infernet', 'inference');
|
|
27
|
+
const STATE_FILE = path.join(STATE_DIR, 'state.json');
|
|
28
|
+
|
|
29
|
+
export function inferenceStateDir() { return STATE_DIR; }
|
|
30
|
+
export function inferenceStateFile() { return STATE_FILE; }
|
|
31
|
+
|
|
32
|
+
export async function readInferenceState() {
|
|
33
|
+
try {
|
|
34
|
+
const raw = await fsp.readFile(STATE_FILE, 'utf8');
|
|
35
|
+
const obj = JSON.parse(raw);
|
|
36
|
+
return obj && typeof obj === 'object' ? obj : {};
|
|
37
|
+
} catch { return {}; }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function readInferenceStateSync() {
|
|
41
|
+
try {
|
|
42
|
+
const raw = fs.readFileSync(STATE_FILE, 'utf8');
|
|
43
|
+
const obj = JSON.parse(raw);
|
|
44
|
+
return obj && typeof obj === 'object' ? obj : {};
|
|
45
|
+
} catch { return {}; }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function writeInferenceState(next) {
|
|
49
|
+
await fsp.mkdir(STATE_DIR, { recursive: true });
|
|
50
|
+
const tmp = `${STATE_FILE}.${process.pid}.${Date.now()}`;
|
|
51
|
+
await fsp.writeFile(tmp, JSON.stringify(next, null, 2));
|
|
52
|
+
await fsp.rename(tmp, STATE_FILE);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Read-modify-write helper. The mutator can be sync or async; it
|
|
57
|
+
* receives the current state (always an object) and returns the
|
|
58
|
+
* next state.
|
|
59
|
+
*/
|
|
60
|
+
export async function patchInferenceState(mutator) {
|
|
61
|
+
const cur = await readInferenceState();
|
|
62
|
+
const next = await mutator({ ...cur });
|
|
63
|
+
await writeInferenceState(next);
|
|
64
|
+
return next;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* True if `pid` is a live process. `process.kill(pid, 0)` throws
|
|
69
|
+
* ESRCH for dead pids and EPERM for foreign processes — we treat
|
|
70
|
+
* EPERM as "alive but not ours" (unusual but possible after a uid
|
|
71
|
+
* change). Anything else → dead.
|
|
72
|
+
*/
|
|
73
|
+
export function isPidAlive(pid) {
|
|
74
|
+
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
75
|
+
try {
|
|
76
|
+
process.kill(pid, 0);
|
|
77
|
+
return true;
|
|
78
|
+
} catch (err) {
|
|
79
|
+
return err?.code === 'EPERM';
|
|
80
|
+
}
|
|
81
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.45",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,11 +30,13 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/config": "0.1.
|
|
35
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
36
|
-
"@infernetprotocol/engine": "0.1.
|
|
37
|
-
"@infernetprotocol/gpu": "0.1.
|
|
33
|
+
"@infernetprotocol/auth": "0.1.45",
|
|
34
|
+
"@infernetprotocol/config": "0.1.45",
|
|
35
|
+
"@infernetprotocol/deploy-providers": "0.1.45",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.45",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.45",
|
|
38
|
+
"@infernetprotocol/discovery": "0.1.45",
|
|
39
|
+
"@infernetprotocol/rpc-adapter": "0.1.45"
|
|
38
40
|
},
|
|
39
41
|
"repository": {
|
|
40
42
|
"type": "git",
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Bundled Python petals client. Spawned by the daemon when it accepts
|
|
3
|
-
* an /v1/petals/inference POST. Fans out across the public Petals
|
|
4
|
-
* swarm using AutoDistributedModelForCausalLM and streams JSON lines
|
|
5
|
-
* (one per token) to stdout. The daemon converts those to SSE frames.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
export const PETALS_CLIENT_PY = `#!/usr/bin/env python3
|
|
9
|
-
"""Petals client — fans out across the swarm + streams tokens.
|
|
10
|
-
|
|
11
|
-
Reads: JSON request from stdin: { model, messages, max_tokens, temperature }
|
|
12
|
-
Writes: JSON lines to stdout: { event: 'token' | 'done' | 'error', ... }
|
|
13
|
-
|
|
14
|
-
Each output line is a single JSON object terminated with \\n so the
|
|
15
|
-
parent process can parse them line-by-line.
|
|
16
|
-
"""
|
|
17
|
-
import json
|
|
18
|
-
import sys
|
|
19
|
-
import time
|
|
20
|
-
|
|
21
|
-
def emit(obj):
|
|
22
|
-
sys.stdout.write(json.dumps(obj) + "\\n")
|
|
23
|
-
sys.stdout.flush()
|
|
24
|
-
|
|
25
|
-
def main():
|
|
26
|
-
try:
|
|
27
|
-
req = json.loads(sys.stdin.read())
|
|
28
|
-
except Exception as e:
|
|
29
|
-
emit({"event": "error", "message": f"bad request: {e}"})
|
|
30
|
-
sys.exit(2)
|
|
31
|
-
|
|
32
|
-
model = req.get("model")
|
|
33
|
-
messages = req.get("messages", [])
|
|
34
|
-
max_tokens = int(req.get("max_tokens") or 256)
|
|
35
|
-
temperature = float(req.get("temperature") or 0.7)
|
|
36
|
-
if not model:
|
|
37
|
-
emit({"event": "error", "message": "model required"})
|
|
38
|
-
sys.exit(2)
|
|
39
|
-
|
|
40
|
-
try:
|
|
41
|
-
import torch
|
|
42
|
-
from transformers import AutoTokenizer
|
|
43
|
-
from petals import AutoDistributedModelForCausalLM
|
|
44
|
-
except ImportError as e:
|
|
45
|
-
emit({"event": "error", "message": f"petals not installed: {e}. Install: pip install -U petals"})
|
|
46
|
-
sys.exit(2)
|
|
47
|
-
|
|
48
|
-
emit({"event": "loading", "model": model})
|
|
49
|
-
tokenizer = AutoTokenizer.from_pretrained(model)
|
|
50
|
-
model_obj = AutoDistributedModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
|
|
51
|
-
|
|
52
|
-
# Compose a single prompt from messages using the tokenizer's chat template.
|
|
53
|
-
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
54
|
-
input_ids = tokenizer.encode(prompt, return_tensors="pt")
|
|
55
|
-
|
|
56
|
-
emit({"event": "ready", "input_tokens": int(input_ids.shape[-1])})
|
|
57
|
-
|
|
58
|
-
# Petals streams via inference_session — emit one JSON line per generated token.
|
|
59
|
-
full_text = ""
|
|
60
|
-
t0 = time.time()
|
|
61
|
-
with model_obj.inference_session(max_length=int(input_ids.shape[-1]) + max_tokens) as session:
|
|
62
|
-
# Prefill
|
|
63
|
-
outputs = model_obj.generate(
|
|
64
|
-
input_ids,
|
|
65
|
-
max_new_tokens=1,
|
|
66
|
-
do_sample=temperature > 0,
|
|
67
|
-
temperature=max(temperature, 1e-3),
|
|
68
|
-
session=session,
|
|
69
|
-
)
|
|
70
|
-
new_id = int(outputs[0, -1])
|
|
71
|
-
token_text = tokenizer.decode([new_id], skip_special_tokens=True)
|
|
72
|
-
full_text += token_text
|
|
73
|
-
emit({"event": "token", "text": token_text})
|
|
74
|
-
|
|
75
|
-
# IPIP-0031 per-layer attribution: after the first generate the
|
|
76
|
-
# session has resolved which DHT peers serve which transformer
|
|
77
|
-
# blocks. Emit that mapping so the control plane can split the
|
|
78
|
-
# CPR receipt across layer-contributing operators by block share.
|
|
79
|
-
try:
|
|
80
|
-
chosen = []
|
|
81
|
-
for s in getattr(session, "chosen_servers", []) or []:
|
|
82
|
-
chosen.append({
|
|
83
|
-
"peer_id": str(getattr(s, "peer_id", "")),
|
|
84
|
-
"start_block": int(getattr(s, "start", getattr(s, "start_block", 0))),
|
|
85
|
-
"end_block": int(getattr(s, "end", getattr(s, "end_block", 0))),
|
|
86
|
-
})
|
|
87
|
-
if chosen:
|
|
88
|
-
emit({"event": "routing", "peers": chosen})
|
|
89
|
-
except Exception as e:
|
|
90
|
-
emit({"event": "log", "warn": f"chosen_servers extract failed: {e}"})
|
|
91
|
-
|
|
92
|
-
for _ in range(max_tokens - 1):
|
|
93
|
-
outputs = model_obj.generate(
|
|
94
|
-
outputs,
|
|
95
|
-
max_new_tokens=1,
|
|
96
|
-
do_sample=temperature > 0,
|
|
97
|
-
temperature=max(temperature, 1e-3),
|
|
98
|
-
session=session,
|
|
99
|
-
)
|
|
100
|
-
new_id = int(outputs[0, -1])
|
|
101
|
-
if new_id == tokenizer.eos_token_id:
|
|
102
|
-
break
|
|
103
|
-
token_text = tokenizer.decode([new_id], skip_special_tokens=True)
|
|
104
|
-
full_text += token_text
|
|
105
|
-
emit({"event": "token", "text": token_text})
|
|
106
|
-
|
|
107
|
-
emit({"event": "done", "text": full_text, "elapsed_s": round(time.time() - t0, 2)})
|
|
108
|
-
|
|
109
|
-
if __name__ == "__main__":
|
|
110
|
-
try:
|
|
111
|
-
main()
|
|
112
|
-
except Exception as e:
|
|
113
|
-
emit({"event": "error", "message": str(e)})
|
|
114
|
-
sys.exit(1)
|
|
115
|
-
`;
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Petals engine adapter — DISTRIBUTED inference across the volunteer
|
|
3
|
-
* swarm (IPIP-0031, follow-up PR).
|
|
4
|
-
*
|
|
5
|
-
* Petals (https://github.com/bigscience-workshop/petals) splits a
|
|
6
|
-
* Llama/Mixtral-style transformer by layer across DHT-discovered nodes.
|
|
7
|
-
* Each node holds a few transformer blocks; activations flow through.
|
|
8
|
-
* Designed for residential bandwidth — exactly the Infernet model.
|
|
9
|
-
*
|
|
10
|
-
* STATUS: scaffolding only. The flow:
|
|
11
|
-
*
|
|
12
|
-
* serve() — operator runs `infernet inference serve --backend petals
|
|
13
|
-
* --model meta-llama/Llama-3.1-70B`. We:
|
|
14
|
-
* 1. spawn `python -m petals.cli.run_server <model>`
|
|
15
|
-
* 2. register the operator as a Petals server in our
|
|
16
|
-
* own registry: row in providers.specs.petals_models
|
|
17
|
-
* 3. heartbeat that the server is alive
|
|
18
|
-
*
|
|
19
|
-
* client() — control plane's chat router, when input_spec.distributed
|
|
20
|
-
* is true, runs a Petals CLIENT (Python subprocess or
|
|
21
|
-
* a thin TS wrapper) that fans out across the swarm.
|
|
22
|
-
* Tokens stream back to the SSE handler exactly like
|
|
23
|
-
* Ollama's stream.
|
|
24
|
-
*
|
|
25
|
-
* The SSE handler in apps/web/app/api/chat/stream/[jobId]/route.js
|
|
26
|
-
* already has the typed events plumbing — when distributed===true,
|
|
27
|
-
* it would call into routePetalsRequest below instead of subscribing
|
|
28
|
-
* to job_events.
|
|
29
|
-
*
|
|
30
|
-
* To-do for the follow-up PR:
|
|
31
|
-
* - bundle a thin Python venv with Petals at infernet setup time
|
|
32
|
-
* - add `infernet inference serve --backend petals` as a top-level command
|
|
33
|
-
* - register petals_models on heartbeat so the dashboard can show
|
|
34
|
-
* "this model is served by 8 nodes via Petals · ~12 tok/s" badges
|
|
35
|
-
* - wire the SSE handler's distributed branch through routePetalsRequest
|
|
36
|
-
* - per-token receipts to operators contributing layers (CPR / IPIP-0007)
|
|
37
|
-
*/
|
|
38
|
-
|
|
39
|
-
import { spawn } from "node:child_process";
|
|
40
|
-
|
|
41
|
-
const DEFAULT_DHT_PREFIX = "/petals/v3-public";
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Start a Petals server contributing this node's VRAM to the swarm.
|
|
45
|
-
* Returns a child handle the daemon can monitor + restart.
|
|
46
|
-
*/
|
|
47
|
-
export function startPetalsServer({ model, numBlocks, port = 31330, prefix = DEFAULT_DHT_PREFIX }) {
|
|
48
|
-
const args = ["-m", "petals.cli.run_server", model];
|
|
49
|
-
if (numBlocks) args.push("--num_blocks", String(numBlocks));
|
|
50
|
-
if (port) args.push("--port", String(port));
|
|
51
|
-
if (prefix) args.push("--initial_peers", prefix);
|
|
52
|
-
const child = spawn("python3", args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
53
|
-
return child;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Run a single inference through a Petals client and stream tokens.
|
|
58
|
-
* Returns an async iterable of { type: 'token' | 'done' | 'error', data }.
|
|
59
|
-
*
|
|
60
|
-
* Currently a stub — wires up the subprocess but doesn't translate the
|
|
61
|
-
* Python output into typed events. Follow-up PR: pip install
|
|
62
|
-
* `petals` in our control-plane container and call from a Python
|
|
63
|
-
* worker, OR use a JS-native bridge once one exists.
|
|
64
|
-
*/
|
|
65
|
-
export async function* routePetalsRequest({ model, messages, maxTokens = 256, temperature = 0.7 }) {
|
|
66
|
-
yield {
|
|
67
|
-
type: "error",
|
|
68
|
-
data: {
|
|
69
|
-
message:
|
|
70
|
-
`Distributed inference via Petals is in scaffolding (IPIP-0031). ` +
|
|
71
|
-
`For now, this request needs to fall back to a single-node provider. ` +
|
|
72
|
-
`Track progress at https://github.com/infernetprotocol/infernet-protocol/issues`
|
|
73
|
-
}
|
|
74
|
-
};
|
|
75
|
-
}
|