@infernetprotocol/cli 0.1.41 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/inference.js +196 -179
- package/commands/register.js +10 -39
- package/commands/rpc.js +133 -0
- package/commands/setup.js +25 -15
- package/commands/start.js +281 -48
- package/commands/train.js +69 -17
- package/commands/uncensored.js +18 -0
- package/index.js +3 -2
- package/lib/chat-executor.js +71 -29
- package/lib/data-crawler.js +118 -9
- package/lib/hf-model.js +23 -47
- package/lib/inference/state.js +81 -0
- package/lib/prompt.js +19 -3
- package/lib/recommender.js +5 -1
- package/lib/vllm.js +140 -0
- package/package.json +7 -6
- package/lib/inference/petals-client.py.js +0 -115
- package/lib/inference/petals-engine.js +0 -75
package/lib/chat-executor.js
CHANGED
|
@@ -93,44 +93,86 @@ class EventBuffer {
|
|
|
93
93
|
// /chat endpoint sometimes doesn't pass a model name). Without this
|
|
94
94
|
// fallback, every model-unspecified job died with
|
|
95
95
|
// "ollama backend: no model — set INFERNET_ENGINE_MODEL or pass model in the job"
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
106
|
-
// Optional override for the per-request num_thread cap.
|
|
107
|
-
// Default (50% of cores) is computed inside the Ollama
|
|
108
|
-
// backend; only forward when the operator explicitly set
|
|
109
|
-
// an override in config.
|
|
110
|
-
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
111
|
-
opts.numThread = eng.ollama_num_thread;
|
|
112
|
-
}
|
|
113
|
-
return createEngine(opts);
|
|
114
|
-
})().catch((err) => {
|
|
96
|
+
// One engine per backend, cached. We route PER MODEL: if the requested model
|
|
97
|
+
// is being served by vLLM (:8000), use the vLLM engine; otherwise the default
|
|
98
|
+
// (config backend or auto-select → Ollama). This is what makes vLLM the actual
|
|
99
|
+
// serving path for hf: models while Ollama keeps handling its own tags.
|
|
100
|
+
const engineCache = new Map(); // cacheKey -> Promise<engine>
|
|
101
|
+
|
|
102
|
+
function buildEngine(cacheKey, opts) {
|
|
103
|
+
if (!engineCache.has(cacheKey)) {
|
|
104
|
+
const p = (async () => createEngine(opts))().catch((err) => {
|
|
115
105
|
// Reset so a transient failure doesn't permanently poison the
|
|
116
106
|
// daemon — next job will retry initialization.
|
|
117
|
-
|
|
107
|
+
engineCache.delete(cacheKey);
|
|
118
108
|
throw err;
|
|
119
109
|
});
|
|
110
|
+
engineCache.set(cacheKey, p);
|
|
120
111
|
}
|
|
121
|
-
return
|
|
112
|
+
return engineCache.get(cacheKey);
|
|
122
113
|
}
|
|
123
114
|
|
|
124
|
-
|
|
125
|
-
|
|
115
|
+
async function defaultEngine() {
|
|
116
|
+
const config = (await loadConfig()) ?? {};
|
|
117
|
+
const eng = config.engine ?? {};
|
|
118
|
+
const opts = {};
|
|
119
|
+
if (eng.backend) opts.backend = eng.backend;
|
|
120
|
+
if (eng.model) opts.defaultModel = eng.model;
|
|
121
|
+
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
122
|
+
// Optional override for the per-request num_thread cap. Default (50% of
|
|
123
|
+
// cores) is computed inside the Ollama backend; only forward when set.
|
|
124
|
+
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
125
|
+
opts.numThread = eng.ollama_num_thread;
|
|
126
|
+
}
|
|
127
|
+
return buildEngine(`default:${eng.backend ?? 'auto'}`, opts);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Best-effort: does Ollama (at host) currently serve this exact tag? */
|
|
131
|
+
async function ollamaHasModel(modelName, host) {
|
|
126
132
|
try {
|
|
127
|
-
const
|
|
128
|
-
|
|
133
|
+
const ctrl = new AbortController();
|
|
134
|
+
const t = setTimeout(() => ctrl.abort(), 800);
|
|
135
|
+
const r = await fetch(`${host}/api/tags`, { signal: ctrl.signal });
|
|
136
|
+
clearTimeout(t);
|
|
137
|
+
if (!r.ok) return false;
|
|
138
|
+
const data = await r.json();
|
|
139
|
+
return (data.models ?? []).some((m) => m.name === modelName || m.model === modelName);
|
|
129
140
|
} catch {
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function getEngineForModel(modelName) {
|
|
146
|
+
// vLLM is the default serving path. Route PER MODEL so both engines coexist:
|
|
147
|
+
// 1. vLLM is serving this model → vLLM (preferred, high throughput)
|
|
148
|
+
// 2. Ollama has this GGUF tag → Ollama (its library keeps working even
|
|
149
|
+
// while vLLM is up serving another model)
|
|
150
|
+
// 3. otherwise → default (auto-select, which prefers vLLM)
|
|
151
|
+
if (modelName) {
|
|
152
|
+
try {
|
|
153
|
+
const { detectVllmModels, VLLM_HOST } = await import('./vllm.js');
|
|
154
|
+
const vModels = await detectVllmModels();
|
|
155
|
+
if (vModels.includes(modelName)) {
|
|
156
|
+
return buildEngine('vllm', { backend: 'vllm', host: VLLM_HOST, defaultModel: modelName });
|
|
157
|
+
}
|
|
158
|
+
} catch {
|
|
159
|
+
// vLLM module/probe failed — try Ollama, then the default.
|
|
160
|
+
}
|
|
161
|
+
const config = (await loadConfig().catch(() => null)) ?? {};
|
|
162
|
+
const ollamaHost = config.engine?.ollamaHost ?? process.env.OLLAMA_HOST ?? 'http://localhost:11434';
|
|
163
|
+
if (await ollamaHasModel(modelName, ollamaHost)) {
|
|
164
|
+
return buildEngine('ollama', { backend: 'ollama', host: ollamaHost, defaultModel: modelName });
|
|
165
|
+
}
|
|
133
166
|
}
|
|
167
|
+
return defaultEngine();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export async function shutdownEngine() {
|
|
171
|
+
const engines = [...engineCache.values()];
|
|
172
|
+
engineCache.clear();
|
|
173
|
+
await Promise.all(engines.map(async (p) => {
|
|
174
|
+
try { (await p).shutdown(); } catch { /* best-effort */ }
|
|
175
|
+
}));
|
|
134
176
|
}
|
|
135
177
|
|
|
136
178
|
/**
|
|
@@ -167,7 +209,7 @@ export async function executeChatJob({ client, job, node }) {
|
|
|
167
209
|
}
|
|
168
210
|
}
|
|
169
211
|
|
|
170
|
-
const engine = await
|
|
212
|
+
const engine = await getEngineForModel(job.model_name);
|
|
171
213
|
const buffer = new EventBuffer(client, job.id);
|
|
172
214
|
|
|
173
215
|
const t0 = Date.now();
|
package/lib/data-crawler.js
CHANGED
|
@@ -2,21 +2,28 @@
|
|
|
2
2
|
* Search-driven training-data collector.
|
|
3
3
|
*
|
|
4
4
|
* Pipeline:
|
|
5
|
-
* 1. Hit
|
|
5
|
+
* 1. Hit infernetprotocol.com/api/v1/search with a query (Nostr-signed).
|
|
6
|
+
* The control plane proxies to the real provider, holds the upstream
|
|
7
|
+
* key, and enforces a per-pubkey daily quota.
|
|
6
8
|
* 2. For each URL: fetch HTML, strip nav/script/style, extract main text.
|
|
7
9
|
* 3. For each page: chunk into Q/A pairs (heading-based) and emit JSONL.
|
|
8
10
|
*
|
|
9
11
|
* The output JSONL uses the chatml `messages` shape, ready to feed into
|
|
10
12
|
* `infernet train run` or any HF/Unsloth/Axolotl pipeline.
|
|
11
13
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
+
* Self-host escape hatch: set INFERNET_DIRECT_VALUESERP=1 with a
|
|
15
|
+
* VALUESERP_API_KEY in env (or `integrations.valueserp.api_key` in config)
|
|
16
|
+
* to bypass the control plane and call valueserp directly.
|
|
14
17
|
*/
|
|
15
18
|
|
|
16
19
|
import fs from "node:fs/promises";
|
|
17
20
|
import path from "node:path";
|
|
21
|
+
import { signRequest, AUTH_HEADER } from "@infernetprotocol/auth";
|
|
18
22
|
import { loadConfig } from "./config.js";
|
|
19
23
|
|
|
24
|
+
const SEARCH_PATH = "/api/v1/search";
|
|
25
|
+
|
|
26
|
+
/** Resolve a direct VALUESERP_API_KEY for the escape hatch. */
|
|
20
27
|
export async function resolveValueSerpKey() {
|
|
21
28
|
if (process.env.VALUESERP_API_KEY) return process.env.VALUESERP_API_KEY;
|
|
22
29
|
const cfg = await loadConfig().catch(() => null);
|
|
@@ -26,12 +33,88 @@ export async function resolveValueSerpKey() {
|
|
|
26
33
|
}
|
|
27
34
|
|
|
28
35
|
/**
|
|
29
|
-
* Search the
|
|
36
|
+
* Search via the control plane (default path). Auth is the node's existing
|
|
37
|
+
* Nostr keypair — operators don't need a separate API key.
|
|
38
|
+
*
|
|
30
39
|
* @param {object} opts
|
|
31
40
|
* @param {string} opts.query
|
|
32
|
-
* @param {string} opts.
|
|
41
|
+
* @param {string} opts.controlPlaneUrl
|
|
42
|
+
* @param {string} opts.publicKey - 64-hex Nostr pubkey
|
|
43
|
+
* @param {string} opts.privateKey - 64-hex Nostr privkey
|
|
33
44
|
* @param {number} [opts.num=20]
|
|
34
45
|
* @param {string} [opts.location]
|
|
46
|
+
* @param {string[]} [opts.domains]
|
|
47
|
+
* @param {number} [opts.timeoutMs=25000]
|
|
48
|
+
*/
|
|
49
|
+
export async function infernetSearch({
|
|
50
|
+
query,
|
|
51
|
+
controlPlaneUrl,
|
|
52
|
+
publicKey,
|
|
53
|
+
privateKey,
|
|
54
|
+
num = 20,
|
|
55
|
+
location,
|
|
56
|
+
domains,
|
|
57
|
+
timeoutMs = 25_000
|
|
58
|
+
} = {}) {
|
|
59
|
+
if (!query) throw new Error("search: query required");
|
|
60
|
+
if (!controlPlaneUrl) throw new Error("search: controlPlaneUrl required");
|
|
61
|
+
if (!publicKey || !privateKey) throw new Error("search: Nostr keypair required (run `infernet init`)");
|
|
62
|
+
|
|
63
|
+
const base = controlPlaneUrl.replace(/\/+$/, "");
|
|
64
|
+
const bodyObj = { query, num };
|
|
65
|
+
if (location) bodyObj.location = location;
|
|
66
|
+
if (domains?.length) bodyObj.domains = domains;
|
|
67
|
+
const bodyText = JSON.stringify(bodyObj);
|
|
68
|
+
|
|
69
|
+
const { header } = signRequest({
|
|
70
|
+
method: "POST",
|
|
71
|
+
path: SEARCH_PATH,
|
|
72
|
+
body: bodyText,
|
|
73
|
+
publicKey,
|
|
74
|
+
privateKey
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const ctrl = new AbortController();
|
|
78
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
79
|
+
let res;
|
|
80
|
+
try {
|
|
81
|
+
res = await fetch(base + SEARCH_PATH, {
|
|
82
|
+
method: "POST",
|
|
83
|
+
headers: {
|
|
84
|
+
"content-type": "application/json",
|
|
85
|
+
[AUTH_HEADER]: header
|
|
86
|
+
},
|
|
87
|
+
body: bodyText,
|
|
88
|
+
signal: ctrl.signal
|
|
89
|
+
});
|
|
90
|
+
} finally {
|
|
91
|
+
clearTimeout(t);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const text = await res.text();
|
|
95
|
+
let payload = null;
|
|
96
|
+
if (text) {
|
|
97
|
+
try { payload = JSON.parse(text); } catch { payload = { raw: text }; }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (!res.ok) {
|
|
101
|
+
const msg = payload?.error ?? `HTTP ${res.status}`;
|
|
102
|
+
const err = new Error(`search failed: ${msg}`);
|
|
103
|
+
err.status = res.status;
|
|
104
|
+
err.body = payload;
|
|
105
|
+
throw err;
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
results: payload?.data ?? [],
|
|
109
|
+
remaining: payload?.remaining,
|
|
110
|
+
limit: payload?.limit,
|
|
111
|
+
resetAt: payload?.reset_at
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Direct ValueSerp call — only used when INFERNET_DIRECT_VALUESERP=1.
|
|
117
|
+
* Kept for self-hosters who don't want to depend on the control plane.
|
|
35
118
|
*/
|
|
36
119
|
export async function valueSerpSearch({ query, apiKey, num = 20, location } = {}) {
|
|
37
120
|
if (!query) throw new Error("valueserp: query required");
|
|
@@ -142,11 +225,15 @@ export function paragraphsToChatml({ url, title, text, query, minChars = 200, ma
|
|
|
142
225
|
* Top-level helper: search + crawl + chunk → write training JSONL.
|
|
143
226
|
* Used by the `infernet train data` CLI surface.
|
|
144
227
|
*
|
|
145
|
-
*
|
|
228
|
+
* Pass `search: { mode: "control-plane", controlPlaneUrl, publicKey, privateKey }`
|
|
229
|
+
* to route through the platform proxy (default), or
|
|
230
|
+
* `search: { mode: "direct", apiKey }` to call the upstream provider yourself.
|
|
231
|
+
*
|
|
232
|
+
* @returns {{ urls: number, examples: number, outPath: string, quota?: object }}
|
|
146
233
|
*/
|
|
147
234
|
export async function buildTrainingDataset({
|
|
148
235
|
query,
|
|
149
|
-
|
|
236
|
+
search,
|
|
150
237
|
outPath,
|
|
151
238
|
num = 20,
|
|
152
239
|
minChars = 200,
|
|
@@ -154,7 +241,29 @@ export async function buildTrainingDataset({
|
|
|
154
241
|
domains = null, // optional whitelist (e.g. ["svelte.dev", "github.com"])
|
|
155
242
|
onProgress = () => {}
|
|
156
243
|
}) {
|
|
157
|
-
|
|
244
|
+
if (!search?.mode) throw new Error("buildTrainingDataset: search.mode is required");
|
|
245
|
+
|
|
246
|
+
let results;
|
|
247
|
+
let quota = null;
|
|
248
|
+
if (search.mode === "direct") {
|
|
249
|
+
results = await valueSerpSearch({ query, apiKey: search.apiKey, num });
|
|
250
|
+
} else if (search.mode === "control-plane") {
|
|
251
|
+
// Server applies the domain filter, but we still pass it so the
|
|
252
|
+
// server can short-circuit irrelevant results before billing them.
|
|
253
|
+
const out = await infernetSearch({
|
|
254
|
+
query,
|
|
255
|
+
num,
|
|
256
|
+
domains,
|
|
257
|
+
controlPlaneUrl: search.controlPlaneUrl,
|
|
258
|
+
publicKey: search.publicKey,
|
|
259
|
+
privateKey: search.privateKey
|
|
260
|
+
});
|
|
261
|
+
results = out.results;
|
|
262
|
+
quota = { remaining: out.remaining, limit: out.limit, resetAt: out.resetAt };
|
|
263
|
+
} else {
|
|
264
|
+
throw new Error(`buildTrainingDataset: unknown search.mode "${search.mode}"`);
|
|
265
|
+
}
|
|
266
|
+
|
|
158
267
|
const filtered = domains
|
|
159
268
|
? results.filter((r) => domains.some((d) => r.domain?.includes(d)))
|
|
160
269
|
: results;
|
|
@@ -179,5 +288,5 @@ export async function buildTrainingDataset({
|
|
|
179
288
|
}
|
|
180
289
|
}
|
|
181
290
|
await out.close();
|
|
182
|
-
return { urls: urlsUsed, examples: totalExamples, outPath };
|
|
291
|
+
return { urls: urlsUsed, examples: totalExamples, outPath, quota };
|
|
183
292
|
}
|
package/lib/hf-model.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { spawn } from 'node:child_process';
|
|
10
|
+
import { existsSync } from 'node:fs';
|
|
10
11
|
import { loadConfig } from './config.js';
|
|
11
12
|
|
|
12
13
|
const HF_API = 'https://huggingface.co/api';
|
|
@@ -123,68 +124,43 @@ export async function fetchHfModelInfo(repoId, token) {
|
|
|
123
124
|
};
|
|
124
125
|
}
|
|
125
126
|
|
|
126
|
-
/**
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
p.on('error', () => resolve(false));
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/** Check whether Python huggingface_hub is importable. */
|
|
136
|
-
async function hfHubAvailable() {
|
|
137
|
-
return new Promise(resolve => {
|
|
138
|
-
const p = spawn('python3', ['-c', 'import huggingface_hub'], { stdio: 'ignore' });
|
|
139
|
-
p.on('exit', code => resolve(code === 0));
|
|
140
|
-
p.on('error', () => resolve(false));
|
|
141
|
-
});
|
|
127
|
+
/** The `hf` CLI the installer puts in the vLLM venv. One place, no fallbacks. */
|
|
128
|
+
function hfCli() {
|
|
129
|
+
const home = process.env.HOME || process.env.USERPROFILE || '';
|
|
130
|
+
const base = process.env.INFERNET_HOME || `${home}/.infernet`;
|
|
131
|
+
return `${base}/vllm-venv/bin/hf`;
|
|
142
132
|
}
|
|
143
133
|
|
|
144
134
|
/**
|
|
145
|
-
* Download a model
|
|
146
|
-
*
|
|
147
|
-
* Returns the local cache path.
|
|
135
|
+
* Download a HuggingFace model via `hf download`. Streams progress and returns
|
|
136
|
+
* the local cache path; surfaces the CLI's stderr on failure.
|
|
148
137
|
*/
|
|
149
138
|
export async function downloadHfModel(repoId, token, { localDir = null } = {}) {
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (!hasHub) {
|
|
139
|
+
const cli = hfCli();
|
|
140
|
+
if (!existsSync(cli)) {
|
|
154
141
|
throw new Error(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
' (or: pip3 install -U huggingface_hub)'
|
|
142
|
+
`HuggingFace CLI not found at ${cli}. Re-run the installer on this NVIDIA ` +
|
|
143
|
+
`host — it installs vLLM + huggingface_hub.`
|
|
158
144
|
);
|
|
159
145
|
}
|
|
160
146
|
|
|
161
|
-
// Build the download command
|
|
162
147
|
const args = ['download', repoId];
|
|
163
148
|
if (token) args.push('--token', token);
|
|
164
149
|
if (localDir) args.push('--local-dir', localDir);
|
|
165
150
|
|
|
166
|
-
|
|
167
|
-
const cmd = hasCli ? 'huggingface-cli' : 'python3';
|
|
168
|
-
const finalArgs = hasCli ? args : [
|
|
169
|
-
'-c',
|
|
170
|
-
`from huggingface_hub import snapshot_download; ` +
|
|
171
|
-
`p = snapshot_download(${JSON.stringify(repoId)}` +
|
|
172
|
-
`${token ? `, token=${JSON.stringify(token)}` : ''}` +
|
|
173
|
-
`${localDir ? `, local_dir=${JSON.stringify(localDir)}` : ''}` +
|
|
174
|
-
`); print(p)`
|
|
175
|
-
];
|
|
176
|
-
|
|
177
|
-
process.stdout.write(`\nDownloading ${repoId} from HuggingFace...\n`);
|
|
178
|
-
if (!hasCli) {
|
|
179
|
-
process.stdout.write('(tip: pip install huggingface_hub for a nicer progress bar)\n\n');
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
const child = spawn(cmd, finalArgs, { stdio: ['ignore', 'inherit', 'inherit'] });
|
|
183
|
-
let localPath = localDir;
|
|
151
|
+
process.stdout.write(`\nDownloading ${repoId} from HuggingFace…\n`);
|
|
184
152
|
|
|
153
|
+
return new Promise((resolve, reject) => {
|
|
154
|
+
const child = spawn(cli, args, { stdio: ['ignore', 'inherit', 'pipe'] });
|
|
155
|
+
let errTail = '';
|
|
156
|
+
child.stderr.on('data', (d) => {
|
|
157
|
+
const s = d.toString();
|
|
158
|
+
process.stderr.write(s);
|
|
159
|
+
errTail = (errTail + s).slice(-1200);
|
|
160
|
+
});
|
|
185
161
|
child.on('exit', code => {
|
|
186
|
-
if (code === 0) resolve(
|
|
187
|
-
else reject(new Error(`
|
|
162
|
+
if (code === 0) resolve(localDir ?? defaultCachePath(repoId));
|
|
163
|
+
else reject(new Error(`hf download exited ${code}${errTail.trim() ? `:\n${errTail.trim()}` : ''}`));
|
|
188
164
|
});
|
|
189
165
|
child.on('error', reject);
|
|
190
166
|
});
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared `~/.infernet/inference/state.json` reader/writer. The file
|
|
3
|
+
* is the single source of truth between `infernet inference …`
|
|
4
|
+
* subcommands and the long-running daemon (which reads it on every
|
|
5
|
+
* heartbeat to assemble specs.rpc / specs.rpc_primary).
|
|
6
|
+
*
|
|
7
|
+
* Shape (all fields optional — the daemon assembles only what's
|
|
8
|
+
* present):
|
|
9
|
+
*
|
|
10
|
+
* {
|
|
11
|
+
* "rpc_slice": { models: [...], host, port, pid, started_at,
|
|
12
|
+
* binary, gpu, vram_gb, ram_gb, max_concurrent },
|
|
13
|
+
* "rpc_primary": { models: [...], gguf_paths: { <model_id>: <path> },
|
|
14
|
+
* version }
|
|
15
|
+
* }
|
|
16
|
+
*
|
|
17
|
+
* Atomic writes via temp-file rename so concurrent subcommands +
|
|
18
|
+
* daemon heartbeats can't see a torn file. Reads tolerate a missing
|
|
19
|
+
* file (returns {}).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import fs from 'node:fs';
|
|
23
|
+
import fsp from 'node:fs/promises';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
|
|
26
|
+
const STATE_DIR = path.join(process.env.HOME ?? '/tmp', '.infernet', 'inference');
|
|
27
|
+
const STATE_FILE = path.join(STATE_DIR, 'state.json');
|
|
28
|
+
|
|
29
|
+
export function inferenceStateDir() { return STATE_DIR; }
|
|
30
|
+
export function inferenceStateFile() { return STATE_FILE; }
|
|
31
|
+
|
|
32
|
+
export async function readInferenceState() {
|
|
33
|
+
try {
|
|
34
|
+
const raw = await fsp.readFile(STATE_FILE, 'utf8');
|
|
35
|
+
const obj = JSON.parse(raw);
|
|
36
|
+
return obj && typeof obj === 'object' ? obj : {};
|
|
37
|
+
} catch { return {}; }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function readInferenceStateSync() {
|
|
41
|
+
try {
|
|
42
|
+
const raw = fs.readFileSync(STATE_FILE, 'utf8');
|
|
43
|
+
const obj = JSON.parse(raw);
|
|
44
|
+
return obj && typeof obj === 'object' ? obj : {};
|
|
45
|
+
} catch { return {}; }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function writeInferenceState(next) {
|
|
49
|
+
await fsp.mkdir(STATE_DIR, { recursive: true });
|
|
50
|
+
const tmp = `${STATE_FILE}.${process.pid}.${Date.now()}`;
|
|
51
|
+
await fsp.writeFile(tmp, JSON.stringify(next, null, 2));
|
|
52
|
+
await fsp.rename(tmp, STATE_FILE);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Read-modify-write helper. The mutator can be sync or async; it
|
|
57
|
+
* receives the current state (always an object) and returns the
|
|
58
|
+
* next state.
|
|
59
|
+
*/
|
|
60
|
+
export async function patchInferenceState(mutator) {
|
|
61
|
+
const cur = await readInferenceState();
|
|
62
|
+
const next = await mutator({ ...cur });
|
|
63
|
+
await writeInferenceState(next);
|
|
64
|
+
return next;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* True if `pid` is a live process. `process.kill(pid, 0)` throws
|
|
69
|
+
* ESRCH for dead pids and EPERM for foreign processes — we treat
|
|
70
|
+
* EPERM as "alive but not ours" (unusual but possible after a uid
|
|
71
|
+
* change). Anything else → dead.
|
|
72
|
+
*/
|
|
73
|
+
export function isPidAlive(pid) {
|
|
74
|
+
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
75
|
+
try {
|
|
76
|
+
process.kill(pid, 0);
|
|
77
|
+
return true;
|
|
78
|
+
} catch (err) {
|
|
79
|
+
return err?.code === 'EPERM';
|
|
80
|
+
}
|
|
81
|
+
}
|
package/lib/prompt.js
CHANGED
|
@@ -17,15 +17,31 @@ import { stdin, stdout } from 'node:process';
|
|
|
17
17
|
export async function question(prompt, options = {}) {
|
|
18
18
|
const { default: defaultValue, secret = false } = options;
|
|
19
19
|
|
|
20
|
+
const suffix = defaultValue ? ` [${secret ? '****' : defaultValue}]` : '';
|
|
21
|
+
const full = `${prompt}${suffix}: `;
|
|
22
|
+
|
|
23
|
+
// Non-interactive stdin (piped `curl | sh` install, `--yes` automation,
|
|
24
|
+
// CI, nohup): there's no terminal to read an answer from. Creating a
|
|
25
|
+
// readline with terminal:true and awaiting rl.question() here NEVER
|
|
26
|
+
// resolves at EOF — the await hangs, the event loop drains, and Node
|
|
27
|
+
// exits 0 mid-function, so callers like `init` silently skip saveConfig
|
|
28
|
+
// and write no config at all. Echo the prompt + chosen default and
|
|
29
|
+
// return it immediately instead.
|
|
30
|
+
if (!stdin.isTTY) {
|
|
31
|
+
stdout.write(
|
|
32
|
+
defaultValue !== undefined
|
|
33
|
+
? `${full}${secret ? '****' : defaultValue} (default, non-interactive)\n`
|
|
34
|
+
: `${full}(non-interactive)\n`
|
|
35
|
+
);
|
|
36
|
+
return defaultValue !== undefined ? defaultValue : '';
|
|
37
|
+
}
|
|
38
|
+
|
|
20
39
|
const rl = readline.createInterface({
|
|
21
40
|
input: stdin,
|
|
22
41
|
output: stdout,
|
|
23
42
|
terminal: true
|
|
24
43
|
});
|
|
25
44
|
|
|
26
|
-
const suffix = defaultValue ? ` [${secret ? '****' : defaultValue}]` : '';
|
|
27
|
-
const full = `${prompt}${suffix}: `;
|
|
28
|
-
|
|
29
45
|
try {
|
|
30
46
|
let answer;
|
|
31
47
|
if (secret) {
|
package/lib/recommender.js
CHANGED
|
@@ -41,6 +41,7 @@ export const CATALOG = [
|
|
|
41
41
|
{ id: "qwen2.5-coder-14b",name:"Qwen2.5 Coder 14B", paramsB: 14, vramMin: 12, ramMin: 32, pullName: "qwen2.5-coder:14b", backend: "ollama", quality: 9, speed: 5, useCases: ["coding","agents"] },
|
|
42
42
|
|
|
43
43
|
// ── 32B (24GB+ VRAM, RTX 4090 / A100 sweet spot) ──
|
|
44
|
+
{ id: "qwen3.5-9b-heretic", name: "Qwen3.5 9B Heretic (uncensored)", paramsB: 9, vramMin: 24, ramMin: 32, pullName: "hf:DavidAU/Qwen3.5-9B-Claude-4.6-HighIQ-THINKING-HERETIC-UNCENSORED", backend: "vllm", quality: 8, speed: 6, useCases: ["chat","uncensored","vision","agents","study"], uncensored: true },
|
|
44
45
|
{ id: "qwen2.5-32b", name: "Qwen2.5 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5:32b", backend: "ollama", quality: 9, speed: 4, useCases: ["chat","coding","study","agents"], notes: "Single-80GB-GPU sweet spot." },
|
|
45
46
|
{ id: "qwen2.5-coder-32b",name:"Qwen2.5 Coder 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5-coder:32b", backend: "ollama", quality: 10, speed: 4, useCases: ["coding","agents"] },
|
|
46
47
|
|
|
@@ -49,7 +50,10 @@ export const CATALOG = [
|
|
|
49
50
|
{ id: "qwen2.5-72b", name: "Qwen2.5 72B", paramsB: 72, vramMin: 40, ramMin: 64, pullName: "qwen2.5:72b", backend: "ollama", quality: 10, speed: 3, useCases: ["chat","coding","study","agents"] },
|
|
50
51
|
|
|
51
52
|
// ── MoE (high VRAM, vLLM only) ──
|
|
52
|
-
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true }
|
|
53
|
+
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true },
|
|
54
|
+
|
|
55
|
+
// ── flagship / cluster-tier (multi-GPU or cloud; won't fit a single card) ──
|
|
56
|
+
{ id: "glm-5.2-uncensored", name: "GLM-5.2 754B FP8 (uncensored)", paramsB: 754, vramMin: 768, ramMin: 768, pullName: "hf:zandenAI/GLM-5.2-FP8-Uncensored", backend: "vllm", quality: 10, speed: 2, useCases: ["chat","uncensored","cybersecurity","agents"], uncensored: true, gated: true }
|
|
53
57
|
];
|
|
54
58
|
|
|
55
59
|
/**
|