@infernetprotocol/cli 0.1.22 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/deploy.js +117 -5
- package/commands/model.js +250 -8
- package/commands/setup.js +44 -7
- package/commands/start.js +2 -5
- package/lib/hf-model.js +54 -2
- package/lib/model-fit.js +49 -5
- package/lib/recommender.js +111 -0
- package/lib/version.js +1 -1
- package/package.json +8 -8
package/commands/deploy.js
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
|
|
22
22
|
import { loadConfig, saveConfig } from "../lib/config.js";
|
|
23
23
|
import { question } from "../lib/prompt.js";
|
|
24
|
-
import { getAdapter, adapters, PROVIDER_KEY_URLS } from "@infernetprotocol/deploy-providers";
|
|
24
|
+
import { getAdapter, adapters, providers, PROVIDER_KEY_URLS } from "@infernetprotocol/deploy-providers";
|
|
25
25
|
import { canonicalize } from "@infernetprotocol/deploy-providers/gpu-normalize";
|
|
26
26
|
import { rankOffers, costBlock, isValidPreset } from "@infernetprotocol/deploy-providers/pricing";
|
|
27
27
|
import { formatFitWarning } from "@infernetprotocol/deploy-providers/model-fit";
|
|
@@ -412,20 +412,132 @@ async function presetSubcommand(presetName, args) {
|
|
|
412
412
|
}
|
|
413
413
|
}
|
|
414
414
|
|
|
415
|
-
//
|
|
415
|
+
// vLLM mode: route through the new provider class so the cloud-init /
|
|
416
|
+
// dockerArgs / health-check pipeline runs end-to-end (IPIP-0019 §9).
|
|
417
|
+
const engine = args.get("engine");
|
|
418
|
+
if (engine === "vllm") {
|
|
419
|
+
return deployWithProviderClass(winner, args);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// Default (Ollama): hand off to the legacy per-provider up-flow.
|
|
416
423
|
const provArgs = makeProviderArgsFromOffer(winner, args);
|
|
417
424
|
return upSubcommand(provArgs);
|
|
418
425
|
}
|
|
419
426
|
|
|
427
|
+
async function deployWithProviderClass(offer, args) {
|
|
428
|
+
const Cls = providers?.[offer.providerId];
|
|
429
|
+
if (!Cls) {
|
|
430
|
+
process.stderr.write(
|
|
431
|
+
`vLLM deploy not yet wired for provider="${offer.providerId}".\n` +
|
|
432
|
+
`Try a preset that picks runpod/tensordock/lambda/vast/digitalocean.\n`
|
|
433
|
+
);
|
|
434
|
+
return 1;
|
|
435
|
+
}
|
|
436
|
+
const apiKey = await resolveApiKey(offer.providerId);
|
|
437
|
+
if (!apiKey) {
|
|
438
|
+
process.stderr.write(`No API key configured for ${offer.providerId}.\n`);
|
|
439
|
+
return 1;
|
|
440
|
+
}
|
|
441
|
+
const provider = new Cls({ apiKey, providerId: offer.providerId });
|
|
442
|
+
|
|
443
|
+
const model = args.get("model");
|
|
444
|
+
if (!model) {
|
|
445
|
+
process.stderr.write(`--model <hf-id> is required when --engine vllm.\n`);
|
|
446
|
+
return 1;
|
|
447
|
+
}
|
|
448
|
+
const vllm = {
|
|
449
|
+
model,
|
|
450
|
+
port: args.get("port") ? Number.parseInt(args.get("port"), 10) : 8000,
|
|
451
|
+
servedModelName: args.get("served-model-name") ?? undefined,
|
|
452
|
+
maxModelLen: args.get("max-model-len") ? Number.parseInt(args.get("max-model-len"), 10) : undefined,
|
|
453
|
+
tensorParallelSize: args.get("tensor-parallel-size") ? Number.parseInt(args.get("tensor-parallel-size"), 10) : undefined,
|
|
454
|
+
gpuMemoryUtilization: args.get("gpu-memory-utilization") ? Number.parseFloat(args.get("gpu-memory-utilization")) : undefined,
|
|
455
|
+
quantization: args.get("quantization") ?? undefined,
|
|
456
|
+
dtype: args.get("dtype") ?? undefined
|
|
457
|
+
};
|
|
458
|
+
|
|
459
|
+
process.stdout.write(`Creating ${offer.providerId} node (engine=vllm, model=${model})…\n`);
|
|
460
|
+
let node;
|
|
461
|
+
try {
|
|
462
|
+
node = await provider.createNode({
|
|
463
|
+
offerId: offer.offerId,
|
|
464
|
+
gpu: offer.gpu?.name,
|
|
465
|
+
gpuCount: offer.gpu?.count ?? 1,
|
|
466
|
+
vramGb: offer.gpu?.vramGb,
|
|
467
|
+
region: offer.region,
|
|
468
|
+
hourlyPrice: offer.pricePerHour,
|
|
469
|
+
engine: "vllm",
|
|
470
|
+
model,
|
|
471
|
+
vllm
|
|
472
|
+
});
|
|
473
|
+
} catch (err) {
|
|
474
|
+
process.stderr.write(`createNode failed: ${err?.message ?? err}\n`);
|
|
475
|
+
return 1;
|
|
476
|
+
}
|
|
477
|
+
process.stdout.write(` local id: ${node.id}\n provider id: ${node.providerNodeId}\n`);
|
|
478
|
+
|
|
479
|
+
process.stdout.write("Waiting for node to come online…\n");
|
|
480
|
+
try {
|
|
481
|
+
node = await provider.waitUntilReady(node);
|
|
482
|
+
} catch (err) {
|
|
483
|
+
process.stderr.write(
|
|
484
|
+
`\nNode was created but did not become ready.\n` +
|
|
485
|
+
`Provider: ${offer.providerId}\n` +
|
|
486
|
+
`Provider ID: ${node.providerNodeId}\n` +
|
|
487
|
+
`Local ID: ${node.id}\n\n` +
|
|
488
|
+
`To clean up:\n infernet deploy destroy ${node.id}\n\n` +
|
|
489
|
+
`(${err?.message ?? err})\n`
|
|
490
|
+
);
|
|
491
|
+
return 1;
|
|
492
|
+
}
|
|
493
|
+
process.stdout.write(` endpoint: ${node.endpointUrl}\n`);
|
|
494
|
+
|
|
495
|
+
process.stdout.write("Bootstrapping vLLM (this can take 5+ minutes — model download + first compile)…\n");
|
|
496
|
+
const health = await provider.bootstrapNode(node, { engine: "vllm", vllm });
|
|
497
|
+
if (!health.ok) {
|
|
498
|
+
process.stderr.write(
|
|
499
|
+
`\nvLLM health-check failed.\n` +
|
|
500
|
+
`Provider: ${offer.providerId}\n` +
|
|
501
|
+
`Provider ID: ${node.providerNodeId}\n` +
|
|
502
|
+
`Local ID: ${node.id}\n` +
|
|
503
|
+
`Reason: ${health.reason ?? health.lastErr}\n\n` +
|
|
504
|
+
`To clean up:\n infernet deploy destroy ${node.id}\n` +
|
|
505
|
+
`To debug:\n infernet deploy logs ${node.id}\n`
|
|
506
|
+
);
|
|
507
|
+
return 1;
|
|
508
|
+
}
|
|
509
|
+
process.stdout.write(`✓ vLLM ready at ${health.healthEndpoint}\n`);
|
|
510
|
+
return 0;
|
|
511
|
+
}
|
|
512
|
+
|
|
420
513
|
async function collectOffersAcrossProviders({ canonicalGpu, maxPrice, request }) {
|
|
421
|
-
const config = (await loadConfig()) ?? {};
|
|
422
514
|
const offers = [];
|
|
423
|
-
|
|
515
|
+
// Union of providers with either a class adapter (IPIP-0019) or a
|
|
516
|
+
// legacy module adapter. The class wins when both exist.
|
|
517
|
+
const allProviders = new Set([...Object.keys(providers ?? {}), ...PROVIDERS]);
|
|
518
|
+
|
|
519
|
+
for (const provider of allProviders) {
|
|
424
520
|
const apiKey = await resolveApiKey(provider);
|
|
425
521
|
if (!apiKey) continue; // skip unconfigured
|
|
522
|
+
|
|
523
|
+
// Prefer the new DeployProvider class (uniform Offer[] shape).
|
|
524
|
+
const Cls = providers?.[provider];
|
|
525
|
+
if (Cls) {
|
|
526
|
+
try {
|
|
527
|
+
const inst = new Cls({ apiKey, providerId: provider });
|
|
528
|
+
const found = await inst.findOffers(request);
|
|
529
|
+
for (const o of found ?? []) offers.push(o);
|
|
530
|
+
continue;
|
|
531
|
+
} catch (err) {
|
|
532
|
+
process.stderr.write(` (${provider}: ${err?.message ?? err})\n`);
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// Fall back to the legacy module adapter shape.
|
|
426
538
|
const adapter = adapters[provider];
|
|
427
539
|
const fn = adapter?.findOffers ?? adapter?.searchOffers;
|
|
428
|
-
if (typeof fn !== "function") continue;
|
|
540
|
+
if (typeof fn !== "function") continue;
|
|
429
541
|
try {
|
|
430
542
|
const found = await fn({ apiKey, ...request });
|
|
431
543
|
for (const o of found ?? []) {
|
package/commands/model.js
CHANGED
|
@@ -19,17 +19,26 @@
|
|
|
19
19
|
|
|
20
20
|
import { spawn } from "node:child_process";
|
|
21
21
|
import { loadConfig, saveConfig, getConfigPath } from "../lib/config.js";
|
|
22
|
-
import { checkModelFits } from "../lib/model-fit.js";
|
|
22
|
+
import { checkModelFits, detectCapacity } from "../lib/model-fit.js";
|
|
23
23
|
import {
|
|
24
24
|
resolveHfToken,
|
|
25
25
|
fetchHfModelInfo,
|
|
26
26
|
downloadHfModel,
|
|
27
27
|
} from "../lib/hf-model.js";
|
|
28
|
+
import { recommendModels } from "../lib/recommender.js";
|
|
28
29
|
|
|
29
30
|
const HELP = `infernet model — manage models served by this node
|
|
30
31
|
|
|
31
32
|
Usage:
|
|
32
33
|
infernet model list List models pulled locally.
|
|
34
|
+
infernet model recommend [flags] Recommend models for your hardware.
|
|
35
|
+
--use-case <chat|coding|study|agents|uncensored|vision>
|
|
36
|
+
--uncensored
|
|
37
|
+
--install install the top pick
|
|
38
|
+
--install-all install everything that fits on disk
|
|
39
|
+
infernet model info hf:<org>/<repo> Inspect a HuggingFace model — size,
|
|
40
|
+
architecture, VRAM estimates — without
|
|
41
|
+
downloading anything.
|
|
33
42
|
infernet model pull <name> Pull an Ollama model (e.g. qwen2.5:7b).
|
|
34
43
|
infernet model pull hf:<org>/<repo> Download a HuggingFace model (vLLM/SGLang).
|
|
35
44
|
infernet model remove <name> Delete a pulled model.
|
|
@@ -37,6 +46,7 @@ Usage:
|
|
|
37
46
|
infernet model show Show the engine default + Ollama host.
|
|
38
47
|
|
|
39
48
|
Examples:
|
|
49
|
+
infernet model info hf:Qwen/Qwen2.5-32B-Instruct
|
|
40
50
|
infernet model pull qwen2.5:7b
|
|
41
51
|
infernet model pull hf:NousResearch/Hermes-3-Llama-3.1-8B
|
|
42
52
|
infernet model use qwen2.5:7b
|
|
@@ -148,6 +158,203 @@ async function cmdList(host) {
|
|
|
148
158
|
return 0;
|
|
149
159
|
}
|
|
150
160
|
|
|
161
|
+
async function cmdRecommend(args) {
|
|
162
|
+
const useCase = args.get("use-case") ?? args.get("for") ?? null;
|
|
163
|
+
const uncensoredOnly = args.has("uncensored");
|
|
164
|
+
const limit = Number.parseInt(args.get("limit") ?? "5", 10);
|
|
165
|
+
const wantInstall = args.has("install") || args.has("i");
|
|
166
|
+
const wantInstallAll = args.has("install-all");
|
|
167
|
+
|
|
168
|
+
const cap = await detectCapacity();
|
|
169
|
+
const vramGb = cap.vram_gb;
|
|
170
|
+
const ramGb = cap.ram_gb;
|
|
171
|
+
|
|
172
|
+
process.stdout.write(`Detected: ${vramGb.toFixed(1)} GB VRAM, ${ramGb.toFixed(1)} GB RAM`);
|
|
173
|
+
process.stdout.write(useCase ? ` (use case: ${useCase})` : "");
|
|
174
|
+
process.stdout.write(uncensoredOnly ? ` (uncensored only)` : "");
|
|
175
|
+
process.stdout.write("\n");
|
|
176
|
+
|
|
177
|
+
const recs = recommendModels({ vramGb, ramGb, useCase, uncensoredOnly, limit });
|
|
178
|
+
if (recs.length === 0) {
|
|
179
|
+
process.stdout.write("No matching models in the catalog. Try without --use-case / --uncensored.\n");
|
|
180
|
+
return 0;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
process.stdout.write(`\nTop ${recs.length} for your hardware:\n\n`);
|
|
184
|
+
recs.forEach((rec, i) => {
|
|
185
|
+
const m = rec.model;
|
|
186
|
+
const fits = rec.fits ? "✓" : "⚠ tight on VRAM";
|
|
187
|
+
const sizeGb = m.paramsB * 0.6; // rough Q4 disk size for catalog estimate
|
|
188
|
+
process.stdout.write(` ${i + 1}. ${m.name} (${m.paramsB}B, ~${sizeGb.toFixed(1)} GB Q4 disk, needs ≥${m.vramMin} GB VRAM) ${fits}\n`);
|
|
189
|
+
if (m.notes) process.stdout.write(` ${m.notes}\n`);
|
|
190
|
+
process.stdout.write(` pull: infernet model pull ${m.pullName}\n\n`);
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
if (!wantInstall && !wantInstallAll) {
|
|
194
|
+
process.stdout.write(`To install the top pick: infernet model recommend --install\n`);
|
|
195
|
+
process.stdout.write(`To install everything that fits: infernet model recommend --install-all\n\n`);
|
|
196
|
+
return 0;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// ---- auto-install path ----
|
|
200
|
+
const fitsOnHardware = recs.filter((r) => r.fits);
|
|
201
|
+
if (fitsOnHardware.length === 0) {
|
|
202
|
+
process.stderr.write("No recommended models actually fit your hardware — refusing to install.\n");
|
|
203
|
+
return 1;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
let toInstall;
|
|
207
|
+
if (wantInstallAll) {
|
|
208
|
+
const diskFreeGb = await detectFreeDiskGb();
|
|
209
|
+
const RESERVE_GB = 10; // leave headroom
|
|
210
|
+
const budget = diskFreeGb - RESERVE_GB;
|
|
211
|
+
process.stdout.write(`Disk free: ${diskFreeGb.toFixed(1)} GB (reserving ${RESERVE_GB} GB headroom → ${budget.toFixed(1)} GB budget)\n`);
|
|
212
|
+
toInstall = [];
|
|
213
|
+
let used = 0;
|
|
214
|
+
for (const r of fitsOnHardware) {
|
|
215
|
+
const sz = r.model.paramsB * 0.6;
|
|
216
|
+
if (used + sz > budget) {
|
|
217
|
+
process.stdout.write(` skipping ${r.model.name} — would exceed disk budget\n`);
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
toInstall.push(r);
|
|
221
|
+
used += sz;
|
|
222
|
+
}
|
|
223
|
+
if (toInstall.length === 0) {
|
|
224
|
+
process.stderr.write(`Disk budget too small for any recommended model.\n`);
|
|
225
|
+
return 1;
|
|
226
|
+
}
|
|
227
|
+
process.stdout.write(`Will install ${toInstall.length} model(s), ~${used.toFixed(1)} GB total.\n\n`);
|
|
228
|
+
} else {
|
|
229
|
+
toInstall = [fitsOnHardware[0]];
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
let lastInstalled = null;
|
|
233
|
+
for (const r of toInstall) {
|
|
234
|
+
process.stdout.write(`\n→ Pulling ${r.model.name} (${r.model.pullName})…\n\n`);
|
|
235
|
+
const code = r.model.pullName.startsWith("hf:")
|
|
236
|
+
? await cmdHfPull(r.model.pullName.slice(3))
|
|
237
|
+
: await cmdPull(host_(), r.model.pullName, { force: true });
|
|
238
|
+
if (code === 0) {
|
|
239
|
+
lastInstalled = r.model;
|
|
240
|
+
} else {
|
|
241
|
+
process.stderr.write(`pull failed for ${r.model.pullName} (exit ${code}); continuing\n`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Set the most recently-installed as the active model so chat works.
|
|
246
|
+
if (lastInstalled) {
|
|
247
|
+
const cfg = (await loadConfig()) ?? {};
|
|
248
|
+
const updated = { ...cfg, engine: { ...(cfg.engine ?? {}), model: lastInstalled.pullName, backend: lastInstalled.backend } };
|
|
249
|
+
await saveConfig(updated);
|
|
250
|
+
process.stdout.write(`\n✓ engine.model = ${lastInstalled.pullName} (${lastInstalled.backend})\n`);
|
|
251
|
+
process.stdout.write(`Try it: infernet chat "hello"\n\n`);
|
|
252
|
+
}
|
|
253
|
+
return 0;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Re-read config to grab the resolved Ollama host. cmdPull needs it. */
|
|
257
|
+
function host_() {
|
|
258
|
+
return process.env.OLLAMA_HOST ?? DEFAULT_HOST;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Best-effort free-disk lookup (GB) on the partition holding $HOME. */
|
|
262
|
+
async function detectFreeDiskGb() {
|
|
263
|
+
try {
|
|
264
|
+
const { execFile } = await import("node:child_process");
|
|
265
|
+
const { promisify } = await import("node:util");
|
|
266
|
+
const pExec = promisify(execFile);
|
|
267
|
+
const { stdout } = await pExec("df", ["-Pk", process.env.HOME ?? "/"], { timeout: 3000 });
|
|
268
|
+
const parts = stdout.trim().split("\n").pop().split(/\s+/);
|
|
269
|
+
const freeKb = Number.parseInt(parts[3], 10);
|
|
270
|
+
return Number.isFinite(freeKb) ? freeKb / 1024 / 1024 : 0;
|
|
271
|
+
} catch {
|
|
272
|
+
return 0;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
async function cmdInfo(name) {
|
|
277
|
+
if (!name) {
|
|
278
|
+
process.stderr.write("error: info requires a model name (e.g. hf:Qwen/Qwen2.5-32B-Instruct)\n");
|
|
279
|
+
return 2;
|
|
280
|
+
}
|
|
281
|
+
const repoId = name.startsWith('hf:') ? name.slice(3) : (name.includes('/') ? name : null);
|
|
282
|
+
if (!repoId) {
|
|
283
|
+
process.stderr.write(
|
|
284
|
+
`info works on HuggingFace models only. Try:\n` +
|
|
285
|
+
` infernet model info hf:Qwen/Qwen2.5-32B-Instruct\n` +
|
|
286
|
+
` infernet model info NousResearch/Hermes-3-Llama-3.1-8B\n`
|
|
287
|
+
);
|
|
288
|
+
return 2;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const token = await resolveHfToken();
|
|
292
|
+
process.stdout.write(`Fetching ${repoId} from HuggingFace…\n`);
|
|
293
|
+
let info;
|
|
294
|
+
try {
|
|
295
|
+
info = await fetchHfModelInfo(repoId, token);
|
|
296
|
+
} catch (err) {
|
|
297
|
+
process.stderr.write(`error: ${err?.message ?? err}\n`);
|
|
298
|
+
return 1;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const fmt = (v, suffix = "") => v == null ? "?" : `${v}${suffix}`;
|
|
302
|
+
process.stdout.write(`\n${info.repoId}\n`);
|
|
303
|
+
process.stdout.write(`${'─'.repeat(Math.min(70, info.repoId.length))}\n`);
|
|
304
|
+
process.stdout.write(` pipeline: ${info.pipeline}\n`);
|
|
305
|
+
process.stdout.write(` architecture: ${info.architectures?.join(", ") ?? info.modelType ?? "?"}\n`);
|
|
306
|
+
process.stdout.write(` model_type: ${fmt(info.modelType)}\n`);
|
|
307
|
+
if (info.isMoe) {
|
|
308
|
+
process.stdout.write(` MoE: ${fmt(info.numExperts)} experts, ${fmt(info.expertsPerTok)} active per token\n`);
|
|
309
|
+
}
|
|
310
|
+
process.stdout.write(` layers: ${fmt(info.numLayers)}\n`);
|
|
311
|
+
process.stdout.write(` hidden_size: ${fmt(info.hiddenSize)}\n`);
|
|
312
|
+
if (info.numHeads) {
|
|
313
|
+
const gqa = info.numKvHeads && info.numKvHeads !== info.numHeads ? ` (GQA, ${info.numKvHeads} kv)` : "";
|
|
314
|
+
process.stdout.write(` attention: ${info.numHeads} heads${gqa}\n`);
|
|
315
|
+
}
|
|
316
|
+
process.stdout.write(` context length: ${info.contextLen ? info.contextLen.toLocaleString() : "?"}\n`);
|
|
317
|
+
process.stdout.write(` vocab: ${info.vocabSize ? info.vocabSize.toLocaleString() : "?"}\n`);
|
|
318
|
+
process.stdout.write(` dtype: ${fmt(info.torchDtype)}\n`);
|
|
319
|
+
if (info.isQuantized) {
|
|
320
|
+
process.stdout.write(` quantized: yes (${info.quantMethod ?? "unknown method"})\n`);
|
|
321
|
+
}
|
|
322
|
+
process.stdout.write(`\n total weights: ${fmt(info.sizeGb, " GB")} (${fmt(info.shardCount)} shards)\n`);
|
|
323
|
+
process.stdout.write(` param count: ~${fmt(info.paramsB, "B")}\n`);
|
|
324
|
+
|
|
325
|
+
if (info.vramEstimateGb) {
|
|
326
|
+
process.stdout.write(`\n VRAM estimate (weights + ~15% overhead, no KV cache):\n`);
|
|
327
|
+
process.stdout.write(` bf16/fp16: ~${info.vramEstimateGb.fp16} GB\n`);
|
|
328
|
+
process.stdout.write(` int8: ~${info.vramEstimateGb.int8} GB\n`);
|
|
329
|
+
process.stdout.write(` int4 (AWQ/GPTQ): ~${info.vramEstimateGb.int4} GB\n`);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
process.stdout.write(`\n recommended backend: ${info.recommendedBackend === 'vllm_or_sglang' ? 'vLLM or SGLang (Ollama not supported)' : 'Ollama or vLLM'}\n`);
|
|
333
|
+
process.stdout.write(` license: ${fmt(info.license)}\n`);
|
|
334
|
+
if (info.downloads) process.stdout.write(` downloads (30d): ${info.downloads.toLocaleString()}\n`);
|
|
335
|
+
|
|
336
|
+
// Single-GPU verdict for common SKUs.
|
|
337
|
+
if (info.vramEstimateGb) {
|
|
338
|
+
process.stdout.write(`\n Fits a single GPU?\n`);
|
|
339
|
+
const skus = [
|
|
340
|
+
{ name: "RTX 4090 24GB", vram: 24 },
|
|
341
|
+
{ name: "L40S / A6000 48GB", vram: 48 },
|
|
342
|
+
{ name: "A100 80GB / H100 80GB", vram: 80 },
|
|
343
|
+
{ name: "H200 141GB", vram: 141 }
|
|
344
|
+
];
|
|
345
|
+
for (const sku of skus) {
|
|
346
|
+
const fp16 = info.vramEstimateGb.fp16 <= sku.vram ? "✓ fp16" : "";
|
|
347
|
+
const int8 = info.vramEstimateGb.int8 <= sku.vram ? (fp16 ? "" : "✓ int8") : "";
|
|
348
|
+
const int4 = info.vramEstimateGb.int4 <= sku.vram ? (fp16 || int8 ? "" : "✓ int4") : "";
|
|
349
|
+
const verdict = fp16 || int8 || int4 || "✗ requires multi-GPU";
|
|
350
|
+
process.stdout.write(` ${sku.name.padEnd(24)} ${verdict}\n`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
process.stdout.write(`\n To download: infernet model pull hf:${info.repoId}\n\n`);
|
|
355
|
+
return 0;
|
|
356
|
+
}
|
|
357
|
+
|
|
151
358
|
async function cmdHfPull(repoId, opts = {}) {
|
|
152
359
|
const token = await resolveHfToken();
|
|
153
360
|
|
|
@@ -203,6 +410,20 @@ async function cmdPull(host, name, opts = {}) {
|
|
|
203
410
|
return cmdHfPull(name.slice(3), opts);
|
|
204
411
|
}
|
|
205
412
|
|
|
413
|
+
// Looks like "Org/Repo" — almost certainly a HuggingFace name. Ollama
|
|
414
|
+
// would just 404 with "pull model manifest: file does not exist", so
|
|
415
|
+
// catch it here and point the operator at the right verb.
|
|
416
|
+
if (name.includes('/') && !name.startsWith('hf:')) {
|
|
417
|
+
process.stderr.write(
|
|
418
|
+
`error: "${name}" looks like a HuggingFace repo, not an Ollama model.\n` +
|
|
419
|
+
`\nOllama can only pull from ollama.com. For HuggingFace, use the hf: prefix:\n` +
|
|
420
|
+
` infernet model pull hf:${name}\n` +
|
|
421
|
+
`\nFor Ollama models, browse https://ollama.com/library — names look like\n` +
|
|
422
|
+
`qwen2.5:7b, llama3.1:8b, dolphin3:8b (no slash, with optional tag).\n`
|
|
423
|
+
);
|
|
424
|
+
return 1;
|
|
425
|
+
}
|
|
426
|
+
|
|
206
427
|
// Quick reachability check before spawning ollama, so we get a friendly
|
|
207
428
|
// error rather than a confusing CLI-not-found if Ollama isn't installed.
|
|
208
429
|
await fetchTags(host);
|
|
@@ -226,13 +447,28 @@ async function cmdPull(host, name, opts = {}) {
|
|
|
226
447
|
try {
|
|
227
448
|
await streamPull(name);
|
|
228
449
|
} catch (err) {
|
|
229
|
-
|
|
230
|
-
process.stderr.write(
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
450
|
+
const msg = err?.message ?? String(err);
|
|
451
|
+
process.stderr.write(`error: ${msg}\n`);
|
|
452
|
+
// "pull model manifest: file does not exist" is Ollama-speak for
|
|
453
|
+
// "registry returned 404 for that name+tag". Surface that clearly.
|
|
454
|
+
if (/manifest.*does not exist|manifest.*not found/i.test(msg)) {
|
|
455
|
+
process.stderr.write(
|
|
456
|
+
`\nThat model + tag combination doesn't exist in the Ollama registry.\n` +
|
|
457
|
+
`Browse exact names at https://ollama.com/library\n` +
|
|
458
|
+
`\nCommon gotchas:\n` +
|
|
459
|
+
` - qwen2.5 only has :0.5b :1.5b :3b :7b :14b :32b :72b (no :8b)\n` +
|
|
460
|
+
` - llama3.1 has :8b and :70b (no :7b)\n` +
|
|
461
|
+
` - dolphin3 has :2b and :8b (no :7b)\n` +
|
|
462
|
+
` - HuggingFace models need the hf: prefix: \`infernet model pull hf:org/repo\`\n`
|
|
463
|
+
);
|
|
464
|
+
} else {
|
|
465
|
+
process.stderr.write(
|
|
466
|
+
`\nNothing pulled. Common causes:\n` +
|
|
467
|
+
` - typo in the model spec (try \`ollama list\` on the registry: https://ollama.com/library)\n` +
|
|
468
|
+
` - missing tag — try the bare model name (e.g. qwen2.5 → qwen2.5:latest)\n` +
|
|
469
|
+
` - private / gated model — pull manually with \`ollama pull ${name}\` to see the raw error\n`
|
|
470
|
+
);
|
|
471
|
+
}
|
|
236
472
|
return 1;
|
|
237
473
|
}
|
|
238
474
|
return 0;
|
|
@@ -327,6 +563,12 @@ export default async function model(args) {
|
|
|
327
563
|
case "list":
|
|
328
564
|
case "ls":
|
|
329
565
|
return await cmdList(host);
|
|
566
|
+
case "info":
|
|
567
|
+
case "inspect":
|
|
568
|
+
return await cmdInfo(arg);
|
|
569
|
+
case "recommend":
|
|
570
|
+
case "suggest":
|
|
571
|
+
return await cmdRecommend(args);
|
|
330
572
|
case "pull":
|
|
331
573
|
case "add":
|
|
332
574
|
return await cmdPull(host, arg, {
|
package/commands/setup.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import { execFile, spawn } from "node:child_process";
|
|
19
|
+
import fsSync from "node:fs";
|
|
19
20
|
import { promisify } from "node:util";
|
|
20
21
|
import { fileURLToPath } from "node:url";
|
|
21
22
|
import { dirname, join } from "node:path";
|
|
@@ -164,10 +165,21 @@ async function confirmRun({ label, command, yes }) {
|
|
|
164
165
|
async function startOllama({ yes }) {
|
|
165
166
|
// Linux: usually has the ollama systemd unit from the installer.
|
|
166
167
|
// macOS: usually opens the Ollama.app. Fall back to `ollama serve &`.
|
|
168
|
+
// Non-systemd Linux (containers, WSL, distros without systemd as
|
|
169
|
+
// PID 1): skip the systemctl noise entirely — it would just print
|
|
170
|
+
// "System has not been booted with systemd as init system" — and go
|
|
171
|
+
// straight to backgrounded `ollama serve`. Detect via the canonical
|
|
172
|
+
// /run/systemd/system probe.
|
|
167
173
|
const isMac = process.platform === "darwin";
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
174
|
+
const hasSystemd = !isMac && fsSync.existsSync("/run/systemd/system");
|
|
175
|
+
let cmd;
|
|
176
|
+
if (isMac) {
|
|
177
|
+
cmd = "open -a Ollama || (nohup ollama serve > /tmp/ollama.log 2>&1 < /dev/null &)";
|
|
178
|
+
} else if (hasSystemd) {
|
|
179
|
+
cmd = "sudo systemctl start ollama || (nohup ollama serve > /tmp/ollama.log 2>&1 < /dev/null &)";
|
|
180
|
+
} else {
|
|
181
|
+
cmd = "nohup ollama serve > /tmp/ollama.log 2>&1 < /dev/null &";
|
|
182
|
+
}
|
|
171
183
|
return confirmRun({
|
|
172
184
|
label: "Start Ollama now",
|
|
173
185
|
command: cmd,
|
|
@@ -175,6 +187,26 @@ async function startOllama({ yes }) {
|
|
|
175
187
|
});
|
|
176
188
|
}
|
|
177
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Poll an Ollama endpoint until it answers /api/tags or we time out.
|
|
192
|
+
* Used after starting Ollama in the background — the spawn returns
|
|
193
|
+
* immediately, but the daemon needs a few seconds to bind.
|
|
194
|
+
*/
|
|
195
|
+
async function waitForOllama(host, { timeoutMs = 30_000, intervalMs = 1_000 } = {}) {
|
|
196
|
+
const deadline = Date.now() + timeoutMs;
|
|
197
|
+
while (Date.now() < deadline) {
|
|
198
|
+
try {
|
|
199
|
+
const ctrl = new AbortController();
|
|
200
|
+
const t = setTimeout(() => ctrl.abort(), 2_000);
|
|
201
|
+
const res = await fetch(new URL("/api/tags", host), { signal: ctrl.signal });
|
|
202
|
+
clearTimeout(t);
|
|
203
|
+
if (res.ok) return true;
|
|
204
|
+
} catch { /* not yet listening */ }
|
|
205
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
206
|
+
}
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
|
|
178
210
|
async function installOllama({ yes }) {
|
|
179
211
|
return confirmRun({
|
|
180
212
|
label: "Install Ollama now (will sudo)",
|
|
@@ -582,13 +614,18 @@ export default async function setup(args) {
|
|
|
582
614
|
);
|
|
583
615
|
const started = await startOllama({ yes });
|
|
584
616
|
if (started) {
|
|
585
|
-
//
|
|
586
|
-
|
|
587
|
-
|
|
617
|
+
// Background `ollama serve &` returns instantly; the
|
|
618
|
+
// daemon needs a few seconds to bind on cold start.
|
|
619
|
+
// Poll for up to 30s before giving up.
|
|
620
|
+
process.stdout.write(` Waiting for Ollama to bind on ${host}...\n`);
|
|
621
|
+
const up = await waitForOllama(host, { timeoutMs: 30_000 });
|
|
622
|
+
if (up) ollamaState = await detectOllama(host, { quiet: true });
|
|
588
623
|
}
|
|
589
624
|
if (!ollamaState.running) {
|
|
590
625
|
process.stdout.write(
|
|
591
|
-
"\n Could not bring Ollama up.
|
|
626
|
+
"\n Could not bring Ollama up. Check the log:\n" +
|
|
627
|
+
" tail -50 /tmp/ollama.log\n" +
|
|
628
|
+
" Then start it manually and re-run `infernet setup`.\n"
|
|
592
629
|
);
|
|
593
630
|
return 1;
|
|
594
631
|
}
|
package/commands/start.js
CHANGED
|
@@ -46,7 +46,7 @@ import { isDaemonAlive } from '../lib/ipc.js';
|
|
|
46
46
|
import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/network.js';
|
|
47
47
|
import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
|
|
48
48
|
import { gatherCoarseSpecs } from './register.js';
|
|
49
|
-
import { checkModelFits } from '../lib/model-fit.js';
|
|
49
|
+
import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
50
50
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
51
51
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
52
52
|
import { pullLatestBinary } from './upgrade.js';
|
|
@@ -515,10 +515,7 @@ async function runDaemon(args, ctx) {
|
|
|
515
515
|
const modelName = String(args?.model);
|
|
516
516
|
const fits = await checkModelFits(modelName);
|
|
517
517
|
if (fits && !fits.ok) {
|
|
518
|
-
throw new Error(
|
|
519
|
-
`${modelName} (≈${fits.size_gb} GB) won't fit on this host ` +
|
|
520
|
-
`(${fits.mode} ceiling ≈ ${fits.ceiling_gb} GB with ${fits.have_gb} GB available)`
|
|
521
|
-
);
|
|
518
|
+
throw new Error(formatFitFailure(modelName, fits));
|
|
522
519
|
}
|
|
523
520
|
result = await ollamaSpawn(['pull', modelName]);
|
|
524
521
|
} else if (command === 'model_remove') {
|
package/lib/hf-model.js
CHANGED
|
@@ -56,14 +56,66 @@ export async function fetchHfModelInfo(repoId, token) {
|
|
|
56
56
|
? ollamaUnsupported.some(t => modelType.toLowerCase().includes(t))
|
|
57
57
|
: shards.length > 50; // large shard count → probably too big for Ollama
|
|
58
58
|
|
|
59
|
+
// The HF API often leaves `size` null on siblings — fall back to
|
|
60
|
+
// model.safetensors.index.json which carries the authoritative
|
|
61
|
+
// total_size for sharded models.
|
|
62
|
+
let trueTotalGb = sizeGb;
|
|
63
|
+
try {
|
|
64
|
+
const idxRes = await fetch(
|
|
65
|
+
`https://huggingface.co/${repoId}/resolve/main/model.safetensors.index.json`,
|
|
66
|
+
{ headers, redirect: 'follow' }
|
|
67
|
+
);
|
|
68
|
+
if (idxRes.ok) {
|
|
69
|
+
const idx = await idxRes.json();
|
|
70
|
+
const sz = idx?.metadata?.total_size;
|
|
71
|
+
if (Number.isFinite(sz) && sz > 0) {
|
|
72
|
+
trueTotalGb = +(sz / 1024 ** 3).toFixed(1);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
} catch { /* best-effort */ }
|
|
76
|
+
|
|
77
|
+
// Architecture pulls for VRAM math + the info command output.
|
|
78
|
+
const numLayers = modelConfig.num_hidden_layers ?? null;
|
|
79
|
+
const hiddenSize = modelConfig.hidden_size ?? null;
|
|
80
|
+
const numHeads = modelConfig.num_attention_heads ?? null;
|
|
81
|
+
const numKvHeads = modelConfig.num_key_value_heads ?? numHeads ?? null;
|
|
82
|
+
const expertsPerTok = modelConfig.num_experts_per_tok ?? null;
|
|
83
|
+
const contextLen = modelConfig.max_position_embeddings ?? null;
|
|
84
|
+
const vocabSize = modelConfig.vocab_size ?? null;
|
|
85
|
+
const architectures = modelConfig.architectures ?? null;
|
|
86
|
+
const isQuantized = !!modelConfig.quantization_config;
|
|
87
|
+
const quantMethod = modelConfig.quantization_config?.quant_method ?? null;
|
|
88
|
+
|
|
89
|
+
// Estimate VRAM at common precisions. Param count derived from disk
|
|
90
|
+
// size + dtype byte-width when explicit param count isn't published.
|
|
91
|
+
const dtypeBytes = (torchDtype === 'float32' || torchDtype === 'fp32') ? 4 : 2;
|
|
92
|
+
const paramsB = trueTotalGb ? +(trueTotalGb * 1024 ** 3 / dtypeBytes / 1e9).toFixed(1) : null;
|
|
93
|
+
const vramEstimateGb = paramsB ? {
|
|
94
|
+
fp16: +(paramsB * 2 * 1.15).toFixed(1), // weights + ~15% overhead
|
|
95
|
+
int8: +(paramsB * 1 * 1.15).toFixed(1),
|
|
96
|
+
int4: +(paramsB * 0.5 * 1.15).toFixed(1)
|
|
97
|
+
} : null;
|
|
98
|
+
|
|
59
99
|
return {
|
|
60
100
|
repoId,
|
|
61
101
|
modelType,
|
|
102
|
+
architectures,
|
|
62
103
|
torchDtype,
|
|
63
104
|
shardCount: shards.length,
|
|
64
|
-
sizeGb,
|
|
105
|
+
sizeGb: trueTotalGb,
|
|
106
|
+
paramsB,
|
|
107
|
+
vramEstimateGb,
|
|
108
|
+
numLayers,
|
|
109
|
+
hiddenSize,
|
|
110
|
+
numHeads,
|
|
111
|
+
numKvHeads,
|
|
112
|
+
contextLen,
|
|
113
|
+
vocabSize,
|
|
65
114
|
numExperts,
|
|
66
|
-
|
|
115
|
+
expertsPerTok,
|
|
116
|
+
isMoe: numExperts !== null || /moe|mixtral|bailing/i.test(modelType ?? ''),
|
|
117
|
+
isQuantized,
|
|
118
|
+
quantMethod,
|
|
67
119
|
recommendedBackend: needsVllmOrSglang ? 'vllm_or_sglang' : 'ollama_or_vllm',
|
|
68
120
|
pipeline: data.pipeline_tag ?? 'text-generation',
|
|
69
121
|
license: data.cardData?.license ?? data.tags?.find(t => t.startsWith('license:'))?.replace('license:', '') ?? null,
|
package/lib/model-fit.js
CHANGED
|
@@ -27,13 +27,16 @@ export const KNOWN_MODEL_SIZES = {
|
|
|
27
27
|
/** Detect available VRAM (GPU) and RAM for fit checks. */
|
|
28
28
|
export async function detectCapacity() {
|
|
29
29
|
try {
|
|
30
|
-
const { detectGpus, detectHost } = await import("@infernetprotocol/gpu");
|
|
30
|
+
const { detectGpus, detectHost, lastDetectionDiagnostics } = await import("@infernetprotocol/gpu");
|
|
31
31
|
const [gpus, host] = await Promise.all([detectGpus(), Promise.resolve(detectHost())]);
|
|
32
32
|
const vramGb = gpus.reduce((a, g) => a + (Number.isFinite(g.vram_mb) ? g.vram_mb / 1024 : 0), 0);
|
|
33
33
|
const ramGb = host.total_ram_mb / 1024;
|
|
34
|
-
|
|
34
|
+
const diagnostics = typeof lastDetectionDiagnostics === "function"
|
|
35
|
+
? lastDetectionDiagnostics()
|
|
36
|
+
: {};
|
|
37
|
+
return { vram_gb: vramGb, ram_gb: ramGb, gpu_count: gpus.length, diagnostics };
|
|
35
38
|
} catch {
|
|
36
|
-
return { vram_gb: 0, ram_gb: 0 };
|
|
39
|
+
return { vram_gb: 0, ram_gb: 0, gpu_count: 0, diagnostics: {} };
|
|
37
40
|
}
|
|
38
41
|
}
|
|
39
42
|
|
|
@@ -57,11 +60,52 @@ export function checkFits({ size_gb, vram_gb, ram_gb }) {
|
|
|
57
60
|
/**
|
|
58
61
|
* Run a capacity check for a named model. Returns null if the model
|
|
59
62
|
* size is unknown (can't check). Returns { ok, mode, have_gb,
|
|
60
|
-
* ceiling_gb, size_gb } otherwise.
|
|
63
|
+
* ceiling_gb, size_gb, gpu_count, diagnostics } otherwise.
|
|
64
|
+
*
|
|
65
|
+
* Honors INFERNET_SKIP_FIT_CHECK=1 — operators on boxes where GPU
|
|
66
|
+
* detection is broken (no nvidia-smi inside container, weird passthrough)
|
|
67
|
+
* can bypass without losing the rest of the install flow.
|
|
61
68
|
*/
|
|
62
69
|
export async function checkModelFits(name) {
|
|
63
70
|
const size_gb = KNOWN_MODEL_SIZES[name];
|
|
64
71
|
if (!Number.isFinite(size_gb)) return null;
|
|
72
|
+
if (process.env.INFERNET_SKIP_FIT_CHECK === "1") {
|
|
73
|
+
return { size_gb, ok: true, mode: "skipped", have_gb: 0, ceiling_gb: null, gpu_count: 0, diagnostics: {} };
|
|
74
|
+
}
|
|
65
75
|
const cap = await detectCapacity();
|
|
66
|
-
return {
|
|
76
|
+
return {
|
|
77
|
+
size_gb,
|
|
78
|
+
gpu_count: cap.gpu_count ?? 0,
|
|
79
|
+
diagnostics: cap.diagnostics ?? {},
|
|
80
|
+
...checkFits({ size_gb, vram_gb: cap.vram_gb, ram_gb: cap.ram_gb })
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Format a fit-check failure into a multi-line, actionable error message
|
|
86
|
+
* with GPU detection diagnostics + override hint. Used by both the CLI
|
|
87
|
+
* (`infernet model pull`) and the daemon (`start.js` model_install).
|
|
88
|
+
*/
|
|
89
|
+
export function formatFitFailure(modelName, fits) {
|
|
90
|
+
const lines = [
|
|
91
|
+
`${modelName} (≈${fits.size_gb} GB) won't fit on this host:`,
|
|
92
|
+
` detected: ${fits.mode} with ${fits.have_gb} GB available`,
|
|
93
|
+
` ceiling: ~${fits.ceiling_gb} GB (${fits.mode === "gpu" ? "85% of VRAM" : "60% of RAM"})`,
|
|
94
|
+
` gpus detected: ${fits.gpu_count ?? 0}`
|
|
95
|
+
];
|
|
96
|
+
const diag = fits.diagnostics ?? {};
|
|
97
|
+
const diagEntries = Object.entries(diag);
|
|
98
|
+
if (diagEntries.length > 0) {
|
|
99
|
+
lines.push(" GPU detection diagnostics:");
|
|
100
|
+
for (const [vendor, reason] of diagEntries) {
|
|
101
|
+
lines.push(` ${vendor}: ${reason}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (fits.mode === "cpu" && (fits.gpu_count ?? 0) === 0) {
|
|
105
|
+
lines.push("");
|
|
106
|
+
lines.push(" This box reports no GPU. If you DO have one but the tool isn't");
|
|
107
|
+
lines.push(" detecting it (containers / odd passthrough), bypass the check:");
|
|
108
|
+
lines.push(" INFERNET_SKIP_FIT_CHECK=1 infernet ...");
|
|
109
|
+
}
|
|
110
|
+
return lines.join("\n");
|
|
67
111
|
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local model recommender.
|
|
3
|
+
*
|
|
4
|
+
* Given the operator's detected (vramGb, ramGb), picks the best-fitting
|
|
5
|
+
* Ollama / HuggingFace models for their hardware. Curated catalog;
|
|
6
|
+
* scoring borrowed from
|
|
7
|
+
* https://github.com/meattacker/local-ai-model-recommender (MIT)
|
|
8
|
+
* adapted for Infernet (adds Dolphin/Hermes/Qwen-Coder, drops UI-only
|
|
9
|
+
* fields).
|
|
10
|
+
*
|
|
11
|
+
* Used by:
|
|
12
|
+
* - `infernet model recommend` CLI surface
|
|
13
|
+
* - `infernet setup` (default-model pick)
|
|
14
|
+
* - `infernet uncensored` (uncensored-only filter)
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** Curated catalog. Each entry can be pulled via the listed pullName. */
|
|
18
|
+
export const CATALOG = [
|
|
19
|
+
// ── ultra-light (CPU / phone-tier GPU) ──
|
|
20
|
+
{ id: "tinyllama", name: "TinyLlama 1.1B", paramsB: 1.1, vramMin: 0, ramMin: 4, pullName: "tinyllama", backend: "ollama", quality: 2, speed: 10, useCases: ["chat","testing"], notes: "Good only for testing the stack." },
|
|
21
|
+
{ id: "llama3.2-1b", name: "Llama 3.2 1B", paramsB: 1, vramMin: 0, ramMin: 4, pullName: "llama3.2:1b", backend: "ollama", quality: 4, speed: 10, useCases: ["chat","summary"], notes: "Light, fast Q&A." },
|
|
22
|
+
{ id: "qwen2.5-1.5b", name: "Qwen2.5 1.5B", paramsB: 1.5, vramMin: 0, ramMin: 4, pullName: "qwen2.5:1.5b", backend: "ollama", quality: 5, speed: 9, useCases: ["chat","study"] },
|
|
23
|
+
|
|
24
|
+
// ── light (small GPU / 16GB RAM) ──
|
|
25
|
+
{ id: "llama3.2-3b", name: "Llama 3.2 3B", paramsB: 3, vramMin: 2, ramMin: 8, pullName: "llama3.2:3b", backend: "ollama", quality: 6, speed: 8, useCases: ["chat","study","agents"] },
|
|
26
|
+
{ id: "qwen2.5-3b", name: "Qwen2.5 3B", paramsB: 3, vramMin: 2, ramMin: 8, pullName: "qwen2.5:3b", backend: "ollama", quality: 6, speed: 8, useCases: ["chat","study"] },
|
|
27
|
+
{ id: "phi4-mini", name: "Phi-4 Mini 3.8B", paramsB: 3.8, vramMin: 4, ramMin: 8, pullName: "phi4-mini", backend: "ollama", quality: 8, speed: 7, useCases: ["chat","study","agents","coding"] },
|
|
28
|
+
{ id: "gemma3-4b", name: "Gemma 3 4B (vision)",paramsB: 4, vramMin: 4, ramMin: 8, pullName: "gemma3:4b", backend: "ollama", quality: 7, speed: 7, useCases: ["chat","vision","study"] },
|
|
29
|
+
|
|
30
|
+
// ── 7-8B (RTX 3060/4060+, 16GB+ RAM) ──
|
|
31
|
+
{ id: "qwen2.5-7b", name: "Qwen2.5 7B", paramsB: 7, vramMin: 6, ramMin: 16, pullName: "qwen2.5:7b", backend: "ollama", quality: 8, speed: 6, useCases: ["chat","study","coding","agents"] },
|
|
32
|
+
{ id: "llama3.1-8b", name: "Llama 3.1 8B", paramsB: 8, vramMin: 6, ramMin: 16, pullName: "llama3.1:8b", backend: "ollama", quality: 8, speed: 6, useCases: ["chat","study","agents"] },
|
|
33
|
+
{ id: "mistral-7b", name: "Mistral 7B", paramsB: 7, vramMin: 6, ramMin: 16, pullName: "mistral:7b", backend: "ollama", quality: 7, speed: 6, useCases: ["chat","study","coding"] },
|
|
34
|
+
{ id: "qwen2.5-coder-7b",name: "Qwen2.5 Coder 7B", paramsB: 7, vramMin: 6, ramMin: 16, pullName: "qwen2.5-coder:7b", backend: "ollama", quality: 9, speed: 6, useCases: ["coding","agents"], notes: "Best 7B coding model." },
|
|
35
|
+
{ id: "dolphin3-8b", name: "Dolphin 3 8B (uncensored)", paramsB: 8, vramMin: 6, ramMin: 16, pullName: "dolphin3:8b", backend: "ollama", quality: 8, speed: 6, useCases: ["chat","uncensored"], uncensored: true, notes: "Eric Hartford's Dolphin series, no alignment." },
|
|
36
|
+
{ id: "deepseek-r1-7b", name: "DeepSeek-R1 7B", paramsB: 7, vramMin: 6, ramMin: 16, pullName: "deepseek-r1:7b", backend: "ollama", quality: 8, speed: 4, useCases: ["study","coding","reasoning"], notes: "Reasoning-focused; slower because it thinks." },
|
|
37
|
+
{ id: "hermes3-llama3-8b",name:"Hermes 3 Llama-3.1 8B (uncensored)", paramsB: 8, vramMin: 8, ramMin: 16, pullName: "hf:NousResearch/Hermes-3-Llama-3.1-8B", backend: "vllm", quality: 8, speed: 6, useCases: ["chat","uncensored","agents"], uncensored: true },
|
|
38
|
+
|
|
39
|
+
// ── 13-14B (16GB+ VRAM) ──
|
|
40
|
+
{ id: "qwen2.5-14b", name: "Qwen2.5 14B", paramsB: 14, vramMin: 12, ramMin: 32, pullName: "qwen2.5:14b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","study","agents"] },
|
|
41
|
+
{ id: "qwen2.5-coder-14b",name:"Qwen2.5 Coder 14B", paramsB: 14, vramMin: 12, ramMin: 32, pullName: "qwen2.5-coder:14b", backend: "ollama", quality: 9, speed: 5, useCases: ["coding","agents"] },
|
|
42
|
+
|
|
43
|
+
// ── 32B (24GB+ VRAM, RTX 4090 / A100 sweet spot) ──
|
|
44
|
+
{ id: "qwen2.5-32b", name: "Qwen2.5 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5:32b", backend: "ollama", quality: 9, speed: 4, useCases: ["chat","coding","study","agents"], notes: "Single-80GB-GPU sweet spot." },
|
|
45
|
+
{ id: "qwen2.5-coder-32b",name:"Qwen2.5 Coder 32B", paramsB: 32, vramMin: 24, ramMin: 48, pullName: "qwen2.5-coder:32b", backend: "ollama", quality: 10, speed: 4, useCases: ["coding","agents"] },
|
|
46
|
+
|
|
47
|
+
// ── 70B (40-80GB VRAM, Q4 fits 24GB) ──
|
|
48
|
+
{ id: "llama3.3-70b", name: "Llama 3.3 70B", paramsB: 70, vramMin: 40, ramMin: 64, pullName: "llama3.3:70b", backend: "ollama", quality: 10, speed: 3, useCases: ["chat","study","coding","agents"], notes: "Frontier-class at home with 80GB." },
|
|
49
|
+
{ id: "qwen2.5-72b", name: "Qwen2.5 72B", paramsB: 72, vramMin: 40, ramMin: 64, pullName: "qwen2.5:72b", backend: "ollama", quality: 10, speed: 3, useCases: ["chat","coding","study","agents"] },
|
|
50
|
+
|
|
51
|
+
// ── MoE (high VRAM, vLLM only) ──
|
|
52
|
+
{ id: "mixtral-8x7b", name: "Mixtral 8x7B", paramsB: 47, vramMin: 30, ramMin: 64, pullName: "dolphin-mixtral:8x7b", backend: "ollama", quality: 9, speed: 5, useCases: ["chat","coding","uncensored"], uncensored: true }
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Score a model against detected hardware + use case.
|
|
57
|
+
* Higher = better fit. Models that don't meet vramMin are scored lower
|
|
58
|
+
* but not excluded — operators can still pull with --force.
|
|
59
|
+
*/
|
|
60
|
+
function scoreModel(m, { vramGb, ramGb, useCase }) {
|
|
61
|
+
let score = 0;
|
|
62
|
+
|
|
63
|
+
// Hardware fit — biggest factor.
|
|
64
|
+
if (m.vramMin === 0 || vramGb >= m.vramMin) {
|
|
65
|
+
// Fits comfortably; reward larger models the more headroom we have
|
|
66
|
+
score += 50 + Math.min(40, (vramGb - m.vramMin) * 1.5);
|
|
67
|
+
} else {
|
|
68
|
+
// Won't fit — heavily penalize, scaled by how short we are
|
|
69
|
+
score -= 80 - Math.min(60, (m.vramMin - vramGb) * 4);
|
|
70
|
+
}
|
|
71
|
+
if (ramGb >= m.ramMin) score += 10;
|
|
72
|
+
else score -= 20;
|
|
73
|
+
|
|
74
|
+
// Quality bias for use-case match
|
|
75
|
+
score += m.quality * 4;
|
|
76
|
+
if (useCase && m.useCases?.includes(useCase)) score += 25;
|
|
77
|
+
|
|
78
|
+
// CPU-only systems prefer speed
|
|
79
|
+
if (vramGb === 0) score += m.speed * 3 - m.paramsB * 2;
|
|
80
|
+
|
|
81
|
+
return score;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Return the top N models for the given hardware + use case.
|
|
86
|
+
* @param {object} opts
|
|
87
|
+
* @param {number} opts.vramGb
|
|
88
|
+
* @param {number} opts.ramGb
|
|
89
|
+
* @param {string} [opts.useCase] chat | coding | study | agents | uncensored | vision
|
|
90
|
+
* @param {boolean} [opts.uncensoredOnly]
|
|
91
|
+
* @param {number} [opts.limit=3]
|
|
92
|
+
* @returns {Array<{model, score, fits}>}
|
|
93
|
+
*/
|
|
94
|
+
export function recommendModels({ vramGb = 0, ramGb = 0, useCase, uncensoredOnly = false, limit = 3 } = {}) {
|
|
95
|
+
return CATALOG
|
|
96
|
+
.filter((m) => uncensoredOnly ? m.uncensored : true)
|
|
97
|
+
.filter((m) => useCase ? m.useCases?.includes(useCase) : true)
|
|
98
|
+
.map((m) => ({
|
|
99
|
+
model: m,
|
|
100
|
+
score: scoreModel(m, { vramGb, ramGb, useCase }),
|
|
101
|
+
fits: vramGb >= m.vramMin && ramGb >= m.ramMin
|
|
102
|
+
}))
|
|
103
|
+
.sort((a, b) => b.score - a.score)
|
|
104
|
+
.slice(0, limit);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Default model for `infernet setup` based on detected hardware. */
|
|
108
|
+
export function pickDefaultModel({ vramGb = 0, ramGb = 0 } = {}) {
|
|
109
|
+
const top = recommendModels({ vramGb, ramGb, limit: 1 })[0];
|
|
110
|
+
return top?.model ?? null;
|
|
111
|
+
}
|
package/lib/version.js
CHANGED
|
@@ -10,7 +10,7 @@ export const CURRENT_VERSION = JSON.parse(
|
|
|
10
10
|
export async function fetchLatestVersion() {
|
|
11
11
|
try {
|
|
12
12
|
const res = await fetch(
|
|
13
|
-
'https://api.github.com/repos/
|
|
13
|
+
'https://api.github.com/repos/infernetprotocol/infernet-protocol/releases/latest',
|
|
14
14
|
{ headers: { accept: 'application/vnd.github+json', 'user-agent': 'infernet-daemon' }, signal: AbortSignal.timeout(10_000) }
|
|
15
15
|
);
|
|
16
16
|
if (!res.ok) return null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.26",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,18 +30,18 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/config": "0.1.
|
|
35
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
36
|
-
"@infernetprotocol/engine": "0.1.
|
|
37
|
-
"@infernetprotocol/gpu": "0.1.
|
|
33
|
+
"@infernetprotocol/auth": "0.1.26",
|
|
34
|
+
"@infernetprotocol/config": "0.1.26",
|
|
35
|
+
"@infernetprotocol/deploy-providers": "0.1.26",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.26",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.26"
|
|
38
38
|
},
|
|
39
39
|
"repository": {
|
|
40
40
|
"type": "git",
|
|
41
|
-
"url": "git+https://github.com/
|
|
41
|
+
"url": "git+https://github.com/InfernetProtocol/infernet-protocol.git",
|
|
42
42
|
"directory": "apps/cli"
|
|
43
43
|
},
|
|
44
|
-
"homepage": "https://github.com/
|
|
44
|
+
"homepage": "https://github.com/InfernetProtocol/infernet-protocol#readme",
|
|
45
45
|
"keywords": [
|
|
46
46
|
"infernet",
|
|
47
47
|
"gpu",
|