@infernetprotocol/cli 0.1.81 → 0.1.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/help.js +1 -0
- package/commands/house-models.js +173 -0
- package/commands/setup.js +32 -0
- package/commands/start.js +68 -2
- package/commands/status.js +9 -1
- package/index.js +4 -2
- package/lib/house-models.js +505 -0
- package/package.json +9 -9
package/commands/help.js
CHANGED
|
@@ -26,6 +26,7 @@ Daemon:
|
|
|
26
26
|
start Start the node daemon (detached by default)
|
|
27
27
|
stop Stop the running daemon
|
|
28
28
|
status Show this node's current state (remote row + live daemon)
|
|
29
|
+
house-models House models this node pulls by default: status / list / sync / off / uncensored off / cap
|
|
29
30
|
stats Live in-memory stats from the running daemon (via IPC)
|
|
30
31
|
logs Show / tail the daemon log (~/.config/infernet/daemon.log)
|
|
31
32
|
service Userland systemd unit (Linux): install / enable / status / logs
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `infernet house-models` — see and control the house models this node pulls.
|
|
3
|
+
*
|
|
4
|
+
* House models are listed at https://infernetprotocol.com/house-models.json;
|
|
5
|
+
* provider daemons pull the ones that fit, by default. See lib/house-models.js.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { loadConfig, saveConfig } from '../lib/config.js';
|
|
9
|
+
import { isDaemonAlive, sendToDaemon } from '../lib/ipc.js';
|
|
10
|
+
import {
|
|
11
|
+
resolveSettings,
|
|
12
|
+
readState,
|
|
13
|
+
formatStatus,
|
|
14
|
+
fetchManifest,
|
|
15
|
+
ollamaTags,
|
|
16
|
+
planSync,
|
|
17
|
+
diskSpace,
|
|
18
|
+
ramGb,
|
|
19
|
+
syncHouseModels
|
|
20
|
+
} from '../lib/house-models.js';
|
|
21
|
+
|
|
22
|
+
const HELP = `infernet house-models — models Infernet publishes for every node to serve
|
|
23
|
+
|
|
24
|
+
Usage:
|
|
25
|
+
infernet house-models [status] What is pulled, disk used, last sync
|
|
26
|
+
infernet house-models list The manifest and whether each model fits here
|
|
27
|
+
infernet house-models sync Sync now (asks the running daemon)
|
|
28
|
+
infernet house-models off | on Turn house models off / back on (off removes
|
|
29
|
+
the ones the daemon pulled; never yours)
|
|
30
|
+
infernet house-models uncensored off | on
|
|
31
|
+
Skip / allow uncensored house models only
|
|
32
|
+
infernet house-models cap <gb>|none Cap the disk house models may use
|
|
33
|
+
|
|
34
|
+
Flags:
|
|
35
|
+
--json Machine-readable output (status, list)
|
|
36
|
+
--help Show this help
|
|
37
|
+
|
|
38
|
+
Environment (beats config.json): INFERNET_HOUSE_MODELS=0,
|
|
39
|
+
INFERNET_UNCENSORED_MODELS=0, INFERNET_HOUSE_MODELS_MAX_GB=<n>.
|
|
40
|
+
|
|
41
|
+
House models are ON by default. The daemon checks
|
|
42
|
+
https://infernetprotocol.com/house-models.json on start and every 6 hours,
|
|
43
|
+
pulls listed models that fit (RAM >= min_ram_gb, and the pull leaves at least
|
|
44
|
+
10% of the disk free, 10-50 GB), one at a time (waiting up to 10 min for a running job), verifies
|
|
45
|
+
each model's digest, and removes house models that leave the list. A model you
|
|
46
|
+
pulled yourself is never removed.
|
|
47
|
+
`;
|
|
48
|
+
|
|
49
|
+
async function setKey(patch) {
|
|
50
|
+
const config = (await loadConfig()) ?? {};
|
|
51
|
+
config.houseModels = { ...(config.houseModels ?? {}), ...patch };
|
|
52
|
+
await saveConfig(config);
|
|
53
|
+
return config;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function nudgeDaemon() {
|
|
57
|
+
if (await isDaemonAlive()) {
|
|
58
|
+
const res = await sendToDaemon('house-models-sync', null, { timeoutMs: 2000 }).catch(() => null);
|
|
59
|
+
if (res?.ok) {
|
|
60
|
+
process.stdout.write('The running daemon is applying it now (see `infernet logs`).\n');
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
process.stdout.write('Takes effect on the daemon\'s next sync (or run `infernet house-models sync`).\n');
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export default async function houseModels(args) {
|
|
68
|
+
if (args.has('help') || args.has('h')) {
|
|
69
|
+
process.stdout.write(HELP);
|
|
70
|
+
return 0;
|
|
71
|
+
}
|
|
72
|
+
const [sub = 'status', arg] = args.positional ?? [];
|
|
73
|
+
const config = (await loadConfig().catch(() => null)) ?? {};
|
|
74
|
+
|
|
75
|
+
if (sub === 'status') {
|
|
76
|
+
const settings = resolveSettings({ config });
|
|
77
|
+
const state = readState();
|
|
78
|
+
const installed = await ollamaTags().catch(() => null);
|
|
79
|
+
if (args.has('json')) {
|
|
80
|
+
process.stdout.write(JSON.stringify({ settings, state }, null, 2) + '\n');
|
|
81
|
+
} else {
|
|
82
|
+
process.stdout.write(formatStatus({ settings, state, installed }) + '\n');
|
|
83
|
+
}
|
|
84
|
+
return 0;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (sub === 'list') {
|
|
88
|
+
const settings = resolveSettings({ config });
|
|
89
|
+
const manifest = await fetchManifest({ url: settings.manifestUrl });
|
|
90
|
+
const installed = await ollamaTags().catch(() => []);
|
|
91
|
+
const disk = await diskSpace();
|
|
92
|
+
const plan = planSync({
|
|
93
|
+
manifest, installed, owned: readState().owned,
|
|
94
|
+
capacity: { ramGb: ramGb(), diskFreeBytes: disk.freeBytes, diskTotalBytes: disk.totalBytes },
|
|
95
|
+
settings
|
|
96
|
+
});
|
|
97
|
+
if (args.has('json')) {
|
|
98
|
+
process.stdout.write(JSON.stringify({ manifest, plan }, null, 2) + '\n');
|
|
99
|
+
return 0;
|
|
100
|
+
}
|
|
101
|
+
const verdict = new Map();
|
|
102
|
+
for (const m of plan.keep) verdict.set(m.ollama_name, 'installed');
|
|
103
|
+
for (const m of plan.pull) verdict.set(m.ollama_name, 'will pull');
|
|
104
|
+
for (const r of plan.remove) verdict.set(r.name, `will remove: ${r.reason}`);
|
|
105
|
+
for (const s of plan.skipped) verdict.set(s.name, `skip: ${s.reason}`);
|
|
106
|
+
process.stdout.write(`manifest: ${settings.manifestUrl}${manifest.updated ? ` (updated ${manifest.updated})` : ''}\n\n`);
|
|
107
|
+
for (const m of manifest.models) {
|
|
108
|
+
process.stdout.write(`${m.ollama_name}${m.uncensored ? ' [uncensored]' : ''}\n`);
|
|
109
|
+
process.stdout.write(` ${(m.size_bytes / 1024 ** 3).toFixed(1)} GB, needs ${m.min_ram_gb} GB RAM — ${m.description}\n`);
|
|
110
|
+
process.stdout.write(` here: ${verdict.get(m.ollama_name) ?? '-'}\n`);
|
|
111
|
+
}
|
|
112
|
+
return 0;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (sub === 'sync') {
|
|
116
|
+
if (await isDaemonAlive()) {
|
|
117
|
+
const res = await sendToDaemon('house-models-sync', null, { timeoutMs: 2000 }).catch((e) => ({ ok: false, error: e?.message }));
|
|
118
|
+
if (res?.ok) {
|
|
119
|
+
process.stdout.write('Sync started in the daemon; follow it with `infernet logs`, check with `infernet house-models`.\n');
|
|
120
|
+
return 0;
|
|
121
|
+
}
|
|
122
|
+
process.stderr.write(`daemon refused: ${res?.error ?? 'no answer'}\n`);
|
|
123
|
+
return 1;
|
|
124
|
+
}
|
|
125
|
+
process.stdout.write('No daemon running; syncing in this process.\n');
|
|
126
|
+
const summary = await syncHouseModels({
|
|
127
|
+
settings: resolveSettings({ config }),
|
|
128
|
+
log: (m) => process.stdout.write(`${m}\n`)
|
|
129
|
+
});
|
|
130
|
+
process.stdout.write(formatStatus({ settings: resolveSettings({ config }), state: readState() }) + '\n');
|
|
131
|
+
return summary.errors.length > 0 ? 1 : 0;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (sub === 'off' || sub === 'on') {
|
|
135
|
+
await setKey({ enabled: sub === 'on' });
|
|
136
|
+
process.stdout.write(`House models turned ${sub} in config.json.\n`);
|
|
137
|
+
await nudgeDaemon();
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (sub === 'uncensored') {
|
|
142
|
+
if (arg !== 'on' && arg !== 'off') {
|
|
143
|
+
process.stderr.write('Usage: infernet house-models uncensored on|off\n');
|
|
144
|
+
return 1;
|
|
145
|
+
}
|
|
146
|
+
await setKey({ uncensored: arg === 'on' });
|
|
147
|
+
process.stdout.write(`Uncensored house models turned ${arg} in config.json.\n`);
|
|
148
|
+
await nudgeDaemon();
|
|
149
|
+
return 0;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (sub === 'cap') {
|
|
153
|
+
if (arg === 'none' || arg === 'off') {
|
|
154
|
+
await setKey({ maxDiskGb: null });
|
|
155
|
+
process.stdout.write('House-models disk cap removed (the 10%-of-disk free reserve, 10-50 GB, still applies).\n');
|
|
156
|
+
} else {
|
|
157
|
+
const gb = Number(arg);
|
|
158
|
+
if (!Number.isFinite(gb) || gb < 0) {
|
|
159
|
+
process.stderr.write('Usage: infernet house-models cap <gb>|none\n');
|
|
160
|
+
return 1;
|
|
161
|
+
}
|
|
162
|
+
await setKey({ maxDiskGb: gb });
|
|
163
|
+
process.stdout.write(`House models may use at most ${gb} GB of disk.\n`);
|
|
164
|
+
}
|
|
165
|
+
await nudgeDaemon();
|
|
166
|
+
return 0;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
process.stderr.write(`unknown subcommand: ${sub}\n\n${HELP}`);
|
|
170
|
+
return 1;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export { HELP };
|
package/commands/setup.js
CHANGED
|
@@ -49,6 +49,10 @@ Flags:
|
|
|
49
49
|
--skip-pull Skip the model-pull step.
|
|
50
50
|
--no-abliterated Don't default to the abliterated (uncensored) model
|
|
51
51
|
when it fits; pick the largest stock Qwen instead.
|
|
52
|
+
--no-house-models Don't pull Infernet house models (saved to config).
|
|
53
|
+
--no-uncensored-models Skip uncensored house models only (saved to config).
|
|
54
|
+
--house-models-max-gb <n>
|
|
55
|
+
Cap the disk house models may use (saved to config).
|
|
52
56
|
--backend <kind> Pin engine backend (vllm | ollama | mojo | stub). Default: auto (prefers vLLM, falls back to Ollama).
|
|
53
57
|
--host <url> Override Ollama host. Default: http://localhost:11434.
|
|
54
58
|
--port <n> P2P port to open in the firewall. Default: ${DEFAULT_P2P_PORT}.
|
|
@@ -853,6 +857,20 @@ export default async function setup(args) {
|
|
|
853
857
|
}
|
|
854
858
|
}
|
|
855
859
|
|
|
860
|
+
// ---- House models: persist any opt-out before the daemon (re)starts ----
|
|
861
|
+
if (isProvider) {
|
|
862
|
+
const patch = {};
|
|
863
|
+
if (args.has("no-house-models")) patch.enabled = false;
|
|
864
|
+
if (args.has("no-uncensored-models")) patch.uncensored = false;
|
|
865
|
+
const capArg = Number(args.get("house-models-max-gb"));
|
|
866
|
+
if (args.get("house-models-max-gb") !== undefined && Number.isFinite(capArg)) patch.maxDiskGb = capArg;
|
|
867
|
+
if (Object.keys(patch).length > 0) {
|
|
868
|
+
const cfgNow = (await loadConfig()) ?? {};
|
|
869
|
+
cfgNow.houseModels = { ...(cfgNow.houseModels ?? {}), ...patch };
|
|
870
|
+
await saveConfig(cfgNow);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
|
|
856
874
|
// ---- Daemon (always restart so it picks up current CLI code; providers only) ----
|
|
857
875
|
if (!skipDaemon && isProvider && configAfterId) {
|
|
858
876
|
n += 1;
|
|
@@ -965,6 +983,20 @@ export default async function setup(args) {
|
|
|
965
983
|
if (code !== 0) warn(" pubkey link did not complete — re-run `infernet pubkey link` manually");
|
|
966
984
|
}
|
|
967
985
|
|
|
986
|
+
if (isProvider) {
|
|
987
|
+
const { resolveSettings } = await import("../lib/house-models.js");
|
|
988
|
+
const hs = resolveSettings({ config: (await loadConfig()) ?? {} });
|
|
989
|
+
process.stdout.write("\n");
|
|
990
|
+
if (hs.enabled) {
|
|
991
|
+
ok(`house models: ON — this node will pull the models listed at ${hs.manifestUrl}`);
|
|
992
|
+
process.stdout.write(` that fit it (RAM, and always leaving 10% of the disk free, 10-50 GB${hs.maxDiskGb != null ? `, capped at ${hs.maxDiskGb} GB` : ""}),\n`);
|
|
993
|
+
process.stdout.write(` in the background, and serve them on Infernet. Uncensored house models: ${hs.uncensored ? "on" : "off"}.\n`);
|
|
994
|
+
process.stdout.write(" Opt out: infernet house-models off (uncensored only: infernet house-models uncensored off)\n");
|
|
995
|
+
} else {
|
|
996
|
+
ok("house models: off (turn on with `infernet house-models on`)");
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
|
|
968
1000
|
process.stdout.write("\n");
|
|
969
1001
|
ok("setup complete");
|
|
970
1002
|
process.stdout.write("\nTry it:\n");
|
package/commands/start.js
CHANGED
|
@@ -65,6 +65,7 @@ import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
|
|
|
65
65
|
import { nodeTokenLogin } from './login.js';
|
|
66
66
|
import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion, readInstalledVersion } from '../lib/version.js';
|
|
67
67
|
import { readInferenceState, readInferenceStateSync, isPidAlive } from '../lib/inference/state.js';
|
|
68
|
+
import { resolveSettings as resolveHouseSettings, syncHouseModels, readState as readHouseState, FIRST_SYNC_DELAY_MS as HOUSE_FIRST_SYNC_DELAY_MS } from '../lib/house-models.js';
|
|
68
69
|
|
|
69
70
|
function readInferenceStateSyncSafe() {
|
|
70
71
|
try { return readInferenceStateSync(); } catch { return {}; }
|
|
@@ -84,8 +85,16 @@ Flags:
|
|
|
84
85
|
--no-p2p Don't bind the P2P TCP listener
|
|
85
86
|
--no-advertise Don't send address/port in heartbeats
|
|
86
87
|
--once Run one heartbeat + one poll and exit (debug)
|
|
88
|
+
--no-house-models Don't pull Infernet house models for this run
|
|
89
|
+
(persist with \`infernet house-models off\`)
|
|
90
|
+
--no-uncensored-models Skip uncensored house models only
|
|
91
|
+
--house-models-max-gb <n> Cap the disk house models may use
|
|
87
92
|
--help Show this help
|
|
88
93
|
|
|
94
|
+
House models: provider nodes pull the models listed at
|
|
95
|
+
https://infernetprotocol.com/house-models.json (on start, then every 6 h) when
|
|
96
|
+
they fit. See \`infernet house-models --help\`.
|
|
97
|
+
|
|
89
98
|
Daemon logs to \`~/.config/infernet/daemon.log\`, exposes an IPC socket for
|
|
90
99
|
live queries (see \`infernet status\`, \`infernet stats\`, \`infernet logs\`).
|
|
91
100
|
`;
|
|
@@ -162,6 +171,10 @@ function passthroughArgs(args) {
|
|
|
162
171
|
if (args.has('no-p2p')) passthrough.push('--no-p2p');
|
|
163
172
|
if (args.has('no-advertise')) passthrough.push('--no-advertise');
|
|
164
173
|
if (args.has('once')) passthrough.push('--once');
|
|
174
|
+
if (args.has('no-house-models')) passthrough.push('--no-house-models');
|
|
175
|
+
if (args.has('no-uncensored-models')) passthrough.push('--no-uncensored-models');
|
|
176
|
+
const houseCap = args.get('house-models-max-gb');
|
|
177
|
+
if (houseCap) passthrough.push('--house-models-max-gb', houseCap);
|
|
165
178
|
return passthrough;
|
|
166
179
|
}
|
|
167
180
|
|
|
@@ -287,6 +300,9 @@ async function runDaemon(args, ctx) {
|
|
|
287
300
|
let heartbeatTimer = null;
|
|
288
301
|
let pollTimer = null;
|
|
289
302
|
let shuttingDown = false;
|
|
303
|
+
// House models (lib/house-models.js): set up after the first heartbeat.
|
|
304
|
+
let runHouseSync = null;
|
|
305
|
+
let houseSyncRunning = false;
|
|
290
306
|
let ipcServer = null;
|
|
291
307
|
let p2pServer = null;
|
|
292
308
|
let healthServer = null;
|
|
@@ -1296,6 +1312,7 @@ async function runDaemon(args, ctx) {
|
|
|
1296
1312
|
},
|
|
1297
1313
|
controlPlaneUrl: config.controlPlane?.url ?? null,
|
|
1298
1314
|
intervals: { heartbeatMs, pollMs },
|
|
1315
|
+
houseModels: node.role === 'provider' ? { syncing: houseSyncRunning, ...readHouseState() } : null,
|
|
1299
1316
|
stats: {
|
|
1300
1317
|
heartbeatsOk: stats.heartbeatsOk,
|
|
1301
1318
|
heartbeatsFailed: stats.heartbeatsFailed,
|
|
@@ -1360,6 +1377,10 @@ async function runDaemon(args, ctx) {
|
|
|
1360
1377
|
case 'ping': return { ok: true, data: { pong: Date.now() } };
|
|
1361
1378
|
case 'status':
|
|
1362
1379
|
case 'stats': return { ok: true, data: snapshot() };
|
|
1380
|
+
case 'house-models-sync':
|
|
1381
|
+
if (!runHouseSync) return { ok: false, error: 'house models run on provider nodes only' };
|
|
1382
|
+
setImmediate(() => { runHouseSync('ipc').catch(() => {}); });
|
|
1383
|
+
return { ok: true, data: { started: !houseSyncRunning } };
|
|
1363
1384
|
case 'shutdown':
|
|
1364
1385
|
setImmediate(() => shutdown('ipc-shutdown'));
|
|
1365
1386
|
return { ok: true, data: { shuttingDown: true } };
|
|
@@ -1630,6 +1651,40 @@ async function runDaemon(args, ctx) {
|
|
|
1630
1651
|
if (typeof trainingTimer.unref === 'function') trainingTimer.unref();
|
|
1631
1652
|
}
|
|
1632
1653
|
|
|
1654
|
+
// House models: Profullstack-published models every provider pulls and
|
|
1655
|
+
// serves by default. Settings are re-read from config.json on every pass so
|
|
1656
|
+
// `infernet house-models off` takes effect without a restart.
|
|
1657
|
+
if (node.role === 'provider') {
|
|
1658
|
+
const startupSettings = resolveHouseSettings({ config, args });
|
|
1659
|
+
const label = (s) => `${s.enabled ? 'on' : 'off'}; uncensored ${s.uncensored ? 'on' : 'off'}${s.maxDiskGb != null ? `; disk cap ${s.maxDiskGb} GB` : ''}`;
|
|
1660
|
+
process.stdout.write(`[house-models] ${label(startupSettings)} — manifest ${startupSettings.manifestUrl}, every ${Math.round(startupSettings.intervalMs / 3_600_000 * 10) / 10}h (opt out: infernet house-models off)\n`);
|
|
1661
|
+
runHouseSync = async (why) => {
|
|
1662
|
+
if (houseSyncRunning || shuttingDown) return null;
|
|
1663
|
+
houseSyncRunning = true;
|
|
1664
|
+
try {
|
|
1665
|
+
const fresh = await loadConfig().catch(() => null);
|
|
1666
|
+
const settings = resolveHouseSettings({ config: fresh ?? config, args });
|
|
1667
|
+
const summary = await syncHouseModels({
|
|
1668
|
+
settings,
|
|
1669
|
+
isBusy: () => stats.activeJobIds.size > 0 || shuttingDown,
|
|
1670
|
+
log: (m) => process.stdout.write(`[${new Date().toISOString()}] [house-models] ${m}\n`)
|
|
1671
|
+
});
|
|
1672
|
+
process.stdout.write(`[${new Date().toISOString()}] [house-models] sync (${why}): pulled ${summary.pulled.length}, removed ${summary.removed.length}, skipped ${summary.skipped.length}, errors ${summary.errors.length}\n`);
|
|
1673
|
+
if (summary.pulled.length > 0 || summary.removed.length > 0) {
|
|
1674
|
+
invalidateSpecsCache();
|
|
1675
|
+
heartbeat().catch(() => {});
|
|
1676
|
+
}
|
|
1677
|
+
return summary;
|
|
1678
|
+
} finally {
|
|
1679
|
+
houseSyncRunning = false;
|
|
1680
|
+
}
|
|
1681
|
+
};
|
|
1682
|
+
const houseFirst = setTimeout(() => { runHouseSync('startup').catch(() => {}); }, HOUSE_FIRST_SYNC_DELAY_MS);
|
|
1683
|
+
if (typeof houseFirst.unref === 'function') houseFirst.unref();
|
|
1684
|
+
const houseTimer = setInterval(() => { runHouseSync('scheduled').catch(() => {}); }, startupSettings.intervalMs);
|
|
1685
|
+
if (typeof houseTimer.unref === 'function') houseTimer.unref();
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1633
1688
|
// Self-update: check the GitHub release feed every 60 seconds. If a newer version is
|
|
1634
1689
|
// available, pull the installer, re-generate any missing keys + re-register
|
|
1635
1690
|
// specs (idempotent), then re-exec so the new code is running immediately.
|
|
@@ -1642,10 +1697,19 @@ async function runDaemon(args, ctx) {
|
|
|
1642
1697
|
const followOnly = process.env.INFERNET_AUTO_UPDATE === 'follow';
|
|
1643
1698
|
const checkForUpdate = async () => {
|
|
1644
1699
|
if (shuttingDown || isUpgrading || Date.now() < updatePausedUntil) return;
|
|
1645
|
-
// Never start an upgrade under a running job; the next check retries.
|
|
1646
|
-
if (stats.activeJobIds.size > 0) return;
|
|
1647
1700
|
const latest = followOnly ? readInstalledVersion() : await fetchLatestVersion();
|
|
1648
1701
|
if (!latest || !isNewerVersion(CURRENT_VERSION, latest)) return;
|
|
1702
|
+
// Never start an upgrade under a running job. But a node that is always
|
|
1703
|
+
// busy used to skip every check and never update (bonita sat on 0.1.77
|
|
1704
|
+
// for four releases under steady chat load), so stop taking new jobs
|
|
1705
|
+
// and upgrade on the first check after the running ones finish.
|
|
1706
|
+
if (stats.activeJobIds.size > 0) {
|
|
1707
|
+
if (!stats.draining) {
|
|
1708
|
+
stats.draining = true;
|
|
1709
|
+
process.stdout.write(`[update] v${latest} available (running v${CURRENT_VERSION}) — finishing ${stats.activeJobIds.size} running job(s) first, taking no new ones\n`);
|
|
1710
|
+
}
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1649
1713
|
isUpgrading = true;
|
|
1650
1714
|
|
|
1651
1715
|
if (followOnly) {
|
|
@@ -1655,6 +1719,7 @@ async function runDaemon(args, ctx) {
|
|
|
1655
1719
|
const ok = await pullLatestBinary();
|
|
1656
1720
|
if (!ok) {
|
|
1657
1721
|
process.stderr.write('[update] installer failed — will retry next check\n');
|
|
1722
|
+
stats.draining = false;
|
|
1658
1723
|
isUpgrading = false;
|
|
1659
1724
|
return;
|
|
1660
1725
|
}
|
|
@@ -1666,6 +1731,7 @@ async function runDaemon(args, ctx) {
|
|
|
1666
1731
|
if (!isNewerVersion(CURRENT_VERSION, installed)) {
|
|
1667
1732
|
updatePausedUntil = Date.now() + UPDATE_PAUSE_MS;
|
|
1668
1733
|
process.stderr.write(`[update] installer left v${installed} here (latest v${latest}); not restarting, next try in ${UPDATE_PAUSE_MS / 3_600_000}h\n`);
|
|
1734
|
+
stats.draining = false;
|
|
1669
1735
|
isUpgrading = false;
|
|
1670
1736
|
return;
|
|
1671
1737
|
}
|
package/commands/status.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { isDaemonAlive, sendToDaemon } from '../lib/ipc.js';
|
|
9
9
|
import { formatEndpoint } from '../lib/network.js';
|
|
10
10
|
import { formatBalance } from './payout.js';
|
|
11
|
+
import { resolveSettings as resolveHouseSettings, readState as readHouseState, formatStatus as formatHouseStatus, ollamaTags } from '../lib/house-models.js';
|
|
11
12
|
|
|
12
13
|
const HELP = `infernet status — show this node's state
|
|
13
14
|
|
|
@@ -59,7 +60,8 @@ export default async function status(args, ctx) {
|
|
|
59
60
|
}
|
|
60
61
|
|
|
61
62
|
if (args.has('json')) {
|
|
62
|
-
|
|
63
|
+
const houseModels = node.role === 'provider' ? { settings: resolveHouseSettings({ config }), state: readHouseState() } : null;
|
|
64
|
+
process.stdout.write(JSON.stringify({ row, daemon, houseModels }, null, 2) + '\n');
|
|
63
65
|
return 0;
|
|
64
66
|
}
|
|
65
67
|
|
|
@@ -97,6 +99,12 @@ export default async function status(args, ctx) {
|
|
|
97
99
|
process.stdout.write(` jobs: completed=${s.jobsCompleted} active=${s.activeJobs}\n`);
|
|
98
100
|
}
|
|
99
101
|
|
|
102
|
+
if (node.role === 'provider') {
|
|
103
|
+
const houseSettings = resolveHouseSettings({ config });
|
|
104
|
+
const installed = await ollamaTags().catch(() => null);
|
|
105
|
+
process.stdout.write('\n' + formatHouseStatus({ settings: houseSettings, state: readHouseState(), installed }) + '\n\n');
|
|
106
|
+
}
|
|
107
|
+
|
|
100
108
|
if (node.role === 'provider') {
|
|
101
109
|
try {
|
|
102
110
|
const earnings = await client.earnings(1);
|
package/index.js
CHANGED
|
@@ -35,6 +35,7 @@ import gpu from './commands/gpu.js';
|
|
|
35
35
|
import firewall from './commands/firewall.js';
|
|
36
36
|
import chat from './commands/chat.js';
|
|
37
37
|
import setup from './commands/setup.js';
|
|
38
|
+
import houseModels from './commands/house-models.js';
|
|
38
39
|
import model from './commands/model.js';
|
|
39
40
|
import tui from './commands/tui.js';
|
|
40
41
|
import doctor from './commands/doctor.js';
|
|
@@ -114,11 +115,12 @@ const COMMANDS = {
|
|
|
114
115
|
// aliases
|
|
115
116
|
uninstall: remove,
|
|
116
117
|
start, status, stop, restart, stats, logs,
|
|
117
|
-
payout, payments, gpu, firewall, chat, setup, model, train, publish, inference, rpc, uncensored, reservation, reservations: reservation, order, orders: order, mcp, credits, invites, redeem, batch, batches: batch, billing, keys, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help
|
|
118
|
+
payout, payments, gpu, firewall, chat, setup, model, train, publish, inference, rpc, uncensored, reservation, reservations: reservation, order, orders: order, mcp, credits, invites, redeem, batch, batches: batch, billing, keys, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help,
|
|
119
|
+
'house-models': houseModels, house: houseModels
|
|
118
120
|
};
|
|
119
121
|
|
|
120
122
|
// Commands that can run without a loaded config.
|
|
121
|
-
const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'publish', 'inference', 'rpc', 'uncensored', 'reservation', 'reservations', 'order', 'orders', 'mcp', 'credits', 'invites', 'redeem', 'batch', 'batches', 'billing', 'keys', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall']);
|
|
123
|
+
const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'publish', 'inference', 'rpc', 'uncensored', 'reservation', 'reservations', 'order', 'orders', 'mcp', 'credits', 'invites', 'redeem', 'batch', 'batches', 'billing', 'keys', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall', 'house-models', 'house']);
|
|
122
124
|
// Commands that need a config but not a control-plane client (none today
|
|
123
125
|
// — kept as a future escape hatch).
|
|
124
126
|
const NO_CLIENT = new Set();
|
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* House models: models Profullstack publishes that every provider node pulls
|
|
3
|
+
* and serves by default (Anthony, 2026-11-02: "our models always deploy to all
|
|
4
|
+
* nodes, everyone's, not just ours").
|
|
5
|
+
*
|
|
6
|
+
* The list is a static JSON manifest in this repo (apps/web/public/
|
|
7
|
+
* house-models.json), served at https://infernetprotocol.com/house-models.json.
|
|
8
|
+
* Adding a model is a PR to that file. The daemon fetches it on start and every
|
|
9
|
+
* few hours, pulls each listed model that fits this machine into Ollama (one at
|
|
10
|
+
* a time, preferring idle moments), verifies the pulled model's digest and
|
|
11
|
+
* size against the manifest, and removes house models that leave the manifest.
|
|
12
|
+
* Pulled models show up in served_models like any other Ollama model.
|
|
13
|
+
*
|
|
14
|
+
* Safety rules, because this downloads onto other people's machines:
|
|
15
|
+
* - The manifest is only fetched over HTTPS from our own domains.
|
|
16
|
+
* - A model the operator pulled themselves is never removed: only models the
|
|
17
|
+
* daemon pulled (recorded in house-models-state.json) are ever deleted.
|
|
18
|
+
* - A pull needs min_ram_gb of RAM and leaves at least a disk reserve free
|
|
19
|
+
* (10% of the disk, at least 10 GB, at most 50 GB), and house models as a
|
|
20
|
+
* whole stay under the operator's disk cap if one is set.
|
|
21
|
+
* - Opt out of everything with `--no-house-models`, `infernet house-models
|
|
22
|
+
* off` or INFERNET_HOUSE_MODELS=0; opt out of uncensored ones only with
|
|
23
|
+
* `--no-uncensored-models`, `infernet house-models uncensored off` or
|
|
24
|
+
* INFERNET_UNCENSORED_MODELS=0. Opting out removes what the daemon pulled.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import fs from 'node:fs/promises';
|
|
28
|
+
import fsSync from 'node:fs';
|
|
29
|
+
import os from 'node:os';
|
|
30
|
+
import path from 'node:path';
|
|
31
|
+
|
|
32
|
+
import { getConfigDir } from './config.js';
|
|
33
|
+
|
|
34
|
+
export const DEFAULT_MANIFEST_URL = 'https://infernetprotocol.com/house-models.json';
|
|
35
|
+
export const ALLOWED_MANIFEST_HOSTS = ['infernetprotocol.com', 'www.infernetprotocol.com', 'c0mpute.com', 'www.c0mpute.com'];
|
|
36
|
+
export const DEFAULT_SYNC_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
37
|
+
export const FIRST_SYNC_DELAY_MS = 2 * 60 * 1000;
|
|
38
|
+
/** A model that failed to pull or verify is not retried for this long. */
|
|
39
|
+
export const FAILURE_BACKOFF_MS = 24 * 60 * 60 * 1000;
|
|
40
|
+
export const DISK_RESERVE_MIN_BYTES = 10 * 1024 ** 3;
|
|
41
|
+
export const DISK_RESERVE_FRACTION = 0.10;
|
|
42
|
+
/** 10% of a 1 TB disk is 100 GB; past this, 10% stops meaning "nearly full". */
|
|
43
|
+
export const DISK_RESERVE_MAX_BYTES = 50 * 1024 ** 3;
|
|
44
|
+
|
|
45
|
+
const GB = 1024 ** 3;
|
|
46
|
+
const NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._\-/:]{0,199}$/;
|
|
47
|
+
const DIGEST_RE = /^(sha256:)?[a-f0-9]{64}$/;
|
|
48
|
+
|
|
49
|
+
const isOff = (v) => typeof v === 'string' && ['0', 'false', 'off', 'no'].includes(v.trim().toLowerCase());
|
|
50
|
+
const isOn = (v) => typeof v === 'string' && ['1', 'true', 'on', 'yes'].includes(v.trim().toLowerCase());
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Ollama lists a model without a tag as `<name>:latest`. Normalize so a
|
|
54
|
+
* manifest source like `hf.co/org/repo` matches the tags entry.
|
|
55
|
+
*/
|
|
56
|
+
export function normalizeOllamaName(name) {
|
|
57
|
+
if (typeof name !== 'string') return '';
|
|
58
|
+
const n = name.trim();
|
|
59
|
+
const last = n.split('/').pop() ?? '';
|
|
60
|
+
return last.includes(':') ? n : `${n}:latest`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function stripDigest(d) {
|
|
64
|
+
return typeof d === 'string' ? d.replace(/^sha256:/, '').toLowerCase() : null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Validate a manifest document. Bad entries are dropped individually (and
|
|
69
|
+
* reported in `rejected`) so one typo cannot stop every other model.
|
|
70
|
+
*/
|
|
71
|
+
export function parseManifest(doc) {
|
|
72
|
+
if (!doc || typeof doc !== 'object' || !Array.isArray(doc.models)) {
|
|
73
|
+
throw new Error('house-models manifest: expected an object with a "models" array');
|
|
74
|
+
}
|
|
75
|
+
if (doc.schema !== undefined && doc.schema !== 1) {
|
|
76
|
+
throw new Error(`house-models manifest: unsupported schema ${doc.schema}`);
|
|
77
|
+
}
|
|
78
|
+
const models = [];
|
|
79
|
+
const rejected = [];
|
|
80
|
+
const seen = new Set();
|
|
81
|
+
for (const raw of doc.models) {
|
|
82
|
+
const why = [];
|
|
83
|
+
const source = typeof raw?.source === 'string' ? raw.source.trim() : '';
|
|
84
|
+
if (!source || !NAME_RE.test(source)) why.push('source missing or not an Ollama/hf.co reference');
|
|
85
|
+
if (source.includes('..')) why.push('source contains ".."');
|
|
86
|
+
const size = Number(raw?.size_bytes);
|
|
87
|
+
if (!Number.isFinite(size) || size <= 0) why.push('size_bytes must be a positive number');
|
|
88
|
+
const minRam = Number(raw?.min_ram_gb);
|
|
89
|
+
if (!Number.isFinite(minRam) || minRam < 0) why.push('min_ram_gb must be a number');
|
|
90
|
+
const minDisk = raw?.min_disk_gb === undefined ? 0 : Number(raw.min_disk_gb);
|
|
91
|
+
if (!Number.isFinite(minDisk) || minDisk < 0) why.push('min_disk_gb must be a number');
|
|
92
|
+
if (raw?.digest != null && !DIGEST_RE.test(String(raw.digest))) why.push('digest must be sha256 hex');
|
|
93
|
+
const key = normalizeOllamaName(source);
|
|
94
|
+
if (seen.has(key)) why.push('duplicate source');
|
|
95
|
+
if (why.length > 0) {
|
|
96
|
+
rejected.push({ name: raw?.name ?? source ?? '(unnamed)', reasons: why });
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
seen.add(key);
|
|
100
|
+
models.push({
|
|
101
|
+
name: typeof raw.name === 'string' && raw.name.trim() ? raw.name.trim() : source,
|
|
102
|
+
source,
|
|
103
|
+
ollama_name: key,
|
|
104
|
+
digest: raw.digest ? `sha256:${stripDigest(raw.digest)}` : null,
|
|
105
|
+
size_bytes: size,
|
|
106
|
+
min_ram_gb: minRam,
|
|
107
|
+
min_disk_gb: minDisk,
|
|
108
|
+
uncensored: raw.uncensored === true,
|
|
109
|
+
description: typeof raw.description === 'string' ? raw.description : '',
|
|
110
|
+
license: typeof raw.license === 'string' ? raw.license : null,
|
|
111
|
+
added: typeof raw.added === 'string' ? raw.added : null
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
return { version: doc.version ?? null, updated: doc.updated ?? null, models, rejected };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Only HTTPS, only our hosts: a manifest from anywhere else is refused. */
|
|
118
|
+
export function assertManifestUrl(url) {
|
|
119
|
+
let u;
|
|
120
|
+
try { u = new URL(url); } catch { throw new Error(`house-models: bad manifest URL ${url}`); }
|
|
121
|
+
if (u.protocol !== 'https:') throw new Error(`house-models: manifest must be https (${url})`);
|
|
122
|
+
if (!ALLOWED_MANIFEST_HOSTS.includes(u.hostname)) {
|
|
123
|
+
throw new Error(`house-models: manifest host ${u.hostname} is not one of ${ALLOWED_MANIFEST_HOSTS.join(', ')}`);
|
|
124
|
+
}
|
|
125
|
+
return u.toString();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export async function fetchManifest({ url = DEFAULT_MANIFEST_URL, fetchImpl = fetch, timeoutMs = 15_000 } = {}) {
|
|
129
|
+
const safe = assertManifestUrl(url);
|
|
130
|
+
const res = await fetchImpl(safe, {
|
|
131
|
+
headers: { accept: 'application/json' },
|
|
132
|
+
redirect: 'error',
|
|
133
|
+
signal: AbortSignal.timeout?.(timeoutMs)
|
|
134
|
+
});
|
|
135
|
+
if (!res.ok) throw new Error(`house-models: GET ${safe} -> HTTP ${res.status}`);
|
|
136
|
+
return parseManifest(await res.json());
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Effective settings: CLI flags beat env, env beats config.json, and the
|
|
141
|
+
* default is ON (house models and uncensored house models both).
|
|
142
|
+
*/
|
|
143
|
+
export function resolveSettings({ config = {}, env = process.env, args = null } = {}) {
|
|
144
|
+
const cfg = config?.houseModels ?? {};
|
|
145
|
+
let enabled = cfg.enabled !== false;
|
|
146
|
+
let uncensored = cfg.uncensored !== false;
|
|
147
|
+
let maxDiskGb = Number.isFinite(Number(cfg.maxDiskGb)) && cfg.maxDiskGb !== null ? Number(cfg.maxDiskGb) : null;
|
|
148
|
+
const sources = { enabled: cfg.enabled === false ? 'config' : 'default', uncensored: cfg.uncensored === false ? 'config' : 'default' };
|
|
149
|
+
|
|
150
|
+
if (isOff(env.INFERNET_HOUSE_MODELS)) { enabled = false; sources.enabled = 'env INFERNET_HOUSE_MODELS'; }
|
|
151
|
+
else if (isOn(env.INFERNET_HOUSE_MODELS)) { enabled = true; sources.enabled = 'env INFERNET_HOUSE_MODELS'; }
|
|
152
|
+
if (isOn(env.INFERNET_NO_HOUSE_MODELS)) { enabled = false; sources.enabled = 'env INFERNET_NO_HOUSE_MODELS'; }
|
|
153
|
+
if (isOff(env.INFERNET_UNCENSORED_MODELS)) { uncensored = false; sources.uncensored = 'env INFERNET_UNCENSORED_MODELS'; }
|
|
154
|
+
else if (isOn(env.INFERNET_UNCENSORED_MODELS)) { uncensored = true; sources.uncensored = 'env INFERNET_UNCENSORED_MODELS'; }
|
|
155
|
+
if (isOn(env.INFERNET_NO_UNCENSORED_MODELS)) { uncensored = false; sources.uncensored = 'env INFERNET_NO_UNCENSORED_MODELS'; }
|
|
156
|
+
const envCap = Number(env.INFERNET_HOUSE_MODELS_MAX_GB);
|
|
157
|
+
if (env.INFERNET_HOUSE_MODELS_MAX_GB !== undefined && env.INFERNET_HOUSE_MODELS_MAX_GB !== '' && Number.isFinite(envCap)) {
|
|
158
|
+
maxDiskGb = envCap;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (args?.has?.('no-house-models')) { enabled = false; sources.enabled = '--no-house-models'; }
|
|
162
|
+
if (args?.has?.('no-uncensored-models')) { uncensored = false; sources.uncensored = '--no-uncensored-models'; }
|
|
163
|
+
const argCap = Number(args?.get?.('house-models-max-gb'));
|
|
164
|
+
if (Number.isFinite(argCap) && args?.get?.('house-models-max-gb') !== undefined) maxDiskGb = argCap;
|
|
165
|
+
|
|
166
|
+
const manifestUrl = env.INFERNET_HOUSE_MODELS_URL || cfg.manifestUrl || DEFAULT_MANIFEST_URL;
|
|
167
|
+
const intervalMin = Number(env.INFERNET_HOUSE_MODELS_INTERVAL_MIN);
|
|
168
|
+
const intervalMs = Number.isFinite(intervalMin) && intervalMin >= 5 ? intervalMin * 60_000 : DEFAULT_SYNC_INTERVAL_MS;
|
|
169
|
+
return { enabled, uncensored, maxDiskGb: maxDiskGb != null && maxDiskGb >= 0 ? maxDiskGb : null, manifestUrl, intervalMs, sources };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Free disk a pull must leave behind: 10% of the disk, but at least 10 GB and
|
|
174
|
+
* at most 50 GB (so a 900 GB disk with 80 GB free can still take a 3 GB model).
|
|
175
|
+
*/
|
|
176
|
+
export function diskReserveBytes(totalBytes) {
|
|
177
|
+
const pct = Number.isFinite(totalBytes) ? totalBytes * DISK_RESERVE_FRACTION : 0;
|
|
178
|
+
return Math.max(DISK_RESERVE_MIN_BYTES, Math.min(pct, DISK_RESERVE_MAX_BYTES));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Decide what to pull and what to remove. Pure: every input is passed in.
|
|
183
|
+
*
|
|
184
|
+
* manifest parseManifest() output
|
|
185
|
+
* installed Ollama /api/tags models: [{ name, digest, size }]
|
|
186
|
+
* owned { [ollama_name]: {...} }: models the daemon pulled itself
|
|
187
|
+
* capacity { ramGb, diskFreeBytes, diskTotalBytes }
|
|
188
|
+
* settings resolveSettings() output
|
|
189
|
+
* failures { [ollama_name]: { at } }: recent pull/verify failures
|
|
190
|
+
*
|
|
191
|
+
* Returns { pull: [entry], remove: [{ name, reason }], keep: [entry], skipped: [{ name, reason }] }.
|
|
192
|
+
*/
|
|
193
|
+
export function planSync({ manifest, installed = [], owned = {}, capacity = {}, settings, failures = {}, now = Date.now() }) {
|
|
194
|
+
const installedByName = new Map(installed.map((m) => [normalizeOllamaName(m.name ?? m.model), m]));
|
|
195
|
+
const plan = { pull: [], remove: [], keep: [], skipped: [] };
|
|
196
|
+
const models = manifest?.models ?? [];
|
|
197
|
+
const listed = new Set(models.map((m) => m.ollama_name));
|
|
198
|
+
|
|
199
|
+
// Owned models that left the manifest go first, whatever else is set.
|
|
200
|
+
for (const name of Object.keys(owned)) {
|
|
201
|
+
if (!listed.has(name) && installedByName.has(name)) {
|
|
202
|
+
plan.remove.push({ name, reason: 'no longer in the house-models manifest' });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const ram = Number(capacity.ramGb);
|
|
207
|
+
let free = Number(capacity.diskFreeBytes);
|
|
208
|
+
const reserve = diskReserveBytes(Number(capacity.diskTotalBytes));
|
|
209
|
+
const capBytes = settings?.maxDiskGb != null ? settings.maxDiskGb * GB : Infinity;
|
|
210
|
+
let used = 0;
|
|
211
|
+
|
|
212
|
+
for (const m of models) {
|
|
213
|
+
const have = installedByName.get(m.ollama_name);
|
|
214
|
+
const ours = Object.prototype.hasOwnProperty.call(owned, m.ollama_name);
|
|
215
|
+
let reason = null;
|
|
216
|
+
if (!settings?.enabled) reason = 'house models are turned off on this node';
|
|
217
|
+
else if (m.uncensored && !settings.uncensored) reason = 'uncensored house models are turned off on this node';
|
|
218
|
+
else if (Number.isFinite(ram) && ram > 0 && ram < m.min_ram_gb) reason = `needs ${m.min_ram_gb} GB RAM, this node has ${ram.toFixed(1)} GB`;
|
|
219
|
+
else if (used + m.size_bytes > capBytes) reason = `over the house-models disk cap of ${settings.maxDiskGb} GB`;
|
|
220
|
+
|
|
221
|
+
if (have) {
|
|
222
|
+
if (reason && ours) plan.remove.push({ name: m.ollama_name, reason });
|
|
223
|
+
else if (reason) plan.skipped.push({ name: m.ollama_name, reason: `${reason} (kept: you installed it yourself)` });
|
|
224
|
+
else { plan.keep.push(m); if (ours) used += m.size_bytes; }
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
if (reason) { plan.skipped.push({ name: m.ollama_name, reason }); continue; }
|
|
228
|
+
const failed = failures[m.ollama_name];
|
|
229
|
+
if (failed && now - Date.parse(failed.at) < FAILURE_BACKOFF_MS) {
|
|
230
|
+
plan.skipped.push({ name: m.ollama_name, reason: `last attempt failed (${failed.error ?? 'error'}); retrying after 24 h` });
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
if (Number.isFinite(free)) {
|
|
234
|
+
const needFree = Math.max(m.min_disk_gb * GB, m.size_bytes + reserve);
|
|
235
|
+
if (free < needFree) {
|
|
236
|
+
plan.skipped.push({
|
|
237
|
+
name: m.ollama_name,
|
|
238
|
+
reason: `not enough free disk: ${(free / GB).toFixed(1)} GB free, needs ${(needFree / GB).toFixed(1)} GB (model + ${(reserve / GB).toFixed(0)} GB reserve)`
|
|
239
|
+
});
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
free -= m.size_bytes;
|
|
243
|
+
}
|
|
244
|
+
used += m.size_bytes;
|
|
245
|
+
plan.pull.push(m);
|
|
246
|
+
}
|
|
247
|
+
return plan;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Check a pulled model against its manifest entry. Ollama's /api/tags digest
|
|
252
|
+
* is the sha256 of the registry manifest, so a pinned digest proves we got
|
|
253
|
+
* exactly the published build. Size must match within 1% (exact for the
|
|
254
|
+
* Ollama registry; the 1% covers registries that recompute small layers).
|
|
255
|
+
*/
|
|
256
|
+
export function verifyPulled(entry, tagsModel) {
|
|
257
|
+
if (!tagsModel) return { ok: false, error: 'model not listed by Ollama after pull' };
|
|
258
|
+
if (entry.digest) {
|
|
259
|
+
const got = stripDigest(tagsModel.digest);
|
|
260
|
+
if (got !== stripDigest(entry.digest)) {
|
|
261
|
+
return { ok: false, error: `digest mismatch: expected ${entry.digest}, got sha256:${got ?? '?'}` };
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
const size = Number(tagsModel.size);
|
|
265
|
+
if (Number.isFinite(size) && size > 0 && Math.abs(size - entry.size_bytes) > entry.size_bytes * 0.01) {
|
|
266
|
+
return { ok: false, error: `size mismatch: expected ${entry.size_bytes} bytes, got ${size}` };
|
|
267
|
+
}
|
|
268
|
+
return { ok: true };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ---------------------------------------------------------------------------
|
|
272
|
+
// State file: what the daemon pulled, the last sync, failures.
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
|
|
275
|
+
export function getStatePath() {
|
|
276
|
+
return path.join(getConfigDir(), 'house-models-state.json');
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
export function readState(p = getStatePath()) {
|
|
280
|
+
try {
|
|
281
|
+
const s = JSON.parse(fsSync.readFileSync(p, 'utf8'));
|
|
282
|
+
return { owned: s.owned ?? {}, failures: s.failures ?? {}, ...s };
|
|
283
|
+
} catch {
|
|
284
|
+
return { owned: {}, failures: {} };
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export async function writeState(state, p = getStatePath()) {
|
|
289
|
+
await fs.mkdir(path.dirname(p), { recursive: true, mode: 0o700 });
|
|
290
|
+
const tmp = `${p}.${process.pid}.tmp`;
|
|
291
|
+
await fs.writeFile(tmp, JSON.stringify(state, null, 2) + '\n', { mode: 0o644 });
|
|
292
|
+
await fs.rename(tmp, p);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// ---------------------------------------------------------------------------
|
|
296
|
+
// Machine probes (Ollama + disk + RAM).
|
|
297
|
+
// ---------------------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
export function ollamaHost(env = process.env) {
|
|
300
|
+
const h = env.OLLAMA_HOST || 'http://localhost:11434';
|
|
301
|
+
return /^https?:\/\//.test(h) ? h : `http://${h}`;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export async function ollamaTags({ host = ollamaHost(), fetchImpl = fetch } = {}) {
|
|
305
|
+
const res = await fetchImpl(new URL('/api/tags', host), { signal: AbortSignal.timeout?.(5000) });
|
|
306
|
+
if (!res.ok) throw new Error(`ollama /api/tags HTTP ${res.status}`);
|
|
307
|
+
const json = await res.json();
|
|
308
|
+
return (Array.isArray(json?.models) ? json.models : []).map((m) => ({
|
|
309
|
+
name: m.name ?? m.model, digest: m.digest ?? null, size: m.size ?? null
|
|
310
|
+
}));
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export async function ollamaPull(name, { host = ollamaHost(), fetchImpl = fetch, onProgress } = {}) {
|
|
314
|
+
const res = await fetchImpl(new URL('/api/pull', host), {
|
|
315
|
+
method: 'POST',
|
|
316
|
+
headers: { 'content-type': 'application/json' },
|
|
317
|
+
body: JSON.stringify({ model: name, name, stream: true })
|
|
318
|
+
});
|
|
319
|
+
if (!res.ok || !res.body) throw new Error(`ollama pull HTTP ${res.status}`);
|
|
320
|
+
const reader = res.body.getReader();
|
|
321
|
+
const dec = new TextDecoder();
|
|
322
|
+
let buf = '';
|
|
323
|
+
let last = null;
|
|
324
|
+
for (;;) {
|
|
325
|
+
const { value, done } = await reader.read();
|
|
326
|
+
if (done) break;
|
|
327
|
+
buf += dec.decode(value, { stream: true });
|
|
328
|
+
let i;
|
|
329
|
+
while ((i = buf.indexOf('\n')) >= 0) {
|
|
330
|
+
const raw = buf.slice(0, i).trim();
|
|
331
|
+
buf = buf.slice(i + 1);
|
|
332
|
+
if (!raw) continue;
|
|
333
|
+
let line;
|
|
334
|
+
try { line = JSON.parse(raw); } catch { continue; }
|
|
335
|
+
if (line.error) throw new Error(`ollama: ${line.error}`);
|
|
336
|
+
last = line;
|
|
337
|
+
onProgress?.(line);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
if (last?.status !== 'success') throw new Error(`ollama pull ended without success (${last?.status ?? 'no output'})`);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export async function ollamaDelete(name, { host = ollamaHost(), fetchImpl = fetch } = {}) {
|
|
344
|
+
const res = await fetchImpl(new URL('/api/delete', host), {
|
|
345
|
+
method: 'DELETE',
|
|
346
|
+
headers: { 'content-type': 'application/json' },
|
|
347
|
+
body: JSON.stringify({ model: name, name })
|
|
348
|
+
});
|
|
349
|
+
if (!res.ok && res.status !== 404) throw new Error(`ollama delete HTTP ${res.status}`);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** Where Ollama keeps blobs, best guess; used only to measure free disk. */
|
|
353
|
+
export function ollamaModelsDir(env = process.env) {
|
|
354
|
+
const candidates = [
|
|
355
|
+
env.OLLAMA_MODELS,
|
|
356
|
+
'/usr/share/ollama/.ollama/models',
|
|
357
|
+
'/var/lib/ollama/.ollama/models',
|
|
358
|
+
'/var/lib/ollama',
|
|
359
|
+
path.join(os.homedir(), '.ollama', 'models')
|
|
360
|
+
].filter(Boolean);
|
|
361
|
+
for (const c of candidates) {
|
|
362
|
+
try { if (fsSync.statSync(c).isDirectory()) return c; } catch { /* next */ }
|
|
363
|
+
}
|
|
364
|
+
return os.homedir();
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
export async function diskSpace(dir = ollamaModelsDir()) {
|
|
368
|
+
try {
|
|
369
|
+
const s = await fs.statfs(dir);
|
|
370
|
+
return { dir, freeBytes: s.bavail * s.bsize, totalBytes: s.blocks * s.bsize };
|
|
371
|
+
} catch {
|
|
372
|
+
return { dir, freeBytes: NaN, totalBytes: NaN };
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
export function ramGb() {
|
|
377
|
+
return os.totalmem() / GB;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// ---------------------------------------------------------------------------
|
|
381
|
+
// One sync pass.
|
|
382
|
+
// ---------------------------------------------------------------------------
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Fetch the manifest, plan, then remove and pull one model at a time.
|
|
386
|
+
* Before each pull it waits up to `busyWaitMs` (10 min) for the node to be
|
|
387
|
+
* idle (`isBusy()`), then pulls anyway: a pull is network and disk work that
|
|
388
|
+
* Ollama does beside inference, and a node that is never idle must still get
|
|
389
|
+
* its house models. Writes the state file. Returns a summary.
|
|
390
|
+
*/
|
|
391
|
+
export async function syncHouseModels({
|
|
392
|
+
settings,
|
|
393
|
+
fetchImpl = fetch,
|
|
394
|
+
host = ollamaHost(),
|
|
395
|
+
statePath = getStatePath(),
|
|
396
|
+
isBusy = () => false,
|
|
397
|
+
busyWaitMs = 10 * 60 * 1000,
|
|
398
|
+
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
399
|
+
log = () => {},
|
|
400
|
+
capacity = null,
|
|
401
|
+
manifest = null
|
|
402
|
+
} = {}) {
|
|
403
|
+
const state = readState(statePath);
|
|
404
|
+
const at = new Date().toISOString();
|
|
405
|
+
const summary = { at, pulled: [], removed: [], skipped: [], errors: [] };
|
|
406
|
+
try {
|
|
407
|
+
const man = manifest ?? await fetchManifest({ url: settings.manifestUrl, fetchImpl });
|
|
408
|
+
for (const r of man.rejected) log(`manifest entry ${r.name} rejected: ${r.reasons.join('; ')}`);
|
|
409
|
+
let installed = await ollamaTags({ host, fetchImpl });
|
|
410
|
+
// Forget ownership of models someone removed by hand.
|
|
411
|
+
const names = new Set(installed.map((m) => normalizeOllamaName(m.name)));
|
|
412
|
+
for (const n of Object.keys(state.owned)) if (!names.has(n)) delete state.owned[n];
|
|
413
|
+
|
|
414
|
+
const disk = capacity?.disk ?? await diskSpace();
|
|
415
|
+
const cap = {
|
|
416
|
+
ramGb: capacity?.ramGb ?? ramGb(),
|
|
417
|
+
diskFreeBytes: disk.freeBytes,
|
|
418
|
+
diskTotalBytes: disk.totalBytes
|
|
419
|
+
};
|
|
420
|
+
const plan = planSync({ manifest: man, installed, owned: state.owned, capacity: cap, settings, failures: state.failures });
|
|
421
|
+
summary.skipped = plan.skipped;
|
|
422
|
+
|
|
423
|
+
for (const r of plan.remove) {
|
|
424
|
+
try {
|
|
425
|
+
await ollamaDelete(r.name, { host, fetchImpl });
|
|
426
|
+
delete state.owned[r.name];
|
|
427
|
+
summary.removed.push(r);
|
|
428
|
+
log(`removed ${r.name}: ${r.reason}`);
|
|
429
|
+
} catch (err) {
|
|
430
|
+
summary.errors.push({ name: r.name, error: err?.message ?? String(err) });
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
for (const m of plan.pull) {
|
|
435
|
+
const waitUntil = Date.now() + busyWaitMs;
|
|
436
|
+
while (isBusy() && Date.now() < waitUntil) await sleep(15_000);
|
|
437
|
+
log(`pulling ${m.ollama_name} (${(m.size_bytes / GB).toFixed(1)} GB)`);
|
|
438
|
+
try {
|
|
439
|
+
let lastLog = 0;
|
|
440
|
+
await ollamaPull(m.source, {
|
|
441
|
+
host, fetchImpl,
|
|
442
|
+
onProgress: (l) => {
|
|
443
|
+
if (Date.now() - lastLog > 60_000 && l.total) {
|
|
444
|
+
lastLog = Date.now();
|
|
445
|
+
log(` ${m.ollama_name}: ${Math.round((l.completed / l.total) * 100)}%`);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
});
|
|
449
|
+
installed = await ollamaTags({ host, fetchImpl });
|
|
450
|
+
const got = installed.find((t) => normalizeOllamaName(t.name) === m.ollama_name);
|
|
451
|
+
const v = verifyPulled(m, got);
|
|
452
|
+
if (!v.ok) {
|
|
453
|
+
await ollamaDelete(m.ollama_name, { host, fetchImpl }).catch(() => {});
|
|
454
|
+
throw new Error(`verification failed, removed: ${v.error}`);
|
|
455
|
+
}
|
|
456
|
+
state.owned[m.ollama_name] = { digest: got.digest ?? null, size_bytes: Number(got.size) || m.size_bytes, pulled_at: new Date().toISOString(), uncensored: m.uncensored };
|
|
457
|
+
delete state.failures[m.ollama_name];
|
|
458
|
+
summary.pulled.push(m.ollama_name);
|
|
459
|
+
log(`pulled and verified ${m.ollama_name}`);
|
|
460
|
+
} catch (err) {
|
|
461
|
+
const error = (err?.message ?? String(err)).slice(0, 300);
|
|
462
|
+
state.failures[m.ollama_name] = { at: new Date().toISOString(), error };
|
|
463
|
+
summary.errors.push({ name: m.ollama_name, error });
|
|
464
|
+
log(`pull ${m.ollama_name} failed: ${error}`);
|
|
465
|
+
}
|
|
466
|
+
await writeState({ ...state, last_sync: summary }, statePath).catch(() => {});
|
|
467
|
+
}
|
|
468
|
+
state.manifest = { url: settings.manifestUrl, version: man.version, updated: man.updated, models: man.models.map((m) => m.ollama_name) };
|
|
469
|
+
} catch (err) {
|
|
470
|
+
summary.errors.push({ name: null, error: err?.message ?? String(err) });
|
|
471
|
+
log(`sync failed: ${err?.message ?? err}`);
|
|
472
|
+
}
|
|
473
|
+
state.settings = { enabled: settings.enabled, uncensored: settings.uncensored, maxDiskGb: settings.maxDiskGb };
|
|
474
|
+
state.last_sync = summary;
|
|
475
|
+
await writeState(state, statePath).catch(() => {});
|
|
476
|
+
return summary;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/** Human summary for `infernet status` / `infernet house-models status`. */
|
|
480
|
+
export function formatStatus({ settings, state, installed = null }) {
|
|
481
|
+
const lines = [];
|
|
482
|
+
const on = settings.enabled ? 'on' : `off (${settings.sources?.enabled ?? 'config'})`;
|
|
483
|
+
const unc = settings.uncensored ? 'on' : `off (${settings.sources?.uncensored ?? 'config'})`;
|
|
484
|
+
const cap = settings.maxDiskGb != null ? `${settings.maxDiskGb} GB` : 'none (always leaves 10% of the disk free, 10-50 GB)';
|
|
485
|
+
lines.push(`House models: ${on}; uncensored: ${unc}; disk cap: ${cap}`);
|
|
486
|
+
const owned = state?.owned ?? {};
|
|
487
|
+
const names = Object.keys(owned);
|
|
488
|
+
const installedNames = installed ? new Set(installed.map((m) => normalizeOllamaName(m.name))) : null;
|
|
489
|
+
let total = 0;
|
|
490
|
+
for (const n of names) {
|
|
491
|
+
const o = owned[n];
|
|
492
|
+
total += Number(o.size_bytes) || 0;
|
|
493
|
+
const gone = installedNames && !installedNames.has(n) ? ' (missing from Ollama)' : '';
|
|
494
|
+
lines.push(` - ${n} ${((Number(o.size_bytes) || 0) / GB).toFixed(1)} GB${o.uncensored ? ' [uncensored]' : ''}${gone}`);
|
|
495
|
+
}
|
|
496
|
+
if (names.length === 0) lines.push(' (none pulled yet)');
|
|
497
|
+
lines.push(` disk used by house models: ${(total / GB).toFixed(1)} GB`);
|
|
498
|
+
const ls = state?.last_sync;
|
|
499
|
+
if (ls?.at) {
|
|
500
|
+
lines.push(` last sync: ${ls.at}${ls.errors?.length ? `, ${ls.errors.length} error(s): ${ls.errors.map((e) => `${e.name ?? 'manifest'}: ${e.error}`).join('; ')}` : ''}`);
|
|
501
|
+
for (const s of ls.skipped ?? []) lines.push(` skipped ${s.name}: ${s.reason}`);
|
|
502
|
+
}
|
|
503
|
+
lines.push(' opt out: infernet house-models off | uncensored only: infernet house-models uncensored off');
|
|
504
|
+
return lines.join('\n');
|
|
505
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.83",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,14 +32,14 @@
|
|
|
32
32
|
"ora": "^9.4.1",
|
|
33
33
|
"react": "^17.0.2",
|
|
34
34
|
"react-blessed": "^0.7.2",
|
|
35
|
-
"@infernetprotocol/auth": "0.1.
|
|
36
|
-
"@infernetprotocol/config": "0.1.
|
|
37
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
38
|
-
"@infernetprotocol/engine": "0.1.
|
|
39
|
-
"@infernetprotocol/gpu": "0.1.
|
|
40
|
-
"@infernetprotocol/liquid-agent": "0.1.
|
|
41
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
42
|
-
"@infernetprotocol/training": "0.1.
|
|
35
|
+
"@infernetprotocol/auth": "0.1.83",
|
|
36
|
+
"@infernetprotocol/config": "0.1.83",
|
|
37
|
+
"@infernetprotocol/deploy-providers": "0.1.83",
|
|
38
|
+
"@infernetprotocol/engine": "0.1.83",
|
|
39
|
+
"@infernetprotocol/gpu": "0.1.83",
|
|
40
|
+
"@infernetprotocol/liquid-agent": "0.1.83",
|
|
41
|
+
"@infernetprotocol/rpc-adapter": "0.1.83",
|
|
42
|
+
"@infernetprotocol/training": "0.1.83"
|
|
43
43
|
},
|
|
44
44
|
"repository": {
|
|
45
45
|
"type": "git",
|