@infernetprotocol/cli 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/commands/chat.js +268 -0
- package/commands/console.js +229 -0
- package/commands/debug.js +222 -0
- package/commands/deploy.js +524 -0
- package/commands/doctor.js +501 -0
- package/commands/firewall.js +39 -0
- package/commands/gpu.js +130 -0
- package/commands/help.js +54 -0
- package/commands/init.js +304 -0
- package/commands/login.js +305 -0
- package/commands/logs.js +93 -0
- package/commands/model.js +359 -0
- package/commands/payments.js +92 -0
- package/commands/payout.js +128 -0
- package/commands/pubkey.js +116 -0
- package/commands/register.js +304 -0
- package/commands/remove.js +230 -0
- package/commands/service.js +255 -0
- package/commands/setup.js +869 -0
- package/commands/start.js +903 -0
- package/commands/stats.js +85 -0
- package/commands/status.js +116 -0
- package/commands/stop.js +81 -0
- package/commands/train.js +563 -0
- package/commands/tui.js +285 -0
- package/commands/uncensored.js +222 -0
- package/commands/update.js +80 -0
- package/commands/upgrade.js +2 -0
- package/examples/app.js +153 -0
- package/index.js +207 -0
- package/lib/chat-executor.js +232 -0
- package/lib/config.js +148 -0
- package/lib/daemonize.js +46 -0
- package/lib/firewall.js +190 -0
- package/lib/hf-model.js +166 -0
- package/lib/identity.js +18 -0
- package/lib/ipc.js +114 -0
- package/lib/model-fit.js +67 -0
- package/lib/model-key.js +52 -0
- package/lib/network.js +240 -0
- package/lib/nip44.js +130 -0
- package/lib/node-client.js +152 -0
- package/lib/peers.js +133 -0
- package/lib/prompt.js +49 -0
- package/lib/remote-chat.js +115 -0
- package/lib/route.js +44 -0
- package/lib/runtime-config.js +130 -0
- package/lib/version.js +32 -0
- package/package.json +57 -0
|
@@ -0,0 +1,903 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `infernet start` — run the node daemon loop.
|
|
3
|
+
*
|
|
4
|
+
* By default, `infernet start` spawns a **detached** background process and
|
|
5
|
+
* returns immediately; logs go to `~/.config/infernet/daemon.log`. Use
|
|
6
|
+
* `--foreground` to run the loop in the current terminal (useful under
|
|
7
|
+
* systemd / Docker / Kubernetes where the supervisor wants the process in
|
|
8
|
+
* the foreground).
|
|
9
|
+
*
|
|
10
|
+
* The running daemon:
|
|
11
|
+
* - Heartbeats every 30s via signed POST /api/v1/node/heartbeat
|
|
12
|
+
* - Polls /api/v1/node/jobs/poll every 15s (providers only), processes
|
|
13
|
+
* any returned jobs, emits streaming events via signed
|
|
14
|
+
* POST /api/v1/node/jobs/:id/events, and closes the loop with
|
|
15
|
+
* POST /api/v1/node/jobs/:id/complete.
|
|
16
|
+
* - Exposes a Unix-domain IPC socket at `~/.config/infernet/daemon.sock`
|
|
17
|
+
* so `infernet status`, `infernet stats`, `infernet stop`, etc. can ask
|
|
18
|
+
* the live process what it's doing.
|
|
19
|
+
* - Handles SIGINT / SIGTERM: sends a final heartbeat with status=offline,
|
|
20
|
+
* removes pid/sock files, exits 0.
|
|
21
|
+
*
|
|
22
|
+
* The daemon never holds a database credential. Every request to the
|
|
23
|
+
* control plane is signed with the node's Nostr privkey.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import fs from 'node:fs/promises';
|
|
27
|
+
import { openSync } from 'node:fs';
|
|
28
|
+
import net from 'node:net';
|
|
29
|
+
import http from 'node:http';
|
|
30
|
+
import { spawn } from 'node:child_process';
|
|
31
|
+
import { chmodSync, unlinkSync } from 'node:fs';
|
|
32
|
+
import { readFileSync } from 'node:fs';
|
|
33
|
+
import { fileURLToPath } from 'node:url';
|
|
34
|
+
import { dirname, join } from 'node:path';
|
|
35
|
+
|
|
36
|
+
import {
|
|
37
|
+
getDaemonPidPath,
|
|
38
|
+
getDaemonSocketPath,
|
|
39
|
+
getDaemonLogPath,
|
|
40
|
+
loadConfig,
|
|
41
|
+
saveConfig,
|
|
42
|
+
fixConfigPermissions
|
|
43
|
+
} from '../lib/config.js';
|
|
44
|
+
import { spawnDetachedDaemon } from '../lib/daemonize.js';
|
|
45
|
+
import { isDaemonAlive } from '../lib/ipc.js';
|
|
46
|
+
import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/network.js';
|
|
47
|
+
import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
|
|
48
|
+
import { gatherCoarseSpecs } from './register.js';
|
|
49
|
+
import { checkModelFits } from '../lib/model-fit.js';
|
|
50
|
+
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
51
|
+
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
52
|
+
import { pullLatestBinary } from './upgrade.js';
|
|
53
|
+
import { nodeTokenLogin } from './login.js';
|
|
54
|
+
import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from '../lib/version.js';
|
|
55
|
+
|
|
56
|
+
const HELP = `infernet start — run the node daemon
|
|
57
|
+
|
|
58
|
+
Usage:
|
|
59
|
+
infernet start [flags]
|
|
60
|
+
|
|
61
|
+
Flags:
|
|
62
|
+
--foreground Run in the current terminal (don't detach)
|
|
63
|
+
--heartbeat-interval <ms> Override heartbeat cadence (default 30000)
|
|
64
|
+
--poll-interval <ms> Override job poll cadence (default 15000)
|
|
65
|
+
--p2p-port <n> TCP port for peer connections (default 46337)
|
|
66
|
+
--no-p2p Don't bind the P2P TCP listener
|
|
67
|
+
--no-advertise Don't send address/port in heartbeats
|
|
68
|
+
--once Run one heartbeat + one poll and exit (debug)
|
|
69
|
+
--help Show this help
|
|
70
|
+
|
|
71
|
+
Daemon logs to \`~/.config/infernet/daemon.log\`, exposes an IPC socket for
|
|
72
|
+
live queries (see \`infernet status\`, \`infernet stats\`, \`infernet logs\`).
|
|
73
|
+
`;
|
|
74
|
+
|
|
75
|
+
const DEFAULT_HEARTBEAT_MS = 30_000;
|
|
76
|
+
const DEFAULT_POLL_MS = 15_000;
|
|
77
|
+
const UPDATE_CHECK_MS = 5 * 60 * 1000; // 5 minutes
|
|
78
|
+
|
|
79
|
+
async function writePidFile(pid) {
|
|
80
|
+
const p = getDaemonPidPath();
|
|
81
|
+
await fs.writeFile(p, String(pid), { mode: 0o600 });
|
|
82
|
+
return p;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function removePidFile() {
|
|
86
|
+
try { await fs.unlink(getDaemonPidPath()); } catch { /* ignore */ }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function removeSocketFile() {
|
|
90
|
+
try { unlinkSync(getDaemonSocketPath()); } catch { /* ignore */ }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
// Entry point — branch between "spawn detached" and "run foreground loop".
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
export default async function start(args, ctx) {
|
|
97
|
+
if (args.has('help') || args.has('h')) {
|
|
98
|
+
process.stdout.write(HELP);
|
|
99
|
+
return 0;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const foreground = args.has('foreground') || process.env.INFERNET_DAEMON_FOREGROUND === '1';
|
|
103
|
+
if (!foreground) {
|
|
104
|
+
return spawnAndReturn(args);
|
|
105
|
+
}
|
|
106
|
+
return runDaemon(args, ctx);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function spawnAndReturn(args) {
|
|
110
|
+
const alreadyAlive = await isDaemonAlive();
|
|
111
|
+
if (alreadyAlive) {
|
|
112
|
+
process.stderr.write('A daemon is already running (socket responsive). Run `infernet stop` first.\n');
|
|
113
|
+
return 1;
|
|
114
|
+
}
|
|
115
|
+
removeSocketFile();
|
|
116
|
+
|
|
117
|
+
const passthrough = [];
|
|
118
|
+
const hb = args.get('heartbeat-interval');
|
|
119
|
+
if (hb) passthrough.push('--heartbeat-interval', hb);
|
|
120
|
+
const poll = args.get('poll-interval');
|
|
121
|
+
if (poll) passthrough.push('--poll-interval', poll);
|
|
122
|
+
const p2pPort = args.get('p2p-port');
|
|
123
|
+
if (p2pPort) passthrough.push('--p2p-port', p2pPort);
|
|
124
|
+
if (args.has('no-p2p')) passthrough.push('--no-p2p');
|
|
125
|
+
if (args.has('no-advertise')) passthrough.push('--no-advertise');
|
|
126
|
+
if (args.has('once')) passthrough.push('--once');
|
|
127
|
+
|
|
128
|
+
const { pid, logPath } = spawnDetachedDaemon(passthrough);
|
|
129
|
+
process.stdout.write(`infernet daemon started (pid ${pid})\n`);
|
|
130
|
+
process.stdout.write(` logs: ${logPath}\n`);
|
|
131
|
+
process.stdout.write(` socket: ${getDaemonSocketPath()}\n`);
|
|
132
|
+
process.stdout.write('Tail logs with `infernet logs -f`, query live state with `infernet status`.\n');
|
|
133
|
+
return 0;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function runDaemon(args, ctx) {
|
|
137
|
+
const { config, client, configPath } = ctx;
|
|
138
|
+
const node = config.node ?? {};
|
|
139
|
+
if (!node.role || !node.nodeId) {
|
|
140
|
+
process.stderr.write('Config is missing node.role/node.nodeId. Run `infernet init` first.\n');
|
|
141
|
+
return 1;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const heartbeatMs = Number.parseInt(args.get('heartbeat-interval') ?? '', 10) || DEFAULT_HEARTBEAT_MS;
|
|
145
|
+
const pollMs = Number.parseInt(args.get('poll-interval') ?? '', 10) || DEFAULT_POLL_MS;
|
|
146
|
+
const once = args.has('once');
|
|
147
|
+
|
|
148
|
+
const p2pDisabled = args.has('no-p2p');
|
|
149
|
+
const noAdvertise = args.has('no-advertise') || node.address === null;
|
|
150
|
+
// Bind port (what we listen on, locally) and advertised port (what we
|
|
151
|
+
// tell the control plane to dial). Same value 99% of the time, but
|
|
152
|
+
// hosting platforms that NAT the container (RunPod, anything with
|
|
153
|
+
// dynamic port mapping) need them split — daemon binds to e.g. 46337
|
|
154
|
+
// inside the container, but the public address:port is something like
|
|
155
|
+
// 213.173.107.42:21517. The cloud-init detects RunPod env vars and
|
|
156
|
+
// exports INFERNET_BIND_PORT (internal) + INFERNET_PUBLIC_PORT
|
|
157
|
+
// (external) separately.
|
|
158
|
+
const bindPort = Number.parseInt(
|
|
159
|
+
args.get('bind-port') ?? process.env.INFERNET_BIND_PORT ?? '',
|
|
160
|
+
10
|
|
161
|
+
) || Number.parseInt(args.get('p2p-port') ?? '', 10) || resolveP2pPort(config);
|
|
162
|
+
const advertisedPort = Number.parseInt(
|
|
163
|
+
args.get('p2p-port') ?? process.env.INFERNET_PUBLIC_PORT ?? '',
|
|
164
|
+
10
|
|
165
|
+
) || resolveP2pPort(config) || bindPort;
|
|
166
|
+
// Back-compat: keep the existing variable name pointing at bindPort,
|
|
167
|
+
// since the rest of the file (P2P listener, formatEndpoint, etc.)
|
|
168
|
+
// wants the bind value.
|
|
169
|
+
const p2pPort = bindPort;
|
|
170
|
+
const advertisedAddress = noAdvertise ? null : (node.address ?? await detectLocalAddress());
|
|
171
|
+
|
|
172
|
+
const startedAt = new Date();
|
|
173
|
+
const stats = {
|
|
174
|
+
heartbeatsOk: 0,
|
|
175
|
+
heartbeatsFailed: 0,
|
|
176
|
+
lastHeartbeatAt: null,
|
|
177
|
+
lastHeartbeatError: null,
|
|
178
|
+
jobsPicked: 0,
|
|
179
|
+
jobsCompleted: 0,
|
|
180
|
+
jobsFailed: 0,
|
|
181
|
+
lastJobAt: null,
|
|
182
|
+
lastPollAt: null,
|
|
183
|
+
pollsOk: 0,
|
|
184
|
+
pollsFailed: 0,
|
|
185
|
+
activeJobIds: new Set()
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
const pidPath = await writePidFile(process.pid);
|
|
189
|
+
const socketPath = getDaemonSocketPath();
|
|
190
|
+
removeSocketFile();
|
|
191
|
+
|
|
192
|
+
// Self-heal: fix config permissions on every daemon start so nodes
|
|
193
|
+
// upgraded before the 0600 enforcement still get corrected.
|
|
194
|
+
fixConfigPermissions().catch(() => {});
|
|
195
|
+
|
|
196
|
+
process.stdout.write('infernet daemon starting\n');
|
|
197
|
+
process.stdout.write(` node_id: ${node.nodeId}\n`);
|
|
198
|
+
process.stdout.write(` role: ${node.role}\n`);
|
|
199
|
+
process.stdout.write(` control: ${config.controlPlane?.url ?? '(not set)'}\n`);
|
|
200
|
+
process.stdout.write(` pid: ${process.pid}\n`);
|
|
201
|
+
process.stdout.write(` pidfile: ${pidPath}\n`);
|
|
202
|
+
process.stdout.write(` socket: ${socketPath}\n`);
|
|
203
|
+
process.stdout.write(` config: ${configPath}\n`);
|
|
204
|
+
process.stdout.write(` heartbeat: ${heartbeatMs}ms\n`);
|
|
205
|
+
process.stdout.write(` poll: ${pollMs}ms\n`);
|
|
206
|
+
if (!p2pDisabled) {
|
|
207
|
+
process.stdout.write(` p2p: ${formatEndpoint(advertisedAddress ?? '-', advertisedPort)}${bindPort !== advertisedPort ? ` (binds locally :${bindPort})` : ''}\n`);
|
|
208
|
+
} else {
|
|
209
|
+
process.stdout.write(' p2p: disabled\n');
|
|
210
|
+
}
|
|
211
|
+
if (noAdvertise) process.stdout.write(' advertise: off (outbound-only)\n');
|
|
212
|
+
process.stdout.write(` version: v${CURRENT_VERSION} (auto-update every ${UPDATE_CHECK_MS / 60_000}min)\n`);
|
|
213
|
+
|
|
214
|
+
let heartbeatTimer = null;
|
|
215
|
+
let pollTimer = null;
|
|
216
|
+
let shuttingDown = false;
|
|
217
|
+
let ipcServer = null;
|
|
218
|
+
let p2pServer = null;
|
|
219
|
+
let healthServer = null;
|
|
220
|
+
let p2pConnections = 0;
|
|
221
|
+
let p2pLastConnectionAt = null;
|
|
222
|
+
|
|
223
|
+
// Specs cache — re-detection (nvidia-smi, /sys/class/infiniband, Ollama
|
|
224
|
+
// /api/tags) is cheap but not free. Refresh every SPECS_TTL_MS so the
|
|
225
|
+
// control plane sees current served-models / hardware without paying
|
|
226
|
+
// the detection cost on every 30s heartbeat.
|
|
227
|
+
const SPECS_TTL_MS = 5 * 60 * 1000;
|
|
228
|
+
let cachedSpecs = null;
|
|
229
|
+
let cachedSpecsAt = 0;
|
|
230
|
+
|
|
231
|
+
// Rolling benchmark — last N completed chat jobs' (tokens, duration)
|
|
232
|
+
// pairs. Used to compute tokens_per_second_avg for the heartbeat,
|
|
233
|
+
// which feeds speed-aware routing on the control plane. Capped so
|
|
234
|
+
// memory stays bounded across long-running daemons.
|
|
235
|
+
const BENCH_RING_MAX = 32;
|
|
236
|
+
const benchRing = [];
|
|
237
|
+
|
|
238
|
+
function recordBench({ token_count, duration_ms }) {
|
|
239
|
+
if (!Number.isFinite(token_count) || !Number.isFinite(duration_ms) || duration_ms <= 0) return;
|
|
240
|
+
if (token_count <= 0) return;
|
|
241
|
+
benchRing.push({ tokens: token_count, ms: duration_ms, at: Date.now() });
|
|
242
|
+
while (benchRing.length > BENCH_RING_MAX) benchRing.shift();
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function benchSummary() {
|
|
246
|
+
if (benchRing.length === 0) return null;
|
|
247
|
+
const totalTokens = benchRing.reduce((a, e) => a + e.tokens, 0);
|
|
248
|
+
const totalMs = benchRing.reduce((a, e) => a + e.ms, 0);
|
|
249
|
+
if (totalMs <= 0) return null;
|
|
250
|
+
const tps = (totalTokens / totalMs) * 1000;
|
|
251
|
+
return {
|
|
252
|
+
tokens_per_second_avg: +tps.toFixed(2),
|
|
253
|
+
samples: benchRing.length,
|
|
254
|
+
window_started_at: new Date(benchRing[0].at).toISOString()
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Reachability self-check — daemon calls /api/probe?host=&port=
|
|
260
|
+
* once after starting the P2P listener. Result cached on the
|
|
261
|
+
* specs object so it ships with heartbeats and surfaces in the
|
|
262
|
+
* dashboard. Refreshed every REACHABILITY_TTL_MS so a router
|
|
263
|
+
* port-forward added later gets picked up without restart.
|
|
264
|
+
*/
|
|
265
|
+
const REACHABILITY_TTL_MS = 10 * 60 * 1000;
|
|
266
|
+
let cachedReachable = null;
|
|
267
|
+
let cachedReachableAt = 0;
|
|
268
|
+
|
|
269
|
+
async function probeReachable() {
|
|
270
|
+
if (noAdvertise || !advertisedAddress || !p2pPort || p2pDisabled) return null;
|
|
271
|
+
const now = Date.now();
|
|
272
|
+
if (cachedReachable !== null && now - cachedReachableAt < REACHABILITY_TTL_MS) {
|
|
273
|
+
return cachedReachable;
|
|
274
|
+
}
|
|
275
|
+
const baseUrl = config.controlPlane?.url;
|
|
276
|
+
if (!baseUrl) return null;
|
|
277
|
+
try {
|
|
278
|
+
const url = new URL("/api/probe", baseUrl);
|
|
279
|
+
url.searchParams.set("host", advertisedAddress);
|
|
280
|
+
// Probe the externally-advertised port — that's what we're
|
|
281
|
+
// claiming is reachable, so that's what we want verified.
|
|
282
|
+
url.searchParams.set("port", String(advertisedPort));
|
|
283
|
+
const res = await fetch(url, { signal: AbortSignal.timeout?.(7000) });
|
|
284
|
+
if (!res.ok) return null;
|
|
285
|
+
const body = await res.json();
|
|
286
|
+
cachedReachable = {
|
|
287
|
+
ok: !!body.reachable,
|
|
288
|
+
error: body.error ?? null,
|
|
289
|
+
checked_at: new Date(now).toISOString()
|
|
290
|
+
};
|
|
291
|
+
cachedReachableAt = now;
|
|
292
|
+
return cachedReachable;
|
|
293
|
+
} catch {
|
|
294
|
+
return null;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Live load snapshot — fresh on every heartbeat, NOT cached. The
|
|
300
|
+
* picker on the control plane uses this to skip saturated nodes
|
|
301
|
+
* and weight by remaining headroom. Cheap to compute (one
|
|
302
|
+
* nvidia-smi shellout, one os.freemem read) — well worth the
|
|
303
|
+
* cost on each 30s heartbeat compared to the cost of routing a
|
|
304
|
+
* job to a saturated provider that 504s mid-stream.
|
|
305
|
+
*/
|
|
306
|
+
async function liveLoad() {
|
|
307
|
+
const host = detectHost();
|
|
308
|
+
const totalRamGb = host.total_ram_mb / 1024;
|
|
309
|
+
const freeRamGb = host.free_ram_mb / 1024;
|
|
310
|
+
let freeVramGb = 0;
|
|
311
|
+
let totalVramGb = 0;
|
|
312
|
+
let maxGpuUtilization = null;
|
|
313
|
+
try {
|
|
314
|
+
const gpus = await detectGpus();
|
|
315
|
+
for (const g of gpus) {
|
|
316
|
+
if (Number.isFinite(g.vram_mb)) totalVramGb += g.vram_mb / 1024;
|
|
317
|
+
if (Number.isFinite(g.vram_mb) && Number.isFinite(g.vram_used_mb)) {
|
|
318
|
+
freeVramGb += Math.max(0, (g.vram_mb - g.vram_used_mb) / 1024);
|
|
319
|
+
}
|
|
320
|
+
if (Number.isFinite(g.utilization)) {
|
|
321
|
+
maxGpuUtilization = Math.max(maxGpuUtilization ?? 0, g.utilization);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
} catch {
|
|
325
|
+
// Best-effort; CPU-only boxes report zero GPU stats.
|
|
326
|
+
}
|
|
327
|
+
return {
|
|
328
|
+
active_jobs: stats.activeJobIds.size,
|
|
329
|
+
load_avg_1m: Array.isArray(host.load_avg) ? +host.load_avg[0]?.toFixed(2) : null,
|
|
330
|
+
ram: {
|
|
331
|
+
total_gb: +totalRamGb.toFixed(2),
|
|
332
|
+
free_gb: +freeRamGb.toFixed(2)
|
|
333
|
+
},
|
|
334
|
+
vram: {
|
|
335
|
+
total_gb: +totalVramGb.toFixed(2),
|
|
336
|
+
free_gb: +freeVramGb.toFixed(2)
|
|
337
|
+
},
|
|
338
|
+
gpu_utilization_max: maxGpuUtilization,
|
|
339
|
+
measured_at: new Date().toISOString()
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
async function freshSpecs() {
|
|
344
|
+
if (node.role !== 'provider') return null;
|
|
345
|
+
const now = Date.now();
|
|
346
|
+
let base;
|
|
347
|
+
if (cachedSpecs && now - cachedSpecsAt < SPECS_TTL_MS) {
|
|
348
|
+
base = cachedSpecs;
|
|
349
|
+
} else {
|
|
350
|
+
try {
|
|
351
|
+
cachedSpecs = await gatherCoarseSpecs();
|
|
352
|
+
cachedSpecsAt = now;
|
|
353
|
+
base = cachedSpecs;
|
|
354
|
+
} catch (err) {
|
|
355
|
+
process.stderr.write(`specs detection failed: ${err?.message ?? err}\n`);
|
|
356
|
+
base = cachedSpecs;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
if (!base) return null;
|
|
360
|
+
// Always re-read load (cheap, must be current). bench rolls
|
|
361
|
+
// independently as jobs complete. reachability has its own
|
|
362
|
+
// TTL so we don't hammer /api/probe on every heartbeat.
|
|
363
|
+
const bench = benchSummary();
|
|
364
|
+
const load = await liveLoad();
|
|
365
|
+
const reachable = await probeReachable();
|
|
366
|
+
|
|
367
|
+
// IPIP-0028: ensure a keypair exists for each served model and
|
|
368
|
+
// advertise their pubkeys so consumers can encrypt to model keys.
|
|
369
|
+
let modelKeys = {};
|
|
370
|
+
try {
|
|
371
|
+
const servedModels = Array.isArray(base.served_models) ? base.served_models : [];
|
|
372
|
+
for (const name of servedModels) {
|
|
373
|
+
const kp = await getOrCreateModelKey(name);
|
|
374
|
+
if (kp?.publicKey) modelKeys[name] = kp.publicKey;
|
|
375
|
+
}
|
|
376
|
+
} catch {
|
|
377
|
+
// non-fatal — fall back to empty model_keys
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
return {
|
|
381
|
+
...base,
|
|
382
|
+
...(bench ? { bench } : {}),
|
|
383
|
+
load,
|
|
384
|
+
...(reachable ? { reachable } : {}),
|
|
385
|
+
...(Object.keys(modelKeys).length > 0 ? { model_keys: modelKeys } : {}),
|
|
386
|
+
public_key: config?.node?.publicKey ?? undefined,
|
|
387
|
+
cli_version: CURRENT_VERSION
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
async function heartbeat() {
|
|
392
|
+
const payload = { status: 'available' };
|
|
393
|
+
if (!noAdvertise) {
|
|
394
|
+
if (advertisedAddress) payload.address = advertisedAddress;
|
|
395
|
+
// Heartbeat advertises the EXTERNAL port (what clients should
|
|
396
|
+
// dial), not the internal bind port. RunPod / NAT'd hosts:
|
|
397
|
+
// bindPort = 46337 (container), advertisedPort = 21517 (edge).
|
|
398
|
+
if (!p2pDisabled) payload.port = advertisedPort;
|
|
399
|
+
}
|
|
400
|
+
// Provider role only: include current specs so the control plane
|
|
401
|
+
// sees fresh CPU / GPU / served_models without requiring a manual
|
|
402
|
+
// `infernet register` after each capability change.
|
|
403
|
+
const specs = await freshSpecs();
|
|
404
|
+
if (specs) payload.specs = specs;
|
|
405
|
+
try {
|
|
406
|
+
await client.heartbeat(payload);
|
|
407
|
+
stats.heartbeatsOk += 1;
|
|
408
|
+
stats.lastHeartbeatAt = new Date().toISOString();
|
|
409
|
+
stats.lastHeartbeatError = null;
|
|
410
|
+
process.stdout.write(`[${stats.lastHeartbeatAt}] heartbeat ok\n`);
|
|
411
|
+
} catch (err) {
|
|
412
|
+
stats.heartbeatsFailed += 1;
|
|
413
|
+
stats.lastHeartbeatError = err?.message ?? String(err);
|
|
414
|
+
process.stderr.write(`heartbeat error: ${stats.lastHeartbeatError}\n`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
async function processJob(job) {
|
|
419
|
+
const t0 = new Date().toISOString();
|
|
420
|
+
stats.jobsPicked += 1;
|
|
421
|
+
stats.activeJobIds.add(job.id);
|
|
422
|
+
stats.lastJobAt = t0;
|
|
423
|
+
process.stdout.write(`[${t0}] picking up job ${job.id} type=${job.type ?? 'inference'} (${job.title ?? 'untitled'})\n`);
|
|
424
|
+
|
|
425
|
+
try {
|
|
426
|
+
let resultPayload;
|
|
427
|
+
if (job.type === 'chat') {
|
|
428
|
+
const result = await executeChatJob({ client, job, node });
|
|
429
|
+
recordBench(result);
|
|
430
|
+
resultPayload = {
|
|
431
|
+
type: 'chat',
|
|
432
|
+
text: result.text,
|
|
433
|
+
token_count: result.token_count,
|
|
434
|
+
duration_ms: result.duration_ms,
|
|
435
|
+
completed_by: node.nodeId
|
|
436
|
+
};
|
|
437
|
+
} else {
|
|
438
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
439
|
+
resultPayload = { stub: true, completed_by: node.nodeId };
|
|
440
|
+
}
|
|
441
|
+
await client.completeJob(job.id, { status: 'completed', result: resultPayload });
|
|
442
|
+
stats.jobsCompleted += 1;
|
|
443
|
+
stats.activeJobIds.delete(job.id);
|
|
444
|
+
process.stdout.write(`[${new Date().toISOString()}] completed job ${job.id} type=${job.type ?? 'inference'}\n`);
|
|
445
|
+
} catch (err) {
|
|
446
|
+
stats.jobsFailed += 1;
|
|
447
|
+
stats.activeJobIds.delete(job.id);
|
|
448
|
+
const msg = err?.message ?? String(err);
|
|
449
|
+
process.stderr.write(`job ${job.id} failed: ${msg}\n`);
|
|
450
|
+
if (job.type === 'chat') {
|
|
451
|
+
await failChatJob({ client, jobId: job.id, message: msg });
|
|
452
|
+
}
|
|
453
|
+
try {
|
|
454
|
+
await client.failJob(job.id, msg);
|
|
455
|
+
} catch (markErr) {
|
|
456
|
+
process.stderr.write(`failJob failed: ${markErr?.message ?? markErr}\n`);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
async function pollJobs() {
|
|
462
|
+
stats.lastPollAt = new Date().toISOString();
|
|
463
|
+
if (node.role !== 'provider') {
|
|
464
|
+
stats.pollsOk += 1;
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
try {
|
|
468
|
+
const result = await client.pollJobs({ limit: 5 });
|
|
469
|
+
stats.pollsOk += 1;
|
|
470
|
+
for (const job of result?.jobs ?? []) {
|
|
471
|
+
// Skip jobs already being processed by this daemon — the
|
|
472
|
+
// server keeps returning 'assigned' jobs until we mark them
|
|
473
|
+
// complete, so without this guard the poll loop queues the
|
|
474
|
+
// same job on every tick while it's in flight.
|
|
475
|
+
if (stats.activeJobIds.has(job.id)) continue;
|
|
476
|
+
await processJob(job);
|
|
477
|
+
}
|
|
478
|
+
} catch (err) {
|
|
479
|
+
stats.pollsFailed += 1;
|
|
480
|
+
process.stderr.write(`job poll error: ${err?.message ?? err}\n`);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Owner-issued node commands (model_install / model_remove). Same
|
|
486
|
+
* outbound-poll pattern as jobs — no inbound connectivity needed.
|
|
487
|
+
* Auth at the server side: only owners (verified via pubkey_links)
|
|
488
|
+
* can write to node_commands; the daemon's signed poll restricts
|
|
489
|
+
* which rows it sees by pubkey match. So a compromised dashboard
|
|
490
|
+
* account can only push commands to nodes that account already
|
|
491
|
+
* owns; a compromised daemon key can only execute commands
|
|
492
|
+
* targeting itself.
|
|
493
|
+
*/
|
|
494
|
+
async function pollNodeCommands() {
|
|
495
|
+
if (node.role !== 'provider') return;
|
|
496
|
+
let result;
|
|
497
|
+
try {
|
|
498
|
+
result = await client.pollCommands(5);
|
|
499
|
+
} catch (err) {
|
|
500
|
+
process.stderr.write(`command poll error: ${err?.message ?? err}\n`);
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
for (const cmd of result?.commands ?? []) {
|
|
504
|
+
await runNodeCommand(cmd);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async function runNodeCommand(cmd) {
|
|
509
|
+
const { id, command, args } = cmd;
|
|
510
|
+
const t0 = new Date().toISOString();
|
|
511
|
+
process.stdout.write(`[${t0}] running command ${id}: ${command} ${JSON.stringify(args)}\n`);
|
|
512
|
+
try {
|
|
513
|
+
let result;
|
|
514
|
+
if (command === 'model_install') {
|
|
515
|
+
const modelName = String(args?.model);
|
|
516
|
+
const fits = await checkModelFits(modelName);
|
|
517
|
+
if (fits && !fits.ok) {
|
|
518
|
+
throw new Error(
|
|
519
|
+
`${modelName} (≈${fits.size_gb} GB) won't fit on this host ` +
|
|
520
|
+
`(${fits.mode} ceiling ≈ ${fits.ceiling_gb} GB with ${fits.have_gb} GB available)`
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
result = await ollamaSpawn(['pull', modelName]);
|
|
524
|
+
} else if (command === 'model_remove') {
|
|
525
|
+
result = await ollamaSpawn(['rm', String(args?.model)]);
|
|
526
|
+
} else {
|
|
527
|
+
throw new Error(`unknown command verb: ${command}`);
|
|
528
|
+
}
|
|
529
|
+
await client.completeCommand(id, { status: 'completed', result });
|
|
530
|
+
process.stdout.write(`[${new Date().toISOString()}] completed command ${id}\n`);
|
|
531
|
+
} catch (err) {
|
|
532
|
+
const msg = err?.message ?? String(err);
|
|
533
|
+
process.stderr.write(`command ${id} failed: ${msg}\n`);
|
|
534
|
+
try {
|
|
535
|
+
await client.completeCommand(id, { status: 'failed', error: msg });
|
|
536
|
+
} catch (markErr) {
|
|
537
|
+
process.stderr.write(`completeCommand failed: ${markErr?.message ?? markErr}\n`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Run an `ollama` subcommand non-interactively, capturing stdout
|
|
544
|
+
* and stderr. Returns { stdout, stderr, code } on success; throws
|
|
545
|
+
* on non-zero exit so the command queue records a failure.
|
|
546
|
+
*/
|
|
547
|
+
function ollamaSpawn(args) {
|
|
548
|
+
return new Promise((resolve, reject) => {
|
|
549
|
+
const child = spawn('ollama', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
550
|
+
let out = '';
|
|
551
|
+
let err = '';
|
|
552
|
+
child.stdout.on('data', (b) => { out += b.toString(); });
|
|
553
|
+
child.stderr.on('data', (b) => { err += b.toString(); });
|
|
554
|
+
child.on('error', (e) => reject(new Error(`ollama spawn error: ${e?.message ?? e}`)));
|
|
555
|
+
child.on('exit', (code) => {
|
|
556
|
+
if (code === 0) resolve({ stdout: out.slice(-4096), stderr: err.slice(-4096), code });
|
|
557
|
+
else reject(new Error(`ollama exited ${code}: ${(err || out).slice(-512)}`));
|
|
558
|
+
});
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function snapshot() {
|
|
563
|
+
return {
|
|
564
|
+
pid: process.pid,
|
|
565
|
+
startedAt: startedAt.toISOString(),
|
|
566
|
+
uptimeMs: Date.now() - startedAt.getTime(),
|
|
567
|
+
node: {
|
|
568
|
+
nodeId: node.nodeId,
|
|
569
|
+
id: node.id,
|
|
570
|
+
role: node.role,
|
|
571
|
+
name: node.name ?? null
|
|
572
|
+
},
|
|
573
|
+
controlPlaneUrl: config.controlPlane?.url ?? null,
|
|
574
|
+
intervals: { heartbeatMs, pollMs },
|
|
575
|
+
stats: {
|
|
576
|
+
heartbeatsOk: stats.heartbeatsOk,
|
|
577
|
+
heartbeatsFailed: stats.heartbeatsFailed,
|
|
578
|
+
lastHeartbeatAt: stats.lastHeartbeatAt,
|
|
579
|
+
lastHeartbeatError: stats.lastHeartbeatError,
|
|
580
|
+
jobsPicked: stats.jobsPicked,
|
|
581
|
+
jobsCompleted: stats.jobsCompleted,
|
|
582
|
+
jobsFailed: stats.jobsFailed,
|
|
583
|
+
activeJobs: stats.activeJobIds.size,
|
|
584
|
+
activeJobIds: Array.from(stats.activeJobIds),
|
|
585
|
+
lastJobAt: stats.lastJobAt,
|
|
586
|
+
pollsOk: stats.pollsOk,
|
|
587
|
+
pollsFailed: stats.pollsFailed,
|
|
588
|
+
lastPollAt: stats.lastPollAt
|
|
589
|
+
},
|
|
590
|
+
p2p: p2pDisabled ? { enabled: false } : {
|
|
591
|
+
enabled: true,
|
|
592
|
+
port: p2pPort,
|
|
593
|
+
address: advertisedAddress,
|
|
594
|
+
endpoint: formatEndpoint(advertisedAddress ?? '-', p2pPort),
|
|
595
|
+
connectionsTotal: p2pConnections,
|
|
596
|
+
lastConnectionAt: p2pLastConnectionAt
|
|
597
|
+
}
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
function startIpcServer() {
|
|
602
|
+
return new Promise((resolve, reject) => {
|
|
603
|
+
const server = net.createServer((sock) => {
|
|
604
|
+
let buf = '';
|
|
605
|
+
sock.setEncoding('utf8');
|
|
606
|
+
sock.on('data', async (chunk) => {
|
|
607
|
+
buf += chunk;
|
|
608
|
+
for (;;) {
|
|
609
|
+
const nl = buf.indexOf('\n');
|
|
610
|
+
if (nl < 0) break;
|
|
611
|
+
const line = buf.slice(0, nl);
|
|
612
|
+
buf = buf.slice(nl + 1);
|
|
613
|
+
let msg;
|
|
614
|
+
try { msg = JSON.parse(line); }
|
|
615
|
+
catch (err) {
|
|
616
|
+
sock.write(JSON.stringify({ ok: false, error: 'bad-json', cause: err?.message ?? String(err) }) + '\n');
|
|
617
|
+
continue;
|
|
618
|
+
}
|
|
619
|
+
const reply = await handleIpc(msg);
|
|
620
|
+
try { sock.write(JSON.stringify(reply) + '\n'); } catch { /* ignore */ }
|
|
621
|
+
}
|
|
622
|
+
});
|
|
623
|
+
sock.on('error', () => {});
|
|
624
|
+
});
|
|
625
|
+
server.once('error', reject);
|
|
626
|
+
server.listen(socketPath, () => {
|
|
627
|
+
try { chmodSync(socketPath, 0o600); } catch { /* ignore */ }
|
|
628
|
+
resolve(server);
|
|
629
|
+
});
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
async function handleIpc(msg) {
|
|
634
|
+
const cmd = msg?.cmd;
|
|
635
|
+
switch (cmd) {
|
|
636
|
+
case 'ping': return { ok: true, data: { pong: Date.now() } };
|
|
637
|
+
case 'status':
|
|
638
|
+
case 'stats': return { ok: true, data: snapshot() };
|
|
639
|
+
case 'shutdown':
|
|
640
|
+
setImmediate(() => shutdown('ipc-shutdown'));
|
|
641
|
+
return { ok: true, data: { shuttingDown: true } };
|
|
642
|
+
default:
|
|
643
|
+
return { ok: false, error: `unknown-cmd: ${cmd ?? '(none)'}` };
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
function startP2pServer() {
|
|
648
|
+
return new Promise((resolve, reject) => {
|
|
649
|
+
const server = net.createServer((sock) => {
|
|
650
|
+
p2pConnections += 1;
|
|
651
|
+
p2pLastConnectionAt = new Date().toISOString();
|
|
652
|
+
sock.setEncoding('utf8');
|
|
653
|
+
let buf = '';
|
|
654
|
+
sock.on('data', async (chunk) => {
|
|
655
|
+
buf += chunk;
|
|
656
|
+
for (;;) {
|
|
657
|
+
const nl = buf.indexOf('\n');
|
|
658
|
+
if (nl < 0) break;
|
|
659
|
+
const line = buf.slice(0, nl);
|
|
660
|
+
buf = buf.slice(nl + 1);
|
|
661
|
+
let msg;
|
|
662
|
+
try { msg = JSON.parse(line); }
|
|
663
|
+
catch (err) {
|
|
664
|
+
try { sock.write(JSON.stringify({ ok: false, error: 'bad-json', cause: err?.message ?? String(err) }) + '\n'); } catch { /* ignore */ }
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
let reply;
|
|
668
|
+
switch (msg?.cmd) {
|
|
669
|
+
case 'ping':
|
|
670
|
+
reply = { ok: true, data: { pong: Date.now(), node_id: node.nodeId } };
|
|
671
|
+
break;
|
|
672
|
+
case 'info':
|
|
673
|
+
reply = { ok: true, data: {
|
|
674
|
+
node_id: node.nodeId,
|
|
675
|
+
role: node.role,
|
|
676
|
+
name: node.name ?? null,
|
|
677
|
+
port: p2pPort
|
|
678
|
+
} };
|
|
679
|
+
break;
|
|
680
|
+
default:
|
|
681
|
+
reply = { ok: false, error: `unknown-cmd: ${msg?.cmd ?? '(none)'}` };
|
|
682
|
+
}
|
|
683
|
+
try { sock.write(JSON.stringify(reply) + '\n'); } catch {}
|
|
684
|
+
}
|
|
685
|
+
});
|
|
686
|
+
sock.on('error', () => {});
|
|
687
|
+
});
|
|
688
|
+
server.once('error', reject);
|
|
689
|
+
server.listen({ port: p2pPort, host: '::', ipv6Only: false }, () => {
|
|
690
|
+
resolve(server);
|
|
691
|
+
});
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
const shutdown = async (signal) => {
|
|
696
|
+
if (shuttingDown) return;
|
|
697
|
+
shuttingDown = true;
|
|
698
|
+
process.stdout.write(`\nReceived ${signal}; shutting down...\n`);
|
|
699
|
+
if (heartbeatTimer) clearInterval(heartbeatTimer);
|
|
700
|
+
if (pollTimer) clearInterval(pollTimer);
|
|
701
|
+
try {
|
|
702
|
+
await client.heartbeat({ status: 'offline' });
|
|
703
|
+
} catch (err) {
|
|
704
|
+
process.stderr.write(`offline heartbeat failed: ${err?.message ?? err}\n`);
|
|
705
|
+
}
|
|
706
|
+
if (ipcServer) { try { ipcServer.close(); } catch {} }
|
|
707
|
+
if (p2pServer) { try { p2pServer.close(); } catch {} }
|
|
708
|
+
if (healthServer) { try { healthServer.close(); } catch {} }
|
|
709
|
+
try { await shutdownEngine(); } catch {}
|
|
710
|
+
removeSocketFile();
|
|
711
|
+
await removePidFile();
|
|
712
|
+
process.stdout.write('bye\n');
|
|
713
|
+
process.exit(0);
|
|
714
|
+
};
|
|
715
|
+
|
|
716
|
+
process.on('SIGINT', () => { shutdown('SIGINT'); });
|
|
717
|
+
process.on('SIGTERM', () => { shutdown('SIGTERM'); });
|
|
718
|
+
|
|
719
|
+
try {
|
|
720
|
+
ipcServer = await startIpcServer();
|
|
721
|
+
process.stdout.write(`IPC listening on ${socketPath}\n`);
|
|
722
|
+
} catch (err) {
|
|
723
|
+
process.stderr.write(`Failed to bind IPC socket at ${socketPath}: ${err?.message ?? err}\n`);
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
// /healthz — bind a tiny HTTP server on $PORT (default 8080) so
|
|
727
|
+
// Docker / Runpod / Kubernetes HEALTHCHECK probes can reach the
|
|
728
|
+
// daemon even though the main daemon's port is 46337 (P2P, often
|
|
729
|
+
// firewalled). Returns 200 + a JSON daemon snapshot.
|
|
730
|
+
const healthPort = Number.parseInt(process.env.PORT ?? '', 10) || 8080;
|
|
731
|
+
try {
|
|
732
|
+
healthServer = http.createServer((req, res) => {
|
|
733
|
+
if (req.url === '/healthz' || req.url === '/health' || req.url === '/') {
|
|
734
|
+
const snap = snapshot();
|
|
735
|
+
const body = JSON.stringify({
|
|
736
|
+
ok: true,
|
|
737
|
+
pid: snap.pid,
|
|
738
|
+
uptime_ms: snap.uptimeMs,
|
|
739
|
+
role: snap.node?.role ?? null,
|
|
740
|
+
node_id: snap.node?.nodeId ?? null,
|
|
741
|
+
control_plane: snap.controlPlaneUrl,
|
|
742
|
+
heartbeats_ok: snap.stats?.heartbeatsOk ?? 0,
|
|
743
|
+
heartbeats_failed: snap.stats?.heartbeatsFailed ?? 0,
|
|
744
|
+
last_heartbeat_at: snap.stats?.lastHeartbeatAt ?? null,
|
|
745
|
+
last_heartbeat_error: snap.stats?.lastHeartbeatError ?? null,
|
|
746
|
+
jobs_completed: snap.stats?.jobsCompleted ?? 0,
|
|
747
|
+
jobs_failed: snap.stats?.jobsFailed ?? 0,
|
|
748
|
+
active_jobs: stats.activeJobIds.size
|
|
749
|
+
});
|
|
750
|
+
res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'no-store' });
|
|
751
|
+
res.end(body);
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
res.writeHead(404, { 'content-type': 'text/plain' });
|
|
755
|
+
res.end('not found\n');
|
|
756
|
+
});
|
|
757
|
+
await new Promise((resolve, reject) => {
|
|
758
|
+
healthServer.once('error', reject);
|
|
759
|
+
healthServer.listen(healthPort, '0.0.0.0', () => {
|
|
760
|
+
healthServer.removeListener('error', reject);
|
|
761
|
+
resolve();
|
|
762
|
+
});
|
|
763
|
+
});
|
|
764
|
+
process.stdout.write(`/healthz listening on 0.0.0.0:${healthPort}\n`);
|
|
765
|
+
} catch (err) {
|
|
766
|
+
// Health server failure is non-fatal — daemon still does its
|
|
767
|
+
// job, we just won't get container-level liveness probes.
|
|
768
|
+
process.stderr.write(`/healthz failed to bind on :${healthPort}: ${err?.message ?? err}\n`);
|
|
769
|
+
healthServer = null;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
if (!p2pDisabled) {
|
|
773
|
+
try {
|
|
774
|
+
p2pServer = await startP2pServer();
|
|
775
|
+
process.stdout.write(`P2P listening on ${formatEndpoint(advertisedAddress ?? '-', p2pPort)} (dual-stack)\n`);
|
|
776
|
+
} catch (err) {
|
|
777
|
+
const cause = err?.code === 'EADDRINUSE'
|
|
778
|
+
? ` (port ${p2pPort} is already in use — pick another with --p2p-port)`
|
|
779
|
+
: err?.code === 'EACCES'
|
|
780
|
+
? ` (permission denied — try a port above 1024)`
|
|
781
|
+
: '';
|
|
782
|
+
process.stderr.write(`Failed to bind P2P listener on port ${p2pPort}${cause}: ${err?.message ?? err}\n`);
|
|
783
|
+
process.stderr.write('If this is a firewall issue, run `infernet firewall` for per-distro commands.\n');
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
// Ensure a valid bearer token exists. Refreshes autonomously using the
|
|
788
|
+
// node keypair so the daemon never needs a manual `infernet login`.
|
|
789
|
+
{
|
|
790
|
+
const cfg = await loadConfig().catch(() => null);
|
|
791
|
+
const expAt = cfg?.auth?.expiresAt ? new Date(cfg.auth.expiresAt) : null;
|
|
792
|
+
const needsRefresh = !cfg?.auth?.bearerToken || !expAt || expAt - Date.now() < 30 * 86400 * 1000;
|
|
793
|
+
if (needsRefresh) {
|
|
794
|
+
nodeTokenLogin(cfg ?? config).catch((err) =>
|
|
795
|
+
process.stderr.write(`[login] auto-refresh failed: ${err?.message ?? err}\n`)
|
|
796
|
+
);
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
await heartbeat();
|
|
801
|
+
await pollJobs();
|
|
802
|
+
await pollNodeCommands();
|
|
803
|
+
|
|
804
|
+
if (once) {
|
|
805
|
+
if (ipcServer) { try { ipcServer.close(); } catch {} }
|
|
806
|
+
if (p2pServer) { try { p2pServer.close(); } catch {} }
|
|
807
|
+
if (healthServer) { try { healthServer.close(); } catch {} }
|
|
808
|
+
removeSocketFile();
|
|
809
|
+
await removePidFile();
|
|
810
|
+
return 0;
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
heartbeatTimer = setInterval(() => {
|
|
814
|
+
heartbeat().catch((err) => process.stderr.write(`heartbeat threw: ${err?.message ?? err}\n`));
|
|
815
|
+
}, heartbeatMs);
|
|
816
|
+
pollTimer = setInterval(() => {
|
|
817
|
+
pollJobs().catch((err) => process.stderr.write(`poll threw: ${err?.message ?? err}\n`));
|
|
818
|
+
}, pollMs);
|
|
819
|
+
// Commands poll less often than jobs — they're rare (model
|
|
820
|
+
// install/remove from the dashboard) and ollama pull can take
|
|
821
|
+
// minutes, so we don't want overlapping pulls of the same model.
|
|
822
|
+
const commandPollMs = Math.max(pollMs * 2, 30_000);
|
|
823
|
+
const commandTimer = setInterval(() => {
|
|
824
|
+
pollNodeCommands().catch((err) =>
|
|
825
|
+
process.stderr.write(`command poll threw: ${err?.message ?? err}\n`)
|
|
826
|
+
);
|
|
827
|
+
}, commandPollMs);
|
|
828
|
+
if (typeof commandTimer.unref === 'function') commandTimer.unref();
|
|
829
|
+
|
|
830
|
+
// Self-update: check npm registry every 5 minutes. If a newer version is
|
|
831
|
+
// available, pull the installer, re-generate any missing keys + re-register
|
|
832
|
+
// specs (idempotent), then re-exec so the new code is running immediately.
|
|
833
|
+
let isUpgrading = false;
|
|
834
|
+
const updateTimer = setInterval(async () => {
|
|
835
|
+
if (shuttingDown || isUpgrading) return;
|
|
836
|
+
const latest = await fetchLatestVersion();
|
|
837
|
+
if (!latest || !isNewerVersion(CURRENT_VERSION, latest)) return;
|
|
838
|
+
isUpgrading = true;
|
|
839
|
+
|
|
840
|
+
process.stdout.write(`[update] v${latest} available (running v${CURRENT_VERSION}) — upgrading\n`);
|
|
841
|
+
const ok = await pullLatestBinary();
|
|
842
|
+
if (!ok) {
|
|
843
|
+
process.stderr.write('[update] installer failed — will retry next check\n');
|
|
844
|
+
isUpgrading = false;
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
// Post-upgrade: run `infernet upgrade --skip-setup` is already done
|
|
849
|
+
// (pullLatestBinary above). Now run setup --yes so keys are regenerated,
|
|
850
|
+
// specs re-registered, and the daemon restarts cleanly via re-exec below.
|
|
851
|
+
process.stdout.write('[update] re-initializing keys and specs\n');
|
|
852
|
+
try {
|
|
853
|
+
const { default: setup } = await import('./setup.js');
|
|
854
|
+
const setupArgs = new Map([['yes', true], ['confirm', true], ['skip-pull', true], ['skip-daemon', true]]);
|
|
855
|
+
await setup(setupArgs, { config, client, configPath });
|
|
856
|
+
} catch (err) {
|
|
857
|
+
process.stderr.write(`[update] post-upgrade setup: ${err?.message ?? err}\n`);
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
// Release any in-flight jobs before we close ports.
|
|
861
|
+
for (const jobId of stats.activeJobIds) {
|
|
862
|
+
try {
|
|
863
|
+
await client.failJob(jobId, 'daemon restarting for self-update');
|
|
864
|
+
} catch { /* best-effort */ }
|
|
865
|
+
}
|
|
866
|
+
stats.activeJobIds.clear();
|
|
867
|
+
|
|
868
|
+
// Tear down servers BEFORE spawning the new process so the new daemon
|
|
869
|
+
// can bind the same ports without EADDRINUSE races.
|
|
870
|
+
shuttingDown = true;
|
|
871
|
+
clearInterval(heartbeatTimer);
|
|
872
|
+
clearInterval(pollTimer);
|
|
873
|
+
clearInterval(commandTimer);
|
|
874
|
+
clearInterval(updateTimer);
|
|
875
|
+
if (ipcServer) try { ipcServer.close(); } catch {}
|
|
876
|
+
if (p2pServer) try { p2pServer.close(); } catch {}
|
|
877
|
+
if (healthServer) try { healthServer.close(); } catch {}
|
|
878
|
+
try { await shutdownEngine(); } catch {}
|
|
879
|
+
removeSocketFile();
|
|
880
|
+
await removePidFile();
|
|
881
|
+
|
|
882
|
+
// Small grace period so OS reclaims the ports before the new process binds.
|
|
883
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
884
|
+
|
|
885
|
+
// Re-exec the daemon with the same args so the new code is live.
|
|
886
|
+
process.stdout.write('[update] restarting daemon with new version\n');
|
|
887
|
+
const logPath = getDaemonLogPath();
|
|
888
|
+
const logFd = openSync(logPath, 'a');
|
|
889
|
+
spawn(process.execPath, process.argv.slice(1), {
|
|
890
|
+
detached: true,
|
|
891
|
+
stdio: ['ignore', logFd, logFd],
|
|
892
|
+
env: process.env
|
|
893
|
+
}).unref();
|
|
894
|
+
|
|
895
|
+
process.exit(0);
|
|
896
|
+
}, UPDATE_CHECK_MS);
|
|
897
|
+
if (typeof updateTimer.unref === 'function') updateTimer.unref();
|
|
898
|
+
|
|
899
|
+
await new Promise(() => {});
|
|
900
|
+
return 0;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
export { HELP };
|