@infernetprotocol/cli 0.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/LICENSE +15 -0
  2. package/commands/chat.js +268 -0
  3. package/commands/console.js +229 -0
  4. package/commands/debug.js +222 -0
  5. package/commands/deploy.js +524 -0
  6. package/commands/doctor.js +501 -0
  7. package/commands/firewall.js +39 -0
  8. package/commands/gpu.js +130 -0
  9. package/commands/help.js +54 -0
  10. package/commands/init.js +304 -0
  11. package/commands/login.js +305 -0
  12. package/commands/logs.js +93 -0
  13. package/commands/model.js +359 -0
  14. package/commands/payments.js +92 -0
  15. package/commands/payout.js +128 -0
  16. package/commands/pubkey.js +116 -0
  17. package/commands/register.js +304 -0
  18. package/commands/remove.js +230 -0
  19. package/commands/service.js +255 -0
  20. package/commands/setup.js +869 -0
  21. package/commands/start.js +903 -0
  22. package/commands/stats.js +85 -0
  23. package/commands/status.js +116 -0
  24. package/commands/stop.js +81 -0
  25. package/commands/train.js +563 -0
  26. package/commands/tui.js +285 -0
  27. package/commands/uncensored.js +222 -0
  28. package/commands/update.js +80 -0
  29. package/commands/upgrade.js +2 -0
  30. package/examples/app.js +153 -0
  31. package/index.js +207 -0
  32. package/lib/chat-executor.js +232 -0
  33. package/lib/config.js +148 -0
  34. package/lib/daemonize.js +46 -0
  35. package/lib/firewall.js +190 -0
  36. package/lib/hf-model.js +166 -0
  37. package/lib/identity.js +18 -0
  38. package/lib/ipc.js +114 -0
  39. package/lib/model-fit.js +67 -0
  40. package/lib/model-key.js +52 -0
  41. package/lib/network.js +240 -0
  42. package/lib/nip44.js +130 -0
  43. package/lib/node-client.js +152 -0
  44. package/lib/peers.js +133 -0
  45. package/lib/prompt.js +49 -0
  46. package/lib/remote-chat.js +115 -0
  47. package/lib/route.js +44 -0
  48. package/lib/runtime-config.js +130 -0
  49. package/lib/version.js +32 -0
  50. package/package.json +57 -0
@@ -0,0 +1,903 @@
1
+ /**
2
+ * `infernet start` — run the node daemon loop.
3
+ *
4
+ * By default, `infernet start` spawns a **detached** background process and
5
+ * returns immediately; logs go to `~/.config/infernet/daemon.log`. Use
6
+ * `--foreground` to run the loop in the current terminal (useful under
7
+ * systemd / Docker / Kubernetes where the supervisor wants the process in
8
+ * the foreground).
9
+ *
10
+ * The running daemon:
11
+ * - Heartbeats every 30s via signed POST /api/v1/node/heartbeat
12
+ * - Polls /api/v1/node/jobs/poll every 15s (providers only), processes
13
+ * any returned jobs, emits streaming events via signed
14
+ * POST /api/v1/node/jobs/:id/events, and closes the loop with
15
+ * POST /api/v1/node/jobs/:id/complete.
16
+ * - Exposes a Unix-domain IPC socket at `~/.config/infernet/daemon.sock`
17
+ * so `infernet status`, `infernet stats`, `infernet stop`, etc. can ask
18
+ * the live process what it's doing.
19
+ * - Handles SIGINT / SIGTERM: sends a final heartbeat with status=offline,
20
+ * removes pid/sock files, exits 0.
21
+ *
22
+ * The daemon never holds a database credential. Every request to the
23
+ * control plane is signed with the node's Nostr privkey.
24
+ */
25
+
26
+ import fs from 'node:fs/promises';
27
+ import { openSync } from 'node:fs';
28
+ import net from 'node:net';
29
+ import http from 'node:http';
30
+ import { spawn } from 'node:child_process';
31
+ import { chmodSync, unlinkSync } from 'node:fs';
32
+ import { readFileSync } from 'node:fs';
33
+ import { fileURLToPath } from 'node:url';
34
+ import { dirname, join } from 'node:path';
35
+
36
+ import {
37
+ getDaemonPidPath,
38
+ getDaemonSocketPath,
39
+ getDaemonLogPath,
40
+ loadConfig,
41
+ saveConfig,
42
+ fixConfigPermissions
43
+ } from '../lib/config.js';
44
+ import { spawnDetachedDaemon } from '../lib/daemonize.js';
45
+ import { isDaemonAlive } from '../lib/ipc.js';
46
+ import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/network.js';
47
+ import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
48
+ import { gatherCoarseSpecs } from './register.js';
49
+ import { checkModelFits } from '../lib/model-fit.js';
50
+ import { detectGpus, detectHost } from '@infernetprotocol/gpu';
51
+ import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
52
+ import { pullLatestBinary } from './upgrade.js';
53
+ import { nodeTokenLogin } from './login.js';
54
+ import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from '../lib/version.js';
55
+
56
+ const HELP = `infernet start — run the node daemon
57
+
58
+ Usage:
59
+ infernet start [flags]
60
+
61
+ Flags:
62
+ --foreground Run in the current terminal (don't detach)
63
+ --heartbeat-interval <ms> Override heartbeat cadence (default 30000)
64
+ --poll-interval <ms> Override job poll cadence (default 15000)
65
+ --p2p-port <n> TCP port for peer connections (default 46337)
66
+ --no-p2p Don't bind the P2P TCP listener
67
+ --no-advertise Don't send address/port in heartbeats
68
+ --once Run one heartbeat + one poll and exit (debug)
69
+ --help Show this help
70
+
71
+ Daemon logs to \`~/.config/infernet/daemon.log\`, exposes an IPC socket for
72
+ live queries (see \`infernet status\`, \`infernet stats\`, \`infernet logs\`).
73
+ `;
74
+
75
+ const DEFAULT_HEARTBEAT_MS = 30_000;
76
+ const DEFAULT_POLL_MS = 15_000;
77
+ const UPDATE_CHECK_MS = 5 * 60 * 1000; // 5 minutes
78
+
79
+ async function writePidFile(pid) {
80
+ const p = getDaemonPidPath();
81
+ await fs.writeFile(p, String(pid), { mode: 0o600 });
82
+ return p;
83
+ }
84
+
85
+ async function removePidFile() {
86
+ try { await fs.unlink(getDaemonPidPath()); } catch { /* ignore */ }
87
+ }
88
+
89
+ function removeSocketFile() {
90
+ try { unlinkSync(getDaemonSocketPath()); } catch { /* ignore */ }
91
+ }
92
+
93
+ // ---------------------------------------------------------------------------
94
+ // Entry point — branch between "spawn detached" and "run foreground loop".
95
+ // ---------------------------------------------------------------------------
96
+ export default async function start(args, ctx) {
97
+ if (args.has('help') || args.has('h')) {
98
+ process.stdout.write(HELP);
99
+ return 0;
100
+ }
101
+
102
+ const foreground = args.has('foreground') || process.env.INFERNET_DAEMON_FOREGROUND === '1';
103
+ if (!foreground) {
104
+ return spawnAndReturn(args);
105
+ }
106
+ return runDaemon(args, ctx);
107
+ }
108
+
109
+ async function spawnAndReturn(args) {
110
+ const alreadyAlive = await isDaemonAlive();
111
+ if (alreadyAlive) {
112
+ process.stderr.write('A daemon is already running (socket responsive). Run `infernet stop` first.\n');
113
+ return 1;
114
+ }
115
+ removeSocketFile();
116
+
117
+ const passthrough = [];
118
+ const hb = args.get('heartbeat-interval');
119
+ if (hb) passthrough.push('--heartbeat-interval', hb);
120
+ const poll = args.get('poll-interval');
121
+ if (poll) passthrough.push('--poll-interval', poll);
122
+ const p2pPort = args.get('p2p-port');
123
+ if (p2pPort) passthrough.push('--p2p-port', p2pPort);
124
+ if (args.has('no-p2p')) passthrough.push('--no-p2p');
125
+ if (args.has('no-advertise')) passthrough.push('--no-advertise');
126
+ if (args.has('once')) passthrough.push('--once');
127
+
128
+ const { pid, logPath } = spawnDetachedDaemon(passthrough);
129
+ process.stdout.write(`infernet daemon started (pid ${pid})\n`);
130
+ process.stdout.write(` logs: ${logPath}\n`);
131
+ process.stdout.write(` socket: ${getDaemonSocketPath()}\n`);
132
+ process.stdout.write('Tail logs with `infernet logs -f`, query live state with `infernet status`.\n');
133
+ return 0;
134
+ }
135
+
136
+ async function runDaemon(args, ctx) {
137
+ const { config, client, configPath } = ctx;
138
+ const node = config.node ?? {};
139
+ if (!node.role || !node.nodeId) {
140
+ process.stderr.write('Config is missing node.role/node.nodeId. Run `infernet init` first.\n');
141
+ return 1;
142
+ }
143
+
144
+ const heartbeatMs = Number.parseInt(args.get('heartbeat-interval') ?? '', 10) || DEFAULT_HEARTBEAT_MS;
145
+ const pollMs = Number.parseInt(args.get('poll-interval') ?? '', 10) || DEFAULT_POLL_MS;
146
+ const once = args.has('once');
147
+
148
+ const p2pDisabled = args.has('no-p2p');
149
+ const noAdvertise = args.has('no-advertise') || node.address === null;
150
+ // Bind port (what we listen on, locally) and advertised port (what we
151
+ // tell the control plane to dial). Same value 99% of the time, but
152
+ // hosting platforms that NAT the container (RunPod, anything with
153
+ // dynamic port mapping) need them split — daemon binds to e.g. 46337
154
+ // inside the container, but the public address:port is something like
155
+ // 213.173.107.42:21517. The cloud-init detects RunPod env vars and
156
+ // exports INFERNET_BIND_PORT (internal) + INFERNET_PUBLIC_PORT
157
+ // (external) separately.
158
+ const bindPort = Number.parseInt(
159
+ args.get('bind-port') ?? process.env.INFERNET_BIND_PORT ?? '',
160
+ 10
161
+ ) || Number.parseInt(args.get('p2p-port') ?? '', 10) || resolveP2pPort(config);
162
+ const advertisedPort = Number.parseInt(
163
+ args.get('p2p-port') ?? process.env.INFERNET_PUBLIC_PORT ?? '',
164
+ 10
165
+ ) || resolveP2pPort(config) || bindPort;
166
+ // Back-compat: keep the existing variable name pointing at bindPort,
167
+ // since the rest of the file (P2P listener, formatEndpoint, etc.)
168
+ // wants the bind value.
169
+ const p2pPort = bindPort;
170
+ const advertisedAddress = noAdvertise ? null : (node.address ?? await detectLocalAddress());
171
+
172
+ const startedAt = new Date();
173
+ const stats = {
174
+ heartbeatsOk: 0,
175
+ heartbeatsFailed: 0,
176
+ lastHeartbeatAt: null,
177
+ lastHeartbeatError: null,
178
+ jobsPicked: 0,
179
+ jobsCompleted: 0,
180
+ jobsFailed: 0,
181
+ lastJobAt: null,
182
+ lastPollAt: null,
183
+ pollsOk: 0,
184
+ pollsFailed: 0,
185
+ activeJobIds: new Set()
186
+ };
187
+
188
+ const pidPath = await writePidFile(process.pid);
189
+ const socketPath = getDaemonSocketPath();
190
+ removeSocketFile();
191
+
192
+ // Self-heal: fix config permissions on every daemon start so nodes
193
+ // upgraded before the 0600 enforcement still get corrected.
194
+ fixConfigPermissions().catch(() => {});
195
+
196
+ process.stdout.write('infernet daemon starting\n');
197
+ process.stdout.write(` node_id: ${node.nodeId}\n`);
198
+ process.stdout.write(` role: ${node.role}\n`);
199
+ process.stdout.write(` control: ${config.controlPlane?.url ?? '(not set)'}\n`);
200
+ process.stdout.write(` pid: ${process.pid}\n`);
201
+ process.stdout.write(` pidfile: ${pidPath}\n`);
202
+ process.stdout.write(` socket: ${socketPath}\n`);
203
+ process.stdout.write(` config: ${configPath}\n`);
204
+ process.stdout.write(` heartbeat: ${heartbeatMs}ms\n`);
205
+ process.stdout.write(` poll: ${pollMs}ms\n`);
206
+ if (!p2pDisabled) {
207
+ process.stdout.write(` p2p: ${formatEndpoint(advertisedAddress ?? '-', advertisedPort)}${bindPort !== advertisedPort ? ` (binds locally :${bindPort})` : ''}\n`);
208
+ } else {
209
+ process.stdout.write(' p2p: disabled\n');
210
+ }
211
+ if (noAdvertise) process.stdout.write(' advertise: off (outbound-only)\n');
212
+ process.stdout.write(` version: v${CURRENT_VERSION} (auto-update every ${UPDATE_CHECK_MS / 60_000}min)\n`);
213
+
214
+ let heartbeatTimer = null;
215
+ let pollTimer = null;
216
+ let shuttingDown = false;
217
+ let ipcServer = null;
218
+ let p2pServer = null;
219
+ let healthServer = null;
220
+ let p2pConnections = 0;
221
+ let p2pLastConnectionAt = null;
222
+
223
+ // Specs cache — re-detection (nvidia-smi, /sys/class/infiniband, Ollama
224
+ // /api/tags) is cheap but not free. Refresh every SPECS_TTL_MS so the
225
+ // control plane sees current served-models / hardware without paying
226
+ // the detection cost on every 30s heartbeat.
227
+ const SPECS_TTL_MS = 5 * 60 * 1000;
228
+ let cachedSpecs = null;
229
+ let cachedSpecsAt = 0;
230
+
231
+ // Rolling benchmark — last N completed chat jobs' (tokens, duration)
232
+ // pairs. Used to compute tokens_per_second_avg for the heartbeat,
233
+ // which feeds speed-aware routing on the control plane. Capped so
234
+ // memory stays bounded across long-running daemons.
235
+ const BENCH_RING_MAX = 32;
236
+ const benchRing = [];
237
+
238
+ function recordBench({ token_count, duration_ms }) {
239
+ if (!Number.isFinite(token_count) || !Number.isFinite(duration_ms) || duration_ms <= 0) return;
240
+ if (token_count <= 0) return;
241
+ benchRing.push({ tokens: token_count, ms: duration_ms, at: Date.now() });
242
+ while (benchRing.length > BENCH_RING_MAX) benchRing.shift();
243
+ }
244
+
245
+ function benchSummary() {
246
+ if (benchRing.length === 0) return null;
247
+ const totalTokens = benchRing.reduce((a, e) => a + e.tokens, 0);
248
+ const totalMs = benchRing.reduce((a, e) => a + e.ms, 0);
249
+ if (totalMs <= 0) return null;
250
+ const tps = (totalTokens / totalMs) * 1000;
251
+ return {
252
+ tokens_per_second_avg: +tps.toFixed(2),
253
+ samples: benchRing.length,
254
+ window_started_at: new Date(benchRing[0].at).toISOString()
255
+ };
256
+ }
257
+
258
+ /**
259
+ * Reachability self-check — daemon calls /api/probe?host=&port=
260
+ * once after starting the P2P listener. Result cached on the
261
+ * specs object so it ships with heartbeats and surfaces in the
262
+ * dashboard. Refreshed every REACHABILITY_TTL_MS so a router
263
+ * port-forward added later gets picked up without restart.
264
+ */
265
+ const REACHABILITY_TTL_MS = 10 * 60 * 1000;
266
+ let cachedReachable = null;
267
+ let cachedReachableAt = 0;
268
+
269
+ async function probeReachable() {
270
+ if (noAdvertise || !advertisedAddress || !p2pPort || p2pDisabled) return null;
271
+ const now = Date.now();
272
+ if (cachedReachable !== null && now - cachedReachableAt < REACHABILITY_TTL_MS) {
273
+ return cachedReachable;
274
+ }
275
+ const baseUrl = config.controlPlane?.url;
276
+ if (!baseUrl) return null;
277
+ try {
278
+ const url = new URL("/api/probe", baseUrl);
279
+ url.searchParams.set("host", advertisedAddress);
280
+ // Probe the externally-advertised port — that's what we're
281
+ // claiming is reachable, so that's what we want verified.
282
+ url.searchParams.set("port", String(advertisedPort));
283
+ const res = await fetch(url, { signal: AbortSignal.timeout?.(7000) });
284
+ if (!res.ok) return null;
285
+ const body = await res.json();
286
+ cachedReachable = {
287
+ ok: !!body.reachable,
288
+ error: body.error ?? null,
289
+ checked_at: new Date(now).toISOString()
290
+ };
291
+ cachedReachableAt = now;
292
+ return cachedReachable;
293
+ } catch {
294
+ return null;
295
+ }
296
+ }
297
+
298
+ /**
299
+ * Live load snapshot — fresh on every heartbeat, NOT cached. The
300
+ * picker on the control plane uses this to skip saturated nodes
301
+ * and weight by remaining headroom. Cheap to compute (one
302
+ * nvidia-smi shellout, one os.freemem read) — well worth the
303
+ * cost on each 30s heartbeat compared to the cost of routing a
304
+ * job to a saturated provider that 504s mid-stream.
305
+ */
306
+ async function liveLoad() {
307
+ const host = detectHost();
308
+ const totalRamGb = host.total_ram_mb / 1024;
309
+ const freeRamGb = host.free_ram_mb / 1024;
310
+ let freeVramGb = 0;
311
+ let totalVramGb = 0;
312
+ let maxGpuUtilization = null;
313
+ try {
314
+ const gpus = await detectGpus();
315
+ for (const g of gpus) {
316
+ if (Number.isFinite(g.vram_mb)) totalVramGb += g.vram_mb / 1024;
317
+ if (Number.isFinite(g.vram_mb) && Number.isFinite(g.vram_used_mb)) {
318
+ freeVramGb += Math.max(0, (g.vram_mb - g.vram_used_mb) / 1024);
319
+ }
320
+ if (Number.isFinite(g.utilization)) {
321
+ maxGpuUtilization = Math.max(maxGpuUtilization ?? 0, g.utilization);
322
+ }
323
+ }
324
+ } catch {
325
+ // Best-effort; CPU-only boxes report zero GPU stats.
326
+ }
327
+ return {
328
+ active_jobs: stats.activeJobIds.size,
329
+ load_avg_1m: Array.isArray(host.load_avg) ? +host.load_avg[0]?.toFixed(2) : null,
330
+ ram: {
331
+ total_gb: +totalRamGb.toFixed(2),
332
+ free_gb: +freeRamGb.toFixed(2)
333
+ },
334
+ vram: {
335
+ total_gb: +totalVramGb.toFixed(2),
336
+ free_gb: +freeVramGb.toFixed(2)
337
+ },
338
+ gpu_utilization_max: maxGpuUtilization,
339
+ measured_at: new Date().toISOString()
340
+ };
341
+ }
342
+
343
+ async function freshSpecs() {
344
+ if (node.role !== 'provider') return null;
345
+ const now = Date.now();
346
+ let base;
347
+ if (cachedSpecs && now - cachedSpecsAt < SPECS_TTL_MS) {
348
+ base = cachedSpecs;
349
+ } else {
350
+ try {
351
+ cachedSpecs = await gatherCoarseSpecs();
352
+ cachedSpecsAt = now;
353
+ base = cachedSpecs;
354
+ } catch (err) {
355
+ process.stderr.write(`specs detection failed: ${err?.message ?? err}\n`);
356
+ base = cachedSpecs;
357
+ }
358
+ }
359
+ if (!base) return null;
360
+ // Always re-read load (cheap, must be current). bench rolls
361
+ // independently as jobs complete. reachability has its own
362
+ // TTL so we don't hammer /api/probe on every heartbeat.
363
+ const bench = benchSummary();
364
+ const load = await liveLoad();
365
+ const reachable = await probeReachable();
366
+
367
+ // IPIP-0028: ensure a keypair exists for each served model and
368
+ // advertise their pubkeys so consumers can encrypt to model keys.
369
+ let modelKeys = {};
370
+ try {
371
+ const servedModels = Array.isArray(base.served_models) ? base.served_models : [];
372
+ for (const name of servedModels) {
373
+ const kp = await getOrCreateModelKey(name);
374
+ if (kp?.publicKey) modelKeys[name] = kp.publicKey;
375
+ }
376
+ } catch {
377
+ // non-fatal — fall back to empty model_keys
378
+ }
379
+
380
+ return {
381
+ ...base,
382
+ ...(bench ? { bench } : {}),
383
+ load,
384
+ ...(reachable ? { reachable } : {}),
385
+ ...(Object.keys(modelKeys).length > 0 ? { model_keys: modelKeys } : {}),
386
+ public_key: config?.node?.publicKey ?? undefined,
387
+ cli_version: CURRENT_VERSION
388
+ };
389
+ }
390
+
391
+ async function heartbeat() {
392
+ const payload = { status: 'available' };
393
+ if (!noAdvertise) {
394
+ if (advertisedAddress) payload.address = advertisedAddress;
395
+ // Heartbeat advertises the EXTERNAL port (what clients should
396
+ // dial), not the internal bind port. RunPod / NAT'd hosts:
397
+ // bindPort = 46337 (container), advertisedPort = 21517 (edge).
398
+ if (!p2pDisabled) payload.port = advertisedPort;
399
+ }
400
+ // Provider role only: include current specs so the control plane
401
+ // sees fresh CPU / GPU / served_models without requiring a manual
402
+ // `infernet register` after each capability change.
403
+ const specs = await freshSpecs();
404
+ if (specs) payload.specs = specs;
405
+ try {
406
+ await client.heartbeat(payload);
407
+ stats.heartbeatsOk += 1;
408
+ stats.lastHeartbeatAt = new Date().toISOString();
409
+ stats.lastHeartbeatError = null;
410
+ process.stdout.write(`[${stats.lastHeartbeatAt}] heartbeat ok\n`);
411
+ } catch (err) {
412
+ stats.heartbeatsFailed += 1;
413
+ stats.lastHeartbeatError = err?.message ?? String(err);
414
+ process.stderr.write(`heartbeat error: ${stats.lastHeartbeatError}\n`);
415
+ }
416
+ }
417
+
418
+ async function processJob(job) {
419
+ const t0 = new Date().toISOString();
420
+ stats.jobsPicked += 1;
421
+ stats.activeJobIds.add(job.id);
422
+ stats.lastJobAt = t0;
423
+ process.stdout.write(`[${t0}] picking up job ${job.id} type=${job.type ?? 'inference'} (${job.title ?? 'untitled'})\n`);
424
+
425
+ try {
426
+ let resultPayload;
427
+ if (job.type === 'chat') {
428
+ const result = await executeChatJob({ client, job, node });
429
+ recordBench(result);
430
+ resultPayload = {
431
+ type: 'chat',
432
+ text: result.text,
433
+ token_count: result.token_count,
434
+ duration_ms: result.duration_ms,
435
+ completed_by: node.nodeId
436
+ };
437
+ } else {
438
+ await new Promise((resolve) => setTimeout(resolve, 500));
439
+ resultPayload = { stub: true, completed_by: node.nodeId };
440
+ }
441
+ await client.completeJob(job.id, { status: 'completed', result: resultPayload });
442
+ stats.jobsCompleted += 1;
443
+ stats.activeJobIds.delete(job.id);
444
+ process.stdout.write(`[${new Date().toISOString()}] completed job ${job.id} type=${job.type ?? 'inference'}\n`);
445
+ } catch (err) {
446
+ stats.jobsFailed += 1;
447
+ stats.activeJobIds.delete(job.id);
448
+ const msg = err?.message ?? String(err);
449
+ process.stderr.write(`job ${job.id} failed: ${msg}\n`);
450
+ if (job.type === 'chat') {
451
+ await failChatJob({ client, jobId: job.id, message: msg });
452
+ }
453
+ try {
454
+ await client.failJob(job.id, msg);
455
+ } catch (markErr) {
456
+ process.stderr.write(`failJob failed: ${markErr?.message ?? markErr}\n`);
457
+ }
458
+ }
459
+ }
460
+
461
+ async function pollJobs() {
462
+ stats.lastPollAt = new Date().toISOString();
463
+ if (node.role !== 'provider') {
464
+ stats.pollsOk += 1;
465
+ return;
466
+ }
467
+ try {
468
+ const result = await client.pollJobs({ limit: 5 });
469
+ stats.pollsOk += 1;
470
+ for (const job of result?.jobs ?? []) {
471
+ // Skip jobs already being processed by this daemon — the
472
+ // server keeps returning 'assigned' jobs until we mark them
473
+ // complete, so without this guard the poll loop queues the
474
+ // same job on every tick while it's in flight.
475
+ if (stats.activeJobIds.has(job.id)) continue;
476
+ await processJob(job);
477
+ }
478
+ } catch (err) {
479
+ stats.pollsFailed += 1;
480
+ process.stderr.write(`job poll error: ${err?.message ?? err}\n`);
481
+ }
482
+ }
483
+
484
+ /**
485
+ * Owner-issued node commands (model_install / model_remove). Same
486
+ * outbound-poll pattern as jobs — no inbound connectivity needed.
487
+ * Auth at the server side: only owners (verified via pubkey_links)
488
+ * can write to node_commands; the daemon's signed poll restricts
489
+ * which rows it sees by pubkey match. So a compromised dashboard
490
+ * account can only push commands to nodes that account already
491
+ * owns; a compromised daemon key can only execute commands
492
+ * targeting itself.
493
+ */
494
+ async function pollNodeCommands() {
495
+ if (node.role !== 'provider') return;
496
+ let result;
497
+ try {
498
+ result = await client.pollCommands(5);
499
+ } catch (err) {
500
+ process.stderr.write(`command poll error: ${err?.message ?? err}\n`);
501
+ return;
502
+ }
503
+ for (const cmd of result?.commands ?? []) {
504
+ await runNodeCommand(cmd);
505
+ }
506
+ }
507
+
508
+ async function runNodeCommand(cmd) {
509
+ const { id, command, args } = cmd;
510
+ const t0 = new Date().toISOString();
511
+ process.stdout.write(`[${t0}] running command ${id}: ${command} ${JSON.stringify(args)}\n`);
512
+ try {
513
+ let result;
514
+ if (command === 'model_install') {
515
+ const modelName = String(args?.model);
516
+ const fits = await checkModelFits(modelName);
517
+ if (fits && !fits.ok) {
518
+ throw new Error(
519
+ `${modelName} (≈${fits.size_gb} GB) won't fit on this host ` +
520
+ `(${fits.mode} ceiling ≈ ${fits.ceiling_gb} GB with ${fits.have_gb} GB available)`
521
+ );
522
+ }
523
+ result = await ollamaSpawn(['pull', modelName]);
524
+ } else if (command === 'model_remove') {
525
+ result = await ollamaSpawn(['rm', String(args?.model)]);
526
+ } else {
527
+ throw new Error(`unknown command verb: ${command}`);
528
+ }
529
+ await client.completeCommand(id, { status: 'completed', result });
530
+ process.stdout.write(`[${new Date().toISOString()}] completed command ${id}\n`);
531
+ } catch (err) {
532
+ const msg = err?.message ?? String(err);
533
+ process.stderr.write(`command ${id} failed: ${msg}\n`);
534
+ try {
535
+ await client.completeCommand(id, { status: 'failed', error: msg });
536
+ } catch (markErr) {
537
+ process.stderr.write(`completeCommand failed: ${markErr?.message ?? markErr}\n`);
538
+ }
539
+ }
540
+ }
541
+
542
+ /**
543
+ * Run an `ollama` subcommand non-interactively, capturing stdout
544
+ * and stderr. Returns { stdout, stderr, code } on success; throws
545
+ * on non-zero exit so the command queue records a failure.
546
+ */
547
+ function ollamaSpawn(args) {
548
+ return new Promise((resolve, reject) => {
549
+ const child = spawn('ollama', args, { stdio: ['ignore', 'pipe', 'pipe'] });
550
+ let out = '';
551
+ let err = '';
552
+ child.stdout.on('data', (b) => { out += b.toString(); });
553
+ child.stderr.on('data', (b) => { err += b.toString(); });
554
+ child.on('error', (e) => reject(new Error(`ollama spawn error: ${e?.message ?? e}`)));
555
+ child.on('exit', (code) => {
556
+ if (code === 0) resolve({ stdout: out.slice(-4096), stderr: err.slice(-4096), code });
557
+ else reject(new Error(`ollama exited ${code}: ${(err || out).slice(-512)}`));
558
+ });
559
+ });
560
+ }
561
+
562
+ function snapshot() {
563
+ return {
564
+ pid: process.pid,
565
+ startedAt: startedAt.toISOString(),
566
+ uptimeMs: Date.now() - startedAt.getTime(),
567
+ node: {
568
+ nodeId: node.nodeId,
569
+ id: node.id,
570
+ role: node.role,
571
+ name: node.name ?? null
572
+ },
573
+ controlPlaneUrl: config.controlPlane?.url ?? null,
574
+ intervals: { heartbeatMs, pollMs },
575
+ stats: {
576
+ heartbeatsOk: stats.heartbeatsOk,
577
+ heartbeatsFailed: stats.heartbeatsFailed,
578
+ lastHeartbeatAt: stats.lastHeartbeatAt,
579
+ lastHeartbeatError: stats.lastHeartbeatError,
580
+ jobsPicked: stats.jobsPicked,
581
+ jobsCompleted: stats.jobsCompleted,
582
+ jobsFailed: stats.jobsFailed,
583
+ activeJobs: stats.activeJobIds.size,
584
+ activeJobIds: Array.from(stats.activeJobIds),
585
+ lastJobAt: stats.lastJobAt,
586
+ pollsOk: stats.pollsOk,
587
+ pollsFailed: stats.pollsFailed,
588
+ lastPollAt: stats.lastPollAt
589
+ },
590
+ p2p: p2pDisabled ? { enabled: false } : {
591
+ enabled: true,
592
+ port: p2pPort,
593
+ address: advertisedAddress,
594
+ endpoint: formatEndpoint(advertisedAddress ?? '-', p2pPort),
595
+ connectionsTotal: p2pConnections,
596
+ lastConnectionAt: p2pLastConnectionAt
597
+ }
598
+ };
599
+ }
600
+
601
+ function startIpcServer() {
602
+ return new Promise((resolve, reject) => {
603
+ const server = net.createServer((sock) => {
604
+ let buf = '';
605
+ sock.setEncoding('utf8');
606
+ sock.on('data', async (chunk) => {
607
+ buf += chunk;
608
+ for (;;) {
609
+ const nl = buf.indexOf('\n');
610
+ if (nl < 0) break;
611
+ const line = buf.slice(0, nl);
612
+ buf = buf.slice(nl + 1);
613
+ let msg;
614
+ try { msg = JSON.parse(line); }
615
+ catch (err) {
616
+ sock.write(JSON.stringify({ ok: false, error: 'bad-json', cause: err?.message ?? String(err) }) + '\n');
617
+ continue;
618
+ }
619
+ const reply = await handleIpc(msg);
620
+ try { sock.write(JSON.stringify(reply) + '\n'); } catch { /* ignore */ }
621
+ }
622
+ });
623
+ sock.on('error', () => {});
624
+ });
625
+ server.once('error', reject);
626
+ server.listen(socketPath, () => {
627
+ try { chmodSync(socketPath, 0o600); } catch { /* ignore */ }
628
+ resolve(server);
629
+ });
630
+ });
631
+ }
632
+
633
+ async function handleIpc(msg) {
634
+ const cmd = msg?.cmd;
635
+ switch (cmd) {
636
+ case 'ping': return { ok: true, data: { pong: Date.now() } };
637
+ case 'status':
638
+ case 'stats': return { ok: true, data: snapshot() };
639
+ case 'shutdown':
640
+ setImmediate(() => shutdown('ipc-shutdown'));
641
+ return { ok: true, data: { shuttingDown: true } };
642
+ default:
643
+ return { ok: false, error: `unknown-cmd: ${cmd ?? '(none)'}` };
644
+ }
645
+ }
646
+
647
+ function startP2pServer() {
648
+ return new Promise((resolve, reject) => {
649
+ const server = net.createServer((sock) => {
650
+ p2pConnections += 1;
651
+ p2pLastConnectionAt = new Date().toISOString();
652
+ sock.setEncoding('utf8');
653
+ let buf = '';
654
+ sock.on('data', async (chunk) => {
655
+ buf += chunk;
656
+ for (;;) {
657
+ const nl = buf.indexOf('\n');
658
+ if (nl < 0) break;
659
+ const line = buf.slice(0, nl);
660
+ buf = buf.slice(nl + 1);
661
+ let msg;
662
+ try { msg = JSON.parse(line); }
663
+ catch (err) {
664
+ try { sock.write(JSON.stringify({ ok: false, error: 'bad-json', cause: err?.message ?? String(err) }) + '\n'); } catch { /* ignore */ }
665
+ continue;
666
+ }
667
+ let reply;
668
+ switch (msg?.cmd) {
669
+ case 'ping':
670
+ reply = { ok: true, data: { pong: Date.now(), node_id: node.nodeId } };
671
+ break;
672
+ case 'info':
673
+ reply = { ok: true, data: {
674
+ node_id: node.nodeId,
675
+ role: node.role,
676
+ name: node.name ?? null,
677
+ port: p2pPort
678
+ } };
679
+ break;
680
+ default:
681
+ reply = { ok: false, error: `unknown-cmd: ${msg?.cmd ?? '(none)'}` };
682
+ }
683
+ try { sock.write(JSON.stringify(reply) + '\n'); } catch {}
684
+ }
685
+ });
686
+ sock.on('error', () => {});
687
+ });
688
+ server.once('error', reject);
689
+ server.listen({ port: p2pPort, host: '::', ipv6Only: false }, () => {
690
+ resolve(server);
691
+ });
692
+ });
693
+ }
694
+
695
+ const shutdown = async (signal) => {
696
+ if (shuttingDown) return;
697
+ shuttingDown = true;
698
+ process.stdout.write(`\nReceived ${signal}; shutting down...\n`);
699
+ if (heartbeatTimer) clearInterval(heartbeatTimer);
700
+ if (pollTimer) clearInterval(pollTimer);
701
+ try {
702
+ await client.heartbeat({ status: 'offline' });
703
+ } catch (err) {
704
+ process.stderr.write(`offline heartbeat failed: ${err?.message ?? err}\n`);
705
+ }
706
+ if (ipcServer) { try { ipcServer.close(); } catch {} }
707
+ if (p2pServer) { try { p2pServer.close(); } catch {} }
708
+ if (healthServer) { try { healthServer.close(); } catch {} }
709
+ try { await shutdownEngine(); } catch {}
710
+ removeSocketFile();
711
+ await removePidFile();
712
+ process.stdout.write('bye\n');
713
+ process.exit(0);
714
+ };
715
+
716
+ process.on('SIGINT', () => { shutdown('SIGINT'); });
717
+ process.on('SIGTERM', () => { shutdown('SIGTERM'); });
718
+
719
+ try {
720
+ ipcServer = await startIpcServer();
721
+ process.stdout.write(`IPC listening on ${socketPath}\n`);
722
+ } catch (err) {
723
+ process.stderr.write(`Failed to bind IPC socket at ${socketPath}: ${err?.message ?? err}\n`);
724
+ }
725
+
726
+ // /healthz — bind a tiny HTTP server on $PORT (default 8080) so
727
+ // Docker / Runpod / Kubernetes HEALTHCHECK probes can reach the
728
+ // daemon even though the main daemon's port is 46337 (P2P, often
729
+ // firewalled). Returns 200 + a JSON daemon snapshot.
730
+ const healthPort = Number.parseInt(process.env.PORT ?? '', 10) || 8080;
731
+ try {
732
+ healthServer = http.createServer((req, res) => {
733
+ if (req.url === '/healthz' || req.url === '/health' || req.url === '/') {
734
+ const snap = snapshot();
735
+ const body = JSON.stringify({
736
+ ok: true,
737
+ pid: snap.pid,
738
+ uptime_ms: snap.uptimeMs,
739
+ role: snap.node?.role ?? null,
740
+ node_id: snap.node?.nodeId ?? null,
741
+ control_plane: snap.controlPlaneUrl,
742
+ heartbeats_ok: snap.stats?.heartbeatsOk ?? 0,
743
+ heartbeats_failed: snap.stats?.heartbeatsFailed ?? 0,
744
+ last_heartbeat_at: snap.stats?.lastHeartbeatAt ?? null,
745
+ last_heartbeat_error: snap.stats?.lastHeartbeatError ?? null,
746
+ jobs_completed: snap.stats?.jobsCompleted ?? 0,
747
+ jobs_failed: snap.stats?.jobsFailed ?? 0,
748
+ active_jobs: stats.activeJobIds.size
749
+ });
750
+ res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'no-store' });
751
+ res.end(body);
752
+ return;
753
+ }
754
+ res.writeHead(404, { 'content-type': 'text/plain' });
755
+ res.end('not found\n');
756
+ });
757
+ await new Promise((resolve, reject) => {
758
+ healthServer.once('error', reject);
759
+ healthServer.listen(healthPort, '0.0.0.0', () => {
760
+ healthServer.removeListener('error', reject);
761
+ resolve();
762
+ });
763
+ });
764
+ process.stdout.write(`/healthz listening on 0.0.0.0:${healthPort}\n`);
765
+ } catch (err) {
766
+ // Health server failure is non-fatal — daemon still does its
767
+ // job, we just won't get container-level liveness probes.
768
+ process.stderr.write(`/healthz failed to bind on :${healthPort}: ${err?.message ?? err}\n`);
769
+ healthServer = null;
770
+ }
771
+
772
+ if (!p2pDisabled) {
773
+ try {
774
+ p2pServer = await startP2pServer();
775
+ process.stdout.write(`P2P listening on ${formatEndpoint(advertisedAddress ?? '-', p2pPort)} (dual-stack)\n`);
776
+ } catch (err) {
777
+ const cause = err?.code === 'EADDRINUSE'
778
+ ? ` (port ${p2pPort} is already in use — pick another with --p2p-port)`
779
+ : err?.code === 'EACCES'
780
+ ? ` (permission denied — try a port above 1024)`
781
+ : '';
782
+ process.stderr.write(`Failed to bind P2P listener on port ${p2pPort}${cause}: ${err?.message ?? err}\n`);
783
+ process.stderr.write('If this is a firewall issue, run `infernet firewall` for per-distro commands.\n');
784
+ }
785
+ }
786
+
787
+ // Ensure a valid bearer token exists. Refreshes autonomously using the
788
+ // node keypair so the daemon never needs a manual `infernet login`.
789
+ {
790
+ const cfg = await loadConfig().catch(() => null);
791
+ const expAt = cfg?.auth?.expiresAt ? new Date(cfg.auth.expiresAt) : null;
792
+ const needsRefresh = !cfg?.auth?.bearerToken || !expAt || expAt - Date.now() < 30 * 86400 * 1000;
793
+ if (needsRefresh) {
794
+ nodeTokenLogin(cfg ?? config).catch((err) =>
795
+ process.stderr.write(`[login] auto-refresh failed: ${err?.message ?? err}\n`)
796
+ );
797
+ }
798
+ }
799
+
800
+ await heartbeat();
801
+ await pollJobs();
802
+ await pollNodeCommands();
803
+
804
+ if (once) {
805
+ if (ipcServer) { try { ipcServer.close(); } catch {} }
806
+ if (p2pServer) { try { p2pServer.close(); } catch {} }
807
+ if (healthServer) { try { healthServer.close(); } catch {} }
808
+ removeSocketFile();
809
+ await removePidFile();
810
+ return 0;
811
+ }
812
+
813
+ heartbeatTimer = setInterval(() => {
814
+ heartbeat().catch((err) => process.stderr.write(`heartbeat threw: ${err?.message ?? err}\n`));
815
+ }, heartbeatMs);
816
+ pollTimer = setInterval(() => {
817
+ pollJobs().catch((err) => process.stderr.write(`poll threw: ${err?.message ?? err}\n`));
818
+ }, pollMs);
819
+ // Commands poll less often than jobs — they're rare (model
820
+ // install/remove from the dashboard) and ollama pull can take
821
+ // minutes, so we don't want overlapping pulls of the same model.
822
+ const commandPollMs = Math.max(pollMs * 2, 30_000);
823
+ const commandTimer = setInterval(() => {
824
+ pollNodeCommands().catch((err) =>
825
+ process.stderr.write(`command poll threw: ${err?.message ?? err}\n`)
826
+ );
827
+ }, commandPollMs);
828
+ if (typeof commandTimer.unref === 'function') commandTimer.unref();
829
+
830
+ // Self-update: check npm registry every 5 minutes. If a newer version is
831
+ // available, pull the installer, re-generate any missing keys + re-register
832
+ // specs (idempotent), then re-exec so the new code is running immediately.
833
+ let isUpgrading = false;
834
+ const updateTimer = setInterval(async () => {
835
+ if (shuttingDown || isUpgrading) return;
836
+ const latest = await fetchLatestVersion();
837
+ if (!latest || !isNewerVersion(CURRENT_VERSION, latest)) return;
838
+ isUpgrading = true;
839
+
840
+ process.stdout.write(`[update] v${latest} available (running v${CURRENT_VERSION}) — upgrading\n`);
841
+ const ok = await pullLatestBinary();
842
+ if (!ok) {
843
+ process.stderr.write('[update] installer failed — will retry next check\n');
844
+ isUpgrading = false;
845
+ return;
846
+ }
847
+
848
+ // Post-upgrade: run `infernet upgrade --skip-setup` is already done
849
+ // (pullLatestBinary above). Now run setup --yes so keys are regenerated,
850
+ // specs re-registered, and the daemon restarts cleanly via re-exec below.
851
+ process.stdout.write('[update] re-initializing keys and specs\n');
852
+ try {
853
+ const { default: setup } = await import('./setup.js');
854
+ const setupArgs = new Map([['yes', true], ['confirm', true], ['skip-pull', true], ['skip-daemon', true]]);
855
+ await setup(setupArgs, { config, client, configPath });
856
+ } catch (err) {
857
+ process.stderr.write(`[update] post-upgrade setup: ${err?.message ?? err}\n`);
858
+ }
859
+
860
+ // Release any in-flight jobs before we close ports.
861
+ for (const jobId of stats.activeJobIds) {
862
+ try {
863
+ await client.failJob(jobId, 'daemon restarting for self-update');
864
+ } catch { /* best-effort */ }
865
+ }
866
+ stats.activeJobIds.clear();
867
+
868
+ // Tear down servers BEFORE spawning the new process so the new daemon
869
+ // can bind the same ports without EADDRINUSE races.
870
+ shuttingDown = true;
871
+ clearInterval(heartbeatTimer);
872
+ clearInterval(pollTimer);
873
+ clearInterval(commandTimer);
874
+ clearInterval(updateTimer);
875
+ if (ipcServer) try { ipcServer.close(); } catch {}
876
+ if (p2pServer) try { p2pServer.close(); } catch {}
877
+ if (healthServer) try { healthServer.close(); } catch {}
878
+ try { await shutdownEngine(); } catch {}
879
+ removeSocketFile();
880
+ await removePidFile();
881
+
882
+ // Small grace period so OS reclaims the ports before the new process binds.
883
+ await new Promise((r) => setTimeout(r, 500));
884
+
885
+ // Re-exec the daemon with the same args so the new code is live.
886
+ process.stdout.write('[update] restarting daemon with new version\n');
887
+ const logPath = getDaemonLogPath();
888
+ const logFd = openSync(logPath, 'a');
889
+ spawn(process.execPath, process.argv.slice(1), {
890
+ detached: true,
891
+ stdio: ['ignore', logFd, logFd],
892
+ env: process.env
893
+ }).unref();
894
+
895
+ process.exit(0);
896
+ }, UPDATE_CHECK_MS);
897
+ if (typeof updateTimer.unref === 'function') updateTimer.unref();
898
+
899
+ await new Promise(() => {});
900
+ return 0;
901
+ }
902
+
903
+ export { HELP };