@infernetprotocol/cli 0.1.60 → 0.1.62

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,7 +21,7 @@
21
21
  import fs from "node:fs/promises";
22
22
  import { loadConfig, getConfigPath, getConfigDir, getDaemonLogPath, fixConfigPermissions } from "../lib/config.js";
23
23
  import { isDaemonAlive, sendToDaemon } from "../lib/ipc.js";
24
- import { spawnDetachedDaemon } from "../lib/daemonize.js";
24
+ import { readLiveSupervisorPid, spawnDetachedSupervisor } from "../lib/supervisor.js";
25
25
  import { createNodeClient } from "../lib/node-client.js";
26
26
  import { submitChatJob, streamChatEvents } from "../lib/remote-chat.js";
27
27
 
@@ -226,7 +226,8 @@ async function checkDaemon() {
226
226
  if (!alive) {
227
227
  // Auto-start: doctor starts the daemon rather than just warning.
228
228
  try {
229
- spawnDetachedDaemon([]);
229
+ // A live supervisor is already bringing the daemon back.
230
+ if (!readLiveSupervisorPid()) spawnDetachedSupervisor([]);
230
231
  } catch (err) {
231
232
  return {
232
233
  status: "fail",
@@ -8,8 +8,12 @@
8
8
  * or anything system-wide. Service file lives under
9
9
  * ~/.config/systemd/user/infernet.service.
10
10
  *
11
- * Linux-only (uses systemd --user). macOS / Windows users continue
12
- * to run `infernet start` directly or wire their own supervisor.
11
+ * Linux uses systemd --user; macOS uses a launchd LaunchAgent
12
+ * (~/Library/LaunchAgents/com.infernetprotocol.node.plist). Elsewhere
13
+ * `infernet start` runs the built-in supervisor (lib/supervisor.js).
14
+ *
15
+ * Both units restart on any non-zero exit (crash, OOM kill, or exit 75 from
16
+ * a self-update) and stay down after a clean `infernet stop` (exit 0).
13
17
  *
14
18
  * Usage:
15
19
  * infernet service install write the unit file + daemon-reload
@@ -22,6 +26,7 @@
22
26
  */
23
27
 
24
28
  import fs from "node:fs/promises";
29
+ import fsSync from "node:fs";
25
30
  import path from "node:path";
26
31
  import os from "node:os";
27
32
  import { spawn, spawnSync } from "node:child_process";
@@ -29,6 +34,7 @@ import { fileURLToPath } from "node:url";
29
34
  import { question } from "../lib/prompt.js";
30
35
 
31
36
  export const SERVICE_NAME = "infernet";
37
+ export const LAUNCHD_LABEL = "com.infernetprotocol.node";
32
38
 
33
39
  const HELP = `infernet service — manage the userland systemd unit
34
40
 
@@ -50,7 +56,7 @@ Flags:
50
56
  -h, --help Show this help.
51
57
 
52
58
  Notes:
53
- - Linux-only (uses systemd --user). Skip on macOS / Windows.
59
+ - Linux: systemd --user. macOS: launchd LaunchAgent (${"com.infernetprotocol.node"}).
54
60
  - Type=simple. ExecStart calls 'infernet start --foreground' so
55
61
  systemd owns the process (no double-fork).
56
62
  - Logs go to journald; tail with 'journalctl --user -u ${SERVICE_NAME} -f'
@@ -60,9 +66,67 @@ Notes:
60
66
  `;
61
67
 
62
68
  export function getServicePath() {
69
+ if (process.platform === "darwin") return getLaunchdPath();
63
70
  return path.join(os.homedir(), ".config", "systemd", "user", `${SERVICE_NAME}.service`);
64
71
  }
65
72
 
73
+ export function getLaunchdPath() {
74
+ return path.join(os.homedir(), "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
75
+ }
76
+
77
+ function xmlEscape(v) {
78
+ return String(v).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
79
+ }
80
+
81
+ /**
82
+ * LaunchAgent plist. KeepAlive.SuccessfulExit=false restarts on any non-zero
83
+ * exit and leaves a clean `infernet stop` (exit 0) alone; RunAtLoad starts it
84
+ * at login. Pure function.
85
+ *
86
+ * @param {{ nodeBin: string, cliEntry: string, logPath: string, pathEnv?: string }} args
87
+ */
88
+ export function buildLaunchdPlist({ nodeBin, cliEntry, logPath, pathEnv } = {}) {
89
+ if (!nodeBin) throw new Error("buildLaunchdPlist: nodeBin is required");
90
+ if (!cliEntry) throw new Error("buildLaunchdPlist: cliEntry is required");
91
+ const args = [nodeBin, cliEntry, "start", "--foreground"]
92
+ .map((a) => ` <string>${xmlEscape(a)}</string>`).join("\n");
93
+ return `<?xml version="1.0" encoding="UTF-8"?>
94
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
95
+ <plist version="1.0">
96
+ <dict>
97
+ <key>Label</key>
98
+ <string>${LAUNCHD_LABEL}</string>
99
+ <key>ProgramArguments</key>
100
+ <array>
101
+ ${args}
102
+ </array>
103
+ <key>EnvironmentVariables</key>
104
+ <dict>
105
+ <key>INFERNET_SUPERVISED</key>
106
+ <string>1</string>
107
+ <key>NODE_ENV</key>
108
+ <string>production</string>
109
+ <key>PATH</key>
110
+ <string>${xmlEscape(pathEnv ?? "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin")}</string>
111
+ </dict>
112
+ <key>RunAtLoad</key>
113
+ <true/>
114
+ <key>KeepAlive</key>
115
+ <dict>
116
+ <key>SuccessfulExit</key>
117
+ <false/>
118
+ </dict>
119
+ <key>ThrottleInterval</key>
120
+ <integer>10</integer>
121
+ <key>StandardOutPath</key>
122
+ <string>${xmlEscape(logPath)}</string>
123
+ <key>StandardErrorPath</key>
124
+ <string>${xmlEscape(logPath)}</string>
125
+ </dict>
126
+ </plist>
127
+ `;
128
+ }
129
+
66
130
  /**
67
131
  * Render the unit file contents from inputs. Pure function — easy to
68
132
  * unit-test, no side effects, no environment lookups.
@@ -84,6 +148,9 @@ export function buildServiceUnit({ nodeBin, cliEntry, environment = {}, descript
84
148
  "Documentation=https://infernetprotocol.com",
85
149
  "After=network-online.target",
86
150
  "Wants=network-online.target",
151
+ // Never give up restarting: a node that crash-loops while the network is
152
+ // down must still come back once it returns.
153
+ "StartLimitIntervalSec=0",
87
154
  "",
88
155
  "[Service]",
89
156
  "Type=simple",
@@ -91,6 +158,8 @@ export function buildServiceUnit({ nodeBin, cliEntry, environment = {}, descript
91
158
  "Restart=on-failure",
92
159
  "RestartSec=10",
93
160
  "Environment=NODE_ENV=production",
161
+ // Self-update exits 75 under a supervisor instead of re-execing (lib/supervisor.js).
162
+ "Environment=INFERNET_SUPERVISED=1",
94
163
  ...envLines,
95
164
  "",
96
165
  "[Install]",
@@ -106,7 +175,79 @@ function resolveCliEntry() {
106
175
  return path.resolve(here, "..", "index.js");
107
176
  }
108
177
 
178
+ /** True when this host can run the unit: systemd --user on Linux, launchd on macOS. */
179
+ export function serviceSupported() {
180
+ if (process.platform === "darwin") return true;
181
+ if (process.platform !== "linux") return false;
182
+ const uid = process.getuid?.();
183
+ return fsSync.existsSync("/run/systemd/system") && uid != null && fsSync.existsSync(`/run/user/${uid}/bus`);
184
+ }
185
+
186
+ /** A system-wide infernet.service (our fleet boxes, root installs) owns the daemon already. */
187
+ export function systemUnitPresent() {
188
+ if (process.platform !== "linux") return false;
189
+ const r = spawnSync("systemctl", ["is-enabled", "--quiet", "infernet.service"], { stdio: "ignore" });
190
+ return r.status === 0
191
+ || fsSync.existsSync("/etc/systemd/system/infernet.service")
192
+ || fsSync.existsSync("/lib/systemd/system/infernet.service");
193
+ }
194
+
195
+ function launchctl(args) {
196
+ return spawnSync("launchctl", args, { stdio: "pipe" });
197
+ }
198
+
199
+ /**
200
+ * Install (if missing) and start the supervisor unit for this platform, so the
201
+ * daemon survives crashes, reboots and self-updates. Never throws.
202
+ *
203
+ * @returns {Promise<{ ok: boolean, how?: string, reason?: string }>}
204
+ */
205
+ export async function ensureService({ log = () => {} } = {}) {
206
+ try {
207
+ if (!serviceSupported()) return { ok: false, reason: "no systemd --user or launchd here" };
208
+ if (systemUnitPresent()) return { ok: false, reason: "system infernet.service manages the daemon" };
209
+ const target = getServicePath();
210
+ const cliEntry = resolveCliEntry();
211
+ await fs.mkdir(path.dirname(target), { recursive: true });
212
+
213
+ if (process.platform === "darwin") {
214
+ const { getDaemonLogPath } = await import("../lib/config.js");
215
+ const plist = buildLaunchdPlist({
216
+ nodeBin: process.execPath, cliEntry, logPath: getDaemonLogPath(), pathEnv: process.env.PATH
217
+ });
218
+ await fs.writeFile(target, plist, { mode: 0o644 });
219
+ const domain = `gui/${process.getuid()}`;
220
+ launchctl(["bootout", `${domain}/${LAUNCHD_LABEL}`]);
221
+ const r = launchctl(["bootstrap", domain, target]);
222
+ if (r.status !== 0) {
223
+ const legacy = launchctl(["load", "-w", target]);
224
+ if (legacy.status !== 0) return { ok: false, reason: `launchctl: ${String(r.stderr || legacy.stderr).trim()}` };
225
+ }
226
+ launchctl(["kickstart", "-k", `${domain}/${LAUNCHD_LABEL}`]);
227
+ log(`launchd agent ${LAUNCHD_LABEL} installed and started`);
228
+ return { ok: true, how: "launchd" };
229
+ }
230
+
231
+ await fs.writeFile(target, buildServiceUnit({ nodeBin: process.execPath, cliEntry }), { mode: 0o644 });
232
+ runSystemctl(["daemon-reload"], { inherit: false });
233
+ const en = runSystemctl(["enable", SERVICE_NAME], { inherit: false });
234
+ if (en.status !== 0) return { ok: false, reason: `systemctl --user enable: ${String(en.stderr).trim()}` };
235
+ const rs = runSystemctl(["restart", SERVICE_NAME], { inherit: false });
236
+ if (rs.status !== 0) return { ok: false, reason: `systemctl --user restart: ${String(rs.stderr).trim()}` };
237
+ // Keep running after logout and start at boot. Allowed for your own user on
238
+ // most distros; harmless when it is refused.
239
+ const linger = spawnSync("loginctl", ["enable-linger", os.userInfo().username], { stdio: "pipe" });
240
+ log(linger.status === 0
241
+ ? "systemd user unit installed, enabled and started (linger on: survives logout and reboot)"
242
+ : "systemd user unit installed, enabled and started (run `loginctl enable-linger $USER` to survive logout)");
243
+ return { ok: true, how: "systemd" };
244
+ } catch (err) {
245
+ return { ok: false, reason: err?.message ?? String(err) };
246
+ }
247
+ }
248
+
109
249
  function ensureLinux() {
250
+ if (process.platform === "darwin") return true;
110
251
  if (process.platform !== "linux") {
111
252
  process.stderr.write(
112
253
  `error: 'infernet service' is Linux-only (your platform: ${process.platform}).\n` +
@@ -218,12 +359,46 @@ function spawnInherited(bin, argv) {
218
359
  });
219
360
  }
220
361
 
362
+ async function darwinCmd(sub) {
363
+ const domain = `gui/${process.getuid()}`;
364
+ const target = getLaunchdPath();
365
+ switch (sub) {
366
+ case "install":
367
+ case "enable": {
368
+ const res = await ensureService({ log: (m) => process.stdout.write(`✓ ${m}\n`) });
369
+ if (!res.ok) process.stderr.write(`error: ${res.reason}\n`);
370
+ return res.ok ? 0 : 1;
371
+ }
372
+ case "disable":
373
+ case "uninstall":
374
+ launchctl(["bootout", `${domain}/${LAUNCHD_LABEL}`]);
375
+ if (sub === "uninstall") { try { await fs.unlink(target); } catch { /* not installed */ } }
376
+ process.stdout.write(`✓ ${sub === "uninstall" ? "removed" : "stopped"} ${LAUNCHD_LABEL}\n`);
377
+ return 0;
378
+ case "status":
379
+ return spawnInherited("launchctl", ["print", `${domain}/${LAUNCHD_LABEL}`]);
380
+ case "logs": {
381
+ const { getDaemonLogPath } = await import("../lib/config.js");
382
+ return spawnInherited("tail", ["-f", getDaemonLogPath()]);
383
+ }
384
+ case "unit": {
385
+ const { getDaemonLogPath } = await import("../lib/config.js");
386
+ process.stdout.write(buildLaunchdPlist({ nodeBin: process.execPath, cliEntry: resolveCliEntry(), logPath: getDaemonLogPath() }));
387
+ return 0;
388
+ }
389
+ default:
390
+ process.stderr.write(HELP);
391
+ return 2;
392
+ }
393
+ }
394
+
221
395
  export default async function service(args) {
222
396
  if (args.has("help") || args.has("h")) {
223
397
  process.stdout.write(HELP);
224
398
  return 0;
225
399
  }
226
400
  const sub = (args.positional ?? [])[0];
401
+ if (process.platform === "darwin") return darwinCmd(sub);
227
402
  switch (sub) {
228
403
  case "install": return installCmd(args);
229
404
  case "uninstall": return uninstallCmd(args);
package/commands/setup.js CHANGED
@@ -33,6 +33,7 @@ import {
33
33
  lastDetectionDiagnostics
34
34
  } from "@infernetprotocol/gpu";
35
35
  import { isDaemonAlive } from "../lib/ipc.js";
36
+ import { ensureService } from "./service.js";
36
37
  import { createNodeClient } from "../lib/node-client.js";
37
38
 
38
39
  const pExec = promisify(execFile);
@@ -53,6 +54,8 @@ Flags:
53
54
  --skip-identity Don't chain into \`infernet init\` if identity is missing.
54
55
  --skip-register Don't chain into \`infernet register\` if not registered.
55
56
  --skip-daemon Don't offer to start the daemon.
57
+ --no-service Don't install the systemd/launchd unit; use the
58
+ built-in supervisor (also INFERNET_NO_SERVICE=1).
56
59
  -h, --help Show this help.
57
60
 
58
61
  By default, every step that wants to install or change something on
@@ -834,7 +837,16 @@ export default async function setup(args) {
834
837
  // Give the OS a moment to release the IPC socket / pidfile.
835
838
  await new Promise((r) => setTimeout(r, 800));
836
839
  }
837
- const startCode = await runSubcommand("start", []);
840
+ // Prefer a real service manager: restarts on crash, at boot, and after
841
+ // self-updates. Containers have their own supervisor (restart policy).
842
+ let startCode = 0;
843
+ const noService = args.has("no-service") || process.env.INFERNET_NO_SERVICE === "1"
844
+ || fsSync.existsSync("/.dockerenv") || fsSync.existsSync("/run/.containerenv");
845
+ const svc = noService ? { ok: false, reason: "disabled" } : await ensureService({ log: (m) => ok(m) });
846
+ if (!svc.ok) {
847
+ if (!noService) process.stdout.write(` no service unit (${svc.reason}); using the built-in supervisor\n`);
848
+ startCode = await runSubcommand("start", []);
849
+ }
838
850
  if (startCode !== 0) {
839
851
  fail(`start exited ${startCode}`);
840
852
  warn(" check `infernet logs` and `infernet status`");
package/commands/start.js CHANGED
@@ -1,8 +1,10 @@
1
1
  /**
2
2
  * `infernet start` — run the node daemon loop.
3
3
  *
4
- * By default, `infernet start` spawns a **detached** background process and
5
- * returns immediately; logs go to `~/.config/infernet/daemon.log`. Use
4
+ * By default, `infernet start` spawns a **detached** supervisor that runs the
5
+ * daemon and restarts it when it dies (lib/supervisor.js), and returns
6
+ * immediately; logs go to `~/.config/infernet/daemon.log`. When the user-level
7
+ * systemd unit is enabled, it starts that instead. Use
6
8
  * `--foreground` to run the loop in the current terminal (useful under
7
9
  * systemd / Docker / Kubernetes where the supervisor wants the process in
8
10
  * the foreground).
@@ -41,7 +43,14 @@ import {
41
43
  saveConfig,
42
44
  fixConfigPermissions
43
45
  } from '../lib/config.js';
44
- import { spawnDetachedDaemon } from '../lib/daemonize.js';
46
+ import {
47
+ EXIT_RESTART,
48
+ isSupervised,
49
+ readLiveSupervisorPid,
50
+ runSupervisor,
51
+ spawnDetachedSupervisor,
52
+ userServiceEnabled
53
+ } from '../lib/supervisor.js';
45
54
  import { isDaemonAlive } from '../lib/ipc.js';
46
55
  import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/network.js';
47
56
  import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
@@ -52,6 +61,7 @@ import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extract
52
61
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
53
62
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
54
63
  import { pullLatestBinary } from './upgrade.js';
64
+ import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
55
65
  import { nodeTokenLogin } from './login.js';
56
66
  import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from '../lib/version.js';
57
67
  import { readInferenceState, readInferenceStateSync, isPidAlive } from '../lib/inference/state.js';
@@ -67,6 +77,7 @@ Usage:
67
77
 
68
78
  Flags:
69
79
  --foreground Run in the current terminal (don't detach)
80
+ --supervise Run the restart-on-crash supervisor in this terminal
70
81
  --heartbeat-interval <ms> Override heartbeat cadence (default 30000)
71
82
  --poll-interval <ms> Override job poll cadence (default 15000)
72
83
  --p2p-port <n> TCP port for peer connections (default 46337)
@@ -108,6 +119,18 @@ function removeSocketFile() {
108
119
  try { unlinkSync(getDaemonSocketPath()); } catch { /* ignore */ }
109
120
  }
110
121
 
122
+ // Exit paths clean up only what is still ours: when another daemon has since started, the pidfile
123
+ // holds its pid and the socket is its listener. Deleting them made that daemon unreachable.
124
+ function ownsPidFile() {
125
+ try { return readFileSync(getDaemonPidPath(), 'utf8').trim() === String(process.pid); } catch { return false; }
126
+ }
127
+
128
+ async function removeOwnPidAndSocket() {
129
+ if (!ownsPidFile()) return;
130
+ removeSocketFile();
131
+ await removePidFile();
132
+ }
133
+
111
134
  // ---------------------------------------------------------------------------
112
135
  // Entry point — branch between "spawn detached" and "run foreground loop".
113
136
  // ---------------------------------------------------------------------------
@@ -117,6 +140,9 @@ export default async function start(args, ctx) {
117
140
  return 0;
118
141
  }
119
142
 
143
+ if (args.has('supervise')) {
144
+ return runSupervisor(passthroughArgs(args));
145
+ }
120
146
  const foreground = args.has('foreground') || process.env.INFERNET_DAEMON_FOREGROUND === '1';
121
147
  if (!foreground) {
122
148
  return spawnAndReturn(args);
@@ -124,14 +150,7 @@ export default async function start(args, ctx) {
124
150
  return runDaemon(args, ctx);
125
151
  }
126
152
 
127
- async function spawnAndReturn(args) {
128
- const alreadyAlive = await isDaemonAlive();
129
- if (alreadyAlive) {
130
- process.stderr.write('A daemon is already running (socket responsive). Run `infernet stop` first.\n');
131
- return 1;
132
- }
133
- removeSocketFile();
134
-
153
+ function passthroughArgs(args) {
135
154
  const passthrough = [];
136
155
  const hb = args.get('heartbeat-interval');
137
156
  if (hb) passthrough.push('--heartbeat-interval', hb);
@@ -142,9 +161,32 @@ async function spawnAndReturn(args) {
142
161
  if (args.has('no-p2p')) passthrough.push('--no-p2p');
143
162
  if (args.has('no-advertise')) passthrough.push('--no-advertise');
144
163
  if (args.has('once')) passthrough.push('--once');
164
+ return passthrough;
165
+ }
166
+
167
+ async function spawnAndReturn(args) {
168
+ const alreadyAlive = await isDaemonAlive();
169
+ if (alreadyAlive) {
170
+ process.stderr.write('A daemon is already running (socket responsive). Run `infernet stop` first.\n');
171
+ return 1;
172
+ }
173
+ const supervisorPid = readLiveSupervisorPid();
174
+ if (supervisorPid) {
175
+ process.stderr.write(`A supervisor is already running (pid ${supervisorPid}) and will bring the daemon back. Run \`infernet stop\` first.\n`);
176
+ return 1;
177
+ }
178
+ // The systemd unit owns the daemon when it is enabled: starting a second
179
+ // one here would fight it over the socket, pidfile and port.
180
+ if (userServiceEnabled()) {
181
+ const { spawnSync } = await import('node:child_process');
182
+ const r = spawnSync('systemctl', ['--user', 'start', 'infernet.service'], { stdio: 'inherit' });
183
+ if (r.status === 0) process.stdout.write('infernet daemon started via systemd (systemctl --user status infernet)\n');
184
+ return r.status ?? 1;
185
+ }
186
+ removeSocketFile();
145
187
 
146
- const { pid, logPath } = spawnDetachedDaemon(passthrough);
147
- process.stdout.write(`infernet daemon started (pid ${pid})\n`);
188
+ const { pid, logPath } = spawnDetachedSupervisor(passthroughArgs(args));
189
+ process.stdout.write(`infernet daemon started under its supervisor (pid ${pid}); it restarts itself if it dies\n`);
148
190
  process.stdout.write(` logs: ${logPath}\n`);
149
191
  process.stdout.write(` socket: ${getDaemonSocketPath()}\n`);
150
192
  process.stdout.write('Tail logs with `infernet logs -f`, query live state with `infernet status`.\n');
@@ -152,6 +194,15 @@ async function spawnAndReturn(args) {
152
194
  }
153
195
 
154
196
  async function runDaemon(args, ctx) {
197
+ // A stray rejection must not take the node down; a thrown exception leaves
198
+ // state unknown, so exit non-zero and let the supervisor start a clean one.
199
+ process.on('unhandledRejection', (reason) => {
200
+ process.stderr.write(`[${new Date().toISOString()}] unhandled rejection (continuing): ${reason?.stack ?? reason}\n`);
201
+ });
202
+ process.on('uncaughtException', (err) => {
203
+ process.stderr.write(`[${new Date().toISOString()}] uncaught exception, exiting for restart: ${err?.stack ?? err}\n`);
204
+ process.exit(1);
205
+ });
155
206
  const { config, client, configPath } = ctx;
156
207
  const node = config.node ?? {};
157
208
  if (!node.role || !node.nodeId) {
@@ -1376,8 +1427,7 @@ async function runDaemon(args, ctx) {
1376
1427
  if (p2pServer) { try { p2pServer.close(); } catch {} }
1377
1428
  if (healthServer) { try { healthServer.close(); } catch {} }
1378
1429
  try { await shutdownEngine(); } catch {}
1379
- removeSocketFile();
1380
- await removePidFile();
1430
+ await removeOwnPidAndSocket();
1381
1431
  process.stdout.write('bye\n');
1382
1432
  process.exit(0);
1383
1433
  };
@@ -1519,8 +1569,7 @@ async function runDaemon(args, ctx) {
1519
1569
  if (ipcServer) { try { ipcServer.close(); } catch {} }
1520
1570
  if (p2pServer) { try { p2pServer.close(); } catch {} }
1521
1571
  if (healthServer) { try { healthServer.close(); } catch {} }
1522
- removeSocketFile();
1523
- await removePidFile();
1572
+ await removeOwnPidAndSocket();
1524
1573
  return 0;
1525
1574
  }
1526
1575
 
@@ -1541,6 +1590,18 @@ async function runDaemon(args, ctx) {
1541
1590
  }, commandPollMs);
1542
1591
  if (typeof commandTimer.unref === 'function') commandTimer.unref();
1543
1592
 
1593
+ // Aggregators verify providers independently (lib/aggregator.js).
1594
+ if (node.role === 'aggregator' && config.controlPlane?.url) {
1595
+ const probeRound = async () => {
1596
+ const r = await runProbeRound({ client, controlPlaneUrl: config.controlPlane.url });
1597
+ stats.lastProbeRound = { at: new Date().toISOString(), ...r };
1598
+ process.stdout.write(`[${stats.lastProbeRound.at}] probe round: ${r.reachable}/${r.probed} reachable${r.error ? ` (${r.error})` : ''}\n`);
1599
+ };
1600
+ setTimeout(() => { probeRound(); }, 5_000).unref?.();
1601
+ const probeTimer = setInterval(() => { probeRound(); }, PROBE_INTERVAL_MS);
1602
+ if (typeof probeTimer.unref === 'function') probeTimer.unref();
1603
+ }
1604
+
1544
1605
  // Open-market training (IPIP-0030). Daemons that opted in via
1545
1606
  // `infernet config set engine.acceptTraining true` (or env var
1546
1607
  // INFERNET_ACCEPT_TRAINING=1) poll for available shards and run them
@@ -1619,12 +1680,19 @@ async function runDaemon(args, ctx) {
1619
1680
  if (p2pServer) try { p2pServer.close(); } catch {}
1620
1681
  if (healthServer) try { healthServer.close(); } catch {}
1621
1682
  try { await withTimeout(shutdownEngine(), SHUTDOWN_DEADLINE_MS, 'engine shutdown'); } catch {}
1622
- removeSocketFile();
1623
- await removePidFile();
1683
+ await removeOwnPidAndSocket();
1624
1684
 
1625
1685
  // Small grace period so OS reclaims the ports before the new process binds.
1626
1686
  await new Promise((r) => setTimeout(r, 500));
1627
1687
 
1688
+ // Under a supervisor, hand the restart to it. A detached re-exec followed by
1689
+ // exit 0 is what killed nodes under systemd: exit 0 ends the unit, and the
1690
+ // control-group kill takes the detached child with it.
1691
+ if (isSupervised()) {
1692
+ process.stdout.write(`[update] exiting ${EXIT_RESTART} so the supervisor starts v${latest}\n`);
1693
+ process.exit(EXIT_RESTART);
1694
+ }
1695
+
1628
1696
  // Re-exec the daemon with the same args so the new code is live.
1629
1697
  process.stdout.write('[update] restarting daemon with new version\n');
1630
1698
  const logPath = getDaemonLogPath();
package/commands/stop.js CHANGED
@@ -7,12 +7,17 @@
7
7
  *
8
8
  * Fallback: read ~/.config/infernet/daemon.pid and send SIGTERM (or a
9
9
  * custom --signal) directly to the process.
10
+ *
11
+ * When `infernet start` left a supervisor running (lib/supervisor.js), it is
12
+ * signalled first: it forwards SIGTERM to the daemon and exits, instead of
13
+ * restarting the daemon we just stopped.
10
14
  */
11
15
 
12
16
  import fs from 'node:fs/promises';
13
17
 
14
18
  import { getDaemonPidPath } from '../lib/config.js';
15
19
  import { isDaemonAlive, sendToDaemon } from '../lib/ipc.js';
20
+ import { readLiveSupervisorPid } from '../lib/supervisor.js';
16
21
 
17
22
  const HELP = `infernet stop — signal the running daemon to exit
18
23
 
@@ -33,6 +38,18 @@ export default async function stop(args) {
33
38
 
34
39
  const explicitSignal = args.get('signal') ?? (args.has('force') ? 'SIGKILL' : null);
35
40
 
41
+ const supervisorPid = readLiveSupervisorPid();
42
+ if (supervisorPid) {
43
+ try {
44
+ process.kill(supervisorPid, explicitSignal ?? 'SIGTERM');
45
+ process.stdout.write(`Stopping supervisor (pid ${supervisorPid}) and its daemon.\n`);
46
+ } catch (err) {
47
+ process.stderr.write(`Failed to signal supervisor ${supervisorPid}: ${err.message}\n`);
48
+ }
49
+ // SIGTERM is forwarded to the daemon; any other signal leaves it to us.
50
+ if (!explicitSignal) return 0;
51
+ }
52
+
36
53
  // Prefer graceful IPC-driven shutdown when available.
37
54
  if (!explicitSignal && (await isDaemonAlive())) {
38
55
  const res = await sendToDaemon('shutdown', null, { timeoutMs: 5000 });
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Aggregator duty, phase 1: independent provider verification.
3
+ *
4
+ * Every round an aggregator reads the public peer list (/api/peers), dials each
5
+ * provider's advertised P2P address, sends `{"cmd":"info"}` and times the reply.
6
+ * The signed report (POST /api/v1/node/aggregator/probes) lets /status say
7
+ * "reachable, verified by N aggregators" instead of trusting a node's own
8
+ * heartbeat. Later phases add coordination of multi-node jobs (IPIP-0028/0033).
9
+ */
10
+
11
+ import net from 'node:net';
12
+
13
+ export const PROBE_INTERVAL_MS = 60_000;
14
+ const PROBE_TIMEOUT_MS = 5_000;
15
+ const PROBE_CONCURRENCY = 8;
16
+ const MAX_PEERS = 500;
17
+
18
+ /** `/ip4/1.2.3.4/tcp/46337` or `/dns4/host/tcp/n` -> { host, port }, else null. */
19
+ export function parseMultiaddr(addr) {
20
+ const m = /^\/(ip4|ip6|dns4|dns6|dns)\/([^/]+)\/tcp\/(\d{1,5})$/.exec(String(addr ?? ''));
21
+ if (!m) return null;
22
+ const port = Number(m[3]);
23
+ if (port < 1 || port > 65535) return null;
24
+ return { host: m[2], port };
25
+ }
26
+
27
+ /**
28
+ * Dial one peer and ask for its identity.
29
+ * @returns {Promise<{ reachable: boolean, rtt_ms: number|null, node_id: string|null, error: string|null }>}
30
+ */
31
+ export function probePeer({ host, port }, { timeoutMs = PROBE_TIMEOUT_MS } = {}) {
32
+ return new Promise((resolve) => {
33
+ const t0 = Date.now();
34
+ let settled = false;
35
+ let buf = '';
36
+ const sock = net.connect({ host, port });
37
+ const done = (result) => {
38
+ if (settled) return;
39
+ settled = true;
40
+ clearTimeout(timer);
41
+ sock.destroy();
42
+ resolve(result);
43
+ };
44
+ const timer = setTimeout(
45
+ () => done({ reachable: false, rtt_ms: null, node_id: null, error: 'timeout' }),
46
+ timeoutMs
47
+ );
48
+ sock.setEncoding('utf8');
49
+ sock.on('connect', () => sock.write(JSON.stringify({ cmd: 'info' }) + '\n'));
50
+ sock.on('data', (chunk) => {
51
+ buf += chunk;
52
+ const nl = buf.indexOf('\n');
53
+ if (nl < 0) {
54
+ if (buf.length > 8192) done({ reachable: false, rtt_ms: null, node_id: null, error: 'bad-reply' });
55
+ return;
56
+ }
57
+ let msg = null;
58
+ try { msg = JSON.parse(buf.slice(0, nl)); } catch { /* handled below */ }
59
+ if (msg?.ok && msg.data) {
60
+ done({ reachable: true, rtt_ms: Date.now() - t0, node_id: String(msg.data.node_id ?? '') || null, error: null });
61
+ } else {
62
+ done({ reachable: false, rtt_ms: null, node_id: null, error: 'bad-reply' });
63
+ }
64
+ });
65
+ sock.on('error', (err) => done({ reachable: false, rtt_ms: null, node_id: null, error: err?.code ?? 'error' }));
66
+ sock.on('close', () => done({ reachable: false, rtt_ms: null, node_id: null, error: 'closed' }));
67
+ });
68
+ }
69
+
70
+ async function mapLimit(items, limit, fn) {
71
+ const out = new Array(items.length);
72
+ let next = 0;
73
+ async function worker() {
74
+ while (next < items.length) {
75
+ const i = next++;
76
+ out[i] = await fn(items[i]);
77
+ }
78
+ }
79
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
80
+ return out;
81
+ }
82
+
83
+ /**
84
+ * One probe round: fetch peers, probe them, report. Never throws.
85
+ * @returns {Promise<{ probed: number, reachable: number, error?: string }>}
86
+ */
87
+ export async function runProbeRound({ client, controlPlaneUrl, fetchImpl = fetch, probe = probePeer }) {
88
+ try {
89
+ const res = await fetchImpl(`${controlPlaneUrl.replace(/\/+$/, '')}/api/peers`, {
90
+ headers: { accept: 'application/json' },
91
+ signal: AbortSignal.timeout(15_000)
92
+ });
93
+ if (!res.ok) return { probed: 0, reachable: 0, error: `peers HTTP ${res.status}` };
94
+ const peers = ((await res.json())?.data ?? [])
95
+ .filter((p) => p?.pubkey && parseMultiaddr(p.multiaddr))
96
+ .slice(0, MAX_PEERS);
97
+ const results = await mapLimit(peers, PROBE_CONCURRENCY, async (p) => ({
98
+ pubkey: p.pubkey,
99
+ ...(await probe(parseMultiaddr(p.multiaddr)))
100
+ }));
101
+ if (results.length > 0) await client.signedFetch('/api/v1/node/aggregator/probes', { probes: results });
102
+ return { probed: results.length, reachable: results.filter((r) => r.reachable).length };
103
+ } catch (err) {
104
+ return { probed: 0, reachable: 0, error: err?.message ?? String(err) };
105
+ }
106
+ }
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Built-in supervisor for the node daemon.
3
+ *
4
+ * `infernet start` used to detach ONE `start --foreground` process with nothing
5
+ * watching it: a crash, an OOM kill or a failed self-update re-exec left the node
6
+ * dead until the operator noticed (most outside nodes, 2026-05..10). Now the
7
+ * detached process is this supervisor, which runs the daemon as a child and
8
+ * restarts it.
9
+ *
10
+ * Exit-code contract (shared with systemd `Restart=on-failure` and launchd
11
+ * `KeepAlive.SuccessfulExit=false`):
12
+ * 0 deliberate stop (`infernet stop`, SIGTERM): stay down
13
+ * EXIT_RESTART (75) restart now (self-update installed a new version)
14
+ * anything else crash: restart with backoff
15
+ */
16
+
17
+ import { spawn, spawnSync } from 'node:child_process';
18
+ import fs from 'node:fs';
19
+ import path from 'node:path';
20
+
21
+ import { getConfigDir, getDaemonLogPath } from './config.js';
22
+
23
+ export const EXIT_RESTART = 75;
24
+
25
+ const BACKOFF_START_MS = 1_000;
26
+ const BACKOFF_MAX_MS = 60_000;
27
+ /** A child that stayed up this long resets the backoff. */
28
+ const HEALTHY_AFTER_MS = 10 * 60_000;
29
+
30
+ export function getSupervisorPidPath() {
31
+ return path.join(getConfigDir(), 'supervisor.pid');
32
+ }
33
+
34
+ /**
35
+ * True when something will restart this daemon after it exits non-zero: our
36
+ * supervisor, our systemd/launchd units, or any systemd unit (INVOCATION_ID).
37
+ */
38
+ export function isSupervised(env = process.env) {
39
+ return env.INFERNET_SUPERVISED === '1'
40
+ || Boolean(env.INVOCATION_ID)
41
+ || (env.XPC_SERVICE_NAME ?? '').startsWith('com.infernetprotocol.');
42
+ }
43
+
44
+ /** Next backoff delay: doubles from 1 s, capped at 60 s; reset after a healthy run. */
45
+ export function nextBackoff(prevMs, ranForMs) {
46
+ if (ranForMs >= HEALTHY_AFTER_MS || !prevMs) return BACKOFF_START_MS;
47
+ return Math.min(prevMs * 2, BACKOFF_MAX_MS);
48
+ }
49
+
50
+ /**
51
+ * Whether the supervisor should start the daemon again after it exited.
52
+ * `signal` is set when the child was killed (OOM killer, kill -9): restart.
53
+ */
54
+ export function shouldRestart(code, signal) {
55
+ if (signal) return true;
56
+ return code !== 0;
57
+ }
58
+
59
+ /** Pid of a live supervisor from supervisor.pid, or null. */
60
+ export function readLiveSupervisorPid() {
61
+ let raw;
62
+ try { raw = fs.readFileSync(getSupervisorPidPath(), 'utf8'); } catch { return null; }
63
+ const pid = Number.parseInt(raw.trim(), 10);
64
+ if (!Number.isFinite(pid) || pid <= 0) return null;
65
+ try { process.kill(pid, 0); return pid; } catch { return null; }
66
+ }
67
+
68
+ /** The user-level systemd unit (`infernet service install`) is installed and enabled. */
69
+ export function userServiceEnabled() {
70
+ if (process.platform !== 'linux') return false;
71
+ const r = spawnSync('systemctl', ['--user', 'is-enabled', '--quiet', 'infernet.service'], { stdio: 'ignore' });
72
+ return r.status === 0;
73
+ }
74
+
75
+ /**
76
+ * Detach a supervisor (`start --supervise`) that runs `start --foreground`.
77
+ * @param {string[]} passthroughArgs
78
+ * @returns {{ pid: number, logPath: string }}
79
+ */
80
+ export function spawnDetachedSupervisor(passthroughArgs = []) {
81
+ fs.mkdirSync(getConfigDir(), { recursive: true, mode: 0o700 });
82
+ const logPath = getDaemonLogPath();
83
+ const out = fs.openSync(logPath, 'a');
84
+ const scriptPath = process.argv[1];
85
+ if (!scriptPath || !path.isAbsolute(scriptPath)) {
86
+ throw new Error('Unable to determine absolute path to the CLI script for daemonization.');
87
+ }
88
+ const child = spawn(
89
+ process.execPath,
90
+ [scriptPath, 'start', '--supervise', ...passthroughArgs],
91
+ { detached: true, stdio: ['ignore', out, out], env: process.env }
92
+ );
93
+ child.unref();
94
+ return { pid: child.pid, logPath };
95
+ }
96
+
97
+ /**
98
+ * Supervisor loop. Runs until the daemon exits 0 or the supervisor is signalled.
99
+ * @param {string[]} passthroughArgs
100
+ * @returns {Promise<number>}
101
+ */
102
+ export async function runSupervisor(passthroughArgs = []) {
103
+ const pidPath = getSupervisorPidPath();
104
+ fs.mkdirSync(getConfigDir(), { recursive: true, mode: 0o700 });
105
+ fs.writeFileSync(pidPath, String(process.pid), { mode: 0o600 });
106
+ const log = (msg) => process.stdout.write(`[supervisor ${new Date().toISOString()}] ${msg}\n`);
107
+
108
+ let child = null;
109
+ let stopping = false;
110
+ let wake = null;
111
+ const stop = (signal) => {
112
+ if (stopping) return;
113
+ stopping = true;
114
+ log(`${signal}: stopping daemon`);
115
+ if (child) { try { child.kill('SIGTERM'); } catch { /* gone */ } }
116
+ if (wake) wake();
117
+ };
118
+ const onTerm = () => stop('SIGTERM');
119
+ const onInt = () => stop('SIGINT');
120
+ const onHup = () => { /* survive the terminal closing */ };
121
+ process.on('SIGTERM', onTerm);
122
+ process.on('SIGINT', onInt);
123
+ process.on('SIGHUP', onHup);
124
+
125
+ let backoff = 0;
126
+ try {
127
+ while (!stopping) {
128
+ const startedAt = Date.now();
129
+ // Re-read argv[1] each time: an upgrade replaces the files at the same path.
130
+ child = spawn(process.execPath, [process.argv[1], 'start', '--foreground', ...passthroughArgs], {
131
+ stdio: ['ignore', 'inherit', 'inherit'],
132
+ env: { ...process.env, INFERNET_DAEMON_FOREGROUND: '1', INFERNET_SUPERVISED: '1' }
133
+ });
134
+ log(`daemon started (pid ${child.pid})`);
135
+ const { code, signal } = await new Promise((resolve) => {
136
+ child.once('exit', (c, s) => resolve({ code: c, signal: s }));
137
+ child.once('error', (err) => {
138
+ log(`spawn failed: ${err?.message ?? err}`);
139
+ resolve({ code: 1, signal: null });
140
+ });
141
+ });
142
+ child = null;
143
+ if (stopping) break;
144
+ if (!shouldRestart(code, signal)) {
145
+ log('daemon exited 0 (stopped on purpose); supervisor exiting');
146
+ break;
147
+ }
148
+ if (code === EXIT_RESTART) {
149
+ backoff = 0;
150
+ log('daemon asked for a restart (update installed)');
151
+ continue;
152
+ }
153
+ backoff = nextBackoff(backoff, Date.now() - startedAt);
154
+ log(`daemon died (${signal ?? `exit ${code}`}); restarting in ${backoff / 1000}s`);
155
+ await new Promise((resolve) => {
156
+ wake = resolve;
157
+ setTimeout(resolve, backoff);
158
+ });
159
+ wake = null;
160
+ }
161
+ } finally {
162
+ process.off('SIGTERM', onTerm);
163
+ process.off('SIGINT', onInt);
164
+ process.off('SIGHUP', onHup);
165
+ try {
166
+ if (fs.readFileSync(pidPath, 'utf8').trim() === String(process.pid)) fs.unlinkSync(pidPath);
167
+ } catch { /* already gone */ }
168
+ }
169
+ return 0;
170
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.60",
3
+ "version": "0.1.62",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,12 +30,12 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/auth": "0.1.60",
34
- "@infernetprotocol/deploy-providers": "0.1.60",
35
- "@infernetprotocol/engine": "0.1.60",
36
- "@infernetprotocol/config": "0.1.60",
37
- "@infernetprotocol/rpc-adapter": "0.1.60",
38
- "@infernetprotocol/gpu": "0.1.60"
33
+ "@infernetprotocol/deploy-providers": "0.1.62",
34
+ "@infernetprotocol/gpu": "0.1.62",
35
+ "@infernetprotocol/rpc-adapter": "0.1.62",
36
+ "@infernetprotocol/engine": "0.1.62",
37
+ "@infernetprotocol/auth": "0.1.62",
38
+ "@infernetprotocol/config": "0.1.62"
39
39
  },
40
40
  "repository": {
41
41
  "type": "git",