@infernetprotocol/cli 0.1.61 → 0.1.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,7 +21,7 @@
21
21
  import fs from "node:fs/promises";
22
22
  import { loadConfig, getConfigPath, getConfigDir, getDaemonLogPath, fixConfigPermissions } from "../lib/config.js";
23
23
  import { isDaemonAlive, sendToDaemon } from "../lib/ipc.js";
24
- import { spawnDetachedDaemon } from "../lib/daemonize.js";
24
+ import { readLiveSupervisorPid, spawnDetachedSupervisor } from "../lib/supervisor.js";
25
25
  import { createNodeClient } from "../lib/node-client.js";
26
26
  import { submitChatJob, streamChatEvents } from "../lib/remote-chat.js";
27
27
 
@@ -226,7 +226,8 @@ async function checkDaemon() {
226
226
  if (!alive) {
227
227
  // Auto-start: doctor starts the daemon rather than just warning.
228
228
  try {
229
- spawnDetachedDaemon([]);
229
+ // A live supervisor is already bringing the daemon back.
230
+ if (!readLiveSupervisorPid()) spawnDetachedSupervisor([]);
230
231
  } catch (err) {
231
232
  return {
232
233
  status: "fail",
@@ -8,8 +8,12 @@
8
8
  * or anything system-wide. Service file lives under
9
9
  * ~/.config/systemd/user/infernet.service.
10
10
  *
11
- * Linux-only (uses systemd --user). macOS / Windows users continue
12
- * to run `infernet start` directly or wire their own supervisor.
11
+ * Linux uses systemd --user; macOS uses a launchd LaunchAgent
12
+ * (~/Library/LaunchAgents/com.infernetprotocol.node.plist). Elsewhere
13
+ * `infernet start` runs the built-in supervisor (lib/supervisor.js).
14
+ *
15
+ * Both units restart on any non-zero exit (crash, OOM kill, or exit 75 from
16
+ * a self-update) and stay down after a clean `infernet stop` (exit 0).
13
17
  *
14
18
  * Usage:
15
19
  * infernet service install write the unit file + daemon-reload
@@ -22,6 +26,7 @@
22
26
  */
23
27
 
24
28
  import fs from "node:fs/promises";
29
+ import fsSync from "node:fs";
25
30
  import path from "node:path";
26
31
  import os from "node:os";
27
32
  import { spawn, spawnSync } from "node:child_process";
@@ -29,6 +34,7 @@ import { fileURLToPath } from "node:url";
29
34
  import { question } from "../lib/prompt.js";
30
35
 
31
36
  export const SERVICE_NAME = "infernet";
37
+ export const LAUNCHD_LABEL = "com.infernetprotocol.node";
32
38
 
33
39
  const HELP = `infernet service — manage the userland systemd unit
34
40
 
@@ -50,7 +56,7 @@ Flags:
50
56
  -h, --help Show this help.
51
57
 
52
58
  Notes:
53
- - Linux-only (uses systemd --user). Skip on macOS / Windows.
59
+ - Linux: systemd --user. macOS: launchd LaunchAgent (${"com.infernetprotocol.node"}).
54
60
  - Type=simple. ExecStart calls 'infernet start --foreground' so
55
61
  systemd owns the process (no double-fork).
56
62
  - Logs go to journald; tail with 'journalctl --user -u ${SERVICE_NAME} -f'
@@ -60,9 +66,67 @@ Notes:
60
66
  `;
61
67
 
62
68
  export function getServicePath() {
69
+ if (process.platform === "darwin") return getLaunchdPath();
63
70
  return path.join(os.homedir(), ".config", "systemd", "user", `${SERVICE_NAME}.service`);
64
71
  }
65
72
 
73
+ export function getLaunchdPath() {
74
+ return path.join(os.homedir(), "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
75
+ }
76
+
77
+ function xmlEscape(v) {
78
+ return String(v).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
79
+ }
80
+
81
+ /**
82
+ * LaunchAgent plist. KeepAlive.SuccessfulExit=false restarts on any non-zero
83
+ * exit and leaves a clean `infernet stop` (exit 0) alone; RunAtLoad starts it
84
+ * at login. Pure function.
85
+ *
86
+ * @param {{ nodeBin: string, cliEntry: string, logPath: string, pathEnv?: string }} args
87
+ */
88
+ export function buildLaunchdPlist({ nodeBin, cliEntry, logPath, pathEnv } = {}) {
89
+ if (!nodeBin) throw new Error("buildLaunchdPlist: nodeBin is required");
90
+ if (!cliEntry) throw new Error("buildLaunchdPlist: cliEntry is required");
91
+ const args = [nodeBin, cliEntry, "start", "--foreground"]
92
+ .map((a) => ` <string>${xmlEscape(a)}</string>`).join("\n");
93
+ return `<?xml version="1.0" encoding="UTF-8"?>
94
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
95
+ <plist version="1.0">
96
+ <dict>
97
+ <key>Label</key>
98
+ <string>${LAUNCHD_LABEL}</string>
99
+ <key>ProgramArguments</key>
100
+ <array>
101
+ ${args}
102
+ </array>
103
+ <key>EnvironmentVariables</key>
104
+ <dict>
105
+ <key>INFERNET_SUPERVISED</key>
106
+ <string>1</string>
107
+ <key>NODE_ENV</key>
108
+ <string>production</string>
109
+ <key>PATH</key>
110
+ <string>${xmlEscape(pathEnv ?? "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin")}</string>
111
+ </dict>
112
+ <key>RunAtLoad</key>
113
+ <true/>
114
+ <key>KeepAlive</key>
115
+ <dict>
116
+ <key>SuccessfulExit</key>
117
+ <false/>
118
+ </dict>
119
+ <key>ThrottleInterval</key>
120
+ <integer>10</integer>
121
+ <key>StandardOutPath</key>
122
+ <string>${xmlEscape(logPath)}</string>
123
+ <key>StandardErrorPath</key>
124
+ <string>${xmlEscape(logPath)}</string>
125
+ </dict>
126
+ </plist>
127
+ `;
128
+ }
129
+
66
130
  /**
67
131
  * Render the unit file contents from inputs. Pure function — easy to
68
132
  * unit-test, no side effects, no environment lookups.
@@ -84,6 +148,9 @@ export function buildServiceUnit({ nodeBin, cliEntry, environment = {}, descript
84
148
  "Documentation=https://infernetprotocol.com",
85
149
  "After=network-online.target",
86
150
  "Wants=network-online.target",
151
+ // Never give up restarting: a node that crash-loops while the network is
152
+ // down must still come back once it returns.
153
+ "StartLimitIntervalSec=0",
87
154
  "",
88
155
  "[Service]",
89
156
  "Type=simple",
@@ -91,6 +158,8 @@ export function buildServiceUnit({ nodeBin, cliEntry, environment = {}, descript
91
158
  "Restart=on-failure",
92
159
  "RestartSec=10",
93
160
  "Environment=NODE_ENV=production",
161
+ // Self-update exits 75 under a supervisor instead of re-execing (lib/supervisor.js).
162
+ "Environment=INFERNET_SUPERVISED=1",
94
163
  ...envLines,
95
164
  "",
96
165
  "[Install]",
@@ -106,7 +175,79 @@ function resolveCliEntry() {
106
175
  return path.resolve(here, "..", "index.js");
107
176
  }
108
177
 
178
+ /** True when this host can run the unit: systemd --user on Linux, launchd on macOS. */
179
+ export function serviceSupported() {
180
+ if (process.platform === "darwin") return true;
181
+ if (process.platform !== "linux") return false;
182
+ const uid = process.getuid?.();
183
+ return fsSync.existsSync("/run/systemd/system") && uid != null && fsSync.existsSync(`/run/user/${uid}/bus`);
184
+ }
185
+
186
+ /** A system-wide infernet.service (our fleet boxes, root installs) owns the daemon already. */
187
+ export function systemUnitPresent() {
188
+ if (process.platform !== "linux") return false;
189
+ const r = spawnSync("systemctl", ["is-enabled", "--quiet", "infernet.service"], { stdio: "ignore" });
190
+ return r.status === 0
191
+ || fsSync.existsSync("/etc/systemd/system/infernet.service")
192
+ || fsSync.existsSync("/lib/systemd/system/infernet.service");
193
+ }
194
+
195
+ function launchctl(args) {
196
+ return spawnSync("launchctl", args, { stdio: "pipe" });
197
+ }
198
+
199
+ /**
200
+ * Install (if missing) and start the supervisor unit for this platform, so the
201
+ * daemon survives crashes, reboots and self-updates. Never throws.
202
+ *
203
+ * @returns {Promise<{ ok: boolean, how?: string, reason?: string }>}
204
+ */
205
+ export async function ensureService({ log = () => {} } = {}) {
206
+ try {
207
+ if (!serviceSupported()) return { ok: false, reason: "no systemd --user or launchd here" };
208
+ if (systemUnitPresent()) return { ok: false, reason: "system infernet.service manages the daemon" };
209
+ const target = getServicePath();
210
+ const cliEntry = resolveCliEntry();
211
+ await fs.mkdir(path.dirname(target), { recursive: true });
212
+
213
+ if (process.platform === "darwin") {
214
+ const { getDaemonLogPath } = await import("../lib/config.js");
215
+ const plist = buildLaunchdPlist({
216
+ nodeBin: process.execPath, cliEntry, logPath: getDaemonLogPath(), pathEnv: process.env.PATH
217
+ });
218
+ await fs.writeFile(target, plist, { mode: 0o644 });
219
+ const domain = `gui/${process.getuid()}`;
220
+ launchctl(["bootout", `${domain}/${LAUNCHD_LABEL}`]);
221
+ const r = launchctl(["bootstrap", domain, target]);
222
+ if (r.status !== 0) {
223
+ const legacy = launchctl(["load", "-w", target]);
224
+ if (legacy.status !== 0) return { ok: false, reason: `launchctl: ${String(r.stderr || legacy.stderr).trim()}` };
225
+ }
226
+ launchctl(["kickstart", "-k", `${domain}/${LAUNCHD_LABEL}`]);
227
+ log(`launchd agent ${LAUNCHD_LABEL} installed and started`);
228
+ return { ok: true, how: "launchd" };
229
+ }
230
+
231
+ await fs.writeFile(target, buildServiceUnit({ nodeBin: process.execPath, cliEntry }), { mode: 0o644 });
232
+ runSystemctl(["daemon-reload"], { inherit: false });
233
+ const en = runSystemctl(["enable", SERVICE_NAME], { inherit: false });
234
+ if (en.status !== 0) return { ok: false, reason: `systemctl --user enable: ${String(en.stderr).trim()}` };
235
+ const rs = runSystemctl(["restart", SERVICE_NAME], { inherit: false });
236
+ if (rs.status !== 0) return { ok: false, reason: `systemctl --user restart: ${String(rs.stderr).trim()}` };
237
+ // Keep running after logout and start at boot. Allowed for your own user on
238
+ // most distros; harmless when it is refused.
239
+ const linger = spawnSync("loginctl", ["enable-linger", os.userInfo().username], { stdio: "pipe" });
240
+ log(linger.status === 0
241
+ ? "systemd user unit installed, enabled and started (linger on: survives logout and reboot)"
242
+ : "systemd user unit installed, enabled and started (run `loginctl enable-linger $USER` to survive logout)");
243
+ return { ok: true, how: "systemd" };
244
+ } catch (err) {
245
+ return { ok: false, reason: err?.message ?? String(err) };
246
+ }
247
+ }
248
+
109
249
  function ensureLinux() {
250
+ if (process.platform === "darwin") return true;
110
251
  if (process.platform !== "linux") {
111
252
  process.stderr.write(
112
253
  `error: 'infernet service' is Linux-only (your platform: ${process.platform}).\n` +
@@ -218,12 +359,46 @@ function spawnInherited(bin, argv) {
218
359
  });
219
360
  }
220
361
 
362
+ async function darwinCmd(sub) {
363
+ const domain = `gui/${process.getuid()}`;
364
+ const target = getLaunchdPath();
365
+ switch (sub) {
366
+ case "install":
367
+ case "enable": {
368
+ const res = await ensureService({ log: (m) => process.stdout.write(`✓ ${m}\n`) });
369
+ if (!res.ok) process.stderr.write(`error: ${res.reason}\n`);
370
+ return res.ok ? 0 : 1;
371
+ }
372
+ case "disable":
373
+ case "uninstall":
374
+ launchctl(["bootout", `${domain}/${LAUNCHD_LABEL}`]);
375
+ if (sub === "uninstall") { try { await fs.unlink(target); } catch { /* not installed */ } }
376
+ process.stdout.write(`✓ ${sub === "uninstall" ? "removed" : "stopped"} ${LAUNCHD_LABEL}\n`);
377
+ return 0;
378
+ case "status":
379
+ return spawnInherited("launchctl", ["print", `${domain}/${LAUNCHD_LABEL}`]);
380
+ case "logs": {
381
+ const { getDaemonLogPath } = await import("../lib/config.js");
382
+ return spawnInherited("tail", ["-f", getDaemonLogPath()]);
383
+ }
384
+ case "unit": {
385
+ const { getDaemonLogPath } = await import("../lib/config.js");
386
+ process.stdout.write(buildLaunchdPlist({ nodeBin: process.execPath, cliEntry: resolveCliEntry(), logPath: getDaemonLogPath() }));
387
+ return 0;
388
+ }
389
+ default:
390
+ process.stderr.write(HELP);
391
+ return 2;
392
+ }
393
+ }
394
+
221
395
  export default async function service(args) {
222
396
  if (args.has("help") || args.has("h")) {
223
397
  process.stdout.write(HELP);
224
398
  return 0;
225
399
  }
226
400
  const sub = (args.positional ?? [])[0];
401
+ if (process.platform === "darwin") return darwinCmd(sub);
227
402
  switch (sub) {
228
403
  case "install": return installCmd(args);
229
404
  case "uninstall": return uninstallCmd(args);
package/commands/setup.js CHANGED
@@ -15,7 +15,7 @@
15
15
  * infernet setup --skip-pull # don't pull a model
16
16
  */
17
17
 
18
- import { execFile, spawn } from "node:child_process";
18
+ import { execFile, spawn, spawnSync } from "node:child_process";
19
19
  import fsSync from "node:fs";
20
20
  import { promisify } from "node:util";
21
21
  import { fileURLToPath } from "node:url";
@@ -33,6 +33,7 @@ import {
33
33
  lastDetectionDiagnostics
34
34
  } from "@infernetprotocol/gpu";
35
35
  import { isDaemonAlive } from "../lib/ipc.js";
36
+ import { ensureService, systemUnitPresent } from "./service.js";
36
37
  import { createNodeClient } from "../lib/node-client.js";
37
38
 
38
39
  const pExec = promisify(execFile);
@@ -53,6 +54,8 @@ Flags:
53
54
  --skip-identity Don't chain into \`infernet init\` if identity is missing.
54
55
  --skip-register Don't chain into \`infernet register\` if not registered.
55
56
  --skip-daemon Don't offer to start the daemon.
57
+ --no-service Don't install the systemd/launchd unit; use the
58
+ built-in supervisor (also INFERNET_NO_SERVICE=1).
56
59
  -h, --help Show this help.
57
60
 
58
61
  By default, every step that wants to install or change something on
@@ -824,7 +827,7 @@ export default async function setup(args) {
824
827
  n += 1;
825
828
  step(n, total, "Daemon");
826
829
  const wasAlive = await isDaemonAlive(500);
827
- if (wasAlive) {
830
+ if (wasAlive && !systemUnitPresent()) {
828
831
  process.stdout.write(" daemon was running — restarting so it picks up current CLI code\n");
829
832
  try {
830
833
  await runSubcommand("stop", []);
@@ -834,7 +837,26 @@ export default async function setup(args) {
834
837
  // Give the OS a moment to release the IPC socket / pidfile.
835
838
  await new Promise((r) => setTimeout(r, 800));
836
839
  }
837
- const startCode = await runSubcommand("start", []);
840
+ // Prefer a real service manager: restarts on crash, at boot, and after
841
+ // self-updates. Containers have their own supervisor (restart policy).
842
+ let startCode = 0;
843
+ const noService = args.has("no-service") || process.env.INFERNET_NO_SERVICE === "1"
844
+ || fsSync.existsSync("/.dockerenv") || fsSync.existsSync("/run/.containerenv");
845
+ if (systemUnitPresent()) {
846
+ // A system-wide infernet.service owns the daemon (fleet boxes, root
847
+ // installs). Starting another one here is how a second daemon leaked
848
+ // next to it for a week on bbs; restart the unit instead.
849
+ const r = spawnSync("systemctl", ["restart", "infernet.service"], { stdio: "pipe" });
850
+ const viaSudo = r.status === 0 ? r : spawnSync("sudo", ["-n", "systemctl", "restart", "infernet.service"], { stdio: "pipe" });
851
+ if (viaSudo.status === 0) ok("restarted infernet.service");
852
+ else warn("infernet.service manages the daemon; restart it with: sudo systemctl restart infernet");
853
+ } else {
854
+ const svc = noService ? { ok: false, reason: "disabled" } : await ensureService({ log: (m) => ok(m) });
855
+ if (!svc.ok) {
856
+ if (!noService) process.stdout.write(` no service unit (${svc.reason}); using the built-in supervisor\n`);
857
+ startCode = await runSubcommand("start", []);
858
+ }
859
+ }
838
860
  if (startCode !== 0) {
839
861
  fail(`start exited ${startCode}`);
840
862
  warn(" check `infernet logs` and `infernet status`");
package/commands/start.js CHANGED
@@ -1,8 +1,10 @@
1
1
  /**
2
2
  * `infernet start` — run the node daemon loop.
3
3
  *
4
- * By default, `infernet start` spawns a **detached** background process and
5
- * returns immediately; logs go to `~/.config/infernet/daemon.log`. Use
4
+ * By default, `infernet start` spawns a **detached** supervisor that runs the
5
+ * daemon and restarts it when it dies (lib/supervisor.js), and returns
6
+ * immediately; logs go to `~/.config/infernet/daemon.log`. When the user-level
7
+ * systemd unit is enabled, it starts that instead. Use
6
8
  * `--foreground` to run the loop in the current terminal (useful under
7
9
  * systemd / Docker / Kubernetes where the supervisor wants the process in
8
10
  * the foreground).
@@ -41,7 +43,14 @@ import {
41
43
  saveConfig,
42
44
  fixConfigPermissions
43
45
  } from '../lib/config.js';
44
- import { spawnDetachedDaemon } from '../lib/daemonize.js';
46
+ import {
47
+ EXIT_RESTART,
48
+ isSupervised,
49
+ readLiveSupervisorPid,
50
+ runSupervisor,
51
+ spawnDetachedSupervisor,
52
+ userServiceEnabled
53
+ } from '../lib/supervisor.js';
45
54
  import { isDaemonAlive } from '../lib/ipc.js';
46
55
  import { resolveP2pPort, detectLocalAddress, formatEndpoint } from '../lib/network.js';
47
56
  import { executeChatJob, failChatJob, shutdownEngine } from '../lib/chat-executor.js';
@@ -52,8 +61,9 @@ import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extract
52
61
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
53
62
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
54
63
  import { pullLatestBinary } from './upgrade.js';
64
+ import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
55
65
  import { nodeTokenLogin } from './login.js';
56
- import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from '../lib/version.js';
66
+ import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion, readInstalledVersion } from '../lib/version.js';
57
67
  import { readInferenceState, readInferenceStateSync, isPidAlive } from '../lib/inference/state.js';
58
68
 
59
69
  function readInferenceStateSyncSafe() {
@@ -67,6 +77,7 @@ Usage:
67
77
 
68
78
  Flags:
69
79
  --foreground Run in the current terminal (don't detach)
80
+ --supervise Run the restart-on-crash supervisor in this terminal
70
81
  --heartbeat-interval <ms> Override heartbeat cadence (default 30000)
71
82
  --poll-interval <ms> Override job poll cadence (default 15000)
72
83
  --p2p-port <n> TCP port for peer connections (default 46337)
@@ -83,6 +94,7 @@ const DEFAULT_HEARTBEAT_MS = 30_000;
83
94
  const DEFAULT_POLL_MS = 15_000;
84
95
  const UPDATE_CHECK_MS = 5 * 60 * 1000; // 5 minutes
85
96
  const UPDATE_DRAIN_MS = 10 * 60 * 1000; // wait for in-flight jobs before restarting
97
+ const UPDATE_PAUSE_MS = 6 * 60 * 60 * 1000; // after an update that installed nothing newer
86
98
  const SHUTDOWN_DEADLINE_MS = 5_000; // hard exit: a half-shut daemon must never outlive its pidfile
87
99
  const OFFLINE_HEARTBEAT_TIMEOUT_MS = 3_000;
88
100
 
@@ -129,6 +141,9 @@ export default async function start(args, ctx) {
129
141
  return 0;
130
142
  }
131
143
 
144
+ if (args.has('supervise')) {
145
+ return runSupervisor(passthroughArgs(args));
146
+ }
132
147
  const foreground = args.has('foreground') || process.env.INFERNET_DAEMON_FOREGROUND === '1';
133
148
  if (!foreground) {
134
149
  return spawnAndReturn(args);
@@ -136,14 +151,7 @@ export default async function start(args, ctx) {
136
151
  return runDaemon(args, ctx);
137
152
  }
138
153
 
139
- async function spawnAndReturn(args) {
140
- const alreadyAlive = await isDaemonAlive();
141
- if (alreadyAlive) {
142
- process.stderr.write('A daemon is already running (socket responsive). Run `infernet stop` first.\n');
143
- return 1;
144
- }
145
- removeSocketFile();
146
-
154
+ function passthroughArgs(args) {
147
155
  const passthrough = [];
148
156
  const hb = args.get('heartbeat-interval');
149
157
  if (hb) passthrough.push('--heartbeat-interval', hb);
@@ -154,9 +162,32 @@ async function spawnAndReturn(args) {
154
162
  if (args.has('no-p2p')) passthrough.push('--no-p2p');
155
163
  if (args.has('no-advertise')) passthrough.push('--no-advertise');
156
164
  if (args.has('once')) passthrough.push('--once');
165
+ return passthrough;
166
+ }
167
+
168
+ async function spawnAndReturn(args) {
169
+ const alreadyAlive = await isDaemonAlive();
170
+ if (alreadyAlive) {
171
+ process.stderr.write('A daemon is already running (socket responsive). Run `infernet stop` first.\n');
172
+ return 1;
173
+ }
174
+ const supervisorPid = readLiveSupervisorPid();
175
+ if (supervisorPid) {
176
+ process.stderr.write(`A supervisor is already running (pid ${supervisorPid}) and will bring the daemon back. Run \`infernet stop\` first.\n`);
177
+ return 1;
178
+ }
179
+ // The systemd unit owns the daemon when it is enabled: starting a second
180
+ // one here would fight it over the socket, pidfile and port.
181
+ if (userServiceEnabled()) {
182
+ const { spawnSync } = await import('node:child_process');
183
+ const r = spawnSync('systemctl', ['--user', 'start', 'infernet.service'], { stdio: 'inherit' });
184
+ if (r.status === 0) process.stdout.write('infernet daemon started via systemd (systemctl --user status infernet)\n');
185
+ return r.status ?? 1;
186
+ }
187
+ removeSocketFile();
157
188
 
158
- const { pid, logPath } = spawnDetachedDaemon(passthrough);
159
- process.stdout.write(`infernet daemon started (pid ${pid})\n`);
189
+ const { pid, logPath } = spawnDetachedSupervisor(passthroughArgs(args));
190
+ process.stdout.write(`infernet daemon started under its supervisor (pid ${pid}); it restarts itself if it dies\n`);
160
191
  process.stdout.write(` logs: ${logPath}\n`);
161
192
  process.stdout.write(` socket: ${getDaemonSocketPath()}\n`);
162
193
  process.stdout.write('Tail logs with `infernet logs -f`, query live state with `infernet status`.\n');
@@ -164,6 +195,15 @@ async function spawnAndReturn(args) {
164
195
  }
165
196
 
166
197
  async function runDaemon(args, ctx) {
198
+ // A stray rejection must not take the node down; a thrown exception leaves
199
+ // state unknown, so exit non-zero and let the supervisor start a clean one.
200
+ process.on('unhandledRejection', (reason) => {
201
+ process.stderr.write(`[${new Date().toISOString()}] unhandled rejection (continuing): ${reason?.stack ?? reason}\n`);
202
+ });
203
+ process.on('uncaughtException', (err) => {
204
+ process.stderr.write(`[${new Date().toISOString()}] uncaught exception, exiting for restart: ${err?.stack ?? err}\n`);
205
+ process.exit(1);
206
+ });
167
207
  const { config, client, configPath } = ctx;
168
208
  const node = config.node ?? {};
169
209
  if (!node.role || !node.nodeId) {
@@ -1551,6 +1591,18 @@ async function runDaemon(args, ctx) {
1551
1591
  }, commandPollMs);
1552
1592
  if (typeof commandTimer.unref === 'function') commandTimer.unref();
1553
1593
 
1594
+ // Aggregators verify providers independently (lib/aggregator.js).
1595
+ if (node.role === 'aggregator' && config.controlPlane?.url) {
1596
+ const probeRound = async () => {
1597
+ const r = await runProbeRound({ client, controlPlaneUrl: config.controlPlane.url });
1598
+ stats.lastProbeRound = { at: new Date().toISOString(), ...r };
1599
+ process.stdout.write(`[${stats.lastProbeRound.at}] probe round: ${r.reachable}/${r.probed} reachable${r.error ? ` (${r.error})` : ''}\n`);
1600
+ };
1601
+ setTimeout(() => { probeRound(); }, 5_000).unref?.();
1602
+ const probeTimer = setInterval(() => { probeRound(); }, PROBE_INTERVAL_MS);
1603
+ if (typeof probeTimer.unref === 'function') probeTimer.unref();
1604
+ }
1605
+
1554
1606
  // Open-market training (IPIP-0030). Daemons that opted in via
1555
1607
  // `infernet config set engine.acceptTraining true` (or env var
1556
1608
  // INFERNET_ACCEPT_TRAINING=1) poll for available shards and run them
@@ -1573,8 +1625,9 @@ async function runDaemon(args, ctx) {
1573
1625
  // available, pull the installer, re-generate any missing keys + re-register
1574
1626
  // specs (idempotent), then re-exec so the new code is running immediately.
1575
1627
  let isUpgrading = false;
1628
+ let updatePausedUntil = 0;
1576
1629
  const updateTimer = setInterval(async () => {
1577
- if (shuttingDown || isUpgrading) return;
1630
+ if (shuttingDown || isUpgrading || Date.now() < updatePausedUntil) return;
1578
1631
  // Never start an upgrade under a running job; the next check retries.
1579
1632
  if (stats.activeJobIds.size > 0) return;
1580
1633
  const latest = await fetchLatestVersion();
@@ -1588,6 +1641,17 @@ async function runDaemon(args, ctx) {
1588
1641
  isUpgrading = false;
1589
1642
  return;
1590
1643
  }
1644
+ // The installer can succeed without changing this install (a git checkout
1645
+ // whose package.json lagged the release, or npm writing to another
1646
+ // prefix). Restarting then would rerun the same update every 5 minutes
1647
+ // forever, so keep serving and try again much later.
1648
+ const installed = readInstalledVersion();
1649
+ if (!isNewerVersion(CURRENT_VERSION, installed)) {
1650
+ updatePausedUntil = Date.now() + UPDATE_PAUSE_MS;
1651
+ process.stderr.write(`[update] installer left v${installed} here (latest v${latest}); not restarting, next try in ${UPDATE_PAUSE_MS / 3_600_000}h\n`);
1652
+ isUpgrading = false;
1653
+ return;
1654
+ }
1591
1655
 
1592
1656
  // Post-upgrade: run `infernet upgrade --skip-setup` is already done
1593
1657
  // (pullLatestBinary above). Now run setup --yes so keys are regenerated,
@@ -1634,6 +1698,14 @@ async function runDaemon(args, ctx) {
1634
1698
  // Small grace period so OS reclaims the ports before the new process binds.
1635
1699
  await new Promise((r) => setTimeout(r, 500));
1636
1700
 
1701
+ // Under a supervisor, hand the restart to it. A detached re-exec followed by
1702
+ // exit 0 is what killed nodes under systemd: exit 0 ends the unit, and the
1703
+ // control-group kill takes the detached child with it.
1704
+ if (isSupervised()) {
1705
+ process.stdout.write(`[update] exiting ${EXIT_RESTART} so the supervisor starts v${latest}\n`);
1706
+ process.exit(EXIT_RESTART);
1707
+ }
1708
+
1637
1709
  // Re-exec the daemon with the same args so the new code is live.
1638
1710
  process.stdout.write('[update] restarting daemon with new version\n');
1639
1711
  const logPath = getDaemonLogPath();
package/commands/stop.js CHANGED
@@ -7,12 +7,17 @@
7
7
  *
8
8
  * Fallback: read ~/.config/infernet/daemon.pid and send SIGTERM (or a
9
9
  * custom --signal) directly to the process.
10
+ *
11
+ * When `infernet start` left a supervisor running (lib/supervisor.js), it is
12
+ * signalled first: it forwards SIGTERM to the daemon and exits, instead of
13
+ * restarting the daemon we just stopped.
10
14
  */
11
15
 
12
16
  import fs from 'node:fs/promises';
13
17
 
14
18
  import { getDaemonPidPath } from '../lib/config.js';
15
19
  import { isDaemonAlive, sendToDaemon } from '../lib/ipc.js';
20
+ import { readLiveSupervisorPid } from '../lib/supervisor.js';
16
21
 
17
22
  const HELP = `infernet stop — signal the running daemon to exit
18
23
 
@@ -33,6 +38,18 @@ export default async function stop(args) {
33
38
 
34
39
  const explicitSignal = args.get('signal') ?? (args.has('force') ? 'SIGKILL' : null);
35
40
 
41
+ const supervisorPid = readLiveSupervisorPid();
42
+ if (supervisorPid) {
43
+ try {
44
+ process.kill(supervisorPid, explicitSignal ?? 'SIGTERM');
45
+ process.stdout.write(`Stopping supervisor (pid ${supervisorPid}) and its daemon.\n`);
46
+ } catch (err) {
47
+ process.stderr.write(`Failed to signal supervisor ${supervisorPid}: ${err.message}\n`);
48
+ }
49
+ // SIGTERM is forwarded to the daemon; any other signal leaves it to us.
50
+ if (!explicitSignal) return 0;
51
+ }
52
+
36
53
  // Prefer graceful IPC-driven shutdown when available.
37
54
  if (!explicitSignal && (await isDaemonAlive())) {
38
55
  const res = await sendToDaemon('shutdown', null, { timeoutMs: 5000 });
@@ -45,7 +45,10 @@ function runShell(cmd) {
45
45
 
46
46
  export async function pullLatestBinary() {
47
47
  process.stdout.write(`\n→ Pulling latest CLI from ${INSTALLER_URL}\n\n`);
48
- const code = await runShell(`curl -fsSL ${INSTALLER_URL} | sh`);
48
+ // INFERNET_AUTOSTART=0: the installer would otherwise run `infernet setup`,
49
+ // which restarts (or, next to infernet.service, duplicates) the daemon from
50
+ // inside its own self-update. Both callers run setup themselves afterwards.
51
+ const code = await runShell(`curl -fsSL ${INSTALLER_URL} | INFERNET_AUTOSTART=0 sh`);
49
52
  if (code !== 0) {
50
53
  process.stderr.write(`\nerror: installer exited ${code}.\n`);
51
54
  return false;
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Aggregator duty, phase 1: independent provider verification.
3
+ *
4
+ * Every round an aggregator reads the public peer list (/api/peers), dials each
5
+ * provider's advertised P2P address, sends `{"cmd":"info"}` and times the reply.
6
+ * The signed report (POST /api/v1/node/aggregator/probes) lets /status say
7
+ * "reachable, verified by N aggregators" instead of trusting a node's own
8
+ * heartbeat. Later phases add coordination of multi-node jobs (IPIP-0028/0033).
9
+ */
10
+
11
+ import net from 'node:net';
12
+
13
+ export const PROBE_INTERVAL_MS = 60_000;
14
+ const PROBE_TIMEOUT_MS = 5_000;
15
+ const PROBE_CONCURRENCY = 8;
16
+ const MAX_PEERS = 500;
17
+
18
+ /** `/ip4/1.2.3.4/tcp/46337` or `/dns4/host/tcp/n` -> { host, port }, else null. */
19
+ export function parseMultiaddr(addr) {
20
+ const m = /^\/(ip4|ip6|dns4|dns6|dns)\/([^/]+)\/tcp\/(\d{1,5})$/.exec(String(addr ?? ''));
21
+ if (!m) return null;
22
+ const port = Number(m[3]);
23
+ if (port < 1 || port > 65535) return null;
24
+ return { host: m[2], port };
25
+ }
26
+
27
+ /**
28
+ * Dial one peer and ask for its identity.
29
+ * @returns {Promise<{ reachable: boolean, rtt_ms: number|null, node_id: string|null, error: string|null }>}
30
+ */
31
+ export function probePeer({ host, port }, { timeoutMs = PROBE_TIMEOUT_MS } = {}) {
32
+ return new Promise((resolve) => {
33
+ const t0 = Date.now();
34
+ let settled = false;
35
+ let buf = '';
36
+ const sock = net.connect({ host, port });
37
+ const done = (result) => {
38
+ if (settled) return;
39
+ settled = true;
40
+ clearTimeout(timer);
41
+ sock.destroy();
42
+ resolve(result);
43
+ };
44
+ const timer = setTimeout(
45
+ () => done({ reachable: false, rtt_ms: null, node_id: null, error: 'timeout' }),
46
+ timeoutMs
47
+ );
48
+ sock.setEncoding('utf8');
49
+ sock.on('connect', () => sock.write(JSON.stringify({ cmd: 'info' }) + '\n'));
50
+ sock.on('data', (chunk) => {
51
+ buf += chunk;
52
+ const nl = buf.indexOf('\n');
53
+ if (nl < 0) {
54
+ if (buf.length > 8192) done({ reachable: false, rtt_ms: null, node_id: null, error: 'bad-reply' });
55
+ return;
56
+ }
57
+ let msg = null;
58
+ try { msg = JSON.parse(buf.slice(0, nl)); } catch { /* handled below */ }
59
+ if (msg?.ok && msg.data) {
60
+ done({ reachable: true, rtt_ms: Date.now() - t0, node_id: String(msg.data.node_id ?? '') || null, error: null });
61
+ } else {
62
+ done({ reachable: false, rtt_ms: null, node_id: null, error: 'bad-reply' });
63
+ }
64
+ });
65
+ sock.on('error', (err) => done({ reachable: false, rtt_ms: null, node_id: null, error: err?.code ?? 'error' }));
66
+ sock.on('close', () => done({ reachable: false, rtt_ms: null, node_id: null, error: 'closed' }));
67
+ });
68
+ }
69
+
70
+ async function mapLimit(items, limit, fn) {
71
+ const out = new Array(items.length);
72
+ let next = 0;
73
+ async function worker() {
74
+ while (next < items.length) {
75
+ const i = next++;
76
+ out[i] = await fn(items[i]);
77
+ }
78
+ }
79
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
80
+ return out;
81
+ }
82
+
83
+ /**
84
+ * One probe round: fetch peers, probe them, report. Never throws.
85
+ * @returns {Promise<{ probed: number, reachable: number, error?: string }>}
86
+ */
87
+ export async function runProbeRound({ client, controlPlaneUrl, fetchImpl = fetch, probe = probePeer }) {
88
+ try {
89
+ const res = await fetchImpl(`${controlPlaneUrl.replace(/\/+$/, '')}/api/peers`, {
90
+ headers: { accept: 'application/json' },
91
+ signal: AbortSignal.timeout(15_000)
92
+ });
93
+ if (!res.ok) return { probed: 0, reachable: 0, error: `peers HTTP ${res.status}` };
94
+ const peers = ((await res.json())?.data ?? [])
95
+ .filter((p) => p?.pubkey && parseMultiaddr(p.multiaddr))
96
+ .slice(0, MAX_PEERS);
97
+ const results = await mapLimit(peers, PROBE_CONCURRENCY, async (p) => ({
98
+ pubkey: p.pubkey,
99
+ ...(await probe(parseMultiaddr(p.multiaddr)))
100
+ }));
101
+ if (results.length > 0) await client.signedFetch('/api/v1/node/aggregator/probes', { probes: results });
102
+ return { probed: results.length, reachable: results.filter((r) => r.reachable).length };
103
+ } catch (err) {
104
+ return { probed: 0, reachable: 0, error: err?.message ?? String(err) };
105
+ }
106
+ }
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Built-in supervisor for the node daemon.
3
+ *
4
+ * `infernet start` used to detach ONE `start --foreground` process with nothing
5
+ * watching it: a crash, an OOM kill or a failed self-update re-exec left the node
6
+ * dead until the operator noticed (most outside nodes, 2026-05..10). Now the
7
+ * detached process is this supervisor, which runs the daemon as a child and
8
+ * restarts it.
9
+ *
10
+ * Exit-code contract (shared with systemd `Restart=on-failure` and launchd
11
+ * `KeepAlive.SuccessfulExit=false`):
12
+ * 0 deliberate stop (`infernet stop`, SIGTERM): stay down
13
+ * EXIT_RESTART (75) restart now (self-update installed a new version)
14
+ * anything else crash: restart with backoff
15
+ */
16
+
17
+ import { spawn, spawnSync } from 'node:child_process';
18
+ import fs from 'node:fs';
19
+ import path from 'node:path';
20
+
21
+ import { getConfigDir, getDaemonLogPath } from './config.js';
22
+
23
+ export const EXIT_RESTART = 75;
24
+
25
+ const BACKOFF_START_MS = 1_000;
26
+ const BACKOFF_MAX_MS = 60_000;
27
+ /** A child that stayed up this long resets the backoff. */
28
+ const HEALTHY_AFTER_MS = 10 * 60_000;
29
+
30
+ export function getSupervisorPidPath() {
31
+ return path.join(getConfigDir(), 'supervisor.pid');
32
+ }
33
+
34
+ /**
35
+ * True when something will restart this daemon after it exits non-zero: our
36
+ * supervisor, our systemd/launchd units, or any systemd unit (INVOCATION_ID).
37
+ */
38
+ export function isSupervised(env = process.env) {
39
+ return env.INFERNET_SUPERVISED === '1'
40
+ || Boolean(env.INVOCATION_ID)
41
+ || (env.XPC_SERVICE_NAME ?? '').startsWith('com.infernetprotocol.');
42
+ }
43
+
44
+ /** Next backoff delay: doubles from 1 s, capped at 60 s; reset after a healthy run. */
45
+ export function nextBackoff(prevMs, ranForMs) {
46
+ if (ranForMs >= HEALTHY_AFTER_MS || !prevMs) return BACKOFF_START_MS;
47
+ return Math.min(prevMs * 2, BACKOFF_MAX_MS);
48
+ }
49
+
50
+ /**
51
+ * Whether the supervisor should start the daemon again after it exited.
52
+ * `signal` is set when the child was killed (OOM killer, kill -9): restart.
53
+ */
54
+ export function shouldRestart(code, signal) {
55
+ if (signal) return true;
56
+ return code !== 0;
57
+ }
58
+
59
+ /** Pid of a live supervisor from supervisor.pid, or null. */
60
+ export function readLiveSupervisorPid() {
61
+ let raw;
62
+ try { raw = fs.readFileSync(getSupervisorPidPath(), 'utf8'); } catch { return null; }
63
+ const pid = Number.parseInt(raw.trim(), 10);
64
+ if (!Number.isFinite(pid) || pid <= 0) return null;
65
+ try { process.kill(pid, 0); return pid; } catch { return null; }
66
+ }
67
+
68
+ /** The user-level systemd unit (`infernet service install`) is installed and enabled. */
69
+ export function userServiceEnabled() {
70
+ if (process.platform !== 'linux') return false;
71
+ const r = spawnSync('systemctl', ['--user', 'is-enabled', '--quiet', 'infernet.service'], { stdio: 'ignore' });
72
+ return r.status === 0;
73
+ }
74
+
75
+ /**
76
+ * Detach a supervisor (`start --supervise`) that runs `start --foreground`.
77
+ * @param {string[]} passthroughArgs
78
+ * @returns {{ pid: number, logPath: string }}
79
+ */
80
+ export function spawnDetachedSupervisor(passthroughArgs = []) {
81
+ fs.mkdirSync(getConfigDir(), { recursive: true, mode: 0o700 });
82
+ const logPath = getDaemonLogPath();
83
+ const out = fs.openSync(logPath, 'a');
84
+ const scriptPath = process.argv[1];
85
+ if (!scriptPath || !path.isAbsolute(scriptPath)) {
86
+ throw new Error('Unable to determine absolute path to the CLI script for daemonization.');
87
+ }
88
+ const child = spawn(
89
+ process.execPath,
90
+ [scriptPath, 'start', '--supervise', ...passthroughArgs],
91
+ { detached: true, stdio: ['ignore', out, out], env: process.env }
92
+ );
93
+ child.unref();
94
+ return { pid: child.pid, logPath };
95
+ }
96
+
97
+ /**
98
+ * Supervisor loop. Runs until the daemon exits 0 or the supervisor is signalled.
99
+ * @param {string[]} passthroughArgs
100
+ * @returns {Promise<number>}
101
+ */
102
+ export async function runSupervisor(passthroughArgs = []) {
103
+ const pidPath = getSupervisorPidPath();
104
+ fs.mkdirSync(getConfigDir(), { recursive: true, mode: 0o700 });
105
+ fs.writeFileSync(pidPath, String(process.pid), { mode: 0o600 });
106
+ const log = (msg) => process.stdout.write(`[supervisor ${new Date().toISOString()}] ${msg}\n`);
107
+
108
+ let child = null;
109
+ let stopping = false;
110
+ let wake = null;
111
+ const stop = (signal) => {
112
+ if (stopping) return;
113
+ stopping = true;
114
+ log(`${signal}: stopping daemon`);
115
+ if (child) { try { child.kill('SIGTERM'); } catch { /* gone */ } }
116
+ if (wake) wake();
117
+ };
118
+ const onTerm = () => stop('SIGTERM');
119
+ const onInt = () => stop('SIGINT');
120
+ const onHup = () => { /* survive the terminal closing */ };
121
+ process.on('SIGTERM', onTerm);
122
+ process.on('SIGINT', onInt);
123
+ process.on('SIGHUP', onHup);
124
+
125
+ let backoff = 0;
126
+ try {
127
+ while (!stopping) {
128
+ const startedAt = Date.now();
129
+ // Re-read argv[1] each time: an upgrade replaces the files at the same path.
130
+ child = spawn(process.execPath, [process.argv[1], 'start', '--foreground', ...passthroughArgs], {
131
+ stdio: ['ignore', 'inherit', 'inherit'],
132
+ env: { ...process.env, INFERNET_DAEMON_FOREGROUND: '1', INFERNET_SUPERVISED: '1' }
133
+ });
134
+ log(`daemon started (pid ${child.pid})`);
135
+ const { code, signal } = await new Promise((resolve) => {
136
+ child.once('exit', (c, s) => resolve({ code: c, signal: s }));
137
+ child.once('error', (err) => {
138
+ log(`spawn failed: ${err?.message ?? err}`);
139
+ resolve({ code: 1, signal: null });
140
+ });
141
+ });
142
+ child = null;
143
+ if (stopping) break;
144
+ if (!shouldRestart(code, signal)) {
145
+ log('daemon exited 0 (stopped on purpose); supervisor exiting');
146
+ break;
147
+ }
148
+ if (code === EXIT_RESTART) {
149
+ backoff = 0;
150
+ log('daemon asked for a restart (update installed)');
151
+ continue;
152
+ }
153
+ backoff = nextBackoff(backoff, Date.now() - startedAt);
154
+ log(`daemon died (${signal ?? `exit ${code}`}); restarting in ${backoff / 1000}s`);
155
+ await new Promise((resolve) => {
156
+ wake = resolve;
157
+ setTimeout(resolve, backoff);
158
+ });
159
+ wake = null;
160
+ }
161
+ } finally {
162
+ process.off('SIGTERM', onTerm);
163
+ process.off('SIGINT', onInt);
164
+ process.off('SIGHUP', onHup);
165
+ try {
166
+ if (fs.readFileSync(pidPath, 'utf8').trim() === String(process.pid)) fs.unlinkSync(pidPath);
167
+ } catch { /* already gone */ }
168
+ }
169
+ return 0;
170
+ }
package/lib/version.js CHANGED
@@ -3,9 +3,21 @@ import { join, dirname } from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
4
 
5
5
  const _dir = dirname(fileURLToPath(import.meta.url));
6
- export const CURRENT_VERSION = JSON.parse(
7
- readFileSync(join(_dir, '../package.json'), 'utf8')
8
- ).version;
6
+ export const CURRENT_VERSION = readInstalledVersion();
7
+
8
+ /**
9
+ * The version on disk right now, read fresh (CURRENT_VERSION is what this
10
+ * process started with). Self-update compares the two after the installer
11
+ * runs: if nothing newer landed, restarting would only repeat the same update
12
+ * every 5 minutes.
13
+ */
14
+ export function readInstalledVersion() {
15
+ try {
16
+ return JSON.parse(readFileSync(join(_dir, '../package.json'), 'utf8')).version;
17
+ } catch {
18
+ return '0.0.0';
19
+ }
20
+ }
9
21
 
10
22
  export async function fetchLatestVersion() {
11
23
  try {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.61",
3
+ "version": "0.1.63",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,12 +30,12 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/config": "0.1.61",
34
- "@infernetprotocol/deploy-providers": "0.1.61",
35
- "@infernetprotocol/engine": "0.1.61",
36
- "@infernetprotocol/gpu": "0.1.61",
37
- "@infernetprotocol/auth": "0.1.61",
38
- "@infernetprotocol/rpc-adapter": "0.1.61"
33
+ "@infernetprotocol/config": "0.1.63",
34
+ "@infernetprotocol/rpc-adapter": "0.1.63",
35
+ "@infernetprotocol/engine": "0.1.63",
36
+ "@infernetprotocol/gpu": "0.1.63",
37
+ "@infernetprotocol/auth": "0.1.63",
38
+ "@infernetprotocol/deploy-providers": "0.1.63"
39
39
  },
40
40
  "repository": {
41
41
  "type": "git",