@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/daemon/daemon.js

@@ -43,36 +43,211 @@ import { FileWatcher } from '../orchestrator/file-watcher.js';
 import { InfrastructureService } from '../infrastructure/infrastructure.service.js';
 import { DockerHealthIndicator } from '../monitoring/docker-health.indicator.js';
 import { AppHealthIndicator } from '../monitoring/app-health.indicator.js';
+import { EventLoopWatch } from '../monitoring/event-loop-watch.js';
+import { EventLoopStallIndicator } from '../monitoring/event-loop-stall.indicator.js';
 import { registerDaemonJobs } from './daemon-scheduler.js';
 import { METRICS_SERVICE_TOKEN as TITAN_METRICS_TOKEN } from '@omnitron-dev/titan-metrics';
-import { ORCHESTRATOR_TOKEN, LOG_MANAGER_TOKEN, STATE_STORE_TOKEN, OMNITRON_DB_TOKEN, AUTH_SERVICE_TOKEN, LOG_COLLECTOR_TOKEN, FLEET_SERVICE_TOKEN, ALERT_SERVICE_TOKEN, HEALTH_CHECK_SERVICE_TOKEN, DISCOVERY_SERVICE_TOKEN, KUBERNETES_SERVICE_TOKEN, PIPELINE_SERVICE_TOKEN, TRACE_COLLECTOR_TOKEN, BACKUP_SERVICE_TOKEN, SECRETS_SERVICE_TOKEN, TELEMETRY_RELAY_TOKEN, DEPLOY_SERVICE_TOKEN, INFRASTRUCTURE_GATE_TOKEN, PROJECT_SERVICE_TOKEN, SLAVE_STORAGE_TOKEN, } from '../shared/tokens.js';
+import { ORCHESTRATOR_TOKEN, LOG_MANAGER_TOKEN, STATE_STORE_TOKEN, OMNITRON_DB_TOKEN, AUTH_SERVICE_TOKEN, LOG_COLLECTOR_TOKEN, FLEET_SERVICE_TOKEN, ALERT_SERVICE_TOKEN, HEALTH_CHECK_SERVICE_TOKEN, DISCOVERY_SERVICE_TOKEN, KUBERNETES_SERVICE_TOKEN, PIPELINE_SERVICE_TOKEN, TRACE_COLLECTOR_TOKEN, BACKUP_SERVICE_TOKEN, SECRETS_SERVICE_TOKEN, AUDIT_SERVICE_TOKEN, DAEMON_STATE_STORE_TOKEN, TELEMETRY_RELAY_TOKEN, DEPLOY_SERVICE_TOKEN, INFRASTRUCTURE_GATE_TOKEN, PROJECT_SERVICE_TOKEN, SLAVE_STORAGE_TOKEN, } from '../shared/tokens.js';
 import { APP_EVENTS, INFRA_EVENTS, DAEMON_EVENTS, PROJECT_EVENTS, STACK_EVENTS, NODE_EVENTS, } from '../shared/events.js';
 import { JWT_SERVICE_TOKEN } from '@omnitron-dev/titan-auth';
-import { AuthenticationManager } from '@omnitron-dev/titan/netron/auth';
+import { AuthenticationManager, AuthorizationManager, } from '@omnitron-dev/titan/netron/auth';
 import { createAuthContextWrapper } from '../services/auth-context.js';
+import { ROLES } from '../shared/roles.js';
+import { expandPath } from '../shared/paths.js';
+/**
+ * Deadline for a request on the daemon's unix socket.
+ *
+ * Same value as `CLI_REQUEST_TIMEOUT` in `daemon-client.ts`: one number for
+ * one socket, stated by both ends rather than inherited from a default meant
+ * for wire requests.
+ */
+const DAEMON_SOCKET_REQUEST_TIMEOUT = 60_000;
+/**
+ * Where the daemon listens. The default is loopback; anything the operator
+ * writes is passed through literally, `0.0.0.0` included.
+ *
+ * Exported so the two call sites below and the tests agree on one
+ * definition. They previously each carried their own copy of
+ * `host !== '0.0.0.0' ? host : '127.0.0.1'`, which is how the two of them
+ * plus the config default managed to disagree about what the default was.
+ */
+export function resolveBindHost(configured) {
+    return configured ?? '127.0.0.1';
+}
+/**
+ * Where a daemon binds its fleet TCP transport.
+ *
+ * A master follows `daemon.host`, whose default is loopback — the safe answer
+ * for the machine an operator is sitting at.
+ *
+ * A slave cannot use that default, because being reachable is the whole of
+ * what a slave is for: the master dials it to check its health and to pull
+ * from it. Bound to loopback it is a daemon nobody can see, and the console
+ * shows it as offline for ever with no way to tell it from a machine that is
+ * genuinely down.
+ *
+ * It was exactly that, and by a longer route than a wrong default. The
+ * generated slave config carried `daemon: { host: '0.0.0.0', … }` — and
+ * `IEcosystemConfig`, the schema of the file it was written into, has no
+ * `daemon` key; the daemon boots from `~/.omnitron/config.json`, which until
+ * now could not carry one either. So the setting was written, ignored, and
+ * every provisioned slave would have bound 127.0.0.1.
+ *
+ * An explicit `host` still wins, for an operator who wants one interface.
+ */
+export function fleetBindHostFor(dc) {
+    if (dc.role === 'slave')
+        return dc.host ?? '0.0.0.0';
+    return resolveBindHost(dc.host);
+}
+/**
+ * The bind address a daemon should END UP with, from what was actually
+ * configured.
+ *
+ * `fleetBindHostFor` above takes the daemon's RUNNING config, and by the time
+ * anything holds one, `host` has been filled in from `DEFAULT_DAEMON_CONFIG`
+ * — so `?? '0.0.0.0'` can never fire and every slave binds loopback. The
+ * information it needs is whether an operator SET a host, and only the saved
+ * file knows that: absent means "no opinion", `127.0.0.1` means "loopback,
+ * deliberately".
+ *
+ * Measured 2026-09-14 on a provisioned slave, after the rule was written and
+ * before this existed:
+ *
+ *     LISTEN 127.0.0.1:9700    ← the fleet port no master can dial
+ *     LISTEN 127.0.0.1:9801
+ *     LISTEN 127.0.0.1:9802
+ *
+ * The rule was right, the value it was given was already decided. This is the
+ * same rule applied where the answer is still open.
+ */
+export function effectiveBindHost(saved) {
+    if (saved?.host)
+        return saved.host;
+    if (saved?.role === 'slave')
+        return '0.0.0.0';
+    return DEFAULT_DAEMON_CONFIG.host;
+}
+/**
+ * Where a daemon binds the surfaces that exist to serve a console: the Netron
+ * HTTP RPC and the WebSocket.
+ *
+ * A master follows `daemon.host`, like everything else. A slave does not,
+ * because on a slave that knob is published for a different reason: a fleet
+ * node has to bind its TCP transport somewhere the master can reach, and a
+ * generated slave config therefore says `0.0.0.0`. The same value then
+ * published two more surfaces — an authentication endpoint and a WebSocket
+ * one that has no rate limit available to it — and a slave serves no console,
+ * so nothing dials either. They were cost without benefit, on a machine whose
+ * whole point is being somewhere else.
+ */
+export function consoleBindHostFor(dc) {
+    return dc.role === 'slave' ? '127.0.0.1' : resolveBindHost(dc.host);
+}
+/** Addresses that reach this host and nowhere else. */
+export function isLoopbackHost(host) {
+    if (!host)
+        return true; // absent means the loopback default above
+    const h = host.trim().toLowerCase();
+    return h === '127.0.0.1' || h === 'localhost' || h === '::1' || h === '[::1]' || h.startsWith('127.');
+}
+/**
+ * Is the daemon still accepting the password its own migration seeded?
+ *
+ * Resolves the auth service lazily and answers `false` on any failure — a
+ * daemon that cannot reach its database has a louder problem than this, and
+ * a warning that fails the boot would be a worse trade than the risk it
+ * describes.
+ */
+async function isSeededAdminPasswordInUse(app, logger) {
+    try {
+        const auth = await app.container.resolveAsync(AUTH_SERVICE_TOKEN);
+        return await auth.isUsingSeededAdminPassword();
+    }
+    catch (err) {
+        logger.warn({ err: err.message }, 'Could not check whether the seeded admin password is still in use');
+        return false;
+    }
+}
+/**
+ * The address other fleet nodes should dial to reach this one.
+ *
+ * `peer.address` is not decoration: `leader-election.ts` calls `requestVote`
+ * and `leaderHeartbeat` against it. A node bound to `0.0.0.0` therefore may
+ * not advertise `0.0.0.0` (not routable) — and must not advertise
+ * `127.0.0.1` either, which is what it did: every other node in the fleet
+ * resolves that to itself, so votes and heartbeats meant for the leader go
+ * to the sender's own loopback. A wrong answer of the right shape.
+ *
+ * When the bind address says nothing about reachability, take the first
+ * non-internal IPv4 the host has. Loopback remains the honest answer only
+ * when there is genuinely nothing else — a single-host deployment, where
+ * no other node can reach this one regardless.
+ */
+export function advertisedAddress(bindHost) {
+    if (bindHost && bindHost !== '0.0.0.0' && bindHost !== '::')
+        return bindHost;
+    for (const addrs of Object.values(os.networkInterfaces())) {
+        for (const a of addrs ?? []) {
+            if (a.family === 'IPv4' && !a.internal)
+                return a.address;
+        }
+    }
+    return '127.0.0.1';
+}
 export class OmnitronDaemon {
     app = null;
     pidManager = null;
+    /** Port this daemon serves `/netron/invoke` on; advertised to fleet peers. */
+    rpcHttpPort = null;
     fileWatcher = null;
     infraService = null;
+    eventLoopWatch = null;
     eventBroadcaster = null;
     syncService = null;
     nodeManagerService = null;
+    slaveConnector = null;
+    meshHandle = null;
     nodeManagerRpcService = null;
+    /**
+     * The live record of release builds this daemon is running.
+     *
+     * Held on the daemon rather than in the container because it is the state
+     * of work in progress: a build that a restart kills leaves a release
+     * directory with no manifest, and that — not this — is the durable trace.
+     */
+    releaseService = null;
     systemWorkerManager = null;
+    /** Everything the health-monitor worker needs to be spawned again. */
+    healthWorkerSpawn = null;
+    healthWorkerRestartTimer = null;
+    healthWorkerRestartAttempts = 0;
+    /** The startup budget the worker is given, so a failure can name it. */
+    healthWorkerStartupTimeout = 0;
+    /** In-daemon check timer, armed only while the worker is down. */
+    fallbackNodeCheckTimer = null;
+    /** Set by the shutdown task so nothing schedules work on the way out. */
+    isShuttingDown = false;
     leaderElection = null;
     configSyncService = null;
+    metricsServer = null;
+    metricsBridge = null;
     dc = DEFAULT_DAEMON_CONFIG;
     /** Daemon config accessor for external use (e.g., DaemonRpcService) */
     get daemonConfig() { return this.dc; }
     async start(config, options, daemonConfig) {
         this.dc = daemonConfig ?? DEFAULT_DAEMON_CONFIG;
-        const pidFile = this.dc.pidFile.replace('~', process.env['HOME'] ?? '');
+        const pidFile = expandPath(this.dc.pidFile);
         this.pidManager = new PidManager(pidFile);
         if (this.pidManager.isRunning()) {
             const existingPid = this.pidManager.getPid();
             throw new Error(`Omnitron daemon already running (PID: ${existingPid})`);
         }
+        // Stale file from a previous crash is normal — clean it up so the
+        // atomic `O_EXCL` write below succeeds. P1-L: doing this AFTER
+        // isRunning() returned false means we only delete known-dead
+        // markers; live daemons get the conflict error above.
+        this.pidManager.remove();
         this.pidManager.write();
         // 1. Create Titan Application (the daemon itself)
         const DaemonModule = createDaemonModule(config, this.dc);
@@ -80,6 +255,25 @@ export class OmnitronDaemon {
             name: 'omnitron',
             version: CLI_VERSION,
         });
+        // P1-M — install our own SIGHUP handler BEFORE Application's
+        // default shutdown handlers attach, so `kill -HUP <daemon>`
+        // triggers a config reload (the Unix convention for long-running
+        // daemons) instead of terminating the supervisor. We don't bind
+        // this through the Application's signal manager because that
+        // path is wired to `runShutdownTasks()`; we install at the
+        // process level and the listener runs first.
+        process.on('SIGHUP', () => {
+            this.reloadConfigOnHup().catch((err) => {
+                // The message already carries which steps applied — printing only
+                // `.message` was fine for the load failure and misleading for the
+                // apply failure, which is the one that leaves work half-done.
+                console.warn(`[omnitron] SIGHUP config reload failed: ${err.message}`);
+                const cause = err.cause;
+                if (cause) {
+                    console.warn(`[omnitron]   caused by: ${cause.stack ?? String(cause)}`);
+                }
+            });
+        });
         // 2. Register transports (Unix, TCP, HTTP, WebSocket)
         await this.registerTransports(this.dc);
         // 3. Resolve and expose RPC services via Netron
@@ -88,6 +282,12 @@ export class OmnitronDaemon {
         await this.app.start();
         const loggerModule = await this.app.container.resolveAsync(LOGGER_SERVICE_TOKEN);
         const logger = loggerModule.logger;
+        // As soon as there is somewhere to say it, so a stall in the boot's own
+        // steps — infrastructure, migrations — is seen too. A node whose loop
+        // stands still answers nothing, its heartbeat included, and until this
+        // nothing on the node could say so. See `monitoring/event-loop-watch.ts`.
+        this.eventLoopWatch = new EventLoopWatch(logger);
+        this.eventLoopWatch.start();
         logger.info({
             socketPath: this.dc.socketPath,
             port: this.dc.port,
@@ -111,9 +311,33 @@ export class OmnitronDaemon {
         {
             const orchestrator = await this.app.container.resolveAsync(ORCHESTRATOR_TOKEN);
             if (this.app.netron) {
-                const socketPath = this.dc.socketPath.replace('~', process.env['HOME'] ?? '');
+                const socketPath = expandPath(this.dc.socketPath);
                 orchestrator.setDaemonNetron(this.app.netron, socketPath);
             }
+            // 8.6. Wire metrics bridge + start /metrics HTTP endpoint.
+            // Bridge subscribes to orchestrator events (restart/crash/escalation),
+            // janitor sweeps, and esbuild reset/build hooks. The HTTP server
+            // refreshes per-app gauges on each scrape via the bridge before
+            // formatting Prometheus text.
+            try {
+                const { MetricsBridge, MetricsServer } = await import('../observability/index.js');
+                const metricsService = await this.app.container.resolveAsync(TITAN_METRICS_TOKEN);
+                const bridge = new MetricsBridge(metricsService, logger);
+                orchestrator.setMetricsBridge(bridge);
+                this.metricsBridge = bridge;
+                const metricsPort = (this.dc.httpPort ?? 9800) + 3;
+                const metricsServer = new MetricsServer({
+                    port: metricsPort,
+                    metrics: metricsService,
+                    bridge,
+                    logger,
+                });
+                await metricsServer.start();
+                this.metricsServer = metricsServer;
+            }
+            catch (err) {
+                logger.warn({ error: err.message }, 'metrics endpoint failed to start — /metrics will be unavailable');
+            }
         }
         // 8.8. Start webapp (nginx container serving Console UI) if enabled
         {
@@ -122,7 +346,24 @@ export class OmnitronDaemon {
             if (savedConfig?.webapp) {
                 try {
                     const { WebappService } = await import('../webapp/webapp.service.js');
-                    const webapp = new WebappService(logger, process.cwd(), (this.dc.httpPort ?? 9800) + 1, this.dc.httpPort ?? 9800);
+                    // Publishing the console while the seeded password is still in
+                    // place hands the whole control plane — start, stop, restart, logs,
+                    // config — to anyone who can reach the port and guess a word that
+                    // is written in the migration. Migration 001 says the password
+                    // "MUST be changed on first login"; nothing enforced it, and a
+                    // comment is not a control.
+                    //
+                    // So the console falls back to loopback rather than refusing to
+                    // start: the operator needs it to change the password, and a
+                    // control plane that will not come up is its own outage. The log
+                    // line says what to do and that this is not the configured bind.
+                    let consoleHost = this.dc.consoleBindHost ?? '127.0.0.1';
+                    if (!isLoopbackHost(consoleHost) && (await isSeededAdminPasswordInUse(this.app, logger))) {
+                        logger.error({ configured: consoleHost, bound: '127.0.0.1' }, 'REFUSING to publish the Console while the seeded admin password is unchanged — ' +
+                            'binding loopback instead. Sign in at http://127.0.0.1 and change it, then restart.');
+                        consoleHost = '127.0.0.1';
+                    }
+                    const webapp = new WebappService(logger, process.cwd(), (this.dc.httpPort ?? 9800) + 1, this.dc.httpPort ?? 9800, consoleHost);
                     await webapp.start();
                 }
                 catch (err) {
@@ -130,14 +371,34 @@ export class OmnitronDaemon {
                 }
             }
         }
+        // 8.9. Fleet monitoring. Started BEFORE managed apps and not awaited.
+        //
+        // It used to be step 11.5, after `startApps`, `startScheduledTasks` and
+        // `startBackgroundServices`. Fleet monitoring is about MACHINES; it has
+        // nothing to do with whether this daemon's managed applications start, and
+        // gating it behind them means one application that fails its startup
+        // leaves the whole fleet unwatched for as long as the supervisor keeps
+        // retrying it. Measured on this host: ten minutes after a daemon start,
+        // with two applications still cycling through failed starts, not a single
+        // node had been checked and nothing had said so.
+        //
+        // Not awaited, for the same reason in the other direction: spawning the
+        // worker has a 30-second startup timeout, and the applications should not
+        // wait behind it. The function handles and logs its own failures, and arms
+        // the in-daemon fallback checks when the worker cannot run.
+        void this.startHealthMonitorWorker(logger).catch((err) => {
+            // It handles its own failures; this is the last resort, because an
+            // unhandled rejection is a crash of the daemon rather than a lost
+            // subsystem.
+            logger.error({ error: err.message }, 'Health monitor startup threw');
+        });
         // 9. Start managed apps (waits for infra gate if apps have `requires`)
         await this.startApps(config, logger);
         // 10. Start background tasks via titan-scheduler (replaces bare setInterval)
         await this.startScheduledTasks(config, logger);
         // 11. Start remaining background services (telemetry, traces, leader election)
         await this.startBackgroundServices(config, logger);
-        // 11.5. Start health monitor system worker (master only)
-        await this.startHealthMonitorWorker(logger);
+        // (Fleet monitoring is started at 8.5, above — before managed apps.)
         // 12. Wire event broadcasting (orchestrator events → WebSocket push)
         await this.wireEventBroadcasting(logger);
         // 13. Register shutdown tasks — Application handles SIGTERM/SIGINT
@@ -160,18 +421,87 @@ export class OmnitronDaemon {
         this.app.netron.registerTransportServer('unix', {
             name: 'daemon-local',
             options: {
-                path: dc.socketPath.replace('~', process.env['HOME'] ?? ''),
+                path: expandPath(dc.socketPath),
                 force: true,
                 mode: 0o600, // Owner-only access for security
+                // Every other party on this socket states a deadline and this one
+                // did not, so `netron.ts:797` — `serverRequestTimeout ??
+                // transportOpts.requestTimeout` — found both undefined and built its
+                // accepted-connection peers with netron's 5 s default, which is a
+                // deadline for a wire request rather than for management-plane work.
+                // The reading of the server option was added deliberately («a
+                // `requestTimeout` set on the server was silently ignored»); the
+                // value was never supplied. Same number as `CLI_REQUEST_TIMEOUT`,
+                // which the daemon's own client uses over this same socket.
+                requestTimeout: DAEMON_SOCKET_REQUEST_TIMEOUT,
+                // The same bridge the HTTP and WebSocket servers below have: it puts
+                // the context the guard admitted a call with — here the implicit
+                // `omnitron-local` set just after this — where the services read
+                // who is calling. This server had none, so for every CLI call the
+                // guard saw `omnitron-local` and the services saw nobody: the audit
+                // trail recorded `system` for 171 of the master's 178 rows, and an
+                // operator's deployment read exactly like the daemon's own boot
+                // autostart. No proxy stands in front of a unix socket, so the
+                // wrapper trusts no forwarded address — its default.
+                invocationWrapper: createAuthContextWrapper(),
             },
         });
-        // TCP — remote fleet communication (cross-server)
+        // Local CLI trust. The Unix socket above is created mode 0600, so any
+        // peer that can open it is provably the same OS user that owns this
+        // daemon — the OS has already authenticated them. Grant those
+        // connections an implicit admin context so the local CLI satisfies the
+        // `@Public({ auth: { roles } })` guards on the daemon RPC services
+        // without a redundant JWT handshake (the CLI talks over this socket
+        // only). Remote transports — TCP (fleet), HTTP/WS (webapp) — are NOT
+        // registered here and keep requiring a validated token, so this widens
+        // nothing beyond same-user local access that the socket mode already
+        // grants.
+        this.app.netron.setTransportAuthContext('unix', {
+            userId: 'omnitron-local',
+            roles: [ROLES.ADMIN],
+            permissions: [],
+            metadata: { source: 'local-unix-socket' },
+        });
+        // TCP — remote fleet communication (cross-server). The default is
+        // 127.0.0.1 so a single-host deployment never exposes daemon control
+        // to the LAN; multi-host fleet operators opt in via `daemon.host`.
+        //
+        // The default belongs in DEFAULT_DAEMON_CONFIG, not here. It used to be
+        // enforced here instead, as `dc.host !== '0.0.0.0' ? dc.host :
+        // '127.0.0.1'` over a config default of `'0.0.0.0'` — absence encoded as
+        // a value from the domain of valid answers. `0.0.0.0` is not a marker for
+        // "unset", it is a bind address with a meaning, and it was the exact
+        // value the comment above it told fleet operators to set. Following the
+        // documentation produced loopback and no fleet, silently. It also
+        // contradicted every neighbouring knob: `consoleBindHost: '0.0.0.0'`
+        // publishes, and so does an app's transport host.
+        //
+        // (The original hazard — pre-fix this defaulted to all-interfaces while
+        // the AuthorizationManager below was missing, giving any LAN host an
+        // unauthenticated `stopAll({force:true})` — is answered by the default
+        // being loopback and by that manager now being wired.)
+        const tcpHost = fleetBindHostFor(dc);
+        if (!isLoopbackHost(tcpHost)) {
+            // Said, not enforced, and the difference is deliberate: silently
+            // rebinding a fleet transport to loopback would take a cluster down,
+            // and a node that cannot be reached is an outage rather than a
+            // hardening. The console above IS rebound, because its whole purpose
+            // is a password prompt and loopback still serves it.
+            void (async () => {
+                const loggerModule = await this.app.container.resolveAsync(LOGGER_SERVICE_TOKEN);
+                const log = loggerModule.logger;
+                if (await isSeededAdminPasswordInUse(this.app, log)) {
+                    log.error({ host: tcpHost, port: dc.port }, 'The daemon fleet transport is published and the seeded admin password is unchanged — ' +
+                        'anyone who can reach this port can sign in and stop every app. Change it now.');
+                }
+            })().catch(() => undefined);
+        }
         this.app.netron.registerTransport('tcp', () => new TcpTransport());
         this.app.netron.registerTransportServer('tcp', {
             name: 'daemon-fleet',
             options: {
                 port: dc.port,
-                host: dc.host,
+                host: tcpHost,
             },
         });
         // Auth — JWT validation + session check + role extraction
@@ -220,18 +550,79 @@ export class OmnitronDaemon {
             },
             tokenCache: { enabled: true, ttl: 10_000 },
         });
-        this.app.netron.configureAuth(authManager);
-        const authContextWrapper = createAuthContextWrapper();
+        // Wire the AuthorizationManager so the `@Public({ auth: { roles } })`
+        // decorators on DaemonRpcService are actually enforced on the wire.
+        // Without an authzManager, `remote-peer.ts:enforceMethodAccess`
+        // early-returns and every method is reachable by anyone who can
+        // mint a valid JWT, regardless of `roles`. The WS origin policy
+        // (`allowedOrigins: true` → same-host) closes the cross-origin
+        // upgrade attack against cookie-mode auth.
+        const authzManager = new AuthorizationManager(authLogger);
+        this.app.netron.configureAuth(authManager, authzManager, {
+            allowedOrigins: true,
+        });
+        // `trustProxy` is shared with the rate limiter deliberately: both answer
+        // the same question — is there a proxy in front whose forwarded headers
+        // we may believe? Two separate switches would eventually disagree.
+        const authContextWrapper = createAuthContextWrapper({
+            trustProxy: dc.httpRateLimit?.trustProxy === true,
+        });
+        // HTTP / WS bind also defaults to 127.0.0.1. The webapp talks to
+        // these over the same loopback; nginx fronts the public surface.
+        // Operators who proxy in from another host opt in via `daemon.host`
+        // (same knob that controls TCP, with the same literal meaning).
+        //
+        // Except on a slave, where that knob has to be published for a different
+        // reason and publishing these with it is pure cost. A fleet node must
+        // bind its TCP transport somewhere the master can reach — that is what
+        // `daemon.host: '0.0.0.0'` in a generated slave config is for — and the
+        // same value then published the Netron HTTP RPC and the WebSocket
+        // alongside it. A slave serves no console: the operator uses the
+        // master's. So the two surfaces it gained are an authentication endpoint
+        // it does not need and a WebSocket one that, as the note below records,
+        // has no rate limit available to it.
+        //
+        // Rebound rather than warned about, which is the opposite of the choice
+        // made for the TCP transport twenty lines up — and for the opposite
+        // reason. Rebinding the fleet port would take a cluster down, because
+        // being reachable is that port's whole job. Rebinding these costs a slave
+        // nothing, because nothing dials them.
+        const localHost = consoleBindHostFor(dc);
         // HTTP — Netron RPC API (internal port, nginx proxies from public port)
+        // The public `httpPort` (9800) is nginx in front of the console; the
+        // daemon's own Netron HTTP listener sits one above it. Remembered here
+        // because the fleet registration below must advertise the port peers can
+        // actually POST `/netron/invoke` to — not the TCP port, and not the
+        // console's.
         const internalHttpPort = (dc.httpPort ?? 9800) + 1;
+        this.rpcHttpPort = internalHttpPort;
         this.app.netron.registerTransport('http', () => new HttpTransport());
+        const rl = dc.httpRateLimit ?? {};
         this.app.netron.registerTransportServer('http', {
             name: 'daemon-http',
             options: {
                 port: internalHttpPort,
-                host: '0.0.0.0',
+                host: localHost,
                 cors: true,
                 invocationWrapper: authContextWrapper,
+                // Volume cap on the HTTP RPC surface, and only that one. Password
+                // guessing is bounded by the per-account lockout in AuthService.
+                //
+                // The WebSocket surface below is NOT covered: netron's WebSocket
+                // transport has no rate-limit option — `maxPayload` is the only
+                // bound it accepts — and the console talks over WebSocket. This
+                // comment used to say the limit bounded "everything else, including
+                // attempts to flood the daemon into unresponsiveness", which
+                // describes a protection that covers one of the two transports an
+                // attacker can reach.
+                rateLimit: {
+                    enabled: rl.enabled ?? true,
+                    windowMs: rl.windowMs ?? 60_000,
+                    maxRequests: rl.maxRequests ?? 3_000,
+                    globalMaxRequests: rl.globalMaxRequests ?? 6_000,
+                    trustProxy: rl.trustProxy ?? false,
+                    ...(rl.whitelist ? { whitelist: rl.whitelist } : {}),
+                },
             },
         });
         // WebSocket — real-time event push to webapp clients
@@ -241,8 +632,18 @@ export class OmnitronDaemon {
             name: 'daemon-ws',
             options: {
                 port: wsPort,
-                host: '0.0.0.0',
+                host: localHost,
+                // Applied per invocation by `remote-peer.ts`, the dispatcher every
+                // socket transport shares — so WS calls get the same request context
+                // HTTP calls do. Until recently only the HTTP server read this
+                // option and this line did nothing.
                 invocationWrapper: authContextWrapper,
+                // The only bound this transport accepts. Left unset it inherits
+                // `ws`'s 100 MB default, which is a lot of memory to hand a single
+                // frame on a control-plane socket that carries RPC arguments and
+                // event pushes. 8 MB is well above anything the console sends and
+                // well below anything worth allocating for a stranger.
+                maxPayload: dc.wsMaxPayload ?? 8 * 1024 * 1024,
             },
         });
     }
@@ -260,11 +661,20 @@ export class OmnitronDaemon {
         const logManager = await container.resolveAsync(LOG_MANAGER_TOKEN);
         const logCollector = await container.resolveAsync(LOG_COLLECTOR_TOKEN);
         const loggerModule = await container.resolveAsync(LOGGER_SERVICE_TOKEN);
+        // Table retention. `logging.maxSize` and `maxFiles` bound the rotated
+        // files on disk and say nothing about the `logs` table, which had no
+        // bound at all — 13 GB on the development host. Off unless configured,
+        // and the number comes from the config rather than a default chosen
+        // inside the collector, because a wrong default there deletes history.
+        logCollector.setRetentionDays(config.logging?.databaseRetentionDays ?? 0, loggerModule.logger.child({ component: 'log-retention' }));
         const rpcService = new DaemonRpcService(orchestrator, titanHealth, logManager, config, this);
         await this.app.netron.peer.exposeService(rpcService);
         // Logs RPC service (works on both master and slave — uses PG or SQLite)
         const logsRpcService = new LogsRpcService(logCollector);
         await this.app.netron.peer.exposeService(logsRpcService);
+        // `getLogs` answers an app this daemon does not run from the stored rows —
+        // where a node's synced logs are.
+        rpcService.setLogStore(logCollector);
         // Metrics RPC service — provided by TitanMetricsModule
         const metricsService = await container.resolveAsync(TITAN_METRICS_TOKEN);
         const { MetricsRpcService: MetricsRpc } = await import('@omnitron-dev/titan-metrics');
@@ -275,10 +685,29 @@ export class OmnitronDaemon {
         const healthCheckService = await container.resolveAsync(HEALTH_CHECK_SERVICE_TOKEN);
         const healthCheckRpcService = new HealthCheckRpc(healthCheckService);
         await this.app.netron.peer.exposeService(healthCheckRpcService);
+        // The audit trail. Master-only — its table lives in the omnitron
+        // database — and every service that records into it takes it as an
+        // optional dependency, so a slave serves the same methods and records
+        // nothing rather than refusing them.
+        let audit;
+        if (!isSlave) {
+            try {
+                audit = await container.resolveAsync(AUDIT_SERVICE_TOKEN);
+                // Revealing an app's secrets is recorded (`revealEnv`).
+                rpcService.setAudit(audit);
+                const { AuditRpcService } = await import('../services/audit.rpc-service.js');
+                await this.app.netron.peer.exposeService(new AuditRpcService(audit));
+            }
+            catch (err) {
+                // A daemon whose database did not come up still runs; it simply
+                // records nothing, and says so once rather than at every action.
+                loggerModule.logger.warn({ error: err.message }, 'No audit trail on this daemon — actions will not be recorded');
+            }
+        }
         // Secrets RPC service (file-based, no DB dependency)
         const { SecretsRpcService: SecretsRpc } = await import('../services/secrets.rpc-service.js');
         const secretsService = await container.resolveAsync(SECRETS_SERVICE_TOKEN);
-        const secretsRpcService = new SecretsRpc(secretsService);
+        const secretsRpcService = new SecretsRpc(secretsService, audit);
         await this.app.netron.peer.exposeService(secretsRpcService);
         // Kubernetes RPC service (no DB dependency)
         const { KubernetesRpcService: K8sRpc } = await import('../services/kubernetes.rpc-service.js');
@@ -290,14 +719,60 @@ export class OmnitronDaemon {
         const backupService = await container.resolveAsync(BACKUP_SERVICE_TOKEN);
         const backupRpcService = new BackupRpc(backupService);
         await this.app.netron.peer.exposeService(backupRpcService);
+        // Re-arm any persisted backup schedules so they survive daemon restarts.
+        await backupService.restoreSchedules().catch((err) => loggerModule.logger.warn({ err: err.message }, 'Failed to restore backup schedules'));
         // Infrastructure RPC service (reports infra state — slave has no infra)
         const { InfrastructureRpcService: InfraRpc } = await import('../services/infrastructure.rpc-service.js');
-        const infraRpcService = new InfraRpc(() => this.infraService);
+        const infraRpcService = new InfraRpc(() => this.infraService, 
+        // Only a node hosts a stack's infrastructure on demand. A master
+        // provisions its own from its own config at boot, and handing it a
+        // second one over RPC would give one machine two reconcilers over one
+        // set of containers.
+        isSlave
+            ? (infraConfig, declaredServices, presetRegistry, serviceOverrides) => {
+                const service = new InfrastructureService(loggerModule.logger.child({ component: 'infra' }), infraConfig, declaredServices, presetRegistry, serviceOverrides, 
+                // A node keeps its own state in SQLite. Provisioning a
+                // control-plane Postgres for it gives the host a database
+                // nobody queries, on default credentials.
+                false);
+                this.infraService = service;
+                return service;
+            }
+            : undefined, 
+        // The node's own vault. Generated service credentials are written
+        // here and read back on every later provision, which is what makes a
+        // generated password usable: a data directory keeps the one it was
+        // initialised with.
+        // Resolved per call, not captured here: a resolution that fails during
+        // startup would otherwise leave credential generation off for the life
+        // of this daemon, and the absence reads as "this node needs none".
+        isSlave
+            ? () => {
+                try {
+                    return container.resolve(SECRETS_SERVICE_TOKEN);
+                }
+                catch {
+                    return undefined;
+                }
+            }
+            : undefined, 
+        // The logger, because the one thing this service says on its own — that
+        // a deployment is running on a default credential — is said at error
+        // level and reaches nobody without it.
+        loggerModule.logger.child({ component: 'infra' }));
         await this.app.netron.peer.exposeService(infraRpcService);
         // Project + Stack management RPC service
         const projectService = await container.resolveAsync(PROJECT_SERVICE_TOKEN);
-        const projectRpcService = new ProjectRpcService(projectService);
+        const projectRpcService = new ProjectRpcService(projectService, audit);
         await this.app.netron.peer.exposeService(projectRpcService);
+        // Releases. Master-only, like the builds themselves: a release is built
+        // from clean clones of BOTH repositories, and only a master has both.
+        if (!isSlave) {
+            const { ReleaseService } = await import('../services/release.service.js');
+            const { ReleaseRpcService } = await import('../services/release.rpc-service.js');
+            this.releaseService = new ReleaseService(loggerModule.logger.child({ component: 'release' }));
+            await this.app.netron.peer.exposeService(new ReleaseRpcService(this.releaseService, audit, projectService));
+        }
         // Sync service (slave→master data replication)
         const syncNodeId = `${os.hostname()}-${dc.port}`;
         let syncDb;
@@ -309,6 +784,53 @@ export class OmnitronDaemon {
             syncDb = await container.resolveAsync(OMNITRON_DB_TOKEN);
         }
         this.syncService = new SyncService(syncDb, loggerModule.logger.child({ component: 'sync' }), syncNodeId, dc.role, dc.sync);
+        // Remote metrics into the store the console actually reads.
+        //
+        // `ingestMetric` wrote them to `metrics_raw`, and nothing in this
+        // repository reads that table — the metrics page queries titan-metrics'
+        // own storage. So once the pipeline carried anything, it would have
+        // carried it somewhere invisible. Recording through the same service the
+        // master uses for its own readings makes a remote node a LABEL on the
+        // existing series instead of a second store of the same concept.
+        //
+        // `node` is a distinct argument in the sink's signature rather than a
+        // key a caller may forget to put in `labels`: a sample recorded without
+        // it merges with the master's own, and the chart then shows two machines
+        // summed under one name.
+        if (!isSlave) {
+            const syncMetrics = await container.resolveAsync(TITAN_METRICS_TOKEN);
+            this.syncService.setMetricsSink((sample) => {
+                try {
+                    syncMetrics.recordTyped('gauge', sample.name, { ...sample.labels, app: sample.app, node: sample.node }, sample.value);
+                }
+                catch (err) {
+                    loggerModule.logger.warn({ node: sample.node, name: sample.name, error: err.message }, 'Failed to record a replicated metric');
+                }
+            });
+        }
+        // The producer the replication pipeline never had.
+        //
+        // `SyncService` buffers, batches, retries with backoff, evicts when over
+        // budget, and the master keeps a dedup ledger for what it has taken —
+        // all of it built, and `buffer`/`bufferBatch` had ZERO callers. A slave
+        // collected its logs into its own SQLite and replicated none of them,
+        // and nothing said so: an empty buffer drains successfully every cycle.
+        //
+        // Wired only for a slave. `bufferBatch` is a no-op for any other role,
+        // so this is belt and braces rather than the guard itself — but a master
+        // has no master to ship to, and doing the work would be pure cost.
+        if (isSlave) {
+            const syncService = this.syncService;
+            logCollector.setSyncSink((entries) => {
+                void syncService
+                    .bufferBatch(entries.map((e) => ({ category: 'logs', payload: e })))
+                    .catch(() => {
+                    // `bufferBatch` logs its own failure. Swallowed here because the
+                    // lines are already in the local table: losing the replica copy
+                    // must not cost the flush that succeeded.
+                });
+            });
+        }
         const syncRpcService = new SyncRpcService(this.syncService);
         await this.app.netron.peer.exposeService(syncRpcService);
         // System info service (real-time OS/hardware metrics)
@@ -317,19 +839,120 @@ export class OmnitronDaemon {
         await this.app.netron.peer.exposeService(systemInfoRpcService);
         // Event Broadcaster RPC service (WebSocket subscriptions)
         this.eventBroadcaster = new EventBroadcasterService(loggerModule.logger);
-        const eventBroadcasterRpcService = new EventBroadcasterRpcService(this.eventBroadcaster);
+        const eventBroadcasterRpcService = new EventBroadcasterRpcService(this.eventBroadcaster, this.app.netron);
         await this.app.netron.peer.exposeService(eventBroadcasterRpcService);
         // Node manager — file-based, no PG (master only — manages infrastructure nodes)
         // Health checks delegated to dedicated system worker (started later after infra is ready)
         if (!isSlave) {
             const { NodeManagerService } = await import('../services/node-manager.service.js');
             const { NodeManagerRpcService } = await import('../services/node-manager.rpc-service.js');
-            const secretsService = await container.resolveAsync(SECRETS_SERVICE_TOKEN);
-            const nodeManager = new NodeManagerService(loggerModule.logger.child({ component: 'nodes' }), secretsService);
+            const { DaemonStateStore: DaemonStateStoreCls } = await import('./daemon-state-store.service.js');
+            const daemonStateStore = await container.resolveAsync(DAEMON_STATE_STORE_TOKEN);
+            const nodeManager = new NodeManagerService(loggerModule.logger.child({ component: 'nodes' }), daemonStateStore, secretsService);
             this.nodeManagerService = nodeManager;
-            const nodeManagerRpcService = new NodeManagerRpcService(nodeManager);
+            const nodeManagerRpcService = new NodeManagerRpcService(nodeManager, audit);
+            // The deployer reaches nodes through the same SSH implementation as the
+            // health checks — the only one that can present a stored password or a
+            // key passphrase.
+            {
+                const { RemoteDeployer } = await import('../services/remote-deployer.service.js');
+                const { ExecutionService } = await import('../execution/execution.service.js');
+                const deployLogger = loggerModule.logger.child({ component: 'deploy' });
+                const remoteDeployer = new RemoteDeployer(deployLogger, new ExecutionService(deployLogger));
+                nodeManagerRpcService.setRemoteDeployer(remoteDeployer);
+                // Upgrading a node from the console: the same build the CLI makes,
+                // made here, then the same two calls. Without this the console could
+                // see every node and update none of them.
+                const { NodeUpgradeService } = await import('../services/node-upgrade.service.js');
+                nodeManagerRpcService.setUpgradeService(new NodeUpgradeService(loggerModule.logger.child({ component: 'node-upgrade' }), {
+                    getNode: (id) => nodeManager.getNode(id),
+                    // The same projection the CLI builds for `fleet upgrade`, made
+                    // here so the console's plan and the terminal's dry run decide
+                    // from identical inputs. Two places computing «what is this
+                    // node running» is how they come to disagree.
+                    listCandidates: () => nodeManager.listNodes().map((n) => ({
+                        nodeId: n.id,
+                        name: n.name,
+                        currentVersion: n.status?.omnitronVersion ?? null,
+                        isLocal: n.isLocal,
+                        // SSH, not the daemon port: an upgrade travels over SSH,
+                        // and a node whose daemon is unreachable is often exactly
+                        // the one worth upgrading.
+                        sshReachable: n.status?.sshConnected ?? null,
+                        // The machine, for telling two registry names for one box
+                        // apart before both get the bundle.
+                        address: `${n.host}:${n.sshPort}`,
+                    })),
+                }, (id) => nodeManager.nodeToDeployTarget(id), () => remoteDeployer, audit));
+            }
             this.nodeManagerRpcService = nodeManagerRpcService;
+            // The local node's own answers, for the three readers that ask every
+            // node about itself. A daemon has no mesh connection to itself, so
+            // `askNode` needs a local source for each.
+            //
+            // On the INSTANCE, after it is assigned, and not on `this.…?.` from
+            // wherever each dependency happens to be constructed: `setSyncService`
+            // was first written next to `new SyncService(...)`, ninety lines above
+            // this assignment and inside the same function, where `?.` swallowed it
+            // silently and the local node would have reported "not wired" forever.
+            // An optional call is only safe where the object is known to exist.
+            nodeManagerRpcService.setSyncService(this.syncService);
             await this.app.netron.peer.exposeService(nodeManagerRpcService);
+            // The mesh: every registered node, connected and replicating.
+            //
+            // `SlaveConnector` maintains the connections and pulls each node's
+            // write-ahead buffer on connect and on every heartbeat, and all of it
+            // worked — but the only caller of `addSlave` was a remote or cluster
+            // stack starting. So the master connected to the nodes something had
+            // been deployed onto, and to no others. A node added through the
+            // console, provisioned, and collecting its own metrics and logs was
+            // never dialled, and buffered locally forever: measured at 47,407
+            // undelivered entries over eleven hours on the first such node, which
+            // reported healthy the whole time.
+            //
+            // Built here rather than inside ProjectService because reaching a node
+            // needs what only this side has — the registry's SSH credentials, for
+            // the token the node's daemon demands and for the tunnel a firewalled
+            // daemon port needs. ProjectService adopts the same instance: one per
+            // master, or two connectors pull the same buffer and race to ack it.
+            try {
+                const { SlaveConnector } = await import('../cluster/slave-connector.js');
+                const { createMeshDialer } = await import('../cluster/mesh-link.js');
+                const { startMesh } = await import('../cluster/mesh.js');
+                const { ExecutionService: Exec } = await import('../execution/execution.service.js');
+                const meshLogger = loggerModule.logger.child({ component: 'mesh' });
+                const connector = new SlaveConnector(meshLogger, this.syncService ?? null, {
+                    dial: createMeshDialer({
+                        logger: meshLogger,
+                        execution: new Exec(meshLogger),
+                        subject: `mesh:${os.hostname()}`,
+                        sshTargetFor: async ({ host }) => {
+                            const node = nodeManager.listNodes().find((n) => n.host === host && !n.isLocal);
+                            return node ? nodeManager.nodeToSshTarget(node) : null;
+                        },
+                    }),
+                });
+                this.slaveConnector = connector;
+                projectService.setSlaveConnector(connector);
+                // So the console can answer "is this node replicating", which is a
+                // different question from "can this master reach it".
+                nodeManagerRpcService.setSlaveConnector(connector);
+                // A stack names a host; the registry holds the credential for it.
+                // Wired here because this is the side that has both — ProjectService
+                // reads config files, the node registry keeps its secrets in the
+                // daemon's vault.
+                projectService.setNodeCredentialResolver(async (host) => {
+                    const node = nodeManager.listNodes().find((n) => n.host === host && !n.isLocal);
+                    return node ? nodeManager.nodeToSshTarget(node) : null;
+                });
+                this.meshHandle = startMesh({ registry: nodeManager, connector, logger: meshLogger });
+            }
+            catch (err) {
+                // A master with no mesh still supervises its own apps, so this does
+                // not stop the daemon — but nothing else reports it, and a silent
+                // failure here looks exactly like a fleet that has nothing to say.
+                loggerModule.logger.error({ error: err.message }, 'Mesh not started — registered nodes will buffer their data locally and replicate nothing');
+            }
         }
         // =====================================================================
         // Master-only RPC services (require PostgreSQL)
@@ -348,6 +971,9 @@ export class OmnitronDaemon {
             const { TelemetryRpcService: TelemetryRpc } = await import('../services/telemetry.rpc-service.js');
             const telemetryRelay = await container.resolveAsync(TELEMETRY_RELAY_TOKEN);
             const telemetryRpcService = new TelemetryRpc(telemetryRelay);
+            // The console asks every node for its relay, and a daemon has no mesh
+            // connection to itself, so the local node's answer comes from here.
+            this.nodeManagerRpcService?.setTelemetryRelay(telemetryRelay);
             await this.app.netron.peer.exposeService(telemetryRpcService);
             // Fleet management RPC service
             const { FleetRpcService: FleetRpc } = await import('../services/fleet.rpc-service.js');
@@ -387,6 +1013,8 @@ export class OmnitronDaemon {
                 ...(dc.cluster.heartbeatInterval != null && { heartbeatInterval: dc.cluster.heartbeatInterval }),
             });
             const clusterRpcService = new ClusterRpc(this.leaderElection);
+            // The local node's own election state, for the cross-node reader.
+            this.nodeManagerRpcService?.setLeaderElection(this.leaderElection);
             await this.app.netron.peer.exposeService(clusterRpcService);
         }
     }
@@ -422,6 +1050,10 @@ export class OmnitronDaemon {
         if (!this.app)
             return;
         const stateStore = await this.app.container.resolveAsync(STATE_STORE_TOKEN);
+        // First-touch init reads from SQLite (and migrates the legacy
+        // JSON snapshot in-place if one exists). Subsequent reads via
+        // `load()` are synchronous from the in-memory cache.
+        await stateStore.init();
         const previousState = stateStore.load();
         if (previousState) {
             const runningApps = previousState.apps.filter((a) => a.status === 'online' || a.status === 'starting');
@@ -481,8 +1113,37 @@ export class OmnitronDaemon {
                 logger.warn('Continuing without internal database — some features will be unavailable');
             }
         }
-        // Step 2: Provision app infrastructure (postgres, redis, minio) if declared
-        // Slave daemons never provision Docker infrastructure
+        // Step 2: Provision app infrastructure (postgres, redis, minio) if declared.
+        //
+        // Two ownership models share this slot historically:
+        //
+        //   (a) Top-level apps declared directly in the daemon config — the
+        //       daemon owns their infra here, marks the gate ready, and apps
+        //       in the top-level `apps: []` proceed.
+        //
+        //   (b) Project/stack mode — infra is owned by `StackInfrastructureManager`
+        //       per `(project, stack)` and provisioned later by
+        //       `projectService.startStack`. Running both passes can collide
+        //       on the SAME service name with different prefixes (D14): the
+        //       daemon path uses prefix `omnitron-` while the stack path
+        //       sets `${project}-${stack}-`. With a non-trivial top-level
+        //       `infrastructure.services` map the daemon would create a
+        //       parallel set of containers that nobody owns and the stack's
+        //       provision later races them on ports.
+        //
+        // Resolution: if any project has enabled stacks, defer infra entirely
+        // to stack provision and mark the gate ready immediately. Top-level
+        // `infrastructure.services` is informational in this mode (used by
+        // stack auto-detection upstream); the daemon does not provision it
+        // again here.
+        const { ProjectRegistry } = await import('../project/registry.js');
+        const projects = ProjectRegistry.open().list();
+        const projectModeActive = projects.some((p) => Array.isArray(p.enabledStacks) && p.enabledStacks.length > 0);
+        if (projectModeActive) {
+            logger.info({ projects: projects.length }, 'project mode active — daemon-level infrastructure deferred to stack provision');
+            gate.markReady({ services: {}, ready: true });
+            return;
+        }
         if (config.infrastructure && !isSlave) {
             try {
                 this.infraService = new InfrastructureService(logger, config.infrastructure);
@@ -502,7 +1163,21 @@ export class OmnitronDaemon {
                     error: reason,
                 });
                 logger.error({ error: reason }, 'Failed to provision app infrastructure');
-                logger.warn('Apps with infrastructure requirements may fail to start');
+                // Fail-fast unless explicitly opted out via `--no-infra` (handled
+                // earlier by skipping this branch). Re-throw so daemon startup
+                // surfaces the actual cause to the operator instead of leaving the
+                // daemon "running" with a half-broken state.
+                const allowDegraded = config.allowDegradedInfra === true;
+                if (!allowDegraded) {
+                    const throwable = new Error(`Daemon infrastructure provisioning failed: ${reason}\n` +
+                        `Stack apps will not start until this is resolved.\n` +
+                        `Fix the underlying issue (Docker socket, image pull, port, …) and run 'omnitron up' again.\n` +
+                        `To deliberately run with no managed infra (external services), pass --no-infra to 'omnitron up'.`);
+                    throwable.code = 'DAEMON_INFRA_PROVISIONING_FAILED';
+                    throwable.cause = err;
+                    throw throwable;
+                }
+                logger.warn('allowDegradedInfra=true — apps with infrastructure requirements may fail to start');
             }
         }
         else {
@@ -521,27 +1196,70 @@ export class OmnitronDaemon {
      */
     async reconcileOrphanContainers(logger, projects, projectService) {
         const { listManagedContainers, stopContainer, removeContainer } = await import('../infrastructure/container-runtime.js');
+        const { decideOrphans } = await import('../infrastructure/orphan-containers.js');
         const managed = await listManagedContainers();
         if (managed.length === 0)
             return;
         // Build set of expected container name prefixes from registered projects/stacks
         const expectedPrefixes = new Set();
         const internalNames = new Set(['omnitron-pg', 'omnitron-nginx']);
+        // Load each project's config FIRST. `listStacks` reads the loaded-config
+        // cache and returns [] for a project whose config has not been read —
+        // and this runs before the per-project load loop further down, whose own
+        // comment says loading "populates internal cache used by listStacks".
+        //
+        // So on every boot where projects come from the registry rather than CWD
+        // auto-detection — the normal case — `expectedPrefixes` was EMPTY and
+        // every managed container looked like an orphan. Measured from this
+        // daemon's own log, 2026-09-12 12:46, ten RUNNING containers removed in
+        // fourteen seconds: postgres, redis, bitcoin, monero (daemon and wallet),
+        // nominatim, tor, tiles, minio, gateway — each logged as "not part of any
+        // registered stack" while being part of the registered stack.
+        //
+        // Named volumes survived (`docker rm -f`, no `-v`), so this cost a cold
+        // restart of every service rather than the chain data. That is luck, not
+        // design.
+        let expectationsComplete = true;
         for (const project of projects) {
+            try {
+                await projectService.loadProjectConfig(project.name);
+            }
+            catch (err) {
+                expectationsComplete = false;
+                logger.warn({ project: project.name, error: err.message }, 'Could not read this project’s stacks — skipping orphan reconciliation rather than guessing');
+                continue;
+            }
             const stacks = projectService.listStacks(project.name);
+            if (stacks.length === 0)
+                expectationsComplete = false;
             for (const stack of stacks) {
                 expectedPrefixes.add(`${project.name}-${stack.name}-`);
             }
         }
+        // Whether anything may be removed is one decision with several ways of
+        // being wrong, and every way it has been wrong so far was a mistake about
+        // what the inputs MEANT. It lives in `decideOrphans`, where it can be
+        // pinned in a test rather than discovered on a host.
+        const decision = decideOrphans({
+            isSlave: this.dc?.role === 'slave',
+            managed: managed.map((c) => ({
+                name: c.name,
+                status: c.status,
+                project: c.project,
+                stack: c.stack,
+            })),
+            projects: projects.map((p) => p.name),
+            expectedPrefixes: [...expectedPrefixes],
+            expectationsComplete,
+            internalNames: [...internalNames],
+        });
+        if (decision.action === 'skip') {
+            logger.warn({ managed: managed.length, expectedPrefixes: expectedPrefixes.size, because: decision.because }, 'Orphan reconciliation skipped');
+            return;
+        }
         let orphanCount = 0;
-        for (const container of managed) {
-            // Always keep internal omnitron containers
-            if (internalNames.has(container.name))
-                continue;
-            // Keep containers that match a known project/stack prefix
-            if ([...expectedPrefixes].some((prefix) => container.name.startsWith(prefix)))
-                continue;
-            // This container is an orphan — stop and remove it
+        for (const name of decision.containers) {
+            const container = managed.find((c) => c.name === name);
             logger.info({ container: container.name, status: container.status }, 'Removing orphan container (not part of any registered stack)');
             try {
                 if (container.status === 'running')
@@ -557,49 +1275,23 @@ export class OmnitronDaemon {
             logger.info({ removed: orphanCount }, 'Orphan container cleanup complete');
         }
     }
+    /**
+     * Apply the internal-database schema.
+     *
+     * Delegates to the shared runner (`database/migration-runner.ts`) so this
+     * path, the infrastructure service and the `migrate` CLI all execute the
+     * same registry through the same engine. A failure here is logged as an
+     * ERROR — the daemon keeps running, but every feature backed by
+     * omnitron-pg (alerts, sessions, logs, metrics, deployments) is degraded
+     * and the operator has to see it.
+     */
     async runOmnitronMigrations(logger) {
+        const { runOmnitronMigrations } = await import('../database/migration-runner.js');
         try {
-            const { Kysely, PostgresDialect, Migrator } = await import('kysely');
-            const pg = await import('pg');
-            const m001 = await import('../database/migrations/001_initial_schema.js');
-            const m002 = await import('../database/migrations/002_metrics_raw.js');
-            const m003 = await import('../database/migrations/003_pipelines_traces.js');
-            const m004 = await import('../database/migrations/004_sync_buffer.js');
-            const m005 = await import('../database/migrations/005_node_health_checks.js');
-            const pool = new pg.default.Pool({
-                host: 'localhost', port: 5480,
-                database: 'omnitron', user: 'omnitron', password: 'omnitron',
-            });
-            const db = new Kysely({ dialect: new PostgresDialect({ pool }) });
-            try {
-                const migrator = new Migrator({
-                    db,
-                    provider: {
-                        async getMigrations() {
-                            return {
-                                '001_initial_schema': { up: m001.up, down: m001.down },
-                                '002_metrics_raw': { up: m002.up, down: m002.down },
-                                '003_pipelines_traces': { up: m003.up, down: m003.down },
-                                '004_sync_buffer': { up: m004.up, down: m004.down },
-                                '005_node_health_checks': { up: m005.up, down: m005.down },
-                            };
-                        },
-                    },
-                });
-                const { results, error } = await migrator.migrateToLatest();
-                const applied = results?.filter((r) => r.status === 'Success') ?? [];
-                if (applied.length > 0) {
-                    logger.info({ migrations: applied.map((r) => r.migrationName) }, `Applied ${applied.length} migration(s)`);
-                }
-                if (error)
-                    logger.warn({ error: String(error) }, 'Migration warning');
-            }
-            finally {
-                await db.destroy();
-            }
+            await runOmnitronMigrations(logger);
         }
         catch (err) {
-            logger.warn({ error: err.message }, 'Could not run migrations — will retry on next start');
+            logger.error({ error: err.message }, 'Omnitron database migrations FAILED — alerts, sessions, logs and metrics will not work until this is resolved');
         }
     }
     // ============================================================================
@@ -615,6 +1307,15 @@ export class OmnitronDaemon {
             titanHealth.registerIndicator(new DockerHealthIndicator(() => this.infraService));
             // Managed apps health indicator
             titanHealth.registerIndicator(new AppHealthIndicator(orchestrator));
+            // How long the loop has stood still, and when — the number the console
+            // reads for a node that went quiet.
+            if (this.eventLoopWatch)
+                titanHealth.registerIndicator(new EventLoopStallIndicator(this.eventLoopWatch));
+            // The console asks every node for its own indicators, and a daemon has
+            // no mesh connection to itself — so the local node's answer comes from
+            // here. Registered beside the indicators rather than in the wiring
+            // above, because this is the object that has just been given them.
+            this.nodeManagerRpcService?.setTitanHealth(titanHealth);
         }
         catch {
             // Health module may not be available — non-critical
@@ -674,6 +1375,31 @@ export class OmnitronDaemon {
             logger.info('No projects registered. Use `omnitron project add <name> <path>` or run from a directory with omnitron.config.ts');
             return;
         }
+        // When there are enabled stacks to resume, wait for Docker first. Under
+        // OS-service supervision the daemon boots at login, seconds before
+        // OrbStack/dockerd finishes starting — on 2026-07-11 the one-shot resume
+        // fired into "Docker is not available" and the platform stayed appless
+        // for days. Bounded wait; on timeout we proceed and the enabled-stacks
+        // reconciler keeps retrying with backoff.
+        const hasEnabledStacks = allProjects.some((p) => (p.enabledStacks?.length ?? 0) > 0);
+        if (hasEnabledStacks) {
+            const { isDockerAvailable } = await import('../infrastructure/container-runtime.js');
+            const deadline = Date.now() + 120_000;
+            let waited = false;
+            while (!(await isDockerAvailable())) {
+                if (Date.now() > deadline) {
+                    logger.warn('Docker still unavailable after 120s — proceeding; reconciler will retry enabled stacks');
+                    break;
+                }
+                if (!waited) {
+                    logger.info('Docker not available yet (host still booting?) — waiting before stack resume');
+                    waited = true;
+                }
+                await new Promise((r) => setTimeout(r, 3_000));
+            }
+            if (waited)
+                logger.info('Docker is available — continuing stack resume');
+        }
         // Reconcile orphan containers: stop/remove any omnitron.managed containers
         // that don't belong to current registered projects/stacks (e.g. leftovers from
         // a previous session or auto-restarted by Docker after a system reboot).
@@ -708,7 +1434,7 @@ export class OmnitronDaemon {
             // Start each enabled stack
             for (const stackName of enabledStacks) {
                 try {
-                    await projectService.startStack(project.name, stackName);
+                    await projectService.startStack(project.name, stackName, { source: 'boot' });
                     totalStacks++;
                     logger.info({ project: project.name, stack: stackName }, 'Stack started');
                 }
@@ -718,6 +1444,10 @@ export class OmnitronDaemon {
             }
         }
         logger.info({ projects: allProjects.length, stacks: totalStacks }, 'All projects and stacks initialized');
+        // Keep converging after boot: retry failed resumes (with backoff) and
+        // re-start enabled stacks that later fall over. One-shot boot resume is
+        // not enough — see the 2026-07-11 Docker-race incident.
+        projectService.startEnabledStacksReconciler();
     }
     // ============================================================================
     // Scheduled Tasks (Phase 2: titan-scheduler replaces bare setInterval)
@@ -734,7 +1464,8 @@ export class OmnitronDaemon {
             const logManager = await container.resolveAsync(LOG_MANAGER_TOKEN);
             // Master-only services (require PG)
             const authService = !isSlave ? await container.resolveAsync(AUTH_SERVICE_TOKEN) : null;
-            const alertService = !isSlave ? await container.resolveAsync(ALERT_SERVICE_TOKEN) : null;
+            // Both roles: a node evaluates its own alarms on its SQLite (daemon.module.ts).
+            const alertService = await container.resolveAsync(ALERT_SERVICE_TOKEN);
             const fleetService = !isSlave ? await container.resolveAsync(FLEET_SERVICE_TOKEN) : null;
             registerDaemonJobs(scheduler, {
                 logger,
@@ -745,6 +1476,10 @@ export class OmnitronDaemon {
                 fleetService,
                 logManager,
                 infraService: this.infraService,
+                // Only a slave replicates. `bufferBatch` is itself a no-op for any
+                // other role, but passing null keeps the job from registering at all
+                // and makes the scheduler's own "skipped, because" line say so.
+                syncService: isSlave ? this.syncService : null,
                 metricsInterval: config.monitoring.metrics.interval,
                 healthCheckInterval: config.monitoring.healthCheck.interval,
             });
@@ -775,16 +1510,18 @@ export class OmnitronDaemon {
                 logger.warn({ error: err.message }, 'Telemetry relay failed to start');
             }
         }
-        // Register self in fleet (master only — requires PG)
+        // Register self in fleet (master only — requires PG). `registerSelf`, so
+        // the service remembers which row is this daemon's: the fleet-heartbeat
+        // job beats that row, and had only the literal 'self' before.
         if (!isSlave) {
             try {
                 const fleetService = await container.resolveAsync(FLEET_SERVICE_TOKEN);
-                await fleetService.registerNode({
+                await fleetService.registerSelf({
                     hostname: os.hostname(),
-                    address: this.dc.host === '0.0.0.0' ? '127.0.0.1' : this.dc.host,
+                    address: advertisedAddress(this.dc.host),
                     port: this.dc.port,
                     role: 'leader',
-                    metadata: { pid: process.pid, version: CLI_VERSION, httpPort: this.dc.httpPort },
+                    metadata: { pid: process.pid, version: CLI_VERSION, httpPort: this.dc.httpPort, rpcPort: this.rpcHttpPort },
                 });
             }
             catch {
@@ -829,77 +1566,167 @@ export class OmnitronDaemon {
     // ============================================================================
     // Health Monitor System Worker
     // ============================================================================
+    /**
+     * Start the health-monitor system worker, and keep it started.
+     *
+     * Three things were wrong with the previous version, and they compounded
+     * into one outage:
+     *
+     *  - It returned SILENTLY when a precondition failed, so a daemon that
+     *    never checked a single node looked exactly like one that checked them
+     *    all. Each branch now says which precondition it was.
+     *  - Nothing watched the worker's process. When it died, the master kept
+     *    the proxy, and every fleet check — the console's Refresh, the Check
+     *    button, `omnitron node check` — answered
+     *    `TitanError: Service with id HealthMonitor@1.0.0 not found`. Observed
+     *    live: a fleet frozen at the worker's last report for two days, with
+     *    the console rendering it as current.
+     *  - The fallback timer existed only inside the catch for a FAILED SPAWN,
+     *    so a worker that started and later crashed had no fallback at all.
+     *
+     * The worker is now respawned with backoff, the proxy is dropped the moment
+     * its process ends, and the daemon runs the checks itself in the meantime.
+     */
     async startHealthMonitorWorker(logger) {
-        if (!this.app || !this.nodeManagerService || this.dc.role === 'slave')
+        if (this.dc.role === 'slave')
+            return; // By design: slaves do not manage a fleet.
+        if (!this.app) {
+            logger.warn({}, 'Health monitor not started — application is not available');
             return;
+        }
+        if (!this.nodeManagerService) {
+            logger.warn({}, 'Health monitor not started — node manager was not constructed');
+            return;
+        }
+        const nodeManager = this.nodeManagerService;
         try {
             const titanPm = await import('@omnitron-dev/titan-pm');
             const { SystemWorkerManager } = await import('../workers/system-worker-manager.js');
             const { fileURLToPath } = await import('node:url');
             const nodePath = await import('node:path');
+            const { omnitronPgConnectionString } = await import('../database/connection.js');
             // Resolve PM from DI
             const pm = await this.app.container.resolveAsync(titanPm.PM_MANAGER_TOKEN);
             const loggerModule = await this.app.container.resolveAsync(LOGGER_SERVICE_TOKEN);
-            this.systemWorkerManager = new SystemWorkerManager(pm, loggerModule.logger.child({ component: 'system-workers' }));
-            // Build health monitor config
-            const hmDefaults = this.dc.healthMonitor ?? {};
-            const healthConfig = {
-                intervalMs: hmDefaults.intervalMs ?? 30_000,
-                concurrency: hmDefaults.concurrency ?? 20,
-                offlineTimeoutMs: hmDefaults.offlineTimeoutMs ?? 90_000,
-                pingTimeout: hmDefaults.pingTimeout ?? 5_000,
-                sshTimeout: hmDefaults.sshTimeout ?? 10_000,
-                omnitronCheckTimeout: hmDefaults.omnitronCheckTimeout ?? 15_000,
-                pingEnabled: hmDefaults.pingEnabled ?? true,
-                retentionDays: hmDefaults.retentionDays ?? 7,
-                dbUrl: 'postgresql://omnitron:omnitron@localhost:5480/omnitron',
-            };
-            // Get worker process path — .ts when running via tsx, .js when compiled
+            const workerManager = new SystemWorkerManager(pm, loggerModule.logger.child({ component: 'system-workers' }));
+            this.systemWorkerManager = workerManager;
+            // Get worker process path. The naive heuristic of "tsx in execArgv → .ts"
+            // is wrong: the daemon binary itself is loaded with `--import tsx/esm`
+            // (so transpilation works for any .ts in the tree), but the *daemon
+            // entry* may still be the compiled .js. We must check what physically
+            // exists alongside the running daemon, not what's in execArgv.
             const thisDir = nodePath.dirname(fileURLToPath(import.meta.url));
-            const isTsx = process.execArgv.some((a) => a.includes('tsx'));
-            const ext = isTsx ? '.ts' : '.js';
-            const workerPath = nodePath.resolve(thisDir, '..', 'workers', `health-monitor-process${ext}`);
-            // Convert nodes to check targets
-            const nodesJson = JSON.stringify(await this.nodeManagerService.getNodeCheckTargets());
-            const configJson = JSON.stringify(healthConfig);
+            const workersDir = nodePath.resolve(thisDir, '..', 'workers');
+            const tsPath = nodePath.join(workersDir, 'health-monitor-process.ts');
+            const jsPath = nodePath.join(workersDir, 'health-monitor-process.js');
+            const fsMod = await import('node:fs');
+            const workerPath = fsMod.existsSync(jsPath)
+                ? jsPath
+                : fsMod.existsSync(tsPath)
+                    ? tsPath
+                    : jsPath; // fall through to .js — error will be clear if missing
             const spawnOpts = {};
             if (process.execArgv.some((a) => a.includes('tsx'))) {
                 spawnOpts.execArgv = ['--import', 'tsx/esm'];
             }
-            spawnOpts.startupTimeout = 30_000;
-            const proxy = await this.systemWorkerManager.spawn('health-monitor', workerPath, { configJson, nodesJson }, spawnOpts);
-            // Wire IPC: worker → master status cache updates via PM's public API
-            const nodeManager = this.nodeManagerService;
-            const eventBroadcaster = this.eventBroadcaster;
-            this.systemWorkerManager.onMessage('health-monitor', (msg) => {
-                if (msg?.type === 'health:status_batch' && Array.isArray(msg.summaries)) {
-                    nodeManager.updateStatusCacheFromWorker(msg.summaries);
-                    // Broadcast node status updates via WebSocket
-                    if (eventBroadcaster) {
-                        const online = msg.summaries.filter((s) => s.status === 'online').length;
-                        const degraded = msg.summaries.filter((s) => s.status === 'degraded').length;
-                        const offline = msg.summaries.filter((s) => s.status === 'offline').length;
-                        eventBroadcaster.broadcast(NODE_EVENTS.CHECK_COMPLETED, {
-                            nodeCount: msg.summaries.length,
-                            onlineCount: online,
-                            degradedCount: degraded,
-                            offlineCount: offline,
-                        });
+            // What the worker has to do before it can report ready is create a
+            // Titan application — a DI container, a module graph and a PG pool.
+            // Managed applications get two to five minutes for that; this was
+            // thirty seconds, and on a loaded host it expired on every attempt.
+            // A deadline shorter than the work turns a healthy subsystem into one
+            // that is started and killed for ever, which is what was observed:
+            // fifty minutes of restart attempts, none of which could have finished.
+            spawnOpts.startupTimeout =
+                this.dc.healthMonitor?.startupTimeoutMs
+                    ?? DEFAULT_DAEMON_CONFIG.healthMonitor?.startupTimeoutMs
+                    ?? 120_000;
+            this.healthWorkerStartupTimeout = spawnOpts.startupTimeout;
+            /**
+             * The worker's configuration.
+             *
+             * Read at every spawn, not once: the operator's check settings live in
+             * the node manager and change at runtime. The fallbacks come from
+             * `DEFAULT_DAEMON_CONFIG` rather than being written out again here —
+             * the inline copies said 30 s and 7 days while the declared defaults
+             * said 60 s and 90 days, so which numbers a deployment got depended on
+             * whether its config file happened to mention `healthMonitor` at all.
+             */
+            const buildConfig = () => {
+                const declared = DEFAULT_DAEMON_CONFIG.healthMonitor ?? {};
+                const configured = this.dc.healthMonitor ?? {};
+                const operator = nodeManager.getCheckConfig();
+                return {
+                    intervalMs: configured.intervalMs ?? declared.intervalMs ?? 60_000,
+                    concurrency: operator.concurrency,
+                    offlineTimeoutMs: configured.offlineTimeoutMs ?? declared.offlineTimeoutMs ?? 90_000,
+                    pingTimeout: operator.pingTimeout,
+                    sshTimeout: operator.sshTimeout,
+                    omnitronCheckTimeout: operator.omnitronCheckTimeout,
+                    pingEnabled: operator.pingEnabled,
+                    retentionDays: configured.retentionDays ?? declared.retentionDays ?? 90,
+                    // Was a hard-coded DSN, password included, in this file — the sixth
+                    // copy of literals `database/connection.ts` exists to be the only
+                    // copy of. A daemon pointed elsewhere by OMNITRON_DATABASE_URL wrote
+                    // its check history to a database nothing else used.
+                    dbUrl: omnitronPgConnectionString(),
+                };
+            };
+            this.healthWorkerSpawn = async () => {
+                const healthConfig = buildConfig();
+                const nodesJson = JSON.stringify(await nodeManager.getNodeCheckTargets());
+                const proxy = await workerManager.spawn('health-monitor', workerPath, { configJson: JSON.stringify(healthConfig), nodesJson }, spawnOpts);
+                // Wire IPC: worker → master status cache updates via PM's public API
+                workerManager.onMessage('health-monitor', (msg) => {
+                    if (msg?.type === 'health:status_batch' && Array.isArray(msg.summaries)) {
+                        nodeManager.updateStatusCacheFromWorker(msg.summaries);
+                        this.broadcastNodeCheckCompleted(msg.summaries);
                     }
+                });
+                if (this.nodeManagerRpcService) {
+                    this.nodeManagerRpcService.setHealthWorkerProxy(proxy);
                 }
+                this.healthWorkerRestartAttempts = 0;
+                this.stopFallbackNodeChecks();
+                logger.info({ intervalMs: healthConfig.intervalMs }, 'Health monitor system worker started');
+            };
+            // A worker that ends — crash or deliberate stop — must take its proxy
+            // with it. `expected` is true only while the daemon is shutting down,
+            // which is also when a respawn would be wrong.
+            workerManager.onExit('health-monitor', (exit) => {
+                this.nodeManagerRpcService?.setHealthWorkerProxy(null);
+                if (exit.expected || this.isShuttingDown)
+                    return;
+                this.startFallbackNodeChecks(logger);
+                this.scheduleHealthMonitorRestart(logger);
+            });
+            // Operator changes to the check settings have to reach the process that
+            // performs the checks. They reached a field on the node manager that
+            // only the daemon's own fallback path read, so turning ping off in the
+            // console turned nothing off.
+            nodeManager.on('checkConfig:changed', () => {
+                void callWorkerMethod(workerManager, 'updateConfig', JSON.stringify(buildConfig())).catch((err) => {
+                    logger.warn({ error: err.message }, 'Failed to push check config to health monitor');
+                });
             });
             // Wire node CRUD events → sync to worker
             const syncNodesToWorker = async () => {
+                if (!workerManager.get('health-monitor'))
+                    return;
                 const targets = await nodeManager.getNodeCheckTargets();
-                proxy.updateNodes(JSON.stringify(targets)).catch(() => { });
+                await callWorkerMethod(workerManager, 'updateNodes', JSON.stringify(targets)).catch((err) => {
+                    logger.warn({ error: err.message }, 'Failed to sync node list to health monitor');
+                });
             };
             nodeManager.on('node:added', syncNodesToWorker);
             nodeManager.on('node:updated', syncNodesToWorker);
             nodeManager.on('node:removed', syncNodesToWorker);
-            // Wire RPC service to use worker proxy + PG repository for direct reads
+            // A node changing state is the event the console subscribes to. Four of
+            // the five NODE_EVENTS channels were declared and never emitted.
+            nodeManager.on('node:health', (t) => {
+                this.broadcastNodeTransition(t);
+            });
+            // Wire PG repository for direct history reads (bypasses worker)
             if (this.nodeManagerRpcService) {
-                this.nodeManagerRpcService.setHealthWorkerProxy(proxy);
-                // Wire PG repository for direct history reads (bypasses worker)
                 try {
                     const db = await this.app.container.resolveAsync(OMNITRON_DB_TOKEN);
                     const { NodeHealthRepository } = await import('../services/node-health.repository.js');
@@ -909,20 +1736,102 @@ export class OmnitronDaemon {
                     logger.warn({}, 'Could not wire health repository — PG may not be ready');
                 }
             }
-            logger.info({ intervalMs: healthConfig.intervalMs }, 'Health monitor system worker started');
+            // The console needs these to size its uptime bars against history that
+            // actually exists.
+            const bootConfig = buildConfig();
+            nodeManager.setHistoryConfig({
+                uptimeIntervalMs: this.dc.healthMonitor?.uptimeIntervalMs
+                    ?? DEFAULT_DAEMON_CONFIG.healthMonitor?.uptimeIntervalMs
+                    ?? 86_400_000,
+                retentionDays: bootConfig.retentionDays,
+            });
+            await this.healthWorkerSpawn();
         }
         catch (err) {
             logger.warn({ error: err.message }, 'Health monitor worker failed to start — falling back to direct checks');
-            // Fallback: start legacy periodic checks if worker fails
-            // This ensures node status still works even without the worker
-            if (this.nodeManagerService) {
-                const intervalMs = this.dc.healthMonitor?.intervalMs ?? 30_000;
-                const timer = setInterval(() => void this.nodeManagerService?.checkAllNodes(), intervalMs);
-                timer.unref();
-                void this.nodeManagerService.checkAllNodes();
-            }
+            this.startFallbackNodeChecks(logger);
+            if (this.healthWorkerSpawn)
+                this.scheduleHealthMonitorRestart(logger);
         }
     }
+    /**
+     * Try the health-monitor worker again, backing off.
+     *
+     * Capped at a minute: the worker is cheap to start and the fleet view is
+     * blind without it, but a worker that cannot start (a missing file, a bad
+     * runtime) must not be respawned in a tight loop for the life of the daemon.
+     */
+    scheduleHealthMonitorRestart(logger) {
+        if (this.healthWorkerRestartTimer || this.isShuttingDown || !this.healthWorkerSpawn)
+            return;
+        const attempt = ++this.healthWorkerRestartAttempts;
+        const delay = Math.min(60_000, 2_000 * 2 ** Math.min(attempt - 1, 5));
+        logger.warn({ attempt, delayMs: delay }, 'Health monitor worker is down — restarting');
+        this.healthWorkerRestartTimer = setTimeout(() => {
+            this.healthWorkerRestartTimer = null;
+            if (this.isShuttingDown || !this.healthWorkerSpawn)
+                return;
+            this.healthWorkerSpawn().catch((err) => {
+                logger.warn({ attempt, startupTimeoutMs: this.healthWorkerStartupTimeout, error: err.message }, 'Health monitor worker restart failed');
+                this.scheduleHealthMonitorRestart(logger);
+            });
+        }, delay);
+        this.healthWorkerRestartTimer.unref();
+    }
+    /**
+     * Run node checks in the daemon while the worker is unavailable.
+     *
+     * Idempotent, and stopped as soon as the worker is back — two check loops
+     * against the same fleet would double every remote host's load and write
+     * two rows per interval.
+     */
+    startFallbackNodeChecks(logger) {
+        if (this.fallbackNodeCheckTimer || !this.nodeManagerService)
+            return;
+        const declared = DEFAULT_DAEMON_CONFIG.healthMonitor ?? {};
+        const intervalMs = this.dc.healthMonitor?.intervalMs ?? declared.intervalMs ?? 60_000;
+        logger.warn({ intervalMs }, 'Checking nodes from the daemon until the health monitor returns');
+        this.fallbackNodeCheckTimer = setInterval(() => {
+            void this.nodeManagerService?.checkAllNodes();
+        }, intervalMs);
+        this.fallbackNodeCheckTimer.unref();
+        void this.nodeManagerService.checkAllNodes();
+    }
+    stopFallbackNodeChecks() {
+        if (!this.fallbackNodeCheckTimer)
+            return;
+        clearInterval(this.fallbackNodeCheckTimer);
+        this.fallbackNodeCheckTimer = null;
+    }
+    /** Aggregate counts from a worker batch → one console event. */
+    broadcastNodeCheckCompleted(summaries) {
+        if (!this.eventBroadcaster)
+            return;
+        const count = (s) => summaries.filter((x) => x.status === s).length;
+        this.eventBroadcaster.broadcast(NODE_EVENTS.CHECK_COMPLETED, {
+            nodeCount: summaries.length,
+            onlineCount: count('online'),
+            degradedCount: count('degraded'),
+            offlineCount: count('offline'),
+        });
+    }
+    /** One node's health changing → the channel named after the new state. */
+    broadcastNodeTransition(transition) {
+        if (!this.eventBroadcaster)
+            return;
+        const payload = {
+            nodeId: transition.nodeId,
+            status: transition.status,
+            ...(transition.previousStatus ? { previousStatus: transition.previousStatus } : {}),
+        };
+        this.eventBroadcaster.broadcast(NODE_EVENTS.STATUS_UPDATED, payload);
+        const channel = transition.status === 'online' ? NODE_EVENTS.WENT_ONLINE
+            : transition.status === 'offline' ? NODE_EVENTS.WENT_OFFLINE
+                : transition.status === 'degraded' ? NODE_EVENTS.WENT_DEGRADED
+                    : null;
+        if (channel)
+            this.eventBroadcaster.broadcast(channel, payload);
+    }
     // ============================================================================
     // Event Broadcasting (Phase 3: orchestrator events → WebSocket push)
     // ============================================================================
@@ -1018,16 +1927,53 @@ export class OmnitronDaemon {
         this.fileWatcher.start(apps);
         return this.fileWatcher.getWatchedApps();
     }
+    /**
+     * Point the file watcher at a reloaded config, if one is running.
+     *
+     * Public because `DaemonRpcService.reloadConfig` is the other half of the
+     * same reload — the RPC and the SIGHUP handler must not diverge on what a
+     * reload updates, which is exactly how the watcher came to be left out of
+     * both.
+     */
+    applyWatchConfig(config) {
+        this.fileWatcher?.applyConfig(config);
+    }
     disableWatch() {
         if (this.fileWatcher) {
             this.fileWatcher.stop();
             this.fileWatcher = null;
         }
     }
+    /**
+     * Whether file watching is running, and over what.
+     *
+     * `enabled` said only that a `FileWatcher` OBJECT existed. Observed on this
+     * host: `{ enabled: true, apps: [] }` — a watcher running over nothing,
+     * reported to an operator as "watching". Everyone downstream read the
+     * boolean and concluded their edits were being picked up.
+     *
+     * The watch set is built from `config.apps` — the daemon's OWN ecosystem
+     * config. Apps that arrive through the project registry are not in it, so a
+     * daemon serving projects watches none of them however the flag reads.
+     * `watching` is the fact an operator actually wants; `enabled` keeps its
+     * old meaning so existing callers do not silently change behaviour, and
+     * `reason` says why the two can disagree.
+     */
     getWatchStatus() {
-        if (!this.fileWatcher)
-            return { enabled: false, apps: [] };
-        return { enabled: true, apps: this.fileWatcher.getWatchedApps() };
+        if (!this.fileWatcher) {
+            return { enabled: false, watching: false, reason: 'the file watcher is not running', apps: [] };
+        }
+        const apps = this.fileWatcher.getWatchedApps();
+        if (apps.length === 0) {
+            return {
+                enabled: true,
+                watching: false,
+                reason: 'the watcher is running but no app matched: the watch set comes from this daemon\'s own ' +
+                    'ecosystem config, and apps registered through a project are not in it',
+                apps,
+            };
+        }
+        return { enabled: true, watching: true, apps };
     }
     async getConfig() {
         const { loadEcosystemConfig } = await import('../config/loader.js');
@@ -1038,7 +1984,7 @@ export class OmnitronDaemon {
     // ============================================================================
     /**
      * Trigger graceful shutdown via Application's lifecycle system.
-     * Called by DaemonRpcService.shutdown() for the `omnitron shutdown` CLI command.
+     * Called by DaemonRpcService.shutdown() for the `omnitron down` CLI command.
      */
     async stop() {
         if (!this.app)
@@ -1047,6 +1993,67 @@ export class OmnitronDaemon {
         // Application doesn't process.exit() for Manual reason — do it explicitly
         process.exit(0);
     }
+    /**
+     * SIGHUP handler — reload the ecosystem config without taking the
+     * daemon down. Follows the Unix convention (`man 7 signal`: "many
+     * daemons interpret SIGHUP as a request to reread their configs").
+     * Reloads via the same code path the `reloadConfig` RPC uses so
+     * orchestrator + RPC service both pick up the new entries.
+     */
+    async reloadConfigOnHup() {
+        if (!this.app)
+            return;
+        // Loading first, on its own: a malformed file must be rejected before
+        // anything is touched, and that failure genuinely means "the daemon is as
+        // it was".
+        const { loadEcosystemConfig } = await import('../config/loader.js');
+        const newConfig = await loadEcosystemConfig();
+        // Applying is the other half, and it has no such guarantee — each
+        // destination takes the config in turn, so a failure in the middle leaves
+        // the daemon split. `applyReloadedConfig` names what applied and what did
+        // not, because "reload failed" reads as "nothing changed" and that is the
+        // wrong reading exactly when it matters.
+        //
+        // The handler used to end in `catch (err) { throw err; }`, which did
+        // nothing whatever. Removing it is trivial; what it was standing in front
+        // of is not.
+        const { applyReloadedConfig } = await import('./config-reload.js');
+        await applyReloadedConfig(newConfig, [
+            {
+                name: 'orchestrator',
+                apply: async (cfg) => {
+                    const orchestrator = await this.app.container.resolveAsync(ORCHESTRATOR_TOKEN);
+                    orchestrator.setConfig(cfg);
+                },
+            },
+            {
+                name: 'file watcher',
+                // The watcher holds its own copy of the app list. Left out, a reload
+                // updated who gets restarted but not who is watched: an app added to
+                // the config was never watched, and one removed from it kept its
+                // watchers and went on triggering restarts.
+                apply: (cfg) => {
+                    this.applyWatchConfig(cfg);
+                },
+            },
+            {
+                name: 'RPC snapshot',
+                // Exposed under DAEMON_SERVICE_ID; the reloadConfig RPC path mutates
+                // `this.config` directly, so we mirror the same write. Unresolvable is
+                // not a failure here — the service may not be exposed yet.
+                apply: async (cfg) => {
+                    const { DAEMON_SERVICE_ID } = await import('../config/defaults.js');
+                    const daemonRpc = await this.app.container
+                        .resolveAsync(DAEMON_SERVICE_ID)
+                        .catch(() => null);
+                    if (daemonRpc) {
+                        daemonRpc.config = cfg;
+                    }
+                },
+            },
+        ]);
+        console.info('[omnitron] SIGHUP — config reloaded');
+    }
     /**
      * Register all daemon shutdown tasks with Application's lifecycle system.
      *
@@ -1071,6 +2078,32 @@ export class OmnitronDaemon {
                 signal: details?.signal,
             });
         });
+        // D13: Application now drives shutdown via LifecycleController, which
+        // emits per-phase events via ApplicationEvent.LifecyclePhaseEvent.
+        // Forward `phase-finish` durations into the metrics bridge so the
+        // `lifecycle_shutdown_phase_duration_seconds` histogram and the
+        // `lifecycle_shutdown_phase_timeouts_total` counter populate.
+        app.on(ApplicationEvent.LifecyclePhaseEvent, (event) => {
+            const bridge = this.metricsBridge;
+            if (!bridge)
+                return;
+            const phase = event?.phase;
+            const kind = event?.kind;
+            const durationMs = typeof event?.durationMs === 'number' ? event.durationMs : 0;
+            if (!phase)
+                return;
+            try {
+                if (kind === 'phase-finish') {
+                    bridge.recordLifecyclePhase(String(phase), durationMs, 'finish');
+                }
+                else if (kind === 'phase-timeout') {
+                    bridge.recordLifecyclePhase(String(phase), durationMs, 'timeout');
+                }
+            }
+            catch {
+                // Metrics are best-effort; never fail shutdown over telemetry.
+            }
+        });
         // --- Priority: First (0) — Immediate cleanup ---
         app.registerShutdownTask('stop-file-watcher-and-webapp', async () => {
             if (this.fileWatcher) {
@@ -1081,6 +2114,13 @@ export class OmnitronDaemon {
                 this.eventBroadcaster.clear();
                 this.eventBroadcaster = null;
             }
+            if (this.metricsServer) {
+                try {
+                    await this.metricsServer.stop();
+                }
+                catch { /* non-critical */ }
+                this.metricsServer = null;
+            }
             // Stop webapp nginx container (non-blocking — don't wait if slow)
             try {
                 const { removeContainer } = await import('../infrastructure/container-runtime.js');
@@ -1088,6 +2128,20 @@ export class OmnitronDaemon {
             }
             catch { /* non-critical */ }
         }, ShutdownPriority.First);
+        // A release build outlives its daemon otherwise.
+        //
+        // Every step runs in its own process group, detached, so `pnpm install`
+        // and the gate suite carry on after the daemon that started them is
+        // gone — holding ports, writing into a release directory nothing will
+        // ever finish, and for a quarter of an hour. Stopped first, with the
+        // file watcher, because ending them is instant and everything after
+        // this is slower.
+        app.registerShutdownTask('stop-release-builds', () => {
+            // It says what it stopped through its own logger; a shutdown task that
+            // resolved the logger module to say one line would be one more thing
+            // that can fail while the daemon is going down.
+            this.releaseService?.stopAll();
+        }, ShutdownPriority.First);
         // --- Priority: VeryHigh (10) — Stop managed apps (fast — 5s per app max) ---
         app.registerShutdownTask('stop-managed-apps', async () => {
             const orchestrator = await container.resolveAsync(ORCHESTRATOR_TOKEN);
@@ -1105,11 +2159,6 @@ export class OmnitronDaemon {
                 await mc.stop();
             }
             catch { /* non-critical */ }
-            try {
-                const alert = await container.resolveAsync(ALERT_SERVICE_TOKEN);
-                alert.stop();
-            }
-            catch { /* non-critical */ }
             try {
                 const fleet = await container.resolveAsync(FLEET_SERVICE_TOKEN);
                 fleet.stopHeartbeatSweep();
@@ -1123,6 +2172,22 @@ export class OmnitronDaemon {
         }, ShutdownPriority.High);
         // --- Priority: Normal (50) — System workers, node manager, sync, cluster ---
         app.registerShutdownTask('stop-system-services', async () => {
+            // The mesh first: stop following the registry, then drop the
+            // connections. Each one may hold an SSH tunnel, and a tunnel that
+            // outlives its daemon holds a socket and a remote session open with
+            // nothing left to notice.
+            try {
+                this.meshHandle?.stop();
+            }
+            catch { /* non-critical */ }
+            this.meshHandle = null;
+            if (this.slaveConnector) {
+                try {
+                    await this.slaveConnector.dispose();
+                }
+                catch { /* non-critical */ }
+                this.slaveConnector = null;
+            }
             if (this.syncService) {
                 try {
                     await this.syncService.stop();
@@ -1140,6 +2205,15 @@ export class OmnitronDaemon {
                     this.configSyncService.stopPeriodicSync();
             }
             catch { /* non-critical */ }
+            // Before stopping the workers: a worker exiting on the way out must not
+            // be read as a crash and rescheduled.
+            this.isShuttingDown = true;
+            this.healthWorkerSpawn = null;
+            if (this.healthWorkerRestartTimer) {
+                clearTimeout(this.healthWorkerRestartTimer);
+                this.healthWorkerRestartTimer = null;
+            }
+            this.stopFallbackNodeChecks();
             if (this.systemWorkerManager) {
                 try {
                     await this.systemWorkerManager.stopAll();
@@ -1186,10 +2260,29 @@ export class OmnitronDaemon {
             }
             catch { /* non-critical */ }
             if (this.infraService) {
-                try {
-                    await this.infraService.teardown();
+                // A node's infrastructure outlives its daemon.
+                //
+                // `teardown()` is `docker rm`, not `docker stop`, and this task runs
+                // on every shutdown — so on a node, restarting the daemon would
+                // remove the stack's Postgres, Redis and MinIO. `omnitron fleet
+                // upgrade` restarts every node's daemon, which would have made a
+                // routine upgrade take the test environment down with it.
+                //
+                // The daemon is the supervisor, not the owner of the data plane's
+                // lifetime. Locally it still tears down — `omnitron down` there means
+                // "I am done for the day" — and on either, `omnitron infra down` is
+                // the command that removes containers on purpose.
+                if (this.dc.role === 'slave') {
+                    // Said on the way out, because the absence of a teardown is
+                    // otherwise indistinguishable from a teardown that failed.
+                    console.info('[omnitron] leaving this node\'s infrastructure running — the daemon does not own its lifetime');
+                }
+                else {
+                    try {
+                        await this.infraService.teardown();
+                    }
+                    catch { /* best-effort */ }
                 }
-                catch { /* best-effort */ }
                 this.infraService = null;
             }
         }, ShutdownPriority.VeryLow);
@@ -1199,4 +2292,22 @@ export class OmnitronDaemon {
         }, ShutdownPriority.Last);
     }
 }
+/**
+ * Call a method on a system worker's proxy.
+ *
+ * `ServiceProxy` is typed with an index signature whose values may be an
+ * async iterable as well as a function, so a direct `worker.updateConfig(…)`
+ * does not type-check. Resolving through one place keeps the cast in one
+ * place, and returns without throwing when the worker is not running — the
+ * only correct answer for "tell the health monitor" when there is none.
+ */
+async function callWorkerMethod(manager, method, ...args) {
+    const worker = manager.get('health-monitor');
+    if (!worker)
+        return undefined;
+    const fn = worker[method];
+    if (typeof fn !== 'function')
+        return undefined;
+    return fn.apply(worker, args);
+}
 //# sourceMappingURL=daemon.js.map