@omnitron-dev/omnitron 0.2.0 → 0.3.0

package/dist/daemon/daemon.d.ts

@@ -28,16 +28,134 @@ export interface DaemonStartOptions {
     /** Disable file watching even if explicitly requested */
     noWatch?: boolean;
 }
+/**
+ * Where the daemon listens. The default is loopback; anything the operator
+ * writes is passed through literally, `0.0.0.0` included.
+ *
+ * Exported so the two call sites below and the tests agree on one
+ * definition. They previously each carried their own copy of
+ * `host !== '0.0.0.0' ? host : '127.0.0.1'`, which is how the two of them
+ * plus the config default managed to disagree about what the default was.
+ */
+export declare function resolveBindHost(configured: string | undefined): string;
+/**
+ * Where a daemon binds its fleet TCP transport.
+ *
+ * A master follows `daemon.host`, whose default is loopback — the safe answer
+ * for the machine an operator is sitting at.
+ *
+ * A slave cannot use that default, because being reachable is the whole of
+ * what a slave is for: the master dials it to check its health and to pull
+ * from it. Bound to loopback it is a daemon nobody can see, and the console
+ * shows it as offline for ever with no way to tell it from a machine that is
+ * genuinely down.
+ *
+ * It was exactly that, and by a longer route than a wrong default. The
+ * generated slave config carried `daemon: { host: '0.0.0.0', … }` — and
+ * `IEcosystemConfig`, the schema of the file it was written into, has no
+ * `daemon` key; the daemon boots from `~/.omnitron/config.json`, which until
+ * now could not carry one either. So the setting was written, ignored, and
+ * every provisioned slave would have bound 127.0.0.1.
+ *
+ * An explicit `host` still wins, for an operator who wants one interface.
+ */
+export declare function fleetBindHostFor(dc: {
+    role?: string;
+    host?: string;
+}): string;
+/**
+ * The bind address a daemon should END UP with, from what was actually
+ * configured.
+ *
+ * `fleetBindHostFor` above takes the daemon's RUNNING config, and by the time
+ * anything holds one, `host` has been filled in from `DEFAULT_DAEMON_CONFIG`
+ * — so `?? '0.0.0.0'` can never fire and every slave binds loopback. The
+ * information it needs is whether an operator SET a host, and only the saved
+ * file knows that: absent means "no opinion", `127.0.0.1` means "loopback,
+ * deliberately".
+ *
+ * Measured 2026-09-14 on a provisioned slave, after the rule was written and
+ * before this existed:
+ *
+ *     LISTEN 127.0.0.1:9700    ← the fleet port no master can dial
+ *     LISTEN 127.0.0.1:9801
+ *     LISTEN 127.0.0.1:9802
+ *
+ * The rule was right, the value it was given was already decided. This is the
+ * same rule applied where the answer is still open.
+ */
+export declare function effectiveBindHost(saved: {
+    role?: string;
+    host?: string;
+} | null): string;
+/**
+ * Where a daemon binds the surfaces that exist to serve a console: the Netron
+ * HTTP RPC and the WebSocket.
+ *
+ * A master follows `daemon.host`, like everything else. A slave does not,
+ * because on a slave that knob is published for a different reason: a fleet
+ * node has to bind its TCP transport somewhere the master can reach, and a
+ * generated slave config therefore says `0.0.0.0`. The same value then
+ * published two more surfaces — an authentication endpoint and a WebSocket
+ * one that has no rate limit available to it — and a slave serves no console,
+ * so nothing dials either. They were cost without benefit, on a machine whose
+ * whole point is being somewhere else.
+ */
+export declare function consoleBindHostFor(dc: {
+    role?: string;
+    host?: string;
+}): string;
+/** Addresses that reach this host and nowhere else. */
+export declare function isLoopbackHost(host: string | undefined): boolean;
+/**
+ * The address other fleet nodes should dial to reach this one.
+ *
+ * `peer.address` is not decoration: `leader-election.ts` calls `requestVote`
+ * and `leaderHeartbeat` against it. A node bound to `0.0.0.0` therefore may
+ * not advertise `0.0.0.0` (not routable) — and must not advertise
+ * `127.0.0.1` either, which is what it did: every other node in the fleet
+ * resolves that to itself, so votes and heartbeats meant for the leader go
+ * to the sender's own loopback. A wrong answer of the right shape.
+ *
+ * When the bind address says nothing about reachability, take the first
+ * non-internal IPv4 the host has. Loopback remains the honest answer only
+ * when there is genuinely nothing else — a single-host deployment, where
+ * no other node can reach this one regardless.
+ */
+export declare function advertisedAddress(bindHost: string | undefined): string;
 export declare class OmnitronDaemon {
     private app;
     private pidManager;
+    /** Port this daemon serves `/netron/invoke` on; advertised to fleet peers. */
+    private rpcHttpPort;
     private fileWatcher;
     private infraService;
+    private eventLoopWatch;
     private eventBroadcaster;
     private syncService;
     private nodeManagerService;
+    private slaveConnector;
+    private meshHandle;
     private nodeManagerRpcService;
+    /**
+     * The live record of release builds this daemon is running.
+     *
+     * Held on the daemon rather than in the container because it is the state
+     * of work in progress: a build that a restart kills leaves a release
+     * directory with no manifest, and that — not this — is the durable trace.
+     */
+    private releaseService;
     private systemWorkerManager;
+    /** Everything the health-monitor worker needs to be spawned again. */
+    private healthWorkerSpawn;
+    private healthWorkerRestartTimer;
+    private healthWorkerRestartAttempts;
+    /** The startup budget the worker is given, so a failure can name it. */
+    private healthWorkerStartupTimeout;
+    /** In-daemon check timer, armed only while the worker is down. */
+    private fallbackNodeCheckTimer;
+    /** Set by the shutdown task so nothing schedules work on the way out. */
+    private isShuttingDown;
     private leaderElection;
     private configSyncService;
     private metricsServer;
@@ -61,6 +179,16 @@ export declare class OmnitronDaemon {
      * with current stack containers are stopped.
      */
     private reconcileOrphanContainers;
+    /**
+     * Apply the internal-database schema.
+     *
+     * Delegates to the shared runner (`database/migration-runner.ts`) so this
+     * path, the infrastructure service and the `migrate` CLI all execute the
+     * same registry through the same engine. A failure here is logged as an
+     * ERROR — the daemon keeps running, but every feature backed by
+     * omnitron-pg (alerts, sessions, logs, metrics, deployments) is degraded
+     * and the operator has to see it.
+     */
     private runOmnitronMigrations;
     private registerHealthIndicators;
     private startFileWatcher;
@@ -77,15 +205,83 @@ export declare class OmnitronDaemon {
     private startApps;
     private startScheduledTasks;
     private startBackgroundServices;
+    /**
+     * Start the health-monitor system worker, and keep it started.
+     *
+     * Three things were wrong with the previous version, and they compounded
+     * into one outage:
+     *
+     *  - It returned SILENTLY when a precondition failed, so a daemon that
+     *    never checked a single node looked exactly like one that checked them
+     *    all. Each branch now says which precondition it was.
+     *  - Nothing watched the worker's process. When it died, the master kept
+     *    the proxy, and every fleet check — the console's Refresh, the Check
+     *    button, `omnitron node check` — answered
+     *    `TitanError: Service with id HealthMonitor@1.0.0 not found`. Observed
+     *    live: a fleet frozen at the worker's last report for two days, with
+     *    the console rendering it as current.
+     *  - The fallback timer existed only inside the catch for a FAILED SPAWN,
+     *    so a worker that started and later crashed had no fallback at all.
+     *
+     * The worker is now respawned with backoff, the proxy is dropped the moment
+     * its process ends, and the daemon runs the checks itself in the meantime.
+     */
     private startHealthMonitorWorker;
+    /**
+     * Try the health-monitor worker again, backing off.
+     *
+     * Capped at a minute: the worker is cheap to start and the fleet view is
+     * blind without it, but a worker that cannot start (a missing file, a bad
+     * runtime) must not be respawned in a tight loop for the life of the daemon.
+     */
+    private scheduleHealthMonitorRestart;
+    /**
+     * Run node checks in the daemon while the worker is unavailable.
+     *
+     * Idempotent, and stopped as soon as the worker is back — two check loops
+     * against the same fleet would double every remote host's load and write
+     * two rows per interval.
+     */
+    private startFallbackNodeChecks;
+    private stopFallbackNodeChecks;
+    /** Aggregate counts from a worker batch → one console event. */
+    private broadcastNodeCheckCompleted;
+    /** One node's health changing → the channel named after the new state. */
+    private broadcastNodeTransition;
     private wireEventBroadcasting;
     enableWatch(apps?: string[]): Promise<Array<{
         name: string;
         directory: string;
     }>>;
+    /**
+     * Point the file watcher at a reloaded config, if one is running.
+     *
+     * Public because `DaemonRpcService.reloadConfig` is the other half of the
+     * same reload — the RPC and the SIGHUP handler must not diverge on what a
+     * reload updates, which is exactly how the watcher came to be left out of
+     * both.
+     */
+    applyWatchConfig(config: IEcosystemConfig): void;
     disableWatch(): void;
+    /**
+     * Whether file watching is running, and over what.
+     *
+     * `enabled` said only that a `FileWatcher` OBJECT existed. Observed on this
+     * host: `{ enabled: true, apps: [] }` — a watcher running over nothing,
+     * reported to an operator as "watching". Everyone downstream read the
+     * boolean and concluded their edits were being picked up.
+     *
+     * The watch set is built from `config.apps` — the daemon's OWN ecosystem
+     * config. Apps that arrive through the project registry are not in it, so a
+     * daemon serving projects watches none of them however the flag reads.
+     * `watching` is the fact an operator actually wants; `enabled` keeps its
+     * old meaning so existing callers do not silently change behaviour, and
+     * `reason` says why the two can disagree.
+     */
     getWatchStatus(): {
         enabled: boolean;
+        watching: boolean;
+        reason?: string;
         apps: Array<{
             name: string;
             directory: string;
@@ -94,9 +290,17 @@ export declare class OmnitronDaemon {
     private getConfig;
     /**
      * Trigger graceful shutdown via Application's lifecycle system.
-     * Called by DaemonRpcService.shutdown() for the `omnitron shutdown` CLI command.
+     * Called by DaemonRpcService.shutdown() for the `omnitron down` CLI command.
      */
     stop(): Promise<void>;
+    /**
+     * SIGHUP handler — reload the ecosystem config without taking the
+     * daemon down. Follows the Unix convention (`man 7 signal`: "many
+     * daemons interpret SIGHUP as a request to reread their configs").
+     * Reloads via the same code path the `reloadConfig` RPC uses so
+     * orchestrator + RPC service both pick up the new entries.
+     */
+    private reloadConfigOnHup;
     /**
      * Register all daemon shutdown tasks with Application's lifecycle system.
      *

package/dist/daemon/daemon.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../../src/daemon/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AA4E1E,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yDAAyD;IACzD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,YAAY,CAAsC;IAC1D,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,kBAAkB,CAAiF;IAC3G,OAAO,CAAC,qBAAqB,CAAwF;IACrH,OAAO,CAAC,mBAAmB,CAAkF;IAC7G,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,aAAa,CAAkE;IACvF,OAAO,CAAC,aAAa,CAAkE;IACvF,OAAO,CAAC,EAAE,CAAwC;IAElD,uEAAuE;IACvE,IAAI,YAAY,IAAI,aAAa,CAAoB;IAE/C,KAAK,CAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,kBAAkB,EAAE,YAAY,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;YA+IlG,kBAAkB;YA6GlB,iBAAiB;YAmLjB,kBAAkB;YA8BlB,kBAAkB;YAwBlB,mBAAmB;IA4IjC;;;;;;;;OAQG;YACW,yBAAyB;YAmDzB,qBAAqB;YAmDrB,wBAAwB;IAqBtC,OAAO,CAAC,gBAAgB;IAuBxB;;;;;;;;;OASG;YACW,SAAS;YAgGT,mBAAmB;YA8CnB,uBAAuB;YA6EvB,wBAAwB;YAgIxB,qBAAqB;IA+F7B,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAgBvF,YAAY,IAAI,IAAI;IAOpB,cAAc,IAAI;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE;YAK1E,SAAS;IASvB;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,qBAAqB;CA8J9B"}
+{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../../src/daemon/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAgG1E,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yDAAyD;IACzD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAEtE;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAG7E;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAAG,MAAM,CAIxF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE/E;AAED,uDAAuD;AACvD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAIhE;AA0BD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAStE;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,UAAU,CAA2B;IAC7C,8EAA8E;IAC9E,OAAO,CAAC,WAAW,CAAuB;IAE1C,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,YAAY,CAAsC;IAC1D,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,kBAAkB,CAAiF;IAC3G,OAAO,CAAC,cAAc,CAAuE;IAC7F,OAAO,CAAC,UAAU,CAAwD;IAC1E,OAAO,CAAC,qBAAqB,CAAwF;IAErH;;;;;;OAMG;IACH,OAAO,CAAC,cAAc,CAAwE;IAC9F,OAAO,CAAC,mBAAmB,CAAkF;IAC7G,sEAAsE;IACtE,OAAO,CAAC,iBAAiB,CAAsC;IAC/D,OAAO,CAAC,wBAAwB,CAA+B;IAC/D,OAAO,CAAC,2BAA2B,CAAK;IACxC,wEAAwE;IACxE,OAAO,CAAC,0BAA0B,CAAK;IACvC,kEAAkE;IAClE,OAAO,CAAC,sBAAsB,CAA+B;IAC7D,yEAAyE;IACzE,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,aAAa,CAAkE;IACvF,OAAO,CAAC,aAAa,CAAkE;IACvF,OAAO,CAAC,EAAE,CAAwC;IAElD,uEAAuE;IACvE,IAAI,YAAY,IAAI,aAAa,CAAoB;IAE/C,KAAK,CAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,kBAAkB,EAAE,YAAY,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CA4N/G;YAMa,kBAAkB;YA8PlB,iBAAiB;YAocjB,kBAAkB;YA8BlB,kBAAkB;YA4BlB,mBAAmB;IA4IjC;;;;;;;;OAQG;YACW,yBAAyB;IAqGvC;;;;;;;;;OASG;YACW,qBAAqB;YAiBrB,wBAAwB;IA+BtC,OAAO,CAAC,gBAAgB;IAuBxB;;;;;;;;;OASG;YACW,SAAS;YA8HT,mBAAmB;YAmDnB,uBAAuB;IA+ErC;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,wBAAwB;IA8LtC;;;;;;OAMG;IACH,OAAO,CAAC,4BAA4B;IAuBpC;;;;;;OAMG;IACH,OAAO,CAAC,uBAAuB;IAc/B,OAAO,CAAC,sBAAsB;IAM9B,gEAAgE;IAChE,OAAO,CAAC,2BAA2B;IAWnC,0EAA0E;IAC1E,OAAO,CAAC,uBAAuB;YAsBjB,qBAAqB;IA+F7B,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CActF;IAED;;;;;;;OAOG;IACH,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAE/C;IAED,YAAY,IAAI,IAAI,CAKnB;IAED;;;;;;;;;;;;;;OAcG;IACH,cAAc,IAAI;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,QAAQ,EAAE,OAAO,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClD,CAgBA;YAEa,SAAS;IASvB;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED;;;;;;OAMG;YACW,iBAAiB;IAyD/B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,qBAAqB;CAkN9B"}