@omnitron-dev/omnitron 0.1.3 → 0.2.0

package/dist/orchestrator/orchestrator.service.js

@@ -23,7 +23,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { PoolStrategy, SupervisionStrategy, RestartDecision, } from '@omnitron-dev/titan-pm';
+import { PoolStrategy, SupervisionStrategy, } from '@omnitron-dev/titan-pm';
 import { EventEmitter } from '@omnitron-dev/eventemitter';
 import { AppHandle } from './app-handle.js';
 import { launchClassic } from './classic-launcher.js';
@@ -32,6 +32,7 @@ import { buildRestartPolicy } from '../supervisor/restart-policy.js';
 import { ServiceRouter } from './service-router.js';
 import { loadBootstrapConfig, clearCacheFor } from './bootstrap-loader.js';
 import { BuildService } from './build-service.js';
+import { ProcessJanitor } from './process-janitor.js';
 import { CLI_VERSION } from '../config/defaults.js';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 // Resolve bootstrap-process path — .ts in dev (tsx), .js when compiled
@@ -55,11 +56,38 @@ function mapStrategy(strategy) {
             return SupervisionStrategy.ONE_FOR_ONE;
     }
 }
+/**
+ * Strip the `${project}/${stack}/` prefix from a handle key (or
+ * leave a bare name untouched). Used to identify "the same physical
+ * app under any registration form" so duplicate registrations from
+ * stack mode + bare-name RPC paths can be deduplicated.
+ *
+ * Examples:
+ *   effectiveAppName('omni/dev/paysys') === 'paysys'
+ *   effectiveAppName('paysys')           === 'paysys'
+ *   effectiveAppName('a/b/c/paysys')     === 'paysys'  (last segment)
+ */
+function effectiveAppName(key) {
+    const idx = key.lastIndexOf('/');
+    return idx >= 0 ? key.slice(idx + 1) : key;
+}
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 export class OrchestratorService extends EventEmitter {
     logger;
     pm;
     stateStore;
     handles = new Map();
+    /**
+     * Per-name in-flight start lock (T#57). When two `startApp()` calls
+     * arrive concurrently for the same app, the first one's promise is
+     * stored here; the second one awaits and returns the same handle
+     * rather than racing through to a parallel `launchBootstrapMode` /
+     * `launchClassicMode` that would spawn a second worker pool and
+     * leak the original. The map is cleared in a `finally` once the
+     * launch settles (success or failure) so the next legitimate call
+     * for the same name isn't permanently locked out.
+     */
+    startingApps = new Map();
     config = null;
     cwd;
     metricsTimer = null;
@@ -70,16 +98,71 @@ export class OrchestratorService extends EventEmitter {
     buildService = null;
     /** Cached build results per app name */
     buildResults = new Map();
+    /**
+     * Per-app restart coalescer state.
+     *
+     * `inFlight` — the currently-running restart promise (so a concurrent
+     *   trigger doesn't kick off a parallel restart and tear down the half-
+     *   started state from the previous one).
+     * `triggerSerial` — monotonically incremented by external trigger
+     *   entry points (BuildService onRebuild, RPC restartApp, manual
+     *   `omnitron restart`, deploy.service). Internal coalescer arms
+     *   that queue themselves DO NOT bump it.
+     * `firedSerial` — value of `triggerSerial` that the most recently
+     *   completed restart consumed. A trailing coalesced restart only
+     *   fires when `triggerSerial > firedSerial`, i.e. a genuinely new
+     *   external trigger arrived during the in-flight window. Two
+     *   coalescer arms colliding can no longer cause a self-perpetuating
+     *   loop because neither bumps `triggerSerial`.
+     * `cooldownUntil` — absolute epoch ms timestamp. Any trigger received
+     *   before this clears is queued instead of fired immediately.
+     *   Prevents the storm where esbuild's onEnd fires for a freshly-
+     *   spawned context milliseconds after the app reports online.
+     */
+    restartCoalescer = new Map();
+    /**
+     * Wall-clock window in ms during which restart triggers from
+     * BuildService.watchApp are deferred after `app:online`. Sized so a
+     * normal bundle build + child spawn + DI bootstrap fits comfortably
+     * inside; anything legitimately changing more often than this is a
+     * developer typing very fast, which the regular debounce handles.
+     */
+    static POST_ONLINE_COOLDOWN_MS = 3_000;
+    /**
+     * Orphan-process reaper. Lazily created on first `startAll` so unit
+     * tests that hit `startApp` directly don't get a periodic timer.
+     */
+    janitor = null;
     /** Daemon's Netron instance for native service routing */
     daemonNetron = null;
     /** Daemon Unix socket path for child-to-daemon Netron connections */
     daemonSocketPath = null;
+    /**
+     * Promise that resolves once the daemon's Netron is wired up. Apps that
+     * declare topology cannot launch until this is ready — instead of failing
+     * with "daemon Netron not available" the launcher awaits this promise
+     * (with a generous timeout) so transient startup-order interleavings
+     * resolve themselves instead of becoming user-visible errors.
+     */
+    daemonNetronReady;
+    resolveDaemonNetronReady = () => undefined;
+    /**
+     * Optional metrics sink. Set by the daemon at boot via
+     * `setMetricsBridge` so the orchestrator can forward janitor sweeps,
+     * esbuild rebuilds, and lifecycle events without coupling itself to
+     * the metrics module. Null means metrics are disabled (tests, slim
+     * bootstraps).
+     */
+    metricsBridge = null;
     constructor(logger, pm, stateStore, cwd) {
         super();
         this.logger = logger;
         this.pm = pm;
         this.stateStore = stateStore;
         this.cwd = cwd ?? process.cwd();
+        this.daemonNetronReady = new Promise((resolve) => {
+            this.resolveDaemonNetronReady = resolve;
+        });
     }
     /**
      * Set the daemon Netron instance and socket path for native topology routing.
@@ -88,28 +171,155 @@ export class OrchestratorService extends EventEmitter {
     setDaemonNetron(netron, socketPath) {
         this.daemonNetron = netron;
         this.daemonSocketPath = socketPath;
+        this.resolveDaemonNetronReady();
         this.logger.info({ socketPath }, 'Daemon Netron set for native topology routing');
     }
+    /**
+     * Wire the metrics bridge. Called once during daemon bootstrap.
+     * After this, janitor sweeps, esbuild rebuilds, and orchestrator
+     * lifecycle events are forwarded to Prometheus.
+     */
+    setMetricsBridge(bridge) {
+        this.metricsBridge = bridge;
+        bridge.attachOrchestrator(this);
+        this.wireOrchestratorTelemetry();
+    }
+    /**
+     * Subscribe to our own EventEmitter to forward state-change events to
+     * the metrics bridge. Idempotent so repeated `setMetricsBridge` calls
+     * (mostly tests) don't multiply listeners.
+     */
+    telemetryWired = false;
+    wireOrchestratorTelemetry() {
+        if (this.telemetryWired)
+            return;
+        this.telemetryWired = true;
+        this.on('app:restart', (app, attempt) => {
+            this.metricsBridge?.recordAppRestart(app, attempt, 'crash');
+        });
+        this.on('app:crash', (app) => {
+            this.metricsBridge?.recordAppCrash(app);
+        });
+        this.on('app:escalated', (app) => {
+            this.metricsBridge?.recordAppEscalated(app);
+        });
+    }
+    /**
+     * Wait until the daemon Netron is available, with a hard ceiling so we
+     * never hang forever if `setDaemonNetron()` was never wired in.
+     */
+    async ensureDaemonNetronReady(timeoutMs = 30_000) {
+        if (this.daemonNetron && this.daemonSocketPath)
+            return;
+        const timeout = new Promise((_resolve, reject) => {
+            const timer = setTimeout(() => reject(new Error(`Daemon Netron not initialized within ${timeoutMs}ms — wiring bug?`)), timeoutMs);
+            timer.unref?.();
+        });
+        await Promise.race([this.daemonNetronReady, timeout]);
+    }
     // ============================================================================
     // Lifecycle
     // ============================================================================
     async startAll(config) {
         this.config = config;
+        // Reap any fork-workers left over from a previous daemon BEFORE
+        // launching apps. Without this, a hard-killed daemon's children
+        // are still alive (now adopted by init); they hold DB connections
+        // and sockets that the new apps will fight for.
+        await this.ensureJanitor().coldStartSweep();
+        // Dependency-aware skip: if app A fails to start, every app that
+        // declares A in `dependsOn` (transitively) is skipped instead of
+        // attempted. Without this, a downstream app would launch, fail to
+        // reach its missing dependency, crash, and chew through its restart
+        // budget — masking the real root cause.
+        const failed = new Set();
+        const blocked = new Set();
         for (const batch of resolveStartupOrder(config.apps)) {
             const enabled = batch.filter((e) => e.enabled !== false);
             await Promise.all(enabled.map(async (e) => {
+                const blockingDep = (e.dependsOn ?? []).find((dep) => failed.has(dep) || blocked.has(dep));
+                if (blockingDep) {
+                    blocked.add(e.name);
+                    this.logger.warn({ app: e.name, blockedBy: blockingDep }, 'Skipping app startup — depends on a failed/blocked app');
+                    return;
+                }
                 try {
                     await this.startApp(e, config);
                 }
                 catch (err) {
+                    failed.add(e.name);
                     this.logger.error({ app: e.name, error: err.message }, 'Failed to start app — continuing with remaining apps');
                 }
             }));
         }
+        if (failed.size > 0 || blocked.size > 0) {
+            this.logger.warn({ failed: [...failed], blocked: [...blocked] }, 'Startup completed with failures — some apps did not launch');
+        }
+        // Periodic sweep handles runtime leaks from partially-failed
+        // restart cycles (esbuild-rebuild → spawn-fail → orphan).
+        this.ensureJanitor().start();
         this.startMetricsPolling(config.monitoring.metrics.interval);
         this.persistState();
     }
+    /**
+     * Lazily create the orphan-process janitor with the live "owned
+     * pids" snapshot wired to the orchestrator's handle map.
+     */
+    ensureJanitor() {
+        if (this.janitor)
+            return this.janitor;
+        this.janitor = new ProcessJanitor({
+            logger: this.logger,
+            getOwnedPids: () => this.collectOwnedPids(),
+            onMetrics: (m) => this.metricsBridge?.recordJanitorSweep(m),
+        });
+        return this.janitor;
+    }
+    /**
+     * Walk every live handle's supervisor children and collect the OS
+     * pids the orchestrator currently considers ours. Cheap — called
+     * every janitor sweep.
+     */
+    collectOwnedPids() {
+        const owned = new Set();
+        for (const handle of this.handles.values()) {
+            const supervisor = handle.supervisor;
+            if (!supervisor)
+                continue;
+            for (const childName of supervisor.getChildNames()) {
+                const processId = supervisor.getChildProcessId(childName);
+                if (!processId)
+                    continue;
+                const workerHandle = this.pm.getWorkerHandle(processId);
+                const pid = workerHandle?.pid;
+                if (typeof pid === 'number')
+                    owned.add(pid);
+            }
+        }
+        return owned;
+    }
     async startApp(entry, config) {
+        // T#57: coalesce concurrent calls for the same app name. Without
+        // this, two near-simultaneous `startApp({ name: 'main' })` calls
+        // both see `this.handles.get('main')` as undefined-or-not-online
+        // (the first one's handle is in status='starting'), both fall
+        // through to `launchBootstrapMode`, and we spawn two parallel
+        // worker pools competing for the same port. Promise coalescing
+        // — return the in-flight promise to the second caller — gives
+        // both callers the same handle and ensures only one launch
+        // actually runs.
+        const inflight = this.startingApps.get(entry.name);
+        if (inflight) {
+            this.logger.debug({ app: entry.name }, 'Coalescing concurrent startApp into in-flight launch (T#57)');
+            return inflight;
+        }
+        const promise = this.startAppInternal(entry, config).finally(() => {
+            this.startingApps.delete(entry.name);
+        });
+        this.startingApps.set(entry.name, promise);
+        return promise;
+    }
+    async startAppInternal(entry, config) {
         // Persist ecosystem config for restartApp — stack mode passes config per-app,
         // but restartApp calls startApp without it.
         if (config && !this.config) {
@@ -118,6 +328,41 @@ export class OrchestratorService extends EventEmitter {
         const cfg = config ?? this.config;
         if (!cfg)
             throw new Error('Ecosystem config not loaded');
+        // Deduplicate by effective app identity. Project-mode registers
+        // apps as `${project}/${stack}/${app}`; non-project paths use the
+        // bare `${app}` short name. They refer to the SAME physical app
+        // and must never coexist as separate handles — the duplicate
+        // would compete for the port and the orphan-reaper's owned-pid
+        // accounting would split between them.
+        //
+        // Policy:
+        //   - exact key already exists, online   → return it (idempotent)
+        //   - exact key already exists, not online → fall through and re-launch
+        //     under the same key
+        //   - DIFFERENT key with the same effective short name → reject:
+        //     operator should restart by canonical name, not register an
+        //     alias.
+        const effective = effectiveAppName(entry.name);
+        for (const [key, h] of this.handles) {
+            if (key === entry.name)
+                continue;
+            if (effectiveAppName(key) !== effective)
+                continue;
+            // Same effective app under a different key.
+            if (h.status === 'online') {
+                this.logger.warn({ existing: key, attempted: entry.name, effective }, 'App already running under a canonical name — refusing duplicate registration');
+                return h;
+            }
+            // Existing handle is errored/stopped — replace silently so a
+            // failed previous launch doesn't permanently block the canonical
+            // form.
+            this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Replacing stale duplicate handle');
+            this.handles.delete(key);
+            // Drop any metrics that were collected under the stale alias so it
+            // doesn't haunt future snapshots as a ghost offline app.
+            if (key !== entry.name)
+                this.metricsBridge?.evictApp(key);
+        }
         const existing = this.handles.get(entry.name);
         if (existing && existing.status === 'online') {
             this.logger.warn({ app: entry.name }, 'App already running');
@@ -145,7 +390,8 @@ export class OrchestratorService extends EventEmitter {
         }
     }
     async stopApp(name, force = false, timeout = 10_000) {
-        const handle = this.handles.get(name);
+        const canonical = this.resolveAppName(name) ?? name;
+        const handle = this.handles.get(canonical);
         if (!handle || handle.status === 'stopped')
             return;
         handle.markStopping();
@@ -178,6 +424,11 @@ export class OrchestratorService extends EventEmitter {
             clearInterval(this.metricsTimer);
             this.metricsTimer = null;
         }
+        // Stop the janitor first so it doesn't race the controlled
+        // shutdown by trying to "reap" children that are mid-stop.
+        if (this.janitor) {
+            this.janitor.stop();
+        }
         let count = 0;
         // When force=true (daemon shutdown), use short timeout per app (3s SIGTERM, then SIGKILL)
         const perAppTimeout = force ? 3_000 : 10_000;
@@ -209,36 +460,171 @@ export class OrchestratorService extends EventEmitter {
         this.buildResults.clear();
         return count;
     }
+    /**
+     * Restart an app, coalescing concurrent triggers.
+     *
+     * The dev-mode hot-reload path can deliver many restart triggers per
+     * second (file save → esbuild rebuild → onEnd → debounced → restart),
+     * and naive parallelism here ends in cross-stomping state mutations
+     * (one restart's stopApp tearing down the other's freshly-spawned
+     * child). This wrapper serialises restarts per app:
+     *
+     *   - If a restart is already in flight, we set the `pending` bit and
+     *     await the current restart. When it completes we run exactly one
+     *     follow-up — extra triggers received in the meantime collapse onto
+     *     the same bit, so a burst of N triggers produces at most 2 actual
+     *     restarts (the current one + one trailing).
+     *   - If we're inside the post-online cooldown window, the trigger is
+     *     queued (via `pending`) and fires after the cooldown clears.
+     *
+     * The hot path is `restartAppNow()` which contains the actual work and
+     * is what tests / non-coalescing callers can hit directly.
+     */
     async restartApp(name) {
+        // External entry — record one new trigger and dispatch.
+        return this.restartAppCoalesced(name, /* fromExternal */ true);
+    }
+    /**
+     * Internal coalescer dispatcher.
+     *
+     * `fromExternal=true` bumps `triggerSerial` (a user / file-watcher /
+     * RPC asked us to restart). `fromExternal=false` is used by the
+     * recursive arms (cooldown-wait, await-inflight) which are NOT new
+     * triggers, just continuations — they must not bump or the trailing
+     * coalesced restart would re-fire forever.
+     */
+    async restartAppCoalesced(name, fromExternal) {
+        const canonical = this.resolveAppName(name) ?? name;
+        const handle = this.handles.get(canonical);
+        if (!handle)
+            throw new Error(`Unknown app: ${name}`);
+        name = canonical;
+        const state = this.getCoalescerState(name);
+        if (fromExternal)
+            state.triggerSerial += 1;
+        const observedSerial = state.triggerSerial;
+        // Cooldown: the app reported online recently. Wait it out, then
+        // proceed as a non-external continuation (we already counted this
+        // trigger when we first arrived).
+        const now = Date.now();
+        if (now < state.cooldownUntil) {
+            const waitMs = state.cooldownUntil - now;
+            this.logger.debug({ app: name, cooldownRemainingMs: waitMs }, 'Restart trigger received during post-online cooldown — queued');
+            await sleep(waitMs);
+            return this.restartAppCoalesced(name, /* fromExternal */ false);
+        }
+        // Already restarting — wait for it, then re-enter as a continuation.
+        // The trailing-fire gate at the end of the in-flight finally block
+        // is what re-fires us if our serial outranks what got consumed.
+        if (state.inFlight) {
+            await state.inFlight.catch(() => { });
+            // If the in-flight restart already consumed our serial (or a
+            // later one), we have nothing new to do — return its result by
+            // re-reading the current handle. Otherwise fall through and
+            // claim the slot ourselves.
+            if (state.firedSerial >= observedSerial) {
+                const refreshed = this.handles.get(name);
+                if (refreshed)
+                    return refreshed;
+            }
+            return this.restartAppCoalesced(name, /* fromExternal */ false);
+        }
+        // Take the slot. Capture which serial this run is consuming —
+        // anything bumped after this is the next run's problem.
+        const consumedSerial = state.triggerSerial;
+        const promise = this.restartAppNow(name);
+        state.inFlight = promise;
+        try {
+            // Cooldown window is started from the `app:online` transition
+            // inside startApp(), not here, so the timer is tied to the moment
+            // the new child reports ready, not the moment our promise resolves.
+            return await promise;
+        }
+        finally {
+            state.inFlight = null;
+            state.firedSerial = consumedSerial;
+            // Trailing coalesced fire — only if a GENUINELY NEW external
+            // trigger arrived between when we claimed the slot and now.
+            // Two coalescer arms colliding can no longer cause this branch
+            // to fire because they don't bump triggerSerial.
+            if (state.triggerSerial > consumedSerial) {
+                this.restartAppCoalesced(name, /* fromExternal */ false).catch((err) => {
+                    this.logger.error({ app: name, error: err.message }, 'Trailing coalesced restart failed');
+                });
+            }
+        }
+    }
+    /**
+     * Internal restart hot path — assumes coalescer has cleared us to proceed.
+     *
+     * Architecture note (Fix #247): we deliberately do NOT tear down the
+     * esbuild watch or invalidate buildResults here. The watch is what
+     * triggered us in the first place — its bundle is already fresh, and
+     * disposing the context creates a window where the next save can be
+     * missed (esbuild's metafile doesn't carry over between contexts).
+     *
+     * `launchBootstrapMode` sees the live buildResult + watch and skips
+     * the cold rebuild, so a restart cycle is just: kill old child →
+     * spawn new child with the bundle that was already on disk. The
+     * bootstrap-config cache is cleared because changes to `bootstrap.ts`
+     * (topology definition) need to take effect on the next start.
+     */
+    async restartAppNow(name) {
         const handle = this.handles.get(name);
         if (!handle)
             throw new Error(`Unknown app: ${name}`);
         const entry = handle.entry;
-        // In dev mode, clear bootstrap config cache so topology changes are picked up
         if (this.devMode && entry.bootstrap) {
-            clearCacheFor(path.resolve(this.cwd, entry.bootstrap));
-            // Stop esbuild watch and clear stale build result — launchBootstrapMode will rebuild + re-watch
-            if (this.buildService) {
-                await this.buildService.unwatchApp(name);
-            }
-            this.buildResults.delete(name);
+            clearCacheFor(path.resolve(entry.cwd ?? this.cwd, entry.bootstrap));
         }
         await this.stopApp(name);
         return this.startApp(entry);
     }
+    getCoalescerState(name) {
+        let state = this.restartCoalescer.get(name);
+        if (!state) {
+            state = { inFlight: null, triggerSerial: 0, firedSerial: 0, cooldownUntil: 0 };
+            this.restartCoalescer.set(name, state);
+        }
+        return state;
+    }
     /**
-     * Resolve raw app name to its handle key.
-     * In stack mode, handles are namespaced (e.g., 'omni/dev/main' for app 'main').
-     * This finds the handle whose key ends with the raw app name.
+     * Start the post-online cooldown for an app. Called from the `app:online`
+     * transition (both initial startApp and subsequent restartApp paths
+     * funnel through this), so a freshly-online app cannot be torn down by
+     * a rebuild signal that was already in flight when it finished booting.
+     */
+    startPostOnlineCooldown(name) {
+        const state = this.getCoalescerState(name);
+        state.cooldownUntil = Date.now() + OrchestratorService.POST_ONLINE_COOLDOWN_MS;
+    }
+    /**
+     * Resolve a user-supplied name to a canonical handle key.
+     *
+     * Inputs accepted:
+     *   - exact canonical key (`omni/dev/paysys`) → returned as-is
+     *   - bare short name (`paysys`) → returns the canonical key whose
+     *     last `/`-segment matches, IF AND ONLY IF the match is unique.
+     *
+     * Returns `undefined` for no match. Throws on ambiguity (multiple
+     * stacks expose the same short name) so the caller can surface a
+     * clear error to the operator instead of operating on the wrong
+     * app.
      */
     resolveAppName(rawName) {
         if (this.handles.has(rawName))
             return rawName;
+        const effective = effectiveAppName(rawName);
+        const matches = [];
         for (const key of this.handles.keys()) {
-            if (key.endsWith(`/${rawName}`))
-                return key;
+            if (effectiveAppName(key) === effective)
+                matches.push(key);
         }
-        return undefined;
+        if (matches.length === 0)
+            return undefined;
+        if (matches.length === 1)
+            return matches[0];
+        throw new Error(`App name '${rawName}' is ambiguous — multiple matches: ${matches.join(', ')}. Use the full canonical name.`);
     }
     /**
      * Reload app with zero-downtime strategy:
@@ -251,9 +637,11 @@ export class OrchestratorService extends EventEmitter {
      * zero-downtime is not possible (single-process, no supervisor).
      */
     async reloadApp(name) {
-        const handle = this.handles.get(name);
+        const canonical = this.resolveAppName(name) ?? name;
+        const handle = this.handles.get(canonical);
         if (!handle)
             throw new Error(`Unknown app: ${name}`);
+        name = canonical;
         // Classic mode or no supervisor — cannot do zero-downtime, fall back to restart
         if (handle.mode !== 'bootstrap' || !handle.supervisor) {
             return this.restartApp(name);
@@ -313,9 +701,11 @@ export class OrchestratorService extends EventEmitter {
     // Scaling — delegates to PM supervisor
     // ============================================================================
     async scaleApp(name, instances) {
-        const handle = this.handles.get(name);
+        const canonical = this.resolveAppName(name) ?? name;
+        const handle = this.handles.get(canonical);
         if (!handle)
             throw new Error(`Unknown app: ${name}`);
+        name = canonical;
         if (!this.config)
             throw new Error('Ecosystem config not loaded');
         if (handle.mode !== 'bootstrap') {
@@ -345,11 +735,18 @@ export class OrchestratorService extends EventEmitter {
         return handles.map((h) => this.toProcessInfo(h));
     }
     getApp(name) {
-        const h = this.handles.get(name);
+        const canonical = this.resolveAppName(name) ?? name;
+        const h = this.handles.get(canonical);
         return h ? this.toProcessInfo(h) : null;
     }
     getHandle(name) {
-        return this.handles.get(name);
+        // Match getAppStatus / getLogs name-resolution semantics — accept either
+        // a fully namespaced handle key ('project/stack/app') or the bare app
+        // name and resolve to the right handle. Without this, CLI commands like
+        // `omnitron env main` fail with "Unknown app: main" even though the
+        // app exists as `omni/dev/main`.
+        const resolved = this.resolveAppName(name);
+        return this.handles.get(resolved ?? name);
     }
     /**
      * List all handle names matching a prefix.
@@ -367,6 +764,35 @@ export class OrchestratorService extends EventEmitter {
     // ============================================================================
     // Monitoring — async, delegates to PM supervisor for bootstrap mode
     // ============================================================================
+    /**
+     * Pull the live DI dependency graph from a running app's first child
+     * worker. Used by the daemon RPC backing `omnitron inspect <app>
+     * --graph`. Returns null if the app isn't running, has no children,
+     * or the worker doesn't expose `getDependencyGraph` (older bootstraps).
+     */
+    async getDependencyGraph(name) {
+        const handle = this.getHandle(name);
+        if (!handle || handle.mode !== 'bootstrap' || !handle.supervisor)
+            return null;
+        const childNames = handle.supervisor.getChildNames();
+        for (const childName of childNames) {
+            const processId = handle.supervisor.getChildProcessId(childName);
+            if (!processId)
+                continue;
+            const workerHandle = this.pm.getWorkerHandle(processId);
+            const proxy = workerHandle?.proxy;
+            if (!proxy?.getDependencyGraph)
+                continue;
+            try {
+                const graph = await proxy.getDependencyGraph();
+                return graph;
+            }
+            catch (err) {
+                this.logger.warn({ app: name, child: childName, error: err.message }, 'getDependencyGraph: child threw — trying next worker');
+            }
+        }
+        return null;
+    }
     async getMetrics(name) {
         const result = {};
         for (const [appName, handle] of this.handles) {
@@ -488,8 +914,13 @@ export class OrchestratorService extends EventEmitter {
     }
     getLogs(name, lines = 100) {
         const result = [];
+        // Stack-mode handles are keyed as `<project>/<stack>/<app>` while CLI
+        // users typically pass just the bare app name. Resolve via the same
+        // matcher used by getAppStatus() so `omnitron logs main` works in
+        // both single-app and stack modes.
+        const resolved = name ? this.resolveAppName(name) ?? name : undefined;
         for (const [appName, handle] of this.handles) {
-            if (name && appName !== name)
+            if (resolved && appName !== resolved)
                 continue;
             result.push({ app: appName, lines: handle.getLogs(lines) });
         }
@@ -535,7 +966,7 @@ export class OrchestratorService extends EventEmitter {
      */
     async launchBootstrapMode(entry, handle, config) {
         handle.markStarting();
-        const bootstrapAbsPath = path.resolve(this.cwd, entry.bootstrap);
+        const bootstrapAbsPath = path.resolve(entry.cwd ?? this.cwd, entry.bootstrap);
         // Load bootstrap definition to check for process topology and requirements
         let topology;
         let definition;
@@ -588,25 +1019,92 @@ export class OrchestratorService extends EventEmitter {
             }
         }
         // Build all entry points via esbuild (dev mode bundles from source .ts)
+        //
+        // On a *restart* (as opposed to a first start), buildResults already
+        // has a fresh entry — esbuild's watcher just produced the bundle
+        // that triggered us. Reusing it skips a redundant cold rebuild
+        // (which was costing ~500ms per restart and racing against the
+        // watcher's own incremental output). The watch context is also
+        // preserved (we never unwatched), so we don't re-arm it either.
         if (definition && this.devMode) {
             try {
                 if (!this.buildService) {
-                    this.buildService = new BuildService(/* isDev */ true);
+                    this.buildService = new BuildService(/* isDev */ true, {
+                        onEsbuildReset: (info) => this.metricsBridge?.recordResilientHandleReset(info),
+                        onBuildResult: ({ app, kind, durationMs, ok }) => this.metricsBridge?.recordEsbuildBuild(app, kind, durationMs, ok),
+                    });
                 }
-                const buildResult = await this.buildService.buildApp(entry.name, bootstrapAbsPath, definition);
-                this.buildResults.set(entry.name, buildResult);
-                this.logger.info({ app: entry.name, modules: buildResult.modulePaths.size }, 'Built entry points via esbuild');
-                // Start esbuild watch for incremental rebuilds on source changes.
-                // On rebuild → restart the app (stop + re-build + re-launch).
-                await this.buildService.watchApp(entry.name, bootstrapAbsPath, definition, () => {
-                    this.logger.info({ app: entry.name }, 'esbuild rebuild detected — restarting');
-                    this.restartApp(entry.name).catch((err) => {
-                        this.logger.error({ app: entry.name, error: err.message }, 'Restart after esbuild rebuild failed');
+                const existing = this.buildResults.get(entry.name);
+                const existingWatch = this.buildService.isWatching?.(entry.name);
+                // Bundle on-disk presence check. A restart can race against
+                // the user (or a script) removing `.omnitron-build/` between
+                // builds — without this stat the orchestrator hands the
+                // worker a path that no longer exists and the child dies
+                // with `ERR_MODULE_NOT_FOUND` before the supervisor can
+                // recover. Cheap stat, big DX win.
+                const bundlePresent = existing
+                    ? fs.existsSync(existing.bootstrapPath)
+                    : false;
+                if (existing && existingWatch && bundlePresent) {
+                    // Restart path — keep the already-built bundle and live watch.
+                    this.logger.debug({ app: entry.name, modules: existing.modulePaths.size }, 'Reusing existing esbuild bundle + watch (restart path)');
+                }
+                else {
+                    if (existing && existingWatch && !bundlePresent) {
+                        // Bundle was deleted out from under us. Clear the watch
+                        // and the cached BuildResult so the next call rebuilds
+                        // from cold and re-arms the watcher cleanly.
+                        this.logger.warn({ app: entry.name, bootstrapPath: existing.bootstrapPath }, 'Cached bundle missing on disk — forcing rebuild');
+                        try {
+                            await this.buildService.unwatchApp?.(entry.name);
+                        }
+                        catch (err) {
+                            this.logger.debug({ err, app: entry.name }, 'unwatchApp failed (non-fatal)');
+                        }
+                        this.buildResults.delete(entry.name);
+                    }
+                    // First-start path — build everything from cold and arm the watcher.
+                    const buildResult = await this.buildService.buildApp(entry.name, bootstrapAbsPath, definition);
+                    this.buildResults.set(entry.name, buildResult);
+                    this.logger.info({ app: entry.name, modules: buildResult.modulePaths.size }, 'Built entry points via esbuild');
+                    // Start esbuild watch for incremental rebuilds on source changes.
+                    // On rebuild → fire a restart trigger; the coalescer takes it
+                    // from there.
+                    await this.buildService.watchApp(entry.name, bootstrapAbsPath, definition, () => {
+                        // Gate: skip if the app is in a non-running state — repeated
+                        // restarts of an escalated/errored process just churn forever
+                        // without forward progress (Fix #248).
+                        // Skip restart when the app is in a terminal non-running
+                        // state. Includes 'crashed' (supervisor escalated after
+                        // exceeding its restart budget) and 'errored' (start
+                        // failed). 'stopped' means operator explicitly stopped it.
+                        // In all three cases, restarting just churns a known-broken
+                        // process; the operator must fix the underlying issue and
+                        // call `omnitron start <app>` to clear the state.
+                        const status = this.getAppStatus(entry.name);
+                        if (status === 'errored' || status === 'crashed' || status === 'stopped') {
+                            this.logger.warn({ app: entry.name, status }, 'esbuild rebuild detected but app is not running — skipping restart');
+                            return;
+                        }
+                        this.logger.info({ app: entry.name }, 'esbuild rebuild detected — restarting');
+                        this.metricsBridge?.recordEsbuildBuild(entry.name, 'rebuild', 0, true);
+                        this.restartApp(entry.name).catch((err) => {
+                            this.logger.error({ app: entry.name, error: err.message }, 'Restart after esbuild rebuild failed');
+                        });
                     });
-                });
+                }
             }
             catch (err) {
-                this.logger.error({ app: entry.name, error: err.message }, 'esbuild failed — child processes will fall back to tsx');
+                // The "fall back to tsx" wording was misleading — the orchestrator
+                // does not inject a tsx loader into spawned children, so without
+                // a build the bootstrap-process will try to `import` the source
+                // .ts file (referenced as `./app.module.js` in the topology) and
+                // fail with a cryptic "Cannot find module" downstream. Surface
+                // the build error directly so the operator sees the actual
+                // problem on `omnitron start/restart`.
+                this.logger.error({ app: entry.name, error: err.message }, 'esbuild build failed — app cannot start without bundled output');
+                throw new Error(`Cannot launch '${entry.name}' in dev mode: esbuild build failed (${err.message}). ` +
+                    `Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build/.`);
             }
         }
         const buildResult = this.buildResults.get(entry.name);
@@ -634,10 +1132,11 @@ export class OrchestratorService extends EventEmitter {
      */
     async launchTopology(entry, handle, config, bootstrapAbsPath, topology, buildResult) {
         const policy = buildRestartPolicy(entry, config);
-        if (!this.daemonNetron || !this.daemonSocketPath) {
-            throw new Error(`Cannot launch topology for '${entry.name}': daemon Netron not available. ` +
-                'Ensure setDaemonNetron() is called before starting apps.');
-        }
+        // Wait for daemon Netron to be wired up. Cheap no-op if already ready;
+        // otherwise blocks (with timeout) instead of immediately failing — the
+        // race between daemon startup and stack auto-resume is benign and just
+        // needs a single rendezvous.
+        await this.ensureDaemonNetronReady();
         const serviceRouter = new ServiceRouter(this.daemonNetron, this.logger);
         const daemonSocketUrl = `unix://${this.daemonSocketPath}`;
         // Topology startup order: providers (topology.expose) before consumers (topology.access).
@@ -712,6 +1211,12 @@ export class OrchestratorService extends EventEmitter {
             // pm.pool() returns a Proxy wrapping ProcessPool — execute() dispatches to pool workers
             const pool = await this.pm.pool(BOOTSTRAP_PROCESS_PATH, poolOptions);
             poolNames.push(procEntry.name);
+            // Track the pool on the AppHandle so `getInfo()` can report the
+            // actual worker count for pool-managed topology entries. Without
+            // this the orchestrator falls back to `supervisor.getChildProcessId()`
+            // which returns null for pool workers (they aren't supervisor
+            // children) and the entry is mis-reported as `status: 'stopped'`.
+            handle.topologyPools.set(procEntry.name, pool);
             // Auto-discover @Service metadata from pool workers and expose via ServiceRouter.
             // No manual `exports` needed — @Service decorator IS the source of truth.
             // Requires explicit `topology: { expose: true }` — opt-in for security.
@@ -781,11 +1286,12 @@ export class OrchestratorService extends EventEmitter {
             window: policy.window ?? config.supervision.window,
             backoff: policy.backoff ?? config.supervision.backoff,
             children,
-            onChildCrash: async (_child, _error) => {
-                // Always restart — supervisor's maxRestarts/window handles escalation.
-                // Critical flag determines whether exceeding max restarts kills the app.
-                return RestartDecision.RESTART;
-            },
+            // No custom onChildCrash — supervisor's built-in budget gate
+            // (process-supervisor.ts:524-531) does what we want:
+            //   * within maxRestarts/window → RESTART
+            //   * over budget → ESCALATE (if child is critical) or IGNORE
+            // Wiring an unconditional RESTART here masked the budget and
+            // turned every crashing child into an infinite respawn storm.
         };
         // Create supervisor WITHOUT starting — wire events first to avoid race condition
         const supervisor = this.pm.createSupervisor(supervisorConfig);
@@ -798,6 +1304,7 @@ export class OrchestratorService extends EventEmitter {
                 handle.markOnline(childPid ?? process.pid);
             }
             this.persistState();
+            this.startPostOnlineCooldown(entry.name);
             this.emit('app:online', entry.name);
         });
         // Set supervisor on handle BEFORE start so attachLogCapture can access it
@@ -876,11 +1383,12 @@ export class OrchestratorService extends EventEmitter {
             window: policy.window ?? config.supervision.window,
             backoff: policy.backoff ?? config.supervision.backoff,
             children: [childConfig],
-            onChildCrash: async (_child, _error) => {
-                // Always restart — supervisor's maxRestarts/window handles escalation.
-                // Critical flag determines whether exceeding max restarts kills the app.
-                return RestartDecision.RESTART;
-            },
+            // No custom onChildCrash — supervisor's built-in budget gate
+            // (process-supervisor.ts:524-531) does what we want:
+            //   * within maxRestarts/window → RESTART
+            //   * over budget → ESCALATE (if child is critical) or IGNORE
+            // Wiring an unconditional RESTART here masked the budget and
+            // turned every crashing child into an infinite respawn storm.
         };
         // Create supervisor WITHOUT starting — wire events first to avoid race condition
         const supervisor = this.pm.createSupervisor(supervisorConfig);
@@ -895,6 +1403,7 @@ export class OrchestratorService extends EventEmitter {
                 handle.markOnline(childPid ?? process.pid);
             }
             this.persistState();
+            this.startPostOnlineCooldown(entry.name);
             this.emit('app:online', entry.name);
         });
         // Set supervisor on handle BEFORE start so attachLogCapture can access it
@@ -1095,19 +1604,41 @@ export class OrchestratorService extends EventEmitter {
         if (handle.topologyProcesses && handle.topologyProcesses.length > 0 && handle.supervisor) {
             info.processes = [];
             for (const topo of handle.topologyProcesses) {
-                const childNames = handle.supervisor.getChildNames();
-                // Match topology entry to supervisor child by name
-                const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
-                const processId = handle.supervisor.getChildProcessId(childName);
+                const isPoolEntry = (topo.instances ?? 1) > 1;
                 let childPid = null;
                 let childStatus = 'stopped';
-                if (processId) {
-                    const wh = this.pm.getWorkerHandle(processId);
-                    childPid = wh?.pid ?? null;
-                    childStatus = childPid ? 'online' : 'stopped';
+                if (isPoolEntry) {
+                    // Pool-managed topology — the supervisor doesn't know about
+                    // these (workers are owned by ProcessPool). Read directly
+                    // from the pool ref we stashed at creation time.
+                    const pool = handle.topologyPools.get(topo.name);
+                    if (pool && pool.size > 0) {
+                        childStatus = 'online';
+                        // Surface the first worker's PID. Pool workers are
+                        // interchangeable so any one is representative; the CLI
+                        // prints "PID" as a quick liveness signal, not a unique
+                        // identifier.
+                        const workerIds = pool.getWorkerIds();
+                        if (workerIds.length > 0) {
+                            const firstId = workerIds[0];
+                            const wh = this.pm.getWorkerHandle(firstId);
+                            childPid = wh?.pid ?? null;
+                        }
+                    }
+                }
+                else {
+                    // Single-instance topology — supervisor manages the child directly.
+                    const childNames = handle.supervisor.getChildNames();
+                    const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
+                    const processId = handle.supervisor.getChildProcessId(childName);
+                    if (processId) {
+                        const wh = this.pm.getWorkerHandle(processId);
+                        childPid = wh?.pid ?? null;
+                        childStatus = childPid ? 'online' : 'stopped';
+                    }
                 }
                 // Derive process type from declarations for backward-compatible DTO
-                const derivedType = topo.transports ? 'server' : (topo.instances ?? 1) > 1 ? 'worker' : 'custom';
+                const derivedType = topo.transports ? 'server' : isPoolEntry ? 'worker' : 'custom';
                 info.processes.push({
                     name: topo.name,
                     type: derivedType,