@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/orchestrator/orchestrator.service.d.ts

@@ -11,7 +11,13 @@
  *
  * Classic mode:
  *   - child_process.fork() with manual exit handling (legacy compat)
- *   - Real CPU/memory metrics via pidusage
+ *
+ * Metrics for both modes come from one `ps -o pid=,rss=,%cpu=` call covering
+ * every managed pid — supervisor children and pool workers alike. This line
+ * used to say "real CPU/memory metrics via pidusage"; `pidusage` is not a
+ * dependency of this package and appears nowhere else in the source. The
+ * sentence outlived the code it described, in the one area where a stale
+ * claim is hardest to catch: metrics that are wrong still look like metrics.
  *
  * Architecture:
  *   pm.supervisor({ children: [{ process: bootstrap-process.js, ... }] })
@@ -28,11 +34,72 @@ import type { IEcosystemConfig, IEcosystemAppEntry, ProcessInfoDto, AppStatus }
 import { AppHandle } from './app-handle.js';
 import type { StateStore } from '../daemon/state-store.js';
 import type { Netron } from '@omnitron-dev/titan/netron';
+/**
+ * Strip the `${project}/${stack}/` prefix from a handle key (or
+ * leave a bare name untouched). Used to identify "the same physical
+ * app under any registration form" so duplicate registrations from
+ * stack mode + bare-name RPC paths can be deduplicated.
+ *
+ * Examples:
+ *   effectiveAppName('omni/dev/payments') === 'payments'
+ *   effectiveAppName('payments')           === 'payments'
+ *   effectiveAppName('a/b/c/payments')     === 'payments'  (last segment)
+ */
+export declare function effectiveAppName(key: string): string;
+/**
+ * Auto-promote a bare `entry.name` to canonical `${project}/${stack}/${name}`
+ * form when the env carries OMNITRON_PROJECT + OMNITRON_STACK. The stack-mode
+ * boot path (ProjectService.startStack) already prefixes the name explicitly;
+ * this helper closes every other entry point that could otherwise register a
+ * bare-name handle which then becomes invisible to the stack-prefix-based
+ * UI lookup (ProjectService.toStackInfo filters on `${project}/${stack}/`).
+ *
+ * Idempotent: if the entry already looks namespaced (or env lacks the keys),
+ * the input is returned unchanged. An entry qualified under a DIFFERENT
+ * project/stack is preserved too — that's operator intent (cross-project
+ * administrative run), not a bug.
+ */
+export declare function ensureNamespacedEntry(entry: IEcosystemAppEntry): IEcosystemAppEntry;
 export declare class OrchestratorService extends EventEmitter {
     private readonly logger;
     private readonly pm;
     private readonly stateStore;
     private readonly handles;
+    /**
+     * Per-name in-flight start lock (T#57). When two `startApp()` calls
+     * arrive concurrently for the same app, the first one's promise is
+     * stored here; the second one awaits and returns the same handle
+     * rather than racing through to a parallel `launchBootstrapMode` /
+     * `launchClassicMode` that would spawn a second worker pool and
+     * leak the original. The map is cleared in a `finally` once the
+     * launch settles (success or failure) so the next legitimate call
+     * for the same name isn't permanently locked out.
+     */
+    private readonly startingApps;
+    /**
+     * Per-app async serial-execution queue covering start / stop / restart
+     * / crash-restart-timer (P0-E + P0-I + P2-B in the audit). Pre-this-
+     * queue each path had its own coalescer:
+     *   - startingApps: same-name start coalescing only — `stopApp` did
+     *     NOT consult it, so stop racing in-flight start left an orphan
+     *     supervisor when stop arrived before start wrote childProcess.
+     *   - restartCoalescer: same-name restart coalescing; independent of
+     *     stop+start.
+     *   - handleClassicCrash setTimeout: a raw timer that could fire
+     *     during the `await stopChild(...)` window and spawn a zombie
+     *     against an app the operator just stopped.
+     * The queue collapses all four into one primitive ordered by
+     * priority (stop > restart > start > crash-restart). The same-key
+     * mutex means stop ALWAYS observes a completed start, and an
+     * explicit stop ALWAYS preempts a pending crash-restart timer.
+     */
+    private readonly lifecycleQueue;
+    /**
+     * Pending post-escalation cool-down timers, keyed by app name.
+     * Held here rather than inside a lifecycle-queue slot so that a
+     * crashed app still answers operator commands while it waits.
+     */
+    private readonly coolDownTimers;
     private config;
     private readonly cwd;
     private metricsTimer;
@@ -43,25 +110,303 @@ export declare class OrchestratorService extends EventEmitter {
     private buildService;
     /** Cached build results per app name */
     private readonly buildResults;
+    /**
+     * Per-app restart coalescer state.
+     *
+     * `inFlight` — the currently-running restart promise (so a concurrent
+     *   trigger doesn't kick off a parallel restart and tear down the half-
+     *   started state from the previous one).
+     * `triggerSerial` — monotonically incremented by external trigger
+     *   entry points (BuildService onRebuild, RPC restartApp, manual
+     *   `omnitron restart`, deploy.service). Internal coalescer arms
+     *   that queue themselves DO NOT bump it.
+     * `firedSerial` — value of `triggerSerial` that the most recently
+     *   completed restart consumed. A trailing coalesced restart only
+     *   fires when `triggerSerial > firedSerial`, i.e. a genuinely new
+     *   external trigger arrived during the in-flight window. Two
+     *   coalescer arms colliding can no longer cause a self-perpetuating
+     *   loop because neither bumps `triggerSerial`.
+     * `cooldownUntil` — absolute epoch ms timestamp. Any trigger received
+     *   before this clears is queued instead of fired immediately.
+     *   Prevents the storm where esbuild's onEnd fires for a freshly-
+     *   spawned context milliseconds after the app reports online.
+     */
+    private readonly restartCoalescer;
+    /**
+     * Wall-clock window in ms during which restart triggers from
+     * BuildService.watchApp are deferred after `app:online`. Sized so a
+     * normal bundle build + child spawn + DI bootstrap fits comfortably
+     * inside; anything legitimately changing more often than this is a
+     * developer typing very fast, which the regular debounce handles.
+     */
+    private static readonly POST_ONLINE_COOLDOWN_MS;
+    /**
+     * Orphan-process reaper. Lazily created on first reconcile so unit
+     * tests that hit `startApp` directly don't get a periodic timer
+     * until the first real reconcile runs (which they bypass by
+     * stubbing `bootReconciled = true`).
+     */
+    private janitor;
+    /**
+     * One-shot guard for cold-start reconciliation. T#63 — every code
+     * path that lands an app on this orchestrator goes through
+     * `ensureBootReconciled()`, regardless of whether the entry-point
+     * is `startAll` (daemon-config) or `startApp` (project/stack
+     * mode). Pre-T#63 the cold-start sweep only fired inside
+     * `startAll`, so project-mode boots silently inherited orphans
+     * from the previous daemon generation.
+     */
+    private bootReconciled;
+    private bootReconcilePromise;
     /** Daemon's Netron instance for native service routing */
     private daemonNetron;
     /** Daemon Unix socket path for child-to-daemon Netron connections */
     private daemonSocketPath;
+    /**
+     * Promise that resolves once the daemon's Netron is wired up. Apps that
+     * declare topology cannot launch until this is ready — instead of failing
+     * with "daemon Netron not available" the launcher awaits this promise
+     * (with a generous timeout) so transient startup-order interleavings
+     * resolve themselves instead of becoming user-visible errors.
+     */
+    private daemonNetronReady;
+    private resolveDaemonNetronReady;
+    /**
+     * Optional metrics sink. Set by the daemon at boot via
+     * `setMetricsBridge` so the orchestrator can forward janitor sweeps,
+     * esbuild rebuilds, and lifecycle events without coupling itself to
+     * the metrics module. Null means metrics are disabled (tests, slim
+     * bootstraps).
+     */
+    private metricsBridge;
     constructor(logger: ILogger, pm: ProcessManager, stateStore: StateStore, cwd?: string);
     /**
      * Set the daemon Netron instance and socket path for native topology routing.
      * Called by the daemon after start when Netron is available.
      */
     setDaemonNetron(netron: Netron, socketPath: string): void;
+    /**
+     * Wire the metrics bridge. Called once during daemon bootstrap.
+     * After this, janitor sweeps, esbuild rebuilds, and orchestrator
+     * lifecycle events are forwarded to Prometheus.
+     */
+    setMetricsBridge(bridge: import('../observability/metrics-bridge.js').MetricsBridge): void;
+    /**
+     * Subscribe to our own EventEmitter to forward state-change events to
+     * the metrics bridge. Idempotent so repeated `setMetricsBridge` calls
+     * (mostly tests) don't multiply listeners.
+     */
+    private telemetryWired;
+    private wireOrchestratorTelemetry;
+    /**
+     * Wait until the daemon Netron is available, with a hard ceiling so we
+     * never hang forever if `setDaemonNetron()` was never wired in.
+     */
+    private ensureDaemonNetronReady;
+    /**
+     * Refresh the orchestrator's view of the ecosystem config. Called
+     * from `daemon.reloadConfig` so a subsequent `restartApp` reads the
+     * new entry definition (mode, env, dependsOn, restart policy)
+     * instead of the stale copy captured at first `startAll`. Pre-fix
+     * the two refs diverged silently after every reload — operators
+     * editing ecosystem.config.ts and running `omnitron reload`
+     * believed the daemon picked up the changes; in fact only daemon's
+     * own RPC service did, and individual app restarts kept running
+     * against the pre-reload definition until the daemon itself
+     * restarted. P0-G in the audit.
+     */
+    setConfig(config: IEcosystemConfig): void;
+    /**
+     * Start metrics sampling if it is not already running.
+     *
+     * Called from every path that can produce a managed process, so no app can
+     * end up unsampled because of how it was started.
+     */
+    private ensureMetricsPolling;
+    /**
+     * One-shot cold-start reconciliation: discover pre-existing managed
+     * processes, reap fork-workers left over from a previous daemon
+     * generation, and arm the periodic janitor sweep. Idempotent —
+     * concurrent callers join the in-flight promise; subsequent calls
+     * after success are no-ops.
+     *
+     * Called from BOTH `startAll` (top-level daemon-config boot) and
+     * `startApp` (project/stack mode boot), so every code path that
+     * lands an app on this orchestrator runs the cleanup. Pre-T#63 the
+     * cold-start sweep was gated on `startAll` and never ran in
+     * project mode, so an orphan from a previous daemon survived
+     * silently and held the listen port hostage from its successor.
+     */
+    private ensureBootReconciled;
     startAll(config: IEcosystemConfig): Promise<void>;
+    /**
+     * Lazily create the orphan-process janitor with the live "owned
+     * pids" snapshot wired to the orchestrator's handle map.
+     */
+    private ensureJanitor;
+    /**
+     * Every OS pid the orchestrator currently considers ours. Cheap — called
+     * on every janitor sweep, and the janitor kills whatever this set omits.
+     *
+     * The rule itself lives in `owned-pids.ts`, where it can be tested: the
+     * janitor's own tests supply this set as a fixture, so they could never
+     * catch it being incomplete — and it was, by a whole class of process.
+     */
+    /**
+     * Who is holding the port an app could not bind, in one sentence.
+     *
+     * Called where a start or restart failed, because that is the moment the
+     * answer is worth having and the moment nobody has it: the error names the
+     * address and never the holder. Measured on this daemon — four
+     * `EADDRINUSE` failures on 3005 in thirty-one seconds, `omnitron list`
+     * reporting the app crashed with no pid, and the port held the whole time
+     * by a child of this very daemon that nothing in its records claimed.
+     *
+     * Returns null when the failure was not a port clash, and says so out loud
+     * when it could not read the machine — a reading that failed and a port
+     * nobody holds are different findings.
+     */
+    explainPortConflict(message: string): Promise<string | null>;
+    /** The parent of one process, or undefined when it cannot be read. */
+    private parentOf;
+    private collectOwnedPids;
+    /**
+     * Every pid the process manager currently vouches for.
+     *
+     * Two windows, and a process is only safe if both are asked. `getForkedPids`
+     * holds a claim from `fork()` until `spawn()` resolves and releases it —
+     * after that its own docstring says "the WorkerHandle registry is the
+     * authority". `listProcesses()` IS that registry, and it is the only source
+     * that answers for a process the orchestrator did not start: the daemon's
+     * system workers go through this same manager but belong to no app handle,
+     * so walking `this.handles` can never see them.
+     *
+     * Terminal records are excluded deliberately. They keep the pid they had,
+     * the OS reuses pids, and a stale one here would vouch for exactly the
+     * leaked worker the janitor exists to reap.
+     */
+    private managerOwnedPids;
     startApp(entry: IEcosystemAppEntry, config?: IEcosystemConfig): Promise<AppHandle>;
+    private startAppInternal;
+    /**
+     * Everything a handle still runs, stopped: its supervisor's children (or its
+     * classic child process) AND its topology pools.
+     *
+     * The pools were missed. A pool's workers are not the supervisor's children
+     * — they are the daemon's, siblings of the server, recorded only in
+     * `topologyPools` — so `supervisor.stop()` never reached them, and nothing
+     * in the orchestrator destroyed a pool: only the daemon's own shutdown did.
+     * Every restart of an app with `instances > 1` left the previous workers
+     * running. On daos/test (2026-09-25): priceverse's `ohlcv-aggregator` and
+     * storage's `transform` in three generations at once.
+     *
+     * The pools go even when the supervisor's stop throws: a stop that failed
+     * half-way must not also leave the half nobody asked about.
+     */
+    private tearDownProcesses;
+    /**
+     * Give back every service an app exposed on the daemon's Netron.
+     *
+     * A registration outlives the process behind it: the daemon keeps answering
+     * `queryInterface` with the full method list and every call through the name
+     * fails at the socket. `takeOverExisting` rescues a name the next launch
+     * re-exposes; one it does not stays advertised for the life of the daemon.
+     */
+    private releaseExposedServices;
     stopApp(name: string, force?: boolean, timeout?: number): Promise<void>;
     stopAll(force?: boolean): Promise<number>;
+    /**
+     * Give a supervised app the definition its next start will use — its
+     * environment, its command. The running process is not touched; the next
+     * `restartApp` starts from this. `false` when nothing by that name runs
+     * here. See `AppHandle.redefine`.
+     */
+    redefineApp(name: string, entry: IEcosystemAppEntry): boolean;
+    /**
+     * Restart an app, coalescing concurrent triggers.
+     *
+     * The dev-mode hot-reload path can deliver many restart triggers per
+     * second (file save → esbuild rebuild → onEnd → debounced → restart),
+     * and naive parallelism here ends in cross-stomping state mutations
+     * (one restart's stopApp tearing down the other's freshly-spawned
+     * child). This wrapper serialises restarts per app:
+     *
+     *   - If a restart is already in flight, we set the `pending` bit and
+     *     await the current restart. When it completes we run exactly one
+     *     follow-up — extra triggers received in the meantime collapse onto
+     *     the same bit, so a burst of N triggers produces at most 2 actual
+     *     restarts (the current one + one trailing).
+     *   - If we're inside the post-online cooldown window, the trigger is
+     *     queued (via `pending`) and fires after the cooldown clears.
+     *
+     * The hot path is `restartAppNow()` which contains the actual work and
+     * is what tests / non-coalescing callers can hit directly.
+     */
     restartApp(name: string): Promise<AppHandle>;
     /**
-     * Resolve raw app name to its handle key.
-     * In stack mode, handles are namespaced (e.g., 'omni/dev/main' for app 'main').
-     * This finds the handle whose key ends with the raw app name.
+     * Internal coalescer dispatcher.
+     *
+     * `fromExternal=true` bumps `triggerSerial` (a user / file-watcher /
+     * RPC asked us to restart). `fromExternal=false` is used by the
+     * recursive arms (cooldown-wait, await-inflight) which are NOT new
+     * triggers, just continuations — they must not bump or the trailing
+     * coalesced restart would re-fire forever.
+     */
+    private restartAppCoalesced;
+    /**
+     * Internal restart hot path — assumes coalescer has cleared us to proceed.
+     *
+     * Architecture note (Fix #247): we deliberately do NOT tear down the
+     * esbuild watch or invalidate buildResults here. The watch is what
+     * triggered us in the first place — its bundle is already fresh, and
+     * disposing the context creates a window where the next save can be
+     * missed (esbuild's metafile doesn't carry over between contexts).
+     *
+     * `launchBootstrapMode` sees the live buildResult + watch and skips
+     * the cold rebuild, so a restart cycle is just: kill old child →
+     * spawn new child with the bundle that was already on disk. The
+     * bootstrap-config cache is cleared because changes to `bootstrap.ts`
+     * (topology definition) need to take effect on the next start.
+     */
+    /**
+     * Start an app this daemon already supervises, using the entry its handle
+     * carries.
+     *
+     * `startApp(entry)` needs an ecosystem entry, which the daemon's own config
+     * is only one source of — a project stack registers handles this daemon
+     * never declared. `restartAppNow` has always taken the entry off the handle
+     * for exactly that reason; this is the same move for a start, so the name
+     * `list` prints is a name `start` accepts.
+     *
+     * Returns `undefined` when the app is unknown, leaving the caller to decide
+     * what to say about a name that names nothing.
+     */
+    startKnownApp(name: string): Promise<AppHandle | undefined>;
+    private restartAppNow;
+    private getCoalescerState;
+    /**
+     * Start the post-online cooldown for an app. Called from the `app:online`
+     * transition (both initial startApp and subsequent restartApp paths
+     * funnel through this), so a freshly-online app cannot be torn down by
+     * a rebuild signal that was already in flight when it finished booting.
+     */
+    private startPostOnlineCooldown;
+    /**
+     * Resolve a user-supplied name to a canonical handle key.
+     *
+     * Inputs accepted:
+     *   - exact canonical key (`omni/dev/payments`) → returned as-is
+     *   - bare short name (`payments`) → returns the canonical key whose
+     *     last `/`-segment matches, IF AND ONLY IF the match is unique.
+     *
+     * A qualified name that matches no handle exactly resolves to nothing: it
+     * names a deployment this daemon does not have, and the nearest app with
+     * the same last segment is a different one.
+     *
+     * Returns `undefined` for no match. Throws on ambiguity (multiple
+     * stacks expose the same short name) so the caller can surface a
+     * clear error to the operator instead of operating on the wrong
+     * app.
      */
     resolveAppName(rawName: string): string | undefined;
     /**
@@ -92,7 +437,45 @@ export declare class OrchestratorService extends EventEmitter {
      */
     listHandleNames(prefix?: string): string[];
     getWorkerHandle(processId: string): import("@omnitron-dev/titan-pm").IWorkerHandle | undefined;
+    /**
+     * Surface the underlying IProcessInfo for a titan-pm processId
+     * so callers (e.g. the daemon's `inspect` RPC) can read uptime
+     * + status without reaching into the process-manager directly.
+     */
+    getChildProcessInfo(processId: string): import("@omnitron-dev/titan-pm").IProcessInfo | undefined;
+    /**
+     * Pull the live DI dependency graph from a running app's first child
+     * worker. Used by the daemon RPC backing `omnitron inspect <app>
+     * --graph`. Returns null if the app isn't running, has no children,
+     * or the worker doesn't expose `getDependencyGraph` (older bootstraps).
+     */
+    getDependencyGraph(name: string): Promise<{
+        nodes: Array<{
+            id: string;
+            label?: string;
+            type?: string;
+        }>;
+        edges: Array<{
+            from: string;
+            to: string;
+            type?: 'dependency' | 'parent';
+        }>;
+    } | null>;
+    /**
+     * `name` is resolved like every other per-app question (`main` finds
+     * `daos/dev/main`); it was compared verbatim, so `omnitron metrics main`
+     * printed «(no data)». An unknown name answers `{}`, and the caller says so.
+     */
     getMetrics(name?: string): Promise<Record<string, IProcessMetrics | null>>;
+    /**
+     * Each app's health, asked of every process it runs — its supervisor
+     * children and its worker pools — and combined (`app-health.ts`).
+     *
+     * `name` is resolved like every other per-app question: `main` finds
+     * `daos/dev/main`. It was compared verbatim, so `omnitron health main`
+     * matched nothing and the answer was empty. An unknown name answers `{}`,
+     * and the caller says so.
+     */
     getHealth(name?: string): Promise<Record<string, IHealthStatus | null>>;
     /**
      * Drain pre-built MetricSample[] from all children.
@@ -144,10 +527,29 @@ export declare class OrchestratorService extends EventEmitter {
      * config needed. The @Service decorator on the worker IS the source of truth.
      */
     private launchTopology;
+    /**
+     * Register each single process's `@Service` classes on the daemon.
+     *
+     * Mirrors the pool path above, including its poll: a child reports ready
+     * before its Application has started, and `@Service` classes are exposed to
+     * Netron during start, so asking once asks a process that is still booting
+     * and gets an empty list back.
+     */
+    private exposeChildTopologyServices;
     /**
      * Launch single-process bootstrap app (no topology — original behavior).
      */
     private launchSingleProcess;
+    /**
+     * A child's start, for both launch paths: the app is online when the start
+     * completes it, and not before (`AppHandle.childStarted`).
+     *
+     * Each path had its own copy of this handler, and both marked the app
+     * online — with that child's pid — on every child's start. `app:online`
+     * went out once per child with it: three STARTED events for one boot of a
+     * three-process app, the first two about an app still coming up.
+     */
+    private onChildStarted;
     /**
      * Wire common supervisor events (crash, restart, escalate) and log capture.
      */
@@ -157,12 +559,143 @@ export declare class OrchestratorService extends EventEmitter {
      * Routes child stdout/stderr lines to AppHandle ring buffer for `getLogs()`.
      */
     private attachLogCapture;
+    /**
+     * Attach log capture to the workers of a topology POOL.
+     *
+     * `attachLogCapture` above is driven by `child:started`, which only
+     * supervisor children emit — a pool worker is not one, so nothing ever
+     * subscribed to its output. Everything those workers print went nowhere:
+     * the OHLCV aggregator and the image transformer have been mute since they
+     * were written, and since titan-pm started holding pre-capture output it was
+     * also a megabyte per worker retained for a reader that never arrived.
+     *
+     * Attached once, at pool creation. A worker replaced later by autoscaling or
+     * a health restart is not covered; these pools are fixed-size, and the
+     * alternative is an event this package does not expose yet.
+     */
+    private attachPoolLogCapture;
+    /** The one place a child's line becomes an app's line. */
+    private childLogForwarder;
+    /**
+     * Write the failure a child reported before it could start to the app's log
+     * handlers, as one structured line.
+     *
+     * `error.childStack` is the stack from inside the child when it managed to
+     * send one — for a module that throws on import that is the entire
+     * diagnosis, and it exists nowhere else once the process is gone.
+     */
+    private persistChildFailure;
+    /**
+     * Write a dead child's captured output to the app's log handlers.
+     *
+     * Emitted as pino-shaped JSON so it lands in the same file, at the same
+     * level, as everything else that app ever logged — a reader tailing
+     * `app.log` after a failed start finds the reason there rather than having
+     * to know that this particular failure hides in a different place.
+     */
+    private persistChildOutput;
+    /**
+     * Subscribe to the WorkerHandle's `exit` event so the orchestrator
+     * keeps the exit code / signal / "expected" flag past the
+     * WorkerHandle's GC. Pre-fix the supervisor's `child:crash` event
+     * fired with an Error but never surfaced the underlying
+     * code/signal — operators had no way to tell SIGKILL from a
+     * thrown TypeError. The captured info lands on
+     * `handle.lastExit` and is surfaced by `omnitron inspect`.
+     */
+    private attachExitCapture;
     private stopChildProcess;
-    private handleClassicCrash;
+    /**
+     * Cancel a pending crash cool-down for `name`, if one is armed.
+     * An explicit stop means "this app is going down now, stay down",
+     * which the cool-down must not undo ten minutes later.
+     */
+    private clearCoolDown;
+    private scheduleRecovery;
     private calculateBackoff;
-    /** Sample CPU/memory for a single OS process via `ps` (async — does not block event loop) */
+    /**
+     * The live OS processes behind one topology entry.
+     *
+     * Single-instance entries run as supervisor children; entries with
+     * `instances > 1` run in a pool the supervisor knows nothing about. Both
+     * shapes are resolved here, in one place, because metrics sampling and
+     * status reporting each used to resolve them separately — and disagreed.
+     * The pool branch asserted `online` from `pool.size > 0` before looking at
+     * a pid at all, so a worker whose process had exited still read as online
+     * (observed live: two pool workers reporting `online` on pids that `kill
+     * -0` said were gone).
+     */
+    private resolveTopologyPids;
+    /**
+     * The live processes behind one topology entry, keyed by the PM process id
+     * as well as the OS pid.
+     *
+     * The pid alone answers "is it alive"; the process id is what any question
+     * ABOUT that process has to be asked with — `pm.getProcess(id)` carries the
+     * spawn time, and the supervisor's restart counter is keyed by child name.
+     * `resolveTopologyPids` used to discard both on the way out, which is why
+     * the `omnitron list` DTO had nothing to report but the parent's numbers.
+     */
+    private resolveTopologyMembers;
+    /**
+     * Sample every process an app owns, and record the per-entry readings on
+     * the handle on the way through.
+     *
+     * A bootstrap app is several OS processes, so "the memory this app uses"
+     * is their sum. The periodic poller used to sample the single pid stored
+     * on the handle — which for a multi-process app is whichever child was
+     * registered last — and overwrite `lastMetrics` with it every five
+     * seconds, while `getMetrics()` right beside it aggregated correctly and
+     * wrote to the same field. The poller won on frequency, so the aggregate
+     * was never what anyone saw: `acme/dev/main` reported 122 MB against 437 MB
+     * actually resident across its three processes.
+     *
+     * ## Why `ps` and not the child's own report
+     *
+     * The aggregating path preferred the PM-level metrics a child reports over
+     * its RPC channel, falling back to `ps`. Those two are not the same
+     * quantities:
+     *
+     *   - `cpu` from the child is `(cpuUsage().user + .system) / 1e6` — CPU
+     *     SECONDS CONSUMED SINCE START, a monotonically growing counter. `ps`
+     *     `%cpu` is a percentage. The dashboard column is labelled `CPU %`,
+     *     so an app running for a day would have read `86400`. Observed
+     *     climbing live: 12.845 to 13.297 across twelve seconds of idling.
+     *   - `memory` from the child is `memoryUsage().heapUsed` — the V8 heap.
+     *     `ps` reports RSS, which is what "memory this app uses" means to
+     *     whoever is sizing a host. Observed: 66 MB heap against 149 MB
+     *     resident for the same process.
+     *
+     * Mixing them per-child, depending on which source answered first, gave a
+     * column that was neither quantity. `ps` is asked for both figures here so
+     * that every number under one label means one thing.
+     */
+    private sampleAppMetrics;
+    /**
+     * Sample several processes in one `ps` call.
+     *
+     * Pids the OS no longer knows are simply absent from the result — an
+     * omission the caller can see, unlike a zeroed reading it cannot tell
+     * apart from an idle process.
+     */
+    private sampleProcessMetricsBatch;
+    /**
+     * Sample CPU/memory for a single OS process via `ps`.
+     *
+     * `null` when it could not be read. This used to return
+     * `{ cpu: 0, memory: 0 }` on any failure, and the caller wrote that
+     * straight over `handle.lastMetrics` — so a `ps` that timed out under load
+     * made a running classic-mode app read as idle, indistinguishable from one
+     * that genuinely was. The batch path above already treated absence as
+     * absence; these two now agree.
+     */
     private sampleProcessMetrics;
-    /** Periodic metrics polling for classic mode processes */
+    /**
+     * Periodic CPU/memory sampling for every managed process.
+     *
+     * Idempotent: re-arming replaces the previous timer rather than stacking a
+     * second one, so `setConfig` can call it on every config reload.
+     */
     private startMetricsPolling;
     private toProcessInfo;
     private persistState;

package/dist/orchestrator/orchestrator.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.service.d.ts","sourceRoot":"","sources":["../../src/orchestrator/orchestrator.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,EAIL,KAAK,cAAc,EAInB,KAAK,eAAe,EACpB,KAAK,aAAa,EAGnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EACV,gBAAgB,EAChB,kBAAkB,EAElB,cAAc,EACd,SAAS,EAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAO5C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AA0BzD,qBAAa,mBAAoB,SAAQ,YAAY;IAuBjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAxB7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IACxD,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsD;IAErF,oFAAoF;IACpF,OAAO,UAAS;IAEhB,kFAAkF;IAClF,OAAO,CAAC,YAAY,CAA6B;IAEjD,wCAAwC;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAkC;IAE/D,0DAA0D;IAC1D,OAAO,CAAC,YAAY,CAAuB;IAE3C,qEAAqE;IACrE,OAAO,CAAC,gBAAgB,CAAuB;gBAG5B,MAAM,EAAE,OAAO,EACf,EAAE,EAAE,cAAc,EAClB,UAAU,EAAE,UAAU,EACvC,GAAG,CAAC,EAAE,MAAM;IAMd;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAUnD,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBjD,QAAQ,CAAC,KAAK,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC;IAqClF,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,UAAQ,EAAE,OAAO,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BrE,OAAO,CAAC,KAAK,UAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA2CvC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAmBlD;;;;OAIG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAQnD;;;;;;;;;OASG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAoDjD;;;OAGG;YACW,kBAAkB;IA2B1B,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAwBnE,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,cAAc,EAAE;IAQpD,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI;IAK3C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAI9C;;;OAGG;IACH,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAM1C,eAAe,CAAC,SAAS,EAAE,MAAM;IAQ3B,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC,CAAC;IAgD1E,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC,CAAC;IAuB7E;;;;OAIG;IACG,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC,CAAC;IA4BnF;;;OAGG;IACG,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlF,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAS5E;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAKjD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;YAQlD,iBAAiB;IAmB/B;;;;;;OAMG;YACW,mBAAmB;IAmHjC;;;;;;;;;;;;OAYG;YACW,cAAc;IAoO5B;;OAEG;YACW,mBAAmB;IA6GjC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA+B5B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;YA+BV,gBAAgB;YA4BhB,kBAAkB;IAkChC,OAAO,CAAC,gBAAgB;IAaxB,6FAA6F;YAC/E,oBAAoB;IAmBlC,0DAA0D;IAC1D,OAAO,CAAC,mBAAmB;IAwB3B,OAAO,CAAC,aAAa;IAoDrB,OAAO,CAAC,YAAY;CAqBrB"}
+{"version":3,"file":"orchestrator.service.d.ts","sourceRoot":"","sources":["../../src/orchestrator/orchestrator.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,EAQL,KAAK,cAAc,EAGnB,KAAK,eAAe,EACpB,KAAK,aAAa,EAKnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EACV,gBAAgB,EAChB,kBAAkB,EAElB,cAAc,EACd,SAAS,EAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAoD5C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AA0BzD;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAGpD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,kBAAkB,GAAG,kBAAkB,CAQnF;AA2CD,qBAAa,mBAAoB,SAAQ,YAAY;IA4IjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,UAAU;IA7I7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IAExD;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAyC;IAEtE;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA+B;IAE9D;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqC;IACpE,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsD;IAErF,oFAAoF;IACpF,OAAO,UAAS;IAEhB,kFAAkF;IAClF,OAAO,CAAC,YAAY,CAA6B;IAEjD,wCAAwC;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAkC;IAE/D;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAQ7B;IAEJ;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IAExD;;;;;OAKG;IACH,OAAO,CAAC,OAAO,CAA+B;IAE9C;;;;;;;;OAQG;IACH,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAA8B;IAE1D,0DAA0D;IAC1D,OAAO,CAAC,YAAY,CAAuB;IAE3C,qEAAqE;IACrE,OAAO,CAAC,gBAAgB,CAAuB;IAE/C;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB,CAAgB;IACzC,OAAO,CAAC,wBAAwB,CAA+B;IAE/D;;;;;;OAMG;IACH,OAAO,CAAC,aAAa,CAA2E;IAEhG,YACmB,MAAM,EAAE,OAAO,EACf,EAAE,EAAE,cAAc,EAClB,UAAU,EAAE,UAAU,EACvC,GAAG,CAAC,EAAE,MAAM,EAOb;IAED;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAKxD;IAED;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,OAAO,oCAAoC,EAAE,aAAa,GAAG,IAAI,CAIzF;IAED;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,yBAAyB;IAcjC;;;OAGG;YACW,uBAAuB;IAgBrC;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAgBxC;IAED;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAK5B;;;;;;;;;;;;;OAaG;YACW,oBAAoB;IAwD5B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqDtD;IAED;;;OAGG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;;;;OAOG;IACH;;;;;;;;;;;;;OAaG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAoCjE;IAED,sEAAsE;YACxD,QAAQ;IAkBtB,OAAO,CAAC,gBAAgB;IAQxB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gBAAgB;IAUlB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCvF;YAEa,gBAAgB;IAoK9B;;;;;;;;;;;;;;OAcG;YACW,iBAAiB;IAsB/B;;;;;;;OAOG;YACW,sBAAsB;IAa9B,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,UAAQ,EAAE,OAAO,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAoB1E;IAEK,OAAO,CAAC,KAAK,UAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CA+C5C;IAED;;;;;OAKG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAK5D;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAGjD;IAED;;;;;;;;OAQG;YACW,mBAAmB;IA6HjC;;;;;;;;;;;;;;OAcG;IACH;;;;;;;;;;;;OAYG;IACG,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAKhE;YAEa,aAAa;IAa3B,OAAO,CAAC,iBAAiB;IASzB;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAyBlD;IAED;;;;;;;;;OASG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAwDhD;IAED;;;OAGG;YACW,kBAAkB;IA2B1B,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAoBlE;IAMD,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,cAAc,EAAE,CAMnD;IAED,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAI1C;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAQ7C;IAED;;;OAGG;IACH,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAIzC;IAED,eAAe,CAAC,SAAS,EAAE,MAAM,8DAEhC;IAED;;;;OAIG;IACH,mBAAmB,CAAC,SAAS,EAAE,MAAM,6DAEpC;IAMD;;;;;OAKG;IACG,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAC9C,KAAK,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC5D,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAA;SAAE,CAAC,CAAC;KAC5E,GAAG,IAAI,CAAC,CA4BR;IAED;;;;OAIG;IACG,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC,CAAC,CAqB/E;IAED;;;;;;;;OAQG;IACG,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC,CAAC,CA+B5E;IAED;;;;OAIG;IACG,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC,CAAC,CA0BlF;IAED;;;OAGG;IACG,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBjF;IAED,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAY3E;IAED;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAGhD;IAED;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,CAE/D;YAMa,iBAAiB;IAmB/B;;;;;;OAMG;YACW,mBAAmB;IAuOjC;;;;;;;;;;;;OAYG;YACW,cAAc;IA+S5B;;;;;;;OAOG;YACW,2BAA2B;IA4EzC;;OAEG;YACW,mBAAmB;IAkHjC;;;;;;;;OAQG;IACH,OAAO,CAAC,cAAc;IAuBtB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsI5B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,oBAAoB;IAmB5B,0DAA0D;IAC1D,OAAO,CAAC,iBAAiB;IA0BzB;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IAmB3B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAqC1B;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;YAkCX,gBAAgB;IA2C9B;;;;OAIG;IACH,OAAO,CAAC,aAAa;YAQP,gBAAgB;IAqI9B,OAAO,CAAC,gBAAgB;IAYxB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,mBAAmB;IAI3B;;;;;;;;;OASG;IACH,OAAO,CAAC,sBAAsB;IAwB9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;YACW,gBAAgB;IA8F9B;;;;;;OAMG;YACW,yBAAyB;IAuBvC;;;;;;;;;OASG;YACW,oBAAoB;IAclC;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAuC3B,OAAO,CAAC,aAAa;IA6FrB,OAAO,CAAC,YAAY;CAqBrB"}