@omnitron-dev/omnitron 0.2.0 → 0.3.0

package/dist/orchestrator/port-conflict.js

@@ -0,0 +1,131 @@
+/**
+ * Who is holding the port an application could not bind.
+ *
+ * Measured on the development daemon, 06:05:56–06:06:27Z, four times in
+ * thirty-one seconds:
+ *
+ *     Restart failed — Netron service failed to start: listen EADDRINUSE:
+ *     address already in use 0.0.0.0:3005. The application cannot serve RPC,
+ *     so it is not started.
+ *
+ * `omnitron list` then reported the app `crashed, RST 5` with both children
+ * `stopped` and no pid, and `omnitron restart` answered `Max restarts
+ * exceeded`. All of that was true of the daemon's records and none of it was
+ * true of the machine: port 3005 was held by a LIVE process whose parent was
+ * the daemon itself — a child of its own it had stopped accounting for. The
+ * application was serving requests the whole time.
+ *
+ * The failure says which address it could not have. It does not say who has
+ * it, and that one fact is the difference between a three-minute
+ * investigation with `lsof` and a line in the log. Three readings compose
+ * it: the address out of the message, the holder out of the operating
+ * system, and the parent of the holder — because "somebody else's server" and
+ * "my own previous instance" call for opposite responses, and they look
+ * identical from inside the error.
+ *
+ * This module only reads and decides. Killing belongs to the janitor, which
+ * already reaps children the daemon has lost track of — and a holder that is
+ * our own unaccounted child is exactly what that sweep exists for.
+ */
+/**
+ * The address an `EADDRINUSE` refers to, or null when the message is about
+ * something else.
+ *
+ * Node writes `listen EADDRINUSE: address already in use 0.0.0.0:3005`, and
+ * every layer above wraps it in a sentence of its own, so the match is on
+ * the address rather than on the shape of the whole message.
+ */
+export function addressInUse(message) {
+    if (!message.includes('EADDRINUSE'))
+        return null;
+    // `host:port`, where the host may be IPv4, `::`, a bracketed IPv6 or a
+    // path-free empty string in some Node versions.
+    const match = /EADDRINUSE[^0-9[]*(\[[0-9a-fA-F:]+\]|[0-9.]+|::)?:(\d{1,5})\b/.exec(message);
+    if (!match)
+        return null;
+    const port = Number(match[2]);
+    if (!Number.isInteger(port) || port <= 0 || port > 65535)
+        return null;
+    return { host: match[1] ?? '', port };
+}
+/** What to run to find the listener, per platform. */
+export function holderCommand(port, platform) {
+    if (platform === 'linux') {
+        // `-H` drops the header; `-p` adds `users:(("node",pid=989927,fd=28))`.
+        return { file: 'ss', args: ['-ltnpH', `sport = :${port}`] };
+    }
+    // `-F` is lsof's machine-readable mode: one field per line, tagged.
+    return { file: 'lsof', args: ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-Fpc'] };
+}
+/**
+ * The listeners in that command's output.
+ *
+ * Both formats are parsed here rather than in the caller so that the shapes
+ * are pinned by tests against real output — `lsof -F` emits `p<pid>` and
+ * `c<command>` on separate lines and repeats them per process, and `ss`
+ * carries every listener's holders inside one `users:((...))` group.
+ */
+export function parseHolders(output, platform) {
+    const holders = [];
+    if (platform === 'linux') {
+        for (const m of output.matchAll(/\(\("([^"]+)",pid=(\d+),/g)) {
+            holders.push({ command: m[1], pid: Number(m[2]) });
+        }
+        return holders;
+    }
+    let pid = null;
+    for (const line of output.split('\n')) {
+        if (line.startsWith('p')) {
+            const parsed = Number(line.slice(1));
+            pid = Number.isInteger(parsed) ? parsed : null;
+            continue;
+        }
+        if (line.startsWith('c') && pid !== null) {
+            holders.push({ pid, command: line.slice(1) });
+            pid = null;
+        }
+    }
+    return holders;
+}
+/**
+ * What the holder is to this daemon.
+ *
+ * `ours-unaccounted` is the case the measurement above describes and the only
+ * one where a restart can be made to work without a person: the process is a
+ * child of this daemon and no app handle, pool or process-manager record
+ * claims it, so it is a leftover of a restart that half-failed.
+ */
+export function explainConflict(input) {
+    const { port, holders, parentOf, daemonPid, owned } = input;
+    if (holders.length === 0) {
+        return {
+            kind: 'unknown',
+            because: `nothing is listening on ${port} now — the holder exited between the failure and this reading`,
+        };
+    }
+    const holder = holders[0];
+    const parent = parentOf(holder.pid);
+    if (parent !== daemonPid) {
+        return {
+            kind: 'foreign',
+            holder,
+            because: `port ${port} is held by pid ${holder.pid} (${holder.command}), whose parent is ` +
+                `${parent ?? 'unknown'} — not this daemon. Nothing here will touch it`,
+        };
+    }
+    if (owned.has(holder.pid)) {
+        return {
+            kind: 'ours-owned',
+            holder,
+            because: `port ${port} is held by pid ${holder.pid} (${holder.command}), a child this daemon ` +
+                `still accounts for — the restart raced its own predecessor's shutdown`,
+        };
+    }
+    return {
+        kind: 'ours-unaccounted',
+        holder,
+        because: `port ${port} is held by pid ${holder.pid} (${holder.command}), a child of this daemon ` +
+            `that no app, pool or process-manager record claims — a leftover of a restart that half-failed`,
+    };
+}
+//# sourceMappingURL=port-conflict.js.map

package/dist/orchestrator/port-conflict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"port-conflict.js","sourceRoot":"","sources":["../../src/orchestrator/port-conflict.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAQH;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IACjD,uEAAuE;IACvE,gDAAgD;IAChD,MAAM,KAAK,GAAG,+DAA+D,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5F,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC;AACxC,CAAC;AAQD,sDAAsD;AACtD,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,QAAyB;IACnE,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,wEAAwE;QACxE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,YAAY,IAAI,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IACD,oEAAoE;IACpE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC;AAClF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,QAAyB;IACpE,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,GAAG,GAAkB,IAAI,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9C,GAAG,GAAG,IAAI,CAAC;QACb,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,KAM/B;IACC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,+DAA+D;SACxG,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAEpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO;YACL,IAAI,EAAE,SAAS;YACf,MAAM;YACN,OAAO,EACL,QAAQ,IAAI,mBAAmB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,OAAO,qBAAqB;gBACjF,GAAG,MAAM,IAAI,SAAS,gDAAgD;SACzE,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,MAAM;YACN,OAAO,EACL,QAAQ,IAAI,mBAAmB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,OAAO,yBAAyB;gBACrF,uEAAuE;SAC1E,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,EAAE,kBAAkB;QACxB,MAAM;QACN,OAAO,EACL,QAAQ,IAAI,mBAAmB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,OAAO,4BAA4B;YACxF,+FAA+F;KAClG,CAAC;AACJ,CAAC"}

package/dist/orchestrator/process-janitor.d.ts

@@ -24,8 +24,10 @@
  *   4. SIGTERM first, then SIGKILL after `gracefulMs`.
  *
  * Cold-start mode: on daemon boot, before any apps register, do a
- * single scan and kill every fork-worker whose ppid is NOT this
- * daemon's PID. They belong to a previous daemon that's now dead.
+ * single scan and kill every fork-worker whose parent is gone — a
+ * previous daemon that really is dead. A worker whose parent is ALIVE
+ * and is not us belongs to somebody else and is left alone, which is
+ * the same rule the periodic sweep applies.
  */
 import type { ILogger } from '@omnitron-dev/titan/module/logger';
 export interface ProcessJanitorOptions {
@@ -62,7 +64,30 @@ export interface ProcessJanitorOptions {
      * shells out to `ps -eo pid,ppid,args` and filters for fork-worker
      * paths.
      */
-    readonly listProcesses?: () => readonly PsRow[];
+    /**
+     * How to read the process table.
+     *
+     * `null` means the table could not be read — distinct from `[]`, which
+     * means it was read and holds no fork-workers. The two used to be one
+     * value and therefore one set of metrics, so a janitor that had not
+     * completed a sweep in hours reported clean ones.
+     */
+    readonly listProcesses?: () => readonly PsRow[] | null | Promise<readonly PsRow[] | null>;
+    /**
+     * How a process is signalled. Defaults to `process.kill`.
+     *
+     * Injectable because a janitor whose kill cannot be intercepted cannot be
+     * tested without killing: a test naming a pid it believes is imaginary
+     * sends a real signal to whatever holds that number on the machine
+     * running the suite. That is not hypothetical — it happened while writing
+     * the tests for this file, to a process holding a port on a live stand,
+     * and this repository has a previous incident of the same shape where a
+     * unit test reached `coldStartSweep` and killed six backends.
+     *
+     * A seam here makes the dangerous path unreachable from a test by
+     * construction, rather than by every future test remembering.
+     */
+    readonly kill?: (pid: number, signal: NodeJS.Signals) => void;
     /**
      * Override for the liveness probe (tests). Real implementation
      * uses `process.kill(pid, 0)`.
@@ -78,6 +103,15 @@ export interface PsRow {
 }
 export interface JanitorSweepMetrics {
     readonly scannedAt: Date;
+    /**
+     * False when `ps` could not be read at all.
+     *
+     * Without this a failed sweep reported `forkWorkersAlive: 0`, `orphansFound:
+     * 0` — a measurement of nothing, indistinguishable from a machine with
+     * nothing to reap. Measured on the development host: 990 sweeps skipped
+     * and every one of them recorded as a clean one.
+     */
+    readonly swept: boolean;
     readonly forkWorkersAlive: number;
     readonly ownedPids: number;
     readonly orphansFound: number;
@@ -88,10 +122,13 @@ export declare class ProcessJanitor {
     private readonly intervalMs;
     private readonly gracefulMs;
     private readonly minProcessAgeSeconds;
+    /** Sweeps that could not read the process table, in a row. */
+    private consecutiveSkips;
     private readonly logger;
     private readonly onMetrics;
     private readonly getOwnedPids;
     private readonly listProcesses;
+    private readonly kill;
     private readonly isAlive;
     private timer;
     private inFlight;
@@ -111,12 +148,49 @@ export declare class ProcessJanitor {
     stop(): void;
     /** Run one sweep on demand. Useful for tests and manual triggering. */
     runSweep(): Promise<JanitorSweepMetrics>;
+    /**
+     * Whether this row's parent is gone.
+     *
+     * One statement, used by both sweeps, because they have already disagreed
+     * about it once: when a parent dies its children are REPARENTED — to
+     * launchd on macOS, to init or a subreaper elsewhere — so an orphan's
+     * `ppid` becomes 1, and pid 1 is alive on every running system. Asking
+     * `isAlive(ppid)` therefore answers "the parent is fine" for precisely the
+     * processes both sweeps exist to reap, and the liveness test is kept only
+     * for the case where a ppid points at something that really has vanished
+     * without reparenting.
+     */
+    private hasDeadParent;
     /**
      * SIGTERM all pids → wait `gracefulMs` → SIGKILL any survivor.
      * Returns the number of pids successfully terminated (including
      * those killed forcefully).
      */
+    /**
+     * SIGTERM, a grace period, then SIGKILL for whatever is still there.
+     *
+     * Three things this used to get wrong, and all three were invisible:
+     *
+     *  - **A process was counted twice.** One that answered `ESRCH` to SIGTERM
+     *    (already gone) scored a success, and then scored a second one in the
+     *    SIGKILL pass for not being alive. `Math.min(success, pids.length)` at
+     *    the end hid it — for the BATCH, not per process: one double-count plus
+     *    one process that genuinely refused to die still clamped to "all
+     *    reaped". A clamp is not a count.
+     *  - **SIGKILL was assumed to have worked.** Success was recorded when the
+     *    signal was SENT. A process that does not die is exactly the one worth
+     *    knowing about, and it was the one that reported success.
+     *  - **The escalation was silent.** Measured 2026-09-14: every orphan
+     *    holding a port ignored SIGTERM and needed SIGKILL. "Died politely" and
+     *    "had to be killed" produced identical logs, so the fact that these
+     *    processes do not respond to TERM could not be learned from the record.
+     */
     private reap;
+    /**
+     * Wait for killed pids to leave the process table, up to a deadline.
+     * Returns whichever are still there when it expires.
+     */
+    private awaitDeath;
 }
 /**
  * Walk `ps -eo pid,ppid,etime,args` and pick rows whose command
@@ -133,7 +207,7 @@ export declare class ProcessJanitor {
  * spawned workers from the orphan reaper while their
  * supervisor.getChildNames() is still empty.
  */
-declare function listForkWorkersFromPs(logger?: ILogger): PsRow[];
+declare function listForkWorkersFromPs(logger?: ILogger): Promise<PsRow[] | null>;
 /**
  * Parse ps's `etime` format → seconds.
  * Supported shapes:

package/dist/orchestrator/process-janitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"process-janitor.d.ts","sourceRoot":"","sources":["../../src/orchestrator/process-janitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAGjE,MAAM,WAAW,qBAAqB;IACpC,4CAA4C;IAC5C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,8BAA8B;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IACjD;;;OAGG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,mBAAmB,KAAK,IAAI,CAAC;IACtD;;;;OAIG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,SAAS,KAAK,EAAE,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CAC7C;AAED,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IACzB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiD;IAC3E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA4B;IACzD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IACvD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,QAAQ,CAAS;gBAEb,OAAO,EAAE,qBAAqB;IAW1C;;;;;;;OAOG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAgBvC,yCAAyC;IACzC,KAAK,IAAI,IAAI;IAoBb,wCAAwC;IACxC,IAAI,IAAI,IAAI;IAOZ,uEAAuE;IACjE,QAAQ,IAAI,OAAO,CAAC,mBAAmB,CAAC;IA+C9C;;;;OAIG;YACW,IAAI;CAgCnB;AAMD;;;;;;;;;;;;;;GAcG;AACH,iBAAS,qBAAqB,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,KAAK,EAAE,CAyBxD;AAED;;;;;;;GAOG;AACH,iBAAS,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAevC;AAOD,iBAAS,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAE5C;AAGD,eAAO,MAAM,QAAQ;;;;CAAwD,CAAC"}
+{"version":3,"file":"process-janitor.d.ts","sourceRoot":"","sources":["../../src/orchestrator/process-janitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAGjE,MAAM,WAAW,qBAAqB;IACpC,4CAA4C;IAC5C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,8BAA8B;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IACjD;;;OAGG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,mBAAmB,KAAK,IAAI,CAAC;IACtD;;;;OAIG;IACH;;;;;;;OAOG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,SAAS,KAAK,EAAE,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IAE1F;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;IAC9D;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CAC7C;AAED,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IACzB;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,8DAA8D;IAC9D,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiD;IAC3E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA4B;IACzD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAmE;IACjG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgD;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,qBAAqB,EAUzC;IAED;;;;;;;OAOG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,CAuDtC;IAED,yCAAyC;IACzC,KAAK,IAAI,IAAI,CAkBZ;IAED,wCAAwC;IACxC,IAAI,IAAI,IAAI,CAKX;IAED,uEAAuE;IACjE,QAAQ,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAoF7C;IAED;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,aAAa;IAIrB;;;;OAIG;IACH;;;;;;;;;;;;;;;;;;OAkBG;YACW,IAAI;IA0DlB;;;OAGG;YACW,UAAU;CAUzB;AA2ED;;;;;;;;;;;;;;GAcG;AACH,iBAAe,qBAAqB,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAyC9E;AAED;;;;;;;GAOG;AACH,iBAAS,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAevC;AAOD,iBAAS,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAE5C;AAGD,eAAO,MAAM,QAAQ;;;;CAAwD,CAAC"}

package/dist/orchestrator/process-janitor.js

@@ -24,20 +24,25 @@
  *   4. SIGTERM first, then SIGKILL after `gracefulMs`.
  *
  * Cold-start mode: on daemon boot, before any apps register, do a
- * single scan and kill every fork-worker whose ppid is NOT this
- * daemon's PID. They belong to a previous daemon that's now dead.
+ * single scan and kill every fork-worker whose parent is gone — a
+ * previous daemon that really is dead. A worker whose parent is ALIVE
+ * and is not us belongs to somebody else and is left alone, which is
+ * the same rule the periodic sweep applies.
  */
-import { execSync } from 'node:child_process';
+import { execFile } from 'node:child_process';
 import { setTimeout as wait } from 'node:timers/promises';
 import { isProcessAlive } from '@omnitron-dev/titan/utils';
 export class ProcessJanitor {
     intervalMs;
     gracefulMs;
     minProcessAgeSeconds;
+    /** Sweeps that could not read the process table, in a row. */
+    consecutiveSkips = 0;
     logger;
     onMetrics;
     getOwnedPids;
     listProcesses;
+    kill;
     isAlive;
     timer = null;
     inFlight = false;
@@ -49,6 +54,7 @@ export class ProcessJanitor {
         this.onMetrics = options.onMetrics;
         this.getOwnedPids = options.getOwnedPids;
         this.listProcesses = options.listProcesses ?? (() => listForkWorkersFromPs(this.logger));
+        this.kill = options.kill ?? ((pid, signal) => process.kill(pid, signal));
         this.isAlive = options.isAlive ?? defaultIsAlive;
     }
     /**
@@ -61,8 +67,47 @@ export class ProcessJanitor {
      */
     async coldStartSweep() {
         const myPid = process.pid;
-        const all = this.listProcesses();
-        const stale = all.filter((row) => row.ppid !== myPid);
+        const listed = await this.listProcesses();
+        if (listed === null) {
+            this.logger?.error?.('janitor: cold start could not read the process table — leftovers from a previous daemon are not being reaped');
+            return 0;
+        }
+        const all = listed;
+        // A foreign parent is not evidence of a dead one.
+        //
+        // This read `row.ppid !== myPid` — reap anything not parented by ME —
+        // on the assumption that only the daemon ever calls this, and only at
+        // boot. Nothing enforces that assumption, and when it is wrong the cost
+        // is every running app on the machine. Measured 2026-09-12: one unit
+        // test constructing an `OrchestratorService` and calling `startApp`
+        // reached `ensureBootReconciled` → `coldStartSweep`, where
+        // `process.pid` was the vitest worker's, and killed all six backends of
+        // the live dev stand. The daemon itself survived, so `omnitron ls`
+        // reported `crashed` for apps nothing had crashed.
+        //
+        // `runSweep` below already had the right rule and this did not: "some
+        // other parent — not ours, leave alone". The two halves of one janitor
+        // disagreed about what an orphan is, and only the cheap half ran at the
+        // moment nothing else was there to object.
+        //
+        // But `!isAlive(ppid)` cannot answer the question this sweep exists for.
+        // When a parent dies its children are REPARENTED — to launchd on macOS,
+        // to init or a subreaper on Linux — so an orphan's `ppid` becomes 1, and
+        // pid 1 is alive on every running system. The test therefore returned
+        // `false` for precisely the processes the docblock above promises to
+        // kill, and the sweep could not reap a single one of them.
+        //
+        // Measured 2026-09-14, minutes after a daemon restart: seven
+        // `fork-worker.js` processes with `ppid = 1`, the oldest four and a half
+        // hours old, still holding their TCP ports. Their supervised replacements
+        // could not bind, failed to start, burned their whole restart budget on
+        // `EADDRINUSE` and gave up — so the console reported the APPLICATIONS as
+        // crashed, and the cause was the supervisor's own leftovers. They also
+        // ignored SIGTERM and needed SIGKILL, which is what `reap` already does.
+        //
+        // `ppid === REAPER_PID` is the orphan signature, and it is the one case
+        // where "not my child" really is evidence of a dead parent.
+        const stale = all.filter((row) => row.ppid !== myPid && this.hasDeadParent(row));
         if (stale.length === 0) {
             this.logger?.debug?.({ scanned: all.length }, 'janitor: cold start — no stale fork-workers');
             return 0;
@@ -104,8 +149,33 @@ export class ProcessJanitor {
     /** Run one sweep on demand. Useful for tests and manual triggering. */
     async runSweep() {
         const owned = this.getOwnedPids();
-        const all = this.listProcesses();
+        const listed = await this.listProcesses();
         const myPid = process.pid;
+        if (listed === null) {
+            // A sweep that could not look is not a sweep that found nothing. Both
+            // used to produce the same metrics and the same silence, so a janitor
+            // disabled for hours was indistinguishable from a tidy machine.
+            this.consecutiveSkips += 1;
+            // The first skip is noise; a run of them is a janitor that has stopped
+            // working, and the number is what makes that visible.
+            const level = this.consecutiveSkips >= 5 ? 'error' : 'warn';
+            this.logger?.[level]?.({ consecutiveSkips: this.consecutiveSkips, intervalMs: this.intervalMs }, this.consecutiveSkips >= 5
+                ? 'janitor: has not completed a sweep in several attempts — orphaned workers are not being reaped'
+                : 'janitor: sweep skipped, could not read the process table');
+            const skipped = {
+                scannedAt: new Date(),
+                swept: false,
+                forkWorkersAlive: 0,
+                ownedPids: owned.size,
+                orphansFound: 0,
+                orphansKilled: 0,
+                killErrors: 0,
+            };
+            this.onMetrics?.(skipped);
+            return skipped;
+        }
+        this.consecutiveSkips = 0;
+        const all = listed;
         const orphans = all.filter((row) => {
             // Owned by the orchestrator → not an orphan.
             if (owned.has(row.pid))
@@ -120,9 +190,17 @@ export class ProcessJanitor {
             // claim it → orphan from a partial restart.
             if (row.ppid === myPid)
                 return true;
-            // Process adopted by init (parent is gone) → orphan from a
+            // Process adopted by the reaper (parent is gone) → orphan from a
             // previous daemon that died. Reap it.
-            if (!this.isAlive(row.ppid))
+            //
+            // This read `!this.isAlive(row.ppid)` and could therefore never fire.
+            // The two halves of this janitor disagreed about what an orphan is:
+            // `coldStartSweep` was corrected to test the reaper pid and this was
+            // left behind, so the branch its own comment describes was unreachable
+            // here for as long as the daemon ran. Measured 2026-09-14 minutes after
+            // a restart: a `fork-worker.js` with `ppid = 1`, five minutes old,
+            // surveyed by every sweep and reaped by none.
+            if (this.hasDeadParent(row))
                 return true;
             // Some other parent — not ours, leave alone.
             return false;
@@ -137,6 +215,7 @@ export class ProcessJanitor {
         }
         const metrics = {
             scannedAt: new Date(),
+            swept: true,
             forkWorkersAlive: all.length,
             ownedPids: owned.size,
             orphansFound: orphans.length,
@@ -146,52 +225,175 @@ export class ProcessJanitor {
         this.onMetrics?.(metrics);
         return metrics;
     }
+    /**
+     * Whether this row's parent is gone.
+     *
+     * One statement, used by both sweeps, because they have already disagreed
+     * about it once: when a parent dies its children are REPARENTED — to
+     * launchd on macOS, to init or a subreaper elsewhere — so an orphan's
+     * `ppid` becomes 1, and pid 1 is alive on every running system. Asking
+     * `isAlive(ppid)` therefore answers "the parent is fine" for precisely the
+     * processes both sweeps exist to reap, and the liveness test is kept only
+     * for the case where a ppid points at something that really has vanished
+     * without reparenting.
+     */
+    hasDeadParent(row) {
+        return row.ppid === REAPER_PID || !this.isAlive(row.ppid);
+    }
     /**
      * SIGTERM all pids → wait `gracefulMs` → SIGKILL any survivor.
      * Returns the number of pids successfully terminated (including
      * those killed forcefully).
      */
+    /**
+     * SIGTERM, a grace period, then SIGKILL for whatever is still there.
+     *
+     * Three things this used to get wrong, and all three were invisible:
+     *
+     *  - **A process was counted twice.** One that answered `ESRCH` to SIGTERM
+     *    (already gone) scored a success, and then scored a second one in the
+     *    SIGKILL pass for not being alive. `Math.min(success, pids.length)` at
+     *    the end hid it — for the BATCH, not per process: one double-count plus
+     *    one process that genuinely refused to die still clamped to "all
+     *    reaped". A clamp is not a count.
+     *  - **SIGKILL was assumed to have worked.** Success was recorded when the
+     *    signal was SENT. A process that does not die is exactly the one worth
+     *    knowing about, and it was the one that reported success.
+     *  - **The escalation was silent.** Measured 2026-09-14: every orphan
+     *    holding a port ignored SIGTERM and needed SIGKILL. "Died politely" and
+     *    "had to be killed" produced identical logs, so the fact that these
+     *    processes do not respond to TERM could not be learned from the record.
+     */
     async reap(pids) {
-        let success = 0;
+        const pending = [];
         for (const pid of pids) {
             try {
-                process.kill(pid, 'SIGTERM');
+                this.kill(pid, 'SIGTERM');
+                pending.push(pid);
             }
             catch (err) {
                 const e = err;
-                if (e.code === 'ESRCH') {
-                    // Already gone — count as success.
-                    success += 1;
-                }
-                else {
+                if (e.code !== 'ESRCH') {
+                    // Not "already gone" — a permission problem, most likely. It will
+                    // not die on SIGKILL either, so it is not pending.
                     this.logger?.warn?.({ pid, err }, 'janitor: SIGTERM failed');
                 }
             }
         }
+        if (pending.length === 0)
+            return pids.length;
         await wait(this.gracefulMs);
-        for (const pid of pids) {
-            if (!this.isAlive(pid)) {
-                success += 1;
-                continue;
-            }
+        const stubborn = pending.filter((pid) => this.isAlive(pid));
+        if (stubborn.length > 0) {
+            this.logger?.warn?.({ pids: stubborn.slice(0, 10), gracefulMs: this.gracefulMs }, 'janitor: processes ignored SIGTERM — escalating to SIGKILL');
+        }
+        for (const pid of stubborn) {
             try {
-                process.kill(pid, 'SIGKILL');
-                success += 1;
+                this.kill(pid, 'SIGKILL');
             }
             catch (err) {
                 const e = err;
-                if (e.code === 'ESRCH')
-                    success += 1;
-                else
+                if (e.code !== 'ESRCH')
                     this.logger?.error?.({ pid, err }, 'janitor: SIGKILL failed');
             }
         }
-        return Math.min(success, pids.length);
+        // Confirm. A signal sent is not a process gone.
+        //
+        // Polled rather than checked once. SIGKILL is delivered immediately but
+        // the process is not reaped until the kernel can tear it down, and one in
+        // uninterruptible sleep — blocked on disk, which is the state a machine
+        // under load puts them in — takes longer than any single sample. The
+        // first version waited 500 ms and then reported at ERROR level that the
+        // process had "survived SIGKILL"; observed 2026-09-14 on a pid that was
+        // gone moments later. An alarm that fires on a normal delay teaches its
+        // reader to disregard the case it exists to report.
+        const survivors = await this.awaitDeath(stubborn);
+        if (survivors.length > 0) {
+            this.logger?.error?.({ pids: survivors.slice(0, 10), waitedMs: KILL_CONFIRM_TIMEOUT_MS }, 'janitor: processes survived SIGKILL — still holding whatever they hold');
+        }
+        return pids.length - survivors.length;
+    }
+    /**
+     * Wait for killed pids to leave the process table, up to a deadline.
+     * Returns whichever are still there when it expires.
+     */
+    async awaitDeath(pids) {
+        if (pids.length === 0)
+            return [];
+        const deadline = Date.now() + KILL_CONFIRM_TIMEOUT_MS;
+        let remaining = pids;
+        while (remaining.length > 0 && Date.now() < deadline) {
+            await wait(KILL_CONFIRM_POLL_MS);
+            remaining = remaining.filter((pid) => this.isAlive(pid));
+        }
+        return remaining;
     }
 }
 // ---------------------------------------------------------------------------
 // Process-table helpers (production defaults)
 // ---------------------------------------------------------------------------
+/**
+ * The pid a dead parent's children are handed to.
+ *
+ * 1 on every Unix this runs on — launchd on macOS, init or a subreaper
+ * elsewhere. A `fork-worker.js` parented here was started by a daemon that no
+ * longer exists.
+ */
+const REAPER_PID = 1;
+/** How long the janitor waits for `ps` before giving up on a sweep. */
+/**
+ * How long `ps` may take.
+ *
+ * This was ten seconds, and `ps -eo pid,ppid,etime,args` over a thousand
+ * processes takes 0.07. The deadline was not protecting against a slow `ps`;
+ * it was firing because the DAEMON'S event loop was too busy to read the
+ * result, and `execFile`'s timer — also on that loop — eventually killed a
+ * process that had already finished.
+ *
+ * Measured on the development host: 990 sweeps killed their own `ps` and
+ * skipped, while 36 fork-workers accumulated as children of a daemon that
+ * claimed none of them. The janitor stops working exactly when the machine
+ * is loaded, which is exactly when orphans appear.
+ *
+ * Sixty seconds is protection against a `ps` that genuinely hangs — an NFS
+ * mount, a wedged process table — and not against a control plane doing
+ * its job. It is a third of the sweep interval, so a slow sweep still
+ * finishes before the next one starts.
+ */
+const PS_TIMEOUT_MS = 60_000;
+/** How long a SIGKILLed process is given to actually leave the process table. */
+const KILL_CONFIRM_TIMEOUT_MS = 5_000;
+/** How often that is rechecked. */
+const KILL_CONFIRM_POLL_MS = 100;
+/**
+ * `ps`, asynchronously and with a deadline.
+ *
+ * This was `execSync('ps -eo …')` with no timeout, called from a 30-second
+ * `setInterval` on the daemon's own event loop. A synchronous spawn blocks
+ * that loop completely: while it runs the daemon answers no RPC, writes no
+ * log line, and services no socket. With no timeout, "while it runs" has no
+ * upper bound.
+ *
+ * That is not hypothetical. Observed 2026-09-14 on the development daemon:
+ * the main thread parked in `SyncProcessRunner::Spawn`, reached from
+ * `Environment::RunTimers`, for eight minutes and counting — `omnitron ping`,
+ * `omnitron ls` and every console request hung, the log stopped mid-second,
+ * and the seventeen supervised app processes ran on with nothing watching
+ * them. One `ps` that does not return takes the entire control plane with it.
+ *
+ * Asynchronous, so a slow `ps` costs a sweep rather than the daemon; and
+ * timed out, so it costs a bounded one.
+ */
+function runPs() {
+    return new Promise((resolve, reject) => {
+        execFile('ps', ['-eo', 'pid,ppid,etime,args'], { encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: PS_TIMEOUT_MS, killSignal: 'SIGKILL' }, (err, stdout) => {
+            if (err)
+                reject(err);
+            else
+                resolve(stdout);
+        });
+    });
+}
 /**
  * Walk `ps -eo pid,ppid,etime,args` and pick rows whose command
  * path contains `titan-pm/dist/fork-worker.js`. We match on the path
@@ -207,14 +409,28 @@ export class ProcessJanitor {
  * spawned workers from the orphan reaper while their
  * supervisor.getChildNames() is still empty.
  */
-function listForkWorkersFromPs(logger) {
+async function listForkWorkersFromPs(logger) {
     let raw;
     try {
-        raw = execSync('ps -eo pid,ppid,etime,args', { encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 });
+        raw = await runPs();
     }
     catch (err) {
-        logger?.warn?.({ err }, 'janitor: ps failed');
-        return [];
+        // A `ps` that was KILLED for exceeding the deadline and one that failed
+        // in a millisecond are the same empty result and, without this, the same
+        // log line — so the condition that used to hang the daemon would leave
+        // no trace that it had happened at all. `killed` is what `execFile` sets
+        // when its own timeout fires.
+        const e = err;
+        if (e.killed) {
+            logger?.error?.({ timeoutMs: PS_TIMEOUT_MS }, 'janitor: ps exceeded its deadline and was killed — sweep skipped');
+        }
+        else {
+            logger?.warn?.({ err: e.message }, 'janitor: ps failed');
+        }
+        // `null`, not `[]`. An empty list is an answer — "nothing is running" —
+        // and returning it for a sweep that could not look reported 990 clean
+        // sweeps on a host where 36 fork-workers were accumulating.
+        return null;
     }
     const rows = [];
     for (const line of raw.split('\n')) {