@omnitron-dev/omnitron 0.2.0 → 0.3.1

package/dist/orchestrator/bootstrap-process.d.ts

@@ -13,6 +13,68 @@
  * queryInterface() to get transparent proxies to sibling services.
  */
 import 'reflect-metadata';
+import type { AppHealthAnswer } from '../config/types.js';
+import type { IProcessTraffic } from '@omnitron-dev/titan-pm';
+/**
+ * A topology proxy for a service that was not there yet.
+ *
+ * When the first `queryInterface` fails, this process used to register a
+ * proxy that THROWS on every call for the life of the process — so a service
+ * that appeared a second later was never seen. For a pool that is rare: pools
+ * register in step 1, before any consumer starts. For a sibling child it is
+ * the normal case, because provider and consumer are children of one
+ * supervisor and the provider can only be asked for its services after they
+ * have all started.
+ *
+ * Measured on quotes: `CollectorWorker` registered on the daemon 7
+ * seconds after the server process asked for it, and the server's health
+ * check answered
+ *
+ *     Topology service 'CollectorWorker' unavailable: Service 'CollectorWorker' not found
+ *
+ * to every call for the rest of that process's life — while the same service,
+ * queried from a fresh client, answered `6 exchanges, 6 connected`.
+ *
+ * So: ask again on use. Once the query succeeds, hand over to the
+ * reconnecting proxy, which is what keeps it alive afterwards. A call made
+ * while it is still absent raises the same sentence as before, naming the
+ * service and the reason it could not be had THIS time — not the one from
+ * start-up.
+ *
+ * @internal exported for tests
+ */
+export declare function createDeferredTopologyProxy(netron: {
+    connect(url: string): Promise<{
+        queryInterface(name: string): Promise<unknown>;
+    }>;
+}, serviceName: string, daemonSocketUrl: string): unknown;
+/**
+ * A topology proxy that survives losing its connection.
+ *
+ * The interface `queryInterface` returns is bound to one socket. When that
+ * socket goes — a daemon restart, a transport hiccup, anything — every later
+ * call rejects with "Socket closed during RPC", forever, because nothing
+ * reconnects and the consumer is holding the same dead object. Consumers
+ * resolve this token once at construction, so the feature stays broken until
+ * the process is restarted.
+ *
+ * Observed exactly that way: pricing's OHLCV aggregation failed on every
+ * five-minute tick with that message while the service on the daemon was
+ * healthy and answering a freshly connected client immediately, and the app
+ * process held zero open connections to the daemon socket.
+ *
+ * So: on a call that fails because the connection is gone, reconnect, ask for
+ * the interface again, and retry once. One reconnect at a time, and a failure
+ * to reconnect surfaces the ORIGINAL error rather than a second one about the
+ * reconnect — the caller asked about their call.
+ *
+ * @internal exported for tests
+ */
+export declare function createReconnectingTopologyProxy(netron: {
+    connect(url: string): Promise<{
+        queryInterface(name: string): Promise<unknown>;
+    }>;
+}, serviceName: string, daemonSocketUrl: string, initial: unknown): unknown;
 declare class BootstrapProcess {
     private definition;
     private entry;
@@ -34,15 +96,30 @@ declare class BootstrapProcess {
         __topologyConnects?: string[];
     }): Promise<void>;
     /**
-     * Health check — delegates to definition hook if present.
+     * Health check — the app's own hook, handed the running Application so it
+     * has something to ask. Without an Application there is nothing to be
+     * healthy: the hook is not called, and the answer says why.
      */
-    checkHealth(): Promise<{
-        status: 'healthy' | 'degraded' | 'unhealthy';
-    }>;
+    checkHealth(): Promise<AppHealthAnswer>;
     /**
      * Graceful shutdown — runs lifecycle hooks and stops the Application.
      */
     shutdown(): Promise<void>;
+    /**
+     * What this process's own transports answered — the app's traffic.
+     *
+     * The process manager asks this inside `__getProcessMetrics`
+     * (`IProcessTrafficReporter`). Before, it reported the call counters of
+     * THIS wrapper's methods, which the supervisor calls for health and
+     * metrics: `omnitron metrics` counted its own questions as the app's
+     * requests and never saw one real one.
+     *
+     * HTTP servers answer a traffic snapshot; a process with several sums the
+     * counts and reports the latency of its busiest server. A process with no
+     * HTTP server (a queue worker) has no traffic to report: `null`, which the
+     * daemon shows as «not reported», not as zero.
+     */
+    reportTraffic(): IProcessTraffic | null;
     /**
      * Returns the resolved HTTP port (useful for pool instances on offset ports).
      */
@@ -60,6 +137,24 @@ declare class BootstrapProcess {
         version?: string;
         methods: string[];
     }>;
+    /**
+     * Invoke a method on one of the Application's own Netron services.
+     *
+     * The pool's PM service is THIS class — `BootstrapApp` — so `pool.execute(m)`
+     * can only reach the methods listed on it. `topology.expose` builds a daemon
+     * proxy out of the method names an app service reports through
+     * `getExposedServices()` above and used to route them straight into
+     * `pool.execute(name)`, which meant every such call asked `BootstrapApp` for
+     * a method it has never had. The daemon exposed the service, the consumer's
+     * `queryInterface` succeeded, and the first call came back "Unknown member" —
+     * from inside the pool, about a different service than the one the caller
+     * named. Nothing in the chain connects the two names, which is why the whole
+     * feature could be wired end to end and still never work.
+     *
+     * This is the hop that was missing: name the service, and the call lands on
+     * the Application's own stub for it.
+     */
+    callExposedService(serviceName: string, method: string, args?: unknown[]): Promise<unknown>;
     /**
      * Returns the DI dependency graph of the running Application's
      * container — nodes per registered token, edges per declared

package/dist/orchestrator/bootstrap-process.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bootstrap-process.d.ts","sourceRoot":"","sources":["../../src/orchestrator/bootstrap-process.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,kBAAkB,CAAC;AAoE1B,cACM,gBAAgB;IACpB,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,IAAI,CAAuB;IAEnC;;;;;;OAMG;IACG,IAAI,CAAC,IAAI,EAAE;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC/B,GAAG,OAAO,CAAC,IAAI,CAAC;IA+KjB;;OAEG;IAEG,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,WAAW,CAAA;KAAE,CAAC;IAW9E;;OAEG;IAEG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B/B;;OAEG;IACH,OAAO,IAAI,MAAM,GAAG,IAAI;IAIxB;;OAEG;IACH,OAAO,IAAI,MAAM,GAAG,IAAI;IAIxB;;;OAGG;IACH,kBAAkB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAKlF;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,OAAO,GAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG;QAC7D,KAAK,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC5D,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAA;SAAE,CAAC,CAAC;KAC5E;IAWD;;;;;;OAMG;YACW,qBAAqB;IAkDnC;;;;;;;;;;;;;;;;;;;;;;OAsBG;YACW,uBAAuB;CAiGtC;AAED,eAAe,gBAAgB,CAAC"}
+{"version":3,"file":"bootstrap-process.d.ts","sourceRoot":"","sources":["../../src/orchestrator/bootstrap-process.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,kBAAkB,CAAC;AAS1B,OAAO,KAAK,EAAE,eAAe,EAAiC,MAAM,oBAAoB,CAAC;AACzF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AA+H9D;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE;IAAE,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;KAAE,CAAC,CAAA;CAAE,EAC7F,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,MAAM,GACtB,OAAO,CA2CT;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE;IAAE,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;KAAE,CAAC,CAAA;CAAE,EAC7F,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,MAAM,EACvB,OAAO,EAAE,OAAO,GACf,OAAO,CAgET;AAED,cACM,gBAAgB;IACpB,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,IAAI,CAAuB;IAEnC;;;;;;OAMG;IACG,IAAI,CAAC,IAAI,EAAE;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC/B,GAAG,OAAO,CAAC,IAAI,CAAC,CAkMhB;IAED;;;;OAIG;IAEG,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC,CAc5C;IAED;;OAEG;IAEG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CA0B9B;IAED;;;;;;;;;;;;;OAaG;IACH,aAAa,IAAI,eAAe,GAAG,IAAI,CAqBtC;IAED;;OAEG;IACH,OAAO,IAAI,MAAM,GAAG,IAAI,CAEvB;IAED;;OAEG;IACH,OAAO,IAAI,MAAM,GAAG,IAAI,CAEvB;IAED;;;OAGG;IACH,kBAAkB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAGjF;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBpG;IAED;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,OAAO,GAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG;QAC7D,KAAK,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC5D,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAA;SAAE,CAAC,CAAC;KAC5E,CASA;IAED;;;;;;OAMG;YACW,qBAAqB;IAqEnC;;;;;;;;;;;;;;;;;;;;;;OAsBG;YACW,uBAAuB;CAgGtC;eAEc,gBAAgB"}

package/dist/orchestrator/bootstrap-process.js

@@ -25,10 +25,13 @@ import 'reflect-metadata';
 import path from 'node:path';
 import { Application } from '@omnitron-dev/titan';
 import { createToken } from '@omnitron-dev/titan/nexus';
+import { Errors } from '@omnitron-dev/titan/errors';
 import { HttpTransport } from '@omnitron-dev/titan/netron/transport/http';
 import { WebSocketTransport } from '@omnitron-dev/titan/netron/transport/websocket';
 import { loadBootstrapConfig } from './bootstrap-loader.js';
+import { compileCustomRoutes } from './http-routes.js';
 import { pathToFileURL } from 'node:url';
+import { queryTopologyService } from './topology-query.js';
 /**
  * Convert an absolute filesystem path to a file:// URL string suitable as
  * the anchor argument for `createRequire(...)`. Trailing slash on `p` is
@@ -40,6 +43,16 @@ function pathToFileUrl(p) {
 // Inline PM decorator metadata — avoids importing the full PM module which
 // deadlocks when loaded from source (.ts) alongside the dist-loaded worker-runtime.
 const PROCESS_METADATA_KEY = Symbol.for('process:metadata');
+// These decorators are declared locally rather than imported from
+// `@omnitron-dev/titan-pm` on purpose: this file is a worker ENTRY POINT, and
+// pulling the package in would load it into every child process. They
+// interoperate through `Symbol.for`, which is realm-wide, so the metadata they
+// write is the same metadata `titan-pm`'s `worker-runtime` scans for.
+//
+// That agreement is the whole contract, and it is not enforced by the type
+// system — `a-local-decorator-still-agrees-with-the-runtime.test.ts` pins it.
+// Note the direction of the last divergence: titan-pm's exported `@OnShutdown`
+// wrote a prototype key nothing reads, and THESE copies were the correct ones.
 const PROCESS_METHOD_METADATA_KEY = Symbol.for('process:method:metadata');
 function Process(options = {}) {
     return (target) => {
@@ -84,6 +97,203 @@ function OnShutdown() {
 }
 // Topology DI token prefix — apps use createToken('topology:{processName}')
 const TOPOLOGY_TOKEN_PREFIX = 'topology:';
+/**
+ * Deadline for a topology RPC.
+ *
+ * A topology call is a worker-pool JOB — aggregate five minutes of candles,
+ * transform an upload — not a wire request, and netron's 5 s default is a
+ * deadline for the latter. `connect()` reads `requestTimeout` back out of the
+ * transport registry, and this path registered the unix transport without
+ * options, so every topology call inherited the 5 s.
+ *
+ * Measured on the stand over two days: 97 `OHLCV … aggregation failed` lines,
+ * 70 of them `RPC request timed out after 5000ms` — and scattered THROUGH
+ * healthy runs, minutes after `topology proxy became available`, between ticks
+ * that succeeded. Not a dead peer: a job that occasionally takes longer than
+ * five seconds. The timeout abandons the caller's wait without stopping the
+ * work, so the aggregation ran on and the operator got an error for it.
+ *
+ * 60 s is what `DaemonClient` already uses over this same socket, and it is
+ * still well inside the 5-minute aggregation interval, so a genuine hang
+ * surfaces within one tick rather than being hidden.
+ *
+ * MEASURED AFTER, because "it is fixed" is a different claim from "it is
+ * better": 2.39 failures/hour before, 0.55 after — a 4.3x reduction, not an
+ * elimination. The survivors still say `after 5000ms`, which means their
+ * `RemotePeer` was built with `requestTimeout` undefined, so some path reaches
+ * the daemon without the options set two lines below. The obvious candidates
+ * are ruled out by reading: `getTransportForAddress` resolves `unix://` to the
+ * transport named `unix`, `getOptions` is keyed by that same name, the
+ * reconnect path uses this same Netron, and the pool leg is 120 s. Finding the
+ * rest needs instrumentation on a live process, not more reading.
+ */
+const TOPOLOGY_REQUEST_TIMEOUT = 60_000;
+/**
+ * Announce which stage of start-up we reached.
+ *
+ * Start-up runs config load → module import → `Application.create` →
+ * hooks → transports → `start()`, and until now it reported nothing until it
+ * was finished. When an app hung — `acme/dev/main` did, repeatedly — the only
+ * artefact was `Worker startup timed out after 30000ms` with
+ * `stderrBytes: 0`: the supervisor knew the child never answered and nothing
+ * knew how far it had got.
+ *
+ * Each stage is written to stderr as a single line. stderr specifically,
+ * because the spawner already captures the last 64 KiB of it into the crash
+ * record — so a timeout now carries a trail ending at the stage that hung,
+ * with no new plumbing between the child and the daemon.
+ *
+ * The prefix is machine-greppable on purpose; `omnitron doctor` reads the
+ * last one to name the stalled stage.
+ */
+function reportStage(stage, detail) {
+    const suffix = detail ? ` ${detail}` : '';
+    process.stderr.write(`[omnitron:boot] ${stage}${suffix}\n`);
+}
+/**
+ * A topology proxy for a service that was not there yet.
+ *
+ * When the first `queryInterface` fails, this process used to register a
+ * proxy that THROWS on every call for the life of the process — so a service
+ * that appeared a second later was never seen. For a pool that is rare: pools
+ * register in step 1, before any consumer starts. For a sibling child it is
+ * the normal case, because provider and consumer are children of one
+ * supervisor and the provider can only be asked for its services after they
+ * have all started.
+ *
+ * Measured on quotes: `CollectorWorker` registered on the daemon 7
+ * seconds after the server process asked for it, and the server's health
+ * check answered
+ *
+ *     Topology service 'CollectorWorker' unavailable: Service 'CollectorWorker' not found
+ *
+ * to every call for the rest of that process's life — while the same service,
+ * queried from a fresh client, answered `6 exchanges, 6 connected`.
+ *
+ * So: ask again on use. Once the query succeeds, hand over to the
+ * reconnecting proxy, which is what keeps it alive afterwards. A call made
+ * while it is still absent raises the same sentence as before, naming the
+ * service and the reason it could not be had THIS time — not the one from
+ * start-up.
+ *
+ * @internal exported for tests
+ */
+export function createDeferredTopologyProxy(netron, serviceName, daemonSocketUrl) {
+    let live = null;
+    let resolving = null;
+    const resolve = async () => {
+        resolving ??= (async () => {
+            try {
+                const peer = await netron.connect(daemonSocketUrl);
+                const handle = await peer.queryInterface(serviceName);
+                live = createReconnectingTopologyProxy(netron, serviceName, daemonSocketUrl, handle);
+            }
+            finally {
+                resolving = null;
+            }
+        })();
+        return resolving;
+    };
+    return new Proxy({}, {
+        get: (_target, prop) => {
+            if (typeof prop === 'symbol' || prop === 'then')
+                return undefined;
+            return async (...args) => {
+                if (!live) {
+                    try {
+                        await resolve();
+                    }
+                    catch (err) {
+                        throw new Error(`Topology service '${serviceName}' unavailable: ${err.message}`, { cause: err });
+                    }
+                }
+                return live[prop](...args);
+            };
+        },
+    });
+}
+/**
+ * A topology proxy that survives losing its connection.
+ *
+ * The interface `queryInterface` returns is bound to one socket. When that
+ * socket goes — a daemon restart, a transport hiccup, anything — every later
+ * call rejects with "Socket closed during RPC", forever, because nothing
+ * reconnects and the consumer is holding the same dead object. Consumers
+ * resolve this token once at construction, so the feature stays broken until
+ * the process is restarted.
+ *
+ * Observed exactly that way: pricing's OHLCV aggregation failed on every
+ * five-minute tick with that message while the service on the daemon was
+ * healthy and answering a freshly connected client immediately, and the app
+ * process held zero open connections to the daemon socket.
+ *
+ * So: on a call that fails because the connection is gone, reconnect, ask for
+ * the interface again, and retry once. One reconnect at a time, and a failure
+ * to reconnect surfaces the ORIGINAL error rather than a second one about the
+ * reconnect — the caller asked about their call.
+ *
+ * @internal exported for tests
+ */
+export function createReconnectingTopologyProxy(netron, serviceName, daemonSocketUrl, initial) {
+    let current = initial;
+    let reconnecting = null;
+    /**
+     * Errors that mean "this handle is no longer the service", i.e. re-query and
+     * try once more.
+     *
+     * The socket patterns are the obvious half. The other half is `Unknown
+     * member: 'x' is not defined in the service interface`, which is what a
+     * reconnect leaves behind when it lands while the worker's service is not
+     * yet registered on the daemon: `queryInterface` returns a definition
+     * without the methods, `current` is replaced by it, and every later call
+     * fails on a NAME rather than on a socket. That error matched nothing here,
+     * so the proxy stayed wrong until the app restarted — two of them on the
+     * stand, and no path back.
+     *
+     * Retrying a genuinely absent method costs one re-query and then raises the
+     * same error, so widening this cannot turn a real mistake into silence.
+     */
+    const needsFreshHandle = (error) => {
+        const message = String(error?.message ?? '');
+        return (/socket closed|connection closed|transport lost|not connected|socket is not open/i.test(message) ||
+            /unknown member|not defined in the service interface/i.test(message));
+    };
+    const reconnect = async () => {
+        reconnecting ??= (async () => {
+            try {
+                const peer = await netron.connect(daemonSocketUrl);
+                current = (await peer.queryInterface(serviceName));
+            }
+            finally {
+                reconnecting = null;
+            }
+        })();
+        return reconnecting;
+    };
+    const invoke = (prop, args) => current[prop](...args);
+    return new Proxy({}, {
+        get: (_target, prop) => {
+            if (typeof prop === 'symbol' || prop === 'then')
+                return undefined;
+            return async (...args) => {
+                try {
+                    return await invoke(prop, args);
+                }
+                catch (error) {
+                    if (!needsFreshHandle(error))
+                        throw error;
+                    try {
+                        await reconnect();
+                    }
+                    catch {
+                        throw error;
+                    }
+                    return await invoke(prop, args);
+                }
+            };
+        },
+    });
+}
 let BootstrapProcess = class BootstrapProcess {
     definition = null;
     entry = null;
@@ -100,7 +310,9 @@ let BootstrapProcess = class BootstrapProcess {
         const { bootstrapPath, processName: resolvedProcessName, portOffset: resolvedPortOffset, bundledModulePath, __daemonSocketUrl: daemonSocketUrl, __topologyConnects: topologyAccess, } = deps;
         // Prefer bundled module path (esbuild pre-built) over tsx runtime
         const hasTsx = !bundledModulePath && (process.execArgv ?? []).some((a) => a.includes('tsx'));
+        reportStage('config:loading', bootstrapPath);
         this.definition = await loadBootstrapConfig(bootstrapPath, { devMode: hasTsx || !!bundledModulePath });
+        reportStage('config:loaded', this.definition.name);
         // Find this process's entry in the topology.
         if (resolvedProcessName) {
             this.entry = this.definition.processes.find((p) => p.name === resolvedProcessName) ?? null;
@@ -118,12 +330,15 @@ let BootstrapProcess = class BootstrapProcess {
         const hooks = this.entry.hooks ?? this.definition.hooks;
         // beforeCreate hook
         if (hooks?.beforeCreate) {
+            reportStage('hook:beforeCreate');
             await hooks.beforeCreate();
         }
         // Import the module — prefer pre-bundled .mjs from BuildService,
         // fall back to resolving from bootstrap directory (tsx or dist/)
         const modulePath = bundledModulePath ?? path.resolve(path.dirname(bootstrapPath), this.entry.module);
+        reportStage('module:importing', modulePath);
         const moduleFile = await import(modulePath);
+        reportStage('module:imported');
         // ────────────────────────────────────────────────────────────────────
         // Container identity guard.
         //
@@ -158,13 +373,17 @@ let BootstrapProcess = class BootstrapProcess {
             ModuleClass = moduleFile;
         }
         // Create Titan Application
+        reportStage('application:creating');
         this.app = await Application.create(ModuleClass, {
             name: this.definition.name,
             version: this.definition.version,
         });
+        reportStage('application:created');
         // afterCreate hook
         if (hooks?.afterCreate) {
+            reportStage('hook:afterCreate');
             await hooks.afterCreate(this.app);
+            reportStage('hook:afterCreate:done');
         }
         // Inject topology proxies via Netron-native connection to daemon.
         // topology.access contains Netron service names (e.g., 'OhlcvAggregatorWorker')
@@ -199,7 +418,7 @@ let BootstrapProcess = class BootstrapProcess {
             }
             // Wire custom routes from process entry
             if (this.entry.customRoutes && this.entry.customRoutes.length > 0) {
-                transportOptions['customRoutes'] = this.entry.customRoutes.map((r) => r.handler);
+                transportOptions['customRoutes'] = compileCustomRoutes(this.entry.customRoutes);
             }
             this.app.netron.registerTransportServer('http', {
                 name: 'http',
@@ -217,7 +436,11 @@ let BootstrapProcess = class BootstrapProcess {
                 ...(wsConfig.path && { pathPrefix: wsConfig.path }),
                 ...(wsConfig.keepAlive && { keepAlive: { enabled: true, ...wsConfig.keepAlive } }),
             };
-            // Wire auth invocationWrapper for WebSocket too
+            // Applied per invocation by `remote-peer.ts`, so an app that
+            // establishes its RLS scope in this wrapper gets that scope on
+            // WebSocket calls too. It did not until recently: only the HTTP server
+            // read the option, and a WS call ran with no scope at all — kysera
+            // fails closed, so the symptom was empty results, not leaked rows.
             if (auth?.invocationWrapper) {
                 wsOptions['invocationWrapper'] = auth.invocationWrapper;
             }
@@ -228,26 +451,39 @@ let BootstrapProcess = class BootstrapProcess {
         }
         // beforeStart hook
         if (hooks?.beforeStart) {
+            reportStage('hook:beforeStart');
             await hooks.beforeStart(this.app);
+            reportStage('hook:beforeStart:done');
         }
         // Start the application
+        reportStage('application:starting');
         await this.app.start();
+        reportStage('application:started');
         // afterStart hook
         if (hooks?.afterStart) {
+            reportStage('hook:afterStart');
             await hooks.afterStart(this.app);
+            reportStage('hook:afterStart:done');
         }
+        reportStage('ready');
     }
     /**
-     * Health check — delegates to definition hook if present.
+     * Health check — the app's own hook, handed the running Application so it
+     * has something to ask. Without an Application there is nothing to be
+     * healthy: the hook is not called, and the answer says why.
      */
     async checkHealth() {
+        if (!this.app) {
+            return {
+                status: 'unhealthy',
+                checks: [{ name: 'application', status: 'fail', message: 'the application has not started in this process' }],
+            };
+        }
         const hooks = this.entry?.hooks ?? this.definition?.hooks;
         if (hooks?.onHealthCheck) {
-            return hooks.onHealthCheck();
+            return hooks.onHealthCheck(this.app);
         }
-        return {
-            status: this.app ? 'healthy' : 'unhealthy',
-        };
+        return { status: 'healthy' };
     }
     /**
      * Graceful shutdown — runs lifecycle hooks and stops the Application.
@@ -279,6 +515,44 @@ let BootstrapProcess = class BootstrapProcess {
             }
         }
     }
+    /**
+     * What this process's own transports answered — the app's traffic.
+     *
+     * The process manager asks this inside `__getProcessMetrics`
+     * (`IProcessTrafficReporter`). Before, it reported the call counters of
+     * THIS wrapper's methods, which the supervisor calls for health and
+     * metrics: `omnitron metrics` counted its own questions as the app's
+     * requests and never saw one real one.
+     *
+     * HTTP servers answer a traffic snapshot; a process with several sums the
+     * counts and reports the latency of its busiest server. A process with no
+     * HTTP server (a queue worker) has no traffic to report: `null`, which the
+     * daemon shows as «not reported», not as zero.
+     */
+    reportTraffic() {
+        const servers = this.app?.netron?.transportServers;
+        if (!servers)
+            return null;
+        let traffic = null;
+        for (const server of servers.values()) {
+            const snapshot = server.getTrafficSnapshot?.();
+            if (!snapshot)
+                continue;
+            if (!traffic) {
+                traffic = { ...snapshot };
+                continue;
+            }
+            traffic.requests += snapshot.requests;
+            traffic.serverErrors += snapshot.serverErrors;
+            traffic.clientErrors += snapshot.clientErrors;
+            traffic.probes += snapshot.probes;
+            traffic.held = (traffic.held ?? 0) + (snapshot.held ?? 0);
+            traffic.active += snapshot.active;
+            if ((snapshot.latency?.count ?? 0) > (traffic.latency?.count ?? 0))
+                traffic.latency = snapshot.latency;
+        }
+        return traffic;
+    }
     /**
      * Returns the resolved HTTP port (useful for pool instances on offset ports).
      */
@@ -300,6 +574,35 @@ let BootstrapProcess = class BootstrapProcess {
             return [];
         return this.app.netron.getServiceMetadata();
     }
+    /**
+     * Invoke a method on one of the Application's own Netron services.
+     *
+     * The pool's PM service is THIS class — `BootstrapApp` — so `pool.execute(m)`
+     * can only reach the methods listed on it. `topology.expose` builds a daemon
+     * proxy out of the method names an app service reports through
+     * `getExposedServices()` above and used to route them straight into
+     * `pool.execute(name)`, which meant every such call asked `BootstrapApp` for
+     * a method it has never had. The daemon exposed the service, the consumer's
+     * `queryInterface` succeeded, and the first call came back "Unknown member" —
+     * from inside the pool, about a different service than the one the caller
+     * named. Nothing in the chain connects the two names, which is why the whole
+     * feature could be wired end to end and still never work.
+     *
+     * This is the hop that was missing: name the service, and the call lands on
+     * the Application's own stub for it.
+     */
+    async callExposedService(serviceName, method, args = []) {
+        if (!this.app?.netron) {
+            throw Errors.internal(`Cannot call ${serviceName}.${method}: the application has no Netron`);
+        }
+        const iface = (await this.app.netron.peer.queryInterface(serviceName));
+        const fn = iface[method];
+        if (typeof fn !== 'function') {
+            throw Errors.badRequest(`Service '${serviceName}' has no method '${method}'. Methods reachable over the topology ` +
+                'proxy are those marked @Public — an undecorated method is absent from the service metadata.');
+        }
+        return fn.call(iface, ...args);
+    }
     /**
      * Returns the DI dependency graph of the running Application's
      * container — nodes per registered token, edges per declared
@@ -334,6 +637,7 @@ let BootstrapProcess = class BootstrapProcess {
         const { UnixSocketTransport } = await import('@omnitron-dev/titan/netron/transport/unix');
         // Register Unix transport on the app's Netron for outgoing client connections
         this.app.netron.registerTransport('unix', () => new UnixSocketTransport());
+        this.app.netron.setTransportOptions('unix', { requestTimeout: TOPOLOGY_REQUEST_TIMEOUT });
         // Connect with timeout — fail fast if daemon socket is unavailable
         const connectTimeout = 10_000;
         const daemonPeer = await Promise.race([
@@ -341,12 +645,23 @@ let BootstrapProcess = class BootstrapProcess {
             new Promise((_, reject) => setTimeout(() => reject(new Error(`Daemon socket connection timeout (${connectTimeout}ms): ${daemonSocketUrl}`)), connectTimeout)),
         ]);
         for (const serviceName of serviceNames) {
+            // Asking once turned a half-second of startup order into a logged
+            // failure: this process connects as soon as it is up, and the pool
+            // that provides the service registers a beat later. Six such lines a
+            // day on the stand, none of them an outcome. `queryTopologyService`
+            // covers the window; a service that is genuinely absent still lands in
+            // the catch below, where the warning means what it says.
+            const queried = await queryTopologyService(daemonPeer, serviceName);
             try {
-                const proxy = await daemonPeer.queryInterface(serviceName);
+                if (!queried.ok)
+                    throw queried.error;
+                const proxy = queried.proxy;
                 // Register under topology:{ServiceName} — consumers inject via
                 // createToken('topology:OhlcvAggregatorWorker')
                 const token = createToken(`${TOPOLOGY_TOKEN_PREFIX}${serviceName}`);
-                this.app.container.register(token, { useValue: proxy });
+                this.app.container.register(token, {
+                    useValue: createReconnectingTopologyProxy(this.app.netron, serviceName, daemonSocketUrl, proxy),
+                });
             }
             catch (err) {
                 const entry = JSON.stringify({
@@ -356,17 +671,15 @@ let BootstrapProcess = class BootstrapProcess {
                     msg: `[topology] Failed to query service '${serviceName}' from daemon: ${err.message}`,
                 });
                 process.stderr.write(entry + '\n');
-                // Register an error-throwing proxy so callers get a clear error at call-time
-                // instead of a cryptic "token not found" DI error.
-                const errorProxy = new Proxy({}, {
-                    get: (_target, prop) => {
-                        if (typeof prop === 'symbol' || prop === 'then')
-                            return undefined;
-                        return () => { throw new Error(`Topology service '${serviceName}' unavailable: ${err.message}`); };
-                    },
-                });
+                // Register a proxy that asks AGAIN on use, rather than one that
+                // throws for the life of the process. A sibling child registers its
+                // services only after every child has started, so "not found" here is
+                // the ordinary case, not a verdict — see
+                // `createDeferredTopologyProxy`.
                 const token = createToken(`${TOPOLOGY_TOKEN_PREFIX}${serviceName}`);
-                this.app.container.register(token, { useValue: errorProxy });
+                this.app.container.register(token, {
+                    useValue: createDeferredTopologyProxy(this.app.netron, serviceName, daemonSocketUrl),
+                });
             }
         }
     }
@@ -395,7 +708,6 @@ let BootstrapProcess = class BootstrapProcess {
      */
     async assertContainerIdentity(moduleResolveDir) {
         const { realpathSync, existsSync, readFileSync } = await import('node:fs');
-        const path = await import('node:path');
         const { createRequire } = await import('node:module');
         /**
          * Resolve the **package root directory** of `pkgName` from `fromUrl`.