@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/observability/metrics-bridge.js

@@ -0,0 +1,295 @@
+/**
+ * Omnitron metrics bridge — single point of contact between the daemon's
+ * runtime hooks (orchestrator events, ProcessJanitor sweeps,
+ * ResilientHandle resets, esbuild rebuilds) and the underlying
+ * `IMetricsService` registry from `@omnitron-dev/titan-metrics`.
+ *
+ * Every issue we've debugged in production was diagnosed REACTIVELY
+ * from log greps: postgres connection exhaustion (we found 207 leaked
+ * fork-workers only after pg_stat_activity told us to look), esbuild
+ * Go sidecar deaths (the build self-heal was silent until we wired up
+ * a dedicated log line), RPC name mismatches (only surfaced when an app
+ * couldn't talk to its dependency at runtime). The metrics this bridge
+ * exposes are what we wished we'd had during each of those incidents.
+ *
+ * Design notes:
+ *
+ *  - Metrics live in the titan-metrics registry; this bridge is the
+ *    DAO that types the writes. Callers don't touch the registry
+ *    directly — they call `metrics.recordEsbuildBuild(...)` etc.
+ *
+ *  - Per-app gauges (`status`, `restart_total`, `uptime_seconds`) are
+ *    refreshed on every Prometheus scrape via `refreshAppGauges()`.
+ *    The HTTP server calls it before formatting text. This avoids
+ *    wiring callbacks into every state mutation in the orchestrator.
+ *
+ *  - Event-driven counters/histograms (orphan kills, esbuild
+ *    rebuilds, lifecycle phases) are recorded eagerly via the
+ *    record* helpers, since events are sparse and we don't want to
+ *    drop them between scrapes.
+ */
+// All known statuses; we emit one row per (app, status) so Grafana can
+// pivot. Keeping the list explicit means a typo in OrchestratorService
+// produces a dropped metric instead of a silent new label value.
+const APP_STATUSES = [
+    'stopped',
+    'starting',
+    'online',
+    'stopping',
+    'errored',
+    'crashed',
+];
+export class MetricsBridge {
+    metrics;
+    logger;
+    orchestrator = null;
+    definitionsRegistered = false;
+    constructor(metrics, logger) {
+        this.metrics = metrics;
+        this.logger = logger;
+        this.registerDefinitions();
+    }
+    /** Wire the orchestrator so per-app gauges can be refreshed at scrape time. */
+    attachOrchestrator(orchestrator) {
+        this.orchestrator = orchestrator;
+    }
+    /**
+     * Refresh per-app gauges. Called by the metrics HTTP server before it
+     * formats text — keeps the gauges current without instrumenting every
+     * status mutation in the orchestrator.
+     */
+    refreshAppGauges() {
+        if (!this.orchestrator)
+            return;
+        const apps = this.orchestrator.list();
+        for (const app of apps) {
+            // One gauge series per (app, status) pair — value 1 if the app
+            // is currently in that status, else 0. Lets Grafana plot
+            // `omnitron_app_status{status="online"}` as a stacked bar.
+            for (const status of APP_STATUSES) {
+                this.gauge('omnitron_app_status', { app: app.name, status }, app.status === status ? 1 : 0);
+            }
+            this.gauge('omnitron_app_restarts', { app: app.name }, app.restarts ?? 0);
+            this.gauge('omnitron_app_uptime_seconds', { app: app.name }, app.uptime ?? 0);
+            if (typeof app.cpu === 'number') {
+                this.gauge('omnitron_app_cpu_percent', { app: app.name }, app.cpu);
+            }
+            if (typeof app.memory === 'number') {
+                this.gauge('omnitron_app_memory_bytes', { app: app.name }, app.memory);
+            }
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Event-driven recorders
+    // -------------------------------------------------------------------------
+    /** Record a janitor sweep — gauges current orphan/owned counts + bumps total counter. */
+    recordJanitorSweep(m) {
+        this.gauge('omnitron_workers_alive', {}, m.forkWorkersAlive);
+        this.gauge('omnitron_workers_owned', {}, m.ownedPids);
+        this.gauge('omnitron_workers_orphans_pending', {}, m.orphansFound - m.orphansKilled);
+        if (m.orphansKilled > 0) {
+            this.counter('omnitron_workers_orphans_killed_total', {}, m.orphansKilled);
+        }
+        if (m.killErrors > 0) {
+            this.counter('omnitron_workers_orphan_kill_errors_total', {}, m.killErrors);
+        }
+    }
+    /** Record a ResilientHandle reset — fires once per fatal-error reset across the daemon. */
+    recordResilientHandleReset(info) {
+        this.counter('resilient_handle_resets_total', { name: info.name }, 1);
+    }
+    /** Record an esbuild build (initial or rebuild). */
+    recordEsbuildBuild(app, kind, durationMs, ok) {
+        this.histogram('omnitron_esbuild_duration_seconds', { app, kind }, durationMs / 1000);
+        this.counter('omnitron_esbuild_builds_total', { app, kind, outcome: ok ? 'success' : 'failure' }, 1);
+    }
+    /** Record an app restart event. Called from orchestrator's `app:restart` listener. */
+    recordAppRestart(app, attempt, reason) {
+        this.counter('omnitron_app_restarts_total', { app, reason }, 1);
+        this.gauge('omnitron_app_restart_attempt', { app }, attempt);
+    }
+    /** Record an app crash. */
+    recordAppCrash(app) {
+        this.counter('omnitron_app_crashes_total', { app }, 1);
+    }
+    /** Record an app escalation (max restarts exceeded). */
+    recordAppEscalated(app) {
+        this.counter('omnitron_app_escalations_total', { app }, 1);
+    }
+    /** Record a lifecycle controller phase event. */
+    recordLifecyclePhase(phase, durationMs, kind) {
+        this.histogram('lifecycle_shutdown_phase_duration_seconds', { phase }, durationMs / 1000);
+        if (kind === 'timeout') {
+            this.counter('lifecycle_shutdown_phase_timeouts_total', { phase }, 1);
+        }
+    }
+    /**
+     * Permanently drop every sample for a named app from both the in-memory
+     * registry and the storage backend. Use when an app is definitively
+     * removed (handle replaced under a different canonical name, app
+     * un-registered) so it cannot linger as a "ghost offline" in subsequent
+     * snapshots.
+     *
+     * Best-effort: swallowed errors so an eviction failure can never block
+     * the orchestrator's primary lifecycle path.
+     */
+    evictApp(app) {
+        void this.metrics.evictApp(app).catch((err) => {
+            this.logger?.warn?.({ app, error: err.message }, 'Failed to evict metrics for app');
+        });
+    }
+    // -------------------------------------------------------------------------
+    // Internals
+    // -------------------------------------------------------------------------
+    /**
+     * Pre-register every metric this bridge writes. The titan-metrics
+     * registry would auto-register them with default help text on first
+     * write, but we want stable, descriptive HELP lines and explicit
+     * label sets in the Prometheus output.
+     */
+    registerDefinitions() {
+        if (this.definitionsRegistered)
+            return;
+        this.definitionsRegistered = true;
+        const registry = this.metrics.getRegistry();
+        registry.define({
+            name: 'omnitron_app_status',
+            type: 'gauge',
+            help: '1 if the app is currently in the labelled status, else 0',
+            labelNames: ['app', 'status'],
+        });
+        registry.define({
+            name: 'omnitron_app_restarts',
+            type: 'gauge',
+            help: 'Current restart counter for the app (resets on full stop)',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'omnitron_app_uptime_seconds',
+            type: 'gauge',
+            help: 'Seconds since the app last entered the online state',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'omnitron_app_cpu_percent',
+            type: 'gauge',
+            help: 'Last sampled CPU percent for the app',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'omnitron_app_memory_bytes',
+            type: 'gauge',
+            help: 'Last sampled RSS memory for the app',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'omnitron_workers_alive',
+            type: 'gauge',
+            help: 'Total fork-worker processes the daemon can see in `ps`',
+        });
+        registry.define({
+            name: 'omnitron_workers_owned',
+            type: 'gauge',
+            help: 'Fork-worker pids the daemon has registered as owned',
+        });
+        registry.define({
+            name: 'omnitron_workers_orphans_pending',
+            type: 'gauge',
+            help: 'Orphan fork-worker pids found in the last sweep but not yet killed',
+        });
+        registry.define({
+            name: 'omnitron_workers_orphans_killed_total',
+            type: 'counter',
+            help: 'Total fork-worker orphans the janitor has killed',
+        });
+        registry.define({
+            name: 'omnitron_workers_orphan_kill_errors_total',
+            type: 'counter',
+            help: 'Total errors when the janitor tried to kill an orphan',
+        });
+        registry.define({
+            name: 'resilient_handle_resets_total',
+            type: 'counter',
+            help: 'ResilientHandle reset count (cached external resource went unhealthy and was re-acquired)',
+            labelNames: ['name'],
+        });
+        registry.define({
+            name: 'omnitron_esbuild_duration_seconds',
+            type: 'histogram',
+            help: 'esbuild build/rebuild duration in seconds',
+            labelNames: ['app', 'kind'],
+        });
+        registry.define({
+            name: 'omnitron_esbuild_builds_total',
+            type: 'counter',
+            help: 'Total esbuild builds and rebuilds, partitioned by outcome',
+            labelNames: ['app', 'kind', 'outcome'],
+        });
+        registry.define({
+            name: 'omnitron_app_restarts_total',
+            type: 'counter',
+            help: 'Total app restarts, partitioned by reason',
+            labelNames: ['app', 'reason'],
+        });
+        registry.define({
+            name: 'omnitron_app_restart_attempt',
+            type: 'gauge',
+            help: 'Most recent restart attempt number for the app',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'omnitron_app_crashes_total',
+            type: 'counter',
+            help: 'Total times the app crashed (process exited with non-zero before going online)',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'omnitron_app_escalations_total',
+            type: 'counter',
+            help: 'Total times the supervisor escalated past max restarts',
+            labelNames: ['app'],
+        });
+        registry.define({
+            name: 'lifecycle_shutdown_phase_duration_seconds',
+            type: 'histogram',
+            help: 'LifecycleController shutdown phase duration in seconds',
+            labelNames: ['phase'],
+        });
+        registry.define({
+            name: 'lifecycle_shutdown_phase_timeouts_total',
+            type: 'counter',
+            help: 'Total LifecycleController phases that hit their timeout',
+            labelNames: ['phase'],
+        });
+    }
+    // Thin record helpers — wrap the unified `recordTyped` entry
+    // point so every metric the bridge emits appears in BOTH the
+    // Prometheus exposition (`/metrics`) AND the storage-backed
+    // `querySeries()` API (T#74). Errors are swallowed so a metrics-
+    // pipeline hiccup never crashes the broader daemon.
+    gauge(name, labels, value) {
+        try {
+            this.metrics.recordTyped('gauge', name, labels, value);
+        }
+        catch (err) {
+            this.logger?.warn?.({ err, name }, 'metrics-bridge: gauge write failed');
+        }
+    }
+    counter(name, labels, delta) {
+        try {
+            this.metrics.recordTyped('counter', name, labels, delta);
+        }
+        catch (err) {
+            this.logger?.warn?.({ err, name }, 'metrics-bridge: counter write failed');
+        }
+    }
+    histogram(name, labels, value) {
+        try {
+            this.metrics.recordTyped('histogram', name, labels, value);
+        }
+        catch (err) {
+            this.logger?.warn?.({ err, name }, 'metrics-bridge: histogram write failed');
+        }
+    }
+}
+//# sourceMappingURL=metrics-bridge.js.map

package/dist/observability/metrics-bridge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-bridge.js","sourceRoot":"","sources":["../../src/observability/metrics-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAQH,uEAAuE;AACvE,uEAAuE;AACvE,iEAAiE;AACjE,MAAM,YAAY,GAAG;IACnB,SAAS;IACT,UAAU;IACV,QAAQ;IACR,UAAU;IACV,SAAS;IACT,SAAS;CAC8B,CAAC;AAE1C,MAAM,OAAO,aAAa;IACP,OAAO,CAAkB;IACzB,MAAM,CAAkD;IACjE,YAAY,GAA+B,IAAI,CAAC;IAChD,qBAAqB,GAAG,KAAK,CAAC;IAEtC,YAAY,OAAwB,EAAE,MAA4C;QAChF,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC7B,CAAC;IAED,+EAA+E;IAC/E,kBAAkB,CAAC,YAAiC;QAClD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,+DAA+D;YAC/D,yDAAyD;YACzD,2DAA2D;YAC3D,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9F,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YACrE,CAAC;YACD,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,yBAAyB;IACzB,4EAA4E;IAE5E,yFAAyF;IACzF,kBAAkB,CAAC,CAAsB;QACvC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAC7D,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;QACrF,IAAI,CAAC,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,uCAAuC,EAAE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,OAAO,CAAC,2CAA2C,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,2FAA2F;IAC3F,0BAA0B,CAAC,IAAe;QACxC,IAAI,CAAC,OAAO,CAAC,+BAA+B,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,oDAAoD;IACpD,kBAAkB,CAAC,GAAW,EAAE,IAAyB,EAAE,UAAkB,EAAE,EAAW;QACxF,IAAI,CAAC,SAAS,CAAC,mCAAmC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC;QACtF,IAAI,CAAC,OAAO,CACV,+BAA+B,EAC/B,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,EAClD,CAAC,CACF,CAAC;IACJ,CAAC;IAED,sFAAsF;IACtF,gBAAgB,CAAC,GAAW,EAAE,OAAe,EAAE,MAAqC;QAClF,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;IAED,2BAA2B;IAC3B,cAAc,CAAC,GAAW;QACxB,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,wDAAwD;IACxD,kBAAkB,CAAC,GAAW;QAC5B,IAAI,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,iDAAiD;IACjD,oBAAoB,CAAC,KAAa,EAAE,UAAkB,EAAE,IAA0B;QAChF,IAAI,CAAC,SAAS,CAAC,2CAA2C,EAAE,EAAE,KAAK,EAAE,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC;QAC1F,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,yCAAyC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,QAAQ,CAAC,GAAW;QAClB,KAAK,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5C,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,EAAE,iCAAiC,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,YAAY;IACZ,4EAA4E;IAE5E;;;;;OAKG;IACK,mBAAmB;QACzB,IAAI,IAAI,CAAC,qBAAqB;YAAE,OAAO;QACvC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAE5C,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,qBAAqB;YAC3B,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,0DAA0D;YAChE,UAAU,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;SAC9B,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,uBAAuB;YAC7B,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,2DAA2D;YACjE,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,6BAA6B;YACnC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,qDAAqD;YAC3D,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,sCAAsC;YAC5C,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,2BAA2B;YACjC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,qCAAqC;YAC3C,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,wBAAwB;YAC9B,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,wDAAwD;SAC/D,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,wBAAwB;YAC9B,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,qDAAqD;SAC5D,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,kCAAkC;YACxC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,oEAAoE;SAC3E,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,uCAAuC;YAC7C,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,kDAAkD;SACzD,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,2CAA2C;YACjD,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,uDAAuD;SAC9D,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,+BAA+B;YACrC,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,2FAA2F;YACjG,UAAU,EAAE,CAAC,MAAM,CAAC;SACrB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,mCAAmC;YACzC,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,2CAA2C;YACjD,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;SAC5B,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,+BAA+B;YACrC,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,2DAA2D;YACjE,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC;SACvC,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,6BAA6B;YACnC,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,2CAA2C;YACjD,UAAU,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;SAC9B,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,8BAA8B;YACpC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,gDAAgD;YACtD,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,4BAA4B;YAClC,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,gFAAgF;YACtF,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,gCAAgC;YACtC,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,wDAAwD;YAC9D,UAAU,EAAE,CAAC,KAAK,CAAC;SACpB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,2CAA2C;YACjD,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,wDAAwD;YAC9D,UAAU,EAAE,CAAC,OAAO,CAAC;SACtB,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC;YACd,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,yDAAyD;YAC/D,UAAU,EAAE,CAAC,OAAO,CAAC;SACtB,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,6DAA6D;IAC7D,4DAA4D;IAC5D,iEAAiE;IACjE,oDAAoD;IAC5C,KAAK,CAAC,IAAY,EAAE,MAA8B,EAAE,KAAa;QACvE,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,oCAAoC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAEO,OAAO,CAAC,IAAY,EAAE,MAA8B,EAAE,KAAa;QACzE,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,sCAAsC,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,MAA8B,EAAE,KAAa;QAC3E,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,wCAAwC,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;CACF"}

package/dist/observability/metrics-server.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Standalone HTTP server for Prometheus scraping.
+ *
+ * Bound to a dedicated port (default `httpPort + 3` ≈ 9803) so that
+ * scraping is isolated from the main RPC port: a stuck Netron handler
+ * can't starve `/metrics`, and scraping can't accidentally generate
+ * RPC load. Two routes only:
+ *
+ *   GET /metrics  → text/plain Prometheus exposition (refreshes per-app
+ *                   gauges via MetricsBridge.refreshAppGauges() before
+ *                   formatting, so the snapshot is always current).
+ *   GET /healthz  → 200 OK / "ok\n" — used by liveness checks.
+ *
+ * Anything else returns 404. We deliberately don't expose more — if
+ * we ever need richer introspection, it goes through the daemon's RPC
+ * surface, which has authentication.
+ */
+import type { IMetricsService } from '@omnitron-dev/titan-metrics';
+import type { ILogger } from '@omnitron-dev/titan/module/logger';
+import type { MetricsBridge } from './metrics-bridge.js';
+export interface MetricsServerOptions {
+    /** Listen port. */
+    port: number;
+    /**
+     * Bind host. **Default `127.0.0.1` (loopback)** as of T#65. The
+     * historical default of `0.0.0.0` exposed `/metrics` on every
+     * network interface — internal aggregated traffic counters,
+     * per-app health, infrastructure topology — to anyone on the same
+     * network. Production scraping by a co-located Prometheus on
+     * loopback is unaffected. Remote scraping must opt in by
+     * setting host = '0.0.0.0' AND configuring a bearer token via
+     * `authToken`.
+     */
+    host?: string;
+    /**
+     * Optional bearer token required on `Authorization` header
+     * (T#65). When set, requests without `Authorization: Bearer
+     * <token>` get 401. `/healthz` is always public so external
+     * health probes work; only `/metrics` is gated. Without an
+     * `authToken` set, the endpoint is unauthenticated — which is
+     * safe ONLY while bound to loopback. The server logs a warning
+     * at start when host is non-loopback AND no token is set.
+     */
+    authToken?: string;
+    /** Metric service backing `/metrics`. */
+    metrics: IMetricsService;
+    /** Bridge for refreshing per-app gauges before formatting. */
+    bridge: MetricsBridge;
+    /** Logger. */
+    logger: ILogger;
+}
+export declare class MetricsServer {
+    private server;
+    private readonly opts;
+    constructor(opts: MetricsServerOptions);
+    start(): Promise<{
+        port: number;
+    }>;
+    stop(): Promise<void>;
+    private handle;
+    /**
+     * T#65: constant-time compare against the configured bearer
+     * token. Accepts `Authorization: Bearer <token>` only. Returns
+     * `true` if the request carries a matching token.
+     */
+    private authorized;
+}
+//# sourceMappingURL=metrics-server.d.ts.map

package/dist/observability/metrics-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-server.d.ts","sourceRoot":"","sources":["../../src/observability/metrics-server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,oBAAoB;IACnC,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,OAAO,EAAE,eAAe,CAAC;IACzB,8DAA8D;IAC9D,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc;IACd,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAuB;IAE5C,YAAY,IAAI,EAAE,oBAAoB,EAErC;IAEK,KAAK,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAgDvC;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAY1B;YAEa,MAAM;IA6CpB;;;;OAIG;IACH,OAAO,CAAC,UAAU;CASnB"}

package/dist/observability/metrics-server.js

@@ -0,0 +1,146 @@
+/**
+ * Standalone HTTP server for Prometheus scraping.
+ *
+ * Bound to a dedicated port (default `httpPort + 3` ≈ 9803) so that
+ * scraping is isolated from the main RPC port: a stuck Netron handler
+ * can't starve `/metrics`, and scraping can't accidentally generate
+ * RPC load. Two routes only:
+ *
+ *   GET /metrics  → text/plain Prometheus exposition (refreshes per-app
+ *                   gauges via MetricsBridge.refreshAppGauges() before
+ *                   formatting, so the snapshot is always current).
+ *   GET /healthz  → 200 OK / "ok\n" — used by liveness checks.
+ *
+ * Anything else returns 404. We deliberately don't expose more — if
+ * we ever need richer introspection, it goes through the daemon's RPC
+ * surface, which has authentication.
+ */
+import http from 'node:http';
+export class MetricsServer {
+    server = null;
+    opts;
+    constructor(opts) {
+        this.opts = opts;
+    }
+    async start() {
+        if (this.server) {
+            return { port: this.server.address().port };
+        }
+        const server = http.createServer((req, res) => {
+            void this.handle(req, res).catch((err) => {
+                this.opts.logger.error({ err, url: req.url }, 'metrics server: handler crash');
+                if (!res.headersSent) {
+                    res.statusCode = 500;
+                    res.end('internal error\n');
+                }
+            });
+        });
+        server.keepAliveTimeout = 5_000;
+        server.headersTimeout = 6_000;
+        // T#65: default bind to loopback so the metrics endpoint isn't
+        // reachable from outside the host without an explicit opt-in.
+        const host = this.opts.host ?? '127.0.0.1';
+        await new Promise((resolve, reject) => {
+            server.once('error', reject);
+            server.listen(this.opts.port, host, () => {
+                server.off('error', reject);
+                resolve();
+            });
+        });
+        this.server = server;
+        const address = server.address();
+        this.opts.logger.info({ port: address.port, host: address.address, authenticated: Boolean(this.opts.authToken) }, 'metrics server: listening on /metrics');
+        // T#65: warn loudly if the operator bound to a non-loopback
+        // address WITHOUT setting an auth token. That combination
+        // hands `/metrics` to anyone on the network — usually not
+        // intended, especially in cloud deploys where the daemon's
+        // host is reachable from peer workloads.
+        const isLoopback = host === '127.0.0.1' || host === '::1' || host === 'localhost' || host === '127.1';
+        if (!isLoopback && !this.opts.authToken) {
+            this.opts.logger.warn({ host, port: address.port }, 'metrics server: bound to non-loopback WITHOUT authToken — `/metrics` is publicly readable (T#65)');
+        }
+        return { port: address.port };
+    }
+    async stop() {
+        if (!this.server)
+            return;
+        const server = this.server;
+        this.server = null;
+        await new Promise((resolve) => {
+            server.close(() => resolve());
+            // close() rejects new connections but waits for in-flight ones to
+            // finish naturally. Force close keep-alive connections so we
+            // don't drag shutdown out — they hold sockets idle for keepAliveTimeout.
+            server.closeIdleConnections?.();
+        });
+        this.opts.logger.info({}, 'metrics server: stopped');
+    }
+    async handle(req, res) {
+        const url = req.url ?? '/';
+        if (req.method !== 'GET') {
+            res.statusCode = 405;
+            res.setHeader('Allow', 'GET');
+            res.end('method not allowed\n');
+            return;
+        }
+        if (url === '/healthz' || url === '/health') {
+            res.statusCode = 200;
+            res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+            res.end('ok\n');
+            return;
+        }
+        if (url === '/metrics' || url.startsWith('/metrics?')) {
+            // T#65: gate on bearer token when configured. `/healthz`
+            // remains public so external health probes still work; only
+            // `/metrics` carries the sensitive aggregate data.
+            if (this.opts.authToken && !this.authorized(req)) {
+                res.statusCode = 401;
+                res.setHeader('WWW-Authenticate', 'Bearer realm="omnitron-metrics"');
+                res.end('unauthorized\n');
+                return;
+            }
+            try {
+                this.opts.bridge.refreshAppGauges();
+            }
+            catch (err) {
+                this.opts.logger.warn({ err }, 'metrics server: refreshAppGauges threw — emitting last-known values');
+            }
+            const text = await this.opts.metrics.getPrometheusText();
+            res.statusCode = 200;
+            res.setHeader('Content-Type', 'text/plain; version=0.0.4; charset=utf-8');
+            res.end(text);
+            return;
+        }
+        res.statusCode = 404;
+        res.end('not found\n');
+    }
+    /**
+     * T#65: constant-time compare against the configured bearer
+     * token. Accepts `Authorization: Bearer <token>` only. Returns
+     * `true` if the request carries a matching token.
+     */
+    authorized(req) {
+        const expected = this.opts.authToken;
+        if (!expected)
+            return true; // no auth configured → public
+        const header = req.headers['authorization'];
+        if (typeof header !== 'string')
+            return false;
+        const m = header.match(/^Bearer\s+(.+)$/);
+        if (!m)
+            return false;
+        return timingSafeEqual(m[1], expected);
+    }
+}
+function timingSafeEqual(a, b) {
+    // Constant-time string compare. Returns false fast on differing
+    // length (the length itself isn't a secret in this protocol), and
+    // OR-accumulates byte differences across the shorter length.
+    if (a.length !== b.length)
+        return false;
+    let diff = 0;
+    for (let i = 0; i < a.length; i++)
+        diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+    return diff === 0;
+}
+//# sourceMappingURL=metrics-server.js.map

package/dist/observability/metrics-server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-server.js","sourceRoot":"","sources":["../../src/observability/metrics-server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,IAAI,MAAM,WAAW,CAAC;AAsC7B,MAAM,OAAO,aAAa;IAChB,MAAM,GAAuB,IAAI,CAAC;IACzB,IAAI,CAAuB;IAE5C,YAAY,IAA0B;QACpC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,EAAE,IAAI,EAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAkB,CAAC,IAAI,EAAE,CAAC;QAC/D,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5C,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACvC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBAC/E,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;oBACrB,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;oBACrB,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAChC,MAAM,CAAC,cAAc,GAAG,KAAK,CAAC;QAE9B,+DAA+D;QAC/D,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;QAE3C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE;gBACvC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAiB,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACnB,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAC1F,uCAAuC,CACxC,CAAC;QAEF,4DAA4D;QAC5D,0DAA0D;QAC1D,0DAA0D;QAC1D,2DAA2D;QAC3D,yCAAyC;QACzC,MAAM,UAAU,GACd,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,OAAO,CAAC;QACrF,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACnB,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,EAC5B,kGAAkG,CACnG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,kEAAkE;YAClE,6DAA6D;YAC7D,yEAAyE;YACzE,MAAM,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,yBAAyB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAyB,EAAE,GAAwB;QACtE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC;QAC3B,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YACzB,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YACrB,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC9B,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAED,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YAC5C,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YACrB,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,2BAA2B,CAAC,CAAC;YAC3D,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChB,OAAO;QACT,CAAC;QAED,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACtD,yDAAyD;YACzD,4DAA4D;YAC5D,mDAAmD;YACnD,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;gBACrB,GAAG,CAAC,SAAS,CAAC,kBAAkB,EAAE,iCAAiC,CAAC,CAAC;gBACrE,GAAG,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;gBAC1B,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACtC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACnB,EAAE,GAAG,EAAE,EACP,qEAAqE,CACtE,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACzD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YACrB,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,0CAA0C,CAAC,CAAC;YAC1E,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACd,OAAO;QACT,CAAC;QAED,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QACrB,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACzB,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAyB;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QACrC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,CAAC,8BAA8B;QAC1D,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAC5C,IAAI,OAAO,MAAM,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC1C,IAAI,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACrB,OAAO,eAAe,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,SAAS,eAAe,CAAC,CAAS,EAAE,CAAS;IAC3C,gEAAgE;IAChE,kEAAkE;IAClE,6DAA6D;IAC7D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO,IAAI,KAAK,CAAC,CAAC;AACpB,CAAC"}