geonix 1.33.2 → 1.35.1

package/README.md

@@ -298,6 +298,7 @@ Each returned part has:
 | `GX_HEALTH_TIMEOUT` | `2000` | Registry health-probe timeout per advertised address (ms) |
 | `GX_SECRET` | — | Encryption key: AES-256-GCM payloads + HMAC-SHA256 subjects. Services without the same key cannot communicate. |
 | `GX_DEBUG_ENDPOINT` | — | Mount path for the debug router (e.g. `/_debug`). Disabled when unset. |
+| `GX_DEBUG_ENDPOINT_AUTH` | — | Basic-auth credentials for the debug router as `username:password`. When set, all debug routes (and the UI) require `Authorization: Basic …`. OPTIONS requests bypass to keep CORS preflight working. |
 
 ## Local transport (`GX_TRANSPORT=local://`)
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "geonix",
-    "version": "1.33.2",
+    "version": "1.35.1",
     "type": "module",
     "description": "",
     "bin": {

package/src/Gateway.js

@@ -10,6 +10,7 @@ import { logger } from "./Logger.js";
 import { readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { dirname, join } from "node:path";
+import { timingSafeEqual } from "node:crypto";
 
 const DEBUG_HTML = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "debug.html"), "utf8");
 
@@ -43,12 +44,46 @@ const requestLogger = (req, res, next) => {
 };
 
 const stats = {
-    requests: 0,
-    proxied: 0,
-    proxied_over_nats: 0,
-    debug_requests: 0,
+    requests: 0,           // total HTTP requests seen by the gateway
+    proxied: 0,            // requests forwarded to a backend via HTTP
+    proxied_over_nats: 0,  // requests forwarded via NATS tunnel (no entry.a)
+    errors: 0,             // proxyHttp threw (network error to backend)
+    debug_requests: 0,     // count of /<debug>/* hits
+    startedAt: Date.now(), // gateway boot timestamp (ms)
+    routerRebuilds: 0,     // how many times the route table was rebuilt
+    byMethod: {},          // { GET: N, POST: N, ... }
+    byStatus: { "2xx": 0, "3xx": 0, "4xx": 0, "5xx": 0, other: 0 },
+    byBackend: {},         // { "host:port": count }
+    ws: { active: 0, total: 0, errors: 0 },
 };
 
+// Latency ring buffer — 1000 most recent request durations in ms. ~8 KB. Reads
+// are O(N log N) for percentile compute; writes are O(1). Cheap enough to leave
+// always-on.
+const LATENCY_BUF_SIZE = 1000;
+const _latencyBuf = new Float64Array(LATENCY_BUF_SIZE);
+let _latencyIdx = 0;
+let _latencyCount = 0;
+function recordLatency(ms) {
+    _latencyBuf[_latencyIdx] = ms;
+    _latencyIdx = (_latencyIdx + 1) % LATENCY_BUF_SIZE;
+    if (_latencyCount < LATENCY_BUF_SIZE) { _latencyCount++; }
+}
+function latencyPercentiles() {
+    if (_latencyCount === 0) { return null; }
+    const slice = Array.from(_latencyBuf.slice(0, _latencyCount)).sort((a, b) => a - b);
+    const pick = (q) => slice[Math.min(Math.floor(slice.length * q), slice.length - 1)];
+    return {
+        sampleSize: _latencyCount,
+        bufferSize: LATENCY_BUF_SIZE,
+        p50: pick(0.50),
+        p95: pick(0.95),
+        p99: pick(0.99),
+        max: slice[slice.length - 1],
+        mean: slice.reduce((a, b) => a + b, 0) / slice.length,
+    };
+}
+
 const defaultOpts = {
     beforeRequest: (_req, _res) => {},
     afterRequest: (_req, _res) => {},
@@ -159,6 +194,20 @@ export class Gateway {
             next();
         });
 
+        // per-request tracking: method count + status bucket + duration on response finish
+        this.#api.use((req, res, next) => {
+            const t0 = process.hrtime.bigint();
+            stats.byMethod[req.method] = (stats.byMethod[req.method] || 0) + 1;
+            res.on("finish", () => {
+                const ms = Number(process.hrtime.bigint() - t0) / 1e6;
+                recordLatency(ms);
+                const code = res.statusCode || 0;
+                const bucket = code >= 500 ? "5xx" : code >= 400 ? "4xx" : code >= 300 ? "3xx" : code >= 200 ? "2xx" : "other";
+                stats.byStatus[bucket] = (stats.byStatus[bucket] || 0) + 1;
+            });
+            next();
+        });
+
         // handle mapped endpoints as service calls
         this.#api.use((req, res, next) => {
             stats.requests++;
@@ -294,6 +343,39 @@ export class Gateway {
     #debugRouter() {
         const router = Router();
 
+        // Basic Auth gate: when GX_DEBUG_ENDPOINT_AUTH is set as "user:pass", all
+        // debug routes require a matching Authorization: Basic <base64> header.
+        // OPTIONS preflight bypasses so cross-origin CORS still works.
+        router.use((req, res, next) => {
+            const expected = process.env.GX_DEBUG_ENDPOINT_AUTH;
+            if (!expected) {
+                return next();
+            }
+            if (req.method === "OPTIONS") {
+                return next();
+            }
+            const challenge = (msg) => {
+                res.set("WWW-Authenticate", 'Basic realm="Geonix Debug", charset="UTF-8"');
+                res.status(401).json({ error: msg });
+            };
+            const header = req.headers.authorization;
+            if (!header || !header.startsWith("Basic ")) {
+                return challenge("authentication required");
+            }
+            let provided;
+            try {
+                provided = Buffer.from(header.slice(6).trim(), "base64").toString("utf8");
+            } catch {
+                return challenge("invalid Authorization header");
+            }
+            const a = Buffer.from(expected);
+            const b = Buffer.from(provided);
+            if (a.length !== b.length || !timingSafeEqual(a, b)) {
+                return challenge("invalid credentials");
+            }
+            next();
+        });
+
         router.use((req, res, next) => {
             stats.debug_requests++;
 
@@ -322,6 +404,24 @@ export class Gateway {
             res.send(stats);
         });
 
+        // Aggregated gateway view — counters + computed latency percentiles + live
+        // gauges that aren't in the global stats object (session table, active proxies,
+        // endpoint count). Single endpoint to back the UI's Gateway tab.
+        router.get("/gateway", (req, res) => {
+            const activeProxies = Object.values(this.#registry).filter((e) => e.proxy).length;
+            res.json({
+                ...stats,
+                uptimeSec: Math.floor((Date.now() - stats.startedAt) / 1000),
+                latency: latencyPercentiles(),
+                sessions: this.#sessions.size,
+                maxSessions: MAX_SESSIONS,
+                activeNatsProxies: activeProxies,
+                endpoints: this.#endpoints.length,
+                port: this.#port,
+                pid: process.pid,
+            });
+        });
+
         router.get("/ui", (req, res) => {
             res.set("Content-Type", "text/html; charset=utf-8");
             res.send(DEBUG_HTML);
@@ -427,20 +527,32 @@ export class Gateway {
     #proxyWebsocket(target, inbound, req) {
         try {
             const backend = new WebSocket(target, { headers: { ...req.headers } });
+            stats.ws.total++;
+            stats.ws.active++;
+            let decremented = false;
+            const decrement = () => {
+                if (decremented) { return; }
+                decremented = true;
+                stats.ws.active = Math.max(0, stats.ws.active - 1);
+            };
 
             backend.on("error", (e) => {
+                stats.ws.errors++;
                 logger.error("proxy.ws.backend.error:", e);
                 inbound.close();
+                decrement();
             });
 
             backend.on("open", () => {
                 backend.on("message", (data, isBinary) => inbound.send(isBinary ? data : data.toString()));
                 inbound.on("message", (data) => backend.send(data));
 
-                backend.on("close", () => inbound.close());
-                inbound.on("close", () => backend.close());
+                backend.on("close", () => { inbound.close(); decrement(); });
+                inbound.on("close", () => { backend.close(); decrement(); });
             });
         } catch (e) {
+            stats.ws.errors++;
+            stats.ws.active = Math.max(0, stats.ws.active - 1);
             logger.error("proxy.ws.backend.error:", e);
             inbound.close();
         }
@@ -455,6 +567,7 @@ export class Gateway {
 
         this.#rebuildRouter = false;
         this.#buildRouterRunning = true;
+        stats.routerRebuilds++;
 
         try {
             const router = Router();
@@ -566,11 +679,13 @@ export class Gateway {
                     router[verb](uri, async (req, res, _next) => {
                         stats.proxied++;
                         const backend = endpoint.backend[endpoint.requests++ % endpoint.backend.length];
+                        stats.byBackend[backend] = (stats.byBackend[backend] || 0) + 1;
 
                         try {
                             logger.debug("proxy.web.to:", backend + req.originalUrl);
                             await proxyHttp(`http://${backend}`, req, res);
                         } catch (e) {
+                            stats.errors++;
                             logger.error("proxy.web.error:", e);
                         }
                     });

package/src/debug.html

@@ -132,6 +132,40 @@
         .hint { color: var(--muted); font-size: 11px; }
         .invoke-grid { display: grid; grid-template-columns: 380px 1fr; gap: 16px; align-items: start; }
         @media (max-width: 900px) { .invoke-grid { grid-template-columns: 1fr; } }
+        /* Sortable tables: th becomes click-to-sort; matrix headers stay text and excluded. */
+        table.sortable th[data-key] { cursor: pointer; user-select: none; }
+        table.sortable th[data-key]:hover { color: var(--text); }
+        table.sortable th.sort-asc::after  { content: " ▲"; color: var(--accent); font-size: 9px; }
+        table.sortable th.sort-desc::after { content: " ▼"; color: var(--accent); font-size: 9px; }
+        .filter {
+            background: var(--bg); color: var(--text);
+            border: 1px solid var(--border); border-radius: 6px;
+            padding: 6px 10px; font: inherit; font-family: var(--mono); font-size: 12px;
+            width: 100%; max-width: 360px; margin-bottom: 12px;
+        }
+        .filter:focus { outline: none; border-color: var(--accent); }
+        /* Version-color pills — hue derived from a hash of the version string */
+        .vpill {
+            display: inline-block; padding: 2px 8px; border-radius: 12px;
+            font-family: var(--mono); font-size: 11px; font-weight: 500;
+            border: 1px solid; cursor: default;
+        }
+        /* Cluster-health card on Overview — clickable tile leading to a relevant tab */
+        .health-row { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 12px; margin-bottom: 16px; }
+        .health-tile {
+            background: var(--panel); border: 1px solid var(--border); border-radius: 8px;
+            padding: 12px 16px; cursor: pointer;
+            display: flex; align-items: center; gap: 12px;
+        }
+        .health-tile:hover { border-color: var(--accent); }
+        .health-tile .dot { width: 10px; height: 10px; border-radius: 50%; flex-shrink: 0; }
+        .health-tile .dot.ok    { background: var(--green); box-shadow: 0 0 8px rgba(63,185,80,0.4); }
+        .health-tile .dot.warn  { background: var(--amber); box-shadow: 0 0 8px rgba(210,153,34,0.4); }
+        .health-tile .dot.err   { background: var(--red);   box-shadow: 0 0 8px rgba(248,81,73,0.4); }
+        .health-tile .dot.muted { background: var(--muted); }
+        .health-tile .body { display: flex; flex-direction: column; flex: 1; }
+        .health-tile .lbl  { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: 0.5px; }
+        .health-tile .val  { font-family: var(--mono); font-size: 14px; }
     </style>
 </head>
 <body>
@@ -139,9 +173,12 @@
         <h1>Geonix Debug<span class="gv" id="gv"></span></h1>
         <nav class="tabs">
             <button data-tab="overview" class="active">Overview</button>
+            <button data-tab="gateway">Gateway</button>
             <button data-tab="registry">Registry</button>
             <button data-tab="reachability">Reachability</button>
             <button data-tab="endpoints">Endpoints</button>
+            <button data-tab="versions">Versions</button>
+            <button data-tab="stats">Stats</button>
             <button data-tab="invoke">Invoke</button>
             <button data-tab="raw">Raw JSON</button>
         </nav>
@@ -153,9 +190,12 @@
 
     <main>
         <section id="overview" class="tab active"><div class="loading">Loading…</div></section>
+        <section id="gateway" class="tab"><div class="loading">Loading…</div></section>
         <section id="registry" class="tab"><div class="loading">Loading…</div></section>
         <section id="reachability" class="tab"><div class="loading">Loading…</div></section>
         <section id="endpoints" class="tab"><div class="loading">Loading…</div></section>
+        <section id="versions" class="tab"><div class="loading">Loading…</div></section>
+        <section id="stats" class="tab"><div class="loading">Loading…</div></section>
         <section id="invoke" class="tab"><div class="loading">Loading…</div></section>
         <section id="raw" class="tab"><div class="loading">Loading…</div></section>
         <section id="instance" class="tab"><div class="loading">Loading…</div></section>
@@ -194,6 +234,117 @@
         return `${Math.round(d / 3_600_000)}h`;
     };
 
+    // -----------------------------------------------------------------------
+    // Version → color helper. Stable hash to a small palette so the same string
+    // always renders with the same color across tabs. Purely visual — version
+    // mismatches are informational, not flagged as problems.
+    // -----------------------------------------------------------------------
+
+    const _vPalette = [
+        ["rgba(88,166,255,0.15)",  "rgba(88,166,255,0.5)",  "#79c0ff"],  // blue
+        ["rgba(166,114,255,0.15)", "rgba(166,114,255,0.5)", "#bc8cff"],  // purple
+        ["rgba(63,185,80,0.15)",   "rgba(63,185,80,0.5)",   "#7ee787"],  // green
+        ["rgba(210,153,34,0.15)",  "rgba(210,153,34,0.5)",  "#e3b341"],  // amber
+        ["rgba(255,123,114,0.15)", "rgba(255,123,114,0.5)", "#ff9492"],  // red
+        ["rgba(86,211,200,0.15)",  "rgba(86,211,200,0.5)",  "#76e3da"],  // teal
+        ["rgba(255,153,204,0.15)", "rgba(255,153,204,0.5)", "#ffadd5"],  // pink
+        ["rgba(160,160,160,0.15)", "rgba(160,160,160,0.5)", "#c9d1d9"],  // grey
+    ];
+    function _hashIdx(s, mod) {
+        let h = 0;
+        for (let i = 0; i < s.length; i++) { h = ((h << 5) - h + s.charCodeAt(i)) | 0; }
+        return Math.abs(h) % mod;
+    }
+    function vpill(value) {
+        if (value == null || value === "") { return '<span class="pill muted">—</span>'; }
+        const [bg, border, fg] = _vPalette[_hashIdx(String(value), _vPalette.length)];
+        return `<span class="vpill" style="background:${bg};border-color:${border};color:${fg};" title="${esc(value)}">${esc(value)}</span>`;
+    }
+
+    // -----------------------------------------------------------------------
+    // Sortable + filterable table helper. Sort state is module-level so it
+    // survives auto-refresh and tab switches. `columns` is an array of
+    // { key, label, type, sortable, format }; `type` selects the comparator.
+    // -----------------------------------------------------------------------
+
+    const sortStates = new Map();
+
+    function compareFor(type) {
+        if (type === "number" || type === "date") {
+            return (a, b) => (Number(a) || 0) - (Number(b) || 0);
+        }
+        if (type === "version") {
+            return (a, b) => {
+                const pa = String(a ?? "").split(".").map((n) => parseInt(n, 10) || 0);
+                const pb = String(b ?? "").split(".").map((n) => parseInt(n, 10) || 0);
+                for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+                    const d = (pa[i] || 0) - (pb[i] || 0);
+                    if (d !== 0) { return d; }
+                }
+                return 0;
+            };
+        }
+        if (type === "addresses" || type === "length") {
+            return (a, b) => (a?.length ?? 0) - (b?.length ?? 0);
+        }
+        return (a, b) => String(a ?? "").localeCompare(String(b ?? ""));
+    }
+
+    function applySort(rows, tableId, columns) {
+        const state = sortStates.get(tableId);
+        if (!state) { return rows; }
+        const col = columns.find((c) => c.key === state.key);
+        if (!col) { return rows; }
+        const cmp = compareFor(col.type || "string");
+        const mult = state.dir === "desc" ? -1 : 1;
+        return [...rows].sort((a, b) => cmp(a[state.key], b[state.key]) * mult);
+    }
+
+    function renderTable(tableId, rows, columns, renderRow) {
+        const state = sortStates.get(tableId);
+        const headers = columns.map((c) => {
+            const sortable = c.sortable !== false;
+            const cls = state?.key === c.key ? `sort-${state.dir}` : "";
+            return sortable
+                ? `<th data-key="${esc(c.key)}" class="${cls}">${esc(c.label)}</th>`
+                : `<th>${esc(c.label)}</th>`;
+        }).join("");
+        const body = rows.map(renderRow).join("");
+        return `<table class="sortable" data-table-id="${esc(tableId)}">
+            <thead><tr>${headers}</tr></thead>
+            <tbody>${body || ""}</tbody>
+        </table>`;
+    }
+
+    function attachSortHandlers(scopeEl, tableId, onChange) {
+        scopeEl.querySelectorAll(`table[data-table-id="${tableId}"] th[data-key]`).forEach((th) => {
+            th.addEventListener("click", () => {
+                const cur = sortStates.get(tableId);
+                const key = th.dataset.key;
+                const dir = cur?.key === key && cur.dir === "asc" ? "desc" : "asc";
+                sortStates.set(tableId, { key, dir });
+                onChange();
+            });
+        });
+    }
+
+    function attachRowClicks(scopeEl) {
+        scopeEl.querySelectorAll("tr.clickable[data-id]").forEach((tr) => {
+            tr.addEventListener("click", () => { window.location.hash = `instance=${encodeURIComponent(tr.dataset.id)}`; });
+        });
+    }
+
+    function applyFilter(rows, query, keys) {
+        const q = (query || "").trim().toLowerCase();
+        if (!q) { return rows; }
+        return rows.filter((r) => keys.some((k) => {
+            const v = r[k];
+            if (v == null) { return false; }
+            if (Array.isArray(v)) { return v.some((x) => String(x).toLowerCase().includes(q)); }
+            return String(v).toLowerCase().includes(q);
+        }));
+    }
+
     // -----------------------------------------------------------------------
     // Routing — `#tab` for tabs, `#instance=<id>` for instance detail
     // -----------------------------------------------------------------------
@@ -219,9 +370,12 @@
         document.getElementById(tab)?.classList.add("active");
         document.querySelector(`nav.tabs button[data-tab="${tab}"]`)?.classList.add("active");
         if (tab === "overview")     { return renderOverview(); }
+        if (tab === "gateway")      { return renderGateway(); }
         if (tab === "registry")     { return renderRegistry(); }
         if (tab === "reachability") { return renderReachability(); }
         if (tab === "endpoints")    { return renderEndpoints(); }
+        if (tab === "versions")     { return renderVersions(); }
+        if (tab === "stats")        { return renderStats(); }
         if (tab === "invoke")       { return renderInvoke(); }
         if (tab === "raw")          { return renderRaw(); }
     }
@@ -256,9 +410,63 @@
             document.getElementById("gv").textContent = ` v${info.geonix}`;
             const instanceCount = Object.keys(reg).length;
             const emptyAddrCount = Object.values(reg).filter((e) => !e.a?.length).length;
+            const gxVersions = new Set(Object.values(reg).map((e) => e.gx ?? "—"));
+
+            // Cluster-health tiles. Version diversity is INFORMATIONAL — neutral dot, not amber.
+            const healthTiles = [
+                {
+                    label: "Geonix versions",
+                    val: `${gxVersions.size} detected`,
+                    dotClass: gxVersions.size <= 1 ? "ok" : "muted",
+                    href: "#versions",
+                },
+                {
+                    label: "Instances with empty .a",
+                    val: String(emptyAddrCount),
+                    dotClass: emptyAddrCount === 0 ? "ok" : "warn",
+                    href: "#registry",
+                },
+                {
+                    label: "Services with >1 version",
+                    val: (() => {
+                        const counts = new Map();
+                        for (const e of Object.values(reg)) {
+                            const c = counts.get(e.n) ?? new Set();
+                            c.add(e.v);
+                            counts.set(e.n, c);
+                        }
+                        return [...counts.values()].filter((s) => s.size > 1).length.toString();
+                    })(),
+                    dotClass: "muted",  // informational, not a problem
+                    href: "#versions",
+                },
+            ];
+
+            // Service-count overview table — sortable
+            const svcCounts = services.map((s) => {
+                const count = Object.values(reg).filter((e) => `${e.n}@${e.v}` === s).length;
+                return { sv: s, count };
+            });
+            const svcColumns = [
+                { key: "sv",    label: "Name@Version", type: "string" },
+                { key: "count", label: "Instances",    type: "number" },
+            ];
+            if (!sortStates.has("overview-services")) { sortStates.set("overview-services", { key: "sv", dir: "asc" }); }
+            const renderSvcRow = (r) => `<tr><td>${esc(r.sv)}</td><td>${esc(r.count)}</td></tr>`;
 
             el.innerHTML = `
                 <h2>Overview</h2>
+                <div class="health-row">
+                    ${healthTiles.map((t) => `
+                        <a class="health-tile" href="${esc(t.href)}">
+                            <span class="dot ${t.dotClass}"></span>
+                            <div class="body">
+                                <span class="lbl">${esc(t.label)}</span>
+                                <span class="val">${esc(t.val)}</span>
+                            </div>
+                        </a>
+                    `).join("")}
+                </div>
                 <div class="cards">
                     <div class="card"><div class="label">Geonix</div><div class="value">${esc(info.geonix)}</div></div>
                     <div class="card"><div class="label">Node</div><div class="value small">${esc(info.node.version)} / ${esc(info.node.platform)} ${esc(info.node.arch)}</div></div>
@@ -268,23 +476,186 @@
                     <div class="card"><div class="label">Endpoints</div><div class="value">${endpoints.length}</div></div>
                     <div class="card"><div class="label">Empty .a</div><div class="value" style="color:${emptyAddrCount > 0 ? "var(--amber)" : "var(--green)"}">${emptyAddrCount}</div></div>
                 </div>
-                <h3>Stats</h3>
+                <h3>Gateway Stats</h3>
                 <div class="cards">
                     ${Object.entries(stats).map(([k, v]) => `
                         <div class="card"><div class="label">${esc(k)}</div><div class="value">${esc(v)}</div></div>
                     `).join("")}
                 </div>
                 <h3>Services</h3>
-                <table>
-                    <thead><tr><th>Name@Version</th><th>Instances</th></tr></thead>
-                    <tbody>
-                        ${services.map((s) => {
-                            const count = Object.values(reg).filter((e) => `${e.n}@${e.v}` === s).length;
-                            return `<tr><td>${esc(s)}</td><td>${count}</td></tr>`;
-                        }).join("")}
-                    </tbody>
-                </table>
+                <div id="overview-svc-wrap"></div>
             `;
+
+            const drawSvc = () => {
+                const sorted = applySort(svcCounts, "overview-services", svcColumns);
+                document.getElementById("overview-svc-wrap").innerHTML = renderTable("overview-services", sorted, svcColumns, renderSvcRow);
+                attachSortHandlers(el, "overview-services", drawSvc);
+            };
+            drawSvc();
+        } catch (e) {
+            el.innerHTML = `<div class="err-banner">${esc(e.message)}</div>`;
+        }
+    }
+
+    // -----------------------------------------------------------------------
+    // Gateway — counters, latency percentiles, and live gauges from /gateway
+    // -----------------------------------------------------------------------
+
+    function fmtDuration(sec) {
+        if (sec == null) { return "—"; }
+        const s = Math.floor(sec);
+        const days = Math.floor(s / 86400);
+        const hrs = Math.floor((s % 86400) / 3600);
+        const min = Math.floor((s % 3600) / 60);
+        const ss = s % 60;
+        if (days > 0) { return `${days}d ${hrs}h ${min}m`; }
+        if (hrs > 0)  { return `${hrs}h ${min}m ${ss}s`; }
+        if (min > 0)  { return `${min}m ${ss}s`; }
+        return `${ss}s`;
+    }
+
+    async function renderGateway() {
+        const el = document.getElementById("gateway");
+        el.innerHTML = '<div class="loading">Loading…</div>';
+        try {
+            const gw = await api("/gateway");
+
+            // Top summary cards
+            const httpTotal = gw.requests || 0;
+            const proxiedShare = httpTotal > 0 ? ((gw.proxied / httpTotal) * 100).toFixed(1) : "—";
+            const errRate = (gw.proxied + gw.proxied_over_nats) > 0
+                ? ((gw.errors / (gw.proxied + gw.proxied_over_nats)) * 100).toFixed(2)
+                : "—";
+
+            // Status code rows
+            const statusOrder = ["2xx", "3xx", "4xx", "5xx", "other"];
+            const statusRows = statusOrder.map((k) => ({
+                bucket: k,
+                count: gw.byStatus?.[k] ?? 0,
+            }));
+            const statusColumns = [
+                { key: "bucket", label: "Bucket", type: "string" },
+                { key: "count",  label: "Count",  type: "number" },
+            ];
+            if (!sortStates.has("gw-status")) { sortStates.set("gw-status", { key: "bucket", dir: "asc" }); }
+            const renderStatusRow = (r) => {
+                const cls = r.bucket === "2xx" ? "ok"
+                          : r.bucket === "3xx" ? "muted"
+                          : r.bucket === "4xx" ? "warn"
+                          : r.bucket === "5xx" ? "err" : "muted";
+                return `<tr><td><span class="pill ${cls}">${esc(r.bucket)}</span></td><td>${esc(r.count)}</td></tr>`;
+            };
+
+            // Methods
+            const methodRows = Object.entries(gw.byMethod || {}).map(([m, c]) => ({ method: m, count: c }));
+            const methodColumns = [
+                { key: "method", label: "Method", type: "string" },
+                { key: "count",  label: "Count",  type: "number" },
+            ];
+            if (!sortStates.has("gw-methods")) { sortStates.set("gw-methods", { key: "count", dir: "desc" }); }
+            const renderMethodRow = (r) => `<tr><td><span class="pill">${esc(r.method)}</span></td><td>${esc(r.count)}</td></tr>`;
+
+            // Backends — most-used first
+            const backendEntries = Object.entries(gw.byBackend || {}).map(([addr, count]) => ({ addr, count }));
+            const backendColumns = [
+                { key: "addr",  label: "Backend",  type: "string" },
+                { key: "count", label: "Requests", type: "number" },
+            ];
+            if (!sortStates.has("gw-backends")) { sortStates.set("gw-backends", { key: "count", dir: "desc" }); }
+            const renderBackendRow = (r) => `<tr>
+                <td><span class="address healthy">${esc(r.addr)}</span></td>
+                <td>${esc(r.count)}</td>
+            </tr>`;
+
+            // Latency
+            const lat = gw.latency;
+            const latencyPanel = lat
+                ? `<div class="kv">
+                    <div class="k">Sample size</div>      <div class="v">${esc(lat.sampleSize)} / ${esc(lat.bufferSize)}</div>
+                    <div class="k">Mean</div>             <div class="v">${lat.mean.toFixed(2)} ms</div>
+                    <div class="k">p50</div>              <div class="v">${lat.p50.toFixed(2)} ms</div>
+                    <div class="k">p95</div>              <div class="v">${lat.p95.toFixed(2)} ms</div>
+                    <div class="k">p99</div>              <div class="v">${lat.p99.toFixed(2)} ms</div>
+                    <div class="k">Max</div>              <div class="v">${lat.max.toFixed(2)} ms</div>
+                </div>`
+                : `<div class="empty">No samples yet.</div>`;
+
+            // WebSocket
+            const wsPanel = `<div class="kv">
+                <div class="k">Active</div>     <div class="v">${esc(gw.ws?.active ?? 0)}</div>
+                <div class="k">Total opened</div><div class="v">${esc(gw.ws?.total ?? 0)}</div>
+                <div class="k">Errors</div>     <div class="v" style="color: ${(gw.ws?.errors ?? 0) > 0 ? 'var(--red)' : 'inherit'};">${esc(gw.ws?.errors ?? 0)}</div>
+            </div>`;
+
+            // Process & gauges
+            const gaugesPanel = `<div class="kv">
+                <div class="k">PID</div>            <div class="v">${esc(gw.pid)}</div>
+                <div class="k">Port</div>           <div class="v">${esc(gw.port)}</div>
+                <div class="k">Uptime</div>         <div class="v">${esc(fmtDuration(gw.uptimeSec))}</div>
+                <div class="k">Endpoints mounted</div><div class="v">${esc(gw.endpoints)}</div>
+                <div class="k">Sessions cached</div><div class="v">${esc(gw.sessions)} / ${esc(gw.maxSessions)}</div>
+                <div class="k">Active NATS proxies</div><div class="v" style="color: ${(gw.activeNatsProxies > 0) ? 'var(--amber)' : 'inherit'};">${esc(gw.activeNatsProxies)}</div>
+                <div class="k">Router rebuilds</div><div class="v">${esc(gw.routerRebuilds)}</div>
+            </div>`;
+
+            el.innerHTML = `
+                <h2>Gateway</h2>
+                <p style="color: var(--muted); margin: 0 0 16px; max-width: 800px;">
+                    Live counters and gauges from this Gateway instance. Latency percentiles are
+                    computed over a rolling window of the most recent ${esc(lat?.bufferSize ?? 1000)} requests.
+                    All tracking is in-process and ~free; nothing is persisted.
+                </p>
+                <div class="cards" style="margin-bottom: 16px;">
+                    <div class="card"><div class="label">Total requests</div><div class="value">${esc(httpTotal)}</div></div>
+                    <div class="card"><div class="label">Proxied (HTTP)</div><div class="value" style="color: var(--green);">${esc(gw.proxied)}</div></div>
+                    <div class="card"><div class="label">Proxied (NATS tunnel)</div><div class="value" style="color: ${gw.proxied_over_nats > 0 ? 'var(--amber)' : 'inherit'};">${esc(gw.proxied_over_nats)}</div></div>
+                    <div class="card"><div class="label">Proxy errors</div><div class="value" style="color: ${gw.errors > 0 ? 'var(--red)' : 'inherit'};">${esc(gw.errors)}</div></div>
+                    <div class="card"><div class="label">Proxied share</div><div class="value">${proxiedShare}${proxiedShare === "—" ? "" : "%"}</div></div>
+                    <div class="card"><div class="label">Error rate</div><div class="value">${errRate}${errRate === "—" ? "" : "%"}</div></div>
+                </div>
+
+                <div class="panel-grid">
+                    <div class="panel"><h3>Process</h3>${gaugesPanel}</div>
+                    <div class="panel"><h3>Latency (rolling)</h3>${latencyPanel}</div>
+                    <div class="panel"><h3>WebSocket sessions</h3>${wsPanel}</div>
+                </div>
+
+                <h3>Status code buckets</h3>
+                <div id="gw-status-wrap"></div>
+
+                <h3>HTTP methods</h3>
+                <div id="gw-methods-wrap"></div>
+
+                <h3>Backends (by request count)</h3>
+                <div id="gw-backends-wrap"></div>
+            `;
+
+            const drawStatus = () => {
+                const sorted = applySort(statusRows, "gw-status", statusColumns);
+                document.getElementById("gw-status-wrap").innerHTML = renderTable("gw-status", sorted, statusColumns, renderStatusRow);
+                attachSortHandlers(el, "gw-status", drawStatus);
+            };
+            const drawMethods = () => {
+                if (methodRows.length === 0) {
+                    document.getElementById("gw-methods-wrap").innerHTML = '<div class="empty">No requests yet.</div>';
+                    return;
+                }
+                const sorted = applySort(methodRows, "gw-methods", methodColumns);
+                document.getElementById("gw-methods-wrap").innerHTML = renderTable("gw-methods", sorted, methodColumns, renderMethodRow);
+                attachSortHandlers(el, "gw-methods", drawMethods);
+            };
+            const drawBackends = () => {
+                if (backendEntries.length === 0) {
+                    document.getElementById("gw-backends-wrap").innerHTML = '<div class="empty">No backend traffic yet.</div>';
+                    return;
+                }
+                const sorted = applySort(backendEntries, "gw-backends", backendColumns);
+                document.getElementById("gw-backends-wrap").innerHTML = renderTable("gw-backends", sorted, backendColumns, renderBackendRow);
+                attachSortHandlers(el, "gw-backends", drawBackends);
+            };
+            drawStatus();
+            drawMethods();
+            drawBackends();
         } catch (e) {
             el.innerHTML = `<div class="err-banner">${esc(e.message)}</div>`;
         }
@@ -294,56 +665,89 @@
     // Registry
     // -----------------------------------------------------------------------
 
+    // sticky filter state per tab
+    const filterStates = { registry: "", stats: "" };
+
     async function renderRegistry() {
         const el = document.getElementById("registry");
         el.innerHTML = '<div class="loading">Loading…</div>';
         try {
             const reg = await api("/registry");
-            const entries = Object.values(reg).sort((a, b) => `${a.n}@${a.v}`.localeCompare(`${b.n}@${b.v}`));
+            const rows = Object.values(reg).map((e) => ({
+                ...e,
+                _aCount: e.a?.length ?? 0,
+                _advCount: e.advertised?.length ?? 0,
+                _methodCount: e.m?.length ?? 0,
+                _expiresIn: Math.max(0, e.timeout - Date.now()),
+            }));
+
+            const columns = [
+                { key: "n",            label: "Name",       type: "string"    },
+                { key: "v",            label: "Version",    type: "version"   },
+                { key: "i",            label: "Instance",   type: "string"    },
+                { key: "_aCount",      label: "Healthy .a", type: "number"    },
+                { key: "_advCount",    label: "Advertised", type: "number"    },
+                { key: "_methodCount", label: "Methods",    type: "number"    },
+                { key: "gx",           label: "gx",         type: "version"   },
+                { key: "_expiresIn",   label: "Expires in", type: "number"    },
+            ];
+
+            // default sort: Name asc
+            if (!sortStates.has("registry")) { sortStates.set("registry", { key: "n", dir: "asc" }); }
+
+            const renderRow = (e) => {
+                const aPill = e._aCount === 0
+                    ? `<span class="pill err">0</span>`
+                    : `<span class="pill ok">${e._aCount}</span>`;
+                return `<tr class="clickable" data-id="${esc(e.i)}">
+                    <td>${esc(e.n)}</td>
+                    <td>${vpill(e.v)}</td>
+                    <td><span class="link">${esc(e.i)}</span></td>
+                    <td>${aPill}
+                        <div class="addresses">${(e.a || []).map((a) => `<span class="address healthy">${esc(a)}</span>`).join("")}</div>
+                    </td>
+                    <td><span class="pill muted">${e._advCount}</span>
+                        <details><summary>show</summary>
+                            <div class="addresses">${(e.advertised || []).map((a) => {
+                                const healthy = (e.a || []).includes(a);
+                                return `<span class="address ${healthy ? "healthy" : "dead"}">${esc(a)}</span>`;
+                            }).join("")}</div>
+                        </details>
+                    </td>
+                    <td><span class="pill muted">${e._methodCount}</span></td>
+                    <td>${vpill(e.gx)}</td>
+                    <td>${e._expiresIn > 0 ? `${Math.round(e._expiresIn / 1000)}s` : `<span class="pill err">expired</span>`}</td>
+                </tr>`;
+            };
+
+            const filterKeys = ["n", "v", "i", "gx"];
+
             el.innerHTML = `
-                <h2>Registry — Gateway's view (${entries.length} instance${entries.length === 1 ? "" : "s"})</h2>
-                <table>
-                    <thead><tr>
-                        <th>Name</th><th>Version</th><th>Instance</th>
-                        <th>Healthy .a</th><th>Advertised</th>
-                        <th>Methods</th><th>gx</th><th>Expires in</th>
-                    </tr></thead>
-                    <tbody>
-                        ${entries.map((e) => {
-                            const aCount = e.a?.length ?? 0;
-                            const advCount = e.advertised?.length ?? 0;
-                            const aPill = aCount === 0
-                                ? `<span class="pill err">0</span>`
-                                : `<span class="pill ok">${aCount}</span>`;
-                            const expiresMs = e.timeout - Date.now();
-                            return `
-                                <tr class="clickable" data-id="${esc(e.i)}">
-                                    <td>${esc(e.n)}</td>
-                                    <td>${esc(e.v)}</td>
-                                    <td><span class="link">${esc(e.i)}</span></td>
-                                    <td>${aPill}
-                                        <div class="addresses">${(e.a || []).map((a) => `<span class="address healthy">${esc(a)}</span>`).join("")}</div>
-                                    </td>
-                                    <td><span class="pill muted">${advCount}</span>
-                                        <details><summary>show</summary>
-                                            <div class="addresses">${(e.advertised || []).map((a) => {
-                                                const healthy = (e.a || []).includes(a);
-                                                return `<span class="address ${healthy ? "healthy" : "dead"}">${esc(a)}</span>`;
-                                            }).join("")}</div>
-                                        </details>
-                                    </td>
-                                    <td><span class="pill muted">${e.m?.length ?? 0}</span></td>
-                                    <td>${esc(e.gx ?? "—")}</td>
-                                    <td>${expiresMs > 0 ? `${Math.round(expiresMs / 1000)}s` : `<span class="pill err">expired</span>`}</td>
-                                </tr>
-                            `;
-                        }).join("")}
-                    </tbody>
-                </table>
+                <h2>Registry — Gateway's view (${rows.length} instance${rows.length === 1 ? "" : "s"})</h2>
+                <input class="filter" id="reg-filter" type="search" placeholder="Filter by name, version, instance id, gx, address…" value="${esc(filterStates.registry)}">
+                <div id="reg-table-wrap"></div>
             `;
-            el.querySelectorAll("tr.clickable").forEach((tr) => {
-                tr.addEventListener("click", () => { window.location.hash = `instance=${encodeURIComponent(tr.dataset.id)}`; });
+
+            const draw = () => {
+                const q = filterStates.registry;
+                const filtered = applyFilter(rows, q, filterKeys).filter((r) =>
+                    // also match against addresses (not on the row top level for filter helper)
+                    !q || filterKeys.some((k) => String(r[k] ?? "").toLowerCase().includes(q.toLowerCase()))
+                          || (r.a || []).some((a) => a.toLowerCase().includes(q.toLowerCase()))
+                          || (r.advertised || []).some((a) => a.toLowerCase().includes(q.toLowerCase())),
+                );
+                const sorted = applySort(filtered, "registry", columns);
+                document.getElementById("reg-table-wrap").innerHTML = renderTable("registry", sorted, columns, renderRow);
+                attachSortHandlers(el, "registry", draw);
+                attachRowClicks(el);
+            };
+
+            document.getElementById("reg-filter").addEventListener("input", (ev) => {
+                filterStates.registry = ev.target.value;
+                draw();
             });
+
+            draw();
         } catch (e) {
             el.innerHTML = `<div class="err-banner">${esc(e.message)}</div>`;
         }
@@ -427,24 +831,317 @@
         el.innerHTML = '<div class="loading">Loading…</div>';
         try {
             const eps = await api("/endpoints");
+            const rows = eps.map((e) => ({
+                _raw: e,
+                verb: e.endpoint?.verb ?? "",
+                url: e.endpoint?.url ?? "",
+                version: e.version ?? "",
+                backendCount: (e.backend || []).length,
+                order: e.options?.order ?? 100,
+                requests: e.requests ?? 0,
+            }));
+
+            const columns = [
+                { key: "verb",         label: "Verb",     type: "string"  },
+                { key: "url",          label: "URL",      type: "string"  },
+                { key: "version",      label: "Version",  type: "version" },
+                { key: "backendCount", label: "Backends", type: "number"  },
+                { key: "order",        label: "Order",    type: "number"  },
+                { key: "requests",     label: "Requests", type: "number"  },
+            ];
+
+            if (!sortStates.has("endpoints")) { sortStates.set("endpoints", { key: "url", dir: "asc" }); }
+
+            const renderRow = (r) => `<tr>
+                <td><span class="pill">${esc(r.verb)}</span></td>
+                <td>${esc(r.url)}</td>
+                <td>${vpill(r.version)}</td>
+                <td><div class="addresses">${(r._raw.backend || []).map((b) => `<span class="address healthy">${esc(b)}</span>`).join("")}</div></td>
+                <td>${esc(r.order)}</td>
+                <td>${esc(r.requests)}</td>
+            </tr>`;
+
             el.innerHTML = `
-                <h2>HTTP Endpoints (${eps.length})</h2>
-                <table>
-                    <thead><tr><th>Verb</th><th>URL</th><th>Version</th><th>Backends</th><th>Order</th><th>Requests</th></tr></thead>
-                    <tbody>
-                        ${eps.map((e) => `
-                            <tr>
-                                <td><span class="pill">${esc(e.endpoint.verb)}</span></td>
-                                <td>${esc(e.endpoint.url)}</td>
-                                <td>${esc(e.version)}</td>
-                                <td><div class="addresses">${(e.backend || []).map((b) => `<span class="address healthy">${esc(b)}</span>`).join("")}</div></td>
-                                <td>${esc(e.options?.order ?? "")}</td>
-                                <td>${esc(e.requests ?? 0)}</td>
-                            </tr>
-                        `).join("")}
-                    </tbody>
-                </table>
+                <h2>HTTP Endpoints (${rows.length})</h2>
+                <div id="ep-table-wrap"></div>
             `;
+            const draw = () => {
+                const sorted = applySort(rows, "endpoints", columns);
+                document.getElementById("ep-table-wrap").innerHTML = renderTable("endpoints", sorted, columns, renderRow);
+                attachSortHandlers(el, "endpoints", draw);
+            };
+            draw();
+        } catch (e) {
+            el.innerHTML = `<div class="err-banner">${esc(e.message)}</div>`;
+        }
+    }
+
+    // -----------------------------------------------------------------------
+    // Versions — informational view of detected geonix and service versions.
+    // Version diversity is NOT flagged as a problem; it's surfaced because it
+    // can be a useful clue when investigating an actual issue.
+    // -----------------------------------------------------------------------
+
+    async function renderVersions() {
+        const el = document.getElementById("versions");
+        el.innerHTML = '<div class="loading">Loading…</div>';
+        try {
+            const reg = await api("/registry");
+            const entries = Object.values(reg);
+
+            // Geonix versions
+            const gxAgg = new Map();
+            for (const e of entries) {
+                const k = e.gx ?? "—";
+                const cur = gxAgg.get(k) ?? { gx: k, instances: 0, services: new Set() };
+                cur.instances++;
+                cur.services.add(e.n);
+                gxAgg.set(k, cur);
+            }
+            const gxRows = [...gxAgg.values()].map((r) => ({
+                gx: r.gx,
+                instances: r.instances,
+                serviceCount: r.services.size,
+                services: [...r.services].sort().join(", "),
+            }));
+
+            // Service versions (one row per Name@Version)
+            const svcAgg = new Map();
+            for (const e of entries) {
+                const key = `${e.n}@@${e.v}`;
+                const cur = svcAgg.get(key) ?? { n: e.n, v: e.v, instances: 0, gxs: new Set() };
+                cur.instances++;
+                cur.gxs.add(e.gx ?? "—");
+                svcAgg.set(key, cur);
+            }
+            const svcRows = [...svcAgg.values()].map((r) => ({
+                n: r.n,
+                v: r.v,
+                instances: r.instances,
+                gxCount: r.gxs.size,
+                gxs: [...r.gxs].sort().join(", "),
+            }));
+
+            // How many distinct versions per service name? Useful clue, not flagged as wrong.
+            const versionCountByName = new Map();
+            for (const r of svcRows) {
+                versionCountByName.set(r.n, (versionCountByName.get(r.n) ?? 0) + 1);
+            }
+
+            const gxColumns = [
+                { key: "gx",           label: "Geonix version",  type: "version" },
+                { key: "instances",    label: "Instances",       type: "number"  },
+                { key: "serviceCount", label: "Distinct services", type: "number" },
+                { key: "services",     label: "Services",        type: "string"  },
+            ];
+            const svcColumns = [
+                { key: "n",         label: "Service",   type: "string"  },
+                { key: "v",         label: "Version",   type: "version" },
+                { key: "instances", label: "Instances", type: "number"  },
+                { key: "gxCount",   label: "gx variety", type: "number" },
+                { key: "gxs",       label: "Geonix versions on this row", type: "string" },
+            ];
+
+            if (!sortStates.has("versions-gx"))  { sortStates.set("versions-gx",  { key: "instances", dir: "desc" }); }
+            if (!sortStates.has("versions-svc")) { sortStates.set("versions-svc", { key: "n", dir: "asc" }); }
+
+            const renderGxRow = (r) => `<tr>
+                <td>${vpill(r.gx)}</td>
+                <td>${esc(r.instances)}</td>
+                <td>${esc(r.serviceCount)}</td>
+                <td style="color: var(--muted); font-size: 12px;">${esc(r.services)}</td>
+            </tr>`;
+            const renderSvcRow = (r) => {
+                const distinct = versionCountByName.get(r.n) ?? 1;
+                const distinctBadge = distinct > 1
+                    ? ` <span class="pill muted" title="this service has ${distinct} distinct versions in the registry">${distinct} versions</span>`
+                    : "";
+                return `<tr>
+                    <td>${esc(r.n)}${distinctBadge}</td>
+                    <td>${vpill(r.v)}</td>
+                    <td>${esc(r.instances)}</td>
+                    <td>${esc(r.gxCount)}</td>
+                    <td style="color: var(--muted); font-size: 12px;">${(r.gxs || "").split(", ").map((g) => vpill(g)).join(" ")}</td>
+                </tr>`;
+            };
+
+            const gxCount = gxRows.length;
+            const svcWithMultipleVersions = [...versionCountByName.values()].filter((c) => c > 1).length;
+
+            el.innerHTML = `
+                <h2>Versions</h2>
+                <p style="color: var(--muted); margin: 0 0 16px; max-width: 800px;">
+                    Snapshot of every Geonix and service version detected in the cluster.
+                    Diversity is informational — multiple versions are not inherently a problem,
+                    but knowing what's running where helps when triaging an actual issue.
+                </p>
+                <div class="cards" style="margin-bottom: 24px;">
+                    <div class="card"><div class="label">Geonix versions detected</div><div class="value">${gxCount}</div></div>
+                    <div class="card"><div class="label">Services in registry</div><div class="value">${versionCountByName.size}</div></div>
+                    <div class="card"><div class="label">Services with multiple versions</div><div class="value">${svcWithMultipleVersions}</div></div>
+                </div>
+                <h3>Geonix versions</h3>
+                <div id="ver-gx-wrap"></div>
+                <h3>Service versions</h3>
+                <div id="ver-svc-wrap"></div>
+            `;
+
+            const drawGx = () => {
+                const sorted = applySort(gxRows, "versions-gx", gxColumns);
+                document.getElementById("ver-gx-wrap").innerHTML = renderTable("versions-gx", sorted, gxColumns, renderGxRow);
+                attachSortHandlers(el, "versions-gx", drawGx);
+            };
+            const drawSvc = () => {
+                const sorted = applySort(svcRows, "versions-svc", svcColumns);
+                document.getElementById("ver-svc-wrap").innerHTML = renderTable("versions-svc", sorted, svcColumns, renderSvcRow);
+                attachSortHandlers(el, "versions-svc", drawSvc);
+            };
+            drawGx();
+            drawSvc();
+        } catch (e) {
+            el.innerHTML = `<div class="err-banner">${esc(e.message)}</div>`;
+        }
+    }
+
+    // -----------------------------------------------------------------------
+    // Stats — cluster-wide aggregation of $getRpcStats across all instances.
+    // -----------------------------------------------------------------------
+
+    async function renderStats() {
+        const el = document.getElementById("stats");
+        el.innerHTML = '<div class="loading">Loading $getRpcStats from every instance…</div>';
+        try {
+            const reg = await api("/registry");
+            const ids = Object.keys(reg);
+            if (ids.length === 0) { el.innerHTML = '<div class="empty">No instances registered.</div>'; return; }
+
+            const fetches = await Promise.all(ids.map((id) =>
+                api(`/services/${id}/$getRpcStats`)
+                    .then((s) => ({ id, ok: true, stats: s }))
+                    .catch((err) => ({ id, ok: false, error: err.message })),
+            ));
+
+            const rows = fetches.map((f) => {
+                const entry = reg[f.id] || {};
+                const s = f.ok ? f.stats : null;
+                const httpAttempts = s?.http?.attempts ?? 0;
+                const httpSuccess  = s?.http?.success  ?? 0;
+                return {
+                    id: f.id,
+                    label: entry.n ? `${entry.n}@${entry.v}` : "—",
+                    n: entry.n ?? "",
+                    v: entry.v ?? "",
+                    ok: f.ok,
+                    error: f.error || "",
+                    httpAttempts,
+                    httpSuccess,
+                    httpFailures: s?.http?.failures ?? 0,
+                    httpNonOk: s?.http?.nonOk ?? 0,
+                    httpFetchErrors: s?.http?.fetchErrors ?? 0,
+                    httpDecodeErrors: s?.http?.decodeErrors ?? 0,
+                    natsAttempts: s?.nats?.attempts ?? 0,
+                    natsSuccess: s?.nats?.success ?? 0,
+                    natsFailures: s?.nats?.failures ?? 0,
+                    noUsableInstance: s?.noUsableInstance ?? 0,
+                    rate: httpAttempts > 0 ? (httpSuccess / httpAttempts) : null,
+                    lastFailAt: s?.lastHttpFailure?.at ?? 0,
+                    lastFail: s?.lastHttpFailure ?? null,
+                };
+            });
+
+            // Cluster totals
+            const sum = (k) => rows.reduce((a, r) => a + (r[k] || 0), 0);
+            const totals = {
+                httpAttempts:     sum("httpAttempts"),
+                httpSuccess:      sum("httpSuccess"),
+                httpFailures:     sum("httpFailures"),
+                httpNonOk:        sum("httpNonOk"),
+                httpFetchErrors:  sum("httpFetchErrors"),
+                httpDecodeErrors: sum("httpDecodeErrors"),
+                natsAttempts:     sum("natsAttempts"),
+                natsSuccess:      sum("natsSuccess"),
+                natsFailures:     sum("natsFailures"),
+                noUsableInstance: sum("noUsableInstance"),
+            };
+            const rate = totals.httpAttempts > 0
+                ? (100 * totals.httpSuccess / totals.httpAttempts).toFixed(2)
+                : "—";
+
+            const columns = [
+                { key: "label",           label: "Service",         type: "string"  },
+                { key: "id",              label: "Instance",        type: "string"  },
+                { key: "httpAttempts",    label: "HTTP attempts",   type: "number"  },
+                { key: "httpSuccess",     label: "HTTP success",    type: "number"  },
+                { key: "httpFailures",    label: "HTTP failures",   type: "number"  },
+                { key: "httpNonOk",       label: "non-OK",          type: "number"  },
+                { key: "httpFetchErrors", label: "fetch err",       type: "number"  },
+                { key: "httpDecodeErrors",label: "decode err",      type: "number"  },
+                { key: "natsAttempts",    label: "NATS attempts",   type: "number"  },
+                { key: "natsFailures",    label: "NATS fail",       type: "number"  },
+                { key: "noUsableInstance",label: "No usable",       type: "number"  },
+                { key: "rate",            label: "HTTP success rate", type: "number" },
+                { key: "lastFailAt",      label: "Last HTTP fail",  type: "number"  },
+            ];
+            if (!sortStates.has("stats-instances")) { sortStates.set("stats-instances", { key: "httpAttempts", dir: "desc" }); }
+
+            const renderRow = (r) => {
+                if (!r.ok) {
+                    return `<tr class="clickable" data-id="${esc(r.id)}">
+                        <td>${esc(r.label)}</td>
+                        <td><span class="link">${esc(r.id.slice(0, 12))}…</span></td>
+                        <td colspan="11" style="color: var(--amber);">${esc(r.error)}</td>
+                    </tr>`;
+                }
+                const rateCell = r.rate == null
+                    ? `<span class="pill muted">—</span>`
+                    : `<span class="pill ${r.rate >= 0.99 ? "ok" : (r.rate >= 0.95 ? "warn" : "err")}">${(r.rate * 100).toFixed(1)}%</span>`;
+                const lastFailCell = r.lastFailAt > 0
+                    ? `<span title="${esc(JSON.stringify(r.lastFail))}">${esc(since(r.lastFailAt))} ago</span>`
+                    : `<span class="pill muted">none</span>`;
+                return `<tr class="clickable" data-id="${esc(r.id)}">
+                    <td>${esc(r.label)}</td>
+                    <td><span class="link">${esc(r.id.slice(0, 12))}…</span></td>
+                    <td>${esc(r.httpAttempts)}</td>
+                    <td style="color: var(--green);">${esc(r.httpSuccess)}</td>
+                    <td style="color: ${r.httpFailures > 0 ? "var(--red)" : "inherit"};">${esc(r.httpFailures)}</td>
+                    <td>${esc(r.httpNonOk)}</td>
+                    <td>${esc(r.httpFetchErrors)}</td>
+                    <td>${esc(r.httpDecodeErrors)}</td>
+                    <td>${esc(r.natsAttempts)}</td>
+                    <td style="color: ${r.natsFailures > 0 ? "var(--red)" : "inherit"};">${esc(r.natsFailures)}</td>
+                    <td>${esc(r.noUsableInstance)}</td>
+                    <td>${rateCell}</td>
+                    <td>${lastFailCell}</td>
+                </tr>`;
+            };
+
+            el.innerHTML = `
+                <h2>Outgoing RPC stats — cluster-wide aggregation</h2>
+                <p style="color: var(--muted); margin: 0 0 16px; max-width: 800px;">
+                    Counts of <code>directRequest</code> dispatches summed across every instance.
+                    HTTP success near 100% with NATS attempts plateauing means same-host calls are
+                    taking the direct HTTP path. A rising NATS attempts column or growing
+                    <code>noUsableInstance</code> count points at the caller side of a routing issue.
+                </p>
+                <div class="cards" style="margin-bottom: 24px;">
+                    <div class="card"><div class="label">HTTP attempts</div><div class="value">${totals.httpAttempts}</div></div>
+                    <div class="card"><div class="label">HTTP success</div><div class="value" style="color: var(--green);">${totals.httpSuccess}</div></div>
+                    <div class="card"><div class="label">HTTP failures</div><div class="value" style="color: ${totals.httpFailures > 0 ? "var(--red)" : "inherit"};">${totals.httpFailures}</div></div>
+                    <div class="card"><div class="label">HTTP success rate</div><div class="value">${rate}${rate === "—" ? "" : "%"}</div></div>
+                    <div class="card"><div class="label">NATS attempts</div><div class="value">${totals.natsAttempts}</div></div>
+                    <div class="card"><div class="label">No usable instance</div><div class="value">${totals.noUsableInstance}</div></div>
+                </div>
+                <h3>Breakdown by instance</h3>
+                <div id="stats-table-wrap"></div>
+            `;
+
+            const draw = () => {
+                const sorted = applySort(rows, "stats-instances", columns);
+                document.getElementById("stats-table-wrap").innerHTML = renderTable("stats-instances", sorted, columns, renderRow);
+                attachSortHandlers(el, "stats-instances", draw);
+                attachRowClicks(el);
+            };
+            draw();
         } catch (e) {
             el.innerHTML = `<div class="err-banner">${esc(e.message)}</div>`;
         }
@@ -746,27 +1443,33 @@ ${esc(argsStr)}</pre>
             if (!view) { return '<div class="empty">no entries</div>'; }
             const ids = Object.keys(view);
             if (ids.length === 0) { return '<div class="empty">no entries</div>'; }
-            return `
-                <table>
-                    <thead><tr><th>Name</th><th>Ver</th><th>Instance</th><th>Healthy .a</th><th>Advertised</th></tr></thead>
-                    <tbody>
-                        ${ids.map((iid) => {
-                            const e = view[iid];
-                            const aCount = e.a?.length ?? 0;
-                            const isSelf = iid === id;
-                            return `
-                                <tr class="clickable" data-id="${esc(iid)}">
-                                    <td>${esc(e.n)}${isSelf ? ' <span class="pill muted">self</span>' : ""}</td>
-                                    <td>${esc(e.v)}</td>
-                                    <td><span class="link">${esc(iid.slice(0, 12))}…</span></td>
-                                    <td>${aCount === 0 ? '<span class="pill err">0</span>' : `<span class="pill ok">${aCount}</span>`}</td>
-                                    <td>${e.advertised?.length ?? 0}</td>
-                                </tr>
-                            `;
-                        }).join("")}
-                    </tbody>
-                </table>
-            `;
+            const rows = ids.map((iid) => {
+                const e = view[iid];
+                return {
+                    n: e.n, v: e.v, gx: e.gx ?? "",
+                    iid,
+                    aCount: e.a?.length ?? 0,
+                    advCount: e.advertised?.length ?? 0,
+                    isSelf: iid === id,
+                };
+            });
+            const columns = [
+                { key: "n",        label: "Name",       type: "string" },
+                { key: "v",        label: "Ver",        type: "version" },
+                { key: "iid",      label: "Instance",   type: "string" },
+                { key: "aCount",   label: "Healthy .a", type: "number" },
+                { key: "advCount", label: "Advertised", type: "number" },
+            ];
+            if (!sortStates.has("instance-regview")) { sortStates.set("instance-regview", { key: "n", dir: "asc" }); }
+            const renderRow = (r) => `<tr class="clickable" data-id="${esc(r.iid)}">
+                <td>${esc(r.n)}${r.isSelf ? ' <span class="pill muted">self</span>' : ""}</td>
+                <td>${vpill(r.v)}</td>
+                <td><span class="link">${esc(r.iid.slice(0, 12))}…</span></td>
+                <td>${r.aCount === 0 ? '<span class="pill err">0</span>' : `<span class="pill ok">${r.aCount}</span>`}</td>
+                <td>${esc(r.advCount)}</td>
+            </tr>`;
+            const sorted = applySort(rows, "instance-regview", columns);
+            return `<div id="instance-regview-wrap">${renderTable("instance-regview", sorted, columns, renderRow)}</div>`;
         };
 
         const panelRegView = regView.ok
@@ -794,6 +1497,10 @@ ${esc(argsStr)}</pre>
         el.querySelectorAll("tr.clickable").forEach((tr) => {
             tr.addEventListener("click", () => { window.location.hash = `instance=${encodeURIComponent(tr.dataset.id)}`; });
         });
+        // Wire sort handlers for the embedded "registry as seen by this instance" table.
+        if (regView.ok && document.getElementById("instance-regview-wrap")) {
+            attachSortHandlers(el, "instance-regview", () => renderInstance(id));
+        }
     }
 
     // -----------------------------------------------------------------------