@serve.zone/dcrouter 17.1.5 → 17.1.6

package/ts/monitoring/classes.metricsmanager.ts

@@ -5,11 +5,20 @@ import { SecurityLogger, SecurityEventType } from '../security/classes.securityl
 import { logger } from '../logger.js';
 import type { IAsnActivity } from '../../ts_interfaces/data/stats.js';
 
+const NETWORK_STATS_CACHE_TTL_MS = 3_000;
+const ASN_INTELLIGENCE_CACHE_TTL_MS = 60_000;
+const ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS = 3_000;
+const OBSERVED_IP_SAMPLE_INTERVAL_MS = 60_000;
+const MAX_ASN_CANDIDATE_IPS = 500;
+const MAX_OBSERVED_IP_SAMPLE_SIZE = 20;
+const MAX_SNAPSHOT_LIMIT = 1_000;
+
 export class MetricsManager {
   private metricsLogger: plugins.smartlog.Smartlog;
   private smartMetrics: plugins.smartmetrics.SmartMetrics;
   private dcRouter: DcRouter;
   private resetInterval?: NodeJS.Timeout;
+  private observedIpSamplerInterval?: NodeJS.Timeout;
   private metricsCache: MetricsCache;
   
   // Constants
@@ -69,7 +78,7 @@ export class MetricsManager {
       }
     });
     this.smartMetrics = new plugins.smartmetrics.SmartMetrics(this.metricsLogger, 'dcrouter');
-    // Initialize metrics cache with 500ms TTL
+    // Individual metrics select an appropriate TTL where their collection cost differs.
     this.metricsCache = new MetricsCache(500);
   }
   
@@ -118,6 +127,11 @@ export class MetricsManager {
       // Prune old time-series buckets every minute (don't wait for lazy query)
       this.pruneOldBuckets();
     }, 60000); // Check every minute
+
+    this.observedIpSamplerInterval = setInterval(() => {
+      this.sampleObservedIps();
+    }, OBSERVED_IP_SAMPLE_INTERVAL_MS);
+    this.observedIpSamplerInterval.unref();
     
     logger.log('info', 'MetricsManager started');
   }
@@ -128,6 +142,10 @@ export class MetricsManager {
       clearInterval(this.resetInterval);
       this.resetInterval = undefined;
     }
+    if (this.observedIpSamplerInterval) {
+      clearInterval(this.observedIpSamplerInterval);
+      this.observedIpSamplerInterval = undefined;
+    }
 
     this.smartMetrics.stop();
 
@@ -295,13 +313,35 @@ export class MetricsManager {
   public async getActiveConnectionSnapshots(
     options: plugins.smartproxy.IActiveConnectionSnapshotOptions = {},
   ): Promise<plugins.smartproxy.IActiveConnectionSnapshot[]> {
-    const cacheKey = `activeConnectionSnapshots:${options.limit ?? 1000}:${options.routeId ?? ''}`;
+    const normalizedOptions = {
+      ...options,
+      limit: Math.min(Math.max(options.limit ?? MAX_SNAPSHOT_LIMIT, 1), MAX_SNAPSHOT_LIMIT),
+    };
+    const afterId = (normalizedOptions as typeof normalizedOptions & { afterId?: string }).afterId ?? '';
+    const cacheKey = `activeConnectionSnapshots:${normalizedOptions.limit}:${normalizedOptions.routeId ?? ''}:${afterId}`;
     return await this.metricsCache.get<plugins.smartproxy.IActiveConnectionSnapshot[]>(cacheKey, async () => {
       if (!this.dcRouter.smartProxy) {
         return [];
       }
-      return this.dcRouter.smartProxy.getActiveConnectionSnapshots(options);
-    }, 500);
+      return this.dcRouter.smartProxy.getActiveConnectionSnapshots(normalizedOptions);
+    }, ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS);
+  }
+
+  public async getActiveConnectionSnapshotPage(
+    options: plugins.smartproxy.IActiveConnectionSnapshotOptions = {},
+  ): Promise<plugins.smartproxy.IActiveConnectionSnapshotPage> {
+    const normalizedOptions = {
+      ...options,
+      limit: Math.min(Math.max(options.limit ?? MAX_SNAPSHOT_LIMIT, 1), MAX_SNAPSHOT_LIMIT),
+    };
+    const afterId = normalizedOptions.afterId ?? '';
+    const cacheKey = `activeConnectionSnapshotPage:${normalizedOptions.limit}:${normalizedOptions.routeId ?? ''}:${afterId}`;
+    return await this.metricsCache.get<plugins.smartproxy.IActiveConnectionSnapshotPage>(cacheKey, async () => {
+      if (!this.dcRouter.smartProxy) {
+        return { snapshots: [], hasMore: false, nextCursor: null };
+      }
+      return this.dcRouter.smartProxy.getActiveConnectionSnapshotPage(normalizedOptions);
+    }, ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS);
   }
 
   // Get connection info from SmartProxy
@@ -560,10 +600,59 @@ export class MetricsManager {
       this.securityMetrics.incidents.shift();
     }
   }
+
+  /**
+   * Feed IP intelligence from maintained data-plane counters, independently of
+   * dashboard reads. Startup order is intentionally tolerated: until both
+   * SmartProxy and SecurityPolicyManager exist, the sample is a no-op.
+   */
+  private sampleObservedIps(): void {
+    const smartProxy = this.dcRouter.smartProxy;
+    const securityPolicyManager = this.dcRouter.securityPolicyManager;
+    if (!smartProxy || !securityPolicyManager) {
+      return;
+    }
+
+    try {
+      const proxyMetrics = smartProxy.getMetrics();
+      const connectionsByIP = proxyMetrics.connections.byIP();
+      const throughputByIP = proxyMetrics.throughput.byIP();
+      const candidates = new Map<string, { connections: number; throughput: number }>();
+
+      for (const [ip, connections] of connectionsByIP) {
+        candidates.set(ip, { connections, throughput: 0 });
+      }
+      for (const [ip, throughput] of throughputByIP) {
+        const candidate = candidates.get(ip) || { connections: 0, throughput: 0 };
+        candidate.throughput = throughput.in + throughput.out;
+        candidates.set(ip, candidate);
+      }
+      for (const ip of proxyMetrics.connections.domainRequestsByIP().keys()) {
+        if (!candidates.has(ip)) {
+          candidates.set(ip, { connections: 0, throughput: 0 });
+        }
+      }
+
+      const observedIps = [...candidates.entries()]
+        .filter(([ip]) => Boolean(ip))
+        .sort((a, b) => (
+          b[1].connections - a[1].connections
+          || b[1].throughput - a[1].throughput
+          || a[0].localeCompare(b[0])
+        ))
+        .slice(0, MAX_OBSERVED_IP_SAMPLE_SIZE)
+        .map(([ip]) => ip);
+
+      if (observedIps.length > 0) {
+        securityPolicyManager.queueObservedIps(observedIps);
+      }
+    } catch (error: unknown) {
+      logger.log('warn', `Observed-IP metrics sample failed: ${(error as Error).message}`);
+    }
+  }
   
   // Get network metrics from SmartProxy
   public async getNetworkStats() {
-    // Use shorter cache TTL for network stats to ensure real-time updates
     return this.metricsCache.get('networkStats', async () => {
       const smartProxy = this.dcRouter.smartProxy;
       const proxyMetrics = smartProxy ? smartProxy.getMetrics() : null;
@@ -571,6 +660,7 @@ export class MetricsManager {
       if (!proxyMetrics) {
         return {
           connectionsByIP: new Map<string, number>(),
+          activeConnections: 0,
           throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 },
           topIPs: [] as Array<{ ip: string; count: number }>,
           topIPsByBandwidth: [] as Array<{ ip: string; count: number; bwIn: number; bwOut: number }>,
@@ -587,21 +677,9 @@ export class MetricsManager {
         };
       }
 
-      const activeConnectionSnapshots = await this.getActiveConnectionSnapshots({ limit: 10000 });
-
-      const connectionsByIP = new Map<string, number>();
-      const connectionsByRoute = new Map<string, number>();
-      const activeConnectionsByDomain = new Map<string, number>();
-
-      for (const snapshot of activeConnectionSnapshots) {
-        connectionsByIP.set(snapshot.sourceIp, (connectionsByIP.get(snapshot.sourceIp) || 0) + 1);
-        if (snapshot.routeId) {
-          connectionsByRoute.set(snapshot.routeId, (connectionsByRoute.get(snapshot.routeId) || 0) + 1);
-        }
-        if (snapshot.domain) {
-          activeConnectionsByDomain.set(snapshot.domain, (activeConnectionsByDomain.get(snapshot.domain) || 0) + 1);
-        }
-      }
+      const connectionsByIP = proxyMetrics.connections.byIP();
+      const connectionsByRoute = proxyMetrics.connections.byRoute();
+      const activeConnections = proxyMetrics.connections.active();
 
       const instantThroughput = proxyMetrics.throughput.instant();
 
@@ -769,7 +847,6 @@ export class MetricsManager {
         ...topIPs.map((item) => item.ip),
         ...topIPsByBandwidth.map((item) => item.ip),
       ])];
-      this.dcRouter.securityPolicyManager?.queueObservedIps(observedIps);
 
       const topASNs = await this.buildTopASNs(observedIps, allIPData);
 
@@ -808,10 +885,6 @@ export class MetricsManager {
       for (const entry of protocolCache) {
         if (entry.domain) allKnownDomains.add(entry.domain);
       }
-      for (const snapshot of activeConnectionSnapshots) {
-        if (snapshot.domain) allKnownDomains.add(snapshot.domain);
-      }
-
       // Build reverse map: concrete domain → canonical route key(s)
       const domainToRoutes = new Map<string, string[]>();
       for (const [routeKey, domains] of routeDomains) {
@@ -882,7 +955,7 @@ export class MetricsManager {
         }
 
         domainAgg.set(domain, {
-          activeConnections: activeConnectionsByDomain.get(domain) ?? Math.round(totalConns),
+          activeConnections: Math.round(totalConns),
           bytesInPerSec: totalIn,
           bytesOutPerSec: totalOut,
           routeCount: routeKeys.length,
@@ -914,6 +987,7 @@ export class MetricsManager {
 
       return {
         connectionsByIP,
+        activeConnections,
         throughputRate,
         topIPs,
         topIPsByBandwidth,
@@ -928,7 +1002,7 @@ export class MetricsManager {
         backendProtocols,
         domainActivity,
       };
-    }, 1000); // 1s cache — matches typical dashboard poll interval
+    }, NETWORK_STATS_CACHE_TTL_MS);
   }
 
   private async buildTopASNs(
@@ -936,14 +1010,21 @@ export class MetricsManager {
     allIPData: Map<string, { count: number; bwIn: number; bwOut: number }>,
   ): Promise<IAsnActivity[]> {
     const manager = this.dcRouter.securityPolicyManager;
-    if (!manager || observedIps.length === 0) {
+    const candidateIps = [...new Set(observedIps.filter(Boolean))]
+      .sort()
+      .slice(0, MAX_ASN_CANDIDATE_IPS);
+    if (!manager || candidateIps.length === 0) {
       return [];
     }
 
-    const intelligenceRecords = await manager.listIpIntelligence({
-      ipAddresses: observedIps,
-      limit: Math.max(100, observedIps.length),
-    });
+    const intelligenceRecords = await this.metricsCache.get(
+      'networkAsnIntelligence',
+      () => manager.listIpIntelligence({
+        ipAddresses: candidateIps,
+        limit: Math.max(100, candidateIps.length),
+      }),
+      ASN_INTELLIGENCE_CACHE_TTL_MS,
+    );
     const asnActivity = new Map<number, IAsnActivity>();
 
     for (const record of intelligenceRecords) {

package/ts/opsserver/handlers/security.handler.ts

@@ -45,7 +45,13 @@ export class SecurityHandler {
         'getActiveConnections',
         async (dataArg, toolsArg) => {
           await requireOpsAuth(this.opsServerRef, dataArg, { scope: 'stats:read' });
-          const connectionInfos = await this.getActiveConnections(dataArg.protocol, dataArg.state);
+          const page = await this.getActiveConnections(
+            dataArg.protocol,
+            dataArg.state,
+            dataArg.limit,
+            dataArg.cursor,
+          );
+          const connectionInfos = page.connections;
           const totalConnections = connectionInfos.reduce((sum, conn) => sum + (conn.connectionCount || 1), 0);
           
           const summary = {
@@ -63,6 +69,7 @@ export class SecurityHandler {
           return {
             connections: connectionInfos,
             summary,
+            pagination: page.pagination,
           };
         }
       )
@@ -349,14 +356,27 @@ export class SecurityHandler {
   
   private async getActiveConnections(
     protocol?: 'http' | 'https' | 'smtp' | 'smtps',
-    state?: string
-  ): Promise<interfaces.data.IConnectionInfo[]> {
+    state?: string,
+    requestedLimit?: number,
+    cursor?: string,
+  ): Promise<{
+    connections: interfaces.data.IConnectionInfo[];
+    pagination: { limit: number; hasMore: boolean; nextCursor?: string };
+  }> {
+    const limit = Math.min(Math.max(Math.floor(requestedLimit ?? 250), 1), 1_000);
     const metricsManager = this.opsServerRef.dcRouterRef.metricsManager;
     if (!metricsManager) {
-      return [];
+      return {
+        connections: [],
+        pagination: { limit, hasMore: false },
+      };
     }
 
-    const snapshots = await metricsManager.getActiveConnectionSnapshots({ limit: 10000 });
+    const page = await metricsManager.getActiveConnectionSnapshotPage({
+      limit,
+      afterId: cursor,
+    });
+    const snapshots = page.snapshots;
     const connections = snapshots.map((snapshot): interfaces.data.IConnectionInfo => ({
       id: String(snapshot.id),
       remoteAddress: snapshot.sourcePort === null
@@ -372,7 +392,7 @@ export class SecurityHandler {
       bytesSent: snapshot.bytesOut,
     }));
 
-    return connections.filter((connection) => {
+    const filteredConnections = connections.filter((connection) => {
       if (protocol && connection.protocol !== protocol) {
         return false;
       }
@@ -381,6 +401,15 @@ export class SecurityHandler {
       }
       return true;
     });
+
+    return {
+      connections: filteredConnections,
+      pagination: {
+        limit,
+        hasMore: page.hasMore,
+        nextCursor: page.nextCursor ?? undefined,
+      },
+    };
   }
 
   private mapSnapshotProtocol(

package/ts/opsserver/handlers/stats.handler.ts

@@ -288,7 +288,6 @@ export class StatsHandler {
             promises.push(
               (async () => {
                 const stats = await this.opsServerRef.dcRouterRef.metricsManager!.getNetworkStats();
-                const serverStats = await this.collectServerStats();
 
                 // Build per-IP bandwidth lookup from throughputByIP
                 const ipBandwidth = new Map<string, { in: number; out: number }>();
@@ -322,7 +321,7 @@ export class StatsHandler {
                     in: stats.totalDataTransferred.bytesIn,
                     out: stats.totalDataTransferred.bytesOut,
                   },
-                  activeConnections: serverStats.activeConnections,
+                  activeConnections: stats.activeConnections,
                   connectionDetails,
                   topEndpoints: stats.topIPs.map(ip => ({
                     endpoint: ip.ip,

package/ts/remoteingress/classes.hub-lifecycle.ts

@@ -223,7 +223,9 @@ export class RemoteIngressHubLifecycle {
       tls: tlsConfig,
       performance: manager.getHubPerformanceConfig(),
     });
-    tunnelManager.setOnTopologyChanged((reason) => this.dcRouterRef.mailDnsSync.requestSync(reason));
+    tunnelManager.setOnTopologyChanged((reason) => (
+      this.dcRouterRef.mailDnsSync.requestEdgeEligibilityCheck(reason)
+    ));
     try {
       await tunnelManager.start();
     } catch (err) {

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '17.1.5',
+  version: '17.1.6',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts_web/appstate/network.ts

@@ -4,6 +4,9 @@ import { appState } from './shared.js';
 import { getActionContext } from './login.js';
 import { runBackgroundRefresh } from './shared.js';
 
+const IP_INTELLIGENCE_REFRESH_INTERVAL_MS = 60_000;
+let lastIpIntelligenceRefreshAt = 0;
+
 export interface INetworkState {
   connections: interfaces.data.IConnectionInfo[];
   connectionsByIP: { [ip: string]: number };
@@ -53,9 +56,17 @@ export const networkStatePart = await appState.getStatePart<INetworkState>(
 );
 
 export function refreshNetworkIpIntelligence(identity: interfaces.data.IIdentity, ipAddresses: string[]): void {
-  const ips = [...new Set(ipAddresses.filter(Boolean))].slice(0, 100);
+  const ips = [...new Set(ipAddresses.map((ip) => ip.trim()).filter(Boolean))]
+    .sort()
+    .slice(0, 100);
   if (ips.length === 0) return;
 
+  const now = Date.now();
+  if (now - lastIpIntelligenceRefreshAt < IP_INTELLIGENCE_REFRESH_INTERVAL_MS) {
+    return;
+  }
+  lastIpIntelligenceRefreshAt = now;
+
   runBackgroundRefresh('networkIpIntelligence', 'IP intelligence refresh failed:', async () => {
     const intelligenceRequest = new plugins.domtools.plugins.typedrequest.TypedRequest<
       interfaces.requests.IReq_ListIpIntelligence

package/ts_web/appstate/ui.ts

@@ -1,5 +1,4 @@
 import { appState } from './shared.js';
-import { networkStatePart, fetchNetworkStatsAction } from './network.js';
 import { certificateStatePart, fetchCertificateOverviewAction } from './certificates.js';
 
 export interface IUiState {
@@ -52,13 +51,6 @@ export const toggleAutoRefreshAction = uiStatePart.createAction(async (statePart
 export const setActiveViewAction = uiStatePart.createAction<string>(async (statePartArg, viewName): Promise<IUiState> => {
   const currentState = statePartArg.getState()!;
 
-  // If switching to network view, ensure we fetch network data
-  if (viewName === 'network' && currentState.activeView !== 'network') {
-    setTimeout(() => {
-      networkStatePart.dispatchAction(fetchNetworkStatsAction, null);
-    }, 100);
-  }
-
   // If switching to the Domains group, ensure we fetch certificate data
   // (Certificates is a subview of Domains).
   if (viewName === 'domains' && currentState.activeView !== 'domains') {
@@ -72,4 +64,3 @@ export const setActiveViewAction = uiStatePart.createAction<string>(async (state
     activeView: viewName,
   };
 });
-