@serve.zone/dcrouter 17.1.4 → 17.1.6

package/ts/monitoring/classes.metricsmanager.ts

@@ -5,11 +5,20 @@ import { SecurityLogger, SecurityEventType } from '../security/classes.securityl
 import { logger } from '../logger.js';
 import type { IAsnActivity } from '../../ts_interfaces/data/stats.js';
 
+const NETWORK_STATS_CACHE_TTL_MS = 3_000;
+const ASN_INTELLIGENCE_CACHE_TTL_MS = 60_000;
+const ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS = 3_000;
+const OBSERVED_IP_SAMPLE_INTERVAL_MS = 60_000;
+const MAX_ASN_CANDIDATE_IPS = 500;
+const MAX_OBSERVED_IP_SAMPLE_SIZE = 20;
+const MAX_SNAPSHOT_LIMIT = 1_000;
+
 export class MetricsManager {
   private metricsLogger: plugins.smartlog.Smartlog;
   private smartMetrics: plugins.smartmetrics.SmartMetrics;
   private dcRouter: DcRouter;
   private resetInterval?: NodeJS.Timeout;
+  private observedIpSamplerInterval?: NodeJS.Timeout;
   private metricsCache: MetricsCache;
   
   // Constants
@@ -69,7 +78,7 @@ export class MetricsManager {
       }
     });
     this.smartMetrics = new plugins.smartmetrics.SmartMetrics(this.metricsLogger, 'dcrouter');
-    // Initialize metrics cache with 500ms TTL
+    // Individual metrics select an appropriate TTL where their collection cost differs.
     this.metricsCache = new MetricsCache(500);
   }
   
@@ -118,6 +127,11 @@ export class MetricsManager {
       // Prune old time-series buckets every minute (don't wait for lazy query)
       this.pruneOldBuckets();
     }, 60000); // Check every minute
+
+    this.observedIpSamplerInterval = setInterval(() => {
+      this.sampleObservedIps();
+    }, OBSERVED_IP_SAMPLE_INTERVAL_MS);
+    this.observedIpSamplerInterval.unref();
     
     logger.log('info', 'MetricsManager started');
   }
@@ -128,6 +142,10 @@ export class MetricsManager {
       clearInterval(this.resetInterval);
       this.resetInterval = undefined;
     }
+    if (this.observedIpSamplerInterval) {
+      clearInterval(this.observedIpSamplerInterval);
+      this.observedIpSamplerInterval = undefined;
+    }
 
     this.smartMetrics.stop();
 
@@ -295,13 +313,35 @@ export class MetricsManager {
   public async getActiveConnectionSnapshots(
     options: plugins.smartproxy.IActiveConnectionSnapshotOptions = {},
   ): Promise<plugins.smartproxy.IActiveConnectionSnapshot[]> {
-    const cacheKey = `activeConnectionSnapshots:${options.limit ?? 1000}:${options.routeId ?? ''}`;
+    const normalizedOptions = {
+      ...options,
+      limit: Math.min(Math.max(options.limit ?? MAX_SNAPSHOT_LIMIT, 1), MAX_SNAPSHOT_LIMIT),
+    };
+    const afterId = (normalizedOptions as typeof normalizedOptions & { afterId?: string }).afterId ?? '';
+    const cacheKey = `activeConnectionSnapshots:${normalizedOptions.limit}:${normalizedOptions.routeId ?? ''}:${afterId}`;
     return await this.metricsCache.get<plugins.smartproxy.IActiveConnectionSnapshot[]>(cacheKey, async () => {
       if (!this.dcRouter.smartProxy) {
         return [];
       }
-      return this.dcRouter.smartProxy.getActiveConnectionSnapshots(options);
-    }, 500);
+      return this.dcRouter.smartProxy.getActiveConnectionSnapshots(normalizedOptions);
+    }, ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS);
+  }
+
+  public async getActiveConnectionSnapshotPage(
+    options: plugins.smartproxy.IActiveConnectionSnapshotOptions = {},
+  ): Promise<plugins.smartproxy.IActiveConnectionSnapshotPage> {
+    const normalizedOptions = {
+      ...options,
+      limit: Math.min(Math.max(options.limit ?? MAX_SNAPSHOT_LIMIT, 1), MAX_SNAPSHOT_LIMIT),
+    };
+    const afterId = normalizedOptions.afterId ?? '';
+    const cacheKey = `activeConnectionSnapshotPage:${normalizedOptions.limit}:${normalizedOptions.routeId ?? ''}:${afterId}`;
+    return await this.metricsCache.get<plugins.smartproxy.IActiveConnectionSnapshotPage>(cacheKey, async () => {
+      if (!this.dcRouter.smartProxy) {
+        return { snapshots: [], hasMore: false, nextCursor: null };
+      }
+      return this.dcRouter.smartProxy.getActiveConnectionSnapshotPage(normalizedOptions);
+    }, ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS);
   }
 
   // Get connection info from SmartProxy
@@ -560,10 +600,59 @@ export class MetricsManager {
       this.securityMetrics.incidents.shift();
     }
   }
+
+  /**
+   * Feed IP intelligence from maintained data-plane counters, independently of
+   * dashboard reads. Startup order is intentionally tolerated: until both
+   * SmartProxy and SecurityPolicyManager exist, the sample is a no-op.
+   */
+  private sampleObservedIps(): void {
+    const smartProxy = this.dcRouter.smartProxy;
+    const securityPolicyManager = this.dcRouter.securityPolicyManager;
+    if (!smartProxy || !securityPolicyManager) {
+      return;
+    }
+
+    try {
+      const proxyMetrics = smartProxy.getMetrics();
+      const connectionsByIP = proxyMetrics.connections.byIP();
+      const throughputByIP = proxyMetrics.throughput.byIP();
+      const candidates = new Map<string, { connections: number; throughput: number }>();
+
+      for (const [ip, connections] of connectionsByIP) {
+        candidates.set(ip, { connections, throughput: 0 });
+      }
+      for (const [ip, throughput] of throughputByIP) {
+        const candidate = candidates.get(ip) || { connections: 0, throughput: 0 };
+        candidate.throughput = throughput.in + throughput.out;
+        candidates.set(ip, candidate);
+      }
+      for (const ip of proxyMetrics.connections.domainRequestsByIP().keys()) {
+        if (!candidates.has(ip)) {
+          candidates.set(ip, { connections: 0, throughput: 0 });
+        }
+      }
+
+      const observedIps = [...candidates.entries()]
+        .filter(([ip]) => Boolean(ip))
+        .sort((a, b) => (
+          b[1].connections - a[1].connections
+          || b[1].throughput - a[1].throughput
+          || a[0].localeCompare(b[0])
+        ))
+        .slice(0, MAX_OBSERVED_IP_SAMPLE_SIZE)
+        .map(([ip]) => ip);
+
+      if (observedIps.length > 0) {
+        securityPolicyManager.queueObservedIps(observedIps);
+      }
+    } catch (error: unknown) {
+      logger.log('warn', `Observed-IP metrics sample failed: ${(error as Error).message}`);
+    }
+  }
   
   // Get network metrics from SmartProxy
   public async getNetworkStats() {
-    // Use shorter cache TTL for network stats to ensure real-time updates
     return this.metricsCache.get('networkStats', async () => {
       const smartProxy = this.dcRouter.smartProxy;
       const proxyMetrics = smartProxy ? smartProxy.getMetrics() : null;
@@ -571,6 +660,7 @@ export class MetricsManager {
       if (!proxyMetrics) {
         return {
           connectionsByIP: new Map<string, number>(),
+          activeConnections: 0,
           throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 },
           topIPs: [] as Array<{ ip: string; count: number }>,
           topIPsByBandwidth: [] as Array<{ ip: string; count: number; bwIn: number; bwOut: number }>,
@@ -587,21 +677,9 @@ export class MetricsManager {
         };
       }
 
-      const activeConnectionSnapshots = await this.getActiveConnectionSnapshots({ limit: 10000 });
-
-      const connectionsByIP = new Map<string, number>();
-      const connectionsByRoute = new Map<string, number>();
-      const activeConnectionsByDomain = new Map<string, number>();
-
-      for (const snapshot of activeConnectionSnapshots) {
-        connectionsByIP.set(snapshot.sourceIp, (connectionsByIP.get(snapshot.sourceIp) || 0) + 1);
-        if (snapshot.routeId) {
-          connectionsByRoute.set(snapshot.routeId, (connectionsByRoute.get(snapshot.routeId) || 0) + 1);
-        }
-        if (snapshot.domain) {
-          activeConnectionsByDomain.set(snapshot.domain, (activeConnectionsByDomain.get(snapshot.domain) || 0) + 1);
-        }
-      }
+      const connectionsByIP = proxyMetrics.connections.byIP();
+      const connectionsByRoute = proxyMetrics.connections.byRoute();
+      const activeConnections = proxyMetrics.connections.active();
 
       const instantThroughput = proxyMetrics.throughput.instant();
 
@@ -769,7 +847,6 @@ export class MetricsManager {
         ...topIPs.map((item) => item.ip),
         ...topIPsByBandwidth.map((item) => item.ip),
       ])];
-      this.dcRouter.securityPolicyManager?.queueObservedIps(observedIps);
 
       const topASNs = await this.buildTopASNs(observedIps, allIPData);
 
@@ -808,10 +885,6 @@ export class MetricsManager {
       for (const entry of protocolCache) {
         if (entry.domain) allKnownDomains.add(entry.domain);
       }
-      for (const snapshot of activeConnectionSnapshots) {
-        if (snapshot.domain) allKnownDomains.add(snapshot.domain);
-      }
-
       // Build reverse map: concrete domain → canonical route key(s)
       const domainToRoutes = new Map<string, string[]>();
       for (const [routeKey, domains] of routeDomains) {
@@ -882,7 +955,7 @@ export class MetricsManager {
         }
 
         domainAgg.set(domain, {
-          activeConnections: activeConnectionsByDomain.get(domain) ?? Math.round(totalConns),
+          activeConnections: Math.round(totalConns),
           bytesInPerSec: totalIn,
           bytesOutPerSec: totalOut,
           routeCount: routeKeys.length,
@@ -914,6 +987,7 @@ export class MetricsManager {
 
       return {
         connectionsByIP,
+        activeConnections,
         throughputRate,
         topIPs,
         topIPsByBandwidth,
@@ -928,7 +1002,7 @@ export class MetricsManager {
         backendProtocols,
         domainActivity,
       };
-    }, 1000); // 1s cache — matches typical dashboard poll interval
+    }, NETWORK_STATS_CACHE_TTL_MS);
   }
 
   private async buildTopASNs(
@@ -936,14 +1010,21 @@ export class MetricsManager {
     allIPData: Map<string, { count: number; bwIn: number; bwOut: number }>,
   ): Promise<IAsnActivity[]> {
     const manager = this.dcRouter.securityPolicyManager;
-    if (!manager || observedIps.length === 0) {
+    const candidateIps = [...new Set(observedIps.filter(Boolean))]
+      .sort()
+      .slice(0, MAX_ASN_CANDIDATE_IPS);
+    if (!manager || candidateIps.length === 0) {
       return [];
     }
 
-    const intelligenceRecords = await manager.listIpIntelligence({
-      ipAddresses: observedIps,
-      limit: Math.max(100, observedIps.length),
-    });
+    const intelligenceRecords = await this.metricsCache.get(
+      'networkAsnIntelligence',
+      () => manager.listIpIntelligence({
+        ipAddresses: candidateIps,
+        limit: Math.max(100, candidateIps.length),
+      }),
+      ASN_INTELLIGENCE_CACHE_TTL_MS,
+    );
     const asnActivity = new Map<number, IAsnActivity>();
 
     for (const record of intelligenceRecords) {

package/ts/opsserver/handlers/security.handler.ts

@@ -45,7 +45,13 @@ export class SecurityHandler {
         'getActiveConnections',
         async (dataArg, toolsArg) => {
           await requireOpsAuth(this.opsServerRef, dataArg, { scope: 'stats:read' });
-          const connectionInfos = await this.getActiveConnections(dataArg.protocol, dataArg.state);
+          const page = await this.getActiveConnections(
+            dataArg.protocol,
+            dataArg.state,
+            dataArg.limit,
+            dataArg.cursor,
+          );
+          const connectionInfos = page.connections;
           const totalConnections = connectionInfos.reduce((sum, conn) => sum + (conn.connectionCount || 1), 0);
           
           const summary = {
@@ -63,6 +69,7 @@ export class SecurityHandler {
           return {
             connections: connectionInfos,
             summary,
+            pagination: page.pagination,
           };
         }
       )
@@ -349,14 +356,27 @@ export class SecurityHandler {
   
   private async getActiveConnections(
     protocol?: 'http' | 'https' | 'smtp' | 'smtps',
-    state?: string
-  ): Promise<interfaces.data.IConnectionInfo[]> {
+    state?: string,
+    requestedLimit?: number,
+    cursor?: string,
+  ): Promise<{
+    connections: interfaces.data.IConnectionInfo[];
+    pagination: { limit: number; hasMore: boolean; nextCursor?: string };
+  }> {
+    const limit = Math.min(Math.max(Math.floor(requestedLimit ?? 250), 1), 1_000);
     const metricsManager = this.opsServerRef.dcRouterRef.metricsManager;
     if (!metricsManager) {
-      return [];
+      return {
+        connections: [],
+        pagination: { limit, hasMore: false },
+      };
     }
 
-    const snapshots = await metricsManager.getActiveConnectionSnapshots({ limit: 10000 });
+    const page = await metricsManager.getActiveConnectionSnapshotPage({
+      limit,
+      afterId: cursor,
+    });
+    const snapshots = page.snapshots;
     const connections = snapshots.map((snapshot): interfaces.data.IConnectionInfo => ({
       id: String(snapshot.id),
       remoteAddress: snapshot.sourcePort === null
@@ -372,7 +392,7 @@ export class SecurityHandler {
       bytesSent: snapshot.bytesOut,
     }));
 
-    return connections.filter((connection) => {
+    const filteredConnections = connections.filter((connection) => {
       if (protocol && connection.protocol !== protocol) {
         return false;
       }
@@ -381,6 +401,15 @@ export class SecurityHandler {
       }
       return true;
     });
+
+    return {
+      connections: filteredConnections,
+      pagination: {
+        limit,
+        hasMore: page.hasMore,
+        nextCursor: page.nextCursor ?? undefined,
+      },
+    };
   }
 
   private mapSnapshotProtocol(

package/ts/opsserver/handlers/stats.handler.ts

@@ -288,7 +288,6 @@ export class StatsHandler {
             promises.push(
               (async () => {
                 const stats = await this.opsServerRef.dcRouterRef.metricsManager!.getNetworkStats();
-                const serverStats = await this.collectServerStats();
 
                 // Build per-IP bandwidth lookup from throughputByIP
                 const ipBandwidth = new Map<string, { in: number; out: number }>();
@@ -322,7 +321,7 @@ export class StatsHandler {
                     in: stats.totalDataTransferred.bytesIn,
                     out: stats.totalDataTransferred.bytesOut,
                   },
-                  activeConnections: serverStats.activeConnections,
+                  activeConnections: stats.activeConnections,
                   connectionDetails,
                   topEndpoints: stats.topIPs.map(ip => ({
                     endpoint: ip.ip,

package/ts/remoteingress/classes.hub-lifecycle.ts

@@ -223,7 +223,9 @@ export class RemoteIngressHubLifecycle {
       tls: tlsConfig,
       performance: manager.getHubPerformanceConfig(),
     });
-    tunnelManager.setOnTopologyChanged((reason) => this.dcRouterRef.mailDnsSync.requestSync(reason));
+    tunnelManager.setOnTopologyChanged((reason) => (
+      this.dcRouterRef.mailDnsSync.requestEdgeEligibilityCheck(reason)
+    ));
     try {
       await tunnelManager.start();
     } catch (err) {

package/ts/security/classes.security-policy-manager.ts

@@ -12,6 +12,8 @@ import type {
 
 export interface ISecurityPolicyManagerOptions {
   intelligenceRefreshMs?: number;
+  retentionSweepIntervalMs?: number;
+  now?: () => number;
   onPolicyChanged?: () => void | Promise<void>;
 }
 
@@ -24,12 +26,24 @@ const OBSERVED_IP_BATCH_LIMIT = 20;
 const OBSERVED_IP_QUEUE_CONCURRENCY = 2;
 const OBSERVED_IP_REQUEUE_THROTTLE_MS = 60_000;
 
+export const IP_INTELLIGENCE_LIST_LIMIT = 500;
+export const IP_INTELLIGENCE_RETENTION_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
+export const IP_INTELLIGENCE_RETENTION_HIGH_WATER = 10_000;
+export const IP_INTELLIGENCE_RETENTION_LOW_WATER = 9_000;
+export const IP_INTELLIGENCE_RETENTION_BATCH_SIZE = 500;
+export const IP_INTELLIGENCE_RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+
 export class SecurityPolicyManager {
   private readonly smartNetwork = new plugins.smartnetwork.SmartNetwork({
     cacheTtl: 24 * 60 * 60 * 1000,
     ipIntelligenceTimeout: 5_000,
   });
   private readonly intelligenceRefreshMs: number;
+  private readonly retentionSweepIntervalMs: number;
+  private readonly now: () => number;
+  private retentionSweepTimer?: ReturnType<typeof setInterval>;
+  private retentionSweepPromise?: Promise<void>;
+  private retentionMutationTail: Promise<void> = Promise.resolve();
   private readonly inFlightObservations = new Map<string, Promise<void>>();
   private readonly queuedObservations = new Set<string>();
   private readonly observationQueue: string[] = [];
@@ -41,18 +55,34 @@ export class SecurityPolicyManager {
 
   constructor(options: ISecurityPolicyManagerOptions = {}) {
     this.intelligenceRefreshMs = options.intelligenceRefreshMs ?? 24 * 60 * 60 * 1000;
+    this.retentionSweepIntervalMs = options.retentionSweepIntervalMs
+      ?? IP_INTELLIGENCE_RETENTION_SWEEP_INTERVAL_MS;
+    this.now = options.now ?? Date.now;
     this.onPolicyChanged = options.onPolicyChanged;
   }
 
   public async start(): Promise<void> {
+    this.isStopping = false;
+    await this.requestRetentionSweep();
+    this.retentionSweepTimer = setInterval(() => {
+      void this.requestRetentionSweep();
+    }, this.retentionSweepIntervalMs);
+    this.retentionSweepTimer.unref?.();
     logger.log('info', 'SecurityPolicyManager started');
   }
 
   public async stop(): Promise<void> {
     this.isStopping = true;
+    if (this.retentionSweepTimer) {
+      clearInterval(this.retentionSweepTimer);
+      this.retentionSweepTimer = undefined;
+    }
     this.observationQueue.length = 0;
     this.queuedObservations.clear();
     await this.smartNetwork.stop();
+    await Promise.allSettled([...this.inFlightObservations.values()]);
+    await this.retentionSweepPromise;
+    await this.retentionMutationTail;
   }
 
   public async observeIps(ips: string[]): Promise<void> {
@@ -88,6 +118,7 @@ export class SecurityPolicyManager {
   }
 
   public async observeIp(ipAddress: string, options: { force?: boolean } = {}): Promise<void> {
+    if (this.isStopping) return;
     const ip = this.normalizeIp(ipAddress);
     if (!ip || !this.isPublicIp(ip)) {
       return;
@@ -110,7 +141,7 @@ export class SecurityPolicyManager {
 
   private async performObserveIp(ip: string, options: { force?: boolean } = {}): Promise<void> {
     try {
-      const now = Date.now();
+      const now = this.now();
       let doc = await IpIntelligenceDoc.findByIp(ip);
       if (doc && !options.force && now - doc.updatedAt < this.intelligenceRefreshMs) {
         if (now - doc.lastSeenAt > 60_000) {
@@ -122,16 +153,18 @@ export class SecurityPolicyManager {
       }
 
       const intelligence = await this.smartNetwork.getIpIntelligence(ip);
-      if (!doc) {
-        doc = new IpIntelligenceDoc();
-        doc.ipAddress = ip;
-        doc.firstSeenAt = now;
+      if (doc) {
+        Object.assign(doc, intelligence);
+        doc.lastSeenAt = now;
+        doc.updatedAt = now;
+        doc.seenCount = (doc.seenCount || 0) + 1;
+        await doc.save();
+      } else {
+        doc = await this.persistNewIntelligence(ip, intelligence, now);
+        if (!doc) {
+          return;
+        }
       }
-      Object.assign(doc, intelligence);
-      doc.lastSeenAt = now;
-      doc.updatedAt = now;
-      doc.seenCount = (doc.seenCount || 0) + 1;
-      await doc.save();
 
       if (await this.matchesAnyReactiveRule(doc)) {
         await this.notifyPolicyChanged();
@@ -147,20 +180,154 @@ export class SecurityPolicyManager {
 
   public async listIpIntelligence(options: { ipAddresses?: string[]; limit?: number } = {}): Promise<IIpIntelligenceRecord[]> {
     const limit = Number.isInteger(options.limit) && options.limit! > 0
-      ? Math.min(options.limit!, 500)
-      : undefined;
-
-    let docs: IpIntelligenceDoc[];
-    if (options.ipAddresses?.length) {
-      const ips = [...new Set(options.ipAddresses.map((ip) => this.normalizeIp(ip)).filter(Boolean) as string[])];
-      const results = await Promise.all(ips.map((ip) => IpIntelligenceDoc.findByIp(ip)));
-      docs = results.filter(Boolean) as IpIntelligenceDoc[];
-    } else {
-      docs = await IpIntelligenceDoc.findAll();
+      ? Math.min(options.limit!, IP_INTELLIGENCE_LIST_LIMIT)
+      : IP_INTELLIGENCE_LIST_LIMIT;
+
+    let ipAddresses: string[] | undefined;
+    if (options.ipAddresses !== undefined) {
+      if (!Array.isArray(options.ipAddresses)) {
+        throw new Error('ipAddresses must be an array');
+      }
+      const normalizedIps = options.ipAddresses.map((ipAddress, index) => {
+        if (typeof ipAddress !== 'string') {
+          throw new Error('ipAddresses[' + index + '] must be a string');
+        }
+        const normalizedIp = this.normalizeIp(ipAddress);
+        if (!normalizedIp) {
+          throw new Error('ipAddresses[' + index + '] is not a valid IP address');
+        }
+        return normalizedIp;
+      });
+      ipAddresses = [...new Set(normalizedIps)];
+      if (ipAddresses.length > IP_INTELLIGENCE_LIST_LIMIT) {
+        throw new Error(
+          'ipAddresses may contain at most ' + IP_INTELLIGENCE_LIST_LIMIT + ' unique addresses',
+        );
+      }
+      if (ipAddresses.length === 0) {
+        return [];
+      }
+    }
+
+    const docs = await IpIntelligenceDoc.findRecent({ ipAddresses, limit });
+    return docs.map((doc) => this.intelligenceFromDoc(doc));
+  }
+
+  private async persistNewIntelligence(
+    ip: string,
+    intelligence: Awaited<ReturnType<plugins.smartnetwork.SmartNetwork['getIpIntelligence']>>,
+    now: number,
+  ): Promise<IpIntelligenceDoc | null> {
+    return await this.withRetentionMutation(async () => {
+      if (this.isStopping) return null;
+      let doc = await IpIntelligenceDoc.findByIp(ip);
+      if (!doc) {
+        let countAfterPrune: number;
+        try {
+          countAfterPrune = await this.pruneIpIntelligence({ forCapacity: true });
+        } catch (error: unknown) {
+          logger.log(
+            'warn',
+            'Skipping persistence for new IP ' + ip
+              + ': cache capacity could not be verified: ' + (error as Error).message,
+          );
+          return null;
+        }
+        if (countAfterPrune >= IP_INTELLIGENCE_RETENTION_HIGH_WATER) {
+          logger.log(
+            'warn',
+            'Skipping persistence for new IP ' + ip
+              + ': cache remains at capacity (' + countAfterPrune + ')',
+          );
+          return null;
+        }
+        doc = new IpIntelligenceDoc();
+        doc.ipAddress = ip;
+        doc.firstSeenAt = now;
+      }
+
+      Object.assign(doc, intelligence);
+      doc.lastSeenAt = now;
+      doc.updatedAt = now;
+      doc.seenCount = (doc.seenCount || 0) + 1;
+      await doc.save();
+      return doc;
+    });
+  }
+
+  private async requestRetentionSweep(): Promise<void> {
+    if (this.retentionSweepPromise) {
+      return await this.retentionSweepPromise;
     }
 
-    const sortedDocs = docs.sort((a, b) => (b.lastSeenAt || 0) - (a.lastSeenAt || 0));
-    return (limit ? sortedDocs.slice(0, limit) : sortedDocs).map((doc) => this.intelligenceFromDoc(doc));
+    const sweepPromise = this.withRetentionMutation(async () => {
+      await this.pruneIpIntelligence({ forCapacity: false });
+    }).catch((error: unknown) => {
+      logger.log(
+        'warn',
+        'IP intelligence retention sweep failed: ' + (error as Error).message,
+      );
+    }).finally(() => {
+      if (this.retentionSweepPromise === sweepPromise) {
+        this.retentionSweepPromise = undefined;
+      }
+    });
+    this.retentionSweepPromise = sweepPromise;
+    return await sweepPromise;
+  }
+
+  private async pruneIpIntelligence(options: { forCapacity: boolean }): Promise<number> {
+    let deletedCount = 0;
+    const expiryCutoff = this.now() - IP_INTELLIGENCE_RETENTION_MAX_AGE_MS;
+
+    while (!this.isStopping) {
+      const expiredIds = await IpIntelligenceDoc.findOldestIds({
+        lastSeenBefore: expiryCutoff,
+        limit: IP_INTELLIGENCE_RETENTION_BATCH_SIZE,
+      });
+      if (expiredIds.length === 0) break;
+      deletedCount += await IpIntelligenceDoc.deleteByDocumentIds(expiredIds);
+      if (expiredIds.length < IP_INTELLIGENCE_RETENTION_BATCH_SIZE) break;
+    }
+
+    let currentCount = await IpIntelligenceDoc.countAll();
+    const shouldPruneExcess = currentCount > IP_INTELLIGENCE_RETENTION_HIGH_WATER
+      || (options.forCapacity && currentCount >= IP_INTELLIGENCE_RETENTION_HIGH_WATER);
+    if (shouldPruneExcess) {
+      let remainingToDelete = currentCount - IP_INTELLIGENCE_RETENTION_LOW_WATER;
+      while (remainingToDelete > 0 && !this.isStopping) {
+        const batchLimit = Math.min(
+          remainingToDelete,
+          IP_INTELLIGENCE_RETENTION_BATCH_SIZE,
+        );
+        const oldestIds = await IpIntelligenceDoc.findOldestIds({ limit: batchLimit });
+        if (oldestIds.length === 0) break;
+        const deletedThisBatch = await IpIntelligenceDoc.deleteByDocumentIds(oldestIds);
+        deletedCount += deletedThisBatch;
+        currentCount -= deletedThisBatch;
+        remainingToDelete -= deletedThisBatch;
+        if (deletedThisBatch === 0 || oldestIds.length < batchLimit) break;
+      }
+    }
+
+    if (deletedCount > 0) {
+      logger.log('info', 'IP intelligence retention removed ' + deletedCount + ' cached record(s)');
+    }
+    return currentCount;
+  }
+
+  private async withRetentionMutation<T>(operation: () => Promise<T>): Promise<T> {
+    const previous = this.retentionMutationTail;
+    let release!: () => void;
+    this.retentionMutationTail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    await previous;
+    try {
+      return await operation();
+    } finally {
+      release();
+    }
   }
 
   public async refreshIpIntelligence(ipAddress: string): Promise<IIpIntelligenceRecord | null> {
@@ -443,10 +610,10 @@ export class SecurityPolicyManager {
 
   private normalizeIp(ipAddress: string): string | undefined {
     const ip = ipAddress.trim();
-    if (ip.startsWith('::ffff:')) {
-      return ip.slice('::ffff:'.length);
-    }
-    return plugins.net.isIP(ip) ? ip : undefined;
+    const normalizedIp = ip.startsWith('::ffff:')
+      ? ip.slice('::ffff:'.length)
+      : ip;
+    return plugins.net.isIP(normalizedIp) ? normalizedIp : undefined;
   }
 
   private normalizeCidr(value: string): string | undefined {

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '17.1.4',
+  version: '17.1.6',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }