@serve.zone/dcrouter 17.1.3 → 17.1.5

package/ts/security/classes.security-policy-manager.ts

@@ -12,6 +12,8 @@ import type {
 
 export interface ISecurityPolicyManagerOptions {
   intelligenceRefreshMs?: number;
+  retentionSweepIntervalMs?: number;
+  now?: () => number;
   onPolicyChanged?: () => void | Promise<void>;
 }
 
@@ -24,12 +26,24 @@ const OBSERVED_IP_BATCH_LIMIT = 20;
 const OBSERVED_IP_QUEUE_CONCURRENCY = 2;
 const OBSERVED_IP_REQUEUE_THROTTLE_MS = 60_000;
 
+export const IP_INTELLIGENCE_LIST_LIMIT = 500;
+export const IP_INTELLIGENCE_RETENTION_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
+export const IP_INTELLIGENCE_RETENTION_HIGH_WATER = 10_000;
+export const IP_INTELLIGENCE_RETENTION_LOW_WATER = 9_000;
+export const IP_INTELLIGENCE_RETENTION_BATCH_SIZE = 500;
+export const IP_INTELLIGENCE_RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+
 export class SecurityPolicyManager {
   private readonly smartNetwork = new plugins.smartnetwork.SmartNetwork({
     cacheTtl: 24 * 60 * 60 * 1000,
     ipIntelligenceTimeout: 5_000,
   });
   private readonly intelligenceRefreshMs: number;
+  private readonly retentionSweepIntervalMs: number;
+  private readonly now: () => number;
+  private retentionSweepTimer?: ReturnType<typeof setInterval>;
+  private retentionSweepPromise?: Promise<void>;
+  private retentionMutationTail: Promise<void> = Promise.resolve();
   private readonly inFlightObservations = new Map<string, Promise<void>>();
   private readonly queuedObservations = new Set<string>();
   private readonly observationQueue: string[] = [];
@@ -41,18 +55,34 @@ export class SecurityPolicyManager {
 
   constructor(options: ISecurityPolicyManagerOptions = {}) {
     this.intelligenceRefreshMs = options.intelligenceRefreshMs ?? 24 * 60 * 60 * 1000;
+    this.retentionSweepIntervalMs = options.retentionSweepIntervalMs
+      ?? IP_INTELLIGENCE_RETENTION_SWEEP_INTERVAL_MS;
+    this.now = options.now ?? Date.now;
     this.onPolicyChanged = options.onPolicyChanged;
   }
 
   public async start(): Promise<void> {
+    this.isStopping = false;
+    await this.requestRetentionSweep();
+    this.retentionSweepTimer = setInterval(() => {
+      void this.requestRetentionSweep();
+    }, this.retentionSweepIntervalMs);
+    this.retentionSweepTimer.unref?.();
     logger.log('info', 'SecurityPolicyManager started');
   }
 
   public async stop(): Promise<void> {
     this.isStopping = true;
+    if (this.retentionSweepTimer) {
+      clearInterval(this.retentionSweepTimer);
+      this.retentionSweepTimer = undefined;
+    }
     this.observationQueue.length = 0;
     this.queuedObservations.clear();
     await this.smartNetwork.stop();
+    await Promise.allSettled([...this.inFlightObservations.values()]);
+    await this.retentionSweepPromise;
+    await this.retentionMutationTail;
   }
 
   public async observeIps(ips: string[]): Promise<void> {
@@ -88,6 +118,7 @@ export class SecurityPolicyManager {
   }
 
   public async observeIp(ipAddress: string, options: { force?: boolean } = {}): Promise<void> {
+    if (this.isStopping) return;
     const ip = this.normalizeIp(ipAddress);
     if (!ip || !this.isPublicIp(ip)) {
       return;
@@ -110,7 +141,7 @@ export class SecurityPolicyManager {
 
   private async performObserveIp(ip: string, options: { force?: boolean } = {}): Promise<void> {
     try {
-      const now = Date.now();
+      const now = this.now();
       let doc = await IpIntelligenceDoc.findByIp(ip);
       if (doc && !options.force && now - doc.updatedAt < this.intelligenceRefreshMs) {
         if (now - doc.lastSeenAt > 60_000) {
@@ -122,16 +153,18 @@ export class SecurityPolicyManager {
       }
 
       const intelligence = await this.smartNetwork.getIpIntelligence(ip);
-      if (!doc) {
-        doc = new IpIntelligenceDoc();
-        doc.ipAddress = ip;
-        doc.firstSeenAt = now;
+      if (doc) {
+        Object.assign(doc, intelligence);
+        doc.lastSeenAt = now;
+        doc.updatedAt = now;
+        doc.seenCount = (doc.seenCount || 0) + 1;
+        await doc.save();
+      } else {
+        doc = await this.persistNewIntelligence(ip, intelligence, now);
+        if (!doc) {
+          return;
+        }
       }
-      Object.assign(doc, intelligence);
-      doc.lastSeenAt = now;
-      doc.updatedAt = now;
-      doc.seenCount = (doc.seenCount || 0) + 1;
-      await doc.save();
 
       if (await this.matchesAnyReactiveRule(doc)) {
         await this.notifyPolicyChanged();
@@ -147,20 +180,154 @@ export class SecurityPolicyManager {
 
   public async listIpIntelligence(options: { ipAddresses?: string[]; limit?: number } = {}): Promise<IIpIntelligenceRecord[]> {
     const limit = Number.isInteger(options.limit) && options.limit! > 0
-      ? Math.min(options.limit!, 500)
-      : undefined;
-
-    let docs: IpIntelligenceDoc[];
-    if (options.ipAddresses?.length) {
-      const ips = [...new Set(options.ipAddresses.map((ip) => this.normalizeIp(ip)).filter(Boolean) as string[])];
-      const results = await Promise.all(ips.map((ip) => IpIntelligenceDoc.findByIp(ip)));
-      docs = results.filter(Boolean) as IpIntelligenceDoc[];
-    } else {
-      docs = await IpIntelligenceDoc.findAll();
+      ? Math.min(options.limit!, IP_INTELLIGENCE_LIST_LIMIT)
+      : IP_INTELLIGENCE_LIST_LIMIT;
+
+    let ipAddresses: string[] | undefined;
+    if (options.ipAddresses !== undefined) {
+      if (!Array.isArray(options.ipAddresses)) {
+        throw new Error('ipAddresses must be an array');
+      }
+      const normalizedIps = options.ipAddresses.map((ipAddress, index) => {
+        if (typeof ipAddress !== 'string') {
+          throw new Error('ipAddresses[' + index + '] must be a string');
+        }
+        const normalizedIp = this.normalizeIp(ipAddress);
+        if (!normalizedIp) {
+          throw new Error('ipAddresses[' + index + '] is not a valid IP address');
+        }
+        return normalizedIp;
+      });
+      ipAddresses = [...new Set(normalizedIps)];
+      if (ipAddresses.length > IP_INTELLIGENCE_LIST_LIMIT) {
+        throw new Error(
+          'ipAddresses may contain at most ' + IP_INTELLIGENCE_LIST_LIMIT + ' unique addresses',
+        );
+      }
+      if (ipAddresses.length === 0) {
+        return [];
+      }
     }
 
-    const sortedDocs = docs.sort((a, b) => (b.lastSeenAt || 0) - (a.lastSeenAt || 0));
-    return (limit ? sortedDocs.slice(0, limit) : sortedDocs).map((doc) => this.intelligenceFromDoc(doc));
+    const docs = await IpIntelligenceDoc.findRecent({ ipAddresses, limit });
+    return docs.map((doc) => this.intelligenceFromDoc(doc));
+  }
+
+  private async persistNewIntelligence(
+    ip: string,
+    intelligence: Awaited<ReturnType<plugins.smartnetwork.SmartNetwork['getIpIntelligence']>>,
+    now: number,
+  ): Promise<IpIntelligenceDoc | null> {
+    return await this.withRetentionMutation(async () => {
+      if (this.isStopping) return null;
+      let doc = await IpIntelligenceDoc.findByIp(ip);
+      if (!doc) {
+        let countAfterPrune: number;
+        try {
+          countAfterPrune = await this.pruneIpIntelligence({ forCapacity: true });
+        } catch (error: unknown) {
+          logger.log(
+            'warn',
+            'Skipping persistence for new IP ' + ip
+              + ': cache capacity could not be verified: ' + (error as Error).message,
+          );
+          return null;
+        }
+        if (countAfterPrune >= IP_INTELLIGENCE_RETENTION_HIGH_WATER) {
+          logger.log(
+            'warn',
+            'Skipping persistence for new IP ' + ip
+              + ': cache remains at capacity (' + countAfterPrune + ')',
+          );
+          return null;
+        }
+        doc = new IpIntelligenceDoc();
+        doc.ipAddress = ip;
+        doc.firstSeenAt = now;
+      }
+
+      Object.assign(doc, intelligence);
+      doc.lastSeenAt = now;
+      doc.updatedAt = now;
+      doc.seenCount = (doc.seenCount || 0) + 1;
+      await doc.save();
+      return doc;
+    });
+  }
+
+  private async requestRetentionSweep(): Promise<void> {
+    if (this.retentionSweepPromise) {
+      return await this.retentionSweepPromise;
+    }
+
+    const sweepPromise = this.withRetentionMutation(async () => {
+      await this.pruneIpIntelligence({ forCapacity: false });
+    }).catch((error: unknown) => {
+      logger.log(
+        'warn',
+        'IP intelligence retention sweep failed: ' + (error as Error).message,
+      );
+    }).finally(() => {
+      if (this.retentionSweepPromise === sweepPromise) {
+        this.retentionSweepPromise = undefined;
+      }
+    });
+    this.retentionSweepPromise = sweepPromise;
+    return await sweepPromise;
+  }
+
+  private async pruneIpIntelligence(options: { forCapacity: boolean }): Promise<number> {
+    let deletedCount = 0;
+    const expiryCutoff = this.now() - IP_INTELLIGENCE_RETENTION_MAX_AGE_MS;
+
+    while (!this.isStopping) {
+      const expiredIds = await IpIntelligenceDoc.findOldestIds({
+        lastSeenBefore: expiryCutoff,
+        limit: IP_INTELLIGENCE_RETENTION_BATCH_SIZE,
+      });
+      if (expiredIds.length === 0) break;
+      deletedCount += await IpIntelligenceDoc.deleteByDocumentIds(expiredIds);
+      if (expiredIds.length < IP_INTELLIGENCE_RETENTION_BATCH_SIZE) break;
+    }
+
+    let currentCount = await IpIntelligenceDoc.countAll();
+    const shouldPruneExcess = currentCount > IP_INTELLIGENCE_RETENTION_HIGH_WATER
+      || (options.forCapacity && currentCount >= IP_INTELLIGENCE_RETENTION_HIGH_WATER);
+    if (shouldPruneExcess) {
+      let remainingToDelete = currentCount - IP_INTELLIGENCE_RETENTION_LOW_WATER;
+      while (remainingToDelete > 0 && !this.isStopping) {
+        const batchLimit = Math.min(
+          remainingToDelete,
+          IP_INTELLIGENCE_RETENTION_BATCH_SIZE,
+        );
+        const oldestIds = await IpIntelligenceDoc.findOldestIds({ limit: batchLimit });
+        if (oldestIds.length === 0) break;
+        const deletedThisBatch = await IpIntelligenceDoc.deleteByDocumentIds(oldestIds);
+        deletedCount += deletedThisBatch;
+        currentCount -= deletedThisBatch;
+        remainingToDelete -= deletedThisBatch;
+        if (deletedThisBatch === 0 || oldestIds.length < batchLimit) break;
+      }
+    }
+
+    if (deletedCount > 0) {
+      logger.log('info', 'IP intelligence retention removed ' + deletedCount + ' cached record(s)');
+    }
+    return currentCount;
+  }
+
+  private async withRetentionMutation<T>(operation: () => Promise<T>): Promise<T> {
+    const previous = this.retentionMutationTail;
+    let release!: () => void;
+    this.retentionMutationTail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    await previous;
+    try {
+      return await operation();
+    } finally {
+      release();
+    }
   }
 
   public async refreshIpIntelligence(ipAddress: string): Promise<IIpIntelligenceRecord | null> {
@@ -299,9 +466,9 @@ export class SecurityPolicyManager {
 
   public async compilePolicy(): Promise<ISecurityCompiledPolicy> {
     const rules = await SecurityBlockRuleDoc.findEnabled();
-    const intelligenceDocs = await IpIntelligenceDoc.findAll();
     const blockedIps = new Set<string>();
     const blockedCidrs = new Set<string>();
+    const reactiveRules: SecurityBlockRuleDoc[] = [];
 
     for (const rule of rules) {
       const normalizedValue = rule.value.trim();
@@ -320,7 +487,32 @@ export class SecurityPolicyManager {
         continue;
       }
 
-      for (const doc of intelligenceDocs) {
+      if (this.getReactiveRuleQuery(rule)) {
+        reactiveRules.push(rule);
+      }
+    }
+
+    if (reactiveRules.length === 0) {
+      return this.compiledPolicyFromSets(blockedIps, blockedCidrs);
+    }
+
+    const candidateDocs = new Map<string, IpIntelligenceDoc>();
+    for (const rule of reactiveRules) {
+      const query = this.getReactiveRuleQuery(rule)!;
+      const docs = query.type === 'asn'
+        ? await IpIntelligenceDoc.findByAsn(query.asn)
+        : await IpIntelligenceDoc.findByOrganization(query.organization, query.matchMode);
+      for (const doc of docs) {
+        const documentId = (doc as IpIntelligenceDoc & { _id?: unknown })._id;
+        const candidateKey = documentId === undefined
+          ? `ip:${doc.ipAddress}`
+          : `id:${String(documentId)}`;
+        candidateDocs.set(candidateKey, doc);
+      }
+    }
+
+    for (const rule of reactiveRules) {
+      for (const doc of candidateDocs.values()) {
         if (!this.ruleMatchesIntelligence(rule, doc)) continue;
         const networkEntries = this.normalizeNetworkEntryList([
           ...(doc.networkCidrs || []),
@@ -336,12 +528,45 @@ export class SecurityPolicyManager {
       }
     }
 
+    return this.compiledPolicyFromSets(blockedIps, blockedCidrs);
+  }
+
+  private compiledPolicyFromSets(
+    blockedIps: Set<string>,
+    blockedCidrs: Set<string>,
+  ): ISecurityCompiledPolicy {
     return {
       blockedIps: [...blockedIps].sort(),
       blockedCidrs: [...blockedCidrs].sort(),
     };
   }
 
+  private getReactiveRuleQuery(rule: SecurityBlockRuleDoc):
+    | { type: 'asn'; asn: number }
+    | { type: 'organization'; organization: string; matchMode: TSecurityBlockRuleMatchMode }
+    | undefined {
+    const value = rule.value.trim();
+    if (!value) return undefined;
+
+    if (rule.type === 'asn') {
+      const normalizedAsn = value.replace(/^as/i, '');
+      if (!/^\d+$/.test(normalizedAsn)) return undefined;
+      const asn = Number(normalizedAsn);
+      if (!Number.isSafeInteger(asn) || String(asn) !== normalizedAsn) return undefined;
+      return { type: 'asn', asn };
+    }
+
+    if (rule.type === 'organization') {
+      return {
+        type: 'organization',
+        organization: value,
+        matchMode: rule.matchMode === 'exact' ? 'exact' : 'contains',
+      };
+    }
+
+    return undefined;
+  }
+
   public async compileSmartProxyPolicy(): Promise<ISecurityCompiledPolicy> {
     return await this.compilePolicy();
   }
@@ -385,10 +610,10 @@ export class SecurityPolicyManager {
 
   private normalizeIp(ipAddress: string): string | undefined {
     const ip = ipAddress.trim();
-    if (ip.startsWith('::ffff:')) {
-      return ip.slice('::ffff:'.length);
-    }
-    return plugins.net.isIP(ip) ? ip : undefined;
+    const normalizedIp = ip.startsWith('::ffff:')
+      ? ip.slice('::ffff:'.length)
+      : ip;
+    return plugins.net.isIP(normalizedIp) ? normalizedIp : undefined;
   }
 
   private normalizeCidr(value: string): string | undefined {

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '17.1.3',
+  version: '17.1.5',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }