@push.rocks/smartproxy 19.6.13 → 19.6.15

package/ts/core/utils/log-deduplicator.ts

@@ -0,0 +1,361 @@
+import { logger } from './logger.js';
+
+interface ILogEvent {
+  level: 'info' | 'warn' | 'error' | 'debug';
+  message: string;
+  data?: any;
+  count: number;
+  firstSeen: number;
+  lastSeen: number;
+}
+
+interface IAggregatedEvent {
+  key: string;
+  events: Map<string, ILogEvent>;
+  flushTimer?: NodeJS.Timeout;
+}
+
+/**
+ * Log deduplication utility to reduce log spam for repetitive events
+ */
+export class LogDeduplicator {
+  private globalFlushTimer?: NodeJS.Timeout;
+  private aggregatedEvents: Map<string, IAggregatedEvent> = new Map();
+  private flushInterval: number = 5000; // 5 seconds
+  private maxBatchSize: number = 100;
+  private rapidEventThreshold: number = 50; // Flush early if this many events in 1 second
+  private lastRapidCheck: number = Date.now();
+  
+  constructor(flushInterval?: number) {
+    if (flushInterval) {
+      this.flushInterval = flushInterval;
+    }
+    
+    // Set up global periodic flush to ensure logs are emitted regularly
+    this.globalFlushTimer = setInterval(() => {
+      this.flushAll();
+    }, this.flushInterval * 2); // Flush everything every 2x the normal interval
+    
+    if (this.globalFlushTimer.unref) {
+      this.globalFlushTimer.unref();
+    }
+  }
+  
+  /**
+   * Log a deduplicated event
+   * @param key - Aggregation key (e.g., 'connection-rejected', 'cleanup-batch')
+   * @param level - Log level
+   * @param message - Log message template
+   * @param data - Additional data
+   * @param dedupeKey - Deduplication key within the aggregation (e.g., IP address, reason)
+   */
+  public log(
+    key: string,
+    level: 'info' | 'warn' | 'error' | 'debug',
+    message: string,
+    data?: any,
+    dedupeKey?: string
+  ): void {
+    const eventKey = dedupeKey || message;
+    const now = Date.now();
+    
+    if (!this.aggregatedEvents.has(key)) {
+      this.aggregatedEvents.set(key, {
+        key,
+        events: new Map(),
+        flushTimer: undefined
+      });
+    }
+    
+    const aggregated = this.aggregatedEvents.get(key)!;
+    
+    if (aggregated.events.has(eventKey)) {
+      const event = aggregated.events.get(eventKey)!;
+      event.count++;
+      event.lastSeen = now;
+      if (data) {
+        event.data = { ...event.data, ...data };
+      }
+    } else {
+      aggregated.events.set(eventKey, {
+        level,
+        message,
+        data,
+        count: 1,
+        firstSeen: now,
+        lastSeen: now
+      });
+    }
+    
+    // Check for rapid events (many events in short time)
+    const totalEvents = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    
+    // If we're getting flooded with events, flush more frequently
+    if (now - this.lastRapidCheck < 1000 && totalEvents >= this.rapidEventThreshold) {
+      this.flush(key);
+      this.lastRapidCheck = now;
+    } else if (aggregated.events.size >= this.maxBatchSize) {
+      // Check if we should flush due to size
+      this.flush(key);
+    } else if (!aggregated.flushTimer) {
+      // Schedule flush
+      aggregated.flushTimer = setTimeout(() => {
+        this.flush(key);
+      }, this.flushInterval);
+      
+      if (aggregated.flushTimer.unref) {
+        aggregated.flushTimer.unref();
+      }
+    }
+    
+    // Update rapid check time
+    if (now - this.lastRapidCheck >= 1000) {
+      this.lastRapidCheck = now;
+    }
+  }
+  
+  /**
+   * Flush aggregated events for a specific key
+   */
+  public flush(key: string): void {
+    const aggregated = this.aggregatedEvents.get(key);
+    if (!aggregated || aggregated.events.size === 0) {
+      return;
+    }
+    
+    if (aggregated.flushTimer) {
+      clearTimeout(aggregated.flushTimer);
+      aggregated.flushTimer = undefined;
+    }
+    
+    // Emit aggregated log based on the key
+    switch (key) {
+      case 'connection-rejected':
+        this.flushConnectionRejections(aggregated);
+        break;
+      case 'connection-cleanup':
+        this.flushConnectionCleanups(aggregated);
+        break;
+      case 'connection-terminated':
+        this.flushConnectionTerminations(aggregated);
+        break;
+      case 'ip-rejected':
+        this.flushIPRejections(aggregated);
+        break;
+      default:
+        this.flushGeneric(aggregated);
+    }
+    
+    // Clear events
+    aggregated.events.clear();
+  }
+  
+  /**
+   * Flush all pending events
+   */
+  public flushAll(): void {
+    for (const key of this.aggregatedEvents.keys()) {
+      this.flush(key);
+    }
+  }
+  
+  private flushConnectionRejections(aggregated: IAggregatedEvent): void {
+    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    const byReason = new Map<string, number>();
+    
+    for (const [, event] of aggregated.events) {
+      const reason = event.data?.reason || 'unknown';
+      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
+    }
+    
+    const reasonSummary = Array.from(byReason.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    logger.log('warn', `[SUMMARY] Rejected ${totalCount} connections in ${Math.round(duration/1000)}s`, {
+      reasons: reasonSummary,
+      uniqueIPs: aggregated.events.size,
+      component: 'connection-dedup'
+    });
+  }
+  
+  private flushConnectionCleanups(aggregated: IAggregatedEvent): void {
+    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    const byReason = new Map<string, number>();
+    
+    for (const [, event] of aggregated.events) {
+      const reason = event.data?.reason || 'normal';
+      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
+    }
+    
+    const reasonSummary = Array.from(byReason.entries())
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5) // Top 5 reasons
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
+    logger.log('info', `Cleaned up ${totalCount} connections`, {
+      reasons: reasonSummary,
+      duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
+      component: 'connection-dedup'
+    });
+  }
+  
+  private flushConnectionTerminations(aggregated: IAggregatedEvent): void {
+    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    const byReason = new Map<string, number>();
+    const byIP = new Map<string, number>();
+    let lastActiveCount = 0;
+    
+    for (const [, event] of aggregated.events) {
+      const reason = event.data?.reason || 'unknown';
+      const ip = event.data?.remoteIP || 'unknown';
+      
+      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
+      
+      // Track by IP
+      if (ip !== 'unknown') {
+        byIP.set(ip, (byIP.get(ip) || 0) + event.count);
+      }
+      
+      // Track the last active connection count
+      if (event.data?.activeConnections !== undefined) {
+        lastActiveCount = event.data.activeConnections;
+      }
+    }
+    
+    const reasonSummary = Array.from(byReason.entries())
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5) // Top 5 reasons
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
+    // Show top IPs if there are many different ones
+    let ipInfo = '';
+    if (byIP.size > 3) {
+      const topIPs = Array.from(byIP.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 3)
+        .map(([ip, count]) => `${ip} (${count})`)
+        .join(', ');
+      ipInfo = `, from ${byIP.size} IPs (top: ${topIPs})`;
+    } else if (byIP.size > 0) {
+      ipInfo = `, IPs: ${Array.from(byIP.keys()).join(', ')}`;
+    }
+    
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    
+    // Special handling for localhost connections (HttpProxy)
+    const localhostCount = byIP.get('::ffff:127.0.0.1') || 0;
+    if (localhostCount > 0 && byIP.size === 1) {
+      // All connections are from localhost (HttpProxy)
+      logger.log('info', `[SUMMARY] ${totalCount} HttpProxy connections terminated in ${Math.round(duration/1000)}s`, {
+        reasons: reasonSummary,
+        activeConnections: lastActiveCount,
+        component: 'connection-dedup'
+      });
+    } else {
+      logger.log('info', `[SUMMARY] ${totalCount} connections terminated in ${Math.round(duration/1000)}s`, {
+        reasons: reasonSummary,
+        activeConnections: lastActiveCount,
+        uniqueReasons: byReason.size,
+        ...(ipInfo ? { ips: ipInfo } : {}),
+        component: 'connection-dedup'
+      });
+    }
+  }
+  
+  private flushIPRejections(aggregated: IAggregatedEvent): void {
+    const byIP = new Map<string, { count: number; reasons: Set<string> }>();
+    
+    for (const [ip, event] of aggregated.events) {
+      if (!byIP.has(ip)) {
+        byIP.set(ip, { count: 0, reasons: new Set() });
+      }
+      const ipData = byIP.get(ip)!;
+      ipData.count += event.count;
+      if (event.data?.reason) {
+        ipData.reasons.add(event.data.reason);
+      }
+    }
+    
+    // Log top offenders
+    const topOffenders = Array.from(byIP.entries())
+      .sort((a, b) => b[1].count - a[1].count)
+      .slice(0, 10)
+      .map(([ip, data]) => `${ip} (${data.count}x, ${Array.from(data.reasons).join('/')})`)
+      .join(', ');
+    
+    const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
+    
+    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
+    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s`, {
+      topOffenders,
+      component: 'ip-dedup'
+    });
+  }
+  
+  private flushGeneric(aggregated: IAggregatedEvent): void {
+    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
+    const level = aggregated.events.values().next().value?.level || 'info';
+    
+    // Special handling for IP cleanup events
+    if (aggregated.key === 'ip-cleanup') {
+      const totalCleaned = Array.from(aggregated.events.values()).reduce((sum, e) => {
+        return sum + (e.data?.cleanedIPs || 0) + (e.data?.cleanedRateLimits || 0);
+      }, 0);
+      
+      if (totalCleaned > 0) {
+        logger.log(level as any, `IP tracking cleanup: removed ${totalCleaned} entries across ${totalCount} cleanup cycles`, {
+          duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
+          component: 'log-dedup'
+        });
+      }
+    } else {
+      logger.log(level as any, `${aggregated.key}: ${totalCount} events`, {
+        uniqueEvents: aggregated.events.size,
+        duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
+        component: 'log-dedup'
+      });
+    }
+  }
+  
+  /**
+   * Cleanup and stop deduplication
+   */
+  public cleanup(): void {
+    this.flushAll();
+    
+    if (this.globalFlushTimer) {
+      clearInterval(this.globalFlushTimer);
+      this.globalFlushTimer = undefined;
+    }
+    
+    for (const aggregated of this.aggregatedEvents.values()) {
+      if (aggregated.flushTimer) {
+        clearTimeout(aggregated.flushTimer);
+      }
+    }
+    this.aggregatedEvents.clear();
+  }
+}
+
+// Global instance for connection-related log deduplication
+export const connectionLogDeduplicator = new LogDeduplicator(5000); // 5 second batches
+
+// Ensure logs are flushed on process exit
+process.on('beforeExit', () => {
+  connectionLogDeduplicator.flushAll();
+});
+
+process.on('SIGINT', () => {
+  connectionLogDeduplicator.cleanup();
+  process.exit(0);
+});
+
+process.on('SIGTERM', () => {
+  connectionLogDeduplicator.cleanup();
+  process.exit(0);
+});

package/ts/core/utils/shared-security-manager.ts

@@ -152,9 +152,10 @@ export class SharedSecurityManager {
    * 
    * @param route - The route to check
    * @param context - The request context
+   * @param routeConnectionCount - Current connection count for this route (optional)
    * @returns Whether access is allowed
    */
-  public isAllowed(route: IRouteConfig, context: IRouteContext): boolean {
+  public isAllowed(route: IRouteConfig, context: IRouteContext, routeConnectionCount?: number): boolean {
     if (!route.security) {
       return true; // No security restrictions
     }
@@ -165,6 +166,14 @@ export class SharedSecurityManager {
       return false;
     }
     
+    // --- Route-level connection limit ---
+    if (route.security.maxConnections !== undefined && routeConnectionCount !== undefined) {
+      if (routeConnectionCount >= route.security.maxConnections) {
+        this.logger?.debug?.(`Route connection limit (${route.security.maxConnections}) exceeded for route ${route.name || 'unnamed'}`);
+        return false;
+      }
+    }
+    
     // --- Rate limiting ---
     if (route.security.rateLimit?.enabled && !this.isWithinRateLimit(route, context)) {
       this.logger?.debug?.(`Rate limit exceeded for route ${route.name || 'unnamed'}`);
@@ -304,6 +313,20 @@ export class SharedSecurityManager {
     // Clean up rate limits
     cleanupExpiredRateLimits(this.rateLimits, this.logger);
     
+    // Clean up IP connection tracking
+    let cleanedIPs = 0;
+    for (const [ip, info] of this.connectionsByIP.entries()) {
+      // Remove IPs with no active connections and no recent timestamps
+      if (info.connections.size === 0 && info.timestamps.length === 0) {
+        this.connectionsByIP.delete(ip);
+        cleanedIPs++;
+      }
+    }
+    
+    if (cleanedIPs > 0 && this.logger?.debug) {
+      this.logger.debug(`Cleaned up ${cleanedIPs} IPs with no active connections`);
+    }
+    
     // IP filter cache doesn't need cleanup (tied to routes)
   }
   

package/ts/proxies/http-proxy/http-proxy.ts

@@ -17,6 +17,8 @@ import { WebSocketHandler } from './websocket-handler.js';
 import { HttpRouter } from '../../routing/router/index.js';
 import { cleanupSocket } from '../../core/utils/socket-utils.js';
 import { FunctionCache } from './function-cache.js';
+import { SecurityManager } from './security-manager.js';
+import { connectionLogDeduplicator } from '../../core/utils/log-deduplicator.js';
 
 /**
  * HttpProxy provides a reverse proxy with TLS termination, WebSocket support,
@@ -43,6 +45,7 @@ export class HttpProxy implements IMetricsTracker {
   private router = new HttpRouter(); // Unified HTTP router
   private routeManager: RouteManager;
   private functionCache: FunctionCache;
+  private securityManager: SecurityManager;
   
   // State tracking
   public socketMap = new plugins.lik.ObjectMap<plugins.net.Socket>();
@@ -113,6 +116,14 @@ export class HttpProxy implements IMetricsTracker {
       maxCacheSize: this.options.functionCacheSize || 1000,
       defaultTtl: this.options.functionCacheTtl || 5000
     });
+    
+    // Initialize security manager
+    this.securityManager = new SecurityManager(
+      this.logger, 
+      [], 
+      this.options.maxConnectionsPerIP || 100, 
+      this.options.connectionRateLimitPerMinute || 300
+    );
 
     // Initialize other components
     this.certificateManager = new CertificateManager(this.options);
@@ -269,14 +280,113 @@ export class HttpProxy implements IMetricsTracker {
    */
   private setupConnectionTracking(): void {
     this.httpsServer.on('connection', (connection: plugins.net.Socket) => {
-      // Check if max connections reached
+      let remoteIP = connection.remoteAddress || '';
+      const connectionId = Math.random().toString(36).substring(2, 15);
+      const isFromSmartProxy = this.options.portProxyIntegration && connection.remoteAddress?.includes('127.0.0.1');
+      
+      // For SmartProxy connections, wait for CLIENT_IP header
+      if (isFromSmartProxy) {
+        let headerBuffer = Buffer.alloc(0);
+        let headerParsed = false;
+        
+        const parseHeader = (data: Buffer) => {
+          if (headerParsed) return data;
+          
+          headerBuffer = Buffer.concat([headerBuffer, data]);
+          const headerStr = headerBuffer.toString();
+          const headerEnd = headerStr.indexOf('\r\n');
+          
+          if (headerEnd !== -1) {
+            const header = headerStr.substring(0, headerEnd);
+            if (header.startsWith('CLIENT_IP:')) {
+              remoteIP = header.substring(10); // Extract IP after "CLIENT_IP:"
+              this.logger.debug(`Extracted client IP from SmartProxy: ${remoteIP}`);
+            }
+            headerParsed = true;
+            
+            // Store the real IP on the connection
+            (connection as any)._realRemoteIP = remoteIP;
+            
+            // Validate the real IP
+            const ipValidation = this.securityManager.validateIP(remoteIP);
+            if (!ipValidation.allowed) {
+              connectionLogDeduplicator.log(
+                'ip-rejected',
+                'warn',
+                `HttpProxy connection rejected (via SmartProxy)`,
+                { remoteIP, reason: ipValidation.reason, component: 'http-proxy' },
+                remoteIP
+              );
+              connection.destroy();
+              return null;
+            }
+            
+            // Track connection by real IP
+            this.securityManager.trackConnectionByIP(remoteIP, connectionId);
+            
+            // Return remaining data after header
+            return headerBuffer.slice(headerEnd + 2);
+          }
+          return null;
+        };
+        
+        // Override the first data handler to parse header
+        const originalEmit = connection.emit;
+        connection.emit = function(event: string, ...args: any[]) {
+          if (event === 'data' && !headerParsed) {
+            const remaining = parseHeader(args[0]);
+            if (remaining && remaining.length > 0) {
+              // Call original emit with remaining data
+              return originalEmit.apply(connection, ['data', remaining]);
+            } else if (headerParsed) {
+              // Header parsed but no remaining data
+              return true;
+            }
+            // Header not complete yet, suppress this data event
+            return true;
+          }
+          return originalEmit.apply(connection, [event, ...args]);
+        } as any;
+      } else {
+        // Direct connection - validate immediately
+        const ipValidation = this.securityManager.validateIP(remoteIP);
+        if (!ipValidation.allowed) {
+          connectionLogDeduplicator.log(
+            'ip-rejected',
+            'warn',
+            `HttpProxy connection rejected`,
+            { remoteIP, reason: ipValidation.reason, component: 'http-proxy' },
+            remoteIP
+          );
+          connection.destroy();
+          return;
+        }
+        
+        // Track connection by IP
+        this.securityManager.trackConnectionByIP(remoteIP, connectionId);
+      }
+      
+      // Then check global max connections
       if (this.socketMap.getArray().length >= this.options.maxConnections) {
-        this.logger.warn(`Max connections (${this.options.maxConnections}) reached, rejecting new connection`);
+        connectionLogDeduplicator.log(
+          'connection-rejected',
+          'warn',
+          'HttpProxy max connections reached',
+          {
+            reason: 'global-limit',
+            currentConnections: this.socketMap.getArray().length,
+            maxConnections: this.options.maxConnections,
+            component: 'http-proxy'
+          },
+          'http-proxy-global-limit'
+        );
         connection.destroy();
         return;
       }
-
-      // Add connection to tracking
+      
+      // Add connection to tracking with metadata
+      (connection as any)._connectionId = connectionId;
+      (connection as any)._remoteIP = remoteIP;
       this.socketMap.add(connection);
       this.connectedClients = this.socketMap.getArray().length;
       
@@ -284,12 +394,12 @@ export class HttpProxy implements IMetricsTracker {
       const localPort = connection.localPort || 0;
       const remotePort = connection.remotePort || 0;
       
-      // If this connection is from a SmartProxy (usually indicated by it coming from localhost)
-      if (this.options.portProxyIntegration && connection.remoteAddress?.includes('127.0.0.1')) {
+      // If this connection is from a SmartProxy
+      if (isFromSmartProxy) {
         this.portProxyConnections++;
-        this.logger.debug(`New connection from SmartProxy (local: ${localPort}, remote: ${remotePort})`);
+        this.logger.debug(`New connection from SmartProxy for client ${remoteIP} (local: ${localPort}, remote: ${remotePort})`);
       } else {
-        this.logger.debug(`New direct connection (local: ${localPort}, remote: ${remotePort})`);
+        this.logger.debug(`New direct connection from ${remoteIP} (local: ${localPort}, remote: ${remotePort})`);
       }
       
       // Setup connection cleanup handlers
@@ -298,12 +408,19 @@ export class HttpProxy implements IMetricsTracker {
           this.socketMap.remove(connection);
           this.connectedClients = this.socketMap.getArray().length;
           
+          // Remove IP tracking
+          const connId = (connection as any)._connectionId;
+          const connIP = (connection as any)._realRemoteIP || (connection as any)._remoteIP;
+          if (connId && connIP) {
+            this.securityManager.removeConnectionByIP(connIP, connId);
+          }
+          
           // If this was a SmartProxy connection, decrement the counter
           if (this.options.portProxyIntegration && connection.remoteAddress?.includes('127.0.0.1')) {
             this.portProxyConnections--;
           }
           
-          this.logger.debug(`Connection closed. ${this.connectedClients} connections remaining`);
+          this.logger.debug(`Connection closed from ${connIP || 'unknown'}. ${this.connectedClients} connections remaining`);
         }
       };
       
@@ -480,6 +597,9 @@ export class HttpProxy implements IMetricsTracker {
     
     // Certificate management cleanup is handled by SmartCertManager
     
+    // Flush any pending deduplicated logs
+    connectionLogDeduplicator.flushAll();
+    
     // Close the HTTPS server
     return new Promise((resolve) => {
       this.httpsServer.close(() => {

package/ts/proxies/http-proxy/models/types.ts

@@ -45,6 +45,10 @@ export interface IHttpProxyOptions {
 
   // Direct route configurations
   routes?: IRouteConfig[];
+  
+  // Rate limiting and security
+  maxConnectionsPerIP?: number; // Maximum simultaneous connections from a single IP
+  connectionRateLimitPerMinute?: number; // Max new connections per minute from a single IP
 }
 
 /**