@push.rocks/smartproxy 25.17.10 → 26.1.0

package/ts/core/utils/log-deduplicator.ts

@@ -1,370 +0,0 @@
-import { logger } from './logger.js';
-
-interface ILogEvent {
-  level: 'info' | 'warn' | 'error' | 'debug';
-  message: string;
-  data?: any;
-  count: number;
-  firstSeen: number;
-  lastSeen: number;
-}
-
-interface IAggregatedEvent {
-  key: string;
-  events: Map<string, ILogEvent>;
-  flushTimer?: NodeJS.Timeout;
-}
-
-/**
- * Log deduplication utility to reduce log spam for repetitive events
- */
-export class LogDeduplicator {
-  private globalFlushTimer?: NodeJS.Timeout;
-  private aggregatedEvents: Map<string, IAggregatedEvent> = new Map();
-  private flushInterval: number = 5000; // 5 seconds
-  private maxBatchSize: number = 100;
-  private rapidEventThreshold: number = 50; // Flush early if this many events in 1 second
-  private lastRapidCheck: number = Date.now();
-  
-  constructor(flushInterval?: number) {
-    if (flushInterval) {
-      this.flushInterval = flushInterval;
-    }
-    
-    // Set up global periodic flush to ensure logs are emitted regularly
-    this.globalFlushTimer = setInterval(() => {
-      this.flushAll();
-    }, this.flushInterval * 2); // Flush everything every 2x the normal interval
-    
-    if (this.globalFlushTimer.unref) {
-      this.globalFlushTimer.unref();
-    }
-  }
-  
-  /**
-   * Log a deduplicated event
-   * @param key - Aggregation key (e.g., 'connection-rejected', 'cleanup-batch')
-   * @param level - Log level
-   * @param message - Log message template
-   * @param data - Additional data
-   * @param dedupeKey - Deduplication key within the aggregation (e.g., IP address, reason)
-   */
-  public log(
-    key: string,
-    level: 'info' | 'warn' | 'error' | 'debug',
-    message: string,
-    data?: any,
-    dedupeKey?: string
-  ): void {
-    const eventKey = dedupeKey || message;
-    const now = Date.now();
-    
-    if (!this.aggregatedEvents.has(key)) {
-      this.aggregatedEvents.set(key, {
-        key,
-        events: new Map(),
-        flushTimer: undefined
-      });
-    }
-    
-    const aggregated = this.aggregatedEvents.get(key)!;
-    
-    if (aggregated.events.has(eventKey)) {
-      const event = aggregated.events.get(eventKey)!;
-      event.count++;
-      event.lastSeen = now;
-      if (data) {
-        event.data = { ...event.data, ...data };
-      }
-    } else {
-      aggregated.events.set(eventKey, {
-        level,
-        message,
-        data,
-        count: 1,
-        firstSeen: now,
-        lastSeen: now
-      });
-    }
-    
-    // Check for rapid events (many events in short time)
-    const totalEvents = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
-    
-    // If we're getting flooded with events, flush more frequently
-    if (now - this.lastRapidCheck < 1000 && totalEvents >= this.rapidEventThreshold) {
-      this.flush(key);
-      this.lastRapidCheck = now;
-    } else if (aggregated.events.size >= this.maxBatchSize) {
-      // Check if we should flush due to size
-      this.flush(key);
-    } else if (!aggregated.flushTimer) {
-      // Schedule flush
-      aggregated.flushTimer = setTimeout(() => {
-        this.flush(key);
-      }, this.flushInterval);
-      
-      if (aggregated.flushTimer.unref) {
-        aggregated.flushTimer.unref();
-      }
-    }
-    
-    // Update rapid check time
-    if (now - this.lastRapidCheck >= 1000) {
-      this.lastRapidCheck = now;
-    }
-  }
-  
-  /**
-   * Flush aggregated events for a specific key
-   */
-  public flush(key: string): void {
-    const aggregated = this.aggregatedEvents.get(key);
-    if (!aggregated || aggregated.events.size === 0) {
-      return;
-    }
-    
-    if (aggregated.flushTimer) {
-      clearTimeout(aggregated.flushTimer);
-      aggregated.flushTimer = undefined;
-    }
-    
-    // Emit aggregated log based on the key
-    switch (key) {
-      case 'connection-rejected':
-        this.flushConnectionRejections(aggregated);
-        break;
-      case 'connection-cleanup':
-        this.flushConnectionCleanups(aggregated);
-        break;
-      case 'connection-terminated':
-        this.flushConnectionTerminations(aggregated);
-        break;
-      case 'ip-rejected':
-        this.flushIPRejections(aggregated);
-        break;
-      default:
-        this.flushGeneric(aggregated);
-    }
-    
-    // Clear events
-    aggregated.events.clear();
-  }
-  
-  /**
-   * Flush all pending events
-   */
-  public flushAll(): void {
-    for (const key of this.aggregatedEvents.keys()) {
-      this.flush(key);
-    }
-  }
-  
-  private flushConnectionRejections(aggregated: IAggregatedEvent): void {
-    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
-    const byReason = new Map<string, number>();
-    
-    for (const [, event] of aggregated.events) {
-      const reason = event.data?.reason || 'unknown';
-      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
-    }
-    
-    const reasonSummary = Array.from(byReason.entries())
-      .sort((a, b) => b[1] - a[1])
-      .map(([reason, count]) => `${reason}: ${count}`)
-      .join(', ');
-    
-    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
-    logger.log('warn', `[SUMMARY] Rejected ${totalCount} connections in ${Math.round(duration/1000)}s`, {
-      reasons: reasonSummary,
-      uniqueIPs: aggregated.events.size,
-      component: 'connection-dedup'
-    });
-  }
-  
-  private flushConnectionCleanups(aggregated: IAggregatedEvent): void {
-    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
-    const byReason = new Map<string, number>();
-    
-    for (const [, event] of aggregated.events) {
-      const reason = event.data?.reason || 'normal';
-      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
-    }
-    
-    const reasonSummary = Array.from(byReason.entries())
-      .sort((a, b) => b[1] - a[1])
-      .slice(0, 5) // Top 5 reasons
-      .map(([reason, count]) => `${reason}: ${count}`)
-      .join(', ');
-    
-    logger.log('info', `Cleaned up ${totalCount} connections`, {
-      reasons: reasonSummary,
-      duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
-      component: 'connection-dedup'
-    });
-  }
-  
-  private flushConnectionTerminations(aggregated: IAggregatedEvent): void {
-    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
-    const byReason = new Map<string, number>();
-    const byIP = new Map<string, number>();
-    let lastActiveCount = 0;
-    
-    for (const [, event] of aggregated.events) {
-      const reason = event.data?.reason || 'unknown';
-      const ip = event.data?.remoteIP || 'unknown';
-      
-      byReason.set(reason, (byReason.get(reason) || 0) + event.count);
-      
-      // Track by IP
-      if (ip !== 'unknown') {
-        byIP.set(ip, (byIP.get(ip) || 0) + event.count);
-      }
-      
-      // Track the last active connection count
-      if (event.data?.activeConnections !== undefined) {
-        lastActiveCount = event.data.activeConnections;
-      }
-    }
-    
-    const reasonSummary = Array.from(byReason.entries())
-      .sort((a, b) => b[1] - a[1])
-      .slice(0, 5) // Top 5 reasons
-      .map(([reason, count]) => `${reason}: ${count}`)
-      .join(', ');
-    
-    // Show top IPs if there are many different ones
-    let ipInfo = '';
-    if (byIP.size > 3) {
-      const topIPs = Array.from(byIP.entries())
-        .sort((a, b) => b[1] - a[1])
-        .slice(0, 3)
-        .map(([ip, count]) => `${ip} (${count})`)
-        .join(', ');
-      ipInfo = `, from ${byIP.size} IPs (top: ${topIPs})`;
-    } else if (byIP.size > 0) {
-      ipInfo = `, IPs: ${Array.from(byIP.keys()).join(', ')}`;
-    }
-    
-    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
-    
-    // Special handling for localhost connections (HttpProxy)
-    const localhostCount = byIP.get('::ffff:127.0.0.1') || 0;
-    if (localhostCount > 0 && byIP.size === 1) {
-      // All connections are from localhost (HttpProxy)
-      logger.log('info', `[SUMMARY] ${totalCount} HttpProxy connections terminated in ${Math.round(duration/1000)}s`, {
-        reasons: reasonSummary,
-        activeConnections: lastActiveCount,
-        component: 'connection-dedup'
-      });
-    } else {
-      logger.log('info', `[SUMMARY] ${totalCount} connections terminated in ${Math.round(duration/1000)}s`, {
-        reasons: reasonSummary,
-        activeConnections: lastActiveCount,
-        uniqueReasons: byReason.size,
-        ...(ipInfo ? { ips: ipInfo } : {}),
-        component: 'connection-dedup'
-      });
-    }
-  }
-  
-  private flushIPRejections(aggregated: IAggregatedEvent): void {
-    const byIP = new Map<string, { count: number; reasons: Set<string> }>();
-    const allReasons = new Map<string, number>();
-    
-    for (const [ip, event] of aggregated.events) {
-      if (!byIP.has(ip)) {
-        byIP.set(ip, { count: 0, reasons: new Set() });
-      }
-      const ipData = byIP.get(ip)!;
-      ipData.count += event.count;
-      if (event.data?.reason) {
-        ipData.reasons.add(event.data.reason);
-        // Track overall reason counts
-        allReasons.set(event.data.reason, (allReasons.get(event.data.reason) || 0) + event.count);
-      }
-    }
-    
-    // Create reason summary
-    const reasonSummary = Array.from(allReasons.entries())
-      .sort((a, b) => b[1] - a[1])
-      .map(([reason, count]) => `${reason}: ${count}`)
-      .join(', ');
-    
-    // Log top offenders
-    const topOffenders = Array.from(byIP.entries())
-      .sort((a, b) => b[1].count - a[1].count)
-      .slice(0, 10)
-      .map(([ip, data]) => `${ip} (${data.count}x, ${Array.from(data.reasons).join('/')})`)
-      .join(', ');
-    
-    const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
-    
-    const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
-    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s (${reasonSummary})`, {
-      topOffenders,
-      component: 'ip-dedup'
-    });
-  }
-  
-  private flushGeneric(aggregated: IAggregatedEvent): void {
-    const totalCount = Array.from(aggregated.events.values()).reduce((sum, e) => sum + e.count, 0);
-    const level = aggregated.events.values().next().value?.level || 'info';
-    
-    // Special handling for IP cleanup events
-    if (aggregated.key === 'ip-cleanup') {
-      const totalCleaned = Array.from(aggregated.events.values()).reduce((sum, e) => {
-        return sum + (e.data?.cleanedIPs || 0) + (e.data?.cleanedRateLimits || 0);
-      }, 0);
-      
-      if (totalCleaned > 0) {
-        logger.log(level as any, `IP tracking cleanup: removed ${totalCleaned} entries across ${totalCount} cleanup cycles`, {
-          duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
-          component: 'log-dedup'
-        });
-      }
-    } else {
-      logger.log(level as any, `${aggregated.key}: ${totalCount} events`, {
-        uniqueEvents: aggregated.events.size,
-        duration: Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen)),
-        component: 'log-dedup'
-      });
-    }
-  }
-  
-  /**
-   * Cleanup and stop deduplication
-   */
-  public cleanup(): void {
-    this.flushAll();
-    
-    if (this.globalFlushTimer) {
-      clearInterval(this.globalFlushTimer);
-      this.globalFlushTimer = undefined;
-    }
-    
-    for (const aggregated of this.aggregatedEvents.values()) {
-      if (aggregated.flushTimer) {
-        clearTimeout(aggregated.flushTimer);
-      }
-    }
-    this.aggregatedEvents.clear();
-  }
-}
-
-// Global instance for connection-related log deduplication
-export const connectionLogDeduplicator = new LogDeduplicator(5000); // 5 second batches
-
-// Ensure logs are flushed on process exit.
-// Only use beforeExit — do NOT call process.exit() from SIGINT/SIGTERM handlers
-// as that kills the host process's graceful shutdown (e.g., dcrouter connection draining).
-process.on('beforeExit', () => {
-  connectionLogDeduplicator.flushAll();
-});
-
-process.on('SIGINT', () => {
-  connectionLogDeduplicator.cleanup();
-});
-
-process.on('SIGTERM', () => {
-  connectionLogDeduplicator.cleanup();
-});

package/ts/core/utils/security-utils.ts

@@ -1,305 +0,0 @@
-import * as plugins from '../../plugins.js';
-import { IpMatcher } from '../routing/matchers/ip.js';
-
-/**
- * Security utilities for IP validation, rate limiting, 
- * authentication, and other security features
- */
-
-/**
- * Result of IP validation
- */
-export interface IIpValidationResult {
-  allowed: boolean;
-  reason?: string;
-}
-
-/**
- * IP connection tracking information
- */
-export interface IIpConnectionInfo {
-  connections: Set<string>;  // ConnectionIDs
-  timestamps: number[];      // Connection timestamps
-  ipVariants: string[];      // Normalized IP variants (e.g., ::ffff:127.0.0.1 and 127.0.0.1)
-}
-
-/**
- * Rate limit tracking
- */
-export interface IRateLimitInfo {
-  count: number;
-  expiry: number;
-}
-
-/**
- * Logger interface for security utilities
- */
-export interface ISecurityLogger {
-  info: (message: string, ...args: any[]) => void;
-  warn: (message: string, ...args: any[]) => void;
-  error: (message: string, ...args: any[]) => void;
-  debug?: (message: string, ...args: any[]) => void;
-}
-
-/**
- * Normalize IP addresses for comparison
- * Handles IPv4-mapped IPv6 addresses (::ffff:127.0.0.1)
- * 
- * @param ip IP address to normalize
- * @returns Array of equivalent IP representations
- */
-export function normalizeIP(ip: string): string[] {
-  if (!ip) return [];
-  
-  // Handle IPv4-mapped IPv6 addresses (::ffff:127.0.0.1)
-  if (ip.startsWith('::ffff:')) {
-    const ipv4 = ip.slice(7);
-    return [ip, ipv4];
-  }
-  
-  // Handle IPv4 addresses by also checking IPv4-mapped form
-  if (/^\d{1,3}(\.\d{1,3}){3}$/.test(ip)) {
-    return [ip, `::ffff:${ip}`];
-  }
-  
-  return [ip];
-}
-
-/**
- * Check if an IP is authorized based on allow and block lists
- *
- * @param ip - The IP address to check
- * @param allowedIPs - Array of allowed IP patterns
- * @param blockedIPs - Array of blocked IP patterns
- * @returns Whether the IP is authorized
- */
-export function isIPAuthorized(
-  ip: string, 
-  allowedIPs: string[] = ['*'], 
-  blockedIPs: string[] = []
-): boolean {
-  // Skip IP validation if no rules
-  if (!ip || (allowedIPs.length === 0 && blockedIPs.length === 0)) {
-    return true;
-  }
-
-  // First check if IP is blocked - blocked IPs take precedence
-  if (blockedIPs.length > 0) {
-    for (const pattern of blockedIPs) {
-      if (IpMatcher.match(pattern, ip)) {
-        return false;
-      }
-    }
-  }
-
-  // If allowed IPs list has wildcard, all non-blocked IPs are allowed
-  if (allowedIPs.includes('*')) {
-    return true;
-  }
-
-  // Then check if IP is allowed in the explicit allow list
-  if (allowedIPs.length > 0) {
-    for (const pattern of allowedIPs) {
-      if (IpMatcher.match(pattern, ip)) {
-        return true;
-      }
-    }
-    // If allowedIPs is specified but no match, deny access
-    return false;
-  }
-
-  // Default allow if no explicit allow list
-  return true;
-}
-
-/**
- * Check if an IP exceeds maximum connections
- *
- * @param ip - The IP address to check
- * @param ipConnectionsMap - Map of IPs to connection info
- * @param maxConnectionsPerIP - Maximum allowed connections per IP
- * @returns Result with allowed status and reason if blocked
- */
-export function checkMaxConnections(
-  ip: string,
-  ipConnectionsMap: Map<string, IIpConnectionInfo>,
-  maxConnectionsPerIP: number
-): IIpValidationResult {
-  if (!ipConnectionsMap.has(ip)) {
-    return { allowed: true };
-  }
-
-  const connectionCount = ipConnectionsMap.get(ip)!.connections.size;
-  
-  if (connectionCount >= maxConnectionsPerIP) {
-    return {
-      allowed: false,
-      reason: `Maximum connections per IP (${maxConnectionsPerIP}) exceeded`
-    };
-  }
-
-  return { allowed: true };
-}
-
-/**
- * Check if an IP exceeds connection rate limit
- *
- * @param ip - The IP address to check
- * @param ipConnectionsMap - Map of IPs to connection info
- * @param rateLimit - Maximum connections per minute
- * @returns Result with allowed status and reason if blocked
- */
-export function checkConnectionRate(
-  ip: string,
-  ipConnectionsMap: Map<string, IIpConnectionInfo>,
-  rateLimit: number
-): IIpValidationResult {
-  const now = Date.now();
-  const minute = 60 * 1000;
-
-  // Get or create connection info
-  if (!ipConnectionsMap.has(ip)) {
-    const info: IIpConnectionInfo = {
-      connections: new Set(),
-      timestamps: [now],
-      ipVariants: normalizeIP(ip)
-    };
-    ipConnectionsMap.set(ip, info);
-    return { allowed: true };
-  }
-
-  // Get timestamps and filter out entries older than 1 minute
-  const info = ipConnectionsMap.get(ip)!;
-  const timestamps = info.timestamps.filter(time => now - time < minute);
-  timestamps.push(now);
-  info.timestamps = timestamps;
-
-  // Check if rate exceeds limit
-  if (timestamps.length > rateLimit) {
-    return {
-      allowed: false,
-      reason: `Connection rate limit (${rateLimit}/min) exceeded`
-    };
-  }
-
-  return { allowed: true };
-}
-
-/**
- * Track a connection for an IP
- *
- * @param ip - The IP address
- * @param connectionId - The connection ID to track
- * @param ipConnectionsMap - Map of IPs to connection info
- */
-export function trackConnection(
-  ip: string,
-  connectionId: string,
-  ipConnectionsMap: Map<string, IIpConnectionInfo>
-): void {
-  if (!ipConnectionsMap.has(ip)) {
-    ipConnectionsMap.set(ip, {
-      connections: new Set([connectionId]),
-      timestamps: [Date.now()],
-      ipVariants: normalizeIP(ip)
-    });
-    return;
-  }
-
-  const info = ipConnectionsMap.get(ip)!;
-  info.connections.add(connectionId);
-}
-
-/**
- * Remove connection tracking for an IP
- *
- * @param ip - The IP address
- * @param connectionId - The connection ID to remove
- * @param ipConnectionsMap - Map of IPs to connection info
- */
-export function removeConnection(
-  ip: string,
-  connectionId: string,
-  ipConnectionsMap: Map<string, IIpConnectionInfo>
-): void {
-  if (!ipConnectionsMap.has(ip)) return;
-
-  const info = ipConnectionsMap.get(ip)!;
-  info.connections.delete(connectionId);
-
-  if (info.connections.size === 0) {
-    ipConnectionsMap.delete(ip);
-  }
-}
-
-/**
- * Clean up expired rate limits
- *
- * @param rateLimits - Map of rate limits to clean up
- * @param logger - Logger for debug messages
- */
-export function cleanupExpiredRateLimits(
-  rateLimits: Map<string, Map<string, IRateLimitInfo>>,
-  logger?: ISecurityLogger
-): void {
-  const now = Date.now();
-  let totalRemoved = 0;
-
-  for (const [routeId, routeLimits] of rateLimits.entries()) {
-    let removed = 0;
-    for (const [key, limit] of routeLimits.entries()) {
-      if (limit.expiry < now) {
-        routeLimits.delete(key);
-        removed++;
-        totalRemoved++;
-      }
-    }
-    
-    if (removed > 0 && logger?.debug) {
-      logger.debug(`Cleaned up ${removed} expired rate limits for route ${routeId}`);
-    }
-  }
-  
-  if (totalRemoved > 0 && logger?.info) {
-    logger.info(`Cleaned up ${totalRemoved} expired rate limits total`);
-  }
-}
-
-/**
- * Generate basic auth header value from username and password
- *
- * @param username - The username
- * @param password - The password
- * @returns Base64 encoded basic auth string
- */
-export function generateBasicAuthHeader(username: string, password: string): string {
-  return `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
-}
-
-/**
- * Parse basic auth header
- *
- * @param authHeader - The Authorization header value
- * @returns Username and password, or null if invalid
- */
-export function parseBasicAuthHeader(
-  authHeader: string
-): { username: string; password: string } | null {
-  if (!authHeader || !authHeader.startsWith('Basic ')) {
-    return null;
-  }
-
-  try {
-    const base64 = authHeader.slice(6); // Remove 'Basic '
-    const decoded = Buffer.from(base64, 'base64').toString();
-    const [username, password] = decoded.split(':');
-    
-    if (!username || !password) {
-      return null;
-    }
-
-    return { username, password };
-  } catch (err) {
-    return null;
-  }
-}