@push.rocks/smartproxy 19.6.6 → 19.6.8

package/ts/proxies/smart-proxy/metrics-collector.ts

@@ -1,258 +1,370 @@
 import * as plugins from '../../plugins.js';
 import type { SmartProxy } from './smart-proxy.js';
-import type { IProxyStats, IProxyStatsExtended } from './models/metrics-types.js';
+import type { 
+  IMetrics, 
+  IThroughputData, 
+  IThroughputHistoryPoint,
+  IByteTracker 
+} from './models/metrics-types.js';
+import { ThroughputTracker } from './throughput-tracker.js';
 import { logger } from '../../core/utils/logger.js';
 
 /**
- * Collects and computes metrics for SmartProxy on-demand
+ * Collects and provides metrics for SmartProxy with clean API
  */
-export class MetricsCollector implements IProxyStatsExtended {
-  // RPS tracking (the only state we need to maintain)
-  private requestTimestamps: number[] = [];
-  private readonly RPS_WINDOW_SIZE = 60000; // 1 minute window
-  private readonly MAX_TIMESTAMPS = 5000; // Maximum timestamps to keep
+export class MetricsCollector implements IMetrics {
+  // Throughput tracking
+  private throughputTracker: ThroughputTracker;
   
-  // Optional caching for performance
-  private cachedMetrics: {
-    timestamp: number;
-    connectionsByRoute?: Map<string, number>;
-    connectionsByIP?: Map<string, number>;
-  } = { timestamp: 0 };
+  // Request tracking
+  private requestTimestamps: number[] = [];
+  private totalRequests: number = 0;
   
-  private readonly CACHE_TTL = 1000; // 1 second cache
+  // Connection byte tracking for per-route/IP metrics
+  private connectionByteTrackers = new Map<string, IByteTracker>();
   
-  // RxJS subscription for connection events
+  // Subscriptions
+  private samplingInterval?: NodeJS.Timeout;
   private connectionSubscription?: plugins.smartrx.rxjs.Subscription;
   
+  // Configuration
+  private readonly sampleIntervalMs: number;
+  private readonly retentionSeconds: number;
+  
   constructor(
-    private smartProxy: SmartProxy
+    private smartProxy: SmartProxy,
+    config?: {
+      sampleIntervalMs?: number;
+      retentionSeconds?: number;
+    }
   ) {
-    // Subscription will be set up in start() method
+    this.sampleIntervalMs = config?.sampleIntervalMs || 1000;
+    this.retentionSeconds = config?.retentionSeconds || 3600;
+    this.throughputTracker = new ThroughputTracker(this.retentionSeconds);
   }
   
-  /**
-   * Get the current number of active connections
-   */
-  public getActiveConnections(): number {
-    return this.smartProxy.connectionManager.getConnectionCount();
-  }
-  
-  /**
-   * Get connection counts grouped by route name
-   */
-  public getConnectionsByRoute(): Map<string, number> {
-    const now = Date.now();
-    
-    // Return cached value if fresh
-    if (this.cachedMetrics.connectionsByRoute && 
-        now - this.cachedMetrics.timestamp < this.CACHE_TTL) {
-      return new Map(this.cachedMetrics.connectionsByRoute);
-    }
+  // Connection metrics implementation
+  public connections = {
+    active: (): number => {
+      return this.smartProxy.connectionManager.getConnectionCount();
+    },
     
-    // Compute fresh value
-    const routeCounts = new Map<string, number>();
-    const connections = this.smartProxy.connectionManager.getConnections();
+    total: (): number => {
+      const stats = this.smartProxy.connectionManager.getTerminationStats();
+      let total = this.smartProxy.connectionManager.getConnectionCount();
+      
+      for (const reason in stats.incoming) {
+        total += stats.incoming[reason];
+      }
+      
+      return total;
+    },
     
-    if (this.smartProxy.settings?.enableDetailedLogging) {
-      logger.log('debug', `MetricsCollector: Computing route connections`, {
-        totalConnections: connections.size,
-        component: 'metrics'
-      });
-    }
+    byRoute: (): Map<string, number> => {
+      const routeCounts = new Map<string, number>();
+      const connections = this.smartProxy.connectionManager.getConnections();
+      
+      for (const [_, record] of connections) {
+        const routeName = (record as any).routeName || 
+                         record.routeConfig?.name || 
+                         'unknown';
+        
+        const current = routeCounts.get(routeName) || 0;
+        routeCounts.set(routeName, current + 1);
+      }
+      
+      return routeCounts;
+    },
     
-    for (const [_, record] of connections) {
-      // Try different ways to get the route name
-      const routeName = (record as any).routeName || 
-                       record.routeConfig?.name || 
-                       (record.routeConfig as any)?.routeName ||
-                       'unknown';
+    byIP: (): Map<string, number> => {
+      const ipCounts = new Map<string, number>();
       
-      if (this.smartProxy.settings?.enableDetailedLogging) {
-        logger.log('debug', `MetricsCollector: Connection route info`, {
-          connectionId: record.id,
-          routeName,
-          hasRouteConfig: !!record.routeConfig,
-          routeConfigName: record.routeConfig?.name,
-          routeConfigKeys: record.routeConfig ? Object.keys(record.routeConfig) : [],
-          component: 'metrics'
-        });
+      for (const [_, record] of this.smartProxy.connectionManager.getConnections()) {
+        const ip = record.remoteIP;
+        const current = ipCounts.get(ip) || 0;
+        ipCounts.set(ip, current + 1);
       }
       
-      const current = routeCounts.get(routeName) || 0;
-      routeCounts.set(routeName, current + 1);
-    }
+      return ipCounts;
+    },
     
-    // Cache and return
-    this.cachedMetrics.connectionsByRoute = routeCounts;
-    this.cachedMetrics.timestamp = now;
-    return new Map(routeCounts);
-  }
+    topIPs: (limit: number = 10): Array<{ ip: string; count: number }> => {
+      const ipCounts = this.connections.byIP();
+      return Array.from(ipCounts.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([ip, count]) => ({ ip, count }));
+    }
+  };
   
-  /**
-   * Get connection counts grouped by IP address
-   */
-  public getConnectionsByIP(): Map<string, number> {
-    const now = Date.now();
+  // Throughput metrics implementation
+  public throughput = {
+    instant: (): IThroughputData => {
+      return this.throughputTracker.getRate(1);
+    },
     
-    // Return cached value if fresh
-    if (this.cachedMetrics.connectionsByIP && 
-        now - this.cachedMetrics.timestamp < this.CACHE_TTL) {
-      return new Map(this.cachedMetrics.connectionsByIP);
-    }
+    recent: (): IThroughputData => {
+      return this.throughputTracker.getRate(10);
+    },
     
-    // Compute fresh value
-    const ipCounts = new Map<string, number>();
-    for (const [_, record] of this.smartProxy.connectionManager.getConnections()) {
-      const ip = record.remoteIP;
-      const current = ipCounts.get(ip) || 0;
-      ipCounts.set(ip, current + 1);
-    }
+    average: (): IThroughputData => {
+      return this.throughputTracker.getRate(60);
+    },
     
-    // Cache and return
-    this.cachedMetrics.connectionsByIP = ipCounts;
-    this.cachedMetrics.timestamp = now;
-    return new Map(ipCounts);
-  }
-  
-  /**
-   * Get the total number of connections since proxy start
-   */
-  public getTotalConnections(): number {
-    // Get from termination stats
-    const stats = this.smartProxy.connectionManager.getTerminationStats();
-    let total = this.smartProxy.connectionManager.getConnectionCount(); // Add active connections
+    custom: (seconds: number): IThroughputData => {
+      return this.throughputTracker.getRate(seconds);
+    },
     
-    // Add all terminated connections
-    for (const reason in stats.incoming) {
-      total += stats.incoming[reason];
-    }
+    history: (seconds: number): Array<IThroughputHistoryPoint> => {
+      return this.throughputTracker.getHistory(seconds);
+    },
     
-    return total;
-  }
+    byRoute: (windowSeconds: number = 60): Map<string, IThroughputData> => {
+      const routeThroughput = new Map<string, IThroughputData>();
+      const now = Date.now();
+      const windowStart = now - (windowSeconds * 1000);
+      
+      // Aggregate bytes by route with proper time calculation
+      const routeData = new Map<string, { bytesIn: number; bytesOut: number; totalDuration: number }>();
+      
+      for (const [_, tracker] of this.connectionByteTrackers) {
+        // Only include connections that were active within the window
+        if (tracker.lastUpdate > windowStart || tracker.startTime > windowStart) {
+          // Calculate the actual duration this connection was active within the window
+          const connectionStart = Math.max(tracker.startTime, windowStart);
+          const connectionEnd = tracker.lastUpdate;
+          const durationInWindow = (connectionEnd - connectionStart) / 1000; // Convert to seconds
+          
+          if (durationInWindow > 0) {
+            const current = routeData.get(tracker.routeName) || { bytesIn: 0, bytesOut: 0, totalDuration: 0 };
+            current.bytesIn += tracker.bytesIn;
+            current.bytesOut += tracker.bytesOut;
+            current.totalDuration += durationInWindow;
+            routeData.set(tracker.routeName, current);
+          }
+        }
+      }
+      
+      // Convert to rates (bytes per second)
+      for (const [route, data] of routeData) {
+        if (data.totalDuration > 0) {
+          routeThroughput.set(route, {
+            in: Math.round(data.bytesIn / data.totalDuration),
+            out: Math.round(data.bytesOut / data.totalDuration)
+          });
+        }
+      }
+      
+      return routeThroughput;
+    },
+    
+    byIP: (windowSeconds: number = 60): Map<string, IThroughputData> => {
+      const ipThroughput = new Map<string, IThroughputData>();
+      const now = Date.now();
+      const windowStart = now - (windowSeconds * 1000);
+      
+      // Aggregate bytes by IP with proper time calculation
+      const ipData = new Map<string, { bytesIn: number; bytesOut: number; totalDuration: number }>();
+      
+      for (const [_, tracker] of this.connectionByteTrackers) {
+        // Only include connections that were active within the window
+        if (tracker.lastUpdate > windowStart || tracker.startTime > windowStart) {
+          // Calculate the actual duration this connection was active within the window
+          const connectionStart = Math.max(tracker.startTime, windowStart);
+          const connectionEnd = tracker.lastUpdate;
+          const durationInWindow = (connectionEnd - connectionStart) / 1000; // Convert to seconds
+          
+          if (durationInWindow > 0) {
+            const current = ipData.get(tracker.remoteIP) || { bytesIn: 0, bytesOut: 0, totalDuration: 0 };
+            current.bytesIn += tracker.bytesIn;
+            current.bytesOut += tracker.bytesOut;
+            current.totalDuration += durationInWindow;
+            ipData.set(tracker.remoteIP, current);
+          }
+        }
+      }
+      
+      // Convert to rates (bytes per second)
+      for (const [ip, data] of ipData) {
+        if (data.totalDuration > 0) {
+          ipThroughput.set(ip, {
+            in: Math.round(data.bytesIn / data.totalDuration),
+            out: Math.round(data.bytesOut / data.totalDuration)
+          });
+        }
+      }
+      
+      return ipThroughput;
+    }
+  };
   
-  /**
-   * Get the current requests per second rate
-   */
-  public getRequestsPerSecond(): number {
-    const now = Date.now();
-    const windowStart = now - this.RPS_WINDOW_SIZE;
+  // Request metrics implementation
+  public requests = {
+    perSecond: (): number => {
+      const now = Date.now();
+      const oneSecondAgo = now - 1000;
+      
+      // Clean old timestamps
+      this.requestTimestamps = this.requestTimestamps.filter(ts => ts > now - 60000);
+      
+      // Count requests in last second
+      const recentRequests = this.requestTimestamps.filter(ts => ts > oneSecondAgo);
+      return recentRequests.length;
+    },
     
-    // Clean old timestamps
-    this.requestTimestamps = this.requestTimestamps.filter(ts => ts > windowStart);
+    perMinute: (): number => {
+      const now = Date.now();
+      const oneMinuteAgo = now - 60000;
+      
+      // Count requests in last minute
+      const recentRequests = this.requestTimestamps.filter(ts => ts > oneMinuteAgo);
+      return recentRequests.length;
+    },
     
-    // Calculate RPS based on window
-    const requestsInWindow = this.requestTimestamps.length;
-    return requestsInWindow / (this.RPS_WINDOW_SIZE / 1000);
-  }
+    total: (): number => {
+      return this.totalRequests;
+    }
+  };
   
-  /**
-   * Record a new request for RPS tracking
-   */
-  public recordRequest(): void {
-    const now = Date.now();
-    this.requestTimestamps.push(now);
+  // Totals implementation
+  public totals = {
+    bytesIn: (): number => {
+      let total = 0;
+      
+      // Sum from all active connections
+      for (const [_, record] of this.smartProxy.connectionManager.getConnections()) {
+        total += record.bytesReceived;
+      }
+      
+      // TODO: Add historical data from terminated connections
+      
+      return total;
+    },
     
-    // Prevent unbounded growth - clean up more aggressively
-    if (this.requestTimestamps.length > this.MAX_TIMESTAMPS) {
-      // Keep only timestamps within the window
-      const cutoff = now - this.RPS_WINDOW_SIZE;
-      this.requestTimestamps = this.requestTimestamps.filter(ts => ts > cutoff);
+    bytesOut: (): number => {
+      let total = 0;
+      
+      // Sum from all active connections
+      for (const [_, record] of this.smartProxy.connectionManager.getConnections()) {
+        total += record.bytesSent;
+      }
+      
+      // TODO: Add historical data from terminated connections
+      
+      return total;
+    },
+    
+    connections: (): number => {
+      return this.connections.total();
     }
-  }
+  };
   
-  /**
-   * Get total throughput (bytes transferred)
-   */
-  public getThroughput(): { bytesIn: number; bytesOut: number } {
-    let bytesIn = 0;
-    let bytesOut = 0;
+  // Percentiles implementation (placeholder for now)
+  public percentiles = {
+    connectionDuration: (): { p50: number; p95: number; p99: number } => {
+      // TODO: Implement percentile calculations
+      return { p50: 0, p95: 0, p99: 0 };
+    },
     
-    // Sum bytes from all active connections
-    for (const [_, record] of this.smartProxy.connectionManager.getConnections()) {
-      bytesIn += record.bytesReceived;
-      bytesOut += record.bytesSent;
+    bytesTransferred: (): { 
+      in: { p50: number; p95: number; p99: number };
+      out: { p50: number; p95: number; p99: number };
+    } => {
+      // TODO: Implement percentile calculations
+      return {
+        in: { p50: 0, p95: 0, p99: 0 },
+        out: { p50: 0, p95: 0, p99: 0 }
+      };
     }
-    
-    return { bytesIn, bytesOut };
-  }
+  };
   
   /**
-   * Get throughput rate (bytes per second) for last minute
+   * Record a new request
    */
-  public getThroughputRate(): { bytesInPerSec: number; bytesOutPerSec: number } {
+  public recordRequest(connectionId: string, routeName: string, remoteIP: string): void {
     const now = Date.now();
-    let recentBytesIn = 0;
-    let recentBytesOut = 0;
+    this.requestTimestamps.push(now);
+    this.totalRequests++;
+    
+    // Initialize byte tracker for this connection
+    this.connectionByteTrackers.set(connectionId, {
+      connectionId,
+      routeName,
+      remoteIP,
+      bytesIn: 0,
+      bytesOut: 0,
+      startTime: now,
+      lastUpdate: now
+    });
     
-    // Calculate bytes transferred in last minute from active connections
-    for (const [_, record] of this.smartProxy.connectionManager.getConnections()) {
-      const connectionAge = now - record.incomingStartTime;
-      if (connectionAge < 60000) { // Connection started within last minute
-        recentBytesIn += record.bytesReceived;
-        recentBytesOut += record.bytesSent;
-      } else {
-        // For older connections, estimate rate based on average
-        const rate = connectionAge / 60000;
-        recentBytesIn += record.bytesReceived / rate;
-        recentBytesOut += record.bytesSent / rate;
+    // Cleanup old request timestamps
+    if (this.requestTimestamps.length > 5000) {
+      // First try to clean up old timestamps (older than 1 minute)
+      const cutoff = now - 60000;
+      this.requestTimestamps = this.requestTimestamps.filter(ts => ts > cutoff);
+      
+      // If still too many, enforce hard cap of 5000 most recent
+      if (this.requestTimestamps.length > 5000) {
+        this.requestTimestamps = this.requestTimestamps.slice(-5000);
       }
     }
-    
-    return {
-      bytesInPerSec: Math.round(recentBytesIn / 60),
-      bytesOutPerSec: Math.round(recentBytesOut / 60)
-    };
   }
   
   /**
-   * Get top IPs by connection count
+   * Record bytes transferred for a connection
    */
-  public getTopIPs(limit: number = 10): Array<{ ip: string; connections: number }> {
-    const ipCounts = this.getConnectionsByIP();
-    const sorted = Array.from(ipCounts.entries())
-      .sort((a, b) => b[1] - a[1])
-      .slice(0, limit)
-      .map(([ip, connections]) => ({ ip, connections }));
+  public recordBytes(connectionId: string, bytesIn: number, bytesOut: number): void {
+    // Update global throughput tracker
+    this.throughputTracker.recordBytes(bytesIn, bytesOut);
     
-    return sorted;
-  }
-  
-  /**
-   * Check if an IP has reached the connection limit
-   */
-  public isIPBlocked(ip: string, maxConnectionsPerIP: number): boolean {
-    const ipCounts = this.getConnectionsByIP();
-    const currentConnections = ipCounts.get(ip) || 0;
-    return currentConnections >= maxConnectionsPerIP;
+    // Update connection-specific tracker
+    const tracker = this.connectionByteTrackers.get(connectionId);
+    if (tracker) {
+      tracker.bytesIn += bytesIn;
+      tracker.bytesOut += bytesOut;
+      tracker.lastUpdate = Date.now();
+    }
   }
   
   /**
-   * Clean up old request timestamps
+   * Clean up tracking for a closed connection
    */
-  private cleanupOldRequests(): void {
-    const cutoff = Date.now() - this.RPS_WINDOW_SIZE;
-    this.requestTimestamps = this.requestTimestamps.filter(ts => ts > cutoff);
+  public removeConnection(connectionId: string): void {
+    this.connectionByteTrackers.delete(connectionId);
   }
   
   /**
-   * Start the metrics collector and set up subscriptions
+   * Start the metrics collector
    */
   public start(): void {
     if (!this.smartProxy.routeConnectionHandler) {
       throw new Error('MetricsCollector: RouteConnectionHandler not available');
     }
     
-    // Subscribe to the newConnectionSubject from RouteConnectionHandler
+    // Start periodic sampling
+    this.samplingInterval = setInterval(() => {
+      this.throughputTracker.takeSample();
+      
+      // Clean up old connection trackers (connections closed more than 5 minutes ago)
+      const cutoff = Date.now() - 300000;
+      for (const [id, tracker] of this.connectionByteTrackers) {
+        if (tracker.lastUpdate < cutoff) {
+          this.connectionByteTrackers.delete(id);
+        }
+      }
+    }, this.sampleIntervalMs);
+    
+    // Subscribe to new connections
     this.connectionSubscription = this.smartProxy.routeConnectionHandler.newConnectionSubject.subscribe({
       next: (record) => {
-        this.recordRequest();
+        const routeName = record.routeConfig?.name || 'unknown';
+        this.recordRequest(record.id, routeName, record.remoteIP);
         
-        // Optional: Log connection details
         if (this.smartProxy.settings?.enableDetailedLogging) {
           logger.log('debug', `MetricsCollector: New connection recorded`, {
             connectionId: record.id,
             remoteIP: record.remoteIP,
-            routeName: record.routeConfig?.name || 'unknown',
+            routeName,
             component: 'metrics'
           });
         }
@@ -269,9 +381,14 @@ export class MetricsCollector implements IProxyStatsExtended {
   }
   
   /**
-   * Stop the metrics collector and clean up resources
+   * Stop the metrics collector
    */
   public stop(): void {
+    if (this.samplingInterval) {
+      clearInterval(this.samplingInterval);
+      this.samplingInterval = undefined;
+    }
+    
     if (this.connectionSubscription) {
       this.connectionSubscription.unsubscribe();
       this.connectionSubscription = undefined;
@@ -281,7 +398,7 @@ export class MetricsCollector implements IProxyStatsExtended {
   }
   
   /**
-   * Alias for stop() for backward compatibility
+   * Alias for stop() for compatibility
    */
   public destroy(): void {
     this.stop();

package/ts/proxies/smart-proxy/models/interfaces.ts

@@ -105,6 +105,13 @@ export interface ISmartProxyOptions {
   useHttpProxy?: number[]; // Array of ports to forward to HttpProxy
   httpProxyPort?: number; // Port where HttpProxy is listening (default: 8443)
 
+  // Metrics configuration
+  metrics?: {
+    enabled?: boolean;
+    sampleIntervalMs?: number;
+    retentionSeconds?: number;
+  };
+
   /**
    * Global ACME configuration options for SmartProxy
    * 
@@ -142,7 +149,7 @@ export interface IConnectionRecord {
   outgoingClosedTime?: number;
   lockedDomain?: string; // Used to lock this connection to the initial SNI
   connectionClosed: boolean; // Flag to prevent multiple cleanup attempts
-  cleanupTimer?: NodeJS.Timeout; // Timer for max lifetime/inactivity
+  cleanupTimer?: NodeJS.Timeout | null; // Timer for max lifetime/inactivity
   alertFallbackTimeout?: NodeJS.Timeout; // Timer for fallback after alert
   lastActivity: number; // Last activity timestamp for inactivity detection
   pendingData: Buffer[]; // Buffer to hold data during connection setup