@push.rocks/smartproxy 19.5.3 → 19.5.5

package/ts/core/utils/binary-heap.ts

@@ -0,0 +1,225 @@
+/**
+ * A binary heap implementation for efficient priority queue operations
+ * Supports O(log n) insert and extract operations
+ */
+export class BinaryHeap<T> {
+  private heap: T[] = [];
+  private keyMap?: Map<string, number>; // For efficient key-based lookups
+
+  constructor(
+    private compareFn: (a: T, b: T) => number,
+    private extractKey?: (item: T) => string
+  ) {
+    if (extractKey) {
+      this.keyMap = new Map();
+    }
+  }
+
+  /**
+   * Get the current size of the heap
+   */
+  public get size(): number {
+    return this.heap.length;
+  }
+
+  /**
+   * Check if the heap is empty
+   */
+  public isEmpty(): boolean {
+    return this.heap.length === 0;
+  }
+
+  /**
+   * Peek at the top element without removing it
+   */
+  public peek(): T | undefined {
+    return this.heap[0];
+  }
+
+  /**
+   * Insert a new item into the heap
+   * O(log n) time complexity
+   */
+  public insert(item: T): void {
+    const index = this.heap.length;
+    this.heap.push(item);
+    
+    if (this.keyMap && this.extractKey) {
+      const key = this.extractKey(item);
+      this.keyMap.set(key, index);
+    }
+    
+    this.bubbleUp(index);
+  }
+
+  /**
+   * Extract the top element from the heap
+   * O(log n) time complexity
+   */
+  public extract(): T | undefined {
+    if (this.heap.length === 0) return undefined;
+    if (this.heap.length === 1) {
+      const item = this.heap.pop()!;
+      if (this.keyMap && this.extractKey) {
+        this.keyMap.delete(this.extractKey(item));
+      }
+      return item;
+    }
+    
+    const result = this.heap[0];
+    const lastItem = this.heap.pop()!;
+    this.heap[0] = lastItem;
+    
+    if (this.keyMap && this.extractKey) {
+      this.keyMap.delete(this.extractKey(result));
+      this.keyMap.set(this.extractKey(lastItem), 0);
+    }
+    
+    this.bubbleDown(0);
+    return result;
+  }
+
+  /**
+   * Extract an element that matches the predicate
+   * O(n) time complexity for search, O(log n) for extraction
+   */
+  public extractIf(predicate: (item: T) => boolean): T | undefined {
+    const index = this.heap.findIndex(predicate);
+    if (index === -1) return undefined;
+    
+    return this.extractAt(index);
+  }
+
+  /**
+   * Extract an element by its key (if extractKey was provided)
+   * O(log n) time complexity
+   */
+  public extractByKey(key: string): T | undefined {
+    if (!this.keyMap || !this.extractKey) {
+      throw new Error('extractKey function must be provided to use key-based extraction');
+    }
+    
+    const index = this.keyMap.get(key);
+    if (index === undefined) return undefined;
+    
+    return this.extractAt(index);
+  }
+
+  /**
+   * Check if a key exists in the heap
+   * O(1) time complexity
+   */
+  public hasKey(key: string): boolean {
+    if (!this.keyMap) return false;
+    return this.keyMap.has(key);
+  }
+
+  /**
+   * Get all elements as an array (does not modify heap)
+   * O(n) time complexity
+   */
+  public toArray(): T[] {
+    return [...this.heap];
+  }
+
+  /**
+   * Clear the heap
+   */
+  public clear(): void {
+    this.heap = [];
+    if (this.keyMap) {
+      this.keyMap.clear();
+    }
+  }
+
+  /**
+   * Extract element at specific index
+   */
+  private extractAt(index: number): T {
+    const item = this.heap[index];
+    
+    if (this.keyMap && this.extractKey) {
+      this.keyMap.delete(this.extractKey(item));
+    }
+    
+    if (index === this.heap.length - 1) {
+      this.heap.pop();
+      return item;
+    }
+    
+    const lastItem = this.heap.pop()!;
+    this.heap[index] = lastItem;
+    
+    if (this.keyMap && this.extractKey) {
+      this.keyMap.set(this.extractKey(lastItem), index);
+    }
+    
+    // Try bubbling up first
+    const parentIndex = Math.floor((index - 1) / 2);
+    if (parentIndex >= 0 && this.compareFn(this.heap[index], this.heap[parentIndex]) < 0) {
+      this.bubbleUp(index);
+    } else {
+      this.bubbleDown(index);
+    }
+    
+    return item;
+  }
+
+  /**
+   * Bubble up element at given index to maintain heap property
+   */
+  private bubbleUp(index: number): void {
+    while (index > 0) {
+      const parentIndex = Math.floor((index - 1) / 2);
+      
+      if (this.compareFn(this.heap[index], this.heap[parentIndex]) >= 0) {
+        break;
+      }
+      
+      this.swap(index, parentIndex);
+      index = parentIndex;
+    }
+  }
+
+  /**
+   * Bubble down element at given index to maintain heap property
+   */
+  private bubbleDown(index: number): void {
+    const length = this.heap.length;
+    
+    while (true) {
+      const leftChild = 2 * index + 1;
+      const rightChild = 2 * index + 2;
+      let smallest = index;
+      
+      if (leftChild < length && 
+          this.compareFn(this.heap[leftChild], this.heap[smallest]) < 0) {
+        smallest = leftChild;
+      }
+      
+      if (rightChild < length && 
+          this.compareFn(this.heap[rightChild], this.heap[smallest]) < 0) {
+        smallest = rightChild;
+      }
+      
+      if (smallest === index) break;
+      
+      this.swap(index, smallest);
+      index = smallest;
+    }
+  }
+
+  /**
+   * Swap two elements in the heap
+   */
+  private swap(i: number, j: number): void {
+    const temp = this.heap[i];
+    this.heap[i] = this.heap[j];
+    this.heap[j] = temp;
+    
+    if (this.keyMap && this.extractKey) {
+      this.keyMap.set(this.extractKey(this.heap[i]), i);
+      this.keyMap.set(this.extractKey(this.heap[j]), j);
+    }
+  }
+}

package/ts/core/utils/enhanced-connection-pool.ts

@@ -0,0 +1,420 @@
+import { LifecycleComponent } from './lifecycle-component.js';
+import { BinaryHeap } from './binary-heap.js';
+import { AsyncMutex } from './async-utils.js';
+import { EventEmitter } from 'events';
+
+/**
+ * Interface for pooled connection
+ */
+export interface IPooledConnection<T> {
+  id: string;
+  connection: T;
+  createdAt: number;
+  lastUsedAt: number;
+  useCount: number;
+  inUse: boolean;
+  metadata?: any;
+}
+
+/**
+ * Configuration options for the connection pool
+ */
+export interface IConnectionPoolOptions<T> {
+  minSize?: number;
+  maxSize?: number;
+  acquireTimeout?: number;
+  idleTimeout?: number;
+  maxUseCount?: number;
+  validateOnAcquire?: boolean;
+  validateOnReturn?: boolean;
+  queueTimeout?: number;
+  connectionFactory: () => Promise<T>;
+  connectionValidator?: (connection: T) => Promise<boolean>;
+  connectionDestroyer?: (connection: T) => Promise<void>;
+  onConnectionError?: (error: Error, connection?: T) => void;
+}
+
+/**
+ * Interface for queued acquire request
+ */
+interface IAcquireRequest<T> {
+  id: string;
+  priority: number;
+  timestamp: number;
+  resolve: (connection: IPooledConnection<T>) => void;
+  reject: (error: Error) => void;
+  timeoutHandle?: NodeJS.Timeout;
+}
+
+/**
+ * Enhanced connection pool with priority queue, backpressure, and lifecycle management
+ */
+export class EnhancedConnectionPool<T> extends LifecycleComponent {
+  private readonly options: Required<Omit<IConnectionPoolOptions<T>, 'connectionValidator' | 'connectionDestroyer' | 'onConnectionError'>> & Pick<IConnectionPoolOptions<T>, 'connectionValidator' | 'connectionDestroyer' | 'onConnectionError'>;
+  private readonly availableConnections: IPooledConnection<T>[] = [];
+  private readonly activeConnections: Map<string, IPooledConnection<T>> = new Map();
+  private readonly waitQueue: BinaryHeap<IAcquireRequest<T>>;
+  private readonly mutex = new AsyncMutex();
+  private readonly eventEmitter = new EventEmitter();
+  
+  private connectionIdCounter = 0;
+  private requestIdCounter = 0;
+  private isClosing = false;
+  
+  // Metrics
+  private metrics = {
+    connectionsCreated: 0,
+    connectionsDestroyed: 0,
+    connectionsAcquired: 0,
+    connectionsReleased: 0,
+    acquireTimeouts: 0,
+    validationFailures: 0,
+    queueHighWaterMark: 0,
+  };
+
+  constructor(options: IConnectionPoolOptions<T>) {
+    super();
+    
+    this.options = {
+      minSize: 0,
+      maxSize: 10,
+      acquireTimeout: 30000,
+      idleTimeout: 300000, // 5 minutes
+      maxUseCount: Infinity,
+      validateOnAcquire: true,
+      validateOnReturn: false,
+      queueTimeout: 60000,
+      ...options,
+    };
+    
+    // Initialize priority queue (higher priority = extracted first)
+    this.waitQueue = new BinaryHeap<IAcquireRequest<T>>(
+      (a, b) => b.priority - a.priority || a.timestamp - b.timestamp,
+      (item) => item.id
+    );
+    
+    // Start maintenance cycle
+    this.startMaintenance();
+    
+    // Initialize minimum connections
+    this.initializeMinConnections();
+  }
+
+  /**
+   * Initialize minimum number of connections
+   */
+  private async initializeMinConnections(): Promise<void> {
+    const promises: Promise<void>[] = [];
+    
+    for (let i = 0; i < this.options.minSize; i++) {
+      promises.push(
+        this.createConnection()
+          .then(conn => {
+            this.availableConnections.push(conn);
+          })
+          .catch(err => {
+            if (this.options.onConnectionError) {
+              this.options.onConnectionError(err);
+            }
+          })
+      );
+    }
+    
+    await Promise.all(promises);
+  }
+
+  /**
+   * Start maintenance timer for idle connection cleanup
+   */
+  private startMaintenance(): void {
+    this.setInterval(() => {
+      this.performMaintenance();
+    }, 30000); // Every 30 seconds
+  }
+
+  /**
+   * Perform maintenance tasks
+   */
+  private async performMaintenance(): Promise<void> {
+    await this.mutex.runExclusive(async () => {
+      const now = Date.now();
+      const toRemove: IPooledConnection<T>[] = [];
+      
+      // Check for idle connections beyond minimum size
+      for (let i = this.availableConnections.length - 1; i >= 0; i--) {
+        const conn = this.availableConnections[i];
+        
+        // Keep minimum connections
+        if (this.availableConnections.length <= this.options.minSize) {
+          break;
+        }
+        
+        // Remove idle connections
+        if (now - conn.lastUsedAt > this.options.idleTimeout) {
+          toRemove.push(conn);
+          this.availableConnections.splice(i, 1);
+        }
+      }
+      
+      // Destroy idle connections
+      for (const conn of toRemove) {
+        await this.destroyConnection(conn);
+      }
+    });
+  }
+
+  /**
+   * Acquire a connection from the pool
+   */
+  public async acquire(priority: number = 0, timeout?: number): Promise<IPooledConnection<T>> {
+    if (this.isClosing) {
+      throw new Error('Connection pool is closing');
+    }
+    
+    return this.mutex.runExclusive(async () => {
+      // Try to get an available connection
+      const connection = await this.tryAcquireConnection();
+      if (connection) {
+        return connection;
+      }
+      
+      // Check if we can create a new connection
+      const totalConnections = this.availableConnections.length + this.activeConnections.size;
+      if (totalConnections < this.options.maxSize) {
+        try {
+          const newConnection = await this.createConnection();
+          return this.checkoutConnection(newConnection);
+        } catch (err) {
+          // Fall through to queue if creation fails
+        }
+      }
+      
+      // Add to wait queue
+      return this.queueAcquireRequest(priority, timeout);
+    });
+  }
+
+  /**
+   * Try to acquire an available connection
+   */
+  private async tryAcquireConnection(): Promise<IPooledConnection<T> | null> {
+    while (this.availableConnections.length > 0) {
+      const connection = this.availableConnections.shift()!;
+      
+      // Check if connection exceeded max use count
+      if (connection.useCount >= this.options.maxUseCount) {
+        await this.destroyConnection(connection);
+        continue;
+      }
+      
+      // Validate connection if required
+      if (this.options.validateOnAcquire && this.options.connectionValidator) {
+        try {
+          const isValid = await this.options.connectionValidator(connection.connection);
+          if (!isValid) {
+            this.metrics.validationFailures++;
+            await this.destroyConnection(connection);
+            continue;
+          }
+        } catch (err) {
+          this.metrics.validationFailures++;
+          await this.destroyConnection(connection);
+          continue;
+        }
+      }
+      
+      return this.checkoutConnection(connection);
+    }
+    
+    return null;
+  }
+
+  /**
+   * Checkout a connection for use
+   */
+  private checkoutConnection(connection: IPooledConnection<T>): IPooledConnection<T> {
+    connection.inUse = true;
+    connection.lastUsedAt = Date.now();
+    connection.useCount++;
+    
+    this.activeConnections.set(connection.id, connection);
+    this.metrics.connectionsAcquired++;
+    
+    this.eventEmitter.emit('acquire', connection);
+    return connection;
+  }
+
+  /**
+   * Queue an acquire request
+   */
+  private queueAcquireRequest(priority: number, timeout?: number): Promise<IPooledConnection<T>> {
+    return new Promise<IPooledConnection<T>>((resolve, reject) => {
+      const request: IAcquireRequest<T> = {
+        id: `req-${this.requestIdCounter++}`,
+        priority,
+        timestamp: Date.now(),
+        resolve,
+        reject,
+      };
+      
+      // Set timeout
+      const timeoutMs = timeout || this.options.queueTimeout;
+      request.timeoutHandle = this.setTimeout(() => {
+        if (this.waitQueue.extractByKey(request.id)) {
+          this.metrics.acquireTimeouts++;
+          reject(new Error(`Connection acquire timeout after ${timeoutMs}ms`));
+        }
+      }, timeoutMs);
+      
+      this.waitQueue.insert(request);
+      this.metrics.queueHighWaterMark = Math.max(
+        this.metrics.queueHighWaterMark, 
+        this.waitQueue.size
+      );
+      
+      this.eventEmitter.emit('enqueue', { queueSize: this.waitQueue.size });
+    });
+  }
+
+  /**
+   * Release a connection back to the pool
+   */
+  public async release(connection: IPooledConnection<T>): Promise<void> {
+    return this.mutex.runExclusive(async () => {
+      if (!connection.inUse || !this.activeConnections.has(connection.id)) {
+        throw new Error('Connection is not active');
+      }
+      
+      this.activeConnections.delete(connection.id);
+      connection.inUse = false;
+      connection.lastUsedAt = Date.now();
+      this.metrics.connectionsReleased++;
+      
+      // Check if connection should be destroyed
+      if (connection.useCount >= this.options.maxUseCount) {
+        await this.destroyConnection(connection);
+        return;
+      }
+      
+      // Validate on return if required
+      if (this.options.validateOnReturn && this.options.connectionValidator) {
+        try {
+          const isValid = await this.options.connectionValidator(connection.connection);
+          if (!isValid) {
+            await this.destroyConnection(connection);
+            return;
+          }
+        } catch (err) {
+          await this.destroyConnection(connection);
+          return;
+        }
+      }
+      
+      // Check if there are waiting requests
+      const request = this.waitQueue.extract();
+      if (request) {
+        this.clearTimeout(request.timeoutHandle!);
+        request.resolve(this.checkoutConnection(connection));
+        this.eventEmitter.emit('dequeue', { queueSize: this.waitQueue.size });
+      } else {
+        // Return to available pool
+        this.availableConnections.push(connection);
+        this.eventEmitter.emit('release', connection);
+      }
+    });
+  }
+
+  /**
+   * Create a new connection
+   */
+  private async createConnection(): Promise<IPooledConnection<T>> {
+    const rawConnection = await this.options.connectionFactory();
+    
+    const connection: IPooledConnection<T> = {
+      id: `conn-${this.connectionIdCounter++}`,
+      connection: rawConnection,
+      createdAt: Date.now(),
+      lastUsedAt: Date.now(),
+      useCount: 0,
+      inUse: false,
+    };
+    
+    this.metrics.connectionsCreated++;
+    this.eventEmitter.emit('create', connection);
+    
+    return connection;
+  }
+
+  /**
+   * Destroy a connection
+   */
+  private async destroyConnection(connection: IPooledConnection<T>): Promise<void> {
+    try {
+      if (this.options.connectionDestroyer) {
+        await this.options.connectionDestroyer(connection.connection);
+      }
+      
+      this.metrics.connectionsDestroyed++;
+      this.eventEmitter.emit('destroy', connection);
+    } catch (err) {
+      if (this.options.onConnectionError) {
+        this.options.onConnectionError(err as Error, connection.connection);
+      }
+    }
+  }
+
+  /**
+   * Get current pool statistics
+   */
+  public getStats() {
+    return {
+      available: this.availableConnections.length,
+      active: this.activeConnections.size,
+      waiting: this.waitQueue.size,
+      total: this.availableConnections.length + this.activeConnections.size,
+      ...this.metrics,
+    };
+  }
+
+  /**
+   * Subscribe to pool events
+   */
+  public on(event: string, listener: Function): void {
+    this.addEventListener(this.eventEmitter, event, listener);
+  }
+
+  /**
+   * Close the pool and cleanup resources
+   */
+  protected async onCleanup(): Promise<void> {
+    this.isClosing = true;
+    
+    // Clear the wait queue
+    while (!this.waitQueue.isEmpty()) {
+      const request = this.waitQueue.extract();
+      if (request) {
+        this.clearTimeout(request.timeoutHandle!);
+        request.reject(new Error('Connection pool is closing'));
+      }
+    }
+    
+    // Wait for active connections to be released (with timeout)
+    const timeout = 30000;
+    const startTime = Date.now();
+    
+    while (this.activeConnections.size > 0 && Date.now() - startTime < timeout) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    
+    // Destroy all connections
+    const allConnections = [
+      ...this.availableConnections,
+      ...this.activeConnections.values(),
+    ];
+    
+    await Promise.all(allConnections.map(conn => this.destroyConnection(conn)));
+    
+    this.availableConnections.length = 0;
+    this.activeConnections.clear();
+  }
+}