@dotdo/postgres 0.1.0

package/src/routing/do-connection-pool.ts

@@ -0,0 +1,828 @@
+/**
+ * DO Connection Pool
+ *
+ * Task: postgres-zjj
+ *
+ * Generic connection pooling layer for high-concurrency scenarios.
+ * Provides connection reuse, health checking, and fair queuing.
+ *
+ * Key features:
+ * - Maintain pool of warm DO connections
+ * - Connection reuse to reduce latency
+ * - Max pool size configuration
+ * - Idle connection timeout
+ * - Health checking of pooled connections
+ * - Fair queuing when pool exhausted
+ */
+
+/**
+ * Interface for a generic poolable connection.
+ * Implementations should handle their own transport (WebSocket, HTTP, etc.)
+ */
+export interface PoolableConnection {
+  /** Unique identifier for this connection */
+  readonly id: string
+
+  /** Check if connection is healthy and ready for use */
+  isHealthy(): boolean
+
+  /** Close the connection and release resources */
+  close(): Promise<void>
+
+  /** Get the timestamp of last activity */
+  getLastActivity(): number
+
+  /** Update the last activity timestamp */
+  touch(): void
+}
+
+/**
+ * Factory function to create new connections
+ */
+export type ConnectionFactory<T extends PoolableConnection> = () => Promise<T>
+
+/**
+ * Health check function for validating connections
+ */
+export type HealthChecker<T extends PoolableConnection> = (conn: T) => Promise<boolean>
+
+/**
+ * Configuration for the DO connection pool
+ */
+export interface DOPoolConfig<T extends PoolableConnection> {
+  /** Factory function to create new connections */
+  factory: ConnectionFactory<T>
+
+  /** Maximum number of connections in the pool */
+  maxSize?: number
+
+  /** Minimum number of warm connections to maintain */
+  minSize?: number
+
+  /** Idle timeout before closing connection (ms) */
+  idleTimeoutMs?: number
+
+  /** Timeout for acquiring a connection (ms) */
+  acquireTimeoutMs?: number
+
+  /** Enable health checking of pooled connections */
+  enableHealthCheck?: boolean
+
+  /** Interval for periodic health checks (ms) */
+  healthCheckIntervalMs?: number
+
+  /** Custom health check function */
+  healthChecker?: HealthChecker<T>
+
+  /** Enable fair queuing when pool is exhausted */
+  fairQueue?: boolean
+
+  /** Maximum number of pending acquire requests in queue */
+  maxQueueSize?: number
+
+  /** Enable automatic scaling of pool size based on demand */
+  autoScale?: boolean
+
+  /** Scale up threshold (percentage of pool in use) */
+  scaleUpThreshold?: number
+
+  /** Scale down threshold (percentage of pool in use) */
+  scaleDownThreshold?: number
+
+  /** Connection validation timeout (ms) */
+  validationTimeoutMs?: number
+
+  /** Enable connection warming on startup */
+  warmOnStart?: boolean
+
+  /** Number of connections to warm on startup */
+  warmCount?: number
+}
+
+/**
+ * Statistics for a single connection
+ */
+export interface ConnectionStats {
+  id: string
+  createdAt: number
+  lastUsedAt: number
+  useCount: number
+  errorCount: number
+  state: 'idle' | 'in_use' | 'validating' | 'closed'
+}
+
+/**
+ * Pool utilization metrics
+ */
+export interface DOPoolMetrics {
+  /** Total number of connections created */
+  totalCreated: number
+
+  /** Total number of connections closed */
+  totalClosed: number
+
+  /** Current pool size */
+  currentSize: number
+
+  /** Number of connections currently in use */
+  inUse: number
+
+  /** Number of idle connections available */
+  available: number
+
+  /** Number of pending acquire requests in queue */
+  pendingRequests: number
+
+  /** Total successful acquisitions */
+  totalAcquisitions: number
+
+  /** Total releases back to pool */
+  totalReleases: number
+
+  /** Total health checks performed */
+  totalHealthChecks: number
+
+  /** Total connections failed health check */
+  totalHealthCheckFailures: number
+
+  /** Average wait time for acquire (ms) */
+  avgWaitTimeMs: number
+
+  /** Maximum wait time observed (ms) */
+  maxWaitTimeMs: number
+
+  /** Total acquire timeouts */
+  totalTimeouts: number
+
+  /** Current queue length */
+  queueLength: number
+
+  /** Pool utilization percentage */
+  utilizationPercent: number
+
+  /** Timestamp of last activity */
+  lastActivityAt: number
+}
+
+/**
+ * Pending acquire request in the queue
+ */
+interface PendingRequest<T extends PoolableConnection> {
+  resolve: (conn: T) => void
+  reject: (error: Error) => void
+  enqueuedAt: number
+  timeoutId: ReturnType<typeof setTimeout>
+}
+
+/**
+ * Internal connection wrapper with metadata
+ */
+interface PooledConnectionWrapper<T extends PoolableConnection> {
+  connection: T
+  createdAt: number
+  lastUsedAt: number
+  useCount: number
+  errorCount: number
+  state: 'idle' | 'in_use' | 'validating' | 'closed'
+}
+
+/**
+ * Generic connection pool for Durable Object connections.
+ * Designed for high-concurrency scenarios with fair queuing.
+ */
+export class DOConnectionPool<T extends PoolableConnection> {
+  private readonly config: Required<DOPoolConfig<T>>
+  private readonly connections: Map<string, PooledConnectionWrapper<T>> = new Map()
+  private readonly idleConnections: Set<string> = new Set()
+  private readonly pendingQueue: PendingRequest<T>[] = []
+  private readonly idleTimers: Map<string, ReturnType<typeof setTimeout>> = new Map()
+
+  private healthCheckInterval: ReturnType<typeof setInterval> | null = null
+  private scaleCheckInterval: ReturnType<typeof setInterval> | null = null
+  private closed = false
+
+  // Metrics tracking
+  private totalCreated = 0
+  private totalClosed = 0
+  private totalAcquisitions = 0
+  private totalReleases = 0
+  private totalHealthChecks = 0
+  private totalHealthCheckFailures = 0
+  private totalTimeouts = 0
+  private waitTimes: number[] = []
+  private lastActivityAt = Date.now()
+
+  constructor(config: DOPoolConfig<T>) {
+    this.config = {
+      factory: config.factory,
+      maxSize: config.maxSize ?? 10,
+      minSize: config.minSize ?? 0,
+      idleTimeoutMs: config.idleTimeoutMs ?? 60000,
+      acquireTimeoutMs: config.acquireTimeoutMs ?? 30000,
+      enableHealthCheck: config.enableHealthCheck ?? true,
+      healthCheckIntervalMs: config.healthCheckIntervalMs ?? 30000,
+      healthChecker: config.healthChecker ?? (async (conn) => conn.isHealthy()),
+      fairQueue: config.fairQueue ?? true,
+      maxQueueSize: config.maxQueueSize ?? 100,
+      autoScale: config.autoScale ?? false,
+      scaleUpThreshold: config.scaleUpThreshold ?? 0.8,
+      scaleDownThreshold: config.scaleDownThreshold ?? 0.2,
+      validationTimeoutMs: config.validationTimeoutMs ?? 5000,
+      warmOnStart: config.warmOnStart ?? false,
+      warmCount: config.warmCount ?? config.minSize ?? 0,
+    }
+
+    // Start health check interval
+    if (this.config.enableHealthCheck && this.config.healthCheckIntervalMs > 0) {
+      this.healthCheckInterval = setInterval(
+        () => this.runHealthCheck(),
+        this.config.healthCheckIntervalMs
+      )
+    }
+
+    // Start auto-scaling check interval
+    if (this.config.autoScale) {
+      this.scaleCheckInterval = setInterval(
+        () => this.checkScaling(),
+        Math.min(this.config.healthCheckIntervalMs, 10000)
+      )
+    }
+  }
+
+  /**
+   * Initialize the pool with warm connections
+   */
+  async initialize(): Promise<void> {
+    if (this.config.warmOnStart && this.config.warmCount > 0) {
+      const warmPromises: Promise<void>[] = []
+      const count = Math.min(this.config.warmCount, this.config.maxSize)
+
+      for (let i = 0; i < count; i++) {
+        warmPromises.push(this.createConnection().then(wrapper => {
+          this.markIdle(wrapper.connection.id)
+        }).catch(() => {
+          // Ignore warm-up failures
+        }))
+      }
+
+      await Promise.all(warmPromises)
+    }
+  }
+
+  /**
+   * Acquire a connection from the pool.
+   * Will reuse an idle connection if available, or create a new one.
+   * If pool is exhausted, request will be queued (with fair ordering).
+   */
+  async acquire(): Promise<T> {
+    if (this.closed) {
+      throw new Error('Pool is closed')
+    }
+
+    this.lastActivityAt = Date.now()
+    const startTime = Date.now()
+
+    // Try to get an idle connection first
+    const idleConn = await this.getIdleConnection()
+    if (idleConn) {
+      this.recordWaitTime(Date.now() - startTime)
+      return idleConn
+    }
+
+    // Try to create a new connection if under max size
+    if (this.connections.size < this.config.maxSize) {
+      try {
+        const wrapper = await this.createConnection()
+        this.markInUse(wrapper.connection.id)
+        this.recordWaitTime(Date.now() - startTime)
+        return wrapper.connection
+      } catch (error) {
+        // If creation fails and queue is enabled, fall through to queue
+        if (!this.config.fairQueue) {
+          throw error
+        }
+      }
+    }
+
+    // Pool is exhausted - queue the request
+    if (!this.config.fairQueue) {
+      throw new Error('Pool exhausted: max connections reached')
+    }
+
+    if (this.pendingQueue.length >= this.config.maxQueueSize) {
+      throw new Error('Pool exhausted: max queue size reached')
+    }
+
+    return this.enqueueRequest(startTime)
+  }
+
+  /**
+   * Release a connection back to the pool.
+   * Connection will be validated and made available for reuse.
+   */
+  async release(conn: T): Promise<void> {
+    if (this.closed) {
+      await conn.close()
+      return
+    }
+
+    const wrapper = this.connections.get(conn.id)
+    if (!wrapper) {
+      // Connection not from this pool
+      await conn.close()
+      return
+    }
+
+    this.lastActivityAt = Date.now()
+    this.totalReleases++
+    wrapper.lastUsedAt = Date.now()
+    conn.touch()
+
+    // Check if there are pending requests (fair queue)
+    if (this.pendingQueue.length > 0) {
+      const pending = this.pendingQueue.shift()!
+      clearTimeout(pending.timeoutId)
+      wrapper.state = 'in_use'
+      wrapper.useCount++
+      pending.resolve(conn)
+      return
+    }
+
+    // Return to idle pool
+    this.markIdle(conn.id)
+  }
+
+  /**
+   * Destroy a connection without returning it to the pool.
+   * Use when connection is known to be bad.
+   */
+  async destroy(conn: T): Promise<void> {
+    const wrapper = this.connections.get(conn.id)
+    if (wrapper) {
+      wrapper.state = 'closed'
+      this.connections.delete(conn.id)
+      this.idleConnections.delete(conn.id)
+      this.clearIdleTimer(conn.id)
+      this.totalClosed++
+    }
+
+    try {
+      await conn.close()
+    } catch {
+      // Ignore close errors
+    }
+
+    // Ensure minimum pool size is maintained
+    await this.ensureMinSize()
+  }
+
+  /**
+   * Close the pool and all connections.
+   */
+  async close(): Promise<void> {
+    this.closed = true
+
+    // Clear intervals
+    if (this.healthCheckInterval) {
+      clearInterval(this.healthCheckInterval)
+      this.healthCheckInterval = null
+    }
+
+    if (this.scaleCheckInterval) {
+      clearInterval(this.scaleCheckInterval)
+      this.scaleCheckInterval = null
+    }
+
+    // Clear all idle timers
+    for (const timer of this.idleTimers.values()) {
+      clearTimeout(timer)
+    }
+    this.idleTimers.clear()
+
+    // Reject all pending requests
+    for (const pending of this.pendingQueue) {
+      clearTimeout(pending.timeoutId)
+      pending.reject(new Error('Pool closed'))
+    }
+    this.pendingQueue.length = 0
+
+    // Close all connections
+    const closePromises: Promise<void>[] = []
+    for (const wrapper of this.connections.values()) {
+      wrapper.state = 'closed'
+      closePromises.push(wrapper.connection.close().catch(() => {}))
+    }
+    await Promise.all(closePromises)
+
+    this.connections.clear()
+    this.idleConnections.clear()
+  }
+
+  /**
+   * Get current pool metrics.
+   */
+  getMetrics(): DOPoolMetrics {
+    const inUse = this.countInUse()
+    const available = this.idleConnections.size
+
+    return {
+      totalCreated: this.totalCreated,
+      totalClosed: this.totalClosed,
+      currentSize: this.connections.size,
+      inUse,
+      available,
+      pendingRequests: this.pendingQueue.length,
+      totalAcquisitions: this.totalAcquisitions,
+      totalReleases: this.totalReleases,
+      totalHealthChecks: this.totalHealthChecks,
+      totalHealthCheckFailures: this.totalHealthCheckFailures,
+      avgWaitTimeMs: this.calculateAvgWaitTime(),
+      maxWaitTimeMs: this.waitTimes.length > 0 ? Math.max(...this.waitTimes) : 0,
+      totalTimeouts: this.totalTimeouts,
+      queueLength: this.pendingQueue.length,
+      utilizationPercent: this.connections.size > 0 ? (inUse / this.connections.size) * 100 : 0,
+      lastActivityAt: this.lastActivityAt,
+    }
+  }
+
+  /**
+   * Get statistics for individual connections.
+   */
+  getConnectionStats(): ConnectionStats[] {
+    const stats: ConnectionStats[] = []
+
+    for (const [id, wrapper] of this.connections) {
+      stats.push({
+        id,
+        createdAt: wrapper.createdAt,
+        lastUsedAt: wrapper.lastUsedAt,
+        useCount: wrapper.useCount,
+        errorCount: wrapper.errorCount,
+        state: wrapper.state,
+      })
+    }
+
+    return stats
+  }
+
+  /**
+   * Get current pool size.
+   */
+  get size(): number {
+    return this.connections.size
+  }
+
+  /**
+   * Get number of available (idle) connections.
+   */
+  get availableCount(): number {
+    return this.idleConnections.size
+  }
+
+  /**
+   * Get number of pending requests in queue.
+   */
+  get queueLength(): number {
+    return this.pendingQueue.length
+  }
+
+  /**
+   * Check if pool is closed.
+   */
+  get isClosed(): boolean {
+    return this.closed
+  }
+
+  // Private methods
+
+  private async createConnection(): Promise<PooledConnectionWrapper<T>> {
+    const connection = await this.config.factory()
+    const wrapper: PooledConnectionWrapper<T> = {
+      connection,
+      createdAt: Date.now(),
+      lastUsedAt: Date.now(),
+      useCount: 0,
+      errorCount: 0,
+      state: 'idle',
+    }
+
+    this.connections.set(connection.id, wrapper)
+    this.totalCreated++
+    return wrapper
+  }
+
+  private async getIdleConnection(): Promise<T | null> {
+    for (const connId of this.idleConnections) {
+      const wrapper = this.connections.get(connId)
+      if (!wrapper || wrapper.state !== 'idle') {
+        this.idleConnections.delete(connId)
+        continue
+      }
+
+      // Validate connection health
+      wrapper.state = 'validating'
+
+      try {
+        const isHealthy = await this.validateConnection(wrapper)
+        if (isHealthy) {
+          this.markInUse(connId)
+          return wrapper.connection
+        } else {
+          // Connection is unhealthy, remove it
+          await this.removeConnection(connId)
+        }
+      } catch {
+        // Validation failed, remove connection
+        await this.removeConnection(connId)
+      }
+    }
+
+    return null
+  }
+
+  private async validateConnection(wrapper: PooledConnectionWrapper<T>): Promise<boolean> {
+    return new Promise<boolean>(async (resolve) => {
+      const timeoutId = setTimeout(() => {
+        resolve(false)
+      }, this.config.validationTimeoutMs)
+
+      try {
+        const isHealthy = await this.config.healthChecker(wrapper.connection)
+        clearTimeout(timeoutId)
+        resolve(isHealthy)
+      } catch {
+        clearTimeout(timeoutId)
+        resolve(false)
+      }
+    })
+  }
+
+  private markIdle(connId: string): void {
+    const wrapper = this.connections.get(connId)
+    if (!wrapper) return
+
+    wrapper.state = 'idle'
+    this.idleConnections.add(connId)
+
+    // Start idle timer
+    this.clearIdleTimer(connId)
+    const timer = setTimeout(() => {
+      this.handleIdleTimeout(connId)
+    }, this.config.idleTimeoutMs)
+    this.idleTimers.set(connId, timer)
+  }
+
+  private markInUse(connId: string): void {
+    const wrapper = this.connections.get(connId)
+    if (!wrapper) return
+
+    wrapper.state = 'in_use'
+    wrapper.useCount++
+    this.idleConnections.delete(connId)
+    this.clearIdleTimer(connId)
+    this.totalAcquisitions++
+  }
+
+  private clearIdleTimer(connId: string): void {
+    const timer = this.idleTimers.get(connId)
+    if (timer) {
+      clearTimeout(timer)
+      this.idleTimers.delete(connId)
+    }
+  }
+
+  private async handleIdleTimeout(connId: string): Promise<void> {
+    // Don't remove if we're at minimum size
+    if (this.connections.size <= this.config.minSize) {
+      // Restart the timer
+      const timer = setTimeout(() => {
+        this.handleIdleTimeout(connId)
+      }, this.config.idleTimeoutMs)
+      this.idleTimers.set(connId, timer)
+      return
+    }
+
+    await this.removeConnection(connId)
+  }
+
+  private async removeConnection(connId: string): Promise<void> {
+    const wrapper = this.connections.get(connId)
+    if (!wrapper) return
+
+    wrapper.state = 'closed'
+    this.connections.delete(connId)
+    this.idleConnections.delete(connId)
+    this.clearIdleTimer(connId)
+    this.totalClosed++
+
+    try {
+      await wrapper.connection.close()
+    } catch {
+      // Ignore close errors
+    }
+  }
+
+  private enqueueRequest(startTime: number): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        const index = this.pendingQueue.findIndex(p => p.resolve === resolve)
+        if (index !== -1) {
+          this.pendingQueue.splice(index, 1)
+        }
+        this.totalTimeouts++
+        reject(new Error('Acquire timeout: pool exhausted'))
+      }, this.config.acquireTimeoutMs)
+
+      const pending: PendingRequest<T> = {
+        resolve: (conn) => {
+          this.recordWaitTime(Date.now() - startTime)
+          resolve(conn)
+        },
+        reject,
+        enqueuedAt: startTime,
+        timeoutId,
+      }
+
+      this.pendingQueue.push(pending)
+    })
+  }
+
+  private async runHealthCheck(): Promise<void> {
+    if (this.closed) return
+
+    const connIds = Array.from(this.idleConnections)
+
+    for (const connId of connIds) {
+      if (this.closed) return
+
+      const wrapper = this.connections.get(connId)
+      if (!wrapper || wrapper.state !== 'idle') continue
+
+      this.totalHealthChecks++
+      wrapper.state = 'validating'
+
+      try {
+        const isHealthy = await this.validateConnection(wrapper)
+        if (!isHealthy) {
+          this.totalHealthCheckFailures++
+          await this.removeConnection(connId)
+        } else {
+          wrapper.state = 'idle'
+        }
+      } catch {
+        this.totalHealthCheckFailures++
+        await this.removeConnection(connId)
+      }
+    }
+
+    // Ensure minimum pool size
+    await this.ensureMinSize()
+  }
+
+  private async ensureMinSize(): Promise<void> {
+    while (this.connections.size < this.config.minSize && !this.closed) {
+      try {
+        const wrapper = await this.createConnection()
+        this.markIdle(wrapper.connection.id)
+      } catch {
+        // Stop trying if creation fails
+        break
+      }
+    }
+  }
+
+  private async checkScaling(): Promise<void> {
+    if (this.closed) return
+
+    const utilization = this.countInUse() / Math.max(this.connections.size, 1)
+
+    // Scale up
+    if (utilization >= this.config.scaleUpThreshold &&
+        this.connections.size < this.config.maxSize) {
+      try {
+        const wrapper = await this.createConnection()
+        this.markIdle(wrapper.connection.id)
+      } catch {
+        // Ignore scale-up failures
+      }
+    }
+
+    // Scale down (only if above minimum and low utilization)
+    if (utilization <= this.config.scaleDownThreshold &&
+        this.connections.size > this.config.minSize) {
+      // Remove one idle connection
+      for (const connId of this.idleConnections) {
+        if (this.connections.size <= this.config.minSize) break
+        await this.removeConnection(connId)
+        break
+      }
+    }
+  }
+
+  private countInUse(): number {
+    let count = 0
+    for (const wrapper of this.connections.values()) {
+      if (wrapper.state === 'in_use') {
+        count++
+      }
+    }
+    return count
+  }
+
+  private recordWaitTime(time: number): void {
+    this.waitTimes.push(time)
+    // Keep only last 1000 measurements
+    if (this.waitTimes.length > 1000) {
+      this.waitTimes.shift()
+    }
+  }
+
+  private calculateAvgWaitTime(): number {
+    if (this.waitTimes.length === 0) return 0
+    return this.waitTimes.reduce((a, b) => a + b, 0) / this.waitTimes.length
+  }
+}
+
+/**
+ * Helper class for creating DO stub connections
+ */
+export class DOStubConnection implements PoolableConnection {
+  readonly id: string
+  private lastActivity: number
+  private _healthy = true
+  private _closed = false
+
+  constructor(
+    id: string,
+    private readonly stub: DurableObjectStub,
+    private readonly closeHandler?: () => Promise<void>
+  ) {
+    this.id = id
+    this.lastActivity = Date.now()
+  }
+
+  /**
+   * Get the underlying DO stub
+   */
+  getStub(): DurableObjectStub {
+    return this.stub
+  }
+
+  isHealthy(): boolean {
+    return this._healthy && !this._closed
+  }
+
+  async close(): Promise<void> {
+    this._closed = true
+    if (this.closeHandler) {
+      await this.closeHandler()
+    }
+  }
+
+  getLastActivity(): number {
+    return this.lastActivity
+  }
+
+  touch(): void {
+    this.lastActivity = Date.now()
+  }
+
+  markUnhealthy(): void {
+    this._healthy = false
+  }
+}
+
+/**
+ * Create a pool of DO stub connections
+ */
+export function createDOPool(
+  namespace: DurableObjectNamespace,
+  options: {
+    idGenerator?: () => DurableObjectId
+    maxSize?: number
+    minSize?: number
+    idleTimeoutMs?: number
+  } = {}
+): DOConnectionPool<DOStubConnection> {
+  let connectionCounter = 0
+
+  const factory: ConnectionFactory<DOStubConnection> = async () => {
+    const id = options.idGenerator
+      ? options.idGenerator()
+      : namespace.newUniqueId()
+
+    const stub = namespace.get(id)
+    const connId = `do-conn-${++connectionCounter}`
+
+    return new DOStubConnection(connId, stub)
+  }
+
+  return new DOConnectionPool({
+    factory,
+    maxSize: options.maxSize ?? 10,
+    minSize: options.minSize ?? 0,
+    idleTimeoutMs: options.idleTimeoutMs ?? 60000,
+    fairQueue: true,
+    enableHealthCheck: true,
+  })
+}