findbug 0.2.0

data/lib/findbug/storage/circuit_breaker.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+module Findbug
+  module Storage
+    # CircuitBreaker prevents cascading failures when Redis is down.
+    #
+    # THE PROBLEM IT SOLVES
+    # =====================
+    #
+    # Imagine Redis goes down during peak traffic:
+    #
+    #   Without circuit breaker:
+    #   - 1000 requests/second
+    #   - Each tries to write to Redis
+    #   - Each waits 1 second for timeout
+    #   - Your app becomes unusable
+    #
+    #   With circuit breaker:
+    #   - After 5 failures, circuit "opens"
+    #   - Next 1000 requests skip Redis immediately
+    #   - Your app stays fast
+    #   - After 30 seconds, we try again
+    #
+    # THE THREE STATES
+    # ================
+    #
+    #   ┌─────────────────────────────────────────────────────────┐
+    #   │                                                         │
+    #   │   ┌──────────┐    failures >= 5    ┌──────────┐        │
+    #   │   │  CLOSED  │ ─────────────────── │   OPEN   │        │
+    #   │   │ (normal) │                     │ (tripped)│        │
+    #   │   └──────────┘                     └──────────┘        │
+    #   │        ▲                                 │             │
+    #   │        │ success                         │ 30 seconds  │
+    #   │        │                                 ▼             │
+    #   │        │                          ┌───────────┐        │
+    #   │        └───────────────────────── │ HALF-OPEN │        │
+    #   │                                   │ (testing) │        │
+    #   │                                   └───────────┘        │
+    #   │                                         │              │
+    #   │                                         │ failure      │
+    #   │                                         ▼              │
+    #   │                                   ┌──────────┐         │
+    #   │                                   │   OPEN   │         │
+    #   │                                   └──────────┘         │
+    #   └─────────────────────────────────────────────────────────┘
+    #
+    # THREAD SAFETY
+    # =============
+    #
+    # This class uses Monitor (a reentrant mutex) to ensure thread safety.
+    # Multiple threads can check/update the circuit state without races.
+    #
+    class CircuitBreaker
+      # How many failures before we trip the circuit
+      FAILURE_THRESHOLD = 5
+
+      # How long to wait before trying again (in seconds)
+      RECOVERY_TIME = 30
+
+      class << self
+        # Check if requests are allowed through
+        #
+        # @return [Boolean] true if we should attempt the operation
+        #
+        # @example
+        #   if CircuitBreaker.allow?
+        #     # try Redis operation
+        #   else
+        #     # skip and log
+        #   end
+        #
+        def allow?
+          synchronize do
+            case state
+            when :closed
+              # Normal operation - allow all requests
+              true
+            when :open
+              if recovery_period_elapsed?
+                # Time to test if Redis is back
+                transition_to(:half_open)
+                true
+              else
+                # Still in cooldown - reject immediately
+                false
+              end
+            when :half_open
+              # We're testing - allow this one request through
+              true
+            end
+          end
+        end
+
+        # Record a successful operation
+        #
+        # Call this after a successful Redis operation.
+        # This resets the failure count and closes the circuit.
+        #
+        def record_success
+          synchronize do
+            @failures = 0
+            transition_to(:closed)
+          end
+        end
+
+        # Record a failed operation
+        #
+        # Call this when a Redis operation fails.
+        # After enough failures, the circuit opens.
+        #
+        def record_failure
+          synchronize do
+            @failures = (@failures || 0) + 1
+
+            if state == :half_open
+              # Failed during testing - back to open
+              transition_to(:open)
+            elsif @failures >= FAILURE_THRESHOLD
+              # Too many failures - trip the circuit
+              transition_to(:open)
+              log_circuit_opened
+            end
+          end
+        end
+
+        # Get current state (for monitoring/debugging)
+        #
+        # @return [Symbol] :closed, :open, or :half_open
+        #
+        def state
+          @state || :closed
+        end
+
+        # Get current failure count (for monitoring)
+        #
+        # @return [Integer] number of consecutive failures
+        #
+        def failure_count
+          @failures || 0
+        end
+
+        # Reset the circuit breaker (for testing)
+        def reset!
+          synchronize do
+            @state = :closed
+            @failures = 0
+            @opened_at = nil
+          end
+        end
+
+        # Execute a block with circuit breaker protection
+        #
+        # @yield the operation to protect
+        # @return [Object, nil] the block's return value, or nil if rejected
+        #
+        # @example
+        #   result = CircuitBreaker.execute do
+        #     redis.lpush("key", "value")
+        #   end
+        #
+        # This is a convenience method that combines allow?/record_success/record_failure.
+        #
+        def execute
+          return nil unless allow?
+
+          begin
+            result = yield
+            record_success
+            result
+          rescue StandardError => e
+            record_failure
+            raise e
+          end
+        end
+
+        private
+
+        def synchronize(&block)
+          @monitor ||= Monitor.new
+          @monitor.synchronize(&block)
+        end
+
+        def transition_to(new_state)
+          old_state = @state
+          @state = new_state
+
+          if new_state == :open
+            @opened_at = Time.now
+          end
+
+          log_state_change(old_state, new_state) if old_state != new_state
+        end
+
+        def recovery_period_elapsed?
+          return true unless @opened_at
+
+          Time.now - @opened_at >= RECOVERY_TIME
+        end
+
+        def log_circuit_opened
+          Findbug.logger.warn(
+            "[Findbug] Circuit breaker opened after #{FAILURE_THRESHOLD} failures. " \
+            "Redis operations will be skipped for #{RECOVERY_TIME} seconds."
+          )
+        end
+
+        def log_state_change(old_state, new_state)
+          return if old_state.nil? # Initial state
+
+          case new_state
+          when :closed
+            Findbug.logger.info("[Findbug] Circuit breaker closed. Redis operations resumed.")
+          when :half_open
+            Findbug.logger.info("[Findbug] Circuit breaker half-open. Testing Redis connection...")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/findbug/storage/connection_pool.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "redis"
+require "connection_pool"
+
+module Findbug
+  module Storage
+    # ConnectionPool manages Redis connections for Findbug.
+    #
+    # WHY A SEPARATE POOL?
+    # ====================
+    #
+    # Your Rails app likely already uses Redis for:
+    # - Sidekiq (job queue)
+    # - Caching (Rails.cache)
+    # - Action Cable (websockets)
+    #
+    # If Findbug shared these connections, we could:
+    # 1. Starve your app of connections during high error rates
+    # 2. Cause Sidekiq jobs to timeout waiting for connections
+    # 3. Create unpredictable latency spikes
+    #
+    # By maintaining our OWN pool, Findbug is isolated.
+    # If our pool is exhausted, only Findbug suffers - your app keeps running.
+    #
+    # HOW CONNECTION POOLING WORKS
+    # ============================
+    #
+    #   Without pooling:
+    #   Thread 1 → create connection → use → close
+    #   Thread 2 → create connection → use → close  (expensive!)
+    #   Thread 3 → create connection → use → close
+    #
+    #   With pooling:
+    #   Thread 1 → borrow connection → use → return to pool
+    #   Thread 2 → borrow connection → use → return to pool
+    #   Thread 3 → borrow connection → use → return to pool
+    #                   ↓
+    #            [Pool of 5 connections]
+    #
+    # The `connection_pool` gem handles:
+    # - Creating connections lazily (only when needed)
+    # - Returning connections automatically (via block)
+    # - Waiting for available connections (with timeout)
+    # - Thread-safety (multiple threads can't corrupt state)
+    #
+    class ConnectionPool
+      class << self
+        # Get a connection from the pool and execute a block
+        #
+        # @yield [Redis] a Redis connection
+        # @return [Object] the return value of the block
+        #
+        # @example
+        #   ConnectionPool.with do |redis|
+        #     redis.lpush("findbug:errors", data.to_json)
+        #   end
+        #
+        # WHY A BLOCK?
+        # ------------
+        # The block pattern ensures connections are ALWAYS returned to the pool.
+        # Even if an exception occurs, the connection goes back.
+        # This prevents connection leaks.
+        #
+        def with(&block)
+          pool.with(&block)
+        end
+
+        # Get the raw pool (for advanced usage)
+        #
+        # @return [::ConnectionPool] the underlying connection pool
+        #
+        def pool
+          @pool ||= create_pool
+        end
+
+        # Shutdown the pool (for cleanup/testing)
+        #
+        # This closes all connections and resets the pool.
+        # Call this when shutting down your app or between tests.
+        #
+        def shutdown!
+          @pool&.shutdown { |redis| redis.close }
+          @pool = nil
+        end
+
+        # Check if a connection can be established
+        #
+        # @return [Boolean] true if Redis is reachable
+        #
+        # This is used by the circuit breaker to test if Redis is back up.
+        #
+        def healthy?
+          with { |redis| redis.ping == "PONG" }
+        rescue StandardError
+          false
+        end
+
+        private
+
+        def create_pool
+          config = Findbug.config
+
+          ::ConnectionPool.new(
+            size: config.redis_pool_size,
+            timeout: config.redis_pool_timeout
+          ) do
+            create_redis_connection(config.redis_url)
+          end
+        end
+
+        def create_redis_connection(url)
+          # Parse the URL and create a Redis connection
+          #
+          # WHY NOT JUST `Redis.new(url: url)`?
+          # -----------------------------------
+          # We add some defensive options:
+          # - connect_timeout: Don't hang if Redis is unreachable
+          # - read_timeout: Don't hang if Redis is slow
+          # - write_timeout: Don't hang on slow writes
+          # - reconnect_attempts: Retry on temporary failures
+          #
+          Redis.new(
+            url: url,
+            connect_timeout: 1.0,   # 1 second to establish connection
+            read_timeout: 1.0,      # 1 second to read response
+            write_timeout: 1.0,     # 1 second to write command
+            reconnect_attempts: 1   # Retry once on connection failure
+          )
+        end
+      end
+    end
+  end
+end

data/lib/findbug/storage/redis_buffer.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "circuit_breaker"
+require_relative "connection_pool"
+
+module Findbug
+  module Storage
+    # RedisBuffer provides fast, non-blocking writes to Redis.
+    #
+    # THIS IS THE KEY TO ZERO PERFORMANCE IMPACT
+    # ==========================================
+    #
+    # Traditional error tracking (synchronous):
+    #
+    #   Request starts
+    #       ↓
+    #   Exception occurs
+    #       ↓
+    #   BLOCKING: Write to database (50-100ms)  ← Your user waits!
+    #       ↓
+    #   Request ends
+    #
+    # Findbug (asynchronous):
+    #
+    #   Request starts
+    #       ↓
+    #   Exception occurs
+    #       ↓
+    #   NON-BLOCKING: Spawn thread to write to Redis (0ms)
+    #       ↓                          ↓
+    #   Request ends            Background: Redis write (1-2ms)
+    #       ↓
+    #   User gets response immediately
+    #
+    # WHY REDIS INSTEAD OF DATABASE?
+    # ==============================
+    #
+    # Redis write: ~1-2ms
+    # Database write: ~50-100ms (with indexes, constraints, etc.)
+    #
+    # Even if we made DB writes async, Redis is still better for buffering because:
+    # 1. It's faster (in-memory)
+    # 2. It handles high write loads gracefully
+    # 3. It has built-in expiration (TTL)
+    # 4. It supports atomic list operations
+    #
+    # The database is for long-term storage. Redis is for the fast buffer.
+    #
+    # WHY Thread.new INSTEAD OF SIDEKIQ?
+    # ==================================
+    #
+    # Sidekiq itself writes to Redis. If we used Sidekiq to buffer our errors:
+    # 1. We'd add Sidekiq job overhead (~5ms)
+    # 2. We'd share Redis connections with Sidekiq
+    # 3. We'd depend on Sidekiq being healthy
+    #
+    # A simple Thread.new is:
+    # 1. Instant (no queue overhead)
+    # 2. Independent of your job system
+    # 3. Simpler (no job serialization)
+    #
+    # We use Sidekiq/ActiveJob later for PERSISTING to DB, not for buffering.
+    #
+    class RedisBuffer
+      # Key prefix for error events
+      ERRORS_KEY = "findbug:errors"
+
+      # Key prefix for performance events
+      PERFORMANCE_KEY = "findbug:performance"
+
+      # Key for tracking stats
+      STATS_KEY = "findbug:stats"
+
+      class << self
+        # Push an error event to the buffer (async, non-blocking)
+        #
+        # @param event_data [Hash] the error event data
+        #
+        # @example
+        #   RedisBuffer.push_error({
+        #     exception_class: "RuntimeError",
+        #     message: "Something went wrong",
+        #     backtrace: [...],
+        #     context: {...}
+        #   })
+        #
+        # IMPORTANT: This returns IMMEDIATELY. The actual write happens
+        # in a background thread. This is what makes us non-blocking.
+        #
+        def push_error(event_data)
+          push_async(ERRORS_KEY, event_data)
+        end
+
+        # Push a performance event to the buffer (async, non-blocking)
+        #
+        # @param event_data [Hash] the performance event data
+        #
+        def push_performance(event_data)
+          push_async(PERFORMANCE_KEY, event_data)
+        end
+
+        # Pop a batch of error events from the buffer
+        #
+        # @param batch_size [Integer] maximum number of events to retrieve
+        # @return [Array<Hash>] array of error events
+        #
+        # This is called by the PersistJob to move data from Redis to DB.
+        # It uses LPOP in a loop to get events atomically.
+        #
+        def pop_errors(batch_size = 100)
+          pop_batch(ERRORS_KEY, batch_size)
+        end
+
+        # Pop a batch of performance events from the buffer
+        #
+        # @param batch_size [Integer] maximum number of events to retrieve
+        # @return [Array<Hash>] array of performance events
+        #
+        def pop_performance(batch_size = 100)
+          pop_batch(PERFORMANCE_KEY, batch_size)
+        end
+
+        # Get buffer statistics (for monitoring)
+        #
+        # @return [Hash] buffer stats including queue lengths
+        #
+        def stats
+          ConnectionPool.with do |redis|
+            {
+              error_queue_length: redis.llen(ERRORS_KEY),
+              performance_queue_length: redis.llen(PERFORMANCE_KEY),
+              circuit_breaker_state: CircuitBreaker.state,
+              circuit_breaker_failures: CircuitBreaker.failure_count
+            }
+          end
+        rescue StandardError => e
+          # Always return circuit breaker state even if Redis is down
+          {
+            error_queue_length: 0,
+            performance_queue_length: 0,
+            circuit_breaker_state: Findbug::Storage::CircuitBreaker.state,
+            circuit_breaker_failures: Findbug::Storage::CircuitBreaker.failure_count,
+            error: "Redis connection failed: #{e.message}"
+          }
+        end
+
+        # Clear all buffers (for testing)
+        def clear!
+          ConnectionPool.with do |redis|
+            redis.del(ERRORS_KEY, PERFORMANCE_KEY)
+          end
+        rescue StandardError
+          # Ignore errors during cleanup
+        end
+
+        private
+
+        # The core async push operation
+        #
+        # WHY THIS PATTERN?
+        # -----------------
+        #
+        # 1. Check circuit breaker BEFORE spawning thread
+        #    - If Redis is down, don't waste resources on threads
+        #
+        # 2. Spawn a new thread for the actual write
+        #    - This returns immediately to the caller
+        #    - The thread runs independently
+        #
+        # 3. Inside the thread, use connection pool
+        #    - Gets a connection from the pool
+        #    - Writes to Redis
+        #    - Returns connection automatically (via block)
+        #
+        # 4. Handle errors gracefully
+        #    - Log but don't crash
+        #    - Update circuit breaker state
+        #
+        def push_async(key, event_data)
+          # Early exit if Findbug is disabled
+          return unless Findbug.enabled?
+
+          # Early exit if circuit breaker is open
+          unless CircuitBreaker.allow?
+            increment_dropped_count
+            return
+          end
+
+          # Spawn a thread for non-blocking write
+          #
+          # WHY Thread.new HERE?
+          # --------------------
+          # Thread.new creates a new Ruby thread that runs independently.
+          # The calling code continues immediately without waiting.
+          #
+          # THREAD SAFETY NOTES:
+          # - event_data is captured by the closure (safe - we're not mutating it)
+          # - ConnectionPool handles thread-safe connection borrowing
+          # - Redis operations are atomic
+          #
+          Thread.new do
+            perform_push(key, event_data)
+          rescue StandardError => e
+            # CRITICAL: Catch ALL errors in the thread
+            # An unhandled exception in a thread will crash the thread silently
+            handle_push_error(e)
+          end
+
+          nil # Return immediately
+        end
+
+        # The actual Redis push (runs in background thread)
+        def perform_push(key, event_data)
+          ConnectionPool.with do |redis|
+            # Add timestamp if not present
+            event_data[:captured_at] ||= Time.now.utc.iso8601(3)
+
+            # LPUSH adds to the LEFT of the list (newest first)
+            # We use JSON encoding for storage
+            redis.lpush(key, event_data.to_json)
+
+            # LTRIM keeps only the first N elements
+            # This prevents unbounded memory growth
+            # If we have more than max_buffer_size events, old ones are dropped
+            max_size = Findbug.config.max_buffer_size
+            redis.ltrim(key, 0, max_size - 1)
+
+            # Record success for circuit breaker
+            CircuitBreaker.record_success
+          end
+        end
+
+        # Handle errors during push
+        def handle_push_error(error)
+          CircuitBreaker.record_failure
+
+          # Log at debug level to avoid log spam during outages
+          Findbug.logger.debug(
+            "[Findbug] Failed to push event to Redis: #{error.message}"
+          )
+        end
+
+        # Pop a batch of events atomically
+        #
+        # WHY NOT LRANGE + LTRIM?
+        # -----------------------
+        # That's not atomic. Between LRANGE and LTRIM, new events could arrive.
+        # We use LPOP in a loop which is atomic per operation.
+        #
+        # WHY NOT RPOPLPUSH?
+        # ------------------
+        # We don't need a backup queue. If persistence fails, the job will
+        # retry and the data is still in the main queue.
+        #
+        def pop_batch(key, batch_size)
+          events = []
+
+          ConnectionPool.with do |redis|
+            batch_size.times do
+              # RPOP gets from the RIGHT (oldest first - FIFO order)
+              json = redis.rpop(key)
+              break unless json
+
+              begin
+                events << JSON.parse(json, symbolize_names: true)
+              rescue JSON::ParserError => e
+                Findbug.logger.error("[Findbug] Failed to parse event: #{e.message}")
+              end
+            end
+          end
+
+          events
+        end
+
+        # Track dropped events (when circuit breaker is open)
+        def increment_dropped_count
+          # We could track this in Redis too, but that defeats the purpose
+          # when Redis is down. Just log it.
+          Findbug.logger.debug("[Findbug] Event dropped (circuit breaker open)")
+        end
+      end
+    end
+  end
+end