brainzlab 0.1.1 → 0.1.2

data/lib/brainzlab/utilities/circuit_breaker.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+module BrainzLab
+  module Utilities
+    # Circuit breaker pattern implementation for resilient external calls
+    # Integrates with Flux for metrics and Reflex for error tracking
+    #
+    # States:
+    # - :closed - Normal operation, requests pass through
+    # - :open - Failing, requests are rejected immediately
+    # - :half_open - Testing, limited requests allowed to check recovery
+    #
+    # @example Basic usage
+    #   breaker = BrainzLab::Utilities::CircuitBreaker.new(
+    #     name: "external_api",
+    #     failure_threshold: 5,
+    #     recovery_timeout: 30
+    #   )
+    #
+    #   breaker.call do
+    #     external_api.request
+    #   end
+    #
+    # @example With fallback
+    #   breaker.call(fallback: -> { cached_value }) do
+    #     external_api.request
+    #   end
+    #
+    class CircuitBreaker
+      STATES = %i[closed open half_open].freeze
+
+      attr_reader :name, :state, :failure_count, :success_count, :last_failure_at
+
+      def initialize(name:, failure_threshold: 5, success_threshold: 2, recovery_timeout: 30, timeout: nil, exclude_exceptions: [])
+        @name = name
+        @failure_threshold = failure_threshold
+        @success_threshold = success_threshold
+        @recovery_timeout = recovery_timeout
+        @timeout = timeout
+        @exclude_exceptions = exclude_exceptions
+
+        @state = :closed
+        @failure_count = 0
+        @success_count = 0
+        @last_failure_at = nil
+        @mutex = Mutex.new
+      end
+
+      # Execute a block with circuit breaker protection
+      def call(fallback: nil)
+        check_state_transition!
+
+        case @state
+        when :open
+          track_rejected
+          if fallback
+            fallback.respond_to?(:call) ? fallback.call : fallback
+          else
+            raise CircuitOpenError, "Circuit '#{@name}' is open"
+          end
+        when :closed, :half_open
+          execute_with_protection(fallback) { yield }
+        end
+      end
+
+      # Force the circuit to a specific state
+      def force_state!(new_state)
+        raise ArgumentError, "Invalid state: #{new_state}" unless STATES.include?(new_state)
+
+        @mutex.synchronize do
+          @state = new_state
+          @failure_count = 0 if new_state == :closed
+          @success_count = 0 if new_state == :half_open
+        end
+
+        track_state_change(new_state)
+      end
+
+      # Reset the circuit breaker
+      def reset!
+        force_state!(:closed)
+        @last_failure_at = nil
+      end
+
+      # Get circuit status
+      def status
+        {
+          name: @name,
+          state: @state,
+          failure_count: @failure_count,
+          success_count: @success_count,
+          failure_threshold: @failure_threshold,
+          success_threshold: @success_threshold,
+          last_failure_at: @last_failure_at,
+          recovery_timeout: @recovery_timeout
+        }
+      end
+
+      # Check if circuit is allowing requests
+      def available?
+        check_state_transition!
+        @state != :open
+      end
+
+      # Class-level registry of circuit breakers
+      class << self
+        def registry
+          @registry ||= {}
+        end
+
+        def get(name)
+          registry[name.to_s]
+        end
+
+        def register(name, **options)
+          registry[name.to_s] = new(name: name, **options)
+        end
+
+        def call(name, **options, &block)
+          breaker = get(name) || register(name, **options)
+          breaker.call(**options.slice(:fallback), &block)
+        end
+
+        def reset_all!
+          registry.each_value(&:reset!)
+        end
+
+        def status_all
+          registry.transform_values(&:status)
+        end
+      end
+
+      private
+
+      def execute_with_protection(fallback)
+        result = if @timeout
+                   Timeout.timeout(@timeout) { yield }
+                 else
+                   yield
+                 end
+
+        record_success
+        result
+      rescue *excluded_exceptions => e
+        # Don't count excluded exceptions as failures
+        raise
+      rescue StandardError => e
+        record_failure(e)
+
+        if fallback
+          fallback.respond_to?(:call) ? fallback.call : fallback
+        else
+          raise
+        end
+      end
+
+      def record_success
+        @mutex.synchronize do
+          if @state == :half_open
+            @success_count += 1
+            if @success_count >= @success_threshold
+              transition_to(:closed)
+            end
+          else
+            @failure_count = 0
+          end
+        end
+
+        track_success
+      end
+
+      def record_failure(error)
+        @mutex.synchronize do
+          @failure_count += 1
+          @last_failure_at = Time.now
+
+          if @state == :half_open
+            transition_to(:open)
+          elsif @failure_count >= @failure_threshold
+            transition_to(:open)
+          end
+        end
+
+        track_failure(error)
+      end
+
+      def check_state_transition!
+        return unless @state == :open && @last_failure_at
+
+        if Time.now - @last_failure_at >= @recovery_timeout
+          @mutex.synchronize do
+            transition_to(:half_open) if @state == :open
+          end
+        end
+      end
+
+      def transition_to(new_state)
+        old_state = @state
+        @state = new_state
+
+        case new_state
+        when :closed
+          @failure_count = 0
+          @success_count = 0
+        when :half_open
+          @success_count = 0
+        when :open
+          # Keep failure count for debugging
+        end
+
+        track_state_change(new_state, old_state)
+      end
+
+      def excluded_exceptions
+        @exclude_exceptions.empty? ? [] : @exclude_exceptions
+      end
+
+      # Metrics tracking
+
+      def track_success
+        return unless BrainzLab.configuration.flux_effectively_enabled?
+
+        BrainzLab::Flux.increment("circuit_breaker.success", tags: { name: @name, state: @state.to_s })
+      end
+
+      def track_failure(error)
+        return unless BrainzLab.configuration.flux_effectively_enabled?
+
+        BrainzLab::Flux.increment("circuit_breaker.failure", tags: {
+          name: @name,
+          state: @state.to_s,
+          error_class: error.class.name
+        })
+      end
+
+      def track_rejected
+        return unless BrainzLab.configuration.flux_effectively_enabled?
+
+        BrainzLab::Flux.increment("circuit_breaker.rejected", tags: { name: @name })
+      end
+
+      def track_state_change(new_state, old_state = nil)
+        return unless BrainzLab.configuration.flux_effectively_enabled?
+
+        BrainzLab::Flux.track("circuit_breaker.state_change", {
+          name: @name,
+          new_state: new_state.to_s,
+          old_state: old_state&.to_s,
+          failure_count: @failure_count
+        })
+
+        # Also add breadcrumb for debugging
+        BrainzLab::Reflex.add_breadcrumb(
+          "Circuit '#{@name}' transitioned to #{new_state}",
+          category: "circuit_breaker",
+          level: new_state == :open ? :warning : :info,
+          data: { name: @name, old_state: old_state, new_state: new_state }
+        )
+      end
+
+      # Error raised when circuit is open
+      class CircuitOpenError < StandardError; end
+    end
+  end
+end

data/lib/brainzlab/utilities/health_check.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+module BrainzLab
+  module Utilities
+    # Health check utility for application health endpoints
+    # Provides checks for database, cache, queues, and external services
+    #
+    # @example Basic usage in Rails routes
+    #   # config/routes.rb
+    #   mount BrainzLab::Utilities::HealthCheck::Engine => "/health"
+    #
+    # @example Manual usage
+    #   result = BrainzLab::Utilities::HealthCheck.run
+    #   result[:status]  # => "healthy" or "unhealthy"
+    #   result[:checks]  # => { database: { status: "ok", latency_ms: 5 }, ... }
+    #
+    class HealthCheck
+      CHECKS = %i[database redis cache queue memory disk].freeze
+
+      class << self
+        # Run all configured health checks
+        def run(checks: nil)
+          checks_to_run = checks || CHECKS
+          results = {}
+          overall_healthy = true
+
+          checks_to_run.each do |check|
+            begin
+              result = send("check_#{check}")
+              results[check] = result
+              overall_healthy = false if result[:status] != "ok"
+            rescue StandardError => e
+              results[check] = { status: "error", message: e.message }
+              overall_healthy = false
+            end
+          end
+
+          {
+            status: overall_healthy ? "healthy" : "unhealthy",
+            timestamp: Time.now.utc.iso8601,
+            checks: results
+          }
+        end
+
+        # Quick check - just returns status
+        def healthy?
+          result = run
+          result[:status] == "healthy"
+        end
+
+        # Database connectivity check
+        def check_database
+          return { status: "skip", message: "ActiveRecord not loaded" } unless defined?(ActiveRecord::Base)
+
+          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          ActiveRecord::Base.connection.execute("SELECT 1")
+          latency = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round(2)
+
+          { status: "ok", latency_ms: latency }
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        # Redis connectivity check
+        def check_redis
+          return { status: "skip", message: "Redis not configured" } unless defined?(Redis)
+
+          redis = find_redis_connection
+          return { status: "skip", message: "No Redis connection found" } unless redis
+
+          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          redis.ping
+          latency = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round(2)
+
+          { status: "ok", latency_ms: latency }
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        # Rails cache check
+        def check_cache
+          return { status: "skip", message: "Rails not loaded" } unless defined?(Rails)
+
+          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          key = "brainzlab_health_check_#{SecureRandom.hex(4)}"
+          Rails.cache.write(key, "ok", expires_in: 10.seconds)
+          value = Rails.cache.read(key)
+          Rails.cache.delete(key)
+          latency = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round(2)
+
+          if value == "ok"
+            { status: "ok", latency_ms: latency }
+          else
+            { status: "error", message: "Cache read/write failed" }
+          end
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        # Queue system check
+        def check_queue
+          if defined?(SolidQueue)
+            check_solid_queue
+          elsif defined?(Sidekiq)
+            check_sidekiq
+          elsif defined?(GoodJob)
+            check_good_job
+          else
+            { status: "skip", message: "No queue system detected" }
+          end
+        end
+
+        # Memory usage check
+        def check_memory
+          mem_info = memory_usage
+
+          status = if mem_info[:percentage] > 90
+                     "warning"
+                   elsif mem_info[:percentage] > 95
+                     "error"
+                   else
+                     "ok"
+                   end
+
+          {
+            status: status,
+            used_mb: mem_info[:used_mb],
+            percentage: mem_info[:percentage]
+          }
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        # Disk space check
+        def check_disk
+          disk_info = disk_usage
+
+          status = if disk_info[:percentage] > 90
+                     "warning"
+                   elsif disk_info[:percentage] > 95
+                     "error"
+                   else
+                     "ok"
+                   end
+
+          {
+            status: status,
+            used_gb: disk_info[:used_gb],
+            available_gb: disk_info[:available_gb],
+            percentage: disk_info[:percentage]
+          }
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        # Register a custom health check
+        def register(name, &block)
+          custom_checks[name.to_sym] = block
+        end
+
+        def custom_checks
+          @custom_checks ||= {}
+        end
+
+        private
+
+        def find_redis_connection
+          # Try common Redis connection sources
+          if defined?(Redis.current) && Redis.current
+            Redis.current
+          elsif defined?(Sidekiq) && Sidekiq.respond_to?(:redis)
+            Sidekiq.redis { |conn| return conn }
+          elsif defined?(Rails) && Rails.application.config.respond_to?(:redis)
+            Rails.application.config.redis
+          end
+        rescue StandardError
+          nil
+        end
+
+        def check_solid_queue
+          return { status: "skip", message: "SolidQueue not loaded" } unless defined?(SolidQueue)
+
+          # Check if processes are running
+          if defined?(SolidQueue::Process)
+            process_count = SolidQueue::Process.where("last_heartbeat_at > ?", 5.minutes.ago).count
+            {
+              status: process_count > 0 ? "ok" : "warning",
+              processes: process_count
+            }
+          else
+            { status: "ok", message: "SolidQueue configured" }
+          end
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        def check_sidekiq
+          return { status: "skip", message: "Sidekiq not loaded" } unless defined?(Sidekiq)
+
+          stats = Sidekiq::Stats.new
+          {
+            status: "ok",
+            processed: stats.processed,
+            failed: stats.failed,
+            queues: stats.queues,
+            workers: stats.workers_size
+          }
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        def check_good_job
+          return { status: "skip", message: "GoodJob not loaded" } unless defined?(GoodJob)
+
+          {
+            status: "ok",
+            pending: GoodJob::Job.where(performed_at: nil).count,
+            running: GoodJob::Job.running.count
+          }
+        rescue StandardError => e
+          { status: "error", message: e.message }
+        end
+
+        def memory_usage
+          # Use /proc/self/status on Linux, ps on macOS
+          if File.exist?("/proc/self/status")
+            status = File.read("/proc/self/status")
+            vm_rss = status.match(/VmRSS:\s+(\d+)\s+kB/)&.captures&.first&.to_i || 0
+            used_mb = (vm_rss / 1024.0).round(2)
+          else
+            # macOS fallback
+            pid = Process.pid
+            output = `ps -o rss= -p #{pid}`.strip
+            used_mb = (output.to_i / 1024.0).round(2)
+          end
+
+          # Estimate percentage (based on typical container memory)
+          max_mb = ENV.fetch("MEMORY_LIMIT_MB", 512).to_i
+          percentage = ((used_mb / max_mb) * 100).round(2)
+
+          { used_mb: used_mb, percentage: percentage }
+        end
+
+        def disk_usage
+          output = `df -k /`.split("\n").last.split
+          total = output[1].to_i / 1024 / 1024.0
+          used = output[2].to_i / 1024 / 1024.0
+          available = output[3].to_i / 1024 / 1024.0
+          percentage = ((used / total) * 100).round(2)
+
+          {
+            used_gb: used.round(2),
+            available_gb: available.round(2),
+            percentage: percentage
+          }
+        end
+      end
+
+      # Rails Engine for mounting health endpoints
+      if defined?(::Rails::Engine)
+        class Engine < ::Rails::Engine
+          isolate_namespace BrainzLab::Utilities::HealthCheck
+
+          routes.draw do
+            get "/", to: "health#show"
+            get "/live", to: "health#live"
+            get "/ready", to: "health#ready"
+          end
+        end
+      end
+
+      # Controller for health endpoints
+      if defined?(ActionController::API)
+        class HealthController < ActionController::API
+          def show
+            result = HealthCheck.run
+            status = result[:status] == "healthy" ? :ok : :service_unavailable
+            render json: result, status: status
+          end
+
+          def live
+            # Liveness probe - just check if the app is running
+            render json: { status: "ok", timestamp: Time.now.utc.iso8601 }
+          end
+
+          def ready
+            # Readiness probe - check critical dependencies
+            result = HealthCheck.run(checks: [:database, :redis])
+            status = result[:status] == "healthy" ? :ok : :service_unavailable
+            render json: result, status: status
+          end
+        end
+      end
+    end
+  end
+end