dead_bro 0.2.7 → 0.2.9

data/lib/dead_bro/configuration.rb

@@ -2,42 +2,125 @@
 
 module DeadBro
   class Configuration
-    attr_accessor :api_key, :open_timeout, :read_timeout, :enabled, :ruby_dev, :memory_tracking_enabled,
-      :allocation_tracking_enabled, :circuit_breaker_enabled, :circuit_breaker_failure_threshold, :circuit_breaker_recovery_timeout,
-      :circuit_breaker_retry_timeout, :sample_rate, :excluded_controllers, :excluded_jobs,
-      :exclusive_controllers, :exclusive_jobs, :deploy_id, :slow_query_threshold_ms, :explain_analyze_enabled,
+    # Local-only settings (not overwritten by API `settings` payloads).
+    # Note: `enabled` may still be updated remotely via apply_remote_settings when the backend
+    # returns it in a response; local configure() values apply until the next remote update.
+    attr_accessor :api_key, :open_timeout, :read_timeout, :enabled, :ruby_dev,
+      :circuit_breaker_enabled, :circuit_breaker_failure_threshold, :circuit_breaker_recovery_timeout,
+      :circuit_breaker_retry_timeout, :deploy_id, :disk_paths, :interfaces_ignore
+
+    # Remote-managed settings (overwritten by backend JSON `settings` on successful API responses)
+    attr_accessor :memory_tracking_enabled, :allocation_tracking_enabled,
+      :sample_rate, :slow_query_threshold_ms, :explain_analyze_enabled,
       :job_queue_monitoring_enabled, :enable_db_stats, :enable_process_stats, :enable_system_stats,
-      :disk_paths, :interfaces_ignore, :max_sql_queries_to_send, :max_logs_to_send
+      :max_sql_queries_to_send, :max_logs_to_send
+
+    # Readers for exclusion lists. Writers are defined below so we can compile
+    # and cache the regex form once, instead of rebuilding it per request.
+    attr_reader :excluded_controllers, :excluded_jobs, :exclusive_controllers, :exclusive_jobs
+
+    # Tracks when we last received settings from the backend (in-memory only)
+    attr_accessor :settings_received_at
+
+    # Last successful heartbeat HTTP response time while disabled (in-memory only)
+    attr_accessor :last_heartbeat_at
+
+    # Throttles heartbeat attempts to HEARTBEAT_INTERVAL (set when a heartbeat request is started)
+    attr_accessor :last_heartbeat_attempt_at
+
+    HEARTBEAT_INTERVAL = 60 # seconds
+
+    REMOTE_SETTING_KEYS = %w[
+      enabled sample_rate memory_tracking_enabled allocation_tracking_enabled
+      explain_analyze_enabled slow_query_threshold_ms max_sql_queries_to_send max_logs_to_send
+      excluded_controllers excluded_jobs exclusive_controllers exclusive_jobs
+      job_queue_monitoring_enabled enable_db_stats enable_process_stats enable_system_stats
+    ].freeze
 
     def initialize
       @api_key = nil
-      @endpoint_url = nil
       @open_timeout = 1.0
       @read_timeout = 1.0
       @enabled = true
       @ruby_dev = false
-      @memory_tracking_enabled = true
-      @allocation_tracking_enabled = false # Disabled by default for performance
       @circuit_breaker_enabled = true
       @circuit_breaker_failure_threshold = 3
-      @circuit_breaker_recovery_timeout = 60 # seconds
-      @circuit_breaker_retry_timeout = 300 # seconds
-      @sample_rate = 100
-      @excluded_controllers = []
-      @excluded_jobs = []
-      @exclusive_controllers = []
-      @exclusive_jobs = []
+      @circuit_breaker_recovery_timeout = 60
+      @circuit_breaker_retry_timeout = 300
       @deploy_id = resolve_deploy_id
-      @slow_query_threshold_ms = 500 # Default: 500ms
-      @explain_analyze_enabled = false # Enable EXPLAIN ANALYZE for slow queries by default
-      @job_queue_monitoring_enabled = false # Disabled by default
-      @enable_db_stats = false
-      @enable_process_stats = false
-      @enable_system_stats = false
       @disk_paths = ["/"]
       @interfaces_ignore = %w[lo lo0 docker0]
-      @max_sql_queries_to_send = 500 # Cap to avoid 413 Request Entity Too Large
+
+      # Remote-managed defaults (used until backend sends real values)
+      @sample_rate = 100
+      @memory_tracking_enabled = true
+      @allocation_tracking_enabled = false
+      @explain_analyze_enabled = false
+      @slow_query_threshold_ms = 500
+      @max_sql_queries_to_send = 500
       @max_logs_to_send = 100
+      self.excluded_controllers = []
+      self.excluded_jobs = []
+      self.exclusive_controllers = []
+      self.exclusive_jobs = []
+      @job_queue_monitoring_enabled = false
+      @enable_db_stats = false
+      @enable_process_stats = false
+      @enable_system_stats = false
+
+      @settings_received_at = nil
+      @last_heartbeat_at = nil
+      @last_heartbeat_attempt_at = nil
+      @settings_mutex = Mutex.new
+    end
+
+    def excluded_controllers=(value)
+      @excluded_controllers = Array(value).map(&:to_s)
+      @compiled_excluded_controllers = compile_patterns(@excluded_controllers)
+    end
+
+    def excluded_jobs=(value)
+      @excluded_jobs = Array(value).map(&:to_s)
+      @compiled_excluded_jobs = compile_patterns(@excluded_jobs)
+    end
+
+    def exclusive_controllers=(value)
+      @exclusive_controllers = Array(value).map(&:to_s)
+      @compiled_exclusive_controllers = compile_patterns(@exclusive_controllers)
+    end
+
+    def exclusive_jobs=(value)
+      @exclusive_jobs = Array(value).map(&:to_s)
+      @compiled_exclusive_jobs = compile_patterns(@exclusive_jobs)
+    end
+
+    # Apply a settings hash received from the backend response.
+    # Only known keys are applied; unknown keys are silently ignored.
+    # Serialized so concurrent HTTP threads do not interleave writes with request-thread reads.
+    def apply_remote_settings(hash)
+      return unless hash.is_a?(Hash)
+
+      @settings_mutex.synchronize do
+        hash.each do |key, value|
+          k = key.to_s
+          next unless REMOTE_SETTING_KEYS.include?(k)
+
+          case k
+          when "sample_rate", "slow_query_threshold_ms", "max_sql_queries_to_send", "max_logs_to_send"
+            send(:"#{k}=", value.to_i)
+          when "enabled", "memory_tracking_enabled", "allocation_tracking_enabled", "explain_analyze_enabled",
+               "job_queue_monitoring_enabled", "enable_db_stats", "enable_process_stats", "enable_system_stats"
+            send(:"#{k}=", !!value)
+          when "excluded_controllers", "excluded_jobs", "exclusive_controllers", "exclusive_jobs"
+            send(:"#{k}=", Array(value).map(&:to_s))
+          end
+        end
+      end
+    end
+
+    def heartbeat_due?
+      return false if api_key.nil?
+      last_heartbeat_attempt_at.nil? || (Time.now.utc - last_heartbeat_attempt_at) >= HEARTBEAT_INTERVAL
     end
 
     def resolve_deploy_id
@@ -45,49 +128,51 @@ module DeadBro
     end
 
     def excluded_controller?(controller_name, action_name = nil)
-      return false if @excluded_controllers.empty?
+      compiled = @compiled_excluded_controllers
+      return false if compiled.nil? || compiled.empty?
 
-      # If action_name is provided, check both controller#action patterns and controller-only patterns
       if action_name
         target = "#{controller_name}##{action_name}"
-        # Check controller#action patterns (patterns containing '#')
-        action_patterns = @excluded_controllers.select { |pat| pat.to_s.include?("#") }
-        if action_patterns.any? { |pat| match_name_or_pattern?(target, pat) }
-          return true
-        end
-        # Check controller-only patterns (patterns without '#')
-        # If the controller itself is excluded, all its actions are excluded
-        controller_patterns = @excluded_controllers.reject { |pat| pat.to_s.include?("#") }
-        if controller_patterns.any? { |pat| match_name_or_pattern?(controller_name, pat) }
-          return true
+        compiled.each do |entry|
+          if entry[:has_hash]
+            return true if match_compiled?(target, entry)
+          elsif match_compiled?(controller_name, entry)
+            return true
+          end
         end
         return false
       end
 
-      # When action_name is nil, only check controller-only patterns (no #)
-      controller_patterns = @excluded_controllers.reject { |pat| pat.to_s.include?("#") }
-      return false if controller_patterns.empty?
-      controller_patterns.any? { |pat| match_name_or_pattern?(controller_name, pat) }
+      compiled.each do |entry|
+        next if entry[:has_hash]
+        return true if match_compiled?(controller_name, entry)
+      end
+      false
     end
 
     def excluded_job?(job_class_name)
-      return false if @excluded_jobs.empty?
-      @excluded_jobs.any? { |pat| match_name_or_pattern?(job_class_name, pat) }
+      compiled = @compiled_excluded_jobs
+      return false if compiled.nil? || compiled.empty?
+      compiled.any? { |entry| match_compiled?(job_class_name, entry) }
     end
 
     def exclusive_job?(job_class_name)
-      return true if @exclusive_jobs.empty? # If not defined, allow all (default behavior)
-      @exclusive_jobs.any? { |pat| match_name_or_pattern?(job_class_name, pat) }
+      compiled = @compiled_exclusive_jobs
+      return true if compiled.nil? || compiled.empty?
+      compiled.any? { |entry| match_compiled?(job_class_name, entry) }
     end
 
     def exclusive_controller?(controller_name, action_name)
-      return true if @exclusive_controllers.empty? # If not defined, allow all (default behavior)
+      compiled = @compiled_exclusive_controllers
+      return true if compiled.nil? || compiled.empty?
       target = "#{controller_name}##{action_name}"
-      @exclusive_controllers.any? { |pat| match_name_or_pattern?(target, pat) }
+      compiled.any? { |entry| match_compiled?(target, entry) }
     end
 
     def should_sample?
       sample_rate = resolve_sample_rate
+      sample_rate = 100 if sample_rate.nil?
+
       return true if sample_rate >= 100
       return false if sample_rate <= 0
 
@@ -95,22 +180,9 @@ module DeadBro
       rand(1..100) <= sample_rate
     end
 
+    # Returns the configured sample_rate only (no ENV fallback). Use DeadBro.configure or remote settings.
     def resolve_sample_rate
-      return @sample_rate unless @sample_rate.nil?
-
-      if ENV["dead_bro_SAMPLE_RATE"]
-        env_value = ENV["dead_bro_SAMPLE_RATE"].to_s.strip
-        # Validate that it's a valid integer string
-        if env_value.match?(/^\d+$/)
-          parsed = env_value.to_i
-          # Ensure it's in valid range (0-100)
-          (parsed >= 0 && parsed <= 100) ? parsed : 100
-        else
-          100 # Invalid format, fall back to default
-        end
-      else
-        100 # default
-      end
+      @sample_rate
     end
 
     def resolve_api_key
@@ -119,34 +191,36 @@ module DeadBro
       ENV["DEAD_BRO_API_KEY"]
     end
 
-    def sample_rate=(value)
-      # Allow nil to use default/resolved value
-      return @sample_rate = nil if value.nil?
+    private
 
-      # Allow 0 to disable sampling, or 1-100 for percentage
-      unless value.is_a?(Integer) && value >= 0 && value <= 100
-        raise ArgumentError, "Sample rate must be an integer between 0 and 100, got: #{value.inspect}"
+    # Turn a list of user-facing patterns into {pattern, has_hash, regex}
+    # entries. Regex is nil when the pattern is a plain literal (cheaper eq
+    # compare). Compiling up-front removes per-request regex allocation.
+    def compile_patterns(patterns)
+      Array(patterns).map do |pat|
+        s = pat.to_s
+        has_hash = s.include?("#")
+        regex = if s.include?("*")
+          if has_hash
+            Regexp.new("\\A" + Regexp.escape(s).gsub("\\*", ".*") + "\\z")
+          else
+            Regexp.new("\\A" + Regexp.escape(s).gsub("\\*", "[^:]*") + "\\z")
+          end
+        end
+        {pattern: s, has_hash: has_hash, regex: regex}
       end
-      @sample_rate = value
+    rescue
+      []
     end
 
-    private
-
-    def match_name_or_pattern?(name, pattern)
-      return false if name.nil? || pattern.nil?
-      pat = pattern.to_s
-      return !!(name.to_s == pat) unless pat.include?("*")
-
-      # For controller action patterns (containing '#'), use .* to match any characters including colons
-      # For controller-only patterns, use [^:]* to match namespace segments
-      regex = if pat.include?("#")
-        # Controller action pattern: allow * to match any characters including colons
-        Regexp.new("^" + Regexp.escape(pat).gsub("\\*", ".*") + "$")
+    def match_compiled?(name, entry)
+      return false if name.nil? || entry.nil?
+      n = name.to_s
+      if entry[:regex]
+        !!(n =~ entry[:regex])
       else
-        # Controller-only pattern: use [^:]* to match namespace segments
-        Regexp.new("^" + Regexp.escape(pat).gsub("\\*", "[^:]*") + "$")
+        n == entry[:pattern]
       end
-      !!(name.to_s =~ regex)
     rescue
       false
     end

data/lib/dead_bro/dispatcher.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "thread"
+
+module DeadBro
+  # Background worker pool that runs HTTP posts for Client off the request
+  # thread. Replaces the previous `Thread.new` per metric. One shared pool per
+  # process; re-initializes after fork (Puma, Unicorn).
+  class Dispatcher
+    DEFAULT_QUEUE_SIZE = 500
+    DEFAULT_WORKERS = 2
+    SHUTDOWN = Object.new
+
+    class << self
+      def instance
+        @instance ||= new
+      end
+
+      # Exposed for tests.
+      def reset!
+        @instance&.shutdown
+        @instance = nil
+      end
+
+      # Test hook — when true, `dispatch` runs the block inline on the caller
+      # thread instead of handing it to a worker. Keeps specs deterministic
+      # without having to stub `Thread.new` or poll for queue drain.
+      attr_accessor :inline
+    end
+
+    def initialize(queue_size: DEFAULT_QUEUE_SIZE, workers: DEFAULT_WORKERS)
+      @queue_size = queue_size
+      @worker_count = workers
+      @mutex = Mutex.new
+      @dropped = 0
+      @shutting_down = false
+      boot_workers(Process.pid)
+      install_at_exit_hook
+    end
+
+    # Schedule a block for background execution. Never blocks the caller: if the
+    # queue is full the job is dropped and `dropped_count` is incremented.
+    def dispatch(&block)
+      return false unless block_given?
+      return false if @shutting_down
+
+      if self.class.inline
+        begin
+          block.call
+        rescue
+          # Match worker semantics — swallow job errors.
+        end
+        return true
+      end
+
+      ensure_workers_alive!
+      @queue.push(block, true) # non-blocking
+      true
+    rescue ThreadError
+      @mutex.synchronize { @dropped += 1 }
+      false
+    end
+
+    def dropped_count
+      @mutex.synchronize { @dropped }
+    end
+
+    def shutdown
+      return if @shutting_down
+      @shutting_down = true
+      workers = @workers || []
+      workers.length.times do
+        begin
+          @queue.push(SHUTDOWN)
+        rescue
+        end
+      end
+      workers.each do |t|
+        begin
+          t.join(2)
+        rescue
+        end
+      end
+    end
+
+    private
+
+    def boot_workers(pid)
+      @pid = pid
+      @queue = SizedQueue.new(@queue_size)
+      @workers = Array.new(@worker_count) do
+        t = Thread.new { run }
+        begin
+          t.name = "dead_bro-dispatcher"
+        rescue
+        end
+        t.abort_on_exception = false
+        t
+      end
+    end
+
+    def ensure_workers_alive!
+      return if @pid == Process.pid && @workers && @workers.all?(&:alive?)
+
+      @mutex.synchronize do
+        return if @pid == Process.pid && @workers && @workers.all?(&:alive?)
+        # Post-fork (new PID) or a worker died — bring up a fresh pool.
+        boot_workers(Process.pid)
+        @shutting_down = false
+      end
+    end
+
+    def install_at_exit_hook
+      at_exit { shutdown }
+    rescue
+    end
+
+    def run
+      loop do
+        job = @queue.pop
+        break if job.equal?(SHUTDOWN)
+        begin
+          job.call
+        rescue
+          # Never let a job crash the worker.
+        end
+      end
+    end
+  end
+end

data/lib/dead_bro/error_middleware.rb

@@ -19,7 +19,7 @@ module DeadBro
         # Use the error class name as the event name
         event_name = exception.class.name.to_s
         event_name = EVENT_NAME if event_name.empty?
-        @client.post_metric(event_name: event_name, payload: payload)
+        @client.post_metric(event_name: event_name, payload: payload, force: true)
       rescue
         # Never let APM reporting interfere with the host app
       end

data/lib/dead_bro/job_subscriber.rb

@@ -17,15 +17,26 @@ module DeadBro
         begin
           job_class_name = data[:job].class.name
           if DeadBro.configuration.excluded_job?(job_class_name)
+            drain_job_tracking
             next
           end
           # If exclusive_jobs is defined and not empty, only track matching jobs
           unless DeadBro.configuration.exclusive_job?(job_class_name)
+            drain_job_tracking
             next
           end
         rescue
         end
 
+        # Skip out via sampling before we build any payload — jobs can be chatty
+        # enough that even the "cheap" stop/analyze work matters under load.
+        # Completions have no exception attached; the exception subscriber below
+        # always sends errors with force: true.
+        unless DeadBro.configuration.should_sample?
+          drain_job_tracking
+          next
+        end
+
         duration_ms = ((finished - started) * 1000.0).round(2)
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
@@ -34,6 +45,11 @@ module DeadBro
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
+            DeadBro::MemoryTrackingSubscriber.start_request_tracking
+          else
+            DeadBro::LightweightMemoryTracker.start_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
+          end
         end
 
         # Get SQL queries executed during this job
@@ -103,13 +119,18 @@ module DeadBro
 
         duration_ms = ((finished - started) * 1000.0).round(2)
         exception = data[:exception_object]
-        job_class = data[:job].class.name
+        data[:job].class.name
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
         unless DeadBro::SqlSubscriber.tracking_active?
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
+            DeadBro::MemoryTrackingSubscriber.start_request_tracking
+          else
+            DeadBro::LightweightMemoryTracker.start_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
+          end
         end
 
         # Get SQL queries executed during this job
@@ -164,12 +185,24 @@ module DeadBro
         }
 
         event_name = exception&.class&.name || "ActiveJob::Exception"
-        client.post_metric(event_name: event_name, payload: payload, error: true)
+        client.post_metric(event_name: event_name, payload: payload, force: true)
       end
     rescue
       # Never raise from instrumentation install
     end
 
+    # Release job-side thread-local tracking state when we've decided not to
+    # build a payload (excluded job / sampled out). Matches Subscriber.drain_request_tracking.
+    def self.drain_job_tracking
+      DeadBro::SqlSubscriber.stop_request_tracking if defined?(DeadBro::SqlSubscriber)
+      DeadBro::LightweightMemoryTracker.stop_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
+      if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
+        DeadBro::MemoryTrackingSubscriber.stop_request_tracking
+      end
+    rescue
+      # Best effort
+    end
+
     private
 
     def self.safe_arguments(arguments)
@@ -215,17 +248,7 @@ module DeadBro
     end
 
     def self.memory_usage_mb
-      if defined?(GC) && GC.respond_to?(:stat)
-        # Get memory usage in MB
-        memory_kb = begin
-          `ps -o rss= -p #{Process.pid}`.to_i
-        rescue
-          0
-        end
-        (memory_kb / 1024.0).round(2)
-      else
-        0
-      end
+      DeadBro::MemoryHelpers.rss_mb
     rescue
       0
     end

data/lib/dead_bro/lightweight_memory_tracker.rb

@@ -43,13 +43,11 @@ module DeadBro
     end
 
     def self.lightweight_memory_usage
-      # Use only GC stats for memory estimation (no system calls)
-      return 0 unless defined?(GC) && GC.respond_to?(:stat)
-
-      gc_stats = GC.stat
-      heap_pages = gc_stats[:heap_allocated_pages] || 0
-      # Rough estimation: 4KB per page
-      (heap_pages * 4) / 1024.0 # Convert to MB
+      # Real RSS, cached for ~1s across threads so this is cheap even on hot
+      # paths. Previous versions multiplied heap_pages by 4KB and labelled the
+      # result as MB — both the unit and the page size were wrong (MRI heap
+      # pages are ~16KB and heap != RSS), so the number was effectively fiction.
+      DeadBro::MemoryHelpers.rss_mb
     rescue
       0
     end

data/lib/dead_bro/logger.rb

@@ -18,8 +18,14 @@ module DeadBro
     COLOR_ERROR = "\033[31m" # Red
     COLOR_FATAL = "\033[35m"  # Magenta
 
+    # Hard cap per-thread buffer size. Prevents unbounded growth when a
+    # request/job logs a lot, or when tracking never gets a chance to flush
+    # (e.g. code running outside a request lifecycle).
+    MAX_LOG_ENTRIES = 500
+
     def initialize
       @thread_logs_key = :dead_bro_logs
+      @thread_logs_dropped_key = :dead_bro_logs_dropped
     end
 
     def debug(message)
@@ -42,29 +48,42 @@ module DeadBro
       log(:fatal, message)
     end
 
-    # Get all logs for the current thread
+    # Get all logs for the current thread. If the buffer was capped, append a
+    # synthetic marker entry so downstream consumers know entries were dropped.
     def logs
-      Thread.current[@thread_logs_key] || []
+      entries = Thread.current[@thread_logs_key] || []
+      dropped = Thread.current[@thread_logs_dropped_key] || 0
+      return entries if dropped.zero?
+
+      entries + [{
+        sev: "warn",
+        msg: "[DeadBro::Logger] #{dropped} log entries dropped (buffer cap #{MAX_LOG_ENTRIES})",
+        time: Time.now.utc.iso8601(3)
+      }]
     end
 
     # Clear logs for the current thread
     def clear
       Thread.current[@thread_logs_key] = []
+      Thread.current[@thread_logs_dropped_key] = 0
     end
 
     private
 
     def log(severity, message)
       timestamp = Time.now.utc
-      log_entry = {
-        sev: severity.to_s,
-        msg: message.to_s,
-        time: timestamp.iso8601(3) # Include milliseconds for better precision
-      }
-
-      # Store in thread-local storage
-      Thread.current[@thread_logs_key] ||= []
-      Thread.current[@thread_logs_key] << log_entry
+
+      buffer = (Thread.current[@thread_logs_key] ||= [])
+      if buffer.length >= MAX_LOG_ENTRIES
+        Thread.current[@thread_logs_dropped_key] =
+          (Thread.current[@thread_logs_dropped_key] || 0) + 1
+      else
+        buffer << {
+          sev: severity.to_s,
+          msg: message.to_s,
+          time: timestamp.iso8601(3) # Include milliseconds for better precision
+        }
+      end
 
       # Print the message immediately
       print_log(severity, message, timestamp)