dead_bro 0.2.8 → 0.2.10

data/lib/dead_bro/elasticsearch_subscriber.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module DeadBro
+  class ElasticsearchSubscriber
+    THREAD_LOCAL_KEY = :dead_bro_elasticsearch_events
+    MAX_TRACKED_EVENTS = 500
+
+    # Install gem-based notification subscriber (request.elasticsearch / request.elastic_transport).
+    # The Net::HTTP path is handled by HttpInstrumentation, which calls .record directly.
+    def self.subscribe!
+      install_notifications_subscription!
+    rescue
+    end
+
+    # Called by HttpInstrumentation when it detects a Net::HTTP request to an ES host.
+    def self.record(method:, path:, status:, duration_ms:)
+      events = Thread.current[THREAD_LOCAL_KEY]
+      return unless events
+      return unless should_continue_tracking?
+
+      events << build_event(method, path, status, duration_ms)
+    rescue
+    end
+
+    def self.start_request_tracking
+      Thread.current[THREAD_LOCAL_KEY] = []
+    end
+
+    def self.stop_request_tracking
+      events = Thread.current[THREAD_LOCAL_KEY]
+      Thread.current[THREAD_LOCAL_KEY] = nil
+      events || []
+    end
+
+    def self.should_continue_tracking?
+      events = Thread.current[THREAD_LOCAL_KEY]
+      return false unless events
+      return false if events.length >= MAX_TRACKED_EVENTS
+
+      start_time = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
+      if start_time
+        elapsed_seconds = Time.now - start_time
+        return false if elapsed_seconds >= DeadBro::MAX_TRACKING_DURATION_SECONDS
+      end
+
+      true
+    end
+
+    def self.extract_operation(method, path)
+      return "unknown" if path.nil?
+
+      clean = path.to_s.split("?").first.to_s
+      m = method.to_s.upcase
+
+      if clean =~ /_search\z/i
+        "search"
+      elsif clean =~ /_msearch\z/i
+        "msearch"
+      elsif clean =~ /_bulk\z/i
+        "bulk"
+      elsif clean =~ /_doc\/[^\/]+\/_update\z/i
+        "update"
+      elsif clean =~ /_update\/[^\/]+\z/i
+        "update"
+      elsif clean =~ /_delete_by_query\z/i
+        "delete_by_query"
+      elsif clean =~ /_count\z/i
+        "count"
+      elsif clean =~ /_mapping\z/i
+        m == "GET" ? "get_mapping" : "put_mapping"
+      elsif clean =~ /_doc\/[^\/]+\z/i
+        case m
+        when "GET"        then "get"
+        when "DELETE"     then "delete"
+        when "POST", "PUT" then "index"
+        else "doc"
+        end
+      elsif clean =~ /_doc\z/i
+        "index"
+      elsif clean =~ /\A\/_cluster\//i
+        "cluster"
+      elsif clean =~ /\A\/_cat\//i
+        "cat"
+      elsif clean =~ /\A\/[^\/]+\z/
+        case m
+        when "PUT"    then "create_index"
+        when "DELETE" then "delete_index"
+        when "HEAD"   then "index_exists"
+        when "GET"    then "get_index"
+        else "index_op"
+        end
+      else
+        m.downcase
+      end
+    rescue
+      "unknown"
+    end
+
+    def self.sanitize_path(path)
+      return "" if path.nil?
+      path.to_s
+        .gsub(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i, "/{id}")
+        .gsub(/\/\d+(?=\/|\z)/, "/{id}")
+    rescue
+      path.to_s
+    end
+
+    class << self
+      private
+
+      def install_notifications_subscription!
+        return unless defined?(::ActiveSupport::Notifications)
+
+        %w[request.elasticsearch request.elastic_transport].each do |event_name|
+          ::ActiveSupport::Notifications.subscribe(event_name) do |_name, started, finished, _id, payload|
+            events = Thread.current[THREAD_LOCAL_KEY]
+            next unless events
+            next unless should_continue_tracking?
+
+            duration_ms = ((finished - started) * 1000.0).round(2)
+            method = payload[:method].to_s.upcase
+            path = payload[:path].to_s
+            events << build_event(method, path, payload[:status], duration_ms)
+          rescue
+          end
+        end
+      rescue
+      end
+
+      def build_event(method, path, status, duration_ms)
+        {
+          method: method.to_s.upcase,
+          path: sanitize_path(path),
+          operation: extract_operation(method, path),
+          status: status,
+          duration_ms: duration_ms
+        }
+      end
+    end
+  end
+end

data/lib/dead_bro/error_middleware.rb

@@ -19,7 +19,7 @@ module DeadBro
         # Use the error class name as the event name
         event_name = exception.class.name.to_s
         event_name = EVENT_NAME if event_name.empty?
-        @client.post_metric(event_name: event_name, payload: payload)
+        @client.post_metric(event_name: event_name, payload: payload, force: true)
       rescue
         # Never let APM reporting interfere with the host app
       end

data/lib/dead_bro/http_instrumentation.rb

@@ -12,6 +12,7 @@ module DeadBro
     def self.install!(client: Client.new)
       install_net_http!(client)
       install_typhoeus!(client) if defined?(::Typhoeus)
+      install_faraday!(client) if defined?(::Faraday)
     rescue
       # Never raise from instrumentation install
     end
@@ -38,23 +39,37 @@ module DeadBro
                 nil
               end
 
-              # Skip instrumentation for our own APM endpoint to prevent infinite loops,
-              # but do NOT alter the original method's return value/control flow.
-              skip_instrumentation = uri && (uri.to_s.include?("localhost") || uri.to_s.include?("aberatii.com"))
+              host = (uri && uri.host) || @address
+              port = (uri && uri.port) || @port
+              is_es_host = DeadBro::HttpInstrumentation.elasticsearch_host?(host, port)
+              # Skip localhost/internal only for non-ES hosts. ES on localhost (e.g. port 9200)
+              # must still be tracked; only skip the deadbro backend itself.
+              skip_instrumentation = !is_es_host && uri && (uri.to_s.include?("localhost") || uri.to_s.include?("aberatii.com"))
 
-              unless skip_instrumentation
+              if is_es_host
+                # Route to elasticsearch subscriber instead of http_outgoing
+                if Thread.current[DeadBro::ElasticsearchSubscriber::THREAD_LOCAL_KEY]
+                  path = (uri && uri.path) || req.path
+                  DeadBro::ElasticsearchSubscriber.record(
+                    method: req.method,
+                    path: path,
+                    status: response && response.code.to_i,
+                    duration_ms: duration_ms
+                  )
+                end
+              elsif !skip_instrumentation
+                lib = DeadBro::HttpInstrumentation.typesense_host?(host, port) ? "typesense" : "net_http"
                 payload = {
-                  library: "net_http",
+                  library: lib,
                   method: req.method,
                   url: uri && uri.to_s,
-                  host: (uri && uri.host) || @address,
+                  host: host,
                   path: (uri && uri.path) || req.path,
                   status: response && response.code.to_i,
                   duration_ms: duration_ms,
                   exception: error && error.class.name
                 }
-                # Accumulate per-request; only send with controller metric
-                if Thread.current[THREAD_LOCAL_KEY] && should_continue_tracking?
+                if Thread.current[THREAD_LOCAL_KEY] && DeadBro::HttpInstrumentation.should_continue_tracking?
                   Thread.current[THREAD_LOCAL_KEY] << payload
                 end
               end
@@ -85,8 +100,6 @@ module DeadBro
                 (respond_to?(:base_url) ? base_url : nil)
               end
 
-              # Skip instrumentation for our own APM endpoint to prevent infinite loops,
-              # but do NOT alter the original method's return value/control flow.
               skip_instrumentation = req_url && (req_url.include?("localhost:3100/apm/v1/metrics") || req_url.include?("deadbro.aberatii.com/apm/v1/metrics"))
 
               unless skip_instrumentation
@@ -97,8 +110,7 @@ module DeadBro
                   status: response && response.code,
                   duration_ms: duration_ms
                 }
-                # Accumulate per-request; only send with controller metric
-                if Thread.current[THREAD_LOCAL_KEY] && should_continue_tracking?
+                if Thread.current[THREAD_LOCAL_KEY] && DeadBro::HttpInstrumentation.should_continue_tracking?
                   Thread.current[THREAD_LOCAL_KEY] << payload
                 end
               end
@@ -112,15 +124,96 @@ module DeadBro
     rescue
     end
 
-    # Check if we should continue tracking based on count and time limits
+    def self.install_faraday!(client)
+      return unless defined?(::Faraday)
+
+      unless defined?(::DeadBro::FaradayMiddleware)
+        middleware_klass = Class.new(::Faraday::Middleware) do
+          def call(env)
+            start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            response = nil
+            begin
+              response = @app.call(env)
+            ensure
+              finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              duration_ms = ((finish_time - start_time) * 1000.0).round(2)
+              begin
+                url = env.url
+                host = url.host.to_s
+                port = url.port
+                url_str = url.to_s
+
+                is_es_host = DeadBro::HttpInstrumentation.elasticsearch_host?(host, port)
+                skip = !is_es_host && (url_str.include?("localhost") || url_str.include?("aberatii.com"))
+
+                if is_es_host
+                  if Thread.current[DeadBro::ElasticsearchSubscriber::THREAD_LOCAL_KEY]
+                    DeadBro::ElasticsearchSubscriber.record(
+                      method: env.method.to_s.upcase,
+                      path: url.path,
+                      status: response&.status,
+                      duration_ms: duration_ms
+                    )
+                  end
+                elsif !skip
+                  lib = DeadBro::HttpInstrumentation.typesense_host?(host, port) ? "typesense" : "faraday"
+                  payload = {
+                    library: lib,
+                    method: env.method.to_s.upcase,
+                    url: url_str,
+                    host: host,
+                    path: url.path,
+                    status: response&.status,
+                    duration_ms: duration_ms
+                  }
+                  key = DeadBro::HttpInstrumentation::THREAD_LOCAL_KEY
+                  if Thread.current[key] && DeadBro::HttpInstrumentation.should_continue_tracking?
+                    Thread.current[key] << payload
+                  end
+                end
+              rescue
+              end
+            end
+          end
+        end
+        ::DeadBro.const_set(:FaradayMiddleware, middleware_klass)
+      end
+
+      unless defined?(::DeadBro::FaradayInstrumentation)
+        instrumentation_mod = Module.new do
+          define_method(:initialize) do |url = nil, options = {}, &block|
+            super(url, options, &block)
+            unless builder.handlers.map(&:klass).include?(::DeadBro::FaradayMiddleware)
+              builder.use(::DeadBro::FaradayMiddleware)
+            end
+          rescue
+          end
+        end
+        ::DeadBro.const_set(:FaradayInstrumentation, instrumentation_mod)
+      end
+
+      ::Faraday::Connection.prepend(::DeadBro::FaradayInstrumentation) unless ::Faraday::Connection.ancestors.include?(::DeadBro::FaradayInstrumentation)
+    rescue
+    end
+
+    def self.elasticsearch_host?(host, port)
+      return false if host.nil?
+      return true if port == 9200
+      h = host.to_s
+      h.end_with?(".elastic.co") || h.end_with?(".es.amazonaws.com") || h.include?("elasticsearch")
+    end
+
+    def self.typesense_host?(host, port)
+      return false if host.nil?
+      port == 8108 || host.to_s.end_with?(".typesense.io")
+    end
+
     def self.should_continue_tracking?
       events = Thread.current[THREAD_LOCAL_KEY]
       return false unless events
 
-      # Check count limit
       return false if events.length >= MAX_TRACKED_EVENTS
 
-      # Check time limit
       start_time = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
       if start_time
         elapsed_seconds = Time.now - start_time

data/lib/dead_bro/job_subscriber.rb

@@ -17,15 +17,26 @@ module DeadBro
         begin
           job_class_name = data[:job].class.name
           if DeadBro.configuration.excluded_job?(job_class_name)
+            drain_job_tracking
             next
           end
           # If exclusive_jobs is defined and not empty, only track matching jobs
           unless DeadBro.configuration.exclusive_job?(job_class_name)
+            drain_job_tracking
             next
           end
         rescue
         end
 
+        # Skip out via sampling before we build any payload — jobs can be chatty
+        # enough that even the "cheap" stop/analyze work matters under load.
+        # Completions have no exception attached; the exception subscriber below
+        # always sends errors with force: true.
+        unless DeadBro.configuration.should_sample?
+          drain_job_tracking
+          next
+        end
+
         duration_ms = ((finished - started) * 1000.0).round(2)
 
         # Ensure tracking was started (fallback if perform_start.active_job didn't fire)
@@ -34,6 +45,11 @@ module DeadBro
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
+            DeadBro::MemoryTrackingSubscriber.start_request_tracking
+          else
+            DeadBro::LightweightMemoryTracker.start_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
+          end
         end
 
         # Get SQL queries executed during this job
@@ -110,6 +126,11 @@ module DeadBro
           DeadBro.logger.clear
           Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
           DeadBro::SqlSubscriber.start_request_tracking
+          if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
+            DeadBro::MemoryTrackingSubscriber.start_request_tracking
+          else
+            DeadBro::LightweightMemoryTracker.start_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
+          end
         end
 
         # Get SQL queries executed during this job
@@ -164,12 +185,24 @@ module DeadBro
         }
 
         event_name = exception&.class&.name || "ActiveJob::Exception"
-        client.post_metric(event_name: event_name, payload: payload, error: true)
+        client.post_metric(event_name: event_name, payload: payload, force: true)
       end
     rescue
       # Never raise from instrumentation install
     end
 
+    # Release job-side thread-local tracking state when we've decided not to
+    # build a payload (excluded job / sampled out). Matches Subscriber.drain_request_tracking.
+    def self.drain_job_tracking
+      DeadBro::SqlSubscriber.stop_request_tracking if defined?(DeadBro::SqlSubscriber)
+      DeadBro::LightweightMemoryTracker.stop_request_tracking if defined?(DeadBro::LightweightMemoryTracker)
+      if DeadBro.configuration.allocation_tracking_enabled && defined?(DeadBro::MemoryTrackingSubscriber)
+        DeadBro::MemoryTrackingSubscriber.stop_request_tracking
+      end
+    rescue
+      # Best effort
+    end
+
     private
 
     def self.safe_arguments(arguments)
@@ -215,17 +248,7 @@ module DeadBro
     end
 
     def self.memory_usage_mb
-      if defined?(GC) && GC.respond_to?(:stat)
-        # Get memory usage in MB
-        memory_kb = begin
-          `ps -o rss= -p #{Process.pid}`.to_i
-        rescue
-          0
-        end
-        (memory_kb / 1024.0).round(2)
-      else
-        0
-      end
+      DeadBro::MemoryHelpers.rss_mb
     rescue
       0
     end

data/lib/dead_bro/lightweight_memory_tracker.rb

@@ -43,13 +43,11 @@ module DeadBro
     end
 
     def self.lightweight_memory_usage
-      # Use only GC stats for memory estimation (no system calls)
-      return 0 unless defined?(GC) && GC.respond_to?(:stat)
-
-      gc_stats = GC.stat
-      heap_pages = gc_stats[:heap_allocated_pages] || 0
-      # Rough estimation: 4KB per page
-      (heap_pages * 4) / 1024.0 # Convert to MB
+      # Real RSS, cached for ~1s across threads so this is cheap even on hot
+      # paths. Previous versions multiplied heap_pages by 4KB and labelled the
+      # result as MB — both the unit and the page size were wrong (MRI heap
+      # pages are ~16KB and heap != RSS), so the number was effectively fiction.
+      DeadBro::MemoryHelpers.rss_mb
     rescue
       0
     end

data/lib/dead_bro/logger.rb

@@ -18,8 +18,14 @@ module DeadBro
     COLOR_ERROR = "\033[31m" # Red
     COLOR_FATAL = "\033[35m"  # Magenta
 
+    # Hard cap per-thread buffer size. Prevents unbounded growth when a
+    # request/job logs a lot, or when tracking never gets a chance to flush
+    # (e.g. code running outside a request lifecycle).
+    MAX_LOG_ENTRIES = 500
+
     def initialize
       @thread_logs_key = :dead_bro_logs
+      @thread_logs_dropped_key = :dead_bro_logs_dropped
     end
 
     def debug(message)
@@ -42,29 +48,42 @@ module DeadBro
       log(:fatal, message)
     end
 
-    # Get all logs for the current thread
+    # Get all logs for the current thread. If the buffer was capped, append a
+    # synthetic marker entry so downstream consumers know entries were dropped.
     def logs
-      Thread.current[@thread_logs_key] || []
+      entries = Thread.current[@thread_logs_key] || []
+      dropped = Thread.current[@thread_logs_dropped_key] || 0
+      return entries if dropped.zero?
+
+      entries + [{
+        sev: "warn",
+        msg: "[DeadBro::Logger] #{dropped} log entries dropped (buffer cap #{MAX_LOG_ENTRIES})",
+        time: Time.now.utc.iso8601(3)
+      }]
     end
 
     # Clear logs for the current thread
     def clear
       Thread.current[@thread_logs_key] = []
+      Thread.current[@thread_logs_dropped_key] = 0
     end
 
     private
 
     def log(severity, message)
       timestamp = Time.now.utc
-      log_entry = {
-        sev: severity.to_s,
-        msg: message.to_s,
-        time: timestamp.iso8601(3) # Include milliseconds for better precision
-      }
-
-      # Store in thread-local storage
-      Thread.current[@thread_logs_key] ||= []
-      Thread.current[@thread_logs_key] << log_entry
+
+      buffer = (Thread.current[@thread_logs_key] ||= [])
+      if buffer.length >= MAX_LOG_ENTRIES
+        Thread.current[@thread_logs_dropped_key] =
+          (Thread.current[@thread_logs_dropped_key] || 0) + 1
+      else
+        buffer << {
+          sev: severity.to_s,
+          msg: message.to_s,
+          time: timestamp.iso8601(3) # Include milliseconds for better precision
+        }
+      end
 
       # Print the message immediately
       print_log(severity, message, timestamp)

data/lib/dead_bro/memory_details.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module DeadBro
+  module MemoryDetails
+    # Maps Ruby internal ObjectSpace type codes to human-readable names.
+    # Types omitted here are filtered out (internal noise).
+    OBJECT_TYPE_NAMES = {
+      T_STRING: "String",
+      T_ARRAY: "Array",
+      T_HASH: "Hash",
+      T_OBJECT: "Object",
+      T_DATA: "C Extension",
+      T_CLASS: "Class",
+      T_MODULE: "Module",
+      T_STRUCT: "Struct",
+      T_MATCH: "MatchData",
+      T_REGEXP: "Regexp",
+      T_SYMBOL: "Symbol",
+      T_FLOAT: "Float",
+      T_FILE: "File",
+      T_BIGNUM: "Integer (big)"
+    }.freeze
+
+    # Noise types never shown to users.
+    SKIP_TYPES = %i[FREE T_IMEMO TOTAL T_NODE T_ICLASS T_ZOMBIE T_MOVED].freeze
+
+    def self.format_object_breakdown(deltas)
+      result = {}
+      deltas.each do |type, count|
+        next if SKIP_TYPES.include?(type)
+        next unless count.positive?
+        name = OBJECT_TYPE_NAMES[type] || type.to_s.sub(/\AT_/, "")
+        result[name] = count
+      end
+      result.sort_by { |_, v| -v }.to_h
+    end
+
+    def self.build(gc_before:, gc_after:, memory_before_mb:, memory_after_mb:,
+                   object_counts_before:, object_counts_after:, large_objects:)
+      memory_delta_mb = (memory_after_mb - memory_before_mb).round(2)
+      gc_collections = (gc_after[:count] || 0) - (gc_before[:count] || 0)
+      heap_pages_added = (gc_after[:heap_allocated_pages] || 0) - (gc_before[:heap_allocated_pages] || 0)
+      new_objects = (gc_after[:total_allocated_objects] || 0) - (gc_before[:total_allocated_objects] || 0)
+
+      raw_deltas = {}
+      if object_counts_before.any? && object_counts_after.any?
+        keys = (object_counts_before.keys + object_counts_after.keys).uniq
+        keys.each do |k|
+          diff = (object_counts_after[k] || 0) - (object_counts_before[k] || 0)
+          raw_deltas[k] = diff unless diff.zero?
+        end
+        raw_deltas = raw_deltas.sort_by { |_, v| -v.abs }.first(20).to_h
+      end
+
+      warnings = []
+      warnings << "Memory grew #{memory_delta_mb}MB — possible leak or large allocation" if memory_delta_mb > 20
+      warnings << "GC ran #{gc_collections} times — many short-lived objects being created" if gc_collections > 5
+      warnings << "Heap grew by #{heap_pages_added} pages — Ruby needed more memory from the OS" if heap_pages_added > 10
+      warnings << "#{large_objects.length} object(s) over 1MB found in memory" if large_objects.any?
+
+      {
+        gc_collections: gc_collections,
+        heap_pages_added: heap_pages_added,
+        new_objects: new_objects,
+        object_breakdown: format_object_breakdown(raw_deltas),
+        large_objects: large_objects,
+        warnings: warnings
+      }
+    end
+  end
+end

data/lib/dead_bro/memory_helpers.rb

@@ -4,6 +4,68 @@ module DeadBro
   module MemoryHelpers
     # Helper methods for memory tracking and leak detection
 
+    RSS_CACHE_TTL_SECONDS = 1.0
+    @rss_cache_mutex = Mutex.new
+    @rss_cache = nil # [value_bytes, captured_at_monotonic]
+
+    # Current process RSS in bytes. Uses /proc/self/status on Linux (cheap read)
+    # and falls back to `ps` elsewhere. Result is cached for 1s across threads
+    # so this is safe to call from every request without flooding the kernel.
+    def self.rss_bytes
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      cached = @rss_cache
+      if cached && (now - cached[1]) < RSS_CACHE_TTL_SECONDS
+        return cached[0]
+      end
+
+      value = read_rss_bytes
+      @rss_cache_mutex.synchronize do
+        # Re-check inside the lock to avoid racing a newer reading.
+        cached = @rss_cache
+        if cached.nil? || (now - cached[1]) >= RSS_CACHE_TTL_SECONDS
+          @rss_cache = [value, now]
+        end
+      end
+      value
+    rescue
+      0
+    end
+
+    def self.rss_mb
+      (rss_bytes.to_f / (1024 * 1024)).round(2)
+    rescue
+      0.0
+    end
+
+    def self.read_rss_bytes
+      if File.readable?("/proc/self/status")
+        read_rss_from_proc_status
+      else
+        read_rss_from_ps
+      end
+    rescue
+      0
+    end
+
+    def self.read_rss_from_proc_status
+      File.foreach("/proc/self/status") do |line|
+        next unless line.start_with?("VmRSS:")
+        kb = line.split[1].to_i
+        return kb * 1024 if kb > 0
+      end
+      0
+    rescue
+      0
+    end
+
+    def self.read_rss_from_ps
+      kb = `ps -o rss= -p #{Process.pid}`.to_i
+      return 0 if kb <= 0
+      kb * 1024
+    rescue
+      0
+    end
+
     # Take a memory snapshot with a custom label
     def self.snapshot(label)
       DeadBro::MemoryTrackingSubscriber.take_memory_snapshot(label)