dead_bro 0.2.0

data/lib/dead_bro/memory_leak_detector.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module DeadBro
+  class MemoryLeakDetector
+    # Track memory patterns over time to detect leaks
+    MEMORY_HISTORY_KEY = :dead_bro_memory_history
+    LEAK_DETECTION_WINDOW = 300 # 5 minutes
+    MEMORY_GROWTH_THRESHOLD = 50 # 50MB growth threshold
+    MIN_SAMPLES_FOR_LEAK_DETECTION = 10
+
+    def self.initialize_history
+      Thread.current[MEMORY_HISTORY_KEY] = {
+        samples: [],
+        last_cleanup: Time.now.utc.to_i,
+        leak_alerts: []
+      }
+    end
+
+    def self.record_memory_sample(sample_data)
+      history = Thread.current[MEMORY_HISTORY_KEY] || initialize_history
+
+      sample = {
+        timestamp: Time.now.utc.to_i,
+        memory_usage: sample_data[:memory_usage] || 0,
+        gc_count: sample_data[:gc_count] || 0,
+        heap_pages: sample_data[:heap_pages] || 0,
+        object_count: sample_data[:object_count] || 0,
+        request_id: sample_data[:request_id],
+        controller: sample_data[:controller],
+        action: sample_data[:action]
+      }
+
+      history[:samples] << sample
+
+      # Clean up old samples
+      cleanup_old_samples(history)
+
+      # Check for memory leaks
+      check_for_memory_leaks(history)
+
+      history
+    end
+
+    def self.cleanup_old_samples(history)
+      cutoff_time = Time.now.utc.to_i - LEAK_DETECTION_WINDOW
+      history[:samples] = history[:samples].select { |sample| sample[:timestamp] > cutoff_time }
+    end
+
+    def self.check_for_memory_leaks(history)
+      samples = history[:samples]
+      return if samples.length < MIN_SAMPLES_FOR_LEAK_DETECTION
+
+      # Calculate memory growth trend
+      memory_values = samples.map { |s| s[:memory_usage] }
+      timestamps = samples.map { |s| s[:timestamp] }
+
+      # Use linear regression to detect upward trend
+      trend = calculate_memory_trend(memory_values, timestamps)
+
+      # Check if memory is growing consistently
+      if trend[:slope] > 0.1 && trend[:r_squared] > 0.7 # Growing with good correlation
+        memory_growth = memory_values.last - memory_values.first
+
+        if memory_growth > MEMORY_GROWTH_THRESHOLD
+          leak_alert = {
+            detected_at: Time.now.utc.to_i,
+            memory_growth_mb: memory_growth.round(2),
+            growth_rate_mb_per_second: trend[:slope],
+            confidence: trend[:r_squared],
+            sample_count: samples.length,
+            time_window_seconds: timestamps.last - timestamps.first,
+            recent_controllers: samples.last(5).map { |s| "#{s[:controller]}##{s[:action]}" }.uniq
+          }
+
+          history[:leak_alerts] << leak_alert
+
+          # Only keep recent leak alerts
+          history[:leak_alerts] = history[:leak_alerts].last(10)
+        end
+      end
+    end
+
+    def self.calculate_memory_trend(memory_values, timestamps)
+      return {slope: 0, r_squared: 0} if memory_values.length < 2
+
+      n = memory_values.length
+      sum_x = timestamps.sum
+      sum_y = memory_values.sum
+      sum_xy = timestamps.zip(memory_values).sum { |x, y| x * y }
+      sum_x2 = timestamps.sum { |x| x * x }
+      memory_values.sum { |y| y * y }
+
+      # Calculate slope (m) and intercept (b) for y = mx + b
+      slope = (n * sum_xy - sum_x * sum_y).to_f / (n * sum_x2 - sum_x * sum_x)
+      intercept = (sum_y - slope * sum_x).to_f / n
+
+      # Calculate R-squared (coefficient of determination)
+      y_mean = sum_y.to_f / n
+      ss_tot = memory_values.sum { |y| (y - y_mean)**2 }
+      ss_res = memory_values.zip(timestamps).sum { |y, x| (y - (slope * x + intercept))**2 }
+      r_squared = (ss_tot > 0) ? 1 - (ss_res / ss_tot) : 0
+
+      {
+        slope: slope,
+        intercept: intercept,
+        r_squared: r_squared
+      }
+    end
+
+    def self.get_memory_analysis
+      history = Thread.current[MEMORY_HISTORY_KEY] || initialize_history
+      samples = history[:samples]
+
+      return {status: "insufficient_data", sample_count: samples.length} if samples.length < 5
+
+      memory_values = samples.map { |s| s[:memory_usage] }
+      gc_counts = samples.map { |s| s[:gc_count] }
+      object_counts = samples.map { |s| s[:object_count] }
+
+      # Calculate basic statistics
+      memory_stats = calculate_stats(memory_values)
+      gc_stats = calculate_stats(gc_counts)
+      object_stats = calculate_stats(object_counts)
+
+      # Detect patterns
+      memory_trend = calculate_memory_trend(memory_values, samples.map { |s| s[:timestamp] })
+
+      # Analyze recent activity
+      recent_samples = samples.last(10)
+      recent_controllers = recent_samples.map { |s| "#{s[:controller]}##{s[:action]}" }.tally
+
+      {
+        status: "analyzed",
+        sample_count: samples.length,
+        time_window_seconds: samples.last[:timestamp] - samples.first[:timestamp],
+        memory_stats: memory_stats,
+        gc_stats: gc_stats,
+        object_stats: object_stats,
+        memory_trend: memory_trend,
+        recent_controllers: recent_controllers,
+        leak_alerts: history[:leak_alerts].last(5),
+        memory_efficiency: calculate_memory_efficiency(samples)
+      }
+    end
+
+    def self.calculate_stats(values)
+      return {} if values.empty?
+
+      {
+        min: values.min,
+        max: values.max,
+        mean: (values.sum.to_f / values.length).round(2),
+        median: values.sort[values.length / 2],
+        std_dev: calculate_standard_deviation(values)
+      }
+    end
+
+    def self.calculate_standard_deviation(values)
+      return 0 if values.length < 2
+
+      mean = values.sum.to_f / values.length
+      variance = values.sum { |v| (v - mean)**2 } / (values.length - 1)
+      Math.sqrt(variance).round(2)
+    end
+
+    def self.calculate_memory_efficiency(samples)
+      return {} if samples.length < 2
+
+      # Calculate memory per object ratio
+      memory_per_object = samples.map do |sample|
+        (sample[:object_count] > 0) ? sample[:memory_usage] / sample[:object_count] : 0
+      end
+
+      # Calculate GC efficiency (objects collected per GC cycle)
+      gc_efficiency = []
+      (1...samples.length).each do |i|
+        gc_delta = samples[i][:gc_count] - samples[i - 1][:gc_count]
+        memory_delta = samples[i][:memory_usage] - samples[i - 1][:memory_usage]
+
+        if gc_delta > 0 && memory_delta < 0
+          gc_efficiency << (-memory_delta / gc_delta).round(2)
+        end
+      end
+
+      {
+        average_memory_per_object_kb: (memory_per_object.sum / memory_per_object.length).round(2),
+        gc_efficiency_mb_per_cycle: gc_efficiency.any? ? (gc_efficiency.sum / gc_efficiency.length).round(2) : 0,
+        memory_volatility: calculate_standard_deviation(samples.map { |s| s[:memory_usage] })
+      }
+    end
+
+    def self.clear_history
+      Thread.current[MEMORY_HISTORY_KEY] = nil
+    end
+  end
+end

data/lib/dead_bro/memory_tracking_subscriber.rb

@@ -0,0 +1,361 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+
+module DeadBro
+  class MemoryTrackingSubscriber
+    # Object allocation events
+    ALLOCATION_EVENT = "object_allocations.active_support"
+
+    THREAD_LOCAL_KEY = :dead_bro_memory_events
+    # Consider objects larger than this many bytes as "large"
+    LARGE_OBJECT_THRESHOLD = 1_000_000 # 1MB threshold for large objects
+
+    # Performance optimization settings
+    ALLOCATION_SAMPLING_RATE = 1 # Track all when enabled (adjust in production)
+    MAX_ALLOCATIONS_PER_REQUEST = 1000 # Limit allocations tracked per request
+    LARGE_OBJECT_SAMPLE_RATE = 0.01 # Sample 1% of live objects to estimate large ones
+    MAX_LARGE_OBJECTS = 50 # Cap number of large objects captured per request
+
+    def self.subscribe!(client: Client.new)
+      # Only enable allocation tracking if explicitly enabled (expensive!)
+      return unless DeadBro.configuration.allocation_tracking_enabled
+      if defined?(ActiveSupport::Notifications) && ActiveSupport::Notifications.notifier.respond_to?(:subscribe)
+        begin
+          # Subscribe to object allocation events with sampling
+          ActiveSupport::Notifications.subscribe(ALLOCATION_EVENT) do |name, started, finished, _unique_id, data|
+            # Sample allocations to reduce overhead
+            next unless rand < ALLOCATION_SAMPLING_RATE
+            track_allocation(data, started, finished)
+          end
+        rescue
+          # Allocation tracking might not be available in all Ruby versions
+        end
+      end
+    rescue
+      # Never raise from instrumentation install
+    end
+
+    def self.start_request_tracking
+      # Only track if memory tracking is enabled
+      return unless DeadBro.configuration.memory_tracking_enabled
+
+      Thread.current[THREAD_LOCAL_KEY] = {
+        allocations: [],
+        memory_snapshots: [],
+        large_objects: [],
+        gc_before: gc_stats,
+        memory_before: memory_usage_mb,
+        start_time: Time.now.utc.to_i,
+        object_counts_before: count_objects_snapshot
+      }
+    end
+
+    def self.stop_request_tracking
+      events = Thread.current[THREAD_LOCAL_KEY]
+      Thread.current[THREAD_LOCAL_KEY] = nil
+
+      if events
+        events[:gc_after] = gc_stats
+        events[:memory_after] = memory_usage_mb
+        events[:end_time] = Time.now.utc.to_i
+        events[:duration_seconds] = events[:end_time] - events[:start_time]
+        events[:object_counts_after] = count_objects_snapshot
+
+        # Fallback large object detection via ObjectSpace sampling
+        if (events[:large_objects].nil? || events[:large_objects].empty?) && object_space_available?
+          events[:large_objects] = sample_large_objects
+        end
+      end
+
+      events || {}
+    end
+
+    def self.track_allocation(data, started, finished)
+      return unless Thread.current[THREAD_LOCAL_KEY]
+
+      # Only track if we have meaningful allocation data
+      return unless data.is_a?(Hash) && data[:count] && data[:size]
+
+      # Limit allocations per request to prevent memory bloat
+      allocations = Thread.current[THREAD_LOCAL_KEY][:allocations]
+      return if allocations.length >= MAX_ALLOCATIONS_PER_REQUEST
+
+      # Simplified allocation tracking (avoid expensive operations)
+      allocation = {
+        class_name: data[:class_name] || "Unknown",
+        count: data[:count],
+        size: data[:size]
+        # Removed expensive fields: duration_ms, timestamp, memory_usage
+      }
+
+      # Track large object allocations (these are rare and important)
+      if data[:size] > LARGE_OBJECT_THRESHOLD
+        large_object = allocation.merge(
+          large_object: true,
+          size_mb: (data[:size] / 1_000_000.0).round(2)
+        )
+        Thread.current[THREAD_LOCAL_KEY][:large_objects] << large_object
+      end
+
+      Thread.current[THREAD_LOCAL_KEY][:allocations] << allocation
+    end
+
+    def self.take_memory_snapshot(label = nil)
+      return unless Thread.current[THREAD_LOCAL_KEY]
+
+      snapshot = {
+        label: label || "snapshot_#{Time.now.to_i}",
+        memory_usage: memory_usage_mb,
+        gc_stats: gc_stats,
+        timestamp: Time.now.utc.to_i,
+        object_count: object_count,
+        heap_pages: heap_pages
+      }
+
+      Thread.current[THREAD_LOCAL_KEY][:memory_snapshots] << snapshot
+    end
+
+    def self.analyze_memory_performance(memory_events)
+      return {} if memory_events.empty?
+
+      allocations = memory_events[:allocations] || []
+      large_objects = memory_events[:large_objects] || []
+      snapshots = memory_events[:memory_snapshots] || []
+
+      # Calculate memory growth
+      memory_growth = 0
+      if memory_events[:memory_before] && memory_events[:memory_after]
+        memory_growth = memory_events[:memory_after] - memory_events[:memory_before]
+      end
+
+      # Calculate allocation totals
+      total_allocations = allocations.sum { |a| a[:count] }
+      total_allocated_size = allocations.sum { |a| a[:size] }
+
+      # Group allocations by class
+      allocations_by_class = allocations.group_by { |a| a[:class_name] }
+        .transform_values { |allocs|
+        {
+          count: allocs.sum { |a| a[:count] },
+          size: allocs.sum { |a| a[:size] }
+        }
+      }
+
+      # Find top allocating classes
+      top_allocating_classes = allocations_by_class.sort_by { |_, data| -data[:size] }.first(10)
+
+      # Analyze large objects
+      large_object_analysis = analyze_large_objects(large_objects)
+
+      # Analyze memory snapshots for trends
+      memory_trends = analyze_memory_trends(snapshots)
+
+      # Calculate GC efficiency
+      gc_efficiency = calculate_gc_efficiency(memory_events[:gc_before], memory_events[:gc_after])
+
+      # Analyze object type deltas (by Ruby object type, not class)
+      object_type_deltas = {}
+      if memory_events[:object_counts_before].is_a?(Hash) && memory_events[:object_counts_after].is_a?(Hash)
+        before = memory_events[:object_counts_before]
+        after = memory_events[:object_counts_after]
+        keys = (before.keys + after.keys).uniq
+        keys.each do |k|
+          object_type_deltas[k] = (after[k] || 0) - (before[k] || 0)
+        end
+      end
+
+      {
+        memory_growth_mb: memory_growth.round(2),
+        total_allocations: total_allocations,
+        total_allocated_size: total_allocated_size,
+        total_allocated_size_mb: (total_allocated_size / 1_000_000.0).round(2),
+        allocations_per_second: (memory_events[:duration_seconds] > 0) ?
+          (total_allocations.to_f / memory_events[:duration_seconds]).round(2) : 0,
+        top_allocating_classes: top_allocating_classes.map { |class_name, data|
+          {
+            class_name: class_name,
+            count: data[:count],
+            size: data[:size],
+            size_mb: (data[:size] / 1_000_000.0).round(2)
+          }
+        },
+        large_objects: large_object_analysis,
+        memory_trends: memory_trends,
+        gc_efficiency: gc_efficiency,
+        memory_snapshots_count: snapshots.count,
+        object_type_deltas: top_object_type_deltas(object_type_deltas, limit: 10)
+      }
+    end
+
+    def self.analyze_large_objects(large_objects)
+      return {} if large_objects.empty?
+
+      {
+        count: large_objects.count,
+        total_size_mb: large_objects.sum { |obj| obj[:size_mb] }.round(2),
+        largest_object_mb: large_objects.max_by { |obj| obj[:size_mb] }[:size_mb],
+        by_class: large_objects.group_by { |obj| obj[:class_name] }
+          .transform_values(&:count)
+      }
+    end
+
+    def self.top_object_type_deltas(deltas, limit: 10)
+      return {} unless deltas.is_a?(Hash)
+      deltas.sort_by { |_, v| -v.abs }.first(limit).to_h
+    end
+
+    def self.object_space_available?
+      defined?(ObjectSpace) && ObjectSpace.respond_to?(:each_object) && ObjectSpace.respond_to?(:memsize_of)
+    end
+
+    def self.sample_large_objects
+      results = []
+      return results unless object_space_available?
+
+      begin
+        # Sample across common heap object types
+        ObjectSpace.each_object do |obj|
+          # Randomly sample to control overhead
+          next unless rand < LARGE_OBJECT_SAMPLE_RATE
+
+          size = begin
+            ObjectSpace.memsize_of(obj)
+          rescue
+            0
+          end
+          next unless size && size > LARGE_OBJECT_THRESHOLD
+
+          klass = begin
+            (obj.respond_to?(:class) && obj.class) ? obj.class.name : "Unknown"
+          rescue
+            "Unknown"
+          end
+          results << {class_name: klass, size: size, size_mb: (size / 1_000_000.0).round(2)}
+
+          break if results.length >= MAX_LARGE_OBJECTS
+        end
+      rescue
+        # Best-effort only
+      end
+
+      # Sort largest first and keep top N
+      results.sort_by { |h| -h[:size] }.first(MAX_LARGE_OBJECTS)
+    end
+
+    def self.count_objects_snapshot
+      if defined?(ObjectSpace) && ObjectSpace.respond_to?(:count_objects)
+        ObjectSpace.count_objects.dup
+      else
+        {}
+      end
+    rescue
+      {}
+    end
+
+    def self.analyze_memory_trends(snapshots)
+      return {} if snapshots.length < 2
+
+      # Calculate memory growth rate between snapshots
+      memory_values = snapshots.map { |s| s[:memory_usage] }
+      memory_growth_rates = []
+
+      (1...memory_values.length).each do |i|
+        growth = memory_values[i] - memory_values[i - 1]
+        time_diff = snapshots[i][:timestamp] - snapshots[i - 1][:timestamp]
+        rate = (time_diff > 0) ? growth / time_diff : 0
+        memory_growth_rates << rate
+      end
+
+      {
+        average_growth_rate_mb_per_second: memory_growth_rates.sum / memory_growth_rates.length,
+        max_growth_rate_mb_per_second: memory_growth_rates.max,
+        memory_volatility: memory_growth_rates.map(&:abs).sum / memory_growth_rates.length,
+        peak_memory_mb: memory_values.max,
+        min_memory_mb: memory_values.min
+      }
+    end
+
+    def self.calculate_gc_efficiency(gc_before, gc_after)
+      return {} unless gc_before && gc_after
+
+      {
+        gc_count_increase: (gc_after[:count] || 0) - (gc_before[:count] || 0),
+        heap_pages_increase: (gc_after[:heap_allocated_pages] || 0) - (gc_before[:heap_allocated_pages] || 0),
+        objects_allocated: (gc_after[:total_allocated_objects] || 0) - (gc_before[:total_allocated_objects] || 0),
+        gc_frequency: (gc_after[:count] && gc_before[:count]) ?
+          (gc_after[:count] - gc_before[:count]).to_f / [gc_after[:count], 1].max : 0
+      }
+    end
+
+    def self.memory_usage_mb
+      # Use cached memory calculation to avoid expensive system calls
+      @memory_cache ||= {}
+      cache_key = Process.pid
+
+      # Cache memory usage for 1 second to avoid repeated system calls
+      if @memory_cache[cache_key] && (Time.now - @memory_cache[cache_key][:timestamp]) < 1
+        return @memory_cache[cache_key][:memory]
+      end
+
+      memory = if defined?(GC) && GC.respond_to?(:stat)
+        # Use GC stats as a proxy for memory usage (much faster than ps)
+        gc_stats = GC.stat
+        # Estimate memory usage from heap pages (rough approximation)
+        heap_pages = gc_stats[:heap_allocated_pages] || 0
+        (heap_pages * 4 * 1024) / (1024 * 1024) # 4KB per page, convert to MB
+      else
+        0
+      end
+
+      @memory_cache[cache_key] = {memory: memory, timestamp: Time.now}
+      memory
+    rescue
+      0
+    end
+
+    def self.gc_stats
+      if defined?(GC) && GC.respond_to?(:stat)
+        stats = GC.stat
+        {
+          count: stats[:count] || 0,
+          heap_allocated_pages: stats[:heap_allocated_pages] || 0,
+          heap_sorted_pages: stats[:heap_sorted_pages] || 0,
+          total_allocated_objects: stats[:total_allocated_objects] || 0,
+          heap_live_slots: stats[:heap_live_slots] || 0,
+          heap_eden_pages: stats[:heap_eden_pages] || 0,
+          heap_tomb_pages: stats[:heap_tomb_pages] || 0
+        }
+      else
+        {}
+      end
+    rescue
+      {}
+    end
+
+    def self.object_count
+      if defined?(GC) && GC.respond_to?(:stat)
+        GC.stat[:heap_live_slots] || 0
+      else
+        0
+      end
+    rescue
+      0
+    end
+
+    def self.heap_pages
+      if defined?(GC) && GC.respond_to?(:stat)
+        GC.stat[:heap_allocated_pages] || 0
+      else
+        0
+      end
+    rescue
+      0
+    end
+
+    # Helper method to take memory snapshots at specific points
+    def self.snapshot_at(label)
+      take_memory_snapshot(label)
+    end
+  end
+end

data/lib/dead_bro/railtie.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+begin
+  require "rails/railtie"
+rescue LoadError
+  # Rails not available, skip railtie definition
+end
+
+# Only define Railtie if Rails is available
+if defined?(Rails) && defined?(Rails::Railtie)
+  module DeadBro
+    class Railtie < ::Rails::Railtie
+
+      initializer "dead_bro.subscribe" do |app|
+        app.config.after_initialize do
+          # Use the shared Client instance for all subscribers
+          shared_client = DeadBro.client
+          
+          DeadBro::Subscriber.subscribe!(client: shared_client)
+          # Install outgoing HTTP instrumentation
+          require "dead_bro/http_instrumentation"
+          DeadBro::HttpInstrumentation.install!(client: shared_client)
+
+          # Install SQL query tracking
+          require "dead_bro/sql_subscriber"
+          DeadBro::SqlSubscriber.subscribe!
+
+          # Install Rails cache tracking
+          require "dead_bro/cache_subscriber"
+          DeadBro::CacheSubscriber.subscribe!
+
+          # Install Redis tracking (if Redis-related events are present)
+          require "dead_bro/redis_subscriber"
+          DeadBro::RedisSubscriber.subscribe!
+
+          # Install view rendering tracking
+          require "dead_bro/view_rendering_subscriber"
+          DeadBro::ViewRenderingSubscriber.subscribe!(client: shared_client)
+
+          # Install lightweight memory tracking (default)
+          require "dead_bro/lightweight_memory_tracker"
+          require "dead_bro/memory_leak_detector"
+          DeadBro::MemoryLeakDetector.initialize_history
+
+          # Install detailed memory tracking only if enabled
+          if DeadBro.configuration.allocation_tracking_enabled
+            require "dead_bro/memory_tracking_subscriber"
+            DeadBro::MemoryTrackingSubscriber.subscribe!(client: shared_client)
+          end
+
+          # Install job tracking if ActiveJob is available
+          if defined?(ActiveJob)
+            require "dead_bro/job_subscriber"
+            require "dead_bro/job_sql_tracking_middleware"
+            DeadBro::JobSqlTrackingMiddleware.subscribe!
+            DeadBro::JobSubscriber.subscribe!(client: shared_client)
+          end
+        rescue
+          # Never raise in Railtie init
+        end
+      end
+
+      # Insert Rack middleware early enough to observe uncaught exceptions
+      initializer "dead_bro.middleware" do |app|
+        require "dead_bro/error_middleware"
+        
+        # Use the shared Client instance for the middleware
+        shared_client = DeadBro.client
+
+        if defined?(::ActionDispatch::DebugExceptions)
+          app.config.middleware.insert_before(::ActionDispatch::DebugExceptions, ::DeadBro::ErrorMiddleware, shared_client)
+        elsif defined?(::ActionDispatch::ShowExceptions)
+          app.config.middleware.insert_before(::ActionDispatch::ShowExceptions, ::DeadBro::ErrorMiddleware, shared_client)
+        else
+          app.config.middleware.use(::DeadBro::ErrorMiddleware, shared_client)
+        end
+      rescue
+        # Never raise in Railtie init
+      end
+
+      # Insert SQL tracking middleware
+      initializer "dead_bro.sql_tracking_middleware" do |app|
+        require "dead_bro/sql_tracking_middleware"
+        app.config.middleware.use(::DeadBro::SqlTrackingMiddleware)
+      rescue
+        # Never raise in Railtie init
+      end
+    end
+  end
+end