apm_bro 0.1.11 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1763fe75b6723866fbf061cdc8a978105e3a8bd1be612c2dc0937865c576a1da
-  data.tar.gz: e1bad4ee3231e2a0783994983986e040cec36314912e30f402c044a41f47bdea
+  metadata.gz: 1104f9fc59e6d12b981c8c9dbd646ccf0e4199afaf2a5792a40e7a928ebc2ac9
+  data.tar.gz: b6798ce81a4e61cfd7e05cdf5974cbf0475610b0355686fbd5fad538eb5a0ea6
 SHA512:
-  metadata.gz: 070b54479382d040aebd53e2927d0ebd0aff1a0e0e3a8b4ad4045cc674cec680fd64f2cb279c18412aeff347a34eb098de6bb4fcf227094d2a11e61c12b33b67
-  data.tar.gz: 17894ab0bb4446d2fb90848ee0f1996683390710e6f81c198fa357f31822a6577fae98b70749feb46f1e2fbba70e71dd38bbb4886252114164b5d8dcac96a652
+  metadata.gz: 233bdd3ec6fe51292f331da900f3fbde862a4e78db6a3bceb1d6b2caeeef28bf9611f382ca0f0052baaa255a39f1fb2f3da57dc915c7def3a42d974df4f451e5
+  data.tar.gz: edff2e3c6da3985dab9b176387dafd65c315b7428e29e72231191c2d495831ba6ca69b74e085a882c614a712a9576b72c8f7fa757edfa5ebcd50ed417cf367ab

data/lib/apm_bro/error_middleware.rb

@@ -51,7 +51,8 @@ module ApmBro
           },
         rails_env: safe_rails_env,
         app: safe_app_name,
-        pid: Process.pid
+        pid: Process.pid,
+        logs: ApmBro.logger.logs
       }
     end
 

data/lib/apm_bro/job_sql_tracking_middleware.rb

@@ -5,7 +5,19 @@ module ApmBro
     def self.subscribe!
       # Start SQL tracking when a job begins - use the start event, not the complete event
       ActiveSupport::Notifications.subscribe("perform_start.active_job") do |name, started, finished, _unique_id, data|
+        # Clear logs for this job
+        ApmBro.logger.clear
         ApmBro::SqlSubscriber.start_request_tracking
+
+        # Start lightweight memory tracking for this job
+        if defined?(ApmBro::LightweightMemoryTracker)
+          ApmBro::LightweightMemoryTracker.start_request_tracking
+        end
+
+        # Start detailed memory tracking when allocation tracking is enabled
+        if ApmBro.configuration.allocation_tracking_enabled && defined?(ApmBro::MemoryTrackingSubscriber)
+          ApmBro::MemoryTrackingSubscriber.start_request_tracking
+        end
       end
     rescue StandardError
       # Never raise from instrumentation install

data/lib/apm_bro/job_subscriber.rb

@@ -23,6 +23,34 @@ module ApmBro
         # Get SQL queries executed during this job
         sql_queries = ApmBro::SqlSubscriber.stop_request_tracking
         
+        # Stop memory tracking and get collected memory data
+        if ApmBro.configuration.allocation_tracking_enabled && defined?(ApmBro::MemoryTrackingSubscriber)
+          detailed_memory = ApmBro::MemoryTrackingSubscriber.stop_request_tracking
+          memory_performance = ApmBro::MemoryTrackingSubscriber.analyze_memory_performance(detailed_memory)
+          # Keep memory_events compact and user-friendly (no large raw arrays)
+          memory_events = {
+            memory_before: detailed_memory[:memory_before],
+            memory_after: detailed_memory[:memory_after],
+            duration_seconds: detailed_memory[:duration_seconds],
+            allocations_count: (detailed_memory[:allocations] || []).length,
+            memory_snapshots_count: (detailed_memory[:memory_snapshots] || []).length,
+            large_objects_count: (detailed_memory[:large_objects] || []).length
+          }
+        else
+          lightweight_memory = ApmBro::LightweightMemoryTracker.stop_request_tracking
+          # Separate raw readings from derived performance metrics to avoid duplicating data
+          memory_events = {
+            memory_before: lightweight_memory[:memory_before],
+            memory_after: lightweight_memory[:memory_after]
+          }
+          memory_performance = {
+            memory_growth_mb: lightweight_memory[:memory_growth_mb],
+            gc_count_increase: lightweight_memory[:gc_count_increase],
+            heap_pages_increase: lightweight_memory[:heap_pages_increase],
+            duration_seconds: lightweight_memory[:duration_seconds]
+          }
+        end
+        
         payload = {
           job_class: data[:job].class.name,
           job_id: data[:job].job_id,
@@ -34,7 +62,10 @@ module ApmBro
           rails_env: safe_rails_env,
           host: safe_host,
           memory_usage: memory_usage_mb,
-          gc_stats: gc_stats
+          gc_stats: gc_stats,
+          memory_events: memory_events,
+          memory_performance: memory_performance,
+          logs: ApmBro.logger.logs
         }
         
         client.post_metric(event_name: name, payload: payload)
@@ -56,6 +87,34 @@ module ApmBro
         # Get SQL queries executed during this job
         sql_queries = ApmBro::SqlSubscriber.stop_request_tracking
         
+        # Stop memory tracking and get collected memory data
+        if ApmBro.configuration.allocation_tracking_enabled && defined?(ApmBro::MemoryTrackingSubscriber)
+          detailed_memory = ApmBro::MemoryTrackingSubscriber.stop_request_tracking
+          memory_performance = ApmBro::MemoryTrackingSubscriber.analyze_memory_performance(detailed_memory)
+          # Keep memory_events compact and user-friendly (no large raw arrays)
+          memory_events = {
+            memory_before: detailed_memory[:memory_before],
+            memory_after: detailed_memory[:memory_after],
+            duration_seconds: detailed_memory[:duration_seconds],
+            allocations_count: (detailed_memory[:allocations] || []).length,
+            memory_snapshots_count: (detailed_memory[:memory_snapshots] || []).length,
+            large_objects_count: (detailed_memory[:large_objects] || []).length
+          }
+        else
+          lightweight_memory = ApmBro::LightweightMemoryTracker.stop_request_tracking
+          # Separate raw readings from derived performance metrics to avoid duplicating data
+          memory_events = {
+            memory_before: lightweight_memory[:memory_before],
+            memory_after: lightweight_memory[:memory_after]
+          }
+          memory_performance = {
+            memory_growth_mb: lightweight_memory[:memory_growth_mb],
+            gc_count_increase: lightweight_memory[:gc_count_increase],
+            heap_pages_increase: lightweight_memory[:heap_pages_increase],
+            duration_seconds: lightweight_memory[:duration_seconds]
+          }
+        end
+        
         payload = {
           job_class: data[:job].class.name,
           job_id: data[:job].job_id,
@@ -70,7 +129,10 @@ module ApmBro
           rails_env: safe_rails_env,
           host: safe_host,
           memory_usage: memory_usage_mb,
-          gc_stats: gc_stats
+          gc_stats: gc_stats,
+          memory_events: memory_events,
+          memory_performance: memory_performance,
+          logs: ApmBro.logger.logs
         }
         
         event_name = exception&.class&.name || "ActiveJob::Exception"

data/lib/apm_bro/logger.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module ApmBro
+  class Logger
+    SEVERITY_LEVELS = {
+      debug: 0,
+      info: 1,
+      warn: 2,
+      error: 3,
+      fatal: 4
+    }.freeze
+
+    # ANSI color codes
+    COLOR_RESET = "\033[0m".freeze
+    COLOR_DEBUG = "\033[36m".freeze  # Cyan
+    COLOR_INFO = "\033[32m".freeze   # Green
+    COLOR_WARN = "\033[33m".freeze   # Yellow
+    COLOR_ERROR = "\033[31m".freeze # Red
+    COLOR_FATAL = "\033[35m".freeze  # Magenta
+
+    def initialize
+      @thread_logs_key = :apm_bro_logs
+    end
+
+    def debug(message)
+      log(:debug, message)
+    end
+
+    def info(message)
+      log(:info, message)
+    end
+
+    def warn(message)
+      log(:warn, message)
+    end
+
+    def error(message)
+      log(:error, message)
+    end
+
+    def fatal(message)
+      log(:fatal, message)
+    end
+
+    # Get all logs for the current thread
+    def logs
+      Thread.current[@thread_logs_key] || []
+    end
+
+    # Clear logs for the current thread
+    def clear
+      Thread.current[@thread_logs_key] = []
+    end
+
+    private
+
+    def log(severity, message)
+      timestamp = Time.now.utc
+      log_entry = {
+        sev: severity.to_s,
+        msg: message.to_s,
+        time: timestamp.iso8601(3) # Include milliseconds for better precision
+      }
+
+      # Store in thread-local storage
+      Thread.current[@thread_logs_key] ||= []
+      Thread.current[@thread_logs_key] << log_entry
+
+      # Print the message immediately
+      print_log(severity, message, timestamp)
+    end
+
+    def print_log(severity, message, timestamp)
+      formatted_message = format_log_message(severity, message, timestamp)
+      
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        # Use Rails logger if available (Rails handles its own color formatting)
+        case severity
+        when :debug
+          Rails.logger.debug(formatted_message)
+        when :info
+          Rails.logger.info(formatted_message)
+        when :warn
+          Rails.logger.warn(formatted_message)
+        when :error
+          Rails.logger.error(formatted_message)
+        when :fatal
+          Rails.logger.fatal(formatted_message)
+        end
+      else
+        # Fallback to stdout with colors
+        colored_message = format_log_message_with_color(severity, message, timestamp)
+        $stdout.puts(colored_message)
+      end
+    rescue StandardError
+      # Never let logging break the application
+      $stdout.puts("[ApmBro] #{severity.to_s.upcase}: #{message}")
+    end
+
+    def format_log_message(severity, message, timestamp)
+      "[ApmBro] #{timestamp.iso8601(3)} #{severity.to_s.upcase}: #{message}"
+    end
+
+    def format_log_message_with_color(severity, message, timestamp)
+      color = color_for_severity(severity)
+      severity_str = severity.to_s.upcase
+      "#{color}[ApmBro] #{timestamp.iso8601(3)} #{severity_str}: #{message}#{COLOR_RESET}"
+    end
+
+    def color_for_severity(severity)
+      case severity
+      when :debug
+        COLOR_DEBUG
+      when :info
+        COLOR_INFO
+      when :warn
+        COLOR_WARN
+      when :error
+        COLOR_ERROR
+      when :fatal
+        COLOR_FATAL
+      else
+        COLOR_RESET
+      end
+    end
+  end
+end
+

data/lib/apm_bro/sql_subscriber.rb

@@ -9,6 +9,7 @@ module ApmBro
     THREAD_LOCAL_KEY = :apm_bro_sql_queries
     THREAD_LOCAL_ALLOC_START_KEY = :apm_bro_sql_alloc_start
     THREAD_LOCAL_ALLOC_RESULTS_KEY = :apm_bro_sql_alloc_results
+    THREAD_LOCAL_BACKTRACE_KEY = :apm_bro_sql_backtraces
 
     def self.subscribe!
       # Subscribe with a start/finish listener to measure allocations per query
@@ -20,13 +21,19 @@ module ApmBro
       end
 
       ActiveSupport::Notifications.subscribe(SQL_EVENT_NAME) do |name, started, finished, _unique_id, data|
+        next if data[:name] == "SCHEMA"
         # Only track queries that are part of the current request
         next unless Thread.current[THREAD_LOCAL_KEY]
         unique_id = _unique_id
         allocations = nil
+        captured_backtrace = nil
         begin
           alloc_results = Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY]
           allocations = alloc_results && alloc_results.delete(unique_id)
+          
+          # Get the captured backtrace from when the query started
+          backtrace_map = Thread.current[THREAD_LOCAL_BACKTRACE_KEY]
+          captured_backtrace = backtrace_map && backtrace_map.delete(unique_id)
         rescue StandardError
         end
 
@@ -36,7 +43,7 @@ module ApmBro
           duration_ms: ((finished - started) * 1000.0).round(2),
           cached: data[:cached] || false,
           connection_id: data[:connection_id],
-          trace: safe_query_trace(data),
+          trace: safe_query_trace(data, captured_backtrace),
           allocations: allocations
         }
         # Add to thread-local storage
@@ -49,6 +56,7 @@ module ApmBro
       Thread.current[THREAD_LOCAL_KEY] = []
       Thread.current[THREAD_LOCAL_ALLOC_START_KEY] = {}
       Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY] = {}
+      Thread.current[THREAD_LOCAL_BACKTRACE_KEY] = {}
     end
 
     def self.stop_request_tracking
@@ -56,6 +64,7 @@ module ApmBro
       Thread.current[THREAD_LOCAL_KEY] = nil
       Thread.current[THREAD_LOCAL_ALLOC_START_KEY] = nil
       Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY] = nil
+      Thread.current[THREAD_LOCAL_BACKTRACE_KEY] = nil
       queries || []
     end
 
@@ -75,7 +84,7 @@ module ApmBro
       sql.length > 1000 ? sql[0..1000] + "..." : sql
     end
 
-    def self.safe_query_trace(data)
+    def self.safe_query_trace(data, captured_backtrace = nil)
       return [] unless data.is_a?(Hash)
 
       # Build trace from available data fields
@@ -86,24 +95,64 @@ module ApmBro
         trace << "#{data[:filename]}:#{data[:line]}:in `#{data[:method]}'"
       end
       
-      # Always try to get the full call stack for better trace information
-      begin
-        # Get the current call stack, skip the first few frames (our own code)
-        caller_stack = caller(3, 0) # Skip 3 frames, get up to 1
-        caller_trace = caller_stack.map do |line|
+      # Use the captured backtrace from when the query started (most accurate)
+      if captured_backtrace && captured_backtrace.is_a?(Array) && !captured_backtrace.empty?
+        # Filter to only include frames that contain "app/" (application code)
+        app_frames = captured_backtrace.select do |frame|
+          frame.include?('/app/')
+        end
+        
+        caller_trace = app_frames.map do |line|
           # Remove any potential sensitive information from file paths
           line.gsub(/\/[^\/]*(password|secret|key|token)[^\/]*\//i, '/[FILTERED]/')
         end
         
-        # Combine the immediate location with the call stack
         trace.concat(caller_trace)
-      rescue StandardError
-        # If caller fails, we still have the immediate location
+      else
+        # Fallback: try to get backtrace from current context
+        begin
+          # Get all available frames - we'll filter to find application code
+          all_frames = Thread.current.backtrace || []
+          
+          if all_frames.empty?
+            # Fallback to caller_locations if backtrace is empty
+            locations = caller_locations(1, 50)
+            all_frames = locations.map { |loc| "#{loc.path}:#{loc.lineno}:in `#{loc.label}'" } if locations
+          end
+          
+          # Filter to only include frames that contain "app/" (application code)
+          app_frames = all_frames.select do |frame|
+            frame.include?('/app/')
+          end
+          
+          caller_trace = app_frames.map do |line|
+            line.gsub(/\/[^\/]*(password|secret|key|token)[^\/]*\//i, '/[FILTERED]/')
+          end
+          
+          trace.concat(caller_trace)
+        rescue StandardError
+          # If backtrace fails, try caller as fallback
+          begin
+            caller_stack = caller(20, 50) # Get more frames to find app/ frames
+            app_frames = caller_stack.select { |frame| frame.include?('/app/') }
+            caller_trace = app_frames.map do |line|
+              line.gsub(/\/[^\/]*(password|secret|key|token)[^\/]*\//i, '/[FILTERED]/')
+            end
+            trace.concat(caller_trace)
+          rescue StandardError
+            # If caller also fails, we still have the immediate location
+          end
+        end
       end
       
-      # If we have a backtrace, use it (but it's usually nil for SQL events)
+      # If we have a backtrace in the data, use it (but it's usually nil for SQL events)
       if data[:backtrace] && data[:backtrace].is_a?(Array)
-        backtrace_trace = data[:backtrace].first(5).map do |line|
+        # Filter to only include frames that contain "app/"
+        app_backtrace = data[:backtrace].select do |line|
+          line.is_a?(String) && line.include?('/app/')
+        end
+        
+        backtrace_trace = app_backtrace.map do |line|
           case line
           when String
             line.gsub(/\/[^\/]*(password|secret|key|token)[^\/]*\//i, '/[FILTERED]/')
@@ -114,8 +163,8 @@ module ApmBro
         trace.concat(backtrace_trace)
       end
       
-      # Remove duplicates and limit the number of frames
-      trace.uniq.first(10).map do |line|
+      # Remove duplicates and return all app/ frames (no limit)
+      trace.uniq.map do |line|
         case line
         when String
           # Remove any potential sensitive information from file paths
@@ -138,6 +187,15 @@ module ApmBro
       begin
         map = (Thread.current[ApmBro::SqlSubscriber::THREAD_LOCAL_ALLOC_START_KEY] ||= {})
         map[id] = GC.stat[:total_allocated_objects] if defined?(GC) && GC.respond_to?(:stat)
+        
+        # Capture the backtrace at query start time (before notification system processes it)
+        # This gives us the actual call stack where the SQL was executed
+        backtrace_map = (Thread.current[ApmBro::SqlSubscriber::THREAD_LOCAL_BACKTRACE_KEY] ||= {})
+        captured_backtrace = Thread.current.backtrace
+        if captured_backtrace && captured_backtrace.is_a?(Array)
+          # Skip the first few frames (our listener code) to get to the actual query execution
+          backtrace_map[id] = captured_backtrace[5..-1] || captured_backtrace
+        end
       rescue StandardError
       end
     end

data/lib/apm_bro/sql_tracking_middleware.rb

@@ -7,6 +7,9 @@ module ApmBro
     end
 
     def call(env)
+      # Clear logs for this request
+      ApmBro.logger.clear
+
       # Start SQL tracking for this request
       if defined?(ApmBro::SqlSubscriber)
         puts "Starting SQL tracking for request: #{env['REQUEST_METHOD']} #{env['PATH_INFO']}"

data/lib/apm_bro/subscriber.rb

@@ -95,7 +95,8 @@ module ApmBro
               exception_class: (exception_class || exception_obj&.class&.name),
               message: (exception_message || exception_obj&.message).to_s[0, 1000],
               backtrace: backtrace,
-              error: true
+              error: true,
+              logs: ApmBro.logger.logs
             }
 
             event_name = (exception_class || exception_obj&.class&.name || "exception").to_s
@@ -134,7 +135,8 @@ module ApmBro
           view_events: view_events,
           view_performance: view_performance,
           memory_events: memory_events,
-          memory_performance: memory_performance
+          memory_performance: memory_performance,
+          logs: ApmBro.logger.logs
         }
         client.post_metric(event_name: name, payload: payload)
       end

data/lib/apm_bro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ApmBro
-  VERSION = "0.1.11"
+  VERSION = "0.1.13"
 end

data/lib/apm_bro.rb

@@ -18,6 +18,7 @@ module ApmBro
   autoload :MemoryHelpers, "apm_bro/memory_helpers"
   autoload :JobSubscriber, "apm_bro/job_subscriber"
   autoload :JobSqlTrackingMiddleware, "apm_bro/job_sql_tracking_middleware"
+  autoload :Logger, "apm_bro/logger"
   begin
     require "apm_bro/railtie"
   rescue LoadError
@@ -45,4 +46,9 @@ module ApmBro
       SecureRandom.uuid
     end
   end
+
+  # Returns the logger instance for storing and retrieving log messages
+  def self.logger
+    @logger ||= Logger.new
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apm_bro
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.13
 platform: ruby
 authors:
 - Emanuel Comsa
@@ -29,6 +29,7 @@ files:
 - lib/apm_bro/job_sql_tracking_middleware.rb
 - lib/apm_bro/job_subscriber.rb
 - lib/apm_bro/lightweight_memory_tracker.rb
+- lib/apm_bro/logger.rb
 - lib/apm_bro/memory_helpers.rb
 - lib/apm_bro/memory_leak_detector.rb
 - lib/apm_bro/memory_tracking_subscriber.rb