apm_bro 0.1.15 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26626be56405c7aa4561db094b0e1bf27b1469cef67275786ec1ece6e28947e3
-  data.tar.gz: c07ec1b39ff873b5cccfa909fb726aa5971bc87b6abdae8a59de4bd4550b898b
+  metadata.gz: 7bf4fc110108b143a2f2e608090ae8c75a5882f8634ef1c01d0ddb949d41e54a
+  data.tar.gz: 7c5bd6915f805032ca63b73ab9325ce64b262d8489c2842b45c28a02e84b961d
 SHA512:
-  metadata.gz: 8c8dc70fec2841c841e71b945f9e6db3aaf40f35bd7e750f3386130829a0733e44e91c16da05b0eeb5e52175bad335af9990942ed66385ff5926460fc98b64f9
-  data.tar.gz: 356a4e343d70b3bac7aee65d285fabe765dbb91d0ba74c4a2b594ca6459e13b7cfc68e8a96fbe43c525f5252e4a3f0692028dbcd38c01a0d7ad0d52b6d4cbda5
+  metadata.gz: 601ed7213bb2dc1b551ba8d8e47944492bd67b727b0747aaad9c60b5ce13fb2b728b218a43eb0775e1479f5684ea1721dd0b9310bd25d2ccfcfcfad8cfc9e439
+  data.tar.gz: b8a2e25e5c0369d7bd58fe058ef7d524bdc42c7cc22739d30f9ad0956338370f63578aa10688869c817148451b46d7b9e1dab8835c4e7f33531cd9146a04ed88

data/README.md

@@ -108,6 +108,54 @@ ApmBro automatically tracks SQL queries executed during each request and job. Ea
 - `cached` - Whether the query was cached
 - `connection_id` - Database connection ID
 - `trace` - Call stack showing where the query was executed
+- `explain_plan` - Query execution plan (when EXPLAIN ANALYZE is enabled, see below)
+
+## Automatic EXPLAIN ANALYZE for Slow Queries
+
+ApmBro can automatically run `EXPLAIN ANALYZE` on slow SQL queries to help you understand query performance and identify optimization opportunities. This feature runs in the background and doesn't block your application requests.
+
+### How It Works
+
+- **Automatic Detection**: When a query exceeds the configured threshold, ApmBro automatically captures its execution plan
+- **Background Execution**: EXPLAIN ANALYZE runs in a separate thread using a dedicated database connection, so it never blocks your application
+- **Database Support**: Works with PostgreSQL, MySQL, SQLite, and other databases
+- **Smart Filtering**: Automatically skips transaction queries (BEGIN, COMMIT, ROLLBACK) and other queries that don't benefit from EXPLAIN
+
+### Configuration
+
+- **`explain_analyze_enabled`** (default: `false`) - Set to `true` to enable automatic EXPLAIN ANALYZE
+- **`slow_query_threshold_ms`** (default: `500`) - Queries taking longer than this threshold will have their execution plan captured
+
+### Example Configuration
+
+```ruby
+ApmBro.configure do |config|
+  config.api_key = ENV['APM_BRO_API_KEY']
+  config.enabled = true
+  
+  # Enable EXPLAIN ANALYZE for queries slower than 500ms
+  config.explain_analyze_enabled = true
+  config.slow_query_threshold_ms = 500
+  
+  # Or use a higher threshold for production
+  # config.slow_query_threshold_ms = 1000  # Only explain queries > 1 second
+end
+```
+
+### What You Get
+
+When a slow query is detected, the `explain_plan` field in the SQL query data will contain:
+- **PostgreSQL**: Full EXPLAIN ANALYZE output with buffer usage statistics
+- **MySQL**: EXPLAIN ANALYZE output showing actual execution times
+- **SQLite**: EXPLAIN QUERY PLAN output
+- **Other databases**: Standard EXPLAIN output
+
+This execution plan helps you:
+- Identify missing indexes
+- Understand query execution order
+- Spot full table scans
+- Optimize JOIN operations
+- Analyze buffer and cache usage (PostgreSQL)
 
 ## View Rendering Tracking
 

data/lib/apm_bro/configuration.rb

@@ -4,7 +4,7 @@ module ApmBro
   class Configuration
     DEFAULT_ENDPOINT_PATH = "/v1/metrics"
 
-    attr_accessor :api_key, :endpoint_url, :open_timeout, :read_timeout, :enabled, :ruby_dev, :memory_tracking_enabled, :allocation_tracking_enabled, :circuit_breaker_enabled, :circuit_breaker_failure_threshold, :circuit_breaker_recovery_timeout, :circuit_breaker_retry_timeout, :sample_rate, :excluded_controllers, :excluded_jobs, :excluded_controller_actions, :deploy_id
+    attr_accessor :api_key, :endpoint_url, :open_timeout, :read_timeout, :enabled, :ruby_dev, :memory_tracking_enabled, :allocation_tracking_enabled, :circuit_breaker_enabled, :circuit_breaker_failure_threshold, :circuit_breaker_recovery_timeout, :circuit_breaker_retry_timeout, :sample_rate, :excluded_controllers, :excluded_jobs, :excluded_controller_actions, :deploy_id, :slow_query_threshold_ms, :explain_analyze_enabled
 
     def initialize
       @api_key = nil
@@ -24,6 +24,8 @@ module ApmBro
       @excluded_jobs = []
       @excluded_controller_actions = []
       @deploy_id = resolve_deploy_id
+      @slow_query_threshold_ms = 500 # Default: 500ms
+      @explain_analyze_enabled = false # Enable EXPLAIN ANALYZE for slow queries by default
     end
 
     def resolve_api_key
@@ -112,15 +114,45 @@ module ApmBro
     end
 
     def resolve_excluded_controller_actions
-      return @excluded_controller_actions if @excluded_controller_actions && !@excluded_controller_actions.empty?
+      # Collect patterns from @excluded_controller_actions
+      patterns = []
+      if @excluded_controller_actions && !@excluded_controller_actions.empty?
+        patterns.concat(Array(@excluded_controller_actions))
+      end
+
+      # Also check @excluded_controllers for patterns containing '#' (controller action patterns)
+      if @excluded_controllers && !@excluded_controllers.empty?
+        action_patterns = Array(@excluded_controllers).select { |pat| pat.to_s.include?("#") }
+        patterns.concat(action_patterns)
+      end
+
+      return patterns if !patterns.empty?
 
       if defined?(Rails)
         list = fetch_from_rails_settings(%w[apm_bro excluded_controller_actions])
-        return Array(list) if list
+        if list
+          rails_patterns = Array(list)
+          # Also check excluded_controllers from Rails settings for action patterns
+          controllers_list = fetch_from_rails_settings(%w[apm_bro excluded_controllers])
+          if controllers_list
+            action_patterns = Array(controllers_list).select { |pat| pat.to_s.include?("#") }
+            rails_patterns.concat(action_patterns)
+          end
+          return rails_patterns if !rails_patterns.empty?
+        end
       end
 
       env = ENV["APM_BRO_EXCLUDED_CONTROLLER_ACTIONS"]
-      return env.split(",").map(&:strip) if env && !env.strip.empty?
+      if env && !env.strip.empty?
+        env_patterns = env.split(",").map(&:strip)
+        # Also check excluded_controllers env var for action patterns
+        controllers_env = ENV["APM_BRO_EXCLUDED_CONTROLLERS"]
+        if controllers_env && !controllers_env.strip.empty?
+          action_patterns = controllers_env.split(",").map(&:strip).select { |pat| pat.include?("#") }
+          env_patterns.concat(action_patterns)
+        end
+        return env_patterns if !env_patterns.empty?
+      end
 
       []
     end
@@ -186,8 +218,16 @@ module ApmBro
       return false if name.nil? || pattern.nil?
       pat = pattern.to_s
       return !!(name.to_s == pat) unless pat.include?("*")
-      # Convert simple wildcard pattern (e.g., "Admin::*") to regex
-      regex = Regexp.new("^" + Regexp.escape(pat).gsub("\\*", "[^:]*") + "$")
+      
+      # For controller action patterns (containing '#'), use .* to match any characters including colons
+      # For controller-only patterns, use [^:]* to match namespace segments
+      if pat.include?("#")
+        # Controller action pattern: allow * to match any characters including colons
+        regex = Regexp.new("^" + Regexp.escape(pat).gsub("\\*", ".*") + "$")
+      else
+        # Controller-only pattern: use [^:]* to match namespace segments
+        regex = Regexp.new("^" + Regexp.escape(pat).gsub("\\*", "[^:]*") + "$")
+      end
       !!(name.to_s =~ regex)
     rescue
       false

data/lib/apm_bro/sql_subscriber.rb

@@ -13,6 +13,7 @@ module ApmBro
     THREAD_LOCAL_ALLOC_START_KEY = :apm_bro_sql_alloc_start
     THREAD_LOCAL_ALLOC_RESULTS_KEY = :apm_bro_sql_alloc_results
     THREAD_LOCAL_BACKTRACE_KEY = :apm_bro_sql_backtraces
+    THREAD_LOCAL_EXPLAIN_PENDING_KEY = :apm_bro_explain_pending
 
     def self.subscribe!
       # Subscribe with a start/finish listener to measure allocations per query
@@ -40,15 +41,26 @@ module ApmBro
         rescue
         end
 
+        duration_ms = ((finished - started) * 1000.0).round(2)
+        original_sql = data[:sql]
+
         query_info = {
-          sql: sanitize_sql(data[:sql]),
+          sql: sanitize_sql(original_sql),
           name: data[:name],
-          duration_ms: ((finished - started) * 1000.0).round(2),
+          duration_ms: duration_ms,
           cached: data[:cached] || false,
           connection_id: data[:connection_id],
           trace: safe_query_trace(data, captured_backtrace),
           allocations: allocations
         }
+
+        # Run EXPLAIN ANALYZE for slow queries in the background
+        if should_explain_query?(duration_ms, original_sql)
+          # Store reference to query_info so we can update it when EXPLAIN completes
+          query_info[:explain_plan] = nil # Placeholder
+          start_explain_analyze_background(original_sql, data[:connection_id], query_info)
+        end
+
         # Add to thread-local storage
         Thread.current[THREAD_LOCAL_KEY] << query_info
       end
@@ -59,17 +71,43 @@ module ApmBro
       Thread.current[THREAD_LOCAL_ALLOC_START_KEY] = {}
       Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY] = {}
       Thread.current[THREAD_LOCAL_BACKTRACE_KEY] = {}
+      Thread.current[THREAD_LOCAL_EXPLAIN_PENDING_KEY] = []
     end
 
     def self.stop_request_tracking
+      # Wait for any pending EXPLAIN ANALYZE queries to complete (with timeout)
+      # This must happen BEFORE we get the queries array reference to ensure
+      # all explain_plan fields are populated
+      wait_for_pending_explains(5.0) # 5 second timeout
+      
+      # Get queries after waiting for EXPLAIN to complete
       queries = Thread.current[THREAD_LOCAL_KEY]
+      
       Thread.current[THREAD_LOCAL_KEY] = nil
       Thread.current[THREAD_LOCAL_ALLOC_START_KEY] = nil
       Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY] = nil
       Thread.current[THREAD_LOCAL_BACKTRACE_KEY] = nil
+      Thread.current[THREAD_LOCAL_EXPLAIN_PENDING_KEY] = nil
       queries || []
     end
 
+    def self.wait_for_pending_explains(timeout_seconds)
+      pending = Thread.current[THREAD_LOCAL_EXPLAIN_PENDING_KEY]
+      return unless pending && !pending.empty?
+      
+      start_time = Time.now
+      pending.each do |thread|
+        remaining_time = timeout_seconds - (Time.now - start_time)
+        break if remaining_time <= 0
+        
+        begin
+          thread.join(remaining_time)
+        rescue => e
+          ApmBro.logger.debug("Error waiting for EXPLAIN ANALYZE: #{e.message}")
+        end
+      end
+    end
+
     def self.sanitize_sql(sql)
       return sql unless sql.is_a?(String)
 
@@ -86,6 +124,215 @@ module ApmBro
       (sql.length > 1000) ? sql[0..1000] + "..." : sql
     end
 
+    def self.should_explain_query?(duration_ms, sql)
+      return false unless ApmBro.configuration.explain_analyze_enabled
+      return false if duration_ms < ApmBro.configuration.slow_query_threshold_ms
+      return false unless sql.is_a?(String)
+      return false if sql.strip.empty?
+      
+      # Skip EXPLAIN for certain query types that don't benefit from it
+      sql_upper = sql.upcase.strip
+      return false if sql_upper.start_with?("EXPLAIN")
+      return false if sql_upper.start_with?("BEGIN")
+      return false if sql_upper.start_with?("COMMIT")
+      return false if sql_upper.start_with?("ROLLBACK")
+      return false if sql_upper.start_with?("SAVEPOINT")
+      return false if sql_upper.start_with?("RELEASE")
+      
+      true
+    end
+
+    def self.start_explain_analyze_background(sql, connection_id, query_info)
+      return unless defined?(ActiveRecord)
+      return unless ActiveRecord::Base.respond_to?(:connection)
+      
+      # Capture the main thread reference to append logs to the correct thread
+      main_thread = Thread.current
+      
+      # Run EXPLAIN in a background thread to avoid blocking the main request
+      explain_thread = Thread.new do
+        connection = nil
+        begin
+          # Use a separate connection to avoid interfering with the main query
+          if ActiveRecord::Base.connection_pool.respond_to?(:checkout)
+            connection = ActiveRecord::Base.connection_pool.checkout
+          else
+            connection = ActiveRecord::Base.connection
+          end
+          
+          # Build EXPLAIN query based on database adapter
+          explain_sql = build_explain_query(sql, connection)
+          
+          # Execute the EXPLAIN query
+          # For PostgreSQL, use select_all which returns ActiveRecord::Result
+          # For other databases, use execute
+          adapter_name = connection.adapter_name.downcase
+          if adapter_name == "postgresql" || adapter_name == "postgis"
+            # PostgreSQL: select_all returns ActiveRecord::Result with rows
+            result = connection.select_all(explain_sql)
+          else
+            # Other databases: use execute
+            result = connection.execute(explain_sql)
+          end
+          
+          # Format the result based on database adapter
+          explain_plan = format_explain_result(result, connection)
+          
+          # Update the query_info with the explain plan
+          # This updates the hash that's already in the queries array
+          if explain_plan && !explain_plan.to_s.strip.empty?
+            query_info[:explain_plan] = explain_plan
+            append_log_to_thread(main_thread, :debug, "Captured EXPLAIN ANALYZE for slow query (#{query_info[:duration_ms]}ms): #{explain_plan[0..1000]}...")
+          else
+            query_info[:explain_plan] = nil
+            append_log_to_thread(main_thread, :debug, "EXPLAIN ANALYZE returned empty result. Result type: #{result.class}, Result: #{result.inspect[0..200]}")
+          end
+        rescue => e
+          # Silently fail - don't let EXPLAIN break the application
+          append_log_to_thread(main_thread, :debug, "Failed to capture EXPLAIN ANALYZE: #{e.message}")
+          query_info[:explain_plan] = nil
+        ensure
+          # Return connection to pool if we checked it out
+          if connection && ActiveRecord::Base.connection_pool.respond_to?(:checkin)
+            ActiveRecord::Base.connection_pool.checkin(connection) rescue nil
+          end
+        end
+      end
+      
+      # Track the thread so we can wait for it when stopping request tracking
+      pending = Thread.current[THREAD_LOCAL_EXPLAIN_PENDING_KEY] ||= []
+      pending << explain_thread
+    rescue => e
+      # Use ApmBro.logger here since we're still in the main thread
+      ApmBro.logger.debug("Failed to start EXPLAIN ANALYZE thread: #{e.message}")
+    end
+
+    # Append a log entry directly to a specific thread's log storage
+    # This is used when logging from background threads to ensure logs
+    # are collected with the main request thread's logs
+    def self.append_log_to_thread(thread, severity, message)
+      timestamp = Time.now.utc
+      log_entry = {
+        sev: severity.to_s,
+        msg: message.to_s,
+        time: timestamp.iso8601(3)
+      }
+
+      # Append to the specified thread's log storage
+      thread[:apm_bro_logs] ||= []
+      thread[:apm_bro_logs] << log_entry
+
+      # Also print the message immediately (using current thread's logger)
+      begin
+        if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          formatted_message = "[ApmBro] #{timestamp.iso8601(3)} #{severity.to_s.upcase}: #{message}"
+          case severity
+          when :debug
+            Rails.logger.debug(formatted_message)
+          when :info
+            Rails.logger.info(formatted_message)
+          when :warn
+            Rails.logger.warn(formatted_message)
+          when :error
+            Rails.logger.error(formatted_message)
+          when :fatal
+            Rails.logger.fatal(formatted_message)
+          end
+        else
+          # Fallback to stdout
+          $stdout.puts("[ApmBro] #{timestamp.iso8601(3)} #{severity.to_s.upcase}: #{message}")
+        end
+      rescue
+        # Never let logging break the application
+        $stdout.puts("[ApmBro] #{severity.to_s.upcase}: #{message}")
+      end
+    end
+
+    def self.build_explain_query(sql, connection)
+      adapter_name = connection.adapter_name.downcase
+      
+      case adapter_name
+      when "postgresql", "postgis"
+        # PostgreSQL supports ANALYZE and BUFFERS
+        "EXPLAIN (ANALYZE, BUFFERS) #{sql}"
+      when "mysql", "mysql2", "trilogy"
+        # MySQL uses different syntax - ANALYZE is a separate keyword
+        "EXPLAIN ANALYZE #{sql}"
+      when "sqlite3"
+        # SQLite supports EXPLAIN QUERY PLAN
+        "EXPLAIN QUERY PLAN #{sql}"
+      else
+        # Generic fallback - just EXPLAIN
+        "EXPLAIN #{sql}"
+      end
+    end
+
+    def self.format_explain_result(result, connection)
+      adapter_name = connection.adapter_name.downcase
+      
+      case adapter_name
+      when "postgresql", "postgis"
+        # PostgreSQL returns ActiveRecord::Result from select_all
+        if result.respond_to?(:rows)
+          # ActiveRecord::Result object - rows is an array of arrays
+          # Each row is [query_plan_string]
+          plan_text = result.rows.map { |row| row.is_a?(Array) ? row.first.to_s : row.to_s }.join("\n")
+          return plan_text unless plan_text.strip.empty?
+        end
+        
+        # Try alternative methods to extract the plan
+        if result.respond_to?(:each) && result.respond_to?(:columns)
+          # ActiveRecord::Result with columns
+          plan_column = result.columns.find { |col| col.downcase.include?("plan") || col.downcase.include?("query") } || result.columns.first
+          plan_text = result.map { |row| 
+            if row.is_a?(Hash)
+              row[plan_column] || row[plan_column.to_sym] || row.values.first
+            else
+              row
+            end
+          }.join("\n")
+          return plan_text unless plan_text.strip.empty?
+        end
+        
+        if result.is_a?(Array)
+          # Array of hashes or arrays
+          plan_text = result.map do |row|
+            if row.is_a?(Hash)
+              row["QUERY PLAN"] || row["query plan"] || row[:query_plan] || row.values.first.to_s
+            elsif row.is_a?(Array)
+              row.first.to_s
+            else
+              row.to_s
+            end
+          end.join("\n")
+          return plan_text unless plan_text.strip.empty?
+        end
+        
+        # Fallback to string representation
+        result.to_s
+      when "mysql", "mysql2", "trilogy"
+        # MySQL returns rows
+        if result.is_a?(Array)
+          result.map { |row| row.is_a?(Hash) ? row.values.join(" | ") : row.to_s }.join("\n")
+        else
+          result.to_s
+        end
+      when "sqlite3"
+        # SQLite returns rows
+        if result.is_a?(Array)
+          result.map { |row| row.is_a?(Hash) ? row.values.join(" | ") : row.to_s }.join("\n")
+        else
+          result.to_s
+        end
+      else
+        # Generic fallback
+        result.to_s
+      end
+    rescue => e
+      # Fallback to string representation
+      result.to_s
+    end
+
     def self.safe_query_trace(data, captured_backtrace = nil)
       return [] unless data.is_a?(Hash)
 

data/lib/apm_bro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ApmBro
-  VERSION = "0.1.15"
+  VERSION = "0.1.16"
 end

data/lib/apm_bro.rb

@@ -51,4 +51,13 @@ module ApmBro
   def self.logger
     @logger ||= Logger.new
   end
+
+  # Returns the current environment (Rails.env or ENV fallback)
+  def self.env
+    if defined?(Rails) && Rails.respond_to?(:env)
+      Rails.env
+    else
+      ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development"
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apm_bro
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.16
 platform: ruby
 authors:
 - Emanuel Comsa