newrelic_rpm 3.1.2 → 3.2.0.beta1

data/CHANGELOG

@@ -1,3 +1,12 @@
+v3.2.0
+  * Fix over-detection of mongrel and unicorn and only start the agent when
+    actual server is running
+  * Improve developer mode backtraces to support ruby 1.9.2, windows, and fix
+    nil de-reference in developer mode
+  * Fixed some cases where Memcache instrumentation was failing to load
+  * Added View instrumentation back for Rails 3.1
+  * Ability to set log destination by NEW_RELIC_LOG env var
+  
 v3.1.2
   * Fixed some thread safety issues
   * Work around for Ruby 1.8.7 Marshal crash bug

data/lib/new_relic/agent.rb

@@ -81,9 +81,11 @@ module NewRelic
     require 'new_relic/agent/worker_loop'
     require 'new_relic/agent/stats_engine'
     require 'new_relic/agent/transaction_sampler'
+    require 'new_relic/agent/sql_sampler'
     require 'new_relic/agent/error_collector'
     require 'new_relic/agent/busy_calculator'
     require 'new_relic/agent/sampler'
+    require 'new_relic/agent/database'
 
     require 'new_relic/agent/instrumentation/controller_instrumentation'
 
@@ -146,9 +148,16 @@ module NewRelic
     alias get_stats_no_scope get_stats
 
     # Get the logger for the agent.  Available after the agent has initialized.
-    # This sends output to the agent log file.
+    # This sends output to the agent log file.  If the agent has not initialized
+    # a standard output logger is returned.
     def logger
-      NewRelic::Control.instance.log
+      control = NewRelic::Control.instance(false)
+      if control
+        control.log
+      else
+        require 'logger'
+        @stdoutlog ||= Logger.new $stdout
+      end
     end
 
     # Call this to manually start the Agent in situations where the Agent does
@@ -272,7 +281,7 @@ module NewRelic
     #    end
     #
     def set_sql_obfuscator(type = :replace, &block)
-      agent.set_sql_obfuscator type, &block
+      NewRelic::Agent::Database.set_sql_obfuscator(type, &block)
     end
 
 

data/lib/new_relic/agent/agent.rb

@@ -35,14 +35,16 @@ module NewRelic
         @metric_ids = {}
         @stats_engine = NewRelic::Agent::StatsEngine.new
         @transaction_sampler = NewRelic::Agent::TransactionSampler.new
+        @sql_sampler = NewRelic::Agent::SqlSampler.new
         @stats_engine.transaction_sampler = @transaction_sampler
+        @stats_engine.sql_sampler = @sql_sampler
         @error_collector = NewRelic::Agent::ErrorCollector.new
         @connect_attempts = 0
 
         @request_timeout = NewRelic::Control.instance.fetch('timeout', 2 * 60)
 
         @last_harvest_time = Time.now
-        @obfuscator = method(:default_sql_obfuscator)
+        @obfuscator = lambda {|sql| NewRelic::Agent::Database.default_sql_obfuscator(sql) }
       end
       
       # contains all the class-level methods for NewRelic::Agent::Agent
@@ -64,6 +66,7 @@ module NewRelic
         attr_reader :stats_engine
         # the transaction sampler that handles recording transactions
         attr_reader :transaction_sampler
+        attr_reader :sql_sampler
         # error collector is a simple collection of recorded errors
         attr_reader :error_collector
         # whether we should record raw, obfuscated, or no sql
@@ -271,29 +274,6 @@ module NewRelic
           Thread.current[:newrelic_untraced].pop if Thread.current[:newrelic_untraced]
         end
 
-        # Sets the sql obfuscator used to clean up sql when sending it
-        # to the server. Possible types are:
-        #
-        # :before => sets the block to run before the existing
-        # obfuscators
-        #
-        # :after => sets the block to run after the existing
-        # obfuscator(s)
-        #
-        # :replace => removes the current obfuscator and replaces it
-        # with the provided block
-        def set_sql_obfuscator(type, &block)
-          if type == :before
-            @obfuscator = NewRelic::ChainedCall.new(block, @obfuscator)
-          elsif type == :after
-            @obfuscator = NewRelic::ChainedCall.new(@obfuscator, block)
-          elsif type == :replace
-            @obfuscator = block
-          else
-            fail "unknown sql_obfuscator type #{type}"
-          end
-        end
-
         # Shorthand to the NewRelic::Agent.logger method
         def log
           NewRelic::Agent.logger
@@ -331,68 +311,6 @@ module NewRelic
             log.info "Application: #{control.app_names.join(", ")}"
           end
           
-          # apdex_f is always 4 times the apdex_t
-          def apdex_f
-            (4 * NewRelic::Control.instance.apdex_t).to_f
-          end
-
-          # If the transaction threshold is set to the string
-          # 'apdex_f', we use 4 times the apdex_t value to record
-          # transactions. This gears well with using apdex since you
-          # will attempt to send any transactions that register as 'failing'
-          def apdex_f_threshold?
-            sampler_config.fetch('transaction_threshold', '') =~ /apdex_f/i
-          end
-
-          # Sets the sql recording configuration by trying to detect
-          # any attempt to disable the sql collection - 'off',
-          # 'false', 'none', and friends. Otherwise, we accept 'raw',
-          # and unrecognized values default to 'obfuscated'
-          def set_sql_recording!
-            record_sql_config = sampler_config.fetch('record_sql', :obfuscated)
-            case record_sql_config.to_s
-            when 'off'
-              @record_sql = :off
-            when 'none'
-              @record_sql = :off
-            when 'false'
-              @record_sql = :off
-            when 'raw'
-              @record_sql = :raw
-            else
-              @record_sql = :obfuscated
-            end
-
-            log_sql_transmission_warning?
-          end
-
-          # Warn the user when we are sending raw sql across the wire
-          # - they should probably be using ssl when this is true
-          def log_sql_transmission_warning?
-            log_if((@record_sql == :raw), :warn, "Agent is configured to send raw SQL to the service")
-          end
-
-          # gets the sampler configuration from the control object's settings
-          def sampler_config
-            control.fetch('transaction_tracer', {})
-          end
-
-          # this entire method should be done on the transaction
-          # sampler object, rather than here. We should pass in the
-          # sampler config.
-          def config_transaction_tracer
-            @should_send_samples = @config_should_send_samples = sampler_config.fetch('enabled', true)
-            @should_send_random_samples = sampler_config.fetch('random_sample', false)
-            @explain_threshold = sampler_config.fetch('explain_threshold', 0.5).to_f
-            @explain_enabled = sampler_config.fetch('explain_enabled', true)
-            set_sql_recording!
-
-            # default to 2.0, string 'apdex_f' will turn into your
-            # apdex * 4
-            @slowest_transaction_threshold = sampler_config.fetch('transaction_threshold', 2.0).to_f
-            @slowest_transaction_threshold = apdex_f if apdex_f_threshold?
-          end
-
           # Connecting in the foreground blocks further startup of the
           # agent until we have a connection - useful in cases where
           # you're trying to log a very-short-running process and want
@@ -548,6 +466,16 @@ module NewRelic
             end
           end
           
+          def check_sql_sampler_status
+            # disable sql sampling if disabled by the server
+            # and we're not in dev mode
+            if control.developer_mode? || @should_send_samples
+              @sql_sampler.enable
+            else
+              @sql_sampler.disable
+            end
+          end
+
           # logs info about the worker loop so users can see when the
           # agent actually begins running in the background
           def log_worker_loop_start
@@ -633,6 +561,7 @@ module NewRelic
                 connect(connection_options)
                 if @connected
                   check_transaction_sampler_status
+                  check_sql_sampler_status
                   log_worker_loop_start
                   create_and_run_worker_loop
                   # never reaches here unless there is a problem or
@@ -796,6 +725,8 @@ module NewRelic
           # are allowed to send errors. Pretty simple, and logs at
           # debug whether errors will or will not be sent.
           def configure_error_collector!(server_enabled)
+            # Reinitialize the error collector
+            @error_collector = NewRelic::Agent::ErrorCollector.new
             # Ask for permission to collect error data
             enabled = if error_collector.config_enabled && server_enabled
                         error_collector.enabled = true
@@ -820,6 +751,24 @@ module NewRelic
             log.info "Transaction sampling enabled, rate = #{@transaction_sampler.sampling_rate}"
           end
 
+          # this entire method should be done on the transaction
+          # sampler object, rather than here. We should pass in the
+          # sampler config.
+          def config_transaction_tracer
+            # Reconfigure the transaction tracer
+            @transaction_sampler.configure!
+            @should_send_samples = @config_should_send_samples = sampler_config.fetch('enabled', true)
+            @should_send_random_samples = sampler_config.fetch('random_sample', false)
+            @explain_threshold = sampler_config.fetch('explain_threshold', 0.5).to_f
+            @explain_enabled = sampler_config.fetch('explain_enabled', true)
+            set_sql_recording!
+
+            # default to 2.0, string 'apdex_f' will turn into your
+            # apdex * 4
+            @slowest_transaction_threshold = sampler_config.fetch('transaction_threshold', 2.0).to_f
+            @slowest_transaction_threshold = apdex_f if apdex_f_threshold?
+          end
+
           # Enables or disables the transaction tracer and sets its
           # options based on the options provided to the
           # method.
@@ -837,6 +786,52 @@ module NewRelic
             end
           end
 
+          # apdex_f is always 4 times the apdex_t
+          def apdex_f
+            (4 * NewRelic::Control.instance.apdex_t).to_f
+          end
+
+          # If the transaction threshold is set to the string
+          # 'apdex_f', we use 4 times the apdex_t value to record
+          # transactions. This gears well with using apdex since you
+          # will attempt to send any transactions that register as 'failing'
+          def apdex_f_threshold?
+            sampler_config.fetch('transaction_threshold', '') =~ /apdex_f/i
+          end
+
+          # Sets the sql recording configuration by trying to detect
+          # any attempt to disable the sql collection - 'off',
+          # 'false', 'none', and friends. Otherwise, we accept 'raw',
+          # and unrecognized values default to 'obfuscated'
+          def set_sql_recording!
+            record_sql_config = sampler_config.fetch('record_sql', :obfuscated)
+            case record_sql_config.to_s
+            when 'off'
+              @record_sql = :off
+            when 'none'
+              @record_sql = :off
+            when 'false'
+              @record_sql = :off
+            when 'raw'
+              @record_sql = :raw
+            else
+              @record_sql = :obfuscated
+            end
+
+            log_sql_transmission_warning?
+          end
+
+          # Warn the user when we are sending raw sql across the wire
+          # - they should probably be using ssl when this is true
+          def log_sql_transmission_warning?
+            log.warn("Agent is configured to send raw SQL to the service") if @record_sql == :raw
+          end
+
+          # gets the sampler configuration from the control object's settings
+          def sampler_config
+            control.fetch('transaction_tracer', {})
+          end
+
           # Asks the collector to tell us which sub-collector we
           # should be reporting to, and then does the name resolution
           # on that host so we don't block on DNS during the normal
@@ -868,7 +863,10 @@ module NewRelic
             @report_period = config_data['data_report_period']
             @url_rules = config_data['url_rules']
             @beacon_configuration = BeaconConfiguration.new(config_data)
+            @server_side_config_enabled = config_data['listen_to_server_config']
 
+            control.merge_server_side_config(config_data) if @server_side_config_enabled
+            config_transaction_tracer
             log_connection!(config_data)
             configure_transaction_tracer!(config_data['collect_traces'], config_data['sample_rate'])
             configure_error_collector!(config_data['collect_errors'])
@@ -1043,6 +1041,20 @@ module NewRelic
           @traces
         end
 
+        def harvest_and_send_slowest_sql
+          # FIXME add the code to try to resend if our connection is down
+          sql_traces = @sql_sampler.harvest
+          unless sql_traces.empty?
+            log.debug "Sending (#{sql_traces.count}) sql traces"
+            begin
+              response = invoke_remote :sql_trace_data, sql_traces
+#              log.debug "Sql trace response: #{response}"
+            rescue
+              @sql_sampler.merge sql_traces 
+            end
+          end
+        end
+
         # This handles getting the transaction traces and then sending
         # them across the wire.  This includes gathering SQL
         # explanations, stripping out stack traces, and normalizing
@@ -1257,6 +1269,7 @@ module NewRelic
             NewRelic::Agent.load_data
             harvest_and_send_errors
             harvest_and_send_slowest_sample
+            harvest_and_send_slowest_sql
             harvest_and_send_timeslice_data
           else
             log.debug "Serializing agent data to disk"
@@ -1289,17 +1302,6 @@ module NewRelic
             log.debug "Bypassing graceful disconnect - agent not connected"
           end
         end
-        def default_sql_obfuscator(sql)
-          sql = sql.dup
-          # This is hardly readable.  Use the unit tests.
-          # remove single quoted strings:
-          sql.gsub!(/'(.*?[^\\'])??'(?!')/, '?')
-          # remove double quoted strings:
-          sql.gsub!(/"(.*?[^\\"])??"(?!")/, '?')
-          # replace all number literals
-          sql.gsub!(/\d+/, "?")
-          sql
-        end
       end
 
       extend ClassMethods

data/lib/new_relic/agent/database.rb

@@ -0,0 +1,203 @@
+require 'singleton'
+
+module NewRelic
+  # columns for a mysql explain plan
+  MYSQL_EXPLAIN_COLUMNS = [
+                           "Id",
+                           "Select Type",
+                           "Table",
+                           "Type",
+                           "Possible Keys",
+                           "Key",
+                           "Key Length",
+                           "Ref",
+                           "Rows",
+                           "Extra"
+                          ].freeze
+  
+  module Agent
+    module Database
+      extend self
+      
+      def obfuscate_sql(sql)
+        Obfuscator.instance.obfuscator.call(sql)
+      end
+      
+      def set_sql_obfuscator(type, &block)
+        Obfuscator.instance.set_sql_obfuscator(type, &block)
+      end
+      
+      def get_connection(config)
+        ConnectionManager.instance.get_connection(config)
+      end
+
+      def close_connections
+        ConnectionManager.instance.close_connections
+      end
+      
+      # Perform this in the runtime environment of a managed
+      # application, to explain the sql statement executed within a
+      # segment of a transaction sample. Returns an array of
+      # explanations (which is an array rows consisting of an array of
+      # strings for each column returned by the the explain query)
+      # Note this happens only for statements whose execution time
+      # exceeds a threshold (e.g. 500ms) and only within the slowest
+      # transaction in a report period, selected for shipment to New
+      # Relic
+      def explain_sql(sql, connection_config)
+        return nil unless sql && connection_config
+        statement = sql.split(";\n")[0] # only explain the first
+        explain_sql = explain_statement(statement, connection_config)
+        return explain_sql || []
+      end
+      
+      def explain_statement(statement, config)
+        if is_select?(statement)
+          handle_exception_in_explain do
+            connection = get_connection(config)
+            plan = nil
+            if connection
+              plan = process_resultset(connection.execute("EXPLAIN #{statement}"))
+            end
+            return plan
+          end
+        end
+      end
+      
+      def process_resultset(items)
+        # The resultset type varies for different drivers.  Only thing you can count on is
+        # that it implements each.  Also: can't use select_rows because the native postgres
+        # driver doesn't know that method.
+        
+        headers = []
+        values = []
+        if items.respond_to?(:each_hash)
+          items.each_hash do |row|
+            headers = row.keys
+            values << headers.map{|h| row[h] }
+          end
+        elsif items.respond_to?(:each)
+          items.each do |row|
+            if row.kind_of?(Hash)
+              headers = row.keys
+              values << headers.map{|h| row[h] }
+            else
+              values << row
+            end
+          end
+        else
+          values = [items]
+        end
+        
+        headers = nil if headers.empty?
+        [headers, values]
+      end
+
+      def handle_exception_in_explain
+        yield
+      rescue Exception => e
+        begin
+          # guarantees no throw from explain_sql
+          NewRelic::Control.instance.log.error("Error getting query plan: #{e.message}")
+          NewRelic::Control.instance.log.debug(e.backtrace.join("\n"))
+        rescue Exception
+          # double exception. throw up your hands
+        end
+      end
+      
+      def is_select?(statement)
+        # split the string into at most two segments on the
+        # system-defined field separator character
+        first_word, rest_of_statement = statement.split($;, 2)
+        (first_word.upcase == 'SELECT')
+      end
+
+      class ConnectionManager
+        include Singleton
+        
+        # Returns a cached connection for a given ActiveRecord
+        # configuration - these are stored or reopened as needed, and if
+        # we cannot get one, we ignore it and move on without explaining
+        # the sql
+        def get_connection(config)
+          @connections ||= {}
+          
+          connection = @connections[config]
+          
+          return connection if connection
+          
+          begin
+            connection = ActiveRecord::Base.send("#{config[:adapter]}_connection", config)
+            @connections[config] = connection
+          rescue => e
+            NewRelic::Agent.agent.log.error("Caught exception #{e} trying to get connection to DB for explain. Control: #{config}")
+            NewRelic::Agent.agent.log.error(e.backtrace.join("\n"))
+            nil
+          end
+        end
+        
+        # Closes all the connections in the internal connection cache
+        def close_connections
+          @connections ||= {}
+          @connections.values.each do |connection|
+            begin
+              connection.disconnect!
+            rescue
+            end
+          end
+          
+          @connections = {}
+        end
+      end
+
+      class Obfuscator
+        include Singleton
+        
+        attr_reader :obfuscator
+        
+        def initialize
+          reset
+        end
+
+        def reset
+          @obfuscator = method(:default_sql_obfuscator)
+        end
+        
+        # Sets the sql obfuscator used to clean up sql when sending it
+        # to the server. Possible types are:
+        #
+        # :before => sets the block to run before the existing
+        # obfuscators
+        #
+        # :after => sets the block to run after the existing
+        # obfuscator(s)
+        #
+        # :replace => removes the current obfuscator and replaces it
+        # with the provided block
+        def set_sql_obfuscator(type, &block)
+          if type == :before
+            @obfuscator = NewRelic::ChainedCall.new(block, @obfuscator)
+          elsif type == :after
+            @obfuscator = NewRelic::ChainedCall.new(@obfuscator, block)
+          elsif type == :replace
+            @obfuscator = block
+          else
+            fail "unknown sql_obfuscator type #{type}"
+          end
+        end
+        
+        def default_sql_obfuscator(sql)
+          sql = sql.dup
+          # This is hardly readable.  Use the unit tests.
+          # remove single quoted strings:
+          sql.gsub!(/'(.*?[^\\'])??'(?!')/, '?')
+          # remove double quoted strings:
+          sql.gsub!(/"(.*?[^\\"])??"(?!")/, '?')
+          # replace all number literals
+          sql.gsub!(/\d+/, "?")
+          sql
+        end
+      end
+    end
+  end
+end