newrelic_rpm 3.7.2.195 → 3.7.3.199

data/CHANGELOG

@@ -1,5 +1,29 @@
 # New Relic Ruby Agent Release Notes #
 
+## v3.7.3 ##
+
+  * Obfuscation for PostgreSQL explain plans
+
+  Fixes an agent bug with PostgreSQL where parameters from the original query
+  could appear in explain plans sent to New Relic servers, even when SQL
+  obfuscation was enabled. Parameters from the query are now masked in explain
+  plans prior to transmission when transaction_tracer.record_sql is set to
+  'obfuscated' (the default setting).
+
+  For more information, see:
+  https://docs.newrelic.com/docs/traces/security-for-postgresql-explain-plans
+
+  * More accurate categorization of SQL statements
+
+  Some SQL SELECT statements that were previously being mis-categorized as
+  'SQL - OTHER' will now correctly be tagged as 'SQL - SELECT'. This
+  particularly affected ActiveRecord users using PostgreSQL.
+
+  * More reliable Typhoeus instrumentation
+
+  Fixed an issue where an exception raised from a user-specified on_complete
+  block would cause our Typhoeus instrumentation to fail to record the request.
+
 ## v3.7.2 ##
 
   * Mongo instrumentation improvements
@@ -25,7 +49,7 @@
 
   * Improved Unicorn 4.8.0 compatibility
 
-  A rare issue that could lead to spurrious traced errors on app startup for
+  A rare issue that could lead to spurious traced errors on app startup for
   applications using Unicorn 4.8.0 has been fixed.
 
   * meta_request gem compatibility

data/Rakefile

@@ -78,3 +78,10 @@ task :record_build, [ :build_number, :stage ] do |t, args|
     f.write("module NewRelic; module VERSION; BUILD='#{build_string}'; end; end\n")
   end
 end
+
+task :console do
+  require 'pry'
+  require 'newrelic_rpm'
+  ARGV.clear
+  Pry.start
+end

data/lib/new_relic/agent/agent.rb

@@ -21,6 +21,7 @@ require 'new_relic/agent/cross_app_monitor'
 require 'new_relic/agent/request_sampler'
 require 'new_relic/agent/sampler_collection'
 require 'new_relic/agent/javascript_instrumentor'
+require 'new_relic/agent/vm/monotonic_gc_profiler'
 require 'new_relic/environment_report'
 
 module NewRelic
@@ -52,6 +53,7 @@ module NewRelic
         @harvest_samplers      = NewRelic::Agent::SamplerCollection.new(@events)
         @javascript_instrumentor = NewRelic::Agent::JavascriptInstrumentor.new(@events)
         @harvester             = NewRelic::Agent::Harvester.new(@events)
+        @monotonic_gc_profiler = NewRelic::Agent::VM::MonotonicGCProfiler.new
 
         @connect_state      = :pending
         @connect_attempts   = 0
@@ -105,6 +107,8 @@ module NewRelic
         # the latter during harvest.
         attr_reader :transaction_rules
         attr_reader :harvest_lock
+        # GC::Profiler.total_time is not monotonic so we wrap it.
+        attr_reader :monotonic_gc_profiler
 
         # fakes out a transaction that did not happen in this process
         # by creating apdex, summary metrics, and recording statistics
@@ -836,23 +840,25 @@ module NewRelic
         end
         include Connect
 
-        # Accepts an array of (metrics, transaction_traces, errors) and merges
-        # it into our current collection of data to send. Can be
-        # dangerous if we re-merge the same data more than once - it
-        # will be sent multiple times.
-        def merge_data_from(data)
-          metrics, transaction_traces, errors = data
-          @stats_engine.merge!(metrics) if metrics
-          if transaction_traces && transaction_traces.respond_to?(:any?) &&
-              transaction_traces.any?
-            @transaction_sampler.merge!(transaction_traces)
+        def container_for_endpoint(endpoint)
+          case endpoint
+          when :metric_data             then @stats_engine
+          when :transaction_sample_data then @transaction_sampler
+          when :error_data              then @error_collector
+          when :analytic_event_data     then @request_sampler
+          when :sql_trace_data          then @sql_sampler
           end
-          if errors && errors.respond_to?(:each)
-            @error_collector.merge!(errors)
+        end
+
+        def merge_data_for_endpoint(endpoint, data)
+          if data && !data.empty?
+            container_for_endpoint(endpoint).merge!(data)
           end
+        rescue => e
+          NewRelic::Agent.logger.error("Error while merging #{endpoint} data from child: ", e)
         end
 
-        public :merge_data_from
+        public :merge_data_for_endpoint
 
         # Connect to the server and validate the license.  If successful,
         # connected? returns true when finished.  If not successful, you can

data/lib/new_relic/agent/configuration/default_source.rb

@@ -147,6 +147,17 @@ module NewRelic
           Proc.new { NewRelic::Agent.config[:developer] }
         end
 
+        def self.profiling_available
+          Proc.new {
+            begin
+              require 'ruby-prof'
+              true
+            rescue LoadError
+              false
+            end
+          }
+        end
+
         def self.monitor_mode
           Proc.new { NewRelic::Agent.config[:enabled] }
         end
@@ -283,6 +294,18 @@ module NewRelic
           :type => Boolean,
           :description => 'Alternative method of enabling developer_mode.'
         },
+        :'profiling.available' => {
+          :default => DefaultSource.profiling_available,
+          :public => false,
+          :type => Boolean,
+          :description => 'Determines if ruby-prof is available for developer mode profiling.'
+        },
+        :'profiling.enabled' => {
+          :default => false,
+          :public => false,
+          :type => Boolean,
+          :description => 'Determines at runtime whether developer mode should be profiling or not.'
+        },
         :apdex_t => {
           :default => 0.5,
           :public => true,

data/lib/new_relic/agent/cross_app_tracing.rb

@@ -153,6 +153,8 @@ module NewRelic
       def inject_request_headers( request )
         cross_app_id = NewRelic::Agent.config[:cross_process_id] or
           raise NewRelic::Agent::CrossAppTracing::Error, "no cross app ID configured"
+
+        NewRelic::Agent::TransactionState.get.is_cross_app_caller = true
         txn_guid = NewRelic::Agent::TransactionState.get.request_guid
         txn_data = NewRelic::JSONWrapper.dump([ txn_guid, false ])
 

data/lib/new_relic/agent/database.rb

@@ -3,6 +3,9 @@
 # See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
 
 require 'singleton'
+require 'new_relic/agent/database/obfuscation_helpers'
+require 'new_relic/agent/database/obfuscator'
+require 'new_relic/agent/database/postgres_explain_obfuscator'
 
 module NewRelic
   # columns for a mysql explain plan
@@ -78,6 +81,18 @@ module NewRelic
         ConnectionManager.instance.close_connections
       end
 
+      # This takes a connection config hash from ActiveRecord or Sequel and
+      # returns a string describing the associated database adapter
+      def adapter_from_config(config)
+        if config[:adapter]
+          return config[:adapter].to_s
+        elsif config[:uri] && config[:uri].to_s =~ /^jdbc:([^:]+):/
+          # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or
+          # jdbc-sqlite3 gems.
+          return $1
+        end
+      end
+
       # Perform this in the runtime environment of a managed
       # application, to explain the sql statement executed within a
       # segment of a transaction sample. Returns an array of
@@ -94,6 +109,8 @@ module NewRelic
         return explain_plan || []
       end
 
+      SUPPORTED_ADAPTERS_FOR_EXPLAIN = %w[postgres postgresql mysql2 mysql sqlite].freeze
+
       def explain_statement(statement, config, &explainer)
         return unless is_select?(statement)
 
@@ -107,40 +124,100 @@ module NewRelic
           return
         end
 
+        adapter = adapter_from_config(config)
+        if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
+          NewRelic::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
+          return
+        end
+
         handle_exception_in_explain do
           start = Time.now
           plan = explainer.call(config, statement)
           ::NewRelic::Agent.record_metric("Supportability/Database/execute_explain_plan", Time.now - start)
-          return process_resultset(plan) if plan
+          return process_resultset(plan, adapter) if plan
         end
       end
 
-      def process_resultset(items)
-        # The resultset type varies for different drivers.  Only thing you can count on is
-        # that it implements each.  Also: can't use select_rows because the native postgres
-        # driver doesn't know that method.
+      def process_resultset(results, adapter)
+        case adapter.to_s
+        when 'postgres', 'postgresql'
+          process_explain_results_postgres(results)
+        when 'mysql2'
+          process_explain_results_mysql2(results)
+        when 'mysql'
+          process_explain_results_mysql(results)
+        when 'sqlite'
+          process_explain_results_sqlite(results)
+        end
+      end
+
+      QUERY_PLAN = 'QUERY PLAN'.freeze
+
+      def process_explain_results_postgres(results)
+        if results.is_a?(String)
+          query_plan_string = results
+        else
+          lines = []
+          results.each { |row| lines << row[QUERY_PLAN] }
+          query_plan_string = lines.join("\n")
+        end
+
+        unless record_sql_method == :raw
+          query_plan_string = NewRelic::Agent::Database::PostgresExplainObfuscator.obfuscate(query_plan_string)
+        end
+        values = query_plan_string.split("\n").map { |line| [line] }
 
+        [[QUERY_PLAN], values]
+      end
+
+      # Sequel returns explain plans as just one big pre-formatted String
+      # In that case, we send a nil headers array, and the single string
+      # wrapped in an array for the values.
+      # Note that we don't use this method for Postgres explain plans, since
+      # they need to be passed through the explain plan obfuscator first.
+      def string_explain_plan_results(results)
+        [nil, [results]]
+      end
+
+      def process_explain_results_mysql(results)
+        return string_explain_plan_results(results) if results.is_a?(String)
         headers = []
-        values = []
-        if items.respond_to?(:each_hash)
-          items.each_hash do |row|
-            headers = row.keys
-            values << headers.map{|h| row[h] }
-          end
-        elsif items.respond_to?(:each)
-          items.each do |row|
-            if row.kind_of?(Hash)
-              headers = row.keys
-              values << headers.map{|h| row[h] }
-            else
-              values << row
-            end
+        values  = []
+        if results.is_a?(Array)
+          # We're probably using the jdbc-mysql gem for JRuby, which will give
+          # us an array of hashes.
+          headers = results.first.keys
+          results.each do |row|
+            values << headers.map { |h| row[h] }
           end
         else
-          values = [items]
+          # We're probably using the native mysql driver gem, which will give us
+          # a Mysql::Result object that responds to each_hash
+          results.each_hash do |row|
+            headers = row.keys
+            values << headers.map { |h| row[h] }
+          end
         end
+        [headers, values]
+      end
+
+      def process_explain_results_mysql2(results)
+        return string_explain_plan_results(results) if results.is_a?(String)
+        headers = results.fields
+        values  = []
+        results.each { |row| values << row }
+        [headers, values]
+      end
 
-        headers = nil if headers.empty?
+      SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
+
+      def process_explain_results_sqlite(results)
+        return string_explain_plan_results(results) if results.is_a?(String)
+        headers = SQLITE_EXPLAIN_COLUMNS
+        values  = []
+        results.each do |row|
+          values << headers.map { |h| row[h] }
+        end
         [headers, values]
       end
 
@@ -157,11 +234,32 @@ module NewRelic
         end
       end
 
+      KNOWN_OPERATIONS = [
+        'alter',
+        'select',
+        'update',
+        'delete',
+        'insert',
+        'create',
+        'show',
+        'set',
+        'exec',
+        'execute',
+        'call'
+      ]
+
+      SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
+
+      def parse_operation_from_query(sql)
+        sql = sql.gsub(SQL_COMMENT_REGEX, '')
+        if sql =~ /(\w+)/
+          op = $1.downcase
+          return op if KNOWN_OPERATIONS.include?(op)
+        end
+      end
+
       def is_select?(statement)
-        # split the string into at most two segments on the
-        # system-defined field separator character
-        first_word, rest_of_statement = statement.split($;, 2)
-        (first_word.upcase == 'SELECT')
+        parse_operation_from_query(statement) == 'select'
       end
 
       def parameterized?(statement)
@@ -204,75 +302,6 @@ module NewRelic
         end
       end
 
-      class Obfuscator
-        include Singleton
-
-        attr_reader :obfuscator
-
-        def initialize
-          reset
-        end
-
-        def reset
-          @obfuscator = method(:default_sql_obfuscator)
-        end
-
-        # Sets the sql obfuscator used to clean up sql when sending it
-        # to the server. Possible types are:
-        #
-        # :before => sets the block to run before the existing
-        # obfuscators
-        #
-        # :after => sets the block to run after the existing
-        # obfuscator(s)
-        #
-        # :replace => removes the current obfuscator and replaces it
-        # with the provided block
-        def set_sql_obfuscator(type, &block)
-          if type == :before
-            @obfuscator = NewRelic::ChainedCall.new(block, @obfuscator)
-          elsif type == :after
-            @obfuscator = NewRelic::ChainedCall.new(@obfuscator, block)
-          elsif type == :replace
-            @obfuscator = block
-          else
-            fail "unknown sql_obfuscator type #{type}"
-          end
-        end
-
-        def default_sql_obfuscator(sql)
-          if sql[-3,3] == '...'
-            return "Query too large (over 16k characters) to safely obfuscate"
-          end
-
-          stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
-          adapter = stmt.adapter
-          obfuscated = remove_escaped_quotes(stmt)
-          obfuscated = obfuscate_single_quote_literals(obfuscated)
-          if !(adapter.to_s =~ /postgres/ || adapter.to_s =~ /sqlite/)
-            obfuscated = obfuscate_double_quote_literals(obfuscated)
-          end
-          obfuscated = obfuscate_numeric_literals(obfuscated)
-          obfuscated.to_s # return back to a regular String
-        end
-
-        def remove_escaped_quotes(sql)
-          sql.gsub(/\\"/, '').gsub(/\\'/, '')
-        end
-
-        def obfuscate_single_quote_literals(sql)
-          sql.gsub(/'(?:[^']|'')*'/, '?')
-        end
-
-        def obfuscate_double_quote_literals(sql)
-          sql.gsub(/"(?:[^"]|"")*"/, '?')
-        end
-
-        def obfuscate_numeric_literals(sql)
-          sql.gsub(/\b\d+\b/, "?")
-        end
-      end
-
       class Statement < String
         attr_accessor :adapter, :config, :explainer
       end

data/lib/new_relic/agent/database/obfuscation_helpers.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module NewRelic
+  module Agent
+    module Database
+      module ObfuscationHelpers
+        NUMERICS = /\b\d+\b/
+        SINGLE_QUOTES = /'(?:[^']|'')*'/
+        DOUBLE_QUOTES = /"(?:[^"]|"")*"/
+
+        def remove_escaped_quotes(sql)
+          sql.gsub(/\\"/, '').gsub(/\\'/, '')
+        end
+
+        def obfuscate_single_quote_literals(sql)
+          sql.gsub(SINGLE_QUOTES, '?')
+        end
+
+        def obfuscate_double_quote_literals(sql)
+          sql.gsub(DOUBLE_QUOTES, '?')
+        end
+
+        def obfuscate_numeric_literals(sql)
+          sql.gsub(NUMERICS, "?")
+        end
+
+        def find_literals(sql)
+          literals = sql.scan(NUMERICS)
+          literals << sql.scan(SINGLE_QUOTES)
+          literals.flatten
+        end
+      end
+    end
+  end
+end