newrelic_rpm 3.7.2.195 → 3.7.3.199

data/lib/new_relic/agent/database/obfuscator.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/database/obfuscation_helpers'
+
+module NewRelic
+  module Agent
+    module Database
+      class Obfuscator
+        include Singleton
+        include ObfuscationHelpers
+
+        attr_reader :obfuscator
+
+        def initialize
+          reset
+        end
+
+        def reset
+          @obfuscator = method(:default_sql_obfuscator)
+        end
+
+        # Sets the sql obfuscator used to clean up sql when sending it
+        # to the server. Possible types are:
+        #
+        # :before => sets the block to run before the existing
+        # obfuscators
+        #
+        # :after => sets the block to run after the existing
+        # obfuscator(s)
+        #
+        # :replace => removes the current obfuscator and replaces it
+        # with the provided block
+        def set_sql_obfuscator(type, &block)
+          if type == :before
+            @obfuscator = NewRelic::ChainedCall.new(block, @obfuscator)
+          elsif type == :after
+            @obfuscator = NewRelic::ChainedCall.new(@obfuscator, block)
+          elsif type == :replace
+            @obfuscator = block
+          else
+            fail "unknown sql_obfuscator type #{type}"
+          end
+        end
+
+        def default_sql_obfuscator(sql)
+          if sql[-3,3] == '...'
+            return "Query too large (over 16k characters) to safely obfuscate"
+          end
+
+          stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
+          adapter = stmt.adapter
+          obfuscated = remove_escaped_quotes(stmt)
+          obfuscated = obfuscate_single_quote_literals(obfuscated)
+          if !(adapter.to_s =~ /postgres/ || adapter.to_s =~ /sqlite/)
+            obfuscated = obfuscate_double_quote_literals(obfuscated)
+          end
+          obfuscated = obfuscate_numeric_literals(obfuscated)
+          obfuscated.to_s # return back to a regular String
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/database/postgres_explain_obfuscator.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/database'
+require 'new_relic/agent/database/obfuscation_helpers'
+
+module NewRelic
+  module Agent
+    module Database
+      module PostgresExplainObfuscator
+        extend self
+
+        extend ObfuscationHelpers
+
+        SINGLE_OR_DOUBLE_QUOTES = Regexp.new([
+          ObfuscationHelpers::SINGLE_QUOTES.source,
+          ObfuscationHelpers::DOUBLE_QUOTES.source
+        ].join('|')).freeze
+        LABEL_LINE_REGEX   = /^([^:\n]*:\s+).*$/.freeze
+
+        def obfuscate(explain)
+          # First, we replace all single-quoted strings.
+          # This is necessary in order to deal with multi-line string constants
+          # embedded in the explain output.
+          #
+          # Note that we look for both single or double quotes but do not
+          # replace double quotes in order to avoid accidentally latching onto a
+          # single quote character embedded within a quoted identifier (such as
+          # a table name).
+          #
+          # Note also that we make no special provisions for backslash-escaped
+          # single quotes (\') because these are canonicalized to two single
+          # quotes ('') in the explain output.
+          explain.gsub!(SINGLE_OR_DOUBLE_QUOTES) do |match|
+            match.start_with?('"') ? match : '?'
+          end
+
+          # Now, mask anything after the first colon (:).
+          # All parts of the query that can appear in the explain output are
+          # prefixed with "<label>: ", so we want to preserve the label, but
+          # remove the rest of the line.
+          explain.gsub!(LABEL_LINE_REGEX,   '\1?')
+          explain
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/datastores/mongo/metric_translator.rb

@@ -52,7 +52,8 @@ module NewRelic
           def self.build_metrics(name, collection, request_type = :web)
             default_metrics = [
               "Datastore/statement/MongoDB/#{collection}/#{name}",
-              "Datastore/operation/MongoDB/#{name}"
+              "Datastore/operation/MongoDB/#{name}",
+              "Datastore/all"
             ]
 
             if request_type == :web

data/lib/new_relic/agent/instrumentation/action_controller_subscriber.rb

@@ -18,7 +18,7 @@ module NewRelic
                         Agent.logger.debug("Error creating Rack::Request object: #{e}")
                         nil
                       end
-            TransactionState.reset(request)
+            TransactionState.request = request
           end
         end
 

data/lib/new_relic/agent/instrumentation/active_record_helper.rb

@@ -34,11 +34,10 @@ module NewRelic
         def metric_for_sql(sql)
           metric = NewRelic::Agent::Transaction.database_metric_name
           if metric.nil?
-            if sql =~ /^(select|update|insert|delete|show)/i
-              # Could not determine the model/operation so let's find a better
-              # metric.  If it doesn't match the regex, it's probably a show
-              # command or some DDL which we'll ignore.
-              metric = "Database/SQL/#{$1.downcase}"
+            operation = NewRelic::Agent::Database.parse_operation_from_query(sql)
+            if operation
+              # Could not determine the model/operation so use a fallback metric
+              metric = "Database/SQL/#{operation}"
             else
               metric = "Database/SQL/other"
             end
@@ -48,17 +47,19 @@ module NewRelic
 
         # Given a metric name such as "ActiveRecord/model/action" this
         # returns an array of rollup metrics:
-        # [ "ActiveRecord/all", "ActiveRecord/action" ]
+        # [ "Datastore/all", "ActiveRecord/all", "ActiveRecord/action" ]
         # If the metric name is in the form of "ActiveRecord/action"
-        # this returns merely: [ "ActiveRecord/all" ]
+        # this returns merely: [ "Datastore/all", "ActiveRecord/all" ]
         def rollup_metrics_for(metric)
-          metrics = []
+          metrics = ["Datastore/all"]
 
           # If we're outside of a web transaction, don't record any rollup
           # database metrics. This is to prevent metrics from background tasks
           # from polluting the metrics used to drive overview graphs.
           if NewRelic::Agent::Transaction.recording_web_transaction?
             metrics << "ActiveRecord/all"
+          else
+            metrics << "Datastore/allOther"
           end
           metrics << "ActiveRecord/#{$1}" if metric =~ /ActiveRecord\/[\w|\:]+\/(\w+)/
 

data/lib/new_relic/agent/instrumentation/controller_instrumentation.rb

@@ -310,8 +310,7 @@ module NewRelic
         # @api public
         #
         def perform_action_with_newrelic_trace(*args, &block)
-          request = newrelic_request(args)
-          NewRelic::Agent::TransactionState.reset(request)
+          NewRelic::Agent::TransactionState.request = newrelic_request(args)
 
           # Skip instrumentation based on the value of 'do_not_trace' and if
           # we aren't calling directly with a block.
@@ -322,9 +321,6 @@ module NewRelic
             end
           end
 
-          control = NewRelic::Control.instance
-          return perform_action_with_newrelic_profile(args, &block) if control.profiling?
-
           txn = _start_transaction(block_given? ? args : [])
           begin
             options = { :force => txn.force_flag, :transaction => true }
@@ -413,39 +409,6 @@ module NewRelic
 
         private
 
-        # Profile the instrumented call.  Dev mode only.  Experimental
-        # - should definitely not be used on production applications
-        def perform_action_with_newrelic_profile(args)
-          txn = _start_transaction(block_given? ? args : [])
-          val = nil
-          options = { :metric => true }
-
-          _, expected_scope = NewRelic::Agent::MethodTracer::TraceExecutionScoped.trace_execution_scoped_header(options, txn.start_time.to_f)
-
-          NewRelic::Agent.disable_all_tracing do
-            # turn on profiling
-            profile = RubyProf.profile do
-              if block_given?
-                val = yield
-              else
-                val = perform_action_without_newrelic_trace(*args)
-              end
-            end
-            NewRelic::Agent.instance.transaction_sampler.notice_profile profile
-          end
-
-          return val
-        ensure
-          end_time = Time.now
-          txn.freeze_name
-          metric_names = Array(recorded_metrics(txn))
-          txn_name = metric_names.shift
-
-          NewRelic::Agent::MethodTracer::TraceExecutionScoped.trace_execution_scoped_footer(txn.start_time.to_f, txn_name, metric_names, expected_scope, options, end_time.to_f)
-
-          txn = Transaction.stop(txn_name, end_time)
-        end
-
         # Write a transaction onto a thread local if there isn't already one there.
         # If there is one, just update it.
         def _start_transaction(args) # :nodoc:

data/lib/new_relic/agent/instrumentation/rubyprof.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+DependencyDetection.defer do
+  named :rubyprof
+
+  depends_on do
+    defined?(::RubyProf)
+  end
+
+  executes do
+    NewRelic::Agent.instance.events.subscribe(:start_transaction) do
+      if NewRelic::Agent.config[:'profiling.enabled']
+        ::RubyProf.start
+      end
+    end
+
+    NewRelic::Agent.instance.events.subscribe(:transaction_finished) do
+      if NewRelic::Agent.config[:'profiling.enabled']
+        trace = NewRelic::Agent::TransactionState.get.transaction.transaction_trace
+        trace.profile = ::RubyProf.stop
+      end
+    end
+  end
+end

data/lib/new_relic/agent/instrumentation/typhoeus.rb

@@ -64,10 +64,11 @@ module NewRelic::Agent::Instrumentation::TyphoeusTracing
     if NewRelic::Agent.is_execution_traced? && !request_is_hydra_enabled?(request)
       wrapped_request = ::NewRelic::Agent::HTTPClients::TyphoeusHTTPRequest.new(request)
       t0, segment = ::NewRelic::Agent::CrossAppTracing.start_trace(wrapped_request)
-      request.on_complete do
+      callback = Proc.new do
         wrapped_response = ::NewRelic::Agent::HTTPClients::TyphoeusHTTPResponse.new(request.response)
         ::NewRelic::Agent::CrossAppTracing.finish_trace(t0, segment, wrapped_request, wrapped_response)
-      end if t0
+      end
+      request.on_complete.unshift(callback)
     end
   end
 end

data/lib/new_relic/agent/pipe_channel_manager.rb

@@ -42,8 +42,22 @@ module NewRelic
       # pipe in the parent and child processes is essential in order for the EOF
       # to be correctly triggered. The ready marker mechanism is used because
       # there's no easy hook for after_fork in the parent process.
+      #
+      # This class provides message framing (separation of individual messages),
+      # but not serialization. Serialization / deserialization is the
+      # responsibility of clients.
+      #
+      # Message framing works like this:
+      #
+      # Each message sent across the pipe is preceded by a length tag that
+      # specifies the length of the message that immediately follows, in bytes.
+      # The length tags are serialized as unsigned big-endian long values, (4
+      # bytes each). This means that the maximum theoretical message size is
+      # 4 GB - much larger than we'd ever need or want for this application.
+      #
       class Pipe
         READY_MARKER = "READY"
+        NUM_LENGTH_BYTES = 4
 
         attr_accessor :in, :out
         attr_reader :last_read, :parent_pid
@@ -62,18 +76,37 @@ module NewRelic
           @in.close unless @in.closed?
         end
 
+        def serialize_message_length(data)
+          [data.bytesize].pack("L>")
+        end
+
+        def deserialize_message_length(data)
+          data.unpack("L>").first
+        end
+
         def write(data)
           @out.close unless @out.closed?
-          @in << NewRelic::LanguageSupport.with_cautious_gc do
-            Marshal.dump(data)
-          end
-          @in << "\n\n"
+          @in << serialize_message_length(data)
+          @in << data
         end
 
         def read
           @in.close unless @in.closed?
           @last_read = Time.now
-          @out.gets("\n\n")
+          length_bytes = @out.read(NUM_LENGTH_BYTES)
+          if length_bytes
+            message_length = deserialize_message_length(length_bytes)
+            if message_length
+              @out.read(message_length)
+            else
+              length_hex = length_bytes.bytes.map { |b| b.to_s(16) }.join(' ')
+              NewRelic::Agent.logger.error("Failed to deserialize message length from pipe. Bytes: [#{length_hex}]")
+              nil
+            end
+          else
+            NewRelic::Agent.logger.error("Failed to read bytes for length from pipe.")
+            nil
+          end
         end
 
         def eof?
@@ -176,15 +209,15 @@ module NewRelic
         def merge_data_from_pipe(pipe_handle)
           pipe = find_pipe_for_handle(pipe_handle)
           raw_payload = pipe.read
-
           if raw_payload && !raw_payload.empty?
-            payload = unmarshal(raw_payload)
-            if payload == Pipe::READY_MARKER
+            if raw_payload == Pipe::READY_MARKER
               pipe.after_fork_in_parent
-            elsif payload
-              NewRelic::Agent.agent.merge_data_from([payload[:stats],
-                                                     payload[:transaction_traces],
-                                                     payload[:error_traces]])
+            else
+              payload = unmarshal(raw_payload)
+              if payload
+                endpoint, items = payload
+                NewRelic::Agent.agent.merge_data_for_endpoint(endpoint, items)
+              end
             end
           end
 

data/lib/new_relic/agent/pipe_service.rb

@@ -28,25 +28,25 @@ module NewRelic
         []
       end
 
-      def analytic_event_data(data)
-        nil
+      def analytic_event_data(events)
+        write_to_pipe(:analytic_event_data, events) if events
       end
 
       def metric_data(unsent_timeslice_data)
-        write_to_pipe(:stats => unsent_timeslice_data)
+        write_to_pipe(:metric_data, unsent_timeslice_data)
         {}
       end
 
       def transaction_sample_data(transactions)
-        write_to_pipe(:transaction_traces => transactions) if transactions
+        write_to_pipe(:transaction_sample_data, transactions) if transactions
       end
 
       def error_data(errors)
-        write_to_pipe(:error_traces => errors) if errors
+        write_to_pipe(:error_data, errors) if errors
       end
 
       def sql_trace_data(sql)
-        write_to_pipe(:sql_traces => sql) if sql
+        write_to_pipe(:sql_trace_data, sql) if sql
       end
 
       def shutdown(time)
@@ -66,8 +66,14 @@ module NewRelic
 
       private
 
-      def write_to_pipe(data)
-        @pipe.write(data) if @pipe
+      def marshal_payload(data)
+        NewRelic::LanguageSupport.with_cautious_gc do
+          Marshal.dump(data)
+        end
+      end
+
+      def write_to_pipe(endpoint, data)
+        @pipe.write(marshal_payload([endpoint, data])) if @pipe
       end
     end
   end

data/lib/new_relic/agent/request_sampler.rb

@@ -16,10 +16,12 @@ class NewRelic::Agent::RequestSampler
   SAMPLE_TYPE              = 'Transaction'
 
   # Strings for static keys of the sample structure
-  TYPE_KEY                 = 'type'
-  TIMESTAMP_KEY            = 'timestamp'
-  NAME_KEY                 = 'name'
-  DURATION_KEY             = 'duration'
+  TYPE_KEY                       = 'type'
+  TIMESTAMP_KEY                  = 'timestamp'
+  NAME_KEY                       = 'name'
+  DURATION_KEY                   = 'duration'
+  GUID_KEY                       = 'nr.guid'
+  REFERRING_TRANSACTION_GUID_KEY = 'nr.referringTransactionGuid'
 
   def initialize( event_listener )
     super()
@@ -113,7 +115,6 @@ class NewRelic::Agent::RequestSampler
   # Event handler for the :transaction_finished event.
   def on_transaction_finished(payload)
     return unless @enabled
-    return unless NewRelic::Agent::Transaction.transaction_type_is_web?(payload[:type])
 
     main_event = create_main_event(payload)
     custom_params = create_custom_parameters(payload)
@@ -122,14 +123,70 @@ class NewRelic::Agent::RequestSampler
     notify_full if is_full && !@notified_full
   end
 
+  def self.map_metric(metric_name, to_add={})
+    to_add.values.each(&:freeze)
+
+    spec = ::NewRelic::MetricSpec.new(metric_name)
+    mappings = OVERVIEW_SPECS.fetch(spec, {})
+    mappings.merge!(to_add)
+
+    OVERVIEW_SPECS[spec] = mappings
+  end
+
+  OVERVIEW_SPECS = {}
+
+  # All Transactions
+  # Don't need to use the transaction-type specific metrics since this is
+  # scoped to just one transaction, so Datastore/all has what we want.
+  map_metric('Datastore/all',         :total_call_time => "databaseDuration")
+  map_metric('Datastore/all',         :call_count      => "databaseCallCount")
+
+  # Web Metrics
+  map_metric('WebFrontend/QueueTime', :total_call_time => "queueDuration")
+  map_metric("GC/cumulative",         :total_call_time => "gcCumulative")
+  map_metric('Memcache/allWeb',       :total_call_time => "memcacheDuration")
+
+  map_metric('External/allWeb',       :total_call_time => "externalDuration")
+  map_metric('External/allWeb',       :call_count      => "externalCallCount")
+
+  # Background Metrics
+  map_metric('Memcache/allOther',     :total_call_time => "memcacheDuration")
+
+  map_metric('External/allOther',     :total_call_time => "externalDuration")
+  map_metric('External/allOther',     :call_count      => "externalCallCount")
+
+  def extract_metrics(stats_hash)
+    result = {}
+    if stats_hash
+      OVERVIEW_SPECS.each do |(metric_spec, extracted_values)|
+        if stats_hash.has_key?(metric_spec)
+          stat = stats_hash[metric_spec]
+          extracted_values.each do |value_name, key_name|
+            result[key_name] = stat.send(value_name)
+          end
+        end
+      end
+    end
+    result
+  end
+
   def create_main_event(payload)
-    sample = payload[:overview_metrics] || {}
+    sample = extract_metrics(payload[:metrics])
     sample.merge!({
         TIMESTAMP_KEY     => float(payload[:start_timestamp]),
         NAME_KEY          => string(payload[:name]),
         DURATION_KEY      => float(payload[:duration]),
         TYPE_KEY          => SAMPLE_TYPE,
       })
+    optionally_append(GUID_KEY, :guid, sample, payload)
+    optionally_append(REFERRING_TRANSACTION_GUID_KEY, :referring_transaction_guid, sample, payload)
+    sample
+  end
+
+  def optionally_append(sample_key, payload_key, sample, payload)
+    if payload.include?(payload_key)
+      sample[sample_key] = string(payload[payload_key])
+    end
   end
 
   def create_custom_parameters(payload)