newrelic_rpm 3.7.0.177 → 3.7.1.180

data/CHANGELOG

@@ -1,10 +1,37 @@
 # New Relic Ruby Agent Release Notes #
 
+## v3.7.1 ##
+
+  * MongoDB support
+
+  The Ruby agent provides support for the mongo gem, versions 1.8 and 1.9!
+  Mongo calls are captured for transaction traces along with their parameters,
+  and time spent in Mongo shows up on the Database tab.
+
+  Support for more Mongo gems and more UI goodness will be coming, so watch
+  http://docs.newrelic.com/docs/ruby/mongo for up-to-date status.
+
+  * Harvest thread restarts for forked and daemonized processes
+
+  Historically framework specific code was necessary for the Ruby agent to
+  successfully report data after an app forked or daemonized. Gems or scripts
+  with daemonizing modes had to wait for agent support or find workarounds.
+
+  With 3.7.1 setting `restart_thread_in_child: true` in your newrelic.yml
+  automatically restarts the agent in child processes without requiring custom
+  code. For now the feature is opt-in, but future releases may default it on.
+
+  * Fix for missing HTTP time
+
+  The agent previously did not include connection establishment time for
+  outgoing Net::HTTP requests. This has been corrected, and reported HTTP
+  timings should now be more accurate.
+
 ## v3.7.0 ##
 
   * Official Rubinius support (for Rubinius >= 2.2.1)
 
-  We're happy to say that all known issues with the Ruby agent running on 
+  We're happy to say that all known issues with the Ruby agent running on
   Rubinius have been resolved as of Rubinius version 2.2.1! See
   http://docs.newrelic.com/docs/ruby/rubinius for the most up-to-date status.
 

data/lib/new_relic/agent.rb

@@ -391,8 +391,7 @@ module NewRelic
     # Yield to the block without collecting any metrics or traces in
     # any of the subsequent calls.  If executed recursively, will keep
     # track of the first entry point and turn on tracing again after
-    # leaving that block.  This uses the thread local
-    # +newrelic_untrace+
+    # leaving that block.  This uses the thread local TransactionState.
     #
     # @api public
     #

data/lib/new_relic/agent/agent.rb

@@ -10,6 +10,7 @@ require 'zlib'
 require 'stringio'
 require 'new_relic/agent/sampled_buffer'
 require 'new_relic/agent/autostart'
+require 'new_relic/agent/harvester'
 require 'new_relic/agent/new_relic_service'
 require 'new_relic/agent/pipe_service'
 require 'new_relic/agent/configuration/manager'
@@ -50,6 +51,7 @@ module NewRelic
         @request_sampler       = NewRelic::Agent::RequestSampler.new(@events)
         @harvest_samplers      = NewRelic::Agent::SamplerCollection.new(@events)
         @javascript_instrumentor = NewRelic::Agent::JavascriptInstrumentor.new(@events)
+        @harvester             = NewRelic::Agent::Harvester.new(@events)
 
         @connect_state      = :pending
         @connect_attempts   = 0
@@ -160,6 +162,8 @@ module NewRelic
         #   connection, this tells me to only try it once so this method returns
         #   quickly if there is some kind of latency with the server.
         def after_fork(options={})
+          @harvester.mark_started
+
           Agent.config.apply_config(NewRelic::Agent::Configuration::ManualSource.new(options), 1)
 
           if channel_id = options[:report_to_channel]
@@ -200,22 +204,42 @@ module NewRelic
         # Options:
         # :force_send  => (true/false) # force the agent to send data
         def shutdown(options={})
-          run_loop_before_exit = Agent.config[:force_send]
           return if not started?
+          ::NewRelic::Agent.logger.info "Starting Agent shutdown"
+
+          stop_worker_loop
+          trap_signals_for_litespeed
+          untraced_graceful_disconnect
+          revert_to_default_configuration
+
+          @started = nil
+          Control.reset
+        end
+
+        def revert_to_default_configuration
+          NewRelic::Agent.config.remove_config do |config|
+            config.class == NewRelic::Agent::Configuration::ManualSource ||
+              config.class == NewRelic::Agent::Configuration::ServerSource
+          end
+        end
+
+        def stop_worker_loop
           if @worker_loop
-            @worker_loop.run_task if run_loop_before_exit
+            @worker_loop.run_task if Agent.config[:force_send]
             @worker_loop.stop
           end
+        end
 
-          ::NewRelic::Agent.logger.info "Starting Agent shutdown"
-
+        def trap_signals_for_litespeed
           # if litespeed, then ignore all future SIGUSR1 - it's
           # litespeed trying to shut us down
           if Agent.config[:dispatcher] == :litespeed
             Signal.trap("SIGUSR1", "IGNORE")
             Signal.trap("SIGTERM", "IGNORE")
           end
+        end
 
+        def untraced_graceful_disconnect
           begin
             NewRelic::Agent.disable_all_tracing do
               graceful_disconnect
@@ -223,12 +247,6 @@ module NewRelic
           rescue => e
             ::NewRelic::Agent.logger.error e
           end
-          NewRelic::Agent.config.remove_config do |config|
-            config.class == NewRelic::Agent::Configuration::ManualSource ||
-              config.class == NewRelic::Agent::Configuration::ServerSource
-          end
-          @started = nil
-          Control.reset
         end
 
         # Tells the statistics engine we are starting a new transaction

data/lib/new_relic/agent/agent_logger.rb

@@ -143,9 +143,10 @@ module NewRelic
       end
 
       def set_log_format!
-        def @log.format_message(severity, timestamp, progname, msg)
-          prefix = @logdev.dev == STDOUT ? '** [NewRelic]' : ''
-          prefix + "[#{timestamp.strftime("%m/%d/%y %H:%M:%S %z")} #{Socket.gethostname} (#{$$})] #{severity} : #{msg}\n"
+        @hostname = Socket.gethostname
+        @prefix = wants_stdout? ? '** [NewRelic]' : ''
+        @log.formatter = Proc.new do |severity, timestamp, progname, msg|
+          "#{@prefix}[#{timestamp.strftime("%m/%d/%y %H:%M:%S %z")} #{@hostname} (#{$$})] #{severity} : #{msg}\n"
         end
       end
 

data/lib/new_relic/agent/audit_logger.rb

@@ -42,7 +42,7 @@ module NewRelic
         path = ensure_log_path
         if path
           @log = ::Logger.new(path)
-          @log.formatter = log_formatter
+          @log.formatter = create_log_formatter
           ::NewRelic::Agent.logger.info("Audit log enabled at '#{path}'")
         else
           @log = NewRelic::Agent::NullLogger.new
@@ -65,14 +65,11 @@ module NewRelic
         path
       end
 
-      def log_formatter
-        if @formatter.nil?
-          @formatter = Logger::Formatter.new
-          def @formatter.call(severity, time, progname, msg)
-            "[#{time} #{Socket.gethostname} (#{$$})] : #{msg}\n"
-          end
+      def create_log_formatter
+        @hostname = Socket.gethostname
+        Proc.new do |severity, time, progname, msg|
+          "[#{time} #{@hostname} (#{$$})] : #{msg}\n"
         end
-        @formatter
       end
     end
   end

data/lib/new_relic/agent/configuration/default_source.rb

@@ -630,6 +630,12 @@ module NewRelic
           :type => Boolean,
           :description => 'Enable or disable sequel instrumentation.'
         },
+        :disable_mongo => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :description => 'Enable or disable MongoDB instrumentation.'
+        },
         :'slow_sql.enabled' => {
           :default => DefaultSource.slow_sql_enabled,
           :public => true,
@@ -654,6 +660,18 @@ module NewRelic
           :type => String,
           :description => "Obfuscation level for slow sql queries (e.g. 'obfuscated', 'raw', 'none')."
         },
+        :'mongo.capture_queries' => {
+          :default => true,
+          :public => true,
+          :type => Boolean,
+          :description => "Enable or disable capturing Mongo queries in transaction traces."
+        },
+        :'mongo.obfuscate_queries' => {
+          :default => true,
+          :public => true,
+          :type => Boolean,
+          :description => "Enable or disable obfuscation of Mongo queries in transaction traces."
+        },
         :'error_collector.enabled' => {
           :default => true,
           :public => true,
@@ -854,6 +872,12 @@ module NewRelic
           :type => Boolean,
           :description => 'Include TT custom params in real user monitoring script in outgoing responses.'
         },
+        :restart_thread_in_children => {
+          :default => false,
+          :public => false,
+          :type => Boolean,
+          :description => 'Controls whether to check on running a transaction whether to respawn the harvest thread.'
+        },
       }.freeze
 
     end

data/lib/new_relic/agent/cross_app_tracing.rb

@@ -36,11 +36,11 @@ module NewRelic
       def trace_http_request( request )
         return yield unless NewRelic::Agent.is_execution_traced?
 
-        t0, segment = start_trace( request )
         begin
+          t0, segment = start_trace( request )
           response = yield
         ensure
-          finish_trace( t0, segment, request, response ) if t0
+          finish_trace( t0, segment, request, response )
         end
 
         return response
@@ -55,21 +55,24 @@ module NewRelic
       #          to make the request (e.g. 'Net::HTTP' or 'Typhoeus')
       # * host - Return a String with the hostname or IP of the host being
       #          communicated with.
-      # * method  - Return a String with the HTTP method name used for this request
+      # * method  - Return a String with the HTTP method name for this request
       # * [](key) - Lookup an HTTP request header by name
       # * []=(key, val) - Set an HTTP request header by name
       # * uri  - Full URI of the request
+      #
+      # This method MUST return a pair. The first item always returns the
+      # starting time of the trace, even if an error occurs. The second item is
+      # the transaction segment if it was sucessfully pushed.
       def start_trace( request )
-        inject_request_headers( request ) if cross_app_enabled?
-
-        # Create a segment and time the call
         t0 = Time.now
-        segment = stats_engine.push_scope( :net_http, t0 )
+
+        inject_request_headers( request ) if cross_app_enabled?
+        segment = stats_engine.push_scope( :http_request, t0 )
 
         return t0, segment
       rescue => err
         NewRelic::Agent.logger.error "Uncaught exception while tracing HTTP request", err
-        return nil
+        return t0, nil
       end
 
 
@@ -98,14 +101,17 @@ module NewRelic
             stats_engine.record_metrics(metrics, duration)
             stats_engine.record_metrics(scoped_metric, duration, :scoped => true)
 
-            # Add TT custom parameters
-            segment.name = scoped_metric
-            add_transaction_trace_parameters(request, response)
+            # If we don't have segment, something failed during start_trace so
+            # the current segment isn't the HTTP call it should have been.
+            if segment
+              segment.name = scoped_metric
+              add_transaction_trace_parameters(request, response)
+            end
           end
         ensure
-          # We always need to pop the scope stack to avoid an inconsistent
-          # state, which will prevent tracing of the whole transaction.
-          stats_engine.pop_scope( segment, scoped_metric, t1 )
+          # If we have a segment, always pop the scope stack to avoid an
+          # inconsistent state, which prevents tracing of whole transaction.
+          stats_engine.pop_scope( segment, scoped_metric, t1 ) if segment
         end
       rescue NewRelic::Agent::CrossAppTracing::Error => err
         NewRelic::Agent.logger.debug "while cross app tracing", err
@@ -132,7 +138,7 @@ module NewRelic
         NewRelic::Agent.config[:"cross_application_tracer.enabled"] || NewRelic::Agent.config[:cross_application_tracing]
       end
 
-      # Memoized fetcher for the cross app encoding key. Raises a
+      # Fetcher for the cross app encoding key. Raises a
       # NewRelic::Agent::CrossAppTracing::Error if the key isn't configured.
       def cross_app_encoding_key
         NewRelic::Agent.config[:encoding_key] or

data/lib/new_relic/agent/datastores/mongo.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module NewRelic
+  module Agent
+    module Datastores
+      module Mongo
+
+        def self.is_supported_version?
+          # No version constant in < 2.0 versions of Mongo :(
+          defined?(::Mongo) &&
+            defined?(::Mongo::MongoClient) &&
+            !is_version2
+        end
+
+        # At present we explicitly don't support version 2.x of the driver yet
+        def self.is_version2
+          defined?(::Mongo::VERSION) &&
+            NewRelic::VersionNumber.new(::Mongo::VERSION) > NewRelic::VersionNumber.new("2.0.0")
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/datastores/mongo/metric_generator.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/datastores/mongo/metric_translator'
+
+module NewRelic
+  module Agent
+    module Datastores
+      module Mongo
+        module MetricGenerator
+          def self.generate_metrics_for(name, payload)
+            if NewRelic::Agent::Transaction.recording_web_transaction?
+              request_type = :web
+            else
+              request_type = :other
+            end
+
+            NewRelic::Agent::Datastores::Mongo::MetricTranslator.metrics_for(name, payload, request_type)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/datastores/mongo/metric_translator.rb

@@ -0,0 +1,189 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/datastores/mongo/obfuscator'
+
+module NewRelic
+  module Agent
+    module Datastores
+      module Mongo
+        module MetricTranslator
+          def self.metrics_for(name, payload, request_type = :web)
+            payload = {} if payload.nil?
+
+            collection = payload[:collection]
+
+            if collection_in_selector?(collection, payload)
+              command_key = command_key_from_selector(payload)
+
+              name = get_name_from_selector(command_key)
+              collection = get_collection_from_selector(command_key, payload)
+              log_if_unknown_command(command_key, payload)
+            end
+
+            if self.find_one?(name, payload)
+              name = 'findOne'
+            elsif self.find_and_remove?(name, payload)
+              name = 'findAndRemove'
+            elsif self.find_and_modify?(name, payload)
+              name = 'findAndModify'
+            elsif self.create_index?(name, payload)
+              name = 'createIndex'
+              collection = self.collection_name_from_index(payload)
+            elsif self.drop_indexes?(name, payload)
+              name = 'dropIndexes'
+            elsif self.drop_index?(name, payload)
+              name = 'dropIndex'
+            elsif self.re_index?(name, payload)
+              name = 'reIndex'
+            elsif self.group?(name, payload)
+              name = 'group'
+              collection = collection_name_from_group_selector(payload)
+            elsif self.rename_collection?(name, payload)
+              name = 'renameCollection'
+              collection = collection_name_from_rename_selector(payload)
+            elsif self.ismaster?(name, payload)
+              name = 'ismaster'
+              collection = collection_name_from_ismaster_selector(payload)
+            end
+
+            build_metrics(name, collection, request_type)
+          end
+
+          def self.build_metrics(name, collection, request_type = :web)
+            default_metrics = [
+              "Datastore/statement/MongoDB/#{collection}/#{name}",
+              "Datastore/operation/MongoDB/#{name}",
+              'ActiveRecord/all'
+            ]
+
+            if request_type == :web
+              default_metrics << 'Datastore/allWeb'
+            else
+              default_metrics << 'Datastore/allOther'
+            end
+
+            default_metrics
+          end
+
+          def self.collection_in_selector?(collection, payload)
+            collection == '$cmd' && payload[:selector]
+          end
+
+          NAMES_IN_SELECTOR = [
+            :findandmodify,
+
+            "aggregate",
+            "count",
+            "group",
+            "mapreduce",
+
+            :distinct,
+
+            :deleteIndexes,
+            :reIndex,
+
+            :ismaster,
+            :collstats,
+            :renameCollection,
+            :drop,
+          ]
+
+          UNKNOWN_COMMAND = "UnknownCommand"
+          UNKNOWN_COLLECTION = "UnknownCollection"
+
+          def self.command_key_from_selector(payload)
+            selector = payload[:selector]
+            NAMES_IN_SELECTOR.find do |check_name|
+              selector.key?(check_name)
+            end
+          end
+
+          def self.get_name_from_selector(command_key)
+            return UNKNOWN_COMMAND unless command_key
+
+            command_key.to_sym
+          end
+
+          def self.get_collection_from_selector(command_key, payload)
+            return UNKNOWN_COLLECTION unless command_key
+
+            payload[:selector][command_key]
+          end
+
+          def self.log_if_unknown_command(command_key, payload)
+            unless command_key
+              NewRelic::Agent.logger.debug("Unknown Mongo command: #{Obfuscator.obfuscate_statement(payload).inspect}")
+              NewRelic::Agent.increment_metric("Supportability/Mongo/UnknownCommand")
+            end
+          end
+
+          def self.find_one?(name, payload)
+            name == :find && payload[:limit] == -1
+          end
+
+          def self.find_and_modify?(name, payload)
+            name == :findandmodify
+          end
+
+          def self.find_and_remove?(name, payload)
+            name == :findandmodify && payload[:selector] && payload[:selector][:remove]
+          end
+
+          def self.create_index?(name, payload)
+            name == :insert && payload[:collection] == "system.indexes"
+          end
+
+          def self.drop_indexes?(name, payload)
+            name == :deleteIndexes && payload[:selector] && payload[:selector][:index] == "*"
+          end
+
+          def self.drop_index?(name, payload)
+            name == :deleteIndexes
+          end
+
+          def self.re_index?(name, payload)
+            name == :reIndex && payload[:selector] && payload[:selector][:reIndex]
+          end
+
+          def self.group?(name, payload)
+            name == :group
+          end
+
+          def self.rename_collection?(name, payload)
+            name == :renameCollection
+          end
+
+          def self.ismaster?(name, payload)
+            name == :ismaster
+          end
+
+          def self.collection_name_from_index(payload)
+            if payload[:documents] && payload[:documents].first[:ns]
+              payload[:documents].first[:ns].split('.').last
+            else
+              'system.indexes'
+            end
+          end
+
+          def self.collection_name_from_group_selector(payload)
+            payload[:selector]["group"]["ns"]
+          end
+
+          def self.collection_name_from_rename_selector(payload)
+            parts = payload[:selector][:renameCollection].split('.')
+            parts.shift
+            parts.join('.')
+          end
+
+          def self.collection_name_from_ismaster_selector(payload)
+            payload[:selector][:ismaster]
+          end
+
+        end
+
+      end
+    end
+  end
+end