newrelic_rpm 3.9.4.245 → 3.9.5.251

data/CHANGELOG

@@ -1,5 +1,62 @@
 # New Relic Ruby Agent Release Notes #
 
+## v3.9.5 ##
+
+  * Per-dyno data on Heroku
+
+  When running on Heroku, data from the agent can now be broken out by dyno
+  name, allowing you to more easily see what's happening on a per-dyno level.
+  Dynos on Heroku are now treated in the same way that distinct hosts on other
+  platforms work.
+
+  By default, 'scheduler' and 'run' dyno names will be aggregated into
+  'scheduler.*' and 'run.*' to avoid unbounded growth in the number of reported
+  hostnames.
+
+  Read more about this feature on our Heroku docs page:
+  https://docs.newrelic.com/docs/agents/ruby-agent/miscellaneous/ruby-agent-heroku
+
+  * HTTP response codes in Insights events
+
+  The Ruby agent will now capture HTTP response codes from Rack applications
+  (including Rails and Sinatra apps) and include them under the httpResponseCode
+  attribute on events sent to Insights.
+
+  * Stricter limits on memory usage of metrics and SQL traces
+
+  The agent now imposes stricter limits on the number of distinct SQL traces and
+  metrics that it will buffer in memory at any point in time, leading to more
+  predictable memory consumption even in exceptional circumstances.
+
+  * Improved reliability of thread profiling
+
+  Several issues that would previously have prevented the successful completion
+  and transmission of thread profiles to New Relic's servers have been fixed.
+
+  These issues were related to the use of recursion in processing thread
+  profiles, and have been addressed by both limiting the maximum depth of the
+  backtraces recorded in thread profiles, and eliminating the agent's use of
+  recursion in processing profile data.
+
+  * Allow tracing Rails view helpers with add_method_tracer
+
+  Previously, attempting to trace a Rails view helper method using
+  add_method_tracer on the view helper module would lead to a NoMethodError
+  when the traced method was called (undefined method `trace_execution_scoped').
+  This has been fixed.
+
+  This issue was an instance of the Ruby 'dynamic module inclusion' or 'double
+  inclusion' problem. Usage of add_method_tracer now no longer relies upon the
+  target class having actually picked up the trace_execution_scoped method from
+  the NewRelic::Agent::MethodTracer module.
+
+  * Improved performance of queue time parsing
+
+  The number of objects allocated while parsing the front-end timestamps on
+  incoming HTTP requests has been significantly reduced.
+
+  Thanks to Aleksei Magusev for the contribution!
+
 ## v3.9.4 ##
 
   * Allow agent to use alternate certificate stores

data/Guardfile

@@ -4,4 +4,5 @@ guard :minitest, :test_folders => ['test/new_relic'], :all_after_pass => false d
   watch(%r{^test/rum/.*})       { "test/new_relic/rack/browser_monitoring_test.rb" }
   watch('test/test_helper.rb')  { "test/new_relic" }
   watch('test/agent_helper.rb') { "test/new_relic" }
+  watch('lib/new_relic/agent/configuration/default_source.rb') { "test/new_relic/agent/configuration/orphan_configuration_test.rb" }
 end

data/lib/new_relic/agent/agent.rb

@@ -770,7 +770,7 @@ module NewRelic
               :agent_version => NewRelic::VERSION::STRING,
               :environment => @environment_report,
               :settings => Agent.config.to_collector_hash,
-              :high_security => Agent.config[:high_security],
+              :high_security => Agent.config[:high_security]
             }
           end
 
@@ -811,8 +811,8 @@ module NewRelic
             Agent.config.replace_or_add_config(server_config)
             log_connection!(config_data) if @service
 
-            @transaction_rules = RulesEngine.from_specs(config_data['transaction_name_rules'])
-            @stats_engine.metric_rules = RulesEngine.from_specs(config_data['metric_name_rules'])
+            @transaction_rules = RulesEngine.create_transaction_rules(config_data)
+            @stats_engine.metric_rules = RulesEngine.create_metric_rules(config_data)
 
             # If you're adding something else here to respond to the server-side config,
             # use Agent.instance.events.subscribe(:finished_configuring) callback instead!

data/lib/new_relic/agent/audit_logger.rb

@@ -35,8 +35,11 @@ module NewRelic
           @log.info("REQUEST: #{uri}")
           @log.info("REQUEST BODY: #{request_body}")
         end
-      rescue SystemCallError => e
-        ::NewRelic::Agent.logger.warn("Failed writing to audit log: #{e}")
+      rescue StandardError, SystemStackError, SystemCallError => e
+        ::NewRelic::Agent.logger.warn("Failed writing to audit log", e)
+      rescue Exception => e
+        ::NewRelic::Agent.logger.warn("Failed writing to audit log with exception. Re-raising in case of interupt.", e)
+        raise
       end
 
       def setup_logger

data/lib/new_relic/agent/configuration/default_source.rb

@@ -659,7 +659,7 @@ module NewRelic
           :default => true,
           :public => true,
           :type => Boolean,
-          :description => 'Enable or disable the collection of explain plans.'
+          :description => 'Enable or disable the collection of explain plans in transaction traces. This setting will also apply to explain plans in Slow SQL traces if slow_sql.explain_enabled is not set separately.'
         },
         :'transaction_tracer.stack_trace_threshold' => {
           :default => 0.5,
@@ -709,7 +709,7 @@ module NewRelic
           :default => DefaultSource.slow_sql_explain_enabled,
           :public => true,
           :type => Boolean,
-          :description => 'Enable or disable the collection of explain plans in slow SQL queries.'
+          :description => 'Enable or disable the collection of explain plans in slow SQL queries. If this setting is omitted, the transaction_tracer.explain_enabled setting will be applied as the default setting for explain plans in Slow SQL as well.'
         },
         :'slow_sql.record_sql' => {
           :default => DefaultSource.slow_sql_record_sql,
@@ -1032,12 +1032,18 @@ module NewRelic
           :type         => Boolean,
           :description  => 'Defines whether the agent will wrap third-party middlewares in instrumentation (regardless of whether they are installed via Rack::Builder or Rails).'
         },
-        :use_heroku_dyno_names => {
-          :default      => false,
-          :public       => false,
+        :'heroku.use_dyno_names' => {
+          :default      => true,
+          :public       => true,
           :type         => Boolean,
           :description  => 'Controls whether or not we use the heroku dyno name as the hostname.'
         },
+        :'heroku.dyno_name_prefixes_to_shorten' => {
+          :default      => ['scheduler', 'run'],
+          :public       => true,
+          :type         => Array,
+          :description  => 'List of prefixes for heroku dyno names (such as "scheduler") to report as hostname without trailing dot and process ID.'
+        },
         :labels => {
           :default      => '',
           :public       => true,

data/lib/new_relic/agent/error_collector.rb

@@ -133,6 +133,19 @@ module NewRelic
           end
         end
 
+        def aggregated_metric_names(txn)
+          metric_names = ["Errors/all"]
+          return metric_names unless txn
+
+          if txn.recording_web_transaction?
+            metric_names << "Errors/allWeb"
+          else
+            metric_names << "Errors/allOther"
+          end
+
+          metric_names
+        end
+
         # Increments a statistic that tracks total error rate
         # Be sure not to double-count same exception. This clears per harvest.
         def increment_error_count!(exception, options={}) #THREAD_LOCAL_ACCESS
@@ -142,7 +155,7 @@ module NewRelic
           return if seen?(txn, exception)
           tag_as_seen(txn, exception)
 
-          metric_names = ["Errors/all"]
+          metric_names  = aggregated_metric_names(txn)
           blamed_metric = blamed_metric_name(txn, options)
           metric_names << blamed_metric if blamed_metric
 

data/lib/new_relic/agent/hostname.rb

@@ -6,12 +6,33 @@ module NewRelic
   module Agent
     module Hostname
       def self.get
-        if (dyno_name = ENV['DYNO']) && ::NewRelic::Agent.config[:use_heroku_dyno_names]
+        dyno_name = ENV['DYNO']
+        if dyno_name && ::NewRelic::Agent.config[:'heroku.use_dyno_names']
+          matching_prefix = heroku_dyno_name_prefix(dyno_name)
+          dyno_name = "#{matching_prefix}.*" if matching_prefix
           dyno_name
         else
           Socket.gethostname
         end
       end
+
+      def self.heroku_dyno_name_prefix(dyno_name)
+        get_dyno_prefixes.find do |dyno_prefix|
+          dyno_name.start_with?(dyno_prefix + ".")
+        end
+      end
+
+      # TODO: Once config transforms are in, use those instead of handcoding
+      def self.get_dyno_prefixes
+        dyno_prefixes = ::NewRelic::Agent.config[:'heroku.dyno_name_prefixes_to_shorten'] || []
+        if dyno_prefixes.is_a?(String)
+          dyno_prefixes = dyno_prefixes.split(',')
+        elsif !dyno_prefixes.respond_to?(:find)
+          ::NewRelic::Agent.logger.warn("Ignoring invalid setting found for 'heroku.dyno_name_prefixes_to_shorten', #{dyno_prefixes}.")
+          dyno_prefixes = []
+        end
+        dyno_prefixes
+      end
     end
   end
 end

data/lib/new_relic/agent/instrumentation/middleware_tracing.rb

@@ -52,10 +52,16 @@ module NewRelic
           begin
             Transaction.start(state, category, opts)
             if target == self
-              traced_call(env)
+              result = traced_call(env)
             else
-              target.call(env)
+              result = target.call(env)
             end
+
+            if result.is_a?(Array)
+              state.current_transaction.http_response_code = result[0]
+            end
+
+            result
           rescue => e
             NewRelic::Agent.notice_error(e)
             raise

data/lib/new_relic/agent/instrumentation/queue_time.rb

@@ -16,13 +16,15 @@ module NewRelic
           ALL_QUEUE_METRIC        = 'WebFrontend/QueueTime'.freeze
           # any timestamps before this are thrown out and the parser
           # will try again with a larger unit (2000/1/1 UTC)
-          EARLIEST_ACCEPTABLE_TIMESTAMP = 946684800
+          EARLIEST_ACCEPTABLE_TIME = Time.at(946684800)
 
           CANDIDATE_HEADERS = [
             REQUEST_START_HEADER,
             QUEUE_START_HEADER,
             MIDDLEWARE_START_HEADER
           ].freeze
+
+          DIVISORS = [1_000_000, 1_000, 1]
         end
 
         module_function
@@ -62,17 +64,18 @@ module NewRelic
         end
 
         def parse_timestamp(string)
-          cut_off = Time.at(EARLIEST_ACCEPTABLE_TIMESTAMP)
-          [1_000_000, 1_000, 1].map do |divisor|
+          DIVISORS.each do |divisor|
             begin
-              Time.at(string.to_f / divisor)
+              t = Time.at(string.to_f / divisor)
+              return t if t > EARLIEST_ACCEPTABLE_TIME
             rescue RangeError
               # On Ruby versions built with a 32-bit time_t, attempting to
               # instantiate a Time object in the far future raises a RangeError,
               # in which case we know we've chosen the wrong divisor.
-              nil
             end
-          end.compact.find { |candidate| candidate > cut_off }
+          end
+
+          nil
         end
       end
     end

data/lib/new_relic/agent/method_tracer.rb

@@ -3,6 +3,8 @@
 # See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
 
 require 'new_relic/control'
+require 'new_relic/agent/method_tracer_helpers'
+
 module NewRelic
   module Agent
     # This module contains class methods added to support installing custom
@@ -49,18 +51,22 @@ module NewRelic
         clazz.extend ClassMethods
       end
 
-      # Deprecated: original method preserved for API backward compatibility.
-      # Use either #trace_execution_scoped or #trace_execution_unscoped
+      # Trace a given block with stats and keep track of the caller.
+      # See NewRelic::Agent::MethodTracer::ClassMethods#add_method_tracer for a description of the arguments.
+      # +metric_names+ is either a single name or an array of metric names.
+      # If more than one metric is passed, the +produce_metric+ option only applies to the first.  The
+      # others are always recorded.  Only the first metric is pushed onto the scope stack.
+      #
+      # Generally you pass an array of metric names if you want to record the metric under additional
+      # categories, but generally this *should never ever be done*.  Most of the time you can aggregate
+      # on the server.
       #
       # @api public
-      # @deprecated
       #
-      def trace_method_execution(metric_names, push_scope, produce_metric, deduct_call_time_from_parent, &block) #:nodoc:
-        if push_scope
-          trace_execution_scoped(metric_names, :metric => produce_metric,
-                                 :deduct_call_time_from_parent => deduct_call_time_from_parent, &block)
-        else
-          trace_execution_unscoped(metric_names, &block)
+      def trace_execution_scoped(metric_names, options={}) #THREAD_LOCAL_ACCESS
+        NewRelic::Agent::MethodTracerHelpers.trace_execution_scoped(metric_names, options) do
+          # Using an implicit block avoids object allocation for a &block param
+          yield
         end
       end
 
@@ -79,7 +85,22 @@ module NewRelic
           yield
         ensure
           duration = (Time.now - t0).to_f              # for some reason this is 3 usec faster than Time - Time
-          stat_engine.tl_record_unscoped_metrics(metric_names, duration)
+          NewRelic::Agent.instance.stats_engine.tl_record_unscoped_metrics(metric_names, duration)
+        end
+      end
+
+      # Deprecated: original method preserved for API backward compatibility.
+      # Use either #trace_execution_scoped or #trace_execution_unscoped
+      #
+      # @api public
+      # @deprecated
+      #
+      def trace_method_execution(metric_names, push_scope, produce_metric, deduct_call_time_from_parent, &block) #:nodoc:
+        if push_scope
+          trace_execution_scoped(metric_names, :metric => produce_metric,
+                                 :deduct_call_time_from_parent => deduct_call_time_from_parent, &block)
+        else
+          trace_execution_unscoped(metric_names, &block)
         end
       end
 
@@ -96,147 +117,27 @@ module NewRelic
 
       alias trace_method_execution_no_scope trace_execution_unscoped #:nodoc:
 
-      # Refactored out of the previous trace_execution_scoped
-      # method, most methods in this module relate to code used in
-      # the #trace_execution_scoped method in this module
-      module TraceExecutionScoped
-        extend self
-
-        # Shorthand to return the NewRelic::Agent.instance
-        def agent_instance
-          NewRelic::Agent.instance
-        end
-
-        # Shorthand to return the current statistics engine
-        def stat_engine
-          agent_instance.stats_engine
-        end
-
-        # returns a scoped metric stat for the specified name
-        def get_stats_scoped(first_name, scoped_metric_only)
-          stat_engine.get_stats(first_name, true, scoped_metric_only)
-        end
-
-        # Shorthand method to get stats from the stat engine
-        def get_stats_unscoped(name)
-          stat_engine.get_stats_no_scope(name)
-        end
-
-        # Helper for setting a hash key if the hash key is nil,
-        # instead of the default ||= behavior which sets if it is
-        # false as well
-        def set_if_nil(hash, key)
-          hash[key] = true if hash[key].nil?
-        end
-
-        # helper for logging errors to the newrelic_agent.log
-        # properly. Logs the error at error level
-        def log_errors(code_area)
-          yield
-        rescue => e
-          ::NewRelic::Agent.logger.error("Caught exception in #{code_area}.", e)
-        end
-
-        # provides the header for our traced execution scoped
-        # method - gets the initial time, sets the tracing flag if
-        # needed, and pushes the frame onto the metric stack
-        # logs any errors that occur and returns the start time and
-        # the frame so that we can check for it later, to maintain
-        # sanity. If the frame stack becomes unbalanced, this
-        # transaction loses meaning.
-        def trace_execution_scoped_header(state, t0)
-          log_errors(:trace_execution_scoped_header) do
-            stack = state.traced_method_stack
-            stack.push_frame(state, :method_tracer, t0)
-          end
-        end
-
-        def record_metrics(state, first_name, other_names, duration, exclusive, options)
-          # The default value for :scoped_metric is true, so set it as true
-          # if it was unset.
-          record_scoped_metric = options.has_key?(:scoped_metric) ? options[:scoped_metric] : true
-
-          if options[:metric]
-            if record_scoped_metric
-              stat_engine.record_scoped_and_unscoped_metrics(state, first_name, other_names, duration, exclusive)
-            else
-              metrics = [first_name].concat(other_names)
-              stat_engine.record_unscoped_metrics(state, metrics, duration, exclusive)
-            end
-          else
-            stat_engine.record_unscoped_metrics(state, other_names, duration, exclusive)
-          end
-        end
-
-        # Handles the end of the #trace_execution_scoped method -
-        # calculating the time taken, popping the tracing flag if
-        # needed, deducting time taken by children, and tracing the
-        # subsidiary unscoped metrics if any
-        #
-        # this method fails safely if the header does not manage to
-        # push the scope onto the stack - it simply does not trace
-        # any metrics.
-        def trace_execution_scoped_footer(state, t0, first_name, metric_names, expected_frame, options, t1=Time.now.to_f)
-          log_errors(:trace_method_execution_footer) do
-            if expected_frame
-              stack = state.traced_method_stack
-              frame = stack.pop_frame(state, expected_frame, first_name, t1, !!options[:metric])
-              duration = t1 - t0
-              if duration < 1_000_000_000 # roughly 31 years
-                if duration < 0
-                  ::NewRelic::Agent.logger.log_once(:warn, "metric_duration_negative:#{first_name}",
-                    "Metric #{first_name} has negative duration: #{duration} s")
-                end
-
-                exclusive = duration - frame.children_time
-                if exclusive < 0
-                  ::NewRelic::Agent.logger.log_once(:warn, "metric_exclusive_negative:#{first_name}",
-                    "Metric #{first_name} has negative exclusive time: duration = #{duration} s, child_time = #{frame.children_time}")
-                end
-
-                record_metrics(state, first_name, metric_names, duration, exclusive, options)
-              else
-                ::NewRelic::Agent.logger.log_once(:warn, "too_huge_metric:#{first_name}",
-                  "Ignoring metric #{first_name} with unacceptably large duration: #{duration} s")
-              end
-            end
-          end
-        end
-      end
-      include TraceExecutionScoped
-
-      # Trace a given block with stats and keep track of the caller.
-      # See NewRelic::Agent::MethodTracer::ClassMethods#add_method_tracer for a description of the arguments.
-      # +metric_names+ is either a single name or an array of metric names.
-      # If more than one metric is passed, the +produce_metric+ option only applies to the first.  The
-      # others are always recorded.  Only the first metric is pushed onto the scope stack.
       #
-      # Generally you pass an array of metric names if you want to record the metric under additional
-      # categories, but generally this *should never ever be done*.  Most of the time you can aggregate
-      # on the server.
+      # This method is deprecated and exists only for backwards-compatibility
+      # reasons. Usages should be replaced with calls to
+      # NewRelic::Agent.record_metric.
       #
       # @api public
+      # @deprecated
       #
-      def trace_execution_scoped(metric_names, options={}) #THREAD_LOCAL_ACCESS
-        state = NewRelic::Agent::TransactionState.tl_get
-        return yield unless state.is_execution_traced?
-
-        metric_names = Array(metric_names)
-        first_name = metric_names.shift
-        return yield if first_name.nil?
-
-        set_if_nil(options, :metric)
-        additional_metrics_callback = options[:additional_metrics_callback]
-        start_time = Time.now.to_f
-        expected_scope = trace_execution_scoped_header(state, start_time)
+      def get_stats_scoped(first_name, scoped_metric_only)
+        NewRelic::Agent.instance.stats_engine.get_stats(first_name, true, scoped_metric_only)
+      end
 
-        begin
-          result = yield
-          metric_names += Array(additional_metrics_callback.call) if additional_metrics_callback
-          result
-        ensure
-          trace_execution_scoped_footer(state, start_time, first_name, metric_names, expected_scope, options)
-        end
+      # This method is deprecated and exists only for backwards-compatibility
+      # reasons. Usages should be replaced with calls to
+      # NewRelic::Agent.record_metric.
+      #
+      # @api public
+      # @deprecated
+      #
+      def get_stats_unscoped(name)
+        NewRelic::Agent.instance.stats_engine.get_stats_no_scope(name)
       end
 
       # Defines methods used at the class level, for adding instrumentation
@@ -324,11 +225,9 @@ module NewRelic
           # instrumentation into effectively one method call overhead
           # when the agent is disabled
           def assemble_code_header(method_name, metric_name_code, options)
-              header = "return #{_untraced_method_name(method_name, metric_name_code)}(*args, &block) unless NewRelic::Agent.tl_is_execution_traced?\n"
-
-              header += options[:code_header].to_s
-
-              header
+            header = "return #{_untraced_method_name(method_name, metric_name_code)}(*args, &block) unless NewRelic::Agent.tl_is_execution_traced?\n"
+            header += options[:code_header].to_s
+            header
           end
 
           # returns an eval-able string that contains the traced
@@ -351,29 +250,9 @@ module NewRelic
           # returns an eval-able string that contains the tracing code
           # for a fully traced metric including scoping
           def method_with_push_scope(method_name, metric_name_code, options)
-            # At this point, we expect 'self' to point to a Class or Module that
-            # has included the NewRelic::Agent::MethodTracer module, which means
-            # it should have an instance method defined called
-            # 'trace_execution_scoped'.
-            #
-            # If this is not the case, assume that
-            # we're relying on the fact that MethodTracer is included into the
-            # Module class by init_plugin, and try to call
-            # trace_execution_scoped as a class rather than instance method.
-            #
-            # The inclusion of MethodTracer into Module will be going away in
-            # the future, so if we detect people relying on that behavior, warn
-            # and record a supportability metric.
-            if self.method_defined?(:trace_execution_scoped)
-              klass = 'self'
-            else
-              NewRelic::Agent.logger.warn("Called add_method_tracer from #{self} without including the NewRelic::Agent::MethodTracer module. This is deprecated and will stop working in the future. Please see http://docs.newrelic.com/docs/ruby/ruby-custom-metric-collection for examples of correct add_method_tracer usage.")
-              NewRelic::Agent.increment_metric("Supportability/usage/improper_add_method_tracer")
-              klass = 'self.class'
-            end
             "def #{_traced_method_name(method_name, metric_name_code)}(*args, &block)
               #{options[:code_header]}
-              result = #{klass}.trace_execution_scoped(\"#{metric_name_code}\",
+              result = ::NewRelic::Agent::MethodTracerHelpers.trace_execution_scoped(\"#{metric_name_code}\",
                         :metric => #{options[:metric]}) do
                 #{_untraced_method_name(method_name, metric_name_code)}(*args, &block)
               end