newrelic_rpm 3.6.3.111 → 3.6.4.113.beta

data/CHANGELOG

@@ -1,6 +1,19 @@
 # New Relic Ruby Agent Release Notes #
 
-## v3.6.3 ##
+## v3.6.4 ##
+
+  * Exception Whitelist
+
+  We've improved exception message handling for applications running in
+  high security mode. Enabling 'high_security' now removes exception messages
+  entirely rather than simply obfuscating any SQL.
+
+  By default this feature affects all exceptions, though you can configure a
+  whitelist of exceptions whose messages should be left intact.
+
+  More details: https://newrelic.com/docs/ruby/ruby-agent-configuration
+
+## 3.6.3 ##
 
   * Better Sinatra Support
 

data/lib/new_relic/agent/agent.rb

@@ -45,8 +45,9 @@ module NewRelic
         @request_sampler       = NewRelic::Agent::RequestSampler.new(@events)
         @harvest_samplers      = NewRelic::Agent::SamplerCollection.new(@events)
 
-        @connect_state = :pending
-        @connect_attempts = 0
+        @connect_state      = :pending
+        @connect_attempts   = 0
+        @environment_report = nil
 
         @last_harvest_time = Time.now
         @obfuscator = lambda {|sql| NewRelic::Agent::Database.default_sql_obfuscator(sql) }
@@ -203,8 +204,7 @@ module NewRelic
           # Clear out stats that are left over from parent process
           reset_stats
 
-          # Don't ever check to see if this is a spawner.  If we're in a forked process
-          # I'm pretty sure we're not also forking new instances.
+          generate_environment_report
           start_worker_thread(options)
         end
 
@@ -463,6 +463,7 @@ module NewRelic
           def check_config_and_start_agent
             return unless monitoring? && has_correct_license_key?
             return if using_forking_dispatcher?
+            generate_environment_report
             connect_in_foreground if Agent.config[:sync_startup]
             start_worker_thread
             install_exit_handler
@@ -711,8 +712,15 @@ module NewRelic
             shutdown
           end
 
+          def generate_environment_report
+            @environment_report = environment_for_connect
+          end
+
           # Checks whether we should send environment info, and if so,
-          # returns the snapshot from the local environment
+          # returns the snapshot from the local environment.
+          # Generating the EnvironmentReport has the potential to trigger
+          # require calls in Rails environments, so this method should only
+          # be called synchronously from on the main thread.
           def environment_for_connect
             Agent.config[:send_environment_info] ? Array(EnvironmentReport.new) : []
           end
@@ -726,7 +734,7 @@ module NewRelic
               :app_name => Agent.config.app_names,
               :language => 'ruby',
               :agent_version => NewRelic::VERSION::STRING,
-              :environment => environment_for_connect,
+              :environment => @environment_report,
               :settings => Agent.config.to_collector_hash,
             }
           end

data/lib/new_relic/agent/configuration/defaults.rb

@@ -57,9 +57,9 @@ module NewRelic
           ::NewRelic::Agent::Autostart.agent_should_start?
         end,
         # Don't autostart the agent if we're in IRB or Rails console.
-        # This config option accepts a comma seperated list of constants.
+        # This config option accepts a comma separated list of constants.
         :'autostart.blacklisted_constants' => 'Rails::Console',
-        # Comma seperated list of executables that you don't want to trigger
+        # Comma separated list of executables that you don't want to trigger
         # agents start. e.g. 'rake,my_ruby_script.rb'
         :'autostart.blacklisted_executables' => 'irb',
         :'autostart.blacklisted_rake_tasks' => 'about,assets:clean,assets:clobber,assets:environment,assets:precompile,db:create,db:drop,db:fixtures:load,db:migrate,db:migrate:status,db:rollback,db:schema:cache:clear,db:schema:cache:dump,db:schema:dump,db:schema:load,db:seed,db:setup,db:structure:dump,db:version,doc:app,log:clear,middleware,notes,notes:custom,rails:template,rails:update,routes,secret,spec,spec:controllers,spec:helpers,spec:models,spec:rcov,stats,test,test:all,test:all:db,test:recent,test:single,test:uncommitted,time:zones:all,tmp:clear,tmp:create',
@@ -69,6 +69,11 @@ module NewRelic
         :monitor_daemons => false,
         :multi_homed     => false,
         :high_security   => false,
+        # Strip messages from all exceptions that are not specified in the whitelist.
+        :'strip_exception_messages.enabled' => Proc.new { self[:high_security] },
+        # Comma separated list of exceptions that should show messages when
+        # strip_exception_messages is enabled (e.g. 'NewException, RelicException').
+        :'strip_exception_messages.whitelist' => '',
 
         :host                   => 'collector.newrelic.com',
         :api_host               => 'rpm.newrelic.com',
@@ -143,7 +148,7 @@ module NewRelic
 
         :marshaller => Proc.new { NewRelic::Agent::NewRelicService::JsonMarshaller.is_supported? ? 'json' : 'pruby' },
 
-        :'request_sampler.enabled'        => false,
+        :'request_sampler.enabled'        => true,
         :'request_sampler.sample_rate_ms' => 50
       ].freeze
     end

data/lib/new_relic/agent/configuration/manager.rb

@@ -15,12 +15,20 @@ module NewRelic
       class Manager
         extend Forwardable
         def_delegators :@cache, :[], :has_key?
-        attr_reader :config_stack # mainly for testing
+        attr_reader :config_stack, :stripped_exceptions_whitelist
 
         def initialize
           @config_stack = [ EnvironmentSource.new, DEFAULTS ]
           @cache = Hash.new {|hash,key| hash[key] = self.fetch(key) }
           @callbacks = Hash.new {|hash,key| hash[key] = [] }
+
+          register_callback(:'strip_exception_messages.whitelist') do |whitelist|
+            if whitelist
+              @stripped_exceptions_whitelist = parse_constant_list(whitelist).compact
+            else
+              @stripped_exceptions_whitelist = []
+            end
+          end
         end
 
         def apply_config(source, level=0)
@@ -146,6 +154,29 @@ module NewRelic
             "Updating config (#{direction}) from #{source.class}. Results:",
             flattened.inspect)
         end
+
+        private
+
+        def parse_constant_list(list)
+          list.split(/\s*,\s*/).map do |class_name|
+            const = constantize(class_name)
+
+            unless const
+              NewRelic::Agent.logger.warn "Configuration referenced undefined constant: #{class_name}"
+            end
+
+            const
+          end
+        end
+
+        def constantize(class_name)
+          namespaces = class_name.split('::')
+
+          namespaces.inject(Object) do |namespace, name|
+            return unless namespace
+            namespace.const_get(name) if namespace.const_defined?(name)
+          end
+        end
       end
     end
   end

data/lib/new_relic/agent/instrumentation/action_controller_subscriber.rb

@@ -72,8 +72,8 @@ module NewRelic
         end
 
         def record_metric_on_parent_transaction(metric, time)
-          Agent.instance.stats_engine.transaction_stats_stack[-2] \
-            .record(metric, time)
+          txn = NewRelic::Agent::Transaction.current
+          txn.parent.stats_hash.record(metric, time)
         end
 
         def record_apdex(event)

data/lib/new_relic/agent/instrumentation/controller_instrumentation.rb

@@ -331,20 +331,16 @@ module NewRelic
             end
 
           ensure
+            end_time = Time.now
+
             txn.freeze_name
-            metrics = recorded_metrics(txn)
-            txn_name = metrics.first
+            metric_names = Array(recorded_metrics(txn))
+            txn_name = metric_names.shift
 
-            metric_names = Array(metrics)
-            first_name = metric_names.shift
-            scope = NewRelic::Agent.instance.stats_engine.scope_name
-            end_time = Time.now
-            NewRelic::Agent::MethodTracer::InstanceMethods::TraceExecutionScoped.trace_execution_scoped_footer(txn.start_time.to_f, first_name, metric_names, expected_scope, scope, options, end_time.to_f)
-            NewRelic::Agent::BusyCalculator.dispatcher_finish
-            # Look for a transaction in the thread local and process it.
-            # Clear the thread local when finished to ensure it only gets called once.
+            NewRelic::Agent::MethodTracer::InstanceMethods::TraceExecutionScoped.trace_execution_scoped_footer(txn.start_time.to_f, txn_name, metric_names, expected_scope, options, end_time.to_f)
+            NewRelic::Agent::BusyCalculator.dispatcher_finish(end_time)
+            txn.record_apdex(txn_name, end_time) unless ignore_apdex?
             txn = Transaction.stop(txn_name, end_time)
-            txn.record_apdex(txn_name) unless ignore_apdex?
 
             NewRelic::Agent::TransactionInfo.get.ignore_end_user = true if ignore_enduser?
           end

data/lib/new_relic/agent/instrumentation/resque.rb

@@ -34,7 +34,8 @@ DependencyDetection.defer do
                 yield(*args)
               end
             ensure
-              NewRelic::Agent.shutdown if NewRelic::LanguageSupport.can_fork?
+              NewRelic::Agent.shutdown if NewRelic::LanguageSupport.can_fork? &&
+                                          (!Resque.respond_to?(:inline) || !Resque.inline)
             end
           end
         end

data/lib/new_relic/agent/method_tracer.rb

@@ -206,8 +206,7 @@ module NewRelic
           end
 
           def has_parent?
-            NewRelic::Agent::Transaction.current &&
-              NewRelic::Agent::Transaction.current.has_parent?
+            !NewRelic::Agent::Transaction.parent.nil?
           end
 
           def metrics_for_parent_transaction(first_name, options)
@@ -226,7 +225,7 @@ module NewRelic
 
             parent_metrics = metrics_for_parent_transaction(first_name, options)
             parent_metrics.each do |metric|
-              stat_engine.transaction_stats_stack[-2].record(metric) do |stats|
+              NewRelic::Agent::Transaction.parent.stats_hash.record(metric) do |stats|
                 stats.record_data_point(duration, exclusive)
               end
             end
@@ -240,7 +239,7 @@ module NewRelic
           # this method fails safely if the header does not manage to
           # push the scope onto the stack - it simply does not trace
           # any metrics.
-          def trace_execution_scoped_footer(t0, first_name, metric_names, expected_scope, scope, options, t1=Time.now.to_f)
+          def trace_execution_scoped_footer(t0, first_name, metric_names, expected_scope, options, t1=Time.now.to_f)
             log_errors("trace_method_execution footer") do
               pop_flag!(options[:force])
               if expected_scope
@@ -267,12 +266,11 @@ module NewRelic
             set_if_nil(options, :deduct_call_time_from_parent)
             metric_names = Array(metric_names)
             first_name = metric_names.shift
-            scope = stat_engine.scope_name
             start_time, expected_scope = trace_execution_scoped_header(options)
             begin
               yield
             ensure
-              trace_execution_scoped_footer(start_time, first_name, metric_names, expected_scope, scope, options)
+              trace_execution_scoped_footer(start_time, first_name, metric_names, expected_scope, options)
             end
           end
 

data/lib/new_relic/agent/pipe_service.rb

@@ -28,6 +28,10 @@ module NewRelic
         []
       end
 
+      def analytic_event_data(data)
+        nil
+      end
+
       def metric_data(last_harvest_time, now, unsent_timeslice_data)
         write_to_pipe(:stats => unsent_timeslice_data)
         {}

data/lib/new_relic/agent/request_sampler.rb

@@ -22,6 +22,11 @@ class NewRelic::Agent::RequestSampler
   # :TODO: Get this from the agent instead?
   DEFAULT_REPORT_FREQUENCY = 60
 
+  # Regardless of whether #throttle is successfully called, we will store
+  # at most this many harvest-cycles worth of samples total, to avoid unbounded
+  # memory growth when there's a low-level failure talking to the collector.
+  MAX_FAILED_REPORT_RETENTION = 10
+
   # The namespace and keys of config values
   CONFIG_NAMESPACE = 'request_sampler'
   SAMPLE_RATE_KEY  = "#{CONFIG_NAMESPACE}.sample_rate_ms".to_sym
@@ -46,8 +51,13 @@ class NewRelic::Agent::RequestSampler
     @sample_rate_ms        = DEFAULT_SAMPLE_RATE_MS
     @normal_sample_rate_ms = @sample_rate_ms
     @last_sample_taken     = nil
-    @last_harvest          = nil
     @samples               = []
+    @notified_max_samples  = false
+
+    @sample_count          = 0
+    @request_count         = 0
+    @sample_count_total    = 0
+    @request_count_total   = 0
 
     event_listener.subscribe( :transaction_finished, &method(:on_transaction_finished) )
     self.register_config_callbacks
@@ -80,13 +90,33 @@ class NewRelic::Agent::RequestSampler
   def reset
     NewRelic::Agent.logger.debug "Resetting RequestSampler"
 
+    request_count = nil
+    sample_count = nil
+
     self.synchronize do
+      sample_count = @samples.size
+      request_count = @request_count
+      @request_count = 0
       @samples.clear
       @sample_rate_ms = @normal_sample_rate_ms
       @last_sample_taken = Time.now
+      @notified_max_samples = false
     end
+
+    record_sampling_rate(request_count, sample_count) if @enabled
   end
 
+  def record_sampling_rate(request_count, sample_count)
+    @request_count_total += request_count
+    @sample_count_total  += sample_count
+
+    NewRelic::Agent.logger.debug("Sampled #{sample_count} / #{request_count} (%.1f %%) requests this cycle" % (sample_count.to_f / request_count * 100.0))
+    NewRelic::Agent.logger.debug("Sampled #{@sample_count_total} / #{@request_count_total} (%.1f %%) requests since startup" % (@sample_count_total.to_f / @request_count_total * 100.0))
+
+    engine = NewRelic::Agent.instance.stats_engine
+    engine.record_supportability_metric_count("RequestSampler/requests", request_count)
+    engine.record_supportability_metric_count("RequestSampler/samples", sample_count)
+  end
 
   #
   # :group: Event handlers
@@ -103,6 +133,8 @@ class NewRelic::Agent::RequestSampler
       end
 
       @normal_sample_rate_ms = rate_ms
+      @max_samples = calculate_max_samples
+      NewRelic::Agent.logger.debug "RequestSampler max_samples set to #{@max_samples}"
       self.reset
     end
 
@@ -140,6 +172,7 @@ class NewRelic::Agent::RequestSampler
   # This method is synchronized.
   def <<( sample )
     self.synchronize do
+      @request_count += 1
       self.add_sample( sample ) if should_sample?
     end
 
@@ -176,10 +209,22 @@ class NewRelic::Agent::RequestSampler
   protected
   #########
 
+  def calculate_max_samples
+    max_samples_per_harvest = (DEFAULT_REPORT_FREQUENCY * 1000.0) / @normal_sample_rate_ms
+    max_samples_per_harvest * MAX_FAILED_REPORT_RETENTION
+  end
+
   # Returns +true+ if a sample added now should be kept based on the sample
   # frequency.
   def should_sample?
     return false unless @last_sample_taken
+    if @samples.size >= @max_samples
+      unless @notified_max_samples
+        NewRelic::Agent.logger.warn("Reached maximum of #{@max_samples} samples, ceasing collection")
+        @notified_max_samples = true
+      end
+      return false
+    end
     return ((Time.now - @last_sample_taken) * 1000).ceil >= @sample_rate_ms
   end
 

data/lib/new_relic/agent/stats_engine/metric_stats.rb

@@ -74,27 +74,28 @@ module NewRelic
         end
 
         # Helper method for timing supportability metrics
-        def record_supportability_metrics_timed(metric)
+        def record_supportability_metric_timed(metric)
           start_time = Time.now
           yield
-          end_time = Time.now
-          duration = (end_time - start_time).to_f
         ensure
-          ::NewRelic::Agent.record_metric(metric, duration)
+          duration = (Time.now - start_time).to_f
+          record_supportability_metric(metric, duration)
         end
 
         # Helper for recording a straight value into the count
-        def record_supportability_metrics_count(value, *metrics)
-          record_metrics(metrics) do |stat|
+        def record_supportability_metric_count(metric, value)
+          record_supportability_metric(metric) do |stat|
             stat.call_count = value
           end
         end
 
         # Helper method for recording supportability metrics consistently
-        def record_supportability_metrics(value, *metrics)
-          real_names = metrics.map { |name| "Supportability/#{name}" }
-          NewRelic::Agent.agent.record_metric(real_names) do |stat|
-            yield stat
+        def record_supportability_metric(metric, value=nil)
+          real_name = "Supportability/#{metric}"
+          if block_given?
+            record_metrics(real_name) { |stat| yield stat }
+          else
+            record_metrics(real_name, value)
           end
         end
 

data/lib/new_relic/agent/stats_engine/stats_hash.rb

@@ -57,6 +57,18 @@ module NewRelic
         end
         self
       end
+
+      def resolve_scopes(resolved_scope)
+        new_stats = self.class.new
+        self.each do |spec, stats|
+          if spec.scope != '' &&
+              spec.scope.to_sym == StatsEngine::SCOPE_PLACEHOLDER
+            spec.scope = resolved_scope
+          end
+          new_stats[spec] = stats
+        end
+        return new_stats
+      end
     end
   end
 end

data/lib/new_relic/agent/stats_engine/transactions.rb

@@ -121,40 +121,13 @@ module Agent
       end
 
       def transaction_stats_hash
-        transaction_stats_stack.last
-      end
-
-      def push_transaction_stats
-        transaction_stats_stack << StatsHash.new
-      end
-
-      def pop_transaction_stats(transaction_name)
-        Thread::current[:newrelic_scope_stack] ||= []
-        stats = transaction_stats_stack.pop
-        merge!(apply_scopes(stats, transaction_name)) if stats
-        stats
-      end
-
-      def apply_scopes(stats_hash, resolved_name)
-        new_stats = StatsHash.new
-        stats_hash.each do |spec, stats|
-          if spec.scope != '' &&
-              spec.scope.to_sym == StatsEngine::SCOPE_PLACEHOLDER
-            spec.scope = resolved_name
-          end
-          new_stats[spec] = stats
-        end
-        return new_stats
+        Transaction.current && Transaction.current.stats_hash
       end
 
       # Returns the current scope stack, memoized to a thread local variable
       def scope_stack
         Thread::current[:newrelic_scope_stack] ||= []
       end
-
-      def transaction_stats_stack
-        Thread.current[:newrelic_transaction_stack] ||= []
-      end
     end
   end
 end