ghazel-newrelic_rpm 3.4.0.2 → 3.5.4

data/lib/new_relic/agent/stats_engine/metric_stats.rb

@@ -8,34 +8,82 @@ module NewRelic
         # A simple mutex-synchronized hash to make sure our statistics
         # are internally consistent even in truly-threaded rubies like JRuby
         class SynchronizedHash < ::Hash
-          include NewRelic::LanguageSupport::SynchronizedHash
-          
+          attr_reader :lock
+
           def initialize
             @lock = Mutex.new
           end
-          
+
+          def initialize_copy(old)
+            super
+            old.each do |key, value|
+              self.store(key, value.dup)
+            end
+          end
+
           def []=(*args)
             @lock.synchronize { super }
+          rescue => e
+            log_error(e)
           end
 
           def clear(*args)
             @lock.synchronize { super }
+          rescue => e
+            log_error(e)
           end
 
           def delete(*args)
             @lock.synchronize { super }
+          rescue => e
+            log_error(e)
           end
 
           def delete_if(*args)
             @lock.synchronize { super }
+          rescue => e
+            log_error(e)
+          end
+
+          def reset
+            values.each { |s| s.reset }
+          end
+
+          def log_error(e)
+            backtraces = Thread.list.map { |t| log_thread(t) }.join("\n\n")
+            NewRelic::Control.instance.log.warn(
+              "SynchronizedHash failure: #{e.class.name}: #{e.message}\n#{backtraces}")
+          end
+
+          def log_thread(t)
+            # Ruby 1.8 doesn't expose backtrace properly, so make sure it's there
+            if t.nil? || !t.respond_to?(:backtrace) || t.backtrace.nil?
+              return "#{t}\n\tNo backtrace for thread" 
+            end
+
+            backtrace = t.backtrace.map { |b| "\t#{b}" }.join("\n")
+            "\t#{t}\n#{backtrace}"
+
+          rescue Exception => e
+            # JRuby 1.7.0 has a nasty habit of raising a
+            # java.lang.NullPointerException when we iterate through threads
+            # asking for backtraces.  This line allows us to swallow java
+            # exceptions without referencing their classes (since they don't
+            # exist in MRI).  It also prevents us from swallowing signals or
+            # other nasty things that can happen when you rescue Exception.
+            NewRelic::Control.instance.log.warn(
+              "Error collecting thread backtraces: #{e.class.name}: #{e.message}")
+            NewRelic::Control.instance.log.debug( e.backtrace.join("\n") )
+
+            raise e if e.class.ancestors.include? Exception
           end
         end
-        
+
         # Returns all of the metric names of all the stats in the engine
         def metrics
           stats_hash.keys.map(&:to_s)
         end
-        
+
         # a simple accessor for looking up a stat with no scope -
         # returns a new stats object if no stats object for that
         # metric exists yet
@@ -64,17 +112,43 @@ module NewRelic
           end
           stats
         end
-        
+
         # Returns a stat if one exists, otherwise returns nil. If you
         # want auto-initialization, use one of get_stats or get_stats_no_scope
         def lookup_stats(metric_name, scope_name = '')
           stats_hash[NewRelic::MetricSpec.new(metric_name, scope_name)]
         end
-        
+
+
+        # Helper method for timing supportability metrics
+        def record_supportability_metrics_timed(metrics)
+          start_time = Time.now
+          yield
+          end_time = Time.now
+          duration = (end_time - start_time).to_f
+        ensure
+          record_supportability_metrics(duration, metrics) do |value, metric|
+            metric.record_data_point(value)
+          end
+        end
+
+        # Helper for recording a straight value into the count
+        def record_supportability_metrics_count(value, *metrics)
+          record_supportability_metrics(value, *metrics) do |value, metric|
+            metric.call_count = value
+          end
+        end
+
+        # Helper method for recording supportability metrics consistently
+        def record_supportability_metrics(value, *metrics)
+          metrics.each do |metric|
+              yield(value, get_stats_no_scope("Supportability/#{metric}"))
+          end
+        end
+
         # This module was extracted from the harvest method and should
         # be refactored
         module Harvest
-          
           # merge data from previous harvests into this stats engine -
           # takes into account the case where there are new stats for
           # that metric, and the case where there is no current data
@@ -135,10 +209,12 @@ module NewRelic
 
           def merge_stats(other_engine_or_hash, metric_ids)
             old_data = get_stats_hash_from(other_engine_or_hash)
-
             timeslice_data = {}
-            stats_hash.each do | metric_spec, stats |
-
+            stats_hash.lock.synchronize do
+              Thread.current['newrelic_stats_hash'] = stats_hash.clone
+              stats_hash.reset
+            end
+            Thread.current['newrelic_stats_hash'].each do |metric_spec, stats|
               metric_spec = coerce_to_metric_spec(metric_spec)
               stats_copy = clone_and_reset_stats(metric_spec, stats)
               merge_old_data!(metric_spec, stats_copy, old_data)
@@ -160,7 +236,6 @@ module NewRelic
         # sacrificing efficiency.
         # +++
         def harvest_timeslice_data(previous_timeslice_data, metric_ids)
-
           poll harvest_samplers
           merge_stats(previous_timeslice_data, metric_ids)
         end
@@ -173,9 +248,9 @@ module NewRelic
 
         # Reset each of the stats, such as when a new passenger instance starts up.
         def reset_stats
-          stats_hash.values.each { |s| s.reset }
+          stats_hash.reset
         end
-        
+
         # returns a memoized SynchronizedHash that holds the actual
         # instances of Stats keyed off their MetricName
         def stats_hash

data/lib/new_relic/agent/stats_engine/samplers.rb

@@ -27,15 +27,19 @@ module Agent
         # start up a thread that will periodically poll for metric samples
         return if periodic_samplers.empty?
 
-        @sampler_thread = Thread.new do
-          while true do
+        @sampler_thread = NewRelic::Agent::Thread.new('Sampler Tasks') do
+          loop do
+            now = Time.now
             begin
               sleep POLL_PERIOD
               poll periodic_samplers
+            ensure
+              NewRelic::Agent.instance.stats_engine \
+                .get_stats_no_scope('Supportability/Samplers') \
+                .record_data_point((Time.now - now).to_f)
             end
           end
         end
-        @sampler_thread['newrelic_label'] = 'Sampler Tasks'
       end
 
       private

data/lib/new_relic/agent/stats_engine/transactions.rb

@@ -13,11 +13,11 @@ module Agent
         @children_time = 0
       end
     end
-    
+
     # Handles pushing and popping elements onto an internal stack that
     # tracks where time should be allocated in Transaction Traces
     module Transactions
-      
+
       # Defines methods that stub out the stats engine methods
       # when the agent is disabled
       module Shim # :nodoc:
@@ -29,29 +29,29 @@ module Agent
         def scope_name; end
         def pop_scope(*args); end
       end
-      
+
       # add a new transaction sampler, unless we're currently in a
       # transaction (then we fail)
       def transaction_sampler= sampler
         fail "Can't add a scope listener midflight in a transaction" if scope_stack.any?
         @transaction_sampler = sampler
       end
-      
+
       # removes a transaction sampler
       def remove_transaction_sampler(l)
         @transaction_sampler = nil
       end
-      
+
       # Pushes a scope onto the transaction stack - this generates a
       # TransactionSample::Segment at the end of transaction execution
       def push_scope(metric, time = Time.now.to_f, deduct_call_time_from_parent = true)
         stack = scope_stack
-        @transaction_sampler.notice_push_scope metric, time if @transaction_sampler
+        @transaction_sampler.notice_push_scope metric, time if sampler_enabled?
         scope = ScopeStackElement.new(metric, deduct_call_time_from_parent)
         stack.push scope
         scope
       end
-      
+
       # Pops a scope off the transaction stack - this updates the
       # transaction sampler that we've finished execution of a traced method
       def pop_scope(expected_scope, duration, time=Time.now.to_f)
@@ -66,10 +66,14 @@ module Agent
             stack.last.children_time += scope.children_time
           end
         end
-        @transaction_sampler.notice_pop_scope(scope.name, time) if @transaction_sampler
+        @transaction_sampler.notice_pop_scope(scope.name, time) if sampler_enabled?
         scope
       end
       
+      def sampler_enabled?
+        @transaction_sampler && Agent.config[:'transaction_tracer.enabled']
+      end
+      
       # Returns the latest ScopeStackElement
       def peek_scope
         scope_stack.last
@@ -86,7 +90,7 @@ module Agent
       def scope_name=(transaction)
         Thread::current[:newrelic_scope_name] = transaction
       end
-      
+
       # Returns the current scope name from the thread local
       def scope_name
         Thread::current[:newrelic_scope_name]
@@ -104,7 +108,11 @@ module Agent
       # and is ignored.
       #
       def end_transaction
-        GCProfiler.capture
+        elapsed = GCProfiler.capture
+        if @transaction_sampler && @transaction_sampler.last_sample
+          @transaction_sampler.last_sample.params[:custom_params] ||= {}
+          @transaction_sampler.last_sample.params[:custom_params][:gc_time] = elapsed
+        end
         stack = scope_stack
 
         if stack && stack.empty?
@@ -114,7 +122,7 @@ module Agent
       end
 
       private
-      
+
       # Returns the current scope stack, memoized to a thread local variable
       def scope_stack
         Thread::current[:newrelic_scope_stack] ||= []

data/lib/new_relic/agent/thread.rb

@@ -0,0 +1,27 @@
+module NewRelic
+  module Agent
+
+    class Thread < ::Thread
+      def initialize(label)
+        NewRelic::Agent.logger.debug("Creating New Relic thread: #{label}")
+        self[:newrelic_label] = label
+        super
+      end
+
+      def self.bucket_thread(thread, profile_agent_code)
+        if thread.key?(:newrelic_label)
+          return profile_agent_code ? :agent : :ignore
+        elsif !thread[:newrelic_metric_frame].nil?
+          thread[:newrelic_metric_frame].request.nil? ? :background : :request
+        else
+          :other
+        end
+      end
+
+      def self.scrub_backtrace(thread, profile_agent_code)
+        return thread.backtrace if profile_agent_code
+        thread.backtrace.select {|t| t !~ /\/newrelic_rpm-\d/ }
+      end
+    end
+  end
+end

data/lib/new_relic/agent/thread_profiler.rb

@@ -0,0 +1,295 @@
+require 'new_relic/agent/thread'
+require 'new_relic/agent/worker_loop'
+
+module NewRelic
+  module Agent
+
+    class ThreadProfiler
+
+      attr_reader :profile
+
+      def self.is_supported?
+        RUBY_VERSION >= "1.9.2"
+      end
+
+      def start(profile_id, duration, interval, profile_agent_code)
+        if !ThreadProfiler.is_supported?
+          log.debug("Not starting thread profile as it isn't supported on this environment")
+          @profile = nil
+        else
+          log.debug("Starting thread profile. profile_id=#{profile_id}, duration=#{duration}")
+          @profile = ThreadProfile.new(profile_id, duration, interval, profile_agent_code)
+          @profile.run
+        end
+      end
+
+      def stop(report_data)
+        @profile.stop unless @profile.nil?
+        @profile = nil if !report_data
+      end
+
+      def harvest
+        profile = @profile
+        @profile = nil
+        profile
+      end
+
+      def respond_to_commands(commands, &notify_results)
+        return if commands.empty? || commands.first.size < 2 
+
+        # Doesn't deal with multiple commands in the return set  as 
+        # we currently only have start/stop of thread profiling
+        command_id = commands.first[0]
+        command = commands.first[1]
+
+        name = command["name"]
+        arguments = command["arguments"]
+
+        case name
+          when "start_profiler"
+            start_unless_running_and_notify(command_id, arguments, &notify_results)
+
+          when "stop_profiler"
+            stop_and_notify(command_id, arguments, &notify_results)
+        end
+      end
+
+      def running?
+        !@profile.nil?
+      end
+
+      def finished?
+        @profile && @profile.finished?
+      end
+
+      private
+
+      def start_unless_running_and_notify(command_id, arguments)
+        profile_id = arguments.fetch("profile_id", -1)
+        duration =   arguments.fetch("duration", 120)
+        interval =   arguments.fetch("sample_period", 0.1)
+        profile_agent_code = arguments.fetch("profile_agent_code", true)
+
+        if running?
+          msg = "Profile already in progress. Ignoring agent command to start another."
+          log.debug(msg)
+          yield(command_id, msg) if block_given?
+        else
+          start(profile_id, duration, interval, profile_agent_code)
+          yield(command_id) if block_given?
+        end
+      end
+
+      def stop_and_notify(command_id, arguments)
+        report_data = arguments.fetch("report_data", true)
+        stop(report_data)
+        yield(command_id) if block_given?
+      end
+
+      def log
+        NewRelic::Agent.logger
+      end
+    end
+
+    class ThreadProfile
+
+      attr_reader :profile_id, 
+        :traces, 
+        :profile_agent_code, :interval, 
+        :poll_count, :sample_count, 
+        :start_time, :stop_time
+
+      def initialize(profile_id, duration, interval, profile_agent_code)
+        @profile_id = profile_id
+        @profile_agent_code = profile_agent_code
+
+        @worker_loop = NewRelic::Agent::WorkerLoop.new(:duration => duration)
+        @interval = interval
+        @finished = false
+
+        @traces = {
+          :agent => [],
+          :background => [],
+          :other => [],
+          :request => []
+        }
+        @flattened_nodes = []
+
+        @poll_count = 0
+        @sample_count = 0
+      end
+
+      def run
+        Thread.new('Thread Profiler') do
+          @start_time = now_in_millis
+
+          @worker_loop.run(@interval) do
+            NewRelic::Agent.instance.stats_engine.
+              record_supportability_metrics_timed("ThreadProfiler/PollingTime") do
+
+              @poll_count += 1
+              Thread.list.each do |t|
+                @sample_count += 1
+
+                bucket = Thread.bucket_thread(t, @profile_agent_code)
+                backtrace = Thread.scrub_backtrace(t, @profile_agent_code)
+                aggregate(backtrace, @traces[bucket]) unless bucket == :ignore
+              end
+            end
+          end
+
+          mark_done
+          log.debug("Finished thread profile. Will send with next harvest.")
+        end
+      end
+
+      def stop
+        @worker_loop.stop
+        mark_done
+        log.debug("Stopping thread profile.")
+      end
+
+      def aggregate(trace, trees=@traces[:request], parent=nil)
+        return nil if trace.nil? || trace.empty?
+        node = Node.new(trace.last)
+        existing = trees.find {|n| n == node}
+
+        if existing.nil?
+          existing = node
+          @flattened_nodes << node
+        end
+
+        if parent
+          parent.add_child(node)
+        else
+          trees << node unless trees.include? node
+        end
+
+        existing.runnable_count += 1
+        aggregate(trace[0..-2], existing.children, existing)
+
+        existing
+      end
+
+      def prune!(count_to_keep)
+        @flattened_nodes.sort!(&:order_for_pruning)
+
+        NewRelic::Agent.instance.stats_engine.
+          record_supportability_metrics_count(@flattened_nodes.size, "ThreadProfiler/NodeCount")
+
+        mark_for_pruning(@flattened_nodes, count_to_keep)
+
+        traces.each { |_, nodes| Node.prune!(nodes) }
+      end
+
+      THREAD_PROFILER_NODES = 20_000
+
+      def to_compressed_array
+        prune!(THREAD_PROFILER_NODES)
+
+        traces = {
+          "OTHER" => @traces[:other].map{|t| t.to_array },
+          "REQUEST" => @traces[:request].map{|t| t.to_array },
+          "AGENT" => @traces[:agent].map{|t| t.to_array },
+          "BACKGROUND" => @traces[:background].map{|t| t.to_array }
+        }
+
+        [[@profile_id,
+          @start_time.to_f, @stop_time.to_f,
+          @poll_count, 
+          ThreadProfile.compress(JSON.dump(traces)),
+          @sample_count, 0]]
+      end
+
+      def now_in_millis
+        Time.now.to_f * 1_000
+      end
+
+      def finished?
+        @finished
+      end
+
+      def mark_done
+        @finished = true
+        @stop_time = now_in_millis
+      end
+
+      def mark_for_pruning(nodes, count_to_keep)
+        to_prune = nodes[count_to_keep..-1] || []
+        to_prune.each { |n| n.to_prune = true }
+      end
+
+      def self.flattened_nodes(nodes)
+        nodes.map { |n| [n, flattened_nodes(n.children)] }.flatten
+      end
+
+      def self.compress(json)
+        compressed = Base64.encode64(Zlib::Deflate.deflate(json, Zlib::DEFAULT_COMPRESSION))
+      end
+
+      def self.parse_backtrace(trace)
+        trace.map do |line|
+          line =~ /(.*)\:(\d+)\:in `(.*)'/
+          { :method => $3, :line_no => $2.to_i, :file => $1 }
+        end
+      end
+
+      class Node
+        attr_reader :file, :method, :line_no, :children
+        attr_accessor :runnable_count, :to_prune, :depth
+
+        def initialize(line, parent=nil)
+          line =~ /(.*)\:(\d+)\:in `(.*)'/
+          @file = $1
+          @method = $3
+          @line_no = $2.to_i
+          @children = []
+          @runnable_count = 0
+          @to_prune = false
+          @depth = 0
+
+          parent.add_child(self) if parent
+        end
+
+        def ==(other)
+          @file == other.file &&
+            @method == other.method &&
+            @line_no == other.line_no
+        end
+
+        def total_count
+          @runnable_count
+        end
+
+        # Descending order on count, ascending on depth of nodes
+        def order_for_pruning(y)
+          [-runnable_count, depth] <=> [-y.runnable_count, y.depth]
+        end
+
+        def to_array
+          [[@file, @method, @line_no],
+            @runnable_count, 0,
+            @children.map {|c| c.to_array}]
+        end
+
+        def add_child(child)
+          child.depth = @depth + 1
+          @children << child unless @children.include? child
+        end
+
+        def prune!
+          Node.prune!(@children)
+        end
+
+        def self.prune!(kids)
+          kids.delete_if { |child| child.to_prune }
+          kids.each { |child| child.prune! }
+        end
+      end
+
+      def log
+        NewRelic::Agent.logger
+      end
+    end
+  end
+end