newrelic_rpm 2.9.9 → 2.10.3

data/lib/new_relic/agent/stats_engine/metric_stats.rb

@@ -0,0 +1,118 @@
+module NewRelic
+module Agent
+  class StatsEngine
+    module MetricStats
+      # The stats hash hashes either a metric name for an unscoped metric,
+      # or a metric_spec for a scoped metric value.
+      def lookup_stat(metric_name)
+        stats_hash[metric_name]
+      end
+
+      def metrics
+        stats_hash.keys.map(&:to_s)
+      end
+      
+      def get_stats_no_scope(metric_name)
+        stats_hash[metric_name] ||= NewRelic::MethodTraceStats.new 
+      end
+      
+      # This version allows a caller to pass a stat class to use
+      #
+      def get_custom_stats(metric_name, stat_class)
+        stats_hash[metric_name] ||= stat_class.new
+      end
+      
+      # If use_scope is true, two chained metrics are created, one with scope and one without
+      # If scoped_metric_only is true, only a scoped metric is created (used by rendering metrics which by definition are per controller only)
+      def get_stats(metric_name, use_scope = true, scoped_metric_only = false)
+        
+        if scoped_metric_only
+          spec = NewRelic::MetricSpec.new metric_name, scope_name
+          stats = stats_hash[spec] ||= NewRelic::MethodTraceStats.new 
+        else  
+          stats = stats_hash[metric_name] ||= NewRelic::MethodTraceStats.new 
+          if use_scope && scope_name && scope_name != metric_name 
+            spec = NewRelic::MetricSpec.new metric_name, scope_name
+            scoped_stats = stats_hash[spec] ||= NewRelic::ScopedMethodTraceStats.new(stats) 
+            stats = scoped_stats
+          end
+        end
+        stats
+      end
+      
+      def lookup_stats(metric_name, scope_name = nil)
+        stats_hash[NewRelic::MetricSpec.new(metric_name, scope_name)] ||
+        stats_hash[metric_name]
+      end
+      # Harvest the timeslice data.  First recombine current statss
+      # with any previously
+      # unsent metrics, clear out stats cache, and return the current
+      # stats. 
+      # ---
+      # Note: this is not synchronized.  There is still some risk in this and
+      # we will revisit later to see if we can make this more robust without
+      # sacrificing efficiency.
+      # +++
+      def harvest_timeslice_data(previous_timeslice_data, metric_ids)
+        timeslice_data = {}
+        poll harvest_samplers
+        stats_hash.keys.each do | metric_spec |
+          
+          
+          # get a copy of the stats collected since the last harvest, and clear
+          # the stats inside our hash table for the next time slice.
+          stats = stats_hash[metric_spec]
+          
+          # we have an optimization for unscoped metrics
+          if !(metric_spec.is_a? NewRelic::MetricSpec)
+            metric_spec = NewRelic::MetricSpec.new metric_spec
+          end
+          
+          if stats.nil? 
+            raise "Nil stats for #{metric_spec.name} (#{metric_spec.scope})"
+          end
+          
+          stats_copy = stats.clone
+          stats.reset
+          
+          # if the previous timeslice data has not been reported (due to an error of some sort)
+          # then we need to merge this timeslice with the previously accumulated - but not sent
+          # data
+          previous_metric_data = previous_timeslice_data[metric_spec]
+          stats_copy.merge! previous_metric_data.stats unless previous_metric_data.nil?
+          stats_copy.round!
+          
+          # don't bother collecting and reporting stats that have zero-values for this timeslice.
+          # significant performance boost and storage savings.
+          unless stats_copy.is_reset?
+
+            id = metric_ids[metric_spec]
+            metric_spec_for_transport = id ? nil : metric_spec 
+            
+            metric_data = NewRelic::MetricData.new(metric_spec_for_transport, stats_copy, id)
+            
+            timeslice_data[metric_spec] = metric_data
+          end
+        end
+        
+        timeslice_data
+      end
+      
+      # Remove all stats.  For test code only.
+      def clear_stats 
+        stats_hash.clear
+        NewRelic::Agent::BusyCalculator.reset
+      end
+      
+      # Reset each of the stats, such as when a new passenger instance starts up.
+      def reset_stats 
+        stats_hash.values.each { |s| s.reset }
+      end
+      
+      def stats_hash
+        @stats_hash ||= {}
+      end
+    end
+  end
+end
+end

data/lib/new_relic/agent/stats_engine/samplers.rb

@@ -0,0 +1,81 @@
+module NewRelic
+module Agent
+  class StatsEngine
+    module Shim # :nodoc:
+      def add_sampler(*args); end
+      def add_harvest_sampler(*args); end
+      def spawn_sampler_thread(*args); end
+    end
+    
+    module Samplers
+      
+      # By default a sampler polls on harvest time, once a minute.  However you can 
+      # override #use_harvest_sampler? to return false and it will sample
+      # every POLL_PERIOD seconds on a background thread.
+      POLL_PERIOD = 20
+      
+      def spawn_sampler_thread
+        
+        return if !@sampler_process.nil? && @sampler_process == $$ 
+        
+        # start up a thread that will periodically poll for metric samples
+        return if periodic_samplers.empty?
+        
+        @sampler_thread = Thread.new do
+          while true do
+            begin
+              sleep POLL_PERIOD
+              poll periodic_samplers
+            end
+          end
+        end
+        @sampler_thread['newrelic_label'] = 'Sampler Tasks'
+        @sampler_process = $$
+      end
+      
+      # Add an instance of Sampler to be invoked about every 10 seconds on a background
+      # thread.
+      def add_sampler sampler
+        periodic_samplers.each do |s|
+          raise "Sampler #{sampler.id} is already registered.  Don't call add_sampler directly anymore." if s.id == sampler.id
+        end
+        periodic_samplers << sampler
+        sampler.stats_engine = self
+        log.debug "Adding sampler #{sampler.id.to_s}"
+      end
+      
+      # Add a sampler to be invoked just before each harvest.
+      def add_harvest_sampler sampler
+        harvest_samplers << sampler
+        sampler.stats_engine = self
+        log.debug "Adding harvest time sampler: #{sampler.id.to_s}"
+      end  
+      
+      private
+      
+      # Call poll on each of the samplers.  Remove
+      # the sampler if it raises.
+      def poll(samplers)
+        samplers.delete_if do |sampled_item|
+          begin 
+            sampled_item.poll
+            false # it's okay.  don't delete it.
+          rescue Exception => e
+            log.error "Removing #{sampled_item} from list"
+            log.error e
+            log.debug e.backtrace.to_s
+            true # remove the sampler
+          end
+        end
+      end
+      
+      def harvest_samplers
+        @harvest_samplers ||= []
+      end
+      def periodic_samplers
+        @periodic_samplers ||= []
+      end
+    end
+  end
+end
+end

data/lib/new_relic/agent/stats_engine/transactions.rb

@@ -0,0 +1,149 @@
+module NewRelic
+module Agent
+  class StatsEngine
+        
+    # Defines methods that stub out the stats engine methods
+    # when the agent is disabled
+    
+    class ScopeStackElement
+      attr_reader :name, :deduct_call_time_from_parent
+      attr_accessor :children_time
+      def initialize(name, deduct_call_time)
+        @name = name
+        @deduct_call_time_from_parent = deduct_call_time
+        @children_time = 0
+      end
+    end
+    
+    module Transactions
+      module Shim # :nodoc:
+        def start_transaction(*args); end
+        def end_transaction; end
+        def push_scope(*args); end
+        def transaction_sampler=(*args); end
+        def scope_name=(*args); end
+        def scope_name; end
+        def pop_scope(*args); end
+      end
+      
+      def transaction_sampler= sampler
+        fail "Can't add a scope listener midflight in a transaction" if scope_stack.any?
+        @transaction_sampler = sampler
+      end
+      
+      def remove_transaction_sampler(l)
+        @transaction_sampler = nil
+      end
+      
+      def push_scope(metric, time = Time.now.to_f, deduct_call_time_from_parent = true)
+        
+        stack = scope_stack
+        if collecting_gc?
+          if stack.empty?
+            # reset the gc time so we only include gc time spent during this call
+            @last_gc_timestamp = GC.time
+            @last_gc_count = GC.collections
+          else
+            capture_gc_time
+          end
+        end
+        @transaction_sampler.notice_push_scope metric, time if @transaction_sampler
+        scope = ScopeStackElement.new(metric, deduct_call_time_from_parent)
+        stack.push scope
+        scope
+      end
+      
+      def pop_scope(expected_scope, duration, time=Time.now.to_f)
+        capture_gc_time if collecting_gc?
+        stack = scope_stack
+        scope = stack.pop
+        fail "unbalanced pop from blame stack, got #{scope ? scope.name : 'nil'}, expected #{expected_scope ? expected_scope.name : 'nil'}" if scope != expected_scope
+        
+        if !stack.empty? 
+          if scope.deduct_call_time_from_parent
+            stack.last.children_time += duration
+          else
+            stack.last.children_time += scope.children_time
+          end
+        end
+        @transaction_sampler.notice_pop_scope(scope.name, time) if @transaction_sampler
+        scope
+      end
+      
+      def peek_scope
+        scope_stack.last
+      end
+      
+      # set the name of the transaction for the current thread, which will be used
+      # to define the scope of all traced methods called on this thread until the
+      # scope stack is empty.  
+      #
+      # currently the transaction name is the name of the controller action that 
+      # is invoked via the dispatcher, but conceivably we could use other transaction
+      # names in the future if the traced application does more than service http request
+      # via controller actions
+      def scope_name=(transaction)
+        Thread::current[:newrelic_scope_name] = transaction
+      end
+      
+      def scope_name
+        Thread::current[:newrelic_scope_name]
+      end
+      
+      # Start a new transaction, unless one is already in progress
+      def start_transaction(name = nil)
+        Thread::current[:newrelic_scope_stack] ||= []
+        self.scope_name = name if name
+      end
+      
+      # Try to clean up gracefully, otherwise we leave things hanging around on thread locals.
+      # If it looks like a transaction is still in progress, then maybe this is an inner transaction
+      # and is ignored.
+      #
+      def end_transaction
+        stack = scope_stack
+        
+        if stack && stack.empty?
+          Thread::current[:newrelic_scope_stack] = nil
+          Thread::current[:newrelic_scope_name] = nil
+        end
+      end
+      
+      private
+      
+      # Make sure we don't do this in a multi-threaded environment
+      def collecting_gc?
+        @@collecting_gc ||= GC.respond_to?(:time) && GC.respond_to?(:collections) && !NewRelic::Control.instance.multi_threaded?  
+      end
+      
+      # Assumes collecting_gc?
+      def capture_gc_time
+        # Skip this if we are already in this segment
+        return if !scope_stack.empty? && scope_stack.last.name == "GC/cumulative"
+        num_calls = GC.collections - @last_gc_count
+        elapsed = (GC.time - @last_gc_timestamp).to_f
+        @last_gc_timestamp = GC.time
+        @last_gc_count = GC.collections
+        if num_calls > 0
+          # µs to seconds
+          elapsed = elapsed / 1000000.0
+          # Allocate the GC time to a scope as if the GC just ended
+          # right now.
+          time = Time.now.to_f
+          gc_scope = push_scope("GC/cumulative", time - elapsed)
+          # GC stats are collected into a blamed metric which allows
+          # us to show the stats controller by controller
+          gc_stats = NewRelic::Agent.get_stats(gc_scope.name, true)  
+          gc_stats.record_multiple_data_points(elapsed, num_calls)
+          pop_scope(gc_scope, elapsed, time)
+        end
+      end
+      
+      def scope_stack
+        Thread::current[:newrelic_scope_stack] ||= []
+      end
+      
+    end
+  end
+end
+end

data/lib/new_relic/agent/transaction_sampler.rb

@@ -1,27 +1,30 @@
-
-module NewRelic::Agent
+module NewRelic
+module Agent
   
   class TransactionSampler
     
+    # Module defining methods stubbed out when the agent is disabled
+    module Shim #:nodoc:
+      def notice_first_scope_push(*args); end
+      def notice_push_scope(*args); end
+      def notice_pop_scope(*args); end
+      def notice_scope_empty(*args); end
+    end
+    
     BUILDER_KEY = :transaction_sample_builder
 
-    attr_accessor :stack_trace_threshold, :random_sampling, :sampling_rate, :last_sample, :samples
+    attr_accessor :stack_trace_threshold, :random_sampling, :sampling_rate
+    attr_reader :samples, :last_sample, :disabled
     
-    def initialize(agent)
+    def initialize
       @samples = []
-      
       @harvest_count = 0
       @max_samples = 100
       @random_sample = nil
       config = NewRelic::Control.instance
       sampler_config = config.fetch('transaction_tracer', {})
+      @segment_limit = sampler_config.fetch('limit_segments', 4000)
       @stack_trace_threshold = sampler_config.fetch('stack_trace_threshold', 0.500).to_f
-      
-      agent.stats_engine.add_scope_stack_listener self
-
-      agent.set_sql_obfuscator(:replace) do |sql| 
-        default_sql_obfuscator(sql)
-      end
       @samples_lock = Mutex.new
     end
     
@@ -31,7 +34,8 @@ module NewRelic::Agent
     end
 
     def disable
-      NewRelic::Agent.instance.stats_engine.remove_scope_stack_listener self
+      @disabled = true
+      NewRelic::Agent.instance.stats_engine.remove_transaction_sampler self
     end
     
     def sampling_rate=(val)
@@ -39,25 +43,8 @@ module NewRelic::Agent
       @harvest_count = rand(val)
     end
     
-    def default_sql_obfuscator(sql)
-      sql = sql.dup
-      # This is hardly readable.  Use the unit tests.
-      # remove single quoted strings:
-      sql.gsub!(/'(.*?[^\\'])??'(?!')/, '?')
-      # remove double quoted strings:
-      sql.gsub!(/"(.*?[^\\"])??"(?!")/, '?')
-      # replace all number literals
-      sql.gsub!(/\d+/, "?")
-      sql
-    end
-    
-    
     def notice_first_scope_push(time)
-      if Thread::current[:record_tt] == false
-        Thread::current[BUILDER_KEY] = nil
-      else
-        Thread::current[BUILDER_KEY] = TransactionSampleBuilder.new(time)
-      end
+      start_builder(time) unless disabled
     end
     
     def notice_push_scope(scope, time=Time.now.to_f)
@@ -92,6 +79,7 @@ module NewRelic::Agent
   
     def notice_pop_scope(scope, time = Time.now.to_f)
       return unless builder
+      raise "frozen already???" if builder.sample.frozen?
       builder.trace_exit(scope, time)
     end
     
@@ -103,36 +91,37 @@ module NewRelic::Agent
       return unless last_builder
 
       last_builder.finish_trace(time)
-      reset_builder
+      clear_builder
+      return if last_builder.ignored?
     
       @samples_lock.synchronize do
         @last_sample = last_builder.sample
         
-        # We sometimes see "unanchored" transaction traces
-        if @last_sample.params[:path]
-          @random_sample = @last_sample if @random_sampling
-                  
-          # ensure we don't collect more than a specified number of samples in memory
-          @samples << @last_sample if NewRelic::Control.instance.developer_mode?
-          @samples.shift while @samples.length > @max_samples
-          
-          if @slowest_sample.nil? || @slowest_sample.duration < @last_sample.duration
-            @slowest_sample = @last_sample
-          end
+        @random_sample = @last_sample if @random_sampling
+        
+        # ensure we don't collect more than a specified number of samples in memory
+        @samples << @last_sample if NewRelic::Control.instance.developer_mode?
+        @samples.shift while @samples.length > @max_samples
+        
+        if @slowest_sample.nil? || @slowest_sample.duration < @last_sample.duration
+          @slowest_sample = @last_sample
         end
       end
     end
     
-    def notice_transaction(path, request, params)
-      return unless builder
-
-      builder.set_transaction_info(path, request, params)
+    def notice_transaction(path, uri=nil, params={})
+      builder.set_transaction_info(path, uri, params) if !disabled && builder
+    end
+    
+    def ignore_transaction
+      builder.ignore_transaction if builder
+    end
+    def notice_profile(profile)
+      builder.set_profile(profile) if builder
     end
     
     def notice_transaction_cpu_time(cpu_time)
-      return unless builder
-
-      builder.set_transaction_cpu_time(cpu_time)
+      builder.set_transaction_cpu_time(cpu_time) if builder
     end
     
         
@@ -142,7 +131,7 @@ module NewRelic::Agent
     MAX_SQL_LENGTH = 16384
     def notice_sql(sql, config, duration)
       return unless builder
-      if Thread::current[:record_sql].nil? || Thread::current[:record_sql]
+      if Thread::current[:record_sql] != false
         segment = builder.current_segment
         if segment
           current_sql = segment[:sql]
@@ -164,6 +153,7 @@ module NewRelic::Agent
     # and clear the collected sample list. 
     
     def harvest(previous = nil, slow_threshold = 2.0)
+      return [] if disabled
       result = []
       previous ||= []
       
@@ -178,6 +168,8 @@ module NewRelic::Agent
           
           if (@harvest_count % @sampling_rate) == 0
             result << @random_sample if @random_sample
+          else
+            @random_sample = nil   # if we don't nil this out, then we won't send the slowest if slowest == @random_sample
           end
         end
         
@@ -193,21 +185,33 @@ module NewRelic::Agent
         end
 
         @random_sample = nil
+        @last_sample = nil
       end
+      # Truncate the samples at 2100 segments. The UI will clamp them at 2000 segments anyway.
+      # This will save us memory and bandwidth.
+      result.each { |sample| sample.truncate(@segment_limit) }
       result
     end
 
     # reset samples without rebooting the web server
     def reset!
       @samples = []
+      @last_sample = nil
     end
 
     private 
-      
+    
+      def start_builder(time=nil)
+        if disabled || Thread::current[:record_tt] == false || !NewRelic::Agent.is_execution_traced?
+          clear_builder
+        else
+          Thread::current[BUILDER_KEY] ||= TransactionSampleBuilder.new(time) 
+        end
+      end
       def builder
         Thread::current[BUILDER_KEY]
       end
-      def reset_builder
+      def clear_builder
         Thread::current[BUILDER_KEY] = nil
       end
       
@@ -217,11 +221,12 @@ module NewRelic::Agent
   # generate the sampled data.  It is a thread-local object, and is not
   # accessed by any other thread so no need for synchronization.
   class TransactionSampleBuilder
-    attr_reader :current_segment
+    attr_reader :current_segment, :sample
     
     include CollectionHelper
     
-    def initialize(time=Time.now.to_f)
+    def initialize(time=nil)
+      time ||= Time.now.to_f
       @sample = NewRelic::TransactionSample.new(time)
       @sample_start = time
       @current_segment = @sample.root_segment
@@ -230,7 +235,12 @@ module NewRelic::Agent
     def sample_id
       @sample.sample_id
     end
-
+    def ignored?
+      @ignore || @sample.params[:path].nil? 
+    end
+    def ignore_transaction
+      @ignore = true
+    end
     def trace_entry(metric_name, time)
       segment = @sample.create_segment(time - @sample_start, metric_name)
       @current_segment.add_called_segment(segment)
@@ -241,7 +251,6 @@ module NewRelic::Agent
       if metric_name != @current_segment.metric_name
         fail "unbalanced entry/exit: #{metric_name} != #{@current_segment.metric_name}"
       end
-      
       @current_segment.end_trace(time - @sample_start)
       @current_segment = @current_segment.parent_segment
     end
@@ -251,15 +260,11 @@ module NewRelic::Agent
       # log forensics and return gracefully
       if @sample.frozen?
         log = NewRelic::Control.instance.log
-        
-        log.warn "Unexpected double-freeze of Transaction Trace Object."
-        log.info "Please send this diagnostic data to New Relic"
-        log.info @sample.to_s
+        log.error "Unexpected double-freeze of Transaction Trace Object: \n#{@sample.to_s}"
         return
       end
-      
       @sample.root_segment.end_trace(time - @sample_start)
-      @sample.params[:custom_params] = normalize_params(NewRelic::Agent.instance.custom_params) 
+      @sample.params[:custom_params] = normalize_params(NewRelic::Agent::Instrumentation::MetricFrame.custom_parameters)
       @sample.freeze
       @current_segment = nil
     end
@@ -280,8 +285,11 @@ module NewRelic::Agent
       @sample.freeze unless sample.frozen?
     end
     
+    def set_profile(profile)
+      @sample.profile = profile
+    end
     
-    def set_transaction_info(path, request, params)
+    def set_transaction_info(path, uri, params)
       @sample.params[:path] = path
       
       if NewRelic::Control.instance.capture_params
@@ -291,19 +299,17 @@ module NewRelic::Agent
         @sample.params[:request_params].delete :controller
         @sample.params[:request_params].delete :action
       end
-      
-      @sample.params[:uri] = request.path if request
+      @sample.params[:uri] ||= uri || params[:uri] 
     end
     
     def set_transaction_cpu_time(cpu_time)
       @sample.params[:cpu_time] = cpu_time
     end
     
-    
     def sample
-      fail "Not finished building" unless @sample.frozen?
       @sample
     end
     
   end
 end
+end