newrelic_rpm 2.8.0

data/lib/new_relic/agent/patch_const_missing.rb

@@ -0,0 +1,31 @@
+# This class is for debugging purposes only.
+#
+class Module
+  @@newrelic_agent_thread = nil
+  def new_relic_const_missing(*args)
+    if Thread.current == @@newrelic_agent_thread
+      msg = "Agent background thread shouldn't be calling const_missing (#{args.inspect})   \n"
+      msg << caller[0..4].join("   \n")
+      NewRelic::Config.instance.log.warn msg 
+    end
+    original_const_missing(*args)
+  end
+  
+  def newrelic_enable_warning
+    Module.class_eval do
+      if !defined?(original_const_missing)
+        alias_method :original_const_missing, :const_missing
+        alias_method :const_missing, :new_relic_const_missing
+      end
+    end
+  end
+  def newrelic_disable_warning
+    Module.class_eval do
+      alias_method :const_missing, :original_const_missing if defined?(original_const_missing)
+    end
+  end
+  
+  def newrelic_set_agent_thread(thread)
+    @@newrelic_agent_thread = thread
+  end
+end

data/lib/new_relic/agent/samplers/cpu.rb

@@ -0,0 +1,29 @@
+module NewRelic::Agent
+  class CPUSampler
+    def initialize
+  
+      agent = NewRelic::Agent.instance
+  
+      agent.stats_engine.add_sampled_metric("CPU/User Time") do | stats |
+        t = Process.times
+        @last_utime ||= t.utime
+        
+        utime = t.utime
+        stats.record_data_point(utime - @last_utime) if (utime - @last_utime) >= 0
+        @last_utime = utime
+      end
+  
+      agent.stats_engine.add_sampled_metric("CPU/System Time") do | stats |
+        t = Process.times
+        @last_stime ||= t.stime
+        
+        stime = t.stime
+        stats.record_data_point(stime - @last_stime) if (stime - @last_stime) >= 0
+        @last_stime = stime
+      end
+    end
+  end
+end
+
+# CPU sampling like this doesn't work for jruby
+NewRelic::Agent::CPUSampler.new unless defined? Java

data/lib/new_relic/agent/samplers/memory.rb

@@ -0,0 +1,55 @@
+module NewRelic::Agent
+  class MemorySampler
+    def initialize
+      if RUBY_PLATFORM =~ /java/
+        platform = %x[uname -s].downcase
+      else
+        platform = RUBY_PLATFORM.downcase
+      end
+      
+      # macos, linux, solaris
+      if platform =~ /darwin|linux/
+        @ps = "ps -o rsz"
+      elsif platform =~ /freebsd/
+        @ps = "ps -o rss"
+      elsif platform =~ /solaris/
+        @ps = "/usr/bin/ps -o rss -p"
+      end
+      if !@ps
+        raise "Unsupported platform for getting memory: #{platform}"
+      end
+      
+      if @ps        
+        @broken = false
+        
+        agent = NewRelic::Agent.instance        
+        agent.stats_engine.add_sampled_metric("Memory/Physical") do |stats|
+          if !@broken
+            begin
+              process = $$
+              memory = `#{@ps} #{process}`.split("\n")[1].to_f / 1024
+
+              # if for some reason the ps command doesn't work on the resident os,
+              # then don't execute it any more.
+              if memory >= 0
+                stats.record_data_point memory
+              else 
+                NewRelic::Agent.instance.log.error "Error attempting to determine resident memory for pid #{process} (got result of #{memory}, this process = #{$$}).  Disabling this metric."
+                NewRelic::Agent.instance.log.error "Faulty command: `#{@ps}`"
+                @broken = true
+              end
+            rescue Exception => e
+              if e.is_a? Errno::ENOMEM
+                NewRelic::Agent.instance.log.error "Got OOM trying to determine process memory usage"
+              else
+                raise e
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+NewRelic::Agent::MemorySampler.new

data/lib/new_relic/agent/samplers/mongrel.rb

@@ -0,0 +1,26 @@
+# NewRelic Instrumentation for Mongrel - tracks the queue length of the mongrel server
+if defined? Mongrel::HttpServer
+  
+  agent = NewRelic::Agent.instance
+  mongrel = nil;
+  ObjectSpace.each_object(Mongrel::HttpServer) do |mongrel_instance|
+    # should only be one mongrel instance in the vm
+    if mongrel
+      agent.log.info("Discovered multiple mongrel instances in one Ruby VM.  "+
+        "This is unexpected and might affect the Accuracy of the Mongrel Request Queue metric.")
+    end
+
+    mongrel = mongrel_instance
+  end
+  
+
+  if mongrel
+    agent.stats_engine.add_sampled_metric("Mongrel/Queue Length") do |stats|
+      qsize = mongrel.workers.list.length
+      qsize -= 1 if NewRelic::Agent::Instrumentation::DispatcherInstrumentation::BusyCalculator.is_busy?
+      qsize = 0 if qsize < 0
+      stats.record_data_point qsize
+    end
+  end
+
+end

data/lib/new_relic/agent/stats_engine.rb

@@ -0,0 +1,241 @@
+
+module NewRelic::Agent
+  class StatsEngine
+    POLL_PERIOD = 10
+    
+    attr_accessor :log
+
+    ScopeStackElement = Struct.new(:name, :children_time, :deduct_call_time_from_parent)
+    
+    class SampledItem
+      def initialize(stats, &callback)
+        @stats = stats
+        @callback = callback
+      end
+      
+      def poll
+        @callback.call @stats
+      end
+    end
+    
+    def initialize(log = Logger.new(STDERR))
+      @stats_hash = {}
+      @sampled_items = []
+      @scope_stack_listener = nil
+      @log = log
+      
+      # Makes the unit tests happy
+      Thread::current[:newrelic_scope_stack] = nil
+      
+      spawn_sampler_thread
+    end
+    
+    def spawn_sampler_thread
+      
+      return if !@sampler_process.nil? && @sampler_process == $$ 
+      
+      # start up a thread that will periodically poll for metric samples
+      @sampler_thread = Thread.new do
+        while true do
+          begin
+            sleep POLL_PERIOD
+            @sampled_items.each do |sampled_item|
+              begin 
+                sampled_item.poll
+              rescue => e
+                log.error e
+                @sampled_items.delete sampled_item
+                log.error "Removing #{sampled_item} from list"
+                log.debug e.backtrace.to_s
+              end
+            end
+          end
+        end
+      end
+
+      @sampler_process = $$
+    end
+    
+    def add_scope_stack_listener(l)
+      fail "Can't add a scope listener midflight in a transaction" if scope_stack.any?
+      @scope_stack_listener =  l
+    end
+    
+    def remove_scope_stack_listener(l)
+      fail "Unknown stack listener trying to be removed" if @scope_stack_listener != l
+      @scope_stack_listener = nil
+    end
+    
+    def push_scope(metric, time = Time.now.to_f, deduct_call_time_from_parent = true)
+      
+      stack = (Thread::current[:newrelic_scope_stack] ||= [])
+      
+      if @scope_stack_listener
+        @scope_stack_listener.notice_first_scope_push(time) if stack.empty? 
+        @scope_stack_listener.notice_push_scope metric, time
+      end
+      
+      scope = ScopeStackElement.new(metric, 0, deduct_call_time_from_parent)
+      stack.push scope
+      
+      scope
+    end
+    
+    def pop_scope(expected_scope, duration, time=Time.now.to_f)
+
+      stack = Thread::current[:newrelic_scope_stack]
+      
+      scope = stack.pop
+      
+      fail "unbalanced pop from blame stack: #{scope.name} != #{expected_scope.name}" if scope != expected_scope
+       
+      stack.last.children_time += duration unless (stack.empty? || !scope.deduct_call_time_from_parent)
+      
+      if !scope.deduct_call_time_from_parent && !stack.empty?
+        stack.last.children_time += scope.children_time
+      end
+      
+      if @scope_stack_listener
+        @scope_stack_listener.notice_pop_scope(scope.name, time)
+        @scope_stack_listener.notice_scope_empty(time) if stack.empty? 
+      end
+      
+      scope
+    end
+    
+    def peek_scope
+      scope_stack.last
+    end
+    
+    def add_sampled_metric(metric_name, &sampler_callback)
+      stats = get_stats(metric_name, false)
+      @sampled_items << SampledItem.new(stats, &sampler_callback)
+    end
+    
+    # set the name of the transaction for the current thread, which will be used
+    # to define the scope of all traced methods called on this thread until the
+    # scope stack is empty.  
+    #
+    # currently the transaction name is the name of the controller action that 
+    # is invoked via the dispatcher, but conceivably we could use other transaction
+    # names in the future if the traced application does more than service http request
+    # via controller actions
+    def transaction_name=(transaction)
+      Thread::current[:newrelic_transaction_name] = transaction
+    end
+    
+    def transaction_name
+      Thread::current[:newrelic_transaction_name]
+    end
+    
+    
+    def lookup_stat(metric_name)
+      return @stats_hash[metric_name]
+    end
+    def metrics
+      return @stats_hash.keys
+    end
+    
+    def get_stats_no_scope(metric_name)
+      stats = @stats_hash[metric_name]
+      if stats.nil?
+        stats = NewRelic::MethodTraceStats.new
+        @stats_hash[metric_name] = stats
+      end
+      stats
+    end
+    
+    def get_stats(metric_name, use_scope = true)
+      stats = @stats_hash[metric_name]
+      if stats.nil?
+        stats = NewRelic::MethodTraceStats.new
+        @stats_hash[metric_name] = stats
+      end
+      
+      if use_scope && transaction_name
+        spec = NewRelic::MetricSpec.new metric_name, transaction_name
+        
+        scoped_stats = @stats_hash[spec]
+        if scoped_stats.nil?
+          scoped_stats = NewRelic::ScopedMethodTraceStats.new stats
+          @stats_hash[spec] = scoped_stats        
+        end
+        
+        stats = scoped_stats
+      end
+      return stats
+    end
+    
+    # Note: this is not synchronized.  There is still some risk in this and
+    # we will revisit later to see if we can make this more robust without
+    # sacrificing efficiency.
+    def harvest_timeslice_data(previous_timeslice_data, metric_ids)
+      timeslice_data = {}
+      @stats_hash.keys.each do |metric_spec|
+        
+        
+        # get a copy of the stats collected since the last harvest, and clear
+        # the stats inside our hash table for the next time slice.
+        stats = @stats_hash[metric_spec]
+
+        # we have an optimization for unscoped metrics
+        if !(metric_spec.is_a? NewRelic::MetricSpec)
+          metric_spec = NewRelic::MetricSpec.new metric_spec
+        end
+
+        if stats.nil? 
+          raise "Nil stats for #{metric_spec.name} (#{metric_spec.scope})"
+        end
+        
+        stats_copy = stats.clone
+        stats.reset
+        
+        # if the previous timeslice data has not been reported (due to an error of some sort)
+        # then we need to merge this timeslice with the previously accumulated - but not sent
+        # data
+        previous_metric_data = previous_timeslice_data[metric_spec]
+        stats_copy.merge! previous_metric_data.stats unless previous_metric_data.nil?
+        
+        stats_copy.round!
+        
+        # don't bother collecting and reporting stats that have zero-values for this timeslice.
+        # significant performance boost and storage savings.
+        unless stats_copy.call_count == 0 && stats_copy.total_call_time == 0.0 && stats_copy.total_exclusive_time == 0.0
+          
+          metric_spec_for_transport = (metric_ids[metric_spec].nil?) ? metric_spec : nil
+          
+          metric_data = NewRelic::MetricData.new(metric_spec_for_transport, stats_copy, metric_ids[metric_spec])
+          
+          timeslice_data[metric_spec] = metric_data
+        end
+      end
+      
+      timeslice_data
+    end
+    
+    
+    def start_transaction
+      Thread::current[:newrelic_scope_stack] = []
+    end
+    
+    
+    # Try to clean up gracefully, otherwise we leave things hanging around on thread locals
+    #
+    def end_transaction
+      stack = Thread::current[:newrelic_scope_stack]
+      
+      if stack
+        @scope_stack_listener.notice_scope_empty(Time.now) if @scope_stack_listener && !stack.empty? 
+        Thread::current[:newrelic_scope_stack] = nil
+      end
+      
+      Thread::current[:newrelic_transaction_name] = nil
+    end
+    
+    private
+    
+      def scope_stack
+        Thread::current[:newrelic_scope_stack] ||= []
+      end
+  end
+end

data/lib/new_relic/agent/synchronize.rb

@@ -0,0 +1,40 @@
+
+module NewRelic::Agent
+  
+  module Synchronize
+    def synchronize_sync
+      @_local_sync ||= Sync.new
+      
+      @_local_sync.synchronize(:EX) do
+        yield
+      end
+    end
+    
+    
+    def synchronize_mutex
+      @_local_mutex ||= Mutex.new
+      
+      @_local_mutex.synchronize do
+        yield
+      end
+    end
+  
+    
+    def synchronize_thread
+      old_val = Thread.critical
+      
+      Thread.critical = true
+      
+      begin
+        yield
+      ensure
+        Thread.critical = old_val
+      end
+    end
+    
+    
+    alias synchronize synchronize_mutex
+    alias synchronize_quick synchronize_mutex
+    alias synchronized_long synchronize_mutex
+  end   
+end

data/lib/new_relic/agent/transaction_sampler.rb

@@ -0,0 +1,281 @@
+
+module NewRelic::Agent
+  
+  
+  class TransactionSampler
+    include Synchronize
+    
+    BUILDER_KEY = :transaction_sample_builder
+    @@capture_params = true
+    
+    def self.capture_params
+      @@capture_params
+    end
+    def self.capture_params=(params)
+      @@capture_params = params
+    end
+    attr_accessor :stack_trace_threshold
+    
+    def initialize(agent)
+      @samples = []
+      
+      @max_samples = 100
+      @stack_trace_threshold = 100000.0
+      agent.stats_engine.add_scope_stack_listener self
+
+      agent.set_sql_obfuscator(:replace) do |sql| 
+        default_sql_obfuscator(sql)
+      end
+    end
+    
+    def disable
+      NewRelic::Agent.instance.stats_engine.remove_scope_stack_listener self
+    end
+    
+    
+    def default_sql_obfuscator(sql)
+#      puts "obfuscate: #{sql}"
+      
+      # remove escaped strings
+      sql = sql.gsub("''", "?")
+      
+      # replace all string literals
+      sql = sql.gsub(/'[^']*'/, "?")
+      
+      # replace all number literals
+      sql = sql.gsub(/\d+/, "?")
+      
+#      puts "result: #{sql}"
+      
+      sql
+    end
+    
+    
+    def notice_first_scope_push(time)
+      if Thread::current[:record_tt] == false
+        Thread::current[BUILDER_KEY] = nil
+      else
+        Thread::current[BUILDER_KEY] = TransactionSampleBuilder.new(time)
+      end
+    end
+    
+    def notice_push_scope(scope, time=Time.now.to_f)
+      
+      return unless builder
+      
+      builder.trace_entry(scope, time)
+      
+      # in developer mode, capture the stack trace with the segment.
+      # this is cpu and memory expensive and therefore should not be
+      # turned on in production mode
+      if NewRelic::Config.instance.developer_mode?
+        segment = builder.current_segment
+        if segment
+          # Strip stack frames off the top that match /new_relic/agent/
+          trace = caller
+          while trace.first =~/\/lib\/new_relic\/agent\//
+            trace.shift
+          end
+          
+          trace = trace[0..39] if trace.length > 40
+          segment[:backtrace] = trace
+        end
+      end
+    end
+    
+    def scope_depth
+      return 0 unless builder
+
+      builder.scope_depth
+    end
+  
+    def notice_pop_scope(scope, time = Time.now.to_f)
+      return unless builder
+      builder.trace_exit(scope, time)
+    end
+    
+    # This is called when we are done with the transaction.  We've
+    # unwound the stack to the top level.
+    def notice_scope_empty(time=Time.now.to_f)
+      
+      last_builder = builder
+      return unless last_builder
+
+      last_builder.finish_trace(time)
+      reset_builder
+    
+      synchronize do
+        sample = last_builder.sample
+      
+        # ensure we don't collect more than a specified number of samples in memory
+        @samples << sample if NewRelic::Config.instance.developer_mode? && sample.params[:path] != nil
+        @samples.shift while @samples.length > @max_samples
+        
+        if @slowest_sample.nil? || @slowest_sample.duration < sample.duration
+          @slowest_sample = sample
+        end
+      end
+    end
+    
+    def notice_transaction(path, request, params)
+      return unless builder
+
+      builder.set_transaction_info(path, request, params)
+    end
+    
+    def notice_transaction_cpu_time(cpu_time)
+      return unless builder
+
+      builder.set_transaction_cpu_time(cpu_time)
+    end
+    
+        
+    # some statements (particularly INSERTS with large BLOBS
+    # may be very large; we should trim them to a maximum usable length
+    # config is the driver configuration for the connection
+    MAX_SQL_LENGTH = 16384
+    def notice_sql(sql, config, duration)
+      return unless builder
+      if Thread::current[:record_sql].nil? || Thread::current[:record_sql]
+        segment = builder.current_segment
+        if segment
+          current_sql = segment[:sql]
+          sql = current_sql + ";\n" + sql if current_sql
+
+          if sql.length > (MAX_SQL_LENGTH - 4)
+            sql = sql[0..MAX_SQL_LENGTH-4] + '...'
+          end
+          
+          segment[:sql] = sql
+          segment[:connection_config] = config
+          segment[:backtrace] = caller.join("\n") if duration >= @stack_trace_threshold 
+        end
+      end
+    end
+    
+    # get the set of collected samples, merging into previous samples,
+    # and clear the collected sample list. 
+    
+    def harvest_slowest_sample(previous_slowest = nil)
+      synchronize do
+        slowest = @slowest_sample
+        @slowest_sample = nil
+
+        return nil unless slowest
+
+        if previous_slowest.nil? || previous_slowest.duration < slowest.duration
+          slowest
+        else
+          previous_slowest
+        end
+      end
+    end
+
+    # get the list of samples without clearing the list.
+    def get_samples
+      synchronize do
+        return @samples.clone
+      end
+    end
+    
+    private 
+      
+      def builder
+        Thread::current[BUILDER_KEY]
+      end
+      def reset_builder
+        Thread::current[BUILDER_KEY] = nil
+      end
+      
+  end
+
+  # a builder is created with every sampled transaction, to dynamically
+  # generate the sampled data.  It is a thread-local object, and is not
+  # accessed by any other thread so no need for synchronization.
+  class TransactionSampleBuilder
+    attr_reader :current_segment
+    
+    include CollectionHelper
+    
+    def initialize(time=Time.now.to_f)
+      @sample = NewRelic::TransactionSample.new(time)
+      @sample_start = time
+      @current_segment = @sample.root_segment
+    end
+
+    def trace_entry(metric_name, time)
+      segment = @sample.create_segment(time - @sample_start, metric_name)
+      @current_segment.add_called_segment(segment)
+      @current_segment = segment
+    end
+
+    def trace_exit(metric_name, time)
+      if metric_name != @current_segment.metric_name
+        fail "unbalanced entry/exit: #{metric_name} != #{@current_segment.metric_name}"
+      end
+      
+      @current_segment.end_trace(time - @sample_start)
+      @current_segment = @current_segment.parent_segment
+    end
+    
+    def finish_trace(time)
+      # This should never get called twice, but in a rare case that we can't reproduce in house it does.
+      # log forensics and return gracefully
+      if @sample.frozen?
+        log = NewRelic::Config.instance.log
+        
+        log.warn "Unexpected double-freeze of Transaction Trace Object."
+        log.info "Please send this diagnostic data to New Relic"
+        log.info @sample.to_s
+        return
+      end
+      
+      @sample.root_segment.end_trace(time - @sample_start)
+      @sample.params[:custom_params] = normalize_params(NewRelic::Agent.instance.custom_params) 
+      @sample.freeze
+      @current_segment = nil
+    end
+    
+    def scope_depth
+      depth = -1        # have to account for the root
+      current = @current_segment
+      
+      while(current)
+        depth += 1
+        current = current.parent_segment
+      end
+      
+      depth
+    end
+    
+    def freeze
+      @sample.freeze unless sample.frozen?
+    end
+    
+    
+    def set_transaction_info(path, request, params)
+      @sample.params[:path] = path
+      
+      if TransactionSampler.capture_params
+        params = normalize_params params
+        
+        @sample.params[:request_params].merge!(params)
+        @sample.params[:request_params].delete :controller
+        @sample.params[:request_params].delete :action
+      end
+      
+      @sample.params[:uri] = request.path if request
+    end
+    
+    def set_transaction_cpu_time(cpu_time)
+      @sample.params[:cpu_time] = cpu_time
+    end
+    
+    
+    def sample
+      fail "Not finished building" unless @sample.frozen?
+      @sample
+    end
+    
+  end
+end