newrelic_rpm 3.10.0.279 → 3.11.0.283

data/test/performance/lib/performance.rb

@@ -17,6 +17,7 @@ require 'performance/reporting'
 require 'performance/table'
 require 'performance/console_reporter'
 require 'performance/json_reporter'
+require 'performance/formatting_helpers'
 
 require 'performance/hako_client'
 require 'performance/hako_reporter'

data/test/performance/lib/performance/baseline_compare_reporter.rb

@@ -55,12 +55,16 @@ module Performance
         old_result = baseline.find { |r| r.identifier == identifier}
         new_result = results.find  { |r| r.identifier == identifier }
 
-        delta = new_result.elapsed - old_result.elapsed
-        percent_delta = delta / old_result.elapsed * 100.0
+        delta = new_result.time_per_iteration - old_result.time_per_iteration
+        percent_delta = delta / old_result.time_per_iteration * 100.0
 
         allocations_before = old_result.measurements[:allocations]
         allocations_after  = new_result.measurements[:allocations]
         if allocations_before && allocations_after
+          # normalize allocation counts to be per-iteration
+          allocations_before /= old_result.iterations
+          allocations_after  /= new_result.iterations
+
           allocations_delta  = allocations_after - allocations_before
           allocations_delta_percent = allocations_delta.to_f / allocations_before * 100
         else
@@ -69,8 +73,8 @@ module Performance
 
         rows << [
           identifier,
-          old_result.elapsed,
-          new_result.elapsed,
+          old_result.time_per_iteration,
+          new_result.time_per_iteration,
           percent_delta,
           allocations_before,
           allocations_after,
@@ -81,12 +85,12 @@ module Performance
       format_percent_delta = Proc.new { |v|
         prefix = v > 0 ? "+" : ""
         sprintf("#{prefix}%.1f%%", v)
-      } 
+      }
 
       table = Table.new(rows) do
         column :name
-        column :before,        "%.2f s"
-        column :after,         "%.2f s"
+        column :before,        &(FormattingHelpers.method(:format_duration))
+        column :after,         &(FormattingHelpers.method(:format_duration))
         column :delta,         &format_percent_delta
         column :allocs_before
         column :allocs_after

data/test/performance/lib/performance/console_reporter.rb

@@ -2,6 +2,8 @@
 # This file is distributed under New Relic's license terms.
 # See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
 
+require 'pathname'
+
 module Performance
   class ConsoleReporter
     include Reporting
@@ -18,25 +20,47 @@ module Performance
       report_failed_results
     end
 
+    def format_measurements(result)
+      measurements = result.measurements.merge(:iterations => result.iterations)
+
+      key_width        = measurements.keys.map(&:size).max
+      formatted_values = measurements.values.map { |v| sprintf("%g", v) }
+      value_width      = formatted_values.map(&:size).max
+
+      rows = measurements.map do |key, value|
+        if key == :iterations
+          "  %#{key_width}s: %#{value_width}g" % [key, value]
+        else
+          per_iteration = value / result.iterations.to_f
+          "  %#{key_width}s: %#{value_width}g (%.2f / iter)" % [key, value, per_iteration]
+        end
+      end
+
+      rows.join("\n") + "\n"
+    end
+
     def report_successful_results(results)
       return if successes.empty?
 
       puts ''
       results.each do |result|
-        puts "#{result.identifier}: #{result.elapsed} s"
+        formatted_duration = FormattingHelpers.format_duration(result.time_per_iteration)
+        puts "#{result.identifier}: %.3f ips (#{formatted_duration} / iteration)" % [result.ips]
         unless @options[:brief]
-          result.measurements.each do |key, value|
-            puts "  %s: %g" % [key, value]
-          end
+          puts format_measurements(result)
         end
         unless result.artifacts.empty?
           puts "  artifacts:"
           result.artifacts.each do |artifact|
-            puts "    #{artifact}"
+            puts "    #{make_relative(artifact)}"
           end
         end
         puts '' if !@options[:brief] || !result.artifacts.empty?
       end
     end
+
+    def make_relative(path)
+      "./#{Pathname.new(path).relative_path_from(Pathname.getwd)}"
+    end
   end
 end

data/test/performance/lib/performance/formatting_helpers.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  module FormattingHelpers
+    def self.format_duration(d)
+      if d < 0.001
+        ds = d * 1000 * 1000
+        unit = "µs"
+      elsif d < 1.0
+        ds = d * 1000
+        unit = "ms"
+      else
+        ds = d
+        unit = "s"
+      end
+
+      sprintf("%.2f %s", ds, unit)
+    end
+  end
+end

data/test/performance/lib/performance/instrumentation/stackprof.rb

@@ -12,8 +12,12 @@ module Performance
         require 'stackprof'
       end
 
+      def mode
+        :wall
+      end
+
       def before(test, test_name)
-        StackProf.start(:mode => :wall)
+        StackProf.start(:mode => mode)
       end
 
       def after(test, test_name)
@@ -32,5 +36,11 @@ module Performance
         @artifacts << output_dot_path
       end
     end
+
+    class StackProfAllocationProfile < StackProfProfile
+      def mode
+        :object
+      end
+    end
   end
 end

data/test/performance/lib/performance/result.rb

@@ -7,7 +7,7 @@ require 'time'
 module Performance
   class Result
     attr_reader :test_name, :measurements, :tags, :timer, :artifacts
-    attr_accessor :exception
+    attr_accessor :exception, :iterations
 
     def initialize(test_case, test_name)
       @test_case = test_case
@@ -15,7 +15,8 @@ module Performance
       @measurements   = {}
       @tags           = {}
       @timer = Timer.new
-      @artifacts = []
+      @iterations = 0
+      @artifacts  = []
     end
 
     def exception=(e)
@@ -58,16 +59,25 @@ module Performance
       t.utc.iso8601
     end
 
+    def ips
+      @iterations.to_f / elapsed
+    end
+
+    def time_per_iteration
+      elapsed / @iterations.to_f
+    end
+
     def to_h
       h = {
-        "suite"     => suite_name,
-        "name"      => @test_name,
+        "suite"        => suite_name,
+        "name"         => @test_name,
         "measurements" => measurements_hash,
-        "tags"         => @tags
+        "tags"         => @tags,
+        "iterations"   => @iterations
       }
       h['exception'] = @exception if @exception
       h['artifacts'] = @artifacts if @artifacts && !@artifacts.empty?
-      h['started_at'] = format_timestamp(@timer.start_timestamp) if @timer.start_timestamp
+      h['started_at']  = format_timestamp(@timer.start_timestamp) if @timer.start_timestamp
       h['finished_at'] = format_timestamp(@timer.stop_timestamp) if @timer.stop_timestamp
       h
     end
@@ -81,6 +91,7 @@ module Performance
       result.tags.merge! hash['tags']
       result.exception = hash['exception']
       result.elapsed = elapsed
+      result.iterations = hash['iterations']
       result.timer.start_timestamp = Time.iso8601(hash['started_at']) if hash['started_at']
       result.timer.stop_timestamp = Time.iso8601(hash['finished_at']) if hash['finished_at']
       result

data/test/performance/lib/performance/runner.rb

@@ -11,7 +11,7 @@ module Performance
     DEFAULTS = {
       :instrumentors    => [],
       :inline           => false,
-      :iterations       => 10000,
+      :iterations       => nil,
       :reporter_classes => ['ConsoleReporter'],
       :brief            => false,
       :tags             => {},
@@ -90,7 +90,8 @@ module Performance
 
     def create_test_case(cls)
       test_case = cls.new
-      test_case.iterations = @options[:iterations]
+      test_case.target_iterations = @options[:iterations] if @options[:iterations]
+      test_case.target_duration   = @options[:duration]   if @options[:duration]
       add_progress_callbacks(test_case) if @options[:progress]
       add_instrumentor_callbacks(test_case)
       add_metadata_callbacks(test_case)
@@ -139,7 +140,10 @@ module Performance
       test_case_name = test_case.class.name
       test_identifier = "#{test_case_name}##{method}"
       runner_script = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'script', 'runner'))
-      cmd = "#{runner_script} -T #{test_identifier} -N #{@options[:iterations]} -A #{@options[:agent_path]} -j -q -I"
+      cmd = "#{runner_script} -T #{test_identifier} -j -q -I"
+      cmd << " -A #{@options[:agent_path]}"
+      cmd << " -N #{@options[:iterations]}" if @options[:iterations]
+      cmd << " -d #{@options[:duration]}"   if @options[:duration]
       output = nil
       IO.popen(cmd) do |io|
         output = io.read

data/test/performance/lib/performance/test_case.rb

@@ -4,6 +4,10 @@
 
 module Performance
   class TestCase
+    DEFAULT_WARMUP_DURATION = 2
+    DEFAULT_DURATION        = 5
+    BATCH_TIME              = 0.1
+
     @subclasses = []
 
     def self.inherited(cls)
@@ -14,13 +18,12 @@ module Performance
       @subclasses
     end
 
-    attr_accessor :iterations
+    attr_accessor :target_iterations, :target_duration
 
     def initialize
       @callbacks = {}
-      @iterations = 10000
-      on(:before_each, &method(:setup))
-      on(:after_each,  &method(:teardown))
+      @target_iterations = nil
+      @target_duration   = DEFAULT_DURATION
     end
 
     def setup; end
@@ -56,29 +59,94 @@ module Performance
       results
     end
 
-    def with_callbacks(name)
-      fire(:before_each, self, name)
-      result = yield
-      fire(:after_each, self, name, result)
+    def with_callbacks(name, warmup=false)
+      fire(:before_each, self, name) unless warmup
+      yield
+      fire(:after_each, self, name, @result) unless warmup
+    end
+
+    # This is used to get a rough approximation of the amount of time required
+    # to call action once. We do this so that we will be able to run the actual
+    # timing loop in chunks of iterations, rather than calling Time.now in
+    # between every iteration.
+    def estimate_time_per_iteration(action, duration)
+      start_time = Time.now
+      deadline = start_time + duration
+      
+      iterations = 0
+      while Time.now < deadline
+        action.call
+        iterations += 1
+      end
+      
+      (Time.now - start_time) / iterations
+    end
+
+    def estimate_iterations_per_unit_time(time_per_iteration, desired_duration)
+      (desired_duration / time_per_iteration).ceil
+    end
+
+    def batch_size_in_iterations(blk)
+      approx_time_per_iteration = nil
+      approx_time_per_iteration = estimate_time_per_iteration(blk, DEFAULT_WARMUP_DURATION)
+      estimate_iterations_per_unit_time(approx_time_per_iteration, BATCH_TIME)
+    end
+
+    def run_block_in_batches(blk, duration, batch_size)
+      deadline = Time.now + duration
+      total_iterations = 0
+      while Time.now < deadline
+        batch_iterations = 0
+        while batch_iterations < batch_size
+          blk.call
+          batch_iterations += 1
+        end
+        total_iterations += batch_iterations
+      end
+      total_iterations
+    end
+
+    def run_block_n_times(blk, n)
+      iterations = 0
+      while iterations < n
+        blk.call
+        iterations += 1
+      end
+      iterations
+    end
+
+    def measure(&blk)
+      total_iterations = 0
+      start_time       = nil
+      elapsed          = nil
+
+      batch_size = batch_size_in_iterations(blk) unless target_iterations
+
+      with_callbacks(@result.test_name, false) do
+        start_time = Time.now
+        if target_iterations
+          total_iterations = run_block_n_times(blk, target_iterations)
+        else
+          total_iterations = run_block_in_batches(blk, target_duration, batch_size)
+        end
+        elapsed = Time.now - start_time
+      end
+
+      @result.iterations            = total_iterations
+      @result.timer.start_timestamp = start_time
+      @result.timer.elapsed         = elapsed
     end
 
     def run(name)
-      result = Result.new(self.class, name)
+      @result = Result.new(self.class, name)
       begin
-        with_callbacks(name) do
-          if self.method(name).arity == 0
-            result.timer.measure do
-              self.send(name)
-            end
-          else
-            self.send(name, result.timer)
-          end
-          result
-        end
+        setup
+        self.send(name)
+        teardown
       rescue StandardError, LoadError => e
-        result.exception = e
+        @result.exception = e
       end
-      result
+      @result
     end
   end
 end

data/test/performance/script/runner

@@ -30,6 +30,11 @@ parser = OptionParser.new do |opts|
     end
   end
 
+  opts.on("-a", "--profile-alloc", "Do profiling around each test for object allocations") do
+    options[:inline] = true
+    options[:instrumentors] = [Performance::Instrumentation::StackProfAllocationProfile.to_s]
+  end
+
   opts.on("-l", "--list", "List all available suites and tests") do
     options[:list] = true
   end
@@ -51,10 +56,17 @@ parser = OptionParser.new do |opts|
     options[:reporter_classes] = ['BaselineCompareReporter']
   end
 
-  opts.on("-N", "--iterations=NUM", "Change the number of iterations for each test") do |iterations|
+  opts.on("-N", "--iterations=NUM",
+    "Set a fixed number of iterations for each test.",
+    "Overrides the -d / --duration option.") do |iterations|
     options[:iterations] = iterations.to_i
   end
 
+  opts.on("-d", "--duration=TIME",
+    "Run each test for TIME seconds. Defaults to 5s.") do |duration|
+    options[:duration] = duration.to_f
+  end
+
   opts.on("-I", "--inline", "Run tests inline - do not isolate each test into a sub-invocation") do |i|
     options[:inline] = true
   end

data/test/performance/suites/active_record.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/instrumentation/active_record_helper'
+
+class ActiveRecordTest < Performance::TestCase
+
+  ActiveRecordHelper = NewRelic::Agent::Instrumentation::ActiveRecordHelper unless defined?(ActiveRecordHelper)
+
+  def setup
+    if ActiveRecordHelper.respond_to?(:metrics_for)
+      @run = ActiveRecordHelper.method(:metrics_for)
+    else
+      # Mimics what was buried in the instrumentation that we replaced with
+      # calls to metrics_for in the new instrumentation. Can run against SHA
+      # 399d8ed for baselining (3.10 tags won't work since the perf testing
+      # changed on dev post 3.10)
+      @run = proc do |name, sql, adapter, *_|
+        metric = ActiveRecordHelper.metric_for_name(name) ||
+                 ActiveRecordHelper.metric_for_sql(sql)
+        remote_service_metric = ActiveRecordHelper.remote_service_metric("host", adapter)
+
+        metrics = [metric, remote_service_metric].compact
+        metrics += ActiveRecordHelper.rollup_metrics_for(metric)
+      end
+    end
+  end
+
+  NAME    = "Model Load"
+  SQL     = "SELECT * FROM star"
+  ADAPTER = "mysql2"
+
+  def test_helper_by_name
+    measure do
+      @run.call(NAME, SQL, ADAPTER)
+    end
+  end
+
+  UNKNOWN_NAME = "Blah"
+
+  def test_helper_by_sql
+    measure do
+      @run.call(UNKNOWN_NAME, SQL, ADAPTER)
+    end
+  end
+end