newrelic_rpm 3.9.3.241 → 3.9.4.245

data/test/performance/lib/performance.rb

@@ -12,11 +12,19 @@ require 'performance/runner'
 require 'performance/test_case'
 require 'performance/timer'
 require 'performance/instrumentor'
+
+require 'performance/reporting'
+require 'performance/table'
 require 'performance/console_reporter'
 require 'performance/json_reporter'
+
 require 'performance/hako_client'
 require 'performance/hako_reporter'
 
+require 'performance/baseline'
+require 'performance/baseline_save_reporter'
+require 'performance/baseline_compare_reporter'
+
 module Performance
   def self.logger
     log_path = ENV['LOG'] || $stderr

data/test/performance/lib/performance/baseline.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class Baseline
+    PERSIST_PATH = File.expand_path("~/.newrelic_rpm_baseline")
+
+    attr_reader :results
+
+    def self.load!
+      self.new.load!
+    end
+
+    def self.save!(results)
+      baseline = self.new
+      results.each { |r| baseline.results << r }
+      baseline.save!
+    end
+
+    def initialize
+      @results = []
+    end
+
+    def load!
+      result_hashes = JSON.parse(File.read(PERSIST_PATH))
+      @results = result_hashes.map { |h| Result.from_hash(h) }
+    end
+
+    def save!
+      File.open(PERSIST_PATH, "w") do |f|
+        f.write(JSON.dump(@results.map(&:to_h)))
+      end
+    end
+  end
+end

data/test/performance/lib/performance/baseline_compare_reporter.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class BaselineCompareReporter
+    include Reporting
+
+    def initialize(results, elapsed, options={})
+      @results = results
+      @elapsed = elapsed
+      @options = options
+    end
+
+    def report
+      report_summary
+
+      begin
+        baseline = Baseline.load!
+      rescue => e
+        puts "Failed to load baseline results: #{e}\n#{e.backtrace.join("\n\t")}"
+        return
+      end
+
+      report_successful_results(baseline, successes) unless successes.empty?
+      report_failed_results
+    end
+
+    def report_successful_results(baseline, results)
+      baseline_identifiers  = baseline.map(&:identifier)
+      new_identifiers       = results.map(&:identifier)
+      missing_from_baseline = new_identifiers - baseline_identifiers
+      missing_from_new      = baseline_identifiers - new_identifiers
+      common_identifiers    = new_identifiers & baseline_identifiers
+
+      if !missing_from_baseline.empty?
+        puts "The following tests were not found in the baseline results:\n"
+        missing_from_baseline.each do |identifier|
+          puts "  #{identifier}"
+        end
+        puts ""
+      end
+
+      if !missing_from_baseline.empty?
+        puts "The following tests were not found in the new results:\n"
+        missing_from_new.each do |identifier|
+          puts "  #{identifier}"
+        end
+        puts ""
+      end
+
+      rows = []
+
+      common_identifiers.each do |identifier|
+        old_result = baseline.find { |r| r.identifier == identifier}
+        new_result = results.find  { |r| r.identifier == identifier }
+
+        delta = new_result.elapsed - old_result.elapsed
+        percent_delta = delta / old_result.elapsed * 100.0
+
+        rows << [
+          identifier,
+          old_result.elapsed,
+          new_result.elapsed,
+          percent_delta
+        ]
+      end
+
+      table = Table.new(rows) do
+        column :name
+        column :before, "%.2f s"
+        column :after,  "%.2f s"
+        column :delta do |v|
+          prefix = v > 0 ? "+" : ''
+          sprintf("#{prefix}%.1f%%", v)
+        end
+      end
+
+      puts table.render
+    end
+  end
+end

data/test/performance/lib/performance/baseline_save_reporter.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class BaselineSaveReporter
+    include Reporting
+
+    def initialize(results, elapsed, options={})
+      @results = results
+      @elapsed = elapsed
+      @options = options
+    end
+
+    def report
+      report_summary
+
+      Baseline.save!(successes)
+      puts "Saved #{successes.size} results as baseline."
+
+      report_failed_results
+    end
+  end
+end

data/test/performance/lib/performance/console_reporter.rb

@@ -4,6 +4,8 @@
 
 module Performance
   class ConsoleReporter
+    include Reporting
+
     def initialize(results, elapsed, options={})
       @results = results
       @elapsed = elapsed
@@ -11,14 +13,14 @@ module Performance
     end
 
     def report
-      failures = @results.select { |result| result.failure? }
-      successes = @results - failures
-      puts "#{@results.size} tests, #{failures.size} failures, #{@elapsed} s total"
-      report_successful_results(successes) if successes.any?
-      report_failed_results(failures) if failures.any?
+      report_summary
+      report_successful_results(successes) unless successes.empty?
+      report_failed_results
     end
 
     def report_successful_results(results)
+      return if successes.empty?
+
       puts ''
       results.each do |result|
         puts "#{result.identifier}: #{result.elapsed} s"
@@ -36,20 +38,5 @@ module Performance
         puts '' if !@options[:brief] || !result.artifacts.empty?
       end
     end
-
-    def report_failed_results(results)
-      puts ''
-      results.each do |failure|
-        puts "FAILED: #{failure.identifier}"
-        e = failure.exception
-        if e
-          puts "#{e['class']}: #{e['message']}"
-          puts failure.exception['backtrace'].map { |l| "    #{l}" }.join("\n")
-        else
-          puts "<No exception recorded>"
-        end
-      end
-      puts ''
-    end
   end
 end

data/test/performance/lib/performance/reporting.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  module Reporting
+    def failures
+      @failures ||= @results.select(&:failure?)
+    end
+
+    def successes
+      @successes ||= @results.reject(&:failure?)
+    end
+
+    def report_summary
+      puts "#{@results.size} tests, #{failures.size} failures, #{@elapsed} s total"
+    end
+
+    def report_failed_results
+      return if failures.empty?
+
+      puts ''
+      failures.each do |failure|
+        puts "FAILED: #{failure.identifier}"
+        e = failure.exception
+        if e
+          puts "#{e['class']}: #{e['message']}"
+          puts failure.exception['backtrace'].map { |l| "    #{l}" }.join("\n")
+        else
+          puts "<No exception recorded>"
+        end
+      end
+      puts ''
+    end
+  end
+end

data/test/performance/lib/performance/table.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class Table
+    class Formatter
+      attr_reader :name
+
+      def initialize(name, format_string=nil, &blk)
+        @name = name
+        @format_string = format_string
+        @format_proc   = blk
+      end
+
+      def measure(value)
+        format(value).size
+      end
+
+      def format(value, width=nil)
+        formatted = if @format_string
+          sprintf(@format_string, value)
+        elsif @format_proc
+          @format_proc.call(value)
+        else
+          value.to_s
+        end
+
+        formatted = justify(value, formatted, width) if width
+        formatted
+      end
+
+      def justify(value, formatted, width)
+        case value
+        when Numeric then formatted.rjust(width)
+        else              formatted.ljust(width)
+        end
+      end
+    end
+
+    class Builder
+      attr_reader :formatters
+
+      def initialize
+        @formatters = []
+      end
+
+      def column(name, format_string=nil, &blk)
+        @formatters << Formatter.new(name, format_string, &blk)
+      end
+    end
+
+    def initialize(rows, &blk)
+      @rows = rows
+
+      builder = Builder.new
+      builder.instance_eval(&blk)
+      @schema = builder.formatters
+    end
+
+    def column_widths
+      widths = Array.new(@schema.size, 0)
+      @schema.each_with_index do |col, idx|
+        widths[idx] = col.name.size
+      end
+      @rows.each do |row|
+        row.each_with_index do |cell, idx|
+          width = @schema[idx].measure(cell)
+          widths[idx] = [widths[idx], width].max
+        end
+      end
+      widths
+    end
+
+    def render_row(parts)
+      "| " + parts.join(" | ") + " |"
+    end
+
+    def render
+      widths = column_widths
+
+      blanks    = widths.map { |w| "-" * w }
+      top       = '+-' + blanks.join('-+-') + '-+'
+      separator = '|-' + blanks.join('-+-') + '-|'
+      bottom    = '+-' + blanks.join('-+-') + '-+'
+
+      text_rows = []
+
+      headers = @schema.zip(widths).map { |(c, w)| c.name.to_s.ljust(w) }
+      text_rows << render_row(headers)
+
+      @rows.each do |row|
+        parts = []
+        row.each_with_index do |v, i|
+          parts << @schema[i].format(v, widths[i])
+        end
+        text_rows << render_row(parts)
+      end
+
+      puts top + "\n"
+      puts text_rows.join("\n" + separator + "\n")
+      puts bottom + "\n"
+    end
+  end
+end

data/test/performance/script/runner

@@ -16,12 +16,6 @@ options = {}
 parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options]"
 
-  opts.on("-p", "--progress", "Print out test names as they run") do |p|
-    options[:progress] = true
-  end
-  opts.on("-i", "--instrumentor=NAME", "Use the named instrumentor") do |name|
-    options[:instrumentors] = [name]
-  end
   opts.on("-P", "--profile", "Do profiling around each test") do
     best_profiling_instrumentor = [
       Performance::Instrumentation::StackProfProfile,
@@ -35,46 +29,68 @@ parser = OptionParser.new do |opts|
       Performance.logger.warn("Could not find a supported instrumentor for profiling.")
     end
   end
+
   opts.on("-s", "--suite=NAME", "Filter to just run a single test suite") do |name|
     options[:suite] = name
   end
+
   opts.on("-n", "--name=NAME", "Filter tests to those matching NAME") do |name|
     options[:name] = name
   end
+
+  opts.on("-B", "--baseline", "Save results as a baseline") do |b|
+    options[:reporter_classes] = ['BaselineSaveReporter']
+  end
+
+  opts.on("-C", "--compare", "Compare results to a saved baseline") do |c|
+    options[:reporter_classes] = ['BaselineCompareReporter']
+  end
+
   opts.on("-N", "--iterations=NUM", "Change the number of iterations for each test") do |iterations|
     options[:iterations] = iterations.to_i
   end
+
   opts.on("-I", "--inline", "Run tests inline - do not isolate each test into a sub-invocation") do |i|
     options[:inline] = true
   end
+
   opts.on("-j", "--json", "Produce JSON output") do |q|
     options[:reporter_classes] = ['JSONReporter']
   end
+
   opts.on("-R", "--reporters=NAMES", "Use the specified reporters (comma-separated list of class names)") do |reporter_names|
     reporter_names = reporter_names.split(',')
     options[:reporter_classes] = reporter_names
   end
+
   opts.on("-r", "--randomize", "Randomize test order") do |r|
     options[:randomize] = r
   end
+
   opts.on("-b", "--brief", "Don't print out details for each test, just the elapsed time") do |b|
     options[:brief] = b
   end
+
   opts.on("-T", "--test=NAME", "Run one specific test, identified by <suite name>#<test_name>") do |identifier|
     options[:identifier] = identifier
   end
+
+  opts.on("-i", "--instrumentor=NAME", "Use the named instrumentor") do |name|
+    options[:instrumentors] = [name]
+  end
+
   opts.on("-q", "--quiet", "Disable diagnostic logging") do
     Performance.log_path = "/dev/null"
   end
+
   opts.on("-L", "--log=PATH", "Log diagnostic information to PATH") do |log_path|
     Performance.log_path = log_path
   end
+
   opts.on("-A", "--agent=PATH", "Run tests against the copy of the agent at PATH") do |path|
     options[:agent_path] = path
   end
-  opts.on("-c", "--compare=VERSIONS", "Run tests against the specified VERSIONS of the agent. Use with -A") do |versions|
-    target_versions = versions.split(',')
-  end
+
   opts.on("-m", "--metadata=METADATA", "Attach metadata to the run. Format: 'key:value'. May be specified multiple times.") do |tag_string|
     key, value = tag_string.split(":", 2)
     options[:tags] ||= {}
@@ -84,14 +100,4 @@ end
 parser.parse!
 
 runner = Performance::Runner.new(options)
-
-if target_versions
-  target_versions.each do |version|
-    Performance.logger.info("Running tests against version #{version}")
-    cmd = "cd #{options[:agent_path]} && git checkout #{version}"
-    system(cmd)
-    runner.run_and_report
-  end
-else
-  runner.run_and_report
-end
+runner.run_and_report