newrelic_rpm 3.6.5.130 → 3.6.6.147

data/test/performance/lib/performance/runner.rb

@@ -0,0 +1,207 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'socket'
+
+module Performance
+  class Runner
+    attr_reader :instrumentors
+
+    DEFAULTS = {
+      :instrumentors    => [],
+      :inline           => false,
+      :iterations       => 10000,
+      :reporter_classes => ['ConsoleReporter'],
+      :brief            => false,
+      :tags             => {},
+      :dir              => File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'suites')),
+      :agent_path       => ENV['AGENT_PATH'] || File.join(File.dirname(__FILE__), '..', '..', '..', '..')
+    }
+
+    def initialize(options={})
+      @options = DEFAULTS.merge(options)
+      create_instrumentors(options[:instrumentors] || [])
+      load_test_files(@options[:dir])
+      @reporter_classes = @options[:reporter_classes].map { |cls| Performance.const_get(cls) }
+      @hostname = Socket.gethostname
+    end
+
+    def artifacts_base_dir
+      File.expand_path(File.join('.', 'artifacts'))
+    end
+
+    def create_instrumentors(names)
+      instrumentor_classes = names.map do |name|
+        Performance::Instrumentation.instrumentor_class_by_name(name)
+      end.compact
+
+      instrumentor_classes |= Instrumentation.default_instrumentors
+
+      @instrumentors = []
+      instrumentor_classes.each do |cls|
+        cls.setup if cls.respond_to?(:setup)
+        @instrumentors << cls.new(artifacts_base_dir)
+      end
+    end
+
+    def load_test_files(dir)
+      Dir.glob(File.join(dir, "**", "*.rb")).each do |filename|
+        require filename
+      end
+    end
+
+    def add_progress_callbacks(test_case)
+      test_case.on(:before_each) do |test_case, name|
+        print "#{name}: "
+      end
+      test_case.on(:after_each) do |test_case, name, result|
+        print "#{result.elapsed} s\n"
+      end
+    end
+
+    def add_instrumentor_callbacks(test_case)
+      test_case.on(:before_each) do |test, test_name|
+        instrumentors.each do |i|
+          i.reset
+          i.before(test, test_name)
+        end
+      end
+      test_case.on(:after_each) do |test, test_name, result|
+        instrumentors.reverse.each { |i| i.after(test, test_name) }
+        instrumentors.each do |i|
+          result.measurements.merge!(i.results)
+          result.artifacts.concat(i.artifacts)
+        end
+      end
+    end
+
+    def add_metadata_callbacks(test_case)
+      test_case.on(:after_each) do |test, test_name, result|
+        result.tags.merge!(
+          :newrelic_rpm_version => @newrelic_rpm_version,
+          :newrelic_rpm_git_sha => @newrelic_rpm_git_sha,
+          :ruby_version         => RUBY_DESCRIPTION,
+          :host                 => @hostname
+        )
+        result.tags.merge!(@options[:tags])
+      end
+    end
+
+    def create_test_case(cls)
+      test_case = cls.new
+      test_case.iterations = @options[:iterations]
+      add_progress_callbacks(test_case) if @options[:progress]
+      add_instrumentor_callbacks(test_case)
+      add_metadata_callbacks(test_case)
+      test_case
+    end
+
+    def methods_for_test_case(test_case)
+      methods = test_case.runnable_test_methods
+      methods = methods.shuffle if @options[:randomize]
+      if @options[:name]
+        filter = Regexp.new(@options[:name])
+        methods = methods.select { |m| m.match(filter) }
+      elsif @options[:identifier]
+        suite, method = @options[:identifier].split('#')
+        methods = methods.select { |m| m == method }
+      end
+      methods
+    end
+
+    def newrelic_rpm_path
+      File.expand_path(File.join(@options[:agent_path], 'lib'))
+    end
+
+    def load_newrelic_rpm
+      unless @loaded_newrelic_rpm
+        path = newrelic_rpm_path
+        $:.unshift(path)
+        require "newrelic_rpm"
+        @newrelic_rpm_version = NewRelic::VERSION::STRING
+        @newrelic_rpm_git_sha = find_newrelic_rpm_git_sha(path)
+        @loaded_newrelic_rpm = true
+      end
+    end
+
+    def find_newrelic_rpm_git_sha(path)
+      build_file_path = File.join(path, 'new_relic', 'build.rb')
+      if File.exist?(build_file_path)
+        build_file_contents = File.read(build_file_path)
+        return $1.strip if build_file_contents =~ /GITSHA: (.*)/
+      else
+        %x((cd '#{path}' && git log --pretty='%h' -n 1)).strip
+      end
+    end
+
+    def run_test_subprocess(test_case, method)
+      test_case_name = test_case.class.name
+      test_identifier = "#{test_case_name}##{method}"
+      runner_script = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'script', 'runner'))
+      cmd = "#{runner_script} -T #{test_identifier} -N #{@options[:iterations]} -A #{@options[:agent_path]} -j -q -I"
+      output = nil
+      IO.popen(cmd) do |io|
+        output = io.read
+      end
+      results = JSON.parse(output)
+      result = Result.from_hash(results.first)
+      result.tags.merge!(@options[:tags])
+      result
+    end
+
+    def run_test_inline(test_case, method)
+      begin
+        load_newrelic_rpm
+        GC.start
+        test_case.run(method)
+      rescue => e
+        result = Result.new(test_case.class.name, method)
+        result.exception = e
+        result
+      end
+    end
+
+    def run_test_case(test_case)
+      methods_for_test_case(test_case).map do |method|
+        if @options[:inline]
+          run_test_inline(test_case, method)
+        else
+          run_test_subprocess(test_case, method)
+        end
+      end
+    end
+
+    def suites_to_run
+      if @options[:identifier]
+        suite, method = @options[:identifier].split('#')
+        TestCase.subclasses.select { |cls| cls.name == suite }
+      elsif @options[:suite]
+        TestCase.subclasses.select { |cls| cls.name == @options[:suite] }
+      else
+        TestCase.subclasses
+      end
+    end
+
+    def run_all_test_cases
+      results = []
+      suites_to_run.each do |cls|
+        test_case = create_test_case(cls)
+        results += run_test_case(test_case)
+      end
+      results
+    end
+
+    def run_and_report
+      t0 = Time.now
+      results = run_all_test_cases
+      report_results(results, Time.now - t0)
+    end
+
+    def report_results(results, elapsed)
+      @reporter_classes.each do |cls|
+        cls.new(results, elapsed, @options).report
+      end
+    end
+  end
+end

data/test/performance/lib/performance/test_case.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class TestCase
+    @subclasses = []
+
+    def self.inherited(cls)
+      @subclasses << cls
+    end
+
+    def self.subclasses
+      @subclasses
+    end
+
+    attr_accessor :iterations
+
+    def initialize
+      @callbacks = {}
+      @iterations = 10000
+      on(:before_each, &method(:setup))
+      on(:after_each,  &method(:teardown))
+    end
+
+    def setup; end
+    def teardown; end
+
+    def on(event, &action)
+      @callbacks[event] ||= []
+      @callbacks[event] << action
+    end
+
+    def fire(event, *args)
+      if @callbacks[event]
+        @callbacks[event].each { |cb| cb.arity > 0 ? cb.call(*args) : cb.call }
+      end
+    end
+
+    def runnable_test_methods
+      self.methods.map { |m| m.to_s }.select { |m| m =~ /^test_/ }
+    end
+
+    def with_callbacks(name)
+      fire(:before_each, self, name)
+      result = yield
+      fire(:after_each, self, name, result)
+    end
+
+    def run(name)
+      result = Result.new(self.class, name)
+      begin
+        with_callbacks(name) do
+          result.timer.measure do
+            self.send(name)
+          end
+          result
+        end
+      rescue => e
+        result.exception = e
+      end
+      result
+    end
+  end
+end

data/test/performance/lib/performance/timer.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class Timer
+    def initialize
+      @start_timestamp = nil
+      @end_timestamp = nil
+    end
+
+    def start(t=Time.now)
+      @start_timestamp = t
+    end
+
+    def stopped?
+      !!@stop_timestamp
+    end
+
+    def stop(t=Time.now)
+      @stop_timestamp = t
+    end
+
+    def measure
+      start
+      yield
+      stop
+    end
+
+    def elapsed
+      if @stop_timestamp && @start_timestamp
+        @stop_timestamp - @start_timestamp
+      else
+        nil
+      end
+    end
+  end
+end

data/test/performance/script/mega-runner

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'performance')
+
+start_commit = ARGV[0]
+stop_commit  = ARGV[1]
+repo_path    = ARGV[2]
+
+def commits_between(start_commit, stop_commit, repo)
+  output = `cd #{repo} && git log --reverse --pretty=%H #{start_commit}..#{stop_commit}`
+  output.split("\n")
+end
+
+def checkout_commit(commit, repo)
+  system("cd #{repo} && git checkout #{commit} >/dev/null 2>&1")
+end
+
+runner_options = {
+  :agent_path => repo_path,
+  :reporter_classes => ['HakoReporter'],
+  :isolate          => true,
+  :tags => {
+    :run_id => 'ben-perf-test'
+  }
+}
+commits = commits_between(start_commit, stop_commit, repo_path)
+
+Performance.logger.info "Testing #{commits.size} commits"
+commits.each_with_index do |commit, i|
+  checkout_commit(commit, repo_path)
+  runner_options[:tags][:sequence_number] = i
+  runner = Performance::Runner.new(runner_options)
+  t0 = Time.now
+  results = runner.run_all_test_cases
+  runner.report_results(results, Time.now - t0)
+end

data/test/performance/script/runner

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'optparse'
+require 'rubygems'
+require 'json'
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'performance')
+
+target_versions = nil
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on("-p", "--progress", "Print out test names as they run") do |p|
+    options[:progress] = true
+  end
+  opts.on("-i", "--instrumentor=NAME", "Use the named instrumentor") do |name|
+    options[:instrumentors] = [name]
+  end
+  opts.on("-n", "--name=NAME", "Filter tests to those matching NAME") do |name|
+    options[:name] = name
+  end
+  opts.on("-N", "--iterations=NUM", "Change the number of iterations for each test") do |iterations|
+    options[:iterations] = iterations.to_i
+  end
+  opts.on("-I", "--inline", "Run tests inline - do not isolate each test into a sub-invocation") do |i|
+    options[:inline] = true
+  end
+  opts.on("-j", "--json", "Produce JSON output") do |q|
+    options[:reporter_classes] = ['JSONReporter']
+  end
+  opts.on("-R", "--reporters=NAMES", "Use the specified reporters (comma-separated list of class names)") do |reporter_names|
+    reporter_names = reporter_names.split(',')
+    options[:reporter_classes] = reporter_names
+  end
+  opts.on("-r", "--randomize", "Randomize test order") do |r|
+    options[:randomize] = r
+  end
+  opts.on("-b", "--brief", "Don't print out details for each test, just the elapsed time") do |b|
+    options[:brief] = b
+  end
+  opts.on("-T", "--test=NAME", "Run one specific test, identified by <suite name>#<test_name>") do |identifier|
+    options[:identifier] = identifier
+  end
+  opts.on("-q", "--quiet", "Disable diagnostic logging") do
+    Performance.log_path = "/dev/null"
+  end
+  opts.on("-L", "--log=PATH", "Log diagnostic information to PATH") do |log_path|
+    Performance.log_path = log_path
+  end
+  opts.on("-A", "--agent=PATH", "Run tests against the copy of the agent at PATH") do |path|
+    options[:agent_path] = path
+  end
+  opts.on("-c", "--compare=VERSIONS", "Run tests against the specified VERSIONS of the agent. Use with -A") do |versions|
+    target_versions = versions.split(',')
+  end
+  opts.on("-m", "--metadata=METADATA", "Attach metadata to the run. Format: 'key:value'. May be specified multiple times.") do |tag_string|
+    key, value = tag_string.split(":", 2)
+    options[:tags] ||= {}
+    options[:tags][key] = value
+  end
+end
+parser.parse!
+
+runner = Performance::Runner.new(options)
+
+if target_versions
+  target_versions.each do |version|
+    Performance.logger.info("Running tests against version #{version}")
+    cmd = "cd #{options[:agent_path]} && git checkout #{version}"
+    system(cmd)
+    runner.run_and_report
+  end
+else
+  runner.run_and_report
+end

data/test/performance/suites/startup.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+class StartupShutdown < Performance::TestCase
+  def test_startup_shutdown
+    (iterations / 100).times do
+      NewRelic::Agent.manual_start
+      NewRelic::Agent.shutdown
+    end
+  end
+end