mmtrix_rpm 1.0.0.1

data/test/performance/lib/performance/test_case.rb

@@ -0,0 +1,152 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class TestCase
+    DEFAULT_WARMUP_DURATION = 2
+    DEFAULT_DURATION        = 5
+    BATCH_TIME              = 0.1
+
+    @subclasses = []
+
+    def self.inherited(cls)
+      @subclasses << cls
+    end
+
+    def self.subclasses
+      @subclasses
+    end
+
+    attr_accessor :target_iterations, :target_duration
+
+    def initialize
+      @callbacks = {}
+      @target_iterations = nil
+      @target_duration   = DEFAULT_DURATION
+    end
+
+    def setup; end
+    def teardown; end
+
+    def self.skip_test(test_method_name, options={})
+      skip_specifiers << [test_method_name, options]
+    end
+
+    def self.skip_specifiers
+      @skip_specifiers ||= []
+    end
+
+    def on(event, &action)
+      @callbacks[event] ||= []
+      @callbacks[event] << action
+    end
+
+    def fire(event, *args)
+      if @callbacks[event]
+        @callbacks[event].each { |cb| cb.arity > 0 ? cb.call(*args) : cb.call }
+      end
+    end
+
+    def runnable_test_methods
+      results = self.methods.map(&:to_s).select { |m| m =~ /^test_/ }
+      self.class.skip_specifiers.each do |specifier|
+        method_name, options = *specifier
+        skipped_platforms = Array(options[:platforms])
+        skipped = Platform.current.match_any?(skipped_platforms)
+        results.delete(method_name.to_s) if skipped
+      end
+      results
+    end
+
+    def with_callbacks(name, warmup=false)
+      fire(:before_each, self, name) unless warmup
+      yield
+      fire(:after_each, self, name, @result) unless warmup
+    end
+
+    # This is used to get a rough approximation of the amount of time required
+    # to call action once. We do this so that we will be able to run the actual
+    # timing loop in chunks of iterations, rather than calling Time.now in
+    # between every iteration.
+    def estimate_time_per_iteration(action, duration)
+      start_time = Time.now
+      deadline = start_time + duration
+      
+      iterations = 0
+      while Time.now < deadline
+        action.call
+        iterations += 1
+      end
+      
+      (Time.now - start_time) / iterations
+    end
+
+    def estimate_iterations_per_unit_time(time_per_iteration, desired_duration)
+      (desired_duration / time_per_iteration).ceil
+    end
+
+    def batch_size_in_iterations(blk)
+      approx_time_per_iteration = nil
+      approx_time_per_iteration = estimate_time_per_iteration(blk, DEFAULT_WARMUP_DURATION)
+      estimate_iterations_per_unit_time(approx_time_per_iteration, BATCH_TIME)
+    end
+
+    def run_block_in_batches(blk, duration, batch_size)
+      deadline = Time.now + duration
+      total_iterations = 0
+      while Time.now < deadline
+        batch_iterations = 0
+        while batch_iterations < batch_size
+          blk.call
+          batch_iterations += 1
+        end
+        total_iterations += batch_iterations
+      end
+      total_iterations
+    end
+
+    def run_block_n_times(blk, n)
+      iterations = 0
+      while iterations < n
+        blk.call
+        iterations += 1
+      end
+      iterations
+    end
+
+    def measure(&blk)
+      total_iterations = 0
+      start_time       = nil
+      elapsed          = nil
+
+      batch_size = batch_size_in_iterations(blk) unless target_iterations
+
+      with_callbacks(@result.test_name, false) do
+        start_time = Time.now
+        if target_iterations
+          total_iterations = run_block_n_times(blk, target_iterations)
+        else
+          total_iterations = run_block_in_batches(blk, target_duration, batch_size)
+        end
+        elapsed = Time.now - start_time
+      end
+
+      @result.iterations            = total_iterations
+      @result.timer.start_timestamp = start_time
+      @result.timer.elapsed         = elapsed
+    end
+
+    def run(name)
+      @result = Result.new(self.class, name)
+      begin
+        setup
+        self.send(name)
+        teardown
+      rescue StandardError, LoadError => e
+        @result.exception = e
+      end
+      @result
+    end
+  end
+end

data/test/performance/lib/performance/timer.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class Timer
+    attr_accessor :start_timestamp, :stop_timestamp, :elapsed
+
+    def initialize
+      @start_timestamp = nil
+      @stop_timestamp = nil
+      @elapsed = 0.0
+      @most_recent_start = nil
+    end
+
+    def start(t=Time.now)
+      @start_timestamp ||= t
+      @most_recent_start = t
+    end
+
+    def stopped?
+      !!@stop_timestamp
+    end
+
+    def stop(t=Time.now)
+      @stop_timestamp = t
+      @elapsed += t - @most_recent_start
+    end
+
+    def measure
+      start
+      yield
+      stop
+    end
+
+    def inspect
+      "<Performance::Timer @start_timestamp=#{start_timestamp.inspect}, @stop_timestamp=#{stop_timestamp.inspect}, elapsed=#{elapsed}>"
+    end
+  end
+end

data/test/performance/lib/performance.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'logger'
+
+$: << File.expand_path(File.dirname(__FILE__))
+
+require 'performance/platform'
+require 'performance/result'
+require 'performance/runner'
+require 'performance/test_case'
+require 'performance/timer'
+require 'performance/instrumentor'
+
+require 'performance/reporting'
+require 'performance/table'
+require 'performance/console_reporter'
+require 'performance/json_reporter'
+require 'performance/formatting_helpers'
+
+require 'performance/hako_client'
+require 'performance/hako_reporter'
+
+require 'performance/baseline'
+require 'performance/baseline_save_reporter'
+require 'performance/baseline_compare_reporter'
+
+module Performance
+  def self.logger
+    log_path = ENV['LOG'] || $stderr
+    @logger ||= Logger.new(log_path)
+  end
+
+  def self.log_path=(path)
+    @logger = Logger.new(path)
+  end
+end

data/test/performance/script/baselines

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'optparse'
+require 'rubygems'
+require 'httparty'
+require 'tmpdir'
+require 'json'
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'performance')
+
+GEM_NAME = 'mmtrix_rpm'
+
+class GemWorkspace
+  attr_reader :path
+
+  def initialize(path=nil)
+    @path = File.expand_path(path || Dir.mktmpdir)
+    FileUtils.mkdir_p(gems_dir)
+    FileUtils.mkdir_p(extracted_gems_dir)
+  end
+
+  def gems_dir
+    File.join(@path, 'gems')
+  end
+
+  def extracted_gems_dir
+    File.join(@path, 'extracted_gems')
+  end
+end
+
+class FetchedGem
+  attr_reader :name, :version
+
+  def initialize(workspace, name, version)
+    @workspace = workspace
+    @name = name
+    @version = version
+  end
+
+  def gem_filename
+    "#{@name}-#{@version}.gem"
+  end
+
+  def path
+    File.join(@workspace.gems_dir, gem_filename)
+  end
+
+  def extracted_path
+    File.join(@workspace.extracted_gems_dir, gem_filename.gsub(/\.gem$/, ''))
+  end
+
+  def uri
+    "https://rubygems.org/downloads/#{gem_filename}"
+  end
+
+  def fetch!
+    File.open(path, "w") do |f|
+      rsp = HTTParty.get(uri)
+      f.write(rsp.body)
+    end
+  end
+
+  def fetched?
+    File.exist?(path)
+  end
+
+  def git_sha
+    `git rev-parse #{@version}`.strip
+  end
+
+  def extract!
+    fetch! unless fetched?
+    cmd = "cd #{@workspace.extracted_gems_dir} && gem unpack '#{path}'"
+    system(cmd)
+  end
+end
+
+versions_url = "https://rubygems.org/api/v1/versions/#{GEM_NAME}.json"
+result = Net::HTTP.get(URI.parse(versions_url))
+
+gem_versions = JSON.parse(result)
+ga_gem_versions = gem_versions.reject { |v| v['prerelease'] }
+
+workspace = GemWorkspace.new(ARGV.first)
+ga_gem_versions.each do |version|
+  gem = FetchedGem.new(workspace, GEM_NAME, version['number'])
+  gem.extract!
+  options = {
+    :agent_path => gem.extracted_path,
+    :reporter_classes => ["ConsoleReporter", "HakoReporter"],
+    :tags => {
+      :series => 'baseline',
+      :mmtrix_rpm_git_sha => gem.git_sha
+    }
+  }
+  runner = Performance::Runner.new(options)
+  runner.run_and_report
+end

data/test/performance/script/mega-runner

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'performance')
+
+start_commit = ARGV[0]
+stop_commit  = ARGV[1]
+repo_path    = ARGV[2]
+
+def commits_between(start_commit, stop_commit, repo)
+  output = `cd #{repo} && git log --reverse --pretty=%H #{start_commit}..#{stop_commit}`
+  output.split("\n")
+end
+
+def checkout_commit(commit, repo)
+  system("cd #{repo} && git checkout #{commit} >/dev/null 2>&1")
+end
+
+runner_options = {
+  :agent_path => repo_path,
+  :reporter_classes => ['HakoReporter'],
+  :isolate          => true,
+  :tags => {
+    :run_id => 'ben-perf-test'
+  }
+}
+commits = commits_between(start_commit, stop_commit, repo_path)
+
+Performance.logger.info "Testing #{commits.size} commits"
+commits.each_with_index do |commit, i|
+  checkout_commit(commit, repo_path)
+  runner_options[:tags][:sequence_number] = i
+  runner = Performance::Runner.new(runner_options)
+  t0 = Time.now
+  results = runner.run_all_test_cases
+  runner.report_results(results, Time.now - t0)
+end

data/test/performance/script/runner

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'optparse'
+require 'rubygems'
+require 'json'
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'performance')
+
+target_versions = nil
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on("-P", "--profile", "Do profiling around each test") do
+    best_profiling_instrumentor = [
+      Performance::Instrumentation::StackProfProfile,
+      Performance::Instrumentation::PerfToolsProfile
+    ].find(&:supported?)
+
+    if best_profiling_instrumentor
+      options[:inline] = true
+      options[:instrumentors] = [best_profiling_instrumentor.to_s]
+    else
+      Performance.logger.warn("Could not find a supported instrumentor for profiling.")
+    end
+  end
+
+  opts.on("-a", "--profile-alloc", "Do profiling around each test for object allocations") do
+    options[:inline] = true
+    options[:instrumentors] = [Performance::Instrumentation::StackProfAllocationProfile.to_s]
+  end
+
+  opts.on("-l", "--list", "List all available suites and tests") do
+    options[:list] = true
+  end
+
+  opts.on("-s", "--suite=NAME", "Filter test suites to run (allows comma separated list)") do |name|
+    options[:suite] ||= []
+    options[:suite].concat(name.split(","))
+  end
+
+  opts.on("-n", "--name=NAME", "Filter tests to those matching NAME") do |name|
+    options[:name] = name
+  end
+
+  opts.on("-B", "--baseline", "Save results as a baseline") do |b|
+    options[:reporter_classes] = ['BaselineSaveReporter']
+  end
+
+  opts.on("-C", "--compare", "Compare results to a saved baseline") do |c|
+    options[:reporter_classes] = ['BaselineCompareReporter']
+  end
+
+  opts.on("-N", "--iterations=NUM",
+    "Set a fixed number of iterations for each test.",
+    "Overrides the -d / --duration option.") do |iterations|
+    options[:iterations] = iterations.to_i
+  end
+
+  opts.on("-d", "--duration=TIME",
+    "Run each test for TIME seconds. Defaults to 5s.") do |duration|
+    options[:duration] = duration.to_f
+  end
+
+  opts.on("-I", "--inline", "Run tests inline - do not isolate each test into a sub-invocation") do |i|
+    options[:inline] = true
+  end
+
+  opts.on("-j", "--json", "Produce JSON output") do |q|
+    options[:reporter_classes] = ['JSONReporter']
+  end
+
+  opts.on("-R", "--reporters=NAMES", "Use the specified reporters (comma-separated list of class names)") do |reporter_names|
+    reporter_names = reporter_names.split(',')
+    options[:reporter_classes] = reporter_names
+  end
+
+  opts.on("-r", "--randomize", "Randomize test order") do |r|
+    options[:randomize] = r
+  end
+
+  opts.on("-b", "--brief", "Don't print out details for each test, just the elapsed time") do |b|
+    options[:brief] = b
+  end
+
+  opts.on("-T", "--test=NAME", "Run one specific test, identified by <suite name>#<test_name>") do |identifier|
+    options[:identifier] = identifier
+  end
+
+  opts.on("-i", "--instrumentor=NAME", "Use the named instrumentor") do |name|
+    options[:instrumentors] = [name]
+  end
+
+  opts.on("-q", "--quiet", "Disable diagnostic logging") do
+    Performance.log_path = "/dev/null"
+  end
+
+  opts.on("-L", "--log=PATH", "Log diagnostic information to PATH") do |log_path|
+    Performance.log_path = log_path
+  end
+
+  opts.on("-A", "--agent=PATH", "Run tests against the copy of the agent at PATH") do |path|
+    options[:agent_path] = path
+  end
+
+  opts.on("-m", "--metadata=METADATA", "Attach metadata to the run. Format: 'key:value'. May be specified multiple times.") do |tag_string|
+    key, value = tag_string.split(":", 2)
+    options[:tags] ||= {}
+    options[:tags][key] = value
+  end
+end
+parser.parse!
+
+runner = Performance::Runner.new(options)
+
+if options[:list]
+  all_tests = runner.list_test_cases
+  tests_by_suite = all_tests.group_by { |t| t[0] }
+  tests_by_suite.each do |(suite, tests)|
+    puts "#{suite}:"
+    tests.each do |(_, test_method)|
+      puts "  #{test_method}"
+    end
+    puts ''
+  end
+else
+  runner.run_and_report
+end

data/test/performance/suites/active_record.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+
+class ActiveRecordTest < Performance::TestCase
+  def setup
+    require 'mmtrix/agent/instrumentation/active_record_helper'
+
+    const_set(:ActiveRecordHelper, Mmtrix::Agent::Instrumentation::ActiveRecordHelper) unless defined?(ActiveRecordHelper)
+
+    if ActiveRecordHelper.respond_to?(:metrics_for)
+      @run = ActiveRecordHelper.method(:metrics_for)
+    else
+      # Mimics what was buried in the instrumentation that we replaced with
+      # calls to metrics_for in the new instrumentation. Can run against SHA
+      # 399d8ed for baselining (3.10 tags won't work since the perf testing
+      # changed on dev post 3.10)
+      @run = proc do |name, sql, adapter, *_|
+        metric = ActiveRecordHelper.metric_for_name(name) ||
+                 ActiveRecordHelper.metric_for_sql(sql)
+        remote_service_metric = ActiveRecordHelper.remote_service_metric("host", adapter)
+
+        metrics = [metric, remote_service_metric].compact
+        metrics += ActiveRecordHelper.rollup_metrics_for(metric)
+      end
+    end
+  end
+
+  NAME    = "Model Load"
+  SQL     = "SELECT * FROM star"
+  ADAPTER = "mysql2"
+
+  def test_helper_by_name
+    measure do
+      @run.call(NAME, SQL, ADAPTER)
+    end
+  end
+
+  UNKNOWN_NAME = "Blah"
+
+  def test_helper_by_sql
+    measure do
+      @run.call(UNKNOWN_NAME, SQL, ADAPTER)
+    end
+  end
+end

data/test/performance/suites/agent_attributes.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'securerandom'
+
+class AgentAttributesTests < Performance::TestCase
+  def setup
+    require 'mmtrix/agent/attribute_filter'
+  end
+
+  ALPHA = "alpha".freeze
+  BETA  = "beta".freeze
+
+  def test_empty_agent_attributes
+    @filter = Mmtrix::Agent::AttributeFilter.new(Mmtrix::Agent.config)
+
+    measure do
+      @filter.apply(ALPHA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+      @filter.apply(BETA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+    end
+  end
+
+  def test_with_attribute_rules
+    with_config(:'attributes.include' => ['alpha'],
+                :'attributes.exclude' => ['beta']) do
+
+      @filter = Mmtrix::Agent::AttributeFilter.new(Mmtrix::Agent.config)
+
+      measure do
+        @filter.apply(ALPHA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+        @filter.apply(BETA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+      end
+    end
+  end
+
+  def test_with_wildcards
+    with_config(:'attributes.include' => ['alpha*'],
+                :'attributes.exclude' => ['beta*']) do
+
+      @filter = Mmtrix::Agent::AttributeFilter.new(Mmtrix::Agent.config)
+
+      measure do
+        @filter.apply(ALPHA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+        @filter.apply(BETA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+      end
+    end
+  end
+
+  def test_with_tons_o_rules
+    with_config(:'attributes.include' => 100.times.map { SecureRandom.hex },
+                :'attributes.exclude' => 100.times.map { SecureRandom.hex }) do
+
+      @filter = Mmtrix::Agent::AttributeFilter.new(Mmtrix::Agent.config)
+
+      measure do
+        @filter.apply(ALPHA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+        @filter.apply(BETA, Mmtrix::Agent::AttributeFilter::DST_ALL)
+      end
+    end
+  end
+end

data/test/performance/suites/config.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+class ConfigPerfTests < Performance::TestCase
+  def setup
+    @config = Mmtrix::Agent::Configuration::Manager.new
+    @config.add_config_for_testing(:my_value => "boo")
+  end
+
+  def test_raw_access
+    measure do
+      v = @config[:my_value]
+    end
+  end
+
+  def test_defaulting_access
+    measure do
+      v = @config[:log_level]
+    end
+  end
+
+  def test_missing_key
+    measure do
+      v = @config[:nope]
+    end
+  end
+
+  def test_blowing_cache
+    measure do
+      @config.reset_cache
+      v = @config[:my_value]
+    end
+  end
+end