mmtrix_rpm 1.0.0.1

data/test/performance/lib/performance/instrumentor.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  module Instrumentation
+    def self.load_instrumentors
+      dir = File.expand_path(File.join(File.dirname(__FILE__), 'instrumentation'))
+      Dir.glob(File.join(dir, "*.rb")).each do |filename|
+        require filename
+      end
+    end
+
+    def self.default_instrumentors
+      Instrumentor.subclasses.select do |cls|
+        cls.on_by_default? && cls.supported?
+      end
+    end
+
+    def self.instrumentor_class_by_name(name)
+      begin
+        cls = self.const_get(name)
+        if cls.supported?
+          cls
+        else
+          Performance.logger.warn("Instrumentor '#{name}' is unsupported on this platform")
+          nil
+        end
+      rescue NameError => e
+        Performance.logger.error("Failed to load instrumentor '#{name}': #{e.inspect}")
+        nil
+      end
+    end
+
+    class Instrumentor
+      def self.inherited(cls)
+        @subclasses ||= []
+        @subclasses << cls
+      end
+
+      def self.subclasses
+        @subclasses || []
+      end
+
+      def self.platforms(*args)
+        @supported_platforms = args
+      end
+
+      def self.supported?
+        (
+          @supported_platforms.nil? ||
+          Platform.current.match_any?(@supported_platforms)
+        )
+      end
+
+      def self.on_by_default
+        @on_by_default = true
+      end
+
+      def self.on_by_default?
+        @on_by_default
+      end
+
+      attr_reader :artifacts
+
+      def initialize(artifacts_dir)
+        @artifacts_dir = artifacts_dir
+        reset
+      end
+
+      def reset
+        @artifacts = []
+      end
+
+      def pretty_name
+        self.class.name.split("::").last
+      end
+
+      def artifacts_dir_for(test_case, test_name)
+        path = File.join(@artifacts_dir, test_case.class.name, test_name)
+        FileUtils.mkdir_p(path)
+        path
+      end
+
+      def artifact_path(test_case, test_name, extension)
+        File.join(artifacts_dir_for(test_case, test_name), "#{pretty_name}.#{extension}")
+      end
+
+      def before(*); end
+      def after(*); end
+      def results; {}; end
+    end
+  end
+end
+
+Performance::Instrumentation.load_instrumentors

data/test/performance/lib/performance/json_reporter.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class JSONReporter
+    def initialize(results, elapsed, options={})
+      @results = results
+    end
+
+    def report
+      puts JSON.dump(@results.map { |result| result.to_h })
+    end
+  end
+end

data/test/performance/lib/performance/platform.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class Platform
+    def self.current
+      @current ||= self.new
+    end
+
+    def jruby?
+      defined?(JRUBY_VERSION)
+    end
+
+    def ree?
+      defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION =~ /MBARI/
+    end
+
+    def match?(p)
+      case p
+      when :jruby   then jruby?
+      when :mri     then !jruby?
+      when :ree     then !jruby? && ree?
+      when :mri_18  then !jruby? && RUBY_VERSION =~ /^1\.8\./
+      when :mri_19  then !jruby? && RUBY_VERSION =~ /^1\.9\./
+      when :mri_193 then !jruby? && RUBY_VERSION =~ /^1\.9\.3/
+      when :mri_20  then !jruby? && RUBY_VERSION =~ /^2\.0\./
+      when :mri_21  then !jruby? && RUBY_VERSION =~ /^2\.1\./
+      when :mri_22  then !jruby? && RUBY_VERSION =~ /^2\.2\./
+      end
+    end
+
+    def match_any?(platforms)
+      platforms.any? { |p| match?(p) }
+    end
+  end
+end

data/test/performance/lib/performance/reporting.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  module Reporting
+    def failures
+      @failures ||= @results.select(&:failure?)
+    end
+
+    def successes
+      @successes ||= @results.reject(&:failure?)
+    end
+
+    def report_summary
+      puts "#{@results.size} tests, #{failures.size} failures, #{@elapsed} s total"
+    end
+
+    def report_failed_results
+      return if failures.empty?
+
+      puts ''
+      failures.each do |failure|
+        puts "FAILED: #{failure.identifier}"
+        e = failure.exception
+        if e
+          puts "#{e['class']}: #{e['message']}"
+          puts failure.exception['backtrace'].map { |l| "    #{l}" }.join("\n")
+        else
+          puts "<No exception recorded>"
+        end
+      end
+      puts ''
+    end
+  end
+end

data/test/performance/lib/performance/result.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'time'
+
+module Performance
+  class Result
+    attr_reader :test_name, :measurements, :tags, :timer, :artifacts
+    attr_accessor :exception, :iterations
+
+    def initialize(test_case, test_name)
+      @test_case = test_case
+      @test_name = test_name
+      @measurements   = {}
+      @tags           = {}
+      @timer = Timer.new
+      @iterations = 0
+      @artifacts  = []
+    end
+
+    def exception=(e)
+      if e.is_a?(Exception)
+        @exception = {
+          'class'     => e.class.name,
+          'message'   => e.message,
+          'backtrace' => e.backtrace
+        }
+      else
+        @exception = e
+      end
+    end
+
+    def elapsed=(elapsed)
+      @elapsed = elapsed
+    end
+
+    def elapsed
+      @elapsed || @timer.elapsed
+    end
+
+    def failure?
+      elapsed.nil? || !@exception.nil?
+    end
+
+    def suite_name
+      @test_case.is_a?(String) ? @test_case : @test_case.name
+    end
+
+    def identifier
+      "#{suite_name}##{@test_name}"
+    end
+
+    def measurements_hash
+      @measurements.merge(:elapsed => elapsed)
+    end
+
+    def format_timestamp(t)
+      t.utc.iso8601
+    end
+
+    def ips
+      @iterations.to_f / elapsed
+    end
+
+    def time_per_iteration
+      elapsed / @iterations.to_f
+    end
+
+    def to_h
+      h = {
+        "suite"        => suite_name,
+        "name"         => @test_name,
+        "measurements" => measurements_hash,
+        "tags"         => @tags,
+        "iterations"   => @iterations
+      }
+      h['exception'] = @exception if @exception
+      h['artifacts'] = @artifacts if @artifacts && !@artifacts.empty?
+      h['started_at']  = format_timestamp(@timer.start_timestamp) if @timer.start_timestamp
+      h['finished_at'] = format_timestamp(@timer.stop_timestamp) if @timer.stop_timestamp
+      h
+    end
+
+    def self.from_hash(hash)
+      elapsed = hash['measurements'].delete('elapsed')
+      result = self.new(hash['suite'], hash['name'])
+      hash['measurements'].each do |key, value|
+        result.measurements[key.to_sym] = value
+      end
+      result.tags.merge! hash['tags']
+      result.exception = hash['exception']
+      result.elapsed = elapsed
+      result.iterations = hash['iterations']
+      result.timer.start_timestamp = Time.iso8601(hash['started_at']) if hash['started_at']
+      result.timer.stop_timestamp = Time.iso8601(hash['finished_at']) if hash['finished_at']
+      result
+    end
+
+    def inspect
+      "<Performance::Result #{identifier}: #{elapsed} s, results=#{@results.inspect}>"
+    end
+  end
+end

data/test/performance/lib/performance/runner.rb

@@ -0,0 +1,221 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'socket'
+
+module Performance
+  class Runner
+    attr_reader :instrumentors
+
+    DEFAULTS = {
+      :instrumentors    => [],
+      :inline           => false,
+      :iterations       => nil,
+      :reporter_classes => ['ConsoleReporter'],
+      :brief            => false,
+      :tags             => {},
+      :dir              => File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'suites')),
+      :agent_path       => ENV['AGENT_PATH'] || File.join(File.dirname(__FILE__), '..', '..', '..', '..')
+    }
+
+    def initialize(options={})
+      @options = DEFAULTS.merge(options)
+      create_instrumentors(options[:instrumentors] || [])
+      load_test_files(@options[:dir])
+      @reporter_classes = @options[:reporter_classes].map { |cls| Performance.const_get(cls) }
+      @hostname = Socket.gethostname
+    end
+
+    def artifacts_base_dir
+      File.expand_path(File.join('.', 'artifacts'))
+    end
+
+    def create_instrumentors(names)
+      instrumentor_classes = names.map do |name|
+        Performance::Instrumentation.instrumentor_class_by_name(name)
+      end.compact
+
+      instrumentor_classes |= Instrumentation.default_instrumentors
+
+      @instrumentors = []
+      instrumentor_classes.each do |cls|
+        cls.setup if cls.respond_to?(:setup)
+        @instrumentors << cls.new(artifacts_base_dir)
+      end
+    end
+
+    def load_test_files(dir)
+      Dir.glob(File.join(dir, "**", "*.rb")).each do |filename|
+        require filename
+      end
+    end
+
+    def add_progress_callbacks(test_case)
+      test_case.on(:before_each) do |_test_case, name|
+        print "#{name}: "
+      end
+      test_case.on(:after_each) do |_test_case, name, result|
+        print "#{result.elapsed} s\n"
+      end
+    end
+
+    def add_instrumentor_callbacks(test_case)
+      test_case.on(:before_each) do |test, test_name|
+        instrumentors.each do |i|
+          i.reset
+          i.before(test, test_name)
+        end
+      end
+      test_case.on(:after_each) do |test, test_name, result|
+        instrumentors.reverse.each { |i| i.after(test, test_name) }
+        instrumentors.each do |i|
+          result.measurements.merge!(i.results)
+          result.artifacts.concat(i.artifacts)
+        end
+      end
+    end
+
+    def add_metadata_callbacks(test_case)
+      test_case.on(:after_each) do |test, test_name, result|
+        result.tags.merge!(
+          :mmtrix_rpm_version => @mmtrix_rpm_version,
+          :mmtrix_rpm_git_sha => @mmtrix_rpm_git_sha,
+          :ruby_version         => RUBY_DESCRIPTION,
+          :host                 => @hostname
+        )
+        result.tags.merge!(@options[:tags])
+      end
+    end
+
+    def create_test_case(cls)
+      test_case = cls.new
+      test_case.target_iterations = @options[:iterations] if @options[:iterations]
+      test_case.target_duration   = @options[:duration]   if @options[:duration]
+      add_progress_callbacks(test_case) if @options[:progress]
+      add_instrumentor_callbacks(test_case)
+      add_metadata_callbacks(test_case)
+      test_case
+    end
+
+    def methods_for_test_case(test_case)
+      methods = test_case.runnable_test_methods
+      methods = methods.shuffle if @options[:randomize]
+      if @options[:name]
+        filter = Regexp.new(@options[:name])
+        methods = methods.select { |m| m.match(filter) }
+      elsif @options[:identifier]
+        _suite, method = @options[:identifier].split('#')
+        methods = methods.select { |m| m == method }
+      end
+      methods
+    end
+
+    def mmtrix_rpm_path
+      File.expand_path(File.join(@options[:agent_path], 'lib'))
+    end
+
+    def load_mmtrix_rpm
+      unless @loaded_mmtrix_rpm
+        path = mmtrix_rpm_path
+        $:.unshift(path)
+        require "mmtrix_rpm"
+        @mmtrix_rpm_version = Mmtrix::VERSION::STRING
+        @mmtrix_rpm_git_sha = find_mmtrix_rpm_git_sha(path)
+        @loaded_mmtrix_rpm = true
+      end
+    end
+
+    def find_mmtrix_rpm_git_sha(path)
+      build_file_path = File.join(path, 'mmtrix', 'build.rb')
+      if File.exist?(build_file_path)
+        build_file_contents = File.read(build_file_path)
+        return $1.strip if build_file_contents =~ /GITSHA: (.*)/
+      else
+        %x((cd '#{path}' && git log --pretty='%h' -n 1)).strip
+      end
+    end
+
+    def run_test_subprocess(test_case, method)
+      test_case_name = test_case.class.name
+      test_identifier = "#{test_case_name}##{method}"
+      runner_script = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'script', 'runner'))
+      cmd = "#{runner_script} -T #{test_identifier} -j -q -I"
+      cmd << " -A #{@options[:agent_path]}"
+      cmd << " -N #{@options[:iterations]}" if @options[:iterations]
+      cmd << " -d #{@options[:duration]}"   if @options[:duration]
+      output = nil
+      IO.popen(cmd) do |io|
+        output = io.read
+      end
+      results = JSON.parse(output)
+      result = Result.from_hash(results.first)
+      result.tags.merge!(@options[:tags])
+      result
+    end
+
+    def run_test_inline(test_case, method)
+      begin
+        load_mmtrix_rpm
+        GC.start
+        test_case.run(method)
+      rescue => e
+        result = Result.new(test_case.class.name, method)
+        result.exception = e
+        result
+      end
+    end
+
+    def run_test_case(test_case)
+      methods_for_test_case(test_case).map do |method|
+        if @options[:inline]
+          run_test_inline(test_case, method)
+        else
+          run_test_subprocess(test_case, method)
+        end
+      end
+    end
+
+    def suites_to_run
+      if @options[:identifier]
+        suite, _method = @options[:identifier].split('#')
+        TestCase.subclasses.select { |cls| cls.name == suite }
+      elsif @options[:suite]
+        TestCase.subclasses.select { |cls| @options[:suite].include?(cls.name) }
+      else
+        TestCase.subclasses
+      end
+    end
+
+    def run_all_test_cases
+      results = []
+      suites_to_run.each do |cls|
+        test_case = create_test_case(cls)
+        results += run_test_case(test_case)
+      end
+      results
+    end
+
+    def run_and_report
+      t0 = Time.now
+      results = run_all_test_cases
+      report_results(results, Time.now - t0)
+    end
+
+    def list_test_cases
+      results = []
+      suites_to_run.each do |cls|
+        methods_for_test_case(cls.new).each do |method|
+          results << [cls, method]
+        end
+      end
+      results
+    end
+
+    def report_results(results, elapsed)
+      @reporter_classes.each do |cls|
+        cls.new(results, elapsed, @options).report
+      end
+    end
+  end
+end

data/test/performance/lib/performance/table.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Performance
+  class Table
+    class Formatter
+      attr_reader :name
+
+      def initialize(name, format_string=nil, &blk)
+        @name = name
+        @format_string = format_string
+        @format_proc   = blk
+      end
+
+      def measure(value)
+        format(value).size
+      end
+
+      def format(value, width=nil)
+        formatted = if @format_string
+          sprintf(@format_string, value)
+        elsif @format_proc
+          @format_proc.call(value)
+        else
+          value.to_s
+        end
+
+        formatted = justify(value, formatted, width) if width
+        formatted
+      end
+
+      def justify(value, formatted, width)
+        case value
+        when Numeric then formatted.rjust(width)
+        else              formatted.ljust(width)
+        end
+      end
+    end
+
+    class Builder
+      attr_reader :formatters
+
+      def initialize
+        @formatters = []
+      end
+
+      def column(name, format_string=nil, &blk)
+        @formatters << Formatter.new(name, format_string, &blk)
+      end
+    end
+
+    def initialize(rows, &blk)
+      @rows = rows
+
+      builder = Builder.new
+      builder.instance_eval(&blk)
+      @schema = builder.formatters
+    end
+
+    def column_widths
+      widths = Array.new(@schema.size, 0)
+      @schema.each_with_index do |col, idx|
+        widths[idx] = col.name.size
+      end
+      @rows.each do |row|
+        row.each_with_index do |cell, idx|
+          width = @schema[idx].measure(cell)
+          widths[idx] = [widths[idx], width].max
+        end
+      end
+      widths
+    end
+
+    def render_row(parts)
+      "| " + parts.join(" | ") + " |"
+    end
+
+    def render
+      widths = column_widths
+
+      blanks    = widths.map { |w| "-" * w }
+      top       = '+-' + blanks.join('-+-') + '-+'
+      separator = '|-' + blanks.join('-+-') + '-|'
+      bottom    = '+-' + blanks.join('-+-') + '-+'
+
+      text_rows = []
+
+      headers = @schema.zip(widths).map { |(c, w)| c.name.to_s.ljust(w) }
+      text_rows << render_row(headers)
+
+      @rows.each do |row|
+        parts = []
+        row.each_with_index do |v, i|
+          parts << @schema[i].format(v, widths[i])
+        end
+        text_rows << render_row(parts)
+      end
+
+      puts top + "\n"
+      puts text_rows.join("\n" + separator + "\n")
+      puts bottom + "\n"
+    end
+  end
+end