motion-benchmark-ips 1.0 → 1.1

data/lib/project/ips/job.rb

@@ -0,0 +1,351 @@
+module Benchmark
+  module IPS
+    # Benchmark jobs.
+    class Job
+      # Microseconds per 100 millisecond.
+      MICROSECONDS_PER_100MS = 100_000
+      # Microseconds per second.
+      MICROSECONDS_PER_SECOND = Timing::MICROSECONDS_PER_SECOND
+      # The percentage of the expected runtime to allow
+      # before reporting a weird runtime
+      MAX_TIME_SKEW = 0.05
+
+      # Two-element arrays, consisting of label and block pairs.
+      # @return [Array<Entry>] list of entries
+      attr_reader :list
+
+      # Determining whether to run comparison utility.
+      # @return [Boolean] true if needs to run compare.
+      attr_reader :compare
+
+      # Determining whether to hold results between Ruby invocations
+      # @return [Boolean]
+      attr_accessor :hold
+
+      # Report object containing information about the run.
+      # @return [Report] the report object.
+      attr_reader :full_report
+
+      # Storing Iterations in time period.
+      # @return [Hash]
+      attr_reader :timing
+
+      # Warmup time setter and getter (in seconds).
+      # @return [Integer]
+      attr_accessor :warmup
+
+      # Calculation time setter and getter (in seconds).
+      # @return [Integer]
+      attr_accessor :time
+
+      # Warmup and calculation iterations.
+      # @return [Integer]
+      attr_accessor :iterations
+
+      # Statistics model.
+      # @return [Object]
+      attr_accessor :stats
+
+      # Confidence.
+      # @return [Integer]
+      attr_accessor :confidence
+
+      # Instantiate the Benchmark::IPS::Job.
+      # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite.
+      # @option opts [Boolean] (false) :quiet Suppress the printing of information.
+      def initialize opts={}
+        @suite = opts[:suite] || nil
+        @stdout = opts[:quiet] ? nil : StdoutReport.new
+        @list = []
+        @compare = false
+        @json_path = false
+        @held_path = nil
+        @held_results = nil
+
+        @timing = Hash.new 1 # default to 1 in case warmup isn't run
+        @full_report = Report.new
+
+        # Default warmup and calculation time in seconds.
+        @warmup = 2
+        @time = 5
+        @iterations = 1
+
+        # Default statistical model
+        @stats = :sd
+        @confidence = 95
+      end
+
+      # Job configuration options, set +@warmup+ and +@time+.
+      # @option opts [Integer] :warmup Warmup time.
+      # @option opts [Integer] :time Calculation time.
+      # @option iterations [Integer] :time Warmup and calculation iterations.
+      def config opts
+        @warmup = opts[:warmup] if opts[:warmup]
+        @time = opts[:time] if opts[:time]
+        @suite = opts[:suite] if opts[:suite]
+        @iterations = opts[:iterations] if opts[:iterations]
+        @stats = opts[:stats] if opts[:stats]
+        @confidence = opts[:confidence] if opts[:confidence]
+      end
+
+      # Return true if job needs to be compared.
+      # @return [Boolean] Need to compare?
+      def compare?
+        @compare
+      end
+
+      # Set @compare to true.
+      def compare!
+        @compare = true
+      end
+
+      # Return true if results are held while multiple Ruby invocations
+      # @return [Boolean] Need to hold results between multiple Ruby invocations?
+      def hold?
+        !!@held_path
+      end
+
+      # Set @hold to true.
+      def hold!(held_path)
+        @held_path = held_path
+      end
+
+      # Return true if job needs to generate json.
+      # @return [Boolean] Need to generate json?
+      def json?
+        !!@json_path
+      end
+
+      # Set @json_path to given path, defaults to "data.json".
+      def json!(path="data.json")
+        @json_path = path
+      end
+
+      # Registers the given label and block pair in the job list.
+      # @param label [String] Label of benchmarked code.
+      # @param str [String] Code to be benchmarked.
+      # @param blk [Proc] Code to be benchmarked.
+      # @raise [ArgumentError] Raises if str and blk are both present.
+      # @raise [ArgumentError] Raises if str and blk are both absent.
+      def item(label="", str=nil, &blk) # :yield:
+        if blk and str
+          raise ArgumentError, "specify a block and a str, but not both"
+        end
+
+        action = str || blk
+        raise ArgumentError, "no block or string" unless action
+
+        @list.push Entry.new(label, action)
+        self
+      end
+      alias_method :report, :item
+
+      # Calculate the cycles needed to run for approx 100ms,
+      # given the number of iterations to run the given time.
+      # @param [Float] time_msec Each iteration's time in ms.
+      # @param [Integer] iters Iterations.
+      # @return [Integer] Cycles per 100ms.
+      def cycles_per_100ms time_msec, iters
+        cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
+        cycles <= 0 ? 1 : cycles
+      end
+
+      # Calculate the time difference of before and after in microseconds.
+      # @param [Time] before time.
+      # @param [Time] after time.
+      # @return [Float] Time difference of before and after.
+      def time_us before, after
+        (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
+      end
+
+      # Calculate the interations per second given the number
+      # of cycles run and the time in microseconds that elapsed.
+      # @param [Integer] cycles Cycles.
+      # @param [Integer] time_us Time in microsecond.
+      # @return [Float] Iteration per second.
+      def iterations_per_sec cycles, time_us
+        MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
+      end
+
+      def held_results?
+        File.exist?(@held_path)
+      end
+
+      def load_held_results
+        require "json"
+        @held_results = Hash[File.open(@held_path).map { |line|
+          result = JSON.parse(line)
+          [result['item'], result]
+        }]
+      end
+
+      def run
+        if @warmup && @warmup != 0 then
+          @stdout.start_warming if @stdout
+          @iterations.times do
+            run_warmup
+          end
+        end
+
+        @stdout.start_running if @stdout
+
+        held = nil
+
+        @iterations.times do |n|
+          held = run_benchmark
+        end
+
+        @stdout.footer if @stdout
+
+        if held
+          puts
+          puts 'Pausing here -- run Ruby again to measure the next benchmark...'
+        end
+      end
+
+      # Run warmup.
+      def run_warmup
+        @list.each do |item|
+          next if hold? && @held_results && @held_results.key?(item.label)
+
+          @suite.warming item.label, @warmup if @suite
+          @stdout.warming item.label, @warmup if @stdout
+
+          Timing.clean_env
+
+          before = Timing.now
+          target = Timing.add_second before, @warmup
+
+          warmup_iter = 0
+
+          while Timing.now < target
+            item.call_times(1)
+            warmup_iter += 1
+          end
+
+          after = Timing.now
+
+          warmup_time_us = Timing.time_us(before, after)
+
+          @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter
+
+          @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
+          @suite.warmup_stats warmup_time_us, @timing[item] if @suite
+
+          break if hold?
+        end
+      end
+
+      # Run calculation.
+      def run_benchmark
+        @list.each do |item|
+          if hold? && @held_results && @held_results.key?(item.label)
+           result = @held_results[item.label]
+            create_report(item.label, result['measured_us'], result['iter'],
+                          create_stats(result['samples']), result['cycles'])
+            next
+          end
+
+          @suite.running item.label, @time if @suite
+          @stdout.running item.label, @time if @stdout
+
+          Timing.clean_env
+
+          iter = 0
+
+          measurements_us = []
+
+          # Running this number of cycles should take around 100ms.
+          cycles = @timing[item]
+
+          target = Timing.add_second Timing.now, @time
+
+          while (before = Timing.now) < target
+            item.call_times cycles
+            after = Timing.now
+
+            # If for some reason the timing said this took no time (O_o)
+            # then ignore the iteration entirely and start another.
+            iter_us = Timing.time_us before, after
+            next if iter_us <= 0.0
+
+            iter += cycles
+
+            measurements_us << iter_us
+          end
+
+          final_time = before
+
+          measured_us = measurements_us.inject(:+)
+
+          samples = measurements_us.map { |time_us|
+            iterations_per_sec cycles, time_us
+          }
+
+          rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)
+
+          if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
+            rep.show_total_time!
+          end
+
+          @stdout.add_report rep, caller(1).first if @stdout
+          @suite.add_report rep, caller(1).first if @suite
+
+          if hold? && item != @list.last
+            File.open @held_path, "a" do |f|
+              require "json"
+              f.write JSON.generate({
+                :item => item.label,
+                :measured_us => measured_us,
+                :iter => iter,
+                :samples => samples,
+                :cycles => cycles
+              })
+              f.write "\n"
+            end
+
+            return true
+          end
+        end
+
+        if hold? && @full_report.entries.size == @list.size
+          File.delete @held_path if File.exist?(@held_path)
+        end
+
+        false
+      end
+
+      def create_stats(samples)
+        case @stats
+          when :sd
+            Stats::SD.new(samples)
+          when :bootstrap
+            Stats::Bootstrap.new(samples, @confidence)
+          else
+            raise "unknown stats #{@stats}"
+        end
+      end
+
+      # Run comparison of entries in +@full_report+.
+      def run_comparison
+        @full_report.run_comparison if compare?
+      end
+
+      # Generate json from +@full_report+.
+      def generate_json
+        @full_report.generate_json @json_path if json?
+      end
+
+      # Create report by add entry to +@full_report+.
+      # @param label [String] Report item label.
+      # @param measured_us [Integer] Measured time in microsecond.
+      # @param iter [Integer] Iterations.
+      # @param samples [Array<Float>] Sampled iterations per second.
+      # @param cycles [Integer] Number of Cycles.
+      # @return [Report::Entry] Entry with data.
+      def create_report(label, measured_us, iter, samples, cycles)
+        @full_report.add_entry label, measured_us, iter, samples, cycles
+      end
+    end
+  end
+end

data/lib/project/ips/job/entry.rb

@@ -0,0 +1,77 @@
+module Benchmark
+  module IPS
+    # Benchmark jobs.
+    class Job
+      # Entries in Benchmark Jobs.
+      class Entry
+        # Instantiate the Benchmark::IPS::Job::Entry.
+        # @param label [#to_s] Label of Benchmarked code.
+        # @param action [String, Proc] Code to be benchmarked.
+        # @raise [ArgumentError] Raises when action is not String or not responding to +call+.
+        def initialize(label, action)
+          @label = label
+
+          if action.kind_of? String
+            compile action
+            @action = self
+            @as_action = true
+          else
+            unless action.respond_to? :call
+              raise ArgumentError, "invalid action, must respond to #call"
+            end
+
+            @action = action
+
+            if action.respond_to? :arity and action.arity > 0
+              @call_loop = true
+            else
+              @call_loop = false
+            end
+
+            @as_action = false
+          end
+        end
+
+        # The label of benchmarking action.
+        # @return [#to_s] Label of action.
+        attr_reader :label
+
+        # The benchmarking action.
+        # @return [String, Proc] Code to be called, could be String / Proc.
+        attr_reader :action
+
+        # Call action by given times, return if +@call_loop+ is present.
+        # @param times [Integer] Times to call +@action+.
+        # @return [Integer] Number of times the +@action+ has been called.
+        def call_times(times)
+          return @action.call(times) if @call_loop
+
+          act = @action
+
+          i = 0
+          while i < times
+            act.call
+            i += 1
+          end
+        end
+
+        # Compile code into +call_times+ method.
+        # @param str [String] Code to be compiled.
+        # @return [Symbol] :call_times.
+        def compile(str)
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(__total);
+              __i = 0
+              while __i < __total
+                #{str};
+                __i += 1
+              end
+            end
+          CODE
+          m.class_eval code
+        end
+      end
+    end
+  end
+end

data/lib/project/ips/job/stdout_report.rb

@@ -0,0 +1,64 @@
+module Benchmark
+  module IPS
+    class Job
+      class StdoutReport
+        def initialize
+          @last_item = nil
+        end
+
+        def start_warming
+          $stdout.puts "Warming up --------------------------------------"
+        end
+
+        def start_running
+          $stdout.puts "Calculating -------------------------------------"
+        end
+
+        def warming(label, _warmup)
+          $stdout.print rjust(label)
+        end
+
+        def warmup_stats(_warmup_time_us, timing)
+          case format
+          when :human
+            $stdout.printf "%s i/100ms\n", Helpers.scale(timing)
+          else
+            $stdout.printf "%10d i/100ms\n", timing
+          end
+        end
+
+        alias_method :running, :warming
+
+        def add_report(item, caller)
+          $stdout.puts " #{item.body}"
+          @last_item = item
+        end
+
+        def footer
+          return unless @last_item
+          footer = @last_item.stats.footer
+          $stdout.puts footer.rjust(40) if footer
+        end
+
+        private
+
+        # @return [Symbol] format used for benchmarking
+        def format
+          Benchmark::IPS.options[:format]
+        end
+
+        # Add padding to label's right if label's length < 20,
+        # Otherwise add a new line and 20 whitespaces.
+        # @return [String] Right justified label.
+        def rjust(label)
+          label = label.to_s
+          if label.size > 20
+            "#{label}\n#{' ' * 20}"
+          else
+            label.rjust(20)
+          end
+        end
+      end
+    end
+  end
+end