benchmark-ips 2.3.0 → 2.11.0

data/lib/benchmark/ips/job.rb

@@ -5,95 +5,11 @@ module Benchmark
       # Microseconds per 100 millisecond.
       MICROSECONDS_PER_100MS = 100_000
       # Microseconds per second.
-      MICROSECONDS_PER_SECOND = 1_000_000
+      MICROSECONDS_PER_SECOND = Timing::MICROSECONDS_PER_SECOND
       # The percentage of the expected runtime to allow
       # before reporting a weird runtime
       MAX_TIME_SKEW = 0.05
-
-      # Entries in Benchmark Jobs.
-      class Entry
-        # Instantiate the Benchmark::IPS::Job::Entry.
-        # @param label [#to_s] Label of Benchmarked code.
-        # @param action [String, Proc] Code to be benchmarked.
-        # @raise [ArgumentError] Raises when action is not String or not responding to +call+.
-        def initialize(label, action)
-          @label = label
-
-          if action.kind_of? String
-            compile action
-            @action = self
-            @as_action = true
-          else
-            unless action.respond_to? :call
-              raise ArgumentError, "invalid action, must respond to #call"
-            end
-
-            @action = action
-
-            if action.respond_to? :arity and action.arity > 0
-              @call_loop = true
-            else
-              @call_loop = false
-            end
-
-            @as_action = false
-          end
-        end
-
-        # The label of benchmarking action.
-        # @return [#to_s] Label of action.
-        attr_reader :label
-
-        # The benchmarking action.
-        # @return [String, Proc] Code to be called, could be String / Proc.
-        attr_reader :action
-
-        # Add padding to label's right if label's length < 20,
-        # Otherwise add a new line and 20 whitespaces.
-        # @return [String] Right justified label.
-        def label_rjust
-          label = @label.to_s
-          if label.size > 20
-            "#{label}\n#{' ' * 20}"
-          else
-            label.rjust(20)
-          end
-        end
-
-        # Call action by given times, return if +@call_loop+ is present.
-        # @param times [Integer] Times to call +@action+.
-        # @return [Integer] Number of times the +@action+ has been called.
-        def call_times(times)
-          return @action.call(times) if @call_loop
-
-          act = @action
-
-          i = 0
-          while i < times
-            act.call
-            i += 1
-          end
-        end
-
-        # Compile code into +call_times+ method.
-        # @param str [String] Code to be compiled.
-        # @return [Symbol] :call_times.
-        def compile(str)
-          m = (class << self; self; end)
-          code = <<-CODE
-            def call_times(__total);
-              __i = 0
-              while __i < __total
-                #{str};
-                __i += 1
-              end
-            end
-          CODE
-          m.class_eval code
-        end
-      end # End of Entry
-
-      # class Job
+      POW_2_30 = 1 << 30
 
       # Two-element arrays, consisting of label and block pairs.
       # @return [Array<Entry>] list of entries
@@ -103,6 +19,10 @@ module Benchmark
       # @return [Boolean] true if needs to run compare.
       attr_reader :compare
 
+      # Determining whether to hold results between Ruby invocations
+      # @return [Boolean]
+      attr_accessor :hold
+
       # Report object containing information about the run.
       # @return [Report] the report object.
       attr_reader :full_report
@@ -119,31 +39,76 @@ module Benchmark
       # @return [Integer]
       attr_accessor :time
 
+      # Warmup and calculation iterations.
+      # @return [Integer]
+      attr_accessor :iterations
+
+      # Statistics model.
+      # @return [Object]
+      attr_accessor :stats
+
+      # Confidence.
+      # @return [Integer]
+      attr_accessor :confidence
+
+      # Silence output
+      # @return [Boolean]
+      attr_reader :quiet
+
+      # Suite
+      # @return [Benchmark::IPS::NoopSuite]
+      attr_reader :suite
+
       # Instantiate the Benchmark::IPS::Job.
-      # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite.
-      # @option opts [Boolean] (false) :quiet Suppress the printing of information.
       def initialize opts={}
-        @suite = opts[:suite] || nil
-        @quiet = opts[:quiet] || false
         @list = []
-        @compare = false
+        @run_single = false
         @json_path = false
+        @compare = false
+        @compare_order = :fastest
+        @held_path = nil
+        @held_results = nil
 
-        @timing = {}
+        @timing = Hash.new 1 # default to 1 in case warmup isn't run
         @full_report = Report.new
 
         # Default warmup and calculation time in seconds.
         @warmup = 2
         @time = 5
+        @iterations = 1
+
+        # Default statistical model
+        @stats = :sd
+        @confidence = 95
+
+        self.quiet = false
       end
 
       # Job configuration options, set +@warmup+ and +@time+.
       # @option opts [Integer] :warmup Warmup time.
       # @option opts [Integer] :time Calculation time.
+      # @option iterations [Integer] :time Warmup and calculation iterations.
       def config opts
         @warmup = opts[:warmup] if opts[:warmup]
         @time = opts[:time] if opts[:time]
         @suite = opts[:suite] if opts[:suite]
+        @iterations = opts[:iterations] if opts[:iterations]
+        @stats = opts[:stats] if opts[:stats]
+        @confidence = opts[:confidence] if opts[:confidence]
+        self.quiet = opts[:quiet] if opts.key?(:quiet)
+        self.suite = opts[:suite]
+      end
+
+      def quiet=(val)
+        @stdout = reporter(quiet: val)
+      end
+
+      def suite=(suite)
+        @suite = suite || Benchmark::IPS::NoopSuite.new
+      end
+
+      def reporter(quiet:)
+        quiet ? NoopReport.new : StdoutReport.new
       end
 
       # Return true if job needs to be compared.
@@ -152,11 +117,40 @@ module Benchmark
         @compare
       end
 
-      # Set @compare to true.
-      def compare!
+      # Run comparison utility.
+      def compare!(order: :fastest)
         @compare = true
+        @compare_order = order
+      end
+
+      # Return true if results are held while multiple Ruby invocations
+      # @return [Boolean] Need to hold results between multiple Ruby invocations?
+      def hold?
+        !!@held_path
+      end
+
+      # Hold after each iteration.
+      # @param held_path [String] File name to store hold file.
+      def hold!(held_path)
+        @held_path = held_path
+        @run_single = true
       end
 
+      # Save interim results. Similar to hold, but all reports are run
+      # The report label must change for each invocation.
+      # One way to achieve this is to include the version in the label.
+      # @param held_path [String] File name to store hold file.
+      def save!(held_path)
+        @held_path = held_path
+        @run_single = false
+      end
+
+      # Return true if items are to be run one at a time.
+      # For the traditional hold, this is true
+      # @return [Boolean] Run just a single item?
+      def run_single?
+        @run_single
+      end
 
       # Return true if job needs to generate json.
       # @return [Boolean] Need to generate json?
@@ -164,15 +158,15 @@ module Benchmark
         !!@json_path
       end
 
-      # Set @json_path to given path, defaults to "data.json".
+      # Generate json to given path, defaults to "data.json".
       def json!(path="data.json")
         @json_path = path
       end
 
       # Registers the given label and block pair in the job list.
       # @param label [String] Label of benchmarked code.
-      # @param str [String] Code to be benchamrked.
-      # @param blk [Proc] Code to be benchamrked.
+      # @param str [String] Code to be benchmarked.
+      # @param blk [Proc] Code to be benchmarked.
       # @raise [ArgumentError] Raises if str and blk are both present.
       # @raise [ArgumentError] Raises if str and blk are both absent.
       def item(label="", str=nil, &blk) # :yield:
@@ -195,8 +189,7 @@ module Benchmark
       # @return [Integer] Cycles per 100ms.
       def cycles_per_100ms time_msec, iters
         cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
-        cycles = 1 if cycles <= 0
-        cycles
+        cycles <= 0 ? 1 : cycles
       end
 
       # Calculate the time difference of before and after in microseconds.
@@ -216,121 +209,189 @@ module Benchmark
         MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
       end
 
-      # Run warmup.
-      def run_warmup
-        @list.each do |item|
-          @suite.warming item.label, @warmup if @suite
+      def load_held_results
+        return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
+        require "json"
+        @held_results = {}
+        JSON.load(IO.read(@held_path)).each do |result|
+          @held_results[result['item']] = result
+          create_report(result['item'], result['measured_us'], result['iter'],
+                        create_stats(result['samples']), result['cycles'])
+        end
+      end
+
+      def save_held_results
+        return unless @held_path
+        require "json"
+        data = full_report.entries.map { |e|
+          {
+            'item' => e.label,
+            'measured_us' => e.microseconds,
+            'iter' => e.iterations,
+            'samples' => e.samples,
+            'cycles' => e.measurement_cycle
+          }
+        }
+        IO.write(@held_path, JSON.generate(data) << "\n")
+      end
+
+      def all_results_have_been_run?
+        @full_report.entries.size == @list.size
+      end
+
+      def clear_held_results
+        File.delete @held_path if File.exist?(@held_path)
+      end
 
-          unless @quiet
-            $stdout.print item.label_rjust
+      def run
+        if @warmup && @warmup != 0 then
+          @stdout.start_warming
+          @iterations.times do
+            run_warmup
           end
+        end
 
-          Timing.clean_env
+        @stdout.start_running
 
-          before = Time.now
-          target = Time.now + @warmup
+        @iterations.times do |n|
+          run_benchmark
+        end
 
-          warmup_iter = 0
+        @stdout.footer
+      end
 
-          while Time.now < target
-            item.call_times(1)
-            warmup_iter += 1
-          end
+      # Run warmup.
+      def run_warmup
+        @list.each do |item|
+          next if run_single? && @held_results && @held_results.key?(item.label)
 
-          after = Time.now
+          @suite.warming item.label, @warmup
+          @stdout.warming item.label, @warmup
 
-          warmup_time_us = time_us before, after
+          Timing.clean_env
 
-          @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter
+          # Run for up to half of the configured warmup time with an increasing
+          # number of cycles to reduce overhead and improve accuracy.
+          # This also avoids running with a constant number of cycles, which a
+          # JIT might speculate on and then have to recompile in #run_benchmark.
+          before = Timing.now
+          target = Timing.add_second before, @warmup / 2.0
 
-          case Benchmark::IPS.options[:format]
-          when :human
-            $stdout.printf "%s i/100ms\n", Helpers.scale(@timing[item]) unless @quiet
-          else
-            $stdout.printf "%10d i/100ms\n", @timing[item] unless @quiet
+          cycles = 1
+          begin
+            t0 = Timing.now
+            item.call_times cycles
+            t1 = Timing.now
+            warmup_iter = cycles
+            warmup_time_us = Timing.time_us(t0, t1)
+
+            # If the number of cycles would go outside the 32-bit signed integers range
+            # then exit the loop to avoid overflows and start the 100ms warmup runs
+            break if cycles >= POW_2_30
+            cycles *= 2
+          end while Timing.now + warmup_time_us * 2 < target
+
+          cycles = cycles_per_100ms warmup_time_us, warmup_iter
+          @timing[item] = cycles
+
+          # Run for the remaining of warmup in a similar way as #run_benchmark.
+          target = Timing.add_second before, @warmup
+          while Timing.now + MICROSECONDS_PER_100MS < target
+            item.call_times cycles
           end
 
-          @suite.warmup_stats warmup_time_us, @timing[item] if @suite
+          @stdout.warmup_stats warmup_time_us, @timing[item]
+          @suite.warmup_stats warmup_time_us, @timing[item]
+
+          break if run_single?
         end
       end
 
       # Run calculation.
-      def run
+      def run_benchmark
         @list.each do |item|
-          @suite.running item.label, @time if @suite
+          next if run_single? && @held_results && @held_results.key?(item.label)
 
-          unless @quiet
-            $stdout.print item.label_rjust
-          end
+          @suite.running item.label, @time
+          @stdout.running item.label, @time
 
           Timing.clean_env
 
           iter = 0
 
-          target = Time.now + @time
-
           measurements_us = []
 
           # Running this number of cycles should take around 100ms.
           cycles = @timing[item]
 
-          while Time.now < target
-            before = Time.now
+          target = Timing.add_second Timing.now, @time
+
+          begin
+            before = Timing.now
             item.call_times cycles
-            after = Time.now
+            after = Timing.now
 
             # If for some reason the timing said this took no time (O_o)
             # then ignore the iteration entirely and start another.
-            iter_us = time_us before, after
+            iter_us = Timing.time_us before, after
             next if iter_us <= 0.0
 
             iter += cycles
 
             measurements_us << iter_us
-          end
+          end while Timing.now < target
 
-          final_time = Time.now
+          final_time = before
 
-          measured_us = measurements_us.inject(0) { |a,i| a + i }
+          measured_us = measurements_us.inject(:+)
 
-          all_ips = measurements_us.map { |time_us|
+          samples = measurements_us.map { |time_us|
             iterations_per_sec cycles, time_us
           }
 
-          avg_ips = Timing.mean(all_ips)
-          sd_ips =  Timing.stddev(all_ips).round
-
-          rep = create_report(item, measured_us, iter, avg_ips, sd_ips, cycles)
+          rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)
 
           if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
             rep.show_total_time!
           end
 
-          $stdout.puts " #{rep.body}" unless @quiet
+          @stdout.add_report rep, caller(1).first
+          @suite.add_report rep, caller(1).first
 
-          @suite.add_report rep, caller(1).first if @suite
+          break if run_single?
+        end
+      end
+
+      def create_stats(samples)
+        case @stats
+          when :sd
+            Stats::SD.new(samples)
+          when :bootstrap
+            Stats::Bootstrap.new(samples, @confidence)
+          else
+            raise "unknown stats #{@stats}"
         end
       end
 
       # Run comparison of entries in +@full_report+.
       def run_comparison
-        @full_report.run_comparison
+        @full_report.run_comparison(@compare_order) if compare?
       end
 
       # Generate json from +@full_report+.
       def generate_json
-        @full_report.generate_json @json_path
+        @full_report.generate_json @json_path if json?
       end
 
       # Create report by add entry to +@full_report+.
-      # @param item [Benchmark::IPS::Job::Entry] Report item.
+      # @param label [String] Report item label.
       # @param measured_us [Integer] Measured time in microsecond.
       # @param iter [Integer] Iterations.
-      # @param avg_ips [Float] Average iterations per second.
-      # @param sd_ips [Float] Standard deviation iterations per second.
+      # @param samples [Array<Float>] Sampled iterations per second.
       # @param cycles [Integer] Number of Cycles.
-      def create_report(item, measured_us, iter, avg_ips, sd_ips, cycles)
-        @full_report.add_entry item.label, measured_us, iter, avg_ips, sd_ips, cycles
+      # @return [Report::Entry] Entry with data.
+      def create_report(label, measured_us, iter, samples, cycles)
+        @full_report.add_entry label, measured_us, iter, samples, cycles
       end
     end
   end

data/lib/benchmark/ips/noop_suite.rb

@@ -0,0 +1,25 @@
+module Benchmark
+  module IPS
+    class NoopSuite
+      def start_warming
+      end
+
+      def start_running
+      end
+
+      def footer
+      end
+
+      def warming(a, b)
+      end
+
+      def warmup_stats(a, b)
+      end
+
+      def add_report(a, b)
+      end
+
+      alias_method :running, :warming
+    end
+  end
+end

data/lib/benchmark/ips/report.rb

@@ -3,7 +3,7 @@
 module Benchmark
   module IPS
 
-    # Report contains benchamrking entries.
+    # Report contains benchmarking entries.
     # Perform operations like add new entry, run comparison between entries.
     class Report
 
@@ -13,15 +13,13 @@ module Benchmark
         # @param [#to_s] label Label of entry.
         # @param [Integer] us Measured time in microsecond.
         # @param [Integer] iters Iterations.
-        # @param [Float] ips Iterations per second.
-        # @param [Float] ips_sd Standard deviation of iterations per second.
+        # @param [Object] stats Statistics.
         # @param [Integer] cycles Number of Cycles.
-        def initialize(label, us, iters, ips, ips_sd, cycles)
+        def initialize(label, us, iters, stats, cycles)
           @label = label
           @microseconds = us
           @iterations = iters
-          @ips = ips
-          @ips_sd = ips_sd
+          @stats = stats
           @measurement_cycle = cycles
           @show_total_time = false
         end
@@ -38,13 +36,25 @@ module Benchmark
         # @return [Integer] number of iterations.
         attr_reader :iterations
 
-        # Iterations per second.
+        # Statistical summary of samples.
+        # @return [Object] statisical summary.
+        attr_reader :stats
+
+        # LEGACY: Iterations per second.
         # @return [Float] number of iterations per second.
-        attr_reader :ips
+        def ips
+          @stats.central_tendency
+        end
 
-        # Standard deviation of iteration per second.
+        # LEGACY: Standard deviation of iteration per second.
         # @return [Float] standard deviation of iteration per second.
-        attr_reader :ips_sd
+        def ips_sd
+          @stats.error
+        end
+
+        def samples
+          @stats.samples
+        end
 
         # Number of Cycles.
         # @return [Integer] number of cycles.
@@ -65,8 +75,8 @@ module Benchmark
 
         # Return entry's standard deviation of iteration per second in percentage.
         # @return [Float] +@ips_sd+ in percentage.
-        def stddev_percentage
-          100.0 * (@ips_sd.to_f / @ips.to_f)
+        def error_percentage
+          @stats.error_percentage
         end
 
         alias_method :runtime, :seconds
@@ -78,7 +88,7 @@ module Benchmark
         def body
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(ips), stddev_percentage]
+            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
@@ -87,7 +97,7 @@ module Benchmark
               left.ljust(20) + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [ips, stddev_percentage]
+            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
 
             if @show_total_time
               left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
@@ -117,8 +127,8 @@ module Benchmark
 
       # class Report
 
-      # Entry to represent each benchamarked code in Report.
-      # @return [Array<Entry>] Entries in Report.
+      # Entry to represent each benchmarked code in Report.
+      # @return [Array<Report::Entry>] Entries in Report.
       attr_reader :entries
 
       # Instantiate the Report.
@@ -131,13 +141,14 @@ module Benchmark
       # @param label [String] Entry label.
       # @param microseconds [Integer] Measured time in microsecond.
       # @param iters [Integer] Iterations.
-      # @param ips [Float] Average Iterations per second.
-      # @param ips_sd [Float] Standard deviation of iterations per second.
+      # @param stats [Object] Statistical results.
       # @param measurement_cycle [Integer] Number of cycles.
-      # @return [Entry] Last added entry.
-      def add_entry label, microseconds, iters, ips, ips_sd, measurement_cycle
-        @entries << Entry.new(label, microseconds, iters, ips, ips_sd, measurement_cycle)
-        @entries.last
+      # @return [Report::Entry] Last added entry.
+      def add_entry label, microseconds, iters, stats, measurement_cycle
+        entry = Entry.new(label, microseconds, iters, stats, measurement_cycle)
+        @entries.delete_if { |e| e.label == label }
+        @entries << entry
+        entry
       end
 
       # Entries data in array for generate json.
@@ -145,28 +156,40 @@ module Benchmark
       #   name:   Entry#label
       #   ips:    Entry#ips
       #   stddev: Entry#ips_sd
-      # @return [Array] Array of entries
+      #   microseconds: Entry#microseconds
+      #   iterations:   Entry#iterations
+      #   cycles:       Entry#measurement_cycles
+      # @return [Array<Hash<Symbol,String|Float|Integer>] Array of hashes
       def data
         @data ||= @entries.collect do |entry|
           {
             :name => entry.label,
-            :ips =>  entry.ips,
-            :stddev => entry.ips_sd
+            :central_tendency =>  entry.stats.central_tendency,
+            :ips =>  entry.stats.central_tendency, # for backwards compatibility
+            :error => entry.stats.error,
+            :stddev => entry.stats.error, # for backwards compatibility
+            :microseconds => entry.microseconds,
+            :iterations => entry.iterations,
+            :cycles => entry.measurement_cycle,
           }
         end
       end
 
       # Run comparison of entries.
-      def run_comparison
-        Benchmark.compare(*@entries)
+      def run_comparison(order)
+        Benchmark.compare(*@entries, order: order)
       end
 
       # Generate json from Report#data to given path.
       # @param path [String] path to generate json.
       def generate_json(path)
-        File.open path, "w" do |f|
-          require "json"
-          f.write JSON.pretty_generate(data)
+        require "json"
+        if path.respond_to?(:write) # STDOUT
+          path.write JSON.pretty_generate(data)
+        else
+          File.open path, "w" do |f|
+            f.write JSON.pretty_generate(data)
+          end
         end
       end
     end