benchmark-ips 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac378dafa55fa25f223374c352d9fce265a73a1d
-  data.tar.gz: edcbad0a3ed40b69793e18b52b40b4d2d73b72fd
+  metadata.gz: 8ae132eff282ec75647b3d3d1301cf42372dcdaf
+  data.tar.gz: cc8d76ab64858a0682f7fb767fa8e958973cdb3d
 SHA512:
-  metadata.gz: 427e884910dc92a0e50ceb825ff36770b70c0dcee9088f56d3e090867986f6c8d12d176c7a671d7b964f15c12c2ae6edd4c87b532c6c68bdd351c0508c03fd90
-  data.tar.gz: 3db5f1f4445eb0b2c4985633a197abf58be95f9a2bb01282bda329c2880916e1c25c5cd0ec94b2a17adbdc667e3e8829dd46fa428ef13dc41cf3327136564d89
+  metadata.gz: 8a79e55f2f50dfda13b04fd852bfa122a41918e2e74a105cf42dc01f7d65042a089c392f5fe6964c7784e57c7d7ef73fa6fb43546ec5e7f881e9b945a9c9cf26
+  data.tar.gz: 31bcb112be6fa82299708cc4dfccbe9ac421774ff7ff53da85d6bac1d61cacdc9ef28ec24e07efd96ffb59fbff0e3aee0eb8f180ce4aa90468e223e87de7537e

data/History.txt

@@ -1,3 +1,38 @@
+=== 2.4.0 / 2016-02-12
+
+* 1 minor features
+  * Add support for hold! and independent invocations.
+
+* 6 bug fixes
+  * Separate messages for warming up and calculating.
+  * Tighten timing loop.
+  * Pass simple types into Job#create_report
+  * More concise sorting
+  * Fix runtime comparison
+  * Use runtime if ips is not available
+
+* 5 doc fixes
+  * Fix typo unsed --> used
+  * Better document Report::Entry
+  * Fix some typos in docs
+  * Don't calculate mean 2 times
+  * Add more tolerance to tests
+
+* 13 merged PRs
+  * Merge pull request #44 from kbrock/job_extract
+  * Merge pull request #45 from kbrock/runtime_only
+  * Merge pull request #47 from kbrock/use_avg
+  * Merge pull request #46 from kbrock/report_stdout
+  * Merge pull request #48 from bquorning/fix-label-for-runtime-comparison
+  * Merge pull request #50 from tjschuck/fix_typo
+  * Merge pull request #51 from bquorning/all-reports-respond-to-ips
+  * Merge pull request #52 from kbrock/document_reports
+  * Merge pull request #53 from kbrock/interface_create_report
+  * Merge pull request #54 from PragTob/patch-2
+  * Merge pull request #55 from chrisseaton/messages
+  * Merge pull request #56 from chrisseaton/independence
+  * Merge pull request #57 from chrisseaton/tighten-loop
+
 === 2.3.0 / 2015-07-20
 
 * 2 minor features:

data/README.md

@@ -136,6 +136,27 @@ Benchmark.ips do |x|
 end
 ```
 
+### Independent benchmarking
+
+If you are comparing multiple implementations of a piece of code you may want
+to benchmark them in separate invocations of Ruby so that the measurements
+are independent of each other. You can do this with the `hold!` command.
+
+```ruby
+Benchmark.ips do |x|
+  
+  ...
+  
+  # Hold results between multiple invocations of Ruby
+  x.hold! 'filename'
+  
+end
+```
+
+This will run only one benchmarks each time you run the command, storing
+results in the specified file. The file is deleted when all results have been
+gathered and the report is shown.
+
 ## REQUIREMENTS:
 
 * None!

data/lib/benchmark/compare.rb

@@ -28,40 +28,23 @@ module Benchmark
 
     # Compare between reports, prints out facts of each report:
     # runtime, comparative speed difference.
-    # @param reports [Array<Report>] Reports to compare.
-    def compare(*reports)
-      return if reports.size < 2
+    # @param entries [Array<Report::Entry>] Reports to compare.
+    def compare(*entries)
+      return if entries.size < 2
 
-      iter = false
-      sorted = reports.sort do |a,b|
-        if a.respond_to? :ips
-          iter = true
-          b.ips <=> a.ips
-        else
-          a.runtime <=> b.runtime
-        end
-      end
+      sorted = entries.sort_by(&:ips).reverse
 
       best = sorted.shift
 
       $stdout.puts "\nComparison:"
 
-      if iter
-        $stdout.printf "%20s: %10.1f i/s\n", best.label, best.ips
-      else
-        $stdout.puts "#{best.rjust(20)}: #{best.runtime}s"
-      end
+      $stdout.printf "%20s: %10.1f i/s\n", best.label, best.ips
 
       sorted.each do |report|
         name = report.label.to_s
 
-        if iter
-          x = (best.ips.to_f / report.ips.to_f)
-          $stdout.printf "%20s: %10.1f i/s - %.2fx slower\n", name, report.ips, x
-        else
-          x = "%.2f" % (report.ips.to_f / best.ips.to_f)
-          $stdout.puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
-        end
+        x = (best.ips.to_f / report.ips.to_f)
+        $stdout.printf "%20s: %10.1f i/s - %.2fx slower\n", name, report.ips, x
       end
 
       $stdout.puts

data/lib/benchmark/ips.rb

@@ -2,6 +2,8 @@
 require 'benchmark/timing'
 require 'benchmark/compare'
 require 'benchmark/ips/report'
+require 'benchmark/ips/job/entry'
+require 'benchmark/ips/job/stdout_report'
 require 'benchmark/ips/job'
 
 # Performance benchmarking library
@@ -11,10 +13,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.3.0"
+    VERSION = "2.4.0"
 
     # CODENAME of current version.
-    CODENAME = "Monsoon BBQ"
+    CODENAME = "Nonlinear Rollercoaster"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -49,26 +51,17 @@ module Benchmark
       job.config job_opts
 
       yield job
-
-      $stdout.puts "Calculating -------------------------------------" unless quiet
+      
+      job.load_held_results if job.hold? && job.held_results?
 
       job.run_warmup
-
-      $stdout.puts "-------------------------------------------------" unless quiet
-
       job.run
 
       $stdout.sync = sync
+      job.run_comparison
+      job.generate_json
 
-      if job.compare?
-        job.run_comparison
-      end
-
-      if job.json?
-        job.generate_json
-      end
-
-      return job.full_report
+      job.full_report
     end
 
     # Set options for running the benchmarks.

data/lib/benchmark/ips/job.rb

@@ -10,91 +10,6 @@ module Benchmark
       # before reporting a weird runtime
       MAX_TIME_SKEW = 0.05
 
-      # Entries in Benchmark Jobs.
-      class Entry
-        # Instantiate the Benchmark::IPS::Job::Entry.
-        # @param label [#to_s] Label of Benchmarked code.
-        # @param action [String, Proc] Code to be benchmarked.
-        # @raise [ArgumentError] Raises when action is not String or not responding to +call+.
-        def initialize(label, action)
-          @label = label
-
-          if action.kind_of? String
-            compile action
-            @action = self
-            @as_action = true
-          else
-            unless action.respond_to? :call
-              raise ArgumentError, "invalid action, must respond to #call"
-            end
-
-            @action = action
-
-            if action.respond_to? :arity and action.arity > 0
-              @call_loop = true
-            else
-              @call_loop = false
-            end
-
-            @as_action = false
-          end
-        end
-
-        # The label of benchmarking action.
-        # @return [#to_s] Label of action.
-        attr_reader :label
-
-        # The benchmarking action.
-        # @return [String, Proc] Code to be called, could be String / Proc.
-        attr_reader :action
-
-        # Add padding to label's right if label's length < 20,
-        # Otherwise add a new line and 20 whitespaces.
-        # @return [String] Right justified label.
-        def label_rjust
-          label = @label.to_s
-          if label.size > 20
-            "#{label}\n#{' ' * 20}"
-          else
-            label.rjust(20)
-          end
-        end
-
-        # Call action by given times, return if +@call_loop+ is present.
-        # @param times [Integer] Times to call +@action+.
-        # @return [Integer] Number of times the +@action+ has been called.
-        def call_times(times)
-          return @action.call(times) if @call_loop
-
-          act = @action
-
-          i = 0
-          while i < times
-            act.call
-            i += 1
-          end
-        end
-
-        # Compile code into +call_times+ method.
-        # @param str [String] Code to be compiled.
-        # @return [Symbol] :call_times.
-        def compile(str)
-          m = (class << self; self; end)
-          code = <<-CODE
-            def call_times(__total);
-              __i = 0
-              while __i < __total
-                #{str};
-                __i += 1
-              end
-            end
-          CODE
-          m.class_eval code
-        end
-      end # End of Entry
-
-      # class Job
-
       # Two-element arrays, consisting of label and block pairs.
       # @return [Array<Entry>] list of entries
       attr_reader :list
@@ -103,6 +18,10 @@ module Benchmark
       # @return [Boolean] true if needs to run compare.
       attr_reader :compare
 
+      # Determining whether to hold results between Ruby invocations
+      # @return [Boolean]
+      attr_accessor :hold
+
       # Report object containing information about the run.
       # @return [Report] the report object.
       attr_reader :full_report
@@ -124,10 +43,12 @@ module Benchmark
       # @option opts [Boolean] (false) :quiet Suppress the printing of information.
       def initialize opts={}
         @suite = opts[:suite] || nil
-        @quiet = opts[:quiet] || false
+        @stdout = opts[:quiet] ? nil : StdoutReport.new
         @list = []
         @compare = false
         @json_path = false
+        @held_path = nil
+        @held_results = nil
 
         @timing = {}
         @full_report = Report.new
@@ -157,6 +78,16 @@ module Benchmark
         @compare = true
       end
 
+      # Return true if results are held while multiple Ruby invocations
+      # @return [Boolean] Need to hold results between multiple Ruby invocations?
+      def hold?
+        !!@held_path
+      end
+
+      # Set @hold to true.
+      def hold!(held_path)
+        @held_path = held_path
+      end
 
       # Return true if job needs to generate json.
       # @return [Boolean] Need to generate json?
@@ -171,8 +102,8 @@ module Benchmark
 
       # Registers the given label and block pair in the job list.
       # @param label [String] Label of benchmarked code.
-      # @param str [String] Code to be benchamrked.
-      # @param blk [Proc] Code to be benchamrked.
+      # @param str [String] Code to be benchmarked.
+      # @param blk [Proc] Code to be benchmarked.
       # @raise [ArgumentError] Raises if str and blk are both present.
       # @raise [ArgumentError] Raises if str and blk are both absent.
       def item(label="", str=nil, &blk) # :yield:
@@ -215,15 +146,27 @@ module Benchmark
       def iterations_per_sec cycles, time_us
         MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
       end
+      
+      def held_results?
+        File.exist?(@held_path)
+      end
+      
+      def load_held_results
+        require "json"
+        @held_results = Hash[File.open(@held_path).map { |line|
+          result = JSON.parse(line)
+          [result['item'], result]
+        }]
+      end
 
       # Run warmup.
       def run_warmup
+        @stdout.start_warming if @stdout
         @list.each do |item|
+          next if hold? && @held_results && @held_results.key?(item.label)
+          
           @suite.warming item.label, @warmup if @suite
-
-          unless @quiet
-            $stdout.print item.label_rjust
-          end
+          @stdout.warming item.label, @warmup if @stdout
 
           Timing.clean_env
 
@@ -243,37 +186,38 @@ module Benchmark
 
           @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter
 
-          case Benchmark::IPS.options[:format]
-          when :human
-            $stdout.printf "%s i/100ms\n", Helpers.scale(@timing[item]) unless @quiet
-          else
-            $stdout.printf "%10d i/100ms\n", @timing[item] unless @quiet
-          end
-
+          @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
           @suite.warmup_stats warmup_time_us, @timing[item] if @suite
+          
+          break if hold?
         end
       end
 
       # Run calculation.
       def run
+        @stdout.start_running if @stdout
         @list.each do |item|
-          @suite.running item.label, @time if @suite
-
-          unless @quiet
-            $stdout.print item.label_rjust
+          if hold? && @held_results && @held_results.key?(item.label)
+           result = @held_results[item.label]
+            create_report(item.label, result['measured_us'], result['iter'],
+              result['avg_ips'], result['sd_ips'], result['cycles'])
+            next
           end
+          
+          @suite.running item.label, @time if @suite
+          @stdout.running item.label, @time if @stdout
 
           Timing.clean_env
 
           iter = 0
 
-          target = Time.now + @time
-
           measurements_us = []
 
           # Running this number of cycles should take around 100ms.
           cycles = @timing[item]
 
+          target = Time.now + @time
+          
           while Time.now < target
             before = Time.now
             item.call_times cycles
@@ -298,39 +242,62 @@ module Benchmark
           }
 
           avg_ips = Timing.mean(all_ips)
-          sd_ips =  Timing.stddev(all_ips).round
+          sd_ips =  Timing.stddev(all_ips, avg_ips).round
 
-          rep = create_report(item, measured_us, iter, avg_ips, sd_ips, cycles)
+          rep = create_report(item.label, measured_us, iter, avg_ips, sd_ips, cycles)
 
           if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
             rep.show_total_time!
           end
 
-          $stdout.puts " #{rep.body}" unless @quiet
-
+          @stdout.add_report rep, caller(1).first if @stdout
           @suite.add_report rep, caller(1).first if @suite
+          
+          if hold? && item != @list.last
+            File.open @held_path, "a" do |f|
+              require "json"
+              f.write JSON.generate({
+                :item => item.label,
+                :measured_us => measured_us,
+                :iter => iter,
+                :avg_ips => avg_ips,
+                :sd_ips => sd_ips,
+                :cycles => cycles
+              })
+              f.write "\n"
+            end
+            
+            puts
+            puts 'Pausing here -- run Ruby again to measure the next benchmark...'
+            break
+          end
+        end
+        
+        if hold? && @full_report.entries.size == @list.size
+          File.delete @held_path if File.exist?(@held_path)
         end
       end
 
       # Run comparison of entries in +@full_report+.
       def run_comparison
-        @full_report.run_comparison
+        @full_report.run_comparison if compare?
       end
 
       # Generate json from +@full_report+.
       def generate_json
-        @full_report.generate_json @json_path
+        @full_report.generate_json @json_path if json?
       end
 
       # Create report by add entry to +@full_report+.
-      # @param item [Benchmark::IPS::Job::Entry] Report item.
+      # @param label [String] Report item label.
       # @param measured_us [Integer] Measured time in microsecond.
       # @param iter [Integer] Iterations.
       # @param avg_ips [Float] Average iterations per second.
       # @param sd_ips [Float] Standard deviation iterations per second.
       # @param cycles [Integer] Number of Cycles.
-      def create_report(item, measured_us, iter, avg_ips, sd_ips, cycles)
-        @full_report.add_entry item.label, measured_us, iter, avg_ips, sd_ips, cycles
+      # @return [Report::Entry] Entry with data.
+      def create_report(label, measured_us, iter, avg_ips, sd_ips, cycles)
+        @full_report.add_entry label, measured_us, iter, avg_ips, sd_ips, cycles
       end
     end
   end

data/lib/benchmark/ips/report.rb

@@ -3,7 +3,7 @@
 module Benchmark
   module IPS
 
-    # Report contains benchamrking entries.
+    # Report contains benchmarking entries.
     # Perform operations like add new entry, run comparison between entries.
     class Report
 
@@ -117,8 +117,8 @@ module Benchmark
 
       # class Report
 
-      # Entry to represent each benchamarked code in Report.
-      # @return [Array<Entry>] Entries in Report.
+      # Entry to represent each benchmarked code in Report.
+      # @return [Array<Report::Entry>] Entries in Report.
       attr_reader :entries
 
       # Instantiate the Report.
@@ -134,10 +134,11 @@ module Benchmark
       # @param ips [Float] Average Iterations per second.
       # @param ips_sd [Float] Standard deviation of iterations per second.
       # @param measurement_cycle [Integer] Number of cycles.
-      # @return [Entry] Last added entry.
+      # @return [Report::Entry] Last added entry.
       def add_entry label, microseconds, iters, ips, ips_sd, measurement_cycle
-        @entries << Entry.new(label, microseconds, iters, ips, ips_sd, measurement_cycle)
-        @entries.last
+        entry = Entry.new(label, microseconds, iters, ips, ips_sd, measurement_cycle)
+        @entries << entry
+        entry
       end
 
       # Entries data in array for generate json.
@@ -145,7 +146,7 @@ module Benchmark
       #   name:   Entry#label
       #   ips:    Entry#ips
       #   stddev: Entry#ips_sd
-      # @return [Array] Array of entries
+      # @return [Array<Hash<Symbol,String|Float>] Array of hashes with :label, :ips, :stddev
       def data
         @data ||= @entries.collect do |entry|
           {

data/lib/benchmark/timing.rb

@@ -43,7 +43,7 @@ module Benchmark
       resamples
     end
 
-    # Recycle unsed objects by starting Garbage Collector.
+    # Recycle used objects by starting Garbage Collector.
     def self.clean_env
       # rbx
       if GC.respond_to? :run

data/test/test_benchmark_ips.rb

@@ -59,7 +59,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.05", rep2.label
     assert_in_delta 20.0, rep2.iterations.to_f, 1.0
-    assert_in_delta 20.0, rep2.ips, 1.0
+    assert_in_delta 20.0, rep2.ips, 2.0
   end
 
   def test_ips_alternate_config
@@ -73,7 +73,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4, rep.iterations
-    assert_in_delta 4.0, rep.ips, 0.2
+    assert_in_delta 4.0, rep.ips, 0.4
   end
 
   def test_ips_old_config

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Evan Phoenix
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-20 00:00:00.000000000 Z
+date: 2016-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.6'
+        version: '5.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.6'
+        version: '5.8'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
 description: A iterations per second enhancement to Benchmark.
 email:
 - evan@phx.io
@@ -63,7 +63,6 @@ extra_rdoc_files:
 - README.md
 files:
 - ".autotest"
-- ".gemtest"
 - History.txt
 - Manifest.txt
 - README.md
@@ -96,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A iterations per second enhancement to Benchmark.