benchmark-bigo 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ddf502a4bf2a5eabc6ec0127d22f1ab16ee69eca
-  data.tar.gz: 43b1ce1e054c451c63a0d9597d73461d4b64be15
+  metadata.gz: f2001993a5384ac4c261dfcfab54718bd699ebe5
+  data.tar.gz: 9126a20c3623302fee2c997c2264d601164adc21
 SHA512:
-  metadata.gz: 6c73e84f56907153c5ff2ec84ecf4da9942431ed477a2408504b0d96e2c5d0be2c3aee8638a117fa5c5da29a76b4c4b150f2221b5362520b66d296201dc19fce
-  data.tar.gz: e7bbe74394cd62b596a51bc2002abe776f36c789ffb7b4a941dc88fb3e8c2fd2a4339dcc008c281e12264385b547fcfd91fd2d4c32a8e19d00405667eec57ea6
+  metadata.gz: b4a97d657456950c926c63a56cf12dd9b2f60b7b793b1e4b4d305b42cd2e03f0dd82d59674b93ccfaa7a06024922a106b29d1e5eadd0dcd85ed845bffcf2e6ca
+  data.tar.gz: f414c0c8f1a215ccca45103e543e60486700a1133544f5ed6799d151610c30d0fe3db312d30ec009e2c6c62db22b9108edf04fdb2cd31018a1a1607e3ba7fc7d

data/README.md

@@ -23,10 +23,7 @@ $ gem install benchmark-bigo
 ```ruby
 require 'benchmark/bigo'
 
-report = Benchmark.bigo do |x|
-  # increments is the total number of data points to collect
-  x.increments = 6
-
+Benchmark.bigo do |x|
   # generator should construct a test object of the given size
   # example of an Array generator
   x.generator {|size| (0...size).to_a.shuffle }
@@ -34,20 +31,23 @@ report = Benchmark.bigo do |x|
   # or you can use the built in array generator
   # x.generate :array
 
-  # specifies how the size of the object should grow
-  #   options: linear, exponential
-  #
-  # specifies that arrays should grow linearly by 1000
-  x.linear 1000
+  # steps is the total number of data points to collect
+  # default is 10
+  x.steps = 6
+
+  # step_size is the size between steps
+  # default is 100
+  x.step_size = 200
+
+  # indicates the starting size of the object to test
+  # default is 100
+  x.min_size = 1000
 
   # report takes a label and a block.
   # block is passed in the generated object and the size of that object
-  x.report("#at") {|generated, size| generated.at rand(size) }
-  x.report("#index") {|generated, size| generated.index rand(size) }
-  x.report("#empty-index") {|generated, size| generated.index 'foo' }
-
-  # save results in JSON format
-  x.data! 'chart_array_simple.json'
+  x.report("#at")           {|array, size| array.at rand(size) }
+  x.report("#index")        {|array, size| array.index rand(size) }
+  x.report("#index-miss")   {|array, size| array.index (size + rand(size)) }
 
   # generate HTML chart using ChartKick
   x.chart! 'chart_array_simple.html'
@@ -55,9 +55,113 @@ report = Benchmark.bigo do |x|
   # for each report, create a comparison chart showing the report
   # and scaled series for O(log n), O(n), O(n log n), and O(n squared)
   x.compare!
+
+  # generate an ASCII chart using gnuplot
+  # works best with only one or two reports
+  # otherwise the lines often overlap each other
+  x.termplot!
+
+  # generate JSON output
+  x.json! 'chart_array_simple.json'
+
+  # generate CSV output
+  x.csv! 'chart_array_simple.csv'
+end
+```
+
+## Generators
+
+In order to test a block of code on an input of varying size, the benchmark tool must know how to generate this varying input. **Generators** are used to create this input to be tested. In many cases the generator creates an object, but this is not the only way generators can be used.
+
+There are a few built-in generators to make it easy to test common objects.
+
+### Array
+
+The Array generator is used by calling
+
+```
+Benchmark.bigo do |x|
+  x.generate :array
+  ...
+end
+```
+
+This generator knows how to create Arrays of varying sizes. The generated Array contains the set of numbers between 0 and (size-1), randomly shuffled. For example, a generated Array of size 5 might look like:
+
+```
+[3,2,4,1,0]
+```
+
+### String
+
+The String generator is used by calling
+
+```
+Benchmark.bigo do |x|
+  x.generate :string
+  ...
 end
 ```
 
+This generator uses `SecureRandom.hex` to create random hexadecimal string of varying sizes. In this case the size indicates the length in *bytes* of the string to be generated. This means that the resulting string will be 2 * size. For example, a generated String of size 10 might look like:
+
+```
+"70fd739c8640b8c9212b"
+```
+
+### Size
+
+This generator is useful when there isn't a direct object to be tested, and instead the benchmark is testing the impact of the size within the report blocks themselves. In this case, the two variables passed to the report block will both be the size value. The example shows how you might test the performance difference for iteration operations.
+
+Example:
+
+```
+Benchmark.bigo do |x|
+  x.generate :size
+
+  x.report("#times") {|size,_|
+
+    size.times do |i|
+      # do things here
+    end
+  }
+
+  x.report("for") {|size,_|
+
+    for i in size
+      # do things here
+    end
+  }
+
+end
+```
+
+### Custom
+
+There are many cases where custom generators are needed. This is the case if the code you wish to benchmark
+runs against a specific type of input. A custom generator is created by specifying a block that accepts a single size parameter and returns an object of that size.
+
+The following generator creates a Hash where each of the keys is the integer of the size and the value is a random string 10 characters long.
+
+```
+Benchmark.bigo do |x|
+  x.generator do |size|
+    h = {}
+    (0...size).each {|i| h[i] = SecureRandom.hex(10)}
+    h
+  end
+  ...
+end
+
+# for size 5
+# => { 0 => "1fb3e5bb88815d824fff",
+#      1 => "63006b6e810502f5bedd",
+#      2 => "7bef4e15a2b4f9e44fa9",
+#      3 => "f78319925f8c0790b409",
+#      4 => "2bfc6a0a91c4949dc6ae"
+#    }
+```
+
 ## Contributing
 
 After checking out the source, run the tests:

data/lib/benchmark/bigo.rb

@@ -1,8 +1,11 @@
 # encoding: utf-8
 require 'erb'
 require 'chartkick'
+require 'csv'
 require 'benchmark/ips'
 require 'benchmark/bigo/report'
+require 'benchmark/bigo/chart'
+require 'benchmark/bigo/termplot'
 require 'benchmark/bigo/job'
 
 module Benchmark
@@ -32,14 +35,7 @@ module Benchmark
       $stdout.puts "-------------------------------------------------" unless quiet
 
       job.run
-
-      if job.data?
-        job.generate_data
-      end
-
-      if job.chart?
-        job.generate_chart
-      end
+      job.generate_output
 
       $stdout.sync = sync
 

data/lib/benchmark/bigo/chart.rb

@@ -0,0 +1,148 @@
+module Benchmark
+
+  module BigO
+    class Chart
+
+      TYPES = [:const, :logn, :n, :nlogn, :n_sq]
+      attr_accessor :sample_size
+
+      def initialize report_data, sizes
+        @data = report_data.freeze
+        @sizes = sizes.freeze
+
+        @sample_size = @sizes.first
+
+        # can't take log of 1,
+        # so it can't be used as the sample
+        if @sample_size == 1
+          @sample_size = @sizes[1]
+        end
+      end
+
+      def generate config={}
+
+        charts = [ { name: 'Growth Chart',
+                     data: @data,
+                     opts: opts_for(@data) } ]
+
+        if config[:compare]
+          for entry_data in @data
+            charts << { name: entry_data[:name],
+                        data: comparison_for(entry_data),
+                        opts: opts_for([entry_data]) }
+          end
+        end
+
+        charts
+      end
+
+      def opts_for data
+        data = [data] unless Array === data
+
+        min = data.collect{|d| d[:data].values.min }.min
+        max = data.collect{|d| d[:data].values.max }.max
+
+        orange = "#f0662d"
+        purple = "#8062a6"
+        light_green = "#7bc545"
+        med_blue = "#0883b2"
+        yellow = "#ffaa00"
+        teal = "#00c7c3"
+
+        {
+          discrete: true,
+          width: "800px",
+          height: "500px",
+          min: (min * 0.8).floor,
+          max: (max * 1.2).ceil,
+          library: {
+            colors: [orange, purple, light_green, med_blue, yellow, teal],
+            xAxis: {type: 'linear', title: {text: "Size"}},
+            yAxis: {type: 'linear', title: {text: "Microseconds per Iteration"}}
+          }
+        }
+      end
+
+      def comparison_for data
+        sample = data[:data][@sample_size]
+
+        comparison = [data]
+
+        TYPES.each do |type|
+          comparison << generate_data_for(type, sample)
+        end
+
+        comparison
+      end
+
+      def generate_data_for type, sample
+
+        # for the given sizes, create a hash from an array
+        # the keys of the hash are the sizes
+        # the values are the generated data for this type of comparison
+        data = Hash[ @sizes.map {|n| [n, data_generator(type, n, sample) ] } ]
+
+        { name: title_for(type), data: data }
+
+      end
+
+      def title_for type
+
+        case type
+        when :const
+          'const'
+        when :logn
+          'log n'
+        when :n
+          'n'
+        when :nlogn
+          'n log n'
+        when :n_sq
+          'n squared'
+        end
+
+      end
+
+      def data_generator type, n, sample
+        factor = factor_for(type, sample)
+
+        case type
+        when :const
+          factor
+        when :logn
+          Math.log10(n) * factor
+
+        when :n
+          n * factor
+
+        when :nlogn
+          n * Math.log10(n) * factor
+
+        when :n_sq
+          n * n * factor
+
+        end
+      end
+
+      # calculate the scaling factor for the given type and sample using sample_size
+      def factor_for type, sample
+        case type
+        when :const
+          sample.to_f
+        when :logn
+          sample.to_f/Math.log10(@sample_size)
+
+        when :n
+          sample.to_f/@sample_size
+
+        when :nlogn
+          sample.to_f/(@sample_size * Math.log10(@sample_size))
+
+        when :n_sq
+          sample.to_f/(@sample_size * @sample_size)
+        end
+      end
+
+    end
+  end
+end

data/lib/benchmark/bigo/job.rb

@@ -35,16 +35,9 @@ module Benchmark
 
       include Chartkick::Helper
 
-      # how many total increments are being measured
-      attr_accessor :increments
+      require 'securerandom'
 
-      # whether to graph the results on a log scale
-      attr_accessor :logscale
-
-      # whether to generate a chart of the results
-      # if nil, do not generate chart
-      # else string is name of file to write chart out to
-      attr_reader :chart
+      attr_accessor :min_size, :steps, :step_size
 
       def initialize opts={}
         super
@@ -53,39 +46,60 @@ module Benchmark
         @generator = nil
 
         # defaults
-        linear
-        @increments = 5
-        @logscale = false
-        @chart = nil
-        @data_file = nil
+        @min_size = 100
+        @step_size = 100
+        @steps = 10
+
+        # whether to generate a chart of the results
+        # if nil, do not generate chart
+        # else string is name of file to write chart out to
+        @chart_file = nil
+
+        # whether to generate json output of the results
+        # if nil, do not generate data
+        # else string is name of file to write data out to
+        @json_file = nil
+
+        # whether to generate csv output of the results
+        # if nil, do not generate data
+        # else string is name of file to write data out to
+        @csv_file = nil
+
+        # whether to generate a plot in the terminal
+        # using gnuplot
+        @term_plot = false
+      end
+
+      def max_size
+        @min_size + (@step_size * (@steps-1))
+        # should also equal step(@steps-1)
       end
 
       def config opts
         super
-        @increments = opts[:increments] if opts[:increments]
-        @logscale = opts[:logscale] if opts[:logscale]
-        @full_report.logscale! if @logscale
+        @min_size = opts[:min_size] if opts[:min_size]
+        @steps = opts[:steps] if opts[:steps]
+        @step_size = opts[:step_size] if opts[:step_size]
       end
 
-      def chart?
-        @chart
+      def chart! filename='chart.html'
+        @chart_file = filename
       end
 
-      def chart! filename='chart.html'
-        @chart = filename
+      def compare?
+        @compare
       end
 
-      def data?
-        @data_file
+      def termplot!
+        @term_plot = true
       end
 
-      def data! filename='data.json'
-        @data_file = filename
+      def json! filename='data.json'
+        @json_file = filename
       end
 
-      def logscale= val
-        @logscale = val
-        @full_report.logscale! if @logscale
+      def csv! filename='data.csv'
+        @csv_file = filename
       end
 
       def generator &blk
@@ -99,43 +113,43 @@ module Benchmark
       # represented by the symbol passed to the method
       def generate sym
 
-        case sym
+        @generator =
+          case sym
 
-        # generates an Array containing shuffled integer values from 0 to size
-        when :array
-          @generator = Proc.new{|size| (0...size).to_a.shuffle }
+            # generates an Array containing shuffled integer values from 0 to size
+          when :array
+            Proc.new {|size| (0...size).to_a.shuffle }
 
-        # when :string
-          # to do
+            # generates a random hex string of length size
+          when :string
+            Proc.new {|size| SecureRandom.hex(size) }
 
-        else
-          raise "#{sym} is not a supported object type"
-        end
-      end
+            # simply returns the size
+            # for performance benefits when handling the
+            # size completely in the report block
+          when :size
+            Proc.new {|size| size }
 
-      # custom incrementer
-      def incrementer &blk
-        @incrementer = blk
-        raise ArgumentError, "no block" unless @incrementer
-      end
+            # when :hash
+            # TODO: hash generator
 
-      # linear incrementer
-      def linear increments=100
-        @incrementer = Proc.new {|i| i * increments }
-        @logscale = false
+          else
+            raise "#{sym} is not a supported object type"
+          end
       end
 
-      # exponential incrementer
-      def exponential base=10
-        @incrementer = Proc.new {|i| base ** (i-1) }
-        @full_report.logscale!
-        @logscale = true
+      # return the size for the nth step
+      # n = 0 returns @min_size
+      def step n
+        @min_size + (n * @step_size)
       end
 
       def sizes
-        (1..@increments).collect do |idx|
-          @incrementer.call(idx).to_i
+        @sizes ||=
+        (0...@steps).collect do |n|
+          step n
         end
+        @sizes
       end
 
       def item label="", str=nil, &blk # :yield:
@@ -157,48 +171,61 @@ module Benchmark
       end
       alias_method :report, :item
 
-      def run_warmup
-        super
-
-        max_timing = @timing.values.max
-        @full_report.per_iterations = 10**Math.log10(max_timing).ceil
+      def generate_output
+        generate_json
+        generate_csv
+        generate_chart
+        generate_termplot
       end
 
-      def generate_data
-        return if @data_file.nil?
+      def generate_json
+        return if @json_file.nil?
 
-        all_data = @full_report.chart_data
+        all_data = @full_report.data
 
-        File.open @data_file, 'w' do |f|
+        File.open @json_file, 'w' do |f|
           f.write JSON.pretty_generate(all_data)
         end
       end
 
-      def generate_chart
-        return if @chart.nil?
+      def generate_csv
+        return if @csv_file.nil?
 
-        all_data = @full_report.chart_data
+        all_data = @full_report.data
+        data_points = all_data.map{|report| report[:data].keys }.flatten.uniq
 
-        charts = []
-        charts << { name: 'Growth Chart', data: all_data, opts: @full_report.chart_opts(all_data) }
-
-        if compare?
-          all_sizes = sizes
-          for chart_data in all_data
-            comparison_data = @full_report.comparison_chart_data chart_data, all_sizes
-            charts << { name: chart_data[:name], data: comparison_data, opts: @full_report.chart_opts(chart_data) }
+        CSV.open @csv_file, 'w' do |csv|
+          header = [''] + data_points
+          csv << header
+          all_data.each do |row|
+            csv << [row[:name]] + row[:data].values
           end
         end
+      end
+
+      def generate_chart
+        return if @chart_file.nil?
+
+        @chart = Chart.new @full_report.data, sizes
+
+        charts = @chart.generate(compare: compare?)
 
         template_file = File.join File.dirname(__FILE__), 'templates/chart.erb'
         template = ERB.new(File.read(template_file))
 
-        File.open @chart, 'w' do |f|
+        File.open @chart_file, 'w' do |f|
           f.write template.result(binding)
         end
 
       end
 
+      def generate_termplot
+        return unless @term_plot
+
+        @plot = TermPlot.new @full_report.data, sizes
+        @plot.generate
+      end
+
     end
   end
 end