benchmark-sweet 0.2.0

data/examples/benchmark_big_split.rb

@@ -0,0 +1,49 @@
+require 'benchmark/sweet'
+require 'active_support/all'
+require "more_core_extensions/all" # [].tabelize
+
+# version 2.3.7
+#
+#  method      | nil_ips                | str_ips       | nil_memsize        | str_memsize
+# -------------+------------------------+---------------+--------------------+-------------
+#  ?split:[]   | 51825798.8 i/s         | 1407946.4 i/s | 40.0 bytes         | 360.0 bytes
+#  &&split||[] | 46730725.8 i/s - 1.11x | 1413355.3 i/s | 40.0 bytes         | 360.0 bytes
+#  to_s.split  | 5685237.1 i/s - 9.12x  | 1396494.3 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+#
+# version 2.4.4
+#
+#  method      | nil_ips                | str_ips       | nil_memsize        | str_memsize
+# -------------+------------------------+---------------+--------------------+-------------
+#  ?split:[]   | 51559454.4 i/s         | 1438780.8 i/s | 40.0 bytes         | 360.0 bytes
+#  &.split||[] | 46446196.0 i/s - 1.11x | 1437665.3 i/s | 40.0 bytes         | 360.0 bytes
+#  &&split||[] | 43356335.6 i/s - 1.19x | 1434466.6 i/s | 40.0 bytes         | 360.0 bytes
+#  to_s.split  | 5835694.7 i/s - 8.84x  | 1427819.0 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+
+NSTRING    = nil
+DELIMITER  = '/'.freeze
+STRING     = "ab/cd/ef/gh".freeze
+
+Benchmark.items(metrics: %w(ips memsize), memory: 3, warmup: 1, time: 3, quiet: false, force: ENV["FORCE"] == "true") do |x|
+  x.metadata version: RUBY_VERSION
+  x.metadata data: "nil" do
+    x.report("to_s.split",  "NSTRING.to_s.split(DELIMITER)")
+    x.report("?split:[]",   "NSTRING ? NSTRING.split(DELIMITER) : []")
+    x.report("&&split||[]", "NSTRING && NSTRING.split(DELIMITER) || []")
+    x.report("&.split||[]", "NSTRING&.split(DELIMITER) || []") if RUBY_VERSION >= "2.4"
+  end
+
+  x.metadata data: "str" do
+    x.report("to_s.split",  "STRING.to_s.split(DELIMITER)")
+    x.report("?split:[]",   "STRING ? STRING.split(DELIMITER) : []")
+    x.report("&&split||[]", "STRING && STRING.split(DELIMITER) || []")
+    x.report("&.split||[]", "STRING&.split(DELIMITER) || []") if RUBY_VERSION >= "2.4"
+  end
+
+  # partition the data by ruby version and whether data is present
+  # that way we're only comparing similar values
+  # note: this is not necessarily the correlation to how the data is displayed
+  x.compare_by :version, :data
+  x.report_with grouping: [:metric, :version], row: :method, column: [:data], value: :comp_short
+
+  x.save_file (ENV["SAVE_FILE"] == "true") ? $0.sub(/\.rb$/, '.json') : ENV["SAVE_FILE"] if ENV["SAVE_FILE"]
+end

data/examples/benchmark_blank.rb

@@ -0,0 +1,56 @@
+require "benchmark/sweet"
+require "active_support/all"
+require "more_core_extensions/all" # [].tabelize
+
+#  method             | NIL                    | EMPTY                  | FULL
+# --------------------+------------------------+------------------------+------------------------
+#  x&.empty?          | 13752904.8 i/s         | 13089322.9 i/s         | 13352488.3 i/s
+#  !x && x.empty?     | 13334422.1 i/s         | 12019275.5 i/s - 1.14x | 11882260.1 i/s - 1.16x
+#  x.blank?           | 11889050.9 i/s - 1.16x | 11673162.3 i/s - 1.18x | 12039255.2 i/s - 1.14x
+#  x.nil? || x.empty? | 11620573.1 i/s - 1.18x | 10676420.2 i/s - 1.29x | 10048254.7 i/s - 1.37x
+#  x.try!(:empty)     | 6240643.7 i/s - 2.20x  | 3962583.3 i/s - 3.47x  | 4071200.6 i/s - 3.38x
+#  x.try(:empty)      | 6044172.3 i/s - 2.28x  | 2385145.0 i/s - 5.76x  | 2454406.3 i/s - 5.60x
+#  x.empty?           |                        | 13743969.3 i/s         | 13754672.4 i/s
+
+ANIL=nil
+EMPTY=[].freeze
+FULL=["a"].freeze
+
+Benchmark.items(metrics: %w(ips)) do |x|
+  x.metadata version: RUBY_VERSION
+  x.metadata data: 'NIL' do
+    x.report("x.nil? || x.empty?") { ANIL.nil? || ANIL.empty? }
+    x.report("!x && x.empty?")     { !ANIL || ANIL.empty? }
+    x.report("x&.empty?")          { ANIL&.empty? }
+    x.report("x.try!(:empty)")     { ANIL.try!(:empty?) }
+    x.report("x.try(:empty)")      { ANIL.try(:empty?) }
+    x.report("x.blank?")           { ANIL.blank? }
+  end
+
+  x.metadata data: 'EMPTY' do
+    x.report("x.nil? || x.empty?") { EMPTY.nil? || EMPTY.empty? }
+    x.report("!x && x.empty?")     { !EMPTY || EMPTY.empty? }
+    x.report("x&.empty?")          { EMPTY&.empty? }
+    x.report("x.try!(:empty)")     { EMPTY.try!(:empty?) }
+    x.report("x.try(:empty)")      { EMPTY.try(:empty?) }
+    x.report("x.blank?")           { EMPTY.blank? }
+    # base case
+    x.report("x.empty?")           { EMPTY.empty? }
+  end
+
+  x.metadata data: 'FULL' do
+    x.report("x.nil? || x.empty?") { FULL.nil? || FULL.empty? }
+    x.report("!x && x.empty?")     { !FULL || FULL.empty? }
+    x.report("x&.empty?")          { FULL&.empty? }
+    x.report("x.try!(:empty)")     { FULL.try!(:empty?) }
+    x.report("x.try(:empty)")      { FULL.try(:empty?) }
+    x.report("x.blank?")           { FULL.blank? }
+    # base case
+    x.report("x.empty?")           { FULL.empty? }
+  end
+
+  x.compare_by :data
+  x.report_with grouping: nil, row: :method, column: :data
+
+  x.save_file (ENV["SAVE_FILE"] == "true") ? $0.sub(/\.rb$/, '.json') : ENV["SAVE_FILE"] if ENV["SAVE_FILE"]
+end

data/examples/benchmark_rpt_database.rb

@@ -0,0 +1,127 @@
+require "benchmark/sweet"
+require "more_core_extensions/all"
+require "active_record"
+
+# For various versions of rails, compare `Model.all.first` vs `Model.all.to_a.first`
+#
+# To work across versions, we need to run this multiple times (with different Gemfile each run)
+#
+# meta data of :version is stored to be able to distinguish the version across each of the runs
+#
+# options:
+#   DATABASE_URL
+#     link to the database
+#
+#     DATABASE_URL=postgres://user@password:localhost/user_benchmark
+#
+#   SAVE_FILE
+#     a save file contains the values from the various invocations.
+#     This uses the method `save_file`, which is optional. But without it, it won't compare across versions.
+#
+#     running this script multiple times will only use the first value obtained
+#     But running this with different metadata (i.e. AR version) will run again and compare across the versions
+#
+#     default             : write to a save file named "{this script's name}.json"
+#     SAVE_FILE=file.json : write to a save file named "file.json"
+#
+#   FORCE
+#     This tells the script to overwrite previously run identical metadata
+#     It uses the `force: true` option to share with benchmark that this behavior is desired
+#
+#     FORCE=true  :overwrite the previous values for this script
+#     FORCE=false : default behavior. don't run multiple times for the same metadata
+#
+
+# version 5.2.1
+#
+# grouping 6.0.2.2 (100 records)
+#
+#  method     | ips                | memsize
+# ------------+--------------------+---------------------
+#  first      | 2945.9 i/s         | 9808 bytes
+#  to_a.first | 1204.7 i/s - 2.45x | 68200 bytes - 6.95x
+#
+#  NOTE: the results are in color
+
+ActiveRecord::Base.establish_connection(ENV.fetch('DATABASE_URL') { "postgres://localhost/user_benchmark" })
+ActiveRecord::Migration.verbose = false
+
+ActiveRecord::Schema.define do
+  create_table :users, force: true do |t|
+    t.string :name
+  end
+  #add_index :users, :name
+
+  create_table :accounts, force: true do |t|
+    t.string :name
+  end
+end
+
+class User < ActiveRecord::Base; end
+class Account < ActiveRecord::Base; end
+
+if User.count == 0
+  puts "Creating 100 users"
+  100.times { |i| User.create name: "user #{i}" }
+end
+
+#in the table cells, it typically displays the value and units. this lambda is adding in colors (based upon best/worst)
+VALUE_TO_S  = ->(m) { m.comp_short("\e[#{m.color}m#{m.central_tendency.round(1)} #{m.units}\e[0m") }
+
+# These are the various items that will be compared
+#
+# metrics is the metric that is actually run
+# memory
+# warmup - run the tests for this many seconds before running the actual benchmark (for ips)
+# time   - run the tests for this many seconds (for ips)
+# quied  - display the ips run information
+# force  - defined above. this is allowing the command line to change this value (default: false)
+Benchmark.items(metrics: %w(ips memsize), memory: 3, warmup: 1, time: 3, quiet: false, force: ENV["FORCE"] == "true") do |x|
+  # for all examples, store this metadata with the row
+  # metadata is stored in the savefile along with the method name and benchmarks.
+  # future runs of the script that have a different version of AR will be stored as separate benchmarks
+  # this allows comparison across multiple versions
+  #
+  # If you are applying a patch for different behavior, or you're running against head, consider using something more comples:
+  #
+  # this can be an array. with something like version: [ActiveRecord.version.to_s, ENV["PATCH"], ENV["SHA"]].compact.join(".")
+  x.metadata version: ActiveRecord.version.to_s
+
+  # compare_by is a display parameter
+  #
+  # this is used to know which metadata is different and which are the same
+  # the best and worst value is determined across this criteria
+  #
+  # so if you want to show a different list for each version, add it to the list
+  # This typically will include the group and either the column or the row header
+  #
+  # defaults to unique by method (and always unique by metric)
+  x.compare_by { |label, _| [label[:count], label[:version]] }
+
+  # these next two cases are marked with count=100. Use row, column or grouping to group these
+  # These values tend to be in the compare_by block. because running on 100 values is different than 0. (but not always the case)
+  x.metadata count: "100" do
+    x.report("first") { User.all.first }
+    x.report("to_a.first") { User.all.to_a.first }
+  end
+
+  x.metadata count: "0" do
+    x.report("first") { Account.all.first }
+    x.report("to_a.first") { Account.all.to_a.first }
+  end
+
+  # Note, often the report title is the same as the code, in those cases just pass the name
+  # x.report("Account.all.to_a.first")
+  # x.report("Account.all.to_a.first")
+
+  # display only. parameters
+  # these can be symbols or lambdas
+  # grouping is the value for determining what data goes into each table (default - only 1 table)
+  # row is the title on the left hand side (default: all metadata)
+  # column header is the metric that was captured (i.e.: ips or metrics) (default: metric name)
+  # value is the text that is displayed in the cell of the table (default: "central_tendancy units")
+  x.report_with grouping: ->(l){ "#{l[:version]} (#{l[:count]} records)"}, row: :method, column: :metric, value: VALUE_TO_S
+
+  # defined above. benchmark sweet pretty much depends upon this json file
+  x.save_file ENV["SAVE_FILE"] ? ENV["SAVE_FILE"] : $0.sub(/\.rb$/, '.json')
+end

data/examples/benchmark_simple_database.rb

@@ -0,0 +1,34 @@
+require "benchmark/sweet"
+require "more_core_extensions/all"
+require "active_record"
+#
+# label               | ips               | queries  | rows
+#---------------------+-------------------+----------+----------------------
+# User.all.first      | 3185.3 i/s        | 1.0 objs | 1.0 objs
+# User.all.to_a.first | 977.1 i/s - 3.26x | 1.0 objs | 100.0 objs - 100.00x
+
+
+ActiveRecord::Base.establish_connection(ENV.fetch('DATABASE_URL') { "postgres://localhost/user_benchmark" })
+ActiveRecord::Migration.verbose = false
+
+ActiveRecord::Schema.define do
+  create_table :users, force: true do |t|
+    t.string :name
+  end
+  #add_index :users, :name
+end
+
+class User < ActiveRecord::Base ;end
+
+if User.count == 0
+  puts "Creating 100 users"
+  100.times { |i| User.create name: "user #{i}" }
+end
+
+Benchmark.items(metrics: %w(ips queries rows)) do |x|
+  x.report("User.all.first")
+  x.report("User.all.to_a.first")
+
+  x.report_with row: :method, column: :metric
+  x.save_file (ENV["SAVE_FILE"] == "true") ? $0.sub(/\.rb$/, '.json') : ENV["SAVE_FILE"] if ENV["SAVE_FILE"]
+end

data/lib/benchmark-sweet.rb

@@ -0,0 +1 @@
+require "benchmark/sweet"

data/lib/benchmark/sweet.rb

@@ -0,0 +1,94 @@
+require "benchmark/sweet/version"
+require "benchmark/sweet/ips"
+require "benchmark/sweet/memory"
+require "benchmark/sweet/queries"
+require "benchmark/sweet/job"
+require "benchmark/sweet/comparison"
+require "benchmark/sweet/item"
+require "benchmark/ips"
+
+module Benchmark
+  module Sweet
+    def items(options = {memory: true, ips: true})
+      job = ::Benchmark::Sweet::Job.new(options)
+
+      yield job
+
+      job.load_entries
+      job.run
+      job.save_entries
+
+      job.run_report
+
+      job # both items and entries are useful
+    end
+
+    # report helper methods
+    # these are the building blocks for the reports printed.
+    # These can be used to create different tables
+
+    # @param base [Array<Comparison>}] array of comparisons
+    # @param grouping [Symbol|Array<Symbol>|Proc] Proc passed to group_by to partition records.
+    #  Accepts Comparison, returns an object to partition. returns nil to filter from the list
+    # @keyword sort [Boolean] true to sort by the grouping value (default false)
+    #   Proc accepts the label to generate custom summary text
+    def self.group(base, grouping, sort: false, &block)
+      if grouping.nil?
+        yield nil, base
+        return
+      end
+
+      grouping = symbol_to_proc(grouping)
+
+      label_records = base.group_by(&grouping).select { |value, comparisons| !value.nil? }
+      label_records = label_records.sort_by(&:first) if sort
+
+      label_records.each(&block)
+    end
+
+    def self.table(base, grouping: nil, sort: false, row: :label, column: :metric, value: :comp_short)
+      header_name = grouping.respond_to?(:call) ? "grouping" : grouping
+      column = symbol_to_proc(column)
+      value = symbol_to_proc(value)
+
+      group(base, grouping, sort: true) do |header_value, table_comparisons|
+        row_key = row.kind_of?(Symbol) || row.kind_of?(String) ? row : "label"
+        table_rows = group(table_comparisons, row, sort: sort).map do |row_header, row_comparisons|
+          row_comparisons.each_with_object({row_key => row_header}) do |comparison, row_data|
+            row_data[column.call(comparison)] = value.call(comparison)
+          end
+        end
+        if block_given?
+          yield header_value, table_rows
+        else
+          print_table(header_name, header_value, table_rows)
+        end
+      end
+    end
+
+    # proc produced: -> (comparison) { comparison.method }
+    def self.symbol_to_proc(field, join: "_")
+      if field.kind_of?(Symbol) || field.kind_of?(String)
+        field_name = field
+        -> v { v[field_name] }
+      elsif field.kind_of?(Array)
+        field_names = field
+        if join
+          -> v { field_names.map { |gn| v[gn].to_s }.join(join) }
+        else
+          -> v { field_names.map { |gn| v[gn] } }
+        end
+      else
+        field
+      end
+    end
+
+    def self.print_table(header_name, header_value, table_rows)
+      require "more_core_extensions" # defines tableize
+      puts "", "#{header_name} #{header_value}", "" if header_value
+      # passing colummns to make sure table keeps the same column order
+      puts table_rows.tableize(:columns => table_rows.first.keys)
+    end
+  end
+  extend Benchmark::Sweet
+end

data/lib/benchmark/sweet/comparison.rb

@@ -0,0 +1,101 @@
+module Benchmark
+  module Sweet
+    class Comparison
+      UNITS = {"ips" => "i/s", "memsize" => "bytes", "memsize_retained" => "bytes"}.freeze
+      attr_reader :label, :metric, :stats, :baseline, :worst
+      attr_reader :offset, :total
+      def initialize(metric, label, stats, offset, total, baseline, worst = nil)
+        @metric = metric
+        @label = label
+        @stats = stats
+        @offset = offset
+        @total = total
+        @baseline = baseline
+        @worst = worst
+      end
+
+      def [](field)
+        case field
+        when :metric      then metric
+        when :comp_short  then comp_short
+        when :comp_string then comp_string
+        when :label       then label  # not sure if this one makes sense
+        else label[field]
+        end
+      end
+
+      def central_tendency ; stats.central_tendency ; end
+      def error ; stats.error ; end
+      def units ; UNITS[metric] || "objs" ; end
+
+      def mode
+        @mode ||= best? ? :best : overlaps? ? :same : diff_error ? :slowerish : :slower
+      end
+
+      def best? ; !baseline || (baseline == stats) ; end
+
+      # @return true if it is basically the same as the best
+      def overlaps?
+        return @overlaps if defined?(@overlaps)
+        @overlaps = slowdown == 1 ||
+                      stats && baseline && (stats.central_tendency == baseline.central_tendency || stats.overlaps?(baseline))
+      end
+
+      def worst?
+        @worst ? stats.overlaps?(@worst) : (total.to_i - 1 == offset.to_i) && slowdown.to_i > 1
+      end
+
+      def slowdown
+        return @slowdown if @slowdown
+        @slowdown, @diff_error = stats.slowdown(baseline)
+        @slowdown
+      end
+
+      def diff_error
+        @diff_error ||= (slowdown ; @diff_error)
+      end
+
+      # quick display
+
+      def comp_string(l_to_s = nil)
+        l_to_s ||= -> l { l.to_s }
+        case mode
+        when :best
+          "%20s: %10.1f %s" % [l_to_s.call(label), central_tendency, units]
+        when :same
+          "%20s: %10.1f %s - same-ish: difference falls within error" % [l_to_s.call(label), central_tendency, units]
+        when :slower 
+          "%20s: %10.1f %s - %.2fx (± %.2f) slower" % [l_to_s.call(label), central_tendency, units, slowdown, error]
+        when :slowerish
+          "%20s: %10.1f %s - %.2fx slower" % [l_to_s.call(label), central_tendency, units, slowdown]
+        end
+      end
+
+      # I tend to call with:
+      #   c.comp_short("\033[#{c.color}m#{c.central_tendency.round(1)} #{c.units}\e[0m") # "\033[31m#{value}\e[0m"
+      def comp_short(value = nil)
+        value ||= "#{central_tendency.round(1)} #{units}"
+        case mode
+        when :best, :same
+          value
+        when :slower 
+          "%s - %.2fx (± %.2f)" % [value, slowdown, error]
+        when :slowerish
+          "%s - %.2fx" % [value, slowdown]
+        end
+      end
+
+      def color
+        if !baseline
+          ";0"
+        elsif best? || overlaps?
+          "32"
+        elsif worst?
+          "31"
+        else
+          ";0"
+        end
+      end
+    end
+  end
+end