benchmark-sweet 0.2.0

data/lib/benchmark/sweet/ips.rb

@@ -0,0 +1,16 @@
+module Benchmark
+  module Sweet
+    module IPS
+      def run_ips
+        require "benchmark/ips"
+        rpt = Benchmark.ips(warmup: options[:warmup], time: options[:time], quiet: options[:quiet]) do |x|
+          items.each { |e| x.item(e.label, e.action || e.block) }
+          #x.compare! if compare
+        end
+        rpt.entries.each do |entry|
+          add_entry(entry.label, "ips", entry.stats)
+        end
+      end
+    end
+  end
+end

data/lib/benchmark/sweet/item.rb

@@ -0,0 +1,30 @@
+module Benchmark
+  module Sweet
+    # borrowed heavily from Benchmark::IPS::Job::Entry
+    # may be able to fallback on that - will need to generate a &block friendly proc for that structure
+    class Item
+      attr_reader :label, :action
+      def initialize(label, action = nil)
+        @label = label
+        @action = action || @label[:method] #raise("Item needs an action")
+      end
+
+      def block
+        @block ||= action.kind_of?(String) ? compile(action) : action
+      end
+
+      # to use with Job::Entry...
+      # def call_once ; call_times(1) ; end
+      # def callback_proc
+      #   lambda(&method(:call_once))
+      # end
+      def compile(str)
+        eval <<-CODE
+          Proc.new do
+            #{str}
+          end
+        CODE
+      end
+    end
+  end
+end

data/lib/benchmark/sweet/job.rb

@@ -0,0 +1,244 @@
+module Benchmark
+  module Sweet
+    # abstract notion of a job
+    class Job
+      include Benchmark::Sweet::IPS
+      include Benchmark::Sweet::Memory
+      include Benchmark::Sweet::Queries
+
+      # metrics calculated by the ips test suite
+      IPS_METRICS      = %w(ips).freeze
+      # metrics calculated by the memory test suite
+      MEMORY_METRICS   = %w(memsize memsize_retained objects objects_retained strings strings_retained).freeze
+      # metrics calculated by the database test suite
+      DATABASE_METRICS = %w(rows queries ignored ignored cached).freeze
+      ALL_METRICS      = (IPS_METRICS + MEMORY_METRICS + DATABASE_METRICS).freeze
+      HIGHER_BETTER    = %w(ips).freeze
+
+      # @returns [Array<Job::Item>] list of report items to run
+      attr_reader :items
+      # @returns [Hash<String,Hash<String,Stat>>] entries[name][metric] = value
+      attr_reader :entries
+
+      # @option options :quiet    [Boolean] true to suppress the display of interim test calculations
+      # @option options :warmup   [Number]  For ips tests, the amount of time to warmup
+      # @option options :time     [Number]  For ips tests, the amount of time to the calculations
+      # @option options :metrics  [String|Symbol,Array<String|Symbol] list of metrics to run
+      # TODO: :confidence
+      attr_reader :options
+
+      # lambda used to group metrics that should be compared
+      # The lambda takes the label as an argument and returns a unique object per comparison group 
+      # NOTE: This lambda takes a label hash as an argument
+      #       While other lambdas in this api take a comparison object
+      # a symbol is assumed to refer to the label
+      # @return [Nil|Lambda] lambda for grouping
+      attr_reader :grouping
+
+      def initialize(options = {})
+        @options = options
+        @options[:metrics] ||= IPS_METRICS + %w()
+        validate_metrics(@options[:metrics])
+        @items = []
+        @entries = {}
+        @symbolize_keys = false
+        # load / save
+        @filename = nil
+        # display
+        @grouping = nil
+        @report_options = {}
+        @report_block = nil
+        # current item metadata
+        @meta = {}
+      end
+
+      def configure(options)
+        @options.merge!(options)
+      end
+
+      # @returns [Boolean] true to run iterations per second tests
+      def ips? ; !(relevant_metric_names & IPS_METRICS).empty? ; end
+      # @returns [Boolean] true to run memory tests
+      def memory? ; !(relevant_metric_names & MEMORY_METRICS).empty? ; end
+      # @returns [Boolean] true to run database queries tests
+      def database? ; !(relevant_metric_names & DATABASE_METRICS).empty? ; end
+
+      # @returns  [Boolean] true to suppress the display of interim test calculations
+      def quiet? ; options[:quiet] ; end
+
+      # @returns  [Boolean] true to run tests for data that has already been processed
+      def force? ; options[:force] ; end
+
+      # @returns [Array<String>] List of metrics to compare
+      def relevant_metric_names ; options[:metrics] ; end
+
+      # items to run (typical benchmark/benchmark-ips use case)
+      def item(label, action = nil, &block)
+        # could use Benchmark::IPS::Job::Entry
+        current_meta = label.kind_of?(Hash) ? @meta.merge(label) : @meta.merge(method: label)
+        @items << Item.new(current_meta, action || block)
+      end
+      alias report item
+
+      def metadata(options)
+        @old_meta = @meta
+        @meta = @meta.merge(options)
+        return unless block_given?
+        yield
+        @meta = @old_meta
+      end
+
+      def save_file(filename)
+        @filename = filename
+      end
+
+      # &block - a lambda that accepts a label and a stats object
+      # returns a unique object for each set of metrics that should be compared with each other
+      #
+      # unfortunatly, this currently has a different signature than all other lambdas
+      # at this time, there are no comparisons created yet. so it is hard to pass one in
+      # example:
+      #   x.compare_by { |label, value| label[:data] }
+      #   x.compare_by :data
+      #
+      def compare_by(*symbol, &block)
+        @grouping = symbol.empty? ? block : Proc.new { |label, value| symbol.map { |s| label[s] } }
+      end
+
+      # Setup the testing framework
+      # TODO: would be easier to debug if these were part of run_report
+      # @keyword :grouping [Symbol|lambda|nil] proc with parameters label, stat that generates grouping names
+      #          defaults to the compare_by value
+      # @keyword :sort [Boolean] true to sort the rows (default false). NOTE: grouping names ARE sorted
+      # @keyword :row [Symbol|lambda] a lambda (default - display the full label)
+      # @keyword :column [Symbol|lambda] (default :metric)
+      # @keyword :value  (default :comp_short - the value and delta)
+      #          for color, consider passing `value: ->(m){ m.comp_short("\033[#{m.color}m#{m[field]}\e[0m") }`
+      def report_with(args = {}, &block)
+        @report_options = args
+        @report_block = block
+        # Assume the display grouping is the same as comparison grouping unless an explicit value was provided
+        if !args.key?(:grouping) && @grouping
+          args[:grouping] = @grouping.respond_to?(:call) ? -> v { @grouping.call(v.label, v.stats) } : @grouping
+        end
+      end
+
+      # if we are using symbols as keys for our labels
+      def labels_have_symbols!
+      end
+
+      # report results
+      def add_entry(label, metric, stat)
+        (@entries[metric] ||= {})[label] = stat.respond_to?(:central_tendency) ? stat : create_stats(stat)
+      end
+
+      def entry_stat(label, metric)
+        @entries.dig(metric, label)
+      end
+
+      def relevant_entries
+        relevant_metric_names.map { |n| [n, @entries[n] ] }
+      end
+      # serialization
+
+      def load_entries(filename = @filename)
+        # ? have ips save / load their own data?
+        return unless filename && File.exist?(filename)
+        require "json"
+
+        JSON.load(IO.read(filename)).each do |v|
+          n = v["name"]
+          n.symbolize_keys!
+          add_entry n, v["metric"], v["samples"]
+        end
+
+      end
+
+      def save_entries(filename = @filename)
+        return unless filename
+        require "json"
+
+        # sanity checking
+        symbol_value = false
+
+        data = @entries.flat_map do |metric_name, metric_values|
+          metric_values.map do |label, stat|
+            # warnings
+            symbol_values ||= label.kind_of?(Hash) && label.values.detect { |v| v.nil? || v.kind_of?(Symbol) }
+            {
+              'name'    => label,
+              'metric'  => metric_name,
+              'samples' => stat.samples,
+              # extra data like measured_us, iter, and others?
+            }
+          end
+        end
+
+        puts "", "Warning: Please use strings or numbers for label hash values (not nils or symbols). Symbols are not JSON friendly." if symbol_value
+        IO.write(filename, JSON.pretty_generate(data) << "\n")
+      end
+
+      def run
+        # run metrics if they are requested and haven't run yet
+        # only run the suites that provide the data the user needs.
+        # if the first node has the data, assumes all do
+        #
+        # TODO: may want to override these values
+        run_ips     if ips?      && (force? || !@entries.dig(IPS_METRICS.first, items.first.label))
+        run_memory  if memory?   && (force? || !@entries.dig(MEMORY_METRICS.first, items.first.label))
+        run_queries if database? && (force? || !@entries.dig(DATABASE_METRICS.first, items.first.label))
+      end
+
+      # ? metric => label(:version, :method) => stats
+      # ? label(:metric, :version, :method) => stats
+      # @returns [Hash<String,Hash<String,Comparison>>] Same as entries, but contains comparisons not Stats
+      def run_report
+        comparison_values.tap do |results|
+          display_report(results)
+        end
+      end
+
+      def display_report(comparisons)
+        if !@report_block || @report_block.arity == 2
+          Benchmark::Sweet.table(comparisons, **@report_options, &@report_block)
+        else
+          @report_block.call(comparisons)
+        end
+      end
+
+      # of note, this groups with @grouping (defined by group_by)
+      # but then all data continues to the next step
+      # this allows you to make comparisons across rows / columns / grouping
+      def comparison_values
+        relevant_entries.flat_map do |metric_name, metric_entries|
+          # TODO: map these to Comparison(metric_name, label, stats) So we only have 1 type of lambda
+          partitioned_metrics = grouping ? metric_entries.group_by(&grouping) : {nil => metric_entries}
+          partitioned_metrics.flat_map do |grouping_name, grouped_metrics|
+            sorted = grouped_metrics.sort_by { |n, e| e.central_tendency }
+            sorted.reverse! if HIGHER_BETTER.include?(metric_name)
+
+            _best_label, best_stats = sorted.first
+            total = sorted.count
+
+            # TODO: fix ranking. i / total doesn't work as well when there is only 1 entry or some entries are the same
+            sorted.each_with_index.map { |(label, stats), i| Comparison.new(metric_name, label, stats, i, total, best_stats) }
+          end
+        end
+      end
+
+      private
+
+      def validate_metrics(metric_options)
+        if !(invalid = metric_options - ALL_METRICS).empty?
+          $stderr.puts "unknown metrics: #{invalid.join(", ")}"
+          $stderr.puts "choose: #{(ALL_METRICS).join(", ")}"
+          raise IllegalArgument, "unknown metric: #{invalid.join(", ")}"
+        end
+      end
+
+      def create_stats(samples)
+        Benchmark::IPS::Stats::SD.new(Array(samples))
+      end
+    end
+  end
+end

data/lib/benchmark/sweet/memory.rb

@@ -0,0 +1,41 @@
+module Benchmark
+  module Sweet
+    module Memory
+      def run_memory
+        # I'd prefer to use benchmark/memory - but not sure if it bought us enough
+        # require "benchmark/memory"
+        # rpt = Benchmark.memory(quiet: options[:quiet]) do |x|
+        #   items.each { |item| x.report(item.label, &item.block) }
+        #   x.compare! if compare
+        # end
+        # rpt.entries.each do |e| 
+        #   add_entry e.label, "memory",           e.measurement.memory.allocated
+        #   add_entry e.label, "memory_retained",  e.measurement.memory.retained
+        #   add_entry e.label, "objects",          e.measurement.objects.allocated
+        #   add_entry e.label, "objects_retained", e.measurement.objects.retained
+        #   add_entry e.label, "string",           e.measurement.string.allocated
+        #   add_entry e.label, "string_retained",  e.measurement.string.retained
+        # end
+        require 'memory_profiler'
+        puts "Memory Profiling----------" unless quiet?
+
+        items.each do |entry|
+          name = entry.label
+
+          $stdout.printf("%20s ", name.to_s) unless quiet?
+          rpts = (options[:memory] || 1).times.map { MemoryProfiler.report(&entry.block) }
+          tot_stat  = add_entry(name, "memsize",          rpts.map(&:total_allocated_memsize))
+          totr_stat = add_entry name, "memsize_retained", rpts.map(&:total_retained_memsize)
+          obj_stat  = add_entry name, "objects",          rpts.map(&:total_allocated) ## ? size
+          objr_stat = add_entry name, "objects_retained", rpts.map(&:total_retained)
+          str_stat  = add_entry(name, "strings",          rpts.map { |rpt| rpt.strings_allocated.size })
+          strr_stat = add_entry(name, "strings_retained", rpts.map { |rpt| rpt.strings_retained.size })
+
+          $stdout.printf("%10s  alloc/ret %10s  strings/ret\n",
+                         "#{tot_stat.central_tendency}/#{totr_stat.central_tendency}",
+                         "#{str_stat.central_tendency}/#{strr_stat.central_tendency}") unless quiet?
+        end
+      end      
+    end
+  end
+end

data/lib/benchmark/sweet/queries.rb

@@ -0,0 +1,60 @@
+module Benchmark
+  module Sweet
+    module Queries
+      def run_queries
+        items.each do |entry|
+          values = ::Benchmark::Sweet::Queries::QueryCounter.count(&entry.block) # { entry.call_times(1) }
+          add_entry entry.label, "rows",    values[:instance_count]
+          add_entry entry.label, "queries", values[:sql_count]
+          add_entry entry.label, "ignored", values[:ignored_count]
+          add_entry entry.label, "cached",  values[:cache_count]
+          unless options[:quiet]
+            printf "%20s: %3d queries %5d ar_objects", entry.label, values[:sql_count], values[:instance_count]
+            printf " (%d ignored)", values[:ignored_count] if values[:ignored_count] > 0
+            puts
+          end
+        end
+      end
+
+      # Derived from code found in http://stackoverflow.com/questions/5490411/counting-the-number-of-queries-performed
+      #
+      # This could get much more elaborate
+      # results could be separated by payload[:statement_name] (sometimes nil) or payload[:class_name]
+      # Could add explains for all queries (and determine index usage)
+      class QueryCounter
+        def self.count(&block)
+          new.count(&block)
+        end
+
+        CACHE_STATEMENT    = "CACHE".freeze
+        IGNORED_STATEMENTS = %w(CACHE SCHEMA).freeze
+        IGNORED_QUERIES    = /^(?:ROLLBACK|BEGIN|COMMIT|SAVEPOINT|RELEASE)/.freeze
+
+        def callback(_name, _start, _finish, _id, payload)
+          if payload[:sql]
+            if payload[:name] == CACHE_STATEMENT
+              @instance[:cache_count] += 1
+            elsif IGNORED_STATEMENTS.include?(payload[:name]) || IGNORED_QUERIES.match(payload[:sql])
+              @instances[:ignored_count] += 1
+            else
+              @instances[:sql_count] += 1
+            end
+          else
+            @instances[:instance_count] += payload[:record_count]
+          end
+        end
+
+        def callback_proc
+          lambda(&method(:callback))
+        end
+
+        # TODO: possibly setup a single subscribe and use a context/thread local to properly count metrics
+        def count(&block)
+          @instances = {cache_count: 0, ignored_count: 0, sql_count: 0, instance_count: 0}
+          ActiveSupport::Notifications.subscribed(callback_proc, /active_record/, &block)
+          @instances
+        end
+      end
+    end
+  end
+end

data/lib/benchmark/sweet/version.rb

@@ -0,0 +1,5 @@
+module Benchmark
+  module Sweet
+    VERSION = "0.2.0"
+  end
+end

metadata

@@ -0,0 +1,189 @@
+--- !ruby/object:Gem::Specification
+name: benchmark-sweet
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Keenan Brock
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.4
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.2
+- !ruby/object:Gem::Dependency
+  name: memory_profiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: more_core_extensions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+    Benchmark Sweet is a suite to run multiple kinds of metrics.
+    It can be configured to run memory, sql query, and ips benchmarks on a common set of code.
+    This data can be collected across multiple runs of the code, to support multiple ruby or
+    gem versions.
+    This also generates more complex comparisons
+email:
+- keenan@thebrocks.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark-sweet.gemspec
+- bin/console
+- bin/setup
+- examples/benchmark_big_small.rb
+- examples/benchmark_big_split.rb
+- examples/benchmark_blank.rb
+- examples/benchmark_rpt_database.rb
+- examples/benchmark_simple_database.rb
+- lib/benchmark-sweet.rb
+- lib/benchmark/sweet.rb
+- lib/benchmark/sweet/comparison.rb
+- lib/benchmark/sweet/ips.rb
+- lib/benchmark/sweet/item.rb
+- lib/benchmark/sweet/job.rb
+- lib/benchmark/sweet/memory.rb
+- lib/benchmark/sweet/queries.rb
+- lib/benchmark/sweet/version.rb
+homepage: https://github.com/kbrock/benchmark-sweet
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/kbrock/benchmark-sweet
+  source_code_uri: http://github.com/kbrock/benchmark-sweet
+  changelog_uri: http://github.com/kbrock/benchmark-sweet/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6.2
+signing_key: 
+specification_version: 4
+summary: Suite to run multiple benchmarks
+test_files: []