benchmark-extensions 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in benchmarker.gemspec
+gemspec
+
+group :development do
+  gem 'rake'
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Lenny Marks
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+Ruby benchmark extensions
+==============================
+
+* ```BME::Runner.run(threads, repetitions, &blk)``` - Run block of code in :threads threads, :repetition times each thread.
+  <pre>
+    BME::Runner.run(2, 2) do |run_id|
+      puts run_id
+    end
+
+    [0][0]
+    [0][1]
+    [1][0]
+    [1][1]
+  </pre>
+* ```BME::ReportAggregator``` - Threadsafe aggregating wrapper around ```#report``` from SDK Benchmark.
+  See: [examples/report_aggregator.rb](examples/report_aggregator.rb)
+* ```benchmark``` executable - A mini-framework with automatic discovery and statistic reporting for pluggable ```simulations```.
+  <pre>
+    $ bundle exec benchmark --help
+    Usage: benchmark [options]
+    -h, --help                       Display this screen
+    -t, --threads NUM                Number of simultaneous threads per thread group
+    -r, --repetitions NUM            Number of repetitions per thread
+    -s, --simulation NAME            Run only specified simulation
+
+    $ bundle exec benchmark -s ManView -t 3 -r 2
+                                                    user     system      total        real
+      [1][0] ManView GA1001                     0.088000   0.000000   0.088000 (  0.086000)
+      [1][0] ManView GA1001                     0.020000   0.000000   0.020000 (  0.020000)
+      .....
+      ManView.history                           0.224000   0.000000   0.224000 (  0.222000)
+      ManView.tracking_info                     0.091000   0.000000   0.091000 (  0.091000)
+      ManView.related_manuscripts               0.209000   0.000000   0.209000 (  0.209000)
+      ManView.contact_info                      0.322000   0.000000   0.322000 (  0.322000)
+      .....
+      ManView                                   3.251001   0.000000   3.251001 (  3.252001)
+      >total:                                   3.251001   0.000000   3.251001 (  3.252001)
+
+      ManView.history(12)                      total(0.22200) avg(0.01850) median(0.00950) std(0.02259) min(0.00500) max(0.08600)
+      ManView.tracking_info(12)                total(0.09100) avg(0.00758) median(0.00450) std(0.00682) min(0.00200) max(0.02000)
+      ManView.related_manuscripts(12)          total(0.20900) avg(0.01742) median(0.01100) std(0.01274) min(0.00800) max(0.04800)
+      ManView.contact_info(12)                 total(0.32200) avg(0.02683) median(0.01650) std(0.02814) min(0.00700) max(0.09900)
+      .....
+      ManView(216)                             total(3.25200) avg(0.01506) median(0.00500) std(0.03255) min(0.00000) max(0.32000)
+  </pre>
+  
+## Plugins
+
+Ruby files in the ```simulations/``` directory are automatically loaded.
+
+A class is plugged in to the framework via the ```simulation``` method. ```simulation``` expects a block that returns a list of elements
+used to instantiate simulation instances.
+An instance of the simulation class will be instantiated for each batch item and all ```simulate_*'```
+methods will be executed and benchmarked. The batch item a simulation instance was instantiated
+with is accessible as ```#subject```.
+
+e.g.
+
+    class ManView
+      simulation 'ManView' do
+        %w(GA1001 AA1002)
+      end
+
+      def simulate_events
+      ....
+
+      def simulate_contact_info
+      ...
+
+      private
+
+      def manuscript
+        @manuscript ||= Manuscript.find(subject)
+      end
+    end
+
+### Customizing
+
+The benchmark script loads ```init.rb``` which is where your customizations should go.
+
+To override simulation methods (i.e. to decorate) you can use ```Simulation.add_helper(SimulationHelpers)```
+to specify a mixin to be added to all simulation classes.
+
+e.g. # init.rb
+
+    module SimulationHelpers
+      def run
+        EOPCommon::DoInSession.execute_new { super }
+      end
+      ....
+    end
+
+    Simulation.add_helper(SimulationHelpers)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/benchmark-extensions.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bme/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'benchmark-extensions'
+  gem.version       = BME::VERSION
+  gem.authors       = ['Lenny Marks']
+  gem.email         = %w(lenny@aps.org)
+  gem.description   = %q{Benchmark extensions}
+  gem.summary       = %q{Benchmark extensions}
+  gem.homepage      = ''
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = %w(benchmark)
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = %w(lib)
+  gem.add_dependency('easystats')
+end

data/bin/benchmark

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+root_dir = File.dirname(ENV['BUNDLE_GEMFILE'])
+
+require 'optparse'
+require 'benchmark'
+
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = 'Usage: benchmark [options]'
+
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit(0)
+  end
+
+  opts.on('-t', '--threads NUM', 'Number of simultaneous threads per thread group') do |v|
+    options[:threads] = v.to_i
+  end
+
+  opts.on('-r', '--repetitions NUM', 'Number of repetitions per thread') do |v|
+    options[:repetitions] = v.to_i
+  end
+
+  opts.on('-s', '--simulation NAME', 'Run only specified simulation') do |v|
+    options[:simulation] = v
+  end
+end
+optparse.parse!
+
+options[:threads] ||= 1
+options[:repetitions] ||= 1
+
+require 'benchmark-extensions'
+require 'bme/simulations'
+
+init_file = "#{root_dir}/init.rb"
+require init_file if init_file
+
+Dir.glob("#{root_dir}/simulations/*.rb").each { |f| require f }
+
+simulations = BME::Simulation.select { |c| options[:simulation] ? c.name == options[:simulation] : true }
+
+labels = simulations.reduce([]) do |l, simulation_class|
+  simulation_class.steps.each do |s|
+    l << "#{simulation_class.label}.#{s}"
+  end
+  l << simulation_class.label
+  l
+end
+
+aggregator = nil
+
+Benchmark.bm(40, *labels, '>total:') do |bm|
+  aggregator = BME::ReportAggregator.new(bm)
+
+  BME::Runner.run(options[:threads], options[:repetitions]) do |run_id|
+    simulations.each do |simulation_class|
+      next if options[:simulation] && simulation_class.name != options[:simulation]
+      simulation_class.run(run_id, aggregator)
+    end
+  end
+
+  aggregator.totals(labels)
+end
+
+puts "\n#{aggregator.stats(40, 5, labels)}"

data/examples/plugins/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'benchmark-extensions', :path => '../..'

data/examples/plugins/init.rb

@@ -0,0 +1,20 @@
+# Fake transaction like class for demonstration purposese
+class Transaction
+  class << self
+    def exec(&blk)
+      blk.call
+    end
+  end
+end
+
+module SimulationHelpers
+   def run
+     Transaction.exec { super }
+   end
+
+   def some_helper
+     'foo'
+   end
+end
+
+BME::Simulation.add_helper(SimulationHelpers)

data/examples/plugins/simulations/manuscript.rb

@@ -0,0 +1,20 @@
+class Manuscript
+  simulation 'Manuscript' do
+    %w(GA1001 GA1002)
+  end
+
+  def simulate_items
+    some_helper # see SimulationHelpers in init.rb
+    manuscript.items
+  end
+
+  def simulate_history
+    manuscript.history
+    sleep(0.1)
+  end
+
+  def manuscript
+    # more typical would be Manuscript.find(subject)
+    @manuscript ||= Struct.new(:order_id, :items, :history).new(:items => [], :history => [])
+  end
+end

data/examples/report_aggregator.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+ENV['BUNDLE_GEMFILE'] = File.expand_path('../../Gemfile', __FILE__)
+
+require 'bundler'
+Bundler.setup
+
+require 'benchmark'
+require 'benchmark-extensions'
+
+labels = %w(groupa.foo groupa.bar groupa.baz groupa)
+labels.concat %w(groupb.foo groupb.bar groupb.baz groupb)
+labels.concat %w(foo bar baz)
+
+aggregator = nil
+
+Benchmark.bm(40, *labels, '>total:') do |bm|
+  aggregator = BME::ReportAggregator.new(bm)
+
+  # to threads 2 repetitions each
+  BME::Runner.run(2, 2) do |run_id|
+    3.times do |i|
+      aggregator.report("#{run_id} #{i} groupa.foo", 'groupa.foo', 'groupa', 'foo') { sleep(0.1) }
+      aggregator.report("#{run_id} #{i} groupa.bar", 'groupa.bar', 'groupa', 'bar') { sleep(0.1) }
+      aggregator.report("#{run_id} #{i} groupa.baz", 'groupa.baz', 'groupa', 'baz') { sleep(0.1) }
+    end
+
+    3.times do |i|
+      aggregator.report("#{run_id} #{i} grouba.foo", 'groupb.foo', 'groupb', 'foo') { sleep(0.1) }
+      aggregator.report("#{run_id} #{i} groupb.bar", 'groupb.bar', 'groupb', 'baz') { sleep(0.1) }
+      aggregator.report("#{run_id} #{i} groupb.baz", 'groupb.baz', 'groupb', 'baz') { sleep(0.1) }
+    end
+  end
+
+  aggregator.totals(labels)
+end
+
+puts "\n#{aggregator.stats(40, 5, labels)}"
+

data/lib/benchmark-extensions.rb

@@ -0,0 +1,8 @@
+require 'bme/version'
+
+module BME
+  autoload :ReportAggregator, 'bme/report_aggregator'
+  autoload :Runner, 'bme/runner'
+end
+
+

data/lib/bme/report_aggregator.rb

@@ -0,0 +1,44 @@
+require 'easystats'
+
+module BME
+  class ReportAggregator
+    attr_reader :keyed_reports, :benchmark, :reports
+
+    def initialize(benchmark)
+      @benchmark = benchmark
+      @keyed_reports = {}
+      @reports = []
+      @mutex = Mutex.new
+    end
+
+    def report(label, *keys, &blk)
+      @mutex.synchronize do
+        reports << r = benchmark.report(label, &blk)
+        keys.each do |k|
+          (keyed_reports[k] ||= []) << r
+        end
+      end
+    end
+
+    def totals(labels)
+      totals = labels.reduce([]) { |l, label| l << tally(keyed_reports.fetch(label)); l }
+      totals << tally(reports)
+      totals
+    end
+
+    def tally(l)
+      l.reduce { |r, total| total += r }
+    end
+
+    def stats(width, precision, labels)
+      buf = ''
+      p = precision
+      labels.each do |k|
+        realtimes = keyed_reports.fetch(k).map(&:real)
+        buf << sprintf("%-#{width}s total(%.#{p}f) avg(%.#{p}f) median(%.#{p}f) std(%.#{p}f) min(%.#{p}f) max(%.#{p}f)  \n",
+                       "#{k}(#{realtimes.size})", realtimes.sum, realtimes.average, realtimes.median, realtimes.standard_deviation, realtimes.min, realtimes.max)
+      end
+      buf
+    end
+  end
+end

data/lib/bme/runner.rb

@@ -0,0 +1,17 @@
+module BME
+  class Runner
+    class << self
+      def run(threads, repetitions, &blk)
+        thread_group = []
+        threads.times do |thread_index|
+          thread_group << Thread.new do
+            repetitions.times do |i|
+              blk.call("[#{thread_index}][#{i}]")
+            end
+          end
+        end
+        thread_group.map(&:join)
+      end
+    end
+  end
+end

data/lib/bme/simulations.rb

@@ -0,0 +1,81 @@
+module BME
+  class Simulation
+    extend Enumerable
+
+    class << self
+      attr_reader :simulations, :helpers
+
+      def add(simulation_class)
+        simulations << simulation_class
+      end
+
+      def each(&blk)
+        simulations.each(&blk)
+      end
+
+      def add_helper(mixin)
+        self.helpers << mixin
+      end
+    end
+
+    @simulations = []
+    @helpers = []
+  end
+
+  module SimulationClassMethods
+    def steps
+      instance_methods.map { |m| m[/^simulate_(.*)/, 1] }.compact
+    end
+
+    def run(run_id, aggregator)
+      batch.each do |*args|
+        s = new
+        s.send(:setup, run_id, aggregator, *args)
+        s.send(:run)
+        s
+      end
+    end
+  end
+
+  module SimulationInstanceMethods
+    attr_reader :run_id, :aggregator, :batch_args
+
+    def subject
+      batch_args.first
+    end
+
+    private
+
+    def qualified_label
+      "#{run_id} #{self.class.label} #{subject}"
+    end
+
+    def setup(run_id, aggregator, *args)
+      @run_id, @aggregator, @batch_args = run_id, aggregator, args
+    end
+
+    def run
+      self.class.steps.each do |step|
+        aggregator.report(qualified_label, self.class.label, "#{self.class.label}.#{step}" ) { send("simulate_#{step}") }
+      end
+    end
+  end
+
+  module SimulationExtension
+    def simulation(label = name, &blk)
+      Simulation.add(self)
+
+      define_singleton_method(:batch, &blk)
+      define_singleton_method(:label) { label }
+
+      extend SimulationClassMethods
+      include SimulationInstanceMethods
+
+      Simulation.helpers.each do |h|
+        include h
+      end
+    end
+  end
+
+  Class.send(:include, SimulationExtension)
+end

data/lib/bme/version.rb

@@ -0,0 +1,3 @@
+module BME
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: benchmark-extensions
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Lenny Marks
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: easystats
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Benchmark extensions
+email:
+- lenny@aps.org
+executables:
+- benchmark
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark-extensions.gemspec
+- bin/benchmark
+- examples/plugins/Gemfile
+- examples/plugins/init.rb
+- examples/plugins/simulations/manuscript.rb
+- examples/report_aggregator.rb
+- lib/benchmark-extensions.rb
+- lib/bme/report_aggregator.rb
+- lib/bme/runner.rb
+- lib/bme/simulations.rb
+- lib/bme/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4387911391081834859
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4387911391081834859
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Benchmark extensions
+test_files: []