benchmark-extensions 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,19 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ .idea
19
+
data/Gemfile ADDED
@@ -0,0 +1,9 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in benchmarker.gemspec
4
+ gemspec
5
+
6
+ group :development do
7
+ gem 'rake'
8
+ end
9
+
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Lenny Marks
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,93 @@
1
+ Ruby benchmark extensions
2
+ ==============================
3
+
4
+ * ```BME::Runner.run(threads, repetitions, &blk)``` - Run block of code in :threads threads, :repetition times each thread.
5
+ <pre>
6
+ BME::Runner.run(2, 2) do |run_id|
7
+ puts run_id
8
+ end
9
+
10
+ [0][0]
11
+ [0][1]
12
+ [1][0]
13
+ [1][1]
14
+ </pre>
15
+ * ```BME::ReportAggregator``` - Threadsafe aggregating wrapper around ```#report``` from SDK Benchmark.
16
+ See: [examples/report_aggregator.rb](examples/report_aggregator.rb)
17
+ * ```benchmark``` executable - A mini-framework with automatic discovery and statistic reporting for pluggable ```simulations```.
18
+ <pre>
19
+ $ bundle exec benchmark --help
20
+ Usage: benchmark [options]
21
+ -h, --help Display this screen
22
+ -t, --threads NUM Number of simultaneous threads per thread group
23
+ -r, --repetitions NUM Number of repetitions per thread
24
+ -s, --simulation NAME Run only specified simulation
25
+
26
+ $ bundle exec benchmark -s ManView -t 3 -r 2
27
+ user system total real
28
+ [1][0] ManView GA1001 0.088000 0.000000 0.088000 ( 0.086000)
29
+ [1][0] ManView GA1001 0.020000 0.000000 0.020000 ( 0.020000)
30
+ .....
31
+ ManView.history 0.224000 0.000000 0.224000 ( 0.222000)
32
+ ManView.tracking_info 0.091000 0.000000 0.091000 ( 0.091000)
33
+ ManView.related_manuscripts 0.209000 0.000000 0.209000 ( 0.209000)
34
+ ManView.contact_info 0.322000 0.000000 0.322000 ( 0.322000)
35
+ .....
36
+ ManView 3.251001 0.000000 3.251001 ( 3.252001)
37
+ >total: 3.251001 0.000000 3.251001 ( 3.252001)
38
+
39
+ ManView.history(12) total(0.22200) avg(0.01850) median(0.00950) std(0.02259) min(0.00500) max(0.08600)
40
+ ManView.tracking_info(12) total(0.09100) avg(0.00758) median(0.00450) std(0.00682) min(0.00200) max(0.02000)
41
+ ManView.related_manuscripts(12) total(0.20900) avg(0.01742) median(0.01100) std(0.01274) min(0.00800) max(0.04800)
42
+ ManView.contact_info(12) total(0.32200) avg(0.02683) median(0.01650) std(0.02814) min(0.00700) max(0.09900)
43
+ .....
44
+ ManView(216) total(3.25200) avg(0.01506) median(0.00500) std(0.03255) min(0.00000) max(0.32000)
45
+ </pre>
46
+
47
+ ## Plugins
48
+
49
+ Ruby files in the ```simulations/``` directory are automatically loaded.
50
+
51
+ A class is plugged in to the framework via the ```simulation``` method. ```simulation``` expects a block that returns a list of elements
52
+ used to instantiate simulation instances.
53
+ An instance of the simulation class will be instantiated for each batch item and all ```simulate_*'```
54
+ methods will be executed and benchmarked. The batch item a simulation instance was instantiated
55
+ with is accessible as ```#subject```.
56
+
57
+ e.g.
58
+
59
+ class ManView
60
+ simulation 'ManView' do
61
+ %w(GA1001 AA1002)
62
+ end
63
+
64
+ def simulate_events
65
+ ....
66
+
67
+ def simulate_contact_info
68
+ ...
69
+
70
+ private
71
+
72
+ def manuscript
73
+ @manuscript ||= Manuscript.find(subject)
74
+ end
75
+ end
76
+
77
+ ### Customizing
78
+
79
+ The benchmark script loads ```init.rb``` which is where your customizations should go.
80
+
81
+ To override simulation methods (i.e. to decorate) you can use ```Simulation.add_helper(SimulationHelpers)```
82
+ to specify a mixin to be added to all simulation classes.
83
+
84
+ e.g. # init.rb
85
+
86
+ module SimulationHelpers
87
+ def run
88
+ EOPCommon::DoInSession.execute_new { super }
89
+ end
90
+ ....
91
+ end
92
+
93
+ Simulation.add_helper(SimulationHelpers)
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,20 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'bme/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = 'benchmark-extensions'
8
+ gem.version = BME::VERSION
9
+ gem.authors = ['Lenny Marks']
10
+ gem.email = %w(lenny@aps.org)
11
+ gem.description = %q{Benchmark extensions}
12
+ gem.summary = %q{Benchmark extensions}
13
+ gem.homepage = ''
14
+
15
+ gem.files = `git ls-files`.split($/)
16
+ gem.executables = %w(benchmark)
17
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
+ gem.require_paths = %w(lib)
19
+ gem.add_dependency('easystats')
20
+ end
@@ -0,0 +1,67 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ root_dir = File.dirname(ENV['BUNDLE_GEMFILE'])
4
+
5
+ require 'optparse'
6
+ require 'benchmark'
7
+
8
+ options = {}
9
+ optparse = OptionParser.new do |opts|
10
+ opts.banner = 'Usage: benchmark [options]'
11
+
12
+ opts.on('-h', '--help', 'Display this screen') do
13
+ puts opts
14
+ exit(0)
15
+ end
16
+
17
+ opts.on('-t', '--threads NUM', 'Number of simultaneous threads per thread group') do |v|
18
+ options[:threads] = v.to_i
19
+ end
20
+
21
+ opts.on('-r', '--repetitions NUM', 'Number of repetitions per thread') do |v|
22
+ options[:repetitions] = v.to_i
23
+ end
24
+
25
+ opts.on('-s', '--simulation NAME', 'Run only specified simulation') do |v|
26
+ options[:simulation] = v
27
+ end
28
+ end
29
+ optparse.parse!
30
+
31
+ options[:threads] ||= 1
32
+ options[:repetitions] ||= 1
33
+
34
+ require 'benchmark-extensions'
35
+ require 'bme/simulations'
36
+
37
+ init_file = "#{root_dir}/init.rb"
38
+ require init_file if init_file
39
+
40
+ Dir.glob("#{root_dir}/simulations/*.rb").each { |f| require f }
41
+
42
+ simulations = BME::Simulation.select { |c| options[:simulation] ? c.name == options[:simulation] : true }
43
+
44
+ labels = simulations.reduce([]) do |l, simulation_class|
45
+ simulation_class.steps.each do |s|
46
+ l << "#{simulation_class.label}.#{s}"
47
+ end
48
+ l << simulation_class.label
49
+ l
50
+ end
51
+
52
+ aggregator = nil
53
+
54
+ Benchmark.bm(40, *labels, '>total:') do |bm|
55
+ aggregator = BME::ReportAggregator.new(bm)
56
+
57
+ BME::Runner.run(options[:threads], options[:repetitions]) do |run_id|
58
+ simulations.each do |simulation_class|
59
+ next if options[:simulation] && simulation_class.name != options[:simulation]
60
+ simulation_class.run(run_id, aggregator)
61
+ end
62
+ end
63
+
64
+ aggregator.totals(labels)
65
+ end
66
+
67
+ puts "\n#{aggregator.stats(40, 5, labels)}"
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'benchmark-extensions', :path => '../..'
@@ -0,0 +1,20 @@
1
+ # Fake transaction like class for demonstration purposese
2
+ class Transaction
3
+ class << self
4
+ def exec(&blk)
5
+ blk.call
6
+ end
7
+ end
8
+ end
9
+
10
+ module SimulationHelpers
11
+ def run
12
+ Transaction.exec { super }
13
+ end
14
+
15
+ def some_helper
16
+ 'foo'
17
+ end
18
+ end
19
+
20
+ BME::Simulation.add_helper(SimulationHelpers)
@@ -0,0 +1,20 @@
1
+ class Manuscript
2
+ simulation 'Manuscript' do
3
+ %w(GA1001 GA1002)
4
+ end
5
+
6
+ def simulate_items
7
+ some_helper # see SimulationHelpers in init.rb
8
+ manuscript.items
9
+ end
10
+
11
+ def simulate_history
12
+ manuscript.history
13
+ sleep(0.1)
14
+ end
15
+
16
+ def manuscript
17
+ # more typical would be Manuscript.find(subject)
18
+ @manuscript ||= Struct.new(:order_id, :items, :history).new(:items => [], :history => [])
19
+ end
20
+ end
@@ -0,0 +1,39 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ ENV['BUNDLE_GEMFILE'] = File.expand_path('../../Gemfile', __FILE__)
4
+
5
+ require 'bundler'
6
+ Bundler.setup
7
+
8
+ require 'benchmark'
9
+ require 'benchmark-extensions'
10
+
11
+ labels = %w(groupa.foo groupa.bar groupa.baz groupa)
12
+ labels.concat %w(groupb.foo groupb.bar groupb.baz groupb)
13
+ labels.concat %w(foo bar baz)
14
+
15
+ aggregator = nil
16
+
17
+ Benchmark.bm(40, *labels, '>total:') do |bm|
18
+ aggregator = BME::ReportAggregator.new(bm)
19
+
20
+ # to threads 2 repetitions each
21
+ BME::Runner.run(2, 2) do |run_id|
22
+ 3.times do |i|
23
+ aggregator.report("#{run_id} #{i} groupa.foo", 'groupa.foo', 'groupa', 'foo') { sleep(0.1) }
24
+ aggregator.report("#{run_id} #{i} groupa.bar", 'groupa.bar', 'groupa', 'bar') { sleep(0.1) }
25
+ aggregator.report("#{run_id} #{i} groupa.baz", 'groupa.baz', 'groupa', 'baz') { sleep(0.1) }
26
+ end
27
+
28
+ 3.times do |i|
29
+ aggregator.report("#{run_id} #{i} grouba.foo", 'groupb.foo', 'groupb', 'foo') { sleep(0.1) }
30
+ aggregator.report("#{run_id} #{i} groupb.bar", 'groupb.bar', 'groupb', 'baz') { sleep(0.1) }
31
+ aggregator.report("#{run_id} #{i} groupb.baz", 'groupb.baz', 'groupb', 'baz') { sleep(0.1) }
32
+ end
33
+ end
34
+
35
+ aggregator.totals(labels)
36
+ end
37
+
38
+ puts "\n#{aggregator.stats(40, 5, labels)}"
39
+
@@ -0,0 +1,8 @@
1
+ require 'bme/version'
2
+
3
+ module BME
4
+ autoload :ReportAggregator, 'bme/report_aggregator'
5
+ autoload :Runner, 'bme/runner'
6
+ end
7
+
8
+
@@ -0,0 +1,44 @@
1
+ require 'easystats'
2
+
3
+ module BME
4
+ class ReportAggregator
5
+ attr_reader :keyed_reports, :benchmark, :reports
6
+
7
+ def initialize(benchmark)
8
+ @benchmark = benchmark
9
+ @keyed_reports = {}
10
+ @reports = []
11
+ @mutex = Mutex.new
12
+ end
13
+
14
+ def report(label, *keys, &blk)
15
+ @mutex.synchronize do
16
+ reports << r = benchmark.report(label, &blk)
17
+ keys.each do |k|
18
+ (keyed_reports[k] ||= []) << r
19
+ end
20
+ end
21
+ end
22
+
23
+ def totals(labels)
24
+ totals = labels.reduce([]) { |l, label| l << tally(keyed_reports.fetch(label)); l }
25
+ totals << tally(reports)
26
+ totals
27
+ end
28
+
29
+ def tally(l)
30
+ l.reduce { |r, total| total += r }
31
+ end
32
+
33
+ def stats(width, precision, labels)
34
+ buf = ''
35
+ p = precision
36
+ labels.each do |k|
37
+ realtimes = keyed_reports.fetch(k).map(&:real)
38
+ buf << sprintf("%-#{width}s total(%.#{p}f) avg(%.#{p}f) median(%.#{p}f) std(%.#{p}f) min(%.#{p}f) max(%.#{p}f) \n",
39
+ "#{k}(#{realtimes.size})", realtimes.sum, realtimes.average, realtimes.median, realtimes.standard_deviation, realtimes.min, realtimes.max)
40
+ end
41
+ buf
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,17 @@
1
+ module BME
2
+ class Runner
3
+ class << self
4
+ def run(threads, repetitions, &blk)
5
+ thread_group = []
6
+ threads.times do |thread_index|
7
+ thread_group << Thread.new do
8
+ repetitions.times do |i|
9
+ blk.call("[#{thread_index}][#{i}]")
10
+ end
11
+ end
12
+ end
13
+ thread_group.map(&:join)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,81 @@
1
+ module BME
2
+ class Simulation
3
+ extend Enumerable
4
+
5
+ class << self
6
+ attr_reader :simulations, :helpers
7
+
8
+ def add(simulation_class)
9
+ simulations << simulation_class
10
+ end
11
+
12
+ def each(&blk)
13
+ simulations.each(&blk)
14
+ end
15
+
16
+ def add_helper(mixin)
17
+ self.helpers << mixin
18
+ end
19
+ end
20
+
21
+ @simulations = []
22
+ @helpers = []
23
+ end
24
+
25
+ module SimulationClassMethods
26
+ def steps
27
+ instance_methods.map { |m| m[/^simulate_(.*)/, 1] }.compact
28
+ end
29
+
30
+ def run(run_id, aggregator)
31
+ batch.each do |*args|
32
+ s = new
33
+ s.send(:setup, run_id, aggregator, *args)
34
+ s.send(:run)
35
+ s
36
+ end
37
+ end
38
+ end
39
+
40
+ module SimulationInstanceMethods
41
+ attr_reader :run_id, :aggregator, :batch_args
42
+
43
+ def subject
44
+ batch_args.first
45
+ end
46
+
47
+ private
48
+
49
+ def qualified_label
50
+ "#{run_id} #{self.class.label} #{subject}"
51
+ end
52
+
53
+ def setup(run_id, aggregator, *args)
54
+ @run_id, @aggregator, @batch_args = run_id, aggregator, args
55
+ end
56
+
57
+ def run
58
+ self.class.steps.each do |step|
59
+ aggregator.report(qualified_label, self.class.label, "#{self.class.label}.#{step}" ) { send("simulate_#{step}") }
60
+ end
61
+ end
62
+ end
63
+
64
+ module SimulationExtension
65
+ def simulation(label = name, &blk)
66
+ Simulation.add(self)
67
+
68
+ define_singleton_method(:batch, &blk)
69
+ define_singleton_method(:label) { label }
70
+
71
+ extend SimulationClassMethods
72
+ include SimulationInstanceMethods
73
+
74
+ Simulation.helpers.each do |h|
75
+ include h
76
+ end
77
+ end
78
+ end
79
+
80
+ Class.send(:include, SimulationExtension)
81
+ end
@@ -0,0 +1,3 @@
1
+ module BME
2
+ VERSION = "0.0.1"
3
+ end
metadata ADDED
@@ -0,0 +1,84 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: benchmark-extensions
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Lenny Marks
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-08-19 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: easystats
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ description: Benchmark extensions
31
+ email:
32
+ - lenny@aps.org
33
+ executables:
34
+ - benchmark
35
+ extensions: []
36
+ extra_rdoc_files: []
37
+ files:
38
+ - .gitignore
39
+ - Gemfile
40
+ - LICENSE.txt
41
+ - README.md
42
+ - Rakefile
43
+ - benchmark-extensions.gemspec
44
+ - bin/benchmark
45
+ - examples/plugins/Gemfile
46
+ - examples/plugins/init.rb
47
+ - examples/plugins/simulations/manuscript.rb
48
+ - examples/report_aggregator.rb
49
+ - lib/benchmark-extensions.rb
50
+ - lib/bme/report_aggregator.rb
51
+ - lib/bme/runner.rb
52
+ - lib/bme/simulations.rb
53
+ - lib/bme/version.rb
54
+ homepage: ''
55
+ licenses: []
56
+ post_install_message:
57
+ rdoc_options: []
58
+ require_paths:
59
+ - lib
60
+ required_ruby_version: !ruby/object:Gem::Requirement
61
+ none: false
62
+ requirements:
63
+ - - ! '>='
64
+ - !ruby/object:Gem::Version
65
+ version: '0'
66
+ segments:
67
+ - 0
68
+ hash: -4387911391081834859
69
+ required_rubygems_version: !ruby/object:Gem::Requirement
70
+ none: false
71
+ requirements:
72
+ - - ! '>='
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ segments:
76
+ - 0
77
+ hash: -4387911391081834859
78
+ requirements: []
79
+ rubyforge_project:
80
+ rubygems_version: 1.8.24
81
+ signing_key:
82
+ specification_version: 3
83
+ summary: Benchmark extensions
84
+ test_files: []