benchmark_driver_monotonic_raw 0.14.13

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.ruby_opts = %w[-w]
+  t.rspec_opts = %w[--profile]
+end
+
+task default: :spec

data/benchmark-driver/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/benchmark-driver/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at mamaveb@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/benchmark-driver/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in benchmark-driver.gemspec
+gemspec
+
+gem 'rake'

data/benchmark-driver/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 bmarkons
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/benchmark-driver/README.md

@@ -0,0 +1,8 @@
+# benchmark-driver.gem
+
+This is an alias to install `benchmark_driver.gem`.
+It was built because its command name is actually `benchmark-driver`.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/benchmark-driver/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/benchmark-driver/benchmark-driver.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "benchmark-driver"
+  spec.version       = "0.1.0"
+  spec.authors       = ["bmarkons"]
+  spec.email         = ["mamaveb@gmail.com"]
+
+  spec.summary       = %q{Alias gem to install benchmark_driver.gem}
+  spec.homepage      = "https://github.com/benchmark-driver/benchmark-driver"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "benchmark_driver"
+end

data/benchmark-driver/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "benchmark/driver"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/benchmark-driver/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/benchmark-driver/lib/benchmark-driver.rb

@@ -0,0 +1 @@
+require 'benchmark_driver'

data/benchmark-driver/lib/benchmark/driver.rb

@@ -0,0 +1 @@
+require 'benchmark_driver'

data/benchmark_driver.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'benchmark_driver/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'benchmark_driver_monotonic_raw'
+  spec.version       = BenchmarkDriver::VERSION
+  spec.authors       = ['Simon Eskildsen']
+  spec.email         = ['sirup@sirupsen.com']
+
+  spec.summary       = 'benchmark_driver using monotonic raw'
+  spec.description   = 'benchmark_drive using monotonic raw'
+  spec.homepage      = 'https://github.com/benchmark-driver/benchmark-driver'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 2.2.0'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rspec-retry'
+end

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'benchmark_driver'
+
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/benchmark-driver

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+$:.unshift File.expand_path('../lib', __dir__)
+
+require 'benchmark_driver'
+require 'optparse'
+require 'shellwords'
+require 'yaml'
+
+# Parse command line options
+config = BenchmarkDriver::Config.new.tap do |c|
+  executables = []
+  bundler = false
+  timeout = false
+
+  parser = OptionParser.new do |o|
+    o.version = BenchmarkDriver::VERSION
+    o.banner = "Usage: #{File.basename($0, '.*')} [options] RUBY|YAML..."
+    o.on('-r', '--runner TYPE', String, 'Specify runner type: ips, time, memory, once (default: ips)') do |d|
+      c.runner_type = d
+    end
+    o.on('-o', '--output TYPE', String, 'Specify output type: compare, simple, markdown, record (default: compare)') do |out|
+      c.output_type = out
+    end
+    o.on('-e', '--executables EXECS', String, 'Ruby executables (e1::path1 arg1; e2::path2 arg2;...)') do |e|
+      e.split(';').each do |name_path|
+        name, path = name_path.split('::', 2)
+        path ||= name # if `::` is not given, regard whole string as path
+        command = path.shellsplit
+        command[0] = File.expand_path(command[0])
+        executables << BenchmarkDriver::Config::Executable.new(name: name, command: command)
+      end
+    end
+    o.on('--rbenv VERSIONS', String, 'Ruby executables in rbenv (x.x.x arg1;y.y.y arg2;...)') do |r|
+      r.split(';').each do |version|
+        executables << BenchmarkDriver::Rbenv.parse_spec(version)
+      end
+    end
+    o.on('--repeat-count NUM', Integer, 'Try benchmark NUM times and use the fastest result or the worst memory usage') do |v|
+      c.repeat_count = v
+    end
+    o.on('--repeat-result TYPE', String, 'Yield "best", "average" or "worst" result with --repeat-count (default: best)') do |v|
+      unless BenchmarkDriver::Repeater::VALID_TYPES.include?(v)
+        raise ArgumentError.new("--repeat-result must be #{BenchmarkDriver::Repeater::VALID_TYPES.join(', ')} but got #{v.inspect}")
+      end
+      c.repeat_result = v
+    end
+    o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
+      bundler = v
+    end
+    o.on('--filter REGEXP', String, 'Filter out benchmarks with given regexp') do |v|
+      c.filters << Regexp.compile(v)
+    end
+    o.on('--run-duration SECONDS', Float, 'Warmup estimates loop_count to run for this duration (default: 3)') do |v|
+      c.run_duration = v
+    end
+    o.on('--timeout SECONDS', Float, 'Timeout ruby command execution with timeout(1)') do |v|
+      timeout = v
+    end if (os = RbConfig::CONFIG['host_os']) && os.match(/linux/) && system('which timeout > /dev/null') # depending on coreutils for now...
+    o.on('-v', '--verbose', 'Verbose mode. Multiple -v options increase visilibity (max: 2)') do |v|
+      c.verbose += 1
+    end
+  end
+  begin
+    c.paths = parser.parse!(ARGV)
+  rescue OptionParser::InvalidArgument => e
+    abort e.message
+  end
+  if c.paths.empty?
+    abort "No YAML file is specified!\n\n#{parser.help}"
+  end
+
+  # Configs that need to be set lazily
+  unless executables.empty?
+    c.executables = executables
+  end
+  c.executables.each do |exec|
+    if bundler
+      exec.command << '-rbundler/setup'
+    end
+    if timeout
+      exec.command = ['timeout', timeout.to_s, *exec.command]
+    end
+  end
+
+  c.freeze
+end
+
+# Parse benchmark job definitions
+jobs = config.paths.flat_map do |path|
+  job = { 'type' => config.runner_type }
+
+  # Treat *.rb as a single-execution benchmark, others are considered as YAML definition
+  if path.end_with?('.rb')
+    name = File.basename(path).sub(/\.rb\z/, '')
+    script = File.read(path)
+    prelude = script.slice!(/\A(^#[^\n]+\n)+/m) || '' # preserve magic comment
+    job.merge!('prelude' => prelude, 'benchmark' => { name => script }, 'loop_count' => 1)
+  else
+    job.merge!(YAML.load_file(path))
+  end
+
+  begin
+    # `working_directory` is YAML-specific special parameter, mainly for "command_stdout"
+    BenchmarkDriver::JobParser.parse(job, default_params: { working_directory: File.dirname(path) })
+  rescue ArgumentError
+    $stderr.puts "benchmark-driver: Failed to parse #{path.dump}."
+    $stderr.puts '  YAML format may be wrong. See error below:'
+    $stderr.puts
+    raise
+  end
+end.select do |job|
+  config.filters.all? do |filter|
+    job.name.match(filter)
+  end
+end
+
+# Run jobs
+BenchmarkDriver::Runner.run(jobs, config: config)

data/lib/benchmark_driver.rb

@@ -0,0 +1,14 @@
+require 'benchmark_driver/bulk_output'
+require 'benchmark_driver/config'
+require 'benchmark_driver/job_parser'
+require 'benchmark_driver/output'
+require 'benchmark_driver/rbenv'
+require 'benchmark_driver/repeater'
+require 'benchmark_driver/ruby_interface'
+require 'benchmark_driver/runner'
+require 'benchmark_driver/version'
+
+require 'benchmark'
+def Benchmark.driver(**args, &block)
+  BenchmarkDriver::RubyInterface.run(**args, &block)
+end

data/lib/benchmark_driver/bulk_output.rb

@@ -0,0 +1,59 @@
+module BenchmarkDriver
+  # This is API for your casual output plugin and NOT internally used by BenchmarkDriver.
+  #
+  # By fully utilizing with_*/report APIs, you can implement streaming-output plugins.
+  # See also: lib/benchmark_driver/output.rb (this class's instance will be `@output`)
+  # But using these APIs can be difficult because the API is not stable yet and it's hard
+  # to deal with the complex state machine.
+  #
+  # If you don't need to output results in a streaming manner, you can create an output
+  # plugin class that inherits `BenchmarkDriver::BulkOutput`, which requires to override
+  # only `#bulk_output` that takes all inputs at once.
+  class BulkOutput
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    # @param [Array<BenchmarkDriver::Job>] jobs
+    # @param [Array<BenchmarkDriver::Context>] contexts
+    def initialize(metrics:, jobs:, contexts:)
+      @metrics = metrics
+    end
+
+    # The main API you need to override if you make a class inherit `BenchmarkDriver::BulkOutput`.
+    # @param [Hash{ BenchmarkDriver::Job => Hash{ BenchmarkDriver::Context => BenchmarkDriver::Result } }] job_context_result
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    def bulk_output(job_context_result:, metrics:)
+      raise NotImplementedError.new("#{self.class} must override #bulk_output")
+    end
+
+    def with_warmup(&block)
+      block.call # noop
+    end
+
+    def with_benchmark(&block)
+      @job_context_result = Hash.new do |hash, job|
+        hash[job] = {}
+      end
+      result = block.call
+      bulk_output(job_context_result: @job_context_result, metrics: @metrics)
+      result
+    end
+
+    # @param [BenchmarkDriver::Job] job
+    def with_job(job, &block)
+      @job = job
+      block.call
+    end
+
+    # @param [BenchmarkDriver::Context] context
+    def with_context(context, &block)
+      @context = context
+      block.call
+    end
+
+    # @param [BenchmarkDriver::Result] result
+    def report(result)
+      if defined?(@job_context_result)
+        @job_context_result[@job][@context] = result
+      end
+    end
+  end
+end

data/lib/benchmark_driver/config.rb

@@ -0,0 +1,59 @@
+require 'benchmark_driver/struct'
+require 'rbconfig'
+
+module BenchmarkDriver
+  # All CLI options
+  Config = ::BenchmarkDriver::Struct.new(
+    :runner_type,   # @param [String]
+    :output_type,   # @param [String]
+    :paths,         # @param [Array<String>]
+    :executables,   # @param [Array<BenchmarkDriver::Config::Executable>]
+    :filters,       # @param [Array<Regexp>]
+    :repeat_count,  # @param [Integer]
+    :repeat_result, # @param [String]
+    :run_duration,  # @param [Float]
+    :timeout,       # @param [Float,nil]
+    :verbose,       # @param [Integer]
+    defaults: {
+      runner_type: 'ips',
+      output_type: 'compare',
+      filters: [],
+      repeat_count: 1,
+      repeat_result: 'best',
+      run_duration: 3.0,
+      verbose: 0,
+    },
+  )
+
+  # Subset of FullConfig passed to JobRunner
+  Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
+    :repeat_count,  # @param [Integer]
+    :repeat_result, # @param [String]
+    :run_duration,  # @param [Float]
+    :timeout,       # @param [Float,nil]
+    :verbose,       # @param [Integer]
+  )
+
+  Config::Executable = ::BenchmarkDriver::Struct.new(
+    :name,    # @param [String]
+    :command, # @param [Array<String>]
+  ) do
+    def initialize(*)
+      super
+      @cache = {} # modifiable storage even after `#freeze`
+    end
+
+    # @return [String] - Return result of `ruby -v`. This is for convenience of output plugins.
+    def description
+      @cache[:description] ||= IO.popen([*command, '-v'], &:read).rstrip
+    end
+
+    # @return [String] - Return RUBY_VERSION
+    def version
+      @cache[:version] ||= IO.popen([*command, '-e', 'print RUBY_VERSION'], &:read)
+    end
+  end
+  Config.defaults[:executables] = [
+    BenchmarkDriver::Config::Executable.new(name: RUBY_VERSION, command: [RbConfig.ruby]),
+  ]
+end