beaker-benchmark 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 922c931b423b9789170751154af7dc4a6a6329ec
+  data.tar.gz: 2acd0c5534b6c7a435b8680f502b768cd8ff3e94
+SHA512:
+  metadata.gz: 887223881be37ff961623e150fbf954232d3229a5d73dab2f9aefefc93467418c8e014408e068774dbadd8958515f23d54a144e5d53727bd8b8c929be2e06105
+  data.tar.gz: 253c9dbb32aa9425cca0be231e4fcb40539a11b020ac43064e971b85e9eca8de3282f8f7f7d7136f09d2163722665dec4a370d071ca0dbe7a38dc06b038e77b6

data/.gitignore

@@ -0,0 +1,25 @@
+*.swp
+log/*
+!.gitignore
+junit
+acceptance-tests
+pkg
+Gemfile.lock
+options.rb
+test.cfg
+.yardoc
+coverage
+.bundle
+.vendor
+_vendor
+tmp/
+doc
+# JetBrains IDEA
+*.iml
+.idea/
+# rbenv file
+.ruby-version
+.ruby-gemset
+# Vagrant folder
+.vagrant/
+.vagrant_files/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--tty

data/.simplecov

@@ -0,0 +1,9 @@
+SimpleCov.configure do
+  add_filter 'spec/'
+  add_filter 'vendor/'
+  add_filter do |file|
+    file.lines_of_code < 10
+  end
+end
+
+SimpleCov.start if ENV['BEAKER_BENCHMARK_COVERAGE']

data/Gemfile

@@ -0,0 +1,27 @@
+source ENV['GEM_SOURCE'] || "https://rubygems.org"
+
+gemspec
+
+
+
+def location_for(place, fake_version = nil)
+  if place =~ /^(git:[^#]*)#(.*)/
+    [fake_version, { :git => $1, :branch => $2, :require => false }].compact
+  elsif place =~ /^file:\/\/(.*)/
+    ['>= 0', { :path => File.expand_path($1), :require => false }]
+  else
+    [place, { :require => false }]
+  end
+end
+
+
+# We don't put beaker in as a test dependency because we
+# don't want to create a transitive dependency
+group :acceptance_testing do
+  gem "beaker", *location_for(ENV['BEAKER_VERSION'] || '~> 3.0')
+end
+
+
+if File.exists? "#{__FILE__}.local"
+  eval(File.read("#{__FILE__}.local"), binding)
+end

data/LICENSE

@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/MAINTAINERS

@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "file_format": "This MAINTAINERS file format is described at https://github.com/puppetlabs/maintainers",
+  "issues": "https://tickets.puppetlabs.com/browse/QA",
+  "internal_list": "https://groups.google.com/a/puppet.com/forum/?hl=en#!forum/silver-team",
+  "people": [
+    {
+      "github": "samwoods1",
+      "email": "sam.woods@puppet.com",
+      "name": "Sam Woods"
+    }
+  ]
+}

data/README.md

@@ -0,0 +1,19 @@
+# beaker-benchmark
+
+A tool for monitoring performance on a puppet infrastructure node while performing some actions.
+
+# Usage
+        # Monitor performance on the master while pe-puppetserver is starting up
+        test_name('measure_perf_on_puppetserver_start') {
+          on(master, 'puppet resource service pe-puppetserver ensure=stopped', true)
+          measure_perf_on('start_pe-puppetserver', master) {
+            on(master, 'puppet resource service pe-puppetserver ensure=running')
+          }
+        }
+
+        # Example output:
+        Action: measure_perf_on_puppetserver_start_start_pe-puppetserver, Duration: 37.463595
+        Avg CPU: 172%, Avg MEM: 1634829, Avg DSK read: 0, Avg DSK Write: 45
+        Additional output if include_processes:
+          Process pid: 14067, command: '/opt/puppetlabs/server/apps/postgresql/bin/postgres -D /opt/puppetlabs/server/data/postgresql/9.6/data -c log_directory=/var/log/puppetlabs/postgresql'
+              Avg CPU: '0', Avg MEM: 48888, Avg DSK read: 0, Avg DSK Write: 48888

data/Rakefile

@@ -0,0 +1,154 @@
+require 'rspec/core/rake_task'
+
+namespace :test do
+
+  namespace :spec do
+
+    desc "Run spec tests"
+    RSpec::Core::RakeTask.new(:run) do |t|
+      t.rspec_opts = ['--color']
+      t.pattern = 'spec/'
+    end
+
+    desc "Run spec tests with coverage"
+    RSpec::Core::RakeTask.new(:coverage) do |t|
+      ENV['BEAKER_BENCHMARK_COVERAGE'] = 'y'
+      t.rspec_opts = ['--color']
+      t.pattern = 'spec/'
+    end
+
+  end
+
+  namespace :acceptance do
+
+    desc <<-EOS
+A quick acceptance test, named because it has no pre-suites to run
+    EOS
+    task :quick do
+
+      sh("beaker",
+         "--hosts", ENV['CONFIG'] || "acceptance/config/nodes/vagrant-ubuntu-1404.yml",
+         "--tests", "acceptance/tests",
+         "--log-level", "debug",
+         "--keyfile", ENV['KEY'] || "#{ENV['HOME']}/.ssh/id_rsa")
+    end
+
+  end
+
+end
+
+# namespace-named default tasks.
+# these are the default tasks invoked when only the namespace is referenced.
+# they're needed because `task :default` in those blocks doesn't work as expected.
+task 'test:spec' => 'test:spec:run'
+task 'test:acceptance' => 'test:acceptance:quick'
+
+# global defaults
+task :test => 'test:spec'
+task :default => :test
+
+###########################################################
+#
+#   Documentation Tasks
+#
+###########################################################
+DOCS_DAEMON = "yard server --reload --daemon --server thin"
+FOREGROUND_SERVER = 'bundle exec yard server --reload --verbose --server thin lib/beaker'
+
+def running?( cmdline )
+  ps = `ps -ef`
+  found = ps.lines.grep( /#{Regexp.quote( cmdline )}/ )
+  if found.length > 1
+    raise StandardError, "Found multiple YARD Servers. Don't know what to do."
+  end
+
+  yes = found.empty? ? false : true
+  return yes, found.first
+end
+
+def pid_from( output )
+  output.squeeze(' ').strip.split(' ')[1]
+end
+
+desc 'Start the documentation server in the foreground'
+task :docs => 'docs:clear' do
+  original_dir = Dir.pwd
+  Dir.chdir( File.expand_path(File.dirname(__FILE__)) )
+  sh FOREGROUND_SERVER
+  Dir.chdir( original_dir )
+end
+
+namespace :docs do
+
+  desc 'Clear the generated documentation cache'
+  task :clear do
+    original_dir = Dir.pwd
+    Dir.chdir( File.expand_path(File.dirname(__FILE__)) )
+    sh 'rm -rf docs'
+    Dir.chdir( original_dir )
+  end
+
+  desc 'Generate static documentation'
+  task :gen => 'docs:clear' do
+    original_dir = Dir.pwd
+    Dir.chdir( File.expand_path(File.dirname(__FILE__)) )
+    output = `bundle exec yard doc`
+    puts output
+    if output =~ /\[warn\]|\[error\]/
+      fail "Errors/Warnings during yard documentation generation"
+    end
+    Dir.chdir( original_dir )
+  end
+
+  desc 'Run the documentation server in the background, alias `bg`'
+  task :background => 'docs:clear' do
+    yes, output = running?( DOCS_DAEMON )
+    if yes
+      puts "Not starting a new YARD Server..."
+      puts "Found one running with pid #{pid_from( output )}."
+    else
+      original_dir = Dir.pwd
+      Dir.chdir( File.expand_path(File.dirname(__FILE__)) )
+      sh "bundle exec #{DOCS_DAEMON}"
+      Dir.chdir( original_dir )
+    end
+  end
+
+  task(:bg) { Rake::Task['docs:background'].invoke }
+
+  desc 'Check the status of the documentation server'
+  task :status do
+    yes, output = running?( DOCS_DAEMON )
+    if yes
+      pid = pid_from( output )
+      puts "Found a YARD Server running with pid #{pid}"
+    else
+      puts "Could not find a running YARD Server."
+    end
+  end
+
+  desc "Stop a running YARD Server"
+  task :stop do
+    yes, output = running?( DOCS_DAEMON )
+    if yes
+      pid = pid_from( output )
+      puts "Found a YARD Server running with pid #{pid}"
+      `kill #{pid}`
+      puts "Stopping..."
+      yes, output = running?( DOCS_DAEMON )
+      if yes
+        `kill -9 #{pid}`
+        yes, output = running?( DOCS_DAEMON )
+        if yes
+          puts "Could not Stop Server!"
+        else
+          puts "Server stopped."
+        end
+      else
+        puts "Server stopped."
+      end
+    else
+      puts "Could not find a running YARD Server"
+    end
+  end
+end

data/acceptance/config/nodes/vagrant-ubuntu-1404.yml

@@ -0,0 +1,8 @@
+HOSTS:
+  ubuntu-server-1404-x64:
+    roles:
+      - master
+    platform: ubuntu-14.04-amd64
+    box: puppetlabs/ubuntu-14.04-64-nocm
+    box_url: https://vagrantcloud.com/puppetlabs/boxes/ubuntu-14.04-64-nocm
+    hypervisor: vagrant

data/acceptance/tests/first.rb

@@ -0,0 +1,8 @@
+
+# Acceptance level testing goes into files in the tests directory like this one,
+# Each file corresponding to a new test made up of individual testing steps
+test_name "Template Acceptance Test Example"
+
+step "Fail fast!"
+
+fail_test("There are no acceptance tests yet!")

data/beaker-benchmark.gemspec

@@ -0,0 +1,37 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require 'beaker-benchmark/version'
+
+Gem::Specification.new do |s|
+  s.name        = "beaker-benchmark"
+  s.version     = Beaker::DSL::BeakerBenchmark::Version::STRING
+  s.authors     = ["Puppetlabs"]
+  s.email       = ["silver-team@puppetlabs.com"]
+  s.homepage    = "https://github.com/puppetlabs/beaker-benchmark"
+  s.summary     = %q{Beaker benchmark Helpers!}
+  s.description = %q{Used to monitor performance on a puppet infrastructure node}
+  s.license     = 'Apache2'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # Testing dependencies
+  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'rspec-its'
+  s.add_development_dependency 'fakefs', '~> 0.6'
+  s.add_development_dependency 'rake', '~> 10.1'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'pry', '~> 0.10'
+
+  # Documentation dependencies
+  s.add_development_dependency 'yard'
+  s.add_development_dependency 'markdown'
+  s.add_development_dependency 'thin'
+
+  # Run time dependencies
+  s.add_runtime_dependency 'stringify-hash', '~> 0.0.0'
+
+end
+

data/bin/beaker-benchmark

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'rubygems' unless defined?(Gem)
+require 'beaker-benchmark'
+
+VERSION_STRING =
+"
+                                 _ .--.
+                                ( `    )
+        beaker-benchmark      .-'      `--,
+                  _..----.. (             )`-.
+                .'_|` _|` _|(  .__,           )
+               /_|  _|  _|  _(        (_,  .-'
+              ;|  _|  _|  _|  '-'__,--'`--'
+              | _|  _|  _|  _| |
+          _   ||  _|  _|  _|  _| %s
+        _( `--.\\_|  _|  _|  _|/
+     .-'       )--,|  _|  _|.`
+    (__, (_      ) )_|  _| /
+      `-.__.\\ _,--'\\|__|__/
+                    ;____;
+                     \\YT/
+                      ||
+                     |\"\"|
+                     '=='
+"
+
+
+
+puts VERSION_STRING % [Beaker::DSL::Helpers::BeakerBenchmark::Version::STRING]
+
+exit 0

data/lib/beaker-benchmark/helpers.rb

@@ -0,0 +1,173 @@
+require 'csv'
+
+module Beaker
+  module DSL
+    module BeakerBenchmark
+      module Helpers
+
+        # Example usage:
+        # test_name('measure_perf_on_puppetserver_start') {
+        #   on(master, 'puppet resource service pe-puppetserver ensure=stopped')
+        #   result = measure_perf_on(master, 'start_pe-puppetserver', true) {
+        #     on(master, 'puppet resource service pe-puppetserver ensure=running')
+        #     # on(master, 'facter fqdn')
+        #   }
+        #   raise("pe-puppetserver restart took longer than 120 seconds: #{result.duration} seconds") if result.duration > 120
+        #   raise("pe-puppetserver restart used an average of more than 1gb of memory: #{result.avg_mem} bytes") if result.avg_mem > 1073741824
+        #   process_cpu = result.processes.select{ |pid, process| process[:cmd] =~ /puppet-server-release.jar/ }.values[0][:avg_cpu]
+        #   raise("pe-puppetserver restart caused pe-puppetserver service to use more than 95% of CPU: #{process_cpu}") if process_cpu > 95
+        # }
+        def measure_perf_on(infrastructure_host, action_name, include_processes=false, process_regex='/opt/puppetlabs', &block)
+          # Append action name to test case name if test name is available
+          action_name = metadata[:case][:name] + "_#{action_name}" if defined? metadata && metadata[:case] && metadata[:case][:name]
+
+          atop_log = 'atop_log.log'
+
+          start_monitoring(infrastructure_host, atop_log, include_processes)
+
+          yield
+
+          stop_monitoring(infrastructure_host, action_name, atop_log, include_processes.nil? ? nil : process_regex)
+        end
+
+        def setup_atop(infrastructure_host)
+          @benchmark_tmpdir = Dir.mktmpdir
+          # Only install atop once per host
+          unless infrastructure_host.check_for_package('atop')
+            add_el_extras(infrastructure_host)
+            infrastructure_host.install_package('atop')
+          end
+        end
+
+        def start_monitoring(infrastructure_host, atop_log, include_processes=false, sample_interval=1)
+          setup_atop(infrastructure_host)
+          additional_args = ''
+          additional_args = ',PRC,PRM,PRD' if include_processes
+          atop_cmd        = "sh -c 'nohup atop -P CPU,SWP,DSK#{additional_args} -i #{sample_interval} > #{atop_log} 2>&1 &'"
+
+          on(infrastructure_host, atop_cmd)
+          @beaker_benchmark_start = Time.now
+        end
+
+        def stop_monitoring(infrastructure_host, action_name, atop_log_name, process_regex='.*')
+          if defined?@beaker_benchmark_start && !@beaker_benchmark_start.nil?
+            duration = Time.now - @beaker_benchmark_start
+          else
+            duration = nil
+          end
+
+          # The atop process sticks around unless killed
+          on infrastructure_host, 'pkill -15 -f atop'
+          set_processes_to_monitor(infrastructure_host, process_regex)
+          parse_atop_log(infrastructure_host, action_name, duration, atop_log_name)
+        end
+
+        def parse_atop_log(infrastructure_host, action_name, duration, atop_log_name)
+          unless infrastructure_host.file_exist?(atop_log_name)
+            raise("atop log does not exist at #{atop_log_name}")
+          end
+
+          scp_from(infrastructure_host, atop_log_name, '/tmp')
+          cpu_usage  = []
+          mem_usage  = []
+          disk_read  = []
+          disk_write = []
+
+          process_cpu = []
+          skip        = true
+          CSV.parse(File.read("/tmp/#{File.basename(atop_log_name)}"), { :col_sep => ' ' }) do |row|
+            #skip the first entry, until the first separator 'SEP'.
+            measure_type = row[0]
+            if skip
+              skip = (measure_type != 'SEP')
+              next
+            end
+            case measure_type
+              when 'CPU'
+                # system + usr + iowait
+                cpu_active = row[8].to_i + row[9].to_i + row[12].to_i
+                # active + idle
+                cpu_total = cpu_active + row[11].to_i
+                cpu_percent = cpu_active * 100 / cpu_total
+                cpu_usage.push(cpu_percent)
+              when 'SWP'
+                mem_usage.push(row[10].to_i)
+              when 'DSK'
+                disk_read.push(row[9].to_i)
+                disk_write.push(row[11].to_i)
+              when 'PRC'
+                add_process_measure(:cpu_usage, row[6], row[16].to_i)
+              when 'PRM'
+                add_process_measure(:mem_usage, row[6], row[11].to_i)
+              when 'PRD'
+                # TODO: investigate why atop always shows disk_read as 0
+                # add_process_measure(:disk_read, row[6], row[11].to_i)
+                add_process_measure(:disk_write, row[6], row[13].to_i)
+            end
+          end
+
+          PerformanceResult.new({ :cpu => cpu_usage, :mem => mem_usage, :disk_read => disk_read, :disk_write => disk_write, :action => action_name, :duration => duration, :processes => @processes_to_monitor, :logger => @logger})
+        end
+
+        def set_processes_to_monitor(infrastructure_host, process_regex)
+          @processes_to_monitor = {}
+          return unless process_regex
+          result = on(infrastructure_host, "ps -eo pid,cmd | grep #{process_regex}").output
+          result.each_line do |line|
+            # use the PID as key and command with args as value
+            # also ignore the ps and grep commands executed above.
+            unless line.include? "grep #{process_regex}"
+              @processes_to_monitor[line.split(' ').first] = { :cmd => line.split(' ')[1..-1].join(' '), :cpu_usage => [], :mem_usage => [], :disk_read => [], :disk_write => [] }
+            end
+          end
+          @logger.info result.stdout
+        end
+
+        def add_process_measure measure_type, pid, value
+          if @processes_to_monitor.keys.include? pid
+            @processes_to_monitor[pid][measure_type].push value
+          end
+        end
+
+        # Example output:
+        #   Action: measure_perf_on_puppetserver_start_start_pe-puppetserver, Duration: 37.463595
+        #   Avg CPU: 72%, Avg MEM: 1634829, Avg DSK read: 0, Avg DSK Write: 45
+        # Additional output if include_processes:
+        #   Process pid: 14067, command: '/opt/puppetlabs/server/apps/postgresql/bin/postgres -D /opt/puppetlabs/server/data/postgresql/9.6/data -c log_directory=/var/log/puppetlabs/postgresql'
+        #       Avg CPU: '1', Avg MEM: 48888, Avg DSK Write: 20
+        class PerformanceResult
+          attr_accessor :avg_cpu, :avg_mem, :avg_disk_read, :avg_disk_write, :action_name, :duration, :processes
+          def initialize(args)
+            @avg_cpu = args[:cpu].empty? ? 0 : args[:cpu].inject{ |sum, el| sum + el } / args[:cpu].size
+            @avg_mem = args[:mem].empty? ? 0 : args[:mem].inject{ |sum, el| sum + el } / args[:mem].size
+            # @avg_disk_read = args[:disk_read].empty? ? 0 : args[:disk_read].inject{ |sum, el| sum + el } / args[:disk_read].size
+            @avg_disk_write = args[:disk_write].empty? ? 0 : args[:disk_write].inject{ |sum, el| sum + el } / args[:disk_write].size
+            @action_name = args[:action]
+            @duration = args[:duration]
+            @processes = args[:processes]
+            @logger = args[:logger]
+
+            @processes.keys.each do |key|
+              @processes[key][:avg_cpu] = @processes[key][:cpu_usage].inject{ |sum, el| sum + el } / @processes[key][:cpu_usage].size unless @processes[key][:cpu_usage].empty?
+              @processes[key][:avg_mem] = @processes[key][:mem_usage].inject{ |sum, el| sum + el } / @processes[key][:mem_usage].size unless @processes[key][:mem_usage].empty?
+              # @processes[key][:avg_disk_read] = @processes[key][:disk_read].inject{ |sum, el| sum + el } / @processes[key][:disk_read].size unless @processes[key][:disk_read].empty?
+              @processes[key][:avg_disk_write] = @processes[key][:disk_write].inject{ |sum, el| sum + el } / @processes[key][:disk_write].size unless @processes[key][:disk_write].empty?
+            end if @processes
+            # TODO: At this point, we need to push these results into bigquery or elasticsearch
+            # so we can normalize results over time and report on increases in trends
+          end
+
+          def log_summary
+            @logger.info "Action: #{@action_name}, Duration: #{@duration}"
+            @logger.info "Avg CPU: #{@avg_cpu}%, Avg MEM: #{@avg_mem}, Avg DSK read: #{@avg_disk_read}, Avg DSK Write: #{@avg_disk_write}"
+            @processes.keys.each do |key|
+              @logger.info "Process pid: #{key}, command: '#{@processes[key][:cmd]}'"
+              @logger.info "   Avg CPU: '#{@processes[key][:avg_cpu]}%', Avg MEM: #{@processes[key][:avg_mem]}, Avg DSK read: #{@processes[key][:avg_disk_read]}, Avg DSK Write: #{@processes[key][:avg_disk_write]}"
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/beaker-benchmark/version.rb

@@ -0,0 +1,11 @@
+module Beaker
+  module DSL
+    module BeakerBenchmark
+
+      module Version
+        STRING = '0.0.1'
+      end
+
+    end
+  end
+end

data/lib/beaker-benchmark.rb

@@ -0,0 +1,22 @@
+require 'stringify-hash'
+require 'beaker-benchmark/helpers'
+require 'beaker-benchmark/version'
+
+
+module Beaker
+  module DSL
+    module BeakerBenchmark
+      include Beaker::DSL::BeakerBenchmark::Helpers
+    end
+  end
+end
+
+
+# Boilerplate DSL inclusion mechanism:
+# First we register our module with the Beaker DSL
+Beaker::DSL.register( Beaker::DSL::BeakerBenchmark )
+# Modules added into a module which has previously been included are not
+# retroactively included in the including class.
+#
+# https://github.com/adrianomitre/retroactive_module_inclusion
+Beaker::TestCase.class_eval { include Beaker::DSL }

data/spec/beaker-benchmark/helpers_spec.rb

@@ -0,0 +1,153 @@
+require 'spec_helper'
+
+class ClassMixedWithDSLHelpers
+  include BeakerTestHelpers
+  include Beaker::DSL::BeakerBenchmark::Helpers
+
+  def logger
+    RSpec::Mocks::Double.new('logger').as_null_object
+  end
+
+end
+
+describe ClassMixedWithDSLHelpers do
+  subject { ClassMixedWithDSLHelpers.new }
+
+  describe 'release conditions' do
+
+    it 'has updated the version number from the original template' do
+      expect( Beaker::DSL::BeakerBenchmark::Version::STRING ).to_not be === '0.0.1rc0'
+    end
+
+    it 'has a MAINTAINERS doc' do
+      expect( File.exist?( 'MAINTAINERS' ) ).to be_truthy
+    end
+
+  end
+
+  describe '.setup_atop' do
+
+    it 'installs atop on first run' do
+
+    end
+
+    it 'sets up epel repo on el hosts on first run' do
+
+    end
+
+    it 'does not install atop after first run' do
+
+    end
+
+
+
+
+  end
+
+  describe '.measure_perf_on' do
+
+    it 'concatenates test name with action name if available' do
+
+    end
+
+    it 'only uses action name if test name is not available' do
+
+    end
+
+  end
+
+  describe '.start_monitoring' do
+
+    it 'executes atop command' do
+
+    end
+
+    it 'executes atop command with additional arguments if include_processes=true' do
+
+    end
+
+    it 'sets appropriate value for @beaker_benchmark_start' do
+
+    end
+
+  end
+
+  describe '.stop_monitoring' do
+
+    it 'sets duration, if @beaker_benchmark_start has a value' do
+
+    end
+
+    it 'defaults duration to 0, if @beaker_benchmark_start has no value' do
+
+    end
+
+    it 'kills the background atop process' do
+
+    end
+
+
+  end
+
+  describe '.set_processes_to_monitor' do
+
+    it 'populates @processes_to_monitor if process_regex matches any running process commands' do
+
+    end
+
+    it 'sets @processes_to_monitor an empty hash if process_regex does not match any running process' do
+
+    end
+
+    it 'sets @processes_to_monitor an empty hash if process_regex is nil' do
+
+    end
+
+    it 'sets @processes_to_monitor to all processes if process_regex is .*' do
+
+    end
+
+  end
+
+  describe '.parse_atop_log' do
+
+    it 'creates new PerformanceResult object with valid values' do
+
+    end
+
+  end
+
+  describe 'PerformanceResult::initialize' do
+
+    it 'properly averages arrays of values' do
+
+    end
+
+    it 'sets overall averages to 0 if no values are set' do
+
+    end
+
+    it 'raises an exception if :mem, :cpu, :disk_read or :disk_write args do not exist' do
+
+    end
+
+    it 'creates a result without no process data if @processes_to_monitor is empty or nil' do
+
+    end
+
+    it 'raises an exception if :mem, :cpu, :disk_read or :disk_write args do not exist' do
+
+    end
+
+  end
+
+  describe 'PerformanceResult::log' do
+
+    it 'prints out the results' do
+
+    end
+
+  end
+end
+
+

data/spec/beaker_test_helpers.rb

@@ -0,0 +1,17 @@
+# These are specifically to mock Beaker methods necessary for testing
+# that will be available during runtime because this is never run separate
+# from Beaker itself.
+#
+# Including Beaker as a dependency would not work as a solution to this issue,
+# since that would make a cycle in the dependency graph, at least until
+# Beaker 3.0 happens and this is no longer a dependency of Beaker's.
+module BeakerTestHelpers
+  include Beaker::DSL
+end
+
+module Beaker
+  module DSL
+    def self.register( helper )
+    end
+  end
+end

data/spec/helpers.rb

@@ -0,0 +1,109 @@
+module TestFileHelpers
+  def create_files file_array
+    file_array.each do |f|
+      FileUtils.mkdir_p File.dirname(f)
+      FileUtils.touch f
+    end
+  end
+
+  def fog_file_contents
+    { :default => { :aws_access_key_id => "IMANACCESSKEY",
+                    :aws_secret_access_key => "supersekritkey",
+                    :aix_hypervisor_server => "aix_hypervisor.labs.net",
+                    :aix_hypervisor_username => "aixer",
+                    :aix_hypervisor_keyfile => "/Users/user/.ssh/id_rsa-acceptance",
+                    :solaris_hypervisor_server => "solaris_hypervisor.labs.net",
+                    :solaris_hypervisor_username => "harness",
+                    :solaris_hypervisor_keyfile => "/Users/user/.ssh/id_rsa-old.private",
+                    :solaris_hypervisor_vmpath => "rpoooool/zs",
+                    :solaris_hypervisor_snappaths => ["rpoooool/USER/z0"],
+                    :vsphere_server => "vsphere.labs.net",
+                    :vsphere_username => "vsphere@labs.com",
+                    :vsphere_password => "supersekritpassword"} }
+  end
+
+end
+
+module HostHelpers
+  HOST_DEFAULTS = { :platform => 'unix',
+                    :snapshot => 'pe',
+                    :box => 'box_name',
+                    :roles => ['agent'],
+                    :snapshot => 'snap',
+                    :ip => 'default.ip.address',
+                    :box => 'default_box_name',
+                    :box_url => 'http://default.box.url',
+  }
+
+  HOST_NAME     = "vm%d"
+  HOST_SNAPSHOT = "snapshot%d"
+  HOST_IP       = "ip.address.for.%s"
+  HOST_BOX      = "%s_of_my_box"
+  HOST_BOX_URL  = "http://address.for.my.box.%s"
+  HOST_TEMPLATE = "%s_has_a_template"
+
+  def logger
+    double( 'logger' ).as_null_object
+  end
+
+  def make_opts
+    opts = StringifyHash.new
+    opts.merge( { :logger => logger,
+                                               :host_config => 'sample.config',
+                                               :type => nil,
+                                               :pooling_api => 'http://vcloud.delivery.puppetlabs.net/',
+                                               :datastore => 'instance0',
+                                               :folder => 'Delivery/Quality Assurance/Staging/Dynamic',
+                                               :resourcepool => 'delivery/Quality Assurance/Staging/Dynamic',
+                                               :gce_project => 'beaker-compute',
+                                               :gce_keyfile => '/path/to/keyfile.p12',
+                                               :gce_password => 'notasecret',
+                                               :gce_email => '12345678910@developer.gserviceaccount.com' } )
+  end
+
+  def generate_result (name, opts )
+    result = double( 'result' )
+    stdout = opts.has_key?(:stdout) ? opts[:stdout] : name
+    stderr = opts.has_key?(:stderr) ? opts[:stderr] : name
+    exit_code = opts.has_key?(:exit_code) ? opts[:exit_code] :  0
+    exit_code = [exit_code].flatten
+    allow( result ).to receive( :stdout ).and_return( stdout )
+    allow( result ).to receive( :stderr ).and_return( stderr )
+    allow( result ).to receive( :exit_code ).and_return( *exit_code )
+    result
+  end
+
+  def make_host_opts name, opts
+    make_opts.merge( { 'HOSTS' => { name => opts } } ).merge( opts )
+  end
+
+  def make_host name, host_hash
+    host_hash = StringifyHash.new.merge(HOST_DEFAULTS.merge(host_hash))
+
+    host = make_opts.merge(host_hash)
+
+    allow(host).to receive( :name ).and_return( name )
+    allow(host).to receive( :to_s ).and_return( name )
+    allow(host).to receive( :exec ).and_return( generate_result( name, host_hash ) )
+    host
+  end
+
+  def make_hosts preset_opts = {}, amt = 3
+    hosts = []
+    (1..amt).each do |num|
+      name = HOST_NAME % num
+      opts = { :snapshot => HOST_SNAPSHOT % num,
+               :ip => HOST_IP % name,
+               :template => HOST_TEMPLATE % name,
+               :box => HOST_BOX % name,
+               :box_url => HOST_BOX_URL % name }.merge( preset_opts )
+      hosts << make_host(name, opts)
+    end
+    hosts
+  end
+
+  def make_instance instance_data = {}
+    OpenStruct.new instance_data
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'simplecov'
+require 'beaker'
+require 'beaker_test_helpers'
+require 'beaker-benchmark'
+require 'helpers'
+
+require 'rspec/its'
+
+RSpec.configure do |config|
+  config.include TestFileHelpers
+  config.include HostHelpers
+end

metadata

@@ -0,0 +1,208 @@
+--- !ruby/object:Gem::Specification
+name: beaker-benchmark
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Puppetlabs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: markdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stringify-hash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+description: Used to monitor performance on a puppet infrastructure node
+email:
+- silver-team@puppetlabs.com
+executables:
+- beaker-benchmark
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".simplecov"
+- Gemfile
+- LICENSE
+- MAINTAINERS
+- README.md
+- Rakefile
+- acceptance/config/nodes/vagrant-ubuntu-1404.yml
+- acceptance/tests/first.rb
+- beaker-benchmark.gemspec
+- bin/beaker-benchmark
+- lib/beaker-benchmark.rb
+- lib/beaker-benchmark/helpers.rb
+- lib/beaker-benchmark/version.rb
+- spec/beaker-benchmark/helpers_spec.rb
+- spec/beaker_test_helpers.rb
+- spec/helpers.rb
+- spec/spec_helper.rb
+homepage: https://github.com/puppetlabs/beaker-benchmark
+licenses:
+- Apache2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Beaker benchmark Helpers!
+test_files:
+- spec/beaker-benchmark/helpers_spec.rb
+- spec/beaker_test_helpers.rb
+- spec/helpers.rb
+- spec/spec_helper.rb