openstudio_measure_tester 0.1.4 → 0.1.5

data/lib/openstudio_measure_tester/runner.rb

@@ -0,0 +1,270 @@
+module OpenStudioMeasureTester
+  # The runner is the workhorse that executes the tests. This class does not invoke Rake and can be run
+  # as a library method or as a CLI call
+  class Runner
+    # Initialize the runner
+    #
+    # @param base_dir [string] Base dir, measures will be recursively tested from this location. Results will be here too.
+    def initialize(base_dir)
+      @base_dir = base_dir
+
+      puts "Base dir is #{base_dir}"
+    end
+
+    # Create and return the location where the tests results need to be stored
+    def test_results_dir
+      dir = "#{@base_dir}/test_results"
+      FileUtils.mkdir_p dir unless Dir.exist? dir
+
+      return dir
+    end
+
+    # Run ERB to create the dashboard
+    def dashboard
+      template = OpenStudioMeasureTester::Dashboard.new(test_results_dir)
+      template.render
+    end
+
+    # Prepare the current directory and the root directory to remove old test results before running
+    # the new tests
+    #
+    # @param orig_results_dir [string] Location on where there results of minitest/coverage will be stored.
+    def pre_process_minitest(orig_results_dir)
+      puts "Current directory is #{@base_dir}"
+      puts "Pre-processed tests run data in #{orig_results_dir}"
+      puts "Test results will be stored in: #{test_results_dir}"
+
+      # There is a bunch of moving that needs to happen with coverage/minitest...
+      FileUtils.rm_rf "#{orig_results_dir}/coverage" if Dir.exist? "#{orig_results_dir}/coverage"
+      FileUtils.rm_rf "#{@base_dir}/coverage" if Dir.exist? "#{@base_dir}/coverage"
+      FileUtils.rm_rf "#{test_results_dir}/coverage" if Dir.exist? "#{test_results_dir}/coverage"
+
+      FileUtils.rm_rf "#{orig_results_dir}/minitest" if Dir.exist? "#{orig_results_dir}/minitest"
+      FileUtils.rm_rf "#{@base_dir}/minitest" if Dir.exist? "#{@base_dir}/minitest"
+      FileUtils.rm_rf "#{test_results_dir}/minitest" if Dir.exist? "#{test_results_dir}/minitest"
+
+      FileUtils.rm_rf "#{orig_results_dir}/test" if Dir.exist? "#{orig_results_dir}/test"
+      FileUtils.rm_rf "#{@base_dir}/test" if Dir.exist? "#{@base_dir}/test"
+      # remove the test directory if it is empty (size == 2 for . and ..)
+      if Dir.exist?("#{test_results_dir}/test") && Dir.entries("#{test_results_dir}/test").size == 2
+        FileUtils.rm_rf "#{test_results_dir}/test"
+      end
+      FileUtils.rm_rf "#{test_results_dir}/minitest" if Dir.exist? "#{test_results_dir}/minitest"
+
+      # Create the test_results directory to store all the results
+      return test_results_dir
+    end
+
+    # Rubocop stores the results (for now) in the test_results directory
+    def pre_process_rubocop
+      # copy over the .rubocop.yml file into the root directory of where this is running.
+      shared_rubocop_file = File.expand_path('../../.rubocop.yml', File.dirname(__FILE__))
+      dest_file = "#{File.expand_path(@base_dir)}/.rubocop.yml"
+      if shared_rubocop_file != dest_file
+        FileUtils.copy(shared_rubocop_file, dest_file)
+      end
+
+      puts "Current directory is #{@base_dir}"
+      # puts "Pre-processing tests run in #{@base_dir}"
+      puts "Test results will be stored in: #{test_results_dir}"
+
+      FileUtils.rm_rf "#{test_results_dir}/rubocop" if Dir.exist? "#{test_results_dir}/rubocop"
+      FileUtils.rm_rf "#{@base_dir}/rubocop" if Dir.exist? "#{@base_dir}/rubocop"
+
+      # Call the test_results dir to create the directory and return it as a string.
+      return test_results_dir
+    end
+
+    # Post process the various results and save them into the base_dir
+    #
+    # @param original_results_directory [string] Location of the results from coverag and minitest
+    def post_process_results(original_results_directory = nil)
+      puts " ========================= Starting Results Post Process ================================"
+      puts "Current directory: #{@base_dir}"
+      puts "Test results will be stored in: #{test_results_dir}"
+
+      results = OpenStudioMeasureTester::OpenStudioTestingResult.new(@base_dir, test_results_dir, original_results_directory)
+      results.save_results # one single file for dashboard
+
+      # call the create dashboard command now that we have results
+      dashboard
+
+      # return the results exit code
+      return results.exit_code
+    end
+
+    # OpenStudio style check preparation
+    #
+    # @param base_dir [string] Base directory where results will be stored. If called from rake it is the location of the Rakefile.
+    # @param measures_dir [string] The current working directory. If called from Rake it is the active directory.
+    def pre_process_style
+      puts "Current directory is #{@base_dir}"
+      puts "Test results will be stored in: #{test_results_dir}"
+
+      FileUtils.rm_rf "#{test_results_dir}/openstudio_style" if Dir.exist? "#{test_results_dir}/openstudio_style"
+      FileUtils.rm_rf "#{@base_dir}/openstudio_style" if Dir.exist? "#{@base_dir}/openstudio_style"
+
+      # Call the test_results dir to create the directory and return it as a string.
+      return test_results_dir
+    end
+
+    def run_style(skip_post_process)
+      puts " ========================= Starting Run for OpenStudio Style ================================"
+      pre_process_style
+
+      # Run the style tests
+      style = OpenStudioMeasureTester::OpenStudioStyle.new(test_results_dir, "#{@base_dir}/**/measure.rb")
+      style.save_results
+
+      if skip_post_process
+        return true
+      else
+        return post_process_results
+      end
+    end
+
+    def run_rubocop(skip_post_process, auto_correct = false)
+      puts " ========================= Starting Run for Rubocop ================================"
+      pre_process_rubocop
+
+      rubocop_results_file = "#{test_results_dir}/rubocop/rubocop-results.xml"
+      # The documentation on running RuboCop from the Ruby source is not really helpful. I reversed engineered this
+      # by putting in puts statements in my local install rubocop gem to see how options were passed.
+      #
+      # https://github.com/bbatsov/rubocop/blob/9bdbaba9dcaa3dedad5e857b440d0d8988b806f5/lib/rubocop/runner.rb#L25
+      require 'rubocop/formatter/checkstyle_formatter'
+      options = {
+        # out and output_path do not actually save the results, has to be appended after the formatter.
+        # out: 'junk.xml',
+        # output_path: 'junk.xml',
+        auto_correct: auto_correct,
+        color: false,
+        formatters: ['simple', ['RuboCop::Formatter::CheckstyleFormatter', rubocop_results_file]]
+      }
+
+      # Load in the ruby config from the root directory
+      config_store = RuboCop::ConfigStore.new
+      # puts "Searching for .rubocop.yml recursively from #{@base_dir}"
+      config_store.for(@base_dir)
+
+      rc_runner = RuboCop::Runner.new(options, config_store)
+      rc_runner.run(["#{File.expand_path(@base_dir)}/**/*.rb"])
+
+      if skip_post_process
+        return true
+      else
+        return post_process_results
+      end
+    end
+
+    # The results of the coverage and minitest are stored in the root of the directory structure (if Rake)
+    def run_test(skip_post_process, original_results_directory, run_coverage = true)
+      puts " ========================= Starting Run for Minitest (and coverage) ============================"
+      # not sure what @base_dir has to be right now
+      pre_process_minitest(original_results_directory)
+
+      # Specify the minitest reporters
+      require 'minitest/reporters'
+      Minitest::Reporters.use! [
+        Minitest::Reporters::HtmlReporter.new,
+        Minitest::Reporters::JUnitReporter.new
+      ]
+
+      if run_coverage
+        # Load in the coverage before loading the test files
+        SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new(
+          [
+            SimpleCov::Formatter::HTMLFormatter
+          ]
+        )
+
+        SimpleCov.start do
+          # Track all files inside of @base_dir
+          track_files "#{@base_dir}/**/*.rb"
+
+          use_merging false
+
+          # Exclude all files outside of @base_dir
+          root_filter = nil
+          add_filter do |src|
+            root_filter ||= /\A#{Regexp.escape(@base_dir + File::SEPARATOR)}/io
+            src.filename !~ root_filter
+          end
+        end
+      end
+
+      num_tests = 0
+      Dir["#{@base_dir}/**/*_Test.rb", "#{@base_dir}/**/*_test.rb"].uniq.each do |file|
+        puts "Loading file for testing: #{file}"
+        load File.expand_path(file)
+        num_tests += 1
+      end
+
+      if num_tests < 1
+        puts 'No tests found'
+
+        if run_coverage
+          # This doesn't seem to be working, it doesn't save off the .resultset.json.
+          begin
+            simplecov_exit_status = SimpleCov.end_now
+          rescue NoMethodError
+            # in case using some other version of SimpleCov
+            SimpleCov.set_exit_exception
+            exit_status = SimpleCov.exit_status_from_exception
+            SimpleCov.result.format!
+            simplecov_exit_status = SimpleCov.process_result(SimpleCov.result, exit_status)
+          end
+        end
+
+        if skip_post_process
+          return true
+        else
+          return post_process_results(original_results_directory)
+        end
+      else
+        puts "Inspected #{num_tests} tests"
+      end
+
+      # Now call run on the loaded files. Note that the Minitest.autorun method has been nulled out in the
+      # openstudio_measure_tester.rb file, so it will not run.
+      begin
+        Minitest.run ['--verbose']
+      rescue StandardError => exception
+        puts
+        puts '!!!!!!!!!!!!!!!!!!!!! Minitest Error Occurred !!!!!!!!!!!!!!!!!!!!!'
+        puts exception.message
+        puts exception.backtrace
+        puts '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
+        puts
+      end
+
+      # Shutdown SimpleCov and collect results
+      # This will set SimpleCov.running to false which will prevent from running again at_exit
+      if run_coverage
+        begin
+          simplecov_exit_status = SimpleCov.end_now
+        rescue NoMethodError
+          # in case using some other version of SimpleCov
+          SimpleCov.set_exit_exception
+          exit_status = SimpleCov.exit_status_from_exception
+          SimpleCov.result.format!
+          simplecov_exit_status = SimpleCov.process_result(SimpleCov.result, exit_status)
+        end
+      end
+
+      if skip_post_process
+        return true
+      else
+        return post_process_results(original_results_directory)
+      end
+    end
+
+    def run_all(original_results_directory)
+      # do not run coverage now since the at_exit is causing exceptions when running (GC?)
+      run_rubocop(true)
+      run_style(true)
+      run_test(true, original_results_directory, false)
+      post_process_results(original_results_directory)
+    end
+  end
+end

data/lib/openstudio_measure_tester/version.rb

@@ -27,5 +27,5 @@
 ########################################################################################################################
 
 module OpenStudioMeasureTester
-  VERSION = '0.1.4'.freeze
+  VERSION = '0.1.5'.freeze
 end

data/openstudio_measure_tester.gemspec

@@ -6,7 +6,7 @@ require 'openstudio_measure_tester/version'
 Gem::Specification.new do |spec|
   spec.name          = 'openstudio_measure_tester'
   spec.version       = OpenStudioMeasureTester::VERSION
-  spec.authors       = ['Nicholas Long']
+  spec.authors       = ['Nicholas Long', 'Katherine Fleming', 'Daniel Macumber', 'Robert Guglielmetti']
   spec.email         = ['nicholas.long@nrel.gov']
 
   spec.summary       = 'Testing framework for OpenStudio measures'
@@ -20,11 +20,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '1.16.1'
+  spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'rspec', '3.7.0'
 
-  spec.add_dependency 'activesupport', '5.1.6'
-  spec.add_dependency 'ci_reporter_minitest', '1.0.0'
   spec.add_dependency 'git', '1.3.0'
   spec.add_dependency 'minitest', '5.4.3'
   spec.add_dependency 'minitest-reporters', '1.2.0'

metadata

@@ -1,29 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: openstudio_measure_tester
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Nicholas Long
+- Katherine Fleming
+- Daniel Macumber
+- Robert Guglielmetti
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-19 00:00:00.000000000 Z
+date: 2018-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.16.1
+        version: '1.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.16.1
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -38,34 +41,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.7.0
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 5.1.6
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 5.1.6
-- !ruby/object:Gem::Dependency
-  name: ci_reporter_minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
@@ -173,6 +148,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop-http---s3-amazonaws-com-openstudio-resources-styles-rubocop-yml"
 - ".rubocop.yml"
 - ".travis.yml"
 - CHANGELOG.md
@@ -181,6 +157,7 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/run_measure_tests
 - bin/setup
 - dashboard/css/bootstrap-grid.css
 - dashboard/css/bootstrap-grid.min.css
@@ -201,8 +178,8 @@ files:
 - lib/openstudio_measure_tester/openstudio_testing_result.rb
 - lib/openstudio_measure_tester/rake_task.rb
 - lib/openstudio_measure_tester/rubocop_result.rb
+- lib/openstudio_measure_tester/runner.rb
 - lib/openstudio_measure_tester/templates/dashboard.html.erb
-- lib/openstudio_measure_tester/test_helper.rb
 - lib/openstudio_measure_tester/version.rb
 - openstudio_measure_tester.gemspec
 homepage: https://openstudio.nrel.gov

data/lib/openstudio_measure_tester/test_helper.rb

@@ -1,43 +0,0 @@
-########################################################################################################################
-#  OpenStudio(R), Copyright (c) 2008-2018, Alliance for Sustainable Energy, LLC. All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
-#  following conditions are met:
-#
-#  (1) Redistributions of source code must retain the above copyright notice, this list of conditions and the following
-#  disclaimer.
-#
-#  (2) Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
-#  following disclaimer in the documentation and/or other materials provided with the distribution.
-#
-#  (3) Neither the name of the copyright holder nor the names of any contributors may be used to endorse or promote
-#  products derived from this software without specific prior written permission from the respective party.
-#
-#  (4) Other than as required in clauses (1) and (2), distributions in any form of modifications or other derivative
-#  works may not use the "OpenStudio" trademark, "OS", "os", or any other confusingly similar designation without
-#  specific prior written permission from Alliance for Sustainable Energy, LLC.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
-#  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER, THE UNITED STATES GOVERNMENT, OR ANY CONTRIBUTORS BE LIABLE FOR
-#  ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-#  AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-########################################################################################################################
-
-$coverage_started = false
-unless $coverage_started
-  require 'simplecov'
-  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new(
-    [
-      SimpleCov::Formatter::HTMLFormatter
-    ]
-  )
-
-  SimpleCov.start
-  $coverage_started = true
-end
-
-require 'minitest/reporters'
-Minitest::Reporters.use! [Minitest::Reporters::SpecReporter.new, Minitest::Reporters::HtmlReporter.new]