openstudio_measure_tester 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c701b7ee1c908a30adce739d1ef8e30a1b804b1
-  data.tar.gz: 2520f8964fdeb8ea55df34d743676c56732dca46
+  metadata.gz: 114965f771ab5429ff6e0a47f9469143723f957a
+  data.tar.gz: 6880af89a1126f2474f503722ce73746074bf04d
 SHA512:
-  metadata.gz: c419ccadea649d88baef183cb66780178dfb3b3a70be6077c08c7f1d7ca8a95657105f1a6a9ae4444e1ffc189df5c9d74ce0d3abc1f03f01783849476dde7369
-  data.tar.gz: 7ec95e37072d0f1af63b8435d1eb283a2a9eb387a02ba0b0e544fc9f460a1ad4c177e618754caedf12d89975966766f046eb3c299f38c21cb7ab1ca6a506db81
+  metadata.gz: 65c48c6e8ebe186696f5a2322dde5899e84bacfadd95510b75cf03e8982bee6137f9ea4d2e36fabb7b103e9211cedb8d228c70ffe6a7f17cc16fadd850299fa2
+  data.tar.gz: aeb836feeafb27e169a38e1569b0f2fe606ba8df68d8133dd2c29627df23b5084d70f8245af706c09aba56621342b30fcb1192477bf2fe37887cc9e162090688

data/.gitignore

@@ -18,7 +18,7 @@
 rspec.html
 Gemfile.lock
 **/.rubocop-http*
-rubocop-results.xml
+/rubocop-results.xml
 !spec/files/rubocop-results.xml
 spec/files/rubocop/rubocop.json
 spec/**/.rubocop.yml

data/.travis.yml

@@ -1,21 +1,15 @@
 sudo: true
 language: ruby
 rvm:
-- 2.2.4
+  - 2.2.4
 matrix:
   include:
   - env: OPENSTUDIO_VERSION=2.4.0 && OPENSTUDIO_SHA=f58a3e1808 && RUBYLIB=/usr/Ruby
   - env: OPENSTUDIO_VERSION=2.5.1 && OPENSTUDIO_SHA=4f268e2854 && RUBYLIB=/usr/Ruby
 before_install: gem install bundler -v 1.16.2
 before_script:
-- curl -sLO https://raw.githubusercontent.com/NREL/OpenStudio-server/develop/docker/deployment/scripts/install_openstudio.sh
-- chmod +x install_openstudio.sh
-- sudo ./install_openstudio.sh $OPENSTUDIO_VERSION $OPENSTUDIO_SHA
+  - curl -sLO https://raw.githubusercontent.com/NREL/OpenStudio-server/develop/docker/deployment/scripts/install_openstudio.sh
+  - chmod +x install_openstudio.sh
+  - sudo ./install_openstudio.sh $OPENSTUDIO_VERSION $OPENSTUDIO_SHA
 script:
-- bundle exec rake
-deploy:
-  - provider: rubygems
-    api_key:
-      secure: IOYqPDXgaX5tZgsaeuSs2Q5hUry0l2RHrXDmuEeu7SElLDQSXS2z8c43eHg6GphtIho9NZgonXeCe1QoXb/GdHsyv2Up1XfaTtPkf7LHJo/zYPF7Yjc1mjGpKRPfk1srH8nfb6w9w7zjFlhk73a/Vh26BvdRCXqAA8W72d0ur2p8OTlGSYjcgLihWI13sSKrfHjTRXhOpv6gkziaY2pmvNWeeYdqOD3CpvlMDAMkHY/KptksaqWdBAT9Jr4gIfAGeGbKzye0OmBBrwMV4n/4KZqNNDyjiGYGoSbOhVEsAOHbeJz6IOoC93ICQi/nLuWBw+T8do79X2/GBIj2/N7eaEo4CcYsxMS0JX+dE4yL+OzQOFkDenq0WAEB7JjtBVW3sBHAIRGO2o2Cma6Ga5+86oSD0f5lAvnhiCejA1Fy9X5pnIMsk1zWlPLydPX4TwIm10JBaRLut+MFq0tL5tJunyIwlZnMgVofVdKYZHS7fVOvqSTQhM+iA0Gx5l7bKZQjpO3m91j14pKqdJij0PJ4oxy7ozc38YSUL85NKXnbf2KwnYVxMGz4yBrSalzMjftu6WfCM8liDTb0ACHby56KbPrlVlixJrrJIefZCLqaHtqL2lLlyVV36EtRbNl3It7DrOQ21HK3raOEuSxht3uuTu9FIdTx0TKraJUyToE6gbw=
-    on:
-      tags: true
+  - bundle exec rake

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# Version 0.1.6
+
+* Do not require git to run
+* Change name of compatibility JSON to the measure name (instead of file name)
+
 # Version 0.1.5
 
 * Run Minitest in foreground

data/README.md

@@ -43,26 +43,26 @@ The OpenStudio Measure Tester is a rubygem that exposes rake tasks for testing O
 
 * Require the OpenStudio-measure-tester gem
 
-```ruby
-require 'openstudio_measure_tester'
-measures_dir = 'spec/test_measures/AddOverhangsByProjectionFactor'
-# all measures (recursively) from measures_dir will be tested
-  
-runner = OpenStudioMeasureTester::Runner.new(measures_dir)
+   ```ruby
+   require 'openstudio_measure_tester'
+   measures_dir = 'spec/test_measures/AddOverhangsByProjectionFactor'
+   # all measures (recursively) from measures_dir will be tested
 
-# base_dir is needed for coverage results as they are written to disk on the at_exit calls
-base_dir = Dir.pwd
- 
-result = runner.run_all(base_dir)
-puts result
-# result will be 0 or 1, 0=success, 1=failure
+   runner = OpenStudioMeasureTester::Runner.new(measures_dir)
 
-runner.run_style(false)
+   # base_dir is needed for coverage results as they are written to disk on the at_exit calls
+   base_dir = Dir.pwd
 
-runner.run_test(false, base_dir)
+   result = runner.run_all(base_dir)
+   puts result
+   # result will be 0 or 1, 0=success, 1=failure
 
-runner.run_rubocop(false)
-```
+   runner.run_style(false)
+
+   runner.run_test(false, base_dir)
+
+   runner.run_rubocop(false)
+   ```
 
 * Results will be saved into the run directory (measures_dir from above).    
 
@@ -85,5 +85,8 @@ export RUBYLIB=/usr/Ruby
 
 # Releasing
 
-New versions of the OpenStudio Measure Tester Gem will be automatically released for any tags. Makes sure to increment 
-the version in `/lib/openstudio_measure_tester/version.rb` and tag the release (preferably off the master branch).
+* Update change log
+* Update version in `/lib/openstudio_measure_tester/version.rb`
+* Merge down to master
+* Release via github
+* run `rake release` from master  

data/dashboard/css/dashboard.css

@@ -107,7 +107,7 @@ body {
 .donut .status {
   display: block;
   position: absolute;
-  top: 50%;
+  top: 43%;
   left: 50%;
   width: 30%;
   height: 0;
@@ -125,7 +125,7 @@ body {
 .donut .adjust-pos {
 
   padding-top: 10%;
-  margin-top: -10%;
+  margin-top: -20%;
 
 }
 

data/dashboard/index.html

@@ -33,22 +33,14 @@
                 <div class="card-block">
                   <div class="card-body">
                     <h5 class="card-title">Tests</h5>
-             
-                    
                       <div class="row pt-5">
-                      
                         <div class="col-sm-4 offset-sm-1 badge-height badge badge-success"><span class='large-text'>0</span></div>
-                      
-                      
                         <div class="col-sm-4 offset-sm-2 badge-height badge badge-success"><span class='large-text'>0</span></div>
-                      
                       </div>
                       <div class="row">
                         <div class="col-sm-4 offset-sm-1 center-text"><span class="badge-text">Failures</span></div>
                         <div class="col-sm-4 offset-sm-2 center-text"><span class="badge-text">Errors</span></div>
                       </div>
-                    
-                 
                   </div>
                 </div>
               </div>
@@ -59,16 +51,12 @@
                   <div class="card-body">
                     <h5 class="card-title">OpenStudio Style</h5>
                     <div>
-                      
                         <div class="row pt-5">
-                        
                           <div class="col-sm-4 offset-sm-4 badge-height badge badge-danger"><span class='large-text'>34</span></div>
-                        
                         </div>
                         <div class="row">
                           <div class="col-sm-4 offset-sm-4 center-text"><span class="badge-text">Errors</span></div>
                         </div>
-                      
                     </div>
                   </div>
                 </div>
@@ -115,16 +103,11 @@
                   <tr>
                     <td>ChangeBuildingLocation</td>
                     <td>  
-                      
                         <span class="badge badge-danger">0</span>
                         <span class="badge badge-warning">0</span> 
-                      
                     </td>
                     <td>
-                          
-                            <span class="badge badge-danger">6</span>
-                          
-                          
+                        <span class="badge badge-danger">6</span>
                     </td>
                     <td></td>
                     <td>79.69 %</td>
@@ -552,7 +535,6 @@
       size(true);
 
       }());
-
     </script>
   </body>
 </html>

data/lib/openstudio_measure_tester.rb

@@ -29,11 +29,16 @@
 require 'openstudio'
 
 require 'pp'
-require 'git'
 require 'rexml/document'
 require 'minitest'
 require 'simplecov'
 
+begin
+require 'git'
+rescue LoadError => error
+  puts 'Could not load git, will not be able to report git information'
+end
+
 # override the default at_exit call
 SimpleCov.at_exit do
 end
@@ -44,17 +49,17 @@ def Minitest.autorun; end
 # Rubocop loads a lot of objects, anyway to minimize would be nice.
 require 'rubocop'
 
-require 'openstudio_measure_tester/core_ext'
-require 'openstudio_measure_tester/version'
-require 'openstudio_measure_tester/openstudio_style'
-require 'openstudio_measure_tester/minitest_result'
-require 'openstudio_measure_tester/coverage'
-require 'openstudio_measure_tester/rubocop_result'
-require 'openstudio_measure_tester/openstudio_testing_result'
-require 'openstudio_measure_tester/dashboard'
-require 'openstudio_measure_tester/runner'
+require_relative 'openstudio_measure_tester/core_ext'
+require_relative 'openstudio_measure_tester/version'
+require_relative 'openstudio_measure_tester/openstudio_style'
+require_relative 'openstudio_measure_tester/minitest_result'
+require_relative 'openstudio_measure_tester/coverage'
+require_relative 'openstudio_measure_tester/rubocop_result'
+require_relative 'openstudio_measure_tester/openstudio_testing_result'
+require_relative 'openstudio_measure_tester/dashboard'
+require_relative 'openstudio_measure_tester/runner'
 
-require 'openstudio_measure_tester/rake_task'
+require_relative 'openstudio_measure_tester/rake_task'
 
 # Set the encoding to UTF-8. OpenStudio Docker images do not have this set by default
 Encoding.default_external = Encoding::UTF_8
@@ -64,15 +69,13 @@ module OpenStudioMeasureTester
   # No action here. Most of this will be rake_tasks at the moment.
 end
 
-
 class Minitest::Test
-	def teardown
-		before = ObjectSpace.count_objects
-    	GC.start
-    	after = ObjectSpace.count_objects
-    	delta = {}
-    	before.each {|k, v| delta[k] = v - after[k] if after.has_key? k }
-    	puts "GC Delta: #{delta}"
-	end
+  def teardown
+    before = ObjectSpace.count_objects
+    GC.start
+    after = ObjectSpace.count_objects
+    delta = {}
+    before.each { |k, v| delta[k] = v - after[k] if after.key? k }
+    puts "GC Delta: #{delta}"
+  end
 end
-

data/lib/openstudio_measure_tester/minitest_result.rb

@@ -35,6 +35,7 @@ module OpenStudioMeasureTester
     attr_reader :total_failures
     attr_reader :total_skipped
     attr_reader :total_tests
+    attr_reader :total_compatibility_errors
     attr_reader :measure_results
     attr_reader :summary
 
@@ -46,6 +47,9 @@ module OpenStudioMeasureTester
       @total_errors = 0
       @total_failures = 0
       @total_skipped = 0
+      @total_compatibility_errors = 0
+      @total_loaded = true
+      @total_load_errors = []
 
       @measure_results = {}
       @summary = {}
@@ -55,37 +59,76 @@ module OpenStudioMeasureTester
     end
 
     def parse_results
-      Dir["#{@path_to_results}/reports/*.xml"].each do |file|
-        puts "Parsing minitest report #{file}"
-        doc = REXML::Document.new(File.open(file)).root
+      # use the compatibility file to find the measure name and other files
+      Dir["#{@path_to_results}/compatibility/*.json"].each do |file|
+        puts "Parsing compatibility report #{file}"
+        json_data = JSON.parse(File.read(file), symbolize_names: true)
 
-        # continue if doc is empty
-        next unless doc
-
-        measure_name = file.split('/')[-1].split('.')[0].split('-')[1].gsub /-?[tT]est\z/, ''
+        # Test if the measure has already been parse, if so, then continue
+        next if @measure_results.keys.include? json_data[:measure_name]
 
         mhash = {}
-        mhash['tested_class'] = measure_name
-
-        # Note: only 1 failure and 1 error possible per test
-        testsuite_element = doc.elements['testsuite']
-        errors, failures = parse_measure(testsuite_element)
-
-        mhash['measure_tests'] = testsuite_element.attributes['tests'].to_i
-        mhash['measure_assertions'] = testsuite_element.attributes['assertions'].to_i
-        mhash['measure_errors'] = testsuite_element.attributes['errors'].to_i
-        mhash['measure_failures'] = testsuite_element.attributes['failures'].to_i
-        mhash['measure_skipped'] = testsuite_element.attributes['skipped'].to_i
+        mhash[:tested_class] = json_data[:measure_name]
+        mhash[:openstudio_version] = json_data[:openstudio_version]
+        mhash[:measure_min_version] = json_data[:measure_min_version]
+        mhash[:measure_max_version] = json_data[:measure_max_version]
+        mhash[:loaded] = json_data[:loaded]
+        if json_data[:load_errors].nil?
+          mhash[:load_errors] = []
+        else
+          mhash[:load_errors] = json_data[:load_errors]
+        end
 
-        mhash['issues'] = { errors: errors, failures: failures }
+        # initialize a bunch of data
+        mhash[:measure_compatibility_errors] = json_data[:compatible] ? 0 : 1
+        mhash[:measure_tests] = 0
+        mhash[:measure_assertions] = 0
+        mhash[:measure_errors] = 0
+        mhash[:measure_failures] = 0
+        mhash[:measure_skipped] = 0
+        mhash[:issues] = {
+            errors: [],
+            failures: [],
+            skipped: [],
+            compatibility_error: json_data[:compatible] ? 0 : 1
+        }
+
+        # find the report XML - if it exists
+        report_xmls = Dir["#{@path_to_results}/reports/TEST-#{json_data[:measure_name]}*.xml"]
+        if report_xmls.count == 1
+          puts "Parsing minitest report #{report_xmls[0]}"
+          doc = REXML::Document.new(File.open(report_xmls[0])).root
+
+          if doc
+            # Note: only 1 failure and 1 error possible per test
+            testsuite_element = doc.elements['testsuite']
+            errors, failures, skipped = parse_measure(testsuite_element)
+
+            mhash[:measure_tests] = testsuite_element.attributes['tests'].to_i
+            mhash[:measure_assertions] = testsuite_element.attributes['assertions'].to_i
+            mhash[:measure_errors] = testsuite_element.attributes['errors'].to_i
+            mhash[:measure_failures] = testsuite_element.attributes['failures'].to_i
+            mhash[:measure_skipped] = testsuite_element.attributes['skipped'].to_i
+
+            mhash[:issues][:errors] = errors
+            mhash[:issues][:failures] = failures
+            mhash[:issues][:skipped] = skipped
+          end
+        else
+          # There are more than one XMLs or there are no XML
+          # No XMLs is typically because the measure was not applicable to then version of OpenStudio
+        end
 
-        @measure_results[measure_name] = mhash
+        @measure_results[mhash[:tested_class]] = mhash
 
-        @total_tests += mhash['measure_tests']
-        @total_assertions += mhash['measure_assertions']
-        @total_errors += mhash['measure_errors']
-        @total_failures += mhash['measure_failures']
-        @total_skipped += mhash['measure_skipped']
+        @total_tests += mhash[:measure_tests]
+        @total_assertions += mhash[:measure_assertions]
+        @total_errors += mhash[:measure_errors]
+        @total_failures += mhash[:measure_failures]
+        @total_skipped += mhash[:measure_skipped]
+        @total_compatibility_errors += mhash[:measure_compatibility_errors]
+        @total_loaded = @total_loaded && mhash[:loaded]
+        @total_load_errors.concat(mhash[:load_errors])
       end
 
       @error_status = true if @total_errors > 0
@@ -94,14 +137,17 @@ module OpenStudioMeasureTester
     def to_file
       # save as a json and have something else parse it/plot it.
 
-      @summary['test_directory'] = @path_to_results
-      @summary['total_tests'] = @total_tests
-      @summary['total_assertions'] = @total_assertions
-      @summary['total_errors'] = @total_errors
-      @summary['total_failures'] = @total_failures
-      @summary['total_skipped'] = @total_skipped
-      @summary['by_measure'] = @measure_results
-
+      @summary[:test_directory] = @path_to_results
+      @summary[:total_tests] = @total_tests
+      @summary[:total_assertions] = @total_assertions
+      @summary[:total_errors] = @total_errors
+      @summary[:total_failures] = @total_failures
+      @summary[:total_skipped] = @total_skipped
+      @summary[:total_compatibility_errors] = @total_compatibility_errors
+      @summary[:total_loaded] = @total_loaded
+      @summary[:total_load_errors] = @total_load_errors
+      @summary[:by_measure] = @measure_results
+      
       # pp @summary
 
       FileUtils.mkdir "#{@path_to_results}/" unless Dir.exist? "#{@path_to_results}/"
@@ -115,16 +161,19 @@ module OpenStudioMeasureTester
     def parse_measure(testsuite_element)
       errors = []
       failures = []
+      skipped = []
 
       testsuite_element.elements.each('testcase') do |testcase|
         if testcase.elements['error']
           errors << testcase.elements['error']
         elsif testcase.elements['failure']
           failures << testcase.elements['failure']
+        elsif testcase.elements['skipped']
+          skipped << 'Skipped test: ' + testcase.elements['skipped'].attributes['type']
         end
       end
 
-      return errors, failures
+      return errors, failures, skipped
     end
   end
 end

data/lib/openstudio_measure_tester/openstudio_style.rb

@@ -39,6 +39,20 @@ module OpenStudioMeasureTester
         type: :deprecated,
         severity: :error,
         file_type: :measure
+      }, {
+        regex: /OpenStudio::Ruleset::WorkspaceUserScript/,
+        check_type: :if_exists,
+        message: 'OpenStudio::Ruleset::WorkspaceUserScript is deprecated, use OpenStudio::Measure::EnergyPlusMeasure instead.',
+        type: :deprecated,
+        severity: :error,
+        file_type: :measure
+      }, {
+        regex: /OpenStudio::Ruleset::ReportingUserScript/,
+        check_type: :if_exists,
+        message: 'OpenStudio::Ruleset::ReportingUserScript is deprecated, use OpenStudio::Measure::ReportingMeasure instead.',
+        type: :deprecated,
+        severity: :error,
+        file_type: :measure
       }, {
         regex: /OpenStudio::Ruleset::OSRunner/,
         check_type: :if_exists,
@@ -154,7 +168,7 @@ module OpenStudioMeasureTester
             measure_info = infoExtractor(measure, OpenStudio::Model::OptionalModel.new, OpenStudio::OptionalWorkspace.new)
           rescue NameError => error
             log_message("Unable to parse info from measure. Error: '#{error}'", :general, :error)
-          rescue => error
+          rescue StandardError => error
             log_message("Unknown error extracting measure info. Error #{error}", :general, :error)
           end
 
@@ -397,7 +411,6 @@ module OpenStudioMeasureTester
         result[:version_modified] = nil
       end
       result[:xml_checksum] = measure.xmlChecksum
-      result[:name] = measure.name
       result[:display_name] = measure.displayName
       result[:class_name] = measure.className
       result[:description] = measure.description

data/lib/openstudio_measure_tester/openstudio_testing_result.rb

@@ -52,7 +52,7 @@ module OpenStudioMeasureTester
         g = Git.open(Dir.pwd)
         config = g.config
         repo_name = config['remote.origin.url'] ? config['remote.origin.url'].split('/').last.chomp('.git') : nil
-        current_branch = g.branch.name ? g.branch.name : nil
+        current_branch = g.branch.name || nil
         logs = g.log
         sha = nil
         if !logs.empty?
@@ -79,7 +79,7 @@ module OpenStudioMeasureTester
       # the @results hash
       filename = "#{@test_results_dir}/openstudio_style/openstudio_style.json"
       if File.exist? filename
-        puts "Found OpenStudio Style results, parsing"
+        puts 'Found OpenStudio Style results, parsing'
         @results['openstudio_style'] = JSON.parse(File.read(filename))
       end
 
@@ -96,7 +96,7 @@ module OpenStudioMeasureTester
       if @test_results_dir != @orig_results_dir
         # coverage
         if Dir.exist? "#{@orig_results_dir}/coverage"
-          puts "Found Coverage results, parsing"
+          puts 'Found Coverage results, parsing'
           FileUtils.rm_rf "#{@test_results_dir}/coverage" if Dir.exist? "#{@test_results_dir}/coverage"
           FileUtils.cp_r "#{@orig_results_dir}/coverage/.", "#{@test_results_dir}/coverage"
           FileUtils.rm_rf "#{@orig_results_dir}/coverage" if Dir.exist? "#{@orig_results_dir}/coverage"
@@ -108,18 +108,21 @@ module OpenStudioMeasureTester
 
         # minitest
         if Dir.exist?("#{@orig_results_dir}/test/html_reports") || Dir.exist?("#{@orig_results_dir}/test/reports")
-          puts "Found Minitest Results, parsing"
-          FileUtils.rm_rf "#{@test_results_dir}/minitest" if Dir.exist? "#{@test_results_dir}/minitest"
+          puts 'Found Minitest Results, parsing'
+          # Do not delete the compatibilty directory which is generated when the test is run
+          FileUtils.rm_rf "#{@test_results_dir}/minitest/html_reports" if Dir.exist? "#{@test_results_dir}/minitest/html_reports"
+          FileUtils.rm_rf "#{@test_results_dir}/minitest/reports" if Dir.exist? "#{@test_results_dir}/minitest/reports"
+
           FileUtils.mkdir_p "#{@test_results_dir}/minitest"
 
           # Copy the files over in case the folder is locked.
           if Dir.exist?("#{@orig_results_dir}/test/html_reports")
-            puts "Moving Minitest HTML results to results directory"
+            puts 'Moving Minitest HTML results to results directory'
             FileUtils.cp_r "#{@orig_results_dir}/test/html_reports/.", "#{@test_results_dir}/minitest/html_reports"
           end
 
           if Dir.exist?("#{@orig_results_dir}/test/reports")
-            puts "Moving Minitest XML results to results directory"
+            puts 'Moving Minitest XML results to results directory'
             FileUtils.cp_r "#{@orig_results_dir}/test/reports/.", "#{@test_results_dir}/minitest/reports"
           end
 
@@ -178,7 +181,7 @@ module OpenStudioMeasureTester
       end
 
       if @results['minitest']
-        if @results['minitest']['total_errors'] > 0 || @results['minitest']['total_failures'] > 0
+        if @results['minitest'][:total_errors] > 0 || @results['minitest'][:total_failures] > 0
           puts 'Unit Test (Minitest) errors/failures found.'
           final_exit_code = 1
         end

data/lib/openstudio_measure_tester/runner.rb

@@ -79,7 +79,7 @@ module OpenStudioMeasureTester
     #
     # @param original_results_directory [string] Location of the results from coverag and minitest
     def post_process_results(original_results_directory = nil)
-      puts " ========================= Starting Results Post Process ================================"
+      puts ' ========================= Starting Results Post Process ================================'
       puts "Current directory: #{@base_dir}"
       puts "Test results will be stored in: #{test_results_dir}"
 
@@ -109,7 +109,7 @@ module OpenStudioMeasureTester
     end
 
     def run_style(skip_post_process)
-      puts " ========================= Starting Run for OpenStudio Style ================================"
+      puts ' ========================= Starting Run for OpenStudio Style ================================'
       pre_process_style
 
       # Run the style tests
@@ -124,7 +124,7 @@ module OpenStudioMeasureTester
     end
 
     def run_rubocop(skip_post_process, auto_correct = false)
-      puts " ========================= Starting Run for Rubocop ================================"
+      puts ' ========================= Starting Run for Rubocop ================================'
       pre_process_rubocop
 
       rubocop_results_file = "#{test_results_dir}/rubocop/rubocop-results.xml"
@@ -159,7 +159,7 @@ module OpenStudioMeasureTester
 
     # The results of the coverage and minitest are stored in the root of the directory structure (if Rake)
     def run_test(skip_post_process, original_results_directory, run_coverage = true)
-      puts " ========================= Starting Run for Minitest (and coverage) ============================"
+      puts ' ========================= Starting Run for Minitest (and coverage) ============================'
       # not sure what @base_dir has to be right now
       pre_process_minitest(original_results_directory)
 
@@ -194,10 +194,76 @@ module OpenStudioMeasureTester
       end
 
       num_tests = 0
+      openstudio_version = OpenStudio::VersionString.new(OpenStudio.openStudioVersion)
       Dir["#{@base_dir}/**/*_Test.rb", "#{@base_dir}/**/*_test.rb"].uniq.each do |file|
-        puts "Loading file for testing: #{file}"
-        load File.expand_path(file)
-        num_tests += 1
+      
+        begin
+          file = File.expand_path(file)
+          measure_dir = File.expand_path(File.join(File.dirname(file), '..'))
+
+          # check measure xml
+          compatible = {
+            compatible: true,
+            message: '',
+            openstudio_version: openstudio_version.str,
+            measure_min_version: 'None',
+            measure_max_version: 'None', 
+            loaded: false,
+            load_errors: []
+          }
+          begin
+            measure = OpenStudio::BCLMeasure.new(measure_dir)
+            compatible[:measure_name] = measure.className
+            measure.files.each do |f|
+              if f.fileName == 'measure.rb'
+                if !f.minCompatibleVersion.empty?
+                  min_version = f.minCompatibleVersion.get
+                  compatible[:measure_min_version] = min_version.str
+                  if openstudio_version < min_version
+                    compatible[:compatible] = false
+                    compatible[:message] = "OpenStudio Version #{openstudio_version.str} < Min Version #{min_version.str}"
+                  end
+                end
+                if !f.maxCompatibleVersion.empty?
+                  max_version = f.maxCompatibleVersion.get
+                  compatible[:measure_max_version] = max_version.str
+                  if openstudio_version > max_version
+                    compatible[:compatible] = false
+                    compatible[:message] = "OpenStudio Version #{openstudio_version.str} > Max Version #{max_version.str}"
+                  end
+                end
+              end
+            end
+          rescue StandardError => exception
+            compatible[:compatible] = false
+            compatible[:message] = exception.message
+          end
+
+          if !compatible[:compatible]
+            puts "Measure not compatible: #{measure_dir}, #{compatible[:message]}"
+            next
+          end
+
+          # load test
+          puts "Loading file for testing: #{file}"
+          begin
+            load file
+            compatible[:loaded] = true
+            num_tests += 1
+          rescue StandardError, LoadError => exception
+            compatible[:load_errors] << exception.message
+          end
+          
+        ensure
+          # Write out the compatibility
+          # write out to a file that the measure is not applicable
+          os_compatible_file = "#{@base_dir}/test_results/minitest/compatibility/#{compatible[:measure_name]}.json"
+          puts os_compatible_file
+          FileUtils.mkdir_p File.dirname(os_compatible_file) unless Dir.exists? File.dirname(os_compatible_file)
+          File.open(os_compatible_file, 'w') do |f|
+            f << JSON.pretty_generate(compatible)
+          end
+        end
       end
 
       if num_tests < 1

data/lib/openstudio_measure_tester/templates/dashboard.html.erb

@@ -39,7 +39,23 @@
                 <div class="card-block">
                   <div class="card-body">
                     <h5 class="card-title">Tests</h5>
-             
+                    <% if @hash['minitest'] && @hash['minitest']['total_load_errors'] %>
+                      <div class="row pt-3">
+                        <% total_load_errors = @hash['minitest']['total_load_errors'].count %>
+                        <% if total_load_errors > 0 %>
+                          <div class="col-sm-3 offset-sm-2 badge-med-height badge badge-danger"><span class='med-text'><%= total_load_errors %></span></div>
+                        <% else %>
+                          <div class="col-sm-3 offset-sm-2 badge-med-height badge badge-success"><span class='med-text'><%= total_load_errors %></span></div>
+                        <% end %>
+                        <div class="col-sm-3 badge-med-height badge-text">Load Errors</div>
+                      </div>
+                    <% else %>
+                      <div class="row pt-3">
+                        <div class="col-sm-6 offset-sm-3 badge-lg-height badge badge-warning badge-text"><i data-feather="alert-triangle" class="size48"></i><br><span class="pt-2">No Data</span> </div>
+                        <div class="col-sm-3 badge-med-height badge-text">Load Errors</div>
+                      </div>
+                    <% end %>
+                    
                     <% if @hash['minitest'] && @hash['minitest']['total_failures'] %>
                       <div class="row pt-3">
                         <% passed = @hash['minitest']['total_tests'] - (@hash['minitest']['total_failures'] + @hash['minitest']['total_errors'] + @hash['minitest']['total_skipped']) %>
@@ -62,6 +78,22 @@
                         <% end %>
                         <div class="col-sm-5 badge-med-height badge-text">Errors</div>
                       </div>
+                      <div class="row pt-2">
+                        <% if @hash['minitest']['total_skipped'] > 0 %>
+                          <div class="col-sm-3 offset-sm-2 badge-med-height badge badge-info"><span class='med-text'><%= @hash['minitest']['total_skipped'] %></span></div>
+                        <% else %>
+                          <div class="col-sm-3 offset-sm-2 badge-med-height badge badge-success"><span class='med-text'><%= @hash['minitest']['total_skipped'] %></span></div>
+                        <% end %>
+                        <div class="col-sm-5 badge-med-height badge-text">Skipped</div>
+                      </div>
+                      <div class="row pt-2">
+                        <% if @hash['minitest']['total_compatibility_errors'] > 0 %>
+                          <div class="col-sm-3 offset-sm-2 badge-med-height badge badge-info"><span class='med-text'><%= @hash['minitest']['total_compatibility_errors'] %></span></div>
+                        <% else %>
+                          <div class="col-sm-3 offset-sm-2 badge-med-height badge badge-success"><span class='med-text'><%= @hash['minitest']['total_compatibility_errors'] %></span></div>
+                        <% end %>
+                        <div class="col-sm-5 badge-med-height badge-text">Incompatible</div>
+                      </div>
                     <% else %>
                       <div class="row pt-5">
                         <div class="col-sm-6 offset-sm-3 badge-lg-height badge badge-warning badge-text"><i data-feather="alert-triangle" class="size48"></i><br><span class="pt-2">No Data</span> </div>
@@ -135,19 +167,30 @@
               <table class="table table-striped table-sm table-text">
                 <thead>
                   <tr>
-                    <th>Measure Name</th>
-                    <th>Tests<br/><span class="small-text">Passed - Failures - Errors</span></th>
-                    <th>OS Style Errors<br/><span class="small-text">Errors - Warnings - Info</span></th>
-                    <th>RuboCop<br/><span class="small-text">Errors - Warnings - Info</span></th>
-                    <th>Coverage</th>
-                    <th>&nbsp;</th>
+                    <th rowspan="2">Measure Name</th>
+                    <th rowspan="2">Loaded</th>
+                    <th rowspan="2">Min OS Version</th>
+                    <th rowspan="2">Max OS Version</th>
+                    <th rowspan="2">Tested OS Version</th>
+                    <th colspan="5">Tests</th>
+                    <th rowspan="2">OS Style Errors<br/><span class="small-text">Errors - Warnings - Info</span></th>
+                    <th rowspan="2">RuboCop<br/><span class="small-text">Errors - Warnings - Info</span></th>
+                    <th rowspan="2">Coverage</th>
+                    <th>&nbsp</th>
+                  </tr>
+                  <tr>
+                    <th>Pass</th>
+                    <th>Fail</th>
+                    <th>Error</th>
+                    <th>Skip</th>
+                    <th>Incom</th>
                   </tr>
                 </thead>
                 <tbody>
                   <!-- use the largest 'by_measure' array, in case the tests weren't all run (coverage & minitest run together) -->
                   <% max = 0 %>
                   <% test_type = ''%>
-                  <% ['coverage', 'openstudio_style', 'rubocop'].each do |type| %>
+                  <% ['coverage', 'openstudio_style', 'minitest','rubocop'].each do |type| %>
                     <% if @hash[type] && @hash[type]['by_measure'] %>
                       <% if @hash[type]['by_measure'].length > max %>
                         <% max = @hash[type]['by_measure'].length %>
@@ -159,8 +202,36 @@
                     <% @hash[test_type]['by_measure'].each do |name, measure| %>
                     <tr>
                       <td><%= name %></td>
+                      <td>
+                        <% if  @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] && @hash['minitest']['by_measure'][name]['loaded'] %>
+                          Yes
+                        <% else %>
+                          False
+                        <% end %>
+                      </td>
+                      <td>
+                        <% if  @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] %>
+                          <%= @hash['minitest']['by_measure'][name]['measure_min_version'] %>
+                        <% else %>
+                          Unknown
+                        <% end %>
+                      </td>
+                      <td>
+                        <% if  @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] %>
+                          <%= @hash['minitest']['by_measure'][name]['measure_max_version'] %>
+                        <% else %>
+                          Unknown
+                        <% end %>
+                      </td>
                       <td>
                         <% if  @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] %>
+                          <%= @hash['minitest']['by_measure'][name]['openstudio_version'] %>
+                        <% else %>
+                          Unknown
+                        <% end %>
+                      </td>
+                      <% if  @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] %>
+                        <td>
                           <% passed =  @hash['minitest']['by_measure'][name]['measure_tests'] - (@hash['minitest']['by_measure'][name]['measure_failures'] + @hash['minitest']['by_measure'][name]['measure_errors'] + @hash['minitest']['by_measure'][name]['measure_skipped']) %>
                           <span class="badge badge-success"><%= passed %></span>
                           <% if @hash['minitest']['by_measure'][name]['measure_failures'] > 0 %>
@@ -168,15 +239,21 @@
                           <% else %>
                             <% label = 'success' %>
                           <% end %>
+                        </td>
+                        <td>
                           <span class="badge badge-<%= label %>"><%= @hash['minitest']['by_measure'][name]['measure_failures'] %></span>
                           <% if @hash['minitest']['by_measure'][name]['measure_errors'] > 0 %>
-                              <% label = 'danger' %>
+                            <% label = 'danger' %>
                           <% else %>
-                              <% label = 'success' %>
+                            <% label = 'success' %>
                           <% end %>
-                          <span class="badge badge-<%= label %>"><%= @hash['minitest']['by_measure'][name]['measure_errors'] %></span>
-                        <% end %>
-                      </td>
+                        </td>
+                        <td><span class="badge badge-<%= label %>"><%= @hash['minitest']['by_measure'][name]['measure_errors'] %></span></td>
+                        <td><span class="badge badge-info"><%= @hash['minitest']['by_measure'][name]['measure_skipped'] %></span></td>
+                        <td><span class="badge badge-info"><%= @hash['minitest']['by_measure'][name]['measure_compatibility_errors'] %></span></td>
+                      <% else %>
+                        <td></td><td></td><td></td><td></td><td></td>
+                      <% end %>
                       <td>
                         <% if @hash['openstudio_style'] && @hash['openstudio_style']['by_measure'][name] %>
                           <% if @hash['openstudio_style']['by_measure'][name]['measure_errors'] > 0 %>
@@ -330,11 +407,44 @@
                   </div>
                 </div>
 
+                <div class="row pb-3">
+                  <div class="col-sm-12">
+                    <h4>Loaded</h4>
+                    <% if @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] %>
+                      <% if @hash['minitest']['by_measure'][name]['loaded'] %>
+                        <div>True</div>
+                      <% else %>
+                        <div>False</div>
+                        <% if @hash['minitest']['by_measure'][name]['load_errors'] %>
+                          <table class="table table-striped">
+                            <thead>
+                              <tr>
+                                <th>Error</th>
+                              </tr>
+                            </thead>
+                            <tbody>
+                              <% @hash['minitest']['by_measure'][name]['load_errors'].each do |load_error| %>
+                                <tr>
+                                  <td><%= load_error %></td>
+                                </tr>
+                              <% end %>
+                            </tbody>
+                          </table>
+                        <% end %>
+                      <% end %>
+                    <% else %>
+                      <div>
+                        <span class="badge badge-danger">NO DATA</span>  Test was not run
+                      </div>
+                    <% end %>
+                  </div>
+                </div>
+                
                 <div class="row pb-3">
                   <div class="col-sm-12">
                     <h4>Tests</h4>
                     <% if @hash['minitest'] && @hash['minitest']['by_measure'] && @hash['minitest']['by_measure'][name] && @hash['minitest']['by_measure'][name]['issues'] %>
-                      <% if @hash['minitest']['by_measure'][name]['issues']['failures'].length > 0 || @hash['minitest']['by_measure'][name]['issues']['errors'].length > 0 %>
+                      <% if @hash['minitest']['by_measure'][name]['issues']['failures'].length > 0 || @hash['minitest']['by_measure'][name]['issues']['errors'].length > 0 || @hash['minitest']['by_measure'][name]['issues']['skipped'].length > 0 %>
                         <table class="table table-striped">
                           <thead>
                             <tr>
@@ -354,6 +464,12 @@
                               <td><span class="badge badge-dark">Error</span></td>
                               <td><%= err %></td>
                             </tr>
+                          <% end %>
+                           <% @hash['minitest']['by_measure'][name]['issues']['skipped'].each do |skip| %>
+                            <tr>
+                              <td><span class="badge badge-info">Skipped</span></td>
+                              <td><%= skip %></td>
+                            </tr>
                           <% end %>
                           </tbody>
                         </table>
@@ -567,7 +683,6 @@
       size(true);
 
       }());
-
     </script>
   </body>
 </html>

data/lib/openstudio_measure_tester/version.rb

@@ -27,5 +27,5 @@
 ########################################################################################################################
 
 module OpenStudioMeasureTester
-  VERSION = '0.1.5'.freeze
+  VERSION = '0.1.6'.freeze
 end

data/openstudio_measure_tester.gemspec

@@ -1,4 +1,3 @@
-
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'openstudio_measure_tester/version'
@@ -23,11 +22,18 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'rspec', '3.7.0'
 
+  if /^2\.2/.match(RUBY_VERSION)
+    spec.add_dependency 'rubocop', '0.54.0'
+  elsif /^2\.0/.match(RUBY_VERSION)
+    spec.add_dependency 'nokogiri', '1.6.8' 
+    spec.add_dependency 'rainbow', '2.2.2' 
+    spec.add_dependency 'rubocop', '0.50.0'
+  end
+  
   spec.add_dependency 'git', '1.3.0'
   spec.add_dependency 'minitest', '5.4.3'
   spec.add_dependency 'minitest-reporters', '1.2.0'
   spec.add_dependency 'rake', '12.3.1'
-  spec.add_dependency 'rubocop', '0.54.0'
   spec.add_dependency 'rubocop-checkstyle_formatter', '0.4'
   spec.add_dependency 'simplecov', '0.16.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openstudio_measure_tester
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Nicholas Long
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-13 00:00:00.000000000 Z
+date: 2018-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -41,6 +41,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.7.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.54.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.54.0
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
@@ -97,20 +111,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 12.3.1
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.54.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.54.0
 - !ruby/object:Gem::Dependency
   name: rubocop-checkstyle_formatter
   requirement: !ruby/object:Gem::Requirement