openstudio-analysis 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35555c57a9856bcb1f7fb5f0a6595e4399bd022a
-  data.tar.gz: 790e9b5480f83fdee93a117d42dcdb98fc8574a9
+  metadata.gz: 9d66a0d90f0f410e7db18fbdc523c191991d5d20
+  data.tar.gz: da6f448c4902518e932ad8a17dd183be1d6ff8d7
 SHA512:
-  metadata.gz: 3d806c1ef888b34e21b20d412dcd314a11c0a2a009cce212a2fdae2e1deffc5e4931202e504a14f212f2e52c8ae4590994da34f5786d0a046f79b7b0e256ae6a
-  data.tar.gz: efaa94dab1049e80d0d42a66125e30851b851a0481dc69dfac9a089195adbd643998e11e2652e584145c253382ab6764f6af60e985160ab8cf6b89a9c99f9eb9
+  metadata.gz: d5440d40d0d9bcaf5d0d07e4f8a88471df0d8063b88c2fd2da335b5d48fd3900fe39c72045b398d8929f594ee5fb40fb3038c96bb9896bfef83e0c55f8c489cb
+  data.tar.gz: 22f2c7ab4cc3bae5e0bccda01921f95c22c7ee3b6bd221e0e75950cc79b760e5ccda9aaf1d660c6d5a249f16fbaf159c3c04e76cbc2fd43285d114353ad871b4

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 OpenStudio Analysis Gem Change Log
 ==================================
 
+Version 0.3.7 (Unreleased)
+-------------
+* Worker initilization and finalization scripts
+* Do not allow the file to proccess if the Measure Display Names are not unique
+
 Version 0.3.6
 -------------
 * Allow multiple measure paths. Will search by order for the measure.

data/README.md

@@ -4,7 +4,7 @@ OpenStudio Analysis Gem
 
 The OpenStudio Analysis Gem is used to communicate files to the OpenStudio Distributed Analysis.
 
-The purpose of this gem is to generate the analysis.json file, analysis.zip, and communicate with the server to upload 
+The purpose of this gem is to generate the analysis.json file, analysis.zip, and communicate with the server to upload
 the simulations.
 
 The gem does not create the cluster. Currently the only supported Cloud platform is
@@ -16,6 +16,4 @@ Instructions
 Testing
 -------
 
-This gem used RSpec for testing.  To test simple run `rspec` at the command line.
-
-
+This gem used RSpec for testing.  To test simply run `rspec` at the command line.

data/Rakefile

@@ -1,54 +1,25 @@
 require 'bundler'
 Bundler.setup
 
-require 'rake'
 require 'rspec/core/rake_task'
 
-$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
-require 'openstudio/analysis/version'
+# Always create spec reports
+require 'ci/reporter/rake/rspec'
 
-task gem: :build
-desc 'build gem locally'
-task :build do
-  system 'gem build openstudio-analysis.gemspec'
-end
-
-desc 'build and install gem locally'
-task install: :build do
-  system "gem install openstudio-analysis-#{OpenStudio::Analysis::VERSION}.gem --no-ri --no-rdoc"
-end
-
-desc 'release gem (this builds, pushes to rubygems, and tags in github'
-task release: :build do
-  # add catch if there are local changes not committed to crash
-  system "git tag -a v#{OpenStudio::Analysis::VERSION} -m 'Tagging #{OpenStudio::Analysis::VERSION}'"
-  system 'git push --tags'
-  system "gem push openstudio-analysis-#{OpenStudio::Analysis::VERSION}.gem"
-  system "rm openstudio-analysis-#{OpenStudio::Analysis::VERSION}.gem"
-end
+# Gem tasks
+require 'bundler/gem_tasks'
 
 RSpec::Core::RakeTask.new('spec:unit') do |spec|
-  spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
+  spec.rspec_opts = %w(--format progress)
   spec.pattern = FileList['spec/openstudio/**/*_spec.rb']
 end
 
 RSpec::Core::RakeTask.new('spec:integration') do |spec|
-  spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
+  spec.rspec_opts = %w(--format progress)
   spec.pattern = FileList['spec/integration/**/*_spec.rb']
 end
 
-task default: 'spec:unit'
-
-desc 'import files from other repos'
-task :import_files do
-  # tbd
-end
-
-desc 'uninstall all openstudio-analysis gems'
-task :uninstall do
+task 'spec:unit' => 'ci:setup:rspec'
+task 'spec:integration' => 'ci:setup:rspec'
 
-  system 'gem uninstall openstudio-analysis -a'
-end
-
-desc 'reinstall the gem (uninstall, build, and reinstall'
-task reinstall: [:uninstall, :install]
+task default: 'spec:unit'

data/lib/openstudio-analysis.rb

@@ -1,14 +1,16 @@
+# Ruby libraries to include
 require 'json'
+require 'securerandom'
+require 'logger'
+require 'pathname'
+
+# gems to always include
 require 'faraday'
-require 'uuid'
 require 'roo'
 require 'erb'
 require 'zip'
 require 'semantic'
 require 'semantic/core_ext'
-require 'logger'
-require 'pp'
-require 'pathname'
 
 # core
 require 'openstudio/analysis/server_api'

data/lib/openstudio/analysis/server_api.rb

@@ -269,7 +269,7 @@ module OpenStudio
         file_path_and_name = nil
 
         response = @conn.get do |r|
-          r.url  '/admin/backup_database?full_backup=true'
+          r.url '/admin/backup_database?full_backup=true'
           r.options.timeout = 3600 # 60 minutes
         end
 
@@ -379,20 +379,20 @@ module OpenStudio
         analysis_id = nil
         if formulation_json
           if options[:reset_uuids]
-            analysis_id = UUID.new.generate
+            analysis_id = SecureRandom.uuid
             formulation_json[:analysis][:uuid] = analysis_id
 
             formulation_json[:analysis][:problem][:workflow].each do |wf|
-              wf[:uuid] = UUID.new.generate
+              wf[:uuid] = SecureRandom.uuid
               if wf[:arguments]
                 wf[:arguments].each do |arg|
-                  arg[:uuid] = UUID.new.generate
+                  arg[:uuid] = SecureRandom.uuid
                 end
               end
               if wf[:variables]
                 wf[:variables].each do |var|
-                  var[:uuid] = UUID.new.generate
-                  var[:argument][:uuid] = UUID.new.generate if var[:argument]
+                  var[:uuid] = SecureRandom.uuid
+                  var[:argument][:uuid] = SecureRandom.uuid if var[:argument]
                 end
               end
             end
@@ -407,7 +407,7 @@ module OpenStudio
             analysis: options
           }
           puts formulation_json
-          analysis_id = UUID.new.generate
+          analysis_id = SecureRandom.uuid
           formulation_json[:analysis][:uuid] = analysis_id
         end
         fail "No analysis id defined in analyis.json #{options[:formulation_file]}" if analysis_id.nil?
@@ -440,7 +440,6 @@ module OpenStudio
             req.options[:timeout] = 1800 # seconds
           end
 
-
           if response.status == 201
             puts 'Successfully uploaded ZIP file'
           else
@@ -463,7 +462,7 @@ module OpenStudio
 
         if options[:reset_uuids]
           dp_hash[:analysis_uuid] = analysis_id
-          dp_hash[:uuid] = UUID.new.generate
+          dp_hash[:uuid] = SecureRandom.uuid
         end
 
         # merge in the analysis_id as it has to be what is in the database

data/lib/openstudio/analysis/translator/excel.rb

@@ -9,6 +9,8 @@ module OpenStudio
         attr_reader :models
         attr_reader :weather_files
         attr_reader :measure_paths
+        attr_reader :worker_inits
+        attr_reader :worker_finals
         attr_reader :export_path
         attr_reader :cluster_name
         attr_reader :variables
@@ -46,6 +48,8 @@ module OpenStudio
           @weather_files = []
           @models = []
           @other_files = []
+          @worker_inits = []
+          @worker_finals = []
           @export_path = './export'
           @measure_paths = []
           @number_of_samples = 0 # todo: remove this
@@ -98,7 +102,6 @@ module OpenStudio
             fail "Measures directory '#{mp}' does not exist" unless Dir.exist?(mp)
           end
 
-
           @models.uniq!
           fail 'No seed models defined in spreadsheet' if @models.empty?
 
@@ -114,12 +117,28 @@ module OpenStudio
           end
 
           # This can be a directory as well
-          @other_files.each do |of|
-            fail "Other files do not exist for: #{of[:path]}" unless File.exist?(of[:path])
+          @other_files.each do |f|
+            fail "Other files do not exist for: #{f[:path]}" unless File.exist?(f[:path])
+          end
+
+          @worker_inits.each do |f|
+            fail "Worker initialization file does not exist for: #{f[:path]}" unless File.exist?(f[:path])
+          end
+
+          @worker_finals.each do |f|
+            fail "Worker finalization file does not exist for: #{f[:path]}" unless File.exist?(f[:path])
           end
 
           FileUtils.mkdir_p(@export_path)
 
+          # verify that the measure display names are unique
+          # puts @variables.inspect
+          measure_display_names = @variables['data'].map { |m| m['enabled'] ? m['display_name'] : nil }.compact
+          measure_display_names_mult = measure_display_names.select { |m| measure_display_names.count(m) > 1 }.uniq
+          if measure_display_names_mult && !measure_display_names_mult.empty?
+            fail "Measure Display Names are not unique for '#{measure_display_names_mult.join('\', \'')}'"
+          end
+
           # verify that all continuous variables have all the data needed and create a name maps
           variable_names = []
           @variables['data'].each do |measure|
@@ -349,9 +368,9 @@ module OpenStudio
         # Package up the seed, weather files, and measures
         def save_analysis_zip(model)
           def add_directory_to_zip(zipfile, local_directory, relative_zip_directory)
-            # pp "Add Directory #{local_directory}"
+            # puts "Add Directory #{local_directory}"
             Dir[File.join("#{local_directory}", '**', '**')].each do |file|
-              # pp "Adding File #{file}"
+              # puts "Adding File #{file}"
               zipfile.add(file.sub(local_directory, relative_zip_directory), file)
             end
             zipfile
@@ -371,7 +390,7 @@ module OpenStudio
 
             # Convert this into a hash which looks like {name: measure_name}. This will allow us to add more
             # fields to the measure, such as which directory is the measure.
-            required_measures = required_measures.map { |value| {name: value} }
+            required_measures = required_measures.map { |value| { name: value } }
 
             # first validate that all the measures exist
             errors = []
@@ -408,10 +427,10 @@ module OpenStudio
                   if File.basename(file) == 'resources' || File.basename(file) == 'lib'
                     add_directory_to_zip(zipfile, file, "./measures/#{measure[:name]}/#{File.basename(file)}")
                   else
-                    # pp "Skipping Directory #{File.basename(file)}"
+                    # puts "Skipping Directory #{File.basename(file)}"
                   end
                 else
-                  # pp "Adding File #{file}"
+                  # puts "Adding File #{file}"
                   # added_measures << measure_dir unless added_measures.include? measure_dir
                   zipfile.add(file.sub(measure_dir_to_add, "./measures/#{measure[:name]}/"), file)
                 end
@@ -427,6 +446,34 @@ module OpenStudio
                 zipfile.add(file.sub(others[:path], "./lib/#{others[:lib_zip_name]}/"), file)
               end
             end
+
+            # puts "Adding in Worker initialize scripts #{@worker_inits}"
+            @worker_inits.each_with_index do |f, index|
+              # this is ordered
+              f[:ordered_file_name] = "#{index.to_s.rjust(2, '0')}_#{File.basename(f[:path])}"
+              f[:index] = index
+
+              zipfile.add(f[:path].sub(f[:path], "./lib/worker_initialize/#{f[:ordered_file_name]}"), f[:path])
+              arg_file = "#{File.basename(f[:ordered_file_name], '.*')}.args"
+              file = Tempfile.new('arg')
+              file.write(f[:args])
+              zipfile.add("./lib/worker_initialize/#{arg_file}", file)
+              file.close
+            end
+
+            # puts "Adding in Worker finalize scripts #{@worker_finals}"
+            @worker_finals.each_with_index do |f, index|
+              # this is ordered
+              f[:ordered_file_name] = "#{index.to_s.rjust(2, '0')}_#{File.basename(f[:path])}"
+              f[:index] = index
+
+              zipfile.add(f[:path].sub(f[:path], "./lib/worker_finalize/#{f[:ordered_file_name]}"), f[:path])
+              arg_file = "#{File.basename(f[:ordered_file_name], '.*')}.args"
+              file = Tempfile.new('arg')
+              file.write(f[:args])
+              zipfile.add("./lib/worker_finalize/#{arg_file}", file)
+              file.close
+            end
           end
         end
 
@@ -474,6 +521,8 @@ module OpenStudio
           b_weather_files = false
           b_models = false
           b_other_libs = false
+          b_worker_init = false
+          b_worker_final = false
 
           rows.each do |row|
             if row[0] == 'Settings'
@@ -484,6 +533,8 @@ module OpenStudio
               b_weather_files = false
               b_models = false
               b_other_libs = false
+              b_worker_init = false
+              b_worker_final = false
               next
             elsif row[0] == 'Running Setup'
               b_settings = false
@@ -493,6 +544,8 @@ module OpenStudio
               b_weather_files = false
               b_models = false
               b_other_libs = false
+              b_worker_init = false
+              b_worker_final = false
               next
             elsif row[0] == 'Problem Definition'
               b_settings = false
@@ -502,6 +555,8 @@ module OpenStudio
               b_weather_files = false
               b_models = false
               b_other_libs = false
+              b_worker_init = false
+              b_worker_final = false
               next
             elsif row[0] == 'Algorithm Setup'
               b_settings = false
@@ -511,6 +566,8 @@ module OpenStudio
               b_weather_files = false
               b_models = false
               b_other_libs = false
+              b_worker_init = false
+              b_worker_final = false
               next
             elsif row[0] == 'Weather Files'
               b_settings = false
@@ -520,6 +577,8 @@ module OpenStudio
               b_weather_files = true
               b_models = false
               b_other_libs = false
+              b_worker_init = false
+              b_worker_final = false
               next
             elsif row[0] == 'Models'
               b_settings = false
@@ -529,6 +588,8 @@ module OpenStudio
               b_weather_files = false
               b_models = true
               b_other_libs = false
+              b_worker_init = false
+              b_worker_final = false
               next
             elsif row[0] == 'Other Library Files'
               b_settings = false
@@ -538,6 +599,30 @@ module OpenStudio
               b_weather_files = false
               b_models = false
               b_other_libs = true
+              b_worker_init = false
+              b_worker_final = false
+              next
+            elsif row[0] =~ /Worker Initialization Scripts/
+              b_settings = false
+              b_run_setup = false
+              b_problem_setup = false
+              b_algorithm_setup = false
+              b_weather_files = false
+              b_models = false
+              b_other_libs = false
+              b_worker_init = true
+              b_worker_final = false
+              next
+            elsif row[0] =~ /Worker Finalization Scripts/
+              b_settings = false
+              b_run_setup = false
+              b_problem_setup = false
+              b_algorithm_setup = false
+              b_weather_files = false
+              b_models = false
+              b_other_libs = false
+              b_worker_init = false
+              b_worker_final = true
               next
             end
 
@@ -555,13 +640,12 @@ module OpenStudio
               # type some of the values that we know
               @settings['proxy_port'] = @settings['proxy_port'].to_i if @settings['proxy_port']
 
-
             elsif b_run_setup
               if row[0] == 'Analysis Name'
                 if row[1]
                   @name = row[1]
                 else
-                  @name = UUID.new.generate
+                  @name = SecureRandom.uuid
                 end
                 @analysis_name = @name.snake_case
               end
@@ -611,7 +695,7 @@ module OpenStudio
               if row[1]
                 tmp_m_name = row[1]
               else
-                tmp_m_name = UUID.new.generate
+                tmp_m_name = SecureRandom.uuid
               end
               # Only add models if the row is flagged
               if row[0] && row[0].downcase == 'model'
@@ -629,7 +713,23 @@ module OpenStudio
               end
 
               @other_files << { lib_zip_name: row[1], path: other_path }
+            elsif b_worker_init
+              worker_init_path = row[1]
+              unless (Pathname.new worker_init_path).absolute?
+                worker_init_path = File.expand_path(File.join(@root_path, worker_init_path))
+              end
+
+              @worker_inits << { name: row[0], path: worker_init_path, args: row[2] }
+            elsif b_worker_final
+              worker_final_path = row[1]
+              unless (Pathname.new worker_final_path).absolute?
+                worker_final_path = File.expand_path(File.join(@root_path, worker_final_path))
+              end
+
+              @worker_finals << { name: row[0], path: worker_final_path, args: row[2] }
             end
+
+            next
           end
 
           # do some last checks
@@ -639,7 +739,7 @@ module OpenStudio
         # parse_variables will parse the XLS spreadsheet and save the data into
         # a higher level JSON file.  The JSON file is historic and it should really
         # be omitted as an intermediate step
-        def   parse_variables
+        def parse_variables
           # clean remove whitespace and unicode chars
           # The parse is a unique format (https://github.com/Empact/roo/blob/master/lib/roo/base.rb#L444)
           # If you add a new column and you want that variable in the hash, then you must add it here.
@@ -818,7 +918,7 @@ module OpenStudio
 
                 # parse the choices/enums
                 if var['type'] == 'enum' || var['type'] == 'choice' # this is now a choice
-                  var['distribution']['enumerations'] = row[:enums].gsub('|', '').split(',').map { |v| v.strip }
+                  var['distribution']['enumerations'] = row[:enums].gsub('|', '').split(',').map(&:strip)
                 elsif var['type'] == 'bool'
                   var['distribution']['enumerations'] = []
                   var['distribution']['enumerations'] << 'true' # TODO: should this be a real bool?
@@ -848,7 +948,7 @@ module OpenStudio
               # generate name id
               # TODO: put this into a logger. puts "Parsing measure #{row[1]}"
               display_name = row[:measure_name_or_var_type]
-              measure_name = display_name.downcase.strip.gsub('-', '_').gsub(' ', '_')
+              measure_name = display_name.downcase.strip.gsub('-', '_').gsub(' ', '_').gsub('__', '_')
               data['data'][measure_index]['display_name'] = display_name
               data['data'][measure_index]['name'] = measure_name
               data['data'][measure_index]['enabled'] = row[:enabled] == 'TRUE' ? true : false