openstudio-workflow 1.0.0.pat1 → 1.0.0

data/lib/openstudio/workflow/util/post_process.rb

@@ -0,0 +1,177 @@
+module OpenStudio
+  module Workflow
+    module Util
+      require 'openstudio/workflow/util/measure'
+      require 'csv'
+
+      # This module serves as a wrapper around various post-processing tasks used to manage outputs
+      # @todo (rhorsey) ummmm. So some of this is pretty ugly. Since @dmacumber had ideas about this maybe he can figure
+      #   out what to do about it all
+      # @todo (nlong) the output adapter restructure will frack up the extraction method royally
+      #
+      module PostProcess
+        # This method loads a sql file into OpenStudio and returns it
+        #
+        # @param [String] sql_file Absolute path to the sql file to be loaded
+        # @return [Object, nil] The OpenStudio::SqlFile object, or nil if it could not be found
+        #
+        def load_sql_file(sql_file)
+          return nil unless File.exist? sql_file
+          OpenStudio::SqlFile.new(@sql_filename)
+        end
+
+        # This method parses all sorts of stuff which something needs
+        #
+        # @param [String] run_dir The directory that the simulation was run in
+        # @return [Hash, Hash] results and objective_function (which may be empty) are returned
+        # @todo (rhorsey) fix the description
+        #
+        def run_extract_inputs_and_outputs(run_dir, logger)
+          # For xml, the measure attributes are in the measure_attributes_xml.json file
+          # TODO: somehow pass the metadata around on which JSONs to suck into the database
+          results = {}
+          # Inputs are in the measure_attributes.json file
+          if File.exist? "#{run_dir}/measure_attributes.json"
+            h = JSON.parse(File.read("#{run_dir}/measure_attributes.json"), symbolize_names: true)
+            h = rename_hash_keys(h, logger)
+            results.merge! h
+          end
+
+          logger.info 'Saving the result hash to file'
+          File.open("#{run_dir}/results.json", 'w') { |f| f << JSON.pretty_generate(results) }
+
+          objective_functions = {}
+          if @registry[:analysis]
+            logger.info 'Iterating over Analysis JSON Output Variables'
+            # Save the objective functions to the object for sending back to the simulation executive
+
+            if @analysis_json[:analysis] && @analysis_json[:analysis][:output_variables]
+              @analysis_json[:analysis][:output_variables].each do |variable|
+                # determine which ones are the objective functions (code smell: todo: use enumerator)
+                if variable[:objective_function]
+                  logger.info "Looking for objective function #{variable[:name]}"
+                  # TODO: move this to cleaner logic. Use ostruct?
+                  k, v = variable[:name].split('.')
+
+                  # look for the objective function key and make sure that it is not nil. False is an okay obj function.
+                  if results.key?(k.to_sym) && !results[k.to_sym][v.to_sym].nil?
+                    objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = results[k.to_sym][v.to_sym]
+                    if variable[:objective_function_target]
+                      logger.info "Found objective function target for #{variable[:name]}"
+                      objective_functions["objective_function_target_#{variable[:objective_function_index] + 1}"] = variable[:objective_function_target].to_f
+                    end
+                    if variable[:scaling_factor]
+                      logger.info "Found scaling factor for #{variable[:name]}"
+                      objective_functions["scaling_factor_#{variable[:objective_function_index] + 1}"] = variable[:scaling_factor].to_f
+                    end
+                    if variable[:objective_function_group]
+                      logger.info "Found objective function group for #{variable[:name]}"
+                      objective_functions["objective_function_group_#{variable[:objective_function_index] + 1}"] = variable[:objective_function_group].to_f
+                    end
+                  else
+                    logger.warn "No results for objective function #{variable[:name]}"
+                    objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = Float::MAX
+                    objective_functions["objective_function_target_#{variable[:objective_function_index] + 1}"] = nil
+                    objective_functions["scaling_factor_#{variable[:objective_function_index] + 1}"] = nil
+                    objective_functions["objective_function_group_#{variable[:objective_function_index] + 1}"] = nil
+                  end
+                end
+              end
+            end
+          end
+
+          return results, objective_functions
+        end
+
+        # Remove any invalid characters in the measure attribute keys. Periods and Pipes are the most problematic
+        #   because mongo does not allow hash keys with periods, and the pipes are used in the map/reduce method that
+        #   was written to speed up the data write in openstudio-server. Also remove any trailing underscores and spaces
+        #
+        # @param [Hash] hash Any hash with potentially problematic characters
+        # @param [Logger] logger Logger to write to
+        #
+        def rename_hash_keys(hash, logger)
+          # @todo should we log the name changes?
+          regex = /[|!@#\$%^&\*\(\)\{\}\\\[\];:'",<.>\/?\+=]+/
+
+          rename_keys = lambda do |h|
+            if Hash === h
+              h.each_key do |key|
+                if key.to_s =~ regex
+                  logger.warn "Renaming result key '#{key}' to remove invalid characters"
+                end
+              end
+              Hash[h.map { |k, v| [k.to_s.gsub(regex, '_').squeeze('_').gsub(/[_\s]+$/, '').chomp.to_sym, rename_keys[v]] }]
+            else
+              h
+            end
+          end
+
+          rename_keys[hash]
+        end
+
+
+        # Save reports to a common directory
+        #
+        # @param [String] run_dir
+        # @param [String] directory
+        # @param [String] logger
+        #
+        def gather_reports(run_dir, directory, logger)
+          logger.info run_dir
+          logger.info directory
+
+          FileUtils.mkdir_p "#{directory}/reports"
+
+          # try to find the energyplus result file
+          eplus_html = "#{run_dir}/eplustbl.htm"
+          if File.exist? eplus_html
+            # do some encoding on the html if possible
+            html = File.read(eplus_html)
+            html = html.force_encoding('ISO-8859-1').encode('utf-8', replace: nil)
+            logger.info "Saving EnergyPlus HTML report to #{directory}/reports/eplustbl.html"
+            File.open("#{directory}/reports/eplustbl.html", 'w') { |f| f << html }
+          end
+
+          # Also, find any "report*.*" files
+          Dir["#{run_dir}/*/report*.*"].each do |report|
+            # get the parent directory of the file and snake case it
+            measure_class_name = File.basename(File.dirname(report))
+            file_ext = File.extname(report)
+            append_str = File.basename(report, '.*')
+            new_file_name = "#{directory}/reports/#{measure_class_name}_#{append_str}#{file_ext}"
+            logger.info "Saving report #{report} to #{new_file_name}"
+            FileUtils.copy report, new_file_name
+          end
+
+          # Remove empty directories in run folder
+          Dir["#{run_dir}/*"].select { |d| File.directory? d }.select { |d| (Dir.entries(d) - %w(. ..)).empty? }.each do |d|
+            logger.info "Removing empty directory #{d}"
+            Dir.rmdir d
+          end
+        end
+
+
+        # A general post-processing step which could be made significantly more modular
+        #
+        # @param [String] run_dir
+        #
+        def cleanup(run_dir, directory, logger)
+
+
+          paths_to_rm = []
+          # paths_to_rm << Pathname.glob("#{run_dir}/*.osm")
+          # paths_to_rm << Pathname.glob("#{run_dir}/*.idf") # keep the idfs
+          # paths_to_rm << Pathname.glob("*.audit")
+          # paths_to_rm << Pathname.glob("*.bnd")
+          # paths_to_rm << Pathname.glob("#{run_dir}/*.eso")
+          paths_to_rm << Pathname.glob("#{run_dir}/*.mtr")
+          paths_to_rm << Pathname.glob("#{run_dir}/*.epw")
+          paths_to_rm << Pathname.glob("#{run_dir}/*.mtd")
+          paths_to_rm << Pathname.glob("#{run_dir}/*.rdd")
+          paths_to_rm.each { |p| FileUtils.rm_rf(p) }
+        end
+      end
+    end
+  end
+end

data/lib/openstudio/workflow/util/weather_file.rb

@@ -0,0 +1,108 @@
+module OpenStudio
+  module Workflow
+    module Util
+      # The current precedence rules for weather files are defined in this module. Best practice is to only use the
+      #  #get_weather_file method, as it will be forward compatible
+      #
+      module WeatherFile
+        # Returns the weather file with precedence
+        #
+        # @param [String] directory The directory to append all relative directories to, see #get_weather_file_from_fs
+        # @param [String] wf The weather file being searched for. If not the name of the file this parameter should be
+        #   the absolute path specifying it's location
+        # @param [Array] wf_search_array The set of precedence ordered relative directories to search for the wf in. A
+        #   typical entry might look like `['files', '../../files', '../../weather']`
+        # @param [Object] model The OpenStudio::Model object to parse, see #get_weather_file_from_osm
+        # @return [String, nil] The weather file with precedence if defined, nil if not, and a failure if the wf is
+        #   defined but not in the filesystem
+        #
+        def get_weather_file(directory, wf, wf_search_array, model, logger = nil)
+          # TODO: this logic needs some updating, weather file should come from current model, found using search paths
+          logger ||= ::Logger.new(STDOUT) unless logger
+          if wf
+            weather_file = get_weather_file_from_fs(directory, wf, wf_search_array, logger)
+            raise 'Could not locate the weather file in the filesystem. Please see the log' if weather_file == false
+          end
+          weather_file = get_weather_file_from_osm(model, logger) if weather_file.nil?
+          raise 'Could not locate the weather file in the filesystem. Please see the log' if weather_file == false
+          logger.warn 'The weather file could not be determined. Please see the log for details' unless weather_file
+          weather_file
+        end
+
+        private
+
+        # Returns the weather file from the model. If the weather file is defined in the model, then
+        # it checks the file paths to check if the model exists. This allows for a user to upload a
+        # weather file in a measure and then have the measure's path be used for the weather file.
+        #
+        # @todo (rhorsey) verify the description of this method, as it seems suspect
+        # @param [Object] model The OpenStudio::Model object to retrieve the weather file from
+        # @return [nil,false, String] If the result is nil the weather file was not defined in the model, if the result
+        #   is false the weather file was set but could not be found on the filesystem, if a string the weather file was
+        #   defined and it's existence verified
+        #
+        def get_weather_file_from_osm(model, logger)
+          wf = nil
+          # grab the weather file out of the OSM if it exists
+          if model.weatherFile.empty?
+            logger.warn 'No weather file defined in the model'
+          else
+            p = model.weatherFile.get.path.get.to_s.gsub('file://', '')
+            wf = if File.exist? p
+                   File.absolute_path(p)
+                 else
+                   # this is the weather file from the OSM model
+                   File.absolute_path(@model.weatherFile.get.path.get.to_s)
+                 end
+            logger.info "The weather file path found in the model object: #{wf}"
+            unless File.exist? wf
+              logger.warn 'The weather file could not be found on the filesystem.'
+              wf = false
+            end
+          end
+          wf
+        end
+
+        # Returns the weather file defined in the OSW
+        #
+        # @param [String] directory The base directory to append all relative directories to
+        # @param [String] wf The weather file being searched for. If not the name of the file this parameter should be
+        #   the absolute path specifying it's location
+        # @param [Array] wf_search_array The set of precedence ordered relative directories to search for the wf in. A
+        #   typical entry might look like `['files', '../../files', '../../weather']`
+        # @return [nil, false, String] If the result is nil the weather file was not defined in the workflow, if the
+        #   result is false the weather file was set but could not be found on the filesystem, if a string the weather
+        #   file was defined and it's existence verified. The order of precedence for paths is as follows: 1 - an
+        #   absolute path defined in wf, 2 - the wf_search_array, should it be defined, joined with the weather file and
+        #   appended to the directory, with each entry in the array searched until the wf is found
+        #
+        def get_weather_file_from_fs(directory, wf, wf_search_array, logger)
+          raise "wf was defined as #{wf}. Please correct" unless wf
+          weather_file = nil
+          if Pathname.new(wf).absolute?
+            weather_file = wf
+          else
+            wf_search_array.each do |wf_dir|
+              logger.warn "The path #{wf_dir} does not exist" unless File.exist? File.join(directory, wf_dir)
+              next unless File.exist? File.join(directory, wf_dir)
+              if Dir.entries(File.join(directory, wf_dir)).include? File.basename(wf)
+                weather_file = File.absolute_path(File.join(directory, wf_dir, wf))
+                break
+              end
+            end
+          end
+          unless weather_file
+            logger.warn 'The weather file was not found on the filesystem'
+            return nil
+          end
+          logger.info "Weather file with precedence in the file system is #{weather_file}"
+          unless File.exist? weather_file
+            logger.warn 'The weather file could not be found on the filesystem'
+            weather_file = false
+          end
+          weather_file
+        end
+      end
+    end
+  end
+end

data/lib/openstudio/workflow/util.rb

@@ -0,0 +1,14 @@
+module OpenStudio
+  module Workflow
+    # Hard load utils for the moment
+    #
+    module Util
+      require 'openstudio/workflow/util/io'
+      require 'openstudio/workflow/util/measure'
+      require 'openstudio/workflow/util/weather_file'
+      require 'openstudio/workflow/util/model'
+      require 'openstudio/workflow/util/energyplus'
+      require 'openstudio/workflow/util/post_process'
+    end
+  end
+end

data/lib/openstudio/workflow/version.rb

@@ -19,6 +19,6 @@
 
 module OpenStudio
   module Workflow
-    VERSION = '1.0.0.pat1'
+    VERSION = '1.0.0'.freeze # Suffixes must have periods (not dashes)
   end
 end