openstudio-workflow 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc2c789cb6f241ce8a9693f4fd52c2cea857a6ab
-  data.tar.gz: 8ead2192e95b45850c00a0b0e048d326eef05ead
+  metadata.gz: 8246584121ca33fc5d5b10e34ff852d0884d6de6
+  data.tar.gz: 979bcb2d5e8445a6cf69cf1e948d091ae1a0cf61
 SHA512:
-  metadata.gz: 31c526b09291097df08aef677a51e1dd46bfb64710c0a3a0fef1b6e2a8f9fc2ab0d23973afcdd651dedfe7f17f77266fcb6ef7d5df67a80d39addb590ef9b358
-  data.tar.gz: acaee1bf5ae39b1f4be3f67b4d292dfe790f0415e1f1aa04552bdae09ad1003ad0b44c45939a4b5f0f4ed76ff76ec1522871a61dd2fd5c508c8848b88685cbec
+  metadata.gz: 843a1e9af4d17e54b28d30d83fe9020502c29ac71e350f3e1625694feea3120a97826dd8ae8c383e3f4bf75ab91ada7b2c67de3d737ed339bdbd099f4ba1517f
+  data.tar.gz: a9463ff16b20e54881097c2fae31ad82f9ed9a1d3398a7fc0987640abaa824fba58923bfd4783c52572d91ca673c967081c0d735e281e7657d30bd193b373773

data/CHANGELOG.md

@@ -4,6 +4,15 @@ OpenStudio::Workflow Change Log
 Unreleased
 --------------
 
+Version 0.0.2
+--------------
+
+* Support reporting measures
+* Reduce logging messages
+* Keep IDF files
+* Remove mtr and eso files after simulation completes
+* If measure changes weather file, then use the new weather file with the analysis.json weather path
+
 Version 0.0.1
 --------------
 

data/README.md

@@ -60,7 +60,7 @@ The workflow manager can also use MongoDB to receive instructions on the workflo
 * Read the analysis.json file to determine the states that are going to run instead of (or inaddition to) passing them into the constructor
 * Implement better error handling with custom exception classes
 * Implement a different measure directory, seed model directory, and weather file directory option
-* Dynamically add other "states" to the workflow
+* ~Dynamically add other "states" to the workflow~
 * Create and change into a unique directory when running measures
 * ~~Implement Error State~~
 * ~~Implement MongoDB Adapter~~
@@ -68,7 +68,6 @@ The workflow manager can also use MongoDB to receive instructions on the workflo
 * Add a results adapter to return a string as the last call based on the source of the call. (e.g. R, command line, C++, etc).
 * Implement a logger in the Adapters, right now they are unable to log
 * Hook up the measure groups based workflows
-* More tests!
 * ~~Add xml workflow item~~
 
 ## Testing and Development
@@ -95,3 +94,41 @@ Run `rspec` or `rake` to execute the tests.
 
 ## Development
 
+If you are testing changes to OpenStudio source code and want to test these on the Vagrant machine you can use the source configuration.  This creates a new virtual machine which can be accessed by adding the name 'source' to the end of standard Vagrant commands.  To set up this machine, build a custom version of OpenStudio, and install that version for testing follow these steps:
+
+* vagrant up source
+* vagrant ssh source
+* sudo apt-get install dpkg-dev git cmake-curses-gui qt5-default libqt5webkit5-dev libboost1.55-all-dev swig ruby2.0 libssl-dev libxt-dev doxygen graphviz
+* sudo ln -s /usr/lib/x86_64-linux-gnu/libruby-2.0.so.2.0.0 /usr/lib/x86_64-linux-gnu/libruby.so.2.0
+** Install clang (OPTIONAL):
+** wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
+** sudo apt-add-repository 'deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.5 main'
+** sudo apt-get update
+** sudo apt-get install clang-3.5 
+** echo 'export CC=/usr/bin/clang-3.5' >> ~/.bashrc
+** echo 'export CXX=/usr/bin/clang++-3.5' >> ~/.bashrc
+** source ~/.bashrc
+** cd /home/vagrant
+* git clone https://github.com/NREL/OpenStudio.git openstudio
+* cd openstudio
+* git checkout your_branch_name
+* mkdir build
+* cd build
+* cmake .. -DBUILD_PACKAGE=TRUE -DCMAKE_INSTALL_PREFIX=$OPENSTUDIO_ROOT -DRUBY_EXECUTABLE=/usr/local/rbenv/versions/2.0.0-p481/bin/ruby
+* make -j4
+
+To install do either:
+* cd OpenStudioCore-prefix/src/OpenStudioCore-build/
+* sudo make install
+
+or:
+* make package
+* sudo ./OpenStudio-1.5.1.02b7131b4c-Linux.sh --prefix=/usr/local --exclude-subdir --skip-license
+
+Next you have to do this:
+* export RUBYLIB=/usr/local/Ruby
+* export LD_LIBRARY_PATH=/usr/local/lib
+
+Then you can test that you are using your build by comparing the output of these two commands:
+* ruby -e "require 'openstudio'" -e "puts OpenStudio::openStudioLongVersion"
+* git rev-parse --short HEAD

data/Rakefile

@@ -12,9 +12,20 @@ require 'rake'
 require 'rspec/core'
 require 'rspec/core/rake_task'
 
-#require 'rubocop/rake_task'
+# require 'rubocop/rake_task'
 RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
   spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
+require 'rubocop/rake_task'
+desc 'Run RuboCop on the lib directory'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.options = ['--no-color', '--out=rubocop-results.xml']
+  task.formatters = ['RuboCop::Formatter::CheckstyleFormatter']
+  task.requires = ['rubocop/formatter/checkstyle_formatter']
+  # don't abort rake on failure
+  task.fail_on_error = false
+end
+
 task default: [:spec]

data/lib/openstudio-workflow.rb

@@ -23,6 +23,7 @@ require 'multi_json'
 require 'colored'
 require 'fileutils'
 require 'json' # needed for a single pretty generate call
+require 'pathname'
 
 begin
   require 'facter'
@@ -33,6 +34,7 @@ end
 require 'openstudio/workflow/version'
 require 'openstudio/workflow/multi_delegator'
 require 'openstudio/workflow/run'
+require 'openstudio/workflow/jobs/lib/apply_measures'
 
 begin
   require 'openstudio'
@@ -42,14 +44,39 @@ rescue LoadError => e
   puts 'OpenStudio did not load, but most functionality is still available. Will try to continue...'.red
 end
 
+# some core extensions
+class String
+  def snake_case
+    gsub(/::/, '/')
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .tr(' -', '__')
+        .downcase
+  end
+end
+
 module OpenStudio
   module Workflow
     extend self
 
     # Create a new workflow instance using the defined adapter and UUID
-    def load(adapter_name, run_directory, options={})
-      defaults = {adapter_options: {}}
+    def load(adapter_name, run_directory, options = {})
+      defaults = { adapter_options: {} }
       options = defaults.merge(options)
+
+      # Convert various paths to absolute paths
+      if options[:adapter_options] && options[:adapter_options][:mongoid_path] &&
+          (Pathname.new options[:adapter_options][:mongoid_path]).absolute? == false
+        options[:adapter_options][:mongoid_path] = File.expand_path options[:adapter_options][:mongoid_path]
+      end
+      if options[:analysis_root_path] &&
+          (Pathname.new options[:analysis_root_path]).absolute? == false
+        options[:analysis_root_path] = File.expand_path options[:analysis_root_path]
+      end
+      unless (Pathname.new run_directory).absolute?
+        # relative to wherever you are running the script
+        run_directory = File.expand_path run_directory
+      end
       adapter = load_adapter adapter_name, options[:adapter_options]
       run_klass = OpenStudio::Workflow::Run.new(adapter, run_directory, options)
       # return the run class
@@ -58,10 +85,10 @@ module OpenStudio
 
     private
 
-    def load_adapter(name, adapter_options={})
+    def load_adapter(name, adapter_options = {})
       require "openstudio/workflow/adapters/#{name.downcase}"
       klass_name = name.to_s.split('_').map(&:capitalize) * ''
-      #pp "#{klass_name} is the adapter class name"
+      # pp "#{klass_name} is the adapter class name"
       klass = OpenStudio::Workflow::Adapters.const_get(klass_name).new(adapter_options)
       klass
     end

data/lib/openstudio/workflow/adapter.rb

@@ -21,26 +21,25 @@
 module OpenStudio
   module Workflow
     class Adapter
-
       attr_accessor :options
 
-      def initialize(options={})
+      def initialize(options = {})
         @options = options
         @log = nil
       end
 
-      #class << self
-      #attr_reader :problem
+      # class << self
+      # attr_reader :problem
 
-      def load(filename, options={})
+      def load(filename, options = {})
         instance.load(filename, options)
       end
 
-      def communicate_started(id, options = {})
+      def communicate_started(id, _options = {})
         instance.communicate_started id
       end
 
-      def get_datapoint(id, options={})
+      def get_datapoint(id, options = {})
         instance.get_datapoint id, options
       end
 
@@ -60,9 +59,9 @@ module OpenStudio
         instance.communicate_failure id
       end
 
-      def get_logger(file, options={})
+      def get_logger(file, options = {})
         instance.get_logger file, options
       end
     end
   end
-end
+end

data/lib/openstudio/workflow/adapters/local.rb

@@ -24,20 +24,19 @@ module OpenStudio
   module Workflow
     module Adapters
       class Local < Adapter
-        def initialize(options={})
-
+        def initialize(options = {})
           super
         end
 
         # Tell the system that the process has started
-        def communicate_started(directory, options = {})
+        def communicate_started(directory, _options = {})
           # Watch out for namespace conflicts (::Time is okay but Time is OpenStudio::Time)
           File.open("#{directory}/started.job", 'w') { |f| f << "Started Workflow #{::Time.now}" }
         end
 
         # Get the data point from the path
-        def get_datapoint(directory, options={})
-          defaults = {datapoint_filename: 'datapoint.json', format: 'json'}
+        def get_datapoint(directory, options = {})
+          defaults = { datapoint_filename: 'datapoint.json', format: 'json' }
           options = defaults.merge(options)
 
           # how do we log within this file?
@@ -51,7 +50,7 @@ module OpenStudio
         # Get the Problem/Analysis definition from the local file
         # TODO: rename this to get_analysis_definintion (or something like that)
         def get_problem(directory, options = {})
-          defaults = {problem_filename: 'problem.json', format: 'json'}
+          defaults = { problem_filename: 'problem.json', format: 'json' }
           options = defaults.merge(options)
 
           if File.exist? "#{directory}/#{options[:problem_filename]}"
@@ -61,7 +60,7 @@ module OpenStudio
           end
         end
 
-        def communicate_intermediate_result(directory)
+        def communicate_intermediate_result(_directory)
           # noop
         end
 
@@ -73,37 +72,51 @@ module OpenStudio
         # zip up the results of the simuation.
         def communicate_failure(directory)
           File.open("#{directory}/failed.job", 'w') { |f| f << "Failed Workflow #{::Time.now}" }
-          #@communicate_module.communicate_failure(@communicate_object, os_directory)
+          # @communicate_module.communicate_failure(@communicate_object, os_directory)
         end
 
         def communicate_results(directory, results)
+          zip_results(directory, 'workflow')
+
           if results.is_a? Hash
             File.open("#{directory}/datapoint_out.json", 'w') { |f| f << JSON.pretty_generate(results) }
           else
             pp "Unknown datapoint result type. Please handle #{results.class}"
-            #data_point_json_path = OpenStudio::Path.new(run_dir) / OpenStudio::Path.new('data_point_out.json')
-            #os_data_point.saveJSON(data_point_json_path, true)
+            # data_point_json_path = OpenStudio::Path.new(run_dir) / OpenStudio::Path.new('data_point_out.json')
+            # os_data_point.saveJSON(data_point_json_path, true)
           end
-          #end
-        end
-
-        # TODO: can this be deprecated in favor a checking the class?
-        def communicate_results_json(eplus_json, analysis_dir)
-          # noop
-        end
-
-        def reload
-          # noop
+          # end
         end
 
         # For the local adapter send back a handle to a file to append the data. For this adapter
         # the log messages are likely to be the same as the run.log messages.
         # TODO: do we really want two local logs from the Local adapter? One is in the run dir and the other is in the root
-        def get_logger(directory, options={})
-          @log ||= File.open("#{directory}/local_adapter.log", "w")
+        def get_logger(directory, _options = {})
+          @log ||= File.open("#{directory}/local_adapter.log", 'w')
           @log
         end
 
+        # TODO: this uses a system call to zip results at the moment
+        def zip_results(directory, _analysis_type = 'workflow')
+          current_dir = Dir.pwd
+          begin
+            # create zip file using a system call
+            # @logger.info "Zipping up data point #{analysis_dir}"
+            if File.directory? directory
+              Dir.chdir(directory)
+              `zip -9 -r --exclude=*.rb* data_point.zip .`
+            end
+
+            # zip up only the reports folder
+            report_dir = 'reports'
+            # @logger.info "Zipping up Analysis Reports Directory #{report_dir}/reports"
+            if File.directory? report_dir
+              `zip -9 -r data_point_reports.zip reports`
+            end
+          ensure
+            Dir.chdir(current_dir)
+          end
+        end
       end
     end
   end

data/lib/openstudio/workflow/adapters/mongo.rb

@@ -37,7 +37,7 @@ module OpenStudio
       class Mongo < Adapter
         attr_reader :datapoint
 
-        def initialize(options={})
+        def initialize(options = {})
           super
 
           require 'mongoid'
@@ -52,7 +52,7 @@ module OpenStudio
         end
 
         # Tell the system that the process has started
-        def communicate_started(directory, options={})
+        def communicate_started(directory, options = {})
           # Watch out for namespace conflicts (::Time is okay but Time is OpenStudio::Time)
           File.open("#{directory}/started.job", 'w') { |f| f << "Started Workflow #{::Time.now}" }
 
@@ -61,40 +61,52 @@ module OpenStudio
           @datapoint.status_message = ''
           @datapoint.run_start_time = ::Time.now
 
-
-          # TODO: use a different method to determine if this is an amazon account
+          # TODO: Get Facter to play well on windows and replace 'socket'
           # TODO: use the ComputeNode model to pull out the information so that we can reuse the methods
           # Determine what the IP address is of the worker node and save in the data point
-          require 'socket'
-          if Socket.gethostname =~ /os-.*/
-            # Maybe use this in the future: /sbin/ifconfig eth1|grep inet|head -1|sed 's/\:/ /'|awk '{print $3}'
-            # Must be on vagrant and just use the hostname to do a lookup
-            map = {
+
+          retries = 0
+          begin
+            require 'socket'
+            if Socket.gethostname =~ /os-.*/
+              # Maybe use this in the future: /sbin/ifconfig eth1|grep inet|head -1|sed 's/\:/ /'|awk '{print $3}'
+              # Must be on vagrant and just use the hostname to do a lookup
+              map = {
                 'os-server' => '192.168.33.10',
                 'os-worker-1' => '192.168.33.11',
                 'os-worker-2' => '192.168.33.12'
-            }
-            @datapoint.ip_address = map[Socket.gethostname]
-            @datapoint.internal_ip_address = @datapoint.ip_address
-
-            # TODO: add back in the instance id
-          elsif Socket.gethostname =~ /instance-data.ec2.internal.*/i
-            # On amazon, you have to hit an API to determine the IP address because
-            # of the internal/external ip addresses
-
-            # NL: add the suppress
-            public_ip_address = `curl -sL http://169.254.169.254/latest/meta-data/public-ipv4`
-            internal_ip_address = `curl -sL http://169.254.169.254/latest/meta-data/local-ipv4`
-            # instance_information = `curl -sL http://169.254.169.254/latest/meta-data/instance-id`
-            # instance_information = `curl -sL http://169.254.169.254/latest/meta-data/ami-id`
-            @datapoint.ip_address = public_ip_address
-            @datapoint.internal_ip_address = internal_ip_address
-            # @datapoint.server_information = instance_information
-          else
-            if Gem.loaded_specs["facter"]
-              # TODO: add hostname via the facter gem (and anything else?)
-              @datapoint.ip_address = Facter.fact(:ipaddress).value
-              @datapoint.internal_ip_address = Facter.fact(:hostname).value
+              }
+              @datapoint.ip_address = map[Socket.gethostname]
+              @datapoint.internal_ip_address = @datapoint.ip_address
+            else
+              if Gem.loaded_specs['facter']
+                # Check if we are on amazon
+                if Facter.fact(:ec2_metadata)
+                  # must be on amazon
+                  m = Facter.fact(:ec2_metadata).value
+
+                  @datapoint.ip_address = m['public-ipv4'] ? m['public-ipv4'] : 'unknown'
+                  @datapoint.internal_ip_address = m['local-ipv4'] ? m['local-ipv4'] : 'unknown'
+                else
+                  @datapoint.ip_address = Facter.fact(:ipaddress).value
+                  @datapoint.internal_ip_address = Facter.fact(:ipaddress).value
+                end
+              end
+            end
+          rescue => e
+            # catch any exceptions. It appears that if a new instance of amazon starts, then it is likely that
+            # the Facter for AWS may not be initialized yet. Retry after waiting for 15 seconds if this happens.
+            # If this fails out, then the only issue with this is that the data point won't be downloaded because
+            # the worker node is not known
+
+            # retry just in case
+            if retries < 30 # try for up to 5 minutes
+              retries += 1
+              sleep 10
+              retry
+            else
+              raise "could not find Facter based data for worker node after #{retries} retries with message #{e.message}"
+              # just do nothing for now
             end
           end
 
@@ -102,22 +114,27 @@ module OpenStudio
         end
 
         # Get the data point from the path
-        def get_datapoint(directory, options={})
+        def get_datapoint(directory, options = {})
           # TODO : make this a conditional on when to create one vs when to error out.
           # keep @datapoint as the model instance
           @datapoint = DataPoint.find_or_create_by(uuid: options[:datapoint_id])
 
           # convert to JSON for the workflow - and rearrange the version (fix THIS)
           datapoint_hash = {}
-          unless @datapoint.nil?
+          if @datapoint.nil?
+            fail 'Could not find datapoint'
+          else
             datapoint_hash[:data_point] = @datapoint.as_document.to_hash
             # TODO: Can i remove this openstudio_version stuff?
-            #datapoint_hash[:openstudio_version] = datapoint_hash[:openstudio_version]
+            # datapoint_hash[:openstudio_version] = datapoint_hash[:openstudio_version]
 
             # TODO: need to figure out how to get symbols from mongo.
-            datapoint_hash = MultiJson.load(MultiJson.dump(datapoint_hash, pretty: true), symbolize_keys: true)
-          else
-            fail "Could not find datapoint"
+            datapoint_hash = MultiJson.load(MultiJson.dump(datapoint_hash), symbolize_keys: true)
+
+            # save to disk for inspection
+            save_dp = File.join(directory, 'data_point.json')
+            FileUtils.rm_f save_dp if File.exist? save_dp
+            File.open(save_dp, 'w') { |f| f << MultiJson.dump(datapoint_hash, pretty: true) }
           end
 
           datapoint_hash
@@ -125,7 +142,7 @@ module OpenStudio
 
         # TODO: cleanup these options.  Make them part of the class. They are just unwieldly here.
         def get_problem(directory, options = {})
-          defaults = {format: 'json'}
+          defaults = { format: 'json' }
           options = defaults.merge(options)
 
           get_datapoint(directory, options) unless @datapoint
@@ -133,7 +150,7 @@ module OpenStudio
           if @datapoint
             analysis = @datapoint.analysis.as_document.to_hash
           else
-            fail "Cannot retrieve problem because datapoint was nil"
+            fail 'Cannot retrieve problem because datapoint was nil'
           end
 
           analysis_hash = {}
@@ -147,11 +164,11 @@ module OpenStudio
           analysis_hash
         end
 
-        def communicate_intermediate_result(directory)
+        def communicate_intermediate_result(_directory)
           # noop
         end
 
-        def communicate_complete(directory)
+        def communicate_complete(_directory)
           @datapoint.run_end_time = ::Time.now
           @datapoint.status = 'completed'
           @datapoint.status_message = 'completed normal'
@@ -175,33 +192,25 @@ module OpenStudio
         def communicate_results(directory, results)
           zip_results(directory, 'workflow')
 
-          #@logger.info 'Saving EnergyPlus JSON file'
+          # @logger.info 'Saving EnergyPlus JSON file'
           if results
-            @datapoint.results ? @datapoint.results.merge!(eplus_json) : @datapoint.results = results
+            @datapoint.results ? @datapoint.results.merge!(results) : @datapoint.results = results
           end
           result = @datapoint.save! # redundant because next method calls save too.
+
           if result
-            #@logger.info 'Successfully saved result to database'
+            # @logger.info 'Successfully saved result to database'
           else
-            #@logger.error 'ERROR saving result to database'
+            # @logger.error 'ERROR saving result to database'
           end
         end
 
-        # TODO: can this be deprecated in favor a checking the class?
-        def communicate_results_json(eplus_json, analysis_dir)
-          # noop
-        end
-
-        # TODO: not needed anymore i think...
-        def reload
-          # noop
-        end
-
         # TODO: Implement the writing to the mongo_db for logging
-        def get_logger(directory, options={})
+        def get_logger(directory, options = {})
           # get the datapoint object
           get_datapoint(directory, options) unless @datapoint
           @log = OpenStudio::Workflow::Adapters::MongoLog.new(@datapoint)
+
           @log
         end
 
@@ -213,47 +222,28 @@ module OpenStudio
           DataPoint.find_or_create_by(uuid: uuid)
         end
 
-        # TODO: this uses a system call to zip results at the moment
-        def zip_results(analysis_dir, analysis_type = 'workflow')
-          eplus_search_path = nil
+        # TODO: this uses a system call to zip results at the moment, replace with rubylib
+        def zip_results(directory, _analysis_type = 'workflow')
           current_dir = Dir.pwd
-          FileUtils.mkdir_p "#{analysis_dir}/reports"
-          case analysis_type
-            when 'workflow'
-              eplus_search_path = "#{analysis_dir}/*run*/eplustbl.htm"
-            when 'runmanager'
-              eplus_search_path = "#{analysis_dir}/*EnergyPlus*/eplustbl.htm"
-          end
-
-          # copy some files into a report folder
-          eplus_html = Dir.glob(eplus_search_path).last || nil
-          if eplus_html
-            #@logger.info "Checking for HTML Report: #{eplus_html}"
-            if File.exist? eplus_html
-              # do some encoding on the html if possible
-              html = File.read(eplus_html)
-              html = html.force_encoding('ISO-8859-1').encode('utf-8', replace: nil)
-              File.open("#{analysis_dir}/reports/eplustbl.html", 'w') { |f| f << html }
+          begin
+            # create zip file using a system call
+            # @logger.info "Zipping up data point #{analysis_dir}"
+            if File.directory? directory
+              Dir.chdir(directory)
+              `zip -9 -r --exclude=*.rb* data_point_#{@datapoint.uuid}.zip .`
             end
-          end
 
-          # create zip file using a system call
-          #@logger.info "Zipping up Analysis Directory #{analysis_dir}"
-          if File.directory? analysis_dir
-            Dir.chdir(analysis_dir)
-            `zip -9 -r --exclude=*.rb* data_point_#{@datapoint.uuid}.zip .`
-          end
-
-          # zip up only the reports folder
-          report_dir = "#{analysis_dir}"
-          #@logger.info "Zipping up Analysis Reports Directory #{report_dir}/reports"
-          if File.directory? report_dir
-            Dir.chdir(report_dir)
-            `zip -r data_point_#{@datapoint.uuid}_reports.zip reports`
+            # zip up only the reports folder
+            report_dir = 'reports'
+            # @logger.info "Zipping up Analysis Reports Directory #{report_dir}/reports"
+            if File.directory? report_dir
+              `zip -9 -r data_point_#{@datapoint.uuid}_reports.zip reports`
+            end
+          ensure
+            Dir.chdir(current_dir)
           end
-          Dir.chdir(current_dir)
         end
       end
     end
   end
-end
+end