urbanopt-scenario 0.2.0 → 0.4.2

data/docs/package.json

@@ -10,17 +10,21 @@
   },
   "author": "NREL",
   "dependencies": {
-    "highlight.js": "^9.15.6",
-    "json-schema-ref-parser": "^6.1.0",
+    "highlight.js": "^10.2.0",
+    "json-schema-ref-parser": "^9.0.6",
     "json-schema-view-js": "git+https://git@github.com/bgschiller/json-schema-view-js.git",
-    "vuepress": "^1.2.0",
-    "webpack-dev-middleware": "^3.6.0"
+    "vuepress": "^1.5.4",
+    "webpack-dev-middleware": "^3.7.2"
   },
   "devDependencies": {
-    "braces": ">=2.3.1",
-    "gh-pages": "^2.0.1",
-    "js-yaml": ">=3.13.1",
-    "serialize-javascript": ">=2.1.1",
-    "set-value": ">=2.0.1"
+    "braces": "^3.0.2",
+    "dot-prop": "^5.3.0",
+    "gh-pages": "^3.1.0",
+    "js-yaml": "^3.14.0",
+    "minimist": ">=1.2.3",
+    "node-forge": ">=0.10.0",
+    "serialize-javascript": "^5.0.1",
+    "set-value": "^3.0.2",
+    "yargs-parser": ">=18.1.1"
   }
 }

data/lib/urbanopt-scenario.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,

data/lib/urbanopt/scenario.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -42,5 +42,6 @@ require 'urbanopt/scenario/scenario_runner_osw'
 require 'urbanopt/scenario/simulation_dir_base'
 require 'urbanopt/scenario/simulation_dir_osw'
 require 'urbanopt/scenario/simulation_mapper_base'
+require 'urbanopt/scenario/scenario_visualization'
 
 require 'urbanopt/scenario/default_reports'

data/lib/urbanopt/scenario/default_reports.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -31,10 +31,5 @@
 ##
 # Retrieve all default_reports classes.
 ##
-require 'urbanopt/scenario/default_reports/construction_cost'
-require 'urbanopt/scenario/default_reports/feature_report'
-require 'urbanopt/scenario/default_reports/logger'
-require 'urbanopt/scenario/default_reports/program'
-require 'urbanopt/scenario/default_reports/reporting_period'
-require 'urbanopt/scenario/default_reports/scenario_report'
-require 'urbanopt/scenario/default_reports/timeseries_csv'
+
+require 'urbanopt/reporting/default_reports'

data/lib/urbanopt/scenario/extension.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,

data/lib/urbanopt/scenario/logger.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,

data/lib/urbanopt/scenario/scenario_base.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,

data/lib/urbanopt/scenario/scenario_csv.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -52,7 +52,6 @@ module URBANopt
 
       def initialize(name, root_dir, run_dir, feature_file, mapper_files_dir, csv_file, num_header_rows)
         super(name, root_dir, run_dir, feature_file)
-
         @mapper_files_dir = mapper_files_dir
         @csv_file = csv_file
         @num_header_rows = num_header_rows
@@ -73,13 +72,27 @@ module URBANopt
 
       # Require all simulation mappers in mapper_files_dir
       def load_mapper_files
-        Dir.glob(File.join(@mapper_files_dir, '/*.rb')).each do |f|
-          begin
-            require(f)
-          rescue LoadError => e
-            @@logger.error(e.message)
-            raise
-          end
+        dirs = Dir.glob(File.join(@mapper_files_dir, '/*.rb'))
+        # order is not guaranteed...attempt to add Baseline first, then High Efficiency
+        ordered_dirs = []
+        bindex = dirs.find_index { |i| i.include? 'Baseline.rb' }
+        if bindex
+          ordered_dirs << dirs[bindex]
+          dirs.delete_at(bindex)
+        end
+        hindex = dirs.find_index { |i| i.include? 'HighEfficiency.rb' }
+        if hindex
+          ordered_dirs << dirs[hindex] if hindex
+          dirs.delete_at(hindex)
+        end
+        # then the rest
+        ordered_dirs += dirs
+
+        ordered_dirs.each do |f|
+          require(f)
+        rescue LoadError => e
+          @@logger.error(e.message)
+          raise
         end
       end
 

data/lib/urbanopt/scenario/scenario_datapoint_base.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -59,6 +59,13 @@ module URBANopt
         @feature.feature_type
       end
 
+      ##
+      # Gets the type of a feature
+      ##
+      def feature_location
+        @feature.feature_location
+      end
+
       ##
       # Return the directory that this datapoint will run in.
       ##

data/lib/urbanopt/scenario/scenario_post_processor_base.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,

data/lib/urbanopt/scenario/scenario_post_processor_default.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -29,12 +29,12 @@
 # *********************************************************************************
 
 require 'urbanopt/scenario/scenario_post_processor_base'
-require 'urbanopt/scenario/default_reports'
-require 'urbanopt/scenario/default_reports/logger'
+require 'urbanopt/reporting/default_reports'
 
 require 'csv'
 require 'json'
 require 'fileutils'
+require 'sqlite3'
 
 module URBANopt
   module Scenario
@@ -47,10 +47,11 @@ module URBANopt
       def initialize(scenario_base)
         super(scenario_base)
 
-        initialization_hash = { directory_name: scenario_base.run_dir, name: scenario_base.name, id: scenario_base.name }
-        @scenario_result = URBANopt::Scenario::DefaultReports::ScenarioReport.new(initialization_hash)
+        @initialization_hash = { directory_name: scenario_base.run_dir, name: scenario_base.name, id: scenario_base.name, root_dir: scenario_base.root_dir }
+        @scenario_result = URBANopt::Reporting::DefaultReports::ScenarioReport.new(@initialization_hash)
+        @default_save_name = 'default_scenario_report'
 
-        @@logger ||= URBANopt::Scenario::DefaultReports.logger
+        @@logger ||= URBANopt::Reporting::DefaultReports.logger
       end
 
       ##
@@ -70,7 +71,7 @@ module URBANopt
       # [parameters:]
       # +simulation_dir+ - _SimulationDirOSW_ - An object on SimulationDirOSW class.
       def add_simulation_dir(simulation_dir)
-        feature_reports = URBANopt::Scenario::DefaultReports::FeatureReport.from_simulation_dir(simulation_dir)
+        feature_reports = URBANopt::Reporting::DefaultReports::FeatureReport.from_simulation_dir(simulation_dir)
 
         feature_reports.each do |feature_report|
           if feature_report.to_hash[:simulation_status] == 'Complete'
@@ -83,12 +84,113 @@ module URBANopt
         return feature_reports
       end
 
+      # Create database file with scenario-level results
+      #   Sum values for each timestep across all features. Save to new table in a new database
+      def create_scenario_db_file(file_name = @default_save_name)
+        new_db_file = File.join(@initialization_hash[:directory_name], "#{file_name}.db")
+        scenario_db = SQLite3::Database.open new_db_file
+        scenario_db.execute "CREATE TABLE IF NOT EXISTS ReportData(
+          TimeIndex INTEGER,
+          Year VARCHAR(255),
+          Month VARCHAR(255),
+          Day VARCHAR(255),
+          Hour VARCHAR(255),
+          Minute VARCHAR(255),
+          Dst INTEGER,
+          ReportDataDictionaryIndex INTEGER,
+          Value INTEGER
+          )"
+
+        values_arr = []
+        feature_list = Pathname.new(@initialization_hash[:directory_name]).children.select(&:directory?) # Folders in the run/scenario directory
+
+        # get scenario CSV
+        scenario_csv = File.join(@initialization_hash[:root_dir], @initialization_hash[:name] + '.csv')
+        if File.exist?(scenario_csv)
+          # csv found
+          feature_ids = CSV.read(scenario_csv, headers: true)
+          feature_list = []
+          # loop through building feature ids from scenario csv
+          feature_ids['Feature Id'].each do |feature|
+            if Dir.exist?(File.join(@initialization_hash[:directory_name], feature))
+              feature_list << File.join(@initialization_hash[:directory_name], feature)
+            else
+              puts "warning: did not find a directory for datapoint #{feature}...skipping"
+            end
+          end
+        else
+          raise "Couldn't find scenario CSV: #{scenario_csv}"
+        end
+        feature_1_name = File.basename(feature_list[0]) # Get name of first feature, so we can read eplusout.sql from there
+        uo_output_sql_file = File.join(@initialization_hash[:directory_name], feature_1_name, 'eplusout.sql')
+        feature_list.each do |feature| # Loop through each feature in the scenario
+          feature_db = SQLite3::Database.open uo_output_sql_file
+          # Doing "db.results_as_hash = true" is prettier, but in this case significantly slower.
+
+          # RDDI == 10 is the timestep value for facility electricity in OS 3.0.1
+          # TODO: Dynamically read RDDI from table RDDI, insted of hardcoding it
+          elec_query = feature_db.query "SELECT ReportData.TimeIndex, Time.Year, Time.Month, Time.Day, Time.Hour, Time.Minute, Time.Dst, ReportData.Value
+          FROM ReportData
+          INNER JOIN Time ON Time.TimeIndex=ReportData.TimeIndex
+          WHERE ReportDataDictionaryIndex == 10
+          ORDER BY ReportData.TimeIndex"
+
+          elec_query.each do |row| # Add up all the values for electricity usage across all Features at this timestep
+            # row[0] == TimeIndex, row[1] == Value
+
+            arr_match = values_arr.find { |v| v[:time_index] == row[0] }
+            if arr_match.nil?
+              # add new row to value_arr
+              values_arr << { time_index: row[0], year: row[1], month: row[2], day: row[3], hour: row[4], minute: row[5], dst: row[6], elec_val: Float(row[7]), gas_val: 0 }
+            else
+              # running sum
+              arr_match[:elec_val] += Float(row[7])
+            end
+          end # End elec_query
+          elec_query.close
+
+          # RDDI == 1382 is the timestep value for facility gas in OS 3.0.1
+          # TODO: Dynamically read RDDI from table RDDI, insted of hardcoding it
+          gas_query = feature_db.query "SELECT ReportData.TimeIndex, Time.Year, Time.Month, Time.Day, Time.Hour, Time.Minute, Time.Dst, ReportData.Value
+          FROM ReportData
+          INNER JOIN Time ON Time.TimeIndex=ReportData.TimeIndex
+          WHERE ReportDataDictionaryIndex == 1382
+          ORDER BY ReportData.TimeIndex"
+
+          gas_query.each do |row|
+            # row[0] == TimeIndex, row[1] == Value
+            arr_match = values_arr.find { |v| v[:time_index] == row[0] }
+            if arr_match.nil?
+              # add new row to value_arr
+              values_arr << { time_index: row[0], year: row[1], month: row[2], day: row[3], hour: row[4], minute: row[5], dst: row[6], gas_val: Float(row[7]), elec_val: 0 }
+            else
+              # running sum
+              arr_match[:gas_val] += Float(row[7])
+            end
+          end # End gas_query
+          gas_query.close
+          feature_db.close
+        end # End feature_list loop
+
+        elec_sql = []
+        gas_sql = []
+        values_arr.each do |i|
+          elec_sql << "(#{i[:time_index]}, #{i[:year]}, #{i[:month]}, #{i[:day]}, #{i[:hour]}, #{i[:minute]}, #{i[:dst]}, 10, #{i[:elec_val]})"
+          gas_sql << "(#{i[:time_index]}, #{i[:year]}, #{i[:month]}, #{i[:day]}, #{i[:hour]}, #{i[:minute]}, #{i[:dst]}, 1382, #{i[:gas_val]})"
+        end
+
+        # Put summed Values into the database
+        scenario_db.execute("INSERT INTO ReportData VALUES #{elec_sql.join(', ')}")
+        scenario_db.execute("INSERT INTO ReportData VALUES #{gas_sql.join(', ')}")
+        scenario_db.close
+      end
+
       ##
       # Save scenario result
       ##
       # [parameters:]
       # +file_name+ - _String_ - Assign a name to the saved scenario results file
-      def save(file_name = 'default_scenario_report')
+      def save(file_name = @default_save_name)
         @scenario_result.save
 
         return @scenario_result

data/lib/urbanopt/scenario/scenario_post_processor_opendss.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -29,8 +29,7 @@
 # *********************************************************************************
 
 # require 'urbanopt/scenario/scenario_post_processor_base'
-require 'urbanopt/scenario/default_reports'
-require 'urbanopt/scenario/default_reports/logger'
+require 'urbanopt/reporting/default_reports'
 
 require 'csv'
 require 'json'
@@ -64,7 +63,7 @@ module URBANopt
         @feature_reports_data = {}
 
         # initialize logger
-        @@logger ||= URBANopt::Scenario::DefaultReports.logger
+        @@logger ||= URBANopt::Reporting::DefaultReports.logger
       end
 
       # load opendss data
@@ -166,9 +165,9 @@ module URBANopt
             File.write(File.join(transformer_dir, 'feature_reports', 'default_feature_report_opendss' + '.csv'), transformer_csv)
 
             # create transformer report
-            transformer_report = URBANopt::Scenario::DefaultReports::FeatureReport.new(id: k, name: k, directory_name: transformer_dir, feature_type: 'Transformer',
-                                                                                       timesteps_per_hour: @scenario_report.timesteps_per_hour,
-                                                                                       simulation_status: 'complete')
+            transformer_report = URBANopt::Reporting::DefaultReports::FeatureReport.new(id: k, name: k, directory_name: transformer_dir, feature_type: 'Transformer',
+                                                                                        timesteps_per_hour: @scenario_report.timesteps_per_hour,
+                                                                                        simulation_status: 'complete')
 
             # assign results to transfomrer report
             transformer_report.power_distribution.over_voltage_hours = over_voltage_hrs

data/lib/urbanopt/scenario/scenario_runner_base.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -55,7 +55,7 @@ module URBANopt
       # +force_clear+ - _Bool_ - Clear Scenario before creating Simulation input files.
       #
       # [return:] _Array_ Returns an array of all SimulationDirs, even those created previously, for Scenario.
-      def run(scenario, force_clear = false)
+      def run(scenario, force_clear = false, options = {})
         raise 'run is not implemented for ScenarioRunnerBase, override in your class'
       end
     end

data/lib/urbanopt/scenario/scenario_runner_osw.rb

@@ -1,5 +1,5 @@
 # *********************************************************************************
-# URBANopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
+# URBANopt (tm), Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC, and other
 # contributors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -55,6 +55,7 @@ module URBANopt
         end
 
         FileUtils.mkdir_p(scenario.run_dir) if !File.exist?(scenario.run_dir)
+        FileUtils.rm_rf(File.join(scenario.run_dir, 'run_status.json')) if File.exist?(File.join(scenario.run_dir, 'run_status.json'))
 
         simulation_dirs = scenario.simulation_dirs
 
@@ -81,10 +82,11 @@ module URBANopt
       # +scenario+ - _ScenarioBase_ - Scenario to create and run SimulationFiles for.
       # +force_clear+ - _Bool_ - Clear Scenario before creating SimulationFiles.
       # [return:] _Array_ Returns array of all SimulationFiles, even those created previously, for Scenario.
-      def run(scenario, force_clear = false)
+      def run(scenario, force_clear = false, options = {})
         # instantiate openstudio runner - use the defaults for now. If need to change then create
         # the runner.conf file (i.e. run `rake openstudio:runner:init`)
-        runner = OpenStudio::Extension::Runner.new(scenario.root_dir)
+        # allow passing gemfile_path and bundle_install_path in options
+        runner = OpenStudio::Extension::Runner.new(scenario.root_dir, [], options)
 
         # create simulation files
         simulation_dirs = create_simulation_files(scenario, force_clear)
@@ -156,18 +158,30 @@ module URBANopt
         #   puts "DATAPOINT FAILURES: #{failures}"
         # end
 
-        # look for other failed datapoints
+        # write results to file and to command line
+        get_results(scenario, simulation_dirs)
+
+        return simulation_dirs
+      end
+
+      def get_results(scenario, simulation_dirs)
+        # look for other failed datapoints (command line display)
+        # also compile datapoint status for latest_run.json file
+        status_arr = []
         failed_sims = []
-        simulation_dirs.each do |simulation_dir|
-          if File.exist?(File.join(simulation_dir.run_dir, 'failed.job'))
-            failed_sims << simulation_dir.run_dir.split('/')[-1]
+        simulation_dirs.each do |sim_dir|
+          if File.exist?(sim_dir.failed_job_path)
+            failed_sims << sim_dir.run_dir.split('/')[-1]
           end
+          status_arr << { "id": sim_dir.feature_id, "status": sim_dir.simulation_status, "mapper_class": sim_dir.mapper_class }
         end
+
+        # write to file
+        File.open(File.join(scenario.run_dir, 'run_status.json'), 'w') { |f| f.write JSON.pretty_generate("timestamp": Time.now.strftime('%Y-%m-%dT%k:%M:%S.%L'), "results": status_arr) }
+
         if !failed_sims.empty?
           puts "FAILED SIMULATION IDs: #{failed_sims.join(',')}"
         end
-
-        return simulation_dirs
       end
     end
   end