openstudio-analysis 1.3.4 → 1.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/openstudio-analysis.yml +40 -40
  3. data/.gitignore +21 -21
  4. data/.rubocop.yml +9 -9
  5. data/CHANGELOG.md +269 -261
  6. data/Gemfile +14 -14
  7. data/README.md +102 -102
  8. data/Rakefile +40 -40
  9. data/lib/openstudio/analysis/algorithm_attributes.rb +47 -47
  10. data/lib/openstudio/analysis/formulation.rb +857 -857
  11. data/lib/openstudio/analysis/server_api.rb +862 -846
  12. data/lib/openstudio/analysis/server_scripts.rb +108 -108
  13. data/lib/openstudio/analysis/support_files.rb +104 -104
  14. data/lib/openstudio/analysis/translator/datapoints.rb +454 -454
  15. data/lib/openstudio/analysis/translator/excel.rb +893 -893
  16. data/lib/openstudio/analysis/translator/workflow.rb +143 -143
  17. data/lib/openstudio/analysis/version.rb +12 -12
  18. data/lib/openstudio/analysis/workflow.rb +279 -279
  19. data/lib/openstudio/analysis/workflow_step.rb +523 -523
  20. data/lib/openstudio/analysis.rb +144 -144
  21. data/lib/openstudio/helpers/hash.rb +10 -10
  22. data/lib/openstudio/helpers/string.rb +36 -36
  23. data/lib/openstudio/helpers/utils.rb +36 -36
  24. data/lib/openstudio/weather/epw.rb +178 -178
  25. data/lib/openstudio-analysis.rb +47 -47
  26. data/openstudio-analysis.gemspec +38 -38
  27. data/update_license.rb +60 -60
  28. metadata +6 -6
data/lib/openstudio/analysis/translator/excel.rb CHANGED
@@ -1,893 +1,893 @@
1
- # *******************************************************************************
2
- # OpenStudio(R), Copyright (c) Alliance for Sustainable Energy, LLC.
3
- # See also https://openstudio.net/license
4
- # *******************************************************************************
5
-
6
- module OpenStudio
7
- module Analysis
8
- module Translator
9
- class Excel
10
- attr_reader :version
11
- attr_reader :settings
12
- attr_reader :variables
13
- attr_reader :outputs
14
- attr_reader :models
15
- attr_reader :weather_files
16
- attr_reader :measure_paths
17
- attr_reader :weather_paths
18
- attr_reader :worker_inits
19
- attr_reader :worker_finals
20
- attr_reader :export_path
21
- attr_reader :cluster_name
22
- attr_reader :variables
23
- attr_reader :algorithm
24
- attr_reader :problem
25
- attr_reader :run_setup
26
- attr_reader :aws_tags
27
-
28
- # remove these once we have classes to construct the JSON file
29
- attr_accessor :name
30
- attr_accessor :cluster_name
31
- attr_reader :analysis_name
32
-
33
- # methods to override instance variables
34
-
35
- # pass in the filename to read
36
- def initialize(xls_filename)
37
- @xls_filename = xls_filename
38
- @root_path = File.expand_path(File.dirname(@xls_filename))
39
-
40
- @xls = nil
41
- # try to read the spreadsheet as a roo object
42
- if File.exist?(@xls_filename)
43
- @xls = Roo::Spreadsheet.open(@xls_filename)
44
- else
45
- raise "File #{@xls_filename} does not exist"
46
- end
47
-
48
- # Initialize some other instance variables
49
- @version = '0.0.1'
50
- @analyses = [] # Array o OpenStudio::Analysis. Use method to access
51
- @name = nil
52
- @analysis_name = nil
53
- @settings = {}
54
- @weather_files = []
55
- @weather_paths = []
56
- @models = []
57
- @other_files = []
58
- @worker_inits = []
59
- @worker_finals = []
60
- @export_path = './export'
61
- @measure_paths = []
62
- @number_of_samples = 0 # TODO: remove this
63
- @problem = {}
64
- @algorithm = {}
65
- @outputs = {}
66
- @run_setup = {}
67
- @aws_tags = []
68
- end
69
-
70
- def process
71
- @setup = parse_setup
72
-
73
- @version = Semantic::Version.new @version
74
- raise "Spreadsheet version #{@version} is no longer supported. Please upgrade your spreadsheet to at least 0.1.9" if @version < '0.1.9'
75
-
76
- @variables = parse_variables
77
-
78
- @outputs = parse_outputs
79
-
80
- # call validate to make sure everything that is needed exists (i.e. directories)
81
- validate_analysis
82
- end
83
-
84
- # Helper methods to remove models and add new ones programatically. Note that these should
85
- # be moved into a general analysis class
86
- def delete_models
87
- @models = []
88
- end
89
-
90
- def add_model(name, display_name, type, path)
91
- @models << {
92
- name: name,
93
- display_name: display_name,
94
- type: type,
95
- path: path
96
- }
97
- end
98
-
99
- def validate_analysis
100
- # Setup the paths and do some error checking
101
- @measure_paths.each do |mp|
102
- raise "Measures directory '#{mp}' does not exist" unless Dir.exist?(mp)
103
- end
104
-
105
- @models.uniq!
106
- raise 'No seed models defined in spreadsheet' if @models.empty?
107
-
108
- @models.each do |model|
109
- raise "Seed model does not exist: #{model[:path]}" unless File.exist?(model[:path])
110
- end
111
-
112
- @weather_files.uniq!
113
- raise 'No weather files found based on what is in the spreadsheet' if @weather_files.empty?
114
-
115
- @weather_files.each do |wf|
116
- raise "Weather file does not exist: #{wf}" unless File.exist?(wf)
117
- end
118
-
119
- # This can be a directory as well
120
- @other_files.each do |f|
121
- raise "Other files do not exist for: #{f[:path]}" unless File.exist?(f[:path])
122
- end
123
-
124
- @worker_inits.each do |f|
125
- raise "Worker initialization file does not exist for: #{f[:path]}" unless File.exist?(f[:path])
126
- end
127
-
128
- @worker_finals.each do |f|
129
- raise "Worker finalization file does not exist for: #{f[:path]}" unless File.exist?(f[:path])
130
- end
131
-
132
- FileUtils.mkdir_p(@export_path)
133
-
134
- # verify that the measure display names are unique
135
- # puts @variables.inspect
136
- measure_display_names = @variables['data'].map { |m| m['enabled'] ? m['display_name'] : nil }.compact
137
- measure_display_names_mult = measure_display_names.select { |m| measure_display_names.count(m) > 1 }.uniq
138
- if measure_display_names_mult && !measure_display_names_mult.empty?
139
- raise "Measure Display Names are not unique for '#{measure_display_names_mult.join('\', \'')}'"
140
- end
141
-
142
- # verify that all continuous variables have all the data needed and create a name map
143
- variable_names = []
144
- @variables['data'].each do |measure|
145
- if measure['enabled']
146
- measure['variables'].each do |variable|
147
- # Determine if row is suppose to be an argument or a variable to be perturbed.
148
- if variable['variable_type'] == 'variable'
149
- variable_names << variable['display_name']
150
-
151
- # make sure that variables have static values
152
- if variable['distribution']['static_value'].nil? || variable['distribution']['static_value'] == ''
153
- raise "Variable #{measure['name']}:#{variable['name']} needs a static value"
154
- end
155
-
156
- if variable['type'] == 'enum' || variable['type'] == 'Choice'
157
- # check something
158
- else # must be an integer or double
159
- if variable['distribution']['type'] == 'discrete_uncertain'
160
- if variable['distribution']['discrete_values'].nil? || variable['distribution']['discrete_values'] == ''
161
- raise "Variable #{measure['name']}:#{variable['name']} needs discrete values"
162
- end
163
- elsif variable['distribution']['type'] == 'integer_sequence'
164
- if variable['distribution']['mean'].nil? || variable['distribution']['mean'] == ''
165
- raise "Variable #{measure['name']}:#{variable['name']} must have a mean/mode"
166
- end
167
- if variable['distribution']['min'].nil? || variable['distribution']['min'] == ''
168
- raise "Variable #{measure['name']}:#{variable['name']} must have a minimum"
169
- end
170
- if variable['distribution']['max'].nil? || variable['distribution']['max'] == ''
171
- raise "Variable #{measure['name']}:#{variable['name']} must have a maximum"
172
- end
173
- else
174
- if variable['distribution']['mean'].nil? || variable['distribution']['mean'] == ''
175
- raise "Variable #{measure['name']}:#{variable['name']} must have a mean"
176
- end
177
- if variable['distribution']['stddev'].nil? || variable['distribution']['stddev'] == ''
178
- raise "Variable #{measure['name']}:#{variable['name']} must have a stddev"
179
- end
180
- end
181
-
182
- if variable['distribution']['mean'].nil? || variable['distribution']['mean'] == ''
183
- raise "Variable #{measure['name']}:#{variable['name']} must have a mean/mode"
184
- end
185
- if variable['distribution']['min'].nil? || variable['distribution']['min'] == ''
186
- raise "Variable #{measure['name']}:#{variable['name']} must have a minimum"
187
- end
188
- if variable['distribution']['max'].nil? || variable['distribution']['max'] == ''
189
- raise "Variable #{measure['name']}:#{variable['name']} must have a maximum"
190
- end
191
- unless variable['type'] == 'string' || variable['type'] =~ /bool/
192
- if variable['distribution']['min'] > variable['distribution']['max']
193
- raise "Variable min is greater than variable max for #{measure['name']}:#{variable['name']}"
194
- end
195
- end
196
-
197
- end
198
- end
199
- end
200
- end
201
- end
202
-
203
- dupes = variable_names.select { |e| variable_names.count(e) > 1 }.uniq
204
- if dupes.count > 0
205
- raise "duplicate variable names found in list #{dupes.inspect}"
206
- end
207
-
208
- # most of the checks will raise a runtime exception, so this true will never be called
209
- true
210
- end
211
-
212
- # convert the data in excel's parsed data into an OpenStudio Analysis Object
213
- #
214
- # @seed_model [Hash] Seed model to set the new analysis to
215
- # @append_model_name [Boolean] Append the name of the seed model to the display name
216
- # @return [Object] An OpenStudio::Analysis
217
- def analysis(seed_model = nil, append_model_name = false)
218
- raise 'There are no seed models defined in the excel file. Please add one.' if @models.empty?
219
- raise "There are more than one seed models defined in the excel file. Call 'analyses' to return the array" if @models.size > 1 && seed_model.nil?
220
-
221
- seed_model = @models.first if seed_model.nil?
222
-
223
- # Use the programmatic interface to make the analysis
224
- # append the model name to the analysis name if requested (normally if there are more than 1 models in the spreadsheet)
225
- display_name = append_model_name ? @name + ' ' + seed_model[:display_name] : @name
226
-
227
- a = OpenStudio::Analysis.create(display_name)
228
-
229
- @variables['data'].each do |measure|
230
- next unless measure['enabled']
231
-
232
- @measure_paths.each do |measure_path|
233
- measure_dir_to_add = "#{measure_path}/#{measure['measure_file_name_directory']}"
234
- if Dir.exist? measure_dir_to_add
235
- if File.exist? "#{measure_dir_to_add}/measure.rb"
236
- measure['local_path_to_measure'] = "#{measure_dir_to_add}/measure.rb"
237
- break
238
- else
239
- raise "Measure in directory '#{measure_dir_to_add}' did not contain a measure.rb file"
240
- end
241
- end
242
- end
243
-
244
- raise "Could not find measure '#{measure['name']}' in directory named '#{measure['measure_file_name_directory']}' in the measure paths '#{@measure_paths.join(', ')}'" unless measure['local_path_to_measure']
245
-
246
- a.workflow.add_measure_from_excel(measure)
247
- end
248
-
249
- @other_files.each do |library|
250
- a.libraries.add(library[:path], library_name: library[:lib_zip_name])
251
- end
252
-
253
- @worker_inits.each do |w|
254
- a.worker_inits.add(w[:path], args: w[:args])
255
- end
256
-
257
- @worker_finals.each do |w|
258
- a.worker_finalizes.add(w[:path], args: w[:args])
259
- end
260
-
261
- # Add in the outputs
262
- @outputs['output_variables'].each do |o|
263
- o = Hash[o.map { |k, v| [k.to_sym, v] }]
264
- a.add_output(o)
265
- end
266
-
267
- a.analysis_type = @problem['analysis_type']
268
- @algorithm.each do |k, v|
269
- a.algorithm.set_attribute(k, v)
270
- end
271
-
272
- # clear out the seed files before adding new ones
273
- a.seed_model = seed_model[:path]
274
-
275
- # clear out the weather files before adding new ones
276
- a.weather_files.clear
277
- @weather_paths.each do |wp|
278
- a.weather_files.add_files(wp)
279
- end
280
-
281
- a
282
- end
283
-
284
- # Return an array of analyses objects of OpenStudio::Analysis::Formulation
285
- def analyses
286
- as = []
287
- @models.map do |model|
288
- as << analysis(model, @models.count > 1)
289
- end
290
-
291
- as
292
- end
293
-
294
- # Method to return the cluster name for backwards compatibility
295
- def cluster_name
296
- @settings['cluster_name']
297
- end
298
-
299
- # save_analysis will iterate over each model that is defined in the spreadsheet and save the
300
- # zip and json file.
301
- def save_analysis
302
- analyses.each do |a|
303
- puts "Saving JSON and ZIP file for #{@name}:#{a.display_name}"
304
- json_file_name = "#{@export_path}/#{a.name}.json"
305
- FileUtils.rm_f(json_file_name) if File.exist?(json_file_name)
306
- # File.open(json_file_name, 'w') { |f| f << JSON.pretty_generate(new_analysis_json) }
307
-
308
- a.save json_file_name
309
- a.save_zip "#{File.dirname(json_file_name)}/#{File.basename(json_file_name, '.*')}.zip"
310
- end
311
- end
312
-
313
- protected
314
-
315
- # parse_setup will pull out the data on the "setup" tab and store it in memory for later use
316
- def parse_setup
317
- rows = @xls.sheet('Setup').parse
318
- b_settings = false
319
- b_run_setup = false
320
- b_problem_setup = false
321
- b_algorithm_setup = false
322
- b_weather_files = false
323
- b_models = false
324
- b_other_libs = false
325
- b_worker_init = false
326
- b_worker_final = false
327
-
328
- rows.each do |row|
329
- if row[0] == 'Settings'
330
- b_settings = true
331
- b_run_setup = false
332
- b_problem_setup = false
333
- b_algorithm_setup = false
334
- b_weather_files = false
335
- b_models = false
336
- b_other_libs = false
337
- b_worker_init = false
338
- b_worker_final = false
339
- next
340
- elsif row[0] == 'Running Setup'
341
- b_settings = false
342
- b_run_setup = true
343
- b_problem_setup = false
344
- b_algorithm_setup = false
345
- b_weather_files = false
346
- b_models = false
347
- b_other_libs = false
348
- b_worker_init = false
349
- b_worker_final = false
350
- next
351
- elsif row[0] == 'Problem Definition'
352
- b_settings = false
353
- b_run_setup = false
354
- b_problem_setup = true
355
- b_algorithm_setup = false
356
- b_weather_files = false
357
- b_models = false
358
- b_other_libs = false
359
- b_worker_init = false
360
- b_worker_final = false
361
- next
362
- elsif row[0] == 'Algorithm Setup'
363
- b_settings = false
364
- b_run_setup = false
365
- b_problem_setup = false
366
- b_algorithm_setup = true
367
- b_weather_files = false
368
- b_models = false
369
- b_other_libs = false
370
- b_worker_init = false
371
- b_worker_final = false
372
- next
373
- elsif row[0] == 'Weather Files'
374
- b_settings = false
375
- b_run_setup = false
376
- b_problem_setup = false
377
- b_algorithm_setup = false
378
- b_weather_files = true
379
- b_models = false
380
- b_other_libs = false
381
- b_worker_init = false
382
- b_worker_final = false
383
- next
384
- elsif row[0] == 'Models'
385
- b_settings = false
386
- b_run_setup = false
387
- b_problem_setup = false
388
- b_algorithm_setup = false
389
- b_weather_files = false
390
- b_models = true
391
- b_other_libs = false
392
- b_worker_init = false
393
- b_worker_final = false
394
- next
395
- elsif row[0] == 'Other Library Files'
396
- b_settings = false
397
- b_run_setup = false
398
- b_problem_setup = false
399
- b_algorithm_setup = false
400
- b_weather_files = false
401
- b_models = false
402
- b_other_libs = true
403
- b_worker_init = false
404
- b_worker_final = false
405
- next
406
- elsif row[0] =~ /Worker Initialization Scripts/
407
- b_settings = false
408
- b_run_setup = false
409
- b_problem_setup = false
410
- b_algorithm_setup = false
411
- b_weather_files = false
412
- b_models = false
413
- b_other_libs = false
414
- b_worker_init = true
415
- b_worker_final = false
416
- next
417
- elsif row[0] =~ /Worker Finalization Scripts/
418
- b_settings = false
419
- b_run_setup = false
420
- b_problem_setup = false
421
- b_algorithm_setup = false
422
- b_weather_files = false
423
- b_models = false
424
- b_other_libs = false
425
- b_worker_init = false
426
- b_worker_final = true
427
- next
428
- end
429
-
430
- next if row[0].nil?
431
-
432
- if b_settings
433
- @version = row[1].chomp if row[0] == 'Spreadsheet Version'
434
- @settings[row[0].to_underscore.to_s] = row[1] if row[0]
435
- if @settings['cluster_name']
436
- @settings['cluster_name'] = @settings['cluster_name'].to_underscore
437
- end
438
-
439
- if row[0] == 'AWS Tag'
440
- @aws_tags << row[1].strip
441
- end
442
-
443
- # type some of the values that we know
444
- @settings['proxy_port'] = @settings['proxy_port'].to_i if @settings['proxy_port']
445
-
446
- elsif b_run_setup
447
- if row[0] == 'Analysis Name'
448
- if row[1]
449
- @name = row[1]
450
- else
451
- @name = SecureRandom.uuid
452
- end
453
- @analysis_name = @name.to_underscore
454
- end
455
- if row[0] == 'Export Directory'
456
- tmp_filepath = row[1]
457
- if (Pathname.new tmp_filepath).absolute?
458
- @export_path = tmp_filepath
459
- else
460
- @export_path = File.expand_path(File.join(@root_path, tmp_filepath))
461
- end
462
- end
463
- if row[0] == 'Measure Directory'
464
- tmp_filepath = row[1]
465
- if (Pathname.new tmp_filepath).absolute?
466
- @measure_paths << tmp_filepath
467
- else
468
- @measure_paths << File.expand_path(File.join(@root_path, tmp_filepath))
469
- end
470
- end
471
- @run_setup[row[0].to_underscore.to_s] = row[1] if row[0]
472
-
473
- # type cast
474
- if @run_setup['allow_multiple_jobs']
475
- raise 'allow_multiple_jobs is no longer a valid option in the Excel file, please delete the row and rerun'
476
- end
477
- if @run_setup['use_server_as_worker']
478
- raise 'use_server_as_worker is no longer a valid option in the Excel file, please delete the row and rerun'
479
- end
480
- elsif b_problem_setup
481
- if row[0]
482
- v = row[1]
483
- v.to_i if v % 1 == 0
484
- @problem[row[0].to_underscore.to_s] = v
485
- end
486
-
487
- elsif b_algorithm_setup
488
- if row[0] && !row[0].empty?
489
- v = row[1]
490
- v = v.to_i if v % 1 == 0
491
- @algorithm[row[0].to_underscore.to_s] = v
492
- end
493
- elsif b_weather_files
494
- if row[0] == 'Weather File'
495
- weather_path = row[1]
496
- unless (Pathname.new weather_path).absolute?
497
- weather_path = File.expand_path(File.join(@root_path, weather_path))
498
- end
499
- @weather_paths << weather_path
500
- @weather_files += Dir.glob(weather_path)
501
- end
502
- elsif b_models
503
- if row[1]
504
- tmp_m_name = row[1]
505
- else
506
- tmp_m_name = SecureRandom.uuid
507
- end
508
- # Only add models if the row is flagged
509
- if row[0]&.casecmp('model')&.zero?
510
- model_path = row[3]
511
- unless (Pathname.new model_path).absolute?
512
- model_path = File.expand_path(File.join(@root_path, model_path))
513
- end
514
- @models << { name: tmp_m_name.to_underscore, display_name: tmp_m_name, type: row[2], path: model_path }
515
- end
516
- elsif b_other_libs
517
- # determine if the path is relative
518
- other_path = row[2]
519
- unless (Pathname.new other_path).absolute?
520
- other_path = File.expand_path(File.join(@root_path, other_path))
521
- end
522
-
523
- @other_files << { lib_zip_name: row[1], path: other_path }
524
- elsif b_worker_init
525
- worker_init_path = row[1]
526
- unless (Pathname.new worker_init_path).absolute?
527
- worker_init_path = File.expand_path(File.join(@root_path, worker_init_path))
528
- end
529
-
530
- @worker_inits << { name: row[0], path: worker_init_path, args: row[2] }
531
- elsif b_worker_final
532
- worker_final_path = row[1]
533
- unless (Pathname.new worker_final_path).absolute?
534
- worker_final_path = File.expand_path(File.join(@root_path, worker_final_path))
535
- end
536
-
537
- @worker_finals << { name: row[0], path: worker_final_path, args: row[2] }
538
- end
539
-
540
- next
541
- end
542
-
543
- # do some last checks
544
- @measure_paths = ['./measures'] if @measure_paths.empty?
545
- end
546
-
547
- # parse_variables will parse the XLS spreadsheet and save the data into
548
- # a higher level JSON file. The JSON file is historic and it should really
549
- # be omitted as an intermediate step
550
- def parse_variables
551
- # clean remove whitespace and unicode chars
552
- # The parse is a unique format (https://github.com/Empact/roo/blob/master/lib/roo/base.rb#L444)
553
- # If you add a new column and you want that variable in the hash, then you must add it here.
554
- # rows = @xls.sheet('Variables').parse(:enabled => "# variable")
555
- # puts rows.inspect
556
-
557
- rows = nil
558
- begin
559
- if @version >= '0.3.3'.to_version
560
- rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
561
- measure_name_or_var_type: /type/i,
562
- measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
563
- measure_file_name_directory: /measure\sdirectory/i,
564
- measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
565
- display_name_short: /parameter\sshort\sdisplay\sname/i,
566
- # sampling_method: /sampling\smethod/i,
567
- variable_type: /variable\stype/i,
568
- units: /units/i,
569
- default_value: /static.default\svalue/i,
570
- enums: /enumerations/i,
571
- min: /min/i,
572
- max: /max/i,
573
- mode: /mean|mode/i,
574
- stddev: /std\sdev/i,
575
- delta_x: /delta.x/i,
576
- discrete_values: /discrete\svalues/i,
577
- discrete_weights: /discrete\sweights/i,
578
- distribution: /distribution/i,
579
- source: /data\ssource/i,
580
- notes: /notes/i,
581
- relation_to_eui: /typical\svar\sto\seui\srelationship/i,
582
- clean: true)
583
- elsif @version >= '0.3.0'.to_version
584
- rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
585
- measure_name_or_var_type: /type/i,
586
- measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
587
- measure_file_name_directory: /measure\sdirectory/i,
588
- measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
589
- # sampling_method: /sampling\smethod/i,
590
- variable_type: /variable\stype/i,
591
- units: /units/i,
592
- default_value: /static.default\svalue/i,
593
- enums: /enumerations/i,
594
- min: /min/i,
595
- max: /max/i,
596
- mode: /mean|mode/i,
597
- stddev: /std\sdev/i,
598
- delta_x: /delta.x/i,
599
- discrete_values: /discrete\svalues/i,
600
- discrete_weights: /discrete\sweights/i,
601
- distribution: /distribution/i,
602
- source: /data\ssource/i,
603
- notes: /notes/i,
604
- relation_to_eui: /typical\svar\sto\seui\srelationship/i,
605
- clean: true)
606
- elsif @version >= '0.2.0'.to_version
607
- rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
608
- measure_name_or_var_type: /type/i,
609
- measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
610
- measure_file_name_directory: /measure\sdirectory/i,
611
- measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
612
- sampling_method: /sampling\smethod/i,
613
- variable_type: /variable\stype/i,
614
- units: /units/i,
615
- default_value: /static.default\svalue/i,
616
- enums: /enumerations/i,
617
- min: /min/i,
618
- max: /max/i,
619
- mode: /mean|mode/i,
620
- stddev: /std\sdev/i,
621
- delta_x: /delta.x/i,
622
- discrete_values: /discrete\svalues/i,
623
- discrete_weights: /discrete\sweights/i,
624
- distribution: /distribution/i,
625
- source: /data\ssource/i,
626
- notes: /notes/i,
627
- relation_to_eui: /typical\svar\sto\seui\srelationship/i,
628
- clean: true)
629
- elsif @version >= '0.1.12'.to_version
630
- rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
631
- measure_name_or_var_type: /type/i,
632
- measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
633
- measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
634
- sampling_method: /sampling\smethod/i,
635
- variable_type: /variable\stype/i,
636
- units: /units/i,
637
- default_value: /static.default\svalue/i,
638
- enums: /enumerations/i,
639
- min: /min/i,
640
- max: /max/i,
641
- mode: /mean|mode/i,
642
- stddev: /std\sdev/i,
643
- delta_x: /delta.x/i,
644
- discrete_values: /discrete\svalues/i,
645
- discrete_weights: /discrete\sweights/i,
646
- distribution: /distribution/i,
647
- source: /data\ssource/i,
648
- notes: /notes/i,
649
- relation_to_eui: /typical\svar\sto\seui\srelationship/i,
650
- clean: true)
651
- elsif @version >= '0.1.11'.to_version
652
- rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
653
- measure_name_or_var_type: /type/i,
654
- measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
655
- measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
656
- sampling_method: /sampling\smethod/i,
657
- variable_type: /variable\stype/i,
658
- units: /units/i,
659
- default_value: /static.default\svalue/i,
660
- enums: /enumerations/i,
661
- min: /min/i,
662
- max: /max/i,
663
- mode: /mean|mode/i,
664
- stddev: /std\sdev/i,
665
- # delta_x: /delta.x/i,
666
- discrete_values: /discrete\svalues/i,
667
- discrete_weights: /discrete\sweights/i,
668
- distribution: /distribution/i,
669
- source: /data\ssource/i,
670
- notes: /notes/i,
671
- relation_to_eui: /typical\svar\sto\seui\srelationship/i,
672
- clean: true)
673
- else
674
- rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
675
- measure_name_or_var_type: /type/i,
676
- measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
677
- measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
678
- sampling_method: /sampling\smethod/i,
679
- variable_type: /variable\stype/i,
680
- units: /units/i,
681
- default_value: /static.default\svalue/i,
682
- enums: /enumerations/i,
683
- min: /min/i,
684
- max: /max/i,
685
- mode: /mean|mode/i,
686
- stddev: /std\sdev/i,
687
- # delta_x: /delta.x/i,
688
- # discrete_values: /discrete\svalues/i,
689
- # discrete_weights: /discrete\sweights/i,
690
- distribution: /distribution/i,
691
- source: /data\ssource/i,
692
- notes: /notes/i,
693
- relation_to_eui: /typical\svar\sto\seui\srelationship/i,
694
- clean: true)
695
- end
696
- rescue StandardError => e
697
- raise "Unable to parse spreadsheet #{@xls_filename} with version #{@version} due to error: #{e.message}"
698
- end
699
-
700
- raise "Could not find the sheet name 'Variables' in excel file #{@root_path}" unless rows
701
-
702
- # map the data to another hash that is more easily processed
703
- data = {}
704
- data['data'] = []
705
-
706
- measure_index = -1
707
- variable_index = -1
708
- measure_name = nil
709
- rows.each_with_index do |row, icnt|
710
- # puts "Parsing line: #{icnt}:#{row}"
711
-
712
- # check if we are a measure - nil means that the cell was blank
713
- if row[:enabled].nil?
714
- if measure_name && data['data'][measure_index]['enabled']
715
- variable_index += 1
716
-
717
- var = {}
718
- var['variable_type'] = row[:measure_name_or_var_type]
719
- var['display_name'] = row[:measure_file_name_or_var_display_name]
720
- var['display_name_short'] = row[:display_name_short] ? row[:display_name_short] : var['display_name']
721
- var['name'] = row[:measure_type_or_parameter_name_in_measure]
722
- var['index'] = variable_index # order of the variable (not sure of its need)
723
- var['type'] = row[:variable_type].downcase
724
- var['units'] = row[:units]
725
- var['distribution'] = {}
726
-
727
- # parse the choices/enums
728
- if var['type'] == 'enum' || var['type'] == 'choice' # this is now a choice
729
- if row[:enums]
730
- var['distribution']['enumerations'] = row[:enums].delete('|').split(',').map(&:strip)
731
- end
732
- elsif var['type'] == 'bool'
733
- var['distribution']['enumerations'] = []
734
- var['distribution']['enumerations'] << 'true' # TODO: should this be a real bool?
735
- var['distribution']['enumerations'] << 'false'
736
- end
737
-
738
- var['distribution']['min'] = row[:min]
739
- var['distribution']['max'] = row[:max]
740
- var['distribution']['mean'] = row[:mode]
741
- var['distribution']['stddev'] = row[:stddev]
742
- var['distribution']['discrete_values'] = row[:discrete_values]
743
- var['distribution']['discrete_weights'] = row[:discrete_weights]
744
- var['distribution']['type'] = row[:distribution]
745
- var['distribution']['static_value'] = row[:default_value]
746
- var['distribution']['delta_x'] = row[:delta_x]
747
-
748
- # type various values correctly
749
- var['distribution']['min'] = typecast_value(var['type'], var['distribution']['min'])
750
- var['distribution']['max'] = typecast_value(var['type'], var['distribution']['max'])
751
- var['distribution']['mean'] = typecast_value(var['type'], var['distribution']['mean'])
752
- var['distribution']['stddev'] = typecast_value(var['type'], var['distribution']['stddev'])
753
- var['distribution']['static_value'] = typecast_value(var['type'], var['distribution']['static_value'])
754
-
755
- # eval the discrete value and weight arrays
756
- case var['type']
757
- when 'bool', 'boolean'
758
- if var['distribution']['discrete_values']
759
- var['distribution']['discrete_values'] = eval(var['distribution']['discrete_values']).map { |v| v.to_s == 'true' }
760
- end
761
- if var['distribution']['discrete_weights'] && var['distribution']['discrete_weights'] != ''
762
- var['distribution']['discrete_weights'] = eval(var['distribution']['discrete_weights'])
763
- end
764
- else
765
- if var['distribution']['discrete_values']
766
- var['distribution']['discrete_values'] = eval(var['distribution']['discrete_values'])
767
- end
768
- if var['distribution']['discrete_weights'] && var['distribution']['discrete_weights'] != ''
769
- var['distribution']['discrete_weights'] = eval(var['distribution']['discrete_weights'])
770
- end
771
- end
772
-
773
- var['distribution']['source'] = row[:source]
774
- var['notes'] = row[:notes]
775
- var['relation_to_eui'] = row[:relation_to_eui]
776
-
777
- data['data'][measure_index]['variables'] << var
778
- end
779
- else
780
- measure_index += 1
781
- variable_index = 0
782
- data['data'][measure_index] = {}
783
-
784
- # generate name id
785
- # TODO: put this into a logger. puts "Parsing measure #{row[1]}"
786
- display_name = row[:measure_name_or_var_type]
787
- measure_name = display_name.downcase.strip.tr('-', '_').tr(' ', '_').gsub('__', '_')
788
- data['data'][measure_index]['display_name'] = display_name
789
- data['data'][measure_index]['name'] = measure_name
790
- data['data'][measure_index]['enabled'] = row[:enabled]
791
- data['data'][measure_index]['measure_file_name'] = row[:measure_file_name_or_var_display_name]
792
- if row[:measure_file_name_directory]
793
- data['data'][measure_index]['measure_file_name_directory'] = row[:measure_file_name_directory]
794
- else
795
- data['data'][measure_index]['measure_file_name_directory'] = row[:measure_file_name_or_var_display_name].to_underscore
796
- end
797
- data['data'][measure_index]['measure_type'] = row[:measure_type_or_parameter_name_in_measure]
798
- data['data'][measure_index]['version'] = @version_id
799
-
800
- data['data'][measure_index]['variables'] = []
801
- end
802
- end
803
-
804
- data
805
- end
806
-
807
- def parse_outputs
808
- rows = nil
809
- if @version >= '0.3.3'.to_version
810
- rows = @xls.sheet('Outputs').parse(display_name: /variable\sdisplay\sname/i,
811
- display_name_short: /short\sdisplay\sname/i,
812
- metadata_id: /taxonomy\sidentifier/i,
813
- name: /^name$/i,
814
- units: /units/i,
815
- visualize: /visualize/i,
816
- export: /export/i,
817
- variable_type: /variable\stype/i,
818
- objective_function: /objective\sfunction/i,
819
- objective_function_target: /objective\sfunction\starget/i,
820
- scaling_factor: /scale/i,
821
- objective_function_group: /objective\sfunction\sgroup/i)
822
- elsif @version >= '0.3.0'.to_version
823
- rows = @xls.sheet('Outputs').parse(display_name: /variable\sdisplay\sname/i,
824
- # display_name_short: /short\sdisplay\sname/i,
825
- metadata_id: /taxonomy\sidentifier/i,
826
- name: /^name$/i,
827
- units: /units/i,
828
- visualize: /visualize/i,
829
- export: /export/i,
830
- variable_type: /variable\stype/i,
831
- objective_function: /objective\sfunction/i,
832
- objective_function_target: /objective\sfunction\starget/i,
833
- scaling_factor: /scale/i,
834
- objective_function_group: /objective\sfunction\sgroup/i)
835
- else
836
- rows = @xls.sheet('Outputs').parse(display_name: /variable\sdisplay\sname/i,
837
- # display_name_short: /short\sdisplay\sname/i,
838
- # metadata_id: /taxonomy\sidentifier/i,
839
- name: /^name$/i,
840
- units: /units/i,
841
- # visualize: /visualize/i,
842
- # export: /export/i,
843
- # variable_type: /variable\stype/i,
844
- objective_function: /objective\sfunction/i,
845
- objective_function_target: /objective\sfunction\starget/i,
846
- scaling_factor: /scale/i,
847
- objective_function_group: /objective/i)
848
-
849
- end
850
-
851
- unless rows
852
- raise "Could not find the sheet name 'Outputs' in excel file #{@root_path}"
853
- end
854
-
855
- data = {}
856
- data['output_variables'] = []
857
-
858
- variable_index = -1
859
- group_index = 1
860
-
861
- rows.each_with_index do |row, icnt|
862
- next if icnt < 1 # skip the first 3 lines of the file
863
-
864
- var = {}
865
- var['display_name'] = row[:display_name]
866
- var['display_name_short'] = row[:display_name_short] ? row[:display_name_short] : row[:display_name]
867
- var['metadata_id'] = row[:metadata_id]
868
- var['name'] = row[:name]
869
- var['units'] = row[:units]
870
- var['visualize'] = row[:visualize]
871
- var['export'] = row[:export]
872
- var['variable_type'] = row[:variable_type].downcase if row[:variable_type]
873
- var['objective_function'] = row[:objective_function]
874
- var['objective_function_target'] = row[:objective_function_target]
875
- var['scaling_factor'] = row[:scaling_factor]
876
-
877
- if var['objective_function']
878
- if row[:objective_function_group].nil?
879
- var['objective_function_group'] = group_index
880
- group_index += 1
881
- else
882
- var['objective_function_group'] = row[:objective_function_group]
883
- end
884
- end
885
- data['output_variables'] << var
886
- end
887
-
888
- data
889
- end
890
- end
891
- end
892
- end
893
- end
1
+ # *******************************************************************************
2
+ # OpenStudio(R), Copyright (c) Alliance for Sustainable Energy, LLC.
3
+ # See also https://openstudio.net/license
4
+ # *******************************************************************************
5
+
6
+ module OpenStudio
7
+ module Analysis
8
+ module Translator
9
+ class Excel
10
+ attr_reader :version
11
+ attr_reader :settings
12
+ attr_reader :variables
13
+ attr_reader :outputs
14
+ attr_reader :models
15
+ attr_reader :weather_files
16
+ attr_reader :measure_paths
17
+ attr_reader :weather_paths
18
+ attr_reader :worker_inits
19
+ attr_reader :worker_finals
20
+ attr_reader :export_path
21
+ attr_reader :cluster_name
22
+ attr_reader :variables
23
+ attr_reader :algorithm
24
+ attr_reader :problem
25
+ attr_reader :run_setup
26
+ attr_reader :aws_tags
27
+
28
+ # remove these once we have classes to construct the JSON file
29
+ attr_accessor :name
30
+ attr_accessor :cluster_name
31
+ attr_reader :analysis_name
32
+
33
+ # methods to override instance variables
34
+
35
+ # pass in the filename to read
36
+ def initialize(xls_filename)
37
+ @xls_filename = xls_filename
38
+ @root_path = File.expand_path(File.dirname(@xls_filename))
39
+
40
+ @xls = nil
41
+ # try to read the spreadsheet as a roo object
42
+ if File.exist?(@xls_filename)
43
+ @xls = Roo::Spreadsheet.open(@xls_filename)
44
+ else
45
+ raise "File #{@xls_filename} does not exist"
46
+ end
47
+
48
+ # Initialize some other instance variables
49
+ @version = '0.0.1'
50
+ @analyses = [] # Array o OpenStudio::Analysis. Use method to access
51
+ @name = nil
52
+ @analysis_name = nil
53
+ @settings = {}
54
+ @weather_files = []
55
+ @weather_paths = []
56
+ @models = []
57
+ @other_files = []
58
+ @worker_inits = []
59
+ @worker_finals = []
60
+ @export_path = './export'
61
+ @measure_paths = []
62
+ @number_of_samples = 0 # TODO: remove this
63
+ @problem = {}
64
+ @algorithm = {}
65
+ @outputs = {}
66
+ @run_setup = {}
67
+ @aws_tags = []
68
+ end
69
+
70
+ def process
71
+ @setup = parse_setup
72
+
73
+ @version = Semantic::Version.new @version
74
+ raise "Spreadsheet version #{@version} is no longer supported. Please upgrade your spreadsheet to at least 0.1.9" if @version < '0.1.9'
75
+
76
+ @variables = parse_variables
77
+
78
+ @outputs = parse_outputs
79
+
80
+ # call validate to make sure everything that is needed exists (i.e. directories)
81
+ validate_analysis
82
+ end
83
+
84
+ # Helper methods to remove models and add new ones programatically. Note that these should
85
+ # be moved into a general analysis class
86
+ def delete_models
87
+ @models = []
88
+ end
89
+
90
+ def add_model(name, display_name, type, path)
91
+ @models << {
92
+ name: name,
93
+ display_name: display_name,
94
+ type: type,
95
+ path: path
96
+ }
97
+ end
98
+
99
+ def validate_analysis
100
+ # Setup the paths and do some error checking
101
+ @measure_paths.each do |mp|
102
+ raise "Measures directory '#{mp}' does not exist" unless Dir.exist?(mp)
103
+ end
104
+
105
+ @models.uniq!
106
+ raise 'No seed models defined in spreadsheet' if @models.empty?
107
+
108
+ @models.each do |model|
109
+ raise "Seed model does not exist: #{model[:path]}" unless File.exist?(model[:path])
110
+ end
111
+
112
+ @weather_files.uniq!
113
+ raise 'No weather files found based on what is in the spreadsheet' if @weather_files.empty?
114
+
115
+ @weather_files.each do |wf|
116
+ raise "Weather file does not exist: #{wf}" unless File.exist?(wf)
117
+ end
118
+
119
+ # This can be a directory as well
120
+ @other_files.each do |f|
121
+ raise "Other files do not exist for: #{f[:path]}" unless File.exist?(f[:path])
122
+ end
123
+
124
+ @worker_inits.each do |f|
125
+ raise "Worker initialization file does not exist for: #{f[:path]}" unless File.exist?(f[:path])
126
+ end
127
+
128
+ @worker_finals.each do |f|
129
+ raise "Worker finalization file does not exist for: #{f[:path]}" unless File.exist?(f[:path])
130
+ end
131
+
132
+ FileUtils.mkdir_p(@export_path)
133
+
134
+ # verify that the measure display names are unique
135
+ # puts @variables.inspect
136
+ measure_display_names = @variables['data'].map { |m| m['enabled'] ? m['display_name'] : nil }.compact
137
+ measure_display_names_mult = measure_display_names.select { |m| measure_display_names.count(m) > 1 }.uniq
138
+ if measure_display_names_mult && !measure_display_names_mult.empty?
139
+ raise "Measure Display Names are not unique for '#{measure_display_names_mult.join('\', \'')}'"
140
+ end
141
+
142
+ # verify that all continuous variables have all the data needed and create a name map
143
+ variable_names = []
144
+ @variables['data'].each do |measure|
145
+ if measure['enabled']
146
+ measure['variables'].each do |variable|
147
+ # Determine if row is suppose to be an argument or a variable to be perturbed.
148
+ if variable['variable_type'] == 'variable'
149
+ variable_names << variable['display_name']
150
+
151
+ # make sure that variables have static values
152
+ if variable['distribution']['static_value'].nil? || variable['distribution']['static_value'] == ''
153
+ raise "Variable #{measure['name']}:#{variable['name']} needs a static value"
154
+ end
155
+
156
+ if variable['type'] == 'enum' || variable['type'] == 'Choice'
157
+ # check something
158
+ else # must be an integer or double
159
+ if variable['distribution']['type'] == 'discrete_uncertain'
160
+ if variable['distribution']['discrete_values'].nil? || variable['distribution']['discrete_values'] == ''
161
+ raise "Variable #{measure['name']}:#{variable['name']} needs discrete values"
162
+ end
163
+ elsif variable['distribution']['type'] == 'integer_sequence'
164
+ if variable['distribution']['mean'].nil? || variable['distribution']['mean'] == ''
165
+ raise "Variable #{measure['name']}:#{variable['name']} must have a mean/mode"
166
+ end
167
+ if variable['distribution']['min'].nil? || variable['distribution']['min'] == ''
168
+ raise "Variable #{measure['name']}:#{variable['name']} must have a minimum"
169
+ end
170
+ if variable['distribution']['max'].nil? || variable['distribution']['max'] == ''
171
+ raise "Variable #{measure['name']}:#{variable['name']} must have a maximum"
172
+ end
173
+ else
174
+ if variable['distribution']['mean'].nil? || variable['distribution']['mean'] == ''
175
+ raise "Variable #{measure['name']}:#{variable['name']} must have a mean"
176
+ end
177
+ if variable['distribution']['stddev'].nil? || variable['distribution']['stddev'] == ''
178
+ raise "Variable #{measure['name']}:#{variable['name']} must have a stddev"
179
+ end
180
+ end
181
+
182
+ if variable['distribution']['mean'].nil? || variable['distribution']['mean'] == ''
183
+ raise "Variable #{measure['name']}:#{variable['name']} must have a mean/mode"
184
+ end
185
+ if variable['distribution']['min'].nil? || variable['distribution']['min'] == ''
186
+ raise "Variable #{measure['name']}:#{variable['name']} must have a minimum"
187
+ end
188
+ if variable['distribution']['max'].nil? || variable['distribution']['max'] == ''
189
+ raise "Variable #{measure['name']}:#{variable['name']} must have a maximum"
190
+ end
191
+ unless variable['type'] == 'string' || variable['type'] =~ /bool/
192
+ if variable['distribution']['min'] > variable['distribution']['max']
193
+ raise "Variable min is greater than variable max for #{measure['name']}:#{variable['name']}"
194
+ end
195
+ end
196
+
197
+ end
198
+ end
199
+ end
200
+ end
201
+ end
202
+
203
+ dupes = variable_names.select { |e| variable_names.count(e) > 1 }.uniq
204
+ if dupes.count > 0
205
+ raise "duplicate variable names found in list #{dupes.inspect}"
206
+ end
207
+
208
+ # most of the checks will raise a runtime exception, so this true will never be called
209
+ true
210
+ end
211
+
212
+ # convert the data in excel's parsed data into an OpenStudio Analysis Object
213
+ #
214
+ # @seed_model [Hash] Seed model to set the new analysis to
215
+ # @append_model_name [Boolean] Append the name of the seed model to the display name
216
+ # @return [Object] An OpenStudio::Analysis
217
+ def analysis(seed_model = nil, append_model_name = false)
218
+ raise 'There are no seed models defined in the excel file. Please add one.' if @models.empty?
219
+ raise "There are more than one seed models defined in the excel file. Call 'analyses' to return the array" if @models.size > 1 && seed_model.nil?
220
+
221
+ seed_model = @models.first if seed_model.nil?
222
+
223
+ # Use the programmatic interface to make the analysis
224
+ # append the model name to the analysis name if requested (normally if there are more than 1 models in the spreadsheet)
225
+ display_name = append_model_name ? @name + ' ' + seed_model[:display_name] : @name
226
+
227
+ a = OpenStudio::Analysis.create(display_name)
228
+
229
+ @variables['data'].each do |measure|
230
+ next unless measure['enabled']
231
+
232
+ @measure_paths.each do |measure_path|
233
+ measure_dir_to_add = "#{measure_path}/#{measure['measure_file_name_directory']}"
234
+ if Dir.exist? measure_dir_to_add
235
+ if File.exist? "#{measure_dir_to_add}/measure.rb"
236
+ measure['local_path_to_measure'] = "#{measure_dir_to_add}/measure.rb"
237
+ break
238
+ else
239
+ raise "Measure in directory '#{measure_dir_to_add}' did not contain a measure.rb file"
240
+ end
241
+ end
242
+ end
243
+
244
+ raise "Could not find measure '#{measure['name']}' in directory named '#{measure['measure_file_name_directory']}' in the measure paths '#{@measure_paths.join(', ')}'" unless measure['local_path_to_measure']
245
+
246
+ a.workflow.add_measure_from_excel(measure)
247
+ end
248
+
249
+ @other_files.each do |library|
250
+ a.libraries.add(library[:path], library_name: library[:lib_zip_name])
251
+ end
252
+
253
+ @worker_inits.each do |w|
254
+ a.worker_inits.add(w[:path], args: w[:args])
255
+ end
256
+
257
+ @worker_finals.each do |w|
258
+ a.worker_finalizes.add(w[:path], args: w[:args])
259
+ end
260
+
261
+ # Add in the outputs
262
+ @outputs['output_variables'].each do |o|
263
+ o = Hash[o.map { |k, v| [k.to_sym, v] }]
264
+ a.add_output(o)
265
+ end
266
+
267
+ a.analysis_type = @problem['analysis_type']
268
+ @algorithm.each do |k, v|
269
+ a.algorithm.set_attribute(k, v)
270
+ end
271
+
272
+ # clear out the seed files before adding new ones
273
+ a.seed_model = seed_model[:path]
274
+
275
+ # clear out the weather files before adding new ones
276
+ a.weather_files.clear
277
+ @weather_paths.each do |wp|
278
+ a.weather_files.add_files(wp)
279
+ end
280
+
281
+ a
282
+ end
283
+
284
+ # Return an array of analyses objects of OpenStudio::Analysis::Formulation
285
+ def analyses
286
+ as = []
287
+ @models.map do |model|
288
+ as << analysis(model, @models.count > 1)
289
+ end
290
+
291
+ as
292
+ end
293
+
294
+ # Method to return the cluster name for backwards compatibility
295
+ def cluster_name
296
+ @settings['cluster_name']
297
+ end
298
+
299
+ # save_analysis will iterate over each model that is defined in the spreadsheet and save the
300
+ # zip and json file.
301
+ def save_analysis
302
+ analyses.each do |a|
303
+ puts "Saving JSON and ZIP file for #{@name}:#{a.display_name}"
304
+ json_file_name = "#{@export_path}/#{a.name}.json"
305
+ FileUtils.rm_f(json_file_name) if File.exist?(json_file_name)
306
+ # File.open(json_file_name, 'w') { |f| f << JSON.pretty_generate(new_analysis_json) }
307
+
308
+ a.save json_file_name
309
+ a.save_zip "#{File.dirname(json_file_name)}/#{File.basename(json_file_name, '.*')}.zip"
310
+ end
311
+ end
312
+
313
+ protected
314
+
315
+ # parse_setup will pull out the data on the "setup" tab and store it in memory for later use
316
+ def parse_setup
317
+ rows = @xls.sheet('Setup').parse
318
+ b_settings = false
319
+ b_run_setup = false
320
+ b_problem_setup = false
321
+ b_algorithm_setup = false
322
+ b_weather_files = false
323
+ b_models = false
324
+ b_other_libs = false
325
+ b_worker_init = false
326
+ b_worker_final = false
327
+
328
+ rows.each do |row|
329
+ if row[0] == 'Settings'
330
+ b_settings = true
331
+ b_run_setup = false
332
+ b_problem_setup = false
333
+ b_algorithm_setup = false
334
+ b_weather_files = false
335
+ b_models = false
336
+ b_other_libs = false
337
+ b_worker_init = false
338
+ b_worker_final = false
339
+ next
340
+ elsif row[0] == 'Running Setup'
341
+ b_settings = false
342
+ b_run_setup = true
343
+ b_problem_setup = false
344
+ b_algorithm_setup = false
345
+ b_weather_files = false
346
+ b_models = false
347
+ b_other_libs = false
348
+ b_worker_init = false
349
+ b_worker_final = false
350
+ next
351
+ elsif row[0] == 'Problem Definition'
352
+ b_settings = false
353
+ b_run_setup = false
354
+ b_problem_setup = true
355
+ b_algorithm_setup = false
356
+ b_weather_files = false
357
+ b_models = false
358
+ b_other_libs = false
359
+ b_worker_init = false
360
+ b_worker_final = false
361
+ next
362
+ elsif row[0] == 'Algorithm Setup'
363
+ b_settings = false
364
+ b_run_setup = false
365
+ b_problem_setup = false
366
+ b_algorithm_setup = true
367
+ b_weather_files = false
368
+ b_models = false
369
+ b_other_libs = false
370
+ b_worker_init = false
371
+ b_worker_final = false
372
+ next
373
+ elsif row[0] == 'Weather Files'
374
+ b_settings = false
375
+ b_run_setup = false
376
+ b_problem_setup = false
377
+ b_algorithm_setup = false
378
+ b_weather_files = true
379
+ b_models = false
380
+ b_other_libs = false
381
+ b_worker_init = false
382
+ b_worker_final = false
383
+ next
384
+ elsif row[0] == 'Models'
385
+ b_settings = false
386
+ b_run_setup = false
387
+ b_problem_setup = false
388
+ b_algorithm_setup = false
389
+ b_weather_files = false
390
+ b_models = true
391
+ b_other_libs = false
392
+ b_worker_init = false
393
+ b_worker_final = false
394
+ next
395
+ elsif row[0] == 'Other Library Files'
396
+ b_settings = false
397
+ b_run_setup = false
398
+ b_problem_setup = false
399
+ b_algorithm_setup = false
400
+ b_weather_files = false
401
+ b_models = false
402
+ b_other_libs = true
403
+ b_worker_init = false
404
+ b_worker_final = false
405
+ next
406
+ elsif row[0] =~ /Worker Initialization Scripts/
407
+ b_settings = false
408
+ b_run_setup = false
409
+ b_problem_setup = false
410
+ b_algorithm_setup = false
411
+ b_weather_files = false
412
+ b_models = false
413
+ b_other_libs = false
414
+ b_worker_init = true
415
+ b_worker_final = false
416
+ next
417
+ elsif row[0] =~ /Worker Finalization Scripts/
418
+ b_settings = false
419
+ b_run_setup = false
420
+ b_problem_setup = false
421
+ b_algorithm_setup = false
422
+ b_weather_files = false
423
+ b_models = false
424
+ b_other_libs = false
425
+ b_worker_init = false
426
+ b_worker_final = true
427
+ next
428
+ end
429
+
430
+ next if row[0].nil?
431
+
432
+ if b_settings
433
+ @version = row[1].chomp if row[0] == 'Spreadsheet Version'
434
+ @settings[row[0].to_underscore.to_s] = row[1] if row[0]
435
+ if @settings['cluster_name']
436
+ @settings['cluster_name'] = @settings['cluster_name'].to_underscore
437
+ end
438
+
439
+ if row[0] == 'AWS Tag'
440
+ @aws_tags << row[1].strip
441
+ end
442
+
443
+ # type some of the values that we know
444
+ @settings['proxy_port'] = @settings['proxy_port'].to_i if @settings['proxy_port']
445
+
446
+ elsif b_run_setup
447
+ if row[0] == 'Analysis Name'
448
+ if row[1]
449
+ @name = row[1]
450
+ else
451
+ @name = SecureRandom.uuid
452
+ end
453
+ @analysis_name = @name.to_underscore
454
+ end
455
+ if row[0] == 'Export Directory'
456
+ tmp_filepath = row[1]
457
+ if (Pathname.new tmp_filepath).absolute?
458
+ @export_path = tmp_filepath
459
+ else
460
+ @export_path = File.expand_path(File.join(@root_path, tmp_filepath))
461
+ end
462
+ end
463
+ if row[0] == 'Measure Directory'
464
+ tmp_filepath = row[1]
465
+ if (Pathname.new tmp_filepath).absolute?
466
+ @measure_paths << tmp_filepath
467
+ else
468
+ @measure_paths << File.expand_path(File.join(@root_path, tmp_filepath))
469
+ end
470
+ end
471
+ @run_setup[row[0].to_underscore.to_s] = row[1] if row[0]
472
+
473
+ # type cast
474
+ if @run_setup['allow_multiple_jobs']
475
+ raise 'allow_multiple_jobs is no longer a valid option in the Excel file, please delete the row and rerun'
476
+ end
477
+ if @run_setup['use_server_as_worker']
478
+ raise 'use_server_as_worker is no longer a valid option in the Excel file, please delete the row and rerun'
479
+ end
480
+ elsif b_problem_setup
481
+ if row[0]
482
+ v = row[1]
483
+ v.to_i if v % 1 == 0
484
+ @problem[row[0].to_underscore.to_s] = v
485
+ end
486
+
487
+ elsif b_algorithm_setup
488
+ if row[0] && !row[0].empty?
489
+ v = row[1]
490
+ v = v.to_i if v % 1 == 0
491
+ @algorithm[row[0].to_underscore.to_s] = v
492
+ end
493
+ elsif b_weather_files
494
+ if row[0] == 'Weather File'
495
+ weather_path = row[1]
496
+ unless (Pathname.new weather_path).absolute?
497
+ weather_path = File.expand_path(File.join(@root_path, weather_path))
498
+ end
499
+ @weather_paths << weather_path
500
+ @weather_files += Dir.glob(weather_path)
501
+ end
502
+ elsif b_models
503
+ if row[1]
504
+ tmp_m_name = row[1]
505
+ else
506
+ tmp_m_name = SecureRandom.uuid
507
+ end
508
+ # Only add models if the row is flagged
509
+ if row[0]&.casecmp('model')&.zero?
510
+ model_path = row[3]
511
+ unless (Pathname.new model_path).absolute?
512
+ model_path = File.expand_path(File.join(@root_path, model_path))
513
+ end
514
+ @models << { name: tmp_m_name.to_underscore, display_name: tmp_m_name, type: row[2], path: model_path }
515
+ end
516
+ elsif b_other_libs
517
+ # determine if the path is relative
518
+ other_path = row[2]
519
+ unless (Pathname.new other_path).absolute?
520
+ other_path = File.expand_path(File.join(@root_path, other_path))
521
+ end
522
+
523
+ @other_files << { lib_zip_name: row[1], path: other_path }
524
+ elsif b_worker_init
525
+ worker_init_path = row[1]
526
+ unless (Pathname.new worker_init_path).absolute?
527
+ worker_init_path = File.expand_path(File.join(@root_path, worker_init_path))
528
+ end
529
+
530
+ @worker_inits << { name: row[0], path: worker_init_path, args: row[2] }
531
+ elsif b_worker_final
532
+ worker_final_path = row[1]
533
+ unless (Pathname.new worker_final_path).absolute?
534
+ worker_final_path = File.expand_path(File.join(@root_path, worker_final_path))
535
+ end
536
+
537
+ @worker_finals << { name: row[0], path: worker_final_path, args: row[2] }
538
+ end
539
+
540
+ next
541
+ end
542
+
543
+ # do some last checks
544
+ @measure_paths = ['./measures'] if @measure_paths.empty?
545
+ end
546
+
547
+ # parse_variables will parse the XLS spreadsheet and save the data into
548
+ # a higher level JSON file. The JSON file is historic and it should really
549
+ # be omitted as an intermediate step
550
+ def parse_variables
551
+ # clean remove whitespace and unicode chars
552
+ # The parse is a unique format (https://github.com/Empact/roo/blob/master/lib/roo/base.rb#L444)
553
+ # If you add a new column and you want that variable in the hash, then you must add it here.
554
+ # rows = @xls.sheet('Variables').parse(:enabled => "# variable")
555
+ # puts rows.inspect
556
+
557
+ rows = nil
558
+ begin
559
+ if @version >= '0.3.3'.to_version
560
+ rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
561
+ measure_name_or_var_type: /type/i,
562
+ measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
563
+ measure_file_name_directory: /measure\sdirectory/i,
564
+ measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
565
+ display_name_short: /parameter\sshort\sdisplay\sname/i,
566
+ # sampling_method: /sampling\smethod/i,
567
+ variable_type: /variable\stype/i,
568
+ units: /units/i,
569
+ default_value: /static.default\svalue/i,
570
+ enums: /enumerations/i,
571
+ min: /min/i,
572
+ max: /max/i,
573
+ mode: /mean|mode/i,
574
+ stddev: /std\sdev/i,
575
+ delta_x: /delta.x/i,
576
+ discrete_values: /discrete\svalues/i,
577
+ discrete_weights: /discrete\sweights/i,
578
+ distribution: /distribution/i,
579
+ source: /data\ssource/i,
580
+ notes: /notes/i,
581
+ relation_to_eui: /typical\svar\sto\seui\srelationship/i,
582
+ clean: true)
583
+ elsif @version >= '0.3.0'.to_version
584
+ rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
585
+ measure_name_or_var_type: /type/i,
586
+ measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
587
+ measure_file_name_directory: /measure\sdirectory/i,
588
+ measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
589
+ # sampling_method: /sampling\smethod/i,
590
+ variable_type: /variable\stype/i,
591
+ units: /units/i,
592
+ default_value: /static.default\svalue/i,
593
+ enums: /enumerations/i,
594
+ min: /min/i,
595
+ max: /max/i,
596
+ mode: /mean|mode/i,
597
+ stddev: /std\sdev/i,
598
+ delta_x: /delta.x/i,
599
+ discrete_values: /discrete\svalues/i,
600
+ discrete_weights: /discrete\sweights/i,
601
+ distribution: /distribution/i,
602
+ source: /data\ssource/i,
603
+ notes: /notes/i,
604
+ relation_to_eui: /typical\svar\sto\seui\srelationship/i,
605
+ clean: true)
606
+ elsif @version >= '0.2.0'.to_version
607
+ rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
608
+ measure_name_or_var_type: /type/i,
609
+ measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
610
+ measure_file_name_directory: /measure\sdirectory/i,
611
+ measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
612
+ sampling_method: /sampling\smethod/i,
613
+ variable_type: /variable\stype/i,
614
+ units: /units/i,
615
+ default_value: /static.default\svalue/i,
616
+ enums: /enumerations/i,
617
+ min: /min/i,
618
+ max: /max/i,
619
+ mode: /mean|mode/i,
620
+ stddev: /std\sdev/i,
621
+ delta_x: /delta.x/i,
622
+ discrete_values: /discrete\svalues/i,
623
+ discrete_weights: /discrete\sweights/i,
624
+ distribution: /distribution/i,
625
+ source: /data\ssource/i,
626
+ notes: /notes/i,
627
+ relation_to_eui: /typical\svar\sto\seui\srelationship/i,
628
+ clean: true)
629
+ elsif @version >= '0.1.12'.to_version
630
+ rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
631
+ measure_name_or_var_type: /type/i,
632
+ measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
633
+ measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
634
+ sampling_method: /sampling\smethod/i,
635
+ variable_type: /variable\stype/i,
636
+ units: /units/i,
637
+ default_value: /static.default\svalue/i,
638
+ enums: /enumerations/i,
639
+ min: /min/i,
640
+ max: /max/i,
641
+ mode: /mean|mode/i,
642
+ stddev: /std\sdev/i,
643
+ delta_x: /delta.x/i,
644
+ discrete_values: /discrete\svalues/i,
645
+ discrete_weights: /discrete\sweights/i,
646
+ distribution: /distribution/i,
647
+ source: /data\ssource/i,
648
+ notes: /notes/i,
649
+ relation_to_eui: /typical\svar\sto\seui\srelationship/i,
650
+ clean: true)
651
+ elsif @version >= '0.1.11'.to_version
652
+ rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
653
+ measure_name_or_var_type: /type/i,
654
+ measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
655
+ measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
656
+ sampling_method: /sampling\smethod/i,
657
+ variable_type: /variable\stype/i,
658
+ units: /units/i,
659
+ default_value: /static.default\svalue/i,
660
+ enums: /enumerations/i,
661
+ min: /min/i,
662
+ max: /max/i,
663
+ mode: /mean|mode/i,
664
+ stddev: /std\sdev/i,
665
+ # delta_x: /delta.x/i,
666
+ discrete_values: /discrete\svalues/i,
667
+ discrete_weights: /discrete\sweights/i,
668
+ distribution: /distribution/i,
669
+ source: /data\ssource/i,
670
+ notes: /notes/i,
671
+ relation_to_eui: /typical\svar\sto\seui\srelationship/i,
672
+ clean: true)
673
+ else
674
+ rows = @xls.sheet('Variables').parse(enabled: /# variable/i,
675
+ measure_name_or_var_type: /type/i,
676
+ measure_file_name_or_var_display_name: /parameter\sdisplay\sname.*/i,
677
+ measure_type_or_parameter_name_in_measure: /parameter\sname\sin\smeasure/i,
678
+ sampling_method: /sampling\smethod/i,
679
+ variable_type: /variable\stype/i,
680
+ units: /units/i,
681
+ default_value: /static.default\svalue/i,
682
+ enums: /enumerations/i,
683
+ min: /min/i,
684
+ max: /max/i,
685
+ mode: /mean|mode/i,
686
+ stddev: /std\sdev/i,
687
+ # delta_x: /delta.x/i,
688
+ # discrete_values: /discrete\svalues/i,
689
+ # discrete_weights: /discrete\sweights/i,
690
+ distribution: /distribution/i,
691
+ source: /data\ssource/i,
692
+ notes: /notes/i,
693
+ relation_to_eui: /typical\svar\sto\seui\srelationship/i,
694
+ clean: true)
695
+ end
696
+ rescue StandardError => e
697
+ raise "Unable to parse spreadsheet #{@xls_filename} with version #{@version} due to error: #{e.message}"
698
+ end
699
+
700
+ raise "Could not find the sheet name 'Variables' in excel file #{@root_path}" unless rows
701
+
702
+ # map the data to another hash that is more easily processed
703
+ data = {}
704
+ data['data'] = []
705
+
706
+ measure_index = -1
707
+ variable_index = -1
708
+ measure_name = nil
709
+ rows.each_with_index do |row, icnt|
710
+ # puts "Parsing line: #{icnt}:#{row}"
711
+
712
+ # check if we are a measure - nil means that the cell was blank
713
+ if row[:enabled].nil?
714
+ if measure_name && data['data'][measure_index]['enabled']
715
+ variable_index += 1
716
+
717
+ var = {}
718
+ var['variable_type'] = row[:measure_name_or_var_type]
719
+ var['display_name'] = row[:measure_file_name_or_var_display_name]
720
+ var['display_name_short'] = row[:display_name_short] ? row[:display_name_short] : var['display_name']
721
+ var['name'] = row[:measure_type_or_parameter_name_in_measure]
722
+ var['index'] = variable_index # order of the variable (not sure of its need)
723
+ var['type'] = row[:variable_type].downcase
724
+ var['units'] = row[:units]
725
+ var['distribution'] = {}
726
+
727
+ # parse the choices/enums
728
+ if var['type'] == 'enum' || var['type'] == 'choice' # this is now a choice
729
+ if row[:enums]
730
+ var['distribution']['enumerations'] = row[:enums].delete('|').split(',').map(&:strip)
731
+ end
732
+ elsif var['type'] == 'bool'
733
+ var['distribution']['enumerations'] = []
734
+ var['distribution']['enumerations'] << 'true' # TODO: should this be a real bool?
735
+ var['distribution']['enumerations'] << 'false'
736
+ end
737
+
738
+ var['distribution']['min'] = row[:min]
739
+ var['distribution']['max'] = row[:max]
740
+ var['distribution']['mean'] = row[:mode]
741
+ var['distribution']['stddev'] = row[:stddev]
742
+ var['distribution']['discrete_values'] = row[:discrete_values]
743
+ var['distribution']['discrete_weights'] = row[:discrete_weights]
744
+ var['distribution']['type'] = row[:distribution]
745
+ var['distribution']['static_value'] = row[:default_value]
746
+ var['distribution']['delta_x'] = row[:delta_x]
747
+
748
+ # type various values correctly
749
+ var['distribution']['min'] = typecast_value(var['type'], var['distribution']['min'])
750
+ var['distribution']['max'] = typecast_value(var['type'], var['distribution']['max'])
751
+ var['distribution']['mean'] = typecast_value(var['type'], var['distribution']['mean'])
752
+ var['distribution']['stddev'] = typecast_value(var['type'], var['distribution']['stddev'])
753
+ var['distribution']['static_value'] = typecast_value(var['type'], var['distribution']['static_value'])
754
+
755
+ # eval the discrete value and weight arrays
756
+ case var['type']
757
+ when 'bool', 'boolean'
758
+ if var['distribution']['discrete_values']
759
+ var['distribution']['discrete_values'] = eval(var['distribution']['discrete_values']).map { |v| v.to_s == 'true' }
760
+ end
761
+ if var['distribution']['discrete_weights'] && var['distribution']['discrete_weights'] != ''
762
+ var['distribution']['discrete_weights'] = eval(var['distribution']['discrete_weights'])
763
+ end
764
+ else
765
+ if var['distribution']['discrete_values']
766
+ var['distribution']['discrete_values'] = eval(var['distribution']['discrete_values'])
767
+ end
768
+ if var['distribution']['discrete_weights'] && var['distribution']['discrete_weights'] != ''
769
+ var['distribution']['discrete_weights'] = eval(var['distribution']['discrete_weights'])
770
+ end
771
+ end
772
+
773
+ var['distribution']['source'] = row[:source]
774
+ var['notes'] = row[:notes]
775
+ var['relation_to_eui'] = row[:relation_to_eui]
776
+
777
+ data['data'][measure_index]['variables'] << var
778
+ end
779
+ else
780
+ measure_index += 1
781
+ variable_index = 0
782
+ data['data'][measure_index] = {}
783
+
784
+ # generate name id
785
+ # TODO: put this into a logger. puts "Parsing measure #{row[1]}"
786
+ display_name = row[:measure_name_or_var_type]
787
+ measure_name = display_name.downcase.strip.tr('-', '_').tr(' ', '_').gsub('__', '_')
788
+ data['data'][measure_index]['display_name'] = display_name
789
+ data['data'][measure_index]['name'] = measure_name
790
+ data['data'][measure_index]['enabled'] = row[:enabled]
791
+ data['data'][measure_index]['measure_file_name'] = row[:measure_file_name_or_var_display_name]
792
+ if row[:measure_file_name_directory]
793
+ data['data'][measure_index]['measure_file_name_directory'] = row[:measure_file_name_directory]
794
+ else
795
+ data['data'][measure_index]['measure_file_name_directory'] = row[:measure_file_name_or_var_display_name].to_underscore
796
+ end
797
+ data['data'][measure_index]['measure_type'] = row[:measure_type_or_parameter_name_in_measure]
798
+ data['data'][measure_index]['version'] = @version_id
799
+
800
+ data['data'][measure_index]['variables'] = []
801
+ end
802
+ end
803
+
804
+ data
805
+ end
806
+
807
+ def parse_outputs
808
+ rows = nil
809
+ if @version >= '0.3.3'.to_version
810
+ rows = @xls.sheet('Outputs').parse(display_name: /variable\sdisplay\sname/i,
811
+ display_name_short: /short\sdisplay\sname/i,
812
+ metadata_id: /taxonomy\sidentifier/i,
813
+ name: /^name$/i,
814
+ units: /units/i,
815
+ visualize: /visualize/i,
816
+ export: /export/i,
817
+ variable_type: /variable\stype/i,
818
+ objective_function: /objective\sfunction/i,
819
+ objective_function_target: /objective\sfunction\starget/i,
820
+ scaling_factor: /scale/i,
821
+ objective_function_group: /objective\sfunction\sgroup/i)
822
+ elsif @version >= '0.3.0'.to_version
823
+ rows = @xls.sheet('Outputs').parse(display_name: /variable\sdisplay\sname/i,
824
+ # display_name_short: /short\sdisplay\sname/i,
825
+ metadata_id: /taxonomy\sidentifier/i,
826
+ name: /^name$/i,
827
+ units: /units/i,
828
+ visualize: /visualize/i,
829
+ export: /export/i,
830
+ variable_type: /variable\stype/i,
831
+ objective_function: /objective\sfunction/i,
832
+ objective_function_target: /objective\sfunction\starget/i,
833
+ scaling_factor: /scale/i,
834
+ objective_function_group: /objective\sfunction\sgroup/i)
835
+ else
836
+ rows = @xls.sheet('Outputs').parse(display_name: /variable\sdisplay\sname/i,
837
+ # display_name_short: /short\sdisplay\sname/i,
838
+ # metadata_id: /taxonomy\sidentifier/i,
839
+ name: /^name$/i,
840
+ units: /units/i,
841
+ # visualize: /visualize/i,
842
+ # export: /export/i,
843
+ # variable_type: /variable\stype/i,
844
+ objective_function: /objective\sfunction/i,
845
+ objective_function_target: /objective\sfunction\starget/i,
846
+ scaling_factor: /scale/i,
847
+ objective_function_group: /objective/i)
848
+
849
+ end
850
+
851
+ unless rows
852
+ raise "Could not find the sheet name 'Outputs' in excel file #{@root_path}"
853
+ end
854
+
855
+ data = {}
856
+ data['output_variables'] = []
857
+
858
+ variable_index = -1
859
+ group_index = 1
860
+
861
+ rows.each_with_index do |row, icnt|
862
+ next if icnt < 1 # skip the first 3 lines of the file
863
+
864
+ var = {}
865
+ var['display_name'] = row[:display_name]
866
+ var['display_name_short'] = row[:display_name_short] ? row[:display_name_short] : row[:display_name]
867
+ var['metadata_id'] = row[:metadata_id]
868
+ var['name'] = row[:name]
869
+ var['units'] = row[:units]
870
+ var['visualize'] = row[:visualize]
871
+ var['export'] = row[:export]
872
+ var['variable_type'] = row[:variable_type].downcase if row[:variable_type]
873
+ var['objective_function'] = row[:objective_function]
874
+ var['objective_function_target'] = row[:objective_function_target]
875
+ var['scaling_factor'] = row[:scaling_factor]
876
+
877
+ if var['objective_function']
878
+ if row[:objective_function_group].nil?
879
+ var['objective_function_group'] = group_index
880
+ group_index += 1
881
+ else
882
+ var['objective_function_group'] = row[:objective_function_group]
883
+ end
884
+ end
885
+ data['output_variables'] << var
886
+ end
887
+
888
+ data
889
+ end
890
+ end
891
+ end
892
+ end
893
+ end