bcl 0.5.3 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,790 +1,864 @@
1
- ######################################################################
2
- # Copyright (c) 2008-2013, Alliance for Sustainable Energy.
3
- # All rights reserved.
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License as published by the Free Software Foundation; either
8
- # version 2.1 of the License, or (at your option) any later version.
9
- #
10
- # This library is distributed in the hope that it will be useful,
11
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
- # Lesser General Public License for more details.
14
- #
15
- # You should have received a copy of the GNU Lesser General Public
16
- # License along with this library; if not, write to the Free Software
17
- # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
- ######################################################################
19
-
20
- require 'net/https'
21
-
22
- module BCL
23
-
24
- class ComponentMethods
25
-
26
- attr_accessor :config
27
- attr_accessor :session
28
- attr_accessor :http
29
- attr_accessor :parsed_measures_path
30
-
31
- def initialize()
32
- @parsed_measures_path = './measures/parsed'
33
- @config = nil
34
- @session = nil
35
- @access_token = nil
36
- @http = nil
37
- @api_version = 2.0
38
- @group_id = nil
39
-
40
- load_config
41
- end
42
-
43
- def login(username=nil, password=nil, url=nil, group_id = nil)
44
- #figure out what url to use
45
- if url.nil?
46
- url = @config[:server][:url]
47
- end
48
- #look for http vs. https
49
- if url.include? "https"
50
- port = 443
51
- else
52
- port = 80
53
- end
54
- #strip out http(s)
55
- url = url.gsub('http://', '')
56
- url = url.gsub('https://', '')
57
-
58
- if username.nil? || password.nil?
59
- # log in via cached creditials
60
- username = @config[:server][:user][:username]
61
- password = @config[:server][:user][:password]
62
- @group_id = group_id || @config[:server][:user][:group]
63
- puts "logging in using credentials in .bcl/config.yml: Connecting to #{url} on port #{port} as #{username}"
64
- else
65
- @group_id = group_id
66
- puts "logging in using credentials in function arguments: Connecting to #{url} on port #{port} as #{username} with group #{@group_id}"
67
- end
68
-
69
- if @group_id.nil?
70
- puts "[WARNING] You did not set a group ID in your config.yml file or pass in a group ID. You can retrieve your group ID from the node number of your group page (e.g., https://bcl.nrel.gov/node/32). Will continue, but you will not be able to upload content."
71
- end
72
-
73
- @http = Net::HTTP.new(url, port)
74
- @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
75
- if port == 443
76
- @http.use_ssl = true
77
- end
78
-
79
- data = %Q({"username":"#{username}","password":"#{password}"})
80
- #data = {"username" => username, "password" => password}
81
-
82
- login_path = "/api/user/login.json"
83
- headers = {'Content-Type' => 'application/json'}
84
-
85
- res = @http.post(login_path, data, headers)
86
-
87
- # for debugging:
88
- #res.each do |key, value|
89
- # puts "#{key}: #{value}"
90
- #end
91
-
92
- if res.code == '200'
93
- puts "Login Successful"
94
-
95
- bnes = ""
96
- bni = ""
97
- junkout = res["set-cookie"].split(";")
98
- junkout.each do |line|
99
- if line =~ /BNES_SESS/
100
- bnes = line.match(/(BNES_SESS.*)/)[0]
101
- end
102
- end
103
-
104
- junkout.each do |line|
105
- if line =~ /BNI/
106
- bni = line.match(/(BNI.*)/)[0]
107
- end
108
- end
109
-
110
- #puts "DATA: #{data}"
111
- session_name = ""
112
- sessid = ""
113
- json = MultiJson.load(res.body)
114
- json.each do |key, val|
115
- if key == 'session_name'
116
- session_name = val
117
- elsif key == 'sessid'
118
- sessid = val
119
- end
120
- end
121
-
122
- @session = session_name + '=' + sessid + ';' + bni + ";" + bnes
123
-
124
- #get access token
125
- token_path = "/services/session/token"
126
- token_headers = {'Content-Type' => 'application/json', 'Cookie' => @session}
127
- #puts "token_headers = #{token_headers.inspect}"
128
- access_token = @http.post(token_path, "", token_headers)
129
- if access_token.code == '200'
130
- @access_token = access_token.body
131
- else
132
- puts "Unable to get access token; uploads will not work"
133
- puts "error code: #{access_token.code}"
134
- puts "error info: #{access_token.body}"
135
- end
136
-
137
- #puts "access_token = *#{@access_token}*"
138
- # puts "cookie = #{@session}"
139
-
140
- res
141
- else
142
-
143
- puts "error code: #{res.code}"
144
- puts "error info: #{res.body}"
145
- puts "continuing as unauthenticated sessions (you can still search and download)"
146
-
147
- res
148
- end
149
- end
150
-
151
- # retrieve, parse, and save metadata for BCL measures
152
- def measure_metadata(search_term = nil, filter_term=nil, return_all_pages = false)
153
- # setup results directory
154
- if !File.exists?(@parsed_measures_path)
155
- FileUtils.mkdir_p(@parsed_measures_path)
156
- end
157
- puts "...storing parsed metadata in #{@parsed_measures_path}"
158
-
159
- # retrieve measures
160
- puts "retrieving measures that match search_term: #{search_term.nil? ? "nil" : search_term} and filters: #{filter_term.nil? ? "nil" : filter_term}"
161
- retrieve_measures(search_term, filter_term, return_all_pages) do |measure|
162
- # parse and save
163
- parse_measure_metadata(measure)
164
- end
165
-
166
- true
167
- end
168
-
169
- # Read in an exisitng measure.rb file and extract the arguments. Note that the measure_name (display name)
170
- # does not exist in the .rb file, so you have to pass this in.
171
- def parse_measure_file(measure_name, measure_filename)
172
- measure_hash = {}
173
- if File.exists? measure_filename
174
- # read in the measure file and extract some information
175
- measure_string = File.read(measure_filename)
176
-
177
- measure_hash[:classname] = measure_string.match(/class (.*) </)[1]
178
- measure_hash[:path] = "#{@parsed_measures_path}/#{measure_hash[:classname]}"
179
- #measure_hash[:display_name] = clean(measure_name)
180
- measure_hash[:name] = measure_hash[:classname].to_underscore
181
- measure_hash[:display_name] = clean(measure_hash[:name].titleize)
182
- if measure_string =~ /OpenStudio::Ruleset::WorkspaceUserScript/
183
- measure_hash[:measure_type] = "EnergyPlusMeasure"
184
- elsif measure_string =~ /OpenStudio::Ruleset::ModelUserScript/
185
- measure_hash[:measure_type] = "RubyMeasure"
186
- elsif measure_string =~ /OpenStudio::Ruleset::ReportingUserScript/
187
- measure_hash[:measure_type] = "ReportingMeasure"
188
- else
189
- raise "measure type is unknown with an inherited class in #{measure_filename}: #{measure_hash.inspect}"
190
- end
191
-
192
- measure_hash[:arguments] = []
193
-
194
- args = measure_string.scan(/(.*).*=.*OpenStudio::Ruleset::OSArgument.*make(.*)Argument\((.*).*\)/)
195
- args.each do |arg|
196
- new_arg = {}
197
- new_arg[:local_variable] = arg[0].strip
198
- new_arg[:variable_type] = arg[1]
199
- arg_params = arg[2].split(",")
200
- new_arg[:name] = arg_params[0].gsub(/"|'/, "")
201
- next if new_arg[:name] == 'info_widget'
202
- choice_vector = arg_params[1] ? arg_params[1].strip : nil
203
-
204
- # local variable name to get other attributes
205
- new_arg[:display_name] = measure_string.match(/#{new_arg[:local_variable]}.setDisplayName\((.*)\)/)[1]
206
- new_arg[:display_name].gsub!(/"|'/, "") if new_arg[:display_name]
207
- new_arg[:display_name] = clean(new_arg[:display_name])
208
-
209
- if measure_string =~ /#{new_arg[:local_variable]}.setDefaultValue/
210
- new_arg[:default_value] = measure_string.match(/#{new_arg[:local_variable]}.setDefaultValue\((.*)\)/)[1]
211
- else
212
- puts "[WARNING] #{measure_hash[:name]}:#{new_arg[:name]} has no default value... will try to continue"
213
- end
214
-
215
- case new_arg[:variable_type]
216
- when "Choice"
217
- # Choices to appear to only be strings?
218
- puts "Choice vector appears to be #{choice_vector}"
219
- new_arg[:default_value].gsub!(/"|'/, "") if new_arg[:default_value]
220
-
221
- # parse the choices from the measure
222
- possible_choices = measure_string.scan(/#{choice_vector}.*<<.*("|')(.*)("|')/)
223
- puts "Possible choices are #{possible_choices}"
224
-
225
- if possible_choices.empty?
226
- new_arg[:choices] = []
227
- else
228
- new_arg[:choices] = possible_choices.map { |c| c[1] }
229
- end
230
-
231
- # if the choices are inherited from the model, then need to just display the default value which
232
- # somehow magically works because that is the display name
233
- if new_arg[:default_value]
234
- new_arg[:choices] << new_arg[:default_value] unless new_arg[:choices].include?(new_arg[:default_value])
235
- end
236
- when "String"
237
- new_arg[:default_value].gsub!(/"|'/, "") if new_arg[:default_value]
238
- when "Bool"
239
- new_arg[:default_value] = new_arg[:default_value].downcase == "true" ? true : false
240
- when "Integer"
241
- new_arg[:default_value] = new_arg[:default_value].to_i if new_arg[:default_value]
242
- when "Double"
243
- new_arg[:default_value] = new_arg[:default_value].to_f if new_arg[:default_value]
244
- else
245
- raise "unknown variable type of #{new_arg[:variable_type]}"
246
- end
247
-
248
- measure_hash[:arguments] << new_arg
249
- end
250
- end
251
-
252
- measure_hash
253
- end
254
-
255
- # Read the measure's information to pull out the metadata and to move into a more friendly directory name.
256
- # option measure is a JSON
257
- def parse_measure_metadata(measure)
258
-
259
- # check for valid measure
260
- if measure[:measure][:name] && measure[:measure][:uuid]
261
-
262
- file_data = download_component(measure[:measure][:uuid])
263
-
264
- if file_data
265
- save_file = File.expand_path("#{@parsed_measures_path}/#{measure[:measure][:name].downcase.gsub(" ", "_")}.zip")
266
- File.open(save_file, 'wb') { |f| f << file_data }
267
-
268
- # unzip file and delete zip.
269
- # TODO check that something was downloaded here before extracting zip
270
- if File.exist? save_file
271
- BCL.extract_zip(save_file, @parsed_measures_path, true)
272
-
273
- # catch a weird case where there is an extra space in an unzip file structure but not in the measure.name
274
- if measure[:measure][:name] == "Add Daylight Sensor at Center of Spaces with a Specified Space Type Assigned"
275
- unless File.exists? "#{@parsed_measures_path}/#{measure[:measure][:name]}"
276
- temp_dir_name = "#{@parsed_measures_path}/Add Daylight Sensor at Center of Spaces with a Specified Space Type Assigned"
277
- FileUtils.move(temp_dir_name, "#{@parsed_measures_path}/#{measure[:measure][:name]}")
278
- end
279
- end
280
-
281
- temp_dir_name = "#{@parsed_measures_path}/#{measure[:measure][:name]}"
282
-
283
- # Read the measure.rb file
284
- #puts "save dir name #{temp_dir_name}"
285
- measure_filename = "#{temp_dir_name}/measure.rb"
286
- measure_hash = parse_measure_file(measure[:measure][:name], measure_filename)
287
-
288
- unless measure_hash.empty?
289
- # move the directory to the class name
290
- FileUtils.rm_rf(measure_hash[:path]) if File.exists?(measure_hash[:path]) && temp_dir_name != measure_hash[:path]
291
- FileUtils.move(temp_dir_name, measure_hash[:path]) unless temp_dir_name == measure_hash[:path]
292
-
293
- # create a new measure.json file for parsing later if need be
294
- File.open("#{measure_hash[:path]}/measure.json", 'w') { |f| f << MultiJson.dump(measure_hash, :pretty => true) }
295
- end
296
- else
297
- puts "Problems downloading #{measure[:measure][:name]}... moving on"
298
- end
299
- end
300
- end
301
- end
302
-
303
- # clean name
304
- def clean(name)
305
- # TODO: save/display errors
306
- errors = ""
307
- m = nil
308
-
309
- clean_name = name
310
- # remove everything btw parentheses
311
- m = clean_name.match(/\((.+?)\)/)
312
- unless m.nil?
313
- errors = errors + " removing parentheses,"
314
- clean_name = clean_name.gsub(/\((.+?)\)/, "")
315
- end
316
-
317
- # remove everything btw brackets
318
- m = nil
319
- m = clean_name.match(/\[(.+?)\]/)
320
- unless m.nil?
321
- errors = errors + " removing brackets,"
322
- clean_name = clean_name.gsub(/\[(.+?)\]/, "")
323
- end
324
-
325
- # remove characters
326
- m = nil
327
- m = clean_name.match(/(\?|\.|\#).+?/)
328
- unless m.nil?
329
- errors = errors + " removing any of following: ?.#"
330
- clean_name = clean_name.gsub(/(\?|\.|\#).+?/, "")
331
- end
332
- clean_name = clean_name.gsub(".", "")
333
- clean_name = clean_name.gsub("?", "")
334
-
335
- clean_name
336
- end
337
-
338
- # retrieve measures for parsing metadata.
339
- # specify a search term to narrow down search or leave nil to retrieve all
340
- # set all_pages to true to iterate over all pages of results
341
- # can't specify filters other than the hard-coded bundle and show_rows
342
- def retrieve_measures(search_term = nil, filter_term=nil, return_all_pages = false, &block)
343
- #raise "Please login before performing this action" if @session.nil?
344
-
345
- #make sure filter_term includes bundle
346
- if filter_term.nil?
347
- filter_term = "fq[]=bundle%3Anrel_measure"
348
- elsif !filter_term.include? "bundle"
349
- filter_term = filter_term + "&fq[]=bundle%3Anrel_measure"
350
- end
351
-
352
-
353
- # use provided search term or nil.
354
- # if return_all_pages is true, iterate over pages of API results. Otherwise only return first 100
355
- results = search(search_term, filter_term, return_all_pages)
356
- puts "#{results[:result].count} results returned"
357
-
358
- results[:result].each do |result|
359
- puts "retrieving measure: #{result[:measure][:name]}"
360
- yield result
361
- end
362
-
363
- end
364
-
365
- # pushes component to the bcl and publishes them (if logged-in as BCL Website Admin user).
366
- # username and password set in ~/.bcl/config.yml file
367
- def push_content(filename_and_path, write_receipt_file, content_type)
368
- raise "Please login before pushing components" if @session.nil?
369
- raise "Do not have a valid access token; try again" if @access_token.nil?
370
- valid = false
371
- res_j = nil
372
- filename = File.basename(filename_and_path)
373
- #TODO remove special characters in the filename; they create firewall errors
374
- #filename = filename.gsub(/\W/,'_').gsub(/___/,'_').gsub(/__/,'_').chomp('_').strip
375
- filepath = File.dirname(filename_and_path) + "/"
376
- file = File.open(filename_and_path, 'rb')
377
- file_b64 = Base64.encode64(file.read)
378
- @data = {
379
- "file" =>
380
- {
381
- "file" => "#{file_b64}",
382
- "filesize" => "#{File.size(filename_and_path)}",
383
- "filename" => filename
384
- },
385
- "node" =>
386
- {
387
- "type" => "#{content_type}",
388
- "field_component_tags" => #TODO remove this field_component_tags once BCL is fixed
389
- {
390
- "und" => "1289"
391
- },
392
- "og_group_ref" =>
393
- {
394
- "und" =>
395
- ["target_id" => @group_id],
396
-
397
- },
398
- "publish" => 1 #NOTE THIS ONLY WORKS IF YOU ARE A BCL SITE ADMIN
399
- }
400
-
401
- }
402
-
403
- path = "/api/content.json"
404
- headers = {'Content-Type' => 'application/json', 'X-CSRF-Token' => @access_token, 'Cookie' => @session}
405
-
406
-
407
- res = @http.post(path, MultiJson.dump(@data), headers)
408
-
409
- res_j = "could not get json from http post response"
410
- if res.code == '200'
411
- puts "200"
412
- res_j = MultiJson.load(res.body)
413
- puts " 200 - Successful Upload"
414
- valid = true
415
-
416
- elsif res.code == '404'
417
- puts " error code: #{res.code} - #{res.body}"
418
- puts " 404 - check these common causes first:"
419
- puts " the filename contains periods (other than the ones before the file extension)"
420
- puts " you are not an 'administrator member' of the group you're trying to upload to"
421
- valid = false
422
- elsif res.code == '500'
423
- puts " error code: #{res.code} - #{res.body}"
424
- raise "server exception"
425
- valid = false
426
- else
427
- puts " error code: #{res.code} - #{res.body}"
428
- valid = false
429
- end
430
-
431
- if valid
432
- #write out a receipt file into the same directory of the component with the same file name as
433
- #the component
434
- if write_receipt_file
435
- File.open(filepath + File.basename(filename, '.tar.gz') + ".receipt", 'w') do |file|
436
- file << Time.now.to_s
437
- end
438
- end
439
- end
440
-
441
- [valid, res_j]
442
-
443
- end
444
-
445
- def push_contents(array_of_components, skip_files_with_receipts, content_type)
446
- logs = []
447
- array_of_components.each do |comp|
448
- receipt_file = File.dirname(comp) + "/" + File.basename(comp, '.tar.gz') + ".receipt"
449
- log_message = ""
450
- if skip_files_with_receipts && File.exists?(receipt_file)
451
- log_message = "skipping because found receipt #{comp}"
452
- puts log_message
453
- else
454
- log_message = "pushing content #{File.basename(comp, '.tar.gz')}"
455
- puts log_message
456
- valid, res = push_content(comp, true, content_type)
457
- log_message += " #{valid} #{res.inspect.chomp}"
458
- end
459
- logs << log_message
460
- end
461
-
462
- logs
463
- end
464
-
465
- # pushes updated content to the bcl and publishes it (if logged-in as BCL Website Admin user).
466
- # username and password set in ~/.bcl/config.yml file
467
- def update_content(filename_and_path, write_receipt_file, uuid)
468
- raise "Please login before pushing components" if @session.nil?
469
- valid = false
470
- res_j = nil
471
- filename = File.basename(filename_and_path)
472
- #TODO remove special characters in the filename; they create firewall errors
473
- #filename = filename.gsub(/\W/,'_').gsub(/___/,'_').gsub(/__/,'_').chomp('_').strip
474
- filepath = File.dirname(filename_and_path) + "/"
475
- file = File.open(filename_and_path, 'rb')
476
- file_b64 = Base64.encode64(file.read)
477
- @data = {
478
- "file" =>
479
- {
480
- "file" => "#{file_b64}",
481
- "filesize" => "#{File.size(filename_and_path)}",
482
- "filename" => filename
483
- },
484
- "node" =>
485
- {
486
- "uuid" => "#{uuid}",
487
- "field_component_tags" => #TODO remove this field_component_tags once BCL is fixed
488
- {
489
- "und" => "1289"
490
- },
491
- "og_group_ref" =>
492
- {
493
- "und" =>
494
- ["target_id" => @group_id],
495
- },
496
- "publish" => 1 #NOTE THIS ONLY WORKS IF YOU ARE A BCL SITE ADMIN
497
- }
498
- }
499
-
500
- path = "/api/content.json"
501
- headers = {'Content-Type' => 'application/json', 'Cookie' => @session, 'X-CSRF-Token' => @access_token}
502
-
503
- res = @http.post(path, MultiJson.dump(@data), headers)
504
-
505
- res_j = "could not get json from http post response"
506
- if res.code == '200'
507
- res_j = MultiJson.load(res.body)
508
- puts " 200 - Successful Upload"
509
- valid = true
510
- elsif res.code == '404'
511
- puts " error code: #{res.code} - #{res.body}"
512
- puts " 404 - check these common causes first:"
513
- puts " the filename contains periods (other than the ones before the file extension)"
514
- puts " you are not an 'administrator member' of the group you're trying to upload to"
515
- valid = false
516
- elsif res.code == '500'
517
- puts " error code: #{res.code} - #{res.body}"
518
- raise "server exception"
519
- valid = false
520
- else
521
- puts " error code: #{res.code} - #{res.body}"
522
- valid = false
523
- end
524
-
525
- if valid
526
- #write out a receipt file into the same directory of the component with the same file name as
527
- #the component
528
- if write_receipt_file
529
- File.open(filepath + File.basename(filename, '.tar.gz') + ".receipt", 'w') do |file|
530
- file << Time.now.to_s
531
- end
532
- end
533
- end
534
-
535
- [valid, res_j]
536
- end
537
-
538
- def update_contents(array_of_components, skip_files_with_receipts)
539
- logs = []
540
- array_of_components.each do |comp|
541
- receipt_file = File.dirname(comp) + "/" + File.basename(comp, '.tar.gz') + ".receipt"
542
- log_message = ""
543
- if skip_files_with_receipts && File.exists?(receipt_file)
544
- log_message = "skipping update because found receipt #{File.basename(comp)}"
545
- puts log_message
546
- else
547
- #extract uuid from the .tar.gz file
548
- uuid = nil
549
- tgz = Zlib::GzipReader.open(comp)
550
- Archive::Tar::Minitar::Reader.open(tgz).each do |entry|
551
- if entry.name == "component.xml" or entry.name == "measure.xml"
552
- xml_file = LibXML::XML::Document.string(entry.read)
553
- uid_node = xml_file.find('uid').first
554
- uuid = uid_node.content
555
- #vid_node = xml_file.find('version_id').first
556
- #vid = vid_node.content
557
- #puts "uuid = #{uuid}; vid = #{vid}"
558
- end
559
- end
560
- if uuid == nil
561
- log_message = "ERROR: uuid not found for #{File.basename(comp)}"
562
- puts log_message
563
- else
564
- log_message = "pushing updated content #{File.basename(comp)}"
565
- puts log_message
566
- valid, res = update_content(comp, true, uuid)
567
- log_message += " #{valid} #{res.inspect.chomp}"
568
- end
569
- end
570
- logs << log_message
571
- end
572
- logs
573
- end
574
-
575
- # Simple method to search bcl and return the result as hash with symbols
576
- # If all = true, iterate over pages of results and return all
577
- # JSON ONLY
578
- def search(search_str=nil, filter_str=nil, all=false)
579
- full_url = "/api/search/"
580
-
581
- #add search term
582
- if !search_str.nil? and search_str != ""
583
- full_url = full_url + search_str
584
- #strip out xml in case it's included. make sure .json is included
585
- full_url = full_url.gsub('.xml', '')
586
- unless search_str.include? ".json"
587
- full_url = full_url + ".json"
588
- end
589
- else
590
- full_url = full_url + "*.json"
591
- end
592
-
593
- #add api_version
594
- if @api_version < 2.0
595
- puts "WARNING: attempting to use search with api_version #{@api_version}. Use API v2.0 for this functionality."
596
- end
597
- full_url = full_url + "?api_version=#{@api_version}"
598
-
599
- #add filters
600
- if !filter_str.nil?
601
- #strip out api_version from filters, if included
602
- if filter_str.include? "api_version="
603
- filter_str = filter_str.gsub(/api_version=\d{1,}/, '')
604
- filter_str = filter_str.gsub(/&api_version=\d{1,}/, '')
605
- end
606
- full_url = full_url + "&" + filter_str
607
- end
608
-
609
- #simple search vs. all results
610
- if !all
611
- puts "search url: #{full_url}"
612
- res = @http.get(full_url)
613
- #return unparsed
614
- MultiJson.load(res.body, :symbolize_keys => true)
615
- else
616
- #iterate over result pages
617
- #modify filter_str for show_rows=200 for maximum returns
618
- if filter_str.include? "show_rows="
619
- full_url = full_url.gsub(/show_rows=\d{1,}/, "show_rows=200")
620
- else
621
- full_url = full_url + "&show_rows=200"
622
- end
623
- #make sure filter_str doesn't already have a page=x
624
- full_url.gsub(/page=\d{1,}/, '')
625
-
626
- pagecnt = 0
627
- continue = 1
628
- results = []
629
- while continue == 1
630
- #retrieve current page
631
- full_url_all = full_url + "&page=#{pagecnt}"
632
- puts "search url: #{full_url_all}"
633
- response = @http.get(full_url_all)
634
- #parse here so you can build results array
635
- res = MultiJson.load(response.body)
636
-
637
- if res["result"].count > 0
638
- pagecnt += 1
639
- res["result"].each do |r|
640
- results << r
641
- end
642
- else
643
- continue = 0
644
- end
645
- end
646
- #return unparsed b/c that is what is expected
647
- formatted_results = {"result" => results}
648
- results_to_return = MultiJson.load(MultiJson.dump(formatted_results), :symbolize_keys => true)
649
- end
650
- end
651
-
652
- # Delete receipt files
653
- def delete_receipts(array_of_components)
654
- array_of_components.each do |comp|
655
- receipt_file = File.dirname(comp) + "/" + File.basename(comp, '.tar.gz') + ".receipt"
656
- if File.exists?(receipt_file)
657
- FileUtils.remove_file(receipt_file)
658
-
659
- end
660
- end
661
- end
662
-
663
- def list_all_measures()
664
- json = search(nil, "fq[]=bundle%3Anrel_measure&show_rows=100")
665
-
666
- json
667
- end
668
-
669
- def download_component(uid)
670
-
671
- begin
672
- result = @http.get("/api/component/download?uids=#{uid}")
673
- puts "DOWNLOADING: /api/component/download?uids=#{uid}"
674
- #puts "RESULTS: #{result.inspect}"
675
- #puts "RESULTS BODY: #{result.body}"
676
-
677
- #look at response code
678
- if result.code == '200'
679
- puts "Download Successful"
680
- result.body ? result.body : nil
681
- else
682
- puts "Download fail. Error code #{result.code}"
683
- nil
684
- end
685
-
686
- rescue
687
- puts "Couldn't download uid(s): #{uid}...skipping"
688
- nil
689
- end
690
-
691
- end
692
-
693
- private
694
-
695
- def load_config()
696
- config_filename = File.expand_path("~/.bcl/config.yml")
697
-
698
- if File.exists?(config_filename)
699
- puts "loading config settings from #{config_filename}"
700
- @config = YAML.load_file(config_filename)
701
- else
702
- #location of template file
703
- FileUtils.mkdir_p(File.dirname(config_filename))
704
- File.open(config_filename, 'w') { |f| f << default_yaml.to_yaml }
705
- File.chmod(0600, config_filename)
706
- puts "******** Please fill in user credentials in #{config_filename} file if you need to upload data **********"
707
- end
708
- end
709
-
710
- def default_yaml
711
- settings = {
712
- :server => {
713
- :url => "https://bcl.nrel.gov",
714
- :user => {
715
- :username => "ENTER_BCL_USERNAME",
716
- :password => "ENTER_BCL_PASSWORD",
717
- :group => "ENTER_GROUP_ID"
718
- }
719
- }
720
- }
721
-
722
- settings
723
- end
724
- end #class ComponentMethods
725
-
726
-
727
- # TODO make this extend the component_xml class (or create a super class around components)
728
-
729
- def BCL.gather_components(component_dir, chunk_size = 0, delete_previousgather = false, destination=nil)
730
- if destination.nil?
731
- @dest_filename = "components"
732
- else
733
- @dest_filename = destination
734
- end
735
- @dest_file_ext = "tar.gz"
736
-
737
- #store the starting directory
738
- current_dir = Dir.pwd
739
-
740
- #an array to hold reporting info about the batches
741
- gather_components_report = []
742
-
743
- #go to the directory containing the components
744
- Dir.chdir(component_dir)
745
-
746
- # delete any old versions of the component chunks
747
- FileUtils.rm_rf("./gather") if delete_previousgather
748
-
749
- #gather all the components into array
750
- targzs = Pathname.glob("./**/*.tar.gz")
751
- tar_cnt = 0
752
- chunk_cnt = 0
753
- targzs.each do |targz|
754
- if chunk_size != 0 && (tar_cnt % chunk_size) == 0
755
- chunk_cnt += 1
756
- end
757
- tar_cnt += 1
758
-
759
- destination_path = "./gather/#{chunk_cnt}"
760
- FileUtils.mkdir_p(destination_path)
761
- destination_file = "#{destination_path}/#{File.basename(targz.to_s)}"
762
- #puts "copying #{targz.to_s} to #{destination_file}"
763
- FileUtils.cp(targz.to_s, destination_file)
764
- end
765
-
766
- #gather all the .tar.gz files into a single tar.gz
767
- (1..chunk_cnt).each do |cnt|
768
- currentdir = Dir.pwd
769
-
770
- paths = []
771
- Pathname.glob("./gather/#{cnt}/*.tar.gz").each do |pt|
772
- paths << File.basename(pt.to_s)
773
- end
774
-
775
- Dir.chdir("./gather/#{cnt}")
776
- destination = "#{@dest_filename}_#{cnt}.#{@dest_file_ext}"
777
- puts "tarring batch #{cnt} of #{chunk_cnt} to #{@dest_filename}_#{cnt}.#{@dest_file_ext}"
778
- BCL.tarball(destination, paths)
779
- Dir.chdir(currentdir)
780
-
781
- #move the tarball back a directory
782
- FileUtils.move("./gather/#{cnt}/#{destination}", "./gather/#{destination}")
783
- end
784
-
785
- Dir.chdir(current_dir)
786
-
787
- end
788
-
789
-
790
- end # module BCL
1
+ ######################################################################
2
+ # Copyright (c) 2008-2014, Alliance for Sustainable Energy.
3
+ # All rights reserved.
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License as published by the Free Software Foundation; either
8
+ # version 2.1 of the License, or (at your option) any later version.
9
+ #
10
+ # This library is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
+ # Lesser General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU Lesser General Public
16
+ # License along with this library; if not, write to the Free Software
17
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
+ ######################################################################
19
+
20
+ module BCL
21
+ class ComponentMethods
22
+ attr_accessor :config
23
+ attr_accessor :parsed_measures_path
24
+ attr_reader :session
25
+ attr_reader :http
26
+ attr_reader :logged_in
27
+
28
+ def initialize
29
+ @parsed_measures_path = './measures/parsed'
30
+ @config = nil
31
+ @session = nil
32
+ @access_token = nil
33
+ @http = nil
34
+ @api_version = 2.0
35
+ @group_id = nil
36
+ @logged_in = false
37
+
38
+ load_config
39
+ end
40
+
41
+ def login(username = nil, password = nil, url = nil, group_id = nil)
42
+ # figure out what url to use
43
+ if url.nil?
44
+ url = @config[:server][:url]
45
+ end
46
+ # look for http vs. https
47
+ if url.include? 'https'
48
+ port = 443
49
+ else
50
+ port = 80
51
+ end
52
+ # strip out http(s)
53
+ url = url.gsub('http://', '')
54
+ url = url.gsub('https://', '')
55
+
56
+ if username.nil? || password.nil?
57
+ # log in via cached creditials
58
+ username = @config[:server][:user][:username]
59
+ password = @config[:server][:user][:password]
60
+ @group_id = group_id || @config[:server][:user][:group]
61
+ puts "logging in using credentials in .bcl/config.yml: Connecting to #{url} on port #{port} as #{username}"
62
+ else
63
+ @group_id = group_id
64
+ puts "logging in using credentials in function arguments: Connecting to #{url} on port #{port} as #{username} with group #{@group_id}"
65
+ end
66
+
67
+ if @group_id.nil?
68
+ puts '[WARNING] You did not set a group ID in your config.yml file or pass in a group ID. You can retrieve your group ID from the node number of your group page (e.g., https://bcl.nrel.gov/node/32). Will continue, but you will not be able to upload content.'
69
+ end
70
+
71
+ @http = Net::HTTP.new(url, port)
72
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
73
+ if port == 443
74
+ @http.use_ssl = true
75
+ end
76
+
77
+ data = %Q({"username":"#{username}","password":"#{password}"})
78
+ # data = {"username" => username, "password" => password}
79
+
80
+ login_path = '/api/user/login.json'
81
+ headers = { 'Content-Type' => 'application/json' }
82
+
83
+ res = @http.post(login_path, data, headers)
84
+
85
+ # for debugging:
86
+ # res.each do |key, value|
87
+ # puts "#{key}: #{value}"
88
+ # end
89
+
90
+ if res.code == '200'
91
+ puts 'Login Successful'
92
+
93
+ bnes = ''
94
+ bni = ''
95
+ junkout = res['set-cookie'].split(';')
96
+ junkout.each do |line|
97
+ if line =~ /BNES_SESS/
98
+ bnes = line.match(/(BNES_SESS.*)/)[0]
99
+ end
100
+ end
101
+
102
+ junkout.each do |line|
103
+ if line =~ /BNI/
104
+ bni = line.match(/(BNI.*)/)[0]
105
+ end
106
+ end
107
+
108
+ # puts "DATA: #{data}"
109
+ session_name = ''
110
+ sessid = ''
111
+ json = MultiJson.load(res.body)
112
+ json.each do |key, val|
113
+ if key == 'session_name'
114
+ session_name = val
115
+ elsif key == 'sessid'
116
+ sessid = val
117
+ end
118
+ end
119
+
120
+ @session = session_name + '=' + sessid + ';' + bni + ';' + bnes
121
+
122
+ # get access token
123
+ token_path = '/services/session/token'
124
+ token_headers = { 'Content-Type' => 'application/json', 'Cookie' => @session }
125
+ # puts "token_headers = #{token_headers.inspect}"
126
+ access_token = @http.post(token_path, '', token_headers)
127
+ if access_token.code == '200'
128
+ @access_token = access_token.body
129
+ else
130
+ puts 'Unable to get access token; uploads will not work'
131
+ puts "error code: #{access_token.code}"
132
+ puts "error info: #{access_token.body}"
133
+ end
134
+
135
+ # puts "access_token = *#{@access_token}*"
136
+ # puts "cookie = #{@session}"
137
+
138
+ res
139
+ else
140
+
141
+ puts "error code: #{res.code}"
142
+ puts "error info: #{res.body}"
143
+ puts 'continuing as unauthenticated sessions (you can still search and download)'
144
+
145
+ res
146
+ end
147
+ end
148
+
149
+ # retrieve, parse, and save metadata for BCL measures
150
+ def measure_metadata(search_term = nil, filter_term = nil, return_all_pages = false)
151
+ # setup results directory
152
+ unless File.exist?(@parsed_measures_path)
153
+ FileUtils.mkdir_p(@parsed_measures_path)
154
+ end
155
+ puts "...storing parsed metadata in #{@parsed_measures_path}"
156
+
157
+ # retrieve measures
158
+ puts "retrieving measures that match search_term: #{search_term.nil? ? 'nil' : search_term} and filters: #{filter_term.nil? ? 'nil' : filter_term}"
159
+ measures = []
160
+ retrieve_measures(search_term, filter_term, return_all_pages) do |m|
161
+ # parse and save
162
+ r = parse_measure_metadata(m)
163
+ measures << r if r
164
+ end
165
+
166
+ measures
167
+ end
168
+
169
+ # Read in an existing measure.rb file and extract the arguments. Note that the measure_name (display name)
170
+ # does not exist in the .rb file, so you have to pass this in.
171
+ def parse_measure_file(_measure_name, measure_filename)
172
+ measure_hash = {}
173
+
174
+ if File.exist? measure_filename
175
+ # read in the measure file and extract some information
176
+ measure_string = File.read(measure_filename)
177
+
178
+ measure_hash[:classname] = measure_string.match(/class (.*) </)[1]
179
+ measure_hash[:name] = measure_hash[:classname].to_underscore
180
+ measure_hash[:display_name] = measure_hash[:name].titleize
181
+ if measure_string =~ /OpenStudio::Ruleset::WorkspaceUserScript/
182
+ measure_hash[:measure_type] = 'EnergyPlusMeasure'
183
+ elsif measure_string =~ /OpenStudio::Ruleset::ModelUserScript/
184
+ measure_hash[:measure_type] = 'RubyMeasure'
185
+ elsif measure_string =~ /OpenStudio::Ruleset::ReportingUserScript/
186
+ measure_hash[:measure_type] = 'ReportingMeasure'
187
+ elsif measure_string =~ /OpenStudio::Ruleset::UtilityUserScript/
188
+ measure_hash[:measure_type] = 'UtilityUserScript'
189
+ else
190
+ fail "measure type is unknown with an inherited class in #{measure_filename}: #{measure_hash.inspect}"
191
+ end
192
+
193
+ measure_hash[:arguments] = []
194
+
195
+ args = measure_string.scan(/(.*).*=.*OpenStudio::Ruleset::OSArgument.*make(.*)Argument\((.*).*\)/)
196
+ args.each do |arg|
197
+ new_arg = {}
198
+ new_arg[:local_variable] = arg[0].strip
199
+ new_arg[:variable_type] = arg[1]
200
+ arg_params = arg[2].split(',')
201
+ new_arg[:name] = arg_params[0].gsub(/"|'/, '')
202
+ next if new_arg[:name] == 'info_widget'
203
+ choice_vector = arg_params[1] ? arg_params[1].strip : nil
204
+
205
+ # local variable name to get other attributes
206
+ new_arg[:display_name] = measure_string.match(/#{new_arg[:local_variable]}.setDisplayName\((.*)\)/)[1]
207
+ new_arg[:display_name].gsub!(/"|'/, '') if new_arg[:display_name]
208
+ p = parse_measure_name(new_arg[:display_name])
209
+ new_arg[:display_name] = p[0]
210
+ new_arg[:units] = p[1]
211
+ new_arg[:description] = p[2]
212
+
213
+ if measure_string =~ /#{new_arg[:local_variable]}.setDefaultValue/
214
+ new_arg[:default_value] = measure_string.match(/#{new_arg[:local_variable]}.setDefaultValue\((.*)\)/)[1]
215
+ else
216
+ puts "[WARNING] #{measure_hash[:name]}:#{new_arg[:name]} has no default value... will try to continue"
217
+ end
218
+
219
+ case new_arg[:variable_type]
220
+ when 'Choice'
221
+ # Choices to appear to only be strings?
222
+ puts "Choice vector appears to be #{choice_vector}"
223
+ new_arg[:default_value].gsub!(/"|'/, '') if new_arg[:default_value]
224
+
225
+ # parse the choices from the measure
226
+ # scan from where the "instance has been created to the measure"
227
+ possible_choices = nil
228
+ possible_choice_block = measure_string # .scan(/#{choice_vector}.*=(.*)#{new_arg[:local_variable]}.*=/mi)
229
+ if possible_choice_block
230
+ # puts "possible_choice_block is #{possible_choice_block}"
231
+ possible_choices = possible_choice_block.scan(/#{choice_vector}.*<<.*(')(.*)(')/)
232
+ possible_choices += possible_choice_block.scan(/#{choice_vector}.*<<.*(")(.*)(")/)
233
+ end
234
+
235
+ # puts "Possible choices are #{possible_choices}"
236
+
237
+ if possible_choices.nil? || possible_choices.empty?
238
+ new_arg[:choices] = []
239
+ else
240
+ new_arg[:choices] = possible_choices.map { |c| c[1] }
241
+ end
242
+
243
+ # if the choices are inherited from the model, then need to just display the default value which
244
+ # somehow magically works because that is the display name
245
+ if new_arg[:default_value]
246
+ new_arg[:choices] << new_arg[:default_value] unless new_arg[:choices].include?(new_arg[:default_value])
247
+ end
248
+ when 'String', 'Path'
249
+ new_arg[:default_value].gsub!(/"|'/, '') if new_arg[:default_value]
250
+ when 'Bool'
251
+ if new_arg[:default_value]
252
+ new_arg[:default_value] = new_arg[:default_value].downcase == 'true' ? true : false
253
+ end
254
+ when 'Integer'
255
+ new_arg[:default_value] = new_arg[:default_value].to_i if new_arg[:default_value]
256
+ when 'Double'
257
+ new_arg[:default_value] = new_arg[:default_value].to_f if new_arg[:default_value]
258
+ else
259
+ fail "unknown variable type of #{new_arg[:variable_type]}"
260
+ end
261
+
262
+ measure_hash[:arguments] << new_arg
263
+ end
264
+ end
265
+
266
+ # check if there is a measure.xml file?
267
+ measure_xml_filename = "#{File.join(File.dirname(measure_filename), File.basename(measure_filename, '.*'))}.xml"
268
+ if File.exist? measure_xml_filename
269
+ f = File.open measure_xml_filename
270
+ doc = Nokogiri::XML(f)
271
+
272
+ # pull out some information
273
+ measure_hash[:name_xml] = doc.xpath('/measure/name').first.content
274
+ measure_hash[:uid] = doc.xpath('/measure/uid').first.content
275
+ measure_hash[:version_id] = doc.xpath('/measure/version_id').first.content
276
+ measure_hash[:modeler_description] = doc.xpath('/measure/modeler_description').first.content
277
+ measure_hash[:description] = doc.xpath('/measure/description').first.content
278
+ measure_hash[:tags] = doc.xpath('/measure/tags/tag').map { |k| k.content }
279
+ f.close
280
+ end
281
+
282
+ measure_hash
283
+ end
284
+
285
+ def translate_measure_hash_to_csv(measure_hash)
286
+ csv = []
287
+ csv << [false, measure_hash[:display_name], measure_hash[:classname], measure_hash[:classname], measure_hash[:measure_type]]
288
+
289
+ measure_hash[:arguments].each do |argument|
290
+ values = []
291
+ values << ''
292
+ values << 'argument'
293
+ values << ''
294
+ values << argument[:display_name]
295
+ values << argument[:name]
296
+ values << argument[:display_name] # this is the default short display name
297
+ values << argument[:variable_type]
298
+ values << argument[:units]
299
+
300
+ # watch out because :default_value can be a boolean
301
+ argument[:default_value].nil? ? values << '' : values << argument[:default_value]
302
+ choices = ''
303
+ if argument[:choices]
304
+ choices << "|#{argument[:choices].join(',')}|" unless argument[:choices].empty?
305
+ end
306
+ values << choices
307
+
308
+ csv << values
309
+ end
310
+
311
+ csv
312
+ end
313
+
314
+ # Read the measure's information to pull out the metadata and to move into a more friendly directory name.
315
+ # argument of measure is a hash
316
+ def parse_measure_metadata(measure)
317
+ m_result = nil
318
+ # check for valid measure
319
+ if measure[:measure][:name] && measure[:measure][:uuid]
320
+
321
+ file_data = download_component(measure[:measure][:uuid])
322
+
323
+ if file_data
324
+ save_file = File.expand_path("#{@parsed_measures_path}/#{measure[:measure][:name].downcase.gsub(' ', '_')}.zip")
325
+ File.open(save_file, 'wb') { |f| f << file_data }
326
+
327
+ # unzip file and delete zip.
328
+ # TODO check that something was downloaded here before extracting zip
329
+ if File.exist? save_file
330
+ BCL.extract_zip(save_file, @parsed_measures_path, true)
331
+
332
+ # catch a weird case where there is an extra space in an unzip file structure but not in the measure.name
333
+ if measure[:measure][:name] == 'Add Daylight Sensor at Center of Spaces with a Specified Space Type Assigned'
334
+ unless File.exist? "#{@parsed_measures_path}/#{measure[:measure][:name]}"
335
+ temp_dir_name = "#{@parsed_measures_path}/Add Daylight Sensor at Center of Spaces with a Specified Space Type Assigned"
336
+ FileUtils.move(temp_dir_name, "#{@parsed_measures_path}/#{measure[:measure][:name]}")
337
+ end
338
+ end
339
+
340
+ temp_dir_name = File.join(@parsed_measures_path, measure[:measure][:name])
341
+
342
+ # Read the measure.rb file
343
+ # puts "save dir name #{temp_dir_name}"
344
+ measure_filename = "#{temp_dir_name}/measure.rb"
345
+ measure_hash = parse_measure_file(measure[:measure][:name], measure_filename)
346
+
347
+ unless measure_hash.empty?
348
+ m_result = measure_hash
349
+ # move the directory to the class name
350
+ new_dir_name = File.join(@parsed_measures_path, measure_hash[:classname])
351
+ FileUtils.rm_rf(new_dir_name) if File.exist?(new_dir_name) && temp_dir_name != measure_hash[:classname]
352
+ FileUtils.move(temp_dir_name, new_dir_name) unless temp_dir_name == measure_hash[:classname]
353
+
354
+ # create a new measure.json file for parsing later if need be
355
+ File.open(File.join(new_dir_name, 'measure.json'), 'w') { |f| f << MultiJson.dump(measure_hash, pretty: true) }
356
+ else
357
+ puts 'Measure Hash was empty... moving on'
358
+ end
359
+ else
360
+ puts "Problems downloading #{measure[:measure][:name]}... moving on"
361
+ end
362
+ end
363
+ end
364
+
365
+ m_result
366
+ end
367
+
368
+ # parse measure name
369
+ def parse_measure_name(name)
370
+ # TODO: save/display errors
371
+ errors = ''
372
+ m = nil
373
+
374
+ clean_name = name
375
+ units = nil
376
+ description = nil
377
+
378
+ # remove everything btw parentheses
379
+ m = clean_name.match(/\((.+?)\)/)
380
+ unless m.nil?
381
+ errors = errors + ' removing parentheses,'
382
+ units = m[1]
383
+ clean_name = clean_name.gsub(/\((.+?)\)/, '')
384
+ end
385
+
386
+ # remove everything btw brackets
387
+ m = nil
388
+ m = clean_name.match(/\[(.+?)\]/)
389
+ unless m.nil?
390
+ errors = errors + ' removing brackets,'
391
+ clean_name = clean_name.gsub(/\[(.+?)\]/, '')
392
+ end
393
+
394
+ # remove characters
395
+ m = nil
396
+ m = clean_name.match(/(\?|\.|\#).+?/)
397
+ unless m.nil?
398
+ errors = errors + ' removing any of following: ?.#'
399
+ clean_name = clean_name.gsub(/(\?|\.|\#).+?/, '')
400
+ end
401
+ clean_name = clean_name.gsub('.', '')
402
+ clean_name = clean_name.gsub('?', '')
403
+
404
+ [clean_name.strip, units, description]
405
+ end
406
+
407
+ # retrieve measures for parsing metadata.
408
+ # specify a search term to narrow down search or leave nil to retrieve all
409
+ # set all_pages to true to iterate over all pages of results
410
+ # can't specify filters other than the hard-coded bundle and show_rows
411
+ def retrieve_measures(search_term = nil, filter_term = nil, return_all_pages = false, &_block)
412
+ # raise "Please login before performing this action" if @session.nil?
413
+
414
+ # make sure filter_term includes bundle
415
+ if filter_term.nil?
416
+ filter_term = 'fq[]=bundle%3Anrel_measure'
417
+ elsif !filter_term.include? 'bundle'
418
+ filter_term = filter_term + '&fq[]=bundle%3Anrel_measure'
419
+ end
420
+
421
+ # use provided search term or nil.
422
+ # if return_all_pages is true, iterate over pages of API results. Otherwise only return first 100
423
+ results = search(search_term, filter_term, return_all_pages)
424
+ puts "#{results[:result].count} results returned"
425
+
426
+ results[:result].each do |result|
427
+ puts "retrieving measure: #{result[:measure][:name]}"
428
+ yield result
429
+ end
430
+ end
431
+
432
+ # evaluate the response from the API in a consistent manner
433
+ def evaluate_api_response(api_response)
434
+ valid = false
435
+ result = { error: 'could not get json from http post response' }
436
+ case api_response.code
437
+ when '200'
438
+ puts " Response Code: #{api_response.code} - #{api_response.body}"
439
+ if api_response.body.empty?
440
+ puts ' 200 BUT ERROR: Returned body was empty. Possible causes:'
441
+ puts ' - BSD tar on Mac OSX vs gnutar'
442
+ result = { error: 'returned 200, but returned body was empty' }
443
+ valid = false
444
+ else
445
+ puts ' 200 - Successful Upload'
446
+ result = MultiJson.load api_response.body
447
+ valid = true
448
+ end
449
+ when '404'
450
+ puts " Response: #{api_response.code} - #{api_response.body}"
451
+ puts ' 404 - check these common causes first:'
452
+ puts ' - the filename contains periods (other than the ones before the file extension)'
453
+ puts " - you are not an 'administrator member' of the group you're trying to upload to"
454
+ result = MultiJson.load api_response.body
455
+ valid = false
456
+ when '406'
457
+ puts " Response: #{api_response.code} - #{api_response.body}"
458
+ puts ' 406 - check these common causes first:'
459
+ puts ' - the UUID of the item that you are uploading is already on the BCL'
460
+ puts ' - the group_id is not correct in the config.yml (go to group on site, and copy the number at the end of the URL)'
461
+ puts " - you are not an 'administrator member' of the group you're trying to upload to"
462
+ result = MultiJson.load api_response.body
463
+ valid = false
464
+ when '500'
465
+ puts " Response: #{api_response.code} - #{api_response.body}"
466
+ fail 'server exception'
467
+ valid = false
468
+ else
469
+ puts " Response: #{api_response.code} - #{api_response.body}"
470
+ valid = false
471
+ end
472
+
473
+ [valid, result]
474
+ end
475
+
476
+ # Construct the post parameter for the API content.json end point.
477
+ # param(@update) is a boolean that triggers whether to use content_type or uuid
478
+ def construct_post_data(filepath, update, content_type_or_uuid)
479
+ # TODO remove special characters in the filename; they create firewall errors
480
+ # filename = filename.gsub(/\W/,'_').gsub(/___/,'_').gsub(/__/,'_').chomp('_').strip
481
+
482
+ file_b64 = Base64.encode64(File.read(filepath))
483
+
484
+ data = {}
485
+ data['file'] = {
486
+ 'file' => file_b64,
487
+ 'filesize' => File.size(filepath).to_s,
488
+ 'filename' => File.basename(filepath)
489
+ }
490
+
491
+ data['node'] = {}
492
+
493
+ # Only include the content type if this is an update
494
+ if update
495
+ data['node']['uuid'] = content_type_or_uuid
496
+ else
497
+ data['node']['type'] = content_type_or_uuid
498
+ end
499
+
500
+ # TODO remove this field_component_tags once BCL is fixed
501
+ data['node']['field_component_tags'] = { 'und' => '1289' }
502
+ data['node']['og_group_ref'] = { 'und' => ['target_id' => @group_id] }
503
+
504
+ # NOTE THIS ONLY WORKS IF YOU ARE A BCL SITE ADMIN
505
+ data['node']['publish'] = '1'
506
+
507
+ data
508
+ end
509
+
510
+ # pushes component to the bcl and publishes them (if logged-in as BCL Website Admin user).
511
+ # username, password, and group_id are set in the ~/.bcl/config.yml file
512
+ def push_content(filename_and_path, write_receipt_file, content_type)
513
+ fail 'Please login before pushing components' if @session.nil?
514
+ fail 'Do not have a valid access token; try again' if @access_token.nil?
515
+
516
+ data = construct_post_data(filename_and_path, false, content_type)
517
+
518
+ path = '/api/content.json'
519
+ headers = { 'Content-Type' => 'application/json', 'X-CSRF-Token' => @access_token, 'Cookie' => @session }
520
+
521
+ res = @http.post(path, MultiJson.dump(data), headers)
522
+
523
+ valid, json = evaluate_api_response(res)
524
+
525
+ if valid
526
+ # write out a receipt file into the same directory of the component with the same file name as
527
+ # the component
528
+ if write_receipt_file
529
+ File.open("#{File.dirname(filename_and_path)}/#{File.basename(filename_and_path, '.tar.gz')}.receipt", 'w') do |file|
530
+ file << Time.now.to_s
531
+ end
532
+ end
533
+ end
534
+
535
+ [valid, json]
536
+ end
537
+
538
+ # pushes updated content to the bcl and publishes it (if logged-in as BCL Website Admin user).
539
+ # username and password set in ~/.bcl/config.yml file
540
+ def update_content(filename_and_path, write_receipt_file, uuid = nil)
541
+ fail 'Please login before pushing components' unless @session
542
+
543
+ # get the UUID if zip or xml file
544
+ version_id = nil
545
+ if uuid.nil?
546
+ puts File.extname(filename_and_path).downcase
547
+ if filename_and_path =~ /^.*.tar.gz$/i
548
+ uuid, version_id = uuid_vid_from_tarball(filename_and_path)
549
+ puts "Parsed uuid out of tar.gz file with value #{uuid}"
550
+ end
551
+ else
552
+ # verify the uuid via regex
553
+ unless uuid =~ /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
554
+ fail "uuid of #{uuid} is invalid"
555
+ end
556
+ end
557
+ fail 'Please pass in a tar.gz file or pass in the uuid' unless uuid
558
+
559
+ data = construct_post_data(filename_and_path, true, uuid)
560
+
561
+ path = '/api/content.json'
562
+ headers = { 'Content-Type' => 'application/json', 'X-CSRF-Token' => @access_token, 'Cookie' => @session }
563
+
564
+ res = @http.post(path, MultiJson.dump(data), headers)
565
+
566
+ valid, json = evaluate_api_response(res)
567
+
568
+ if valid
569
+ # write out a receipt file into the same directory of the component with the same file name as
570
+ # the component
571
+ if write_receipt_file
572
+ File.open("#{File.dirname(filename_and_path)}/#{File.basename(filename_and_path, '.tar.gz')}.receipt", 'w') do |file|
573
+ file << Time.now.to_s
574
+ end
575
+ end
576
+ end
577
+
578
+ [valid, json]
579
+ end
580
+
581
+ def push_contents(array_of_components, skip_files_with_receipts, content_type)
582
+ logs = []
583
+ array_of_components.each do |comp|
584
+ receipt_file = File.dirname(comp) + '/' + File.basename(comp, '.tar.gz') + '.receipt'
585
+ log_message = ''
586
+ if skip_files_with_receipts && File.exist?(receipt_file)
587
+ log_message = "skipping because found receipt #{comp}"
588
+ puts log_message
589
+ else
590
+ log_message = "pushing content #{File.basename(comp, '.tar.gz')}"
591
+ puts log_message
592
+ valid, res = push_content(comp, true, content_type)
593
+ log_message += " #{valid} #{res.inspect.chomp}"
594
+ end
595
+ logs << log_message
596
+ end
597
+
598
+ logs
599
+ end
600
+
601
+ # Unpack the tarball in memory and extract the XML file to read the UUID and Version ID
602
+ def uuid_vid_from_tarball(path_to_tarball)
603
+ uuid = nil
604
+ vid = nil
605
+
606
+ fail "File does not exist #{path_to_tarball}" unless File.exist? path_to_tarball
607
+ tgz = Zlib::GzipReader.open(path_to_tarball)
608
+ Archive::Tar::Minitar::Reader.open(tgz).each do |entry|
609
+ # If taring with tar zcf ameasure.tar.gz -C measure_dir .
610
+ if entry.name =~ /^.{0,2}component.xml$/ || entry.name =~ /^.{0,2}measure.xml$/
611
+ xml_file = Nokogiri::XML(entry.read)
612
+
613
+ # pull out some information
614
+ if entry.name =~ /component/
615
+ u = xml_file.xpath('/component/uid').first
616
+ v = xml_file.xpath('/component/version_id').first
617
+ else
618
+ u = xml_file.xpath('/measure/uid').first
619
+ v = xml_file.xpath('/measure/version_id').first
620
+ end
621
+ fail "Could not find UUID in XML file #{path_to_tarball}" unless u
622
+ # Don't error on version not existing.
623
+
624
+ uuid = u.content
625
+ vid = v ? v.content : nil
626
+
627
+ # puts "uuid = #{uuid}; vid = #{vid}"
628
+ end
629
+ end
630
+
631
+ [uuid, vid]
632
+ end
633
+
634
+ def update_contents(array_of_tarball_components, skip_files_with_receipts)
635
+ logs = []
636
+ array_of_tarball_components.each do |comp|
637
+ receipt_file = File.dirname(comp) + '/' + File.basename(comp, '.tar.gz') + '.receipt'
638
+ log_message = ''
639
+ if skip_files_with_receipts && File.exist?(receipt_file)
640
+ log_message = "skipping update because found receipt #{File.basename(comp)}"
641
+ puts log_message
642
+ else
643
+ uuid, vid = uuid_vid_from_tarball(comp)
644
+ if uuid.nil?
645
+ log_message = "ERROR: uuid not found for #{File.basename(comp)}"
646
+ puts log_message
647
+ else
648
+ log_message = "pushing updated content #{File.basename(comp)}"
649
+ puts log_message
650
+ valid, res = update_content(comp, true, uuid)
651
+ log_message += " #{valid} #{res.inspect.chomp}"
652
+ end
653
+ end
654
+ logs << log_message
655
+ end
656
+ logs
657
+ end
658
+
659
+ # Simple method to search bcl and return the result as hash with symbols
660
+ # If all = true, iterate over pages of results and return all
661
+ # JSON ONLY
662
+ def search(search_str = nil, filter_str = nil, all = false)
663
+ full_url = '/api/search/'
664
+
665
+ # add search term
666
+ if !search_str.nil? and search_str != ''
667
+ full_url = full_url + search_str
668
+ # strip out xml in case it's included. make sure .json is included
669
+ full_url = full_url.gsub('.xml', '')
670
+ unless search_str.include? '.json'
671
+ full_url = full_url + '.json'
672
+ end
673
+ else
674
+ full_url = full_url + '*.json'
675
+ end
676
+
677
+ # add api_version
678
+ if @api_version < 2.0
679
+ puts "WARNING: attempting to use search with api_version #{@api_version}. Use API v2.0 for this functionality."
680
+ end
681
+ full_url = full_url + "?api_version=#{@api_version}"
682
+
683
+ # add filters
684
+ unless filter_str.nil?
685
+ # strip out api_version from filters, if included
686
+ if filter_str.include? 'api_version='
687
+ filter_str = filter_str.gsub(/api_version=\d{1,}/, '')
688
+ filter_str = filter_str.gsub(/&api_version=\d{1,}/, '')
689
+ end
690
+ full_url = full_url + '&' + filter_str
691
+ end
692
+
693
+ # simple search vs. all results
694
+ if !all
695
+ puts "search url: #{full_url}"
696
+ res = @http.get(full_url)
697
+ # return unparsed
698
+ MultiJson.load(res.body, symbolize_keys: true)
699
+ else
700
+ # iterate over result pages
701
+ # modify filter_str for show_rows=200 for maximum returns
702
+ if filter_str.include? 'show_rows='
703
+ full_url = full_url.gsub(/show_rows=\d{1,}/, 'show_rows=200')
704
+ else
705
+ full_url = full_url + '&show_rows=200'
706
+ end
707
+ # make sure filter_str doesn't already have a page=x
708
+ full_url.gsub(/page=\d{1,}/, '')
709
+
710
+ pagecnt = 0
711
+ continue = 1
712
+ results = []
713
+ while continue == 1
714
+ # retrieve current page
715
+ full_url_all = full_url + "&page=#{pagecnt}"
716
+ puts "search url: #{full_url_all}"
717
+ response = @http.get(full_url_all)
718
+ # parse here so you can build results array
719
+ res = MultiJson.load(response.body)
720
+
721
+ if res['result'].count > 0
722
+ pagecnt += 1
723
+ res['result'].each do |r|
724
+ results << r
725
+ end
726
+ else
727
+ continue = 0
728
+ end
729
+ end
730
+ # return unparsed b/c that is what is expected
731
+ formatted_results = { 'result' => results }
732
+ results_to_return = MultiJson.load(MultiJson.dump(formatted_results), symbolize_keys: true)
733
+ end
734
+ end
735
+
736
+ # Delete receipt files
737
+ def delete_receipts(array_of_components)
738
+ array_of_components.each do |comp|
739
+ receipt_file = File.dirname(comp) + '/' + File.basename(comp, '.tar.gz') + '.receipt'
740
+ if File.exist?(receipt_file)
741
+ FileUtils.remove_file(receipt_file)
742
+
743
+ end
744
+ end
745
+ end
746
+
747
+ def list_all_measures
748
+ json = search(nil, 'fq[]=bundle%3Anrel_measure&show_rows=100')
749
+
750
+ json
751
+ end
752
+
753
+ def download_component(uid)
754
+ result = @http.get("/api/component/download?uids=#{uid}")
755
+ puts "DOWNLOADING: /api/component/download?uids=#{uid}"
756
+ # puts "RESULTS: #{result.inspect}"
757
+ # puts "RESULTS BODY: #{result.body}"
758
+ # look at response code
759
+ if result.code == '200'
760
+ puts 'Download Successful'
761
+ result.body ? result.body : nil
762
+ else
763
+ puts "Download fail. Error code #{result.code}"
764
+ nil
765
+ end
766
+ rescue
767
+ puts "Couldn't download uid(s): #{uid}...skipping"
768
+ nil
769
+ end
770
+
771
+ private
772
+
773
+ def load_config
774
+ config_filename = File.expand_path('~/.bcl/config.yml')
775
+
776
+ if File.exist?(config_filename)
777
+ puts "loading config settings from #{config_filename}"
778
+ @config = YAML.load_file(config_filename)
779
+ else
780
+ # location of template file
781
+ FileUtils.mkdir_p(File.dirname(config_filename))
782
+ File.open(config_filename, 'w') { |f| f << default_yaml.to_yaml }
783
+ File.chmod(0600, config_filename)
784
+ puts "******** Please fill in user credentials in #{config_filename} file if you need to upload data **********"
785
+ end
786
+ end
787
+
788
+ def default_yaml
789
+ settings = {
790
+ server: {
791
+ url: 'https://bcl.nrel.gov',
792
+ user: {
793
+ username: 'ENTER_BCL_USERNAME',
794
+ password: 'ENTER_BCL_PASSWORD',
795
+ group: 'ENTER_GROUP_ID'
796
+ }
797
+ }
798
+ }
799
+
800
+ settings
801
+ end
802
+ end # class ComponentMethods
803
+
804
+ # TODO make this extend the component_xml class (or create a super class around components)
805
+
806
+ def self.gather_components(component_dir, chunk_size = 0, delete_previousgather = false, destination = nil)
807
+ if destination.nil?
808
+ @dest_filename = 'components'
809
+ else
810
+ @dest_filename = destination
811
+ end
812
+ @dest_file_ext = 'tar.gz'
813
+
814
+ # store the starting directory
815
+ current_dir = Dir.pwd
816
+
817
+ # an array to hold reporting info about the batches
818
+ gather_components_report = []
819
+
820
+ # go to the directory containing the components
821
+ Dir.chdir(component_dir)
822
+
823
+ # delete any old versions of the component chunks
824
+ FileUtils.rm_rf('./gather') if delete_previousgather
825
+
826
+ # gather all the components into array
827
+ targzs = Pathname.glob('./**/*.tar.gz')
828
+ tar_cnt = 0
829
+ chunk_cnt = 0
830
+ targzs.each do |targz|
831
+ if chunk_size != 0 && (tar_cnt % chunk_size) == 0
832
+ chunk_cnt += 1
833
+ end
834
+ tar_cnt += 1
835
+
836
+ destination_path = "./gather/#{chunk_cnt}"
837
+ FileUtils.mkdir_p(destination_path)
838
+ destination_file = "#{destination_path}/#{File.basename(targz.to_s)}"
839
+ # puts "copying #{targz.to_s} to #{destination_file}"
840
+ FileUtils.cp(targz.to_s, destination_file)
841
+ end
842
+
843
+ # gather all the .tar.gz files into a single tar.gz
844
+ (1..chunk_cnt).each do |cnt|
845
+ currentdir = Dir.pwd
846
+
847
+ paths = []
848
+ Pathname.glob("./gather/#{cnt}/*.tar.gz").each do |pt|
849
+ paths << File.basename(pt.to_s)
850
+ end
851
+
852
+ Dir.chdir("./gather/#{cnt}")
853
+ destination = "#{@dest_filename}_#{cnt}.#{@dest_file_ext}"
854
+ puts "tarring batch #{cnt} of #{chunk_cnt} to #{@dest_filename}_#{cnt}.#{@dest_file_ext}"
855
+ BCL.tarball(destination, paths)
856
+ Dir.chdir(currentdir)
857
+
858
+ # move the tarball back a directory
859
+ FileUtils.move("./gather/#{cnt}/#{destination}", "./gather/#{destination}")
860
+ end
861
+
862
+ Dir.chdir(current_dir)
863
+ end
864
+ end # module BCL