bcl 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bcl/base_xml.rb +33 -16
- data/lib/bcl/component.rb +388 -390
- data/lib/bcl/component_from_spreadsheet.rb +34 -37
- data/lib/bcl/component_methods.rb +864 -790
- data/lib/bcl/component_spreadsheet.rb +295 -309
- data/lib/bcl/core_ext.rb +38 -6
- data/lib/bcl/master_taxonomy.rb +533 -552
- data/lib/bcl/tar_ball.rb +78 -80
- data/lib/bcl/version.rb +2 -2
- data/lib/bcl.rb +41 -34
- metadata +84 -14
@@ -1,790 +1,864 @@
|
|
1
|
-
######################################################################
|
2
|
-
# Copyright (c) 2008-
|
3
|
-
# All rights reserved.
|
4
|
-
#
|
5
|
-
# This library is free software; you can redistribute it and/or
|
6
|
-
# modify it under the terms of the GNU Lesser General Public
|
7
|
-
# License as published by the Free Software Foundation; either
|
8
|
-
# version 2.1 of the License, or (at your option) any later version.
|
9
|
-
#
|
10
|
-
# This library is distributed in the hope that it will be useful,
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
-
# Lesser General Public License for more details.
|
14
|
-
#
|
15
|
-
# You should have received a copy of the GNU Lesser General Public
|
16
|
-
# License along with this library; if not, write to the Free Software
|
17
|
-
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
18
|
-
######################################################################
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
@
|
33
|
-
@
|
34
|
-
@
|
35
|
-
@
|
36
|
-
@
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
port =
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
#
|
88
|
-
#
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
puts "
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
puts
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
end
|
165
|
-
|
166
|
-
|
167
|
-
end
|
168
|
-
|
169
|
-
# Read in an
|
170
|
-
# does not exist in the .rb file, so you have to pass this in.
|
171
|
-
def parse_measure_file(
|
172
|
-
measure_hash = {}
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
measure_hash[:
|
179
|
-
|
180
|
-
measure_hash[:
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
args
|
196
|
-
|
197
|
-
new_arg
|
198
|
-
new_arg[:
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
new_arg[:display_name].
|
207
|
-
new_arg[:display_name]
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
#
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
puts "
|
428
|
-
|
429
|
-
end
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
puts
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
if valid
|
526
|
-
#write out a receipt file into the same directory of the component with the same file name as
|
527
|
-
#the component
|
528
|
-
if write_receipt_file
|
529
|
-
File.open(
|
530
|
-
file << Time.now.to_s
|
531
|
-
end
|
532
|
-
end
|
533
|
-
end
|
534
|
-
|
535
|
-
[valid,
|
536
|
-
end
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
uuid =
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
end
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
end
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
788
|
-
|
789
|
-
|
790
|
-
|
1
|
+
######################################################################
|
2
|
+
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
3
|
+
# All rights reserved.
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License as published by the Free Software Foundation; either
|
8
|
+
# version 2.1 of the License, or (at your option) any later version.
|
9
|
+
#
|
10
|
+
# This library is distributed in the hope that it will be useful,
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
+
# Lesser General Public License for more details.
|
14
|
+
#
|
15
|
+
# You should have received a copy of the GNU Lesser General Public
|
16
|
+
# License along with this library; if not, write to the Free Software
|
17
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
18
|
+
######################################################################
|
19
|
+
|
20
|
+
module BCL
|
21
|
+
class ComponentMethods
|
22
|
+
attr_accessor :config
|
23
|
+
attr_accessor :parsed_measures_path
|
24
|
+
attr_reader :session
|
25
|
+
attr_reader :http
|
26
|
+
attr_reader :logged_in
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
@parsed_measures_path = './measures/parsed'
|
30
|
+
@config = nil
|
31
|
+
@session = nil
|
32
|
+
@access_token = nil
|
33
|
+
@http = nil
|
34
|
+
@api_version = 2.0
|
35
|
+
@group_id = nil
|
36
|
+
@logged_in = false
|
37
|
+
|
38
|
+
load_config
|
39
|
+
end
|
40
|
+
|
41
|
+
def login(username = nil, password = nil, url = nil, group_id = nil)
|
42
|
+
# figure out what url to use
|
43
|
+
if url.nil?
|
44
|
+
url = @config[:server][:url]
|
45
|
+
end
|
46
|
+
# look for http vs. https
|
47
|
+
if url.include? 'https'
|
48
|
+
port = 443
|
49
|
+
else
|
50
|
+
port = 80
|
51
|
+
end
|
52
|
+
# strip out http(s)
|
53
|
+
url = url.gsub('http://', '')
|
54
|
+
url = url.gsub('https://', '')
|
55
|
+
|
56
|
+
if username.nil? || password.nil?
|
57
|
+
# log in via cached creditials
|
58
|
+
username = @config[:server][:user][:username]
|
59
|
+
password = @config[:server][:user][:password]
|
60
|
+
@group_id = group_id || @config[:server][:user][:group]
|
61
|
+
puts "logging in using credentials in .bcl/config.yml: Connecting to #{url} on port #{port} as #{username}"
|
62
|
+
else
|
63
|
+
@group_id = group_id
|
64
|
+
puts "logging in using credentials in function arguments: Connecting to #{url} on port #{port} as #{username} with group #{@group_id}"
|
65
|
+
end
|
66
|
+
|
67
|
+
if @group_id.nil?
|
68
|
+
puts '[WARNING] You did not set a group ID in your config.yml file or pass in a group ID. You can retrieve your group ID from the node number of your group page (e.g., https://bcl.nrel.gov/node/32). Will continue, but you will not be able to upload content.'
|
69
|
+
end
|
70
|
+
|
71
|
+
@http = Net::HTTP.new(url, port)
|
72
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
73
|
+
if port == 443
|
74
|
+
@http.use_ssl = true
|
75
|
+
end
|
76
|
+
|
77
|
+
data = %Q({"username":"#{username}","password":"#{password}"})
|
78
|
+
# data = {"username" => username, "password" => password}
|
79
|
+
|
80
|
+
login_path = '/api/user/login.json'
|
81
|
+
headers = { 'Content-Type' => 'application/json' }
|
82
|
+
|
83
|
+
res = @http.post(login_path, data, headers)
|
84
|
+
|
85
|
+
# for debugging:
|
86
|
+
# res.each do |key, value|
|
87
|
+
# puts "#{key}: #{value}"
|
88
|
+
# end
|
89
|
+
|
90
|
+
if res.code == '200'
|
91
|
+
puts 'Login Successful'
|
92
|
+
|
93
|
+
bnes = ''
|
94
|
+
bni = ''
|
95
|
+
junkout = res['set-cookie'].split(';')
|
96
|
+
junkout.each do |line|
|
97
|
+
if line =~ /BNES_SESS/
|
98
|
+
bnes = line.match(/(BNES_SESS.*)/)[0]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
junkout.each do |line|
|
103
|
+
if line =~ /BNI/
|
104
|
+
bni = line.match(/(BNI.*)/)[0]
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# puts "DATA: #{data}"
|
109
|
+
session_name = ''
|
110
|
+
sessid = ''
|
111
|
+
json = MultiJson.load(res.body)
|
112
|
+
json.each do |key, val|
|
113
|
+
if key == 'session_name'
|
114
|
+
session_name = val
|
115
|
+
elsif key == 'sessid'
|
116
|
+
sessid = val
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
@session = session_name + '=' + sessid + ';' + bni + ';' + bnes
|
121
|
+
|
122
|
+
# get access token
|
123
|
+
token_path = '/services/session/token'
|
124
|
+
token_headers = { 'Content-Type' => 'application/json', 'Cookie' => @session }
|
125
|
+
# puts "token_headers = #{token_headers.inspect}"
|
126
|
+
access_token = @http.post(token_path, '', token_headers)
|
127
|
+
if access_token.code == '200'
|
128
|
+
@access_token = access_token.body
|
129
|
+
else
|
130
|
+
puts 'Unable to get access token; uploads will not work'
|
131
|
+
puts "error code: #{access_token.code}"
|
132
|
+
puts "error info: #{access_token.body}"
|
133
|
+
end
|
134
|
+
|
135
|
+
# puts "access_token = *#{@access_token}*"
|
136
|
+
# puts "cookie = #{@session}"
|
137
|
+
|
138
|
+
res
|
139
|
+
else
|
140
|
+
|
141
|
+
puts "error code: #{res.code}"
|
142
|
+
puts "error info: #{res.body}"
|
143
|
+
puts 'continuing as unauthenticated sessions (you can still search and download)'
|
144
|
+
|
145
|
+
res
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
# retrieve, parse, and save metadata for BCL measures
|
150
|
+
def measure_metadata(search_term = nil, filter_term = nil, return_all_pages = false)
|
151
|
+
# setup results directory
|
152
|
+
unless File.exist?(@parsed_measures_path)
|
153
|
+
FileUtils.mkdir_p(@parsed_measures_path)
|
154
|
+
end
|
155
|
+
puts "...storing parsed metadata in #{@parsed_measures_path}"
|
156
|
+
|
157
|
+
# retrieve measures
|
158
|
+
puts "retrieving measures that match search_term: #{search_term.nil? ? 'nil' : search_term} and filters: #{filter_term.nil? ? 'nil' : filter_term}"
|
159
|
+
measures = []
|
160
|
+
retrieve_measures(search_term, filter_term, return_all_pages) do |m|
|
161
|
+
# parse and save
|
162
|
+
r = parse_measure_metadata(m)
|
163
|
+
measures << r if r
|
164
|
+
end
|
165
|
+
|
166
|
+
measures
|
167
|
+
end
|
168
|
+
|
169
|
+
# Read in an existing measure.rb file and extract the arguments. Note that the measure_name (display name)
|
170
|
+
# does not exist in the .rb file, so you have to pass this in.
|
171
|
+
def parse_measure_file(_measure_name, measure_filename)
|
172
|
+
measure_hash = {}
|
173
|
+
|
174
|
+
if File.exist? measure_filename
|
175
|
+
# read in the measure file and extract some information
|
176
|
+
measure_string = File.read(measure_filename)
|
177
|
+
|
178
|
+
measure_hash[:classname] = measure_string.match(/class (.*) </)[1]
|
179
|
+
measure_hash[:name] = measure_hash[:classname].to_underscore
|
180
|
+
measure_hash[:display_name] = measure_hash[:name].titleize
|
181
|
+
if measure_string =~ /OpenStudio::Ruleset::WorkspaceUserScript/
|
182
|
+
measure_hash[:measure_type] = 'EnergyPlusMeasure'
|
183
|
+
elsif measure_string =~ /OpenStudio::Ruleset::ModelUserScript/
|
184
|
+
measure_hash[:measure_type] = 'RubyMeasure'
|
185
|
+
elsif measure_string =~ /OpenStudio::Ruleset::ReportingUserScript/
|
186
|
+
measure_hash[:measure_type] = 'ReportingMeasure'
|
187
|
+
elsif measure_string =~ /OpenStudio::Ruleset::UtilityUserScript/
|
188
|
+
measure_hash[:measure_type] = 'UtilityUserScript'
|
189
|
+
else
|
190
|
+
fail "measure type is unknown with an inherited class in #{measure_filename}: #{measure_hash.inspect}"
|
191
|
+
end
|
192
|
+
|
193
|
+
measure_hash[:arguments] = []
|
194
|
+
|
195
|
+
args = measure_string.scan(/(.*).*=.*OpenStudio::Ruleset::OSArgument.*make(.*)Argument\((.*).*\)/)
|
196
|
+
args.each do |arg|
|
197
|
+
new_arg = {}
|
198
|
+
new_arg[:local_variable] = arg[0].strip
|
199
|
+
new_arg[:variable_type] = arg[1]
|
200
|
+
arg_params = arg[2].split(',')
|
201
|
+
new_arg[:name] = arg_params[0].gsub(/"|'/, '')
|
202
|
+
next if new_arg[:name] == 'info_widget'
|
203
|
+
choice_vector = arg_params[1] ? arg_params[1].strip : nil
|
204
|
+
|
205
|
+
# local variable name to get other attributes
|
206
|
+
new_arg[:display_name] = measure_string.match(/#{new_arg[:local_variable]}.setDisplayName\((.*)\)/)[1]
|
207
|
+
new_arg[:display_name].gsub!(/"|'/, '') if new_arg[:display_name]
|
208
|
+
p = parse_measure_name(new_arg[:display_name])
|
209
|
+
new_arg[:display_name] = p[0]
|
210
|
+
new_arg[:units] = p[1]
|
211
|
+
new_arg[:description] = p[2]
|
212
|
+
|
213
|
+
if measure_string =~ /#{new_arg[:local_variable]}.setDefaultValue/
|
214
|
+
new_arg[:default_value] = measure_string.match(/#{new_arg[:local_variable]}.setDefaultValue\((.*)\)/)[1]
|
215
|
+
else
|
216
|
+
puts "[WARNING] #{measure_hash[:name]}:#{new_arg[:name]} has no default value... will try to continue"
|
217
|
+
end
|
218
|
+
|
219
|
+
case new_arg[:variable_type]
|
220
|
+
when 'Choice'
|
221
|
+
# Choices to appear to only be strings?
|
222
|
+
puts "Choice vector appears to be #{choice_vector}"
|
223
|
+
new_arg[:default_value].gsub!(/"|'/, '') if new_arg[:default_value]
|
224
|
+
|
225
|
+
# parse the choices from the measure
|
226
|
+
# scan from where the "instance has been created to the measure"
|
227
|
+
possible_choices = nil
|
228
|
+
possible_choice_block = measure_string # .scan(/#{choice_vector}.*=(.*)#{new_arg[:local_variable]}.*=/mi)
|
229
|
+
if possible_choice_block
|
230
|
+
# puts "possible_choice_block is #{possible_choice_block}"
|
231
|
+
possible_choices = possible_choice_block.scan(/#{choice_vector}.*<<.*(')(.*)(')/)
|
232
|
+
possible_choices += possible_choice_block.scan(/#{choice_vector}.*<<.*(")(.*)(")/)
|
233
|
+
end
|
234
|
+
|
235
|
+
# puts "Possible choices are #{possible_choices}"
|
236
|
+
|
237
|
+
if possible_choices.nil? || possible_choices.empty?
|
238
|
+
new_arg[:choices] = []
|
239
|
+
else
|
240
|
+
new_arg[:choices] = possible_choices.map { |c| c[1] }
|
241
|
+
end
|
242
|
+
|
243
|
+
# if the choices are inherited from the model, then need to just display the default value which
|
244
|
+
# somehow magically works because that is the display name
|
245
|
+
if new_arg[:default_value]
|
246
|
+
new_arg[:choices] << new_arg[:default_value] unless new_arg[:choices].include?(new_arg[:default_value])
|
247
|
+
end
|
248
|
+
when 'String', 'Path'
|
249
|
+
new_arg[:default_value].gsub!(/"|'/, '') if new_arg[:default_value]
|
250
|
+
when 'Bool'
|
251
|
+
if new_arg[:default_value]
|
252
|
+
new_arg[:default_value] = new_arg[:default_value].downcase == 'true' ? true : false
|
253
|
+
end
|
254
|
+
when 'Integer'
|
255
|
+
new_arg[:default_value] = new_arg[:default_value].to_i if new_arg[:default_value]
|
256
|
+
when 'Double'
|
257
|
+
new_arg[:default_value] = new_arg[:default_value].to_f if new_arg[:default_value]
|
258
|
+
else
|
259
|
+
fail "unknown variable type of #{new_arg[:variable_type]}"
|
260
|
+
end
|
261
|
+
|
262
|
+
measure_hash[:arguments] << new_arg
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
# check if there is a measure.xml file?
|
267
|
+
measure_xml_filename = "#{File.join(File.dirname(measure_filename), File.basename(measure_filename, '.*'))}.xml"
|
268
|
+
if File.exist? measure_xml_filename
|
269
|
+
f = File.open measure_xml_filename
|
270
|
+
doc = Nokogiri::XML(f)
|
271
|
+
|
272
|
+
# pull out some information
|
273
|
+
measure_hash[:name_xml] = doc.xpath('/measure/name').first.content
|
274
|
+
measure_hash[:uid] = doc.xpath('/measure/uid').first.content
|
275
|
+
measure_hash[:version_id] = doc.xpath('/measure/version_id').first.content
|
276
|
+
measure_hash[:modeler_description] = doc.xpath('/measure/modeler_description').first.content
|
277
|
+
measure_hash[:description] = doc.xpath('/measure/description').first.content
|
278
|
+
measure_hash[:tags] = doc.xpath('/measure/tags/tag').map { |k| k.content }
|
279
|
+
f.close
|
280
|
+
end
|
281
|
+
|
282
|
+
measure_hash
|
283
|
+
end
|
284
|
+
|
285
|
+
def translate_measure_hash_to_csv(measure_hash)
|
286
|
+
csv = []
|
287
|
+
csv << [false, measure_hash[:display_name], measure_hash[:classname], measure_hash[:classname], measure_hash[:measure_type]]
|
288
|
+
|
289
|
+
measure_hash[:arguments].each do |argument|
|
290
|
+
values = []
|
291
|
+
values << ''
|
292
|
+
values << 'argument'
|
293
|
+
values << ''
|
294
|
+
values << argument[:display_name]
|
295
|
+
values << argument[:name]
|
296
|
+
values << argument[:display_name] # this is the default short display name
|
297
|
+
values << argument[:variable_type]
|
298
|
+
values << argument[:units]
|
299
|
+
|
300
|
+
# watch out because :default_value can be a boolean
|
301
|
+
argument[:default_value].nil? ? values << '' : values << argument[:default_value]
|
302
|
+
choices = ''
|
303
|
+
if argument[:choices]
|
304
|
+
choices << "|#{argument[:choices].join(',')}|" unless argument[:choices].empty?
|
305
|
+
end
|
306
|
+
values << choices
|
307
|
+
|
308
|
+
csv << values
|
309
|
+
end
|
310
|
+
|
311
|
+
csv
|
312
|
+
end
|
313
|
+
|
314
|
+
# Read the measure's information to pull out the metadata and to move into a more friendly directory name.
|
315
|
+
# argument of measure is a hash
|
316
|
+
def parse_measure_metadata(measure)
|
317
|
+
m_result = nil
|
318
|
+
# check for valid measure
|
319
|
+
if measure[:measure][:name] && measure[:measure][:uuid]
|
320
|
+
|
321
|
+
file_data = download_component(measure[:measure][:uuid])
|
322
|
+
|
323
|
+
if file_data
|
324
|
+
save_file = File.expand_path("#{@parsed_measures_path}/#{measure[:measure][:name].downcase.gsub(' ', '_')}.zip")
|
325
|
+
File.open(save_file, 'wb') { |f| f << file_data }
|
326
|
+
|
327
|
+
# unzip file and delete zip.
|
328
|
+
# TODO check that something was downloaded here before extracting zip
|
329
|
+
if File.exist? save_file
|
330
|
+
BCL.extract_zip(save_file, @parsed_measures_path, true)
|
331
|
+
|
332
|
+
# catch a weird case where there is an extra space in an unzip file structure but not in the measure.name
|
333
|
+
if measure[:measure][:name] == 'Add Daylight Sensor at Center of Spaces with a Specified Space Type Assigned'
|
334
|
+
unless File.exist? "#{@parsed_measures_path}/#{measure[:measure][:name]}"
|
335
|
+
temp_dir_name = "#{@parsed_measures_path}/Add Daylight Sensor at Center of Spaces with a Specified Space Type Assigned"
|
336
|
+
FileUtils.move(temp_dir_name, "#{@parsed_measures_path}/#{measure[:measure][:name]}")
|
337
|
+
end
|
338
|
+
end
|
339
|
+
|
340
|
+
temp_dir_name = File.join(@parsed_measures_path, measure[:measure][:name])
|
341
|
+
|
342
|
+
# Read the measure.rb file
|
343
|
+
# puts "save dir name #{temp_dir_name}"
|
344
|
+
measure_filename = "#{temp_dir_name}/measure.rb"
|
345
|
+
measure_hash = parse_measure_file(measure[:measure][:name], measure_filename)
|
346
|
+
|
347
|
+
unless measure_hash.empty?
|
348
|
+
m_result = measure_hash
|
349
|
+
# move the directory to the class name
|
350
|
+
new_dir_name = File.join(@parsed_measures_path, measure_hash[:classname])
|
351
|
+
FileUtils.rm_rf(new_dir_name) if File.exist?(new_dir_name) && temp_dir_name != measure_hash[:classname]
|
352
|
+
FileUtils.move(temp_dir_name, new_dir_name) unless temp_dir_name == measure_hash[:classname]
|
353
|
+
|
354
|
+
# create a new measure.json file for parsing later if need be
|
355
|
+
File.open(File.join(new_dir_name, 'measure.json'), 'w') { |f| f << MultiJson.dump(measure_hash, pretty: true) }
|
356
|
+
else
|
357
|
+
puts 'Measure Hash was empty... moving on'
|
358
|
+
end
|
359
|
+
else
|
360
|
+
puts "Problems downloading #{measure[:measure][:name]}... moving on"
|
361
|
+
end
|
362
|
+
end
|
363
|
+
end
|
364
|
+
|
365
|
+
m_result
|
366
|
+
end
|
367
|
+
|
368
|
+
# parse measure name
|
369
|
+
def parse_measure_name(name)
|
370
|
+
# TODO: save/display errors
|
371
|
+
errors = ''
|
372
|
+
m = nil
|
373
|
+
|
374
|
+
clean_name = name
|
375
|
+
units = nil
|
376
|
+
description = nil
|
377
|
+
|
378
|
+
# remove everything btw parentheses
|
379
|
+
m = clean_name.match(/\((.+?)\)/)
|
380
|
+
unless m.nil?
|
381
|
+
errors = errors + ' removing parentheses,'
|
382
|
+
units = m[1]
|
383
|
+
clean_name = clean_name.gsub(/\((.+?)\)/, '')
|
384
|
+
end
|
385
|
+
|
386
|
+
# remove everything btw brackets
|
387
|
+
m = nil
|
388
|
+
m = clean_name.match(/\[(.+?)\]/)
|
389
|
+
unless m.nil?
|
390
|
+
errors = errors + ' removing brackets,'
|
391
|
+
clean_name = clean_name.gsub(/\[(.+?)\]/, '')
|
392
|
+
end
|
393
|
+
|
394
|
+
# remove characters
|
395
|
+
m = nil
|
396
|
+
m = clean_name.match(/(\?|\.|\#).+?/)
|
397
|
+
unless m.nil?
|
398
|
+
errors = errors + ' removing any of following: ?.#'
|
399
|
+
clean_name = clean_name.gsub(/(\?|\.|\#).+?/, '')
|
400
|
+
end
|
401
|
+
clean_name = clean_name.gsub('.', '')
|
402
|
+
clean_name = clean_name.gsub('?', '')
|
403
|
+
|
404
|
+
[clean_name.strip, units, description]
|
405
|
+
end
|
406
|
+
|
407
|
+
# retrieve measures for parsing metadata.
|
408
|
+
# specify a search term to narrow down search or leave nil to retrieve all
|
409
|
+
# set all_pages to true to iterate over all pages of results
|
410
|
+
# can't specify filters other than the hard-coded bundle and show_rows
|
411
|
+
def retrieve_measures(search_term = nil, filter_term = nil, return_all_pages = false, &_block)
|
412
|
+
# raise "Please login before performing this action" if @session.nil?
|
413
|
+
|
414
|
+
# make sure filter_term includes bundle
|
415
|
+
if filter_term.nil?
|
416
|
+
filter_term = 'fq[]=bundle%3Anrel_measure'
|
417
|
+
elsif !filter_term.include? 'bundle'
|
418
|
+
filter_term = filter_term + '&fq[]=bundle%3Anrel_measure'
|
419
|
+
end
|
420
|
+
|
421
|
+
# use provided search term or nil.
|
422
|
+
# if return_all_pages is true, iterate over pages of API results. Otherwise only return first 100
|
423
|
+
results = search(search_term, filter_term, return_all_pages)
|
424
|
+
puts "#{results[:result].count} results returned"
|
425
|
+
|
426
|
+
results[:result].each do |result|
|
427
|
+
puts "retrieving measure: #{result[:measure][:name]}"
|
428
|
+
yield result
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
432
|
+
# evaluate the response from the API in a consistent manner
|
433
|
+
def evaluate_api_response(api_response)
|
434
|
+
valid = false
|
435
|
+
result = { error: 'could not get json from http post response' }
|
436
|
+
case api_response.code
|
437
|
+
when '200'
|
438
|
+
puts " Response Code: #{api_response.code} - #{api_response.body}"
|
439
|
+
if api_response.body.empty?
|
440
|
+
puts ' 200 BUT ERROR: Returned body was empty. Possible causes:'
|
441
|
+
puts ' - BSD tar on Mac OSX vs gnutar'
|
442
|
+
result = { error: 'returned 200, but returned body was empty' }
|
443
|
+
valid = false
|
444
|
+
else
|
445
|
+
puts ' 200 - Successful Upload'
|
446
|
+
result = MultiJson.load api_response.body
|
447
|
+
valid = true
|
448
|
+
end
|
449
|
+
when '404'
|
450
|
+
puts " Response: #{api_response.code} - #{api_response.body}"
|
451
|
+
puts ' 404 - check these common causes first:'
|
452
|
+
puts ' - the filename contains periods (other than the ones before the file extension)'
|
453
|
+
puts " - you are not an 'administrator member' of the group you're trying to upload to"
|
454
|
+
result = MultiJson.load api_response.body
|
455
|
+
valid = false
|
456
|
+
when '406'
|
457
|
+
puts " Response: #{api_response.code} - #{api_response.body}"
|
458
|
+
puts ' 406 - check these common causes first:'
|
459
|
+
puts ' - the UUID of the item that you are uploading is already on the BCL'
|
460
|
+
puts ' - the group_id is not correct in the config.yml (go to group on site, and copy the number at the end of the URL)'
|
461
|
+
puts " - you are not an 'administrator member' of the group you're trying to upload to"
|
462
|
+
result = MultiJson.load api_response.body
|
463
|
+
valid = false
|
464
|
+
when '500'
|
465
|
+
puts " Response: #{api_response.code} - #{api_response.body}"
|
466
|
+
fail 'server exception'
|
467
|
+
valid = false
|
468
|
+
else
|
469
|
+
puts " Response: #{api_response.code} - #{api_response.body}"
|
470
|
+
valid = false
|
471
|
+
end
|
472
|
+
|
473
|
+
[valid, result]
|
474
|
+
end
|
475
|
+
|
476
|
+
# Construct the post parameter for the API content.json end point.
|
477
|
+
# param(@update) is a boolean that triggers whether to use content_type or uuid
|
478
|
+
def construct_post_data(filepath, update, content_type_or_uuid)
|
479
|
+
# TODO remove special characters in the filename; they create firewall errors
|
480
|
+
# filename = filename.gsub(/\W/,'_').gsub(/___/,'_').gsub(/__/,'_').chomp('_').strip
|
481
|
+
|
482
|
+
file_b64 = Base64.encode64(File.read(filepath))
|
483
|
+
|
484
|
+
data = {}
|
485
|
+
data['file'] = {
|
486
|
+
'file' => file_b64,
|
487
|
+
'filesize' => File.size(filepath).to_s,
|
488
|
+
'filename' => File.basename(filepath)
|
489
|
+
}
|
490
|
+
|
491
|
+
data['node'] = {}
|
492
|
+
|
493
|
+
# Only include the content type if this is an update
|
494
|
+
if update
|
495
|
+
data['node']['uuid'] = content_type_or_uuid
|
496
|
+
else
|
497
|
+
data['node']['type'] = content_type_or_uuid
|
498
|
+
end
|
499
|
+
|
500
|
+
# TODO remove this field_component_tags once BCL is fixed
|
501
|
+
data['node']['field_component_tags'] = { 'und' => '1289' }
|
502
|
+
data['node']['og_group_ref'] = { 'und' => ['target_id' => @group_id] }
|
503
|
+
|
504
|
+
# NOTE THIS ONLY WORKS IF YOU ARE A BCL SITE ADMIN
|
505
|
+
data['node']['publish'] = '1'
|
506
|
+
|
507
|
+
data
|
508
|
+
end
|
509
|
+
|
510
|
+
# pushes component to the bcl and publishes them (if logged-in as BCL Website Admin user).
|
511
|
+
# username, password, and group_id are set in the ~/.bcl/config.yml file
|
512
|
+
def push_content(filename_and_path, write_receipt_file, content_type)
|
513
|
+
fail 'Please login before pushing components' if @session.nil?
|
514
|
+
fail 'Do not have a valid access token; try again' if @access_token.nil?
|
515
|
+
|
516
|
+
data = construct_post_data(filename_and_path, false, content_type)
|
517
|
+
|
518
|
+
path = '/api/content.json'
|
519
|
+
headers = { 'Content-Type' => 'application/json', 'X-CSRF-Token' => @access_token, 'Cookie' => @session }
|
520
|
+
|
521
|
+
res = @http.post(path, MultiJson.dump(data), headers)
|
522
|
+
|
523
|
+
valid, json = evaluate_api_response(res)
|
524
|
+
|
525
|
+
if valid
|
526
|
+
# write out a receipt file into the same directory of the component with the same file name as
|
527
|
+
# the component
|
528
|
+
if write_receipt_file
|
529
|
+
File.open("#{File.dirname(filename_and_path)}/#{File.basename(filename_and_path, '.tar.gz')}.receipt", 'w') do |file|
|
530
|
+
file << Time.now.to_s
|
531
|
+
end
|
532
|
+
end
|
533
|
+
end
|
534
|
+
|
535
|
+
[valid, json]
|
536
|
+
end
|
537
|
+
|
538
|
+
# pushes updated content to the bcl and publishes it (if logged-in as BCL Website Admin user).
|
539
|
+
# username and password set in ~/.bcl/config.yml file
|
540
|
+
def update_content(filename_and_path, write_receipt_file, uuid = nil)
|
541
|
+
fail 'Please login before pushing components' unless @session
|
542
|
+
|
543
|
+
# get the UUID if zip or xml file
|
544
|
+
version_id = nil
|
545
|
+
if uuid.nil?
|
546
|
+
puts File.extname(filename_and_path).downcase
|
547
|
+
if filename_and_path =~ /^.*.tar.gz$/i
|
548
|
+
uuid, version_id = uuid_vid_from_tarball(filename_and_path)
|
549
|
+
puts "Parsed uuid out of tar.gz file with value #{uuid}"
|
550
|
+
end
|
551
|
+
else
|
552
|
+
# verify the uuid via regex
|
553
|
+
unless uuid =~ /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
|
554
|
+
fail "uuid of #{uuid} is invalid"
|
555
|
+
end
|
556
|
+
end
|
557
|
+
fail 'Please pass in a tar.gz file or pass in the uuid' unless uuid
|
558
|
+
|
559
|
+
data = construct_post_data(filename_and_path, true, uuid)
|
560
|
+
|
561
|
+
path = '/api/content.json'
|
562
|
+
headers = { 'Content-Type' => 'application/json', 'X-CSRF-Token' => @access_token, 'Cookie' => @session }
|
563
|
+
|
564
|
+
res = @http.post(path, MultiJson.dump(data), headers)
|
565
|
+
|
566
|
+
valid, json = evaluate_api_response(res)
|
567
|
+
|
568
|
+
if valid
|
569
|
+
# write out a receipt file into the same directory of the component with the same file name as
|
570
|
+
# the component
|
571
|
+
if write_receipt_file
|
572
|
+
File.open("#{File.dirname(filename_and_path)}/#{File.basename(filename_and_path, '.tar.gz')}.receipt", 'w') do |file|
|
573
|
+
file << Time.now.to_s
|
574
|
+
end
|
575
|
+
end
|
576
|
+
end
|
577
|
+
|
578
|
+
[valid, json]
|
579
|
+
end
|
580
|
+
|
581
|
+
def push_contents(array_of_components, skip_files_with_receipts, content_type)
|
582
|
+
logs = []
|
583
|
+
array_of_components.each do |comp|
|
584
|
+
receipt_file = File.dirname(comp) + '/' + File.basename(comp, '.tar.gz') + '.receipt'
|
585
|
+
log_message = ''
|
586
|
+
if skip_files_with_receipts && File.exist?(receipt_file)
|
587
|
+
log_message = "skipping because found receipt #{comp}"
|
588
|
+
puts log_message
|
589
|
+
else
|
590
|
+
log_message = "pushing content #{File.basename(comp, '.tar.gz')}"
|
591
|
+
puts log_message
|
592
|
+
valid, res = push_content(comp, true, content_type)
|
593
|
+
log_message += " #{valid} #{res.inspect.chomp}"
|
594
|
+
end
|
595
|
+
logs << log_message
|
596
|
+
end
|
597
|
+
|
598
|
+
logs
|
599
|
+
end
|
600
|
+
|
601
|
+
# Unpack the tarball in memory and extract the XML file to read the UUID and Version ID
|
602
|
+
def uuid_vid_from_tarball(path_to_tarball)
|
603
|
+
uuid = nil
|
604
|
+
vid = nil
|
605
|
+
|
606
|
+
fail "File does not exist #{path_to_tarball}" unless File.exist? path_to_tarball
|
607
|
+
tgz = Zlib::GzipReader.open(path_to_tarball)
|
608
|
+
Archive::Tar::Minitar::Reader.open(tgz).each do |entry|
|
609
|
+
# If taring with tar zcf ameasure.tar.gz -C measure_dir .
|
610
|
+
if entry.name =~ /^.{0,2}component.xml$/ || entry.name =~ /^.{0,2}measure.xml$/
|
611
|
+
xml_file = Nokogiri::XML(entry.read)
|
612
|
+
|
613
|
+
# pull out some information
|
614
|
+
if entry.name =~ /component/
|
615
|
+
u = xml_file.xpath('/component/uid').first
|
616
|
+
v = xml_file.xpath('/component/version_id').first
|
617
|
+
else
|
618
|
+
u = xml_file.xpath('/measure/uid').first
|
619
|
+
v = xml_file.xpath('/measure/version_id').first
|
620
|
+
end
|
621
|
+
fail "Could not find UUID in XML file #{path_to_tarball}" unless u
|
622
|
+
# Don't error on version not existing.
|
623
|
+
|
624
|
+
uuid = u.content
|
625
|
+
vid = v ? v.content : nil
|
626
|
+
|
627
|
+
# puts "uuid = #{uuid}; vid = #{vid}"
|
628
|
+
end
|
629
|
+
end
|
630
|
+
|
631
|
+
[uuid, vid]
|
632
|
+
end
|
633
|
+
|
634
|
+
def update_contents(array_of_tarball_components, skip_files_with_receipts)
|
635
|
+
logs = []
|
636
|
+
array_of_tarball_components.each do |comp|
|
637
|
+
receipt_file = File.dirname(comp) + '/' + File.basename(comp, '.tar.gz') + '.receipt'
|
638
|
+
log_message = ''
|
639
|
+
if skip_files_with_receipts && File.exist?(receipt_file)
|
640
|
+
log_message = "skipping update because found receipt #{File.basename(comp)}"
|
641
|
+
puts log_message
|
642
|
+
else
|
643
|
+
uuid, vid = uuid_vid_from_tarball(comp)
|
644
|
+
if uuid.nil?
|
645
|
+
log_message = "ERROR: uuid not found for #{File.basename(comp)}"
|
646
|
+
puts log_message
|
647
|
+
else
|
648
|
+
log_message = "pushing updated content #{File.basename(comp)}"
|
649
|
+
puts log_message
|
650
|
+
valid, res = update_content(comp, true, uuid)
|
651
|
+
log_message += " #{valid} #{res.inspect.chomp}"
|
652
|
+
end
|
653
|
+
end
|
654
|
+
logs << log_message
|
655
|
+
end
|
656
|
+
logs
|
657
|
+
end
|
658
|
+
|
659
|
+
# Simple method to search bcl and return the result as hash with symbols
|
660
|
+
# If all = true, iterate over pages of results and return all
|
661
|
+
# JSON ONLY
|
662
|
+
def search(search_str = nil, filter_str = nil, all = false)
|
663
|
+
full_url = '/api/search/'
|
664
|
+
|
665
|
+
# add search term
|
666
|
+
if !search_str.nil? and search_str != ''
|
667
|
+
full_url = full_url + search_str
|
668
|
+
# strip out xml in case it's included. make sure .json is included
|
669
|
+
full_url = full_url.gsub('.xml', '')
|
670
|
+
unless search_str.include? '.json'
|
671
|
+
full_url = full_url + '.json'
|
672
|
+
end
|
673
|
+
else
|
674
|
+
full_url = full_url + '*.json'
|
675
|
+
end
|
676
|
+
|
677
|
+
# add api_version
|
678
|
+
if @api_version < 2.0
|
679
|
+
puts "WARNING: attempting to use search with api_version #{@api_version}. Use API v2.0 for this functionality."
|
680
|
+
end
|
681
|
+
full_url = full_url + "?api_version=#{@api_version}"
|
682
|
+
|
683
|
+
# add filters
|
684
|
+
unless filter_str.nil?
|
685
|
+
# strip out api_version from filters, if included
|
686
|
+
if filter_str.include? 'api_version='
|
687
|
+
filter_str = filter_str.gsub(/api_version=\d{1,}/, '')
|
688
|
+
filter_str = filter_str.gsub(/&api_version=\d{1,}/, '')
|
689
|
+
end
|
690
|
+
full_url = full_url + '&' + filter_str
|
691
|
+
end
|
692
|
+
|
693
|
+
# simple search vs. all results
|
694
|
+
if !all
|
695
|
+
puts "search url: #{full_url}"
|
696
|
+
res = @http.get(full_url)
|
697
|
+
# return unparsed
|
698
|
+
MultiJson.load(res.body, symbolize_keys: true)
|
699
|
+
else
|
700
|
+
# iterate over result pages
|
701
|
+
# modify filter_str for show_rows=200 for maximum returns
|
702
|
+
if filter_str.include? 'show_rows='
|
703
|
+
full_url = full_url.gsub(/show_rows=\d{1,}/, 'show_rows=200')
|
704
|
+
else
|
705
|
+
full_url = full_url + '&show_rows=200'
|
706
|
+
end
|
707
|
+
# make sure filter_str doesn't already have a page=x
|
708
|
+
full_url.gsub(/page=\d{1,}/, '')
|
709
|
+
|
710
|
+
pagecnt = 0
|
711
|
+
continue = 1
|
712
|
+
results = []
|
713
|
+
while continue == 1
|
714
|
+
# retrieve current page
|
715
|
+
full_url_all = full_url + "&page=#{pagecnt}"
|
716
|
+
puts "search url: #{full_url_all}"
|
717
|
+
response = @http.get(full_url_all)
|
718
|
+
# parse here so you can build results array
|
719
|
+
res = MultiJson.load(response.body)
|
720
|
+
|
721
|
+
if res['result'].count > 0
|
722
|
+
pagecnt += 1
|
723
|
+
res['result'].each do |r|
|
724
|
+
results << r
|
725
|
+
end
|
726
|
+
else
|
727
|
+
continue = 0
|
728
|
+
end
|
729
|
+
end
|
730
|
+
# return unparsed b/c that is what is expected
|
731
|
+
formatted_results = { 'result' => results }
|
732
|
+
results_to_return = MultiJson.load(MultiJson.dump(formatted_results), symbolize_keys: true)
|
733
|
+
end
|
734
|
+
end
|
735
|
+
|
736
|
+
# Delete receipt files
|
737
|
+
def delete_receipts(array_of_components)
|
738
|
+
array_of_components.each do |comp|
|
739
|
+
receipt_file = File.dirname(comp) + '/' + File.basename(comp, '.tar.gz') + '.receipt'
|
740
|
+
if File.exist?(receipt_file)
|
741
|
+
FileUtils.remove_file(receipt_file)
|
742
|
+
|
743
|
+
end
|
744
|
+
end
|
745
|
+
end
|
746
|
+
|
747
|
+
def list_all_measures
|
748
|
+
json = search(nil, 'fq[]=bundle%3Anrel_measure&show_rows=100')
|
749
|
+
|
750
|
+
json
|
751
|
+
end
|
752
|
+
|
753
|
+
def download_component(uid)
|
754
|
+
result = @http.get("/api/component/download?uids=#{uid}")
|
755
|
+
puts "DOWNLOADING: /api/component/download?uids=#{uid}"
|
756
|
+
# puts "RESULTS: #{result.inspect}"
|
757
|
+
# puts "RESULTS BODY: #{result.body}"
|
758
|
+
# look at response code
|
759
|
+
if result.code == '200'
|
760
|
+
puts 'Download Successful'
|
761
|
+
result.body ? result.body : nil
|
762
|
+
else
|
763
|
+
puts "Download fail. Error code #{result.code}"
|
764
|
+
nil
|
765
|
+
end
|
766
|
+
rescue
|
767
|
+
puts "Couldn't download uid(s): #{uid}...skipping"
|
768
|
+
nil
|
769
|
+
end
|
770
|
+
|
771
|
+
private
|
772
|
+
|
773
|
+
def load_config
|
774
|
+
config_filename = File.expand_path('~/.bcl/config.yml')
|
775
|
+
|
776
|
+
if File.exist?(config_filename)
|
777
|
+
puts "loading config settings from #{config_filename}"
|
778
|
+
@config = YAML.load_file(config_filename)
|
779
|
+
else
|
780
|
+
# location of template file
|
781
|
+
FileUtils.mkdir_p(File.dirname(config_filename))
|
782
|
+
File.open(config_filename, 'w') { |f| f << default_yaml.to_yaml }
|
783
|
+
File.chmod(0600, config_filename)
|
784
|
+
puts "******** Please fill in user credentials in #{config_filename} file if you need to upload data **********"
|
785
|
+
end
|
786
|
+
end
|
787
|
+
|
788
|
+
def default_yaml
|
789
|
+
settings = {
|
790
|
+
server: {
|
791
|
+
url: 'https://bcl.nrel.gov',
|
792
|
+
user: {
|
793
|
+
username: 'ENTER_BCL_USERNAME',
|
794
|
+
password: 'ENTER_BCL_PASSWORD',
|
795
|
+
group: 'ENTER_GROUP_ID'
|
796
|
+
}
|
797
|
+
}
|
798
|
+
}
|
799
|
+
|
800
|
+
settings
|
801
|
+
end
|
802
|
+
end # class ComponentMethods
|
803
|
+
|
804
|
+
# TODO make this extend the component_xml class (or create a super class around components)
|
805
|
+
|
806
|
+
def self.gather_components(component_dir, chunk_size = 0, delete_previousgather = false, destination = nil)
|
807
|
+
if destination.nil?
|
808
|
+
@dest_filename = 'components'
|
809
|
+
else
|
810
|
+
@dest_filename = destination
|
811
|
+
end
|
812
|
+
@dest_file_ext = 'tar.gz'
|
813
|
+
|
814
|
+
# store the starting directory
|
815
|
+
current_dir = Dir.pwd
|
816
|
+
|
817
|
+
# an array to hold reporting info about the batches
|
818
|
+
gather_components_report = []
|
819
|
+
|
820
|
+
# go to the directory containing the components
|
821
|
+
Dir.chdir(component_dir)
|
822
|
+
|
823
|
+
# delete any old versions of the component chunks
|
824
|
+
FileUtils.rm_rf('./gather') if delete_previousgather
|
825
|
+
|
826
|
+
# gather all the components into array
|
827
|
+
targzs = Pathname.glob('./**/*.tar.gz')
|
828
|
+
tar_cnt = 0
|
829
|
+
chunk_cnt = 0
|
830
|
+
targzs.each do |targz|
|
831
|
+
if chunk_size != 0 && (tar_cnt % chunk_size) == 0
|
832
|
+
chunk_cnt += 1
|
833
|
+
end
|
834
|
+
tar_cnt += 1
|
835
|
+
|
836
|
+
destination_path = "./gather/#{chunk_cnt}"
|
837
|
+
FileUtils.mkdir_p(destination_path)
|
838
|
+
destination_file = "#{destination_path}/#{File.basename(targz.to_s)}"
|
839
|
+
# puts "copying #{targz.to_s} to #{destination_file}"
|
840
|
+
FileUtils.cp(targz.to_s, destination_file)
|
841
|
+
end
|
842
|
+
|
843
|
+
# gather all the .tar.gz files into a single tar.gz
|
844
|
+
(1..chunk_cnt).each do |cnt|
|
845
|
+
currentdir = Dir.pwd
|
846
|
+
|
847
|
+
paths = []
|
848
|
+
Pathname.glob("./gather/#{cnt}/*.tar.gz").each do |pt|
|
849
|
+
paths << File.basename(pt.to_s)
|
850
|
+
end
|
851
|
+
|
852
|
+
Dir.chdir("./gather/#{cnt}")
|
853
|
+
destination = "#{@dest_filename}_#{cnt}.#{@dest_file_ext}"
|
854
|
+
puts "tarring batch #{cnt} of #{chunk_cnt} to #{@dest_filename}_#{cnt}.#{@dest_file_ext}"
|
855
|
+
BCL.tarball(destination, paths)
|
856
|
+
Dir.chdir(currentdir)
|
857
|
+
|
858
|
+
# move the tarball back a directory
|
859
|
+
FileUtils.move("./gather/#{cnt}/#{destination}", "./gather/#{destination}")
|
860
|
+
end
|
861
|
+
|
862
|
+
Dir.chdir(current_dir)
|
863
|
+
end
|
864
|
+
end # module BCL
|