data_miner 0.3.4 → 0.3.5
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/data_miner.gemspec +2 -2
- data/lib/data_miner/attribute.rb +1 -1
- data/lib/data_miner/dictionary.rb +2 -2
- data/lib/data_miner.rb +7 -0
- data/test/data_miner_test.rb +244 -11
- data/test/test_helper.rb +116 -0
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.5
|
data/data_miner.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{data_miner}
|
8
|
-
s.version = "0.3.
|
8
|
+
s.version = "0.3.5"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
|
12
|
-
s.date = %q{2010-03-
|
12
|
+
s.date = %q{2010-03-08}
|
13
13
|
s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
|
14
14
|
s.email = %q{seamus@abshere.net}
|
15
15
|
s.extra_rdoc_files = [
|
data/lib/data_miner/attribute.rb
CHANGED
@@ -92,7 +92,7 @@ module DataMiner
|
|
92
92
|
|
93
93
|
def dictionary(import)
|
94
94
|
raise "shouldn't ask for this" unless wants_dictionary?(import) # don't try to initialize if there are no dictionary options
|
95
|
-
Dictionary.new dictionary_options(import)
|
95
|
+
@_dictionary ||= Dictionary.new dictionary_options(import)
|
96
96
|
end
|
97
97
|
|
98
98
|
# {
|
@@ -3,8 +3,8 @@ module DataMiner
|
|
3
3
|
attr_accessor :key_name, :value_name, :sprintf, :table
|
4
4
|
|
5
5
|
def initialize(options = {})
|
6
|
-
@key_name = options[:
|
7
|
-
@value_name = options[:
|
6
|
+
@key_name = options[:input]
|
7
|
+
@value_name = options[:output]
|
8
8
|
@sprintf = options[:sprintf] || '%s'
|
9
9
|
@table = RemoteTable.new(:url => options[:url])
|
10
10
|
end
|
data/lib/data_miner.rb
CHANGED
@@ -49,9 +49,16 @@ end
|
|
49
49
|
|
50
50
|
ActiveRecord::Base.class_eval do
|
51
51
|
def self.data_miner(&block)
|
52
|
+
unless table_exists?
|
53
|
+
logger.error "[DataMiner gem] Database table `#{table_name}` doesn't exist. DataMiner probably won't work properly until you run a migration or otherwise fix the schema."
|
54
|
+
return
|
55
|
+
end
|
56
|
+
|
52
57
|
DataMiner.classes.add self
|
53
58
|
DataMiner.create_tables
|
54
59
|
DataMiner::Target.find_or_create_by_name name
|
60
|
+
|
61
|
+
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
55
62
|
|
56
63
|
# this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
|
57
64
|
class_eval { cattr_accessor :data_miner_config }
|
data/test/data_miner_test.rb
CHANGED
@@ -178,7 +178,6 @@ end
|
|
178
178
|
|
179
179
|
class AutomobileMakeYear < ActiveRecord::Base
|
180
180
|
set_primary_key :row_hash
|
181
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
182
181
|
|
183
182
|
belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
|
184
183
|
belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'automobile_model_year_id'
|
@@ -241,7 +240,6 @@ end
|
|
241
240
|
|
242
241
|
class AutomobileMakeFleetYear < ActiveRecord::Base
|
243
242
|
set_primary_key :row_hash
|
244
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
245
243
|
belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
|
246
244
|
belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'automobile_model_year_id'
|
247
245
|
belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'automobile_make_year_id'
|
@@ -262,7 +260,6 @@ end
|
|
262
260
|
|
263
261
|
class AutomobileModelYear < ActiveRecord::Base
|
264
262
|
set_primary_key :year
|
265
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
266
263
|
|
267
264
|
has_many :make_years, :class_name => 'AutomobileMakeYear'
|
268
265
|
has_many :variants, :class_name => 'AutomobileVariant'
|
@@ -278,7 +275,6 @@ end
|
|
278
275
|
|
279
276
|
class AutomobileFuelType < ActiveRecord::Base
|
280
277
|
set_primary_key :code
|
281
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
282
278
|
|
283
279
|
data_miner do
|
284
280
|
unique_index 'code'
|
@@ -319,7 +315,6 @@ end
|
|
319
315
|
|
320
316
|
class AutomobileModel < ActiveRecord::Base
|
321
317
|
set_primary_key :row_hash
|
322
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
323
318
|
|
324
319
|
has_many :variants, :class_name => 'AutomobileVariant'
|
325
320
|
belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
|
@@ -331,7 +326,6 @@ end
|
|
331
326
|
|
332
327
|
class AutomobileMake < ActiveRecord::Base
|
333
328
|
set_primary_key :name
|
334
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
335
329
|
|
336
330
|
has_many :make_years, :class_name => 'AutomobileMakeYear'
|
337
331
|
has_many :models, :class_name => 'AutomobileModel'
|
@@ -352,7 +346,6 @@ end
|
|
352
346
|
|
353
347
|
class AutomobileVariant < ActiveRecord::Base
|
354
348
|
set_primary_key :row_hash
|
355
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
356
349
|
|
357
350
|
belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
|
358
351
|
belongs_to :model, :class_name => 'AutomobileModel', :foreign_key => 'automobile_model_id'
|
@@ -541,7 +534,6 @@ end
|
|
541
534
|
|
542
535
|
class Country < ActiveRecord::Base
|
543
536
|
set_primary_key :iso_3166
|
544
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
545
537
|
|
546
538
|
data_miner do
|
547
539
|
unique_index 'iso_3166'
|
@@ -562,7 +554,6 @@ end
|
|
562
554
|
|
563
555
|
class Airport < ActiveRecord::Base
|
564
556
|
set_primary_key :iata_code
|
565
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
566
557
|
belongs_to :country
|
567
558
|
|
568
559
|
data_miner do
|
@@ -582,7 +573,6 @@ end
|
|
582
573
|
|
583
574
|
class CensusRegion < ActiveRecord::Base
|
584
575
|
set_primary_key :number
|
585
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
586
576
|
|
587
577
|
data_miner do
|
588
578
|
unique_index 'number'
|
@@ -600,6 +590,243 @@ class CensusRegion < ActiveRecord::Base
|
|
600
590
|
end
|
601
591
|
end
|
602
592
|
|
593
|
+
class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
|
594
|
+
set_primary_key :department_of_energy_identifier
|
595
|
+
|
596
|
+
data_miner do
|
597
|
+
# conversions are NOT performed here, since we first have to zero out legitimate skips
|
598
|
+
# otherwise you will get values like "999 pounds = 453.138778 kilograms" (where 999 is really a legit skip)
|
599
|
+
import :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv', :headers => :upcase do |attr|
|
600
|
+
unique_index 'department_of_energy_identifier'
|
601
|
+
|
602
|
+
attr.store 'department_of_energy_identifier', :field_name => 'DOEID'
|
603
|
+
|
604
|
+
attr.store 'residence_class', :field_name => 'TYPEHUQ', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/typehuq/typehuq.csv' }
|
605
|
+
attr.store 'construction_year', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Date in the middle (synthetic)', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/yearmade/yearmade.csv' }
|
606
|
+
attr.store 'construction_period', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/yearmade/yearmade.csv' }
|
607
|
+
attr.store 'urbanity', :field_name => 'URBRUR', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/urbrur/urbrur.csv' }
|
608
|
+
attr.store 'dishwasher_use', :field_name => 'DWASHUSE', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/dwashuse/dwashuse.csv' }
|
609
|
+
attr.store 'central_ac_use', :field_name => 'USECENAC', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/usecenac/usecenac.csv' }
|
610
|
+
attr.store 'window_ac_use', :field_name => 'USEWWAC', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/usewwac/usewwac.csv' }
|
611
|
+
attr.store 'clothes_washer_use', :field_name => 'WASHLOAD', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/washload/washload.csv' }
|
612
|
+
attr.store 'clothes_dryer_use', :field_name => 'DRYRUSE', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/dryruse/dryruse.csv' }
|
613
|
+
|
614
|
+
attr.store 'census_division_number', :field_name => 'DIVISION'
|
615
|
+
attr.store 'floorspace', :field_name => 'TOTSQFT'
|
616
|
+
attr.store 'residents', :field_name => 'NHSLDMEM'
|
617
|
+
attr.store 'ownership', :field_name => 'KOWNRENT'
|
618
|
+
attr.store 'thermostat_programmability', :field_name => 'PROTHERM'
|
619
|
+
attr.store 'refrigerator_count', :field_name => 'NUMFRIG'
|
620
|
+
attr.store 'freezer_count', :field_name => 'NUMFREEZ'
|
621
|
+
attr.store 'heating_degree_days', :field_name => 'HD65'
|
622
|
+
attr.store 'cooling_degree_days', :field_name => 'CD65'
|
623
|
+
attr.store 'annual_energy_from_fuel_oil_for_heating_space', :field_name => 'BTUFOSPH'
|
624
|
+
attr.store 'annual_energy_from_fuel_oil_for_heating_water', :field_name => 'BTUFOWTH'
|
625
|
+
attr.store 'annual_energy_from_fuel_oil_for_appliances', :field_name => 'BTUFOAPL'
|
626
|
+
attr.store 'annual_energy_from_natural_gas_for_heating_space', :field_name => 'BTUNGSPH'
|
627
|
+
attr.store 'annual_energy_from_natural_gas_for_heating_water', :field_name => 'BTUNGWTH'
|
628
|
+
attr.store 'annual_energy_from_natural_gas_for_appliances', :field_name => 'BTUNGAPL'
|
629
|
+
attr.store 'annual_energy_from_propane_for_heating_space', :field_name => 'BTULPSPH'
|
630
|
+
attr.store 'annual_energy_from_propane_for_heating_water', :field_name => 'BTULPWTH'
|
631
|
+
attr.store 'annual_energy_from_propane_for_appliances', :field_name => 'BTULPAPL'
|
632
|
+
attr.store 'annual_energy_from_wood', :field_name => 'BTUWOOD'
|
633
|
+
attr.store 'annual_energy_from_kerosene', :field_name => 'BTUKER'
|
634
|
+
attr.store 'annual_energy_from_electricity_for_clothes_driers', :field_name => 'BTUELCDR'
|
635
|
+
attr.store 'annual_energy_from_electricity_for_dishwashers', :field_name => 'BTUELDWH'
|
636
|
+
attr.store 'annual_energy_from_electricity_for_freezers', :field_name => 'BTUELFZZ'
|
637
|
+
attr.store 'annual_energy_from_electricity_for_refrigerators', :field_name => 'BTUELRFG'
|
638
|
+
attr.store 'annual_energy_from_electricity_for_air_conditioners', :field_name => 'BTUELCOL'
|
639
|
+
attr.store 'annual_energy_from_electricity_for_heating_space', :field_name => 'BTUELSPH'
|
640
|
+
attr.store 'annual_energy_from_electricity_for_heating_water', :field_name => 'BTUELWTH'
|
641
|
+
attr.store 'annual_energy_from_electricity_for_other_appliances', :field_name => 'BTUELAPL'
|
642
|
+
attr.store 'weighting', :field_name => 'NWEIGHT'
|
643
|
+
attr.store 'total_rooms', :field_name => 'TOTROOMS'
|
644
|
+
attr.store 'bathrooms', :field_name => 'NCOMBATH'
|
645
|
+
attr.store 'halfbaths', :field_name => 'NHAFBATH'
|
646
|
+
attr.store 'heated_garage', :field_name => 'GARGHEAT'
|
647
|
+
attr.store 'attached_1car_garage', :field_name => 'GARAGE1C'
|
648
|
+
attr.store 'detached_1car_garage', :field_name => 'DGARG1C'
|
649
|
+
attr.store 'attached_2car_garage', :field_name => 'GARAGE2C'
|
650
|
+
attr.store 'detached_2car_garage', :field_name => 'DGARG2C'
|
651
|
+
attr.store 'attached_3car_garage', :field_name => 'GARAGE3C'
|
652
|
+
attr.store 'detached_3car_garage', :field_name => 'DGARG3C'
|
653
|
+
attr.store 'lights_on_1_to_4_hours', :field_name => 'LGT1'
|
654
|
+
attr.store 'efficient_lights_on_1_to_4_hours', :field_name => 'LGT1EE'
|
655
|
+
attr.store 'lights_on_4_to_12_hours', :field_name => 'LGT4'
|
656
|
+
attr.store 'efficient_lights_on_4_to_12_hours', :field_name => 'LGT4EE'
|
657
|
+
attr.store 'lights_on_over_12_hours', :field_name => 'LGT12'
|
658
|
+
attr.store 'efficient_lights_on_over_12_hours', :field_name => 'LGT12EE'
|
659
|
+
attr.store 'outdoor_all_night_lights', :field_name => 'NOUTLGTNT'
|
660
|
+
attr.store 'outdoor_all_night_gas_lights', :field_name => 'NGASLIGHT'
|
661
|
+
end
|
662
|
+
|
663
|
+
# process :zero_out_legitimate_skips
|
664
|
+
|
665
|
+
# process :convert_units_after_zeroing_legitimate_skips
|
666
|
+
|
667
|
+
# process :derive_rooms
|
668
|
+
|
669
|
+
# process :derive_lighting_use
|
670
|
+
|
671
|
+
# process :derive_lighting_efficiency
|
672
|
+
|
673
|
+
# CensusDivision needs its own dataminer
|
674
|
+
# attr.store 'census_division', :field_name => 'DIVISION', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/data_helpers/raw/master/division/division.csv' }
|
675
|
+
|
676
|
+
# this is basically process :derive_census_region
|
677
|
+
# step.derive :census_region_id, :set => '(SELECT census_regions.id FROM census_regions INNER JOIN census_divisions ON census_regions.id = census_divisions.census_region_id WHERE census_divisions.id = residence_survey_responses.census_division_id)'
|
678
|
+
|
679
|
+
# process :derive_residence_air_conditioner_use_id
|
680
|
+
|
681
|
+
# process :derive_residence_clothes_drier_use_id
|
682
|
+
end
|
683
|
+
|
684
|
+
class << self
|
685
|
+
# # continuous variables for which legitimate skip is effectively zero
|
686
|
+
# attr.affect :annual_energy_from_electricity_for_air_conditioners
|
687
|
+
# attr.affect :annual_energy_from_electricity_for_clothes_driers
|
688
|
+
# attr.affect :annual_energy_from_electricity_for_dishwashers
|
689
|
+
# attr.affect :annual_energy_from_electricity_for_freezers
|
690
|
+
# attr.affect :annual_energy_from_electricity_for_heating_space
|
691
|
+
# attr.affect :annual_energy_from_electricity_for_heating_water
|
692
|
+
# attr.affect :annual_energy_from_electricity_for_other_appliances
|
693
|
+
# attr.affect :annual_energy_from_electricity_for_refrigerators
|
694
|
+
# attr.affect :annual_energy_from_fuel_oil_for_appliances
|
695
|
+
# attr.affect :annual_energy_from_fuel_oil_for_heating_space
|
696
|
+
# attr.affect :annual_energy_from_fuel_oil_for_heating_water
|
697
|
+
# attr.affect :annual_energy_from_kerosene
|
698
|
+
# attr.affect :annual_energy_from_propane_for_appliances
|
699
|
+
# attr.affect :annual_energy_from_propane_for_heating_space
|
700
|
+
# attr.affect :annual_energy_from_propane_for_heating_water
|
701
|
+
# attr.affect :annual_energy_from_natural_gas_for_appliances
|
702
|
+
# attr.affect :annual_energy_from_natural_gas_for_heating_space
|
703
|
+
# attr.affect :annual_energy_from_natural_gas_for_heating_water
|
704
|
+
# attr.affect :annual_energy_from_wood
|
705
|
+
# attr.affect :lights_on_1_to_4_hours
|
706
|
+
# attr.affect :lights_on_over_12_hours
|
707
|
+
# attr.affect :efficient_lights_on_over_12_hours
|
708
|
+
# attr.affect :efficient_lights_on_1_to_4_hours
|
709
|
+
# attr.affect :lights_on_4_to_12_hours
|
710
|
+
# attr.affect :efficient_lights_on_4_to_12_hours
|
711
|
+
# attr.affect :outdoor_all_night_gas_lights
|
712
|
+
# attr.affect :outdoor_all_night_lights
|
713
|
+
# # booleans for which legitimate skip is effectively zero
|
714
|
+
# attr.affect :thermostat_programmability
|
715
|
+
# attr.affect :detached_1car_garage
|
716
|
+
# attr.affect :detached_2car_garage
|
717
|
+
# attr.affect :detached_3car_garage
|
718
|
+
# attr.affect :attached_1car_garage
|
719
|
+
# attr.affect :attached_2car_garage
|
720
|
+
# attr.affect :attached_3car_garage
|
721
|
+
# attr.affect :heated_garage
|
722
|
+
def zero_out_legitimate_skips
|
723
|
+
max = maximum(attr_name, :select => "CONVERT(#{attr_name}, UNSIGNED INTEGER)")
|
724
|
+
if /^9+$/.match(max.to_i.to_s) # the max is all 999's... it must be a LEGITIMATE SKIP
|
725
|
+
logger.info "Zeroing #{attr_name} if it's #{max}"
|
726
|
+
update_all("#{attr_name} = 0", "#{attr_name} = #{max}")
|
727
|
+
end
|
728
|
+
end
|
729
|
+
|
730
|
+
# attr.affect :annual_energy_from_fuel_oil_for_heating_space, :from => :kbtus, :to => :joules
|
731
|
+
# attr.affect :annual_energy_from_fuel_oil_for_heating_water, :from => :kbtus, :to => :joules
|
732
|
+
# attr.affect :annual_energy_from_fuel_oil_for_appliances, :from => :kbtus, :to => :joules
|
733
|
+
# attr.affect :annual_energy_from_natural_gas_for_heating_space, :from => :kbtus, :to => :joules
|
734
|
+
# attr.affect :annual_energy_from_natural_gas_for_heating_water, :from => :kbtus, :to => :joules
|
735
|
+
# attr.affect :annual_energy_from_natural_gas_for_appliances, :from => :kbtus, :to => :joules
|
736
|
+
# attr.affect :annual_energy_from_propane_for_heating_space, :from => :kbtus, :to => :joules
|
737
|
+
# attr.affect :annual_energy_from_propane_for_heating_water, :from => :kbtus, :to => :joules
|
738
|
+
# attr.affect :annual_energy_from_propane_for_appliances, :from => :kbtus, :to => :joules
|
739
|
+
# attr.affect :annual_energy_from_wood, :from => :kbtus, :to => :joules
|
740
|
+
# attr.affect :annual_energy_from_kerosene, :from => :kbtus, :to => :joules
|
741
|
+
# attr.affect :annual_energy_from_electricity_for_clothes_driers, :from => :kbtus, :to => :joules
|
742
|
+
# attr.affect :annual_energy_from_electricity_for_dishwashers, :from => :kbtus, :to => :joules
|
743
|
+
# attr.affect :annual_energy_from_electricity_for_freezers, :from => :kbtus, :to => :joules
|
744
|
+
# attr.affect :annual_energy_from_electricity_for_refrigerators, :from => :kbtus, :to => :joules
|
745
|
+
# attr.affect :annual_energy_from_electricity_for_air_conditioners, :from => :kbtus, :to => :joules
|
746
|
+
# attr.affect :annual_energy_from_electricity_for_heating_space, :from => :kbtus, :to => :joules
|
747
|
+
# attr.affect :annual_energy_from_electricity_for_heating_water, :from => :kbtus, :to => :joules
|
748
|
+
# attr.affect :annual_energy_from_electricity_for_other_appliances, :from => :kbtus, :to => :joules
|
749
|
+
# attr.affect :floorspace, :from => :square_feet, :to => :square_metres
|
750
|
+
def convert_units_after_zeroing_legitimate_skips
|
751
|
+
update_all("#{attr_name} = #{attr_name} * #{Conversions::Unit.exchange_rate(attr_options[:from], attr_options[:to])}")
|
752
|
+
end
|
753
|
+
|
754
|
+
def derive_rooms
|
755
|
+
'total_rooms + bathrooms/2 + halfbaths/4 + heated_garage*(attached_1car_garage + detached_1car_garage + 2*(attached_2car_garage + detached_2car_garage) + 3*(attached_3car_garage + detached_3car_garage))'
|
756
|
+
|
757
|
+
end
|
758
|
+
|
759
|
+
def derive_lighting_use
|
760
|
+
'2*(lights_on_1_to_4_hours + efficient_lights_on_1_to_4_hours) + 8*(lights_on_4_to_12_hours + efficient_lights_on_4_to_12_hours) + 16*(lights_on_over_12_hours + efficient_lights_on_over_12_hours) + 12*(outdoor_all_night_lights + outdoor_all_night_gas_lights)'
|
761
|
+
end
|
762
|
+
|
763
|
+
# will be null if lighting_use is zero
|
764
|
+
def derive_lighting_efficiency
|
765
|
+
'(2*efficient_lights_on_1_to_4_hours + 8*efficient_lights_on_4_to_12_hours + 16*efficient_lights_on_over_12_hours) / lighting_use'
|
766
|
+
end
|
767
|
+
|
768
|
+
def derive_residence_air_conditioner_use_id
|
769
|
+
find_in_batches do |batch|
|
770
|
+
batch.each do |record|
|
771
|
+
ce = record.usecenac.to_i
|
772
|
+
ww = record.usewwac.to_i
|
773
|
+
if ce == 3 or ww == 3
|
774
|
+
selector = 3
|
775
|
+
elsif ce == 2 or ww == 2
|
776
|
+
selector = 2
|
777
|
+
elsif ce == 1 or ww == 1
|
778
|
+
selector = 1
|
779
|
+
elsif ce == 0 or ww == 0
|
780
|
+
selector = 0
|
781
|
+
elsif ce == 9 or ww == 9
|
782
|
+
selector = 9
|
783
|
+
else
|
784
|
+
raise "something's wrong. usecenac => #{ce}, usewwac => #{ww}"
|
785
|
+
end
|
786
|
+
record.air_conditioner_use = ResidenceAirConditionerUse.find_by_code(selector)
|
787
|
+
record.save if record.changed?
|
788
|
+
end
|
789
|
+
end
|
790
|
+
end
|
791
|
+
|
792
|
+
def derive_residence_clothes_drier_use_id
|
793
|
+
find_in_batches do |batch|
|
794
|
+
batch.each do |record|
|
795
|
+
dr = record.dryruse.to_i
|
796
|
+
wa = record.washload.to_i
|
797
|
+
selector = case dr
|
798
|
+
when 9
|
799
|
+
9
|
800
|
+
when 1
|
801
|
+
wa
|
802
|
+
when 2
|
803
|
+
if wa == 9
|
804
|
+
9
|
805
|
+
elsif [ 2, 3, 4, 5 ].include?(wa)
|
806
|
+
wa - 1
|
807
|
+
else
|
808
|
+
1
|
809
|
+
end
|
810
|
+
when 3
|
811
|
+
if wa == 9
|
812
|
+
9
|
813
|
+
elsif [ 3, 4, 5 ].include?(wa)
|
814
|
+
wa - 2
|
815
|
+
else
|
816
|
+
1
|
817
|
+
end
|
818
|
+
else
|
819
|
+
raise "A something's wrong. dryruse => #{dr}, washload => #{wa}"
|
820
|
+
end
|
821
|
+
record.clothes_drier_use = ResidenceClothesDrierUse.find_by_code(selector)
|
822
|
+
raise "B something's wrong. dryruse => #{dr}, washload => #{wa}" if record.clothes_drier_use.nil?
|
823
|
+
record.save if record.changed?
|
824
|
+
end
|
825
|
+
end
|
826
|
+
end
|
827
|
+
end
|
828
|
+
end
|
829
|
+
|
603
830
|
class DataMinerTest < Test::Unit::TestCase
|
604
831
|
should "be idempotent" do
|
605
832
|
Country.data_miner_config.run
|
@@ -619,7 +846,6 @@ class DataMinerTest < Test::Unit::TestCase
|
|
619
846
|
assert_raises DataMiner::MissingHashColumn do
|
620
847
|
class IncompleteCountry < ActiveRecord::Base
|
621
848
|
set_table_name 'countries'
|
622
|
-
belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
|
623
849
|
|
624
850
|
data_miner do
|
625
851
|
# no unique index
|
@@ -691,4 +917,11 @@ class DataMinerTest < Test::Unit::TestCase
|
|
691
917
|
assert a != b
|
692
918
|
assert_equal b, Country.first.data_miner_last_run
|
693
919
|
end
|
920
|
+
|
921
|
+
unless ENV['FAST'] == 'true'
|
922
|
+
should "import using a dictionary" do
|
923
|
+
DataMiner.run :class_names => %w{ ResidentialEnergyConsumptionSurveyResponse }
|
924
|
+
assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
|
925
|
+
end
|
926
|
+
end
|
694
927
|
end
|
data/test/test_helper.rb
CHANGED
@@ -164,4 +164,120 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
|
|
164
164
|
t.integer 'data_miner_last_run_id'
|
165
165
|
end
|
166
166
|
execute "ALTER TABLE automobile_models ADD PRIMARY KEY (row_hash);"
|
167
|
+
|
168
|
+
create_table "residential_energy_consumption_survey_responses", :options => 'ENGINE=InnoDB default charset=utf8', :id => false, :force => true do |t|
|
169
|
+
t.integer "department_of_energy_identifier"
|
170
|
+
|
171
|
+
t.string "residence_class"
|
172
|
+
t.date "construction_year"
|
173
|
+
t.string "construction_period"
|
174
|
+
t.string "urbanity"
|
175
|
+
t.string "dishwasher_use"
|
176
|
+
t.string "central_ac_use"
|
177
|
+
t.string "window_ac_use"
|
178
|
+
t.string "clothes_washer_use"
|
179
|
+
t.string "clothes_dryer_use"
|
180
|
+
|
181
|
+
# not done
|
182
|
+
t.integer "census_division_number"
|
183
|
+
t.string "census_division"
|
184
|
+
t.string "census_region"
|
185
|
+
# ------
|
186
|
+
|
187
|
+
# not done
|
188
|
+
t.integer "residence_air_conditioner_use_id"
|
189
|
+
t.integer "residence_clothes_drier_use_id"
|
190
|
+
# ---
|
191
|
+
|
192
|
+
t.float "rooms"
|
193
|
+
t.float "floorspace"
|
194
|
+
t.integer "residents"
|
195
|
+
t.boolean "ownership"
|
196
|
+
t.boolean "thermostat_programmability"
|
197
|
+
t.integer "refrigerator_count"
|
198
|
+
t.integer "freezer_count"
|
199
|
+
t.float "annual_energy_from_fuel_oil_for_heating_space"
|
200
|
+
t.float "annual_energy_from_fuel_oil_for_heating_water"
|
201
|
+
t.float "annual_energy_from_fuel_oil_for_appliances"
|
202
|
+
t.float "annual_energy_from_natural_gas_for_heating_space"
|
203
|
+
t.float "annual_energy_from_natural_gas_for_heating_water"
|
204
|
+
t.float "annual_energy_from_natural_gas_for_appliances"
|
205
|
+
t.float "annual_energy_from_propane_for_heating_space"
|
206
|
+
t.float "annual_energy_from_propane_for_heating_water"
|
207
|
+
t.float "annual_energy_from_propane_for_appliances"
|
208
|
+
t.float "annual_energy_from_wood"
|
209
|
+
t.float "annual_energy_from_kerosene"
|
210
|
+
t.float "annual_energy_from_electricity_for_clothes_driers"
|
211
|
+
t.float "annual_energy_from_electricity_for_dishwashers"
|
212
|
+
t.float "annual_energy_from_electricity_for_freezers"
|
213
|
+
t.float "annual_energy_from_electricity_for_refrigerators"
|
214
|
+
t.float "annual_energy_from_electricity_for_air_conditioners"
|
215
|
+
t.float "annual_energy_from_electricity_for_heating_space"
|
216
|
+
t.float "annual_energy_from_electricity_for_heating_water"
|
217
|
+
t.float "annual_energy_from_electricity_for_other_appliances"
|
218
|
+
t.float "weighting"
|
219
|
+
t.float "lighting_use"
|
220
|
+
t.float "lighting_efficiency"
|
221
|
+
t.integer "heating_degree_days"
|
222
|
+
t.integer "cooling_degree_days"
|
223
|
+
t.integer "total_rooms"
|
224
|
+
t.integer "bathrooms"
|
225
|
+
t.integer "halfbaths"
|
226
|
+
t.integer "heated_garage"
|
227
|
+
t.integer "attached_1car_garage"
|
228
|
+
t.integer "detached_1car_garage"
|
229
|
+
t.integer "attached_2car_garage"
|
230
|
+
t.integer "detached_2car_garage"
|
231
|
+
t.integer "attached_3car_garage"
|
232
|
+
t.integer "detached_3car_garage"
|
233
|
+
t.integer "lights_on_1_to_4_hours"
|
234
|
+
t.integer "efficient_lights_on_1_to_4_hours"
|
235
|
+
t.integer "lights_on_4_to_12_hours"
|
236
|
+
t.integer "efficient_lights_on_4_to_12_hours"
|
237
|
+
t.integer "lights_on_over_12_hours"
|
238
|
+
t.integer "efficient_lights_on_over_12_hours"
|
239
|
+
t.integer "outdoor_all_night_lights"
|
240
|
+
t.integer "outdoor_all_night_gas_lights"
|
241
|
+
|
242
|
+
t.datetime "created_at"
|
243
|
+
t.datetime "updated_at"
|
244
|
+
t.integer 'data_miner_touch_count'
|
245
|
+
t.integer 'data_miner_last_run_id'
|
246
|
+
end
|
247
|
+
execute "ALTER TABLE residential_energy_consumption_survey_responses ADD PRIMARY KEY (department_of_energy_identifier);"
|
248
|
+
|
249
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_clothes_driers", "annual_energy_from_electricity_for_dishwashers", "annual_energy_from_electricity_for_freezers", "annual_energy_from_electricity_for_refrigerators", "annual_energy_from_electricity_for_air_conditioners", "annual_energy_from_electricity_for_heating_space", "annual_energy_from_electricity_for_heating_water", "annual_energy_from_electricity_for_other_appliances", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu3501626657"
|
250
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_clothes_driers", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_annu1433274229"
|
251
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_clothes_driers", "weighting", "residence_clothes_drier_use_id"], :name => "index_residence_survey_responses_on_annu1262382397"
|
252
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_dishwashers", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu4218458677"
|
253
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_dishwashers", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_annu119061746"
|
254
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_dishwashers", "weighting", "residence_dishwasher_use_id"], :name => "index_residence_survey_responses_on_annu3439036757"
|
255
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_freezers", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu3327447874"
|
256
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_freezers", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_annu1386319236"
|
257
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_refrigerators", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu73542686"
|
258
|
+
# add_index "residence_survey_responses", ["annual_energy_from_electricity_for_refrigerators", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_annu3936186192"
|
259
|
+
# add_index "residence_survey_responses", ["annual_energy_from_fuel_oil_for_heating_space", "annual_energy_from_fuel_oil_for_heating_water", "annual_energy_from_fuel_oil_for_appliances", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu2746016586"
|
260
|
+
# add_index "residence_survey_responses", ["annual_energy_from_kerosene", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu2598214"
|
261
|
+
# add_index "residence_survey_responses", ["annual_energy_from_kerosene", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_annu502197058"
|
262
|
+
# add_index "residence_survey_responses", ["annual_energy_from_natural_gas_for_heating_space", "annual_energy_from_natural_gas_for_heating_water", "annual_energy_from_natural_gas_for_appliances", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu830199308"
|
263
|
+
# add_index "residence_survey_responses", ["annual_energy_from_propane_for_heating_space", "annual_energy_from_propane_for_heating_water", "annual_energy_from_propane_for_appliances", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu4097984181"
|
264
|
+
# add_index "residence_survey_responses", ["annual_energy_from_wood", "weighting", "floorspace", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_annu250862876"
|
265
|
+
# add_index "residence_survey_responses", ["annual_energy_from_wood", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_annu3742395500"
|
266
|
+
# add_index "residence_survey_responses", ["floorspace", "annual_energy_from_electricity_for_clothes_driers", "annual_energy_from_electricity_for_dishwashers", "annual_energy_from_electricity_for_freezers", "annual_energy_from_electricity_for_refrigerators", "annual_energy_from_electricity_for_air_conditioners", "annual_energy_from_electricity_for_heating_space", "annual_energy_from_electricity_for_heating_water", "annual_energy_from_electricity_for_other_appliances", "weighting"], :name => "index_residence_survey_responses_on_floo1081052200"
|
267
|
+
# add_index "residence_survey_responses", ["floorspace", "annual_energy_from_fuel_oil_for_heating_space", "annual_energy_from_fuel_oil_for_heating_water", "annual_energy_from_fuel_oil_for_appliances", "weighting"], :name => "index_residence_survey_responses_on_floo2042532749"
|
268
|
+
# add_index "residence_survey_responses", ["floorspace", "annual_energy_from_natural_gas_for_heating_space", "annual_energy_from_natural_gas_for_heating_water", "annual_energy_from_natural_gas_for_appliances", "weighting"], :name => "index_residence_survey_responses_on_floo4150514738"
|
269
|
+
# add_index "residence_survey_responses", ["floorspace", "annual_energy_from_propane_for_heating_space", "annual_energy_from_propane_for_heating_water", "annual_energy_from_propane_for_appliances", "weighting"], :name => "index_residence_survey_responses_on_floo2054994085"
|
270
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents", "heating_degree_days", "census_region_id", "residence_class_id", "ownership", "cooling_degree_days", "residence_urbanity_id"], :name => "index_residence_survey_responses_on_floo2191768676"
|
271
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents", "heating_degree_days", "census_region_id", "residence_class_id", "ownership", "cooling_degree_days"], :name => "index_residence_survey_responses_on_floo1971465492"
|
272
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents", "heating_degree_days", "census_region_id", "residence_class_id", "ownership"], :name => "index_residence_survey_responses_on_floo4007566201"
|
273
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents", "heating_degree_days", "census_region_id", "residence_class_id"], :name => "index_residence_survey_responses_on_floo1574191187"
|
274
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents", "heating_degree_days", "census_region_id"], :name => "index_residence_survey_responses_on_floo259916455"
|
275
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents", "heating_degree_days"], :name => "index_residence_survey_responses_on_floo2330810762"
|
276
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year", "residents"], :name => "index_residence_survey_responses_on_floo3429600394"
|
277
|
+
# add_index "residence_survey_responses", ["floorspace", "construction_year"], :name => "index_residence_survey_responses_on_floo809808213"
|
278
|
+
# add_index "residence_survey_responses", ["floorspace"], :name => "index_residence_survey_responses_on_floorspace"
|
279
|
+
# add_index "residence_survey_responses", ["lighting_efficiency", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_ligh1681825374"
|
280
|
+
# add_index "residence_survey_responses", ["lighting_use", "weighting", "floorspace"], :name => "index_residence_survey_responses_on_ligh3781776396"
|
281
|
+
# add_index "residence_survey_responses", ["refrigerator_count"], :name => "index_residence_survey_responses_on_refr2806359993"
|
282
|
+
# add_index "residence_survey_responses", ["residence_clothes_drier_use_id"], :name => "index_residence_survey_responses_on_resi3713455541"
|
167
283
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_miner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Seamus Abshere
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2010-03-
|
13
|
+
date: 2010-03-08 00:00:00 -05:00
|
14
14
|
default_executable:
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|