earth-ruby19 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +5 -0
- data/Gemfile.lock +138 -0
- data/LICENSE +20 -0
- data/README.markdown +38 -0
- data/lib/earth.rb +146 -0
- data/lib/earth/active_record_ext.rb +9 -0
- data/lib/earth/air.rb +13 -0
- data/lib/earth/air/aircraft.rb +21 -0
- data/lib/earth/air/aircraft/data_miner.rb +184 -0
- data/lib/earth/air/aircraft_class.rb +10 -0
- data/lib/earth/air/aircraft_class/data_miner.rb +42 -0
- data/lib/earth/air/aircraft_manufacturer.rb +9 -0
- data/lib/earth/air/aircraft_manufacturer/data_miner.rb +20 -0
- data/lib/earth/air/airline.rb +16 -0
- data/lib/earth/air/airline/data_miner.rb +57 -0
- data/lib/earth/air/airport.rb +44 -0
- data/lib/earth/air/airport/data_miner.rb +80 -0
- data/lib/earth/air/data_miner.rb +15 -0
- data/lib/earth/air/flight_configuration.rb +7 -0
- data/lib/earth/air/flight_configuration/data_miner.rb +16 -0
- data/lib/earth/air/flight_distance_class.rb +7 -0
- data/lib/earth/air/flight_distance_class/data_miner.rb +16 -0
- data/lib/earth/air/flight_domesticity.rb +6 -0
- data/lib/earth/air/flight_domesticity/data_miner.rb +57 -0
- data/lib/earth/air/flight_fuel_type.rb +12 -0
- data/lib/earth/air/flight_fuel_type/data_miner.rb +12 -0
- data/lib/earth/air/flight_propulsion.rb +7 -0
- data/lib/earth/air/flight_propulsion/data_miner.rb +16 -0
- data/lib/earth/air/flight_seat_class.rb +12 -0
- data/lib/earth/air/flight_seat_class/data_miner.rb +36 -0
- data/lib/earth/air/flight_segment.rb +29 -0
- data/lib/earth/air/flight_segment/data_miner.rb +330 -0
- data/lib/earth/air/flight_service.rb +7 -0
- data/lib/earth/air/flight_service/data_miner.rb +16 -0
- data/lib/earth/all.rb +11 -0
- data/lib/earth/automobile.rb +8 -0
- data/lib/earth/automobile/automobile_fuel_type.rb +18 -0
- data/lib/earth/automobile/automobile_fuel_type/data_miner.rb +45 -0
- data/lib/earth/automobile/automobile_make.rb +14 -0
- data/lib/earth/automobile/automobile_make/data_miner.rb +68 -0
- data/lib/earth/automobile/automobile_make_fleet_year.rb +15 -0
- data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +29 -0
- data/lib/earth/automobile/automobile_make_year.rb +14 -0
- data/lib/earth/automobile/automobile_make_year/data_miner.rb +45 -0
- data/lib/earth/automobile/automobile_model.rb +14 -0
- data/lib/earth/automobile/automobile_model/data_miner.rb +38 -0
- data/lib/earth/automobile/automobile_model_year.rb +15 -0
- data/lib/earth/automobile/automobile_model_year/data_miner.rb +51 -0
- data/lib/earth/automobile/automobile_size_class.rb +14 -0
- data/lib/earth/automobile/automobile_size_class/data_miner.rb +43 -0
- data/lib/earth/automobile/automobile_variant.rb +17 -0
- data/lib/earth/automobile/automobile_variant/data_miner.rb +460 -0
- data/lib/earth/automobile/data_miner.rb +8 -0
- data/lib/earth/base.rb +7 -0
- data/lib/earth/bus.rb +1 -0
- data/lib/earth/bus/bus_class.rb +19 -0
- data/lib/earth/bus/bus_class/data_miner.rb +41 -0
- data/lib/earth/bus/data_miner.rb +1 -0
- data/lib/earth/conversions_ext.rb +45 -0
- data/lib/earth/data_miner.rb +10 -0
- data/lib/earth/diet.rb +2 -0
- data/lib/earth/diet/data_miner.rb +2 -0
- data/lib/earth/diet/diet_class.rb +15 -0
- data/lib/earth/diet/diet_class/data_miner.rb +36 -0
- data/lib/earth/diet/food_group.rb +17 -0
- data/lib/earth/diet/food_group/data_miner.rb +26 -0
- data/lib/earth/fuel.rb +2 -0
- data/lib/earth/fuel/data_miner.rb +2 -0
- data/lib/earth/fuel/fuel_price.rb +13 -0
- data/lib/earth/fuel/fuel_price/data_miner.rb +20 -0
- data/lib/earth/fuel/fuel_type.rb +18 -0
- data/lib/earth/fuel/fuel_type/data_miner.rb +37 -0
- data/lib/earth/hospitality.rb +1 -0
- data/lib/earth/hospitality/data_miner.rb +1 -0
- data/lib/earth/hospitality/lodging_class.rb +9 -0
- data/lib/earth/hospitality/lodging_class/data_miner.rb +30 -0
- data/lib/earth/industry.rb +10 -0
- data/lib/earth/industry/industry.rb +23 -0
- data/lib/earth/industry/industry_product.rb +22 -0
- data/lib/earth/industry/industry_product_line.rb +20 -0
- data/lib/earth/industry/industry_sector.rb +20 -0
- data/lib/earth/industry/merchant.rb +18 -0
- data/lib/earth/industry/merchant_category.rb +22 -0
- data/lib/earth/industry/merchant_category_industry.rb +20 -0
- data/lib/earth/industry/product_line.rb +22 -0
- data/lib/earth/industry/product_line_industry_product.rb +20 -0
- data/lib/earth/industry/sector.rb +19 -0
- data/lib/earth/inflectors.rb +9 -0
- data/lib/earth/locality.rb +10 -0
- data/lib/earth/locality/census_division.rb +22 -0
- data/lib/earth/locality/census_division/data_miner.rb +64 -0
- data/lib/earth/locality/census_region.rb +13 -0
- data/lib/earth/locality/census_region/data_miner.rb +17 -0
- data/lib/earth/locality/climate_division.rb +17 -0
- data/lib/earth/locality/climate_division/data_miner.rb +20 -0
- data/lib/earth/locality/country.rb +13 -0
- data/lib/earth/locality/country/data_miner.rb +19 -0
- data/lib/earth/locality/data_miner.rb +10 -0
- data/lib/earth/locality/egrid_region.rb +15 -0
- data/lib/earth/locality/egrid_region/data_miner.rb +35 -0
- data/lib/earth/locality/egrid_subregion.rb +16 -0
- data/lib/earth/locality/egrid_subregion/data_miner.rb +65 -0
- data/lib/earth/locality/petroleum_administration_for_defense_district.rb +13 -0
- data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb +21 -0
- data/lib/earth/locality/state.rb +22 -0
- data/lib/earth/locality/state/data_miner.rb +37 -0
- data/lib/earth/locality/urbanity.rb +10 -0
- data/lib/earth/locality/urbanity/data_miner.rb +15 -0
- data/lib/earth/locality/zip_code.rb +23 -0
- data/lib/earth/locality/zip_code/data_miner.rb +43 -0
- data/lib/earth/pet.rb +4 -0
- data/lib/earth/pet/breed.rb +15 -0
- data/lib/earth/pet/breed/data_miner.rb +25 -0
- data/lib/earth/pet/breed_gender.rb +14 -0
- data/lib/earth/pet/breed_gender/data_miner.rb +21 -0
- data/lib/earth/pet/data_miner.rb +4 -0
- data/lib/earth/pet/gender.rb +10 -0
- data/lib/earth/pet/gender/data_miner.rb +13 -0
- data/lib/earth/pet/species.rb +40 -0
- data/lib/earth/pet/species/data_miner.rb +42 -0
- data/lib/earth/rail.rb +1 -0
- data/lib/earth/rail/data_miner.rb +1 -0
- data/lib/earth/rail/rail_class.rb +16 -0
- data/lib/earth/rail/rail_class/data_miner.rb +36 -0
- data/lib/earth/residence.rb +8 -0
- data/lib/earth/residence/air_conditioner_use.rb +13 -0
- data/lib/earth/residence/air_conditioner_use/data_miner.rb +22 -0
- data/lib/earth/residence/clothes_machine_use.rb +10 -0
- data/lib/earth/residence/clothes_machine_use/data_miner.rb +28 -0
- data/lib/earth/residence/data_miner.rb +8 -0
- data/lib/earth/residence/dishwasher_use.rb +10 -0
- data/lib/earth/residence/dishwasher_use/data_miner.rb +28 -0
- data/lib/earth/residence/residence_appliance.rb +16 -0
- data/lib/earth/residence/residence_appliance/data_miner.rb +20 -0
- data/lib/earth/residence/residence_class.rb +16 -0
- data/lib/earth/residence/residence_class/data_miner.rb +15 -0
- data/lib/earth/residence/residence_fuel_price.rb +18 -0
- data/lib/earth/residence/residence_fuel_price/data_miner.rb +200 -0
- data/lib/earth/residence/residence_fuel_type.rb +32 -0
- data/lib/earth/residence/residence_fuel_type/data_miner.rb +18 -0
- data/lib/earth/residence/residential_energy_consumption_survey_response.rb +39 -0
- data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +283 -0
- data/spec/lib/earth_spec.rb +25 -0
- data/spec/spec_helper.rb +11 -0
- data/vendor/geokit-rails/CHANGELOG.rdoc +46 -0
- data/vendor/geokit-rails/MIT-LICENSE +20 -0
- data/vendor/geokit-rails/README.markdown +561 -0
- data/vendor/geokit-rails/Rakefile +18 -0
- data/vendor/geokit-rails/about.yml +9 -0
- data/vendor/geokit-rails/assets/api_keys_template +61 -0
- data/vendor/geokit-rails/init.rb +1 -0
- data/vendor/geokit-rails/install.rb +14 -0
- data/vendor/geokit-rails/lib/geokit-rails.rb +24 -0
- data/vendor/geokit-rails/lib/geokit-rails/acts_as_mappable.rb +456 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/abstract.rb +31 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/mysql.rb +22 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/postgresql.rb +22 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/sqlserver.rb +43 -0
- data/vendor/geokit-rails/lib/geokit-rails/defaults.rb +22 -0
- data/vendor/geokit-rails/lib/geokit-rails/geocoder_control.rb +16 -0
- data/vendor/geokit-rails/lib/geokit-rails/ip_geocode_lookup.rb +46 -0
- data/vendor/geokit-rails/test/acts_as_mappable_test.rb +474 -0
- data/vendor/geokit-rails/test/boot.rb +25 -0
- data/vendor/geokit-rails/test/database.yml +20 -0
- data/vendor/geokit-rails/test/fixtures/companies.yml +7 -0
- data/vendor/geokit-rails/test/fixtures/custom_locations.yml +54 -0
- data/vendor/geokit-rails/test/fixtures/locations.yml +54 -0
- data/vendor/geokit-rails/test/fixtures/mock_addresses.yml +17 -0
- data/vendor/geokit-rails/test/fixtures/mock_families.yml +2 -0
- data/vendor/geokit-rails/test/fixtures/mock_houses.yml +9 -0
- data/vendor/geokit-rails/test/fixtures/mock_organizations.yml +5 -0
- data/vendor/geokit-rails/test/fixtures/mock_people.yml +5 -0
- data/vendor/geokit-rails/test/fixtures/stores.yml +0 -0
- data/vendor/geokit-rails/test/ip_geocode_lookup_test.rb +77 -0
- data/vendor/geokit-rails/test/models/company.rb +3 -0
- data/vendor/geokit-rails/test/models/custom_location.rb +12 -0
- data/vendor/geokit-rails/test/models/location.rb +4 -0
- data/vendor/geokit-rails/test/models/mock_address.rb +4 -0
- data/vendor/geokit-rails/test/models/mock_family.rb +3 -0
- data/vendor/geokit-rails/test/models/mock_house.rb +3 -0
- data/vendor/geokit-rails/test/models/mock_organization.rb +4 -0
- data/vendor/geokit-rails/test/models/mock_person.rb +4 -0
- data/vendor/geokit-rails/test/models/store.rb +3 -0
- data/vendor/geokit-rails/test/schema.rb +60 -0
- data/vendor/geokit-rails/test/tasks.rake +31 -0
- data/vendor/geokit-rails/test/test_helper.rb +23 -0
- metadata +476 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
class AutomobileVariant < ActiveRecord::Base
|
|
2
|
+
set_primary_key :row_hash
|
|
3
|
+
|
|
4
|
+
has_many :automobiles, :foreign_key => 'variant_id'
|
|
5
|
+
belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
|
|
6
|
+
belongs_to :model, :class_name => 'AutomobileModel', :foreign_key => 'model_name'
|
|
7
|
+
belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'model_year_name'
|
|
8
|
+
belongs_to :fuel_type, :class_name => 'AutomobileFuelType', :foreign_key => 'fuel_type_code'
|
|
9
|
+
|
|
10
|
+
data_miner do
|
|
11
|
+
tap "Brighter Planet's sanitized automobile variant data", Earth.taps_server
|
|
12
|
+
|
|
13
|
+
process "pull dependencies" do
|
|
14
|
+
run_data_miner_on_belongs_to_associations
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
AutomobileVariant.class_eval do
|
|
2
|
+
class << self
|
|
3
|
+
def transmission_is_blank?(row)
|
|
4
|
+
row['transmission'].blank?
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
def is_a_2007_gmc_or_chevrolet?(row)
|
|
8
|
+
row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def is_a_porsche?(row)
|
|
12
|
+
row['make'].upcase == 'PORSCHE'
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def is_not_a_porsche?(row)
|
|
16
|
+
!is_a_porsche? row
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def is_a_mercedes_benz?(row)
|
|
20
|
+
row['make'] =~ /MERCEDES/i
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def is_a_lexus?(row)
|
|
24
|
+
row['make'].upcase == 'LEXUS'
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def is_a_bmw?(row)
|
|
28
|
+
row['make'].upcase == 'BMW'
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def is_a_ford?(row)
|
|
32
|
+
row['make'].upcase == 'FORD'
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def is_a_rolls_royce_and_model_contains_bentley?(row)
|
|
36
|
+
is_a_rolls_royce?(row) and model_contains_bentley?(row)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def is_a_bentley?(row)
|
|
40
|
+
row['make'].upcase == 'BENTLEY'
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def is_a_rolls_royce?(row)
|
|
44
|
+
row['make'] =~ /ROLLS/i
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def is_a_turbo_brooklands?(row)
|
|
48
|
+
row['model'] =~ /TURBO R\/RL BKLDS/i
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def model_contains_maybach?(row)
|
|
52
|
+
row['model'] =~ /MAYBACH/i
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def model_contains_bentley?(row)
|
|
56
|
+
row['model'] =~ /BENTLEY/i
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def source_code
|
|
60
|
+
IO.read __FILE__
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# updated with 2010 names
|
|
65
|
+
TRANSMISSIONS = {
|
|
66
|
+
'A' => 'Automatic', # prefix
|
|
67
|
+
'M' => 'Manual', # prefix
|
|
68
|
+
'L' => 'Automatic', # prefix
|
|
69
|
+
'S' => 'Semi-Automatic', # prefix
|
|
70
|
+
'AM' => 'Automated Manual',
|
|
71
|
+
'C' => 'Continuously Variable',
|
|
72
|
+
'SA' => 'Semi-Automatic',
|
|
73
|
+
'CVT' => 'Continuously Variable',
|
|
74
|
+
'OT' => 'Other'
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
ENGINE_TYPES = {
|
|
78
|
+
'(GUZZLER)' => nil, # "gas guzzler"
|
|
79
|
+
'(POLICE)' => nil, # police automobile_variant
|
|
80
|
+
'(MPFI)' => 'injection',
|
|
81
|
+
'(MPI*)' => 'injection',
|
|
82
|
+
'(SPFI)' => 'injection',
|
|
83
|
+
'(FFS)' => 'injection',
|
|
84
|
+
'(TURBO)' => 'turbo',
|
|
85
|
+
'(TRBO)' => 'turbo',
|
|
86
|
+
'(TC*)' => 'turbo',
|
|
87
|
+
'(FFS,TRBO)' => %w(injection turbo),
|
|
88
|
+
'(S-CHARGE)' => 'supercharger',
|
|
89
|
+
'(SC*)' => 'supercharger',
|
|
90
|
+
'(DIESEL)' => nil, # diesel
|
|
91
|
+
'(DSL)' => nil, # diesel
|
|
92
|
+
'(ROTARY)' => nil, # rotary
|
|
93
|
+
'(VARIABLE)' => nil, # variable displacement
|
|
94
|
+
'(NO-CAT)' => nil, # no catalytic converter
|
|
95
|
+
'(OHC)' => nil, # overhead camshaft
|
|
96
|
+
'(OHV)' => nil, # overhead valves
|
|
97
|
+
'(16-VALVE)' => nil, # 16V
|
|
98
|
+
'(305)' => nil, # 305 cubic inch displacement
|
|
99
|
+
'(307)' => nil, # 307 cubic inch displacement
|
|
100
|
+
'(M-ENG)' => nil,
|
|
101
|
+
'(W-ENG)' => nil,
|
|
102
|
+
'(GM-BUICK)' => nil,
|
|
103
|
+
'(GM-CHEV)' => nil,
|
|
104
|
+
'(GM-OLDS)' => nil,
|
|
105
|
+
'(GM-PONT)' => nil,
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
class AutomobileVariant::ParserB
|
|
109
|
+
attr_accessor :year
|
|
110
|
+
def initialize(options = {})
|
|
111
|
+
@year = options[:year]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def apply(row)
|
|
115
|
+
row.merge!({
|
|
116
|
+
'make' => row['carline_mfr_name'], # make it line up with the errata
|
|
117
|
+
'model' => row['carline_name'], # ditto
|
|
118
|
+
'transmission' => TRANSMISSIONS[row['model_trans'][0, 1]],
|
|
119
|
+
'speeds' => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
|
|
120
|
+
'turbo' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
|
|
121
|
+
'supercharger' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
|
|
122
|
+
'injection' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
|
|
123
|
+
'displacement' => _displacement(row['opt_disp']),
|
|
124
|
+
'year' => year
|
|
125
|
+
})
|
|
126
|
+
row
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def _displacement(str)
|
|
130
|
+
str = str.gsub(/[\(\)]/, '').strip
|
|
131
|
+
if str =~ /^(.+)L$/
|
|
132
|
+
$1.to_f
|
|
133
|
+
elsif str =~ /^(.+)CC$/
|
|
134
|
+
$1.to_f / 1000
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def add_hints!(bus)
|
|
139
|
+
bus[:format] = :fixed_width
|
|
140
|
+
bus[:cut] = '13-' if year == 1995
|
|
141
|
+
bus[:schema_name] = :fuel_economy_guide_b
|
|
142
|
+
bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
|
|
143
|
+
Slither.define :fuel_economy_guide_b do |d|
|
|
144
|
+
d.rows do |row|
|
|
145
|
+
row.trap { true } # there's only one section
|
|
146
|
+
row.column 'active_year' , 4, :type => :integer # ACTIVE YEAR
|
|
147
|
+
row.column 'state_code' , 1, :type => :string # STATE CODE: F=49-STATE,C=CALIFORNIA
|
|
148
|
+
row.column 'carline_clss' , 2, :type => :integer # CARLINE CLASS CODE
|
|
149
|
+
row.column 'carline_mfr_code' , 3, :type => :integer # CARLINE MANUFACTURER CODE
|
|
150
|
+
row.column 'carline_name' , 28, :type => :string # CARLINE NAME
|
|
151
|
+
row.column 'disp_cub_in' , 4, :type => :integer # DISP CUBIC INCHES
|
|
152
|
+
row.column 'fuel_system' , 2, :type => :string # FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
|
|
153
|
+
row.column 'model_trans' , 6, :type => :string # TRANSMISSION TYPE
|
|
154
|
+
row.column 'no_cyc' , 2, :type => :integer # NUMBER OF ENGINE CYLINDERS
|
|
155
|
+
row.column 'date_time' , 12, :type => :string # DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
|
|
156
|
+
row.column 'release_date' , 6, :type => :string # RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
|
|
157
|
+
row.column 'vi_mfr_code' , 3, :type => :integer # VI MANUFACTURER CODE
|
|
158
|
+
row.column 'carline_code' , 5, :type => :integer # CARLINE CODE
|
|
159
|
+
row.column 'basic_eng_id' , 5, :type => :integer # BASIC ENGINE INDEX
|
|
160
|
+
row.column 'carline_mfr_name' , 32, :type => :string # CARLINE MANUFACTURER NAME
|
|
161
|
+
row.column 'suppress_code' , 1, :type => :integer # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
|
|
162
|
+
row.column 'est_city_mpg' , 3, :type => :integer # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
|
|
163
|
+
row.spacer 2
|
|
164
|
+
row.column 'highway_mpg' , 3, :type => :integer # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
|
|
165
|
+
row.spacer 2
|
|
166
|
+
row.column 'combined_mpg' , 3, :type => :integer # COMBINED MILES PER GALLON
|
|
167
|
+
row.spacer 2
|
|
168
|
+
row.column 'unadj_city_mpg' , 3, :type => :integer # UNADJUSTED CITY MILES PER GALLON
|
|
169
|
+
row.spacer 2
|
|
170
|
+
row.column 'unadj_hwy_mpg' , 3, :type => :integer # UNADJUSTED HIGHWAY MILES PER GALLON
|
|
171
|
+
row.spacer 2
|
|
172
|
+
row.column 'unadj_comb_mpg' , 3, :type => :integer # UNADJUSTED COMBINED MILES PER GALLON
|
|
173
|
+
row.spacer 2
|
|
174
|
+
row.column 'ave_anl_fuel' , 6, :type => :integer # "$" in col 147, Annual Fuel Cost starting col 148 in I5
|
|
175
|
+
row.column 'opt_disp' , 8, :type => :string # OPTIONAL DISPLACEMENT
|
|
176
|
+
row.column 'engine_desc1' , 10, :type => :string # ENGINE DESCRIPTION 1
|
|
177
|
+
row.column 'engine_desc2' , 10, :type => :string # ENGINE DESCRIPTION 2
|
|
178
|
+
row.column 'engine_desc3' , 10, :type => :string # ENGINE DESCRIPTION 3
|
|
179
|
+
row.column 'body_type_2d' , 10, :type => :string # BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
180
|
+
row.column 'body_type_4d' , 10, :type => :string # BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
181
|
+
row.column 'body_type_hbk' , 10, :type => :string # BODY TYPE HBK - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
182
|
+
row.column 'puerto_rico' , 1, :type => :string # '*' IF FOR PUERTO RICO SALES ONLY
|
|
183
|
+
row.column 'overdrive' , 4, :type => :string # OVERDRIVE: ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
|
|
184
|
+
row.column 'drive_system' , 3, :type => :string # FWD=FRONT WHEEL DRIVE, RWD=REAR, 4WD=4-WHEEL
|
|
185
|
+
row.column 'filler' , 1, :type => :string # NOT USED
|
|
186
|
+
row.column 'fuel_type' , 1, :type => :string # R=REGULAR(UNLEADED), P=PREMIUM, D=DIESEL
|
|
187
|
+
row.column 'trans_desc' , 15, :type => :string # TRANSMISSION DESCRIPTORS
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
class AutomobileVariant::ParserC
|
|
193
|
+
attr_accessor :year
|
|
194
|
+
def initialize(options = {})
|
|
195
|
+
@year = options[:year]
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def add_hints!(bus)
|
|
199
|
+
# File will decide format based on filename
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def apply(row)
|
|
203
|
+
row.merge!({
|
|
204
|
+
'make' => row['Manufacturer'], # make it line up with the errata
|
|
205
|
+
'model' => row['carline name'], # ditto
|
|
206
|
+
'drive' => row['drv'] + 'WD',
|
|
207
|
+
'transmission' => TRANSMISSIONS[row['trans'][-3, 1]], # only using prefix, probably a FIXME
|
|
208
|
+
'speeds' => (row['trans'][-2, 1] == 'V') ? '1' : row['trans'][-2, 1],
|
|
209
|
+
'turbo' => row['T'] == 'T',
|
|
210
|
+
'supercharger' => row['S'] == 'S',
|
|
211
|
+
'injection' => true,
|
|
212
|
+
'year' => year
|
|
213
|
+
})
|
|
214
|
+
row
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
class AutomobileVariant::ParserD
|
|
218
|
+
attr_accessor :year
|
|
219
|
+
def initialize(options = {})
|
|
220
|
+
@year = options[:year]
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def add_hints!(bus)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def apply(row)
|
|
227
|
+
row.merge!({
|
|
228
|
+
'make' => row['MFR'], # make it line up with the errata
|
|
229
|
+
'model' => row['CAR LINE'], # ditto
|
|
230
|
+
'drive' => row['DRIVE SYS'] + 'WD',
|
|
231
|
+
'transmission' => TRANSMISSIONS[row['TRANS'][-3, 1]], # only using prefix, probably a FIXME
|
|
232
|
+
'speeds' => (row['TRANS'][-2, 1] == 'V') ? '1' : row['TRANS'][-2, 1],
|
|
233
|
+
'turbo' => row['TURBO'] == 'T',
|
|
234
|
+
'supercharger' => row['SPCHGR'] == 'S',
|
|
235
|
+
'injection' => true,
|
|
236
|
+
'year' => year
|
|
237
|
+
})
|
|
238
|
+
row
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
class AutomobileVariant::ParserE
|
|
242
|
+
OLD_FUEL_CODES = {
|
|
243
|
+
'CNG' => 'C',
|
|
244
|
+
'DU' => 'D',
|
|
245
|
+
'G' => 'R',
|
|
246
|
+
'GP' => 'P',
|
|
247
|
+
'GPR' => 'P'
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
attr_accessor :year
|
|
251
|
+
def initialize(options = {})
|
|
252
|
+
@year = options[:year]
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def add_hints!(bus)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def apply(row)
|
|
259
|
+
row.merge!({
|
|
260
|
+
'make' => row['Division'], # make it line up with the errata
|
|
261
|
+
'model' => row['Carline'], # ditto
|
|
262
|
+
'drive' => row['Drive Sys'] + 'WD',
|
|
263
|
+
'transmission' => TRANSMISSIONS[row['Trans']],
|
|
264
|
+
'speeds' => row['# Gears'],
|
|
265
|
+
'turbo' => row['Air Aspir Method'] == 'TC',
|
|
266
|
+
'supercharger' => row['Air Aspir Method'] == 'SC',
|
|
267
|
+
'injection' => true,
|
|
268
|
+
'year' => year,
|
|
269
|
+
'fuel_type_code' => OLD_FUEL_CODES[row['Fuel Usage - Conventional Fuel']]
|
|
270
|
+
})
|
|
271
|
+
row
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
data_miner do
|
|
276
|
+
schema Earth.database_options do
|
|
277
|
+
string 'row_hash'
|
|
278
|
+
string 'name' # short name!
|
|
279
|
+
string 'make_name'
|
|
280
|
+
string 'model_name' # make + model
|
|
281
|
+
string 'make_year_name' # make + year
|
|
282
|
+
string 'model_year_name' # make + model + year
|
|
283
|
+
integer 'year'
|
|
284
|
+
float 'fuel_efficiency_city'
|
|
285
|
+
string 'fuel_efficiency_city_units'
|
|
286
|
+
float 'fuel_efficiency_highway'
|
|
287
|
+
string 'fuel_efficiency_highway_units'
|
|
288
|
+
string 'fuel_type_code'
|
|
289
|
+
string 'transmission'
|
|
290
|
+
string 'drive'
|
|
291
|
+
boolean 'turbo'
|
|
292
|
+
boolean 'supercharger'
|
|
293
|
+
integer 'cylinders'
|
|
294
|
+
float 'displacement'
|
|
295
|
+
float 'raw_fuel_efficiency_city'
|
|
296
|
+
string 'raw_fuel_efficiency_city_units'
|
|
297
|
+
float 'raw_fuel_efficiency_highway'
|
|
298
|
+
string 'raw_fuel_efficiency_highway_units'
|
|
299
|
+
integer 'carline_mfr_code'
|
|
300
|
+
integer 'vi_mfr_code'
|
|
301
|
+
integer 'carline_code'
|
|
302
|
+
integer 'carline_class_code'
|
|
303
|
+
boolean 'injection'
|
|
304
|
+
string 'carline_class_name'
|
|
305
|
+
string 'speeds'
|
|
306
|
+
index 'make_name'
|
|
307
|
+
index 'model_name'
|
|
308
|
+
index 'make_year_name'
|
|
309
|
+
index 'model_year_name'
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# # 1985---1997
|
|
313
|
+
(85..97).each do |yy|
|
|
314
|
+
filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
|
|
315
|
+
import("19#{ yy } Fuel Economy Guide",
|
|
316
|
+
:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
|
|
317
|
+
:filename => filename,
|
|
318
|
+
:transform => { :class => AutomobileVariant::ParserB, :year => "19#{yy}".to_i },
|
|
319
|
+
:errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
|
320
|
+
key 'row_hash'
|
|
321
|
+
store 'name', :field_name => 'model'
|
|
322
|
+
store 'make_name', :field_name => 'make'
|
|
323
|
+
store 'year'
|
|
324
|
+
store 'fuel_type_code', :field_name => 'fuel_type'
|
|
325
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
|
326
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
|
327
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
328
|
+
store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
329
|
+
store 'cylinders', :field_name => 'no_cyc'
|
|
330
|
+
store 'drive', :field_name => 'drive_system'
|
|
331
|
+
store 'carline_mfr_code'
|
|
332
|
+
store 'vi_mfr_code'
|
|
333
|
+
store 'carline_code'
|
|
334
|
+
store 'carline_class_code', :field_name => 'carline_clss'
|
|
335
|
+
store 'transmission'
|
|
336
|
+
store 'speeds'
|
|
337
|
+
store 'turbo'
|
|
338
|
+
store 'supercharger'
|
|
339
|
+
store 'injection'
|
|
340
|
+
store 'displacement'
|
|
341
|
+
end
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# 1998--2005
|
|
345
|
+
{
|
|
346
|
+
1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
|
|
347
|
+
1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
|
|
348
|
+
2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
|
|
349
|
+
2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
|
|
350
|
+
2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
|
|
351
|
+
2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
|
|
352
|
+
2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
|
|
353
|
+
2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
|
|
354
|
+
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
355
|
+
import "#{ year } Fuel Economy Guide",
|
|
356
|
+
options.merge(:transform => { :class => AutomobileVariant::ParserC, :year => year },
|
|
357
|
+
:errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
|
358
|
+
key 'row_hash'
|
|
359
|
+
store 'name', :field_name => 'model'
|
|
360
|
+
store 'make_name', :field_name => 'make'
|
|
361
|
+
store 'fuel_type_code', :field_name => 'fl'
|
|
362
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
|
363
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
|
364
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
365
|
+
store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
366
|
+
store 'cylinders', :field_name => 'cyl'
|
|
367
|
+
store 'displacement', :field_name => 'displ'
|
|
368
|
+
store 'carline_class_code', :field_name => 'cls' if year >= 2000
|
|
369
|
+
store 'carline_class_name', :field_name => 'Class'
|
|
370
|
+
store 'year'
|
|
371
|
+
store 'transmission'
|
|
372
|
+
store 'speeds'
|
|
373
|
+
store 'turbo'
|
|
374
|
+
store 'supercharger'
|
|
375
|
+
store 'injection'
|
|
376
|
+
store 'drive'
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# 2006--2009
|
|
381
|
+
{
|
|
382
|
+
2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
|
|
383
|
+
2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
|
|
384
|
+
2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
|
|
385
|
+
2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
|
|
386
|
+
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
387
|
+
import "#{ year } Fuel Economy Guide",
|
|
388
|
+
options.merge(:transform => { :class => AutomobileVariant::ParserD, :year => year },
|
|
389
|
+
:errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
|
390
|
+
key 'row_hash'
|
|
391
|
+
store 'name', :field_name => 'model'
|
|
392
|
+
store 'make_name', :field_name => 'make'
|
|
393
|
+
store 'fuel_type_code', :field_name => 'FUEL TYPE'
|
|
394
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
|
395
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
|
396
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
397
|
+
store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
398
|
+
store 'cylinders', :field_name => 'NUMB CYL'
|
|
399
|
+
store 'displacement', :field_name => 'DISPLACEMENT'
|
|
400
|
+
store 'carline_class_code', :field_name => 'CLS'
|
|
401
|
+
store 'carline_class_name', :field_name => 'CLASS'
|
|
402
|
+
store 'year'
|
|
403
|
+
store 'transmission'
|
|
404
|
+
store 'speeds'
|
|
405
|
+
store 'turbo'
|
|
406
|
+
store 'supercharger'
|
|
407
|
+
store 'injection'
|
|
408
|
+
store 'drive'
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# 2010--?
|
|
413
|
+
# sabshere 5/17/10 apparently needs update
|
|
414
|
+
# {
|
|
415
|
+
# 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
|
|
416
|
+
# }.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
417
|
+
# import "#{ year } Fuel Economy Guide",
|
|
418
|
+
# options.merge(:transform => { :class => AutomobileVariant::ParserE, :year => year },
|
|
419
|
+
# :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
|
420
|
+
# key 'row_hash'
|
|
421
|
+
# store 'name', :field_name => 'model'
|
|
422
|
+
# store 'make_name', :field_name => 'make'
|
|
423
|
+
# store 'fuel_type_code'
|
|
424
|
+
# store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
|
425
|
+
# store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
|
426
|
+
# store 'raw_fuel_efficiency_highway', :field_name => 'Hwy Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
427
|
+
# store 'raw_fuel_efficiency_city', :field_name => 'City Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
428
|
+
# store 'cylinders', :field_name => '# Cyl'
|
|
429
|
+
# store 'displacement', :field_name => 'Eng Displ'
|
|
430
|
+
# store 'carline_class_code', :field_name => 'Carline Class'
|
|
431
|
+
# store 'carline_class_name', :field_name => 'Carline Class Desc'
|
|
432
|
+
# store 'year'
|
|
433
|
+
# store 'transmission'
|
|
434
|
+
# store 'speeds'
|
|
435
|
+
# store 'turbo'
|
|
436
|
+
# store 'supercharger'
|
|
437
|
+
# store 'injection'
|
|
438
|
+
# store 'drive'
|
|
439
|
+
# end
|
|
440
|
+
# end
|
|
441
|
+
|
|
442
|
+
process "Derive model and model year names" do
|
|
443
|
+
update_all "model_name = CONCAT(make_name, ' ', name)"
|
|
444
|
+
update_all "make_year_name = CONCAT(make_name, ' ', year)"
|
|
445
|
+
update_all "model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
process "Calculate adjusted fuel efficiency using the latest EPA equations" do
|
|
449
|
+
update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
|
|
450
|
+
update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
%w{ AutomobileMake AutomobileModelYear AutomobileModel }.each do |synthetic_resource|
|
|
454
|
+
process "Synthesize #{synthetic_resource}" do
|
|
455
|
+
synthetic_resource.constantize.run_data_miner!
|
|
456
|
+
end
|
|
457
|
+
end
|
|
458
|
+
end
|
|
459
|
+
end
|
|
460
|
+
|