earth-ruby19 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +5 -0
- data/Gemfile.lock +138 -0
- data/LICENSE +20 -0
- data/README.markdown +38 -0
- data/lib/earth.rb +146 -0
- data/lib/earth/active_record_ext.rb +9 -0
- data/lib/earth/air.rb +13 -0
- data/lib/earth/air/aircraft.rb +21 -0
- data/lib/earth/air/aircraft/data_miner.rb +184 -0
- data/lib/earth/air/aircraft_class.rb +10 -0
- data/lib/earth/air/aircraft_class/data_miner.rb +42 -0
- data/lib/earth/air/aircraft_manufacturer.rb +9 -0
- data/lib/earth/air/aircraft_manufacturer/data_miner.rb +20 -0
- data/lib/earth/air/airline.rb +16 -0
- data/lib/earth/air/airline/data_miner.rb +57 -0
- data/lib/earth/air/airport.rb +44 -0
- data/lib/earth/air/airport/data_miner.rb +80 -0
- data/lib/earth/air/data_miner.rb +15 -0
- data/lib/earth/air/flight_configuration.rb +7 -0
- data/lib/earth/air/flight_configuration/data_miner.rb +16 -0
- data/lib/earth/air/flight_distance_class.rb +7 -0
- data/lib/earth/air/flight_distance_class/data_miner.rb +16 -0
- data/lib/earth/air/flight_domesticity.rb +6 -0
- data/lib/earth/air/flight_domesticity/data_miner.rb +57 -0
- data/lib/earth/air/flight_fuel_type.rb +12 -0
- data/lib/earth/air/flight_fuel_type/data_miner.rb +12 -0
- data/lib/earth/air/flight_propulsion.rb +7 -0
- data/lib/earth/air/flight_propulsion/data_miner.rb +16 -0
- data/lib/earth/air/flight_seat_class.rb +12 -0
- data/lib/earth/air/flight_seat_class/data_miner.rb +36 -0
- data/lib/earth/air/flight_segment.rb +29 -0
- data/lib/earth/air/flight_segment/data_miner.rb +330 -0
- data/lib/earth/air/flight_service.rb +7 -0
- data/lib/earth/air/flight_service/data_miner.rb +16 -0
- data/lib/earth/all.rb +11 -0
- data/lib/earth/automobile.rb +8 -0
- data/lib/earth/automobile/automobile_fuel_type.rb +18 -0
- data/lib/earth/automobile/automobile_fuel_type/data_miner.rb +45 -0
- data/lib/earth/automobile/automobile_make.rb +14 -0
- data/lib/earth/automobile/automobile_make/data_miner.rb +68 -0
- data/lib/earth/automobile/automobile_make_fleet_year.rb +15 -0
- data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +29 -0
- data/lib/earth/automobile/automobile_make_year.rb +14 -0
- data/lib/earth/automobile/automobile_make_year/data_miner.rb +45 -0
- data/lib/earth/automobile/automobile_model.rb +14 -0
- data/lib/earth/automobile/automobile_model/data_miner.rb +38 -0
- data/lib/earth/automobile/automobile_model_year.rb +15 -0
- data/lib/earth/automobile/automobile_model_year/data_miner.rb +51 -0
- data/lib/earth/automobile/automobile_size_class.rb +14 -0
- data/lib/earth/automobile/automobile_size_class/data_miner.rb +43 -0
- data/lib/earth/automobile/automobile_variant.rb +17 -0
- data/lib/earth/automobile/automobile_variant/data_miner.rb +460 -0
- data/lib/earth/automobile/data_miner.rb +8 -0
- data/lib/earth/base.rb +7 -0
- data/lib/earth/bus.rb +1 -0
- data/lib/earth/bus/bus_class.rb +19 -0
- data/lib/earth/bus/bus_class/data_miner.rb +41 -0
- data/lib/earth/bus/data_miner.rb +1 -0
- data/lib/earth/conversions_ext.rb +45 -0
- data/lib/earth/data_miner.rb +10 -0
- data/lib/earth/diet.rb +2 -0
- data/lib/earth/diet/data_miner.rb +2 -0
- data/lib/earth/diet/diet_class.rb +15 -0
- data/lib/earth/diet/diet_class/data_miner.rb +36 -0
- data/lib/earth/diet/food_group.rb +17 -0
- data/lib/earth/diet/food_group/data_miner.rb +26 -0
- data/lib/earth/fuel.rb +2 -0
- data/lib/earth/fuel/data_miner.rb +2 -0
- data/lib/earth/fuel/fuel_price.rb +13 -0
- data/lib/earth/fuel/fuel_price/data_miner.rb +20 -0
- data/lib/earth/fuel/fuel_type.rb +18 -0
- data/lib/earth/fuel/fuel_type/data_miner.rb +37 -0
- data/lib/earth/hospitality.rb +1 -0
- data/lib/earth/hospitality/data_miner.rb +1 -0
- data/lib/earth/hospitality/lodging_class.rb +9 -0
- data/lib/earth/hospitality/lodging_class/data_miner.rb +30 -0
- data/lib/earth/industry.rb +10 -0
- data/lib/earth/industry/industry.rb +23 -0
- data/lib/earth/industry/industry_product.rb +22 -0
- data/lib/earth/industry/industry_product_line.rb +20 -0
- data/lib/earth/industry/industry_sector.rb +20 -0
- data/lib/earth/industry/merchant.rb +18 -0
- data/lib/earth/industry/merchant_category.rb +22 -0
- data/lib/earth/industry/merchant_category_industry.rb +20 -0
- data/lib/earth/industry/product_line.rb +22 -0
- data/lib/earth/industry/product_line_industry_product.rb +20 -0
- data/lib/earth/industry/sector.rb +19 -0
- data/lib/earth/inflectors.rb +9 -0
- data/lib/earth/locality.rb +10 -0
- data/lib/earth/locality/census_division.rb +22 -0
- data/lib/earth/locality/census_division/data_miner.rb +64 -0
- data/lib/earth/locality/census_region.rb +13 -0
- data/lib/earth/locality/census_region/data_miner.rb +17 -0
- data/lib/earth/locality/climate_division.rb +17 -0
- data/lib/earth/locality/climate_division/data_miner.rb +20 -0
- data/lib/earth/locality/country.rb +13 -0
- data/lib/earth/locality/country/data_miner.rb +19 -0
- data/lib/earth/locality/data_miner.rb +10 -0
- data/lib/earth/locality/egrid_region.rb +15 -0
- data/lib/earth/locality/egrid_region/data_miner.rb +35 -0
- data/lib/earth/locality/egrid_subregion.rb +16 -0
- data/lib/earth/locality/egrid_subregion/data_miner.rb +65 -0
- data/lib/earth/locality/petroleum_administration_for_defense_district.rb +13 -0
- data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb +21 -0
- data/lib/earth/locality/state.rb +22 -0
- data/lib/earth/locality/state/data_miner.rb +37 -0
- data/lib/earth/locality/urbanity.rb +10 -0
- data/lib/earth/locality/urbanity/data_miner.rb +15 -0
- data/lib/earth/locality/zip_code.rb +23 -0
- data/lib/earth/locality/zip_code/data_miner.rb +43 -0
- data/lib/earth/pet.rb +4 -0
- data/lib/earth/pet/breed.rb +15 -0
- data/lib/earth/pet/breed/data_miner.rb +25 -0
- data/lib/earth/pet/breed_gender.rb +14 -0
- data/lib/earth/pet/breed_gender/data_miner.rb +21 -0
- data/lib/earth/pet/data_miner.rb +4 -0
- data/lib/earth/pet/gender.rb +10 -0
- data/lib/earth/pet/gender/data_miner.rb +13 -0
- data/lib/earth/pet/species.rb +40 -0
- data/lib/earth/pet/species/data_miner.rb +42 -0
- data/lib/earth/rail.rb +1 -0
- data/lib/earth/rail/data_miner.rb +1 -0
- data/lib/earth/rail/rail_class.rb +16 -0
- data/lib/earth/rail/rail_class/data_miner.rb +36 -0
- data/lib/earth/residence.rb +8 -0
- data/lib/earth/residence/air_conditioner_use.rb +13 -0
- data/lib/earth/residence/air_conditioner_use/data_miner.rb +22 -0
- data/lib/earth/residence/clothes_machine_use.rb +10 -0
- data/lib/earth/residence/clothes_machine_use/data_miner.rb +28 -0
- data/lib/earth/residence/data_miner.rb +8 -0
- data/lib/earth/residence/dishwasher_use.rb +10 -0
- data/lib/earth/residence/dishwasher_use/data_miner.rb +28 -0
- data/lib/earth/residence/residence_appliance.rb +16 -0
- data/lib/earth/residence/residence_appliance/data_miner.rb +20 -0
- data/lib/earth/residence/residence_class.rb +16 -0
- data/lib/earth/residence/residence_class/data_miner.rb +15 -0
- data/lib/earth/residence/residence_fuel_price.rb +18 -0
- data/lib/earth/residence/residence_fuel_price/data_miner.rb +200 -0
- data/lib/earth/residence/residence_fuel_type.rb +32 -0
- data/lib/earth/residence/residence_fuel_type/data_miner.rb +18 -0
- data/lib/earth/residence/residential_energy_consumption_survey_response.rb +39 -0
- data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +283 -0
- data/spec/lib/earth_spec.rb +25 -0
- data/spec/spec_helper.rb +11 -0
- data/vendor/geokit-rails/CHANGELOG.rdoc +46 -0
- data/vendor/geokit-rails/MIT-LICENSE +20 -0
- data/vendor/geokit-rails/README.markdown +561 -0
- data/vendor/geokit-rails/Rakefile +18 -0
- data/vendor/geokit-rails/about.yml +9 -0
- data/vendor/geokit-rails/assets/api_keys_template +61 -0
- data/vendor/geokit-rails/init.rb +1 -0
- data/vendor/geokit-rails/install.rb +14 -0
- data/vendor/geokit-rails/lib/geokit-rails.rb +24 -0
- data/vendor/geokit-rails/lib/geokit-rails/acts_as_mappable.rb +456 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/abstract.rb +31 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/mysql.rb +22 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/postgresql.rb +22 -0
- data/vendor/geokit-rails/lib/geokit-rails/adapters/sqlserver.rb +43 -0
- data/vendor/geokit-rails/lib/geokit-rails/defaults.rb +22 -0
- data/vendor/geokit-rails/lib/geokit-rails/geocoder_control.rb +16 -0
- data/vendor/geokit-rails/lib/geokit-rails/ip_geocode_lookup.rb +46 -0
- data/vendor/geokit-rails/test/acts_as_mappable_test.rb +474 -0
- data/vendor/geokit-rails/test/boot.rb +25 -0
- data/vendor/geokit-rails/test/database.yml +20 -0
- data/vendor/geokit-rails/test/fixtures/companies.yml +7 -0
- data/vendor/geokit-rails/test/fixtures/custom_locations.yml +54 -0
- data/vendor/geokit-rails/test/fixtures/locations.yml +54 -0
- data/vendor/geokit-rails/test/fixtures/mock_addresses.yml +17 -0
- data/vendor/geokit-rails/test/fixtures/mock_families.yml +2 -0
- data/vendor/geokit-rails/test/fixtures/mock_houses.yml +9 -0
- data/vendor/geokit-rails/test/fixtures/mock_organizations.yml +5 -0
- data/vendor/geokit-rails/test/fixtures/mock_people.yml +5 -0
- data/vendor/geokit-rails/test/fixtures/stores.yml +0 -0
- data/vendor/geokit-rails/test/ip_geocode_lookup_test.rb +77 -0
- data/vendor/geokit-rails/test/models/company.rb +3 -0
- data/vendor/geokit-rails/test/models/custom_location.rb +12 -0
- data/vendor/geokit-rails/test/models/location.rb +4 -0
- data/vendor/geokit-rails/test/models/mock_address.rb +4 -0
- data/vendor/geokit-rails/test/models/mock_family.rb +3 -0
- data/vendor/geokit-rails/test/models/mock_house.rb +3 -0
- data/vendor/geokit-rails/test/models/mock_organization.rb +4 -0
- data/vendor/geokit-rails/test/models/mock_person.rb +4 -0
- data/vendor/geokit-rails/test/models/store.rb +3 -0
- data/vendor/geokit-rails/test/schema.rb +60 -0
- data/vendor/geokit-rails/test/tasks.rake +31 -0
- data/vendor/geokit-rails/test/test_helper.rb +23 -0
- metadata +476 -0
@@ -0,0 +1,17 @@
|
|
1
|
+
class AutomobileVariant < ActiveRecord::Base
|
2
|
+
set_primary_key :row_hash
|
3
|
+
|
4
|
+
has_many :automobiles, :foreign_key => 'variant_id'
|
5
|
+
belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
|
6
|
+
belongs_to :model, :class_name => 'AutomobileModel', :foreign_key => 'model_name'
|
7
|
+
belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'model_year_name'
|
8
|
+
belongs_to :fuel_type, :class_name => 'AutomobileFuelType', :foreign_key => 'fuel_type_code'
|
9
|
+
|
10
|
+
data_miner do
|
11
|
+
tap "Brighter Planet's sanitized automobile variant data", Earth.taps_server
|
12
|
+
|
13
|
+
process "pull dependencies" do
|
14
|
+
run_data_miner_on_belongs_to_associations
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,460 @@
|
|
1
|
+
AutomobileVariant.class_eval do
|
2
|
+
class << self
|
3
|
+
def transmission_is_blank?(row)
|
4
|
+
row['transmission'].blank?
|
5
|
+
end
|
6
|
+
|
7
|
+
def is_a_2007_gmc_or_chevrolet?(row)
|
8
|
+
row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
|
9
|
+
end
|
10
|
+
|
11
|
+
def is_a_porsche?(row)
|
12
|
+
row['make'].upcase == 'PORSCHE'
|
13
|
+
end
|
14
|
+
|
15
|
+
def is_not_a_porsche?(row)
|
16
|
+
!is_a_porsche? row
|
17
|
+
end
|
18
|
+
|
19
|
+
def is_a_mercedes_benz?(row)
|
20
|
+
row['make'] =~ /MERCEDES/i
|
21
|
+
end
|
22
|
+
|
23
|
+
def is_a_lexus?(row)
|
24
|
+
row['make'].upcase == 'LEXUS'
|
25
|
+
end
|
26
|
+
|
27
|
+
def is_a_bmw?(row)
|
28
|
+
row['make'].upcase == 'BMW'
|
29
|
+
end
|
30
|
+
|
31
|
+
def is_a_ford?(row)
|
32
|
+
row['make'].upcase == 'FORD'
|
33
|
+
end
|
34
|
+
|
35
|
+
def is_a_rolls_royce_and_model_contains_bentley?(row)
|
36
|
+
is_a_rolls_royce?(row) and model_contains_bentley?(row)
|
37
|
+
end
|
38
|
+
|
39
|
+
def is_a_bentley?(row)
|
40
|
+
row['make'].upcase == 'BENTLEY'
|
41
|
+
end
|
42
|
+
|
43
|
+
def is_a_rolls_royce?(row)
|
44
|
+
row['make'] =~ /ROLLS/i
|
45
|
+
end
|
46
|
+
|
47
|
+
def is_a_turbo_brooklands?(row)
|
48
|
+
row['model'] =~ /TURBO R\/RL BKLDS/i
|
49
|
+
end
|
50
|
+
|
51
|
+
def model_contains_maybach?(row)
|
52
|
+
row['model'] =~ /MAYBACH/i
|
53
|
+
end
|
54
|
+
|
55
|
+
def model_contains_bentley?(row)
|
56
|
+
row['model'] =~ /BENTLEY/i
|
57
|
+
end
|
58
|
+
|
59
|
+
def source_code
|
60
|
+
IO.read __FILE__
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# updated with 2010 names
|
65
|
+
TRANSMISSIONS = {
|
66
|
+
'A' => 'Automatic', # prefix
|
67
|
+
'M' => 'Manual', # prefix
|
68
|
+
'L' => 'Automatic', # prefix
|
69
|
+
'S' => 'Semi-Automatic', # prefix
|
70
|
+
'AM' => 'Automated Manual',
|
71
|
+
'C' => 'Continuously Variable',
|
72
|
+
'SA' => 'Semi-Automatic',
|
73
|
+
'CVT' => 'Continuously Variable',
|
74
|
+
'OT' => 'Other'
|
75
|
+
}
|
76
|
+
|
77
|
+
ENGINE_TYPES = {
|
78
|
+
'(GUZZLER)' => nil, # "gas guzzler"
|
79
|
+
'(POLICE)' => nil, # police automobile_variant
|
80
|
+
'(MPFI)' => 'injection',
|
81
|
+
'(MPI*)' => 'injection',
|
82
|
+
'(SPFI)' => 'injection',
|
83
|
+
'(FFS)' => 'injection',
|
84
|
+
'(TURBO)' => 'turbo',
|
85
|
+
'(TRBO)' => 'turbo',
|
86
|
+
'(TC*)' => 'turbo',
|
87
|
+
'(FFS,TRBO)' => %w(injection turbo),
|
88
|
+
'(S-CHARGE)' => 'supercharger',
|
89
|
+
'(SC*)' => 'supercharger',
|
90
|
+
'(DIESEL)' => nil, # diesel
|
91
|
+
'(DSL)' => nil, # diesel
|
92
|
+
'(ROTARY)' => nil, # rotary
|
93
|
+
'(VARIABLE)' => nil, # variable displacement
|
94
|
+
'(NO-CAT)' => nil, # no catalytic converter
|
95
|
+
'(OHC)' => nil, # overhead camshaft
|
96
|
+
'(OHV)' => nil, # overhead valves
|
97
|
+
'(16-VALVE)' => nil, # 16V
|
98
|
+
'(305)' => nil, # 305 cubic inch displacement
|
99
|
+
'(307)' => nil, # 307 cubic inch displacement
|
100
|
+
'(M-ENG)' => nil,
|
101
|
+
'(W-ENG)' => nil,
|
102
|
+
'(GM-BUICK)' => nil,
|
103
|
+
'(GM-CHEV)' => nil,
|
104
|
+
'(GM-OLDS)' => nil,
|
105
|
+
'(GM-PONT)' => nil,
|
106
|
+
}
|
107
|
+
|
108
|
+
class AutomobileVariant::ParserB
|
109
|
+
attr_accessor :year
|
110
|
+
def initialize(options = {})
|
111
|
+
@year = options[:year]
|
112
|
+
end
|
113
|
+
|
114
|
+
def apply(row)
|
115
|
+
row.merge!({
|
116
|
+
'make' => row['carline_mfr_name'], # make it line up with the errata
|
117
|
+
'model' => row['carline_name'], # ditto
|
118
|
+
'transmission' => TRANSMISSIONS[row['model_trans'][0, 1]],
|
119
|
+
'speeds' => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
|
120
|
+
'turbo' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
|
121
|
+
'supercharger' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
|
122
|
+
'injection' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
|
123
|
+
'displacement' => _displacement(row['opt_disp']),
|
124
|
+
'year' => year
|
125
|
+
})
|
126
|
+
row
|
127
|
+
end
|
128
|
+
|
129
|
+
def _displacement(str)
|
130
|
+
str = str.gsub(/[\(\)]/, '').strip
|
131
|
+
if str =~ /^(.+)L$/
|
132
|
+
$1.to_f
|
133
|
+
elsif str =~ /^(.+)CC$/
|
134
|
+
$1.to_f / 1000
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def add_hints!(bus)
|
139
|
+
bus[:format] = :fixed_width
|
140
|
+
bus[:cut] = '13-' if year == 1995
|
141
|
+
bus[:schema_name] = :fuel_economy_guide_b
|
142
|
+
bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
|
143
|
+
Slither.define :fuel_economy_guide_b do |d|
|
144
|
+
d.rows do |row|
|
145
|
+
row.trap { true } # there's only one section
|
146
|
+
row.column 'active_year' , 4, :type => :integer # ACTIVE YEAR
|
147
|
+
row.column 'state_code' , 1, :type => :string # STATE CODE: F=49-STATE,C=CALIFORNIA
|
148
|
+
row.column 'carline_clss' , 2, :type => :integer # CARLINE CLASS CODE
|
149
|
+
row.column 'carline_mfr_code' , 3, :type => :integer # CARLINE MANUFACTURER CODE
|
150
|
+
row.column 'carline_name' , 28, :type => :string # CARLINE NAME
|
151
|
+
row.column 'disp_cub_in' , 4, :type => :integer # DISP CUBIC INCHES
|
152
|
+
row.column 'fuel_system' , 2, :type => :string # FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
|
153
|
+
row.column 'model_trans' , 6, :type => :string # TRANSMISSION TYPE
|
154
|
+
row.column 'no_cyc' , 2, :type => :integer # NUMBER OF ENGINE CYLINDERS
|
155
|
+
row.column 'date_time' , 12, :type => :string # DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
|
156
|
+
row.column 'release_date' , 6, :type => :string # RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
|
157
|
+
row.column 'vi_mfr_code' , 3, :type => :integer # VI MANUFACTURER CODE
|
158
|
+
row.column 'carline_code' , 5, :type => :integer # CARLINE CODE
|
159
|
+
row.column 'basic_eng_id' , 5, :type => :integer # BASIC ENGINE INDEX
|
160
|
+
row.column 'carline_mfr_name' , 32, :type => :string # CARLINE MANUFACTURER NAME
|
161
|
+
row.column 'suppress_code' , 1, :type => :integer # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
|
162
|
+
row.column 'est_city_mpg' , 3, :type => :integer # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
|
163
|
+
row.spacer 2
|
164
|
+
row.column 'highway_mpg' , 3, :type => :integer # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
|
165
|
+
row.spacer 2
|
166
|
+
row.column 'combined_mpg' , 3, :type => :integer # COMBINED MILES PER GALLON
|
167
|
+
row.spacer 2
|
168
|
+
row.column 'unadj_city_mpg' , 3, :type => :integer # UNADJUSTED CITY MILES PER GALLON
|
169
|
+
row.spacer 2
|
170
|
+
row.column 'unadj_hwy_mpg' , 3, :type => :integer # UNADJUSTED HIGHWAY MILES PER GALLON
|
171
|
+
row.spacer 2
|
172
|
+
row.column 'unadj_comb_mpg' , 3, :type => :integer # UNADJUSTED COMBINED MILES PER GALLON
|
173
|
+
row.spacer 2
|
174
|
+
row.column 'ave_anl_fuel' , 6, :type => :integer # "$" in col 147, Annual Fuel Cost starting col 148 in I5
|
175
|
+
row.column 'opt_disp' , 8, :type => :string # OPTIONAL DISPLACEMENT
|
176
|
+
row.column 'engine_desc1' , 10, :type => :string # ENGINE DESCRIPTION 1
|
177
|
+
row.column 'engine_desc2' , 10, :type => :string # ENGINE DESCRIPTION 2
|
178
|
+
row.column 'engine_desc3' , 10, :type => :string # ENGINE DESCRIPTION 3
|
179
|
+
row.column 'body_type_2d' , 10, :type => :string # BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
180
|
+
row.column 'body_type_4d' , 10, :type => :string # BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
181
|
+
row.column 'body_type_hbk' , 10, :type => :string # BODY TYPE HBK - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
182
|
+
row.column 'puerto_rico' , 1, :type => :string # '*' IF FOR PUERTO RICO SALES ONLY
|
183
|
+
row.column 'overdrive' , 4, :type => :string # OVERDRIVE: ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
|
184
|
+
row.column 'drive_system' , 3, :type => :string # FWD=FRONT WHEEL DRIVE, RWD=REAR, 4WD=4-WHEEL
|
185
|
+
row.column 'filler' , 1, :type => :string # NOT USED
|
186
|
+
row.column 'fuel_type' , 1, :type => :string # R=REGULAR(UNLEADED), P=PREMIUM, D=DIESEL
|
187
|
+
row.column 'trans_desc' , 15, :type => :string # TRANSMISSION DESCRIPTORS
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
class AutomobileVariant::ParserC
|
193
|
+
attr_accessor :year
|
194
|
+
def initialize(options = {})
|
195
|
+
@year = options[:year]
|
196
|
+
end
|
197
|
+
|
198
|
+
def add_hints!(bus)
|
199
|
+
# File will decide format based on filename
|
200
|
+
end
|
201
|
+
|
202
|
+
def apply(row)
|
203
|
+
row.merge!({
|
204
|
+
'make' => row['Manufacturer'], # make it line up with the errata
|
205
|
+
'model' => row['carline name'], # ditto
|
206
|
+
'drive' => row['drv'] + 'WD',
|
207
|
+
'transmission' => TRANSMISSIONS[row['trans'][-3, 1]], # only using prefix, probably a FIXME
|
208
|
+
'speeds' => (row['trans'][-2, 1] == 'V') ? '1' : row['trans'][-2, 1],
|
209
|
+
'turbo' => row['T'] == 'T',
|
210
|
+
'supercharger' => row['S'] == 'S',
|
211
|
+
'injection' => true,
|
212
|
+
'year' => year
|
213
|
+
})
|
214
|
+
row
|
215
|
+
end
|
216
|
+
end
|
217
|
+
class AutomobileVariant::ParserD
|
218
|
+
attr_accessor :year
|
219
|
+
def initialize(options = {})
|
220
|
+
@year = options[:year]
|
221
|
+
end
|
222
|
+
|
223
|
+
def add_hints!(bus)
|
224
|
+
end
|
225
|
+
|
226
|
+
def apply(row)
|
227
|
+
row.merge!({
|
228
|
+
'make' => row['MFR'], # make it line up with the errata
|
229
|
+
'model' => row['CAR LINE'], # ditto
|
230
|
+
'drive' => row['DRIVE SYS'] + 'WD',
|
231
|
+
'transmission' => TRANSMISSIONS[row['TRANS'][-3, 1]], # only using prefix, probably a FIXME
|
232
|
+
'speeds' => (row['TRANS'][-2, 1] == 'V') ? '1' : row['TRANS'][-2, 1],
|
233
|
+
'turbo' => row['TURBO'] == 'T',
|
234
|
+
'supercharger' => row['SPCHGR'] == 'S',
|
235
|
+
'injection' => true,
|
236
|
+
'year' => year
|
237
|
+
})
|
238
|
+
row
|
239
|
+
end
|
240
|
+
end
|
241
|
+
class AutomobileVariant::ParserE
|
242
|
+
OLD_FUEL_CODES = {
|
243
|
+
'CNG' => 'C',
|
244
|
+
'DU' => 'D',
|
245
|
+
'G' => 'R',
|
246
|
+
'GP' => 'P',
|
247
|
+
'GPR' => 'P'
|
248
|
+
}
|
249
|
+
|
250
|
+
attr_accessor :year
|
251
|
+
def initialize(options = {})
|
252
|
+
@year = options[:year]
|
253
|
+
end
|
254
|
+
|
255
|
+
def add_hints!(bus)
|
256
|
+
end
|
257
|
+
|
258
|
+
def apply(row)
|
259
|
+
row.merge!({
|
260
|
+
'make' => row['Division'], # make it line up with the errata
|
261
|
+
'model' => row['Carline'], # ditto
|
262
|
+
'drive' => row['Drive Sys'] + 'WD',
|
263
|
+
'transmission' => TRANSMISSIONS[row['Trans']],
|
264
|
+
'speeds' => row['# Gears'],
|
265
|
+
'turbo' => row['Air Aspir Method'] == 'TC',
|
266
|
+
'supercharger' => row['Air Aspir Method'] == 'SC',
|
267
|
+
'injection' => true,
|
268
|
+
'year' => year,
|
269
|
+
'fuel_type_code' => OLD_FUEL_CODES[row['Fuel Usage - Conventional Fuel']]
|
270
|
+
})
|
271
|
+
row
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
data_miner do
|
276
|
+
schema Earth.database_options do
|
277
|
+
string 'row_hash'
|
278
|
+
string 'name' # short name!
|
279
|
+
string 'make_name'
|
280
|
+
string 'model_name' # make + model
|
281
|
+
string 'make_year_name' # make + year
|
282
|
+
string 'model_year_name' # make + model + year
|
283
|
+
integer 'year'
|
284
|
+
float 'fuel_efficiency_city'
|
285
|
+
string 'fuel_efficiency_city_units'
|
286
|
+
float 'fuel_efficiency_highway'
|
287
|
+
string 'fuel_efficiency_highway_units'
|
288
|
+
string 'fuel_type_code'
|
289
|
+
string 'transmission'
|
290
|
+
string 'drive'
|
291
|
+
boolean 'turbo'
|
292
|
+
boolean 'supercharger'
|
293
|
+
integer 'cylinders'
|
294
|
+
float 'displacement'
|
295
|
+
float 'raw_fuel_efficiency_city'
|
296
|
+
string 'raw_fuel_efficiency_city_units'
|
297
|
+
float 'raw_fuel_efficiency_highway'
|
298
|
+
string 'raw_fuel_efficiency_highway_units'
|
299
|
+
integer 'carline_mfr_code'
|
300
|
+
integer 'vi_mfr_code'
|
301
|
+
integer 'carline_code'
|
302
|
+
integer 'carline_class_code'
|
303
|
+
boolean 'injection'
|
304
|
+
string 'carline_class_name'
|
305
|
+
string 'speeds'
|
306
|
+
index 'make_name'
|
307
|
+
index 'model_name'
|
308
|
+
index 'make_year_name'
|
309
|
+
index 'model_year_name'
|
310
|
+
end
|
311
|
+
|
312
|
+
# # 1985---1997
|
313
|
+
(85..97).each do |yy|
|
314
|
+
filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
|
315
|
+
import("19#{ yy } Fuel Economy Guide",
|
316
|
+
:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
|
317
|
+
:filename => filename,
|
318
|
+
:transform => { :class => AutomobileVariant::ParserB, :year => "19#{yy}".to_i },
|
319
|
+
:errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
320
|
+
key 'row_hash'
|
321
|
+
store 'name', :field_name => 'model'
|
322
|
+
store 'make_name', :field_name => 'make'
|
323
|
+
store 'year'
|
324
|
+
store 'fuel_type_code', :field_name => 'fuel_type'
|
325
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
326
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
327
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
328
|
+
store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
329
|
+
store 'cylinders', :field_name => 'no_cyc'
|
330
|
+
store 'drive', :field_name => 'drive_system'
|
331
|
+
store 'carline_mfr_code'
|
332
|
+
store 'vi_mfr_code'
|
333
|
+
store 'carline_code'
|
334
|
+
store 'carline_class_code', :field_name => 'carline_clss'
|
335
|
+
store 'transmission'
|
336
|
+
store 'speeds'
|
337
|
+
store 'turbo'
|
338
|
+
store 'supercharger'
|
339
|
+
store 'injection'
|
340
|
+
store 'displacement'
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
# 1998--2005
|
345
|
+
{
|
346
|
+
1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
|
347
|
+
1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
|
348
|
+
2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
|
349
|
+
2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
|
350
|
+
2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
|
351
|
+
2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
|
352
|
+
2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
|
353
|
+
2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
|
354
|
+
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
355
|
+
import "#{ year } Fuel Economy Guide",
|
356
|
+
options.merge(:transform => { :class => AutomobileVariant::ParserC, :year => year },
|
357
|
+
:errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
358
|
+
key 'row_hash'
|
359
|
+
store 'name', :field_name => 'model'
|
360
|
+
store 'make_name', :field_name => 'make'
|
361
|
+
store 'fuel_type_code', :field_name => 'fl'
|
362
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
363
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
364
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
365
|
+
store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
366
|
+
store 'cylinders', :field_name => 'cyl'
|
367
|
+
store 'displacement', :field_name => 'displ'
|
368
|
+
store 'carline_class_code', :field_name => 'cls' if year >= 2000
|
369
|
+
store 'carline_class_name', :field_name => 'Class'
|
370
|
+
store 'year'
|
371
|
+
store 'transmission'
|
372
|
+
store 'speeds'
|
373
|
+
store 'turbo'
|
374
|
+
store 'supercharger'
|
375
|
+
store 'injection'
|
376
|
+
store 'drive'
|
377
|
+
end
|
378
|
+
end
|
379
|
+
|
380
|
+
# 2006--2009
|
381
|
+
{
|
382
|
+
2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
|
383
|
+
2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
|
384
|
+
2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
|
385
|
+
2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
|
386
|
+
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
387
|
+
import "#{ year } Fuel Economy Guide",
|
388
|
+
options.merge(:transform => { :class => AutomobileVariant::ParserD, :year => year },
|
389
|
+
:errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
390
|
+
key 'row_hash'
|
391
|
+
store 'name', :field_name => 'model'
|
392
|
+
store 'make_name', :field_name => 'make'
|
393
|
+
store 'fuel_type_code', :field_name => 'FUEL TYPE'
|
394
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
395
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
396
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
397
|
+
store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
398
|
+
store 'cylinders', :field_name => 'NUMB CYL'
|
399
|
+
store 'displacement', :field_name => 'DISPLACEMENT'
|
400
|
+
store 'carline_class_code', :field_name => 'CLS'
|
401
|
+
store 'carline_class_name', :field_name => 'CLASS'
|
402
|
+
store 'year'
|
403
|
+
store 'transmission'
|
404
|
+
store 'speeds'
|
405
|
+
store 'turbo'
|
406
|
+
store 'supercharger'
|
407
|
+
store 'injection'
|
408
|
+
store 'drive'
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
412
|
+
# 2010--?
|
413
|
+
# sabshere 5/17/10 apparently needs update
|
414
|
+
# {
|
415
|
+
# 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
|
416
|
+
# }.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
417
|
+
# import "#{ year } Fuel Economy Guide",
|
418
|
+
# options.merge(:transform => { :class => AutomobileVariant::ParserE, :year => year },
|
419
|
+
# :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
|
420
|
+
# key 'row_hash'
|
421
|
+
# store 'name', :field_name => 'model'
|
422
|
+
# store 'make_name', :field_name => 'make'
|
423
|
+
# store 'fuel_type_code'
|
424
|
+
# store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
|
425
|
+
# store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
|
426
|
+
# store 'raw_fuel_efficiency_highway', :field_name => 'Hwy Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
427
|
+
# store 'raw_fuel_efficiency_city', :field_name => 'City Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
428
|
+
# store 'cylinders', :field_name => '# Cyl'
|
429
|
+
# store 'displacement', :field_name => 'Eng Displ'
|
430
|
+
# store 'carline_class_code', :field_name => 'Carline Class'
|
431
|
+
# store 'carline_class_name', :field_name => 'Carline Class Desc'
|
432
|
+
# store 'year'
|
433
|
+
# store 'transmission'
|
434
|
+
# store 'speeds'
|
435
|
+
# store 'turbo'
|
436
|
+
# store 'supercharger'
|
437
|
+
# store 'injection'
|
438
|
+
# store 'drive'
|
439
|
+
# end
|
440
|
+
# end
|
441
|
+
|
442
|
+
process "Derive model and model year names" do
|
443
|
+
update_all "model_name = CONCAT(make_name, ' ', name)"
|
444
|
+
update_all "make_year_name = CONCAT(make_name, ' ', year)"
|
445
|
+
update_all "model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
|
446
|
+
end
|
447
|
+
|
448
|
+
process "Calculate adjusted fuel efficiency using the latest EPA equations" do
|
449
|
+
update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
|
450
|
+
update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
|
451
|
+
end
|
452
|
+
|
453
|
+
%w{ AutomobileMake AutomobileModelYear AutomobileModel }.each do |synthetic_resource|
|
454
|
+
process "Synthesize #{synthetic_resource}" do
|
455
|
+
synthetic_resource.constantize.run_data_miner!
|
456
|
+
end
|
457
|
+
end
|
458
|
+
end
|
459
|
+
end
|
460
|
+
|