earth-ruby19 0.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (187) hide show
  1. data/Gemfile +5 -0
  2. data/Gemfile.lock +138 -0
  3. data/LICENSE +20 -0
  4. data/README.markdown +38 -0
  5. data/lib/earth.rb +146 -0
  6. data/lib/earth/active_record_ext.rb +9 -0
  7. data/lib/earth/air.rb +13 -0
  8. data/lib/earth/air/aircraft.rb +21 -0
  9. data/lib/earth/air/aircraft/data_miner.rb +184 -0
  10. data/lib/earth/air/aircraft_class.rb +10 -0
  11. data/lib/earth/air/aircraft_class/data_miner.rb +42 -0
  12. data/lib/earth/air/aircraft_manufacturer.rb +9 -0
  13. data/lib/earth/air/aircraft_manufacturer/data_miner.rb +20 -0
  14. data/lib/earth/air/airline.rb +16 -0
  15. data/lib/earth/air/airline/data_miner.rb +57 -0
  16. data/lib/earth/air/airport.rb +44 -0
  17. data/lib/earth/air/airport/data_miner.rb +80 -0
  18. data/lib/earth/air/data_miner.rb +15 -0
  19. data/lib/earth/air/flight_configuration.rb +7 -0
  20. data/lib/earth/air/flight_configuration/data_miner.rb +16 -0
  21. data/lib/earth/air/flight_distance_class.rb +7 -0
  22. data/lib/earth/air/flight_distance_class/data_miner.rb +16 -0
  23. data/lib/earth/air/flight_domesticity.rb +6 -0
  24. data/lib/earth/air/flight_domesticity/data_miner.rb +57 -0
  25. data/lib/earth/air/flight_fuel_type.rb +12 -0
  26. data/lib/earth/air/flight_fuel_type/data_miner.rb +12 -0
  27. data/lib/earth/air/flight_propulsion.rb +7 -0
  28. data/lib/earth/air/flight_propulsion/data_miner.rb +16 -0
  29. data/lib/earth/air/flight_seat_class.rb +12 -0
  30. data/lib/earth/air/flight_seat_class/data_miner.rb +36 -0
  31. data/lib/earth/air/flight_segment.rb +29 -0
  32. data/lib/earth/air/flight_segment/data_miner.rb +330 -0
  33. data/lib/earth/air/flight_service.rb +7 -0
  34. data/lib/earth/air/flight_service/data_miner.rb +16 -0
  35. data/lib/earth/all.rb +11 -0
  36. data/lib/earth/automobile.rb +8 -0
  37. data/lib/earth/automobile/automobile_fuel_type.rb +18 -0
  38. data/lib/earth/automobile/automobile_fuel_type/data_miner.rb +45 -0
  39. data/lib/earth/automobile/automobile_make.rb +14 -0
  40. data/lib/earth/automobile/automobile_make/data_miner.rb +68 -0
  41. data/lib/earth/automobile/automobile_make_fleet_year.rb +15 -0
  42. data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +29 -0
  43. data/lib/earth/automobile/automobile_make_year.rb +14 -0
  44. data/lib/earth/automobile/automobile_make_year/data_miner.rb +45 -0
  45. data/lib/earth/automobile/automobile_model.rb +14 -0
  46. data/lib/earth/automobile/automobile_model/data_miner.rb +38 -0
  47. data/lib/earth/automobile/automobile_model_year.rb +15 -0
  48. data/lib/earth/automobile/automobile_model_year/data_miner.rb +51 -0
  49. data/lib/earth/automobile/automobile_size_class.rb +14 -0
  50. data/lib/earth/automobile/automobile_size_class/data_miner.rb +43 -0
  51. data/lib/earth/automobile/automobile_variant.rb +17 -0
  52. data/lib/earth/automobile/automobile_variant/data_miner.rb +460 -0
  53. data/lib/earth/automobile/data_miner.rb +8 -0
  54. data/lib/earth/base.rb +7 -0
  55. data/lib/earth/bus.rb +1 -0
  56. data/lib/earth/bus/bus_class.rb +19 -0
  57. data/lib/earth/bus/bus_class/data_miner.rb +41 -0
  58. data/lib/earth/bus/data_miner.rb +1 -0
  59. data/lib/earth/conversions_ext.rb +45 -0
  60. data/lib/earth/data_miner.rb +10 -0
  61. data/lib/earth/diet.rb +2 -0
  62. data/lib/earth/diet/data_miner.rb +2 -0
  63. data/lib/earth/diet/diet_class.rb +15 -0
  64. data/lib/earth/diet/diet_class/data_miner.rb +36 -0
  65. data/lib/earth/diet/food_group.rb +17 -0
  66. data/lib/earth/diet/food_group/data_miner.rb +26 -0
  67. data/lib/earth/fuel.rb +2 -0
  68. data/lib/earth/fuel/data_miner.rb +2 -0
  69. data/lib/earth/fuel/fuel_price.rb +13 -0
  70. data/lib/earth/fuel/fuel_price/data_miner.rb +20 -0
  71. data/lib/earth/fuel/fuel_type.rb +18 -0
  72. data/lib/earth/fuel/fuel_type/data_miner.rb +37 -0
  73. data/lib/earth/hospitality.rb +1 -0
  74. data/lib/earth/hospitality/data_miner.rb +1 -0
  75. data/lib/earth/hospitality/lodging_class.rb +9 -0
  76. data/lib/earth/hospitality/lodging_class/data_miner.rb +30 -0
  77. data/lib/earth/industry.rb +10 -0
  78. data/lib/earth/industry/industry.rb +23 -0
  79. data/lib/earth/industry/industry_product.rb +22 -0
  80. data/lib/earth/industry/industry_product_line.rb +20 -0
  81. data/lib/earth/industry/industry_sector.rb +20 -0
  82. data/lib/earth/industry/merchant.rb +18 -0
  83. data/lib/earth/industry/merchant_category.rb +22 -0
  84. data/lib/earth/industry/merchant_category_industry.rb +20 -0
  85. data/lib/earth/industry/product_line.rb +22 -0
  86. data/lib/earth/industry/product_line_industry_product.rb +20 -0
  87. data/lib/earth/industry/sector.rb +19 -0
  88. data/lib/earth/inflectors.rb +9 -0
  89. data/lib/earth/locality.rb +10 -0
  90. data/lib/earth/locality/census_division.rb +22 -0
  91. data/lib/earth/locality/census_division/data_miner.rb +64 -0
  92. data/lib/earth/locality/census_region.rb +13 -0
  93. data/lib/earth/locality/census_region/data_miner.rb +17 -0
  94. data/lib/earth/locality/climate_division.rb +17 -0
  95. data/lib/earth/locality/climate_division/data_miner.rb +20 -0
  96. data/lib/earth/locality/country.rb +13 -0
  97. data/lib/earth/locality/country/data_miner.rb +19 -0
  98. data/lib/earth/locality/data_miner.rb +10 -0
  99. data/lib/earth/locality/egrid_region.rb +15 -0
  100. data/lib/earth/locality/egrid_region/data_miner.rb +35 -0
  101. data/lib/earth/locality/egrid_subregion.rb +16 -0
  102. data/lib/earth/locality/egrid_subregion/data_miner.rb +65 -0
  103. data/lib/earth/locality/petroleum_administration_for_defense_district.rb +13 -0
  104. data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb +21 -0
  105. data/lib/earth/locality/state.rb +22 -0
  106. data/lib/earth/locality/state/data_miner.rb +37 -0
  107. data/lib/earth/locality/urbanity.rb +10 -0
  108. data/lib/earth/locality/urbanity/data_miner.rb +15 -0
  109. data/lib/earth/locality/zip_code.rb +23 -0
  110. data/lib/earth/locality/zip_code/data_miner.rb +43 -0
  111. data/lib/earth/pet.rb +4 -0
  112. data/lib/earth/pet/breed.rb +15 -0
  113. data/lib/earth/pet/breed/data_miner.rb +25 -0
  114. data/lib/earth/pet/breed_gender.rb +14 -0
  115. data/lib/earth/pet/breed_gender/data_miner.rb +21 -0
  116. data/lib/earth/pet/data_miner.rb +4 -0
  117. data/lib/earth/pet/gender.rb +10 -0
  118. data/lib/earth/pet/gender/data_miner.rb +13 -0
  119. data/lib/earth/pet/species.rb +40 -0
  120. data/lib/earth/pet/species/data_miner.rb +42 -0
  121. data/lib/earth/rail.rb +1 -0
  122. data/lib/earth/rail/data_miner.rb +1 -0
  123. data/lib/earth/rail/rail_class.rb +16 -0
  124. data/lib/earth/rail/rail_class/data_miner.rb +36 -0
  125. data/lib/earth/residence.rb +8 -0
  126. data/lib/earth/residence/air_conditioner_use.rb +13 -0
  127. data/lib/earth/residence/air_conditioner_use/data_miner.rb +22 -0
  128. data/lib/earth/residence/clothes_machine_use.rb +10 -0
  129. data/lib/earth/residence/clothes_machine_use/data_miner.rb +28 -0
  130. data/lib/earth/residence/data_miner.rb +8 -0
  131. data/lib/earth/residence/dishwasher_use.rb +10 -0
  132. data/lib/earth/residence/dishwasher_use/data_miner.rb +28 -0
  133. data/lib/earth/residence/residence_appliance.rb +16 -0
  134. data/lib/earth/residence/residence_appliance/data_miner.rb +20 -0
  135. data/lib/earth/residence/residence_class.rb +16 -0
  136. data/lib/earth/residence/residence_class/data_miner.rb +15 -0
  137. data/lib/earth/residence/residence_fuel_price.rb +18 -0
  138. data/lib/earth/residence/residence_fuel_price/data_miner.rb +200 -0
  139. data/lib/earth/residence/residence_fuel_type.rb +32 -0
  140. data/lib/earth/residence/residence_fuel_type/data_miner.rb +18 -0
  141. data/lib/earth/residence/residential_energy_consumption_survey_response.rb +39 -0
  142. data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +283 -0
  143. data/spec/lib/earth_spec.rb +25 -0
  144. data/spec/spec_helper.rb +11 -0
  145. data/vendor/geokit-rails/CHANGELOG.rdoc +46 -0
  146. data/vendor/geokit-rails/MIT-LICENSE +20 -0
  147. data/vendor/geokit-rails/README.markdown +561 -0
  148. data/vendor/geokit-rails/Rakefile +18 -0
  149. data/vendor/geokit-rails/about.yml +9 -0
  150. data/vendor/geokit-rails/assets/api_keys_template +61 -0
  151. data/vendor/geokit-rails/init.rb +1 -0
  152. data/vendor/geokit-rails/install.rb +14 -0
  153. data/vendor/geokit-rails/lib/geokit-rails.rb +24 -0
  154. data/vendor/geokit-rails/lib/geokit-rails/acts_as_mappable.rb +456 -0
  155. data/vendor/geokit-rails/lib/geokit-rails/adapters/abstract.rb +31 -0
  156. data/vendor/geokit-rails/lib/geokit-rails/adapters/mysql.rb +22 -0
  157. data/vendor/geokit-rails/lib/geokit-rails/adapters/postgresql.rb +22 -0
  158. data/vendor/geokit-rails/lib/geokit-rails/adapters/sqlserver.rb +43 -0
  159. data/vendor/geokit-rails/lib/geokit-rails/defaults.rb +22 -0
  160. data/vendor/geokit-rails/lib/geokit-rails/geocoder_control.rb +16 -0
  161. data/vendor/geokit-rails/lib/geokit-rails/ip_geocode_lookup.rb +46 -0
  162. data/vendor/geokit-rails/test/acts_as_mappable_test.rb +474 -0
  163. data/vendor/geokit-rails/test/boot.rb +25 -0
  164. data/vendor/geokit-rails/test/database.yml +20 -0
  165. data/vendor/geokit-rails/test/fixtures/companies.yml +7 -0
  166. data/vendor/geokit-rails/test/fixtures/custom_locations.yml +54 -0
  167. data/vendor/geokit-rails/test/fixtures/locations.yml +54 -0
  168. data/vendor/geokit-rails/test/fixtures/mock_addresses.yml +17 -0
  169. data/vendor/geokit-rails/test/fixtures/mock_families.yml +2 -0
  170. data/vendor/geokit-rails/test/fixtures/mock_houses.yml +9 -0
  171. data/vendor/geokit-rails/test/fixtures/mock_organizations.yml +5 -0
  172. data/vendor/geokit-rails/test/fixtures/mock_people.yml +5 -0
  173. data/vendor/geokit-rails/test/fixtures/stores.yml +0 -0
  174. data/vendor/geokit-rails/test/ip_geocode_lookup_test.rb +77 -0
  175. data/vendor/geokit-rails/test/models/company.rb +3 -0
  176. data/vendor/geokit-rails/test/models/custom_location.rb +12 -0
  177. data/vendor/geokit-rails/test/models/location.rb +4 -0
  178. data/vendor/geokit-rails/test/models/mock_address.rb +4 -0
  179. data/vendor/geokit-rails/test/models/mock_family.rb +3 -0
  180. data/vendor/geokit-rails/test/models/mock_house.rb +3 -0
  181. data/vendor/geokit-rails/test/models/mock_organization.rb +4 -0
  182. data/vendor/geokit-rails/test/models/mock_person.rb +4 -0
  183. data/vendor/geokit-rails/test/models/store.rb +3 -0
  184. data/vendor/geokit-rails/test/schema.rb +60 -0
  185. data/vendor/geokit-rails/test/tasks.rake +31 -0
  186. data/vendor/geokit-rails/test/test_helper.rb +23 -0
  187. metadata +476 -0
@@ -0,0 +1,17 @@
1
+ class AutomobileVariant < ActiveRecord::Base
2
+ set_primary_key :row_hash
3
+
4
+ has_many :automobiles, :foreign_key => 'variant_id'
5
+ belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
6
+ belongs_to :model, :class_name => 'AutomobileModel', :foreign_key => 'model_name'
7
+ belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'model_year_name'
8
+ belongs_to :fuel_type, :class_name => 'AutomobileFuelType', :foreign_key => 'fuel_type_code'
9
+
10
+ data_miner do
11
+ tap "Brighter Planet's sanitized automobile variant data", Earth.taps_server
12
+
13
+ process "pull dependencies" do
14
+ run_data_miner_on_belongs_to_associations
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,460 @@
1
+ AutomobileVariant.class_eval do
2
+ class << self
3
+ def transmission_is_blank?(row)
4
+ row['transmission'].blank?
5
+ end
6
+
7
+ def is_a_2007_gmc_or_chevrolet?(row)
8
+ row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
9
+ end
10
+
11
+ def is_a_porsche?(row)
12
+ row['make'].upcase == 'PORSCHE'
13
+ end
14
+
15
+ def is_not_a_porsche?(row)
16
+ !is_a_porsche? row
17
+ end
18
+
19
+ def is_a_mercedes_benz?(row)
20
+ row['make'] =~ /MERCEDES/i
21
+ end
22
+
23
+ def is_a_lexus?(row)
24
+ row['make'].upcase == 'LEXUS'
25
+ end
26
+
27
+ def is_a_bmw?(row)
28
+ row['make'].upcase == 'BMW'
29
+ end
30
+
31
+ def is_a_ford?(row)
32
+ row['make'].upcase == 'FORD'
33
+ end
34
+
35
+ def is_a_rolls_royce_and_model_contains_bentley?(row)
36
+ is_a_rolls_royce?(row) and model_contains_bentley?(row)
37
+ end
38
+
39
+ def is_a_bentley?(row)
40
+ row['make'].upcase == 'BENTLEY'
41
+ end
42
+
43
+ def is_a_rolls_royce?(row)
44
+ row['make'] =~ /ROLLS/i
45
+ end
46
+
47
+ def is_a_turbo_brooklands?(row)
48
+ row['model'] =~ /TURBO R\/RL BKLDS/i
49
+ end
50
+
51
+ def model_contains_maybach?(row)
52
+ row['model'] =~ /MAYBACH/i
53
+ end
54
+
55
+ def model_contains_bentley?(row)
56
+ row['model'] =~ /BENTLEY/i
57
+ end
58
+
59
+ def source_code
60
+ IO.read __FILE__
61
+ end
62
+ end
63
+
64
+ # updated with 2010 names
65
+ TRANSMISSIONS = {
66
+ 'A' => 'Automatic', # prefix
67
+ 'M' => 'Manual', # prefix
68
+ 'L' => 'Automatic', # prefix
69
+ 'S' => 'Semi-Automatic', # prefix
70
+ 'AM' => 'Automated Manual',
71
+ 'C' => 'Continuously Variable',
72
+ 'SA' => 'Semi-Automatic',
73
+ 'CVT' => 'Continuously Variable',
74
+ 'OT' => 'Other'
75
+ }
76
+
77
+ ENGINE_TYPES = {
78
+ '(GUZZLER)' => nil, # "gas guzzler"
79
+ '(POLICE)' => nil, # police automobile_variant
80
+ '(MPFI)' => 'injection',
81
+ '(MPI*)' => 'injection',
82
+ '(SPFI)' => 'injection',
83
+ '(FFS)' => 'injection',
84
+ '(TURBO)' => 'turbo',
85
+ '(TRBO)' => 'turbo',
86
+ '(TC*)' => 'turbo',
87
+ '(FFS,TRBO)' => %w(injection turbo),
88
+ '(S-CHARGE)' => 'supercharger',
89
+ '(SC*)' => 'supercharger',
90
+ '(DIESEL)' => nil, # diesel
91
+ '(DSL)' => nil, # diesel
92
+ '(ROTARY)' => nil, # rotary
93
+ '(VARIABLE)' => nil, # variable displacement
94
+ '(NO-CAT)' => nil, # no catalytic converter
95
+ '(OHC)' => nil, # overhead camshaft
96
+ '(OHV)' => nil, # overhead valves
97
+ '(16-VALVE)' => nil, # 16V
98
+ '(305)' => nil, # 305 cubic inch displacement
99
+ '(307)' => nil, # 307 cubic inch displacement
100
+ '(M-ENG)' => nil,
101
+ '(W-ENG)' => nil,
102
+ '(GM-BUICK)' => nil,
103
+ '(GM-CHEV)' => nil,
104
+ '(GM-OLDS)' => nil,
105
+ '(GM-PONT)' => nil,
106
+ }
107
+
108
+ class AutomobileVariant::ParserB
109
+ attr_accessor :year
110
+ def initialize(options = {})
111
+ @year = options[:year]
112
+ end
113
+
114
+ def apply(row)
115
+ row.merge!({
116
+ 'make' => row['carline_mfr_name'], # make it line up with the errata
117
+ 'model' => row['carline_name'], # ditto
118
+ 'transmission' => TRANSMISSIONS[row['model_trans'][0, 1]],
119
+ 'speeds' => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
120
+ 'turbo' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
121
+ 'supercharger' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
122
+ 'injection' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
123
+ 'displacement' => _displacement(row['opt_disp']),
124
+ 'year' => year
125
+ })
126
+ row
127
+ end
128
+
129
+ def _displacement(str)
130
+ str = str.gsub(/[\(\)]/, '').strip
131
+ if str =~ /^(.+)L$/
132
+ $1.to_f
133
+ elsif str =~ /^(.+)CC$/
134
+ $1.to_f / 1000
135
+ end
136
+ end
137
+
138
+ def add_hints!(bus)
139
+ bus[:format] = :fixed_width
140
+ bus[:cut] = '13-' if year == 1995
141
+ bus[:schema_name] = :fuel_economy_guide_b
142
+ bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
143
+ Slither.define :fuel_economy_guide_b do |d|
144
+ d.rows do |row|
145
+ row.trap { true } # there's only one section
146
+ row.column 'active_year' , 4, :type => :integer # ACTIVE YEAR
147
+ row.column 'state_code' , 1, :type => :string # STATE CODE: F=49-STATE,C=CALIFORNIA
148
+ row.column 'carline_clss' , 2, :type => :integer # CARLINE CLASS CODE
149
+ row.column 'carline_mfr_code' , 3, :type => :integer # CARLINE MANUFACTURER CODE
150
+ row.column 'carline_name' , 28, :type => :string # CARLINE NAME
151
+ row.column 'disp_cub_in' , 4, :type => :integer # DISP CUBIC INCHES
152
+ row.column 'fuel_system' , 2, :type => :string # FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
153
+ row.column 'model_trans' , 6, :type => :string # TRANSMISSION TYPE
154
+ row.column 'no_cyc' , 2, :type => :integer # NUMBER OF ENGINE CYLINDERS
155
+ row.column 'date_time' , 12, :type => :string # DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
156
+ row.column 'release_date' , 6, :type => :string # RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
157
+ row.column 'vi_mfr_code' , 3, :type => :integer # VI MANUFACTURER CODE
158
+ row.column 'carline_code' , 5, :type => :integer # CARLINE CODE
159
+ row.column 'basic_eng_id' , 5, :type => :integer # BASIC ENGINE INDEX
160
+ row.column 'carline_mfr_name' , 32, :type => :string # CARLINE MANUFACTURER NAME
161
+ row.column 'suppress_code' , 1, :type => :integer # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
162
+ row.column 'est_city_mpg' , 3, :type => :integer # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
163
+ row.spacer 2
164
+ row.column 'highway_mpg' , 3, :type => :integer # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
165
+ row.spacer 2
166
+ row.column 'combined_mpg' , 3, :type => :integer # COMBINED MILES PER GALLON
167
+ row.spacer 2
168
+ row.column 'unadj_city_mpg' , 3, :type => :integer # UNADJUSTED CITY MILES PER GALLON
169
+ row.spacer 2
170
+ row.column 'unadj_hwy_mpg' , 3, :type => :integer # UNADJUSTED HIGHWAY MILES PER GALLON
171
+ row.spacer 2
172
+ row.column 'unadj_comb_mpg' , 3, :type => :integer # UNADJUSTED COMBINED MILES PER GALLON
173
+ row.spacer 2
174
+ row.column 'ave_anl_fuel' , 6, :type => :integer # "$" in col 147, Annual Fuel Cost starting col 148 in I5
175
+ row.column 'opt_disp' , 8, :type => :string # OPTIONAL DISPLACEMENT
176
+ row.column 'engine_desc1' , 10, :type => :string # ENGINE DESCRIPTION 1
177
+ row.column 'engine_desc2' , 10, :type => :string # ENGINE DESCRIPTION 2
178
+ row.column 'engine_desc3' , 10, :type => :string # ENGINE DESCRIPTION 3
179
+ row.column 'body_type_2d' , 10, :type => :string # BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
180
+ row.column 'body_type_4d' , 10, :type => :string # BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
181
+ row.column 'body_type_hbk' , 10, :type => :string # BODY TYPE HBK - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
182
+ row.column 'puerto_rico' , 1, :type => :string # '*' IF FOR PUERTO RICO SALES ONLY
183
+ row.column 'overdrive' , 4, :type => :string # OVERDRIVE: ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
184
+ row.column 'drive_system' , 3, :type => :string # FWD=FRONT WHEEL DRIVE, RWD=REAR, 4WD=4-WHEEL
185
+ row.column 'filler' , 1, :type => :string # NOT USED
186
+ row.column 'fuel_type' , 1, :type => :string # R=REGULAR(UNLEADED), P=PREMIUM, D=DIESEL
187
+ row.column 'trans_desc' , 15, :type => :string # TRANSMISSION DESCRIPTORS
188
+ end
189
+ end
190
+ end
191
+ end
192
+ class AutomobileVariant::ParserC
193
+ attr_accessor :year
194
+ def initialize(options = {})
195
+ @year = options[:year]
196
+ end
197
+
198
+ def add_hints!(bus)
199
+ # File will decide format based on filename
200
+ end
201
+
202
+ def apply(row)
203
+ row.merge!({
204
+ 'make' => row['Manufacturer'], # make it line up with the errata
205
+ 'model' => row['carline name'], # ditto
206
+ 'drive' => row['drv'] + 'WD',
207
+ 'transmission' => TRANSMISSIONS[row['trans'][-3, 1]], # only using prefix, probably a FIXME
208
+ 'speeds' => (row['trans'][-2, 1] == 'V') ? '1' : row['trans'][-2, 1],
209
+ 'turbo' => row['T'] == 'T',
210
+ 'supercharger' => row['S'] == 'S',
211
+ 'injection' => true,
212
+ 'year' => year
213
+ })
214
+ row
215
+ end
216
+ end
217
+ class AutomobileVariant::ParserD
218
+ attr_accessor :year
219
+ def initialize(options = {})
220
+ @year = options[:year]
221
+ end
222
+
223
+ def add_hints!(bus)
224
+ end
225
+
226
+ def apply(row)
227
+ row.merge!({
228
+ 'make' => row['MFR'], # make it line up with the errata
229
+ 'model' => row['CAR LINE'], # ditto
230
+ 'drive' => row['DRIVE SYS'] + 'WD',
231
+ 'transmission' => TRANSMISSIONS[row['TRANS'][-3, 1]], # only using prefix, probably a FIXME
232
+ 'speeds' => (row['TRANS'][-2, 1] == 'V') ? '1' : row['TRANS'][-2, 1],
233
+ 'turbo' => row['TURBO'] == 'T',
234
+ 'supercharger' => row['SPCHGR'] == 'S',
235
+ 'injection' => true,
236
+ 'year' => year
237
+ })
238
+ row
239
+ end
240
+ end
241
+ class AutomobileVariant::ParserE
242
+ OLD_FUEL_CODES = {
243
+ 'CNG' => 'C',
244
+ 'DU' => 'D',
245
+ 'G' => 'R',
246
+ 'GP' => 'P',
247
+ 'GPR' => 'P'
248
+ }
249
+
250
+ attr_accessor :year
251
+ def initialize(options = {})
252
+ @year = options[:year]
253
+ end
254
+
255
+ def add_hints!(bus)
256
+ end
257
+
258
+ def apply(row)
259
+ row.merge!({
260
+ 'make' => row['Division'], # make it line up with the errata
261
+ 'model' => row['Carline'], # ditto
262
+ 'drive' => row['Drive Sys'] + 'WD',
263
+ 'transmission' => TRANSMISSIONS[row['Trans']],
264
+ 'speeds' => row['# Gears'],
265
+ 'turbo' => row['Air Aspir Method'] == 'TC',
266
+ 'supercharger' => row['Air Aspir Method'] == 'SC',
267
+ 'injection' => true,
268
+ 'year' => year,
269
+ 'fuel_type_code' => OLD_FUEL_CODES[row['Fuel Usage - Conventional Fuel']]
270
+ })
271
+ row
272
+ end
273
+ end
274
+
275
+ data_miner do
276
+ schema Earth.database_options do
277
+ string 'row_hash'
278
+ string 'name' # short name!
279
+ string 'make_name'
280
+ string 'model_name' # make + model
281
+ string 'make_year_name' # make + year
282
+ string 'model_year_name' # make + model + year
283
+ integer 'year'
284
+ float 'fuel_efficiency_city'
285
+ string 'fuel_efficiency_city_units'
286
+ float 'fuel_efficiency_highway'
287
+ string 'fuel_efficiency_highway_units'
288
+ string 'fuel_type_code'
289
+ string 'transmission'
290
+ string 'drive'
291
+ boolean 'turbo'
292
+ boolean 'supercharger'
293
+ integer 'cylinders'
294
+ float 'displacement'
295
+ float 'raw_fuel_efficiency_city'
296
+ string 'raw_fuel_efficiency_city_units'
297
+ float 'raw_fuel_efficiency_highway'
298
+ string 'raw_fuel_efficiency_highway_units'
299
+ integer 'carline_mfr_code'
300
+ integer 'vi_mfr_code'
301
+ integer 'carline_code'
302
+ integer 'carline_class_code'
303
+ boolean 'injection'
304
+ string 'carline_class_name'
305
+ string 'speeds'
306
+ index 'make_name'
307
+ index 'model_name'
308
+ index 'make_year_name'
309
+ index 'model_year_name'
310
+ end
311
+
312
+ # # 1985---1997
313
+ (85..97).each do |yy|
314
+ filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
315
+ import("19#{ yy } Fuel Economy Guide",
316
+ :url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
317
+ :filename => filename,
318
+ :transform => { :class => AutomobileVariant::ParserB, :year => "19#{yy}".to_i },
319
+ :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
320
+ key 'row_hash'
321
+ store 'name', :field_name => 'model'
322
+ store 'make_name', :field_name => 'make'
323
+ store 'year'
324
+ store 'fuel_type_code', :field_name => 'fuel_type'
325
+ store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
326
+ store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
327
+ store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
328
+ store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
329
+ store 'cylinders', :field_name => 'no_cyc'
330
+ store 'drive', :field_name => 'drive_system'
331
+ store 'carline_mfr_code'
332
+ store 'vi_mfr_code'
333
+ store 'carline_code'
334
+ store 'carline_class_code', :field_name => 'carline_clss'
335
+ store 'transmission'
336
+ store 'speeds'
337
+ store 'turbo'
338
+ store 'supercharger'
339
+ store 'injection'
340
+ store 'displacement'
341
+ end
342
+ end
343
+
344
+ # 1998--2005
345
+ {
346
+ 1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
347
+ 1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
348
+ 2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
349
+ 2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
350
+ 2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
351
+ 2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
352
+ 2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
353
+ 2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
354
+ }.sort { |a, b| a.first <=> b.first }.each do |year, options|
355
+ import "#{ year } Fuel Economy Guide",
356
+ options.merge(:transform => { :class => AutomobileVariant::ParserC, :year => year },
357
+ :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
358
+ key 'row_hash'
359
+ store 'name', :field_name => 'model'
360
+ store 'make_name', :field_name => 'make'
361
+ store 'fuel_type_code', :field_name => 'fl'
362
+ store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
363
+ store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
364
+ store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
365
+ store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
366
+ store 'cylinders', :field_name => 'cyl'
367
+ store 'displacement', :field_name => 'displ'
368
+ store 'carline_class_code', :field_name => 'cls' if year >= 2000
369
+ store 'carline_class_name', :field_name => 'Class'
370
+ store 'year'
371
+ store 'transmission'
372
+ store 'speeds'
373
+ store 'turbo'
374
+ store 'supercharger'
375
+ store 'injection'
376
+ store 'drive'
377
+ end
378
+ end
379
+
380
+ # 2006--2009
381
+ {
382
+ 2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
383
+ 2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
384
+ 2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
385
+ 2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
386
+ }.sort { |a, b| a.first <=> b.first }.each do |year, options|
387
+ import "#{ year } Fuel Economy Guide",
388
+ options.merge(:transform => { :class => AutomobileVariant::ParserD, :year => year },
389
+ :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
390
+ key 'row_hash'
391
+ store 'name', :field_name => 'model'
392
+ store 'make_name', :field_name => 'make'
393
+ store 'fuel_type_code', :field_name => 'FUEL TYPE'
394
+ store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
395
+ store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
396
+ store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
397
+ store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
398
+ store 'cylinders', :field_name => 'NUMB CYL'
399
+ store 'displacement', :field_name => 'DISPLACEMENT'
400
+ store 'carline_class_code', :field_name => 'CLS'
401
+ store 'carline_class_name', :field_name => 'CLASS'
402
+ store 'year'
403
+ store 'transmission'
404
+ store 'speeds'
405
+ store 'turbo'
406
+ store 'supercharger'
407
+ store 'injection'
408
+ store 'drive'
409
+ end
410
+ end
411
+
412
+ # 2010--?
413
+ # sabshere 5/17/10 apparently needs update
414
+ # {
415
+ # 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
416
+ # }.sort { |a, b| a.first <=> b.first }.each do |year, options|
417
+ # import "#{ year } Fuel Economy Guide",
418
+ # options.merge(:transform => { :class => AutomobileVariant::ParserE, :year => year },
419
+ # :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
420
+ # key 'row_hash'
421
+ # store 'name', :field_name => 'model'
422
+ # store 'make_name', :field_name => 'make'
423
+ # store 'fuel_type_code'
424
+ # store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
425
+ # store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
426
+ # store 'raw_fuel_efficiency_highway', :field_name => 'Hwy Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
427
+ # store 'raw_fuel_efficiency_city', :field_name => 'City Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
428
+ # store 'cylinders', :field_name => '# Cyl'
429
+ # store 'displacement', :field_name => 'Eng Displ'
430
+ # store 'carline_class_code', :field_name => 'Carline Class'
431
+ # store 'carline_class_name', :field_name => 'Carline Class Desc'
432
+ # store 'year'
433
+ # store 'transmission'
434
+ # store 'speeds'
435
+ # store 'turbo'
436
+ # store 'supercharger'
437
+ # store 'injection'
438
+ # store 'drive'
439
+ # end
440
+ # end
441
+
442
+ process "Derive model and model year names" do
443
+ update_all "model_name = CONCAT(make_name, ' ', name)"
444
+ update_all "make_year_name = CONCAT(make_name, ' ', year)"
445
+ update_all "model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
446
+ end
447
+
448
+ process "Calculate adjusted fuel efficiency using the latest EPA equations" do
449
+ update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
450
+ update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
451
+ end
452
+
453
+ %w{ AutomobileMake AutomobileModelYear AutomobileModel }.each do |synthetic_resource|
454
+ process "Synthesize #{synthetic_resource}" do
455
+ synthetic_resource.constantize.run_data_miner!
456
+ end
457
+ end
458
+ end
459
+ end
460
+