earth-ruby19 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. data/Gemfile +5 -0
  2. data/Gemfile.lock +138 -0
  3. data/LICENSE +20 -0
  4. data/README.markdown +38 -0
  5. data/lib/earth.rb +146 -0
  6. data/lib/earth/active_record_ext.rb +9 -0
  7. data/lib/earth/air.rb +13 -0
  8. data/lib/earth/air/aircraft.rb +21 -0
  9. data/lib/earth/air/aircraft/data_miner.rb +184 -0
  10. data/lib/earth/air/aircraft_class.rb +10 -0
  11. data/lib/earth/air/aircraft_class/data_miner.rb +42 -0
  12. data/lib/earth/air/aircraft_manufacturer.rb +9 -0
  13. data/lib/earth/air/aircraft_manufacturer/data_miner.rb +20 -0
  14. data/lib/earth/air/airline.rb +16 -0
  15. data/lib/earth/air/airline/data_miner.rb +57 -0
  16. data/lib/earth/air/airport.rb +44 -0
  17. data/lib/earth/air/airport/data_miner.rb +80 -0
  18. data/lib/earth/air/data_miner.rb +15 -0
  19. data/lib/earth/air/flight_configuration.rb +7 -0
  20. data/lib/earth/air/flight_configuration/data_miner.rb +16 -0
  21. data/lib/earth/air/flight_distance_class.rb +7 -0
  22. data/lib/earth/air/flight_distance_class/data_miner.rb +16 -0
  23. data/lib/earth/air/flight_domesticity.rb +6 -0
  24. data/lib/earth/air/flight_domesticity/data_miner.rb +57 -0
  25. data/lib/earth/air/flight_fuel_type.rb +12 -0
  26. data/lib/earth/air/flight_fuel_type/data_miner.rb +12 -0
  27. data/lib/earth/air/flight_propulsion.rb +7 -0
  28. data/lib/earth/air/flight_propulsion/data_miner.rb +16 -0
  29. data/lib/earth/air/flight_seat_class.rb +12 -0
  30. data/lib/earth/air/flight_seat_class/data_miner.rb +36 -0
  31. data/lib/earth/air/flight_segment.rb +29 -0
  32. data/lib/earth/air/flight_segment/data_miner.rb +330 -0
  33. data/lib/earth/air/flight_service.rb +7 -0
  34. data/lib/earth/air/flight_service/data_miner.rb +16 -0
  35. data/lib/earth/all.rb +11 -0
  36. data/lib/earth/automobile.rb +8 -0
  37. data/lib/earth/automobile/automobile_fuel_type.rb +18 -0
  38. data/lib/earth/automobile/automobile_fuel_type/data_miner.rb +45 -0
  39. data/lib/earth/automobile/automobile_make.rb +14 -0
  40. data/lib/earth/automobile/automobile_make/data_miner.rb +68 -0
  41. data/lib/earth/automobile/automobile_make_fleet_year.rb +15 -0
  42. data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +29 -0
  43. data/lib/earth/automobile/automobile_make_year.rb +14 -0
  44. data/lib/earth/automobile/automobile_make_year/data_miner.rb +45 -0
  45. data/lib/earth/automobile/automobile_model.rb +14 -0
  46. data/lib/earth/automobile/automobile_model/data_miner.rb +38 -0
  47. data/lib/earth/automobile/automobile_model_year.rb +15 -0
  48. data/lib/earth/automobile/automobile_model_year/data_miner.rb +51 -0
  49. data/lib/earth/automobile/automobile_size_class.rb +14 -0
  50. data/lib/earth/automobile/automobile_size_class/data_miner.rb +43 -0
  51. data/lib/earth/automobile/automobile_variant.rb +17 -0
  52. data/lib/earth/automobile/automobile_variant/data_miner.rb +460 -0
  53. data/lib/earth/automobile/data_miner.rb +8 -0
  54. data/lib/earth/base.rb +7 -0
  55. data/lib/earth/bus.rb +1 -0
  56. data/lib/earth/bus/bus_class.rb +19 -0
  57. data/lib/earth/bus/bus_class/data_miner.rb +41 -0
  58. data/lib/earth/bus/data_miner.rb +1 -0
  59. data/lib/earth/conversions_ext.rb +45 -0
  60. data/lib/earth/data_miner.rb +10 -0
  61. data/lib/earth/diet.rb +2 -0
  62. data/lib/earth/diet/data_miner.rb +2 -0
  63. data/lib/earth/diet/diet_class.rb +15 -0
  64. data/lib/earth/diet/diet_class/data_miner.rb +36 -0
  65. data/lib/earth/diet/food_group.rb +17 -0
  66. data/lib/earth/diet/food_group/data_miner.rb +26 -0
  67. data/lib/earth/fuel.rb +2 -0
  68. data/lib/earth/fuel/data_miner.rb +2 -0
  69. data/lib/earth/fuel/fuel_price.rb +13 -0
  70. data/lib/earth/fuel/fuel_price/data_miner.rb +20 -0
  71. data/lib/earth/fuel/fuel_type.rb +18 -0
  72. data/lib/earth/fuel/fuel_type/data_miner.rb +37 -0
  73. data/lib/earth/hospitality.rb +1 -0
  74. data/lib/earth/hospitality/data_miner.rb +1 -0
  75. data/lib/earth/hospitality/lodging_class.rb +9 -0
  76. data/lib/earth/hospitality/lodging_class/data_miner.rb +30 -0
  77. data/lib/earth/industry.rb +10 -0
  78. data/lib/earth/industry/industry.rb +23 -0
  79. data/lib/earth/industry/industry_product.rb +22 -0
  80. data/lib/earth/industry/industry_product_line.rb +20 -0
  81. data/lib/earth/industry/industry_sector.rb +20 -0
  82. data/lib/earth/industry/merchant.rb +18 -0
  83. data/lib/earth/industry/merchant_category.rb +22 -0
  84. data/lib/earth/industry/merchant_category_industry.rb +20 -0
  85. data/lib/earth/industry/product_line.rb +22 -0
  86. data/lib/earth/industry/product_line_industry_product.rb +20 -0
  87. data/lib/earth/industry/sector.rb +19 -0
  88. data/lib/earth/inflectors.rb +9 -0
  89. data/lib/earth/locality.rb +10 -0
  90. data/lib/earth/locality/census_division.rb +22 -0
  91. data/lib/earth/locality/census_division/data_miner.rb +64 -0
  92. data/lib/earth/locality/census_region.rb +13 -0
  93. data/lib/earth/locality/census_region/data_miner.rb +17 -0
  94. data/lib/earth/locality/climate_division.rb +17 -0
  95. data/lib/earth/locality/climate_division/data_miner.rb +20 -0
  96. data/lib/earth/locality/country.rb +13 -0
  97. data/lib/earth/locality/country/data_miner.rb +19 -0
  98. data/lib/earth/locality/data_miner.rb +10 -0
  99. data/lib/earth/locality/egrid_region.rb +15 -0
  100. data/lib/earth/locality/egrid_region/data_miner.rb +35 -0
  101. data/lib/earth/locality/egrid_subregion.rb +16 -0
  102. data/lib/earth/locality/egrid_subregion/data_miner.rb +65 -0
  103. data/lib/earth/locality/petroleum_administration_for_defense_district.rb +13 -0
  104. data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb +21 -0
  105. data/lib/earth/locality/state.rb +22 -0
  106. data/lib/earth/locality/state/data_miner.rb +37 -0
  107. data/lib/earth/locality/urbanity.rb +10 -0
  108. data/lib/earth/locality/urbanity/data_miner.rb +15 -0
  109. data/lib/earth/locality/zip_code.rb +23 -0
  110. data/lib/earth/locality/zip_code/data_miner.rb +43 -0
  111. data/lib/earth/pet.rb +4 -0
  112. data/lib/earth/pet/breed.rb +15 -0
  113. data/lib/earth/pet/breed/data_miner.rb +25 -0
  114. data/lib/earth/pet/breed_gender.rb +14 -0
  115. data/lib/earth/pet/breed_gender/data_miner.rb +21 -0
  116. data/lib/earth/pet/data_miner.rb +4 -0
  117. data/lib/earth/pet/gender.rb +10 -0
  118. data/lib/earth/pet/gender/data_miner.rb +13 -0
  119. data/lib/earth/pet/species.rb +40 -0
  120. data/lib/earth/pet/species/data_miner.rb +42 -0
  121. data/lib/earth/rail.rb +1 -0
  122. data/lib/earth/rail/data_miner.rb +1 -0
  123. data/lib/earth/rail/rail_class.rb +16 -0
  124. data/lib/earth/rail/rail_class/data_miner.rb +36 -0
  125. data/lib/earth/residence.rb +8 -0
  126. data/lib/earth/residence/air_conditioner_use.rb +13 -0
  127. data/lib/earth/residence/air_conditioner_use/data_miner.rb +22 -0
  128. data/lib/earth/residence/clothes_machine_use.rb +10 -0
  129. data/lib/earth/residence/clothes_machine_use/data_miner.rb +28 -0
  130. data/lib/earth/residence/data_miner.rb +8 -0
  131. data/lib/earth/residence/dishwasher_use.rb +10 -0
  132. data/lib/earth/residence/dishwasher_use/data_miner.rb +28 -0
  133. data/lib/earth/residence/residence_appliance.rb +16 -0
  134. data/lib/earth/residence/residence_appliance/data_miner.rb +20 -0
  135. data/lib/earth/residence/residence_class.rb +16 -0
  136. data/lib/earth/residence/residence_class/data_miner.rb +15 -0
  137. data/lib/earth/residence/residence_fuel_price.rb +18 -0
  138. data/lib/earth/residence/residence_fuel_price/data_miner.rb +200 -0
  139. data/lib/earth/residence/residence_fuel_type.rb +32 -0
  140. data/lib/earth/residence/residence_fuel_type/data_miner.rb +18 -0
  141. data/lib/earth/residence/residential_energy_consumption_survey_response.rb +39 -0
  142. data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +283 -0
  143. data/spec/lib/earth_spec.rb +25 -0
  144. data/spec/spec_helper.rb +11 -0
  145. data/vendor/geokit-rails/CHANGELOG.rdoc +46 -0
  146. data/vendor/geokit-rails/MIT-LICENSE +20 -0
  147. data/vendor/geokit-rails/README.markdown +561 -0
  148. data/vendor/geokit-rails/Rakefile +18 -0
  149. data/vendor/geokit-rails/about.yml +9 -0
  150. data/vendor/geokit-rails/assets/api_keys_template +61 -0
  151. data/vendor/geokit-rails/init.rb +1 -0
  152. data/vendor/geokit-rails/install.rb +14 -0
  153. data/vendor/geokit-rails/lib/geokit-rails.rb +24 -0
  154. data/vendor/geokit-rails/lib/geokit-rails/acts_as_mappable.rb +456 -0
  155. data/vendor/geokit-rails/lib/geokit-rails/adapters/abstract.rb +31 -0
  156. data/vendor/geokit-rails/lib/geokit-rails/adapters/mysql.rb +22 -0
  157. data/vendor/geokit-rails/lib/geokit-rails/adapters/postgresql.rb +22 -0
  158. data/vendor/geokit-rails/lib/geokit-rails/adapters/sqlserver.rb +43 -0
  159. data/vendor/geokit-rails/lib/geokit-rails/defaults.rb +22 -0
  160. data/vendor/geokit-rails/lib/geokit-rails/geocoder_control.rb +16 -0
  161. data/vendor/geokit-rails/lib/geokit-rails/ip_geocode_lookup.rb +46 -0
  162. data/vendor/geokit-rails/test/acts_as_mappable_test.rb +474 -0
  163. data/vendor/geokit-rails/test/boot.rb +25 -0
  164. data/vendor/geokit-rails/test/database.yml +20 -0
  165. data/vendor/geokit-rails/test/fixtures/companies.yml +7 -0
  166. data/vendor/geokit-rails/test/fixtures/custom_locations.yml +54 -0
  167. data/vendor/geokit-rails/test/fixtures/locations.yml +54 -0
  168. data/vendor/geokit-rails/test/fixtures/mock_addresses.yml +17 -0
  169. data/vendor/geokit-rails/test/fixtures/mock_families.yml +2 -0
  170. data/vendor/geokit-rails/test/fixtures/mock_houses.yml +9 -0
  171. data/vendor/geokit-rails/test/fixtures/mock_organizations.yml +5 -0
  172. data/vendor/geokit-rails/test/fixtures/mock_people.yml +5 -0
  173. data/vendor/geokit-rails/test/fixtures/stores.yml +0 -0
  174. data/vendor/geokit-rails/test/ip_geocode_lookup_test.rb +77 -0
  175. data/vendor/geokit-rails/test/models/company.rb +3 -0
  176. data/vendor/geokit-rails/test/models/custom_location.rb +12 -0
  177. data/vendor/geokit-rails/test/models/location.rb +4 -0
  178. data/vendor/geokit-rails/test/models/mock_address.rb +4 -0
  179. data/vendor/geokit-rails/test/models/mock_family.rb +3 -0
  180. data/vendor/geokit-rails/test/models/mock_house.rb +3 -0
  181. data/vendor/geokit-rails/test/models/mock_organization.rb +4 -0
  182. data/vendor/geokit-rails/test/models/mock_person.rb +4 -0
  183. data/vendor/geokit-rails/test/models/store.rb +3 -0
  184. data/vendor/geokit-rails/test/schema.rb +60 -0
  185. data/vendor/geokit-rails/test/tasks.rake +31 -0
  186. data/vendor/geokit-rails/test/test_helper.rb +23 -0
  187. metadata +476 -0
@@ -0,0 +1,17 @@
1
+ class AutomobileVariant < ActiveRecord::Base
2
+ set_primary_key :row_hash
3
+
4
+ has_many :automobiles, :foreign_key => 'variant_id'
5
+ belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
6
+ belongs_to :model, :class_name => 'AutomobileModel', :foreign_key => 'model_name'
7
+ belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'model_year_name'
8
+ belongs_to :fuel_type, :class_name => 'AutomobileFuelType', :foreign_key => 'fuel_type_code'
9
+
10
+ data_miner do
11
+ tap "Brighter Planet's sanitized automobile variant data", Earth.taps_server
12
+
13
+ process "pull dependencies" do
14
+ run_data_miner_on_belongs_to_associations
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,460 @@
1
+ AutomobileVariant.class_eval do
2
+ class << self
3
+ def transmission_is_blank?(row)
4
+ row['transmission'].blank?
5
+ end
6
+
7
+ def is_a_2007_gmc_or_chevrolet?(row)
8
+ row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
9
+ end
10
+
11
+ def is_a_porsche?(row)
12
+ row['make'].upcase == 'PORSCHE'
13
+ end
14
+
15
+ def is_not_a_porsche?(row)
16
+ !is_a_porsche? row
17
+ end
18
+
19
+ def is_a_mercedes_benz?(row)
20
+ row['make'] =~ /MERCEDES/i
21
+ end
22
+
23
+ def is_a_lexus?(row)
24
+ row['make'].upcase == 'LEXUS'
25
+ end
26
+
27
+ def is_a_bmw?(row)
28
+ row['make'].upcase == 'BMW'
29
+ end
30
+
31
+ def is_a_ford?(row)
32
+ row['make'].upcase == 'FORD'
33
+ end
34
+
35
+ def is_a_rolls_royce_and_model_contains_bentley?(row)
36
+ is_a_rolls_royce?(row) and model_contains_bentley?(row)
37
+ end
38
+
39
+ def is_a_bentley?(row)
40
+ row['make'].upcase == 'BENTLEY'
41
+ end
42
+
43
+ def is_a_rolls_royce?(row)
44
+ row['make'] =~ /ROLLS/i
45
+ end
46
+
47
+ def is_a_turbo_brooklands?(row)
48
+ row['model'] =~ /TURBO R\/RL BKLDS/i
49
+ end
50
+
51
+ def model_contains_maybach?(row)
52
+ row['model'] =~ /MAYBACH/i
53
+ end
54
+
55
+ def model_contains_bentley?(row)
56
+ row['model'] =~ /BENTLEY/i
57
+ end
58
+
59
+ def source_code
60
+ IO.read __FILE__
61
+ end
62
+ end
63
+
64
+ # updated with 2010 names
65
+ TRANSMISSIONS = {
66
+ 'A' => 'Automatic', # prefix
67
+ 'M' => 'Manual', # prefix
68
+ 'L' => 'Automatic', # prefix
69
+ 'S' => 'Semi-Automatic', # prefix
70
+ 'AM' => 'Automated Manual',
71
+ 'C' => 'Continuously Variable',
72
+ 'SA' => 'Semi-Automatic',
73
+ 'CVT' => 'Continuously Variable',
74
+ 'OT' => 'Other'
75
+ }
76
+
77
+ ENGINE_TYPES = {
78
+ '(GUZZLER)' => nil, # "gas guzzler"
79
+ '(POLICE)' => nil, # police automobile_variant
80
+ '(MPFI)' => 'injection',
81
+ '(MPI*)' => 'injection',
82
+ '(SPFI)' => 'injection',
83
+ '(FFS)' => 'injection',
84
+ '(TURBO)' => 'turbo',
85
+ '(TRBO)' => 'turbo',
86
+ '(TC*)' => 'turbo',
87
+ '(FFS,TRBO)' => %w(injection turbo),
88
+ '(S-CHARGE)' => 'supercharger',
89
+ '(SC*)' => 'supercharger',
90
+ '(DIESEL)' => nil, # diesel
91
+ '(DSL)' => nil, # diesel
92
+ '(ROTARY)' => nil, # rotary
93
+ '(VARIABLE)' => nil, # variable displacement
94
+ '(NO-CAT)' => nil, # no catalytic converter
95
+ '(OHC)' => nil, # overhead camshaft
96
+ '(OHV)' => nil, # overhead valves
97
+ '(16-VALVE)' => nil, # 16V
98
+ '(305)' => nil, # 305 cubic inch displacement
99
+ '(307)' => nil, # 307 cubic inch displacement
100
+ '(M-ENG)' => nil,
101
+ '(W-ENG)' => nil,
102
+ '(GM-BUICK)' => nil,
103
+ '(GM-CHEV)' => nil,
104
+ '(GM-OLDS)' => nil,
105
+ '(GM-PONT)' => nil,
106
+ }
107
+
108
+ class AutomobileVariant::ParserB
109
+ attr_accessor :year
110
+ def initialize(options = {})
111
+ @year = options[:year]
112
+ end
113
+
114
+ def apply(row)
115
+ row.merge!({
116
+ 'make' => row['carline_mfr_name'], # make it line up with the errata
117
+ 'model' => row['carline_name'], # ditto
118
+ 'transmission' => TRANSMISSIONS[row['model_trans'][0, 1]],
119
+ 'speeds' => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
120
+ 'turbo' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
121
+ 'supercharger' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
122
+ 'injection' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
123
+ 'displacement' => _displacement(row['opt_disp']),
124
+ 'year' => year
125
+ })
126
+ row
127
+ end
128
+
129
+ def _displacement(str)
130
+ str = str.gsub(/[\(\)]/, '').strip
131
+ if str =~ /^(.+)L$/
132
+ $1.to_f
133
+ elsif str =~ /^(.+)CC$/
134
+ $1.to_f / 1000
135
+ end
136
+ end
137
+
138
+ def add_hints!(bus)
139
+ bus[:format] = :fixed_width
140
+ bus[:cut] = '13-' if year == 1995
141
+ bus[:schema_name] = :fuel_economy_guide_b
142
+ bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
143
+ Slither.define :fuel_economy_guide_b do |d|
144
+ d.rows do |row|
145
+ row.trap { true } # there's only one section
146
+ row.column 'active_year' , 4, :type => :integer # ACTIVE YEAR
147
+ row.column 'state_code' , 1, :type => :string # STATE CODE: F=49-STATE,C=CALIFORNIA
148
+ row.column 'carline_clss' , 2, :type => :integer # CARLINE CLASS CODE
149
+ row.column 'carline_mfr_code' , 3, :type => :integer # CARLINE MANUFACTURER CODE
150
+ row.column 'carline_name' , 28, :type => :string # CARLINE NAME
151
+ row.column 'disp_cub_in' , 4, :type => :integer # DISP CUBIC INCHES
152
+ row.column 'fuel_system' , 2, :type => :string # FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
153
+ row.column 'model_trans' , 6, :type => :string # TRANSMISSION TYPE
154
+ row.column 'no_cyc' , 2, :type => :integer # NUMBER OF ENGINE CYLINDERS
155
+ row.column 'date_time' , 12, :type => :string # DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
156
+ row.column 'release_date' , 6, :type => :string # RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
157
+ row.column 'vi_mfr_code' , 3, :type => :integer # VI MANUFACTURER CODE
158
+ row.column 'carline_code' , 5, :type => :integer # CARLINE CODE
159
+ row.column 'basic_eng_id' , 5, :type => :integer # BASIC ENGINE INDEX
160
+ row.column 'carline_mfr_name' , 32, :type => :string # CARLINE MANUFACTURER NAME
161
+ row.column 'suppress_code' , 1, :type => :integer # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
162
+ row.column 'est_city_mpg' , 3, :type => :integer # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
163
+ row.spacer 2
164
+ row.column 'highway_mpg' , 3, :type => :integer # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
165
+ row.spacer 2
166
+ row.column 'combined_mpg' , 3, :type => :integer # COMBINED MILES PER GALLON
167
+ row.spacer 2
168
+ row.column 'unadj_city_mpg' , 3, :type => :integer # UNADJUSTED CITY MILES PER GALLON
169
+ row.spacer 2
170
+ row.column 'unadj_hwy_mpg' , 3, :type => :integer # UNADJUSTED HIGHWAY MILES PER GALLON
171
+ row.spacer 2
172
+ row.column 'unadj_comb_mpg' , 3, :type => :integer # UNADJUSTED COMBINED MILES PER GALLON
173
+ row.spacer 2
174
+ row.column 'ave_anl_fuel' , 6, :type => :integer # "$" in col 147, Annual Fuel Cost starting col 148 in I5
175
+ row.column 'opt_disp' , 8, :type => :string # OPTIONAL DISPLACEMENT
176
+ row.column 'engine_desc1' , 10, :type => :string # ENGINE DESCRIPTION 1
177
+ row.column 'engine_desc2' , 10, :type => :string # ENGINE DESCRIPTION 2
178
+ row.column 'engine_desc3' , 10, :type => :string # ENGINE DESCRIPTION 3
179
+ row.column 'body_type_2d' , 10, :type => :string # BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
180
+ row.column 'body_type_4d' , 10, :type => :string # BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
181
+ row.column 'body_type_hbk' , 10, :type => :string # BODY TYPE HBK - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
182
+ row.column 'puerto_rico' , 1, :type => :string # '*' IF FOR PUERTO RICO SALES ONLY
183
+ row.column 'overdrive' , 4, :type => :string # OVERDRIVE: ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
184
+ row.column 'drive_system' , 3, :type => :string # FWD=FRONT WHEEL DRIVE, RWD=REAR, 4WD=4-WHEEL
185
+ row.column 'filler' , 1, :type => :string # NOT USED
186
+ row.column 'fuel_type' , 1, :type => :string # R=REGULAR(UNLEADED), P=PREMIUM, D=DIESEL
187
+ row.column 'trans_desc' , 15, :type => :string # TRANSMISSION DESCRIPTORS
188
+ end
189
+ end
190
+ end
191
+ end
192
+ class AutomobileVariant::ParserC
193
+ attr_accessor :year
194
+ def initialize(options = {})
195
+ @year = options[:year]
196
+ end
197
+
198
+ def add_hints!(bus)
199
+ # File will decide format based on filename
200
+ end
201
+
202
+ def apply(row)
203
+ row.merge!({
204
+ 'make' => row['Manufacturer'], # make it line up with the errata
205
+ 'model' => row['carline name'], # ditto
206
+ 'drive' => row['drv'] + 'WD',
207
+ 'transmission' => TRANSMISSIONS[row['trans'][-3, 1]], # only using prefix, probably a FIXME
208
+ 'speeds' => (row['trans'][-2, 1] == 'V') ? '1' : row['trans'][-2, 1],
209
+ 'turbo' => row['T'] == 'T',
210
+ 'supercharger' => row['S'] == 'S',
211
+ 'injection' => true,
212
+ 'year' => year
213
+ })
214
+ row
215
+ end
216
+ end
217
+ class AutomobileVariant::ParserD
218
+ attr_accessor :year
219
+ def initialize(options = {})
220
+ @year = options[:year]
221
+ end
222
+
223
+ def add_hints!(bus)
224
+ end
225
+
226
+ def apply(row)
227
+ row.merge!({
228
+ 'make' => row['MFR'], # make it line up with the errata
229
+ 'model' => row['CAR LINE'], # ditto
230
+ 'drive' => row['DRIVE SYS'] + 'WD',
231
+ 'transmission' => TRANSMISSIONS[row['TRANS'][-3, 1]], # only using prefix, probably a FIXME
232
+ 'speeds' => (row['TRANS'][-2, 1] == 'V') ? '1' : row['TRANS'][-2, 1],
233
+ 'turbo' => row['TURBO'] == 'T',
234
+ 'supercharger' => row['SPCHGR'] == 'S',
235
+ 'injection' => true,
236
+ 'year' => year
237
+ })
238
+ row
239
+ end
240
+ end
241
+ class AutomobileVariant::ParserE
242
+ OLD_FUEL_CODES = {
243
+ 'CNG' => 'C',
244
+ 'DU' => 'D',
245
+ 'G' => 'R',
246
+ 'GP' => 'P',
247
+ 'GPR' => 'P'
248
+ }
249
+
250
+ attr_accessor :year
251
+ def initialize(options = {})
252
+ @year = options[:year]
253
+ end
254
+
255
+ def add_hints!(bus)
256
+ end
257
+
258
+ def apply(row)
259
+ row.merge!({
260
+ 'make' => row['Division'], # make it line up with the errata
261
+ 'model' => row['Carline'], # ditto
262
+ 'drive' => row['Drive Sys'] + 'WD',
263
+ 'transmission' => TRANSMISSIONS[row['Trans']],
264
+ 'speeds' => row['# Gears'],
265
+ 'turbo' => row['Air Aspir Method'] == 'TC',
266
+ 'supercharger' => row['Air Aspir Method'] == 'SC',
267
+ 'injection' => true,
268
+ 'year' => year,
269
+ 'fuel_type_code' => OLD_FUEL_CODES[row['Fuel Usage - Conventional Fuel']]
270
+ })
271
+ row
272
+ end
273
+ end
274
+
275
+ data_miner do
276
+ schema Earth.database_options do
277
+ string 'row_hash'
278
+ string 'name' # short name!
279
+ string 'make_name'
280
+ string 'model_name' # make + model
281
+ string 'make_year_name' # make + year
282
+ string 'model_year_name' # make + model + year
283
+ integer 'year'
284
+ float 'fuel_efficiency_city'
285
+ string 'fuel_efficiency_city_units'
286
+ float 'fuel_efficiency_highway'
287
+ string 'fuel_efficiency_highway_units'
288
+ string 'fuel_type_code'
289
+ string 'transmission'
290
+ string 'drive'
291
+ boolean 'turbo'
292
+ boolean 'supercharger'
293
+ integer 'cylinders'
294
+ float 'displacement'
295
+ float 'raw_fuel_efficiency_city'
296
+ string 'raw_fuel_efficiency_city_units'
297
+ float 'raw_fuel_efficiency_highway'
298
+ string 'raw_fuel_efficiency_highway_units'
299
+ integer 'carline_mfr_code'
300
+ integer 'vi_mfr_code'
301
+ integer 'carline_code'
302
+ integer 'carline_class_code'
303
+ boolean 'injection'
304
+ string 'carline_class_name'
305
+ string 'speeds'
306
+ index 'make_name'
307
+ index 'model_name'
308
+ index 'make_year_name'
309
+ index 'model_year_name'
310
+ end
311
+
312
+ # # 1985---1997
313
+ (85..97).each do |yy|
314
+ filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
315
+ import("19#{ yy } Fuel Economy Guide",
316
+ :url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
317
+ :filename => filename,
318
+ :transform => { :class => AutomobileVariant::ParserB, :year => "19#{yy}".to_i },
319
+ :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
320
+ key 'row_hash'
321
+ store 'name', :field_name => 'model'
322
+ store 'make_name', :field_name => 'make'
323
+ store 'year'
324
+ store 'fuel_type_code', :field_name => 'fuel_type'
325
+ store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
326
+ store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
327
+ store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
328
+ store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
329
+ store 'cylinders', :field_name => 'no_cyc'
330
+ store 'drive', :field_name => 'drive_system'
331
+ store 'carline_mfr_code'
332
+ store 'vi_mfr_code'
333
+ store 'carline_code'
334
+ store 'carline_class_code', :field_name => 'carline_clss'
335
+ store 'transmission'
336
+ store 'speeds'
337
+ store 'turbo'
338
+ store 'supercharger'
339
+ store 'injection'
340
+ store 'displacement'
341
+ end
342
+ end
343
+
344
+ # 1998--2005
345
+ {
346
+ 1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
347
+ 1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
348
+ 2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
349
+ 2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
350
+ 2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
351
+ 2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
352
+ 2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
353
+ 2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
354
+ }.sort { |a, b| a.first <=> b.first }.each do |year, options|
355
+ import "#{ year } Fuel Economy Guide",
356
+ options.merge(:transform => { :class => AutomobileVariant::ParserC, :year => year },
357
+ :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
358
+ key 'row_hash'
359
+ store 'name', :field_name => 'model'
360
+ store 'make_name', :field_name => 'make'
361
+ store 'fuel_type_code', :field_name => 'fl'
362
+ store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
363
+ store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
364
+ store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
365
+ store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
366
+ store 'cylinders', :field_name => 'cyl'
367
+ store 'displacement', :field_name => 'displ'
368
+ store 'carline_class_code', :field_name => 'cls' if year >= 2000
369
+ store 'carline_class_name', :field_name => 'Class'
370
+ store 'year'
371
+ store 'transmission'
372
+ store 'speeds'
373
+ store 'turbo'
374
+ store 'supercharger'
375
+ store 'injection'
376
+ store 'drive'
377
+ end
378
+ end
379
+
380
+ # 2006--2009
381
+ {
382
+ 2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
383
+ 2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
384
+ 2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
385
+ 2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
386
+ }.sort { |a, b| a.first <=> b.first }.each do |year, options|
387
+ import "#{ year } Fuel Economy Guide",
388
+ options.merge(:transform => { :class => AutomobileVariant::ParserD, :year => year },
389
+ :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
390
+ key 'row_hash'
391
+ store 'name', :field_name => 'model'
392
+ store 'make_name', :field_name => 'make'
393
+ store 'fuel_type_code', :field_name => 'FUEL TYPE'
394
+ store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
395
+ store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
396
+ store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
397
+ store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
398
+ store 'cylinders', :field_name => 'NUMB CYL'
399
+ store 'displacement', :field_name => 'DISPLACEMENT'
400
+ store 'carline_class_code', :field_name => 'CLS'
401
+ store 'carline_class_name', :field_name => 'CLASS'
402
+ store 'year'
403
+ store 'transmission'
404
+ store 'speeds'
405
+ store 'turbo'
406
+ store 'supercharger'
407
+ store 'injection'
408
+ store 'drive'
409
+ end
410
+ end
411
+
412
+ # 2010--?
413
+ # sabshere 5/17/10 apparently needs update
414
+ # {
415
+ # 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
416
+ # }.sort { |a, b| a.first <=> b.first }.each do |year, options|
417
+ # import "#{ year } Fuel Economy Guide",
418
+ # options.merge(:transform => { :class => AutomobileVariant::ParserE, :year => year },
419
+ # :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
420
+ # key 'row_hash'
421
+ # store 'name', :field_name => 'model'
422
+ # store 'make_name', :field_name => 'make'
423
+ # store 'fuel_type_code'
424
+ # store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
425
+ # store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
426
+ # store 'raw_fuel_efficiency_highway', :field_name => 'Hwy Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
427
+ # store 'raw_fuel_efficiency_city', :field_name => 'City Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
428
+ # store 'cylinders', :field_name => '# Cyl'
429
+ # store 'displacement', :field_name => 'Eng Displ'
430
+ # store 'carline_class_code', :field_name => 'Carline Class'
431
+ # store 'carline_class_name', :field_name => 'Carline Class Desc'
432
+ # store 'year'
433
+ # store 'transmission'
434
+ # store 'speeds'
435
+ # store 'turbo'
436
+ # store 'supercharger'
437
+ # store 'injection'
438
+ # store 'drive'
439
+ # end
440
+ # end
441
+
442
+ process "Derive model and model year names" do
443
+ update_all "model_name = CONCAT(make_name, ' ', name)"
444
+ update_all "make_year_name = CONCAT(make_name, ' ', year)"
445
+ update_all "model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
446
+ end
447
+
448
+ process "Calculate adjusted fuel efficiency using the latest EPA equations" do
449
+ update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
450
+ update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
451
+ end
452
+
453
+ %w{ AutomobileMake AutomobileModelYear AutomobileModel }.each do |synthetic_resource|
454
+ process "Synthesize #{synthetic_resource}" do
455
+ synthetic_resource.constantize.run_data_miner!
456
+ end
457
+ end
458
+ end
459
+ end
460
+