data_miner 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/data_miner/attribute.rb +2 -1
- data/test/data_miner/attribute_test.rb +55 -0
- data/test/data_miner_test.rb +13 -539
- data/test/support/aircraft.rb +99 -0
- data/test/support/airport.rb +14 -0
- data/test/support/automobile_fuel_type.rb +40 -0
- data/test/support/automobile_variant.rb +368 -0
- data/test/support/country.rb +15 -0
- data/test/test_helper.rb +6 -3
- metadata +15 -3
data/lib/data_miner/attribute.rb
CHANGED
|
@@ -84,6 +84,7 @@ module DataMiner
|
|
|
84
84
|
return value if value.is_a? ActiveRecord::Base # carry through trapdoor
|
|
85
85
|
value = value_in_dictionary value if wants_dictionary?
|
|
86
86
|
value = synthesize.call(row) if wants_synthesize?
|
|
87
|
+
value = nil if value.blank? and wants_nullification?
|
|
87
88
|
value
|
|
88
89
|
end
|
|
89
90
|
|
|
@@ -150,7 +151,7 @@ module DataMiner
|
|
|
150
151
|
options.has_key? :static
|
|
151
152
|
end
|
|
152
153
|
def wants_nullification?
|
|
153
|
-
nullify
|
|
154
|
+
nullify == true
|
|
154
155
|
end
|
|
155
156
|
def wants_chars?
|
|
156
157
|
chars.present?
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
|
|
3
|
+
class AttributeTest < Test::Unit::TestCase
|
|
4
|
+
context '#value_from_row' do
|
|
5
|
+
setup do
|
|
6
|
+
@airport = Airport.new
|
|
7
|
+
end
|
|
8
|
+
context 'nullify is true' do
|
|
9
|
+
setup do
|
|
10
|
+
@attribute = DataMiner::Attribute.new @airport, 'latitude', :nullify => true
|
|
11
|
+
end
|
|
12
|
+
should 'return nil if field is blank' do
|
|
13
|
+
assert_nil @attribute.value_from_row(
|
|
14
|
+
'name' => 'DTW',
|
|
15
|
+
'city' => 'Warren',
|
|
16
|
+
'country_name' => 'US',
|
|
17
|
+
'latitude' => '',
|
|
18
|
+
'longitude' => ''
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
should 'return the value if field is not blank' do
|
|
22
|
+
assert_equal '12.34', @attribute.value_from_row(
|
|
23
|
+
'name' => 'DTW',
|
|
24
|
+
'city' => 'Warren',
|
|
25
|
+
'country_name' => 'US',
|
|
26
|
+
'latitude' => '12.34',
|
|
27
|
+
'longitude' => ''
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
context 'nullify is false' do
|
|
32
|
+
setup do
|
|
33
|
+
@attribute = DataMiner::Attribute.new @airport, 'latitude'
|
|
34
|
+
end
|
|
35
|
+
should 'return the value if field is not blank' do
|
|
36
|
+
assert_equal '12.34', @attribute.value_from_row(
|
|
37
|
+
'name' => 'DTW',
|
|
38
|
+
'city' => 'Warren',
|
|
39
|
+
'country_name' => 'US',
|
|
40
|
+
'latitude' => '12.34',
|
|
41
|
+
'longitude' => ''
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
should 'return blank if field is blank' do
|
|
45
|
+
assert_equal '', @attribute.value_from_row(
|
|
46
|
+
'name' => 'DTW',
|
|
47
|
+
'city' => 'Warren',
|
|
48
|
+
'country_name' => 'US',
|
|
49
|
+
'latitude' => '',
|
|
50
|
+
'longitude' => ''
|
|
51
|
+
)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
data/test/data_miner_test.rb
CHANGED
|
@@ -1,446 +1,5 @@
|
|
|
1
1
|
require 'test_helper'
|
|
2
2
|
|
|
3
|
-
class AutomobileFuelType < ActiveRecord::Base
|
|
4
|
-
set_primary_key :code
|
|
5
|
-
|
|
6
|
-
data_miner do
|
|
7
|
-
import(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
|
8
|
-
:filename => 'Gd6-dsc.txt',
|
|
9
|
-
:format => :fixed_width,
|
|
10
|
-
:crop => 21..26, # inclusive
|
|
11
|
-
:cut => '2-',
|
|
12
|
-
:select => lambda { |row| /\A[A-Z]/.match row[:code] },
|
|
13
|
-
:schema => [[ 'code', 2, { :type => :string } ],
|
|
14
|
-
[ 'spacer', 2 ],
|
|
15
|
-
[ 'name', 52, { :type => :string } ]]) do
|
|
16
|
-
key 'code'
|
|
17
|
-
store 'name'
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
import :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_fuel_type.csv' do
|
|
21
|
-
key 'code'
|
|
22
|
-
store 'name'
|
|
23
|
-
store 'annual_distance'
|
|
24
|
-
store 'emission_factor'
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
# pull electricity emission factor from residential electricity
|
|
28
|
-
import(:url => 'http://spreadsheets.google.com/pub?key=rukxnmuhhsOsrztTrUaFCXQ',
|
|
29
|
-
:select => lambda { |row| row['code'] == 'El' }) do
|
|
30
|
-
key 'code'
|
|
31
|
-
store 'name'
|
|
32
|
-
store 'emission_factor'
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
# still need distance estimate for electric cars
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
CODES = {
|
|
39
|
-
:electricity => 'El',
|
|
40
|
-
:diesel => 'D'
|
|
41
|
-
}
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
class AutomobileVariant < ActiveRecord::Base
|
|
45
|
-
set_primary_key :row_hash
|
|
46
|
-
|
|
47
|
-
module FuelEconomyGuide
|
|
48
|
-
TRANSMISSIONS = {
|
|
49
|
-
'A' => 'automatic',
|
|
50
|
-
'M' => 'manual',
|
|
51
|
-
'L' => 'automatic', # Lockup/automatic
|
|
52
|
-
'S' => 'semiautomatic', # Semiautomatic
|
|
53
|
-
'C' => 'manual' # TODO verify for VW Syncro
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
ENGINE_TYPES = {
|
|
57
|
-
'(GUZZLER)' => nil, # "gas guzzler"
|
|
58
|
-
'(POLICE)' => nil, # police automobile_variant
|
|
59
|
-
'(MPFI)' => 'injection',
|
|
60
|
-
'(MPI*)' => 'injection',
|
|
61
|
-
'(SPFI)' => 'injection',
|
|
62
|
-
'(FFS)' => 'injection',
|
|
63
|
-
'(TURBO)' => 'turbo',
|
|
64
|
-
'(TRBO)' => 'turbo',
|
|
65
|
-
'(TC*)' => 'turbo',
|
|
66
|
-
'(FFS,TRBO)' => %w(injection turbo),
|
|
67
|
-
'(S-CHARGE)' => 'supercharger',
|
|
68
|
-
'(SC*)' => 'supercharger',
|
|
69
|
-
'(DIESEL)' => nil, # diesel
|
|
70
|
-
'(DSL)' => nil, # diesel
|
|
71
|
-
'(ROTARY)' => nil, # rotary
|
|
72
|
-
'(VARIABLE)' => nil, # variable displacement
|
|
73
|
-
'(NO-CAT)' => nil, # no catalytic converter
|
|
74
|
-
'(OHC)' => nil, # overhead camshaft
|
|
75
|
-
'(OHV)' => nil, # overhead valves
|
|
76
|
-
'(16-VALVE)' => nil, # 16V
|
|
77
|
-
'(305)' => nil, # 305 cubic inch displacement
|
|
78
|
-
'(307)' => nil, # 307 cubic inch displacement
|
|
79
|
-
'(M-ENG)' => nil,
|
|
80
|
-
'(W-ENG)' => nil,
|
|
81
|
-
'(GM-BUICK)' => nil,
|
|
82
|
-
'(GM-CHEV)' => nil,
|
|
83
|
-
'(GM-OLDS)' => nil,
|
|
84
|
-
'(GM-PONT)' => nil,
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
class ParserB
|
|
88
|
-
attr_accessor :year
|
|
89
|
-
def initialize(options = {})
|
|
90
|
-
@year = options[:year]
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def apply(row)
|
|
94
|
-
row.merge!({
|
|
95
|
-
'make' => row['carline_mfr_name'], # make it line up with the errata
|
|
96
|
-
'model' => row['carline_name'], # ditto
|
|
97
|
-
'transmission' => TRANSMISSIONS[row['model_trans'][0, 1]],
|
|
98
|
-
'speeds' => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
|
|
99
|
-
'turbo' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
|
|
100
|
-
'supercharger' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
|
|
101
|
-
'injection' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
|
|
102
|
-
'displacement' => _displacement(row['opt_disp']),
|
|
103
|
-
'year' => year
|
|
104
|
-
})
|
|
105
|
-
row
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def _displacement(str)
|
|
109
|
-
str = str.gsub(/[\(\)]/, '').strip
|
|
110
|
-
if str =~ /^(.+)L$/
|
|
111
|
-
$1.to_f
|
|
112
|
-
elsif str =~ /^(.+)CC$/
|
|
113
|
-
$1.to_f / 1000
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def add_hints!(bus)
|
|
118
|
-
bus[:format] = :fixed_width
|
|
119
|
-
bus[:cut] = '13-' if year == 1995
|
|
120
|
-
bus[:schema_name] = :fuel_economy_guide_b
|
|
121
|
-
bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
|
|
122
|
-
Slither.define :fuel_economy_guide_b do |d|
|
|
123
|
-
d.rows do |row|
|
|
124
|
-
row.trap { true } # there's only one section
|
|
125
|
-
row.column 'active_year' , 4, :type => :integer # ACTIVE YEAR
|
|
126
|
-
row.column 'state_code' , 1, :type => :string # STATE CODE: F=49-STATE,C=CALIFORNIA
|
|
127
|
-
row.column 'carline_clss' , 2, :type => :integer # CARLINE CLASS CODE
|
|
128
|
-
row.column 'carline_mfr_code' , 3, :type => :integer # CARLINE MANUFACTURER CODE
|
|
129
|
-
row.column 'carline_name' , 28, :type => :string # CARLINE NAME
|
|
130
|
-
row.column 'disp_cub_in' , 4, :type => :integer # DISP CUBIC INCHES
|
|
131
|
-
row.column 'fuel_system' , 2, :type => :string # FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
|
|
132
|
-
row.column 'model_trans' , 6, :type => :string # TRANSMISSION TYPE
|
|
133
|
-
row.column 'no_cyc' , 2, :type => :integer # NUMBER OF ENGINE CYLINDERS
|
|
134
|
-
row.column 'date_time' , 12, :type => :string # DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
|
|
135
|
-
row.column 'release_date' , 6, :type => :string # RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
|
|
136
|
-
row.column 'vi_mfr_code' , 3, :type => :integer # VI MANUFACTURER CODE
|
|
137
|
-
row.column 'carline_code' , 5, :type => :integer # CARLINE CODE
|
|
138
|
-
row.column 'basic_eng_id' , 5, :type => :integer # BASIC ENGINE INDEX
|
|
139
|
-
row.column 'carline_mfr_name' , 32, :type => :string # CARLINE MANUFACTURER NAME
|
|
140
|
-
row.column 'suppress_code' , 1, :type => :integer # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
|
|
141
|
-
row.column 'est_city_mpg' , 3, :type => :integer # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
|
|
142
|
-
row.spacer 2
|
|
143
|
-
row.column 'highway_mpg' , 3, :type => :integer # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
|
|
144
|
-
row.spacer 2
|
|
145
|
-
row.column 'combined_mpg' , 3, :type => :integer # COMBINED MILES PER GALLON
|
|
146
|
-
row.spacer 2
|
|
147
|
-
row.column 'unadj_city_mpg' , 3, :type => :integer # UNADJUSTED CITY MILES PER GALLON
|
|
148
|
-
row.spacer 2
|
|
149
|
-
row.column 'unadj_hwy_mpg' , 3, :type => :integer # UNADJUSTED HIGHWAY MILES PER GALLON
|
|
150
|
-
row.spacer 2
|
|
151
|
-
row.column 'unadj_comb_mpg' , 3, :type => :integer # UNADJUSTED COMBINED MILES PER GALLON
|
|
152
|
-
row.spacer 2
|
|
153
|
-
row.column 'ave_anl_fuel' , 6, :type => :integer # "$" in col 147, Annual Fuel Cost starting col 148 in I5
|
|
154
|
-
row.column 'opt_disp' , 8, :type => :string # OPTIONAL DISPLACEMENT
|
|
155
|
-
row.column 'engine_desc1' , 10, :type => :string # ENGINE DESCRIPTION 1
|
|
156
|
-
row.column 'engine_desc2' , 10, :type => :string # ENGINE DESCRIPTION 2
|
|
157
|
-
row.column 'engine_desc3' , 10, :type => :string # ENGINE DESCRIPTION 3
|
|
158
|
-
row.column 'body_type_2d' , 10, :type => :string # BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
159
|
-
row.column 'body_type_4d' , 10, :type => :string # BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
160
|
-
row.column 'body_type_hbk' , 10, :type => :string # BODY TYPE HBK - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
161
|
-
row.column 'puerto_rico' , 1, :type => :string # '*' IF FOR PUERTO RICO SALES ONLY
|
|
162
|
-
row.column 'overdrive' , 4, :type => :string # OVERDRIVE: ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
|
|
163
|
-
row.column 'drive_system' , 3, :type => :string # FWD=FRONT WHEEL DRIVE, RWD=REAR, 4WD=4-WHEEL
|
|
164
|
-
row.column 'filler' , 1, :type => :string # NOT USED
|
|
165
|
-
row.column 'fuel_type' , 1, :type => :string # R=REGULAR(UNLEADED), P=PREMIUM, D=DIESEL
|
|
166
|
-
row.column 'trans_desc' , 15, :type => :string # TRANSMISSION DESCRIPTORS
|
|
167
|
-
end
|
|
168
|
-
end
|
|
169
|
-
end
|
|
170
|
-
end
|
|
171
|
-
class ParserC
|
|
172
|
-
attr_accessor :year
|
|
173
|
-
def initialize(options = {})
|
|
174
|
-
@year = options[:year]
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
def add_hints!(bus)
|
|
178
|
-
# File will decide format based on filename
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def apply(row)
|
|
182
|
-
row.merge!({
|
|
183
|
-
'make' => row['Manufacturer'], # make it line up with the errata
|
|
184
|
-
'model' => row['carline name'], # ditto
|
|
185
|
-
'drive' => row['drv'] + 'WD',
|
|
186
|
-
'transmission' => TRANSMISSIONS[row['trans'][-3, 1]],
|
|
187
|
-
'speeds' => (row['trans'][-2, 1] == 'V') ? 'variable' : row['trans'][-2, 1],
|
|
188
|
-
'turbo' => row['T'] == 'T',
|
|
189
|
-
'supercharger' => row['S'] == 'S',
|
|
190
|
-
'injection' => true,
|
|
191
|
-
'year' => year
|
|
192
|
-
})
|
|
193
|
-
row
|
|
194
|
-
end
|
|
195
|
-
end
|
|
196
|
-
class ParserD
|
|
197
|
-
attr_accessor :year
|
|
198
|
-
def initialize(options = {})
|
|
199
|
-
@year = options[:year]
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
def add_hints!(bus)
|
|
203
|
-
bus[:reject] = lambda { |row| row.values.first.blank? } if year == 2007
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
def apply(row)
|
|
207
|
-
row.merge!({
|
|
208
|
-
'make' => row['MFR'], # make it line up with the errata
|
|
209
|
-
'model' => row['CAR LINE'], # ditto
|
|
210
|
-
'drive' => row['DRIVE SYS'] + 'WD',
|
|
211
|
-
'transmission' => TRANSMISSIONS[row['TRANS'][-3, 1]],
|
|
212
|
-
'speeds' => (row['TRANS'][-2, 1] == 'V') ? 'variable' : row['TRANS'][-2, 1],
|
|
213
|
-
'turbo' => row['TURBO'] == 'T',
|
|
214
|
-
'supercharger' => row['SPCHGR'] == 'S',
|
|
215
|
-
'injection' => true,
|
|
216
|
-
'year' => year
|
|
217
|
-
})
|
|
218
|
-
row
|
|
219
|
-
end
|
|
220
|
-
end
|
|
221
|
-
end
|
|
222
|
-
|
|
223
|
-
class Guru
|
|
224
|
-
# the following matching methods are needed by the errata
|
|
225
|
-
# per https://brighterplanet.sifterapp.com/projects/30/issues/750/comments
|
|
226
|
-
|
|
227
|
-
def transmission_is_blank?(row)
|
|
228
|
-
row['transmission'].blank?
|
|
229
|
-
end
|
|
230
|
-
|
|
231
|
-
def is_a_2007_gmc_or_chevrolet?(row)
|
|
232
|
-
row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
|
|
233
|
-
end
|
|
234
|
-
|
|
235
|
-
def is_a_porsche?(row)
|
|
236
|
-
row['make'].upcase == 'PORSCHE'
|
|
237
|
-
end
|
|
238
|
-
|
|
239
|
-
def is_not_a_porsche?(row)
|
|
240
|
-
!is_a_porsche? row
|
|
241
|
-
end
|
|
242
|
-
|
|
243
|
-
def is_a_mercedes_benz?(row)
|
|
244
|
-
row['make'] =~ /MERCEDES/i
|
|
245
|
-
end
|
|
246
|
-
|
|
247
|
-
def is_a_lexus?(row)
|
|
248
|
-
row['make'].upcase == 'LEXUS'
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
def is_a_bmw?(row)
|
|
252
|
-
row['make'].upcase == 'BMW'
|
|
253
|
-
end
|
|
254
|
-
|
|
255
|
-
def is_a_ford?(row)
|
|
256
|
-
row['make'].upcase == 'FORD'
|
|
257
|
-
end
|
|
258
|
-
|
|
259
|
-
def is_a_rolls_royce_and_model_contains_bentley?(row)
|
|
260
|
-
is_a_rolls_royce?(row) and model_contains_bentley?(row)
|
|
261
|
-
end
|
|
262
|
-
|
|
263
|
-
def is_a_bentley?(row)
|
|
264
|
-
row['make'].upcase == 'BENTLEY'
|
|
265
|
-
end
|
|
266
|
-
|
|
267
|
-
def is_a_rolls_royce?(row)
|
|
268
|
-
row['make'] =~ /ROLLS/i
|
|
269
|
-
end
|
|
270
|
-
|
|
271
|
-
def is_a_turbo_brooklands?(row)
|
|
272
|
-
row['model'] =~ /TURBO R\/RL BKLDS/i
|
|
273
|
-
end
|
|
274
|
-
|
|
275
|
-
def model_contains_maybach?(row)
|
|
276
|
-
row['model'] =~ /MAYBACH/i
|
|
277
|
-
end
|
|
278
|
-
|
|
279
|
-
def model_contains_bentley?(row)
|
|
280
|
-
row['model'] =~ /BENTLEY/i
|
|
281
|
-
end
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
errata = Errata.new :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv',
|
|
285
|
-
:responder => AutomobileVariant::Guru.new
|
|
286
|
-
|
|
287
|
-
data_miner do
|
|
288
|
-
# 1985---1997
|
|
289
|
-
(85..97).each do |yy|
|
|
290
|
-
filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
|
|
291
|
-
import(:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
|
|
292
|
-
:filename => filename,
|
|
293
|
-
:transform => { :class => FuelEconomyGuide::ParserB, :year => "19#{yy}".to_i },
|
|
294
|
-
:errata => errata) do
|
|
295
|
-
key 'row_hash'
|
|
296
|
-
store 'make_name', :field_name => 'make'
|
|
297
|
-
store 'model_name', :field_name => 'model'
|
|
298
|
-
store 'year'
|
|
299
|
-
store 'fuel_type_code', :field_name => 'fuel_type'
|
|
300
|
-
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
|
|
301
|
-
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre # ditto
|
|
302
|
-
store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
303
|
-
store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
304
|
-
store 'cylinders', :field_name => 'no_cyc'
|
|
305
|
-
store 'drive', :field_name => 'drive_system'
|
|
306
|
-
store 'carline_mfr_code'
|
|
307
|
-
store 'vi_mfr_code'
|
|
308
|
-
store 'carline_code'
|
|
309
|
-
store 'carline_class_code', :field_name => 'carline_clss'
|
|
310
|
-
store 'transmission'
|
|
311
|
-
store 'speeds'
|
|
312
|
-
store 'turbo'
|
|
313
|
-
store 'supercharger'
|
|
314
|
-
store 'injection'
|
|
315
|
-
store 'displacement'
|
|
316
|
-
end
|
|
317
|
-
end
|
|
318
|
-
|
|
319
|
-
# 1998--2005
|
|
320
|
-
{
|
|
321
|
-
1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
|
|
322
|
-
1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
|
|
323
|
-
2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
|
|
324
|
-
2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
|
|
325
|
-
2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
|
|
326
|
-
2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
|
|
327
|
-
2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
|
|
328
|
-
2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
|
|
329
|
-
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
330
|
-
import options.merge(:transform => { :class => FuelEconomyGuide::ParserC, :year => year },
|
|
331
|
-
:errata => errata) do
|
|
332
|
-
key 'row_hash'
|
|
333
|
-
store 'make_name', :field_name => 'make'
|
|
334
|
-
store 'model_name', :field_name => 'model'
|
|
335
|
-
store 'fuel_type_code', :field_name => 'fl'
|
|
336
|
-
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
|
|
337
|
-
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre # ditto
|
|
338
|
-
store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
339
|
-
store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
340
|
-
store 'cylinders', :field_name => 'cyl'
|
|
341
|
-
store 'displacement', :field_name => 'displ'
|
|
342
|
-
store 'carline_class_code', :field_name => 'cls' if year >= 2000
|
|
343
|
-
store 'carline_class_name', :field_name => 'Class'
|
|
344
|
-
store 'year'
|
|
345
|
-
store 'transmission'
|
|
346
|
-
store 'speeds'
|
|
347
|
-
store 'turbo'
|
|
348
|
-
store 'supercharger'
|
|
349
|
-
store 'injection'
|
|
350
|
-
store 'drive'
|
|
351
|
-
end
|
|
352
|
-
end
|
|
353
|
-
|
|
354
|
-
# 2006--2010
|
|
355
|
-
{
|
|
356
|
-
2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
|
|
357
|
-
2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
|
|
358
|
-
2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
|
|
359
|
-
2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
|
|
360
|
-
# 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
|
|
361
|
-
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
362
|
-
import options.merge(:transform => { :class => FuelEconomyGuide::ParserD, :year => year },
|
|
363
|
-
:errata => errata) do
|
|
364
|
-
key 'row_hash'
|
|
365
|
-
store 'make_name', :field_name => 'make'
|
|
366
|
-
store 'model_name', :field_name => 'model'
|
|
367
|
-
store 'fuel_type_code', :field_name => 'FUEL TYPE'
|
|
368
|
-
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
|
|
369
|
-
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre # ditto
|
|
370
|
-
store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
371
|
-
store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
372
|
-
store 'cylinders', :field_name => 'NUMB CYL'
|
|
373
|
-
store 'displacement', :field_name => 'DISPLACEMENT'
|
|
374
|
-
store 'carline_class_code', :field_name => 'CLS'
|
|
375
|
-
store 'carline_class_name', :field_name => 'CLASS'
|
|
376
|
-
store 'year'
|
|
377
|
-
store 'transmission'
|
|
378
|
-
store 'speeds'
|
|
379
|
-
store 'turbo'
|
|
380
|
-
store 'supercharger'
|
|
381
|
-
store 'injection'
|
|
382
|
-
store 'drive'
|
|
383
|
-
end
|
|
384
|
-
end
|
|
385
|
-
|
|
386
|
-
# associate :make, :key => :original_automobile_make_name, :foreign_key => :name
|
|
387
|
-
# derive :automobile_model_id # creates models by name
|
|
388
|
-
# associate :fuel_type, :key => :original_automobile_fuel_type_code, :foreign_key => :code
|
|
389
|
-
|
|
390
|
-
process 'Set adjusted fuel economy' do
|
|
391
|
-
update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
|
|
392
|
-
update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
|
|
393
|
-
end
|
|
394
|
-
end
|
|
395
|
-
|
|
396
|
-
def name
|
|
397
|
-
extra = []
|
|
398
|
-
extra << "V#{cylinders}" if cylinders
|
|
399
|
-
extra << "#{displacement}L" if displacement
|
|
400
|
-
extra << "turbo" if turbo
|
|
401
|
-
extra << "FI" if injection
|
|
402
|
-
extra << "#{speeds}spd" if speeds.present?
|
|
403
|
-
extra << transmission if transmission.present?
|
|
404
|
-
extra << "(#{fuel_type.name})" if fuel_type
|
|
405
|
-
extra.join(' ')
|
|
406
|
-
end
|
|
407
|
-
|
|
408
|
-
def fuel_economy_description
|
|
409
|
-
[ fuel_efficiency_city, fuel_efficiency_highway ].map { |f| f.kilometres_per_litre.to(:miles_per_gallon).round }.join('/')
|
|
410
|
-
end
|
|
411
|
-
end
|
|
412
|
-
|
|
413
|
-
class Country < ActiveRecord::Base
|
|
414
|
-
set_primary_key :iso_3166
|
|
415
|
-
|
|
416
|
-
data_miner do
|
|
417
|
-
import 'The official ISO country list', :url => 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';' do
|
|
418
|
-
key 'iso_3166', :field_number => 1
|
|
419
|
-
store 'name', :field_number => 0
|
|
420
|
-
end
|
|
421
|
-
|
|
422
|
-
import 'A Princeton dataset with better capitalization', :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do
|
|
423
|
-
key 'iso_3166', :field_name => 'country code'
|
|
424
|
-
store 'name', :field_name => 'country'
|
|
425
|
-
end
|
|
426
|
-
end
|
|
427
|
-
end
|
|
428
|
-
|
|
429
|
-
class Airport < ActiveRecord::Base
|
|
430
|
-
set_primary_key :iata_code
|
|
431
|
-
|
|
432
|
-
data_miner do
|
|
433
|
-
import :url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat', :headers => false, :select => lambda { |row| row[4].present? } do
|
|
434
|
-
key 'iata_code', :field_number => 4
|
|
435
|
-
store 'name', :field_number => 1
|
|
436
|
-
store 'city', :field_number => 2
|
|
437
|
-
store 'country_name', :field_number => 3
|
|
438
|
-
store 'latitude', :field_number => 6
|
|
439
|
-
store 'longitude', :field_number => 7
|
|
440
|
-
end
|
|
441
|
-
end
|
|
442
|
-
end
|
|
443
|
-
|
|
444
3
|
class TappedAirport < ActiveRecord::Base
|
|
445
4
|
set_primary_key :iata_code
|
|
446
5
|
|
|
@@ -947,103 +506,6 @@ class T100FlightSegment < ActiveRecord::Base
|
|
|
947
506
|
end
|
|
948
507
|
end
|
|
949
508
|
|
|
950
|
-
require 'loose_tight_dictionary'
|
|
951
|
-
class Aircraft < ActiveRecord::Base
|
|
952
|
-
set_primary_key :icao_code
|
|
953
|
-
|
|
954
|
-
def self.bts_dictionary
|
|
955
|
-
@_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.bts.gov/programs/airline_information/accounting_and_reporting_directives/csv/number_260.csv', :select => lambda { |record| record['Aircraft Type'].to_i.between?(1, 998) and record['Manufacturer'].present? }),
|
|
956
|
-
:tightenings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
|
|
957
|
-
:identities => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
|
|
958
|
-
:blockings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
|
|
959
|
-
:left_reader => lambda { |record| record['Manufacturer'] + ' ' + record['Model'] },
|
|
960
|
-
:right_reader => lambda { |record| record['Manufacturer'] + ' ' + record['Long Name'] }
|
|
961
|
-
end
|
|
962
|
-
|
|
963
|
-
class BtsAircraftTypeCodeMatcher
|
|
964
|
-
def match(left_record)
|
|
965
|
-
right_record = Aircraft.bts_dictionary.left_to_right left_record
|
|
966
|
-
right_record['Aircraft Type'] if right_record
|
|
967
|
-
end
|
|
968
|
-
end
|
|
969
|
-
|
|
970
|
-
class BtsNameMatcher
|
|
971
|
-
def match(left_record)
|
|
972
|
-
right_record = Aircraft.bts_dictionary.left_to_right left_record
|
|
973
|
-
right_record['Manufacturer'] + ' ' + right_record['Long Name'] if right_record
|
|
974
|
-
end
|
|
975
|
-
end
|
|
976
|
-
|
|
977
|
-
class Guru
|
|
978
|
-
# for errata
|
|
979
|
-
def is_attributed_to_boeing?(row)
|
|
980
|
-
row['Manufacturer'] =~ /BOEING/i
|
|
981
|
-
end
|
|
982
|
-
|
|
983
|
-
def is_attributed_to_cessna?(row)
|
|
984
|
-
row['Manufacturer'] =~ /CESSNA/i
|
|
985
|
-
end
|
|
986
|
-
|
|
987
|
-
def is_attributed_to_fokker?(row)
|
|
988
|
-
row['Manufacturer'] =~ /FOKKER/i
|
|
989
|
-
end
|
|
990
|
-
|
|
991
|
-
def is_not_attributed_to_aerospatiale?(row)
|
|
992
|
-
not row['Manufacturer'] =~ /AEROSPATIALE/i
|
|
993
|
-
end
|
|
994
|
-
|
|
995
|
-
def is_not_attributed_to_cessna?(row)
|
|
996
|
-
not row['Manufacturer'] =~ /CESSNA/i
|
|
997
|
-
end
|
|
998
|
-
|
|
999
|
-
def is_not_attributed_to_learjet?(row)
|
|
1000
|
-
not row['Manufacturer'] =~ /LEAR/i
|
|
1001
|
-
end
|
|
1002
|
-
|
|
1003
|
-
def is_not_attributed_to_dehavilland?(row)
|
|
1004
|
-
not row['Manufacturer'] =~ /DE ?HAVILLAND/i
|
|
1005
|
-
end
|
|
1006
|
-
|
|
1007
|
-
def is_not_attributed_to_mcdonnell_douglas?(row)
|
|
1008
|
-
not row['Manufacturer'] =~ /MCDONNELL DOUGLAS/i
|
|
1009
|
-
end
|
|
1010
|
-
|
|
1011
|
-
def is_not_a_dc_plane?(row)
|
|
1012
|
-
not row['Model'] =~ /DC/i
|
|
1013
|
-
end
|
|
1014
|
-
|
|
1015
|
-
def is_a_crj_900?(row)
|
|
1016
|
-
row['Designator'].downcase == 'crj9'
|
|
1017
|
-
end
|
|
1018
|
-
end
|
|
1019
|
-
|
|
1020
|
-
data_miner do
|
|
1021
|
-
# ('A'..'Z').each do |letter|
|
|
1022
|
-
# Note: for the purposes of testing, only importing "D"
|
|
1023
|
-
%w{ D }.each do |letter|
|
|
1024
|
-
import("ICAO codes starting with letter #{letter} used by the FAA",
|
|
1025
|
-
:url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
|
|
1026
|
-
:encoding => 'US-ASCII',
|
|
1027
|
-
:errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
|
|
1028
|
-
:responder => Aircraft::Guru.new),
|
|
1029
|
-
:row_xpath => '//table/tr[2]/td/table/tr',
|
|
1030
|
-
:column_xpath => 'td') do
|
|
1031
|
-
key 'icao_code', :field_name => 'Designator'
|
|
1032
|
-
store 'bts_name', :matcher => Aircraft::BtsNameMatcher.new
|
|
1033
|
-
store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher.new
|
|
1034
|
-
store 'manufacturer_name', :field_name => 'Manufacturer'
|
|
1035
|
-
store 'name', :field_name => 'Model'
|
|
1036
|
-
end
|
|
1037
|
-
|
|
1038
|
-
import 'Brighter Planet aircraft class codes',
|
|
1039
|
-
:url => 'http://static.brighterplanet.com/science/data/transport/air/bts_aircraft_type/bts_aircraft_types-brighter_planet_aircraft_classes.csv' do
|
|
1040
|
-
key 'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
|
|
1041
|
-
store 'brighter_planet_aircraft_class_code'
|
|
1042
|
-
end
|
|
1043
|
-
end
|
|
1044
|
-
end
|
|
1045
|
-
end
|
|
1046
|
-
|
|
1047
509
|
# note that this depends on stuff in Aircraft
|
|
1048
510
|
class AircraftDeux < ActiveRecord::Base
|
|
1049
511
|
set_primary_key :icao_code
|
|
@@ -1166,7 +628,19 @@ end
|
|
|
1166
628
|
|
|
1167
629
|
# todo: have somebody properly organize these
|
|
1168
630
|
class DataMinerTest < Test::Unit::TestCase
|
|
1169
|
-
if ENV['
|
|
631
|
+
if ENV['WIP']
|
|
632
|
+
context 'with nullify option' do
|
|
633
|
+
should 'treat blank fields as null values' do
|
|
634
|
+
Aircraft.delete_all
|
|
635
|
+
Aircraft.data_miner_runs.delete_all
|
|
636
|
+
Aircraft.run_data_miner!
|
|
637
|
+
assert_greater_than 0, Aircraft.count
|
|
638
|
+
assert_false Aircraft.where(:brighter_planet_aircraft_class_code => nil).empty?
|
|
639
|
+
end
|
|
640
|
+
end
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
if ENV['ALL'] == 'true'
|
|
1170
644
|
should 'directly create a table for the model' do
|
|
1171
645
|
if AutomobileMakeFleetYear.table_exists?
|
|
1172
646
|
ActiveRecord::Base.connection.execute 'DROP TABLE automobile_make_fleet_years;'
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
require 'loose_tight_dictionary'
|
|
2
|
+
|
|
3
|
+
class Aircraft < ActiveRecord::Base
|
|
4
|
+
set_primary_key :icao_code
|
|
5
|
+
set_table_name 'aircraft'
|
|
6
|
+
|
|
7
|
+
def self.bts_dictionary
|
|
8
|
+
@_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.bts.gov/programs/airline_information/accounting_and_reporting_directives/csv/number_260.csv', :select => lambda { |record| record['Aircraft Type'].to_i.between?(1, 998) and record['Manufacturer'].present? }),
|
|
9
|
+
:tightenings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
|
|
10
|
+
:identities => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
|
|
11
|
+
:blockings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
|
|
12
|
+
:left_reader => lambda { |record| record['Manufacturer'] + ' ' + record['Model'] },
|
|
13
|
+
:right_reader => lambda { |record| record['Manufacturer'] + ' ' + record['Long Name'] }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
class BtsAircraftTypeCodeMatcher
|
|
17
|
+
def match(left_record)
|
|
18
|
+
right_record = Aircraft.bts_dictionary.left_to_right left_record
|
|
19
|
+
right_record['Aircraft Type'] if right_record
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
class BtsNameMatcher
|
|
24
|
+
def match(left_record)
|
|
25
|
+
right_record = Aircraft.bts_dictionary.left_to_right left_record
|
|
26
|
+
right_record['Manufacturer'] + ' ' + right_record['Long Name'] if right_record
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
class Guru
|
|
31
|
+
# for errata
|
|
32
|
+
def is_attributed_to_boeing?(row)
|
|
33
|
+
row['Manufacturer'] =~ /BOEING/i
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def is_attributed_to_cessna?(row)
|
|
37
|
+
row['Manufacturer'] =~ /CESSNA/i
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def is_attributed_to_fokker?(row)
|
|
41
|
+
row['Manufacturer'] =~ /FOKKER/i
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def is_not_attributed_to_aerospatiale?(row)
|
|
45
|
+
not row['Manufacturer'] =~ /AEROSPATIALE/i
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def is_not_attributed_to_cessna?(row)
|
|
49
|
+
not row['Manufacturer'] =~ /CESSNA/i
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def is_not_attributed_to_learjet?(row)
|
|
53
|
+
not row['Manufacturer'] =~ /LEAR/i
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def is_not_attributed_to_dehavilland?(row)
|
|
57
|
+
not row['Manufacturer'] =~ /DE ?HAVILLAND/i
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def is_not_attributed_to_mcdonnell_douglas?(row)
|
|
61
|
+
not row['Manufacturer'] =~ /MCDONNELL DOUGLAS/i
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def is_not_a_dc_plane?(row)
|
|
65
|
+
not row['Model'] =~ /DC/i
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def is_a_crj_900?(row)
|
|
69
|
+
row['Designator'].downcase == 'crj9'
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
data_miner do
|
|
74
|
+
# ('A'..'Z').each do |letter|
|
|
75
|
+
# Note: for the purposes of testing, only importing "D"
|
|
76
|
+
%w{ D }.each do |letter|
|
|
77
|
+
import("ICAO codes starting with letter #{letter} used by the FAA",
|
|
78
|
+
:url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
|
|
79
|
+
:encoding => 'US-ASCII',
|
|
80
|
+
:errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
|
|
81
|
+
:responder => Aircraft::Guru.new),
|
|
82
|
+
:row_xpath => '//table/tr[2]/td/table/tr',
|
|
83
|
+
:column_xpath => 'td') do
|
|
84
|
+
key 'icao_code', :field_name => 'Designator'
|
|
85
|
+
store 'bts_name', :matcher => Aircraft::BtsNameMatcher.new, :nullify => true
|
|
86
|
+
store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher.new, :nullify => true
|
|
87
|
+
store 'manufacturer_name', :field_name => 'Manufacturer', :nullify => true
|
|
88
|
+
store 'name', :field_name => 'Model', :nullify => true
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
import 'Brighter Planet aircraft class codes',
|
|
92
|
+
:url => 'http://static.brighterplanet.com/science/data/transport/air/bts_aircraft_type/bts_aircraft_types-brighter_planet_aircraft_classes.csv' do
|
|
93
|
+
key 'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
|
|
94
|
+
store 'brighter_planet_aircraft_class_code', :nullify => true
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class Airport < ActiveRecord::Base
|
|
2
|
+
set_primary_key :iata_code
|
|
3
|
+
|
|
4
|
+
data_miner do
|
|
5
|
+
import :url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat', :headers => false, :select => lambda { |row| row[4].present? } do
|
|
6
|
+
key 'iata_code', :field_number => 4
|
|
7
|
+
store 'name', :field_number => 1
|
|
8
|
+
store 'city', :field_number => 2
|
|
9
|
+
store 'country_name', :field_number => 3
|
|
10
|
+
store 'latitude', :field_number => 6, :nullify => true
|
|
11
|
+
store 'longitude', :field_number => 7, :nullify => true
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
class AutomobileFuelType < ActiveRecord::Base
|
|
2
|
+
set_primary_key :code
|
|
3
|
+
|
|
4
|
+
data_miner do
|
|
5
|
+
import(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
|
6
|
+
:filename => 'Gd6-dsc.txt',
|
|
7
|
+
:format => :fixed_width,
|
|
8
|
+
:crop => 21..26, # inclusive
|
|
9
|
+
:cut => '2-',
|
|
10
|
+
:select => lambda { |row| /\A[A-Z]/.match row[:code] },
|
|
11
|
+
:schema => [[ 'code', 2, { :type => :string } ],
|
|
12
|
+
[ 'spacer', 2 ],
|
|
13
|
+
[ 'name', 52, { :type => :string } ]]) do
|
|
14
|
+
key 'code'
|
|
15
|
+
store 'name'
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
import :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_fuel_type.csv' do
|
|
19
|
+
key 'code'
|
|
20
|
+
store 'name'
|
|
21
|
+
store 'annual_distance'
|
|
22
|
+
store 'emission_factor'
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# pull electricity emission factor from residential electricity
|
|
26
|
+
import(:url => 'http://spreadsheets.google.com/pub?key=rukxnmuhhsOsrztTrUaFCXQ',
|
|
27
|
+
:select => lambda { |row| row['code'] == 'El' }) do
|
|
28
|
+
key 'code'
|
|
29
|
+
store 'name'
|
|
30
|
+
store 'emission_factor'
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# still need distance estimate for electric cars
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
CODES = {
|
|
37
|
+
:electricity => 'El',
|
|
38
|
+
:diesel => 'D'
|
|
39
|
+
}
|
|
40
|
+
end
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
class AutomobileVariant < ActiveRecord::Base
|
|
2
|
+
set_primary_key :row_hash
|
|
3
|
+
|
|
4
|
+
module FuelEconomyGuide
|
|
5
|
+
TRANSMISSIONS = {
|
|
6
|
+
'A' => 'automatic',
|
|
7
|
+
'M' => 'manual',
|
|
8
|
+
'L' => 'automatic', # Lockup/automatic
|
|
9
|
+
'S' => 'semiautomatic', # Semiautomatic
|
|
10
|
+
'C' => 'manual' # TODO verify for VW Syncro
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
ENGINE_TYPES = {
|
|
14
|
+
'(GUZZLER)' => nil, # "gas guzzler"
|
|
15
|
+
'(POLICE)' => nil, # police automobile_variant
|
|
16
|
+
'(MPFI)' => 'injection',
|
|
17
|
+
'(MPI*)' => 'injection',
|
|
18
|
+
'(SPFI)' => 'injection',
|
|
19
|
+
'(FFS)' => 'injection',
|
|
20
|
+
'(TURBO)' => 'turbo',
|
|
21
|
+
'(TRBO)' => 'turbo',
|
|
22
|
+
'(TC*)' => 'turbo',
|
|
23
|
+
'(FFS,TRBO)' => %w(injection turbo),
|
|
24
|
+
'(S-CHARGE)' => 'supercharger',
|
|
25
|
+
'(SC*)' => 'supercharger',
|
|
26
|
+
'(DIESEL)' => nil, # diesel
|
|
27
|
+
'(DSL)' => nil, # diesel
|
|
28
|
+
'(ROTARY)' => nil, # rotary
|
|
29
|
+
'(VARIABLE)' => nil, # variable displacement
|
|
30
|
+
'(NO-CAT)' => nil, # no catalytic converter
|
|
31
|
+
'(OHC)' => nil, # overhead camshaft
|
|
32
|
+
'(OHV)' => nil, # overhead valves
|
|
33
|
+
'(16-VALVE)' => nil, # 16V
|
|
34
|
+
'(305)' => nil, # 305 cubic inch displacement
|
|
35
|
+
'(307)' => nil, # 307 cubic inch displacement
|
|
36
|
+
'(M-ENG)' => nil,
|
|
37
|
+
'(W-ENG)' => nil,
|
|
38
|
+
'(GM-BUICK)' => nil,
|
|
39
|
+
'(GM-CHEV)' => nil,
|
|
40
|
+
'(GM-OLDS)' => nil,
|
|
41
|
+
'(GM-PONT)' => nil,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
class ParserB
|
|
45
|
+
attr_accessor :year
|
|
46
|
+
def initialize(options = {})
|
|
47
|
+
@year = options[:year]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def apply(row)
|
|
51
|
+
row.merge!({
|
|
52
|
+
'make' => row['carline_mfr_name'], # make it line up with the errata
|
|
53
|
+
'model' => row['carline_name'], # ditto
|
|
54
|
+
'transmission' => TRANSMISSIONS[row['model_trans'][0, 1]],
|
|
55
|
+
'speeds' => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
|
|
56
|
+
'turbo' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
|
|
57
|
+
'supercharger' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
|
|
58
|
+
'injection' => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
|
|
59
|
+
'displacement' => _displacement(row['opt_disp']),
|
|
60
|
+
'year' => year
|
|
61
|
+
})
|
|
62
|
+
row
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def _displacement(str)
|
|
66
|
+
str = str.gsub(/[\(\)]/, '').strip
|
|
67
|
+
if str =~ /^(.+)L$/
|
|
68
|
+
$1.to_f
|
|
69
|
+
elsif str =~ /^(.+)CC$/
|
|
70
|
+
$1.to_f / 1000
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def add_hints!(bus)
|
|
75
|
+
bus[:format] = :fixed_width
|
|
76
|
+
bus[:cut] = '13-' if year == 1995
|
|
77
|
+
bus[:schema_name] = :fuel_economy_guide_b
|
|
78
|
+
bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
|
|
79
|
+
Slither.define :fuel_economy_guide_b do |d|
|
|
80
|
+
d.rows do |row|
|
|
81
|
+
row.trap { true } # there's only one section
|
|
82
|
+
row.column 'active_year' , 4, :type => :integer # ACTIVE YEAR
|
|
83
|
+
row.column 'state_code' , 1, :type => :string # STATE CODE: F=49-STATE,C=CALIFORNIA
|
|
84
|
+
row.column 'carline_clss' , 2, :type => :integer # CARLINE CLASS CODE
|
|
85
|
+
row.column 'carline_mfr_code' , 3, :type => :integer # CARLINE MANUFACTURER CODE
|
|
86
|
+
row.column 'carline_name' , 28, :type => :string # CARLINE NAME
|
|
87
|
+
row.column 'disp_cub_in' , 4, :type => :integer # DISP CUBIC INCHES
|
|
88
|
+
row.column 'fuel_system' , 2, :type => :string # FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
|
|
89
|
+
row.column 'model_trans' , 6, :type => :string # TRANSMISSION TYPE
|
|
90
|
+
row.column 'no_cyc' , 2, :type => :integer # NUMBER OF ENGINE CYLINDERS
|
|
91
|
+
row.column 'date_time' , 12, :type => :string # DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
|
|
92
|
+
row.column 'release_date' , 6, :type => :string # RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
|
|
93
|
+
row.column 'vi_mfr_code' , 3, :type => :integer # VI MANUFACTURER CODE
|
|
94
|
+
row.column 'carline_code' , 5, :type => :integer # CARLINE CODE
|
|
95
|
+
row.column 'basic_eng_id' , 5, :type => :integer # BASIC ENGINE INDEX
|
|
96
|
+
row.column 'carline_mfr_name' , 32, :type => :string # CARLINE MANUFACTURER NAME
|
|
97
|
+
row.column 'suppress_code' , 1, :type => :integer # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
|
|
98
|
+
row.column 'est_city_mpg' , 3, :type => :integer # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
|
|
99
|
+
row.spacer 2
|
|
100
|
+
row.column 'highway_mpg' , 3, :type => :integer # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
|
|
101
|
+
row.spacer 2
|
|
102
|
+
row.column 'combined_mpg' , 3, :type => :integer # COMBINED MILES PER GALLON
|
|
103
|
+
row.spacer 2
|
|
104
|
+
row.column 'unadj_city_mpg' , 3, :type => :integer # UNADJUSTED CITY MILES PER GALLON
|
|
105
|
+
row.spacer 2
|
|
106
|
+
row.column 'unadj_hwy_mpg' , 3, :type => :integer # UNADJUSTED HIGHWAY MILES PER GALLON
|
|
107
|
+
row.spacer 2
|
|
108
|
+
row.column 'unadj_comb_mpg' , 3, :type => :integer # UNADJUSTED COMBINED MILES PER GALLON
|
|
109
|
+
row.spacer 2
|
|
110
|
+
row.column 'ave_anl_fuel' , 6, :type => :integer # "$" in col 147, Annual Fuel Cost starting col 148 in I5
|
|
111
|
+
row.column 'opt_disp' , 8, :type => :string # OPTIONAL DISPLACEMENT
|
|
112
|
+
row.column 'engine_desc1' , 10, :type => :string # ENGINE DESCRIPTION 1
|
|
113
|
+
row.column 'engine_desc2' , 10, :type => :string # ENGINE DESCRIPTION 2
|
|
114
|
+
row.column 'engine_desc3' , 10, :type => :string # ENGINE DESCRIPTION 3
|
|
115
|
+
row.column 'body_type_2d' , 10, :type => :string # BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
116
|
+
row.column 'body_type_4d' , 10, :type => :string # BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
117
|
+
row.column 'body_type_hbk' , 10, :type => :string # BODY TYPE HBK - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
|
|
118
|
+
row.column 'puerto_rico' , 1, :type => :string # '*' IF FOR PUERTO RICO SALES ONLY
|
|
119
|
+
row.column 'overdrive' , 4, :type => :string # OVERDRIVE: ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
|
|
120
|
+
row.column 'drive_system' , 3, :type => :string # FWD=FRONT WHEEL DRIVE, RWD=REAR, 4WD=4-WHEEL
|
|
121
|
+
row.column 'filler' , 1, :type => :string # NOT USED
|
|
122
|
+
row.column 'fuel_type' , 1, :type => :string # R=REGULAR(UNLEADED), P=PREMIUM, D=DIESEL
|
|
123
|
+
row.column 'trans_desc' , 15, :type => :string # TRANSMISSION DESCRIPTORS
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
class ParserC
|
|
129
|
+
attr_accessor :year
|
|
130
|
+
def initialize(options = {})
|
|
131
|
+
@year = options[:year]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def add_hints!(bus)
|
|
135
|
+
# File will decide format based on filename
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def apply(row)
|
|
139
|
+
row.merge!({
|
|
140
|
+
'make' => row['Manufacturer'], # make it line up with the errata
|
|
141
|
+
'model' => row['carline name'], # ditto
|
|
142
|
+
'drive' => row['drv'] + 'WD',
|
|
143
|
+
'transmission' => TRANSMISSIONS[row['trans'][-3, 1]],
|
|
144
|
+
'speeds' => (row['trans'][-2, 1] == 'V') ? 'variable' : row['trans'][-2, 1],
|
|
145
|
+
'turbo' => row['T'] == 'T',
|
|
146
|
+
'supercharger' => row['S'] == 'S',
|
|
147
|
+
'injection' => true,
|
|
148
|
+
'year' => year
|
|
149
|
+
})
|
|
150
|
+
row
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
class ParserD
|
|
154
|
+
attr_accessor :year
|
|
155
|
+
def initialize(options = {})
|
|
156
|
+
@year = options[:year]
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def add_hints!(bus)
|
|
160
|
+
bus[:reject] = lambda { |row| row.values.first.blank? } if year == 2007
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def apply(row)
|
|
164
|
+
row.merge!({
|
|
165
|
+
'make' => row['MFR'], # make it line up with the errata
|
|
166
|
+
'model' => row['CAR LINE'], # ditto
|
|
167
|
+
'drive' => row['DRIVE SYS'] + 'WD',
|
|
168
|
+
'transmission' => TRANSMISSIONS[row['TRANS'][-3, 1]],
|
|
169
|
+
'speeds' => (row['TRANS'][-2, 1] == 'V') ? 'variable' : row['TRANS'][-2, 1],
|
|
170
|
+
'turbo' => row['TURBO'] == 'T',
|
|
171
|
+
'supercharger' => row['SPCHGR'] == 'S',
|
|
172
|
+
'injection' => true,
|
|
173
|
+
'year' => year
|
|
174
|
+
})
|
|
175
|
+
row
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
class Guru
|
|
181
|
+
# the following matching methods are needed by the errata
|
|
182
|
+
# per https://brighterplanet.sifterapp.com/projects/30/issues/750/comments
|
|
183
|
+
|
|
184
|
+
def transmission_is_blank?(row)
|
|
185
|
+
row['transmission'].blank?
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def is_a_2007_gmc_or_chevrolet?(row)
|
|
189
|
+
row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def is_a_porsche?(row)
|
|
193
|
+
row['make'].upcase == 'PORSCHE'
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def is_not_a_porsche?(row)
|
|
197
|
+
!is_a_porsche? row
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def is_a_mercedes_benz?(row)
|
|
201
|
+
row['make'] =~ /MERCEDES/i
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def is_a_lexus?(row)
|
|
205
|
+
row['make'].upcase == 'LEXUS'
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def is_a_bmw?(row)
|
|
209
|
+
row['make'].upcase == 'BMW'
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def is_a_ford?(row)
|
|
213
|
+
row['make'].upcase == 'FORD'
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def is_a_rolls_royce_and_model_contains_bentley?(row)
|
|
217
|
+
is_a_rolls_royce?(row) and model_contains_bentley?(row)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def is_a_bentley?(row)
|
|
221
|
+
row['make'].upcase == 'BENTLEY'
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def is_a_rolls_royce?(row)
|
|
225
|
+
row['make'] =~ /ROLLS/i
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def is_a_turbo_brooklands?(row)
|
|
229
|
+
row['model'] =~ /TURBO R\/RL BKLDS/i
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def model_contains_maybach?(row)
|
|
233
|
+
row['model'] =~ /MAYBACH/i
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def model_contains_bentley?(row)
|
|
237
|
+
row['model'] =~ /BENTLEY/i
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
errata = Errata.new :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv',
|
|
242
|
+
:responder => AutomobileVariant::Guru.new
|
|
243
|
+
|
|
244
|
+
data_miner do
|
|
245
|
+
# 1985---1997
|
|
246
|
+
(85..97).each do |yy|
|
|
247
|
+
filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
|
|
248
|
+
import(:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
|
|
249
|
+
:filename => filename,
|
|
250
|
+
:transform => { :class => FuelEconomyGuide::ParserB, :year => "19#{yy}".to_i },
|
|
251
|
+
:errata => errata) do
|
|
252
|
+
key 'row_hash'
|
|
253
|
+
store 'make_name', :field_name => 'make'
|
|
254
|
+
store 'model_name', :field_name => 'model'
|
|
255
|
+
store 'year'
|
|
256
|
+
store 'fuel_type_code', :field_name => 'fuel_type'
|
|
257
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
|
|
258
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre # ditto
|
|
259
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
260
|
+
store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
261
|
+
store 'cylinders', :field_name => 'no_cyc'
|
|
262
|
+
store 'drive', :field_name => 'drive_system'
|
|
263
|
+
store 'carline_mfr_code'
|
|
264
|
+
store 'vi_mfr_code'
|
|
265
|
+
store 'carline_code'
|
|
266
|
+
store 'carline_class_code', :field_name => 'carline_clss'
|
|
267
|
+
store 'transmission'
|
|
268
|
+
store 'speeds'
|
|
269
|
+
store 'turbo'
|
|
270
|
+
store 'supercharger'
|
|
271
|
+
store 'injection'
|
|
272
|
+
store 'displacement'
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# 1998--2005
|
|
277
|
+
{
|
|
278
|
+
1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
|
|
279
|
+
1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
|
|
280
|
+
2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
|
|
281
|
+
2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
|
|
282
|
+
2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
|
|
283
|
+
2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
|
|
284
|
+
2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
|
|
285
|
+
2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
|
|
286
|
+
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
287
|
+
import options.merge(:transform => { :class => FuelEconomyGuide::ParserC, :year => year },
|
|
288
|
+
:errata => errata) do
|
|
289
|
+
key 'row_hash'
|
|
290
|
+
store 'make_name', :field_name => 'make'
|
|
291
|
+
store 'model_name', :field_name => 'model'
|
|
292
|
+
store 'fuel_type_code', :field_name => 'fl'
|
|
293
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
|
|
294
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre # ditto
|
|
295
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
296
|
+
store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
297
|
+
store 'cylinders', :field_name => 'cyl'
|
|
298
|
+
store 'displacement', :field_name => 'displ'
|
|
299
|
+
store 'carline_class_code', :field_name => 'cls' if year >= 2000
|
|
300
|
+
store 'carline_class_name', :field_name => 'Class'
|
|
301
|
+
store 'year'
|
|
302
|
+
store 'transmission'
|
|
303
|
+
store 'speeds'
|
|
304
|
+
store 'turbo'
|
|
305
|
+
store 'supercharger'
|
|
306
|
+
store 'injection'
|
|
307
|
+
store 'drive'
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# 2006--2010
|
|
312
|
+
{
|
|
313
|
+
2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
|
|
314
|
+
2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
|
|
315
|
+
2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
|
|
316
|
+
2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
|
|
317
|
+
# 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
|
|
318
|
+
}.sort { |a, b| a.first <=> b.first }.each do |year, options|
|
|
319
|
+
import options.merge(:transform => { :class => FuelEconomyGuide::ParserD, :year => year },
|
|
320
|
+
:errata => errata) do
|
|
321
|
+
key 'row_hash'
|
|
322
|
+
store 'make_name', :field_name => 'make'
|
|
323
|
+
store 'model_name', :field_name => 'model'
|
|
324
|
+
store 'fuel_type_code', :field_name => 'FUEL TYPE'
|
|
325
|
+
store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
|
|
326
|
+
store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre # ditto
|
|
327
|
+
store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
328
|
+
store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
|
|
329
|
+
store 'cylinders', :field_name => 'NUMB CYL'
|
|
330
|
+
store 'displacement', :field_name => 'DISPLACEMENT'
|
|
331
|
+
store 'carline_class_code', :field_name => 'CLS'
|
|
332
|
+
store 'carline_class_name', :field_name => 'CLASS'
|
|
333
|
+
store 'year'
|
|
334
|
+
store 'transmission'
|
|
335
|
+
store 'speeds'
|
|
336
|
+
store 'turbo'
|
|
337
|
+
store 'supercharger'
|
|
338
|
+
store 'injection'
|
|
339
|
+
store 'drive'
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# associate :make, :key => :original_automobile_make_name, :foreign_key => :name
|
|
344
|
+
# derive :automobile_model_id # creates models by name
|
|
345
|
+
# associate :fuel_type, :key => :original_automobile_fuel_type_code, :foreign_key => :code
|
|
346
|
+
|
|
347
|
+
process 'Set adjusted fuel economy' do
|
|
348
|
+
update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
|
|
349
|
+
update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
|
|
350
|
+
end
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
def name
|
|
354
|
+
extra = []
|
|
355
|
+
extra << "V#{cylinders}" if cylinders
|
|
356
|
+
extra << "#{displacement}L" if displacement
|
|
357
|
+
extra << "turbo" if turbo
|
|
358
|
+
extra << "FI" if injection
|
|
359
|
+
extra << "#{speeds}spd" if speeds.present?
|
|
360
|
+
extra << transmission if transmission.present?
|
|
361
|
+
extra << "(#{fuel_type.name})" if fuel_type
|
|
362
|
+
extra.join(' ')
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
def fuel_economy_description
|
|
366
|
+
[ fuel_efficiency_city, fuel_efficiency_highway ].map { |f| f.kilometres_per_litre.to(:miles_per_gallon).round }.join('/')
|
|
367
|
+
end
|
|
368
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
class Country < ActiveRecord::Base
|
|
2
|
+
set_primary_key :iso_3166
|
|
3
|
+
|
|
4
|
+
data_miner do
|
|
5
|
+
import 'The official ISO country list', :url => 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';' do
|
|
6
|
+
key 'iso_3166', :field_number => 1
|
|
7
|
+
store 'name', :field_number => 0
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
import 'A Princeton dataset with better capitalization', :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do
|
|
11
|
+
key 'iso_3166', :field_name => 'country code'
|
|
12
|
+
store 'name', :field_name => 'country'
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
data/test/test_helper.rb
CHANGED
|
@@ -6,6 +6,8 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
|
6
6
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
7
7
|
require 'data_miner'
|
|
8
8
|
|
|
9
|
+
ENV['WIP'] = true if ENV['ALL'] == 'true'
|
|
10
|
+
|
|
9
11
|
ActiveRecord::Base.establish_connection(
|
|
10
12
|
'adapter' => 'mysql',
|
|
11
13
|
'database' => 'data_miner_test',
|
|
@@ -13,11 +15,12 @@ ActiveRecord::Base.establish_connection(
|
|
|
13
15
|
'password' => 'password'
|
|
14
16
|
)
|
|
15
17
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
Dir.glob(File.expand_path('support/*.rb', File.dirname(__FILE__))).each do |lib|
|
|
19
|
+
require lib
|
|
18
20
|
end
|
|
19
21
|
|
|
20
|
-
|
|
22
|
+
ActiveSupport::Inflector.inflections do |inflect|
|
|
23
|
+
inflect.uncountable %w{ aircraft aircraft_deux census_division_deux census_division_trois }
|
|
21
24
|
end
|
|
22
25
|
|
|
23
26
|
ActiveRecord::Schema.define(:version => 20090819143429) do
|
metadata
CHANGED
|
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
|
5
5
|
segments:
|
|
6
6
|
- 0
|
|
7
7
|
- 5
|
|
8
|
-
-
|
|
9
|
-
version: 0.5.
|
|
8
|
+
- 6
|
|
9
|
+
version: 0.5.6
|
|
10
10
|
platform: ruby
|
|
11
11
|
authors:
|
|
12
12
|
- Seamus Abshere
|
|
@@ -16,7 +16,7 @@ autorequire:
|
|
|
16
16
|
bindir: bin
|
|
17
17
|
cert_chain: []
|
|
18
18
|
|
|
19
|
-
date: 2010-
|
|
19
|
+
date: 2010-11-01 00:00:00 -04:00
|
|
20
20
|
default_executable:
|
|
21
21
|
dependencies:
|
|
22
22
|
- !ruby/object:Gem::Dependency
|
|
@@ -270,6 +270,12 @@ files:
|
|
|
270
270
|
- lib/data_miner/run.rb
|
|
271
271
|
- lib/data_miner/schema.rb
|
|
272
272
|
- lib/data_miner/tap.rb
|
|
273
|
+
- test/data_miner/attribute_test.rb
|
|
274
|
+
- test/support/airport.rb
|
|
275
|
+
- test/support/country.rb
|
|
276
|
+
- test/support/automobile_fuel_type.rb
|
|
277
|
+
- test/support/aircraft.rb
|
|
278
|
+
- test/support/automobile_variant.rb
|
|
273
279
|
- test/data_miner_test.rb
|
|
274
280
|
- test/test_helper.rb
|
|
275
281
|
has_rdoc: true
|
|
@@ -307,5 +313,11 @@ signing_key:
|
|
|
307
313
|
specification_version: 3
|
|
308
314
|
summary: Mine remote data into your ActiveRecord models.
|
|
309
315
|
test_files:
|
|
316
|
+
- test/data_miner/attribute_test.rb
|
|
317
|
+
- test/support/airport.rb
|
|
318
|
+
- test/support/country.rb
|
|
319
|
+
- test/support/automobile_fuel_type.rb
|
|
320
|
+
- test/support/aircraft.rb
|
|
321
|
+
- test/support/automobile_variant.rb
|
|
310
322
|
- test/data_miner_test.rb
|
|
311
323
|
- test/test_helper.rb
|