earth 0.5.4 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (131) hide show
  1. data/Gemfile +1 -0
  2. data/TODO +2 -0
  3. data/bin/earth_tester.rb +62 -9
  4. data/earth.gemspec +7 -4
  5. data/lib/earth.rb +18 -17
  6. data/lib/earth/air.rb +2 -0
  7. data/lib/earth/air/aircraft.rb +76 -19
  8. data/lib/earth/air/aircraft/data_miner.rb +1 -63
  9. data/lib/earth/air/aircraft_class.rb +48 -12
  10. data/lib/earth/air/aircraft_class/data_miner.rb +3 -46
  11. data/lib/earth/air/aircraft_fuel_use_equation.rb +11 -13
  12. data/lib/earth/air/aircraft_instance.rb +9 -0
  13. data/lib/earth/air/aircraft_instance_seat_class.rb +12 -0
  14. data/lib/earth/air/airline.rb +6 -8
  15. data/lib/earth/air/airline/data_miner.rb +3 -3
  16. data/lib/earth/air/airport.rb +8 -10
  17. data/lib/earth/air/airport/data_miner.rb +1 -1
  18. data/lib/earth/air/bts_aircraft.rb +3 -5
  19. data/lib/earth/air/flight_distance_class.rb +4 -6
  20. data/lib/earth/air/flight_seat_class.rb +5 -7
  21. data/lib/earth/air/flight_segment.rb +58 -42
  22. data/lib/earth/air/flight_segment/data_miner.rb +11 -26
  23. data/lib/earth/automobile.rb +1 -0
  24. data/lib/earth/automobile/automobile_fuel.rb +78 -25
  25. data/lib/earth/automobile/automobile_fuel/data_miner.rb +5 -72
  26. data/lib/earth/automobile/automobile_make.rb +14 -6
  27. data/lib/earth/automobile/automobile_make/data_miner.rb +20 -35
  28. data/lib/earth/automobile/automobile_make_fleet_year.rb +39 -10
  29. data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +0 -29
  30. data/lib/earth/automobile/automobile_make_model.rb +30 -9
  31. data/lib/earth/automobile/automobile_make_model/data_miner.rb +11 -26
  32. data/lib/earth/automobile/automobile_make_model_year.rb +41 -12
  33. data/lib/earth/automobile/automobile_make_model_year/data_miner.rb +14 -34
  34. data/lib/earth/automobile/automobile_make_model_year_variant.rb +56 -38
  35. data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb +15 -61
  36. data/lib/earth/automobile/automobile_make_year.rb +15 -8
  37. data/lib/earth/automobile/automobile_make_year/data_miner.rb +17 -51
  38. data/lib/earth/automobile/automobile_size_class.rb +78 -14
  39. data/lib/earth/automobile/automobile_size_class/data_miner.rb +0 -66
  40. data/lib/earth/automobile/automobile_size_class_year.rb +38 -10
  41. data/lib/earth/automobile/automobile_size_class_year/data_miner.rb +0 -30
  42. data/lib/earth/automobile/automobile_type_fuel_age.rb +60 -11
  43. data/lib/earth/automobile/automobile_type_fuel_age/data_miner.rb +10 -65
  44. data/lib/earth/automobile/automobile_type_fuel_control.rb +41 -10
  45. data/lib/earth/automobile/automobile_type_fuel_control/data_miner.rb +0 -33
  46. data/lib/earth/automobile/automobile_type_fuel_year.rb +60 -15
  47. data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb +1 -52
  48. data/lib/earth/automobile/automobile_type_fuel_year_age.rb +64 -12
  49. data/lib/earth/automobile/automobile_type_fuel_year_age/data_miner.rb +1 -59
  50. data/lib/earth/automobile/automobile_type_fuel_year_control.rb +31 -10
  51. data/lib/earth/automobile/automobile_type_fuel_year_control/data_miner.rb +2 -32
  52. data/lib/earth/automobile/automobile_type_year.rb +51 -10
  53. data/lib/earth/automobile/automobile_type_year/data_miner.rb +1 -43
  54. data/lib/earth/automobile/data_miner.rb +1 -0
  55. data/lib/earth/bus/bus_class.rb +108 -28
  56. data/lib/earth/bus/bus_class/data_miner.rb +0 -88
  57. data/lib/earth/bus/bus_fuel.rb +83 -15
  58. data/lib/earth/bus/bus_fuel/data_miner.rb +0 -76
  59. data/lib/earth/bus/bus_fuel_control.rb +8 -10
  60. data/lib/earth/bus/bus_fuel_year_control.rb +7 -9
  61. data/lib/earth/bus/bus_fuel_year_control/data_miner.rb +1 -5
  62. data/lib/earth/computation/computation_carrier.rb +11 -5
  63. data/lib/earth/computation/computation_carrier/data_miner.rb +0 -7
  64. data/lib/earth/computation/computation_carrier_instance_class.rb +25 -8
  65. data/lib/earth/computation/computation_carrier_instance_class/data_miner.rb +0 -18
  66. data/lib/earth/computation/computation_carrier_region.rb +5 -7
  67. data/lib/earth/diet/diet_class.rb +14 -16
  68. data/lib/earth/diet/food_group.rb +7 -9
  69. data/lib/earth/fuel/fuel.rb +14 -16
  70. data/lib/earth/fuel/fuel/data_miner.rb +7 -3
  71. data/lib/earth/fuel/fuel_price.rb +4 -6
  72. data/lib/earth/fuel/fuel_type.rb +12 -14
  73. data/lib/earth/fuel/fuel_year.rb +57 -16
  74. data/lib/earth/fuel/fuel_year/data_miner.rb +0 -43
  75. data/lib/earth/fuel/greenhouse_gas.rb +46 -9
  76. data/lib/earth/fuel/greenhouse_gas/data_miner.rb +0 -38
  77. data/lib/earth/hospitality/lodging_class.rb +10 -12
  78. data/lib/earth/industry/industry.rb +3 -5
  79. data/lib/earth/industry/industry_product.rb +6 -8
  80. data/lib/earth/industry/industry_product_line.rb +5 -7
  81. data/lib/earth/industry/industry_sector.rb +5 -7
  82. data/lib/earth/industry/merchant.rb +4 -6
  83. data/lib/earth/industry/merchant_category.rb +3 -5
  84. data/lib/earth/industry/merchant_category_industry.rb +5 -7
  85. data/lib/earth/industry/product_line.rb +5 -7
  86. data/lib/earth/industry/product_line_industry_product.rb +5 -7
  87. data/lib/earth/industry/sector.rb +5 -7
  88. data/lib/earth/locality/census_division.rb +21 -23
  89. data/lib/earth/locality/census_region.rb +3 -5
  90. data/lib/earth/locality/climate_division.rb +5 -7
  91. data/lib/earth/locality/country.rb +13 -15
  92. data/lib/earth/locality/egrid_region.rb +14 -5
  93. data/lib/earth/locality/egrid_region/data_miner.rb +0 -10
  94. data/lib/earth/locality/egrid_subregion.rb +71 -18
  95. data/lib/earth/locality/egrid_subregion/data_miner.rb +0 -59
  96. data/lib/earth/locality/petroleum_administration_for_defense_district.rb +6 -8
  97. data/lib/earth/locality/state.rb +6 -8
  98. data/lib/earth/locality/urbanity.rb +2 -4
  99. data/lib/earth/locality/urbanity/data_miner.rb +7 -3
  100. data/lib/earth/locality/zip_code.rb +8 -10
  101. data/lib/earth/pet/breed.rb +5 -7
  102. data/lib/earth/pet/breed_gender.rb +6 -8
  103. data/lib/earth/pet/gender.rb +2 -4
  104. data/lib/earth/pet/species.rb +15 -17
  105. data/lib/earth/rail/rail_class.rb +64 -14
  106. data/lib/earth/rail/rail_class/data_miner.rb +0 -51
  107. data/lib/earth/residence/air_conditioner_use.rb +4 -6
  108. data/lib/earth/residence/air_conditioner_use/data_miner.rb +7 -3
  109. data/lib/earth/residence/clothes_machine_use.rb +4 -6
  110. data/lib/earth/residence/clothes_machine_use/data_miner.rb +11 -9
  111. data/lib/earth/residence/dishwasher_use.rb +4 -6
  112. data/lib/earth/residence/dishwasher_use/data_miner.rb +11 -9
  113. data/lib/earth/residence/residence_appliance.rb +4 -6
  114. data/lib/earth/residence/residence_class.rb +2 -4
  115. data/lib/earth/residence/residence_class/data_miner.rb +7 -3
  116. data/lib/earth/residence/residence_fuel_price.rb +12 -14
  117. data/lib/earth/residence/residence_fuel_price/data_miner.rb +7 -0
  118. data/lib/earth/residence/residence_fuel_type.rb +6 -8
  119. data/lib/earth/residence/residential_energy_consumption_survey_response.rb +91 -93
  120. data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +9 -8
  121. data/lib/earth/shipping/carrier.rb +57 -10
  122. data/lib/earth/shipping/carrier/data_miner.rb +0 -48
  123. data/lib/earth/shipping/carrier_mode.rb +41 -10
  124. data/lib/earth/shipping/carrier_mode/data_miner.rb +0 -33
  125. data/lib/earth/shipping/shipment_mode.rb +30 -7
  126. data/lib/earth/shipping/shipment_mode/data_miner.rb +0 -23
  127. data/lib/earth/utils.rb +45 -0
  128. data/lib/earth/version.rb +1 -1
  129. data/spec/earth/air/aircraft_spec.rb +1 -1
  130. data/vendor/clean_find_in_batches/init.rb +35 -0
  131. metadata +74 -47
@@ -9,9 +9,19 @@ AutomobileMakeModelYear.class_eval do
9
9
  end
10
10
 
11
11
  process "Derive model year names from automobile make model year variants" do
12
- INSERT_IGNORE %{INTO automobile_make_model_years(name, make_name, model_name, make_model_name, year, make_year_name)
13
- SELECT automobile_make_model_year_variants.make_model_year_name, automobile_make_model_year_variants.make_name, automobile_make_model_year_variants.name, automobile_make_model_year_variants.make_model_name, automobile_make_model_year_variants.year, automobile_make_model_year_variants.make_year_name FROM automobile_make_model_year_variants WHERE LENGTH(automobile_make_model_year_variants.make_name) > 0 AND LENGTH(automobile_make_model_year_variants.make_model_name) > 0
14
- }
12
+ ::Earth::Utils.insert_ignore(
13
+ :src => AutomobileMakeModelYearVariant,
14
+ :dest => AutomobileMakeModelYear,
15
+ :cols => {
16
+ :make_model_year_name => :name,
17
+ :make_name => :make_name,
18
+ :name => :model_name,
19
+ :make_model_name => :make_model_name,
20
+ :year => :year,
21
+ :make_year_name => :make_year_name
22
+ }
23
+ # :where => 'LENGTH(src.make_name) > 0 AND LENGTH(src.make_model_name) > 0'
24
+ )
15
25
  end
16
26
 
17
27
  # FIXME TODO make this a method on AutomobileMakeModelYear?
@@ -28,37 +38,7 @@ AutomobileMakeModelYear.class_eval do
28
38
  # sabshere 12/6/10 careful, don't use AutomobileMakeModelYearVariant.where here or you will be forced into projecting *
29
39
  relation = variants.project(variants[:"fuel_efficiency_#{i}"].average).where(conditional_relation).where(null_check)
30
40
  update_all "fuel_efficiency_#{i} = (#{relation.to_sql})"
31
- update_all "fuel_efficiency_#{i}_units = 'kilometres_per_litre'"
32
- end
33
- end
34
-
35
- verify "Year should be from 1985 to 2011" do
36
- AutomobileMakeModelYear.all.each do |model_year|
37
- unless model_year.year.to_i > 1984 and model_year.year.to_i < 2012
38
- raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be from 1985 to 2011)"
39
- end
40
- end
41
- end
42
-
43
- verify "Fuel efficiencies should be greater than zero" do
44
- AutomobileMakeModelYear.all.each do |model_year|
45
- %w{ city highway }.each do |type|
46
- fuel_efficiency = model_year.send(:"fuel_efficiency_#{type}")
47
- unless fuel_efficiency.to_f > 0
48
- raise "Invalid fuel efficiency #{type} for AutomobileMakeModelYear #{model_year.name}: #{fuel_efficiency} (should be > 0)"
49
- end
50
- end
51
- end
52
- end
53
-
54
- verify "Fuel efficiency units should be kilometres per litre" do
55
- AutomobileMakeModelYear.all.each do |model_year|
56
- %w{ city highway }.each do |type|
57
- units = model_year.send(:"fuel_efficiency_#{type}_units")
58
- unless units == "kilometres_per_litre"
59
- raise "Invalid fuel efficiency #{type} units for AutomobileMakeModelYear #{model_year.name}: #{units} (should be kilometres_per_litre)"
60
- end
61
- end
41
+ update_all "fuel_efficiency_#{i}_units" => 'kilometres_per_litre'
62
42
  end
63
43
  end
64
44
  end
@@ -6,41 +6,59 @@ class AutomobileMakeModelYearVariant < ActiveRecord::Base
6
6
  belongs_to :make_model_year, :class_name => 'AutomobileMakeModelYear', :foreign_key => 'make_model_year_name'
7
7
  belongs_to :fuel, :class_name => 'AutomobileFuel', :foreign_key => 'fuel_code', :primary_key => 'code'
8
8
 
9
- force_schema do
10
- string 'row_hash'
11
- string 'name' # short name!
12
- string 'make_name'
13
- string 'make_model_name' # make + model
14
- string 'make_year_name' # make + year
15
- string 'make_model_year_name' # make + model + year
16
- integer 'year'
17
- float 'fuel_efficiency'
18
- string 'fuel_efficiency_units'
19
- float 'fuel_efficiency_city'
20
- string 'fuel_efficiency_city_units'
21
- float 'fuel_efficiency_highway'
22
- string 'fuel_efficiency_highway_units'
23
- string 'fuel_code'
24
- string 'transmission'
25
- string 'drive'
26
- boolean 'turbo'
27
- boolean 'supercharger'
28
- integer 'cylinders'
29
- float 'displacement'
30
- float 'raw_fuel_efficiency_city'
31
- string 'raw_fuel_efficiency_city_units'
32
- float 'raw_fuel_efficiency_highway'
33
- string 'raw_fuel_efficiency_highway_units'
34
- integer 'carline_mfr_code'
35
- integer 'vi_mfr_code'
36
- integer 'carline_code'
37
- integer 'carline_class_code'
38
- boolean 'injection'
39
- string 'carline_class_name'
40
- string 'speeds'
41
- index 'make_name'
42
- index 'make_model_name'
43
- index 'make_year_name'
44
- index 'make_model_year_name'
45
- end
46
- end
9
+ col :row_hash
10
+ col :name # short name!
11
+ col :make_name
12
+ col :make_model_name # make + model
13
+ col :make_year_name # make + year
14
+ col :make_model_year_name # make + model + year
15
+ col :year, :type => :integer
16
+ col :fuel_efficiency, :type => :float
17
+ col :fuel_efficiency_units
18
+ col :fuel_efficiency_city, :type => :float
19
+ col :fuel_efficiency_city_units
20
+ col :fuel_efficiency_highway, :type => :float
21
+ col :fuel_efficiency_highway_units
22
+ col :fuel_code
23
+ col :transmission
24
+ col :drive
25
+ col :turbo, :type => :boolean
26
+ col :supercharger, :type => :boolean
27
+ col :cylinders, :type => :integer
28
+ col :displacement, :type => :float
29
+ col :raw_fuel_efficiency_city, :type => :float
30
+ col :raw_fuel_efficiency_city_units
31
+ col :raw_fuel_efficiency_highway, :type => :float
32
+ col :raw_fuel_efficiency_highway_units
33
+ col :carline_mfr_code, :type => :integer
34
+ col :vi_mfr_code, :type => :integer
35
+ col :carline_code, :type => :integer
36
+ col :carline_class_code, :type => :integer
37
+ col :injection, :type => :boolean
38
+ col :carline_class_name
39
+ col :speeds
40
+ add_index :make_name
41
+ add_index :make_model_name
42
+ add_index :make_year_name
43
+ add_index :make_model_year_name
44
+
45
+ # verify "Fuel code should appear in AutomobileFuel" do
46
+ # if (violators = connection.select_values("SELECT DISTINCT fuel_code FROM #{quoted_table_name} WHERE fuel_code NOT IN (SELECT code FROM #{AutomobileFuel.quoted_table_name})")).any?
47
+ # raise "Invalid fuel code(s): #{violators.join(', ')}"
48
+ # end
49
+ # true
50
+ # end
51
+ #
52
+ # verify "Fuel efficiencies should be greater than zero" do
53
+ # [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |field|
54
+ # if AutomobileMakeModelYearVariant.where(field => nil).any?
55
+ # raise "Invalid #{field} in automobile_make_model_year_variants: nil is not > 0"
56
+ # else
57
+ # min = AutomobileMakeModelYearVariant.minimum(field)
58
+ # unless min > 0
59
+ # raise "Invalid #{field} in automobile_make_model_year_variants: #{min} is not > 0"
60
+ # end
61
+ # end
62
+ # end
63
+ # end
64
+ end
@@ -314,7 +314,7 @@ AutomobileMakeModelYearVariant.class_eval do
314
314
  2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
315
315
  2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
316
316
  2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
317
- }.sort { |a, b| a.first <=> b.first }.each do |year, options|
317
+ }.each do |year, options|
318
318
  import "#{ year } Fuel Economy Guide",
319
319
  options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserC, :year => year },
320
320
  :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
@@ -348,7 +348,7 @@ AutomobileMakeModelYearVariant.class_eval do
348
348
  2007 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/2007_FE_guide_ALL_no_sales_May_01_2007.csv' },
349
349
  2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
350
350
  2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
351
- }.sort { |a, b| a.first <=> b.first }.each do |year, options|
351
+ }.each do |year, options|
352
352
  import "#{ year } Fuel Economy Guide",
353
353
  options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserD, :year => year },
354
354
  :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
@@ -379,7 +379,7 @@ AutomobileMakeModelYearVariant.class_eval do
379
379
  {
380
380
  2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEGuide for DOE-all rel dates-no-sales-02-22-2011public.xlsx' },
381
381
  2011 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/11data.zip', :filename => '2011FEGuide-for DOE rel-dates before 1-23-2011-no-sales-01-10-2011_All_public.xlsx' }
382
- }.sort { |a, b| a.first <=> b.first }.each do |year, options|
382
+ }.each do |year, options|
383
383
  import "#{ year } Fuel Economy Guide",
384
384
  options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserE, :year => year },
385
385
  :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
@@ -407,22 +407,16 @@ AutomobileMakeModelYearVariant.class_eval do
407
407
  end
408
408
 
409
409
  process "Derive model and model year names" do
410
- if ActiveRecord::Base.connection.adapter_name == 'sqlite'
411
- update_all "make_model_name = make_name || ' ' || name"
412
- update_all "make_year_name = make_name || ' ' || year"
413
- update_all "make_model_year_name = make_name || ' ' || name || ' ' || year"
414
- else
415
- update_all "make_model_name = CONCAT(make_name, ' ', name)"
416
- update_all "make_year_name = CONCAT(make_name, ' ', year)"
417
- update_all "make_model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
418
- end
410
+ update_all "make_model_name = make_name || ' ' || name"
411
+ update_all "make_year_name = make_name || ' ' || year"
412
+ update_all "make_model_year_name = make_name || ' ' || name || ' ' || year"
419
413
  end
420
414
 
421
415
  # Note: need to divide by 0.425143707 b/c equation is designed for miles / gallon not km / l
422
416
  # Note: EPA seems to adjust differently for plug-in hybrid electric vehicles (e.g. Leaf and Volt)
423
417
  process "Calculate adjusted fuel efficiency using the latest EPA equations from EPA Fuel Economy Trends report Appendix A including conversion from miles per gallon to kilometres per litre" do
424
- update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
425
- update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
418
+ update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))', 'raw_fuel_efficiency_city > 0'
419
+ update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))', 'raw_fuel_efficiency_highway > 0'
426
420
  end
427
421
 
428
422
  # This will be useful later for calculating MakeModel and Make fuel efficiency based on Variant
@@ -430,58 +424,18 @@ AutomobileMakeModelYearVariant.class_eval do
430
424
  # This results in a deviation from EPA fuel economy label values which use a historical 55/45 weighting
431
425
  process "Calculate combined adjusted fuel efficiency using the latest EPA equation" do
432
426
  update_all "fuel_efficiency = 1 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway))"
433
- update_all "fuel_efficiency_units = 'kilometres_per_litre'"
434
427
  end
435
428
 
436
- # FIXME TODO this step fails if the tables to be synthesized do not already exist
437
- %w{ AutomobileMakeModelYear AutomobileMakeModel }.each do |synthetic_resource|
438
- process "Synthesize #{synthetic_resource}" do
439
- synthetic_resource.constantize.run_data_miner!
440
- end
429
+ process "Set units" do
430
+ update_all :fuel_efficiency_units => 'kilometres_per_litre'
431
+ update_all :fuel_efficiency_city_units => 'kilometres_per_litre'
432
+ update_all :fuel_efficiency_highway_units => 'kilometres_per_litre'
441
433
  end
442
434
 
443
- verify "Year should be from 1985 to 2011" do
444
- connection.select_values("SELECT DISTINCT year FROM automobile_make_model_year_variants").each do |year|
445
- unless year > 1984 and year < 2012
446
- raise "Invalid year in automobile_make_model_year_variants: #{year} is not from 1985 to 2011"
447
- end
448
- end
449
- end
450
-
451
- process "Ensure AutomobileFuel is populated" do
435
+ process "Ensure related tables are populated" do
436
+ AutomobileMakeModelYear.run_data_miner!
437
+ AutomobileMakeModel.run_data_miner!
452
438
  AutomobileFuel.run_data_miner!
453
439
  end
454
-
455
- verify "Fuel code should appear in AutomobileFuel" do
456
- valid_codes = connection.select_values("SELECT DISTINCT code FROM automobile_fuels")
457
- connection.select_values("SELECT DISTINCT fuel_code FROM automobile_make_model_year_variants").each do |fuel_code|
458
- unless valid_codes.include?(fuel_code)
459
- raise "Invalide fuel code in automobile_make_model_year_variants: #{fuel_code} is not in #{valid_codes}"
460
- end
461
- end
462
- end
463
-
464
- verify "Fuel efficiencies should be greater than zero" do
465
- [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |field|
466
- if AutomobileMakeModelYearVariant.where(field => nil).any?
467
- raise "Invalid #{field} in automobile_make_model_year_variants: nil is not > 0"
468
- else
469
- min = AutomobileMakeModelYearVariant.minimum(field)
470
- unless min > 0
471
- raise "Invalid #{field} in automobile_make_model_year_variants: #{min} is not > 0"
472
- end
473
- end
474
- end
475
- end
476
-
477
- verify "Fuel efficiency units should be kilometres per litre" do
478
- %w{ fuel_efficiency_units fuel_efficiency_city_units fuel_efficiency_highway_units }.each do |field|
479
- connection.select_values("SELECT DISTINCT #{field} FROM automobile_make_model_year_variants").each do |value|
480
- unless value == 'kilometres_per_litre'
481
- raise "Invalid #{field} in automobile_make_model_year_variants: #{value} is not 'kilometres_per_litre'"
482
- end
483
- end
484
- end
485
- end
486
440
  end
487
441
  end
@@ -4,12 +4,19 @@ class AutomobileMakeYear < ActiveRecord::Base
4
4
  belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
5
5
  has_many :fleet_years, :class_name => 'AutomobileMakeFleetYear', :foreign_key => 'make_year_name'
6
6
 
7
- force_schema do
8
- string 'name'
9
- string 'make_name'
10
- integer 'year'
11
- float 'fuel_efficiency'
12
- string 'fuel_efficiency_units'
13
- integer 'volume' # This will sometimes be null because not all make_years have CAFE data
14
- end
7
+ col :name
8
+ col :make_name
9
+ col :year, :type => :integer
10
+ col :fuel_efficiency, :type => :float
11
+ col :fuel_efficiency_units
12
+ col :volume, :type => :integer # This will sometimes be null because not all make_years have CAFE data
13
+
14
+ # FIXME TODO verify make_name and volume?
15
+
16
+ # verify "Fuel efficiency should be greater than zero" do
17
+ # if violator = first(:conditions => ['fuel_efficiency IS NULL OR fuel_efficiency <= 0'])
18
+ # raise "Invalid fuel efficiency for AutomobileMakeYear #{record.name}: #{record.fuel_efficiency} (should be > 0)"
19
+ # end
20
+ # true
21
+ # end
15
22
  end
@@ -10,19 +10,16 @@ AutomobileMakeYear.class_eval do
10
10
  end
11
11
 
12
12
  process "Derive manufacturer names and years from automobile make model year variants" do
13
- INSERT_IGNORE %{INTO automobile_make_years(name, make_name, year)
14
- SELECT
15
- automobile_make_model_year_variants.make_year_name,
16
- automobile_make_model_year_variants.make_name,
17
- automobile_make_model_year_variants.year
18
- FROM automobile_make_model_year_variants
19
- WHERE automobile_make_model_year_variants.make_year_name IS NOT NULL
20
- AND LENGTH(automobile_make_model_year_variants.make_year_name) > 0
21
- AND automobile_make_model_year_variants.make_name IS NOT NULL
22
- AND LENGTH(automobile_make_model_year_variants.make_name) > 0
23
- AND automobile_make_model_year_variants.year IS NOT NULL
24
- AND LENGTH(automobile_make_model_year_variants.year) > 0
25
- }
13
+ ::Earth::Utils.insert_ignore(
14
+ :src => AutomobileMakeModelYearVariant,
15
+ :dest => AutomobileMakeYear,
16
+ :cols => {
17
+ :make_year_name => :name,
18
+ :make_name => :make_name,
19
+ :year => :year
20
+ }
21
+ # :where => 'src.make_name IS NOT NULL AND LENGTH(src.make_name) > 0 AND src.year IS NOT NULL AND LENGTH(src.year) > 0'
22
+ )
26
23
  end
27
24
 
28
25
  # FIXME TODO make this a method on AutomobileMakeYear?
@@ -32,52 +29,21 @@ AutomobileMakeYear.class_eval do
32
29
  conditional_relation = make_years[:name].eq(make_fleet_years[:make_year_name])
33
30
  relation = AutomobileMakeFleetYear.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
34
31
  update_all "fuel_efficiency = (#{relation.to_sql})"
35
- update_all "fuel_efficiency_units = 'kilometres_per_litre'"
36
32
  end
37
33
 
38
34
  process "Calculate fuel effeciency from automobile make model year variants for makes without CAFE data" do
39
- connection.execute %{
40
- UPDATE automobile_make_years
41
- SET automobile_make_years.fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_make_years.name = automobile_make_model_year_variants.make_year_name)
42
- WHERE automobile_make_years.fuel_efficiency IS NULL
43
- }
44
- connection.execute %{
45
- UPDATE automobile_make_years
46
- SET automobile_make_years.fuel_efficiency_units = 'kilometres_per_litre'
47
- WHERE automobile_make_years.fuel_efficiency_units IS NULL
48
- }
35
+ update_all(
36
+ %{fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_make_years.name = automobile_make_model_year_variants.make_year_name)},
37
+ %{fuel_efficiency IS NULL}
38
+ )
49
39
  end
50
40
 
51
41
  process "Calculate sales volume from automobile make fleet years for makes with CAFE data" do
52
- connection.execute %{
53
- UPDATE automobile_make_years SET automobile_make_years.volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)
54
- }
55
- end
56
-
57
- # FIXME TODO verify make_name and volume?
58
-
59
- verify "Year should be from 1985 to 2010" do
60
- AutomobileMakeYear.all.each do |make_year|
61
- unless make_year.year > 1984 and make_year.year < 2011
62
- raise "Invalid year for AutomobileMakeYear #{make_year.name}: #{make_year.year} (should be from 1985 to 2010)"
63
- end
64
- end
42
+ update_all %{volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)}
65
43
  end
66
44
 
67
- verify "Fuel efficiency should be greater than zero" do
68
- AutomobileMakeYear.all.each do |make_year|
69
- unless make_year.fuel_efficiency > 0
70
- raise "Invalid fuel efficiency for AutomobileMakeYear #{make_year.name}: #{make_year.fuel_efficiency} (should be > 0)"
71
- end
72
- end
73
- end
74
-
75
- verify "Fuel efficiency units should be kilometres per litre" do
76
- AutomobileMakeYear.all.each do |make_year|
77
- unless make_year.fuel_efficiency_units == "kilometres_per_litre"
78
- raise "Invalid fuel efficiency units for AutomobileMakeYear #{make_year.name}: #{make_year.fuel_efficiency_units} (should be kilometres_per_litre)"
79
- end
80
- end
45
+ process 'Set units' do
46
+ update_all "fuel_efficiency_units = 'kilometres_per_litre'"
81
47
  end
82
48
  end
83
49
  end
@@ -6,18 +6,82 @@ class AutomobileSizeClass < ActiveRecord::Base
6
6
  :conventional_fuel_efficiency_city_multiplier => 0.987,
7
7
  :conventional_fuel_efficiency_highway_multiplier => 0.996
8
8
 
9
- force_schema do
10
- string 'name'
11
- string 'type_name'
12
- float 'annual_distance'
13
- string 'annual_distance_units'
14
- float 'fuel_efficiency_city'
15
- string 'fuel_efficiency_city_units'
16
- float 'fuel_efficiency_highway'
17
- string 'fuel_efficiency_highway_units'
18
- float 'hybrid_fuel_efficiency_city_multiplier'
19
- float 'hybrid_fuel_efficiency_highway_multiplier'
20
- float 'conventional_fuel_efficiency_city_multiplier'
21
- float 'conventional_fuel_efficiency_highway_multiplier'
22
- end
9
+ col :name
10
+ col :type_name
11
+ col :annual_distance, :type => :float
12
+ col :annual_distance_units
13
+ col :fuel_efficiency_city, :type => :float
14
+ col :fuel_efficiency_city_units
15
+ col :fuel_efficiency_highway, :type => :float
16
+ col :fuel_efficiency_highway_units
17
+ col :hybrid_fuel_efficiency_city_multiplier, :type => :float
18
+ col :hybrid_fuel_efficiency_highway_multiplier, :type => :float
19
+ col :conventional_fuel_efficiency_city_multiplier, :type => :float
20
+ col :conventional_fuel_efficiency_highway_multiplier, :type => :float
21
+
22
+ # FIXME TODO verify type_name?
23
+
24
+ # verify "Annual distance should be greater than zero" do
25
+ # AutomobileSizeClass.all.each do |size_class|
26
+ # unless size_class.annual_distance > 0
27
+ # raise "Invalid annual distance for AutomobileSizeClass #{size_class.name}: #{size_class.annual_distance} (should be > 0)"
28
+ # end
29
+ # end
30
+ # end
31
+ #
32
+ # verify "Annual distance units should be kilometres" do
33
+ # AutomobileSizeClass.all.each do |size_class|
34
+ # unless size_class.annual_distance_units == "kilometres"
35
+ # raise "Invalid annual distance units for AutomobileSizeClass #{size_class.name}: #{size_class.annual_distance_units} (should be kilometres)"
36
+ # end
37
+ # end
38
+ # end
39
+ #
40
+ # verify "Fuel efficiencies should be greater than zero" do
41
+ # AutomobileSizeClass.all.each do |size_class|
42
+ # %w{ city highway }.each do |type|
43
+ # fuel_efficiency = size_class.send(:"fuel_efficiency_#{type}")
44
+ # unless fuel_efficiency > 0
45
+ # raise "Invalid fuel efficiency #{type} for AutomobileSizeClass #{size_class.name}: #{fuel_efficiency} (should be > 0)"
46
+ # end
47
+ # end
48
+ # end
49
+ # end
50
+ #
51
+ # verify "Fuel efficiency units should be kilometres per litre" do
52
+ # AutomobileSizeClass.all.each do |size_class|
53
+ # %w{ city highway }.each do |type|
54
+ # units = size_class.send(:"fuel_efficiency_#{type}_units")
55
+ # unless units == "kilometres_per_litre"
56
+ # raise "Invalid fuel efficiency #{type} units for AutomobileSizeClass #{size_class.name}: #{units} (should be kilometres_per_litre)"
57
+ # end
58
+ # end
59
+ # end
60
+ # end
61
+ #
62
+ # verify "Any fuel efficiency multipliers should be greater than zero" do
63
+ # AutomobileSizeClass.all.each do |size_class|
64
+ # %w{ hybrid conventional }.each do |hybridity|
65
+ # %w{ city highway }.each do |type|
66
+ # multiplier = size_class.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
67
+ # if multiplier.present?
68
+ # unless multiplier > 0
69
+ # raise "Invalid #{hybridity} fuel efficiency #{type} multiplier for AutomobileSizeClass #{size_class.name}: #{multiplier} (should be > 0)"
70
+ # end
71
+ # end
72
+ # end
73
+ # end
74
+ # end
75
+ # end
76
+ #
77
+ # verify "Fallback fuel efficiency multipliers should be greater than zero" do
78
+ # %w{ hybrid conventional }.each do |hybridity|
79
+ # %w{ city highway }.each do |type|
80
+ # multiplier = AutomobileSizeClass.fallback.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
81
+ # unless multiplier > 0
82
+ # raise "Invalid AutomobileSizeClass fallback #{hybridity} fuel efficiency #{type} multiplier: #{multiplier} (should be > 0)"
83
+ # end
84
+ # end
85
+ # end
86
+ # end
23
87
  end