earth 0.5.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. data/Gemfile +1 -0
  2. data/TODO +2 -0
  3. data/bin/earth_tester.rb +62 -9
  4. data/earth.gemspec +7 -4
  5. data/lib/earth.rb +18 -17
  6. data/lib/earth/air.rb +2 -0
  7. data/lib/earth/air/aircraft.rb +76 -19
  8. data/lib/earth/air/aircraft/data_miner.rb +1 -63
  9. data/lib/earth/air/aircraft_class.rb +48 -12
  10. data/lib/earth/air/aircraft_class/data_miner.rb +3 -46
  11. data/lib/earth/air/aircraft_fuel_use_equation.rb +11 -13
  12. data/lib/earth/air/aircraft_instance.rb +9 -0
  13. data/lib/earth/air/aircraft_instance_seat_class.rb +12 -0
  14. data/lib/earth/air/airline.rb +6 -8
  15. data/lib/earth/air/airline/data_miner.rb +3 -3
  16. data/lib/earth/air/airport.rb +8 -10
  17. data/lib/earth/air/airport/data_miner.rb +1 -1
  18. data/lib/earth/air/bts_aircraft.rb +3 -5
  19. data/lib/earth/air/flight_distance_class.rb +4 -6
  20. data/lib/earth/air/flight_seat_class.rb +5 -7
  21. data/lib/earth/air/flight_segment.rb +58 -42
  22. data/lib/earth/air/flight_segment/data_miner.rb +11 -26
  23. data/lib/earth/automobile.rb +1 -0
  24. data/lib/earth/automobile/automobile_fuel.rb +78 -25
  25. data/lib/earth/automobile/automobile_fuel/data_miner.rb +5 -72
  26. data/lib/earth/automobile/automobile_make.rb +14 -6
  27. data/lib/earth/automobile/automobile_make/data_miner.rb +20 -35
  28. data/lib/earth/automobile/automobile_make_fleet_year.rb +39 -10
  29. data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +0 -29
  30. data/lib/earth/automobile/automobile_make_model.rb +30 -9
  31. data/lib/earth/automobile/automobile_make_model/data_miner.rb +11 -26
  32. data/lib/earth/automobile/automobile_make_model_year.rb +41 -12
  33. data/lib/earth/automobile/automobile_make_model_year/data_miner.rb +14 -34
  34. data/lib/earth/automobile/automobile_make_model_year_variant.rb +56 -38
  35. data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb +15 -61
  36. data/lib/earth/automobile/automobile_make_year.rb +15 -8
  37. data/lib/earth/automobile/automobile_make_year/data_miner.rb +17 -51
  38. data/lib/earth/automobile/automobile_size_class.rb +78 -14
  39. data/lib/earth/automobile/automobile_size_class/data_miner.rb +0 -66
  40. data/lib/earth/automobile/automobile_size_class_year.rb +38 -10
  41. data/lib/earth/automobile/automobile_size_class_year/data_miner.rb +0 -30
  42. data/lib/earth/automobile/automobile_type_fuel_age.rb +60 -11
  43. data/lib/earth/automobile/automobile_type_fuel_age/data_miner.rb +10 -65
  44. data/lib/earth/automobile/automobile_type_fuel_control.rb +41 -10
  45. data/lib/earth/automobile/automobile_type_fuel_control/data_miner.rb +0 -33
  46. data/lib/earth/automobile/automobile_type_fuel_year.rb +60 -15
  47. data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb +1 -52
  48. data/lib/earth/automobile/automobile_type_fuel_year_age.rb +64 -12
  49. data/lib/earth/automobile/automobile_type_fuel_year_age/data_miner.rb +1 -59
  50. data/lib/earth/automobile/automobile_type_fuel_year_control.rb +31 -10
  51. data/lib/earth/automobile/automobile_type_fuel_year_control/data_miner.rb +2 -32
  52. data/lib/earth/automobile/automobile_type_year.rb +51 -10
  53. data/lib/earth/automobile/automobile_type_year/data_miner.rb +1 -43
  54. data/lib/earth/automobile/data_miner.rb +1 -0
  55. data/lib/earth/bus/bus_class.rb +108 -28
  56. data/lib/earth/bus/bus_class/data_miner.rb +0 -88
  57. data/lib/earth/bus/bus_fuel.rb +83 -15
  58. data/lib/earth/bus/bus_fuel/data_miner.rb +0 -76
  59. data/lib/earth/bus/bus_fuel_control.rb +8 -10
  60. data/lib/earth/bus/bus_fuel_year_control.rb +7 -9
  61. data/lib/earth/bus/bus_fuel_year_control/data_miner.rb +1 -5
  62. data/lib/earth/computation/computation_carrier.rb +11 -5
  63. data/lib/earth/computation/computation_carrier/data_miner.rb +0 -7
  64. data/lib/earth/computation/computation_carrier_instance_class.rb +25 -8
  65. data/lib/earth/computation/computation_carrier_instance_class/data_miner.rb +0 -18
  66. data/lib/earth/computation/computation_carrier_region.rb +5 -7
  67. data/lib/earth/diet/diet_class.rb +14 -16
  68. data/lib/earth/diet/food_group.rb +7 -9
  69. data/lib/earth/fuel/fuel.rb +14 -16
  70. data/lib/earth/fuel/fuel/data_miner.rb +7 -3
  71. data/lib/earth/fuel/fuel_price.rb +4 -6
  72. data/lib/earth/fuel/fuel_type.rb +12 -14
  73. data/lib/earth/fuel/fuel_year.rb +57 -16
  74. data/lib/earth/fuel/fuel_year/data_miner.rb +0 -43
  75. data/lib/earth/fuel/greenhouse_gas.rb +46 -9
  76. data/lib/earth/fuel/greenhouse_gas/data_miner.rb +0 -38
  77. data/lib/earth/hospitality/lodging_class.rb +10 -12
  78. data/lib/earth/industry/industry.rb +3 -5
  79. data/lib/earth/industry/industry_product.rb +6 -8
  80. data/lib/earth/industry/industry_product_line.rb +5 -7
  81. data/lib/earth/industry/industry_sector.rb +5 -7
  82. data/lib/earth/industry/merchant.rb +4 -6
  83. data/lib/earth/industry/merchant_category.rb +3 -5
  84. data/lib/earth/industry/merchant_category_industry.rb +5 -7
  85. data/lib/earth/industry/product_line.rb +5 -7
  86. data/lib/earth/industry/product_line_industry_product.rb +5 -7
  87. data/lib/earth/industry/sector.rb +5 -7
  88. data/lib/earth/locality/census_division.rb +21 -23
  89. data/lib/earth/locality/census_region.rb +3 -5
  90. data/lib/earth/locality/climate_division.rb +5 -7
  91. data/lib/earth/locality/country.rb +13 -15
  92. data/lib/earth/locality/egrid_region.rb +14 -5
  93. data/lib/earth/locality/egrid_region/data_miner.rb +0 -10
  94. data/lib/earth/locality/egrid_subregion.rb +71 -18
  95. data/lib/earth/locality/egrid_subregion/data_miner.rb +0 -59
  96. data/lib/earth/locality/petroleum_administration_for_defense_district.rb +6 -8
  97. data/lib/earth/locality/state.rb +6 -8
  98. data/lib/earth/locality/urbanity.rb +2 -4
  99. data/lib/earth/locality/urbanity/data_miner.rb +7 -3
  100. data/lib/earth/locality/zip_code.rb +8 -10
  101. data/lib/earth/pet/breed.rb +5 -7
  102. data/lib/earth/pet/breed_gender.rb +6 -8
  103. data/lib/earth/pet/gender.rb +2 -4
  104. data/lib/earth/pet/species.rb +15 -17
  105. data/lib/earth/rail/rail_class.rb +64 -14
  106. data/lib/earth/rail/rail_class/data_miner.rb +0 -51
  107. data/lib/earth/residence/air_conditioner_use.rb +4 -6
  108. data/lib/earth/residence/air_conditioner_use/data_miner.rb +7 -3
  109. data/lib/earth/residence/clothes_machine_use.rb +4 -6
  110. data/lib/earth/residence/clothes_machine_use/data_miner.rb +11 -9
  111. data/lib/earth/residence/dishwasher_use.rb +4 -6
  112. data/lib/earth/residence/dishwasher_use/data_miner.rb +11 -9
  113. data/lib/earth/residence/residence_appliance.rb +4 -6
  114. data/lib/earth/residence/residence_class.rb +2 -4
  115. data/lib/earth/residence/residence_class/data_miner.rb +7 -3
  116. data/lib/earth/residence/residence_fuel_price.rb +12 -14
  117. data/lib/earth/residence/residence_fuel_price/data_miner.rb +7 -0
  118. data/lib/earth/residence/residence_fuel_type.rb +6 -8
  119. data/lib/earth/residence/residential_energy_consumption_survey_response.rb +91 -93
  120. data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +9 -8
  121. data/lib/earth/shipping/carrier.rb +57 -10
  122. data/lib/earth/shipping/carrier/data_miner.rb +0 -48
  123. data/lib/earth/shipping/carrier_mode.rb +41 -10
  124. data/lib/earth/shipping/carrier_mode/data_miner.rb +0 -33
  125. data/lib/earth/shipping/shipment_mode.rb +30 -7
  126. data/lib/earth/shipping/shipment_mode/data_miner.rb +0 -23
  127. data/lib/earth/utils.rb +45 -0
  128. data/lib/earth/version.rb +1 -1
  129. data/spec/earth/air/aircraft_spec.rb +1 -1
  130. data/vendor/clean_find_in_batches/init.rb +35 -0
  131. metadata +74 -47
@@ -9,9 +9,19 @@ AutomobileMakeModelYear.class_eval do
9
9
  end
10
10
 
11
11
  process "Derive model year names from automobile make model year variants" do
12
- INSERT_IGNORE %{INTO automobile_make_model_years(name, make_name, model_name, make_model_name, year, make_year_name)
13
- SELECT automobile_make_model_year_variants.make_model_year_name, automobile_make_model_year_variants.make_name, automobile_make_model_year_variants.name, automobile_make_model_year_variants.make_model_name, automobile_make_model_year_variants.year, automobile_make_model_year_variants.make_year_name FROM automobile_make_model_year_variants WHERE LENGTH(automobile_make_model_year_variants.make_name) > 0 AND LENGTH(automobile_make_model_year_variants.make_model_name) > 0
14
- }
12
+ ::Earth::Utils.insert_ignore(
13
+ :src => AutomobileMakeModelYearVariant,
14
+ :dest => AutomobileMakeModelYear,
15
+ :cols => {
16
+ :make_model_year_name => :name,
17
+ :make_name => :make_name,
18
+ :name => :model_name,
19
+ :make_model_name => :make_model_name,
20
+ :year => :year,
21
+ :make_year_name => :make_year_name
22
+ }
23
+ # :where => 'LENGTH(src.make_name) > 0 AND LENGTH(src.make_model_name) > 0'
24
+ )
15
25
  end
16
26
 
17
27
  # FIXME TODO make this a method on AutomobileMakeModelYear?
@@ -28,37 +38,7 @@ AutomobileMakeModelYear.class_eval do
28
38
  # sabshere 12/6/10 careful, don't use AutomobileMakeModelYearVariant.where here or you will be forced into projecting *
29
39
  relation = variants.project(variants[:"fuel_efficiency_#{i}"].average).where(conditional_relation).where(null_check)
30
40
  update_all "fuel_efficiency_#{i} = (#{relation.to_sql})"
31
- update_all "fuel_efficiency_#{i}_units = 'kilometres_per_litre'"
32
- end
33
- end
34
-
35
- verify "Year should be from 1985 to 2011" do
36
- AutomobileMakeModelYear.all.each do |model_year|
37
- unless model_year.year.to_i > 1984 and model_year.year.to_i < 2012
38
- raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be from 1985 to 2011)"
39
- end
40
- end
41
- end
42
-
43
- verify "Fuel efficiencies should be greater than zero" do
44
- AutomobileMakeModelYear.all.each do |model_year|
45
- %w{ city highway }.each do |type|
46
- fuel_efficiency = model_year.send(:"fuel_efficiency_#{type}")
47
- unless fuel_efficiency.to_f > 0
48
- raise "Invalid fuel efficiency #{type} for AutomobileMakeModelYear #{model_year.name}: #{fuel_efficiency} (should be > 0)"
49
- end
50
- end
51
- end
52
- end
53
-
54
- verify "Fuel efficiency units should be kilometres per litre" do
55
- AutomobileMakeModelYear.all.each do |model_year|
56
- %w{ city highway }.each do |type|
57
- units = model_year.send(:"fuel_efficiency_#{type}_units")
58
- unless units == "kilometres_per_litre"
59
- raise "Invalid fuel efficiency #{type} units for AutomobileMakeModelYear #{model_year.name}: #{units} (should be kilometres_per_litre)"
60
- end
61
- end
41
+ update_all "fuel_efficiency_#{i}_units" => 'kilometres_per_litre'
62
42
  end
63
43
  end
64
44
  end
@@ -6,41 +6,59 @@ class AutomobileMakeModelYearVariant < ActiveRecord::Base
6
6
  belongs_to :make_model_year, :class_name => 'AutomobileMakeModelYear', :foreign_key => 'make_model_year_name'
7
7
  belongs_to :fuel, :class_name => 'AutomobileFuel', :foreign_key => 'fuel_code', :primary_key => 'code'
8
8
 
9
- force_schema do
10
- string 'row_hash'
11
- string 'name' # short name!
12
- string 'make_name'
13
- string 'make_model_name' # make + model
14
- string 'make_year_name' # make + year
15
- string 'make_model_year_name' # make + model + year
16
- integer 'year'
17
- float 'fuel_efficiency'
18
- string 'fuel_efficiency_units'
19
- float 'fuel_efficiency_city'
20
- string 'fuel_efficiency_city_units'
21
- float 'fuel_efficiency_highway'
22
- string 'fuel_efficiency_highway_units'
23
- string 'fuel_code'
24
- string 'transmission'
25
- string 'drive'
26
- boolean 'turbo'
27
- boolean 'supercharger'
28
- integer 'cylinders'
29
- float 'displacement'
30
- float 'raw_fuel_efficiency_city'
31
- string 'raw_fuel_efficiency_city_units'
32
- float 'raw_fuel_efficiency_highway'
33
- string 'raw_fuel_efficiency_highway_units'
34
- integer 'carline_mfr_code'
35
- integer 'vi_mfr_code'
36
- integer 'carline_code'
37
- integer 'carline_class_code'
38
- boolean 'injection'
39
- string 'carline_class_name'
40
- string 'speeds'
41
- index 'make_name'
42
- index 'make_model_name'
43
- index 'make_year_name'
44
- index 'make_model_year_name'
45
- end
46
- end
9
+ col :row_hash
10
+ col :name # short name!
11
+ col :make_name
12
+ col :make_model_name # make + model
13
+ col :make_year_name # make + year
14
+ col :make_model_year_name # make + model + year
15
+ col :year, :type => :integer
16
+ col :fuel_efficiency, :type => :float
17
+ col :fuel_efficiency_units
18
+ col :fuel_efficiency_city, :type => :float
19
+ col :fuel_efficiency_city_units
20
+ col :fuel_efficiency_highway, :type => :float
21
+ col :fuel_efficiency_highway_units
22
+ col :fuel_code
23
+ col :transmission
24
+ col :drive
25
+ col :turbo, :type => :boolean
26
+ col :supercharger, :type => :boolean
27
+ col :cylinders, :type => :integer
28
+ col :displacement, :type => :float
29
+ col :raw_fuel_efficiency_city, :type => :float
30
+ col :raw_fuel_efficiency_city_units
31
+ col :raw_fuel_efficiency_highway, :type => :float
32
+ col :raw_fuel_efficiency_highway_units
33
+ col :carline_mfr_code, :type => :integer
34
+ col :vi_mfr_code, :type => :integer
35
+ col :carline_code, :type => :integer
36
+ col :carline_class_code, :type => :integer
37
+ col :injection, :type => :boolean
38
+ col :carline_class_name
39
+ col :speeds
40
+ add_index :make_name
41
+ add_index :make_model_name
42
+ add_index :make_year_name
43
+ add_index :make_model_year_name
44
+
45
+ # verify "Fuel code should appear in AutomobileFuel" do
46
+ # if (violators = connection.select_values("SELECT DISTINCT fuel_code FROM #{quoted_table_name} WHERE fuel_code NOT IN (SELECT code FROM #{AutomobileFuel.quoted_table_name})")).any?
47
+ # raise "Invalid fuel code(s): #{violators.join(', ')}"
48
+ # end
49
+ # true
50
+ # end
51
+ #
52
+ # verify "Fuel efficiencies should be greater than zero" do
53
+ # [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |field|
54
+ # if AutomobileMakeModelYearVariant.where(field => nil).any?
55
+ # raise "Invalid #{field} in automobile_make_model_year_variants: nil is not > 0"
56
+ # else
57
+ # min = AutomobileMakeModelYearVariant.minimum(field)
58
+ # unless min > 0
59
+ # raise "Invalid #{field} in automobile_make_model_year_variants: #{min} is not > 0"
60
+ # end
61
+ # end
62
+ # end
63
+ # end
64
+ end
@@ -314,7 +314,7 @@ AutomobileMakeModelYearVariant.class_eval do
314
314
  2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
315
315
  2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
316
316
  2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
317
- }.sort { |a, b| a.first <=> b.first }.each do |year, options|
317
+ }.each do |year, options|
318
318
  import "#{ year } Fuel Economy Guide",
319
319
  options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserC, :year => year },
320
320
  :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
@@ -348,7 +348,7 @@ AutomobileMakeModelYearVariant.class_eval do
348
348
  2007 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/2007_FE_guide_ALL_no_sales_May_01_2007.csv' },
349
349
  2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
350
350
  2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
351
- }.sort { |a, b| a.first <=> b.first }.each do |year, options|
351
+ }.each do |year, options|
352
352
  import "#{ year } Fuel Economy Guide",
353
353
  options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserD, :year => year },
354
354
  :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
@@ -379,7 +379,7 @@ AutomobileMakeModelYearVariant.class_eval do
379
379
  {
380
380
  2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEGuide for DOE-all rel dates-no-sales-02-22-2011public.xlsx' },
381
381
  2011 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/11data.zip', :filename => '2011FEGuide-for DOE rel-dates before 1-23-2011-no-sales-01-10-2011_All_public.xlsx' }
382
- }.sort { |a, b| a.first <=> b.first }.each do |year, options|
382
+ }.each do |year, options|
383
383
  import "#{ year } Fuel Economy Guide",
384
384
  options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserE, :year => year },
385
385
  :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
@@ -407,22 +407,16 @@ AutomobileMakeModelYearVariant.class_eval do
407
407
  end
408
408
 
409
409
  process "Derive model and model year names" do
410
- if ActiveRecord::Base.connection.adapter_name == 'sqlite'
411
- update_all "make_model_name = make_name || ' ' || name"
412
- update_all "make_year_name = make_name || ' ' || year"
413
- update_all "make_model_year_name = make_name || ' ' || name || ' ' || year"
414
- else
415
- update_all "make_model_name = CONCAT(make_name, ' ', name)"
416
- update_all "make_year_name = CONCAT(make_name, ' ', year)"
417
- update_all "make_model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
418
- end
410
+ update_all "make_model_name = make_name || ' ' || name"
411
+ update_all "make_year_name = make_name || ' ' || year"
412
+ update_all "make_model_year_name = make_name || ' ' || name || ' ' || year"
419
413
  end
420
414
 
421
415
  # Note: need to divide by 0.425143707 b/c equation is designed for miles / gallon not km / l
422
416
  # Note: EPA seems to adjust differently for plug-in hybrid electric vehicles (e.g. Leaf and Volt)
423
417
  process "Calculate adjusted fuel efficiency using the latest EPA equations from EPA Fuel Economy Trends report Appendix A including conversion from miles per gallon to kilometres per litre" do
424
- update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
425
- update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
418
+ update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))', 'raw_fuel_efficiency_city > 0'
419
+ update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))', 'raw_fuel_efficiency_highway > 0'
426
420
  end
427
421
 
428
422
  # This will be useful later for calculating MakeModel and Make fuel efficiency based on Variant
@@ -430,58 +424,18 @@ AutomobileMakeModelYearVariant.class_eval do
430
424
  # This results in a deviation from EPA fuel economy label values which use a historical 55/45 weighting
431
425
  process "Calculate combined adjusted fuel efficiency using the latest EPA equation" do
432
426
  update_all "fuel_efficiency = 1 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway))"
433
- update_all "fuel_efficiency_units = 'kilometres_per_litre'"
434
427
  end
435
428
 
436
- # FIXME TODO this step fails if the tables to be synthesized do not already exist
437
- %w{ AutomobileMakeModelYear AutomobileMakeModel }.each do |synthetic_resource|
438
- process "Synthesize #{synthetic_resource}" do
439
- synthetic_resource.constantize.run_data_miner!
440
- end
429
+ process "Set units" do
430
+ update_all :fuel_efficiency_units => 'kilometres_per_litre'
431
+ update_all :fuel_efficiency_city_units => 'kilometres_per_litre'
432
+ update_all :fuel_efficiency_highway_units => 'kilometres_per_litre'
441
433
  end
442
434
 
443
- verify "Year should be from 1985 to 2011" do
444
- connection.select_values("SELECT DISTINCT year FROM automobile_make_model_year_variants").each do |year|
445
- unless year > 1984 and year < 2012
446
- raise "Invalid year in automobile_make_model_year_variants: #{year} is not from 1985 to 2011"
447
- end
448
- end
449
- end
450
-
451
- process "Ensure AutomobileFuel is populated" do
435
+ process "Ensure related tables are populated" do
436
+ AutomobileMakeModelYear.run_data_miner!
437
+ AutomobileMakeModel.run_data_miner!
452
438
  AutomobileFuel.run_data_miner!
453
439
  end
454
-
455
- verify "Fuel code should appear in AutomobileFuel" do
456
- valid_codes = connection.select_values("SELECT DISTINCT code FROM automobile_fuels")
457
- connection.select_values("SELECT DISTINCT fuel_code FROM automobile_make_model_year_variants").each do |fuel_code|
458
- unless valid_codes.include?(fuel_code)
459
- raise "Invalide fuel code in automobile_make_model_year_variants: #{fuel_code} is not in #{valid_codes}"
460
- end
461
- end
462
- end
463
-
464
- verify "Fuel efficiencies should be greater than zero" do
465
- [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |field|
466
- if AutomobileMakeModelYearVariant.where(field => nil).any?
467
- raise "Invalid #{field} in automobile_make_model_year_variants: nil is not > 0"
468
- else
469
- min = AutomobileMakeModelYearVariant.minimum(field)
470
- unless min > 0
471
- raise "Invalid #{field} in automobile_make_model_year_variants: #{min} is not > 0"
472
- end
473
- end
474
- end
475
- end
476
-
477
- verify "Fuel efficiency units should be kilometres per litre" do
478
- %w{ fuel_efficiency_units fuel_efficiency_city_units fuel_efficiency_highway_units }.each do |field|
479
- connection.select_values("SELECT DISTINCT #{field} FROM automobile_make_model_year_variants").each do |value|
480
- unless value == 'kilometres_per_litre'
481
- raise "Invalid #{field} in automobile_make_model_year_variants: #{value} is not 'kilometres_per_litre'"
482
- end
483
- end
484
- end
485
- end
486
440
  end
487
441
  end
@@ -4,12 +4,19 @@ class AutomobileMakeYear < ActiveRecord::Base
4
4
  belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
5
5
  has_many :fleet_years, :class_name => 'AutomobileMakeFleetYear', :foreign_key => 'make_year_name'
6
6
 
7
- force_schema do
8
- string 'name'
9
- string 'make_name'
10
- integer 'year'
11
- float 'fuel_efficiency'
12
- string 'fuel_efficiency_units'
13
- integer 'volume' # This will sometimes be null because not all make_years have CAFE data
14
- end
7
+ col :name
8
+ col :make_name
9
+ col :year, :type => :integer
10
+ col :fuel_efficiency, :type => :float
11
+ col :fuel_efficiency_units
12
+ col :volume, :type => :integer # This will sometimes be null because not all make_years have CAFE data
13
+
14
+ # FIXME TODO verify make_name and volume?
15
+
16
+ # verify "Fuel efficiency should be greater than zero" do
17
+ # if violator = first(:conditions => ['fuel_efficiency IS NULL OR fuel_efficiency <= 0'])
18
+ # raise "Invalid fuel efficiency for AutomobileMakeYear #{record.name}: #{record.fuel_efficiency} (should be > 0)"
19
+ # end
20
+ # true
21
+ # end
15
22
  end
@@ -10,19 +10,16 @@ AutomobileMakeYear.class_eval do
10
10
  end
11
11
 
12
12
  process "Derive manufacturer names and years from automobile make model year variants" do
13
- INSERT_IGNORE %{INTO automobile_make_years(name, make_name, year)
14
- SELECT
15
- automobile_make_model_year_variants.make_year_name,
16
- automobile_make_model_year_variants.make_name,
17
- automobile_make_model_year_variants.year
18
- FROM automobile_make_model_year_variants
19
- WHERE automobile_make_model_year_variants.make_year_name IS NOT NULL
20
- AND LENGTH(automobile_make_model_year_variants.make_year_name) > 0
21
- AND automobile_make_model_year_variants.make_name IS NOT NULL
22
- AND LENGTH(automobile_make_model_year_variants.make_name) > 0
23
- AND automobile_make_model_year_variants.year IS NOT NULL
24
- AND LENGTH(automobile_make_model_year_variants.year) > 0
25
- }
13
+ ::Earth::Utils.insert_ignore(
14
+ :src => AutomobileMakeModelYearVariant,
15
+ :dest => AutomobileMakeYear,
16
+ :cols => {
17
+ :make_year_name => :name,
18
+ :make_name => :make_name,
19
+ :year => :year
20
+ }
21
+ # :where => 'src.make_name IS NOT NULL AND LENGTH(src.make_name) > 0 AND src.year IS NOT NULL AND LENGTH(src.year) > 0'
22
+ )
26
23
  end
27
24
 
28
25
  # FIXME TODO make this a method on AutomobileMakeYear?
@@ -32,52 +29,21 @@ AutomobileMakeYear.class_eval do
32
29
  conditional_relation = make_years[:name].eq(make_fleet_years[:make_year_name])
33
30
  relation = AutomobileMakeFleetYear.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
34
31
  update_all "fuel_efficiency = (#{relation.to_sql})"
35
- update_all "fuel_efficiency_units = 'kilometres_per_litre'"
36
32
  end
37
33
 
38
34
  process "Calculate fuel effeciency from automobile make model year variants for makes without CAFE data" do
39
- connection.execute %{
40
- UPDATE automobile_make_years
41
- SET automobile_make_years.fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_make_years.name = automobile_make_model_year_variants.make_year_name)
42
- WHERE automobile_make_years.fuel_efficiency IS NULL
43
- }
44
- connection.execute %{
45
- UPDATE automobile_make_years
46
- SET automobile_make_years.fuel_efficiency_units = 'kilometres_per_litre'
47
- WHERE automobile_make_years.fuel_efficiency_units IS NULL
48
- }
35
+ update_all(
36
+ %{fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_make_years.name = automobile_make_model_year_variants.make_year_name)},
37
+ %{fuel_efficiency IS NULL}
38
+ )
49
39
  end
50
40
 
51
41
  process "Calculate sales volume from automobile make fleet years for makes with CAFE data" do
52
- connection.execute %{
53
- UPDATE automobile_make_years SET automobile_make_years.volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)
54
- }
55
- end
56
-
57
- # FIXME TODO verify make_name and volume?
58
-
59
- verify "Year should be from 1985 to 2010" do
60
- AutomobileMakeYear.all.each do |make_year|
61
- unless make_year.year > 1984 and make_year.year < 2011
62
- raise "Invalid year for AutomobileMakeYear #{make_year.name}: #{make_year.year} (should be from 1985 to 2010)"
63
- end
64
- end
42
+ update_all %{volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)}
65
43
  end
66
44
 
67
- verify "Fuel efficiency should be greater than zero" do
68
- AutomobileMakeYear.all.each do |make_year|
69
- unless make_year.fuel_efficiency > 0
70
- raise "Invalid fuel efficiency for AutomobileMakeYear #{make_year.name}: #{make_year.fuel_efficiency} (should be > 0)"
71
- end
72
- end
73
- end
74
-
75
- verify "Fuel efficiency units should be kilometres per litre" do
76
- AutomobileMakeYear.all.each do |make_year|
77
- unless make_year.fuel_efficiency_units == "kilometres_per_litre"
78
- raise "Invalid fuel efficiency units for AutomobileMakeYear #{make_year.name}: #{make_year.fuel_efficiency_units} (should be kilometres_per_litre)"
79
- end
80
- end
45
+ process 'Set units' do
46
+ update_all "fuel_efficiency_units = 'kilometres_per_litre'"
81
47
  end
82
48
  end
83
49
  end
@@ -6,18 +6,82 @@ class AutomobileSizeClass < ActiveRecord::Base
6
6
  :conventional_fuel_efficiency_city_multiplier => 0.987,
7
7
  :conventional_fuel_efficiency_highway_multiplier => 0.996
8
8
 
9
- force_schema do
10
- string 'name'
11
- string 'type_name'
12
- float 'annual_distance'
13
- string 'annual_distance_units'
14
- float 'fuel_efficiency_city'
15
- string 'fuel_efficiency_city_units'
16
- float 'fuel_efficiency_highway'
17
- string 'fuel_efficiency_highway_units'
18
- float 'hybrid_fuel_efficiency_city_multiplier'
19
- float 'hybrid_fuel_efficiency_highway_multiplier'
20
- float 'conventional_fuel_efficiency_city_multiplier'
21
- float 'conventional_fuel_efficiency_highway_multiplier'
22
- end
9
+ col :name
10
+ col :type_name
11
+ col :annual_distance, :type => :float
12
+ col :annual_distance_units
13
+ col :fuel_efficiency_city, :type => :float
14
+ col :fuel_efficiency_city_units
15
+ col :fuel_efficiency_highway, :type => :float
16
+ col :fuel_efficiency_highway_units
17
+ col :hybrid_fuel_efficiency_city_multiplier, :type => :float
18
+ col :hybrid_fuel_efficiency_highway_multiplier, :type => :float
19
+ col :conventional_fuel_efficiency_city_multiplier, :type => :float
20
+ col :conventional_fuel_efficiency_highway_multiplier, :type => :float
21
+
22
+ # FIXME TODO verify type_name?
23
+
24
+ # verify "Annual distance should be greater than zero" do
25
+ # AutomobileSizeClass.all.each do |size_class|
26
+ # unless size_class.annual_distance > 0
27
+ # raise "Invalid annual distance for AutomobileSizeClass #{size_class.name}: #{size_class.annual_distance} (should be > 0)"
28
+ # end
29
+ # end
30
+ # end
31
+ #
32
+ # verify "Annual distance units should be kilometres" do
33
+ # AutomobileSizeClass.all.each do |size_class|
34
+ # unless size_class.annual_distance_units == "kilometres"
35
+ # raise "Invalid annual distance units for AutomobileSizeClass #{size_class.name}: #{size_class.annual_distance_units} (should be kilometres)"
36
+ # end
37
+ # end
38
+ # end
39
+ #
40
+ # verify "Fuel efficiencies should be greater than zero" do
41
+ # AutomobileSizeClass.all.each do |size_class|
42
+ # %w{ city highway }.each do |type|
43
+ # fuel_efficiency = size_class.send(:"fuel_efficiency_#{type}")
44
+ # unless fuel_efficiency > 0
45
+ # raise "Invalid fuel efficiency #{type} for AutomobileSizeClass #{size_class.name}: #{fuel_efficiency} (should be > 0)"
46
+ # end
47
+ # end
48
+ # end
49
+ # end
50
+ #
51
+ # verify "Fuel efficiency units should be kilometres per litre" do
52
+ # AutomobileSizeClass.all.each do |size_class|
53
+ # %w{ city highway }.each do |type|
54
+ # units = size_class.send(:"fuel_efficiency_#{type}_units")
55
+ # unless units == "kilometres_per_litre"
56
+ # raise "Invalid fuel efficiency #{type} units for AutomobileSizeClass #{size_class.name}: #{units} (should be kilometres_per_litre)"
57
+ # end
58
+ # end
59
+ # end
60
+ # end
61
+ #
62
+ # verify "Any fuel efficiency multipliers should be greater than zero" do
63
+ # AutomobileSizeClass.all.each do |size_class|
64
+ # %w{ hybrid conventional }.each do |hybridity|
65
+ # %w{ city highway }.each do |type|
66
+ # multiplier = size_class.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
67
+ # if multiplier.present?
68
+ # unless multiplier > 0
69
+ # raise "Invalid #{hybridity} fuel efficiency #{type} multiplier for AutomobileSizeClass #{size_class.name}: #{multiplier} (should be > 0)"
70
+ # end
71
+ # end
72
+ # end
73
+ # end
74
+ # end
75
+ # end
76
+ #
77
+ # verify "Fallback fuel efficiency multipliers should be greater than zero" do
78
+ # %w{ hybrid conventional }.each do |hybridity|
79
+ # %w{ city highway }.each do |type|
80
+ # multiplier = AutomobileSizeClass.fallback.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
81
+ # unless multiplier > 0
82
+ # raise "Invalid AutomobileSizeClass fallback #{hybridity} fuel efficiency #{type} multiplier: #{multiplier} (should be > 0)"
83
+ # end
84
+ # end
85
+ # end
86
+ # end
23
87
  end