earth 0.3.11 → 0.3.13

Sign up to get free protection for your applications and to get access to all the features.
Files changed (109) hide show
  1. data/Gemfile.lock +7 -3
  2. data/features/automobile_fuel_type.feature +8 -18
  3. data/features/automobile_make_fleet_year.feature +4 -4
  4. data/features/automobile_make_model_year.feature +4 -4
  5. data/features/automobile_make_model_year_variant.feature +4 -4
  6. data/features/automobile_make_year.feature +4 -4
  7. data/features/automobile_size_class_year.feature +34 -0
  8. data/features/automobile_type_fuel_age.feature +54 -0
  9. data/features/automobile_type_fuel_control.feature +34 -0
  10. data/features/automobile_type_fuel_year.feature +54 -0
  11. data/features/automobile_type_fuel_year_control.feature +34 -0
  12. data/features/automobile_type_year.feature +44 -0
  13. data/features/computation_platform.feature +14 -0
  14. data/features/data_center_company.feature +14 -0
  15. data/features/egrid_region.feature +6 -0
  16. data/features/egrid_subregion.feature +4 -4
  17. data/features/flight_segment.feature +14 -0
  18. data/features/fuel_year.feature +44 -0
  19. data/features/greenhouse_gas.feature +44 -0
  20. data/features/server_type.feature +34 -0
  21. data/features/server_type_alias.feature +14 -0
  22. data/features/support/imports/automobile_make_fleet_year_bad.csv +1 -2
  23. data/features/support/imports/automobile_make_fleet_year_good.csv +1 -2
  24. data/features/support/imports/automobile_size_class_year_bad.csv +2 -0
  25. data/features/support/imports/automobile_size_class_year_good.csv +2 -0
  26. data/features/support/imports/automobile_type_fuel_age_bad.csv +3 -0
  27. data/features/support/imports/automobile_type_fuel_age_good.csv +3 -0
  28. data/features/support/imports/automobile_type_fuel_control_bad.csv +3 -0
  29. data/features/support/imports/automobile_type_fuel_control_good.csv +3 -0
  30. data/features/support/imports/automobile_type_fuel_year_bad.csv +2 -0
  31. data/features/support/imports/automobile_type_fuel_year_control_bad.csv +3 -0
  32. data/features/support/imports/automobile_type_fuel_year_control_good.csv +3 -0
  33. data/features/support/imports/automobile_type_fuel_year_good.csv +2 -0
  34. data/features/support/imports/automobile_type_year_bad.csv +2 -0
  35. data/features/support/imports/automobile_type_year_good.csv +2 -0
  36. data/features/support/imports/computation_platform_bad.csv +3 -0
  37. data/features/support/imports/computation_platform_good.csv +3 -0
  38. data/features/support/imports/data_center_company_bad.csv +3 -0
  39. data/features/support/imports/data_center_company_good.csv +3 -0
  40. data/features/support/imports/egrid_subregion_good.csv +2 -2
  41. data/features/support/imports/flight_segment_bad.csv +8 -0
  42. data/features/support/imports/flight_segment_good.csv +8 -0
  43. data/features/support/imports/fuel_year_bad.csv +2 -0
  44. data/features/support/imports/fuel_year_good.csv +2 -0
  45. data/features/support/imports/greenhouse_gas_bad.csv +2 -0
  46. data/features/support/imports/greenhouse_gas_good.csv +2 -0
  47. data/features/support/imports/server_type_alias_bad.csv +3 -0
  48. data/features/support/imports/server_type_alias_good.csv +3 -0
  49. data/features/support/imports/server_type_bad.csv +3 -0
  50. data/features/support/imports/server_type_good.csv +3 -0
  51. data/lib/earth/air/aircraft.rb +2 -3
  52. data/lib/earth/air/aircraft/data_miner.rb +2 -4
  53. data/lib/earth/air/aircraft_class/data_miner.rb +3 -1
  54. data/lib/earth/air/airport/data_miner.rb +4 -3
  55. data/lib/earth/air/flight_distance_class/data_miner.rb +2 -0
  56. data/lib/earth/air/flight_seat_class/data_miner.rb +4 -4
  57. data/lib/earth/air/flight_segment/data_miner.rb +7 -0
  58. data/lib/earth/all.rb +1 -0
  59. data/lib/earth/automobile.rb +6 -0
  60. data/lib/earth/automobile/automobile_fuel_type/data_miner.rb +68 -25
  61. data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +5 -6
  62. data/lib/earth/automobile/automobile_make_model/data_miner.rb +1 -2
  63. data/lib/earth/automobile/automobile_make_model_year/data_miner.rb +3 -4
  64. data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb +14 -10
  65. data/lib/earth/automobile/automobile_make_year/data_miner.rb +2 -2
  66. data/lib/earth/automobile/automobile_size_class/data_miner.rb +28 -26
  67. data/lib/earth/automobile/automobile_size_class_year.rb +7 -0
  68. data/lib/earth/automobile/automobile_size_class_year/data_miner.rb +56 -0
  69. data/lib/earth/automobile/automobile_type_fuel_age.rb +7 -0
  70. data/lib/earth/automobile/automobile_type_fuel_age/data_miner.rb +195 -0
  71. data/lib/earth/automobile/automobile_type_fuel_control.rb +7 -0
  72. data/lib/earth/automobile/automobile_type_fuel_control/data_miner.rb +57 -0
  73. data/lib/earth/automobile/automobile_type_fuel_year.rb +7 -0
  74. data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb +164 -0
  75. data/lib/earth/automobile/automobile_type_fuel_year_control.rb +7 -0
  76. data/lib/earth/automobile/automobile_type_fuel_year_control/data_miner.rb +46 -0
  77. data/lib/earth/automobile/automobile_type_year.rb +7 -0
  78. data/lib/earth/automobile/automobile_type_year/data_miner.rb +55 -0
  79. data/lib/earth/automobile/data_miner.rb +6 -0
  80. data/lib/earth/computation.rb +4 -0
  81. data/lib/earth/computation/computation_platform.rb +9 -0
  82. data/lib/earth/computation/computation_platform/data_miner.rb +24 -0
  83. data/lib/earth/computation/data_center_company.rb +7 -0
  84. data/lib/earth/computation/data_center_company/data_miner.rb +22 -0
  85. data/lib/earth/computation/data_miner.rb +4 -0
  86. data/lib/earth/computation/server_type.rb +9 -0
  87. data/lib/earth/computation/server_type/data_miner.rb +49 -0
  88. data/lib/earth/computation/server_type_alias.rb +10 -0
  89. data/lib/earth/computation/server_type_alias/data_miner.rb +30 -0
  90. data/lib/earth/data_miner.rb +1 -0
  91. data/lib/earth/fuel.rb +2 -0
  92. data/lib/earth/fuel/data_miner.rb +2 -0
  93. data/lib/earth/fuel/fuel_year.rb +7 -0
  94. data/lib/earth/fuel/fuel_year/data_miner.rb +76 -0
  95. data/lib/earth/fuel/greenhouse_gas.rb +7 -0
  96. data/lib/earth/fuel/greenhouse_gas/data_miner.rb +59 -0
  97. data/lib/earth/locality/country/data_miner.rb +1 -1
  98. data/lib/earth/locality/egrid_region/data_miner.rb +8 -0
  99. data/lib/earth/locality/egrid_subregion/data_miner.rb +3 -3
  100. data/lib/earth/locality/urbanity/data_miner.rb +2 -2
  101. data/lib/earth/locality/zip_code/data_miner.rb +2 -3
  102. data/lib/earth/residence/air_conditioner_use/data_miner.rb +2 -2
  103. data/lib/earth/residence/clothes_machine_use/data_miner.rb +3 -3
  104. data/lib/earth/residence/dishwasher_use/data_miner.rb +1 -1
  105. data/lib/earth/residence/residence_class/data_miner.rb +2 -2
  106. data/lib/earth/residence/residential_energy_consumption_survey_response.rb +4 -4
  107. data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +3 -4
  108. data/spec/earth_spec.rb +4 -5
  109. metadata +196 -81
@@ -11,5 +11,7 @@ FlightDistanceClass.class_eval do
11
11
  key 'name'
12
12
  store 'distance', :units_field_name => 'distance_units'
13
13
  end
14
+
15
+ # FIXME TODO verify this
14
16
  end
15
17
  end
@@ -3,15 +3,16 @@ FlightSeatClass.class_eval do
3
3
  schema Earth.database_options do
4
4
  string 'name'
5
5
  float 'multiplier'
6
- integer 'seats'
7
6
  end
8
7
 
9
8
  import "a list of Brighter Planet-defined seat classes and pre-calculated multipliers",
10
- :url => 'http://static.brighterplanet.com/science/data/transport/air/seat_classes/seat_classes.csv' do
9
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdG5zSER1QmFVVkhNcTM2cmhCMEJtWVE&hl=en&single=true&gid=0&output=csv' do
11
10
  key 'name'
12
11
  store 'multiplier'
13
12
  end
14
-
13
+
14
+ # FIXME TODO verify this
15
+
15
16
  # sabshere 5/21/10 in case we ever need this
16
17
  # class << self
17
18
  # def refresh
@@ -33,4 +34,3 @@ FlightSeatClass.class_eval do
33
34
  # end
34
35
  end
35
36
  end
36
-
@@ -314,5 +314,12 @@ FlightSegment.class_eval do
314
314
  process "Derive average seats per departure" do
315
315
  update_all 'seats = total_seats / departures_performed', 'departures_performed > 0'
316
316
  end
317
+
318
+ # FIXME TODO make this verification check actual aircraft codes in Aircraft
319
+ verify 'All segments have an associated aircraft' do
320
+ FlightSegment.where(:aircraft_bts_code => [nil, '']).first.nil?
321
+ end
322
+
323
+ # FIXME TODO finish verification
317
324
  end
318
325
  end
@@ -1,6 +1,7 @@
1
1
  require 'earth/air'
2
2
  require 'earth/automobile'
3
3
  require 'earth/bus'
4
+ require 'earth/computation'
4
5
  require 'earth/diet'
5
6
  require 'earth/fuel'
6
7
  require 'earth/hospitality'
@@ -5,4 +5,10 @@ require 'earth/automobile/automobile_make_year'
5
5
  require 'earth/automobile/automobile_make_model'
6
6
  require 'earth/automobile/automobile_make_model_year'
7
7
  require 'earth/automobile/automobile_size_class'
8
+ require 'earth/automobile/automobile_size_class_year'
8
9
  require 'earth/automobile/automobile_make_model_year_variant'
10
+ require 'earth/automobile/automobile_type_year'
11
+ require 'earth/automobile/automobile_type_fuel_age'
12
+ require 'earth/automobile/automobile_type_fuel_control'
13
+ require 'earth/automobile/automobile_type_fuel_year'
14
+ require 'earth/automobile/automobile_type_fuel_year_control'
@@ -3,54 +3,97 @@ AutomobileFuelType.class_eval do
3
3
  schema Earth.database_options do
4
4
  string 'code'
5
5
  string 'name'
6
+ string 'fuel_name_for_distance'
7
+ string 'fuel_name_for_efs'
8
+ float 'blend_portion'
6
9
  float 'emission_factor'
7
10
  string 'emission_factor_units'
8
11
  float 'annual_distance'
9
12
  string 'annual_distance_units'
10
13
  end
11
14
 
12
- import("the fuels used in the 2000 EPA fuel economy guide",
13
- :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
14
- :filename => 'Gd6-dsc.txt',
15
- :format => :fixed_width,
16
- :crop => 21..26, # inclusive
17
- :cut => '2-',
18
- :select => lambda { |row| /\A[A-Z]/.match row[:code] },
19
- :schema => [[ 'code', 2, { :type => :string } ],
20
- [ 'spacer', 2 ],
21
- [ 'name', 52, { :type => :string } ]]) do
15
+ import "a list of fuel type codes and attributes",
16
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDlqeU9vQkVkNG1NZXV4WklKTjJkU3c&hl=en&single=true&gid=0&output=csv' do
22
17
  key 'code'
23
18
  store 'name'
19
+ store 'fuel_name_for_distance'
20
+ store 'fuel_name_for_efs'
21
+ store 'blend_portion'
24
22
  end
25
23
 
26
- import "a pre-calculated emission factor and average annual distance for each fuel",
27
- :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDlqeU9vQkVkNG1NZXV4WklKTjJkU3c&hl=en&single=true&gid=0&output=csv' do
28
- key 'code'
29
- store 'name'
30
- store 'annual_distance', :units_field_name => 'annual_distance_units'
31
- store 'emission_factor', :units_field_name => 'emission_factor_units'
24
+ process "Calculate annual distance from AutomobileTypeFuelAge" do
25
+ AutomobileTypeFuelAge.run_data_miner!
26
+ ages = AutomobileTypeFuelAge.arel_table
27
+ types = AutomobileFuelType.arel_table
28
+ conditional_relation = ages[:fuel_common_name].eq(types[:fuel_name_for_distance])
29
+ update_all "annual_distance = (#{AutomobileTypeFuelAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql})"
30
+ update_all "annual_distance_units = 'kilometres'"
32
31
  end
33
32
 
34
- verify "Annual distance should be greater than zero" do
33
+ process "Calculate emission factor from FuelYear, AutomobileTypeFuelYear, and GreenhouseGas" do
34
+ FuelYear.run_data_miner!
35
+ AutomobileTypeFuelYear.run_data_miner!
36
+ GreenhouseGas.run_data_miner!
37
+
38
+ co2_gwp = GreenhouseGas.find_by_abbreviation("co2").global_warming_potential
39
+ ch4_gwp = GreenhouseGas.find_by_abbreviation("ch4").global_warming_potential
40
+ n2o_gwp = GreenhouseGas.find_by_abbreviation("n2o").global_warming_potential
41
+
42
+ latest_year = FuelYear.all.collect{|x| x.year }.max
43
+
44
+ fuel_years = FuelYear.arel_table
45
+ type_fuel_years = AutomobileTypeFuelYear.arel_table
46
+ fuel_types = AutomobileFuelType.arel_table
47
+
35
48
  AutomobileFuelType.all.each do |fuel_type|
36
- unless fuel_type.annual_distance > 0.0
37
- raise "Invalid annual_distance for AutomobileFuelType #{fuel_type.name}: #{fuel_type.annual_distance} (should be > 0)"
49
+ current_name = fuel_type.fuel_name_for_efs
50
+
51
+ unless current_name.nil?
52
+ co2_factor_sql = fuel_years.project(fuel_years[:co2_emission_factor]).where(fuel_years[:fuel_common_name].eq(fuel_types[:fuel_name_for_efs]).and(fuel_years[:year].eq(latest_year))).to_sql
53
+ ch4_factor_sql = AutomobileTypeFuelYear.weighted_average_relation(:ch4_emission_factor, :weighted_by => :total_travel).where(type_fuel_years[:fuel_common_name].eq(fuel_types[:fuel_name_for_efs]).and(type_fuel_years[:year].eq(latest_year))).to_sql
54
+ n2o_factor_sql = AutomobileTypeFuelYear.weighted_average_relation(:n2o_emission_factor, :weighted_by => :total_travel).where(type_fuel_years[:fuel_common_name].eq(fuel_types[:fuel_name_for_efs]).and(type_fuel_years[:year].eq(latest_year))).to_sql
55
+ hfc_factor_sql = AutomobileTypeFuelYear.weighted_average_relation(:hfc_emission_factor, :weighted_by => :total_travel).where(type_fuel_years[:fuel_common_name].eq(fuel_types[:fuel_name_for_efs]).and(type_fuel_years[:year].eq(latest_year))).to_sql
56
+
57
+ connection.execute %{
58
+ UPDATE automobile_fuel_types
59
+ SET emission_factor = (((#{co2_factor_sql}) * #{co2_gwp} * (1 - blend_portion)) + ((#{ch4_factor_sql}) * #{ch4_gwp}) + ((#{n2o_factor_sql}) * #{n2o_gwp}) + (#{hfc_factor_sql}))
60
+ WHERE automobile_fuel_types.fuel_name_for_efs = '#{current_name}'
61
+ }
62
+ connection.execute %{
63
+ UPDATE automobile_fuel_types
64
+ SET emission_factor_units = 'kilograms_co2e_per_litre'
65
+ WHERE automobile_fuel_types.fuel_name_for_efs = '#{current_name}'
66
+ }
38
67
  end
39
68
  end
40
69
  end
41
70
 
42
- verify "Annual distance units should never be missing" do
71
+ process "Calculate emission factor for electricity from EgridSubregion and EgridRegion" do
72
+ EgridSubregion.run_data_miner!
73
+ EgridRegion.run_data_miner!
74
+ electricity_ef = (EgridSubregion.find_by_abbreviation("US").electricity_emission_factor / (1 - EgridRegion.find_by_name("US").loss_factor))
75
+ connection.execute %{
76
+ UPDATE automobile_fuel_types
77
+ SET emission_factor = #{electricity_ef}
78
+ WHERE automobile_fuel_types.name = 'electricity'
79
+ }
80
+ end
81
+
82
+ verify "Annual distance and emission factor should be greater than zero" do
43
83
  AutomobileFuelType.all.each do |fuel_type|
44
- if fuel_type.annual_distance_units.nil?
45
- raise "Missing annual distance units for AutomobileFuelType #{fuel_type.name}"
84
+ %w{ annual_distance emission_factor }.each do |attribute|
85
+ value = fuel_type.send(:"#{attribute}")
86
+ unless value > 0
87
+ raise "Invalid #{attribute} for AutomobileFuelType #{fuel_type.name}: #{value} (should be > 0)"
88
+ end
46
89
  end
47
90
  end
48
91
  end
49
92
 
50
- verify "Emission factor should be zero or more" do
93
+ verify "Annual distance units should be kilometres" do
51
94
  AutomobileFuelType.all.each do |fuel_type|
52
- unless fuel_type.emission_factor >= 0.0
53
- raise "Invalid emission_factor for AutomobileFuelType #{fuel_type.name}: #{fuel_type.emission_factor} (should be >= 0)"
95
+ unless fuel_type.annual_distance_units == "kilometres"
96
+ raise "Missing annual distance units for AutomobileFuelType #{fuel_type.name}"
54
97
  end
55
98
  end
56
99
  end
@@ -10,10 +10,9 @@ AutomobileMakeFleetYear.class_eval do
10
10
  string 'fuel_efficiency_units'
11
11
  integer 'volume'
12
12
  end
13
-
14
- # CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
13
+
15
14
  import "annual corporate average fuel economy data for domestic and imported vehicle fleets from the NHTSA",
16
- :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/make_fleet_years.csv',
15
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEdXWXB6dkVLWkowLXhYSFVUT01sS2c&hl=en&single=true&gid=0&output=csv',
17
16
  :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv',
18
17
  :select => lambda { |row| row['volume'].to_i > 0 } do
19
18
  key 'name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['year_content'], row['fleet'][2,2] ].join ' ' }
@@ -25,10 +24,10 @@ AutomobileMakeFleetYear.class_eval do
25
24
  store 'volume'
26
25
  end
27
26
 
28
- verify "Year should be between 1978 and 2007" do
27
+ verify "Year should be from 1978 to 2009" do
29
28
  AutomobileMakeFleetYear.all.each do |fleet_year|
30
- unless fleet_year.year > 1977 and fleet_year.year < 2008
31
- raise "Invalid year for AutomobileMakeFleetYear #{fleet_year.name}: #{fleet_year.year} (should be between 1978 and 2007)"
29
+ unless fleet_year.year > 1977 and fleet_year.year < 2010
30
+ raise "Invalid year for AutomobileMakeFleetYear #{fleet_year.name}: #{fleet_year.year} (should be from 1978 to 2009)"
32
31
  end
33
32
  end
34
33
  end
@@ -12,7 +12,7 @@ AutomobileMakeModel.class_eval do
12
12
  float 'fuel_efficiency_highway'
13
13
  string 'fuel_efficiency_highway_units'
14
14
  end
15
-
15
+
16
16
  process "Derive model names from automobile make model year variants" do
17
17
  AutomobileMakeModelYearVariant.run_data_miner!
18
18
  connection.execute %{
@@ -23,7 +23,6 @@ AutomobileMakeModel.class_eval do
23
23
 
24
24
  # TODO not weighted until we get weightings on auto variants
25
25
  process "Derive average fuel economy from automobile make model year variants" do
26
- AutomobileMakeModelYearVariant.run_data_miner!
27
26
  models = AutomobileMakeModel.arel_table
28
27
  variants = AutomobileMakeModelYearVariant.arel_table
29
28
  conditional_relation = models[:name].eq(variants[:make_model_name])
@@ -15,7 +15,7 @@ AutomobileMakeModelYear.class_eval do
15
15
  float 'fuel_efficiency_highway'
16
16
  string 'fuel_efficiency_highway_units'
17
17
  end
18
-
18
+
19
19
  process "Derive model year names from automobile make model year variants" do
20
20
  AutomobileMakeModelYearVariant.run_data_miner!
21
21
  connection.execute %{
@@ -29,7 +29,6 @@ AutomobileMakeModelYear.class_eval do
29
29
  # even without volume-weighting, the values are much better.
30
30
  # for example, 20km/l for a toyota prius 2006 vs. 13km/l if you use make years
31
31
  process "Calculate city and highway fuel efficiency from automobile make model year variants" do
32
- AutomobileMakeModelYearVariant.run_data_miner!
33
32
  model_years = AutomobileMakeModelYear.arel_table
34
33
  variants = AutomobileMakeModelYearVariant.arel_table
35
34
  conditional_relation = model_years[:name].eq(variants[:make_model_year_name])
@@ -42,10 +41,10 @@ AutomobileMakeModelYear.class_eval do
42
41
  end
43
42
  end
44
43
 
45
- verify "Year should be between 1985 and 2010" do
44
+ verify "Year should be from 1985 to 2010" do
46
45
  AutomobileMakeModelYear.all.each do |model_year|
47
46
  unless model_year.year.to_i > 1984 and model_year.year.to_i < 2011
48
- raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be between 1985 and 2010)"
47
+ raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be from 1985 to 2010)"
49
48
  end
50
49
  end
51
50
  end
@@ -9,7 +9,7 @@ AutomobileMakeModelYearVariant.class_eval do
9
9
  end
10
10
 
11
11
  def is_a_porsche?(row)
12
- row['make'] and row['make'].upcase == 'PORSCHE'
12
+ row['make'].to_s.upcase == 'PORSCHE'
13
13
  end
14
14
 
15
15
  def is_not_a_porsche?(row)
@@ -17,19 +17,19 @@ AutomobileMakeModelYearVariant.class_eval do
17
17
  end
18
18
 
19
19
  def is_a_mercedes_benz?(row)
20
- row['make'] and row['make'] =~ /MERCEDES/i
20
+ row['make'].to_s =~ /MERCEDES/i
21
21
  end
22
22
 
23
23
  def is_a_lexus?(row)
24
- row['make'] and row['make'].upcase == 'LEXUS'
24
+ row['make'].to_s.upcase == 'LEXUS'
25
25
  end
26
26
 
27
27
  def is_a_bmw?(row)
28
- row['make'] and row['make'].upcase == 'BMW'
28
+ row['make'].to_s.upcase == 'BMW'
29
29
  end
30
30
 
31
31
  def is_a_ford?(row)
32
- row['make'] and row['make'].upcase == 'FORD'
32
+ row['make'].to_s.upcase == 'FORD'
33
33
  end
34
34
 
35
35
  def is_a_rolls_royce_and_model_contains_bentley?(row)
@@ -37,11 +37,11 @@ AutomobileMakeModelYearVariant.class_eval do
37
37
  end
38
38
 
39
39
  def is_a_bentley?(row)
40
- row['make'] and row['make'].upcase == 'BENTLEY'
40
+ row['make'].to_s.upcase == 'BENTLEY'
41
41
  end
42
42
 
43
43
  def is_a_rolls_royce?(row)
44
- row['make'] and row['make'] =~ /ROLLS/i
44
+ row['make'].to_s =~ /ROLLS/i
45
45
  end
46
46
 
47
47
  def is_a_turbo_brooklands?(row)
@@ -140,7 +140,7 @@ AutomobileMakeModelYearVariant.class_eval do
140
140
  bus[:cut] = '13-' if year == 1995
141
141
  bus[:schema_name] = :fuel_economy_guide_b
142
142
  bus[:select] = lambda do |row|
143
- (row['suppress_code'].blank? or row['suppress_code'].to_i == 0) and row['state_code'] == 'F'
143
+ (row['suppress_code'].blank? or row['suppress_code'].to_f == 0) and row['state_code'] == 'F'
144
144
  end
145
145
  Slither.define :fuel_economy_guide_b do |d|
146
146
  d.rows do |row|
@@ -314,6 +314,8 @@ AutomobileMakeModelYearVariant.class_eval do
314
314
  end
315
315
 
316
316
  # 1985---1997
317
+ # FIXME TODO one Jaguar in the 1990 FEG has no model name
318
+ # FIXME TODO 14 records in the 1995 FEG are missing fuel efficiencies
317
319
  (85..97).each do |yy|
318
320
  filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
319
321
  import("19#{ yy } Fuel Economy Guide",
@@ -382,6 +384,7 @@ AutomobileMakeModelYearVariant.class_eval do
382
384
  end
383
385
 
384
386
  # 2006--2009
387
+ # FIXME TODO the 2007 FEG import fails: ArgumentError: unknown encoding name - WINDOWS-936
385
388
  {
386
389
  2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
387
390
  2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
@@ -459,16 +462,17 @@ AutomobileMakeModelYearVariant.class_eval do
459
462
  update_all "fuel_efficiency_units = 'kilometres_per_litre'"
460
463
  end
461
464
 
465
+ # FIXME TODO this step fails if the tables to be synthesized do not already exist
462
466
  %w{ AutomobileMakeModelYear AutomobileMakeModel }.each do |synthetic_resource|
463
467
  process "Synthesize #{synthetic_resource}" do
464
468
  synthetic_resource.constantize.run_data_miner!
465
469
  end
466
470
  end
467
471
 
468
- verify "Year should be between 1985 and 2010" do
472
+ verify "Year should be from 1985 to 2010" do
469
473
  AutomobileMakeModelYearVariant.all.each do |variant|
470
474
  unless variant.year > 1984 and variant.year < 2011
471
- raise "Invalid year for AutomobileMakeModelYearVariant #{variant.row_hash}: #{variant.year} (should be between 1985 and 2010)"
475
+ raise "Invalid year for AutomobileMakeModelYearVariant #{variant.row_hash}: #{variant.year} (should be from 1985 to 2010)"
472
476
  end
473
477
  end
474
478
  end
@@ -54,10 +54,10 @@ AutomobileMakeYear.class_eval do
54
54
  }
55
55
  end
56
56
 
57
- verify "Year should be between 1985 and 2010" do
57
+ verify "Year should be from 1985 to 2010" do
58
58
  AutomobileMakeYear.all.each do |make_year|
59
59
  unless make_year.year > 1984 and make_year.year < 2011
60
- raise "Invalid year for AutomobileMakeYear #{make_year.name}: #{make_year.year} (should be between 1985 and 2010)"
60
+ raise "Invalid year for AutomobileMakeYear #{make_year.name}: #{make_year.year} (should be from 1985 to 2010)"
61
61
  end
62
62
  end
63
63
  end
@@ -1,37 +1,39 @@
1
1
  AutomobileSizeClass.class_eval do
2
2
  data_miner do
3
3
  schema Earth.database_options do
4
- string 'name'
5
- string 'emblem'
6
- float 'annual_distance'
7
- string 'annual_distance_units'
8
- float 'fuel_efficiency_city'
9
- string 'fuel_efficiency_city_units'
10
- float 'fuel_efficiency_highway'
11
- string 'fuel_efficiency_highway_units'
12
- float 'hybrid_fuel_efficiency_city_multiplier'
13
- float 'hybrid_fuel_efficiency_highway_multiplier'
14
- float 'conventional_fuel_efficiency_city_multiplier'
15
- float 'conventional_fuel_efficiency_highway_multiplier'
4
+ string 'name'
5
+ string 'type_name'
6
+ float 'annual_distance'
7
+ string 'annual_distance_units'
8
+ float 'fuel_efficiency_city'
9
+ string 'fuel_efficiency_city_units'
10
+ float 'fuel_efficiency_highway'
11
+ string 'fuel_efficiency_highway_units'
12
+ float 'hybrid_fuel_efficiency_city_multiplier'
13
+ float 'hybrid_fuel_efficiency_highway_multiplier'
14
+ float 'conventional_fuel_efficiency_city_multiplier'
15
+ float 'conventional_fuel_efficiency_highway_multiplier'
16
16
  end
17
-
18
- import "a list of size classes and pre-calculated annual distances and fuel efficiencies",
19
- :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_size_class.csv' do
17
+
18
+ import "A list of size classes and pre-calculated fuel efficiencies",
19
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlRUE5IcWlrRENhN0EtUldPTy1rX1E&single=true&gid=0&output=csv' do
20
20
  key 'name'
21
- store 'annual_distance', :units => :kilometres
22
- store 'fuel_efficiency_city', :units => :kilometres_per_litre
23
- store 'fuel_efficiency_highway', :units => :kilometres_per_litre
21
+ store 'type_name'
22
+ store 'fuel_efficiency_city', :units_field_name => 'fuel_efficiency_city_units'
23
+ store 'fuel_efficiency_highway', :units_field_name => 'fuel_efficiency_highway_units'
24
+ end
25
+
26
+ process "Calculate annual distance from AutomobileTypeFuelAge" do
27
+ AutomobileTypeFuelAge.run_data_miner!
28
+ ages = AutomobileTypeFuelAge.arel_table
29
+ classes = AutomobileSizeClass.arel_table
30
+ conditional_relation = ages[:type_name].eq(classes[:type_name])
31
+ update_all "annual_distance = (#{AutomobileTypeFuelAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql})"
32
+ update_all "annual_distance_units = 'kilometres'"
24
33
  end
25
-
26
- # Ian 5/27/2010 I'm pretty sure we don't need emblems in middleware
27
- # import "",
28
- # :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_size_class_emblems.csv' do
29
- # key 'name'
30
- # store 'emblem'
31
- # end
32
34
 
33
35
  import "pre-calculated hybridity multipliers",
34
- :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/vehicle_classes/fuel_efficiency_multipliers.csv' do
36
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGt2NnhXLXUxNFRJSzczU3BkSHB3enc&hl=en&single=true&gid=0&output=csv' do
35
37
  key 'name'
36
38
  store 'hybrid_fuel_efficiency_city_multiplier'
37
39
  store 'hybrid_fuel_efficiency_highway_multiplier'
@@ -0,0 +1,7 @@
1
+ class AutomobileSizeClassYear < ActiveRecord::Base
2
+ set_primary_key :name
3
+
4
+ data_miner do
5
+ tap "Brighter Planet's sanitized automobile size class year data", Earth.taps_server
6
+ end
7
+ end
@@ -0,0 +1,56 @@
1
+ AutomobileSizeClassYear.class_eval do
2
+ data_miner do
3
+ schema Earth.database_options do
4
+ string 'name'
5
+ string 'size_class_name'
6
+ integer 'year'
7
+ string 'type_name'
8
+ float 'fuel_efficiency_city'
9
+ string 'fuel_efficiency_city_units'
10
+ float 'fuel_efficiency_highway'
11
+ string 'fuel_efficiency_highway_units'
12
+ end
13
+
14
+ import "automobile size class year fuel efficiencies from the 2010 EPA Fuel Economy Trends report",
15
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDZOLWdDdlZja04xZkJYc3NjeGxQamc&hl=en&single=true&gid=0&output=csv' do
16
+ key 'name'
17
+ store 'size_class_name'
18
+ store 'year'
19
+ store 'type_name'
20
+ store 'fuel_efficiency_city', :units_field_name => 'fuel_efficiency_city_units'
21
+ store 'fuel_efficiency_highway', :units_field_name => 'fuel_efficiency_highway_units'
22
+ end
23
+
24
+ # FIXME TODO verify that type_name appears in AutomobileTypeFuelAges
25
+
26
+ verify "Year should be from 1975 to 2010" do
27
+ AutomobileSizeClassYear.all.each do |record|
28
+ unless record.year > 1974 and record.year < 2011
29
+ raise "Invalid year for AutomobileSizeClassYear #{record.name}: #{record.year} (should be from 1975 to 2010)"
30
+ end
31
+ end
32
+ end
33
+
34
+ verify "Fuel efficiencies should be greater than zero" do
35
+ AutomobileSizeClassYear.all.each do |year|
36
+ %w{ fuel_efficiency_city fuel_efficiency_highway }.each do |attribute|
37
+ value = year.send(:"#{attribute}")
38
+ unless value > 0
39
+ raise "Invalid #{attribute} for AutomobileSizeClassYear #{year.name}: #{value} (should be > 0)"
40
+ end
41
+ end
42
+ end
43
+ end
44
+
45
+ verify "Fuel efficiency units should be kilometres per litre" do
46
+ AutomobileSizeClassYear.all.each do |year|
47
+ %w{ fuel_efficiency_city_units fuel_efficiency_highway_units }.each do |attribute|
48
+ value = year.send(:"#{attribute}")
49
+ unless value == "kilometres_per_litre"
50
+ raise "Invalid #{attribute} for AutomobileSizeClassYear #{year.name}: #{value} (should be kilometres_per_litre)"
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end