earth 0.7.0 → 0.11.0

data/lib/earth/automobile/automobile_make_year.rb

@@ -1,22 +1,10 @@
 class AutomobileMakeYear < ActiveRecord::Base
   set_primary_key :name
   
-  belongs_to :make,      :class_name => 'AutomobileMake',          :foreign_key => 'make_name'
-  has_many :fleet_years, :class_name => 'AutomobileMakeFleetYear', :foreign_key => 'make_year_name'
-
   col :name
   col :make_name
   col :year, :type => :integer
   col :fuel_efficiency, :type => :float
   col :fuel_efficiency_units
   col :volume, :type => :integer # This will sometimes be null because not all make_years have CAFE data
-  
-  # FIXME TODO verify make_name and volume?
-  
-  # verify "Fuel efficiency should be greater than zero" do
-  #   if violator = first(:conditions => ['fuel_efficiency IS NULL OR fuel_efficiency <= 0'])
-  #     raise "Invalid fuel efficiency for AutomobileMakeYear #{record.name}: #{record.fuel_efficiency} (should be > 0)"
-  #   end
-  #   true
-  # end
 end

data/lib/earth/automobile/automobile_make_year/data_miner.rb

@@ -4,9 +4,9 @@ AutomobileMakeYear.class_eval do
       delete_all
     end
     
-    process "Ensure AutomobileMakeModelYearVariant and AutomobileMakeFleetYear are populated" do
+    process "Ensure AutomobileMakeModelYearVariant and AutomobileMakeYearFleet are populated" do
       AutomobileMakeModelYearVariant.run_data_miner!
-      AutomobileMakeFleetYear.run_data_miner!
+      AutomobileMakeYearFleet.run_data_miner!
     end
     
     process "Derive manufacturer names and years from automobile make model year variants" do
@@ -14,36 +14,35 @@ AutomobileMakeYear.class_eval do
         :src => AutomobileMakeModelYearVariant,
         :dest => AutomobileMakeYear,
         :cols => {
-          :make_year_name => :name,
+          [:make_name, :year] => :name,
           :make_name => :make_name,
           :year => :year
         }
-        # :where => 'src.make_name IS NOT NULL AND LENGTH(src.make_name) > 0 AND src.year IS NOT NULL AND LENGTH(src.year) > 0'
       )
     end
     
-    # FIXME TODO make this a method on AutomobileMakeYear?
-    process "Calculate fuel efficiency from make fleet years for makes with CAFE data" do
-      make_fleet_years = AutomobileMakeFleetYear.arel_table
-      make_years = AutomobileMakeYear.arel_table
-      conditional_relation = make_years[:name].eq(make_fleet_years[:make_year_name])
-      relation = AutomobileMakeFleetYear.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
-      update_all "fuel_efficiency = (#{relation.to_sql})"
+    process "Calculate fuel efficiency and sales volume from make year fleets for makes with CAFE data" do
+      make_years = arel_table
+      year_fleets = AutomobileMakeYearFleet.arel_table
+      conditional_relation = make_years[:make_name].eq(year_fleets[:make_name]).and(make_years[:year].eq(year_fleets[:year]))
+      fuel_efficiency_relation = AutomobileMakeYearFleet.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
+      volume_relation = year_fleets.project(year_fleets[:volume].sum).where(conditional_relation)
+      update_all(%{
+        fuel_efficiency = (#{fuel_efficiency_relation.to_sql}),
+        fuel_efficiency_units = 'kilometres_per_litre',
+        volume = (#{volume_relation.to_sql})
+      })
     end
     
     process "Calculate fuel effeciency from automobile make model year variants for makes without CAFE data" do
-      update_all(
-        %{fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_make_years.name = automobile_make_model_year_variants.make_year_name)},
-        %{fuel_efficiency IS NULL}
-      )
-    end
-    
-    process "Calculate sales volume from automobile make fleet years for makes with CAFE data" do
-      update_all %{volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)}
-    end
-    
-    process 'Set units' do
-      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
+      make_years = arel_table
+      variants = AutomobileMakeModelYearVariant.arel_table
+      conditional_relation = make_years[:make_name].eq(variants[:make_name]).and(make_years[:year].eq(variants[:year]))
+      relation = variants.project(variants[:fuel_efficiency].average).where(conditional_relation)
+      where(:fuel_efficiency => nil).update_all(%{
+        fuel_efficiency = (#{relation.to_sql}),
+        fuel_efficiency_units = 'kilometres_per_litre'
+      })
     end
   end
 end

data/lib/earth/automobile/automobile_make_year_fleet.rb

@@ -0,0 +1,11 @@
+class AutomobileMakeYearFleet < ActiveRecord::Base
+  set_primary_key :name
+  
+  col :name
+  col :make_name
+  col :year, :type => :integer
+  col :fleet
+  col :fuel_efficiency, :type => :float
+  col :fuel_efficiency_units
+  col :volume, :type => :integer
+end

data/lib/earth/automobile/{automobile_make_fleet_year → automobile_make_year_fleet}/data_miner.rb

@@ -1,11 +1,10 @@
-AutomobileMakeFleetYear.class_eval do
+AutomobileMakeYearFleet.class_eval do
   data_miner do
     import "annual corporate average fuel economy data for domestic and imported vehicle fleets from the NHTSA",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEdXWXB6dkVLWkowLXhYSFVUT01sS2c&hl=en&gid=0&output=csv',
            :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv' },
            :select => lambda { |row| row['volume'].to_i > 0 } do
       key   'name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['year_content'], row['fleet'][2,2] ].join ' ' }
-      store 'make_year_name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['year_content'] ].join ' ' }
       store 'make_name', :field_name => 'manufacturer_name'
       store 'year', :field_name => 'year_content'
       store 'fleet', :chars => 2..3 # zero-based

data/lib/earth/automobile/automobile_model.rb

@@ -0,0 +1,5 @@
+class AutomobileModel < ActiveRecord::Base
+  set_primary_key :name
+  
+  col :name
+end

data/lib/earth/automobile/automobile_model/data_miner.rb

@@ -0,0 +1,19 @@
+AutomobileModel.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
+    process "Ensure AutomobileMakeModelYearVariant is populated" do
+      AutomobileMakeModelYearVariant.run_data_miner!
+    end
+    
+    process "Derive model names from automobile make model year variants" do
+      ::Earth::Utils.insert_ignore(
+        :src => AutomobileMakeModelYearVariant,
+        :dest => AutomobileModel,
+        :cols => { :model_name => :name }
+      )
+    end
+  end
+end

data/lib/earth/automobile/automobile_size_class.rb

@@ -1,6 +1,7 @@
 class AutomobileSizeClass < ActiveRecord::Base
   set_primary_key :name
   
+  # FIXME TODO clean up size class in MakeModelYearVariant, derive size class for MakeModelYear, and calculate this from MakeModelYear
   falls_back_on :hybrid_fuel_efficiency_city_multiplier => 1.651, # https://brighterplanet.sifterapp.com/issue/667
                 :hybrid_fuel_efficiency_highway_multiplier => 1.213,
                 :conventional_fuel_efficiency_city_multiplier => 0.987,

data/lib/earth/automobile/automobile_size_class/data_miner.rb

@@ -1,6 +1,5 @@
 AutomobileSizeClass.class_eval do
   data_miner do
-    # FIXME TODO pull this from AutomobileSizeClassYear
     import "a list of size classes and pre-calculated fuel efficiencies",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlRUE5IcWlrRENhN0EtUldPTy1rX1E&gid=0&output=csv' do
       key 'name'
@@ -9,17 +8,44 @@ AutomobileSizeClass.class_eval do
       store 'fuel_efficiency_highway', :units_field_name => 'fuel_efficiency_highway_units'
     end
     
+    # FIXME TODO derive everything from AutomobileSizeClassYear and AutomobileTypeFuelYearAge
+    # process "Start from scratch" do
+    #   delete_all
+    # end
+    # 
+    # process "Ensure AutomobileSizeClassYear is populated" do
+    #   AutomobileSizeClassYear.run_data_miner!
+    #   AutomobileTypeFuelYearAge.run_data_miner!
+    # end
+    # 
+    # process "Derive size classes from AutomobileSizeClassYear" do
+    #   ::Earth::Utils.insert_ignore(
+    #     :src => AutomobileSizeClassYear,
+    #     :dest => AutomobileSizeClass,
+    #     :cols => {
+    #       :size_class_name => :name,
+    #       :type_name => :type_name,
+    #     }
+    #   )
+    # end
+    # 
+    # process "somehow calculate fuel efficiency from size class year and type fuel year age" do
+    #   
+    # end
+    
     process "Ensure AutomobileTypeFuelYearAge is populated" do
       AutomobileTypeFuelYearAge.run_data_miner!
     end
     
     # FIXME TODO make this a method on AutomobileSizeClass?
     process "Calculate annual distance from AutomobileTypeFuelYearAge" do
+      classes = arel_table
       ages = AutomobileTypeFuelYearAge.arel_table
-      classes = AutomobileSizeClass.arel_table
       conditional_relation = ages[:type_name].eq(classes[:type_name])
-      update_all "annual_distance = (#{AutomobileTypeFuelYearAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql})"
-      update_all "annual_distance_units = 'kilometres'"
+      update_all(%{
+        annual_distance = (#{AutomobileTypeFuelYearAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql}),
+        annual_distance_units = 'kilometres'
+      })
     end
     
     import "pre-calculated fuel efficiency multipliers",

data/lib/earth/automobile/automobile_type_fuel_year.rb

@@ -1,7 +1,7 @@
 class AutomobileTypeFuelYear < ActiveRecord::Base
   set_primary_key :name
   
-  has_many :year_controls, :class_name => 'AutomobileTypeFuelYearControl', :foreign_key => 'type_fuel_year_name'
+  # FIXME TODO keep this until fix AutomobileFuel import so that it doesn't call type_fuel_year.type_year
   belongs_to :type_year, :class_name => 'AutomobileTypeYear', :foreign_key => 'type_year_name'
   
   col :name

data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb

@@ -50,12 +50,10 @@ AutomobileTypeFuelYear.class_eval do
     
     process "Convert total travel from billion miles to kilometres" do
       conversion_factor = 1_000_000_000.miles.to(:kilometres)
-      connection.execute %{
-        UPDATE automobile_type_fuel_years
-        SET total_travel = 1.0 * total_travel * #{conversion_factor},
-            total_travel_units = 'kilometres'
-        WHERE total_travel_units = 'billion_miles'
-      }
+      where(:total_travel_units => 'billion_miles').update_all(%{
+        total_travel = 1.0 * total_travel * #{conversion_factor},
+        total_travel_units = 'kilometres'
+      })
     end
     
     import "fuel consumption derived from the 2010 EPA GHG Inventory",
@@ -74,19 +72,32 @@ AutomobileTypeFuelYear.class_eval do
     end
     
     process "Calculate CH4 and N2O emision factors from AutomobileTypeFuelYearControl and AutomobileTypeFuelControl" do
-      AutomobileTypeFuelYear.all.each do |record|
-        record.ch4_emission_factor = record.year_controls.map do |year_control|
-          year_control.total_travel_percent.to_f * year_control.control.ch4_emission_factor
-        end.sum * record.total_travel / record.fuel_consumption
-        
-        record.n2o_emission_factor = record.year_controls.map do |year_control|
-          year_control.total_travel_percent.to_f * year_control.control.n2o_emission_factor
-        end.sum * record.total_travel / record.fuel_consumption
-        
-        record.ch4_emission_factor_units = 'kilograms_per_litre'
-        record.n2o_emission_factor_units = 'kilograms_per_litre'
-        
-        record.save!
+      # FIXME TODO tried to do this with arel but '*' method is not defined
+      # fuel_years = arel_table
+      # year_controls = AutomobileTypeFuelYearControl.arel_table
+      # controls = AutomobileTypeFuelControl.arel_table
+      # year_controls.project((year_controls[:total_travel_percent] * controls[:ch4_emission_factor]).sum)
+      #   .join(controls)
+      #   .on(year_controls[:type_name].eq(controls[:type_name])
+      #   .and(year_controls[:fuel_common_name].eq(controls[:fuel_common_name]))
+      #   .and(year_controls[:control_name].eq(controls[:control_name])))
+      %w{ ch4 n2o }.each do |gas|
+        emission_factor = %{
+          ( SELECT SUM(1.0 * t1.total_travel_percent * t2.#{gas}_emission_factor)
+            FROM #{AutomobileTypeFuelYearControl.quoted_table_name} AS t1
+            INNER JOIN #{AutomobileTypeFuelControl.quoted_table_name} AS t2
+              ON t1.type_name = t2.type_name
+              AND t1.fuel_common_name = t2.fuel_common_name
+              AND t1.control_name = t2.control_name
+            WHERE
+              t1.type_name = #{quoted_table_name}.type_name
+              AND t1.fuel_common_name = #{quoted_table_name}.fuel_common_name
+              AND t1.year = #{quoted_table_name}.year )
+        }
+        update_all(%{
+          #{gas}_emission_factor = 1.0 * #{emission_factor} * total_travel / fuel_consumption,
+          #{gas}_emission_factor_units = 'kilograms_per_litre'
+        })
       end
     end
   end

data/lib/earth/automobile/automobile_type_fuel_year_age.rb

@@ -1,14 +1,11 @@
 class AutomobileTypeFuelYearAge < ActiveRecord::Base
   set_primary_key :name
   
-  belongs_to :type_fuel_year, :class_name => 'AutomobileTypeFuelYear', :foreign_key => 'type_fuel_year_name'
-
   col :name
   col :type_name
   col :fuel_common_name
   col :year, :type => :integer
   col :age, :type => :integer
-  col :type_fuel_year_name
   col :total_travel_percent, :type => :float
   col :annual_distance, :type => :float
   col :annual_distance_units

data/lib/earth/automobile/automobile_type_fuel_year_age/data_miner.rb

@@ -90,16 +90,10 @@ AutomobileTypeFuelYearAge.class_eval do
     
     process "Convert annual distance from miles to kilometres" do
       conversion_factor = 1.miles.to(:kilometres)
-      connection.execute %{
-        UPDATE automobile_type_fuel_year_ages
-        SET annual_distance = 1.0 * annual_distance * #{conversion_factor},
-            annual_distance_units = 'kilometres'
-        WHERE annual_distance_units = 'miles'
-      }
-    end
-    
-    process "Derive type fuel year name for association with AutomobileTypeFuelYear" do
-      update_all "type_fuel_year_name = type_name || ' ' || fuel_common_name || ' ' || year"
+      where(:annual_distance_units => 'miles').update_all(%{
+        annual_distance = 1.0 * annual_distance * #{conversion_factor},
+        annual_distance_units = 'kilometres'
+      })
     end
     
     process "Ensure AutomobileTypeFuelYear is populated" do
@@ -107,11 +101,15 @@ AutomobileTypeFuelYearAge.class_eval do
     end
     
     process "Calculate number of vehicles from total travel percent and AutomobileTypeFuelYear" do
-      total_travel = "(SELECT t1.total_travel FROM #{AutomobileTypeFuelYear.quoted_table_name} AS t1 WHERE t1.name = #{quoted_table_name}.type_fuel_year_name)"
-      update_all(
-        %{vehicles = 1.0 * total_travel_percent * #{total_travel} / annual_distance},
-        %{annual_distance > 0}
-      )
+      total_travel = %{
+        SELECT t1.total_travel
+        FROM #{AutomobileTypeFuelYear.quoted_table_name} AS t1
+        WHERE
+          t1.type_name = #{quoted_table_name}.type_name
+          AND t1.fuel_common_name = #{quoted_table_name}.fuel_common_name
+          AND t1.year = #{quoted_table_name}.year
+      }
+      where("annual_distance > 0").update_all "vehicles = 1.0 * total_travel_percent * (#{total_travel}) / annual_distance"
     end
   end
 end

data/lib/earth/automobile/automobile_type_fuel_year_control.rb

@@ -1,15 +1,11 @@
 class AutomobileTypeFuelYearControl < ActiveRecord::Base
   set_primary_key :name
   
-  belongs_to :control, :class_name => 'AutomobileTypeFuelControl', :foreign_key => 'type_fuel_control_name'
-  
   col :name
   col :type_name
   col :fuel_common_name
   col :year, :type => :integer
   col :control_name
-  col :type_fuel_control_name
-  col :type_fuel_year_name
   col :total_travel_percent, :type => :float
   
   # %w{ type_name fuel_common_name control_name type_fuel_control_name type_fuel_year_name }.each do |attribute|

data/lib/earth/automobile/automobile_type_fuel_year_control/data_miner.rb

@@ -9,13 +9,5 @@ AutomobileTypeFuelYearControl.class_eval do
       store 'control_name'
       store 'total_travel_percent'
     end
-    
-    process "Derive type fuel control name for association with AutomobileTypeFuelControl" do
-      update_all "type_fuel_control_name = type_name || ' ' || fuel_common_name || ' ' || control_name"
-    end
-    
-    process "Derive type fuel year name for association with AutomobileTypeFuelYear" do
-      update_all "type_fuel_year_name = type_name || ' ' || fuel_common_name || ' ' || year"
-    end
   end
 end

data/lib/earth/automobile/automobile_type_year.rb

@@ -1,8 +1,9 @@
 class AutomobileTypeYear < ActiveRecord::Base
   set_primary_key :name
   
+  # FIXME TODO keep this until fix AutomobileFuel fallback hfc emission factor so that it doesn't call type_year.type_fuel_year
   has_many :type_fuel_years, :class_name => 'AutomobileTypeFuelYear', :foreign_key => 'type_year_name'
-
+  
   col :name
   col :type_name
   col :year, :type => :integer

data/lib/earth/automobile/automobile_type_year/data_miner.rb

@@ -12,16 +12,18 @@ AutomobileTypeYear.class_eval do
       AutomobileTypeFuelYear.run_data_miner!
     end
     
-    process "Set units" do
-      update_all :hfc_emission_factor_units => 'kilograms_co2e_per_litre'
-    end
-    
     process "Calculate HFC emission factor from AutomobileTypeFuelYear" do
-      total_fuel_consumption = "(SELECT SUM(src.fuel_consumption) FROM #{AutomobileTypeFuelYear.quoted_table_name} AS src WHERE src.type_year_name = #{quoted_table_name}.name)"
-      update_all(
-        %{hfc_emission_factor = 1.0 * hfc_emissions / #{total_fuel_consumption}},
-        %{#{total_fuel_consumption} > 0}
-      )
-    end    
+      total_fuel_consumption = %{
+        SELECT SUM(src.fuel_consumption)
+        FROM #{AutomobileTypeFuelYear.quoted_table_name} AS src
+        WHERE
+          src.type_name = #{quoted_table_name}.type_name
+          AND src.year = #{quoted_table_name}.year
+      }
+      where("(#{total_fuel_consumption}) > 0").update_all(%{
+        hfc_emission_factor = 1.0 * hfc_emissions / (#{total_fuel_consumption}),
+        hfc_emission_factor_units = 'kilograms_co2e_per_litre'
+      })
+    end
   end
 end

data/lib/earth/automobile/automobile_year.rb

@@ -0,0 +1,5 @@
+class AutomobileYear < ActiveRecord::Base
+  set_primary_key :year
+  
+  col :year, :type => :integer
+end

data/lib/earth/automobile/automobile_year/data_miner.rb

@@ -0,0 +1,15 @@
+AutomobileYear.class_eval do
+  data_miner do
+    process "Ensure AutomobileMakeModelYearVariant is populated" do
+      AutomobileMakeModelYearVariant.run_data_miner!
+    end
+    
+    process "Derive year names from automobile make model year variants" do
+      ::Earth::Utils.insert_ignore(
+        :src => AutomobileMakeModelYearVariant,
+        :dest => AutomobileYear,
+        :cols => { :year => :year }
+      )
+    end
+  end
+end

data/lib/earth/automobile/data_miner.rb

@@ -1,16 +1,18 @@
 require 'earth/automobile/automobile_fuel/data_miner'
 require 'earth/automobile/automobile_make/data_miner'
-require 'earth/automobile/automobile_make_fleet_year/data_miner'
 require 'earth/automobile/automobile_make_year/data_miner'
+require 'earth/automobile/automobile_make_year_fleet/data_miner'
 require 'earth/automobile/automobile_make_model/data_miner'
 require 'earth/automobile/automobile_make_model_year/data_miner'
+require 'earth/automobile/automobile_make_model_year_variant/data_miner'
 require 'earth/automobile/automobile_size_class/data_miner'
 require 'earth/automobile/automobile_size_class_year/data_miner'
-require 'earth/automobile/automobile_make_model_year_variant/data_miner'
 require 'earth/automobile/automobile_type_year/data_miner'
-require 'earth/automobile/automobile_type_fuel_age/data_miner'
 require 'earth/automobile/automobile_type_fuel_control/data_miner'
 require 'earth/automobile/automobile_type_fuel_year/data_miner'
 require 'earth/automobile/automobile_type_fuel_year_age/data_miner'
 require 'earth/automobile/automobile_type_fuel_year_control/data_miner'
+require 'earth/automobile/automobile_model/data_miner'
+require 'earth/automobile/automobile_year/data_miner'
+
 require 'earth/fuel/data_miner'