earth 0.5.4 → 0.6.0

data/lib/earth/automobile/automobile_fuel/data_miner.rb

@@ -1,7 +1,7 @@
 AutomobileFuel.class_eval do
   data_miner do
     import "a list of pure automobile fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE9xTEdueFM2R0diNTgxUlk1QXFSb2c&gid=0&output=html' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE9xTEdueFM2R0diNTgxUlk1QXFSb2c&gid=0&output=csv' do
       key   'name'
       store 'code'
       store 'base_fuel_name'
@@ -28,7 +28,7 @@ AutomobileFuel.class_eval do
     end
     
     process "Derive annual distance" do
-      AutomobileFuel.all.each do |record|
+      find_each do |record|
         scope = record.type_fuel_year_ages.where(:year => record.type_fuel_year_ages.maximum('year'))
         record.annual_distance = scope.weighted_average(:annual_distance, :weighted_by => :vehicles)
         record.annual_distance_units = scope.first.annual_distance_units
@@ -37,7 +37,7 @@ AutomobileFuel.class_eval do
     end
     
     process "Derive co2 emission factor and co2 biogenic emission factors" do
-      AutomobileFuel.all.each do |record|
+      find_each do |record|
         if record.blend_fuel.present?
           record.co2_emission_factor = (record.base_fuel.co2_emission_factor * (1 - record.blend_portion)) + (record.blend_fuel.co2_emission_factor * record.blend_portion)
           record.co2_biogenic_emission_factor = (record.base_fuel.co2_biogenic_emission_factor * (1 - record.blend_portion)) + (record.blend_fuel.co2_biogenic_emission_factor * record.blend_portion)
@@ -52,7 +52,7 @@ AutomobileFuel.class_eval do
     end
     
     process "Derive ch4, n2o, and hfc emission factors" do
-      AutomobileFuel.all.each do |record|
+      find_each do |record|
         scope = record.type_fuel_years.where(:year => record.type_fuel_years.maximum('year'))
         
         record.ch4_emission_factor = scope.weighted_average(:ch4_emission_factor, :weighted_by => :total_travel) * GreenhouseGas[:ch4].global_warming_potential
@@ -81,73 +81,6 @@ AutomobileFuel.class_eval do
     
     # FIXME TODO verify code somehow
     
-    %w{ base_fuel_name distance_key ef_key }.each do |attribute|
-      verify "#{attribute.humanize} should never be missing" do
-        AutomobileFuel.all.each do |fuel|
-          value = fuel.send(:"#{attribute}")
-          unless value.present?
-            puts "Missing #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}"
-            raise "Missing #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}"
-          end
-        end
-      end
-    end
-    
-    # FIXME TODO verify that base_fuel_name and blend_fuel_name are found in Fuel if present
-    # FIXME TODO verify that distance_key is found in AutomobileTypeFuelYearAge
-    # FIXME TODO verify that ef_key is found in AutomobileTypeFuelYear
-    
-    verify "Blend portion should be from 0 to 1 if present" do
-      AutomobileFuel.all.each do |fuel|
-        if fuel.blend_portion.present?
-          unless fuel.blend_portion >=0 and fuel.blend_portion <= 1
-            puts "Invalid blend portion for AutomobileFuel #{fuel.name}: #{fuel.blend_portion} (should be from 0 to 1)"
-            raise "Invalid blend portion for AutomobileFuel #{fuel.name}: #{fuel.blend_portion} (should be from 0 to 1)"
-          end
-        end
-      end
-    end
-    
-    ["co2_emission_factor", "co2_biogenic_emission_factor"].each do |attribute|
-      verify "#{attribute.humanize} should be 0 or more" do
-        AutomobileFuel.all.each do |fuel|
-          value = fuel.send(attribute)
-          unless value >= 0
-            puts "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{value} (should be 0 or more)"
-            raise "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{value} (should be 0 or more)"
-          end
-        end
-      end
-    end
-    
-    ["ch4_emission_factor", "n2o_emission_factor", "hfc_emission_factor"].each do |attribute|
-      verify "#{attribute.humanize} should be > 0" do
-        AutomobileFuel.all.each do |fuel|
-          value = fuel.send(attribute)
-          unless value > 0
-            puts "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{value} (should be > 0)"
-            raise "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{value} (should be > 0)"
-          end
-        end
-      end
-    end
-    
-    [["co2_emission_factor_units", "kilograms_per_litre"],
-     ["co2_biogenic_emission_factor_units", "kilograms_per_litre"],
-     ["ch4_emission_factor_units", "kilograms_co2e_per_litre"],
-     ["n2o_emission_factor_units", "kilograms_co2e_per_litre"],
-     ["hfc_emission_factor_units", "kilograms_co2e_per_litre"]].each do |pair|
-      attribute = pair[0]
-      proper_units = pair[1]
-      verify "#{attribute.humanize} should be #{proper_units.humanize.downcase}" do
-        AutomobileFuel.all.each do |fuel|
-          units = fuel.send(attribute)
-          unless units == proper_units
-            puts "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{units} (should be #{proper_units})"
-            fail
-          end
-        end
-      end
-    end
+
   end
 end

data/lib/earth/automobile/automobile_make.rb

@@ -6,9 +6,17 @@ class AutomobileMake < ActiveRecord::Base
   has_many :fleet_years,              :class_name => 'AutomobileMakeFleetYear',        :foreign_key => 'make_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_name'
 
-  force_schema do
-    string  'name'
-    float   'fuel_efficiency'
-    string  'fuel_efficiency_units'
-  end
-end
+  col :name
+  col :fuel_efficiency, :type => :float
+  col :fuel_efficiency_units
+
+  # TODO convert to table_warnings
+  # verify "Fuel efficiency should be greater than zero" do
+  #   AutomobileMake.all.each do |make|
+  #     unless make.fuel_efficiency > 0
+  #       raise "Invalid fuel efficiency for AutomobileMake #{make.name}: #{make.fuel_efficiency} (should be > 0)"
+  #     end
+  #   end
+  # end
+  
+end

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -10,20 +10,24 @@ AutomobileMake.class_eval do
     end
     
     process "Derive manufacturer names from automobile make model year variants" do
-      INSERT_IGNORE %{INTO automobile_makes(name)
-        SELECT DISTINCT automobile_make_model_year_variants.make_name
-        FROM automobile_make_model_year_variants
-        WHERE automobile_make_model_year_variants.make_name IS NOT NULL
-        AND LENGTH(automobile_make_model_year_variants.make_name) > 0
-      }
+      ::Earth::Utils.insert_ignore(
+        :src => AutomobileMakeModelYearVariant,
+        :dest => AutomobileMake,
+        :cols => {
+          :make_name => :name,
+        }
+      )
     end
     
     # sabshere 1/31/11 add Avanti, DaimlerChrysler, IHC, Tesla, etc.
     process "Derive extra manufacturer names from CAFE data" do
-      INSERT_IGNORE %{INTO automobile_makes(name)
-        SELECT DISTINCT automobile_make_fleet_years.make_name
-        FROM automobile_make_fleet_years
-      }
+      ::Earth::Utils.insert_ignore(
+        :src => AutomobileMakeFleetYear,
+        :dest => AutomobileMake,
+        :cols => {
+          :make_name => :name,
+        }
+      )
     end
     
     # FIXME TODO make this a method on AutomobileMake?
@@ -33,36 +37,17 @@ AutomobileMake.class_eval do
       conditional_relation = makes[:name].eq(make_fleet_years[:make_name])
       relation = AutomobileMakeFleetYear.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
       update_all "fuel_efficiency = (#{relation.to_sql})"
-      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
     end
     
     process "Calculate fuel effeciency from automobile make model year variants for makes without CAFE data" do
-      connection.execute %{
-        UPDATE automobile_makes
-        SET automobile_makes.fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_makes.name = automobile_make_model_year_variants.make_name)
-        WHERE automobile_makes.fuel_efficiency IS NULL
-      }
-      connection.execute %{
-        UPDATE automobile_makes
-        SET automobile_makes.fuel_efficiency_units = 'kilometres_per_litre'
-        WHERE automobile_makes.fuel_efficiency_units IS NULL
-      }
+      update_all(
+        %{fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_makes.name = automobile_make_model_year_variants.make_name)},
+        'fuel_efficiency IS NULL'
+      )
     end
     
-    verify "Fuel efficiency should be greater than zero" do
-      AutomobileMake.all.each do |make|
-        unless make.fuel_efficiency > 0
-          raise "Invalid fuel efficiency for AutomobileMake #{make.name}: #{make.fuel_efficiency} (should be > 0)"
-        end
-      end
-    end
-    
-    verify "Fuel efficiency units should be kilometres per litre" do
-      AutomobileMake.all.each do |make|
-        unless make.fuel_efficiency_units == "kilometres_per_litre"
-          raise "Invalid fuel efficiency units for AutomobileMake #{make.name}: #{make.fuel_efficiency_units} (should be kilometres_per_litre)"
-        end
-      end
+    process "Set units" do
+      update_all :fuel_efficiency_units => 'kilometres_per_litre'
     end
   end
 end

data/lib/earth/automobile/automobile_make_fleet_year.rb

@@ -4,14 +4,43 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
   belongs_to :make,      :class_name => 'AutomobileMake',     :foreign_key => 'make_name'
   belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'make_year_name'
 
-  force_schema do
-    string   'name'
-    string   'make_year_name'
-    string   'make_name'
-    string   'fleet'
-    integer  'year'
-    float    'fuel_efficiency'
-    string   'fuel_efficiency_units'
-    integer  'volume'
-  end
+  col :name
+  col :make_year_name
+  col :make_name
+  col :fleet
+  col :year, :type => :integer
+  col :fuel_efficiency, :type => :float
+  col :fuel_efficiency_units
+  col :volume, :type => :integer
+
+  # TODO convert to table_warnings
+  # verify "Year should be from 1978 to 2011" do
+  #   connection.select_values("SELECT DISTINCT year FROM automobile_make_fleet_years").each do |year|
+  #     year = year.to_i
+  #     unless year > 1977 and year < 2012
+  #       raise "Invalid year in automobile_make_fleet_years: #{year} is not from 1978 to 2011"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Fuel efficiency and volume should be greater than zero" do
+  #   [:fuel_efficiency, :volume].each do |field|
+  #     if AutomobileMakeFleetYear.where(field => nil).any?
+  #       raise "Invalid #{field} in automobile_make_fleet_years: nil is not > 0"
+  #     else
+  #       min = AutomobileMakeFleetYear.minimum(field)
+  #       unless min > 0
+  #         raise "Invalid #{field} in automobile_make_fleet_years: #{min} is not > 0"
+  #       end
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Fuel efficiency units should be kilometres per litre" do
+  #   connection.select_values("SELECT DISTINCT fuel_efficiency_units FROM automobile_make_fleet_years").each do |units|
+  #     unless units == "kilometres_per_litre"
+  #       raise "Invalid fuel efficiency units in automobile_make_fleet_years: #{units} is not 'kilometres_per_litre'"
+  #     end
+  #   end
+  # end
 end

data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb

@@ -12,34 +12,5 @@ AutomobileMakeFleetYear.class_eval do
       store 'fuel_efficiency', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
       store 'volume'
     end
-    
-    verify "Year should be from 1978 to 2010" do
-      connection.select_values("SELECT DISTINCT year FROM automobile_make_fleet_years").each do |year|
-        unless year > 1977 and year < 2011
-          raise "Invalid year in automobile_make_fleet_years: #{year} is not from 1978 to 2010"
-        end
-      end
-    end
-    
-    verify "Fuel efficiency and volume should be greater than zero" do
-      [:fuel_efficiency, :volume].each do |field|
-        if AutomobileMakeFleetYear.where(field => nil).any?
-          raise "Invalid #{field} in automobile_make_fleet_years: nil is not > 0"
-        else
-          min = AutomobileMakeFleetYear.minimum(field)
-          unless min > 0
-            raise "Invalid #{field} in automobile_make_fleet_years: #{min} is not > 0"
-          end
-        end
-      end
-    end
-    
-    verify "Fuel efficiency units should be kilometres per litre" do
-      connection.select_values("SELECT DISTINCT fuel_efficiency_units FROM automobile_make_fleet_years").each do |units|
-        unless units == "kilometres_per_litre"
-          raise "Invalid fuel efficiency units in automobile_make_fleet_years: #{units} is not 'kilometres_per_litre'"
-        end
-      end
-    end
   end
 end

data/lib/earth/automobile/automobile_make_model.rb

@@ -4,13 +4,34 @@ class AutomobileMakeModel < ActiveRecord::Base
   belongs_to :make,                   :class_name => 'AutomobileMake',                 :foreign_key => 'make_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_model_name'
 
-  force_schema do
-    string  'name' # make + model
-    string  'make_name'
-    string  'model_name' # model only
-    float   'fuel_efficiency_city'
-    string  'fuel_efficiency_city_units'
-    float   'fuel_efficiency_highway'
-    string  'fuel_efficiency_highway_units'
-  end
+  col :name # make + model
+  col :make_name
+  col :model_name # model only
+  col :fuel_efficiency_city, :type => :float
+  col :fuel_efficiency_city_units
+  col :fuel_efficiency_highway, :type => :float
+  col :fuel_efficiency_highway_units
+
+  # TODO convert to table_warnings
+  # verify "Fuel efficiencies should be greater than zero" do
+  #   AutomobileMakeModel.all.each do |model|
+  #     %w{ city highway }.each do |type|
+  #       fuel_efficiency = model.send(:"fuel_efficiency_#{type}")
+  #       unless fuel_efficiency > 0
+  #         raise "Invalid fuel efficiency #{type} for AutomobileMakeModel #{model.name}: #{fuel_efficiency} (should be > 0)"
+  #       end
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Fuel efficiency units should be kilometres per litre" do
+  #   AutomobileMakeModel.all.each do |model|
+  #     %w{ city highway }.each do |type|
+  #       units = model.send(:"fuel_efficiency_#{type}_units")
+  #       unless units == "kilometres_per_litre"
+  #         raise "Invalid fuel efficiency #{type} units for AutomobileMakeModel #{model.name}: #{units} (should be kilometres_per_litre)"
+  #       end
+  #     end
+  #   end
+  # end
 end

data/lib/earth/automobile/automobile_make_model/data_miner.rb

@@ -9,9 +9,16 @@ AutomobileMakeModel.class_eval do
     end
     
     process "Derive model names from automobile make model year variants" do
-      INSERT_IGNORE %{INTO automobile_make_models(name, make_name, model_name)
-        SELECT automobile_make_model_year_variants.make_model_name, automobile_make_model_year_variants.make_name, automobile_make_model_year_variants.name FROM automobile_make_model_year_variants WHERE LENGTH(automobile_make_model_year_variants.make_name) > 0 AND LENGTH(automobile_make_model_year_variants.name) > 0
-      }
+      ::Earth::Utils.insert_ignore(
+        :src => AutomobileMakeModelYearVariant,
+        :dest => AutomobileMakeModel,
+        :cols => {
+          :make_model_name => :name,
+          :make_name => :make_name,
+          :name => :model_name
+        }
+        # :where => 'LENGTH(src.make_name) > 0 AND LENGTH(src.make_model_name) > 0'
+      )
     end
     
     # FIXME TODO make this a method on AutomobileMakeModel?
@@ -25,29 +32,7 @@ AutomobileMakeModel.class_eval do
         # sabshere 12/6/10 careful, don't use AutomobileMakeModelYearVariant.where here or you will be forced into projecting *
         relation = variants.project(variants[:"fuel_efficiency_#{i}"].average).where(conditional_relation).where(null_check)
         update_all "fuel_efficiency_#{i} = (#{relation.to_sql})"
-        update_all "fuel_efficiency_#{i}_units = 'kilometres_per_litre'"
-      end
-    end
-    
-    verify "Fuel efficiencies should be greater than zero" do
-      AutomobileMakeModel.all.each do |model|
-        %w{ city highway }.each do |type|
-          fuel_efficiency = model.send(:"fuel_efficiency_#{type}")
-          unless fuel_efficiency > 0
-            raise "Invalid fuel efficiency #{type} for AutomobileMakeModel #{model.name}: #{fuel_efficiency} (should be > 0)"
-          end
-        end
-      end
-    end
-    
-    verify "Fuel efficiency units should be kilometres per litre" do
-      AutomobileMakeModel.all.each do |model|
-        %w{ city highway }.each do |type|
-          units = model.send(:"fuel_efficiency_#{type}_units")
-          unless units == "kilometres_per_litre"
-            raise "Invalid fuel efficiency #{type} units for AutomobileMakeModel #{model.name}: #{units} (should be kilometres_per_litre)"
-          end
-        end
+        update_all "fuel_efficiency_#{i}_units" => 'kilometres_per_litre'
       end
     end
   end

data/lib/earth/automobile/automobile_make_model_year.rb

@@ -4,16 +4,45 @@ class AutomobileMakeModelYear < ActiveRecord::Base
   belongs_to :make_year,              :class_name => 'AutomobileMakeYear',             :foreign_key => 'make_year_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_model_year_name'
   
-  force_schema do
-    string   'name' # make + model + year
-    string   'make_name'
-    string   'model_name'
-    string   'make_model_name'
-    integer  'year'
-    string   'make_year_name'
-    float    'fuel_efficiency_city'
-    string   'fuel_efficiency_city_units'
-    float    'fuel_efficiency_highway'
-    string   'fuel_efficiency_highway_units'
-  end
+  col :name # make + model + year
+  col :make_name
+  col :model_name
+  col :make_model_name
+  col :year, :type => :integer
+  col :make_year_name
+  col :fuel_efficiency_city, :type => :float
+  col :fuel_efficiency_city_units
+  col :fuel_efficiency_highway, :type => :float
+  col :fuel_efficiency_highway_units
+
+  # TODO convert to table_warnings
+  # verify "Year should be from 1985 to 2011" do
+  #   AutomobileMakeModelYear.all.each do |model_year|
+  #     unless model_year.year.to_i > 1984 and model_year.year.to_i < 2012
+  #       raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be from 1985 to 2011)"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Fuel efficiencies should be greater than zero" do
+  #   AutomobileMakeModelYear.all.each do |model_year|
+  #     %w{ city highway }.each do |type|
+  #       fuel_efficiency = model_year.send(:"fuel_efficiency_#{type}")
+  #       unless fuel_efficiency.to_f > 0
+  #         raise "Invalid fuel efficiency #{type} for AutomobileMakeModelYear #{model_year.name}: #{fuel_efficiency} (should be > 0)"
+  #       end
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Fuel efficiency units should be kilometres per litre" do
+  #   AutomobileMakeModelYear.all.each do |model_year|
+  #     %w{ city highway }.each do |type|
+  #       units = model_year.send(:"fuel_efficiency_#{type}_units")
+  #       unless units == "kilometres_per_litre"
+  #         raise "Invalid fuel efficiency #{type} units for AutomobileMakeModelYear #{model_year.name}: #{units} (should be kilometres_per_litre)"
+  #       end
+  #     end
+  #   end
+  # end
 end