earth-ruby19 0.2.3

data/lib/earth/locality/egrid_region/data_miner.rb

@@ -0,0 +1,35 @@
+EgridRegion.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'loss_factor'
+    end
+    
+    # NOTE: the following import uses an 18 Mb zip - don't know if two imports will cause it to be downloaded twice...
+    # 
+    # import "eGRID regions and loss factors derived from eGRID 2007 data",
+    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
+    #        :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
+    #        :sheet => 'STIE05',
+    #        :skip => 4,
+    #        :select => lambda { |row| row['eGRID2007 2005 file State sequence number'].to_i.between?(1, 51) } do
+    #   key   'name', :field_name => 'Grid region (E=Eastern grid, W=Western grid, AK=Alaska, HI=Hawaii, TX=Texas)'
+    #   store 'loss_factor', :field_name => '2005 grid gross loss factor'
+    # end
+    # 
+    # import "the US average grid loss factor derived eGRID 2007 data"
+    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
+    #        :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
+    #        :sheet => 'USGC',
+    #        :skip => 5 do
+    #   key # the name should be 'US'
+    #   store # store the result of (USTNGN05 + USTNFI05 - USTCON05) / USTNGN05
+    # end
+    
+    import "eGRID regions and loss factors derived from eGRID 2007 data",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHoydC1MdDY0YVZkRE5zN0huOUZYbnc&hl=en&single=true&gid=0&output=csv' do
+      key 'name'
+      store 'loss_factor'
+    end
+  end
+end

data/lib/earth/locality/egrid_subregion.rb

@@ -0,0 +1,16 @@
+class EgridSubregion < ActiveRecord::Base
+  set_primary_key :abbreviation
+  
+  has_many :zip_codes, :foreign_key => 'egrid_subregion_abbreviation'
+  belongs_to :egrid_region, :foreign_key => 'egrid_region_name'
+  
+  falls_back_on :electricity_emission_factor => 1.404.pounds.to(:kilograms) # kg CO2 / kWh https://brighterplanet.sifterapp.com/projects/30/issues/437?return_uri=%2Fprojects%2F30%2Fissues%3Fa%3D79%26s%3D1-2
+  
+  data_miner do
+    tap "Brighter Planet's egrid subregion data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/locality/egrid_subregion/data_miner.rb

@@ -0,0 +1,65 @@
+EgridSubregion.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'abbreviation'
+      string 'name'
+      float  'electricity_emission_factor'
+      string 'electricity_emission_factor_units'
+      string 'nerc_abbreviation'
+      string 'egrid_region_name'
+    end
+    
+    # process "define some unit conversions" do
+    #   Conversions.register :pounds_per_megawatt_hour, :kilograms_per_kilowatt_hour, 0.00045359237
+    #   Conversions.register :pounds_per_gigawatt_hour, :kilograms_per_kilowatt_hour, 0.00000045359237
+    # end
+    # 
+    # NOTE: the following import uses an 18 Mb zip - don't know if two imports will cause it to be downloaded twice...
+    # 
+    # import "eGRID regions and electricity emission factors derived from eGRID 2007 data",
+    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
+    #        :filename => 'eGRID2007V1_1_year05_aggregation.xls',
+    #        :sheet => 'SRL05',
+    #        :skip => 3,
+    #        :select => lambda { |row| row['eGRID2007 2005 file eGRID subregion location (operator)-based sequence number'].to_i.between?(1, 26) } do
+    #   key   'abbreviation', :field_name => 'eGRID subregion acronym'
+    #   store 'name', :field_name => 'eGRID subregion name associated with eGRID subregion acronym'
+    #   store 'nerc_abbreviation', :field_name => 'NERC region acronym associated with the eGRID subregion acronym'
+    #   store 'electricity_ef_co2', :field_name => 'eGRID subregion annual CO2 output emission rate (lb/MWh)', :from_units => :pounds_per_megawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+    #   store 'electricity_ef_ch4', :field_name => 'eGRID subregion annual CH4 output emission rate (lb/GWh)', :from_units => :pounds_per_gigawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+    #   store 'electricity_ef_n2o', :field_name => 'eGRID subregion annual N2O output emission rate (lb/GWh)', :from_units => :pounds_per_gigawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+    # end
+    # 
+    # import "US average electricity emission factors derived from eGRID 2007 data",
+    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
+    #        :filename => 'eGRID2007V1_1_year05_aggregation.xls',
+    #        :sheet => 'US05',
+    #        :skip => 3,
+    #        :select => lambda { |row| row['eGRID2007 2005 file US sequence number'].to_i.is?(1) } do
+    #   key   # the single row should be keyed 'US'
+    #   store 'name' # the single row should be named 'United States Average'
+    #   store 'electricity_ef_co2', :field_name => 'US annual CO2 output emission rate (lb/MWh)', :from_units => :pounds_per_megawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+    #   store 'electricity_ef_ch4', :field_name => 'US annual CH4 output emission rate (lb/GWh)', :from_units => :pounds_per_gigawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+    #   store 'electricity_ef_n2o', :field_name => 'US annual N2O output emission rate (lb/GWh)', :from_units => :pounds_per_gigawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+    # end
+    # 
+    # import "the eGRID regions associated with each subregion" do
+    #        :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGRORTJNSWRMQ1puRVprYlAtZHhDaFE&hl=en&single=true&gid=0&output=csv' do
+    #   key 'abbreviation'
+    #   store 'egrid_region_name'
+    # end
+    # 
+    # process "Calculate CO2e emission factor"
+    #   # multiply each gas ef by the gas GWP and sum
+    # end
+    
+    import "a list of eGRID subregions and emissions factors derived from eGRID 2007 data",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGRORTJNSWRMQ1puRVprYlAtZHhDaFE&hl=en&single=true&gid=0&output=csv' do
+      key   'abbreviation'
+      store 'name'
+      store 'nerc_abbreviation'
+      store 'egrid_region_name'
+      store 'electricity_emission_factor', :units_field_name => 'electricity_emission_factor_units'
+    end
+  end
+end

data/lib/earth/locality/petroleum_administration_for_defense_district.rb

@@ -0,0 +1,13 @@
+class PetroleumAdministrationForDefenseDistrict < ActiveRecord::Base
+  set_primary_key :code
+  
+  data_miner do
+    tap "Brighter Planet's PADD info", Earth.taps_server
+  end
+  
+  def name
+    str = "PAD District #{district_code} (#{district_name})"
+    str << " Subdistrict #{district_code}#{subdistrict_code} (#{subdistrict_name})" if subdistrict_code
+    str
+  end
+end

data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb

@@ -0,0 +1,21 @@
+PetroleumAdministrationForDefenseDistrict.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'code'
+      string   'district_code'
+      string   'district_name'
+      string   'subdistrict_code'
+      string   'subdistrict_name'
+    end
+    
+    import "a list of PADD districts and states derived from the EIA PADD definitions",
+           :url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA' do
+      key   'code',             :field_name => 'Code'
+      store 'district_code',    :field_name => 'PAD district code'
+      store 'subdistrict_code', :field_name => 'PAD subdistrict code'
+      store 'district_name',    :field_name => 'PAD district name'
+      store 'subdistrict_name', :field_name => 'PAD subdistrict name'
+    end
+  end
+end
+

data/lib/earth/locality/state.rb

@@ -0,0 +1,22 @@
+class State < ActiveRecord::Base
+  set_primary_key :postal_abbreviation
+  
+  has_many :climate_divisions, :foreign_key => 'state_postal_abbreviation'
+  has_many :zip_codes, :foreign_key => 'state_postal_abbreviation'
+  belongs_to :census_division, :foreign_key => 'census_division_number'
+  belongs_to :petroleum_administration_for_defense_district, :foreign_key => 'petroleum_administration_for_defense_district_code'
+  has_one :census_region, :through => :census_division
+  
+  def country
+    Country.united_states
+  end
+
+  data_miner do
+    tap "Brighter Planet's sanitized states", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+
+end

data/lib/earth/locality/state/data_miner.rb

@@ -0,0 +1,37 @@
+State.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'postal_abbreviation'
+      integer  'fips_code'
+      string   'name'
+      string   'census_division_number'
+      string   'petroleum_administration_for_defense_district_code'
+    end
+    
+    # state names, ansi codes, and postal abbreviations
+    import 'the U.S. Census State ANSI Code file',
+           :url => 'http://www.census.gov/geo/www/ansi/state.txt',
+           :delimiter => '|' do
+      key   'postal_abbreviation', :field_name => 'STUSAB'
+      store 'fips_code', :field_name => 'STATE'
+      store 'name', :field_name => 'STATE_NAME'
+    end
+
+    # census divisions
+    import 'the U.S. Census Geographic Terms and Definitions',
+           :url => 'http://www.census.gov/popest/geographic/codes02.csv',
+           :skip => 9,
+           :select => lambda { |row| row['FIPS CODE STATE'].to_s.strip != 'X' and row['FIPS CODE COUNTY'].to_s.strip == 'X'} do
+      key   'fips_code', :field_name => 'FIPS CODE STATE'
+      store 'census_division_number', :field_name => 'Division'
+    end
+    
+    # PADD
+    import 'a list of state Petroleum Administration for Defense Districts',
+           :url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA' do
+      key   'postal_abbreviation', :field_name => 'State'
+      store 'petroleum_administration_for_defense_district_code', :field_name => 'Code'
+    end
+  end
+end
+

data/lib/earth/locality/urbanity.rb

@@ -0,0 +1,10 @@
+class Urbanity < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+
+  data_miner do
+    tap "Brighter Planet's sanitized urbanity data", Earth.taps_server
+  end
+end

data/lib/earth/locality/urbanity/data_miner.rb

@@ -0,0 +1,15 @@
+Urbanity.class_eval do
+  data_miner do
+    schema do
+      string 'name'
+    end
+
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO urbanities(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.urbanity_id FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.urbanity_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/locality/zip_code.rb

@@ -0,0 +1,23 @@
+class ZipCode < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :egrid_subregion, :foreign_key => 'egrid_subregion_abbreviation'
+  belongs_to :climate_division, :foreign_key => 'climate_division_name'
+  belongs_to :state, :foreign_key => 'state_postal_abbreviation'
+#  has_one :census_division, :through => :state
+#  has_one :census_region, :through => :state
+  has_many :residences
+
+  
+  acts_as_mappable :default_units => :miles,  # FIXME imperial
+                   :lat_column_name => :latitude,
+                   :lng_column_name => :longitude
+
+  data_miner do
+    tap "Brighter Planet's sanitized zip codes", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/locality/zip_code/data_miner.rb

@@ -0,0 +1,43 @@
+ZipCode.class_eval do
+  # sabshere 9/20/10 this isn't called anywhere
+  # def set_latitude_and_longitude
+  #   return if latitude.present? and longitude.present?
+  #   a = Geokit::Geocoders::YahooGeocoder.geocode name
+  #   update_attributes! :latitude => a.lat, :longitude => a.lng
+  # end
+  
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      string   'state_postal_abbreviation'
+      string   'description'
+      string   'latitude'
+      string   'longitude'
+      string   'egrid_subregion_abbreviation'
+      string   'climate_division_name'
+    end
+    
+    import 'the Mapping Hacks zipcode database',
+           :url => 'http://mappinghacks.com/data/zipcode.zip',
+           :filename => 'zipcode.csv' do
+      key   'name', :field_name => 'zip', :sprintf => '%05d'
+      store 'state_postal_abbreviation', :field_name => 'state'
+      store 'description', :field_name => 'city'
+      store 'latitude'
+      store 'longitude'
+    end
+    
+    import 'a list of zipcodes and eGRID Subregions',
+           :url => 'http://static.brighterplanet.com/science/data/electricity/egrid/models_export/zip_subregions.csv' do
+      key   'name', :field_name => 'zip', :sprintf => '%05d'
+      store 'egrid_subregion_abbreviation', :field_name => 'primary_subregion'
+    end
+    
+    import 'a list of zipcodes and Climate Divisions',
+           :url => 'http://static.brighterplanet.com/science/data/geography/zip_code_name-climate_division_name.csv' do
+      key   'name', :field_name => 'zip_code_name', :sprintf => '%05d'
+      store 'climate_division_name'
+    end
+  end
+end
+

data/lib/earth/pet.rb

@@ -0,0 +1,4 @@
+require 'earth/pet/breed'
+require 'earth/pet/breed_gender'
+require 'earth/pet/gender'
+require 'earth/pet/species'

data/lib/earth/pet/breed.rb

@@ -0,0 +1,15 @@
+class Breed < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :pets, :foreign_key => 'breed_id'
+  has_many :breed_genders, :foreign_key => 'breed_name'
+  belongs_to :species, :foreign_key => 'species_name'
+  
+  data_miner do
+    tap "Brighter Planet's breed data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/pet/breed/data_miner.rb

@@ -0,0 +1,25 @@
+Breed.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      string  'species_name'
+      float    'weight'
+      string   'weight_units'
+    end
+    
+    import "Brighter Planet's list of cat and dog breeds",
+           :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv',
+           :encoding => 'ISO-8859-1',
+           :select => lambda { |row| row['gender'].blank? } do
+      key 'name'
+      store 'species_name', :field_name => 'species'
+      store 'weight', :from_units => :pounds, :to_units => :kilograms
+    end
+    
+    import "a list of horse breeds", :url => 'http://www.freebase.com/type/exporttypeinstances/base/horses/horse_breed?page=0&filter_mode=type&filter_view=table&show%01p%3D%2Ftype%2Fobject%2Fname%01index=0&show%01p%3D%2Fcommon%2Ftopic%2Fimage%01index=1&show%01p%3D%2Fcommon%2Ftopic%2Farticle%01index=2&sort%01p%3D%2Ftype%2Fobject%2Ftype%01p%3Dlink%01p%3D%2Ftype%2Flink%2Ftimestamp%01index=false&=&exporttype=csv-8' do
+      key 'name'
+      store 'species_name', :static => 'horse'
+    end
+  end
+end
+

data/lib/earth/pet/breed_gender.rb

@@ -0,0 +1,14 @@
+class BreedGender < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :breed, :foreign_key => 'breed_name'
+  belongs_to :gender, :foreign_key => 'gender_name'
+  
+  data_miner do
+    tap "Brighter Planet's breed gender data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/pet/breed_gender/data_miner.rb

@@ -0,0 +1,21 @@
+BreedGender.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      string   'breed_name'
+      string   'gender_name'
+      float    'weight'
+      string   'weight_units'
+    end
+    
+    import "Brighter Planet's list of cat and dog breeds, genders, and weights",
+           :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv',
+           :select => lambda { |row| row['gender'].present? } do
+      key 'name', :synthesize => lambda { |row| row['name'] + ' ' + row['gender'] }
+      store 'breed_name', :field_name => 'name'
+      store 'gender_name', :field_name => 'gender'
+      store 'weight', :from_units => :pounds, :to_units => :kilograms
+    end
+  end
+end
+

data/lib/earth/pet/data_miner.rb

@@ -0,0 +1,4 @@
+require 'earth/pet/breed/data_miner'
+require 'earth/pet/breed_gender/data_miner'
+require 'earth/pet/gender/data_miner'
+require 'earth/pet/species/data_miner'

data/lib/earth/pet/gender.rb

@@ -0,0 +1,10 @@
+class Gender < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :pets
+  has_many :breed_genders, :foreign_key => 'gender_name'
+  
+  data_miner do
+    tap "Brighter Planet's gender info", Earth.taps_server
+  end
+end

data/lib/earth/pet/gender/data_miner.rb

@@ -0,0 +1,13 @@
+Gender.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'name'
+    end
+    import "a list of genders derived from pet breed genders",
+           :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv',
+           :select => lambda { |row| row['gender'].present? } do
+      key 'name', :field_name => 'gender'
+    end
+  end
+end
+

data/lib/earth/pet/species.rb

@@ -0,0 +1,40 @@
+class Species < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :pets
+  
+  scope :thoroughly_researched, :conditions => 'marginal_dietary_requirement IS NOT NULL'
+  
+  falls_back_on :diet_emission_intensity => lambda { weighted_average :diet_emission_intensity, :weighted_by => :population }, # kg CO2 / joule
+                :marginal_dietary_requirement => lambda { (thoroughly_researched.map(&:weighted_diet_size).sum / thoroughly_researched.sum(:population) ) / thoroughly_researched.weighted_average(:weight, :weighted_by => :population)}, # joules
+                :fixed_dietary_requirement => 0, # force a zero intercept to be respectful of our tiny tiny animal friends
+                :weight => lambda { weighted_average :weight, :weighted_by => :population } # kg
+
+  data_miner do
+    tap "Brighter Planet's species data", Earth.taps_server
+  end
+  
+  class << self
+    def [](name)
+      find_by_name name.to_s
+    end
+  end
+  
+  def diet_size
+    return unless weight and marginal_dietary_requirement and fixed_dietary_requirement
+    weight * marginal_dietary_requirement + fixed_dietary_requirement
+  end
+  
+  def weighted_diet_size
+    return unless _diet_size = diet_size and _population = population
+    _diet_size * _population
+  end
+  
+  def to_s
+    name
+  end
+  
+  def cat?
+    eql? self.class[:cat]
+  end
+end