earth 0.0.18

data/lib/earth/industry/product_line.rb

@@ -0,0 +1,10 @@
+class ProductLine < ActiveRecord::Base
+  set_primary_key :ps_code
+  
+  has_many :product_lines_sectors, :foreign_key => 'ps_code'
+  has_many :sectors, :through => :product_lines_sectors
+  
+  data_miner do
+    tap "Brighter Planet's product line data", Earth.taps_server
+  end
+end

data/lib/earth/industry/product_line/data_miner.rb

@@ -0,0 +1,18 @@
+ProductLine.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'ps_code'
+      string 'description'
+      string 'broadline' # FIXME TODO do we need this?
+      string 'parent'    # FIXME TODO do we need this?
+    end
+    
+    import "the U.S. Census list of Products and Services codes for the wholesale and retail trade sectors",
+           :url => 'http://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGxOUzBId0VuWTB0U0dOUHExOVlWTlE&hl=en&single=true&gid=0&output=csv' do
+      key 'ps_code'
+      store 'description'
+      store 'broadline'
+      store 'parent'
+    end
+  end
+end

data/lib/earth/industry/product_lines_sectors.rb

@@ -0,0 +1,6 @@
+class ProductLinesSectors < ActiveRecord::Base
+  set_primary_key :row_hash
+  
+  belongs_to :product_line, :foreign_key => 'ps_code'
+  belongs_to :sector,       :foreign_key => 'io_code'
+end

data/lib/earth/industry/sector.rb

@@ -0,0 +1,3 @@
+class Sector < ActiveRecord::Base
+  set_primary_key :io_code
+end

data/lib/earth/inflectors.rb

@@ -0,0 +1,9 @@
+ActiveSupport::Inflector.inflections do |inflect|
+  inflect.uncountable %w(aircraft aircraft Aircraft airline_aircraft AirlineAircraft)
+  inflect.uncountable 'commons'
+  inflect.uncountable 'food'
+  inflect.uncountable 'shelter'
+  inflect.uncountable 'transportation'
+  inflect.uncountable 'press_coverage'
+  inflect.irregular 'foot', 'feet'
+end

data/lib/earth/locality.rb

@@ -0,0 +1,10 @@
+require 'earth/locality/census_division'
+require 'earth/locality/census_region'
+require 'earth/locality/climate_division'
+require 'earth/locality/country'
+require 'earth/locality/egrid_region'
+require 'earth/locality/egrid_subregion'
+require 'earth/locality/petroleum_administration_for_defense_district'
+require 'earth/locality/state'
+require 'earth/locality/urbanity'
+require 'earth/locality/zip_code'

data/lib/earth/locality/census_division.rb

@@ -0,0 +1,17 @@
+class CensusDivision < ActiveRecord::Base
+  set_primary_key :number
+  
+  belongs_to :census_region, :foreign_key => 'census_region_number'
+  has_many :states, :foreign_key => 'census_division_number'
+  has_many :zip_codes, :through => :states
+  has_many :climate_divisions, :through => :states
+  has_many :residential_energy_consumption_survey_responses, :foreign_key => 'census_division_number'
+  
+  data_miner do
+    tap "Brighter Planet's sanitized census divisions", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/locality/census_division/data_miner.rb

@@ -0,0 +1,21 @@
+CensusDivision.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      integer  'number'
+      string   'name'
+      string   'census_region_name'
+      integer  'census_region_number'
+    end
+    
+    import 'the U.S. Census Geographic Terms and Definitions',
+           :url => 'http://www.census.gov/popest/geographic/codes02.csv',
+           :skip => 9,
+           :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
+      key   'number', :field_name => 'Division'
+      store 'name', :field_name => 'Name'
+      store 'census_region_number', :field_name => 'Region'
+      store 'census_region_name', :field_name => 'Region', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_regions.csv' }
+    end
+  end
+end
+

data/lib/earth/locality/census_region.rb

@@ -0,0 +1,13 @@
+class CensusRegion < ActiveRecord::Base
+  set_primary_key :number
+  
+  has_many :census_divisions, :foreign_key => 'census_region_number'
+  has_many :states, :through => :census_divisions
+  # has_many :climate_divisions, :through => :census_divisions
+  # has_many :zip_codes, :through => :census_divisions
+  has_many :residential_energy_consumption_survey_responses, :foreign_key => 'census_region_number'
+  
+  data_miner do
+    tap "Brighter Planet's sanitized census regions", Earth.taps_server
+  end
+end

data/lib/earth/locality/census_region/data_miner.rb

@@ -0,0 +1,17 @@
+CensusRegion.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      integer  'number'
+      string   'name'
+    end
+    
+    import 'the U.S. Census Geographic Terms and Definitions',
+           :url => 'http://www.census.gov/popest/geographic/codes02.csv',
+           :skip => 9,
+           :select => lambda { |row| row['Region'].to_i > 0 and row['Division'].to_s.strip == 'X'} do
+      key   'number', :field_name => 'Region'
+      store 'name', :field_name => 'Name'
+    end
+  end
+end
+

data/lib/earth/locality/climate_division.rb

@@ -0,0 +1,17 @@
+class ClimateDivision < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :zip_codes, :foreign_key => 'climate_division_name'
+  belongs_to :state, :foreign_key => 'state_postal_abbreviation'
+  
+  RADIUS = 750
+
+  data_miner do
+    tap "Brighter Planet's sanitized climate divisions", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+
+end

data/lib/earth/locality/climate_division/data_miner.rb

@@ -0,0 +1,20 @@
+ClimateDivision.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'heating_degree_days'
+      float    'cooling_degree_days'
+      string   'state_postal_abbreviation'
+    end
+    
+    import "a list of climate divisions and their average heating and cooling degree days",
+           :url => 'http://static.brighterplanet.com/science/data/climate/climate_divisions/climate_divisions.csv' do
+      key 'name'
+      store 'heating_degree_days'
+      store 'cooling_degree_days'
+      store 'state_postal_abbreviation'
+    end
+    #associate :state, :key => :state_postal_abbreviation, :foreign_key => :postal_abbreviation
+  end
+end
+

data/lib/earth/locality/country.rb

@@ -0,0 +1,13 @@
+class Country < ActiveRecord::Base
+  set_primary_key :iso_3166_code
+  
+  data_miner do
+    tap "Brighter Planet's sanitized countries list", Earth.taps_server
+  end
+  
+  class << self
+    def united_states
+      find_by_iso_3166_code('US')
+    end
+  end
+end

data/lib/earth/locality/country/data_miner.rb

@@ -0,0 +1,19 @@
+Country.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'iso_3166_code'
+      string   'name'
+    end
+    
+    import 'the official ISO country list',
+           :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
+           :skip => 2,
+           :headers => false,
+           :delimiter => ';',
+           :encoding => 'ISO-8859-1' do
+      key   'iso_3166_code', :field_number => 1
+      store 'name', :field_number => 0
+    end
+  end
+end
+

data/lib/earth/locality/data_miner.rb

@@ -0,0 +1,10 @@
+require 'earth/locality/census_division/data_miner'
+require 'earth/locality/census_region/data_miner'
+require 'earth/locality/climate_division/data_miner'
+require 'earth/locality/country/data_miner'
+require 'earth/locality/egrid_region/data_miner'
+require 'earth/locality/egrid_subregion/data_miner'
+require 'earth/locality/petroleum_administration_for_defense_district/data_miner'
+require 'earth/locality/state/data_miner'
+require 'earth/locality/urbanity/data_miner'
+require 'earth/locality/zip_code/data_miner'

data/lib/earth/locality/egrid_region.rb

@@ -0,0 +1,15 @@
+class EgridRegion < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :egrid_subregions, :foreign_key => 'egrid_region_name'
+  
+  data_miner do
+    tap "Brighter Planet's egrid region data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+  
+  falls_back_on :loss_factor => 0.061311 # Ian
+end

data/lib/earth/locality/egrid_region/data_miner.rb

@@ -0,0 +1,19 @@
+EgridRegion.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'loss_factor'
+    end
+    
+    import "eGRID regions and loss factors derived from eGRID 2007 data",
+           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
+           :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
+           :sheet => 'STIE05',
+           :select => lambda { |row| row['eGRID2007 2005 file State sequence number'].to_i.between?(1, 51) },
+           :skip => 4 do
+      key   'name', :field_name => 'Grid region (E=Eastern grid, W=Western grid, AK=Alaska, HI=Hawaii, TX=Texas)'
+      store 'loss_factor', :field_name => '2005 grid gross loss factor'
+    end
+  end
+end
+

data/lib/earth/locality/egrid_subregion.rb

@@ -0,0 +1,16 @@
+class EgridSubregion < ActiveRecord::Base
+  set_primary_key :abbreviation
+  
+  has_many :zip_codes, :foreign_key => 'egrid_subregion_abbreviation'
+  belongs_to :egrid_region, :foreign_key => 'egrid_region_name'
+  
+  falls_back_on :electricity_emission_factor => 1.404.pounds.to(:kilograms) # kg CO2 / kWh https://brighterplanet.sifterapp.com/projects/30/issues/437?return_uri=%2Fprojects%2F30%2Fissues%3Fa%3D79%26s%3D1-2
+  
+  data_miner do
+    tap "Brighter Planet's egrid subregion data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/locality/egrid_subregion/data_miner.rb

@@ -0,0 +1,26 @@
+EgridSubregion.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'abbreviation'
+      string 'name'
+      float  'electricity_emission_factor'
+      string 'electricity_emission_factor_units'
+      string 'nerc_abbreviation'
+      string 'egrid_region_name'
+    end
+    
+    process "Define some unit conversions" do
+      Conversions.register :kilograms_per_kilowatt_hour, :kilograms_per_megawatt_hour, 1_000.0
+    end
+    
+    import "a list of eGRID subregions and pre-calculated emissions factors",
+           :url => 'http://static.brighterplanet.com/science/data/electricity/egrid/models_export/egrid_subregions.csv' do
+      key   'abbreviation'
+      store 'name'
+      store 'electricity_emission_factor', :from_units => :kilograms_per_megawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+      store 'nerc_abbreviation'
+      store 'egrid_region_name'
+    end
+  end
+end
+

data/lib/earth/locality/petroleum_administration_for_defense_district.rb

@@ -0,0 +1,13 @@
+class PetroleumAdministrationForDefenseDistrict < ActiveRecord::Base
+  set_primary_key :code
+  
+  data_miner do
+    tap "Brighter Planet's PADD info", Earth.taps_server
+  end
+  
+  def name
+    str = "PAD District #{district_code} (#{district_name})"
+    str << " Subdistrict #{district_code}#{subdistrict_code} (#{subdistrict_name})" if subdistrict_code
+    str
+  end
+end

data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb

@@ -0,0 +1,21 @@
+PetroleumAdministrationForDefenseDistrict.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'code'
+      string   'district_code'
+      string   'district_name'
+      string   'subdistrict_code'
+      string   'subdistrict_name'
+    end
+    
+    import "a list of PADD districts and states derived from the EIA PADD definitions",
+           :url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA' do
+      key   'code',             :field_name => 'Code'
+      store 'district_code',    :field_name => 'PAD district code'
+      store 'subdistrict_code', :field_name => 'PAD subdistrict code'
+      store 'district_name',    :field_name => 'PAD district name'
+      store 'subdistrict_name', :field_name => 'PAD subdistrict name'
+    end
+  end
+end
+

data/lib/earth/locality/state.rb

@@ -0,0 +1,22 @@
+class State < ActiveRecord::Base
+  set_primary_key :postal_abbreviation
+  
+  has_many :climate_divisions, :foreign_key => 'state_postal_abbreviation'
+  has_many :zip_codes, :foreign_key => 'state_postal_abbreviation'
+  belongs_to :census_division, :foreign_key => 'census_division_number'
+  belongs_to :petroleum_administration_for_defense_district, :foreign_key => 'petroleum_administration_for_defense_district_code'
+  has_one :census_region, :through => :census_division
+  
+  def country
+    Country.united_states
+  end
+
+  data_miner do
+    tap "Brighter Planet's sanitized states", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+
+end

data/lib/earth/locality/state/data_miner.rb

@@ -0,0 +1,37 @@
+State.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'postal_abbreviation'
+      integer  'fips_code'
+      string   'name'
+      string   'census_division_number'
+      string   'petroleum_administration_for_defense_district_code'
+    end
+    
+    # state names, ansi codes, and postal abbreviations
+    import 'the U.S. Census State ANSI Code file',
+           :url => 'http://www.census.gov/geo/www/ansi/state.txt',
+           :delimiter => '|' do
+      key   'postal_abbreviation', :field_name => 'STUSAB'
+      store 'fips_code', :field_name => 'STATE'
+      store 'name', :field_name => 'STATE_NAME'
+    end
+
+    # census divisions
+    import 'the U.S. Census Geographic Terms and Definitions',
+           :url => 'http://www.census.gov/popest/geographic/codes02.csv',
+           :skip => 9,
+           :select => lambda { |row| row['FIPS CODE STATE'].to_s.strip != 'X' and row['FIPS CODE COUNTY'].to_s.strip == 'X'} do
+      key   'fips_code', :field_name => 'FIPS CODE STATE'
+      store 'census_division_number', :field_name => 'Division'
+    end
+    
+    # PADD
+    import 'a list of state Petroleum Administration for Defense Districts',
+           :url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA' do
+      key   'postal_abbreviation', :field_name => 'State'
+      store 'petroleum_administration_for_defense_district_code', :field_name => 'Code'
+    end
+  end
+end
+

data/lib/earth/locality/urbanity.rb

@@ -0,0 +1,20 @@
+class Urbanity < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+
+  data_miner do
+    schema do
+      string :name
+    end
+
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO urbanities(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.urbanity_id FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.urbanity_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/locality/urbanity/data_miner.rb

@@ -0,0 +1,4 @@
+Urbanity.class_eval do
+#  data_miner do
+#...
+end