earth 0.11.16 → 0.11.17

data/lib/earth/fuel/fuel/data_miner.rb

@@ -14,7 +14,7 @@ Fuel.class_eval do
     
     import "liquid fuels with non-variable characteristics",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGJmYkdtajZyV3Byb0lrd21xLVhXUGc&output=csv',
-           :select => lambda {|row| row['energy_content_units'] == 'million_btu_per_barrel'} do
+           :select => proc {|row| row['energy_content_units'] == 'million_btu_per_barrel'} do
       key 'name'
       store 'physical_units', :static => 'litre'
       store 'energy_content', :from_units => :million_btu_per_barrel,        :to_units => :megajoules_per_litre
@@ -25,7 +25,7 @@ Fuel.class_eval do
     
     import "gaseous fuels with non-variable characteristics",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGJmYkdtajZyV3Byb0lrd21xLVhXUGc&output=csv',
-           :select => lambda {|row| row['energy_content_units'] == 'btu_per_cubic_foot'} do
+           :select => proc {|row| row['energy_content_units'] == 'btu_per_cubic_foot'} do
       key 'name'
       store 'physical_units', :static => 'cubic_metre'
       store 'energy_content', :from_units => :btus_per_cubic_foot,           :to_units => :megajoules_per_cubic_metre

data/lib/earth/fuel/fuel_price/data_miner.rb

@@ -6,7 +6,7 @@ FuelPrice.class_eval do
   data_miner do
     import 'fuel prices derived from the EIA',
            :url => 'http://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlSdXJoOFB5aEpHenJQbTVJdS1pMVE&gid=0&output=csv',
-           :select => lambda { |row| row['fuel_type_name'].present? } do
+           :select => proc { |row| row['fuel_type_name'].present? } do
       key   'name', :field_name => 'fuel_type_name'
       store 'price', :units_field_name => 'price_units'
     end

data/lib/earth/industry/cbecs_energy_intensity/data_miner.rb

@@ -128,7 +128,7 @@ CbecsEnergyIntensity.class_eval do
     }
   })
 
-  const_set(:NAICS_CODE_SYNTHESIZER, lambda { |row|
+  const_set(:NAICS_CODE_SYNTHESIZER, proc { |row|
     case row[0].to_s
     when /Education/
       61

data/lib/earth/industry/naics_2002_naics_2007_concordance/data_miner.rb

@@ -8,9 +8,9 @@ Naics2002Naics2007Concordance.class_eval do
            :skip => 3,
            :headers => %w{ 2002_code 2002_title 2007_code 2007_title } do
       key 'row_hash'
-      store 'naics_2002_code', :synthesize => lambda { |row| "%06d" % row['2002_code'].to_i }
-      store 'naics_2007_code', :synthesize => lambda { |row| "%06d" % row['2007_code'].to_i }
-      store 'naics_2002_note', :synthesize => lambda { |row| extract_note row['2002_title'] }
+      store 'naics_2002_code', :synthesize => proc { |row| "%06d" % row['2002_code'].to_i }
+      store 'naics_2007_code', :synthesize => proc { |row| "%06d" % row['2007_code'].to_i }
+      store 'naics_2002_note', :synthesize => proc { |row| Naics2002Naics2007Concordance.extract_note row['2002_title'] }
     end
   end
 end

data/lib/earth/industry/naics_2002_sic_1987_concordance/data_miner.rb

@@ -5,11 +5,11 @@ Naics2002Sic1987Concordance.class_eval do
   data_miner do
     import "the US Census SIC 1987 to NAICS 2002 concordance",
            :url => 'http://www.census.gov/eos/www/naics/concordances/1987_SIC_to_2002_NAICS.xls',
-           :select => lambda {|row| row['SIC'].to_i > 0} do
+           :select => proc {|row| row['SIC'].to_i > 0 } do
       key 'row_hash'
-      store 'naics_2002_code', :synthesize => lambda { |row| "%06d" % row['2002 NAICS'].to_i }
-      store 'sic_1987_code',   :synthesize => lambda { |row| "%04d" % row['SIC'].to_i }
-      store 'sic_note', :synthesize => lambda { |row| extract_note row['SIC Title (and note)'] }
+      store 'naics_2002_code', :synthesize => proc { |row| "%06d" % row['2002 NAICS'].to_i }
+      store 'sic_1987_code',   :synthesize => proc { |row| "%04d" % row['SIC'].to_i }
+      store 'sic_note', :synthesize => proc { |row| Naics2002Sic1987Concordance.extract_note row['SIC Title (and note)'] }
     end
   end
 end

data/lib/earth/industry/sic_1987/data_miner.rb

@@ -2,9 +2,9 @@ Sic1987.class_eval do
   data_miner do
     import "SIC 1987 codes derived from US Census SIC 1987 to NAICS 2002 concordance",
            :url => 'http://www.census.gov/eos/www/naics/concordances/1987_SIC_to_2002_NAICS.xls',
-           :select => lambda {|row| row['SIC'].to_i > 0} do
-      key 'code', :synthesize => lambda { |row| "%04d" % row['SIC'].to_i }
-      store 'description', :synthesize => lambda { |row| format_description row['SIC Title (and note)'] }
+           :select => proc {|row| row['SIC'].to_i > 0} do
+      key 'code', :synthesize => proc { |row| "%04d" % row['SIC'].to_i }
+      store 'description', :synthesize => proc { |row| Sic1987.format_description row['SIC Title (and note)'] }
     end
   end
 end

data/lib/earth/locality/census_region/data_miner.rb

@@ -4,7 +4,7 @@ CensusRegion.class_eval do
            :url => 'http://www.census.gov/popest/about/geo/state_geocodes_v2009.txt',
            :skip => 6,
            :headers => %w{ Region Division FIPS Name },
-           :select => ::Proc.new { |row| row['Region'].to_i > 0 and row['Division'].to_i == 0 }) do
+           :select => proc { |row| row['Region'].to_i > 0 and row['Division'].to_i == 0 }) do
       key   'number', :field_name => 'Region'
       store 'name', :field_name => 'Name'
     end

data/lib/earth/locality/country/data_miner.rb

@@ -21,7 +21,7 @@ Country.class_eval do
     import "heating and cooling degree day data from WRI CAIT",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDN4MkRTSWtWRjdfazhRdWllTkVSMkE&output=csv',
            :select => Proc.new { |record| record['country'] != 'European Union (27)' },
-           :errata => { :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDNSMUtCV0h4cUF4UnBKZlNkczlNbFE&output=csv' } do
+           :errata => { :url => "file://#{Earth::ERRATA_DIR}/country/wri_errata.csv" } do
       key 'name', :field_name => 'country'
       store 'heating_degree_days', :units => :degrees_celsius
       store 'cooling_degree_days', :units => :degrees_celsius
@@ -91,7 +91,7 @@ Country.class_eval do
     import "calculate national electricity emission factors from Brander et al. (2011)",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDZmWHFjLVdBZGRBdGxVdDdqd1YtYWc&output=csv' do
       key 'iso_3166_code', :field_name => 'country_iso_3166_code'
-      store 'electricity_emission_factor', :synthesize => lambda { |row|
+      store 'electricity_emission_factor', :synthesize => proc { |row|
         (
           row['electricity_co2_emission_factor'].to_f +
           (row['electricity_ch4_emission_factor'].to_f * GreenhouseGas[:ch4].global_warming_potential) +

data/lib/earth/locality/egrid_region/data_miner.rb

@@ -4,7 +4,7 @@ EgridRegion.class_eval do
            :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2010V1_1_STIE_USGC.xls',
            :sheet => 'STIE07',
            :skip => 4,
-           :select => lambda { |row| row['eGRID2010 year 2007 file state sequence number'].to_i.between?(1, 51) } do
+           :select => proc { |row| row['eGRID2010 year 2007 file state sequence number'].to_i.between?(1, 51) } do
       key   'name', :field_name => 'Grid region (E=Eastern grid, W=Western grid, AK=Alaska, HI=Hawaii, TX=Texas)'
       store 'country_name', :static => 'US'
       store 'loss_factor', :field_name => 'Year 2007 grid gross loss factor'

data/lib/earth/locality/egrid_subregion/data_miner.rb

@@ -6,7 +6,7 @@ EgridSubregion.class_eval do
            :filename => 'eGRID2010V1_1_year07_AGGREGATION.xls',
            :sheet => 'SRL07',
            :skip => 4,
-           :select => lambda { |row| row['SEQSRL07'].to_i.between?(1, 26) } do
+           :select => proc { |row| row['SEQSRL07'].to_i.between?(1, 26) } do
       key   'abbreviation', :field_name => 'SUBRGN'
       store 'name', :field_name => 'SRNAME'
       store 'nerc_abbreviation',               :field_name => 'NERC'

data/lib/earth/locality/state/data_miner.rb

@@ -4,7 +4,7 @@ State.class_eval do
     import 'the U.S. Census State ANSI Code file',
            :url => 'http://www.census.gov/geo/www/ansi/state.txt',
            :delimiter => '|',
-           :select => lambda { |record| record['STATE'].to_i < 60 } do
+           :select => proc { |record| record['STATE'].to_i < 60 } do
       key   'postal_abbreviation', :field_name => 'STUSAB'
       store 'fips_code',           :field_name => 'STATE'
       store 'name',                :field_name => 'STATE_NAME'
@@ -15,7 +15,7 @@ State.class_eval do
            :url => 'http://www.census.gov/popest/about/geo/state_geocodes_v2009.txt',
            :skip => 8,
            :headers => ['Region', 'Division', 'State FIPS', 'Name'],
-           :select => ::Proc.new { |row| row['State FIPS'].to_i > 0 } do
+           :select => proc { |row| row['State FIPS'].to_i > 0 } do
       key   'fips_code', :field_name => 'State FIPS'
       store 'census_division_number', :field_name => 'Division'
     end

data/lib/earth/locality/zip_code.rb

@@ -1,4 +1,4 @@
-require ::File.join(Earth.vendor_dir, 'geokit-rails', 'lib', 'geokit-rails')
+require ::File.join(Earth::VENDOR_DIR, 'geokit-rails', 'lib', 'geokit-rails')
 
 class ZipCode < ActiveRecord::Base
   self.primary_key = "name"

data/lib/earth/pet/breed/data_miner.rb

@@ -3,7 +3,7 @@ Breed.class_eval do
     import "Brighter Planet's list of cat and dog breeds",
            :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv',
            :encoding => 'ISO-8859-1',
-           :select => lambda { |row| row['gender'].blank? } do
+           :select => proc { |row| row['gender'].blank? } do
       key 'name'
       store 'species_name', :field_name => 'species'
       store 'weight', :from_units => :pounds, :to_units => :kilograms

data/lib/earth/pet/breed_gender/data_miner.rb

@@ -3,8 +3,8 @@ BreedGender.class_eval do
     import "Brighter Planet's list of cat and dog breeds, genders, and weights",
            :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv',
            :encoding => 'ISO-8859-1',
-           :select => lambda { |row| row['gender'].present? } do
-      key 'name', :synthesize => lambda { |row| row['name'] + ' ' + row['gender'] }
+           :select => proc { |row| row['gender'].present? } do
+      key 'name', :synthesize => proc { |row| row['name'] + ' ' + row['gender'] }
       store 'breed_name', :field_name => 'name'
       store 'gender_name', :field_name => 'gender'
       store 'weight', :from_units => :pounds, :to_units => :kilograms

data/lib/earth/pet/gender/data_miner.rb

@@ -2,7 +2,7 @@ Gender.class_eval do
   data_miner do
     import "a list of genders derived from pet breed genders",
            :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv',
-           :select => lambda { |row| row['gender'].present? } do
+           :select => proc { |row| row['gender'].present? } do
       key 'name', :field_name => 'gender'
     end
   end

data/lib/earth/rail/country_rail_class/data_miner.rb

@@ -3,7 +3,7 @@ CountryRailClass.class_eval do
   data_miner do
     import "a list of country-specific rail classes and their pre-calculated characteristics",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdFg3YVptS2FYdHBsb2kyZFRxSm5FdlE&output=csv' do
-      key   'name', :synthesize => lambda { |record| [record['country_iso_3166_code'], record['rail_class_name']].join(' ') }
+      key   'name', :synthesize => proc { |record| [record['country_iso_3166_code'], record['rail_class_name']].join(' ') }
       store 'country_iso_3166_code'
       store 'rail_class_name'
       store 'passengers'

data/lib/earth/rail/country_rail_traction/data_miner.rb

@@ -4,7 +4,7 @@ CountryRailTraction.class_eval do
   data_miner do
     import "european rail traction data from the UIC",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHA1RmZ0SlN6ZkVOenRodGFBQ1N0Q2c&output=csv' do
-      key 'name', :synthesize => lambda { |record| [record['country_iso_3166_code'], record['rail_traction_name']].join(' ') }
+      key 'name', :synthesize => proc { |record| [record['country_iso_3166_code'], record['rail_traction_name']].join(' ') }
       store 'country_iso_3166_code'
       store 'rail_traction_name'
       store 'electricity_intensity', :nullify => :true, :units_field_name => 'electricity_intensity_units'

data/lib/earth/rail/country_rail_traction_class/data_miner.rb

@@ -4,7 +4,7 @@ CountryRailTractionClass.class_eval do
   data_miner do
     import "european rail traction class data from the UIC",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDBiV0VPdkg2UkNSV0pVdWNNcG9fMHc&output=csv' do
-      key 'name', :synthesize => lambda { |record| [record['country_iso_3166_code'], record['rail_traction_name'], record['rail_class_name']].join(' ') }
+      key 'name', :synthesize => proc { |record| [record['country_iso_3166_code'], record['rail_traction_name'], record['rail_class_name']].join(' ') }
       store 'country_iso_3166_code'
       store 'rail_traction_name'
       store 'rail_class_name'

data/lib/earth/rail/national_transit_database_record/data_miner.rb

@@ -3,7 +3,7 @@ NationalTransitDatabaseRecord.class_eval do
   data_miner do
     import "US National Transit Database service data",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdF9Hbktvc3BMNDRWZHhDRU9uMVZfWVE&output=csv' do
-      key 'name', :synthesize => lambda { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }
+      key 'name', :synthesize => proc { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }
       store 'company_id', :field_name => 'trs_id'
       store 'mode_code'
       store 'service_type',       :field_name => 'service_code', :dictionary => { :input => 'code', :output => 'name', :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDBTZENxb3A1SzJaNTVEVGZ5cTdaMVE&output=csv' }
@@ -15,11 +15,11 @@ NationalTransitDatabaseRecord.class_eval do
     
     import "US National Transit Database fuel consumption data",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDItVVR5NjA2Y3FCVjgza25Ccy0zS2c&output=csv' do
-      key 'name', :synthesize => lambda { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }
+      key 'name', :synthesize => proc { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }
       store 'company_id', :field_name => 'trs_id'
       store 'mode_code'
       store 'service_type', :field_name => 'service_code', :dictionary => { :input => 'code', :output => 'name', :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDBTZENxb3A1SzJaNTVEVGZ5cTdaMVE&output=csv' }
-      store 'electricity',  :synthesize => lambda { |row| row['electricity_kwh'].to_f + row['battery_kwh'].to_f }, :units => 'kilowatt_hours'
+      store 'electricity',  :synthesize => proc { |row| row['electricity_kwh'].to_f + row['battery_kwh'].to_f }, :units => 'kilowatt_hours'
       store 'diesel',       :field_name => 'diesel_gallons',    :from_units => :gallons, :to_units => :litres
       store 'gasoline',     :field_name => 'gasoline_gallons',  :from_units => :gallons, :to_units => :litres
       store 'lpg',          :field_name => 'lpg_gallons',       :from_units => :gallons, :to_units => :litres

data/lib/earth/residence/residence_fuel_price/data_miner.rb

@@ -113,7 +113,7 @@ ResidenceFuelPrice.class_eval do
     # electricity in dollars per kWh
     import 'residential electricity prices from the EIA',
            :url => 'http://www.eia.doe.gov/cneaf/electricity/page/sales_revenue.xls',
-           :select => lambda { |row| row['Year'].to_s.first(4).to_i > 1989 } do
+           :select => proc { |row| row['Year'].to_s.first(4).to_i > 1989 } do
       key 'row_hash'
       store 'residence_fuel_type_name', :static => 'electricity'
       store 'locatable_id', :field_name => 'State' # postal abbrev
@@ -127,10 +127,10 @@ ResidenceFuelPrice.class_eval do
     # natural gas in dollars per cubic metre
     # breaks if date-performance is enabled because DateTime.parse(...1899...) dies
     import 'residential natural gas prices from the EIA',
-           :url => 'http://tonto.eia.doe.gov/dnav/ng/xls/ng_pri_sum_a_EPG0_FWA_DMcf_a.xls',
+           :url => 'http://www.eia.gov/dnav/ng/xls/ng_pri_sum_a_EPG0_FWA_DMcf_a.xls',
            :sheet => 'Data 1',
            :skip => 2,
-           :select => lambda { |row| row['year'].to_i > 1989 },
+           :select => proc { |row| row['year'].to_i > 1989 },
            :transform => { :class => NaturalGasParser } do
       key 'row_hash'
       store 'residence_fuel_type_name', :static => 'natural gas'
@@ -150,7 +150,7 @@ ResidenceFuelPrice.class_eval do
     #        :url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_RESID_A_EPPR_PTA_CPGAL_M.xls',
     #        :sheet => 'Data 1',
     #        :skip => 2,
-    #        :select => lambda { |row| row['year'].to_i > 1989 },
+    #        :select => proc { |row| row['year'].to_i > 1989 },
     #        :transform => { :class => FuelOilParser } do
     #   key 'row_hash'
     #   store 'residence_fuel_type_name', :static => 'fuel oil'
@@ -169,7 +169,7 @@ ResidenceFuelPrice.class_eval do
     #        :url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_PROP_A_EPLLPA_PRT_CPGAL_M.xls',
     #        :sheet => 'Data 1',
     #        :skip => 2,
-    #        :select => lambda { |row| row['year'].to_i > 1989 },
+    #        :select => proc { |row| row['year'].to_i > 1989 },
     #        :transform => { :class => PropaneParser } do
     #   key 'row_hash'
     #   store 'residence_fuel_type_name', :static => 'propane'

data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb

@@ -63,16 +63,16 @@ ResidentialEnergyConsumptionSurveyResponse.class_eval do
       store 'outdoor_all_night_lights', :field_name => 'NOUTLGTNT'
       store 'outdoor_all_night_gas_lights', :field_name => 'NGASLIGHT'
       # integers where we treat anything other than true (for example legitimate skip or "occupied without paying rent") as 0
-      store 'heated_garage', :synthesize => lambda { |row| row['GARGHEAT'] == '1' ? 1 : 0 }
-      store 'attached_1car_garage', :synthesize => lambda { |row| row['GARAGE1C'] == '1' ? 1 : 0 }
-      store 'detached_1car_garage', :synthesize => lambda { |row| row['DGARG1C'] == '1' ? 1 : 0 }
-      store 'attached_2car_garage', :synthesize => lambda { |row| row['GARAGE2C'] == '1' ? 1 : 0 }
-      store 'detached_2car_garage', :synthesize => lambda { |row| row['DGARG2C'] == '1' ? 1 : 0 }
-      store 'attached_3car_garage', :synthesize => lambda { |row| row['GARAGE3C'] == '1' ? 1 : 0 }
-      store 'detached_3car_garage', :synthesize => lambda { |row| row['DGARG3C'] == '1' ? 1 : 0 }
+      store 'heated_garage', :synthesize => proc { |row| row['GARGHEAT'] == '1' ? 1 : 0 }
+      store 'attached_1car_garage', :synthesize => proc { |row| row['GARAGE1C'] == '1' ? 1 : 0 }
+      store 'detached_1car_garage', :synthesize => proc { |row| row['DGARG1C'] == '1' ? 1 : 0 }
+      store 'attached_2car_garage', :synthesize => proc { |row| row['GARAGE2C'] == '1' ? 1 : 0 }
+      store 'detached_2car_garage', :synthesize => proc { |row| row['DGARG2C'] == '1' ? 1 : 0 }
+      store 'attached_3car_garage', :synthesize => proc { |row| row['GARAGE3C'] == '1' ? 1 : 0 }
+      store 'detached_3car_garage', :synthesize => proc { |row| row['DGARG3C'] == '1' ? 1 : 0 }
       # booleans where we treat anything other than true (for example legitimate skip or "occupied without paying rent") as false
-      store 'ownership', :synthesize => lambda { |row| row['KOWNRENT'] == '1' }
-      store 'thermostat_programmability', :synthesize => lambda { |row| row['PROTHERM'] == '1' }
+      store 'ownership', :synthesize => proc { |row| row['KOWNRENT'] == '1' }
+      store 'thermostat_programmability', :synthesize => proc { |row| row['PROTHERM'] == '1' }
     end
 
     # Rather than nullify the continuous variables that EIA identifies as LEGITIMATE SKIPS, we convert them to zero

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.11.16"
+  VERSION = "0.11.17"
 end

data/spec/earth/air/aircraft_spec.rb

@@ -1,17 +1,23 @@
 require 'spec_helper'
 require 'earth/air/aircraft'
-require 'earth/air/aircraft/data_miner'
 
 describe Aircraft do
+  before :all do
+    Aircraft.auto_upgrade!
+  end
+  
   describe 'import', :data_miner => true do
-    it 'should treat empty cells as null' do
-      Aircraft.auto_upgrade!
+    before do
+      require 'earth/air/aircraft/data_miner'
+    end
+    
+    it 'should import data' do
       Aircraft.run_data_miner!
-      Aircraft.count.should > 1
-      Aircraft.where(:brighter_planet_aircraft_class_code => nil).should_not be_empty
+      Aircraft.all.count.should == 437
     end
   end
   
+  # FIXME TODO
   # describe 'fuzzy matching' do
   #   it 'should match some example aircraft properly' do
   #     user_input = "Boeing 727-100RE"

data/spec/earth/air/airport_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+require 'earth/air/airport'
+
+describe Airport do
+  before :all do
+    Airport.auto_upgrade!
+  end
+  
+  describe "when importing data", :data_miner => true do
+    before do
+      require 'earth/air/airport/data_miner'
+    end
+    
+    it "imports all airports" do
+      Airport.run_data_miner!
+      Airport.count.should == 5324
+    end
+  end
+  
+  describe "verify imported data", :sanity => true do
+    it "should have name" do
+      Airport.where(:name => nil).count.should == 0
+    end
+    
+    it "should have city" do
+      Airport.where(:city => nil).count.should == 0
+    end
+    
+    it "should have country code" do
+      Airport.where(:country_iso_3166_code => nil).count.should == 0
+    end
+    
+    it "should have lat/lng" do
+      Airport.where(:latitude => nil).count.should == 0
+      Airport.where(:longitude => nil).count.should == 0
+    end
+    
+    # FIXME TODO check for duplicate cities in same country
+  end
+end

data/spec/earth/air/bts_aircraft_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'earth/air/bts_aircraft'
+
+describe BtsAircraft do
+  before :all do
+    BtsAircraft.auto_upgrade!
+  end
+  
+  describe 'import', :data_miner => true do
+    before do
+      require 'earth/air/bts_aircraft/data_miner'
+    end
+    
+    it 'should import data' do
+      BtsAircraft.run_data_miner!
+      BtsAircraft.all.count.should == 378
+    end
+  end
+end

data/spec/earth/air/flight_segment_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+require 'earth/air/flight_segment'
+
+describe FlightSegment do
+  before :all do
+    FlightSegment.auto_upgrade!
+  end
+  
+  describe 'import', :data_miner => true do
+    before do
+      require 'earth/air/flight_segment/data_miner'
+    end
+    
+    it 'should import data' do
+      FlightSegment.run_data_miner!
+      FlightSegment.all.count.should == 1_149_003
+    end
+  end
+  
+  describe "verify imported data", :sanity => true do
+    it "should have year from 2009 to present" do
+      FlightSegment.where("year IS NULL OR year < 2009 OR year > #{::Time.now.year}").count.should == 0
+    end
+    
+    it "should have origin airport in airports" do
+      # FIXME TODO
+    end
+    
+    it "should have destination airport in airports" do
+      # FIXME TODO
+    end
+    
+    it "should have origin country iso code in countries" do
+      # FIXME TODO
+    end
+    
+    it "should have destination country iso code in countries" do
+      # FIXME TODO
+    end
+    
+    it "should have airline name" do
+      FlightSegment.where(:airline_name => nil).count.should == 0
+    end
+    
+    it "should have aircraft description" do
+      FlightSegment.where(:aircraft_description => nil).count.should == 0
+    end
+  end
+end