data_miner 1.3.8 → 2.0.1

data/test/support/airport.rb

@@ -1,16 +0,0 @@
-class Airport < ActiveRecord::Base
-  set_primary_key :iata_code
-  
-  data_miner do
-    import :url => 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat',
-           :headers => false,
-           :select => lambda { |row| row[4].present? } do
-      key 'iata_code', :field_number => 4
-      store 'name', :field_number => 1
-      store 'city', :field_number => 2
-      store 'country_name', :field_number => 3
-      store 'latitude', :field_number => 6, :nullify => true
-      store 'longitude', :field_number => 7, :nullify => true
-    end
-  end
-end

data/test/support/automobile_fuel_type.rb

@@ -1,40 +0,0 @@
-class AutomobileFuelType < ActiveRecord::Base
-  set_primary_key :code
-  
-  data_miner do
-    import(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
-                :filename => 'Gd6-dsc.txt',
-                :format => :fixed_width,
-                :crop => 21..26, # inclusive
-                :cut => '2-',
-                :select => lambda { |row| /\A[A-Z]/.match row[:code] },
-                :schema => [[ 'code',   2, { :type => :string }  ],
-                            [ 'spacer', 2 ],
-                            [ 'name',   52, { :type => :string } ]]) do
-      key 'code'
-      store 'name'
-    end
-
-    import :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_fuel_type.csv' do
-      key 'code'
-      store 'name'
-      store 'annual_distance'
-      store 'emission_factor'
-    end
-
-    # pull electricity emission factor from residential electricity
-    import(:url => 'http://spreadsheets.google.com/pub?key=rukxnmuhhsOsrztTrUaFCXQ',
-                :select => lambda { |row| row['code'] == 'El' }) do
-      key 'code'
-      store 'name'
-      store 'emission_factor'
-    end
-    
-    # still need distance estimate for electric cars
-  end
-  
-  CODES = {
-    :electricity => 'El',
-    :diesel => 'D'
-  }
-end

data/test/support/automobile_variant.rb

@@ -1,362 +0,0 @@
-class AutomobileVariant < ActiveRecord::Base
-  set_primary_key :row_hash
-
-  module FuelEconomyGuide
-    TRANSMISSIONS = {
-      'A' => 'automatic', 
-      'M' => 'manual', 
-      'L' => 'automatic',     # Lockup/automatic
-      'S' => 'semiautomatic', # Semiautomatic
-      'C' => 'manual' # TODO verify for VW Syncro
-    }
-
-    ENGINE_TYPES = {
-      '(GUZZLER)' => nil, # "gas guzzler"
-      '(POLICE)' => nil, # police automobile_variant
-      '(MPFI)' => 'injection',
-      '(MPI*)' => 'injection',
-      '(SPFI)' => 'injection',
-      '(FFS)' => 'injection',
-      '(TURBO)' => 'turbo',
-      '(TRBO)' => 'turbo',
-      '(TC*)' => 'turbo',
-      '(FFS,TRBO)' => %w(injection turbo),
-      '(S-CHARGE)' => 'supercharger',
-      '(SC*)' => 'supercharger',
-      '(DIESEL)' => nil, # diesel
-      '(DSL)' => nil, # diesel
-      '(ROTARY)' => nil, # rotary
-      '(VARIABLE)' => nil, # variable displacement
-      '(NO-CAT)' => nil, # no catalytic converter
-      '(OHC)' => nil, # overhead camshaft
-      '(OHV)' => nil, # overhead valves
-      '(16-VALVE)' => nil, # 16V
-      '(305)' => nil, # 305 cubic inch displacement
-      '(307)' => nil, # 307 cubic inch displacement
-      '(M-ENG)' => nil,
-      '(W-ENG)' => nil,
-      '(GM-BUICK)' => nil,
-      '(GM-CHEV)' => nil,
-      '(GM-OLDS)' => nil,
-      '(GM-PONT)' => nil,
-    }
-
-    class ParserB
-      require 'fixed_width'
-      ::FixedWidth.define :fuel_economy_guide_b do |d|
-        d.rows do |row|
-          row.trap { true } # there's only one section
-          row.column 'active_year'      , 4,    :type => :integer  #   ACTIVE YEAR
-          row.column 'state_code'       , 1,    :type => :string  #   STATE CODE:  F=49-STATE,C=CALIFORNIA
-          row.column 'carline_clss'     , 2,    :type => :integer  #   CARLINE CLASS CODE
-          row.column 'carline_mfr_code' , 3,    :type => :integer  #   CARLINE MANUFACTURER CODE
-          row.column 'carline_name'     , 28,   :type => :string  #   CARLINE NAME
-          row.column 'disp_cub_in'      , 4,    :type => :integer   #  DISP CUBIC INCHES
-          row.column 'fuel_system'      , 2,    :type => :string   #  FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
-          row.column 'model_trans'      , 6,    :type => :string   #  TRANSMISSION TYPE
-          row.column 'no_cyc'           , 2,    :type => :integer   #  NUMBER OF ENGINE CYLINDERS
-          row.column 'date_time'        , 12,   :type => :string  #   DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
-          row.column 'release_date'     , 6,    :type => :string   #  RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
-          row.column 'vi_mfr_code'      , 3,    :type => :integer   #  VI MANUFACTURER CODE
-          row.column 'carline_code'     , 5,    :type => :integer   #  CARLINE CODE
-          row.column 'basic_eng_id'     , 5,    :type => :integer   #  BASIC ENGINE INDEX
-          row.column 'carline_mfr_name' , 32,   :type => :string  #   CARLINE MANUFACTURER NAME
-          row.column 'suppress_code'    , 1,    :type => :integer    # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
-          row.column 'est_city_mpg'     , 3,    :type => :integer    # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
-          row.spacer 2
-          row.column 'highway_mpg'      , 3,    :type => :integer    # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
-          row.spacer 2
-          row.column 'combined_mpg'     , 3,    :type => :integer    # COMBINED MILES PER GALLON
-          row.spacer 2
-          row.column 'unadj_city_mpg'   , 3,    :type => :integer    # UNADJUSTED  CITY MILES PER GALLON
-          row.spacer 2
-          row.column 'unadj_hwy_mpg'    , 3,    :type => :integer    # UNADJUSTED HIGHWAY MILES PER GALLON
-          row.spacer 2
-          row.column 'unadj_comb_mpg'   , 3,    :type => :integer    # UNADJUSTED COMBINED MILES PER GALLON
-          row.spacer 2
-          row.column 'ave_anl_fuel'     , 6,    :type => :integer    # "$" in col 147, Annual Fuel Cost starting col 148 in I5
-          row.column 'opt_disp'         , 8,    :type => :string    # OPTIONAL DISPLACEMENT
-          row.column 'engine_desc1'     , 10,   :type => :string   #  ENGINE DESCRIPTION 1
-          row.column 'engine_desc2'     , 10,   :type => :string   #  ENGINE DESCRIPTION 2
-          row.column 'engine_desc3'     , 10,   :type => :string   #  ENGINE DESCRIPTION 3
-          row.column 'body_type_2d'     , 10,   :type => :string   #  BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
-          row.column 'body_type_4d'     , 10,   :type => :string   #  BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
-          row.column 'body_type_hbk'    , 10,   :type => :string   #  BODY TYPE HBK    - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
-          row.column 'puerto_rico'      , 1,    :type => :string    # '*' IF FOR PUERTO RICO SALES ONLY
-          row.column 'overdrive'        , 4,    :type => :string    # OVERDRIVE:  ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
-          row.column 'drive_system'     , 3,    :type => :string    # FWD=FRONT WHEEL DRIVE, RWD=REAR,  4WD=4-WHEEL
-          row.column 'filler'           , 1,    :type => :string    # NOT USED
-          row.column 'fuel_type'        , 1,    :type => :string    # R=REGULAR(UNLEADED), P=PREMIUM,  D=DIESEL
-          row.column 'trans_desc'       , 15,   :type => :string   #  TRANSMISSION DESCRIPTORS
-        end
-      end
-      attr_accessor :year
-      def initialize(options = {})
-        options = options.stringify_keys
-        @year = options['year']
-      end
-
-      def apply(row)
-        row.merge!({
-          'make'           => row['carline_mfr_name'], # make it line up with the errata
-          'model'          => row['carline_name'],     # ditto
-          'transmission'   => TRANSMISSIONS[row['model_trans'][0, 1]],
-          'speeds'         => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
-          'turbo'          => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
-          'supercharger'   => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
-          'injection'      => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
-          'displacement'   => _displacement(row['opt_disp']),
-          'year'           => year
-        })
-        row
-      end
-
-      def _displacement(str)
-        str = str.gsub(/[\(\)]/, '').strip
-        if str =~ /^(.+)L$/
-          $1.to_f
-        elsif str =~ /^(.+)CC$/
-          $1.to_f / 1000
-        end
-      end
-
-    end
-    class ParserC
-      attr_accessor :year
-      def initialize(options = {})
-        options = options.stringify_keys
-        @year = options['year']
-      end
-
-      def apply(row)
-        row.merge!({
-          'make'           => row['Manufacturer'], # make it line up with the errata
-          'model'          => row['carline name'], # ditto
-          'drive'          => row['drv'] + 'WD',
-          'transmission'   => TRANSMISSIONS[row['trans'][-3, 1]],
-          'speeds'         => (row['trans'][-2, 1] == 'V') ? 'variable' : row['trans'][-2, 1],
-          'turbo'          => row['T'] == 'T',
-          'supercharger'   => row['S'] == 'S',
-          'injection'      => true,
-          'year'           => year
-        })
-        row
-      end
-    end
-    class ParserD
-      attr_accessor :year
-      def initialize(options = {})
-        options = options.stringify_keys
-        @year = options['year']
-      end
-
-      def apply(row)
-        row.merge!({
-          'make'           => row['MFR'],          # make it line up with the errata
-          'model'          => row['CAR LINE'],     # ditto
-          'drive'          => row['DRIVE SYS'] + 'WD',
-          'transmission'   => TRANSMISSIONS[row['TRANS'][-3, 1]],
-          'speeds'         => (row['TRANS'][-2, 1] == 'V') ? 'variable' : row['TRANS'][-2, 1],
-          'turbo'          => row['TURBO'] == 'T',
-          'supercharger'   => row['SPCHGR'] == 'S',
-          'injection'      => true,
-          'year'           => year
-        })
-        row
-      end
-    end
-  end
-  
-  class Guru
-    # the following matching methods are needed by the errata
-    # per https://brighterplanet.sifterapp.com/projects/30/issues/750/comments
-
-    def transmission_is_blank?(row)
-      row['transmission'].blank?
-    end
-
-    def is_a_2007_gmc_or_chevrolet?(row)
-      row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
-    end
-
-    def is_a_porsche?(row)
-      row['make'].upcase == 'PORSCHE'
-    end
-
-    def is_not_a_porsche?(row)
-      !is_a_porsche? row
-    end
-
-    def is_a_mercedes_benz?(row)
-      row['make'] =~ /MERCEDES/i
-    end
-
-    def is_a_lexus?(row)
-      row['make'].upcase == 'LEXUS'
-    end
-
-    def is_a_bmw?(row)
-      row['make'].upcase == 'BMW'
-    end
-
-    def is_a_ford?(row)
-      row['make'].upcase == 'FORD'
-    end
-
-    def is_a_rolls_royce_and_model_contains_bentley?(row)
-      is_a_rolls_royce?(row) and model_contains_bentley?(row)
-    end
-
-    def is_a_bentley?(row)
-      row['make'].upcase == 'BENTLEY'
-    end
-
-    def is_a_rolls_royce?(row)
-      row['make'] =~ /ROLLS/i
-    end
-
-    def is_a_turbo_brooklands?(row)
-      row['model'] =~ /TURBO R\/RL BKLDS/i
-    end
-
-    def model_contains_maybach?(row)
-      row['model'] =~ /MAYBACH/i
-    end
-    
-    def model_contains_bentley?(row)
-      row['model'] =~ /BENTLEY/i
-    end
-  end
-  
-  errata = { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv', :responder => 'AutomobileVariant::Guru' }
-  
-  data_miner do
-    # 1985---1997
-    (85..97).each do |yy|
-      filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
-      import(:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
-             :filename => filename,
-             :transform => { :class => FuelEconomyGuide::ParserB, :year => "19#{yy}".to_i },
-             :format => :fixed_width,
-             :cut => (yy == 95) ? '13-' : nil,
-             :schema_name => :fuel_economy_guide_b,
-             :select => lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' },
-             :errata => errata) do
-        key 'row_hash'
-        store 'make_name', :field_name => 'make'
-        store 'model_name', :field_name => 'model'
-        store 'year'
-        store 'fuel_type_code', :field_name => 'fuel_type'
-        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
-        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre    # ditto
-        store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'cylinders', :field_name => 'no_cyc'
-        store 'drive', :field_name => 'drive_system'
-        store 'carline_mfr_code'
-        store 'vi_mfr_code'
-        store 'carline_code'
-        store 'carline_class_code', :field_name => 'carline_clss'
-        store 'transmission'
-        store 'speeds'
-        store 'turbo'
-        store 'supercharger'
-        store 'injection'
-        store 'displacement'
-      end
-    end
-    
-    # 1998--2005
-    {
-      1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
-      1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
-      2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
-      2001 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' }, # parseexcel 0.5.2 can't read Excel 5.0 { :url => 'http://www.fueleconomy.gov/FEG/epadata/01data.zip', :filename => '01guide0918.xls' }
-      2002 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls' },
-      2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
-      2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
-      2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
-    }.sort { |a, b| a.first <=> b.first }.each do |year, options|
-      import options.merge(:transform => { :class => FuelEconomyGuide::ParserC, :year => year },
-                           :errata => errata) do
-        key 'row_hash'
-        store 'make_name', :field_name => 'make'
-        store 'model_name', :field_name => 'model'
-        store 'fuel_type_code', :field_name => 'fl'
-        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
-        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre    # ditto
-        store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'cylinders', :field_name => 'cyl'
-        store 'displacement', :field_name => 'displ'
-        store 'carline_class_code', :field_name => 'cls' if year >= 2000
-        store 'carline_class_name', :field_name => 'Class'
-        store 'year'
-        store 'transmission'
-        store 'speeds'
-        store 'turbo'
-        store 'supercharger'
-        store 'injection'
-        store 'drive'
-      end
-    end
-    
-    # 2006--2010
-    {
-      2006 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/06data.zip', :filename => '2006_FE_Guide_14-Nov-2005_download.csv' },
-      2007 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/07data.zip', :filename => '2007_FE_guide_ALL_no_sales_May_01_2007.xls' },
-      2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
-      2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
-      # 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
-    }.sort { |a, b| a.first <=> b.first }.each do |year, options|
-      import options.merge(:transform => { :class => FuelEconomyGuide::ParserD, :year => year },
-                           :reject => (year == 2007) ? lambda { |row| row.values.first.blank? } : nil,
-                           :errata => errata) do
-        key 'row_hash'
-        store 'make_name', :field_name => 'make'
-        store 'model_name', :field_name => 'model'
-        store 'fuel_type_code', :field_name => 'FUEL TYPE'
-        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
-        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre    # ditto
-        store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'cylinders', :field_name => 'NUMB CYL'
-        store 'displacement', :field_name => 'DISPLACEMENT'
-        store 'carline_class_code', :field_name => 'CLS'
-        store 'carline_class_name', :field_name => 'CLASS'
-        store 'year'
-        store 'transmission'
-        store 'speeds'
-        store 'turbo'
-        store 'supercharger'
-        store 'injection'
-        store 'drive'
-      end
-    end
-    
-    # associate :make, :key => :original_automobile_make_name, :foreign_key => :name
-    # derive :automobile_model_id # creates models by name
-    # associate :fuel_type, :key => :original_automobile_fuel_type_code, :foreign_key => :code
-    
-    process 'Set adjusted fuel economy' do
-      update_all 'fuel_efficiency_city = 1 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))'
-      update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
-    end
-  end
-  
-  def name
-    extra = []
-    extra << "V#{cylinders}" if cylinders
-    extra << "#{displacement}L" if displacement
-    extra << "turbo" if turbo
-    extra << "FI" if injection
-    extra << "#{speeds}spd" if speeds.present?
-    extra << transmission if transmission.present?
-    extra << "(#{fuel_type.name})" if fuel_type
-    extra.join(' ')
-  end
-  
-  def fuel_economy_description
-    [ fuel_efficiency_city, fuel_efficiency_highway ].map { |f| f.kilometres_per_litre.to(:miles_per_gallon).round }.join('/')
-  end
-end

data/test/support/country.rb

@@ -1,15 +0,0 @@
-class Country < ActiveRecord::Base
-  set_primary_key :iso_3166
-  
-  data_miner do
-    import 'The official ISO country list', :url => 'http://www.iso.org/iso/list-en1-semic-3.txt', :encoding => 'ISO-8859-1', :skip => 2, :headers => false, :delimiter => ';' do
-      key 'iso_3166', :field_number => 1
-      store 'name', :field_number => 0
-    end
-    
-    import 'A Princeton dataset with better capitalization', :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do
-      key 'iso_3166', :field_name => 'country code'
-      store 'name', :field_name => 'country'
-    end
-  end
-end

data/test/support/test_database.rb

@@ -1,311 +0,0 @@
-module TestDatabase
-  extend self
-
-  def connect
-    @connection ||= ActiveRecord::Base.establish_connection(
-      'adapter' => 'mysql',
-      'database' => 'data_miner_test',
-      'username' => 'root',
-      'password' => 'password'
-    )
-  end
-
-  def load_schema
-    connect
-
-    ActiveRecord::Schema.define(:version => 20090819143429) do
-      create_table "t100_flight_segments", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.integer  "departures_performed"
-        t.integer  "payload"
-        t.integer  "seats"
-        t.integer  "passengers"
-        t.integer  "freight"
-        t.integer  "mail"
-        t.integer  "ramp_to_ramp"
-        t.integer  "air_time"
-        t.float    "load_factor"
-        t.float    "freight_share"
-        t.integer  "distance"
-        t.integer  "departures_scheduled"
-        t.string   "unique_carrier"
-        t.integer  "dot_airline_id"
-        t.string   "unique_carrier_name"
-        t.string   "unique_carrier_entity"
-        t.string   "region"
-        t.string   "carrier"
-        t.string   "carrier_name"
-        t.integer  "carrier_group"
-        t.integer  "carrier_group_new"
-        t.string   "origin_airport_iata"
-        t.string   "origin_city_name"
-        t.integer  "origin_city_num"
-        t.string   "origin_state_abr"
-        t.string   "origin_state_fips"
-        t.string   "origin_state_nm"
-        t.string   "origin_country_iso_3166"
-        t.string   "origin_country_name"
-        t.integer  "origin_wac"
-        t.string   "dest_airport_iata"
-        t.string   "dest_city_name"
-        t.integer  "dest_city_num"
-        t.string   "dest_state_abr"
-        t.string   "dest_state_fips"
-        t.string   "dest_state_nm"
-        t.string   "dest_country_iso_3166"
-        t.string   "dest_country_name"
-        t.integer  "dest_wac"
-        t.integer  "bts_aircraft_group"
-        t.integer  "bts_aircraft_type"
-        t.integer  "bts_aircraft_config"
-        t.integer  "year"
-        t.integer  "quarter"
-        t.integer  "month"
-        t.integer  "bts_distance_group"
-        t.string   "bts_service_class"
-        t.string   "data_source"
-        t.float    "seats_per_departure"
-        
-        t.string 'payload_units'
-        t.string 'freight_units'
-        t.string 'mail_units'
-        t.string 'distance_units'
-        
-        t.datetime "created_at"
-        t.datetime "updated_at"
-        
-        t.string   "row_hash"
-      end
-      execute 'ALTER TABLE t100_flight_segments ADD PRIMARY KEY (row_hash);'
-
-      create_table 'tapped_airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string 'i_am_just_here_to_get_in_the_way'
-      end
-      
-      create_table 'airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string   'iata_code'
-        t.string   'name'
-        t.string   'city'
-        t.string   'country_name'
-        t.float    'latitude'
-        t.float    'longitude'
-        t.datetime 'created_at'
-        t.datetime 'updated_at'
-      end
-      execute 'ALTER TABLE airports ADD PRIMARY KEY (iata_code);'
-      
-      create_table "countries", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string   "iso_3166"
-        t.string   "name"
-        t.datetime "created_at"
-        t.datetime "updated_at"
-      end
-      execute "ALTER TABLE countries ADD PRIMARY KEY (iso_3166);"
-      
-      create_table "census_regions", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.integer  "number"
-        t.string   "name"
-        t.datetime "updated_at"
-        t.datetime "created_at"
-      end
-      execute "ALTER TABLE census_regions ADD PRIMARY KEY (number);"
-      
-      create_table 'census_divisions', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.integer  'number'
-        t.string   'name'
-        t.datetime 'updated_at'
-        t.datetime 'created_at'
-        t.string   'census_region_name'
-        t.integer  'census_region_number'
-        
-      end
-      execute 'ALTER TABLE census_divisions ADD PRIMARY KEY (number);'
-      
-      create_table 'census_division_deux', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.integer  'number'
-        t.string   'name'
-        t.datetime 'updated_at'
-        t.datetime 'created_at'
-        t.string   'census_region_name'
-        t.integer  'census_region_number'
-        
-      end
-      execute 'ALTER TABLE census_division_deux ADD PRIMARY KEY (number);'
-      
-      create_table 'crosscalling_census_divisions', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.integer  'number'
-        t.string   'name'
-        t.datetime 'updated_at'
-        t.datetime 'created_at'
-        t.string   'census_region_name'
-        t.integer  'census_region_number'
-        
-      end
-      execute 'ALTER TABLE crosscalling_census_divisions ADD PRIMARY KEY (number);'
-      
-      create_table "automobile_variants", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.float    "fuel_efficiency_city"
-        t.float    "fuel_efficiency_highway"
-        t.string   "make_name"
-        t.string   "model_name"
-        t.string   "year"
-        t.string   "fuel_type_code"
-        t.datetime "updated_at"
-        t.datetime "created_at"
-        t.string   "transmission"
-        t.string   "drive"
-        t.boolean  "turbo"
-        t.boolean  "supercharger"
-        t.integer  "cylinders"
-        t.float    "displacement"
-        t.float    "raw_fuel_efficiency_city"
-        t.float    "raw_fuel_efficiency_highway"
-        t.integer  "carline_mfr_code"
-        t.integer  "vi_mfr_code"
-        t.integer  "carline_code"
-        t.integer  "carline_class_code"
-        t.boolean  "injection"
-        t.string   "carline_class_name"
-        t.string   "speeds"
-        
-        t.string 'raw_fuel_efficiency_highway_units'
-        t.string 'raw_fuel_efficiency_city_units'
-        t.string 'fuel_efficiency_highway_units'
-        t.string 'fuel_efficiency_city_units'
-        
-        t.string   "row_hash"
-      end
-      execute "ALTER TABLE automobile_variants ADD PRIMARY KEY (row_hash);"
-      
-      create_table "automobile_fuel_types", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string   "name"
-        t.datetime "created_at"
-        t.datetime "updated_at"
-        t.float    "emission_factor"
-        t.string   "emission_factor_units"
-        t.float    "annual_distance"
-        t.string   "annual_distance_units"
-        t.string   "code"
-      end
-      execute "ALTER TABLE automobile_fuel_types ADD PRIMARY KEY (code);"
-
-      create_table "residential_energy_consumption_survey_responses", :options => 'ENGINE=InnoDB default charset=utf8', :id => false, :force => true do |t|
-        t.integer  "department_of_energy_identifier"
-
-        t.string   "residence_class"
-        t.date     "construction_year"
-        t.string   "construction_period"
-        t.string   "urbanity"
-        t.string   "dishwasher_use"
-        t.string   "central_ac_use"
-        t.string   "window_ac_use"
-        t.string   "clothes_washer_use"
-        t.string   "clothes_dryer_use"
-
-        t.integer "census_division_number"
-        t.string "census_division_name"
-        t.integer "census_region_number"
-        t.string "census_region_name"
-        
-        t.float    "rooms"
-        t.float    "floorspace"
-        t.integer  "residents"
-        t.boolean  "ownership"
-        t.boolean  "thermostat_programmability"
-        t.integer  "refrigerator_count"
-        t.integer  "freezer_count"    
-        t.float    "annual_energy_from_fuel_oil_for_heating_space"
-        t.float    "annual_energy_from_fuel_oil_for_heating_water"
-        t.float    "annual_energy_from_fuel_oil_for_appliances"
-        t.float    "annual_energy_from_natural_gas_for_heating_space"
-        t.float    "annual_energy_from_natural_gas_for_heating_water"
-        t.float    "annual_energy_from_natural_gas_for_appliances"
-        t.float    "annual_energy_from_propane_for_heating_space"
-        t.float    "annual_energy_from_propane_for_heating_water"
-        t.float    "annual_energy_from_propane_for_appliances"
-        t.float    "annual_energy_from_wood"
-        t.float    "annual_energy_from_kerosene"
-        t.float    "annual_energy_from_electricity_for_clothes_driers"
-        t.float    "annual_energy_from_electricity_for_dishwashers"
-        t.float    "annual_energy_from_electricity_for_freezers"
-        t.float    "annual_energy_from_electricity_for_refrigerators"
-        t.float    "annual_energy_from_electricity_for_air_conditioners"
-        t.float    "annual_energy_from_electricity_for_heating_space"
-        t.float    "annual_energy_from_electricity_for_heating_water"
-        t.float    "annual_energy_from_electricity_for_other_appliances"
-        t.float    "weighting"
-        t.float    "lighting_use"
-        t.float    "lighting_efficiency"
-        t.integer  "heating_degree_days"
-        t.integer  "cooling_degree_days"
-        t.integer  "total_rooms"
-        t.integer  "bathrooms"
-        t.integer  "halfbaths"
-        t.integer  "heated_garage"
-        t.integer  "attached_1car_garage"
-        t.integer  "detached_1car_garage"
-        t.integer  "attached_2car_garage"
-        t.integer  "detached_2car_garage"
-        t.integer  "attached_3car_garage"
-        t.integer  "detached_3car_garage"
-        t.integer  "lights_on_1_to_4_hours"
-        t.integer  "efficient_lights_on_1_to_4_hours"
-        t.integer  "lights_on_4_to_12_hours"
-        t.integer  "efficient_lights_on_4_to_12_hours"
-        t.integer  "lights_on_over_12_hours"
-        t.integer  "efficient_lights_on_over_12_hours"
-        t.integer  "outdoor_all_night_lights"
-        t.integer  "outdoor_all_night_gas_lights"
-        
-        t.datetime "created_at"
-        t.datetime "updated_at"
-      end
-      execute "ALTER TABLE residential_energy_consumption_survey_responses ADD PRIMARY KEY (department_of_energy_identifier);"
-      
-      create_table 'aircraft', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string 'icao_code'
-        t.string   'manufacturer_name'
-        t.string   'name'
-        
-        t.string   "bts_name"
-        t.string "bts_aircraft_type_code"
-
-        t.string   'brighter_planet_aircraft_class_code'
-        # t.float    'm3'
-        # t.float    'm2'
-        # t.float    'm1'
-        # t.float    'endpoint_fuel'
-        t.datetime 'updated_at'
-        t.datetime 'created_at'
-      end
-      execute 'ALTER TABLE aircraft ADD PRIMARY KEY (icao_code);'
-      
-      create_table 'aircraft_deux', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string 'icao_code'
-        t.string   'manufacturer_name'
-        t.string   'name'
-        
-        t.string   "bts_name"
-        t.string "bts_aircraft_type_code"
-
-        # t.string   'brighter_planet_aircraft_class_code'
-        # t.float    'm3'
-        # t.float    'm2'
-        # t.float    'm1'
-        # t.float    'endpoint_fuel'
-        t.datetime 'updated_at'
-        t.datetime 'created_at'
-      end
-      execute 'ALTER TABLE aircraft_deux ADD PRIMARY KEY (icao_code);'
-    end
-
-    DataMiner::Run.create_tables
-  end
-
-  def load_models
-    load_schema
-
-    Dir.glob(File.expand_path('*.rb', File.dirname(__FILE__))).each do |lib|
-      require lib
-    end
-  end
-end