data_miner 0.4.4 → 0.4.5

data/Rakefile

@@ -10,7 +10,7 @@ begin
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/data_miner"
     gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    gem.add_dependency 'remote_table', '~>0.2.5'
+    gem.add_dependency 'remote_table', '~>0.2.6'
     gem.add_dependency 'activerecord', '~>2.3.4'
     gem.add_dependency 'activesupport', '~>2.3.4'
     gem.add_dependency 'andand', '~>1.3.1'

data/VERSION

@@ -1 +1 @@
-0.4.4
+0.4.5

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.4"
+  s.version = "0.4.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-03-24}
+  s.date = %q{2010-03-26}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -51,7 +51,7 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<remote_table>, ["~> 0.2.5"])
+      s.add_runtime_dependency(%q<remote_table>, ["~> 0.2.6"])
       s.add_runtime_dependency(%q<activerecord>, ["~> 2.3.4"])
       s.add_runtime_dependency(%q<activesupport>, ["~> 2.3.4"])
       s.add_runtime_dependency(%q<andand>, ["~> 1.3.1"])
@@ -60,7 +60,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<blockenspiel>, ["~> 0.3.2"])
       s.add_runtime_dependency(%q<log4r>, ["~> 1.1.7"])
     else
-      s.add_dependency(%q<remote_table>, ["~> 0.2.5"])
+      s.add_dependency(%q<remote_table>, ["~> 0.2.6"])
       s.add_dependency(%q<activerecord>, ["~> 2.3.4"])
       s.add_dependency(%q<activesupport>, ["~> 2.3.4"])
       s.add_dependency(%q<andand>, ["~> 1.3.1"])
@@ -70,7 +70,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<log4r>, ["~> 1.1.7"])
     end
   else
-    s.add_dependency(%q<remote_table>, ["~> 0.2.5"])
+    s.add_dependency(%q<remote_table>, ["~> 0.2.6"])
     s.add_dependency(%q<activerecord>, ["~> 2.3.4"])
     s.add_dependency(%q<activesupport>, ["~> 2.3.4"])
     s.add_dependency(%q<andand>, ["~> 1.3.1"])

data/lib/data_miner/attribute.rb

@@ -83,7 +83,7 @@ module DataMiner
     end
     
     def do_convert(row, value)
-      logger.error "[data_miner gem] If you use :from_units, you need to set :to_units (#{resource.name}##{name})" unless wants_units?
+      DataMiner.log_or_raise "If you use :from_units, you need to set :to_units (#{resource.name}##{name})" unless wants_units?
       value.to_f.convert((from_units || unit_from_source(row)), to_units)
     end
     

data/lib/data_miner/configuration.rb

@@ -72,11 +72,11 @@ module DataMiner
       import_runnables.each do |runnable|
         runnable.attributes.each do |_, attribute|
           if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
-            DataMiner.logger.error %{
+            DataMiner.log_or_raise %{
 
 ================================
 
-[data_miner gem] You don't have a valid unit definition for #{resource.name}##{attribute.name}.
+You don't have a valid unit definition for #{resource.name}##{attribute.name}.
 
 You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.map(&:to_sym).inspect }.
 
@@ -93,7 +93,7 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
       missing_columns = Array.new
       import_runnables.each do |runnable|
         runnable.attributes.each do |_, attribute|
-          DataMiner.logger.error "[data_miner gem] You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
+          DataMiner.log_or_raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
           unless resource.column_names.include? attribute.name
             missing_columns << attribute.name
           end
@@ -104,11 +104,11 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
       end
       missing_columns.uniq!
       if missing_columns.any?
-        DataMiner.logger.error %{
+        DataMiner.log_or_raise %{
 
 ================================
 
-[data_miner gem] On #{resource}, it looks like you're missing some columns...
+On #{resource}, it looks like you're missing some columns...
 
 Please run this...
 

data/lib/data_miner/import.rb

@@ -16,7 +16,7 @@ module DataMiner
       @position_in_run = position_in_run
       @description = description
       @errata = Errata.new(:url => options[:errata], :klass => resource) if options[:errata]
-      @table = RemoteTable.new(options.slice(:url, :filename, :post_data, :format, :skip, :cut, :schema, :schema_name, :trap, :select, :reject, :sheet, :delimiter, :headers, :transform, :crop, :encoding))
+      @table = RemoteTable.new(options.slice(:url, :filename, :form_data, :format, :skip, :cut, :schema, :schema_name, :trap, :select, :reject, :sheet, :delimiter, :headers, :transform, :crop, :encoding, :compression, :glob))
     end
 
     def inspect
@@ -28,12 +28,12 @@ module DataMiner
     end
     
     def store(attr_name, attr_options = {})
-      DataMiner.logger.error "[data_miner gem] You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
+      DataMiner.log_or_raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
       attributes[attr_name] = Attribute.new self, attr_name, attr_options
     end
     
     def key(attr_name, attr_options = {})
-      DataMiner.logger.error "[data_miner gem] You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
+      DataMiner.log_or_raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
       @key = attr_name
       store attr_name, attr_options
     end

data/lib/data_miner.rb

@@ -32,6 +32,17 @@ module DataMiner
       
       self.logger = Logger.new 'data_miner'
       logger.add info_outputter, error_outputter
+      ActiveRecord::Base.logger = logger
+      ActiveRecord::Base.colorize_logging = false
+    end
+  end
+  
+  def self.log_or_raise(message)
+    message = "[data_miner gem] #{message}"
+    if ENV['RAILS_ENV'] == 'production'
+      logger.error message
+    else
+      raise message
     end
   end
   
@@ -51,7 +62,7 @@ end
 ActiveRecord::Base.class_eval do
   def self.data_miner(&block)
     unless table_exists?
-      logger.error "[DataMiner gem] Database table `#{table_name}` doesn't exist. DataMiner probably won't work properly until you run a migration or otherwise fix the schema."
+      DataMiner.log_or_raise "Database table `#{table_name}` doesn't exist. DataMiner probably won't work properly until you run a migration or otherwise fix the schema."
       return
     end
     

data/test/data_miner_test.rb

@@ -481,7 +481,7 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
     
     # conversions are NOT performed here, since we first have to zero out legitimate skips
     # otherwise you will get values like "999 pounds = 453.138778 kilograms" (where 999 is really a legit skip)
-    import 'RECs 2005 (but not converting units to metric just yet)', :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv', :headers => :upcase do
+    import 'RECs 2005 (but not converting units to metric just yet)', :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv' do
       key 'department_of_energy_identifier', :field_name => 'DOEID'
       
       store 'residence_class', :field_name => 'TYPEHUQ', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/typehuq/typehuq.csv' }
@@ -636,6 +636,252 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
   end
 end
 
+# T-100 Segment (All Carriers):  http://www.transtats.bts.gov/Fields.asp?Table_ID=293
+class T100FlightSegment < ActiveRecord::Base
+  set_primary_key :row_hash
+  URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
+  FORM_DATA = %{
+    UserTableName=T_100_Segment__All_Carriers&
+    DBShortName=Air_Carriers&
+    RawDataTable=T_T100_SEGMENT_ALL_CARRIER&
+    sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D__MONTH_NUMBER__+AND+YEAR%3D__YEAR__&
+    varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&
+    grouplist=&
+    suml=&
+    sumRegion=&
+    filter1=title%3D&
+    filter2=title%3D&
+    geo=All%A0&
+    time=__MONTH_NAME__&
+    timename=Month&
+    GEOGRAPHY=All&
+    XYEAR=__YEAR__&
+    FREQUENCY=__MONTH_NUMBER__&
+    AllVars=All&
+    VarName=DEPARTURES_SCHEDULED&
+    VarDesc=DepScheduled&
+    VarType=Num&
+    VarName=DEPARTURES_PERFORMED&
+    VarDesc=DepPerformed&
+    VarType=Num&
+    VarName=PAYLOAD&
+    VarDesc=Payload&
+    VarType=Num&
+    VarName=SEATS&
+    VarDesc=Seats&
+    VarType=Num&
+    VarName=PASSENGERS&
+    VarDesc=Passengers&
+    VarType=Num&
+    VarName=FREIGHT&
+    VarDesc=Freight&
+    VarType=Num&
+    VarName=MAIL&
+    VarDesc=Mail&
+    VarType=Num&
+    VarName=DISTANCE&
+    VarDesc=Distance&
+    VarType=Num&
+    VarName=RAMP_TO_RAMP&
+    VarDesc=RampToRamp&
+    VarType=Num&
+    VarName=AIR_TIME&
+    VarDesc=AirTime&
+    VarType=Num&
+    VarName=UNIQUE_CARRIER&
+    VarDesc=UniqueCarrier&
+    VarType=Char&
+    VarName=AIRLINE_ID&
+    VarDesc=AirlineID&
+    VarType=Num&
+    VarName=UNIQUE_CARRIER_NAME&
+    VarDesc=UniqueCarrierName&
+    VarType=Char&
+    VarName=UNIQUE_CARRIER_ENTITY&
+    VarDesc=UniqCarrierEntity&
+    VarType=Char&
+    VarName=REGION&
+    VarDesc=CarrierRegion&
+    VarType=Char&
+    VarName=CARRIER&
+    VarDesc=Carrier&
+    VarType=Char&
+    VarName=CARRIER_NAME&
+    VarDesc=CarrierName&
+    VarType=Char&
+    VarName=CARRIER_GROUP&
+    VarDesc=CarrierGroup&
+    VarType=Num&
+    VarName=CARRIER_GROUP_NEW&
+    VarDesc=CarrierGroupNew&
+    VarType=Num&
+    VarName=ORIGIN&
+    VarDesc=Origin&
+    VarType=Char&
+    VarName=ORIGIN_CITY_NAME&
+    VarDesc=OriginCityName&
+    VarType=Char&
+    VarName=ORIGIN_CITY_NUM&
+    VarDesc=OriginCityNum&
+    VarType=Num&
+    VarName=ORIGIN_STATE_ABR&
+    VarDesc=OriginState&
+    VarType=Char&
+    VarName=ORIGIN_STATE_FIPS&
+    VarDesc=OriginStateFips&
+    VarType=Char&
+    VarName=ORIGIN_STATE_NM&
+    VarDesc=OriginStateName&
+    VarType=Char&
+    VarName=ORIGIN_COUNTRY&
+    VarDesc=OriginCountry&
+    VarType=Char&
+    VarName=ORIGIN_COUNTRY_NAME&
+    VarDesc=OriginCountryName&
+    VarType=Char&
+    VarName=ORIGIN_WAC&
+    VarDesc=OriginWac&
+    VarType=Num&
+    VarName=DEST&
+    VarDesc=Dest&
+    VarType=Char&
+    VarName=DEST_CITY_NAME&
+    VarDesc=DestCityName&
+    VarType=Char&
+    VarName=DEST_CITY_NUM&
+    VarDesc=DestCityNum&
+    VarType=Num&
+    VarName=DEST_STATE_ABR&
+    VarDesc=DestState&
+    VarType=Char&
+    VarName=DEST_STATE_FIPS&
+    VarDesc=DestStateFips&
+    VarType=Char&
+    VarName=DEST_STATE_NM&
+    VarDesc=DestStateName&
+    VarType=Char&
+    VarName=DEST_COUNTRY&
+    VarDesc=DestCountry&
+    VarType=Char&
+    VarName=DEST_COUNTRY_NAME&
+    VarDesc=DestCountryName&
+    VarType=Char&
+    VarName=DEST_WAC&
+    VarDesc=DestWac&
+    VarType=Num&
+    VarName=AIRCRAFT_GROUP&
+    VarDesc=AircraftGroup&
+    VarType=Num&
+    VarName=AIRCRAFT_TYPE&
+    VarDesc=AircraftType&
+    VarType=Char&
+    VarName=AIRCRAFT_CONFIG&
+    VarDesc=AircraftConfig&
+    VarType=Num&
+    VarName=YEAR&
+    VarDesc=Year&
+    VarType=Num&
+    VarName=QUARTER&
+    VarDesc=Quarter&
+    VarType=Num&
+    VarName=MONTH&
+    VarDesc=Month&
+    VarType=Num&
+    VarName=DISTANCE_GROUP&
+    VarDesc=DistanceGroup&
+    VarType=Num&
+    VarName=CLASS&
+    VarDesc=Class&
+    VarType=Char&
+    VarName=DATA_SOURCE&
+    VarDesc=DataSource&
+    VarType=Char
+  }.gsub /[\s]+/,''
+
+  data_miner do
+    months = Hash.new
+    # (2008..2009).each do |year|
+    (2008..2008).each do |year|
+      # (1..12).each do |month|
+      (1..1).each do |month|
+        time = Time.gm year, month
+        form_data = FORM_DATA.dup
+        form_data.gsub! '__YEAR__', time.year.to_s
+        form_data.gsub! '__MONTH_NUMBER__', time.month.to_s
+        form_data.gsub! '__MONTH_NAME__', time.strftime('%B')
+        months[time] = form_data
+      end
+    end
+    months.each do |month, form_data|
+      import "T100 data from #{month.strftime('%B %Y')}",
+           :url => URL,
+           :form_data => form_data,
+           :compression => :zip,
+           :glob => '/*.csv' do
+        key 'row_hash'
+        store 'departures_scheduled', :field_name => 'DEPARTURES_SCHEDULED'
+        store 'departures_performed', :field_name => 'DEPARTURES_PERFORMED'
+        store 'payload', :field_name => 'PAYLOAD', :from_units => :pounds, :to_units => :kilograms
+        store 'seats', :field_name => 'SEATS'
+        store 'passengers', :field_name => 'PASSENGERS'
+        store 'freight', :field_name => 'FREIGHT', :from_units => :pounds, :to_units => :kilograms
+        store 'mail', :field_name => 'MAIL', :from_units => :pounds, :to_units => :kilograms
+        store 'distance', :field_name => 'DISTANCE', :from_units => :miles, :to_units => :kilometres
+        store 'ramp_to_ramp', :field_name => 'RAMP_TO_RAMP'
+        store 'air_time', :field_name => 'AIR_TIME'
+        store 'unique_carrier', :field_name => 'UNIQUE_CARRIER'
+        store 'dot_airline_id', :field_name => 'AIRLINE_ID'
+        store 'unique_carrier_name', :field_name => 'UNIQUE_CARRIER_NAME'
+        store 'unique_carrier_entity', :field_name => 'UNIQUE_CARRIER_ENTITY'
+        store 'region', :field_name => 'REGION'
+        store 'carrier', :field_name => 'CARRIER'
+        store 'carrier_name', :field_name => 'CARRIER_NAME'
+        store 'carrier_group', :field_name => 'CARRIER_GROUP'
+        store 'carrier_group_new', :field_name => 'CARRIER_GROUP_NEW'
+        store 'origin_airport_iata', :field_name => 'ORIGIN'
+        store 'origin_city_name', :field_name => 'ORIGIN_CITY_NAME'
+        store 'origin_city_num', :field_name => 'ORIGIN_CITY_NUM'
+        store 'origin_state_abr', :field_name => 'ORIGIN_STATE_ABR'
+        store 'origin_state_fips', :field_name => 'ORIGIN_STATE_FIPS'
+        store 'origin_state_nm', :field_name => 'ORIGIN_STATE_NM'
+        store 'origin_country_iso_3166', :field_name => 'ORIGIN_COUNTRY'
+        store 'origin_country_name', :field_name => 'ORIGIN_COUNTRY_NAME'
+        store 'origin_wac', :field_name => 'ORIGIN_WAC'
+        store 'dest_airport_iata', :field_name => 'DEST'
+        store 'dest_city_name', :field_name => 'DEST_CITY_NAME'
+        store 'dest_city_num', :field_name => 'DEST_CITY_NUM'
+        store 'dest_state_abr', :field_name => 'DEST_STATE_ABR'
+        store 'dest_state_fips', :field_name => 'DEST_STATE_FIPS'
+        store 'dest_state_nm', :field_name => 'DEST_STATE_NM'
+        store 'dest_country_iso_3166', :field_name => 'DEST_COUNTRY'
+        store 'dest_country_name', :field_name => 'DEST_COUNTRY_NAME'
+        store 'dest_wac', :field_name => 'DEST_WAC'
+        store 'bts_aircraft_group', :field_name => 'AIRCRAFT_GROUP'
+        store 'bts_aircraft_type', :field_name => 'AIRCRAFT_TYPE'
+        store 'bts_aircraft_config', :field_name => 'AIRCRAFT_CONFIG'
+        store 'year', :field_name => 'YEAR'
+        store 'quarter', :field_name => 'QUARTER'
+        store 'month', :field_name => 'MONTH'
+        store 'bts_distance_group', :field_name => 'DISTANCE_GROUP'
+        store 'bts_service_class', :field_name => 'CLASS'
+        store 'data_source', :field_name => 'DATA_SOURCE'
+      end
+    end
+    
+    process 'Derive freight share as a fraction of payload' do
+      update_all 'freight_share = (freight + mail) / payload', 'payload > 0'
+    end
+
+    process 'Derive load factor, which is passengers divided by the total seats available' do
+      update_all 'load_factor = passengers / seats', 'passengers <= seats'
+    end
+    
+    process 'Derive average seats per departure' do
+      update_all 'seats_per_departure = seats / departures_performed', 'departures_performed > 0'
+    end
+  end
+end
+
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
   if ENV['FAST'] == 'true'
@@ -733,9 +979,14 @@ class DataMinerTest < Test::Unit::TestCase
       assert AutomobileVariant.count('make_name LIKE "%tesla"') > 0
     end
     
-    # should "mine residence survey day" do
-    #   ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
-    #   assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
-    # end
+    should "mine T100 flight segments" do
+      T100FlightSegment.run_data_miner!
+      assert T100FlightSegment.count('dest_country_name LIKE "%United States"') > 0
+    end
+    
+    should "mine residence survey day" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
+    end
   end
 end

data/test/test_helper.rb

@@ -18,6 +18,71 @@ class Test::Unit::TestCase
 end
 
 ActiveRecord::Schema.define(:version => 20090819143429) do
+  create_table "t100_flight_segments", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.integer  "departures_performed"
+    t.integer  "payload"
+    t.integer  "seats"
+    t.integer  "passengers"
+    t.integer  "freight"
+    t.integer  "mail"
+    t.integer  "ramp_to_ramp"
+    t.integer  "air_time"
+    t.float    "load_factor"
+    t.float    "freight_share"
+    t.integer  "distance"
+    t.integer  "departures_scheduled"
+    t.string   "unique_carrier"
+    t.integer  "dot_airline_id"
+    t.string   "unique_carrier_name"
+    t.string   "unique_carrier_entity"
+    t.string   "region"
+    t.string   "carrier"
+    t.string   "carrier_name"
+    t.integer  "carrier_group"
+    t.integer  "carrier_group_new"
+    t.string   "origin_airport_iata"
+    t.string   "origin_city_name"
+    t.integer  "origin_city_num"
+    t.string   "origin_state_abr"
+    t.string   "origin_state_fips"
+    t.string   "origin_state_nm"
+    t.string   "origin_country_iso_3166"
+    t.string   "origin_country_name"
+    t.integer  "origin_wac"
+    t.string   "dest_airport_iata"
+    t.string   "dest_city_name"
+    t.integer  "dest_city_num"
+    t.string   "dest_state_abr"
+    t.string   "dest_state_fips"
+    t.string   "dest_state_nm"
+    t.string   "dest_country_iso_3166"
+    t.string   "dest_country_name"
+    t.integer  "dest_wac"
+    t.integer  "bts_aircraft_group"
+    t.integer  "bts_aircraft_type"
+    t.integer  "bts_aircraft_config"
+    t.integer  "year"
+    t.integer  "quarter"
+    t.integer  "month"
+    t.integer  "bts_distance_group"
+    t.string   "bts_service_class"
+    t.string   "data_source"
+    t.float    "seats_per_departure"
+    
+    t.string 'payload_units'
+    t.string 'freight_units'
+    t.string 'mail_units'
+    t.string 'distance_units'
+    
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    
+    t.string   "row_hash"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
+  end
+  execute 'ALTER TABLE t100_flight_segments ADD PRIMARY KEY (row_hash);'
+  
   create_table 'airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
     t.string   'iata_code'
     t.string   'name'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-24 00:00:00 -04:00
+date: 2010-03-26 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,7 +21,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.2.5
+        version: 0.2.6
     version: 
 - !ruby/object:Gem::Dependency 
   name: activerecord