data_miner 1.0.1 → 1.1.0

data/CHANGELOG

@@ -1,3 +1,9 @@
+1.1.0
+* fixed dependency issues
+1.0.0
+* bundler and gemspec instead of jeweler
+* clear up memory leaks and destructive argument borking
+* mostly backwards compatible (but no add_hints! in remote_table transforms, for example)
 0.2.6
 * Upgrade to remote_table 0.1.6 to handle UTF-8 CSVs and long urls.
 0.3.0

data/data_miner.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
   
-  s.add_dependency 'remote_table', '>=1.0.3'
+  s.add_dependency 'remote_table', '>=1.1.0'
   s.add_dependency 'escape', '>=0.0.4'
   s.add_dependency 'activerecord', '>=2.3.4'
   s.add_dependency 'activesupport', '>=2.3.4'
@@ -31,5 +31,9 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'test-unit'
   s.add_development_dependency 'shoulda'
   s.add_development_dependency 'mysql'
-  s.add_development_dependency 'ruby-debug'
+  if RUBY_VERSION >= '1.9'
+    s.add_development_dependency 'ruby-debug19'
+  else
+    s.add_development_dependency 'ruby-debug'
+  end
 end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
 end

data/test/helper.rb

@@ -1,15 +1,9 @@
 require 'rubygems'
 require 'bundler'
-unless RUBY_VERSION >= '1.9'
-  gem 'fastercsv'
-  require 'fastercsv'
-end
 Bundler.setup
 require 'test/unit'
 require 'shoulda'
-unless RUBY_VERSION >= '1.9'
-  require 'ruby-debug'
-end
+require 'ruby-debug'
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'data_miner'

data/test/support/aircraft.rb

@@ -33,6 +33,10 @@ class Aircraft < ActiveRecord::Base
       row['Manufacturer'] =~ /BOEING/i
     end
     
+    def is_not_attributed_to_airbus?(row)
+      row['Manufacturer'] =~ /AIRBUS/i
+    end
+    
     def is_attributed_to_cessna?(row)
       row['Manufacturer'] =~ /CESSNA/i
     end
@@ -77,8 +81,7 @@ class Aircraft < ActiveRecord::Base
       import("ICAO codes starting with letter #{letter} used by the FAA",
               :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
               :encoding => 'US-ASCII',
-              :errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
-                                    :responder => Aircraft::Guru.new),
+              :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :responder => 'Aircraft::Guru' },
               :row_xpath => '//table/tr[2]/td/table/tr',
               :column_xpath => 'td') do
         key 'icao_code', :field_name => 'Designator'

data/test/support/automobile_variant.rb

@@ -42,9 +42,58 @@ class AutomobileVariant < ActiveRecord::Base
     }
 
     class ParserB
+      require 'slither'
+      ::Slither.define :fuel_economy_guide_b do |d|
+        d.rows do |row|
+          row.trap { true } # there's only one section
+          row.column 'active_year'      , 4,    :type => :integer  #   ACTIVE YEAR
+          row.column 'state_code'       , 1,    :type => :string  #   STATE CODE:  F=49-STATE,C=CALIFORNIA
+          row.column 'carline_clss'     , 2,    :type => :integer  #   CARLINE CLASS CODE
+          row.column 'carline_mfr_code' , 3,    :type => :integer  #   CARLINE MANUFACTURER CODE
+          row.column 'carline_name'     , 28,   :type => :string  #   CARLINE NAME
+          row.column 'disp_cub_in'      , 4,    :type => :integer   #  DISP CUBIC INCHES
+          row.column 'fuel_system'      , 2,    :type => :string   #  FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
+          row.column 'model_trans'      , 6,    :type => :string   #  TRANSMISSION TYPE
+          row.column 'no_cyc'           , 2,    :type => :integer   #  NUMBER OF ENGINE CYLINDERS
+          row.column 'date_time'        , 12,   :type => :string  #   DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
+          row.column 'release_date'     , 6,    :type => :string   #  RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
+          row.column 'vi_mfr_code'      , 3,    :type => :integer   #  VI MANUFACTURER CODE
+          row.column 'carline_code'     , 5,    :type => :integer   #  CARLINE CODE
+          row.column 'basic_eng_id'     , 5,    :type => :integer   #  BASIC ENGINE INDEX
+          row.column 'carline_mfr_name' , 32,   :type => :string  #   CARLINE MANUFACTURER NAME
+          row.column 'suppress_code'    , 1,    :type => :integer    # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
+          row.column 'est_city_mpg'     , 3,    :type => :integer    # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
+          row.spacer 2
+          row.column 'highway_mpg'      , 3,    :type => :integer    # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
+          row.spacer 2
+          row.column 'combined_mpg'     , 3,    :type => :integer    # COMBINED MILES PER GALLON
+          row.spacer 2
+          row.column 'unadj_city_mpg'   , 3,    :type => :integer    # UNADJUSTED  CITY MILES PER GALLON
+          row.spacer 2
+          row.column 'unadj_hwy_mpg'    , 3,    :type => :integer    # UNADJUSTED HIGHWAY MILES PER GALLON
+          row.spacer 2
+          row.column 'unadj_comb_mpg'   , 3,    :type => :integer    # UNADJUSTED COMBINED MILES PER GALLON
+          row.spacer 2
+          row.column 'ave_anl_fuel'     , 6,    :type => :integer    # "$" in col 147, Annual Fuel Cost starting col 148 in I5
+          row.column 'opt_disp'         , 8,    :type => :string    # OPTIONAL DISPLACEMENT
+          row.column 'engine_desc1'     , 10,   :type => :string   #  ENGINE DESCRIPTION 1
+          row.column 'engine_desc2'     , 10,   :type => :string   #  ENGINE DESCRIPTION 2
+          row.column 'engine_desc3'     , 10,   :type => :string   #  ENGINE DESCRIPTION 3
+          row.column 'body_type_2d'     , 10,   :type => :string   #  BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
+          row.column 'body_type_4d'     , 10,   :type => :string   #  BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
+          row.column 'body_type_hbk'    , 10,   :type => :string   #  BODY TYPE HBK    - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
+          row.column 'puerto_rico'      , 1,    :type => :string    # '*' IF FOR PUERTO RICO SALES ONLY
+          row.column 'overdrive'        , 4,    :type => :string    # OVERDRIVE:  ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
+          row.column 'drive_system'     , 3,    :type => :string    # FWD=FRONT WHEEL DRIVE, RWD=REAR,  4WD=4-WHEEL
+          row.column 'filler'           , 1,    :type => :string    # NOT USED
+          row.column 'fuel_type'        , 1,    :type => :string    # R=REGULAR(UNLEADED), P=PREMIUM,  D=DIESEL
+          row.column 'trans_desc'       , 15,   :type => :string   #  TRANSMISSION DESCRIPTORS
+        end
+      end
       attr_accessor :year
       def initialize(options = {})
-        @year = options[:year]
+        options = options.stringify_keys
+        @year = options['year']
       end
 
       def apply(row)
@@ -71,68 +120,12 @@ class AutomobileVariant < ActiveRecord::Base
         end
       end
 
-      def add_hints!(bus)
-        bus[:format] = :fixed_width
-        bus[:cut] = '13-' if year == 1995
-        bus[:schema_name] = :fuel_economy_guide_b
-        bus[:select] = lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' }
-        Slither.define :fuel_economy_guide_b do |d|
-          d.rows do |row|
-            row.trap { true } # there's only one section
-            row.column 'active_year'      , 4,    :type => :integer  #   ACTIVE YEAR
-            row.column 'state_code'       , 1,    :type => :string  #   STATE CODE:  F=49-STATE,C=CALIFORNIA
-            row.column 'carline_clss'     , 2,    :type => :integer  #   CARLINE CLASS CODE
-            row.column 'carline_mfr_code' , 3,    :type => :integer  #   CARLINE MANUFACTURER CODE
-            row.column 'carline_name'     , 28,   :type => :string  #   CARLINE NAME
-            row.column 'disp_cub_in'      , 4,    :type => :integer   #  DISP CUBIC INCHES
-            row.column 'fuel_system'      , 2,    :type => :string   #  FUEL SYSTEM: 'FI' FOR FUEL INJECTION, 2-DIGIT INTEGER VALUE FOR #OF VENTURIES IF CARBURETOR SYSTEM.
-            row.column 'model_trans'      , 6,    :type => :string   #  TRANSMISSION TYPE
-            row.column 'no_cyc'           , 2,    :type => :integer   #  NUMBER OF ENGINE CYLINDERS
-            row.column 'date_time'        , 12,   :type => :string  #   DATE AND TIME RECORD ENTERED -YYMMDDHHMMSS (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND)
-            row.column 'release_date'     , 6,    :type => :string   #  RELEASE DATE - YYMMDD (YEAR, MONTH, DAY)
-            row.column 'vi_mfr_code'      , 3,    :type => :integer   #  VI MANUFACTURER CODE
-            row.column 'carline_code'     , 5,    :type => :integer   #  CARLINE CODE
-            row.column 'basic_eng_id'     , 5,    :type => :integer   #  BASIC ENGINE INDEX
-            row.column 'carline_mfr_name' , 32,   :type => :string  #   CARLINE MANUFACTURER NAME
-            row.column 'suppress_code'    , 1,    :type => :integer    # SUPPRESSION CODE (NO SUPPRESSED RECORD IF FOR PUBLIC ACCESS)
-            row.column 'est_city_mpg'     , 3,    :type => :integer    # ESTIMATED (CITY) MILES PER GALLON - 90% OF UNADJUSTED VALUE
-            row.spacer 2
-            row.column 'highway_mpg'      , 3,    :type => :integer    # ESTIMATED (HWY) MILES PER GALLON - 78% OF UNADJUSTED VALUE
-            row.spacer 2
-            row.column 'combined_mpg'     , 3,    :type => :integer    # COMBINED MILES PER GALLON
-            row.spacer 2
-            row.column 'unadj_city_mpg'   , 3,    :type => :integer    # UNADJUSTED  CITY MILES PER GALLON
-            row.spacer 2
-            row.column 'unadj_hwy_mpg'    , 3,    :type => :integer    # UNADJUSTED HIGHWAY MILES PER GALLON
-            row.spacer 2
-            row.column 'unadj_comb_mpg'   , 3,    :type => :integer    # UNADJUSTED COMBINED MILES PER GALLON
-            row.spacer 2
-            row.column 'ave_anl_fuel'     , 6,    :type => :integer    # "$" in col 147, Annual Fuel Cost starting col 148 in I5
-            row.column 'opt_disp'         , 8,    :type => :string    # OPTIONAL DISPLACEMENT
-            row.column 'engine_desc1'     , 10,   :type => :string   #  ENGINE DESCRIPTION 1
-            row.column 'engine_desc2'     , 10,   :type => :string   #  ENGINE DESCRIPTION 2
-            row.column 'engine_desc3'     , 10,   :type => :string   #  ENGINE DESCRIPTION 3
-            row.column 'body_type_2d'     , 10,   :type => :string   #  BODY TYPE 2 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '2DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
-            row.column 'body_type_4d'     , 10,   :type => :string   #  BODY TYPE 4 DOOR - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM '4DR-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
-            row.column 'body_type_hbk'    , 10,   :type => :string   #  BODY TYPE HBK    - IF THE BODY TYPE APPLIES IT WILL TAKE THE FORM 'HBK-PPP/LL' WHERE PPP=PASSENGER INTERIOR VOLUME AND LL=LUGGAGE INTERIOR VOLUME.
-            row.column 'puerto_rico'      , 1,    :type => :string    # '*' IF FOR PUERTO RICO SALES ONLY
-            row.column 'overdrive'        , 4,    :type => :string    # OVERDRIVE:  ' OD ' FOR OVERDRIVE, 'EOD ' FOR ELECTRICALLY OPERATED OVERDRIVE AND 'AEOD' FOR AUTOMATIC OVERDRIVE
-            row.column 'drive_system'     , 3,    :type => :string    # FWD=FRONT WHEEL DRIVE, RWD=REAR,  4WD=4-WHEEL
-            row.column 'filler'           , 1,    :type => :string    # NOT USED
-            row.column 'fuel_type'        , 1,    :type => :string    # R=REGULAR(UNLEADED), P=PREMIUM,  D=DIESEL
-            row.column 'trans_desc'       , 15,   :type => :string   #  TRANSMISSION DESCRIPTORS
-          end
-        end
-      end
     end
     class ParserC
       attr_accessor :year
       def initialize(options = {})
-        @year = options[:year]
-      end
-
-      def add_hints!(bus)
-        # File will decide format based on filename
+        options = options.stringify_keys
+        @year = options['year']
       end
 
       def apply(row)
@@ -153,11 +146,8 @@ class AutomobileVariant < ActiveRecord::Base
     class ParserD
       attr_accessor :year
       def initialize(options = {})
-        @year = options[:year]
-      end
-
-      def add_hints!(bus)
-        bus[:reject] = lambda { |row| row.values.first.blank? } if year == 2007
+        options = options.stringify_keys
+        @year = options['year']
       end
 
       def apply(row)
@@ -238,8 +228,7 @@ class AutomobileVariant < ActiveRecord::Base
     end
   end
   
-  errata = Errata.new :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv',
-                      :responder => AutomobileVariant::Guru.new
+  errata = { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv', :responder => 'AutomobileVariant::Guru' }
   
   data_miner do
     # 1985---1997
@@ -248,6 +237,10 @@ class AutomobileVariant < ActiveRecord::Base
       import(:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
              :filename => filename,
              :transform => { :class => FuelEconomyGuide::ParserB, :year => "19#{yy}".to_i },
+             :format => :fixed_width,
+             :cut => (yy == 95) ? '13-' : nil,
+             :schema_name => :fuel_economy_guide_b,
+             :select => lambda { |row| row['supress_code'].blank? and row['state_code'] == 'F' },
              :errata => errata) do
         key 'row_hash'
         store 'make_name', :field_name => 'make'
@@ -317,6 +310,7 @@ class AutomobileVariant < ActiveRecord::Base
       # 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
     }.sort { |a, b| a.first <=> b.first }.each do |year, options|
       import options.merge(:transform => { :class => FuelEconomyGuide::ParserD, :year => year },
+                           :reject => (year == 2007) ? lambda { |row| row.values.first.blank? } : nil,
                            :errata => errata) do
         key 'row_hash'
         store 'make_name', :field_name => 'make'

data/test/test_old_syntax.rb

@@ -548,7 +548,7 @@ class AircraftDeux < ActiveRecord::Base
       import("ICAO codes starting with letter #{letter} used by the FAA",
               :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
               :encoding => 'windows-1252',
-              :errata => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
+              :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw' },
               :row_xpath => '//table/tr[2]/td/table/tr',
               :column_xpath => 'td') do
         key 'icao_code', :field_name => 'Designator'
@@ -588,7 +588,7 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
 
     # CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
     import :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/make_fleet_years.csv',
-           :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv',
+           :errata => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv' },
            :select => lambda { |row| row['volume'].to_i > 0 } do
       key   'name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['fleet'][2,2], row['year_content'] ].join ' ' }
       store 'make_name', :field_name => 'manufacturer_name'
@@ -661,7 +661,7 @@ class TestOldSyntax < Test::Unit::TestCase
         end
       end
       assert_kind_of DataMiner::Import, AutomobileFuelType.data_miner_config.steps.first
-      assert_equal 'http://example.com', AutomobileFuelType.data_miner_config.steps.first.table.package.url
+      assert_equal 'http://example.com', AutomobileFuelType.data_miner_config.steps.first.table.url
       assert_equal 1, AutomobileFuelType.data_miner_config.step_counter
     end
     should "stop and finish if it gets a DataMiner::Finish" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: false
   segments: 
   - 1
-  - 0
   - 1
-  version: 1.0.1
+  - 0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -28,12 +28,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 17
+        hash: 19
         segments: 
         - 1
+        - 1
         - 0
-        - 3
-        version: 1.0.3
+        version: 1.1.0
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency