data_miner 2.4.1 → 2.5.0

data/CHANGELOG

@@ -1,3 +1,18 @@
+2.5.0 / 2013-03-15
+
+* Breaking changes
+
+  * Lock to roo 1.10.1 and activerecord > 3
+
+* Enhancements
+
+  * If you don't define a `key` in a data_miner block, then just keep appending records - thanks @towerhe
+  * Got rid of ridiculous "install unicode-utils" warning
+
+* Bug fixes
+
+  * Don't die because an Alchemist::NumericConversion is passed to a function that expects a plain Float - thanks @towerhe
+
 2.4.1 / 2012-07-26
 
 * Enhancements

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Brighter Planet
+Copyright (c) 2013 Seamus Abshere
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.markdown

@@ -18,7 +18,7 @@ The killer combination for us is:
 1. [`active_record_inline_schema`](https://github.com/seamusabshere/active_record_inline_schema) - define table structure
 2. [`remote_table`](https://github.com/seamusabshere/remote_table) - download data and parse it
 3. [`errata`](https://github.com/seamusabshere/errata) - apply corrections in a transparent way
-4. [`data_miner`](https://github.com/seamusabshere/remote_table) (this library!) - import data idempotently
+4. [`data_miner`](https://github.com/seamusabshere/data_miner) (this library!) - import data idempotently
 
 ## Documentation
 
@@ -115,13 +115,13 @@ And many more - look for the `data_miner.rb` file that corresponds to each model
 * Andy Rossmeissl <andy@rossmeissl.net>
 * Derek Kastner <dkastner@gmail.com>
 * Ian Hough <ijhough@gmail.com>
+* Tower He <towerhe@gmail.com>
 
 ## Wishlist
 
 * Make the tests real unit tests
 * sql steps shouldn't shell out if binaries are missing
-* csv import step that uses pg_restore, mysqlimport, etc.
 
 ## Copyright
 
-Copyright (c) 2012 Brighter Planet. See LICENSE for details.
+Copyright (c) 2013 Seamus Abshere

data/data_miner.gemspec

@@ -4,8 +4,8 @@ require File.expand_path("../lib/data_miner/version", __FILE__)
 Gem::Specification.new do |s|
   s.name        = "data_miner"
   s.version     = DataMiner::VERSION
-  s.authors     = ["Seamus Abshere", "Andy Rossmeissl", "Derek Kastner"]
-  s.email       = ["seamus@abshere.net"]
+  s.authors     = ["Seamus Abshere", "Andy Rossmeissl", "Derek Kastner", "Ian Hough", "Tower He"]
+  s.email       = ["seamus@abshere.net", "rossmeissl@gmail.com", "dkastner@gmail.com", "ijhough@gmail.com", "towerhe@gmail.com"]
   s.homepage    = "https://github.com/seamusabshere/data_miner"
   s.summary     = %{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.}
   s.description = %q{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. You can also convert units.}
@@ -18,14 +18,15 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   
   s.add_runtime_dependency 'aasm'
-  s.add_runtime_dependency 'active_record_inline_schema', '>=0.5.6'
-  s.add_runtime_dependency 'activerecord', '>=2.3.4'
-  s.add_runtime_dependency 'activesupport', '>=2.3.4'
+  s.add_runtime_dependency 'active_record_inline_schema', '>=0.6.1'
+  s.add_runtime_dependency 'activerecord', '> 3'
+  s.add_runtime_dependency 'activesupport', '> 3'
   s.add_runtime_dependency 'errata', '>=1.0.1'
   s.add_runtime_dependency 'remote_table', '>=2.0.2'
   s.add_runtime_dependency 'upsert', '>=0.3.1'
   s.add_runtime_dependency 'posix-spawn'
   s.add_runtime_dependency 'unix_utils'
+  s.add_runtime_dependency 'roo', '1.10.1'
 
   s.add_development_dependency 'dkastner-alchemist'
   s.add_development_dependency 'conversions'

data/lib/data_miner/step/import.rb

@@ -19,7 +19,7 @@ class DataMiner
       # Description of what this step does.
       # @return [String]
       attr_reader :description
-      
+
       # @private
       def initialize(script, description, settings, &blk)
         settings = settings.symbolize_keys
@@ -83,27 +83,9 @@ class DataMiner
 
       # @private
       def start
-        if not validate? and (storing_primary_key? or table_has_autoincrementing_primary_key?)
-          c = ActiveRecord::Base.connection_pool.checkout
-          Upsert.stream(c, model.table_name) do |upsert|
-            table.each do |row|
-              selector = { @key => attributes[@key].read(row) }
-              document = attributes.except(@key).inject({}) do |memo, (_, attr)|
-                memo.merge! attr.updates(row)
-                memo
-              end
-              upsert.row selector, document
-            end
-          end
-          ActiveRecord::Base.connection_pool.checkin c
-        else
-          table.each do |row|
-            record = model.send "find_or_initialize_by_#{@key}", attributes[@key].read(row)
-            attributes.each { |_, attr| attr.set_from_row record, row }
-            record.save!
-          end
-        end
+        upsert_enabled? ? save_with_upsert : save_with_activerecord
         refresh
+
         nil
       end
 
@@ -115,6 +97,33 @@ class DataMiner
 
       private
 
+      def upsert_enabled?
+        (not validate?) and (storing_primary_key? or table_has_autoincrementing_primary_key?)
+      end
+
+      def save_with_upsert
+        c = ActiveRecord::Base.connection_pool.checkout
+        Upsert.stream(c, model.table_name) do |upsert|
+          table.each do |row|
+            selector = @key ? { @key => attributes[@key].read(row) } : { model.primary_key => nil }
+            document = attributes.except(@key).inject({}) do |memo, (_, attr)|
+              memo.merge! attr.updates(row)
+              memo
+            end
+            upsert.row selector, document
+          end
+        end
+        ActiveRecord::Base.connection_pool.checkin c
+      end
+
+      def save_with_activerecord
+        table.each do |row|
+          record = @key ? model.send("find_or_initialize_by_#{@key}", attributes[@key].read(row)) : model.new
+          attributes.each { |_, attr| attr.set_from_row record, row }
+          record.save!
+        end
+      end
+
       def table_has_autoincrementing_primary_key?
         return @table_has_autoincrementing_primary_key_query.first if @table_has_autoincrementing_primary_key_query.is_a?(Array)
         c = ActiveRecord::Base.connection_pool.checkout
@@ -140,7 +149,7 @@ class DataMiner
 
       def storing_primary_key?
         return @storing_primary_key_query.first if @storing_primary_key_query.is_a?(Array)
-        @storing_primary_key_query = [attributes.has_key?(model.primary_key.to_sym)]
+        @storing_primary_key_query = [model.primary_key && attributes.has_key?(model.primary_key.to_sym)]
         @storing_primary_key_query.first
       end
 

data/lib/data_miner/step/sql.rb

@@ -53,7 +53,7 @@ class DataMiner
       private
 
       def config
-        @config ||= if ActiveRecord::Base.respond_to?(:connection_config)
+        if ActiveRecord::Base.respond_to?(:connection_config)
           ActiveRecord::Base.connection_config
         else
           ActiveRecord::Base.connection_pool.spec.config
@@ -82,7 +82,7 @@ class DataMiner
           ::Process.waitpid pid
         end
         unless $?.success?
-          raise RuntimeError, "[data_miner] Failed: #{argv.join(' ').inspect}"
+          raise RuntimeError, "[data_miner] Failed: ARGV #{argv.join(' ').inspect}"
         end
         nil
       end
@@ -90,25 +90,24 @@ class DataMiner
       alias :mysql2 :mysql
 
       def postgresql(path)
-        connect = []
-        connect << ['--username', config[:username]] if config[:username]
-        connect << ['--password', config[:password]] if config[:password]
-        connect << ['--host',     config[:host]]     if config[:host]
-        connect << ['--port',     config[:port]]     if config[:port]
+        env = {}
+        env['PGHOST']     = config['host']          if config['host']
+        env['PGPORT']     = config['port'].to_s     if config['port']
+        env['PGPASSWORD'] = config['password'].to_s if config['password']
+        env['PGUSER']     = config['username'].to_s if config['username']
 
         argv = [
           'psql',
-          connect,
           '--quiet',
           '--dbname', config[:database],
           '--file',   path
         ].flatten
         
-        child = POSIX::Spawn::Child.new(*argv)
+        child = POSIX::Spawn::Child.new(*([env]+argv))
         $stderr.puts child.out
         $stderr.puts child.err
         unless child.success?
-          raise RuntimeError, "[data_miner] Failed: #{argv.join(' ').inspect} (#{child.err.inspect})"
+          raise RuntimeError, "[data_miner] Failed: ENV #{env.inspect} ARGV #{argv.join(' ').inspect} (#{child.err.inspect})"
         end
         nil
       end

data/lib/data_miner/unit_converter/alchemist.rb

@@ -4,7 +4,7 @@ class DataMiner
   class UnitConverter
     class Alchemist < UnitConverter
       def convert(value, from, to)
-        value.to_f.send(from).to.send(to)
+        value.to_f.send(from).to.send(to).to_f
       end
     end
   end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.4.1'
+  VERSION = '2.5.0'
 end

data/lib/data_miner.rb

@@ -10,7 +10,7 @@ if RUBY_VERSION >= '1.9'
   begin
     require 'unicode_utils/downcase'
   rescue LoadError
-    Kernel.warn '[data_miner] You may wish to include unicode_utils in your Gemfile to improve accuracy of downcasing'
+    # oh well.
   end
 end
 

data/test/data_miner/step/test_import.rb

@@ -2,7 +2,9 @@ require 'helper'
 init_database
 require 'earth'
 
-Earth.init :residence, :electricity, :hospitality, :load_data_miner => true, :apply_schemas => true
+require 'earth/residence'
+require 'earth/electricity'
+require 'earth/hospitality'
 
 class PetBlue < ActiveRecord::Base
   data_miner do

data/test/data_miner/unit_converter/test_alchemist.rb

@@ -2,12 +2,19 @@ require 'helper'
 
 describe 'DataMiner::UnitConverter::Alchemist' do
   before do
+    @original_converter = DataMiner.unit_converter
     DataMiner.unit_converter = :alchemist
   end
 
+  after do
+    DataMiner.unit_converter = @original_converter
+  end
+
   describe '#convert' do
     it 'converts a value from one unit to another' do
-      DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
+      value = DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
+      assert value.is_a?(Float)
+      value.must_be_close_to 7.71617918
     end
   end
 end

data/test/data_miner/unit_converter/test_conversions.rb

@@ -2,14 +2,19 @@ require 'helper'
 
 describe 'DataMiner::UnitConverter::Conversions' do
   before do
-    #DataMiner.unit_converter = :conversions
+    @original_converter = DataMiner.unit_converter
+    DataMiner.unit_converter = :conversions
+  end
+
+  after do
+    DataMiner.unit_converter = @original_converter
   end
 
   describe '#convert' do
     it 'converts a value from one unit to another' do
-      # can't load both alchemist and conversions in same test run
-      # see test/test_unit_conversion for coverage of this adapter
-      #DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
+      value = DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
+      assert value.is_a?(Float)
+      value.must_be_close_to 7.71617918
     end
   end
 end

data/test/helper.rb

@@ -10,8 +10,7 @@ end
 require 'minitest/spec'
 require 'minitest/autorun'
 require 'minitest/reporters'
-MiniTest::Unit.runner = MiniTest::SuiteRunner.new
-MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
+MiniTest::Reporters.use!
 
 require 'active_record'
 require 'logger'

data/test/support/breed.rb

@@ -5,7 +5,7 @@ class Breed < ActiveRecord::Base
     def update_average_age!
       # make sure pet is populated
       Pet.run_data_miner!
-      update_all %{"average_age" = (SELECT AVG("pets"."age") FROM "pets" WHERE "pets"."breed_id" = "breeds"."name")}
+      update_all %{average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
     end
   end
   self.primary_key = "name"

data/test/test_data_miner.rb

@@ -80,6 +80,10 @@ describe DataMiner do
       Pet.run_data_miner!
       Pet.find('Pierre').weight.must_be_close_to 1.9958 # 4.4 pounds in kilograms
     end
+    it "doesn't convert nil to 0 when converting units" do
+      Pet.run_data_miner!
+      Pet.find('Nemo').age.must_be_nil
+    end
     it "sets units" do
       Pet.run_data_miner!
       Pet.find('Pierre').age_units.must_equal 'years'
@@ -119,6 +123,38 @@ describe DataMiner do
         Pet3.run_data_miner!
       end.must_raise RuntimeError, /exist/i
     end
+  end
+
+  describe 'when the key attribute is not defined' do
+    class PetFunny < ActiveRecord::Base
+      self.primary_key = false
+      col :name
+      col :breed
+      col :color
+
+      data_miner do
+        import 'without a key', url: "file://#{PETS_FUNNY}" do
+          store :name
+          store :breed
+          store :color
+        end
+      end
+    end
+    PetFunny.auto_upgrade!
+
+    before { PetFunny.delete_all }
+
+    it 'imports the example data' do
+      PetFunny.run_data_miner!
+      PetFunny.must_be :exists?
+    end
+
+    it 'imports new example data for each run' do
+      PetFunny.run_data_miner!
+      first_count = PetFunny.count
 
+      PetFunny.run_data_miner!
+      PetFunny.count.must_equal first_count * 2
+    end
   end
 end

metadata

@@ -1,17 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.5.0
   prerelease: 
 platform: ruby
 authors:
 - Seamus Abshere
 - Andy Rossmeissl
 - Derek Kastner
+- Ian Hough
+- Tower He
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-27 00:00:00.000000000 Z
+date: 2013-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -36,7 +38,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.5.6
+        version: 0.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,39 +46,39 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.5.6
+        version: 0.6.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ! '>'
       - !ruby/object:Gem::Version
-        version: 2.3.4
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ! '>'
       - !ruby/object:Gem::Version
-        version: 2.3.4
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ! '>'
       - !ruby/object:Gem::Version
-        version: 2.3.4
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ! '>'
       - !ruby/object:Gem::Version
-        version: 2.3.4
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: errata
   requirement: !ruby/object:Gem::Requirement
@@ -157,6 +159,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.1
 - !ruby/object:Gem::Dependency
   name: dkastner-alchemist
   requirement: !ruby/object:Gem::Requirement
@@ -370,6 +388,10 @@ description: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and
   units.
 email:
 - seamus@abshere.net
+- rossmeissl@gmail.com
+- dkastner@gmail.com
+- ijhough@gmail.com
+- towerhe@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -441,7 +463,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: data_miner
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import