data_miner 2.3.4 → 2.4.0

data/CHANGELOG

@@ -1,3 +1,15 @@
+2.4.0 / 2012-07-26
+
+* Breaking changes
+
+  * Entirely remove taps support - it doesn't preserve foreign key constraints and adds a lot of complexity
+
+* Enhancements
+
+  * New "sql" step - executes SQL provided as a string OR a local/remote SQL file provided as a URL
+  * Die with a useful error message if a column specified in an import step doesn't exist - thanks @chrisle https://github.com/seamusabshere/data_miner/issues/17
+  * Allow setting :validate => true on import steps - thanks @chrisle https://github.com/seamusabshere/data_miner/issues/18
+
 2.3.4 / 2012-07-06
 
 * Bug fixes

data/README.markdown

@@ -30,8 +30,17 @@ You define <code>data_miner</code> blocks in your ActiveRecord models. For examp
 
     class Country < ActiveRecord::Base
       self.primary_key = 'iso_3166_code'
+
+      # the "col" class method is provided by a different library - active_record_inline_schema
+      col :iso_3166_code                            # alpha-2 2-letter like GB
+      col :iso_3166_numeric_code, :type => :integer # numeric like 826; aka UN M49 code
+      col :iso_3166_alpha_3_code                    # 3-letter like GBR
+      col :name
   
       data_miner do
+        # auto_upgrade! is provided by active_record_inline_schema
+        process :auto_upgrade!
+
         import("OpenGeoCode.org's Country Codes to Country Names list",
                :url => 'http://opengeocode.org/download/countrynames.txt',
                :format => :delimited,
@@ -107,6 +116,10 @@ And many more - look for the `data_miner.rb` file that corresponds to each model
 * Derek Kastner <dkastner@gmail.com>
 * Ian Hough <ijhough@gmail.com>
 
+## Wishlist
+
+* Make the tests real unit tests
+
 ## Copyright
 
 Copyright (c) 2012 Brighter Planet. See LICENSE for details.

data/data_miner.gemspec

@@ -24,6 +24,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'errata', '>=1.0.1'
   s.add_runtime_dependency 'remote_table', '>=2.0.2'
   s.add_runtime_dependency 'upsert', '>=0.3.1'
+  s.add_runtime_dependency 'posix-spawn'
+  s.add_runtime_dependency 'unix_utils'
 
   s.add_development_dependency 'dkastner-alchemist'
   s.add_development_dependency 'conversions'
@@ -34,6 +36,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'minitest-reporters'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yard'
+  s.add_development_dependency 'rdiscount'
   if RUBY_PLATFORM == 'java'
     s.add_development_dependency 'jruby-openssl'
     s.add_development_dependency 'activerecord-jdbcsqlite3-adapter'

data/lib/data_miner/active_record_class_methods.rb

@@ -63,7 +63,7 @@ class DataMiner
     #
     # @see DataMiner::Script#import Creating an import step by calling DataMiner::Script#import from inside a data miner script
     # @see DataMiner::Script#process Creating a process step by calling DataMiner::Script#process from inside a data miner script
-    # @see DataMiner::Script#tap Creating a tap step by calling DataMiner::Script#tap from inside a data miner script
+    # @see DataMiner::Script#sql Creating a sql step by calling DataMiner::Script#sql from inside a data miner script
     #
     # @example Creating steps
     #   class MyModel < ActiveRecord::Base
@@ -71,6 +71,7 @@ class DataMiner
     #       process [...]
     #       import [...]
     #       import [...yes, it's ok to have more than one import step...]
+    #       sql [...]
     #       process [...]
     #       [...etc...]
     #     end

data/lib/data_miner/attribute.rb

@@ -239,6 +239,9 @@ class DataMiner
 
     # @private
     def read(row)
+      unless column_exists?
+        raise RuntimeError, "[data_miner] Table #{model.table_name} does not have column #{name.inspect}"
+      end
       if matcher and matcher_output = matcher.match(row)
         return matcher_output
       end
@@ -342,6 +345,11 @@ class DataMiner
       step.model
     end
 
+    def column_exists?
+      return @column_exists_boolean if defined?(@column_exists_boolean)
+      @column_exists_boolean = model.column_names.include? name.to_s
+    end
+
     def text_column?
       return @text_column_boolean if defined?(@text_column_boolean)
       @text_column_boolean = model.columns_hash[name.to_s].text?

data/lib/data_miner/run.rb

@@ -100,9 +100,11 @@ class DataMiner
         fail!
         raise $!
       ensure
-        self.row_count_after = model.count
-        if DataMiner.per_column_statistics?
-          ColumnStatistic.take self
+        if model.table_exists?
+          self.row_count_after = model.count
+          if DataMiner.per_column_statistics?
+            ColumnStatistic.take self
+          end
         end
         self.stopped_at = ::Time.now.utc
         save!

data/lib/data_miner/script.rb

@@ -92,47 +92,32 @@ class DataMiner
       append(:process, method_id_or_description, &blk)
     end
 
-    # Use https://github.com/ricardochimal/taps to pull table structure and data.
-    #
-    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
-    # @see DataMiner::Step::Tap The actual Tap class.
-    #
-    # @param [String] description A description of the taps source.
-    # @param [String] source The taps URL, including username, password, domain, and port.
-    # @param [optional, Hash] options
-    # @option options [String] :source_table_name (model.table_name) The source table name, if different.
+    # Import rows into your model.
     #
-    # @note The source table name will default to the model's table name. If it's different, use the +:source_table_name+ option.
-    # @note +taps+ needs to be installed on your system and in your PATH, but it doesn't have to be in your Gemfile. Sometimes having it in your Gemfile will cause Heroku deploys (etc.) to fail because it requires +sqlite3+.
+    # As long as...
+    # 1. you +key+ on the primary key, or
+    # 2. the table has an auto-increment primary key, or
+    # 3. you DON'T enable +:validate+
+    # ... then things will be sped up using the {https://github.com/seamusabshere/upsert upsert library} in streaming mode.
     #
-    # @example Tapping Brighter Planet's reference data web service
-    #   data_miner do
-    #     [...]
-    #     tap "Brighter Planet's reference data", "http://carbon:neutral@data.brighterplanet.com:5000"
-    #     [...]
-    #   end
-    #
-    # @return [nil]
-    def tap(description, source, options = {})
-      append :tap, description, source, options
-    end
-
-    # Import rows into your model.
+    # Otherwise, native +ActiveRecord+ constuctors and validations will be used.
     #
     # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
     # @see DataMiner::Step::Import The actual Import class.
     #
     # @param [String] description A description of the data source.
-    # @param [Hash] table_and_errata_settings Settings, including URL of the data source, that are used to download/parse (using RemoteTable) and (sometimes) correct (using Errata) the data.
-    # @option table_and_errata_settings [String] :url The URL of the data source. Passed directly to +RemoteTable.new+.
-    # @option table_and_errata_settings [Hash] :errata The +:responder+ and +:url+ settings that will be passed to +Errata.new+.
-    # @option table_and_errata_settings [*] anything Any other setting will be passed to +RemoteTable.new+.
+    # @param [Hash] settings Settings, including URL of the data source, that are used to download/parse (using RemoteTable) and (sometimes) correct (using Errata) the data.
+    # @option settings [String] :url The URL of the data source. Passed directly to +RemoteTable.new+.
+    # @option settings [Hash] :errata The +:responder+ and +:url+ settings that will be passed to +Errata.new+.
+    # @option settings [TrueClass,FalseClass] :validate Whether to always run +ActiveRecord+ validations.
+    # @option settings [*] anything Any other setting will be passed to +RemoteTable.new+.
     #
     # @yield [] A block defining how to +key+ the import (to make it idempotent) and which columns to +store+.
     #
-    # @note Be sure to check out https://github.com/seamusabshere/remote_table and https://github.com/seamusabshere/errata for available +table_and_errata_settings+.
+    # @note Be sure to check out https://github.com/seamusabshere/remote_table and https://github.com/seamusabshere/errata for available +settings+.
     # @note There are hundreds of +import+ examples in https://github.com/brighterplanet/earth. The {file:README.markdown README} points to a few (at the bottom.)
     # @note We often use string primary keys to make idempotency easier. https://github.com/seamusabshere/active_record_inline_schema supports defining these inline.
+    # @note Enabling +:validate+ may slow down importing large files because it precludes bulk loading using https://github.com/seamusabshere/upsert.
     #
     # @example From the README
     #   data_miner do
@@ -152,8 +137,26 @@ class DataMiner
     #   end
     #
     # @return [nil]
-    def import(description, table_and_errata_settings, &blk)
-      append(:import, description, table_and_errata_settings, &blk)
+    def import(description, settings, &blk)
+      append(:import, description, settings, &blk)
+    end
+
+    # Execute SQL, provided either as a string or a URL.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Step::Sql The actual Sql class.
+    #
+    # @note +url_or_statement+ is auto-detected by looking for +%r{^[^\s]*/[^\*]}+ (non-spaces followed by a slash followed by non-asterisk). Therefore if you're passing a local file path and want it to be treated like a URL, make it absolute.
+    #
+    # @param [String] description What this step does.
+    # @param [String] url_or_statement SQL statement as a String or location of the SQL file as a URL.
+    #
+    # @example Rapidly get a list of countries from Brighter Planet's Reference Data web service
+    #   data_miner do
+    #     sql "Brighter Planet's countries", 'http://data.brighterplanet.com/countries.sql'
+    #   end
+    def sql(description, url_or_statement)
+      append(:sql, description, url_or_statement)
     end
 
     # Prepend a step to a script unless it's already there. Mostly for internal use.
@@ -237,7 +240,11 @@ class DataMiner
         args = ["#{klass.name.demodulize} step with no description"]
       end
       initializer = [self] + args + [options]
-      klass.new(*initializer, &blk)
+      if block_given?
+        klass.new(*initializer, &blk)
+      else
+        klass.new(*initializer)
+      end
     end
   end
 end

data/lib/data_miner/step/import.rb

@@ -16,31 +16,29 @@ class DataMiner
       # @return [Array<DataMiner::Attribute>]
       attr_reader :attributes
 
-      # @private
-      attr_reader :script
-
       # Description of what this step does.
       # @return [String]
       attr_reader :description
       
       # @private
-      def initialize(script, description, table_and_errata_settings, &blk)
-        table_and_errata_settings = table_and_errata_settings.symbolize_keys
-        if table_and_errata_settings.has_key?(:table)
+      def initialize(script, description, settings, &blk)
+        settings = settings.symbolize_keys
+        if settings.has_key?(:table)
           raise ::ArgumentError, %{[data_miner] :table is no longer an allowed setting.}
         end
-        if (errata_settings = table_and_errata_settings[:errata]) and not errata_settings.is_a?(::Hash)
+        if (errata_settings = settings[:errata]) and not errata_settings.is_a?(::Hash)
           raise ::ArgumentError, %{[data_miner] :errata must be a hash of initialization settings to Errata}
         end
         @script = script
         @attributes = ::ActiveSupport::OrderedHash.new
+        @validate_query = !!settings[:validate]
         @description = description
-        if table_and_errata_settings.has_key? :errata
-          errata_settings = table_and_errata_settings[:errata].symbolize_keys
+        if settings.has_key? :errata
+          errata_settings = settings[:errata].symbolize_keys
           errata_settings[:responder] ||= model
-          table_and_errata_settings[:errata] = errata_settings
+          settings[:errata] = errata_settings
         end
-        @table_settings = table_and_errata_settings.dup
+        @table_settings = settings.dup
         @table_settings[:streaming] = true
         @table_mutex = ::Mutex.new
         instance_eval(&blk)
@@ -85,7 +83,7 @@ class DataMiner
 
       # @private
       def start
-        if storing_primary_key? or table_has_autoincrementing_primary_key?
+        if not validate? and (storing_primary_key? or table_has_autoincrementing_primary_key?)
           c = ActiveRecord::Base.connection_pool.checkout
           Upsert.stream(c, model.table_name) do |upsert|
             table.each do |row|
@@ -109,6 +107,12 @@ class DataMiner
         nil
       end
 
+      # @private
+      # Whether to run ActiveRecord validations. Slows things down because Upsert isn't used.
+      def validate?
+        @validate_query == true
+      end
+
       private
 
       def table_has_autoincrementing_primary_key?

data/lib/data_miner/step/process.rb

@@ -7,9 +7,6 @@ class DataMiner
     # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
     # @see DataMiner::Script#process Creating a process step by calling DataMiner::Script#process from inside a data miner script
     class Process < Step
-      # @private
-      attr_reader :script
-
       # The method to be called on the model class.
       # @return [Symbol]
       attr_reader :method_id
@@ -25,7 +22,7 @@ class DataMiner
       alias :block_description :description
 
       # @private
-      def initialize(script, method_id_or_description, ignored_options = {}, &blk)
+      def initialize(script, method_id_or_description, ignored_options = nil, &blk)
         @script = script
         if block_given?
           @description = method_id_or_description

data/lib/data_miner/step/sql.rb

@@ -0,0 +1,117 @@
+require 'csv'
+require 'tmpdir'
+require 'posix/spawn'
+require 'unix_utils'
+
+class DataMiner
+  class Step
+    # A step that executes a SQL, either from a string or as retrieved from a URL.
+    #
+    # Create these by calling +sql+ inside a +data_miner+ block.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Script#sql Creating a sql step by calling DataMiner::Script#sql from inside a data miner script
+    class Sql < Step
+      URL_DETECTOR = %r{^[^\s]*/[^\*]}
+
+      # Description of what this step does.
+      # @return [String]
+      attr_reader :description
+
+      # Location of the SQL file.
+      # @return [String]
+      attr_reader :url
+
+      # String containing the SQL.
+      # @return [String]
+      attr_reader :statement
+      
+      # @private
+      def initialize(script, description, url_or_statement, ignored_options = nil)
+        @script = script
+        @description = description
+        if url_or_statement =~ URL_DETECTOR
+          @url = url_or_statement
+        else
+          @statement = url_or_statement
+        end
+      end
+
+      # @private
+      def start
+        if statement
+          c = ActiveRecord::Base.connection_pool.checkout
+          c.execute statement
+          ActiveRecord::Base.connection_pool.checkin c
+        else
+          tmp_path = UnixUtils.curl url
+          send config[:adapter], tmp_path
+          File.unlink tmp_path
+        end
+      end
+
+      private
+
+      def config
+        @config ||= if ActiveRecord::Base.respond_to?(:connection_config)
+          ActiveRecord::Base.connection_config
+        else
+          ActiveRecord::Base.connection_pool.spec.config
+        end
+      end
+
+      def mysql(path)
+        connect = if config[:socket]
+          [ '--socket', config[:socket] ]
+        else
+          [ '--host', config.fetch(:host, '127.0.0.1'), '--port', config.fetch(:port, 3306).to_s ]
+        end
+        
+        argv = [
+          'mysql',
+          '--compress',
+          '--user', config[:username],
+          "-p#{config[:password]}",
+          connect,
+          '--default-character-set', 'utf8',
+          config[:database]
+        ].flatten
+
+        File.open(path) do |f|
+          pid = POSIX::Spawn.spawn(*(argv+[{:in => f}]))
+          ::Process.waitpid pid
+        end
+        unless $?.success?
+          raise RuntimeError, "[data_miner] Failed: #{argv.join(' ').inspect}"
+        end
+        nil
+      end
+
+      alias :mysql2 :mysql
+
+      def postgresql(path)
+        connect = []
+        connect << ['--username', config[:username]] if config[:username]
+        connect << ['--password', config[:password]] if config[:password]
+        connect << ['--host',     config[:host]]     if config[:host]
+        connect << ['--port',     config[:port]]     if config[:port]
+
+        argv = [
+          'psql',
+          connect,
+          '--quiet',
+          '--dbname', config[:database],
+          '--file',   path
+        ].flatten
+        
+        child = POSIX::Spawn::Child.new(*argv)
+        $stderr.puts child.out
+        $stderr.puts child.err
+        unless child.success?
+          raise RuntimeError, "[data_miner] Failed: #{argv.join(' ').inspect} (#{child.err.inspect})"
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/data_miner/step.rb

@@ -1,5 +1,8 @@
 class DataMiner
   class Step
+    # @private
+    attr_reader :script
+
     # @private
     def ==(other)
       other.class == self.class and other.description == description

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.3.4'
+  VERSION = '2.4.0'
 end

data/lib/data_miner.rb

@@ -20,8 +20,8 @@ require 'data_miner/script'
 require 'data_miner/dictionary'
 require 'data_miner/step'
 require 'data_miner/step/import'
-require 'data_miner/step/tap'
 require 'data_miner/step/process'
+require 'data_miner/step/sql'
 require 'data_miner/run'
 require 'data_miner/unit_converter'
 
@@ -44,7 +44,7 @@ class DataMiner
 
     # @private
     def compress_whitespace(str)
-      str.gsub(INNER_SPACE, ' ').strip
+      str.gsub(INNER_SPACE, ONE_SPACE).strip
     end
 
     # Set the unit converter.
@@ -66,6 +66,7 @@ class DataMiner
   end
 
   INNER_SPACE = /[ ]+/
+  ONE_SPACE = ' '
 
   include ::Singleton
 

data/test/data_miner/step/test_sql.rb

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+require 'helper'
+init_database
+
+class BreedBlue < ActiveRecord::Base
+  self.table_name = 'breeds'
+  self.primary_key = 'name'
+  data_miner do
+    sql "Brighter Planet's list of breeds (as a URL)", 'http://data.brighterplanet.com/breeds.sql'
+  end
+end
+
+class BreedRed < ActiveRecord::Base
+  self.table_name = 'breeds'
+  self.primary_key = 'name'
+  data_miner do
+    sql "Brighter Planet's list of breeds (as a URL)", 'http://data.brighterplanet.com/breeds.sql'
+    sql "Mess up weights", %{UPDATE breeds SET weight = 999}
+  end
+end
+
+describe DataMiner::Step::Sql do
+  before do
+    BreedBlue.delete_all rescue nil
+  end
+  it "can be provided as a URL" do
+    BreedBlue.run_data_miner!
+    BreedBlue.where(:name => 'Affenpinscher').count.must_equal 1
+    BreedBlue.where(:name => 'Württemberger').count.must_equal 1
+    BreedBlue.find('Afghan Hound').weight.must_be_close_to 24.9476
+  end
+  it "can be provided as a string" do
+    BreedRed.run_data_miner!
+    BreedRed.where(:name => 'Affenpinscher').count.must_equal 1
+    BreedRed.where(:name => 'Württemberger').count.must_equal 1
+    BreedRed.find('Afghan Hound').weight.must_be_close_to 999
+  end
+end

data/test/helper.rb

@@ -59,8 +59,10 @@ def init_models
   require 'support/breed'
   require 'support/pet'
   require 'support/pet2'
+  require 'support/pet3'
   Pet.auto_upgrade!
   Pet2.auto_upgrade!
+  Pet3.auto_upgrade!
 
   ActiveRecord::Base.descendants.each do |model|
     model.attr_accessible nil

data/test/support/pet3.rb

@@ -0,0 +1,9 @@
+class Pet3 < ActiveRecord::Base
+  col :a
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{PETS}") do
+      key :b, :field_name => 'name'
+    end
+  end
+end

data/test/test_data_miner.rb

@@ -7,6 +7,8 @@ describe DataMiner do
   describe "when used to import example data about pets" do
     before do
       Pet.delete_all
+      Pet2.delete_all
+      Pet3.delete_all
       DataMiner::Run.delete_all
       DataMiner::Run::ColumnStatistic.delete_all
     end
@@ -112,5 +114,11 @@ describe DataMiner do
       Pet2.run_data_miner!
       Pet2.find('Jerry').breed_id.must_equal 'Beagle-Basset'
     end
+    it "dies if a column specified in an import step doesn't exist" do
+      lambda do
+        Pet3.run_data_miner!
+      end.must_raise RuntimeError, /exist/i
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.3.4
+  version: 2.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-07 00:00:00.000000000 Z
+date: 2012-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -125,6 +125,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.3.1
+- !ruby/object:Gem::Dependency
+  name: posix-spawn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unix_utils
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dkastner-alchemist
   requirement: !ruby/object:Gem::Requirement
@@ -269,6 +301,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdiscount
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -344,12 +392,13 @@ files:
 - lib/data_miner/step.rb
 - lib/data_miner/step/import.rb
 - lib/data_miner/step/process.rb
-- lib/data_miner/step/tap.rb
+- lib/data_miner/step/sql.rb
 - lib/data_miner/unit_converter.rb
 - lib/data_miner/unit_converter/alchemist.rb
 - lib/data_miner/unit_converter/conversions.rb
 - lib/data_miner/version.rb
 - test/data_miner/step/test_import.rb
+- test/data_miner/step/test_sql.rb
 - test/data_miner/test_attribute.rb
 - test/data_miner/unit_converter/test_alchemist.rb
 - test/data_miner/unit_converter/test_conversions.rb
@@ -362,6 +411,7 @@ files:
 - test/support/data_miner_without_unit_converter.rb
 - test/support/pet.rb
 - test/support/pet2.rb
+- test/support/pet3.rb
 - test/support/pet_color_dictionary.en.csv
 - test/support/pet_color_dictionary.es.csv
 - test/support/pets.csv
@@ -369,7 +419,6 @@ files:
 - test/test_data_miner.rb
 - test/test_data_miner_run_column_statistic.rb
 - test/test_earth_import.rb
-- test/test_earth_tap.rb
 - test/test_safety.rb
 - test/test_unit_conversion.rb
 homepage: https://github.com/seamusabshere/data_miner
@@ -399,6 +448,7 @@ summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and imp
   XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
 test_files:
 - test/data_miner/step/test_import.rb
+- test/data_miner/step/test_sql.rb
 - test/data_miner/test_attribute.rb
 - test/data_miner/unit_converter/test_alchemist.rb
 - test/data_miner/unit_converter/test_conversions.rb
@@ -411,6 +461,7 @@ test_files:
 - test/support/data_miner_without_unit_converter.rb
 - test/support/pet.rb
 - test/support/pet2.rb
+- test/support/pet3.rb
 - test/support/pet_color_dictionary.en.csv
 - test/support/pet_color_dictionary.es.csv
 - test/support/pets.csv
@@ -418,7 +469,6 @@ test_files:
 - test/test_data_miner.rb
 - test/test_data_miner_run_column_statistic.rb
 - test/test_earth_import.rb
-- test/test_earth_tap.rb
 - test/test_safety.rb
 - test/test_unit_conversion.rb
 has_rdoc: 

data/lib/data_miner/step/tap.rb

@@ -1,167 +0,0 @@
-require 'uri'
-
-class DataMiner
-  class Step
-    # A step that uses https://github.com/ricardochimal/taps to import table structure and data.
-    #
-    # Create these by calling +tap+ inside a +data_miner+ block.
-    #
-    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
-    # @see DataMiner::Script#tap Creating a tap step by calling DataMiner::Script#tap from inside a data miner script
-    class Tap < Step
-      DEFAULT_PORTS = {
-        :mysql => 3306,
-        :mysql2 => 3306,
-        :postgres => 5432
-      }
-      
-      DEFAULT_USERNAMES = {
-        :mysql => 'root',
-        :mysql2 => 'root',
-        :postgres => ''
-      }
-      
-      DEFAULT_PASSWORDS = {}
-      DEFAULT_PASSWORDS.default = ''
-      
-      DEFAULT_HOSTS = {}
-      DEFAULT_HOSTS.default = '127.0.0.1'
-
-      # @private
-      attr_reader :script
-
-      # A description of the tapped data source.
-      # @return [String]
-      attr_reader :description
-
-      # The URL of the tapped data source, including username, password, domain, and port number.
-      # @return [String]
-      attr_reader :source
-
-      # Connection options that will be passed to the +taps pull command+. Defaults to the ActiveRecord connection config, if available.
-      # @return [Hash]
-      attr_reader :database_options
-
-      # Source table name. Defaults to the table name of the model.
-      # @return [String]
-      attr_reader :source_table_name
-
-      # @private
-      def initialize(script, description, source, options = {})
-        options = options.symbolize_keys
-        @script = script
-        @description = description
-        @source = source
-        @source_table_name = options.delete(:source_table_name) || model.table_name
-        @database_options = options.reverse_merge script.model.connection.instance_variable_get(:@config).symbolize_keys
-      end
-      
-      # @private
-      def start
-        [ source_table_name, model.table_name ].each do |possible_obstacle|
-          if connection.table_exists? possible_obstacle
-            connection.drop_table possible_obstacle
-          end
-        end
-        taps_pull
-        if needs_table_rename?
-          connection.rename_table source_table_name, model.table_name
-        end
-        nil
-      end
-      
-      # @return [String] The name of the current database.
-      def database
-        unless database = database_options[:database]
-          raise ::ArgumentError, %{[data_miner] Can't infer database name from options or ActiveRecord config.}
-        end
-        database
-      end
-      
-      # @return [String] The database username.
-      def username
-        database_options[:username] || DEFAULT_USERNAMES[adapter.to_sym]
-      end
-
-      # @return [String] The database password.
-      def password
-        database_options[:password] || DEFAULT_PASSWORDS[adapter.to_sym]
-      end
-
-      # @return [String] The database port number.
-      def port
-        database_options[:port] || DEFAULT_PORTS[adapter.to_sym]
-      end
-
-      # @return [String] The database hostname.
-      def host
-        database_options[:host] || DEFAULT_HOSTS[adapter.to_sym]
-      end
-
-      private
-
-      def connection
-        model.connection
-      end
-      
-      def needs_table_rename?
-        source_table_name != model.table_name
-      end
-      
-      def adapter
-        case connection.adapter_name
-        when /mysql2/i
-          'mysql2'
-        when /mysql/i
-          'mysql'
-        when /postgres/i
-          'postgres'
-        when /sqlite/i
-          'sqlite'
-        end
-      end
-
-      # "user:pass"
-      # "user"
-      # nil
-      def userinfo
-        if username.present?
-          [username, password].select(&:present?).join(':')
-        end
-      end
-      
-      def db_url
-        case adapter
-        when 'sqlite'
-          "sqlite://#{database}"
-        else
-          ::URI::Generic.new(adapter, userinfo, host, port, nil, "/#{database}", nil, nil, nil).to_s
-        end
-      end
-
-      # Note that you probably shouldn't put taps into your Gemfile, because it depends on sequel and other gems that may not compile on Heroku (etc.)
-      #
-      # This class automatically detects if you have Bundler installed, and if so, executes the `taps` binary with a "clean" environment (i.e. one that will not pay attention to the fact that taps is not in your Gemfile)
-      def taps_pull
-        args = [
-          'taps',
-          'pull',
-          db_url,
-          source,
-          '--indexes-first',
-          '--tables',
-          source_table_name
-        ]
-        
-        # https://github.com/carlhuda/bundler/issues/1579
-        if defined?(::Bundler)
-          ::Bundler.with_clean_env do
-            ::Kernel.system args.join(' ')
-          end
-        else
-          ::Kernel.system args.join(' ')
-        end
-      end
-    end
-  end
-end

data/test/test_earth_tap.rb

@@ -1,26 +0,0 @@
-# -*- encoding: utf-8 -*-
-require 'helper'
-init_database
-require 'earth'
-
-# use earth, which has a plethora of real-world data_miner blocks
-Earth.init :locality, :pet, :load_data_miner => false, :apply_schemas => true
-
-DataMiner.run %w{Country Breed}
-
-describe DataMiner do
-  describe "being used by the Earth library's tap steps" do
-    describe "for pets" do
-      it "can pull breed and species" do
-        Breed.find('Golden Retriever').species.must_equal Species.find('dog')
-      end
-    end
-    describe "for localities" do
-      it "can handle non-latin characters" do
-        Country.find('DE').name.must_equal 'Germany'
-        Country.find('AX').name.must_equal 'Åland Islands'
-        Country.find('CI').name.must_equal "Côte d'Ivoire"
-      end
-    end
-  end
-end