data_miner 2.0.1 → 2.0.2

data/lib/data_miner/step/process.rb

@@ -1,34 +1,52 @@
-class DataMiner::Step::Process
-  attr_reader :script
-  attr_reader :method_id
-  attr_reader :description
-  attr_reader :blk
+class DataMiner
+  class Step
+    # A step that executes a single class method on the model or an arbitrary code block.
+    #
+    # Create these by calling +process+ inside a +data_miner+ block.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Script#process
+    class Process < Step
+      # @private
+      attr_reader :script
 
-  alias :block_description :description
+      # The method to be called on the model class.
+      # @return [Symbol]
+      attr_reader :method_id
 
-  def initialize(script, method_id_or_description, ignored_options = {}, &blk)
-    @script = script
-    if block_given?
-      @description = method_id_or_description
-      @blk = blk
-    else
-      @description = method_id_or_description
-      @method_id = method_id_or_description
-    end
-  end
-  
-  def model
-    script.model
-  end
-  
-  def perform
-    DataMiner::Script.uniq do
-      if blk
-        model.instance_eval(&blk)
-      else
-        model.send method_id
+      # A description of what the block does. Doesn't exist when a single class method is specified using a Symbol.
+      # @return [String]
+      attr_reader :description
+
+      # The block of arbitrary code to be run.
+      # @return [Proc]
+      attr_reader :blk
+
+      alias :block_description :description
+
+      # @private
+      def initialize(script, method_id_or_description, ignored_options = {}, &blk)
+        @script = script
+        if block_given?
+          @description = method_id_or_description
+          @blk = blk
+        else
+          @description = method_id_or_description
+          @method_id = method_id_or_description
+        end
+      end
+      
+      # @private
+      def perform
+        DataMiner::Script.uniq do
+          if blk
+            model.instance_eval(&blk)
+          else
+            model.send method_id
+          end
+        end
+        nil
       end
     end
-    nil
   end
 end

data/lib/data_miner/step/tap.rb

@@ -1,134 +1,167 @@
 require 'uri'
-# Note that you probably shouldn't put taps into your Gemfile, because it depends on sequel and other gems that may not compile on Heroku (etc.)
-#
-# This class automatically detects if you have Bundler installed, and if so, executes the `taps` binary with a "clean" environment (i.e. one that will not pay attention to the fact that taps is not in your Gemfile)
-class DataMiner::Step::Tap
-  DEFAULT_PORTS = {
-    :mysql => 3306,
-    :mysql2 => 3306,
-    :postgres => 5432
-  }
-  
-  DEFAULT_USERNAMES = {
-    :mysql => 'root',
-    :mysql2 => 'root',
-    :postgres => ''
-  }
-  
-  DEFAULT_PASSWORDS = {}
-  DEFAULT_PASSWORDS.default = ''
-  
-  DEFAULT_HOSTS = {}
-  DEFAULT_HOSTS.default = '127.0.0.1'
 
-  attr_reader :script
-  attr_reader :description
-  attr_reader :source
-  attr_reader :database_options
-  attr_reader :source_table_name
+class DataMiner
+  class Step
+    # A step that uses https://github.com/ricardochimal/taps to import table structure and data.
+    #
+    # Create these by calling +tap+ inside a +data_miner+ block.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Script#tap
+    class Tap < Step
+      DEFAULT_PORTS = {
+        :mysql => 3306,
+        :mysql2 => 3306,
+        :postgres => 5432
+      }
+      
+      DEFAULT_USERNAMES = {
+        :mysql => 'root',
+        :mysql2 => 'root',
+        :postgres => ''
+      }
+      
+      DEFAULT_PASSWORDS = {}
+      DEFAULT_PASSWORDS.default = ''
+      
+      DEFAULT_HOSTS = {}
+      DEFAULT_HOSTS.default = '127.0.0.1'
 
-  def initialize(script, description, source, options = {})
-    options = options.symbolize_keys
-    @script = script
-    @description = description
-    @source = source
-    @database_options = options.except(:source_table_name).reverse_merge(active_record_config)
-    @source_table_name = options.fetch :source_table_name, model.table_name
-  end
-  
-  def model
-    script.model
-  end
-  
-  def perform
-    [ source_table_name, model.table_name ].each do |possible_obstacle|
-      if connection.table_exists? possible_obstacle
-        connection.drop_table possible_obstacle
+      # @private
+      attr_reader :script
+
+      # A description of the tapped data source.
+      # @return [String]
+      attr_reader :description
+
+      # The URL of the tapped data source, including username, password, domain, and port number.
+      # @return [String]
+      attr_reader :source
+
+      # Connection options that will be passed to the +taps pull command+. Defaults to the ActiveRecord connection config, if available.
+      # @return [Hash]
+      attr_reader :database_options
+
+      # Source table name. Defaults to the table name of the model.
+      # @return [String]
+      attr_reader :source_table_name
+
+      # @private
+      def initialize(script, description, source, options = {})
+        options = options.symbolize_keys
+        @script = script
+        @description = description
+        @source = source
+        @source_table_name = options.delete(:source_table_name) || model.table_name
+        @database_options = options.reverse_merge script.model.connection.instance_variable_get(:@config).symbolize_keys
+      end
+      
+      # @private
+      def perform
+        [ source_table_name, model.table_name ].each do |possible_obstacle|
+          if connection.table_exists? possible_obstacle
+            connection.drop_table possible_obstacle
+          end
+        end
+        taps_pull
+        if needs_table_rename?
+          connection.rename_table source_table_name, model.table_name
+        end
+        nil
+      end
+      
+      # @return [String] The name of the current database.
+      def database
+        unless database = database_options[:database]
+          raise ::ArgumentError, %{[data_miner] Can't infer database name from options or ActiveRecord config.}
+        end
+        database
+      end
+      
+      # @return [String] The database username.
+      def username
+        database_options[:username] || DEFAULT_USERNAMES[adapter.to_sym]
       end
-    end
-    taps_pull
-    if needs_table_rename?
-      connection.rename_table source_table_name, model.table_name
-    end
-    nil
-  end
-  
-  # sabshere 1/25/11 what if there were multiple connections
-  # blockenspiel doesn't like to delegate this to #model
-  def connection
-    ::ActiveRecord::Base.connection
-  end
-  
-  def needs_table_rename?
-    source_table_name != model.table_name
-  end
-  
-  def adapter
-    case connection.adapter_name
-    when /mysql2/i
-      'mysql2'
-    when /mysql/i
-      'mysql'
-    when /postgres/i
-      'postgres'
-    when /sqlite/i
-      'sqlite'
-    end
-  end
 
-  # never optional
-  def database
-    database_options[:database]
-  end
-  
-  %w{ username password port host }.each do |x|
-    module_eval %{
-      def #{x}
-        database_options[:#{x}] || DEFAULT_#{x.upcase}S[adapter.to_sym]
+      # @return [String] The database password.
+      def password
+        database_options[:password] || DEFAULT_PASSWORDS[adapter.to_sym]
       end
-    }
-  end
-  
-  # "user:pass"
-  # "user"
-  # nil
-  def userinfo
-    if username.present?
-      [username, password].select(&:present?).join(':')
-    end
-  end
-  
-  def db_url
-    case adapter
-    when 'sqlite'
-      "sqlite://#{database}"
-    else
-      ::URI::Generic.new(adapter, userinfo, host, port, nil, "/#{database}", nil, nil, nil).to_s
-    end
-  end
 
-  def active_record_config
-    connection.instance_variable_get(:@config).symbolize_keys
-  end
-  
-  def taps_pull
-    args = [
-      'taps',
-      'pull',
-      db_url,
-      source,
-      '--indexes-first',
-      '--tables',
-      source_table_name
-    ]
-    
-    # https://github.com/carlhuda/bundler/issues/1579
-    if defined?(::Bundler)
-      ::Bundler.with_clean_env do
-        ::Kernel.system args.join(' ')
+      # @return [String] The database port number.
+      def port
+        database_options[:port] || DEFAULT_PORTS[adapter.to_sym]
+      end
+
+      # @return [String] The database hostname.
+      def host
+        database_options[:host] || DEFAULT_HOSTS[adapter.to_sym]
+      end
+
+      private
+
+      def connection
+        model.connection
+      end
+      
+      def needs_table_rename?
+        source_table_name != model.table_name
+      end
+      
+      def adapter
+        case connection.adapter_name
+        when /mysql2/i
+          'mysql2'
+        when /mysql/i
+          'mysql'
+        when /postgres/i
+          'postgres'
+        when /sqlite/i
+          'sqlite'
+        end
+      end
+
+      # "user:pass"
+      # "user"
+      # nil
+      def userinfo
+        if username.present?
+          [username, password].select(&:present?).join(':')
+        end
+      end
+      
+      def db_url
+        case adapter
+        when 'sqlite'
+          "sqlite://#{database}"
+        else
+          ::URI::Generic.new(adapter, userinfo, host, port, nil, "/#{database}", nil, nil, nil).to_s
+        end
+      end
+
+      # Note that you probably shouldn't put taps into your Gemfile, because it depends on sequel and other gems that may not compile on Heroku (etc.)
+      #
+      # This class automatically detects if you have Bundler installed, and if so, executes the `taps` binary with a "clean" environment (i.e. one that will not pay attention to the fact that taps is not in your Gemfile)
+      def taps_pull
+        args = [
+          'taps',
+          'pull',
+          db_url,
+          source,
+          '--indexes-first',
+          '--tables',
+          source_table_name
+        ]
+        
+        # https://github.com/carlhuda/bundler/issues/1579
+        if defined?(::Bundler)
+          ::Bundler.with_clean_env do
+            ::Kernel.system args.join(' ')
+          end
+        else
+          ::Kernel.system args.join(' ')
+        end
       end
-    else
-      ::Kernel.system args.join(' ')
     end
   end
 end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.0.1'
+  VERSION = '2.0.2'
 end

data/test/test_safety.rb

@@ -7,36 +7,72 @@ Earth.init :locality, :pet, :load_data_miner => true, :apply_schemas => true
 
 describe DataMiner do
   describe "when being run in a multi-threaded environment" do
+    before do
+      @old_thread_abort_on_exception = Thread.abort_on_exception
+      Thread.abort_on_exception = false
+    end
+
+    after do
+      Thread.abort_on_exception = @old_thread_abort_on_exception
+    end
+
     it "tries not to duplicate data" do
-      begin
-        old_thread_abort_on_exception = Thread.abort_on_exception
-        Thread.abort_on_exception = false
-        Breed.delete_all
-        Breed.run_data_miner!
-        reference_count = Breed.count
-        Breed.delete_all
-        threads = (0..2).map do |i|
-          Thread.new do
-            $stderr.write "Thread #{i} starting\n"
-            Breed.run_data_miner!
-            $stderr.write "Thread #{i} done\n"
-          end
+      Breed.delete_all
+      Breed.run_data_miner!
+      reference_count = Breed.count
+      Breed.delete_all
+      threads = (0..2).map do |i|
+        Thread.new do
+          # $stderr.write "Thread #{i} starting\n"
+          Breed.run_data_miner!
+          # $stderr.write "Thread #{i} done\n"
+        end
+      end
+      exceptions = []
+      threads.each do |t|
+        begin
+          t.join
+        rescue
+          exceptions << $!
         end
-        exceptions = []
-        threads.each do |t|
-          begin
-            t.join
-          rescue
-            exceptions << $!
+      end
+      exceptions.length.must_equal 2
+      exceptions.each do |exception|
+        exception.must_be_kind_of LockMethod::Locked
+      end
+      Breed.count.must_equal reference_count
+    end
+
+    it "allows you to clear locks if necessary" do
+      threads = (0..2).map do |i|
+        Thread.new do
+          # $stderr.write "Thread #{i} starting\n"
+          case i
+          when 0
+            Breed.run_data_miner! 
+          when 1
+            sleep 0.3
+            DataMiner::Run.clear_locks
+            Breed.run_data_miner!
+          when 2
+            # i will hit a lock!
+            sleep 0.6
+            Breed.run_data_miner!
           end
+          # $stderr.write "Thread #{i} done\n"
         end
-        exceptions.length.must_equal 2
-        exceptions.each do |exception|
-          exception.must_be_kind_of LockMethod::Locked
+      end
+      exceptions = []
+      threads.each do |t|
+        begin
+          t.join
+        rescue
+          exceptions << $!
         end
-        Breed.count.must_equal reference_count
-      ensure
-        Thread.abort_on_exception = old_thread_abort_on_exception
+      end
+      exceptions.length.must_equal 1
+      exceptions.each do |exception|
+        exception.must_be_kind_of LockMethod::Locked
       end
     end
   end