data_miner 3.0.0.alpha → 3.0.0.beta

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/CHANGELOG

@@ -1,3 +1,15 @@
+3.0.0.beta / 2013-07-26
+
+* Enhancements
+
+  * test steps with after: and every: options, referring to row counts in the previous step
+  * :limit option for import steps - oh yeah
+  * :random_skip option - goes nicely with :limit if you're debugging
+
+* Bug fixes
+
+  * Make sure to stringify data_miner(:append) option
+
 3.0.0.alpha / 2013-07-24
 
 * breaking changes

data/data_miner.gemspec

@@ -25,7 +25,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'posix-spawn'
   s.add_runtime_dependency 'unix_utils'
   s.add_runtime_dependency 'roo', '>=1.10.3'
-
+  s.add_runtime_dependency 'rspec-expectations'
+  
+  s.add_development_dependency 'rspec'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'active_record_inline_schema'
   s.add_development_dependency 'fuzzy_match'

data/lib/data_miner.rb

@@ -21,6 +21,7 @@ require 'data_miner/step'
 require 'data_miner/step/import'
 require 'data_miner/step/process'
 require 'data_miner/step/sql'
+require 'data_miner/step/test'
 
 # A singleton class that holds global configuration for data mining.
 #

data/lib/data_miner/active_record_class_methods.rb

@@ -91,6 +91,7 @@ class DataMiner
     #
     # @return [nil]
     def data_miner(options = {}, &blk)
+      options = options.stringify_keys
       unless options['append']
         @data_miner_script = nil
       end

data/lib/data_miner/script.rb

@@ -92,6 +92,36 @@ class DataMiner
       append(:process, method_id_or_description, &blk)
     end
 
+    # A step that runs tests and stops the data miner on failures.
+    #
+    # rspec-expectations are automatically included.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Step::Test The actual Test class.
+    #
+    # @param [String] description A description of what the block does.
+    # @param [Hash] settings Settings
+    # @option settings [String] :after After how many rows of the previous step to run the tests.
+    # @yield [] Tests to be run
+    #
+    # @example Tests
+    #   data_miner do
+    #     [...]
+    #     test "make sure something works" do
+    #       expect(Pet.count).to be > 10
+    #     end
+    #     [...]
+    #     test "make sure something works", after: 20 do
+    #       [...]
+    #     end
+    #     [...]
+    #   end
+    #
+    # @return [nil]
+    def test(description, settings = {}, &blk)
+      append(:test, description, settings, &blk)
+    end
+
     # Import rows into your model.
     #
     # As long as...
@@ -217,6 +247,11 @@ class DataMiner
         Script.current_stack.clear
       end
       Script.current_stack << model_name
+      steps.each do |step|
+        steps.each do |other|
+          other.register step
+        end
+      end
       steps.each do |step|
         step.start
         model.reset_column_information

data/lib/data_miner/step.rb

@@ -12,5 +12,21 @@ class DataMiner
     def model
       script.model
     end
+
+    def pos
+      script.steps.index self
+    end
+
+    def register(step)
+      # noop
+    end
+
+    def notify(*args)
+      # noop
+    end
+
+    def target?(*args)
+      false
+    end
   end
 end

data/lib/data_miner/step/import.rb

@@ -20,6 +20,17 @@ class DataMiner
       # @return [String]
       attr_reader :description
 
+      # Max number of rows to import.
+      # @return [Numeric]
+      attr_reader :limit
+
+      # Number from zero to one representing what percentage of rows to skip. Defaults to 0, of course :)
+      # @return [Numeric]
+      attr_reader :random_skip
+
+      # @private
+      attr_reader :listeners
+
       # @private
       def initialize(script, description, settings, &blk)
         settings = settings.stringify_keys
@@ -41,6 +52,9 @@ class DataMiner
         @table_settings = settings.dup
         @table_settings['streaming'] = true
         @table_mutex = ::Mutex.new
+        @limit = settings.fetch 'limit', (1.0/0)
+        @random_skip = settings['random_skip']
+        @listeners = []
         instance_eval(&blk)
       end
 
@@ -94,6 +108,12 @@ class DataMiner
         @validate_query == true
       end
 
+      def register(step)
+        if step.target?(self)
+          listeners << step
+        end
+      end
+
       private
 
       def upsert_enabled?
@@ -110,7 +130,9 @@ class DataMiner
         count = 0
         Upsert.stream(c, model.table_name) do |upsert|
           table.each do |row|
+            next if random_skip and random_skip > Kernel.rand
             $stderr.puts "#{count}..." if count_every > 0 and count % count_every == 0
+            break if count > limit
             count += 1
             selector = @key ? { @key => attributes[@key].read(row) } : { model.primary_key => nil }
             document = attrs_except_key.inject({}) do |memo, attr|
@@ -125,6 +147,9 @@ class DataMiner
               memo
             end
             upsert.row selector, document
+            listeners.select! do |listener|
+              listener.notify self, count
+            end
           end
         end
         model.connection_pool.checkin c
@@ -133,11 +158,16 @@ class DataMiner
       def save_with_find_or_initialize
         count = 0
         table.each do |row|
+          next if random_skip and random_skip > Kernel.rand
           $stderr.puts "#{count}..." if count_every > 0 and count % count_every == 0
+          break if count > limit
           count += 1
           record = @key ? model.send("find_or_initialize_by_#{@key}", attributes[@key].read(row)) : model.new
           attributes.each { |_, attr| attr.set_from_row record, row }
           record.save!
+          listeners.select! do |listener|
+            listener.notify self, count
+          end
         end
       end
 

data/lib/data_miner/step/test.rb

@@ -0,0 +1,77 @@
+require 'rspec-expectations'
+
+class DataMiner
+  class Step
+    # A step that runs tests and stops the data miner on failures.
+    #
+    # Create these by calling +test+ inside a +data_miner+ block.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Script#test Creating a test step by calling DataMiner::Script#test from inside a data miner script
+    class Test < Step
+      include ::RSpec::Expectations
+      include ::RSpec::Matchers
+
+      # A description of what the block does. Doesn't exist when a single class method is specified using a Symbol.
+      # @return [String]
+      attr_reader :description
+
+      # The block of arbitrary code to be run.
+      # @return [Proc]
+      attr_reader :blk
+
+      # After how many rows of the previous step to run the tests.
+      # @return [Numeric]
+      attr_reader :after
+
+      # Every how many rows to run tests
+      # @return [Numeric]
+      attr_reader :every
+
+      alias :block_description :description
+
+      # @private
+      def initialize(script, description, settings, &blk)
+        @script = script
+        @description = description
+        @blk = blk
+        @after = settings[:after]
+        @every = settings[:every]
+        raise "can't do both after and every" if after and every
+      end
+      
+      # @private
+      def start
+        if inline?
+          eval_catching_errors
+        end
+        nil
+      end
+
+      def target?(step)
+        !inline? and (step.pos == pos - 1)
+      end
+
+      def notify(step, count)
+        if count % (after || every) == 0
+          eval_catching_errors
+          !after # if it's an after, return false, so that we stop getting informed
+        else
+          true
+        end
+      end
+
+      private
+
+      def inline?
+        not (after or every)
+      end
+
+      def eval_catching_errors
+        DataMiner::Script.uniq { instance_eval(&blk) }
+      rescue ::RSpec::Expectations::ExpectationNotMetError
+        raise RuntimeError, "FAILED: #{description} (#{$!.inspect})"
+      end
+    end
+  end
+end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '3.0.0.alpha'
+  VERSION = '3.0.0.beta'
 end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+require 'bundler/setup'
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+require_relative '../test/support/database'
+init_database
+init_models
+
+require 'data_miner'

data/spec/test_step_spec.rb

@@ -0,0 +1,134 @@
+require 'spec_helper'
+
+class PetTest1 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :favorite_food
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{Pet::PETS}") do
+      key :name
+      store :favorite_food
+    end
+    test "Jerry likes cheese" do
+      expect(PetTest1.find('Jerry').favorite_food).to eq 'cheese'
+    end
+  end
+end
+
+class PetTest2 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :favorite_food
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{Pet::PETS}") do
+      key :name
+      store :favorite_food
+    end
+    test "Jerry likes veggies" do
+      expect(PetTest2.find('Jerry').favorite_food).to eq 'veggies'
+    end
+  end
+end
+
+class PetTest3 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :favorite_food
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{Pet::PETS}") do
+      key :name
+      store :favorite_food
+    end
+    test "First few have somebody named Pierre", after: 2 do
+      expect(PetTest3.count).to eq 2
+      expect(PetTest3.where(name: 'Pierre').count).to be > 0
+    end
+  end
+end
+
+class PetTest4 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :favorite_food
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{Pet::PETS}") do
+      key :name
+      store :favorite_food
+    end
+    test "First few have somebody named Johnny", after: 2 do
+      expect(PetTest4.count).to eq 2 # that's where we are
+      expect(PetTest4.where(name: 'Johnny').count).to be > 0
+    end
+  end
+end
+
+$pet_test_5_i = 0
+class PetTest5 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :favorite_food
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{Pet::PETS}") do
+      key :name
+      store :favorite_food
+    end
+    test "Everybody has a name", every: 2 do
+      $pet_test_5_i += 1
+      expect(PetTest5.count).to eq $pet_test_5_i*2
+      expect(PetTest5.where(name: nil).count).to be 0
+    end
+  end
+end
+
+$pet_test_6_i = 0
+class PetTest6 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :favorite_food
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{Pet::PETS}") do
+      key :name
+      store :favorite_food
+    end
+    test "Everybody has a favorite food", every: 2 do
+      $pet_test_6_i += 1
+      expect(PetTest6.count).to eq $pet_test_6_i*2
+      expect(PetTest6.where(favorite_food: nil).count).to be 0
+    end
+  end
+end
+
+describe DataMiner::Step::Test do
+  it "keeps going if it passes" do
+    PetTest1.run_data_miner!
+    expect(PetTest1.count).to be > 0
+  end
+  it "stops on failure" do
+    expect { PetTest2.run_data_miner! }.to raise_error(/Jerry.*veggies/)
+    expect(PetTest2.count).to be > 0 # still populated tho
+  end
+  it "can be run in the middle of the previous step" do
+    PetTest3.run_data_miner!
+    expect(PetTest3.count).to be > 0
+  end
+  it "can be run in the middle of the previous step - failing" do
+    expect { PetTest4.run_data_miner! }.to raise_error(/First few.*Johnny/)
+    expect(PetTest4.count).to be 2 # stopped after 2
+  end
+  it "can be run every 2" do
+    PetTest5.run_data_miner!
+    expect($pet_test_5_i).to eq 2
+    expect(PetTest5.count).to be 5
+  end
+  it "can be run every 2 - failing" do
+    expect { PetTest6.run_data_miner! }.to raise_error(/Everybody has a favorite food/)
+    expect($pet_test_6_i).to eq 2
+    expect(PetTest6.count).to be $pet_test_6_i*2
+  end
+end

data/test/helper.rb

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'bundler/setup'
 
 if Bundler.definition.specs['debugger'].first
@@ -12,53 +11,6 @@ require 'minitest/autorun'
 require 'minitest/reporters'
 MiniTest::Reporters.use!
 
-require 'active_record'
-require 'logger'
-ActiveRecord::Base.logger = Logger.new $stderr
-ActiveRecord::Base.logger.level = (ENV['VERBOSE'] == 'true') ? Logger::DEBUG : Logger::INFO
-
-ActiveRecord::Base.mass_assignment_sanitizer = :strict
-
-require 'active_record_inline_schema'
+require_relative 'support/database'
 
 require 'data_miner'
-
-def init_database
-  case ENV['DATABASE']
-  when /postgr/i
-    system %{dropdb test_data_miner}
-    system %{createdb test_data_miner}
-    ActiveRecord::Base.establish_connection(
-      'adapter' => 'postgresql',
-      'encoding' => 'utf8',
-      'database' => 'test_data_miner',
-      'username' => `whoami`.chomp
-    )
-  when /sqlite/i
-    ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
-  else
-    system %{mysql -u root -ppassword -e "DROP DATABASE test_data_miner"}
-    system %{mysql -u root -ppassword -e "CREATE DATABASE test_data_miner CHARSET utf8"}
-    ActiveRecord::Base.establish_connection(
-      'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
-      'encoding' => 'utf8',
-      'database' => 'test_data_miner',
-      'username' => 'root',
-      'password' => 'password'
-    )
-  end
-end
-
-def init_models
-  require 'support/breed'
-  require 'support/pet'
-  require 'support/pet2'
-  require 'support/pet3'
-  Pet.auto_upgrade!
-  Pet2.auto_upgrade!
-  Pet3.auto_upgrade!
-
-  ActiveRecord::Base.descendants.each do |model|
-    model.attr_accessible nil
-  end
-end

data/test/support/database.rb

@@ -0,0 +1,50 @@
+require 'active_record'
+require 'logger'
+ActiveRecord::Base.logger = Logger.new $stderr
+ActiveRecord::Base.logger.level = (ENV['VERBOSE'] == 'true') ? Logger::DEBUG : Logger::INFO
+
+ActiveRecord::Base.mass_assignment_sanitizer = :strict
+
+require 'active_record_inline_schema'
+
+require 'data_miner'
+
+def init_database
+  case ENV['DATABASE']
+  when /postgr/i
+    system %{dropdb test_data_miner}
+    system %{createdb test_data_miner}
+    ActiveRecord::Base.establish_connection(
+      'adapter' => 'postgresql',
+      'encoding' => 'utf8',
+      'database' => 'test_data_miner',
+      'username' => `whoami`.chomp
+    )
+  when /sqlite/i
+    ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+  else
+    system %{mysql -u root -ppassword -e "DROP DATABASE test_data_miner"}
+    system %{mysql -u root -ppassword -e "CREATE DATABASE test_data_miner CHARSET utf8"}
+    ActiveRecord::Base.establish_connection(
+      'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
+      'encoding' => 'utf8',
+      'database' => 'test_data_miner',
+      'username' => 'root',
+      'password' => 'password'
+    )
+  end
+end
+
+def init_models
+  require_relative 'breed'
+  require_relative 'pet'
+  require_relative 'pet2'
+  require_relative 'pet3'
+  Pet.auto_upgrade!
+  Pet2.auto_upgrade!
+  Pet3.auto_upgrade!
+
+  ActiveRecord::Base.descendants.each do |model|
+    model.attr_accessible nil
+  end
+end

data/test/support/pet.rb

@@ -15,9 +15,15 @@ class Pet < ActiveRecord::Base
   col :command_phrase
   col :emphatic_command_phrase
   belongs_to :breed
+
+
+
   data_miner do
+
     process :auto_upgrade!
+    
     process :run_data_miner_on_parent_associations!
+
     import("A list of pets", :url => "file://#{PETS}") do
       key :name
       store :age
@@ -31,5 +37,6 @@ class Pet < ActiveRecord::Base
         (row['command_phrase'] + "!!!!!") if row['command_phrase']
       end
     end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 3.0.0.alpha
+  version: 3.0.0.beta
   prerelease: 6
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-25 00:00:00.000000000 Z
+date: 2013-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -143,6 +143,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.10.3
+- !ruby/object:Gem::Dependency
+  name: rspec-expectations
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -333,6 +365,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - .yardopts
 - CHANGELOG
 - Gemfile
@@ -348,7 +381,10 @@ files:
 - lib/data_miner/step/import.rb
 - lib/data_miner/step/process.rb
 - lib/data_miner/step/sql.rb
+- lib/data_miner/step/test.rb
 - lib/data_miner/version.rb
+- spec/spec_helper.rb
+- spec/test_step_spec.rb
 - test/data_miner/step/test_sql.rb
 - test/data_miner/test_attribute.rb
 - test/helper.rb
@@ -356,6 +392,7 @@ files:
 - test/support/breed_by_license_number.csv
 - test/support/breeds.xls
 - test/support/data_miner_with_alchemist.rb
+- test/support/database.rb
 - test/support/pet.rb
 - test/support/pet2.rb
 - test/support/pet3.rb
@@ -390,6 +427,8 @@ specification_version: 3
 summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
   XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
 test_files:
+- spec/spec_helper.rb
+- spec/test_step_spec.rb
 - test/data_miner/step/test_sql.rb
 - test/data_miner/test_attribute.rb
 - test/helper.rb
@@ -397,6 +436,7 @@ test_files:
 - test/support/breed_by_license_number.csv
 - test/support/breeds.xls
 - test/support/data_miner_with_alchemist.rb
+- test/support/database.rb
 - test/support/pet.rb
 - test/support/pet2.rb
 - test/support/pet3.rb