data_miner 2.5.2 → 3.0.0.alpha

data/test/test_data_miner.rb

@@ -9,8 +9,6 @@ describe DataMiner do
       Pet.delete_all
       Pet2.delete_all
       Pet3.delete_all
-      DataMiner::Run.delete_all
-      DataMiner::Run::ColumnStatistic.delete_all
     end
     it "it does not depend on mass-assignment" do
       lambda do
@@ -37,18 +35,6 @@ describe DataMiner do
       Pet.run_data_miner!
       Pet.find('Jerry').color_id.must_equal 'brown/black'
     end
-    it "refreshes the dictionary for every run" do
-      Pet.run_data_miner!
-      Pet.find('Jerry').color_id.must_equal 'brown/black'
-      begin
-        FileUtils.mv COLOR_DICTIONARY_ENGLISH, "#{COLOR_DICTIONARY_ENGLISH}.bak"
-        FileUtils.cp COLOR_DICTIONARY_SPANISH, COLOR_DICTIONARY_ENGLISH # oops! somebody swapped in a spanish dictionary
-        Pet.run_data_miner!
-        Pet.find('Jerry').color_id.must_equal 'morron/negro'
-      ensure
-        FileUtils.mv "#{COLOR_DICTIONARY_ENGLISH}.bak", COLOR_DICTIONARY_ENGLISH
-      end
-    end
     it "refreshes the data source for every run" do
       Pet.run_data_miner!
       Pet.find('Jerry').breed_id.must_equal 'Beagle'
@@ -76,44 +62,24 @@ describe DataMiner do
       Pet.find('Amigo').height.must_equal 300.5
       Pet.find('Johnny').height.must_equal 4000.0
     end
-    it "performs unit conversions" do
-      Pet.run_data_miner!
-      Pet.find('Pierre').weight.must_be_close_to 1.9958 # 4.4 pounds in kilograms
-    end
-    it "doesn't convert nil to 0 when converting units" do
+    it "uses blocks to synthesize values" do
       Pet.run_data_miner!
-      Pet.find('Nemo').age.must_be_nil
+      Pet.find('Jerry').emphatic_command_phrase.must_equal 'che!!!!!'
     end
-    it "sets units" do
+    it "runs the result of synthesize through the standard cleaners" do
       Pet.run_data_miner!
-      Pet.find('Pierre').age_units.must_equal 'years'
-      Pet.find('Pierre').weight_units.must_equal 'kilograms'
-      Pet.find('Pierre').height_units.must_equal 'millimetres'
+      Pet.find('Johnny').emphatic_command_phrase.must_equal 'oh ok !!!!!'
     end
     it "always nullifies numeric columns when blank/nil is the input" do
       Pet.run_data_miner!
       Pet.find('Amigo').weight.must_be_nil
     end
-    it "doesn't nullify string columns by default" do
-      Pet.run_data_miner!
-      Pet.find('Amigo').command_phrase.must_equal ''
-      Pet.find('Johnny').command_phrase.must_equal ''
-    end
-    it "nullifies string columns on demand" do
+    it "does nullify blank string columns by default" do
       Pet.run_data_miner!
+      Pet.find('Amigo').command_phrase.must_be_nil
       Pet.find('Jerry').favorite_food.must_equal 'cheese'
       Pet.find('Johnny').favorite_food.must_be_nil
     end
-    it "doesn't set units if the input was blank/null" do
-      Pet.run_data_miner!
-      Pet.find('Amigo').weight.must_be_nil
-      Pet.find('Amigo').weight_units.must_be_nil
-    end
-    it "keeps a row count before and after" do
-      Pet.run_data_miner!
-      Pet.data_miner_runs.first.row_count_before.must_equal 0
-      Pet.data_miner_runs.first.row_count_after.must_equal 5
-    end
     it "can import based on keys other than the primary key" do
       Pet2.run_data_miner!
       Pet2.find('Jerry').breed_id.must_equal 'Beagle-Basset'

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.5.2
-  prerelease: 
+  version: 3.0.0.alpha
+  prerelease: 6
 platform: ruby
 authors:
 - Seamus Abshere
@@ -13,40 +13,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-05 00:00:00.000000000 Z
+date: 2013-07-25 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: aasm
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: active_record_inline_schema
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.6.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.6.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -176,23 +144,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.10.3
 - !ruby/object:Gem::Dependency
-  name: dkastner-alchemist
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: conversions
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -208,7 +160,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: earth
+  name: active_record_inline_schema
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -239,22 +191,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: lock_method
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -384,8 +320,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
-  XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. You can also convert
-  units.
+  XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. Uses Upsert internally
+  for speed.
 email:
 - seamus@abshere.net
 - rossmeissl@gmail.com
@@ -407,30 +343,19 @@ files:
 - lib/data_miner.rb
 - lib/data_miner/active_record_class_methods.rb
 - lib/data_miner/attribute.rb
-- lib/data_miner/dictionary.rb
-- lib/data_miner/run.rb
-- lib/data_miner/run/column_statistic.rb
 - lib/data_miner/script.rb
 - lib/data_miner/step.rb
 - lib/data_miner/step/import.rb
 - lib/data_miner/step/process.rb
 - lib/data_miner/step/sql.rb
-- lib/data_miner/unit_converter.rb
-- lib/data_miner/unit_converter/alchemist.rb
-- lib/data_miner/unit_converter/conversions.rb
 - lib/data_miner/version.rb
-- test/data_miner/step/test_import.rb
 - test/data_miner/step/test_sql.rb
 - test/data_miner/test_attribute.rb
-- test/data_miner/unit_converter/test_alchemist.rb
-- test/data_miner/unit_converter/test_conversions.rb
 - test/helper.rb
 - test/support/breed.rb
 - test/support/breed_by_license_number.csv
 - test/support/breeds.xls
 - test/support/data_miner_with_alchemist.rb
-- test/support/data_miner_with_conversions.rb
-- test/support/data_miner_without_unit_converter.rb
 - test/support/pet.rb
 - test/support/pet2.rb
 - test/support/pet3.rb
@@ -439,10 +364,6 @@ files:
 - test/support/pets.csv
 - test/support/pets_funny.csv
 - test/test_data_miner.rb
-- test/test_data_miner_run_column_statistic.rb
-- test/test_earth_import.rb
-- test/test_safety.rb
-- test/test_unit_conversion.rb
 homepage: https://github.com/seamusabshere/data_miner
 licenses: []
 post_install_message: 
@@ -458,9 +379,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: data_miner
 rubygems_version: 1.8.25
@@ -469,18 +390,13 @@ specification_version: 3
 summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
   XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
 test_files:
-- test/data_miner/step/test_import.rb
 - test/data_miner/step/test_sql.rb
 - test/data_miner/test_attribute.rb
-- test/data_miner/unit_converter/test_alchemist.rb
-- test/data_miner/unit_converter/test_conversions.rb
 - test/helper.rb
 - test/support/breed.rb
 - test/support/breed_by_license_number.csv
 - test/support/breeds.xls
 - test/support/data_miner_with_alchemist.rb
-- test/support/data_miner_with_conversions.rb
-- test/support/data_miner_without_unit_converter.rb
 - test/support/pet.rb
 - test/support/pet2.rb
 - test/support/pet3.rb
@@ -489,8 +405,4 @@ test_files:
 - test/support/pets.csv
 - test/support/pets_funny.csv
 - test/test_data_miner.rb
-- test/test_data_miner_run_column_statistic.rb
-- test/test_earth_import.rb
-- test/test_safety.rb
-- test/test_unit_conversion.rb
 has_rdoc: 

data/lib/data_miner/dictionary.rb

@@ -1,84 +0,0 @@
-require 'remote_table'
-
-class DataMiner
-  # An easy way to translate data before importing it using an intermediate table.
-  class Dictionary
-    DEFAULT_CASE_SENSITIVE = true
-
-    # What field in the dictionary holds the lookup key.
-    #
-    # In other words, the column we scan down to find an entry.
-    #
-    # @return [String]
-    attr_reader :key_name
-
-    # What field in the dictionary holds the final value.
-    #
-    # @return [String]
-    attr_reader :value_name
-
-    # A +sprintf+-style format to be applied.
-    # @return [String]
-    attr_reader :sprintf
-
-    # The URL of the dictionary. It must be a CSV.
-    # @return [String]
-    attr_reader :url
-
-    # Whether to be case-sensitive with lookups. Defaults to false.
-    # @return [TrueClass, FalseClass]
-    attr_reader :case_sensitive
-
-    # @private
-    def initialize(options = {})
-      options = options.symbolize_keys
-      @url = options[:url]
-      @key_name = options[:input].to_s
-      @value_name = options[:output].to_s
-      @sprintf = options[:sprintf]
-      @case_sensitive = options.fetch :case_sensitive, DEFAULT_CASE_SENSITIVE
-      @table_mutex = ::Mutex.new
-    end
-    
-    # Look up a translation for a value.
-    #
-    # @return [nil, String]
-    def lookup(value)
-      normalized_value = normalize_for_comparison value
-      if match = table.detect { |entry| entry[key_name] == normalized_value }
-        match[value_name].to_s
-      end
-    end
-    
-    private
-
-    def table
-      @table || @table_mutex.synchronize do
-        @table ||= ::RemoteTable.new(url).map do |entry|
-          entry[key_name] = normalize_for_comparison entry[key_name]
-          entry
-        end
-      end
-    end
-    
-    def refresh
-      @table = nil
-    end
-
-    def normalize_for_comparison(str)
-      if sprintf
-        if sprintf.end_with?('f')
-          str = str.to_f
-        elsif sprintf.end_with?('d')
-          str = str.to_i
-        end
-        str = sprintf % str
-      end
-      str = DataMiner.compress_whitespace str
-      unless case_sensitive
-        str = DataMiner.downcase str
-      end
-      str
-    end
-  end
-end

data/lib/data_miner/run.rb

@@ -1,144 +0,0 @@
-require 'aasm'
-require 'active_record_inline_schema'
-
-require 'data_miner/run/column_statistic'
-
-class DataMiner
-  # A record of what happened when you ran a data miner script.
-  #
-  # To create the table, use +DataMiner::Run.auto_upgrade!+, possibly in +db/seeds.rb+ or a database migration.
-  class Run < ::ActiveRecord::Base
-    class << self
-      # If a previous run died and you have manually enabled locking, you may find yourself getting +LockMethod::Locked+ exceptions.
-      #
-      # @note Starting in 2.1.0, runs are no longer locked by default. This method remains in case you want to re-apply locking.
-      # 
-      # @param [String] model_names What locks to clear.
-      #
-      # @return [nil]
-      #
-      # @example Re-enable locking (since it was turned off by default in 2.1.0)
-      #   require 'data_miner'
-      #   require 'lock_method'
-      #   DataMiner::Run.lock_method :start
-      def clear_locks(model_names = DataMiner.model_names)
-        return unless defined?(::LockMethod)
-        model_names.each do |model_name|
-          dummy = new
-          dummy.model_name = model_name
-          dummy.lock_method_clear :start
-        end
-        nil
-      end
-    end
-
-    # Raise this exception to skip the current run without causing it to fail.
-    #
-    # @example Avoid running certain data miner scripts too often (because they take too long).
-    #   class FlightSegment < ActiveRecord::Base
-    #     data_miner do
-    #       [...]
-    #       process "don't run this more than once an hour" do
-    #         if (last_ran_at = data_miner_runs.first(:order => 'created_at DESC').try(:created_at)) and (Time.now.utc - last_ran_at) < 3600
-    #           raise DataMiner::Run::Skip
-    #         end
-    #       end
-    #       [...]
-    #     end
-    #   end
-    class Skip < ::Exception
-    end
-
-    INITIAL_STATE = :limbo
-
-    self.table_name = 'data_miner_runs'
-
-    col :model_name
-    col :aasm_state
-    col :created_at, :type => :datetime
-    col :stopped_at, :type => :datetime
-    col :updated_at, :type => :datetime
-    col :error, :type => :text
-    col :row_count_before, :type => :integer
-    col :row_count_after, :type => :integer
-    add_index :model_name
-    add_index :aasm_state
-
-    validates_presence_of :model_name
-
-    has_many :column_statistics, :class_name => 'DataMiner::Run::ColumnStatistic', :order => 'id ASC'
-
-    include ::AASM
-    aasm_initial_state INITIAL_STATE
-    aasm_state :limbo
-    aasm_state :skipped
-    aasm_state :succeeded
-    aasm_state :failed
-    aasm_event(:succeed) { transitions :from => :limbo, :to => :succeeded }
-    aasm_event(:skip)    { transitions :from => :limbo, :to => :skipped }
-    aasm_event(:fail)    { transitions :from => :limbo, :to => :failed }
-
-    # @private
-    def start
-      model = model_name.constantize
-      if model.table_exists?
-        self.row_count_before = model.count
-      end
-      save!
-      if DataMiner.per_column_statistics?
-        ColumnStatistic.take self
-      end
-      begin
-        catch :data_miner_succeed do
-          yield
-        end
-        succeed!
-      rescue Skip
-        skip!
-      rescue
-        self.error = "#{$!.message}\n#{$!.backtrace.join("\n")}"
-        fail!
-        raise $!
-      ensure
-        if model.connection.respond_to?(:schema_cache)
-          model.connection.schema_cache.clear!
-        end
-        model.reset_column_information
-        if model.table_exists?
-          self.row_count_after = model.count
-          if DataMiner.per_column_statistics?
-            ColumnStatistic.take self
-          end
-        end
-        self.stopped_at = ::Time.now.utc
-        save!
-        DataMiner.logger.info %{[data_miner] #{model_name} #{aasm_current_state.to_s.upcase} (#{(stopped_at-created_at).round(2)}s)}
-      end
-      self
-    end
-
-    # Get the column statistics for a particular column before this run started.
-    #
-    # @param [String] column_name The column you want to know about.
-    #
-    # @return [ColumnStatistic]
-    def initial_column_statistics(column_name)
-      column_statistics.where(:column_name => column_name.to_s).first
-    end
-
-    # Get the column statistics for a particular column after this run finished.
-    #
-    # @param [String] column_name The column you want to know about.
-    #
-    # @return [ColumnStatistic]
-    def final_column_statistics(column_name)
-      column_statistics.where(:column_name => column_name.to_s).last
-    end
-
-    # @private
-    def as_lock
-      database_name = Run.connection.instance_variable_get(:@config).try(:[], :database)
-      [database_name, model_name]
-    end
-  end
-end