data_miner 2.5.2 → 3.0.0.alpha

data/CHANGELOG

@@ -1,3 +1,21 @@
+3.0.0.alpha / 2013-07-24
+
+* breaking changes
+
+  * :dictionary is now just a plain Hash-like object (responds to []) - and no longer attempts to refresh between runs  
+  * no more unit conversions
+  * always nullifies blank strings
+  * always overwrites columns whether there was a non-null value there before or not (no more :overwrite option)
+  * don't use synthesize option, just pass a block
+  * synthesized values get the same whitespace compression and stripping as other values
+  * not tested against the Earth library, which has particular requirements and makes testing too complicated
+  * doesn't keep DataMiner::Run or DataMiner::Run::ColumnStatistic records
+  * no more :matcher option
+
+* enhancements
+
+  * if you have a postgres hstore column called "foo", you can do store 'foo.bar'
+
 2.5.2 / 2013-07-05
 
 * Bug fixes

data/Gemfile

@@ -2,8 +2,6 @@ source :rubygems
 
 gemspec
 
-gem 'conversions'
-
 if RUBY_VERSION >= '1.9'
   gem 'unicode_utils'
 end

data/data_miner.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.email       = ["seamus@abshere.net", "rossmeissl@gmail.com", "dkastner@gmail.com", "ijhough@gmail.com", "towerhe@gmail.com"]
   s.homepage    = "https://github.com/seamusabshere/data_miner"
   s.summary     = %{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.}
-  s.description = %q{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. You can also convert units.}
+  s.description = %q{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. Uses Upsert internally for speed.}
 
   s.rubyforge_project = "data_miner"
 
@@ -17,8 +17,6 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
   
-  s.add_runtime_dependency 'aasm'
-  s.add_runtime_dependency 'active_record_inline_schema', '>=0.6.1'
   s.add_runtime_dependency 'activerecord', '> 3'
   s.add_runtime_dependency 'activesupport', '> 3'
   s.add_runtime_dependency 'errata', '>=1.0.1'
@@ -28,11 +26,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'unix_utils'
   s.add_runtime_dependency 'roo', '>=1.10.3'
 
-  s.add_development_dependency 'dkastner-alchemist'
-  s.add_development_dependency 'conversions'
-  s.add_development_dependency 'earth'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'active_record_inline_schema'
   s.add_development_dependency 'fuzzy_match'
-  s.add_development_dependency 'lock_method'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'minitest-reporters'
   s.add_development_dependency 'rake'

data/lib/data_miner.rb

@@ -17,13 +17,10 @@ end
 require 'data_miner/active_record_class_methods'
 require 'data_miner/attribute'
 require 'data_miner/script'
-require 'data_miner/dictionary'
 require 'data_miner/step'
 require 'data_miner/step/import'
 require 'data_miner/step/process'
 require 'data_miner/step/sql'
-require 'data_miner/run'
-require 'data_miner/unit_converter'
 
 # A singleton class that holds global configuration for data mining.
 #
@@ -46,23 +43,6 @@ class DataMiner
     def compress_whitespace(str)
       str.gsub(INNER_SPACE, ONE_SPACE).strip
     end
-
-    # Set the unit converter.
-    #
-    # @note As of 2012-05-30, there are problems with the alchemist gem and the use of the conversions gem instead is recommended.
-    #
-    # @param [Symbol,nil] conversion_library Either +:alchemist+ or +:conversions+
-    #
-    # @return [nil]
-    def unit_converter=(conversion_library)
-      @unit_converter = UnitConverter.load conversion_library
-      nil
-    end
-
-    # @return [#convert,nil] The user-selected unit converter or nil.
-    def unit_converter
-      @unit_converter
-    end
   end
 
   INNER_SPACE = /[ ]+/
@@ -76,13 +56,14 @@ class DataMiner
   #
   # @param [optional, Array<String>] model_names Names of models to be run.
   #
-  # @return [Array<DataMiner::Run>]
+  # @return nil
   def start(model_names = DataMiner.model_names)
     Script.uniq do
       model_names.map do |model_name|
         model_name.constantize.run_data_miner!
       end
     end
+    nil
   end
 
   # legacy
@@ -115,16 +96,6 @@ class DataMiner
     end
   end
 
-  # Whether per-column stats like max, min, average, standard deviation, etc are enabled.
-  def per_column_statistics?
-    @per_column_statistics == true
-  end
-
-  # Turn on or off per-column stats.
-  def per_column_statistics=(boolean)
-    @per_column_statistics = boolean
-  end
-
   class << self
     delegate(*DataMiner.instance_methods(false), :to => :instance)
   end

data/lib/data_miner/active_record_class_methods.rb

@@ -12,16 +12,9 @@ class DataMiner
       end
     end
     
-    # Access to recordkeeping.
-    #
-    # @return [ActiveRecord::Relation] Records of running the data miner script.
-    def data_miner_runs
-      DataMiner::Run.scoped :conditions => { :model_name => name }
-    end
-
     # Run this model's script.
     #
-    # @return [DataMiner::Run]
+    # @return nil
     def run_data_miner!
       data_miner_script.start
     end
@@ -45,13 +38,14 @@ class DataMiner
     #     end
     #   end
     #
-    # @return [Array<DataMiner::Run>]
+    # @return nil
     def run_data_miner_on_parent_associations!
       reflect_on_all_associations(:belongs_to).reject do |assoc|
-        assoc.options[:polymorphic]
+        assoc.options['polymorphic']
       end.map do |non_polymorphic_belongs_to_assoc|
         non_polymorphic_belongs_to_assoc.klass.run_data_miner!
       end
+      nil
     end
     
     # Define a data miner script.
@@ -97,7 +91,7 @@ class DataMiner
     #
     # @return [nil]
     def data_miner(options = {}, &blk)
-      unless options[:append]
+      unless options['append']
         @data_miner_script = nil
       end
       data_miner_script.append_block blk

data/lib/data_miner/attribute.rb

@@ -9,60 +9,32 @@ class DataMiner
       # @private
       def check_options(options)
         errors = []
-        if options[:dictionary].is_a?(Dictionary)
-          errors << %{:dictionary must be a Hash of options}
+        if options.has_key?('dictionary') and not options['dictionary'].respond_to?(:[])
+          errors << %{:dictionary must respond to [], like a Hash does}
         end
         if (invalid_option_keys = options.keys - VALID_OPTIONS).any?
           errors << %{Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence}}
         end
-        units_options = options.select { |k, _| k.to_s.include?('units') }
-        if units_options.any? and DataMiner.unit_converter.nil?
-          errors << %{You must set DataMiner.unit_converter to :alchemist or :conversions if you wish to convert units}
-        end
-        if units_options.any? and VALID_UNIT_DEFINITION_SETS.none? { |d| d.all? { |required_option| options[required_option].present? } }
-          errors << %{#{units_options.inspect} is not a valid set of units definitions. Please supply a set like #{VALID_UNIT_DEFINITION_SETS.map(&:inspect).to_sentence}".}
-        end
         errors
       end
     end
 
     VALID_OPTIONS = [
-      :from_units,
-      :to_units,
-      :static,
-      :dictionary,
-      :matcher,
-      :field_name,
-      :delimiter,
-      :split,
-      :units,
-      :sprintf,
-      :nullify, # deprecated
-      :nullify_blank_strings,
-      :overwrite,
-      :upcase,
-      :units_field_name,
-      :units_field_number,
-      :field_number,
-      :chars,
-      :synthesize,
-    ]
-
-    VALID_UNIT_DEFINITION_SETS = [
-      [:units],                         # no conversion
-      [:from_units, :to_units],         # yes
-      [:units_field_name],              # no
-      [:units_field_name, :to_units],   # yes
-      [:units_field_number],            # no
-      [:units_field_number, :to_units], # yes
+      'static',
+      'dictionary',
+      'field_name',
+      'delimiter',
+      'split',
+      'sprintf',
+      'upcase',
+      'field_number',
+      'chars',
     ]
 
     DEFAULT_SPLIT_PATTERN = /\s+/
     DEFAULT_SPLIT_KEEP = 0
     DEFAULT_DELIMITER = ', '
-    DEFAULT_NULLIFY_BLANK_STRINGS = false
     DEFAULT_UPCASE = false
-    DEFAULT_OVERWRITE = true
 
     # activerecord-3.2.6/lib/active_record/connection_adapters/column.rb
     TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON', 'yes', 'YES', 'y', 'Y']
@@ -72,24 +44,18 @@ class DataMiner
     attr_reader :step
     
     # Local column name.
-    # @return [Symbol]
+    # @return [String]
     attr_reader :name
     
-    # Synthesize a value by passing a proc that will receive +row+ and should return a final value.
+    # The block passed to a store argument. Synthesize a value by passing a proc that will receive +row+ and should return a final value.
+    #
+    # Unlike past versions of DataMiner, you pass this as a block, not with the :synthesize option.
     #
     # +row+ will be a +Hash+ with string keys or (less often) an +Array+
     #
     # @return [Proc]
     attr_reader :synthesize
     
-    # An object that will be sent +#match(row)+ and should return a final value.
-    #
-    # Can be specified as a String which will be constantized into a class and an object of that class instantized with no arguments.
-    #
-    # +row+ will be a +Hash+ with string keys or (less often) an +Array+
-    # @return [Object]
-    attr_reader :matcher
-    
     # Index of where to find the data in the row, starting from zero.
     #
     # If you pass a +Range+, then multiple fields will be joined together.
@@ -97,10 +63,6 @@ class DataMiner
     # @return [Integer, Range]
     attr_reader :field_number
 
-    # Where to find the data in the row.
-    # @return [Symbol]
-    attr_reader :field_name
-
     # A delimiter to be used when joining fields together into a single final value. Used when +:field_number+ is a +Range+. Defaults to DEFAULT_DELIMITER.
     # @return [String]
     attr_reader :delimiter
@@ -117,26 +79,6 @@ class DataMiner
     # @return [Hash]
     attr_reader :split
 
-    # Final units. May invoke a conversion using https://rubygems.org/gems/alchemist
-    #
-    # If a local column named +[name]_units+ exists, it will be populated with this value.
-    #
-    # @return [Symbol]
-    attr_reader :to_units
-
-    # Initial units. May invoke a conversion using a conversion gem like https://rubygems.org/gems/alchemist
-    # Be sure to set DataMiner.unit_converter
-    # @return [Symbol]
-    attr_reader :from_units
-
-    # If every row specifies its own units, index of where to find the units. Zero-based.
-    # @return [Integer]
-    attr_reader :units_field_number
-
-    # If every row specifies its own units, where to find the units.
-    # @return [Symbol]
-    attr_reader :units_field_name
-
     # A +sprintf+-style format to apply.
     # @return [String]
     attr_reader :sprintf
@@ -145,93 +87,81 @@ class DataMiner
     # @return [String,Numeric,TrueClass,FalseClass,Object]
     attr_reader :static
 
-    # Only meaningful for string columns. Whether to store blank input ("    ") as NULL. Defaults to DEFAULT_NULLIFY_BLANK_STRINGS.
-    # @return [TrueClass,FalseClass]
-    attr_reader :nullify_blank_strings
-
     # Whether to upcase value. Defaults to DEFAULT_UPCASE.
     # @return [TrueClass,FalseClass]
     attr_reader :upcase
 
-    # Whether to overwrite the value in a local column if it is not null. Defaults to DEFAULT_OVERWRITE.
-    # @return [TrueClass,FalseClass]
-    attr_reader :overwrite
+    # Dictionary for translating.
+    #
+    # You pass a Hash or something that responds to []
+    #
+    # @return [#[]]
+    attr_reader :dictionary
 
     # @private
-    def initialize(step, name, options = {})
-      options = options.symbolize_keys
+    def initialize(step, name, options = {}, &blk)
+      options = options.stringify_keys
       if (errors = Attribute.check_options(options)).any?
         raise ::ArgumentError, %{[data_miner] Errors on #{inspect}: #{errors.join(';')}}
       end
       @step = step
-      @name = name.to_sym
-      @synthesize = options[:synthesize]
-      if @dictionary_boolean = options.has_key?(:dictionary)
-        @dictionary_settings = options[:dictionary]
-      end
-      @matcher = options[:matcher].is_a?(::String) ? options[:matcher].constantize.new : options[:matcher]
-      if @static_boolean = options.has_key?(:static)
-        @static = options[:static]
+      @name = name.to_s
+      @synthesize = blk if block_given?
+      @dictionary = options['dictionary']
+      if @static_boolean = options.has_key?('static')
+        @static = options['static']
       end
-      @field_number = options[:field_number]
-      @field_name = options.fetch(:field_name, name).to_sym
-      @delimiter = options.fetch :delimiter, DEFAULT_DELIMITER
-      @chars = options[:chars]
-      if split = options[:split]
-        @split = split.symbolize_keys
-      end
-      @nullify_blank_strings = if options.has_key?(:nullify)
-        # deprecated
-        options[:nullify]
-      else
-        options.fetch :nullify_blank_strings, DEFAULT_NULLIFY_BLANK_STRINGS
+      @field_number = options['field_number']
+      @field_name_settings = options['field_name']
+      @delimiter = options.fetch 'delimiter', DEFAULT_DELIMITER
+      @chars = options['chars']
+      if split = options['split']
+        @split = split.stringify_keys
       end
-      @upcase = options.fetch :upcase, DEFAULT_UPCASE
-      @from_units = options[:from_units]
-      @to_units = options[:to_units] || options[:units]
-      @sprintf = options[:sprintf]
-      @overwrite = options.fetch :overwrite, DEFAULT_OVERWRITE
-      @units_field_name = options[:units_field_name]
-      @units_field_number = options[:units_field_number]
-      @convert_boolean = (@from_units.present? or (@to_units.present? and (@units_field_name.present? or @units_field_number.present?)))
-      @persist_units_boolean = (@to_units.present? or @units_field_name.present? or @units_field_number.present?)
-      @dictionary_mutex = ::Mutex.new
+      @upcase = options.fetch 'upcase', DEFAULT_UPCASE
+      @sprintf = options['sprintf']
     end
 
-    # Dictionary for translating.
-    #
-    # You pass a +Hash+ of options which is used to initialize a +DataMiner::Dictionary+.
-    #
-    # @return [DataMiner::Dictionary]
-    def dictionary
-      @dictionary || @dictionary_mutex.synchronize do
-        @dictionary ||= Dictionary.new(@dictionary_settings)
+    # @private
+    def hstore_column
+      return @hstore_column if defined?(@hstore_column)
+      @hstore_column = name.split('.', 2)[0]
+    end
+
+    # @private
+    def hstore_key
+      return @hstore_key if defined?(@hstore_key)
+      @hstore_key = name.split('.', 2)[1]
+    end
+
+    # Where to find the data in the row.
+    # @return [String]
+    def field_name
+      return @field_name if defined?(@field_name)
+      @field_name = if @field_name_settings
+        @field_name_settings.to_s
+      elsif hstore?
+        hstore_key
+      else
+        name
       end
     end
 
     # # @private
-    # TODO make sure that nil handling is replicated when using upsert
     def set_from_row(local_record, remote_row)
-      previously_nil = local_record.send(name).nil?
-      currently_nil = false
-      if previously_nil or overwrite
-        new_value = read remote_row
-        local_record.send "#{name}=", new_value
-        currently_nil = new_value.nil?
-      end
-      if not currently_nil and persist_units? and (final_to_units = (to_units || read_units(remote_row)))
-        local_record.send "#{name}_units=", final_to_units
+      new_value = read remote_row
+      if hstore?
+        local_record.send(hstore_column)[hstore_key] = new_value
+      else
+        local_record.send("#{name}=", new_value)
       end
     end
 
     # @private
     def updates(remote_row)
       v = read remote_row
-      if persist_units?
-        v_units = unless v.nil?
-          to_units || read_units(remote_row)
-        end
-        { name => v, "#{name}_units" => v_units }
+      if hstore?
+        { hstore_column => { hstore_key => v } }
       else
         { name => v }
       end
@@ -239,27 +169,23 @@ class DataMiner
 
     # @private
     def read(row)
-      unless column_exists?
-        raise RuntimeError, "[data_miner] Table #{model.table_name} does not have column #{name.inspect}"
-      end
-      if matcher and matcher_output = matcher.match(row)
-        return matcher_output
-      end
-      if synthesize
-        return synthesize.call(row)
+      if not column_exists?
+        raise RuntimeError, "[data_miner] Table #{model.table_name} does not have column #{(hstore? ? hstore_column : name).inspect}"
       end
       value = if static?
         static
+      elsif synthesize
+        synthesize.call(row)
       elsif field_number
         if field_number.is_a?(::Range)
           field_number.map { |n| row[n] }.join(delimiter)
         else
           row[field_number]
         end
-      elsif field_name == :row_hash
+      elsif field_name == 'row_hash'
         row.row_hash
       elsif row.is_a?(::Hash) or row.is_a?(::ActiveSupport::OrderedHash)
-        row[field_name.to_s] # remote_table hash keys are always strings
+        row[field_name] # remote_table hash keys are always strings
       end
       if value.nil?
         return
@@ -296,47 +222,29 @@ class DataMiner
         value = value[chars]
       end
       if split
-        pattern = split.fetch :pattern, DEFAULT_SPLIT_PATTERN
-        keep = split.fetch :keep, DEFAULT_SPLIT_KEEP
+        pattern = split.fetch 'pattern', DEFAULT_SPLIT_PATTERN
+        keep = split.fetch 'keep', DEFAULT_SPLIT_KEEP
         value = value.to_s.split(pattern)[keep].to_s
       end
-      if value.blank? and (not text_column? or nullify_blank_strings)
+      if value.blank? # TODO false is "blank"
         return
       end
       value = DataMiner.compress_whitespace value
       if upcase
         value = DataMiner.upcase value
       end
-      if convert?
-        value = convert_units value, row
-      end
       if sprintf
-        if sprintf.end_with?('f')
-          value = value.to_f
-        elsif sprintf.end_with?('d')
-          value = value.to_i
-        end
-        value = sprintf % value
+        value = sprintf % value.to_f
       end
-      if dictionary?
-        value = dictionary.lookup(value)
+      if dictionary
+        value = dictionary[value]
       end
       value
     end
 
-    # @private
-    def convert_units(value, row)
-      final_from_units = from_units || read_units(row)
-      final_to_units = to_units || read_units(row)
-      unless final_from_units and final_to_units
-        raise RuntimeError, "[data_miner] Missing units: from=#{final_from_units.inspect}, to=#{final_to_units.inspect}"
-      end
-      DataMiner.unit_converter.convert value, final_from_units, final_to_units
-    end
-
-    # @private
-    def refresh
-      @dictionary = nil
+    def hstore?
+      return @hstore_boolean if defined?(@hstore_boolean)
+      @hstore_boolean = name.include?('.')
     end
 
     private
@@ -347,48 +255,42 @@ class DataMiner
 
     def column_exists?
       return @column_exists_boolean if defined?(@column_exists_boolean)
-      @column_exists_boolean = model.column_names.include? name.to_s
+      if hstore?
+        @column_exists_boolean = model.column_names.include? hstore_column
+      else
+        @column_exists_boolean = model.column_names.include? name
+      end
     end
 
     def text_column?
       return @text_column_boolean if defined?(@text_column_boolean)
-      @text_column_boolean = model.columns_hash[name.to_s].text?
+      if hstore?
+        @text_column_boolean = true
+      else
+        @text_column_boolean = model.columns_hash[name].text?
+      end
     end
     
     def number_column?
       return @number_column_boolean if defined?(@number_column_boolean)
-      @number_column_boolean = model.columns_hash[name.to_s].number?
+      if hstore?
+        @number_column_boolean = false
+      else
+        @number_column_boolean = model.columns_hash[name].number?
+      end
     end
 
     def boolean_column?
       return @boolean_column_boolean if defined?(@boolean_column_boolean)
-      @boolean_column_boolean = (model.columns_hash[name.to_s].type == :boolean)
+      if hstore?
+        @boolean_column_boolean = false
+      else
+        @boolean_column_boolean = (model.columns_hash[name].type == :boolean)
+      end
     end
 
     def static?
       @static_boolean
     end
-
-    def dictionary?
-      @dictionary_boolean
-    end
-
-    def convert?
-      @convert_boolean
-    end
-
-    def persist_units?
-      @persist_units_boolean
-    end
-
-    def read_units(row)
-      if units = row[units_field_name || units_field_number]
-        DataMiner.compress_whitespace(units).underscore
-      end
-    end
-
-    def free
-      @dictionary = nil
-    end
   end
 end