data_miner 2.1.2 → 2.2.0

data/CHANGELOG

@@ -1,3 +1,14 @@
+2.2.0 / 2012-06-11
+
+* Breaking changes
+
+  * You must specify DataMiner.units_convert = {:alchemist,:conversions}
+
+* Enhancements
+
+  * Swappable unit conversion libraries [@dkastner]
+  * Intelligent parsing of commas and periods in number fields [@ihough]
+
 2.1.2 / 2012-05-22
 
 * Breaking changes

data/Gemfile

@@ -2,15 +2,8 @@ source :rubygems
 
 gemspec
 
-# development dependencies
-gem 'fuzzy_match'
-gem 'minitest'
-gem 'minitest-reporters'
-gem 'mysql2'
-gem 'rake'
-gem 'yard'
-gem 'earth'
-gem 'lock_method'
+gem 'conversions'
+
 if RUBY_VERSION >= '1.9'
   gem 'unicode_utils'
 end

data/data_miner.gemspec

@@ -17,11 +17,21 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
   
-  s.add_runtime_dependency 'remote_table', '>=1.2.2'
+  s.add_runtime_dependency 'aasm'
+  s.add_runtime_dependency 'active_record_inline_schema'
   s.add_runtime_dependency 'activerecord', '>=2.3.4'
   s.add_runtime_dependency 'activesupport', '>=2.3.4'
-  s.add_runtime_dependency 'conversions', '>=1.4.4'
   s.add_runtime_dependency 'errata', '>=1.0.1'
-  s.add_runtime_dependency 'active_record_inline_schema'
-  s.add_runtime_dependency 'aasm'
+  s.add_runtime_dependency 'remote_table', '>=1.2.2'
+
+  s.add_development_dependency 'dkastner-alchemist'
+  s.add_development_dependency 'conversions'
+  s.add_development_dependency 'earth'
+  s.add_development_dependency 'fuzzy_match'
+  s.add_development_dependency 'lock_method'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'minitest-reporters'
+  s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'yard'
 end

data/lib/data_miner.rb

@@ -23,6 +23,7 @@ require 'data_miner/step/import'
 require 'data_miner/step/tap'
 require 'data_miner/step/process'
 require 'data_miner/run'
+require 'data_miner/unit_converter'
 
 # A singleton class that holds global configuration for data mining.
 #
@@ -45,6 +46,23 @@ class DataMiner
     def compress_whitespace(str)
       str.gsub(INNER_SPACE, ' ').strip
     end
+
+    # Set the unit converter.
+    #
+    # @note As of 2012-05-30, there are problems with the alchemist gem and the use of the conversions gem instead is recommended.
+    #
+    # @param [Symbol,nil] conversion_library Either +:alchemist+ or +:conversions+
+    #
+    # @return [nil]
+    def unit_converter=(conversion_library)
+      @unit_converter = UnitConverter.load conversion_library
+      nil
+    end
+
+    # @return [#convert,nil] The user-selected unit converter or nil.
+    def unit_converter
+      @unit_converter
+    end
   end
 
   INNER_SPACE = /[ ]+/

data/lib/data_miner/attribute.rb

@@ -1,5 +1,3 @@
-require 'conversions'
-
 class DataMiner
   # A mapping between a local model column and a remote data source column.
   #
@@ -17,13 +15,23 @@ class DataMiner
         if (invalid_option_keys = options.keys - VALID_OPTIONS).any?
           errors << %{Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence}}
         end
-        if (units_options = options.select { |k, _| k.to_s.include?('units') }).any? and VALID_UNIT_DEFINITION_SETS.none? { |d| d.all? { |required_option| options[required_option].present? } }
+        units_options = options.select { |k, _| k.to_s.include?('units') }
+        if units_options.any? and DataMiner.unit_converter.nil?
+          errors << %{You must set DataMiner.unit_converter to :alchemist or :conversions if you wish to convert units}
+        end
+        if units_options.any? and VALID_UNIT_DEFINITION_SETS.none? { |d| d.all? { |required_option| options[required_option].present? } }
           errors << %{#{units_options.inspect} is not a valid set of units definitions. Please supply a set like #{VALID_UNIT_DEFINITION_SETS.map(&:inspect).to_sentence}".}
         end
         errors
       end
     end
-
+    
+    def number_column?
+      return @number_column_query[0] if @number_column_query.is_a?(Array)
+      @number_column_query = [model.columns_hash[name.to_s].number?]
+      @number_column_query[0]
+    end
+    
     VALID_OPTIONS = [
       :from_units,
       :to_units,
@@ -47,12 +55,12 @@ class DataMiner
     ]
 
     VALID_UNIT_DEFINITION_SETS = [
-      [:units],
-      [:from_units, :to_units],
-      [:units_field_name],
-      [:units_field_name, :to_units],
-      [:units_field_number],
-      [:units_field_number, :to_units],
+      [:units],                         # no conversion
+      [:from_units, :to_units],         # yes
+      [:units_field_name],              # no
+      [:units_field_name, :to_units],   # yes
+      [:units_field_number],            # no
+      [:units_field_number, :to_units], # yes
     ]
 
     DEFAULT_SPLIT_PATTERN = /\s+/
@@ -111,14 +119,15 @@ class DataMiner
     # @return [Hash]
     attr_reader :split
 
-    # Final units. May invoke a conversion using https://github.com/seamusabshere/conversions
+    # Final units. May invoke a conversion using https://rubygems.org/gems/alchemist
     #
     # If a local column named +[name]_units+ exists, it will be populated with this value.
     #
     # @return [Symbol]
     attr_reader :to_units
 
-    # Initial units. May invoke a conversion using https://github.com/seamusabshere/conversions
+    # Initial units. May invoke a conversion using a conversion gem like https://rubygems.org/gems/alchemist
+    # Be sure to set DataMiner.unit_converter
     # @return [Symbol]
     attr_reader :from_units
 
@@ -186,6 +195,8 @@ class DataMiner
       @overwrite = options.fetch :overwrite, DEFAULT_OVERWRITE
       @units_field_name = options[:units_field_name]
       @units_field_number = options[:units_field_number]
+      @convert_boolean = (@from_units.present? or (@to_units.present? and (@units_field_name.present? or @units_field_number.present?)))
+      @persist_units_boolean = (@to_units.present? or @units_field_name.present? or @units_field_number.present?)
       @dictionary_mutex = ::Mutex.new
     end
 
@@ -211,7 +222,7 @@ class DataMiner
         currently_nil = new_value.nil?
       end
 
-      if not currently_nil and units? and (final_to_units = (to_units || read_units(remote_row)))
+      if not currently_nil and persist_units? and (final_to_units = (to_units || read_units(remote_row)))
         local_record.send "#{name}_units=", final_to_units
       end
     end
@@ -244,6 +255,21 @@ class DataMiner
         return value
       end
       value = value.to_s
+      if number_column?
+        period_position = value.rindex '.'
+        comma_position = value.rindex ','
+        # assume that ',' is a thousands separator and '.' is a decimal point unless we have evidence to the contrary
+        if period_position and comma_position and comma_position > period_position
+          # uncommon euro style 1.000,53
+          value = value.delete('.').gsub(',', '.')
+        elsif comma_position and comma_position > (value.length - 4)
+          # uncommon euro style 1000,53
+          value = value.gsub(',', '.')
+        elsif comma_position
+          # more common 1,000[.00] style - still don't want commas
+          value = value.delete(',')
+        end
+      end
       if chars
         value = value[chars]
       end
@@ -252,7 +278,7 @@ class DataMiner
         keep = split.fetch :keep, DEFAULT_SPLIT_KEEP
         value = value.to_s.split(pattern)[keep].to_s
       end
-      if value.blank? and (not stringlike_column? or nullify_blank_strings)
+      if value.blank? and (not text_column? or nullify_blank_strings)
         return
       end
       value = DataMiner.compress_whitespace value
@@ -260,12 +286,7 @@ class DataMiner
         value = DataMiner.upcase value
       end
       if convert?
-        final_from_units = from_units || read_units(row)
-        final_to_units = to_units || read_units(row)
-        if final_from_units.blank? or final_to_units.blank?
-          raise ::RuntimeError, "[data_miner] Missing units (from=#{final_from_units.inspect}, to=#{final_to_units.inspect}"
-        end
-        value = value.to_f.convert final_from_units, final_to_units
+        value = convert_units value, row
       end
       if sprintf
         if sprintf.end_with?('f')
@@ -281,6 +302,16 @@ class DataMiner
       value
     end
 
+    # @private
+    def convert_units(value, row)
+      final_from_units = from_units || read_units(row)
+      final_to_units = to_units || read_units(row)
+      unless final_from_units and final_to_units
+        raise RuntimeError, "[data_miner] Missing units: from=#{final_from_units.inspect}, to=#{final_to_units.inspect}"
+      end
+      DataMiner.unit_converter.convert value, final_from_units, final_to_units
+    end
+
     # @private
     def refresh
       @dictionary = nil
@@ -292,10 +323,10 @@ class DataMiner
       step.model
     end
 
-    def stringlike_column?
-      return @stringlike_column_query[0] if @stringlike_column_query.is_a?(::Array)
-      @stringlike_column_query = [model.columns_hash[name.to_s].type == :string]
-      @stringlike_column_query[0]
+    def text_column?
+      return @text_column_query[0] if @text_column_query.is_a?(Array)
+      @text_column_query = [model.columns_hash[name.to_s].text?]
+      @text_column_query[0]
     end
 
     def static?
@@ -307,11 +338,11 @@ class DataMiner
     end
 
     def convert?
-      from_units.present? or units_field_name.present? or units_field_number.present?
+      @convert_boolean
     end
 
-    def units?
-      to_units.present? or units_field_name.present? or units_field_number.present?
+    def persist_units?
+      @persist_units_boolean
     end
 
     def read_units(row)

data/lib/data_miner/unit_converter.rb

@@ -0,0 +1,12 @@
+class DataMiner
+  class UnitConverter
+    class << self
+      def load(type)
+        if type
+          require "data_miner/unit_converter/#{type}"
+          const_get(type.to_s.camelcase).new
+        end
+      end
+    end
+  end
+end

data/lib/data_miner/unit_converter/alchemist.rb

@@ -0,0 +1,11 @@
+require 'alchemist'
+
+class DataMiner
+  class UnitConverter
+    class Alchemist < UnitConverter
+      def convert(value, from, to)
+        value.to_f.send(from).to.send(to)
+      end
+    end
+  end
+end

data/lib/data_miner/unit_converter/conversions.rb

@@ -0,0 +1,11 @@
+require 'conversions'
+
+class DataMiner
+  class UnitConverter
+    class Conversions < UnitConverter
+      def convert(value, from, to)
+        value.to_f.convert from, to
+      end
+    end
+  end
+end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.1.2'
+  VERSION = '2.2.0'
 end

data/test/data_miner/test_attribute.rb

@@ -0,0 +1,36 @@
+require 'helper'
+
+describe DataMiner::Attribute do
+  before do
+    DataMiner.unit_converter = :alchemist
+  end
+
+  describe '#convert?' do
+    it 'returns true if from_units is set' do
+      attribute = DataMiner::Attribute.new :foo, 'bar', :from_units => :pounds, :to_units => :kilograms
+      assert attribute.send(:convert?)
+    end
+    it 'returns true if to_units and units_field_name are set' do
+      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_name => 'bar', :to_units => :kilograms
+      assert attribute.send(:convert?)
+    end
+    it 'returns true if to_units and units_field_number are set' do
+      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_number => 3, :to_units => :kilograms
+      assert attribute.send(:convert?)
+    end
+    it 'returns false if units_field_name only is set' do
+      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_name => 'bar'
+      refute attribute.send(:convert?)
+    end
+    it 'returns false if units_field_number only is set' do
+      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_number => 'bar'
+      refute attribute.send(:convert?)
+    end
+    it 'raises if no converter and units are used' do
+      DataMiner.unit_converter = nil
+      lambda {
+        DataMiner::Attribute.new :foo, 'bar', :from_units => :pounds, :to_units => :kilograms
+      }.must_raise ArgumentError, /unit_converter/
+    end
+  end
+end

data/test/data_miner/unit_converter/test_alchemist.rb

@@ -0,0 +1,13 @@
+require 'helper'
+
+describe 'DataMiner::UnitConverter::Alchemist' do
+  before do
+    DataMiner.unit_converter = :alchemist
+  end
+
+  describe '#convert' do
+    it 'converts a value from one unit to another' do
+      DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
+    end
+  end
+end

data/test/data_miner/unit_converter/test_conversions.rb

@@ -0,0 +1,15 @@
+require 'helper'
+
+describe 'DataMiner::UnitConverter::Conversions' do
+  before do
+    #DataMiner.unit_converter = :conversions
+  end
+
+  describe '#convert' do
+    it 'converts a value from one unit to another' do
+      # can't load both alchemist and conversions in same test run
+      # see test/test_unit_conversion for coverage of this adapter
+      #DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
+    end
+  end
+end

data/test/helper.rb

@@ -11,11 +11,6 @@ require 'minitest/reporters'
 MiniTest::Unit.runner = MiniTest::SuiteRunner.new
 MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
 
-cmd = %{mysql -u root -ppassword -e "DROP DATABASE data_miner_test; CREATE DATABASE data_miner_test CHARSET utf8"}
-$stderr.puts "Running `#{cmd}`..."
-system cmd
-$stderr.puts "Done."
-
 require 'active_record'
 require 'logger'
 ActiveRecord::Base.logger = Logger.new $stderr
@@ -31,68 +26,26 @@ ActiveRecord::Base.establish_connection(
 ActiveRecord::Base.mass_assignment_sanitizer = :strict
 
 require 'data_miner'
-DataMiner::Run.auto_upgrade!
-DataMiner::Run::ColumnStatistic.auto_upgrade!
-DataMiner::Run.clear_locks
 
-PETS = File.expand_path('../support/pets.csv', __FILE__)
-PETS_FUNNY = File.expand_path('../support/pets_funny.csv', __FILE__)
-COLOR_DICTIONARY_ENGLISH = File.expand_path('../support/pet_color_dictionary.en.csv', __FILE__)
-COLOR_DICTIONARY_SPANISH = File.expand_path('../support/pet_color_dictionary.es.csv', __FILE__)
-BREEDS = File.expand_path('../support/breeds.xls', __FILE__)
+def init_database(unit_converter = :conversions)
+  cmd = %{mysql -u root -ppassword -e "DROP DATABASE data_miner_test; CREATE DATABASE data_miner_test CHARSET utf8"}
+  $stderr.puts "Running `#{cmd}`..."
+  system cmd
+  $stderr.puts "Done."
 
-class Pet < ActiveRecord::Base
-  self.primary_key = "name"
-  col :name
-  col :breed_id
-  col :color_id
-  col :age, :type => :integer
-  col :age_units
-  col :weight, :type => :float
-  col :weight_units
-  col :height, :type => :integer
-  col :height_units
-  col :favorite_food
-  col :command_phrase
-  belongs_to :breed
-  data_miner do
-    process :auto_upgrade!
-    process :run_data_miner_on_parent_associations!
-    import("A list of pets", :url => "file://#{PETS}") do
-      key :name
-      store :age, :units_field_name => 'age_units'
-      store :breed_id, :field_name => :breed, :nullify_blank_strings => true
-      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
-      store :weight, :from_units => :pounds, :to_units => :kilograms
-      store :favorite_food, :nullify_blank_strings => true
-      store :command_phrase
-      store :height, :units => :centimetres
-    end
-  end
-end
+  DataMiner::Run.auto_upgrade!
+  DataMiner::Run::ColumnStatistic.auto_upgrade!
+  DataMiner::Run.clear_locks
 
-class Breed < ActiveRecord::Base
-  class << self
-    def update_average_age!
-      # make sure pet is populated
-      Pet.run_data_miner!
-      update_all %{breeds.average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
-    end
-  end
-  self.primary_key = "name"
-  col :name
-  col :average_age, :type => :float
-  data_miner do
-    process :auto_upgrade!
-    import("A list of breeds", :url => "file://#{BREEDS}") do
-      key :name, :field_name => 'Breed name'
-    end
-    process :update_average_age!
-  end
+  DataMiner.unit_converter = unit_converter
 end
 
-ActiveRecord::Base.descendants.each do |model|
-  model.attr_accessible nil
-end
+def init_models
+  require 'support/breed'
+  require 'support/pet'
+  Pet.auto_upgrade!
 
-Pet.auto_upgrade!
+  ActiveRecord::Base.descendants.each do |model|
+    model.attr_accessible nil
+  end
+end

data/test/support/breed.rb

@@ -0,0 +1,21 @@
+BREEDS = File.expand_path('../breeds.xls', __FILE__)
+
+class Breed < ActiveRecord::Base
+  class << self
+    def update_average_age!
+      # make sure pet is populated
+      Pet.run_data_miner!
+      update_all %{breeds.average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
+    end
+  end
+  self.primary_key = "name"
+  col :name
+  col :average_age, :type => :float
+  data_miner do
+    process :auto_upgrade!
+    import("A list of breeds", :url => "file://#{BREEDS}") do
+      key :name, :field_name => 'Breed name'
+    end
+    process :update_average_age!
+  end
+end

data/test/support/data_miner_with_alchemist.rb

@@ -0,0 +1,13 @@
+require 'helper'
+
+describe 'DataMiner with Alchemist' do
+  before do
+    init_database(:alchemist)
+    init_models
+    Pet.run_data_miner!
+  end
+
+  it 'converts convertible units' do
+    Pet.find('Pierre').weight.must_be_close_to 4.4.pounds.to.kilograms.to_f
+  end
+end

data/test/support/data_miner_with_conversions.rb

@@ -0,0 +1,16 @@
+require 'helper'
+
+require 'conversions'
+Conversions.register :years, :years, 1
+
+describe 'DataMiner with Conversions' do
+  before do
+    init_database(:conversions)
+    init_models
+    Pet.run_data_miner!
+  end
+
+  it 'converts convertible units' do
+    Pet.find('Pierre').weight.must_be_close_to 4.4.pounds.to(:kilograms)
+  end
+end

data/test/support/data_miner_without_unit_converter.rb

@@ -0,0 +1,51 @@
+require 'helper'
+
+class MyPet < ActiveRecord::Base
+  PETS = File.expand_path('../pets.csv', __FILE__)
+  COLOR_DICTIONARY_ENGLISH = File.expand_path('../pet_color_dictionary.en.csv', __FILE__)
+
+  self.primary_key = "name"
+  col :name
+  col :color_id
+  col :age, :type => :integer
+  col :age_units
+  col :weight, :type => :float
+  col :weight_units
+  col :height, :type => :integer
+  col :height_units
+  col :favorite_food
+  col :command_phrase
+
+  data_miner do
+    process :auto_upgrade!
+    import("A list of pets", :url => "file://#{PETS}") do
+      key :name
+      store :age
+      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
+      store :weight
+      store :favorite_food, :nullify_blank_strings => true
+      store :command_phrase
+      store :height, :units => :centimetres
+    end
+  end
+end
+
+describe 'DataMiner with Conversions' do
+  it 'happens when DataMiner.unit_converter is nil' do
+    DataMiner.unit_converter.must_be_nil
+  end
+
+  it 'converts convertible units' do
+    init_database(nil)
+    MyPet.run_data_miner!
+    MyPet.find('Pierre').weight.must_equal 4.4
+  end
+
+  it 'raises an error if conversions are attempted' do
+    init_database(nil)
+    lambda do
+      init_models
+      Pet.run_data_miner!
+    end.must_raise DataMiner::Attribute::NoConverterSet
+  end
+end

data/test/support/pet.rb

@@ -0,0 +1,34 @@
+PETS = File.expand_path('../pets.csv', __FILE__)
+PETS_FUNNY = File.expand_path('../pets_funny.csv', __FILE__)
+COLOR_DICTIONARY_ENGLISH = File.expand_path('../pet_color_dictionary.en.csv', __FILE__)
+COLOR_DICTIONARY_SPANISH = File.expand_path('../pet_color_dictionary.es.csv', __FILE__)
+
+class Pet < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :breed_id
+  col :color_id
+  col :age, :type => :integer
+  col :age_units
+  col :weight, :type => :float
+  col :weight_units
+  col :height, :type => :float
+  col :height_units
+  col :favorite_food
+  col :command_phrase
+  belongs_to :breed
+  data_miner do
+    process :auto_upgrade!
+    process :run_data_miner_on_parent_associations!
+    import("A list of pets", :url => "file://#{PETS}") do
+      key :name
+      store :age, :units_field_name => 'age_units'
+      store :breed_id, :field_name => :breed, :nullify_blank_strings => true
+      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
+      store :weight, :from_units => :pounds, :to_units => :kilograms
+      store :favorite_food, :nullify_blank_strings => true
+      store :command_phrase
+      store :height, :units => :millimetres
+    end
+  end
+end

data/test/support/pets.csv

@@ -1,6 +1,6 @@
 name,breed,color,age,age_units,weight,height,favorite_food,command_phrase
-Pierre,Tabby,GO,4,years,4.4,30,tomato,"eh"
-Jerry,Beagle,BR/BL,5,years,10,30,cheese,"che"
-Amigo,Spanish Lizarto,GR/BU,17,years,"   ",3,crickets,"    "
-Johnny,Beagle,BR/BL,2,years,20,45,"   ",
+Pierre,Tabby,GO,4,years,4.4,"3.000,5",tomato,"eh"
+Jerry,Beagle,BR/BL,5,years,10,"3,000.0",cheese,"che"
+Amigo,Spanish Lizarto,GR/BU,17,years,"   ","300,5",crickets,"    "
+Johnny,Beagle,BR/BL,2,years,20,"4,000","   ",
 Nemo,,,,,,,,

data/test/test_data_miner.rb

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
+init_database
+init_models
 
 describe DataMiner do
   describe "when used to import example data about pets" do
@@ -65,15 +67,22 @@ describe DataMiner do
       Breed.run_data_miner!
       Breed.find('Beagle').average_age.must_equal((5+2)/2.0)
     end
+    it "properly interprets numbers using comma or period separators" do
+      Pet.run_data_miner!
+      Pet.find('Pierre').height.must_equal 3000.5
+      Pet.find('Jerry').height.must_equal 3000.0
+      Pet.find('Amigo').height.must_equal 300.5
+      Pet.find('Johnny').height.must_equal 4000.0
+    end
     it "performs unit conversions" do
       Pet.run_data_miner!
-      Pet.find('Pierre').weight.must_be_close_to(4.4.pounds.to(:kilograms), 0.00001)
+      Pet.find('Pierre').weight.must_be_close_to 1.9958 # 4.4 pounds in kilograms
     end
     it "sets units" do
       Pet.run_data_miner!
       Pet.find('Pierre').age_units.must_equal 'years'
       Pet.find('Pierre').weight_units.must_equal 'kilograms'
-      Pet.find('Pierre').height_units.must_equal 'centimetres'
+      Pet.find('Pierre').height_units.must_equal 'millimetres'
     end
     it "always nullifies numeric columns when blank/nil is the input" do
       Pet.run_data_miner!

data/test/test_data_miner_run_column_statistic.rb

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
+init_database
+init_models
 
 describe DataMiner::Run::ColumnStatistic do
   describe "when advanced statistics are enabled" do

data/test/test_earth_import.rb

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
+init_database
+init_models
 require 'earth'
 
 # use earth, which has a plethora of real-world data_miner blocks

data/test/test_earth_tap.rb

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
+init_database
+init_models
 require 'earth'
 
 # use earth, which has a plethora of real-world data_miner blocks

data/test/test_safety.rb

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
+init_database
+init_models
 require 'earth'
 
 require 'lock_method'

data/test/test_unit_conversion.rb

@@ -0,0 +1,16 @@
+require 'helper'
+
+describe 'DataMiner unit conversion' do
+  it "blows up if you don't specify a converter" do
+    output = `ruby -I#{File.dirname(__FILE__)} #{File.expand_path('../support/data_miner_without_unit_converter.rb', __FILE__)}`
+    refute $?.success?, output
+  end
+  it 'can convert with alchemist' do
+    output = `ruby -I#{File.dirname(__FILE__)} #{File.expand_path('../support/data_miner_with_alchemist.rb', __FILE__)}`
+    assert $?.success?, output
+  end
+  it 'can convert with conversions' do
+    output = `ruby -I#{File.dirname(__FILE__)} #{File.expand_path('../support/data_miner_with_conversions.rb', __FILE__)}`
+    assert $?.success?, output
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,16 +11,16 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-22 00:00:00.000000000 Z
+date: 2012-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: remote_table
+  name: aasm
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,23 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: active_record_inline_schema
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -62,13 +78,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 2.3.4
 - !ruby/object:Gem::Dependency
-  name: conversions
+  name: errata
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.4.4
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -76,15 +92,15 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.4.4
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
-  name: errata
+  name: remote_table
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 1.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -92,16 +108,16 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
-  name: active_record_inline_schema
+  name: dkastner-alchemist
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -110,14 +126,142 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aasm
+  name: conversions
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: earth
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fuzzy_match
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: lock_method
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -153,9 +297,20 @@ files:
 - lib/data_miner/step/import.rb
 - lib/data_miner/step/process.rb
 - lib/data_miner/step/tap.rb
+- lib/data_miner/unit_converter.rb
+- lib/data_miner/unit_converter/alchemist.rb
+- lib/data_miner/unit_converter/conversions.rb
 - lib/data_miner/version.rb
+- test/data_miner/test_attribute.rb
+- test/data_miner/unit_converter/test_alchemist.rb
+- test/data_miner/unit_converter/test_conversions.rb
 - test/helper.rb
+- test/support/breed.rb
 - test/support/breeds.xls
+- test/support/data_miner_with_alchemist.rb
+- test/support/data_miner_with_conversions.rb
+- test/support/data_miner_without_unit_converter.rb
+- test/support/pet.rb
 - test/support/pet_color_dictionary.en.csv
 - test/support/pet_color_dictionary.es.csv
 - test/support/pets.csv
@@ -165,6 +320,7 @@ files:
 - test/test_earth_import.rb
 - test/test_earth_tap.rb
 - test/test_safety.rb
+- test/test_unit_conversion.rb
 homepage: https://github.com/seamusabshere/data_miner
 licenses: []
 post_install_message: 
@@ -185,14 +341,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: data_miner
-rubygems_version: 1.8.21
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
   XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
 test_files:
+- test/data_miner/test_attribute.rb
+- test/data_miner/unit_converter/test_alchemist.rb
+- test/data_miner/unit_converter/test_conversions.rb
 - test/helper.rb
+- test/support/breed.rb
 - test/support/breeds.xls
+- test/support/data_miner_with_alchemist.rb
+- test/support/data_miner_with_conversions.rb
+- test/support/data_miner_without_unit_converter.rb
+- test/support/pet.rb
 - test/support/pet_color_dictionary.en.csv
 - test/support/pet_color_dictionary.es.csv
 - test/support/pets.csv
@@ -202,4 +366,5 @@ test_files:
 - test/test_earth_import.rb
 - test/test_earth_tap.rb
 - test/test_safety.rb
+- test/test_unit_conversion.rb
 has_rdoc: