data_miner 2.0.2 → 2.0.3

data/.yardopts

@@ -0,0 +1,2 @@
+--no-private
+--readme README.markdown

data/CHANGELOG

@@ -1,3 +1,15 @@
+2.0.3 / 2012-05-07
+
+* Enhancements
+
+  * Rename :nullify to :nullify_blank_strings to clarify that it only affects textual columns and means "treat blank input as null".
+  * Don't try to set units for a column that is currently nil (thanks @ihough)
+
+* Bug fixes
+
+  * Don't blow up if mass-assignment rules are strict.
+  * Don't accidentally set a numeric column to 0.0 when the input is blank or nil.
+
 2.0.2 / 2012-05-04
 
 * Breaking changes
@@ -8,6 +20,7 @@
 * Enhancements
 
   * Real documentation!
+  * Make it easier to clear locks with DataMiner::Run.clear_locks
   * Replace class-level mutexes with simple Thread.exclusive calls
   * Simplified DataMiner::Dictionary
 

data/lib/data_miner/active_record_class_methods.rb

@@ -62,9 +62,9 @@ class DataMiner
     #
     # @yield [] The block defining the steps.
     #
-    # @see DataMiner::Script#import
-    # @see DataMiner::Script#process
-    # @see DataMiner::Script#tap
+    # @see DataMiner::Script#import Creating an import step by calling DataMiner::Script#import from inside a data miner script
+    # @see DataMiner::Script#process Creating a process step by calling DataMiner::Script#process from inside a data miner script
+    # @see DataMiner::Script#tap Creating a tap step by calling DataMiner::Script#tap from inside a data miner script
     #
     # @example Creating steps
     #   class MyModel < ActiveRecord::Base

data/lib/data_miner/attribute.rb

@@ -4,8 +4,8 @@ class DataMiner
   # A mapping between a local model column and a remote data source column.
   #
   # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
-  # @see DataMiner::Step::Import#store
-  # @see DataMiner::Step::Import#key
+  # @see DataMiner::Step::Import#store Telling an import step to store a column with DataMiner::Step::Import#store
+  # @see DataMiner::Step::Import#key Telling an import step to key on a column with DataMiner::Step::Import#key
   class Attribute
     class << self
       # @private
@@ -35,7 +35,8 @@ class DataMiner
       :split,
       :units,
       :sprintf,
-      :nullify,
+      :nullify, # deprecated
+      :nullify_blank_strings,
       :overwrite,
       :upcase,
       :units_field_name,
@@ -57,7 +58,7 @@ class DataMiner
     DEFAULT_SPLIT_PATTERN = /\s+/
     DEFAULT_SPLIT_KEEP = 0
     DEFAULT_DELIMITER = ', '
-    DEFAULT_NULLIFY = false
+    DEFAULT_NULLIFY_BLANK_STRINGS = false
     DEFAULT_UPCASE = false
     DEFAULT_OVERWRITE = true
 
@@ -137,9 +138,9 @@ class DataMiner
     # @return [String,Numeric,TrueClass,FalseClass,Object]
     attr_reader :static
 
-    # Whether to nullify the value in a local column if it was not previously null. Defaults to DEFAULT_NULLIFY.
+    # Only meaningful for string columns. Whether to store blank input ("    ") as NULL. Defaults to DEFAULT_NULLIFY_BLANK_STRINGS.
     # @return [TrueClass,FalseClass]
-    attr_reader :nullify
+    attr_reader :nullify_blank_strings
 
     # Whether to upcase value. Defaults to DEFAULT_UPCASE.
     # @return [TrueClass,FalseClass]
@@ -156,7 +157,7 @@ class DataMiner
         raise ::ArgumentError, %{[data_miner] Errors on #{inspect}: #{errors.join(';')}}
       end
       @step = step
-      @name = name
+      @name = name.to_sym
       @synthesize = options[:synthesize]
       if @dictionary_boolean = options.has_key?(:dictionary)
         @dictionary_settings = options[:dictionary]
@@ -172,7 +173,12 @@ class DataMiner
       if split = options[:split]
         @split = split.symbolize_keys
       end
-      @nullify = options.fetch :nullify, DEFAULT_NULLIFY
+      @nullify_blank_strings = if options.has_key?(:nullify)
+        # deprecated
+        options[:nullify]
+      else
+        options.fetch :nullify_blank_strings, DEFAULT_NULLIFY_BLANK_STRINGS
+      end
       @upcase = options.fetch :upcase, DEFAULT_UPCASE
       @from_units = options[:from_units]
       @to_units = options[:to_units] || options[:units]
@@ -196,10 +202,16 @@ class DataMiner
 
     # @private
     def set_from_row(local_record, remote_row)
-      if overwrite or local_record.send(name).nil?
-        local_record.send "#{name}=", read(remote_row)
+      previously_nil = local_record.send(name).nil?
+      currently_nil = false
+
+      if previously_nil or overwrite
+        new_value = read remote_row
+        local_record.send "#{name}=", new_value
+        currently_nil = new_value.nil?
       end
-      if units? and ((final_to_units = (to_units || read_units(remote_row))) or nullify)
+
+      if not currently_nil and units? and (final_to_units = (to_units || read_units(remote_row)))
         local_record.send "#{name}_units=", final_to_units
       end
     end
@@ -240,10 +252,10 @@ class DataMiner
         keep = split.fetch :keep, DEFAULT_SPLIT_KEEP
         value = value.to_s.split(pattern)[keep].to_s
       end
-      value = DataMiner.compress_whitespace value
-      if nullify and value.blank?
+      if value.blank? and (not stringlike_column? or nullify_blank_strings)
         return
       end
+      value = DataMiner.compress_whitespace value
       if upcase
         value = DataMiner.upcase value
       end
@@ -280,6 +292,12 @@ class DataMiner
       step.model
     end
 
+    def stringlike_column?
+      return @stringlike_column_query[0] if @stringlike_column_query.is_a?(::Array)
+      @stringlike_column_query = [model.columns_hash[name.to_s].type == :string]
+      @stringlike_column_query[0]
+    end
+
     def static?
       @static_boolean
     end

data/lib/data_miner/run.rb

@@ -20,6 +20,13 @@ class DataMiner
         end
         nil
       end
+
+      # @private
+      def perform(model_name, &blk)
+        run = new
+        run.model_name = model_name
+        run.perform(&blk)
+      end
     end
     # Raise this exception to skip the current run without causing it to fail.
     #

data/lib/data_miner/script.rb

@@ -130,7 +130,7 @@ class DataMiner
     # @yield [] A block defining how to +key+ the import (to make it idempotent) and which columns to +store+.
     #
     # @note Be sure to check out https://github.com/seamusabshere/remote_table and https://github.com/seamusabshere/errata for available +table_and_errata_settings+.
-    # @note There are hundreds of +import+ examples in https://github.com/brighterplanet/earth
+    # @note There are hundreds of +import+ examples in https://github.com/brighterplanet/earth. The {file:README.markdown README} points to a few (at the bottom.)
     # @note We often use string primary keys to make idempotency easier. https://github.com/seamusabshere/active_record_inline_schema supports defining these inline.
     #
     # @example From the README
@@ -213,7 +213,7 @@ class DataMiner
         Script.current_stack.clear
       end
       Script.current_stack << model_name
-      Run.new(:model_name => model_name).perform do
+      Run.perform(model_name) do
         steps.each do |step|
           step.perform
           model.reset_column_information

data/lib/data_miner/step/import.rb

@@ -8,7 +8,8 @@ class DataMiner
     # Create these by calling +import+ inside a +data_miner+ block.
     #
     # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
-    # @see DataMiner::Script#import
+    # @see DataMiner::Script#import Creating an import step by calling DataMiner::Script#import from inside a data miner script
+    # @see DataMiner::Attribute The Attribute class, which maps local columns and remote data fields from within an import step
     class Import < Step
       # The mappings of local columns to remote data source fields.
       # @return [Array<DataMiner::Attribute>]

data/lib/data_miner/step/process.rb

@@ -5,7 +5,7 @@ class DataMiner
     # Create these by calling +process+ inside a +data_miner+ block.
     #
     # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
-    # @see DataMiner::Script#process
+    # @see DataMiner::Script#process Creating a process step by calling DataMiner::Script#process from inside a data miner script
     class Process < Step
       # @private
       attr_reader :script

data/lib/data_miner/step/tap.rb

@@ -7,7 +7,7 @@ class DataMiner
     # Create these by calling +tap+ inside a +data_miner+ block.
     #
     # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
-    # @see DataMiner::Script#tap
+    # @see DataMiner::Script#tap Creating a tap step by calling DataMiner::Script#tap from inside a data miner script
     class Tap < Step
       DEFAULT_PORTS = {
         :mysql => 3306,

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.0.2'
+  VERSION = '2.0.3'
 end

data/test/helper.rb

@@ -30,3 +30,4 @@ ActiveRecord::Base.establish_connection(
 
 require 'data_miner'
 DataMiner::Run.auto_upgrade!
+DataMiner::Run.clear_locks

data/test/support/pets.csv

@@ -1,5 +1,5 @@
-name,breed,color,age
-Pierre,Tabby,GO,4
-Jerry,Beagle,BR/BL,5
-Amigo,Spanish Lizarto,GR/BU,17
-Johnny,Beagle,BR/BL,2
+name,breed,color,age,age_units,weight,height,favorite_food,command_phrase
+Pierre,Tabby,GO,4,years,4.4,30,tomato,"eh"
+Jerry,Beagle,BR/BL,5,years,10,30,cheese,"che"
+Amigo,Spanish Lizarto,GR/BU,17,years,"   ",3,crickets,"    "
+Johnny,Beagle,BR/BL,2,years,20,45,"   ",

data/test/test_data_miner.rb

@@ -13,15 +13,26 @@ class Pet < ActiveRecord::Base
   col :breed_id
   col :color_id
   col :age, :type => :integer
+  col :age_units
+  col :weight, :type => :float
+  col :weight_units
+  col :height, :type => :integer
+  col :height_units
+  col :favorite_food
+  col :command_phrase
   belongs_to :breed
   data_miner do
     process :auto_upgrade!
     process :run_data_miner_on_parent_associations!
     import("A list of pets", :url => "file://#{PETS}") do
       key :name
-      store :age
+      store :age, :units_field_name => 'age_units'
       store :breed_id, :field_name => :breed
       store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
+      store :weight, :from_units => :pounds, :to_units => :kilograms
+      store :favorite_food, :nullify_blank_strings => true
+      store :command_phrase
+      store :height, :units => :centimetres
     end
   end
 end
@@ -46,6 +57,11 @@ class Breed < ActiveRecord::Base
   end
 end
 
+ActiveRecord::Base.mass_assignment_sanitizer = :strict
+ActiveRecord::Base.descendants.each do |model|
+  model.attr_accessible nil
+end
+
 Pet.auto_upgrade!
 
 describe DataMiner do
@@ -53,6 +69,17 @@ describe DataMiner do
     before do
       Pet.delete_all
     end
+    it "it does not depend on mass-assignment" do
+      lambda do
+        Pet.new(:name => 'hello').save!
+      end.must_raise(ActiveModel::MassAssignmentSecurity::Error)
+      lambda do
+        Pet.new(:color_id => 'hello').save!
+      end.must_raise(ActiveModel::MassAssignmentSecurity::Error)
+      lambda do
+        Pet.new(:age => 'hello').save!
+      end.must_raise(ActiveModel::MassAssignmentSecurity::Error)
+    end
     it "is idempotent given a key" do
       Pet.run_data_miner!
       first_count = Pet.count
@@ -99,5 +126,34 @@ describe DataMiner do
       Breed.run_data_miner!
       Breed.find('Beagle').average_age.must_equal((5+2)/2.0)
     end
+    it "performs unit conversions" do
+      Pet.run_data_miner!
+      Pet.find('Pierre').weight.must_be_close_to(4.4.pounds.to(:kilograms), 0.00001)
+    end
+    it "sets units" do
+      Pet.run_data_miner!
+      Pet.find('Pierre').age_units.must_equal 'years'
+      Pet.find('Pierre').weight_units.must_equal 'kilograms'
+      Pet.find('Pierre').height_units.must_equal 'centimetres'
+    end
+    it "always nullifies numeric columns when blank/nil is the input" do
+      Pet.run_data_miner!
+      Pet.find('Amigo').weight.must_be_nil
+    end
+    it "doesn't nullify string columns by default" do
+      Pet.run_data_miner!
+      Pet.find('Amigo').command_phrase.must_equal ''
+      Pet.find('Johnny').command_phrase.must_equal ''
+    end
+    it "nullifies string columns on demand" do
+      Pet.run_data_miner!
+      Pet.find('Jerry').favorite_food.must_equal 'cheese'
+      Pet.find('Johnny').favorite_food.must_be_nil
+    end
+    it "doesn't set units if the input was blank/null" do
+      Pet.run_data_miner!
+      Pet.find('Amigo').weight.must_be_nil
+      Pet.find('Amigo').weight_units.must_be_nil
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-04 00:00:00.000000000 Z
+date: 2012-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: remote_table
@@ -151,6 +151,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .yardopts
 - CHANGELOG
 - Gemfile
 - LICENSE