activerecord-import 0.17.2 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5482a79b112f9689eee0261baa906028abd8575f
-  data.tar.gz: 2f3e76132e6110a5ed17db800f173b5291ddfe8d
+  metadata.gz: e1c586e3530cab4dd5f201737dd31c9634079ba1
+  data.tar.gz: 1eb5d9c977a304bbff53ce32ffd61a0620426cc0
 SHA512:
-  metadata.gz: 509fe999b0343506f4b1263909d24079394d82d8d3aa16fa5210f35a88b92f40954141949ee7a5f979a81f6105838f0471c5c8cdcfa75a30a5abd38fc96cd1f8
-  data.tar.gz: 517d7de4678ddf9361647f18d08d95c2256e244ae3e6241413abde6b83992e62f4640279c42abdea8bbc710855800840fd2bf941bdca1b447693f2d82cbaa987
+  metadata.gz: cd502299bf521c3f9c6f98a2e4eb55ba8b88f0715401a7b959eedbfd26c0a455524642e3e76dbb429931fa006b519376a9be307daa12dd5b73e14f1c48ada29c
+  data.tar.gz: 6a980c019a2db629a2ba1772b6630119ac8a04da23dc06c091cdb378a75ceeaf37ad17e5104d7c51f2982cf0253ec6485a538327655aea2e0d29a480364549ce

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.2.4
+  - 2.3.3
 
 env:
   global:
@@ -13,6 +13,7 @@ env:
     - AR_VERSION=4.1
     - AR_VERSION=4.2
     - AR_VERSION=5.0
+    - AR_VERSION=5.1
 
 matrix:
   include:
@@ -26,6 +27,9 @@ matrix:
         - bundle exec rake test:jdbcmysql
         - bundle exec rake test:jdbcpostgresql
 
+  allow_failures:
+    - env: AR_VERSION=5.1
+
   fast_finish: true
 
 before_script:
@@ -37,7 +41,7 @@ before_script:
   - cp test/travis/database.yml test/database.yml
 
 addons:
-  postgresql: "9.4"
+  postgresql: "9.5"
   apt:
     sources:
       - travis-ci/sqlite3
@@ -56,4 +60,4 @@ script:
   - bundle exec rake test:sqlite3
   - bundle exec rubocop
 
-sudo: false
+sudo: required

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## Changes in 0.18.0
+
+### New Features
+
+* Uniqueness validation is bypassed when validating models since
+  it cannot be guaranteed if there are duplicates in a batch.
+  Thanks to @jkowens via \#301.
+* Allow for custom timestamp columns. Thanks to @mojidabckuu, @jkowens
+  via \#401.
+ 
+### Fixes
+
+* Fix ActiveRecord 5 issue coercing boolean values when serializing
+  for the database. Thanks to @rjrobinson, @jkowens via \#403.
+
 ## Changes in 0.17.2
 
 ### Fixes

data/Gemfile

@@ -40,6 +40,7 @@ end
 
 platforms :ruby do
   gem "pry-byebug"
+  gem "rb-readline"
 end
 
 version = ENV['AR_VERSION'] || "4.2"

data/README.markdown

@@ -21,26 +21,6 @@ and then the reviews:
 That would be about 4M SQL insert statements vs 3, which results in vastly improved performance. In our case, it converted
 an 18 hour batch process to <2 hrs.
 
-### Rails 5.0
-
-Use activerecord-import 0.11.0 or higher.
-
-### Rails 4.0
-
-Use activerecord-import 0.4.0 or higher.
-
-### Rails 3.2.x up to, but not including 4.0
-
-Use activerecord-import 0.3.0 or higher.
-
-### Rails 3.1.x up to, but not including 3.2
-
-Use the latest in the activerecord-import 0.3.x series.
-
-### Rails 3.0.x up to, but not including 3.1
-
-Use activerecord-import 0.2.11. As of activerecord-import 0.3.0 we are relying on functionality that was introduced in Rails 3.1. Since Rails 3.0.x is no longer a supported version of Rails we have decided to drop support as well.
-
 ### More Information : Usage and Examples in Wiki
 
 For more information on activerecord-import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
@@ -54,7 +34,7 @@ When `ActiveRecord::Import.require_adapter("fake_name")` is called the require w
   require 'activerecord-import/active_record/adapters/fake_name_adapter'
 ```
 
-This allows an external gem to dyanmically add an adapter without the need to add any file/code to the core activerecord-import gem.
+This allows an external gem to dynamically add an adapter without the need to add any file/code to the core activerecord-import gem.
 
 ### Load Path Setup
 To understand how rubygems loads code you can reference the following:

data/gemfiles/5.1.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 5.1.0.rc1'

data/lib/activerecord-import/import.rb

@@ -22,6 +22,44 @@ module ActiveRecord::Import #:nodoc:
       super "Missing column for value <#{name}> at index #{index}"
     end
   end
+
+  class Validator
+    def initialize(validators, options = {})
+      @validators = validators
+      @options    = options
+    end
+
+    def valid_model?(model)
+      validation_context = @options[:validate_with_context]
+      validation_context ||= (model.new_record? ? :create : :update)
+
+      current_context = model.send(:validation_context)
+      model.send(:validation_context=, validation_context)
+      model.errors.clear
+
+      @validators.each do |v|
+        if validation_context == v.options.fetch(:on, validation_context)
+          v.validate(model) if validate?(v, model)
+        end
+      end
+
+      model.send(:validation_context=, current_context)
+      model.errors.empty?
+    end
+
+    def validate?(validator, model)
+      evaluate = lambda do |condition|
+        case condition
+        when String then model.instance_eval(condition)
+        when Symbol then model.send(condition)
+        when Proc then model.instance_eval(&condition)
+        end
+      end
+
+      Array(validator.options[:if]).map(&evaluate).compact.all? &&
+        !Array(validator.options[:unless]).map(&evaluate).compact.any?
+    end
+  end
 end
 
 class ActiveRecord::Associations::CollectionProxy
@@ -97,19 +135,6 @@ end
 
 class ActiveRecord::Base
   class << self
-    # use tz as set in ActiveRecord::Base
-    tproc = lambda do
-      ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
-    end
-
-    AREXT_RAILS_COLUMNS = {
-      create: { "created_on" => tproc,
-                "created_at" => tproc },
-      update: { "updated_on" => tproc,
-                "updated_at" => tproc }
-    }.freeze
-    AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys
-
     # Returns true if the current database connection adapter
     # supports import functionality, otherwise returns false.
     def supports_import?(*args)
@@ -453,10 +478,10 @@ class ActiveRecord::Base
 
       return_obj = if is_validating
         if models
-          import_with_validations( column_names, array_of_attributes, options ) do |failed|
+          import_with_validations( column_names, array_of_attributes, options ) do |validator, failed|
             models.each_with_index do |model, i|
               model = model.dup if options[:recursive]
-              next if model.valid?(options[:validate_with_context])
+              next if validator.valid_model? model
               raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
               array_of_attributes[i] = nil
               failed << model
@@ -487,10 +512,6 @@ class ActiveRecord::Base
       return_obj
     end
 
-    # TODO import_from_table needs to be implemented.
-    def import_from_table( options ) # :nodoc:
-    end
-
     # Imports the passed in +column_names+ and +array_of_attributes+
     # given the passed in +options+ Hash with validations. Returns an
     # object with the methods +failed_instances+ and +num_inserts+.
@@ -501,8 +522,11 @@ class ActiveRecord::Base
     def import_with_validations( column_names, array_of_attributes, options = {} )
       failed_instances = []
 
+      validators = self.validators.reject { |v| v.is_a? ActiveRecord::Validations::UniquenessValidator }
+      validator = ActiveRecord::Import::Validator.new(validators, options)
+
       if block_given?
-        yield failed_instances
+        yield validator, failed_instances
       else
         # create instances for each of our column/value sets
         arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
@@ -512,7 +536,7 @@ class ActiveRecord::Base
         model = new
         arr.each_with_index do |hsh, i|
           hsh.each_pair { |k, v| model[k] = v }
-          next if model.valid?(options[:validate_with_context])
+          next if validator.valid_model? model
           raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
           array_of_attributes[i] = nil
           failure = model.dup
@@ -671,7 +695,6 @@ class ActiveRecord::Base
       # connection and type_caster get called a *lot* in this high intensity loop.
       # Reuse the same ones w/in the loop, otherwise they would keep being re-retreived (= lots of time for large imports)
       connection_memo = connection
-      type_caster_memo = type_caster if respond_to?(:type_caster)
 
       array_of_attributes.map do |arr|
         my_values = arr.each_with_index.map do |val, j|
@@ -681,11 +704,13 @@ class ActiveRecord::Base
           if val.nil? && column.name == primary_key && !sequence_name.blank?
             connection_memo.next_value_for_sequence(sequence_name)
           elsif column
-            if defined?(type_caster_memo) && type_caster_memo.respond_to?(:type_cast_for_database) # Rails 5.0 and higher
-              connection_memo.quote(type_caster_memo.type_cast_for_database(column.name, val))
-            elsif column.respond_to?(:type_cast_from_user)                                         # Rails 4.2 and higher
+            if respond_to?(:type_caster)                                         # Rails 5.0 and higher
+              type = type_for_attribute(column.name)
+              val = type.type == :boolean ? type.cast(val) : type.serialize(val)
+              connection_memo.quote(val)
+            elsif column.respond_to?(:type_cast_from_user)                       # Rails 4.2
               connection_memo.quote(column.type_cast_from_user(val), column)
-            else                                                                                   # Rails 3.2, 4.0 and 4.1
+            else                                                                 # Rails 3.2, 4.0 and 4.1
               if serialized_attributes.include?(column.name)
                 val = serialized_attributes[column.name].dump(val)
               end
@@ -698,39 +723,38 @@ class ActiveRecord::Base
     end
 
     def add_special_rails_stamps( column_names, array_of_attributes, options )
-      timestamps = {}
+      timestamp_columns = {}
+      timestamps        = {}
 
-      AREXT_RAILS_COLUMNS[:create].each_pair do |key, blk|
-        next unless self.column_names.include?(key)
-        value = blk.call
-        timestamps[key] = value
-
-        index = column_names.index(key) || column_names.index(key.to_sym)
-        if index
-          # replace every instance of the array of attributes with our value
-          array_of_attributes.each { |arr| arr[index] = value if arr[index].nil? }
-        else
-          column_names << key
-          array_of_attributes.each { |arr| arr << value }
-        end
+      if respond_to?(:all_timestamp_attributes_in_model, true) # Rails 5.1 and higher
+        timestamp_columns[:create] = timestamp_attributes_for_create_in_model
+        timestamp_columns[:update] = timestamp_attributes_for_update_in_model
+      else
+        instance = new
+        timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
+        timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
       end
 
-      AREXT_RAILS_COLUMNS[:update].each_pair do |key, blk|
-        next unless self.column_names.include?(key)
-        value = blk.call
-        timestamps[key] = value
-
-        index = column_names.index(key) || column_names.index(key.to_sym)
-        if index
-          # replace every instance of the array of attributes with our value
-          array_of_attributes.each { |arr| arr[index] = value }
-        else
-          column_names << key
-          array_of_attributes.each { |arr| arr << value }
-        end
+      # use tz as set in ActiveRecord::Base
+      timestamp = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
+
+      [:create, :update].each do |action|
+        timestamp_columns[action].each do |column|
+          column = column.to_s
+          timestamps[column] = timestamp
+
+          index = column_names.index(column) || column_names.index(column.to_sym)
+          if index
+            # replace every instance of the array of attributes with our value
+            array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil? || action == :update }
+          else
+            column_names << column
+            array_of_attributes.each { |arr| arr << timestamp }
+          end
 
-        if supports_on_duplicate_key_update?
-          connection.add_column_for_on_duplicate_key_update(key, options)
+          if supports_on_duplicate_key_update? && action == :update
+            connection.add_column_for_on_duplicate_key_update(column, options)
+          end
         end
       end
 

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "0.17.2".freeze
+    VERSION = "0.18.0".freeze
   end
 end

data/test/import_test.rb

@@ -88,21 +88,23 @@ describe "#import" do
     end
   end
 
-  describe "with composite primary keys" do
-    it "should import models successfully" do
-      tags = [Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')]
+  unless ENV["SKIP_COMPOSITE_PK"]
+    describe "with composite primary keys" do
+      it "should import models successfully" do
+        tags = [Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')]
 
-      assert_difference "Tag.count", +1 do
-        Tag.import tags
+        assert_difference "Tag.count", +1 do
+          Tag.import tags
+        end
       end
-    end
 
-    it "should import array of values successfully" do
-      columns = [:tag_id, :publisher_id, :tag]
-      values = [[1, 1, 'Mystery'], [2, 1, 'Science']]
+      it "should import array of values successfully" do
+        columns = [:tag_id, :publisher_id, :tag]
+        values = [[1, 1, 'Mystery'], [2, 1, 'Science']]
 
-      assert_difference "Tag.count", +2 do
-        Tag.import columns, values, validate: false
+        assert_difference "Tag.count", +2 do
+          Tag.import columns, values, validate: false
+        end
       end
     end
   end
@@ -159,6 +161,13 @@ describe "#import" do
         end
       end
 
+      it "should ignore uniqueness validators" do
+        Topic.import columns, valid_values, validate: true
+        assert_difference "Topic.count", +2 do
+          Topic.import columns, valid_values, validate: true
+        end
+      end
+
       it "should not import invalid data" do
         assert_no_difference "Topic.count" do
           Topic.import columns, invalid_values, validate: true
@@ -589,6 +598,30 @@ describe "#import" do
     end
   end
 
+  context 'importing arrays of values with boolean fields' do
+    let(:columns) { [:author_name, :title, :for_sale] }
+
+    it 'should be able to coerce integers as boolean fields' do
+      Book.delete_all if Book.count > 0
+      values = [['Author #1', 'Book #1', 0], ['Author #2', 'Book #2', 1]]
+      assert_difference "Book.count", +2 do
+        Book.import columns, values
+      end
+      assert_equal false, Book.first.for_sale
+      assert_equal true, Book.last.for_sale
+    end
+
+    it 'should be able to coerce strings as boolean fields' do
+      Book.delete_all if Book.count > 0
+      values = [['Author #1', 'Book #1', 'false'], ['Author #2', 'Book #2', 'true']]
+      assert_difference "Book.count", +2 do
+        Book.import columns, values
+      end
+      assert_equal false, Book.first.for_sale
+      assert_equal true, Book.last.for_sale
+    end
+  end
+
   describe "importing when model has default_scope" do
     it "doesn't import the default scope values" do
       assert_difference "Widget.unscoped.count", +2 do

data/test/models/tag.rb

@@ -1,4 +1,4 @@
 class Tag < ActiveRecord::Base
-  self.primary_keys = :tag_id, :publisher_id
+  self.primary_keys = :tag_id, :publisher_id unless ENV["SKIP_COMPOSITE_PK"]
   has_many :books, inverse_of: :tag
 end

data/test/models/topic.rb

@@ -1,6 +1,7 @@
 class Topic < ActiveRecord::Base
   validates_presence_of :author_name
   validates :title, numericality: { only_integer: true }, on: :context_test
+  validates :title, uniqueness: true
 
   has_many :books, inverse_of: :topic
   belongs_to :parent, class_name: "Topic"

data/test/schema/generic_schema.rb

@@ -161,12 +161,14 @@ ActiveRecord::Schema.define do
 
   add_index :cars, :Name, unique: true
 
-  execute %(
-  CREATE TABLE IF NOT EXISTS tags (
-        tag_id    INT NOT NULL,
-        publisher_id INT NOT NULL,
-        tag       VARCHAR(50),
-        PRIMARY KEY (tag_id, publisher_id)
-    );
-  ).split.join(' ').strip
+  unless ENV["SKIP_COMPOSITE_PK"]
+    execute %(
+    CREATE TABLE IF NOT EXISTS tags (
+          tag_id    INT NOT NULL,
+          publisher_id INT NOT NULL,
+          tag       VARCHAR(50),
+          PRIMARY KEY (tag_id, publisher_id)
+      );
+    ).split.join(' ').strip
+  end
 end

data/test/schema/postgresql_schema.rb

@@ -1,5 +1,6 @@
 ActiveRecord::Schema.define do
   execute('CREATE extension IF NOT EXISTS "hstore";')
+  execute('CREATE extension IF NOT EXISTS "pgcrypto";')
   execute('CREATE extension IF NOT EXISTS "uuid-ossp";')
 
   create_table :vendors, id: :uuid, force: :cascade do |t|

data/test/support/postgresql/import_examples.rb

@@ -176,7 +176,7 @@ def should_support_postgresql_upsert_functionality
       # would be associated with the wrong parent.
       it ":on_duplicate_key_ignore is ignored" do
         assert_raise ActiveRecord::RecordNotUnique do
-          Topic.import mixed_topics, recursive: true, on_duplicate_key_ignore: true
+          Topic.import mixed_topics, recursive: true, on_duplicate_key_ignore: true, validate: false
         end
       end
     end

data/test/support/shared_examples/on_duplicate_key_ignore.rb

@@ -12,15 +12,17 @@ def should_support_on_duplicate_key_ignore
         end
       end
 
-      context "with composite primary keys" do
-        it "should import array of values successfully" do
-          columns = [:tag_id, :publisher_id, :tag]
-          values = [[1, 1, 'Mystery'], [1, 1, 'Science']]
+      unless ENV["SKIP_COMPOSITE_PK"]
+        context "with composite primary keys" do
+          it "should import array of values successfully" do
+            columns = [:tag_id, :publisher_id, :tag]
+            values = [[1, 1, 'Mystery'], [1, 1, 'Science']]
 
-          assert_difference "Tag.count", +1 do
-            Tag.import columns, values, on_duplicate_key_ignore: true, validate: false
+            assert_difference "Tag.count", +1 do
+              Tag.import columns, values, on_duplicate_key_ignore: true, validate: false
+            end
+            assert_equal 'Mystery', Tag.first.tag
           end
-          assert_equal 'Mystery', Tag.first.tag
         end
       end
     end

data/test/support/shared_examples/on_duplicate_key_update.rb

@@ -77,15 +77,17 @@ def should_support_basic_on_duplicate_key_update
         end
       end
 
-      context "with composite primary keys" do
-        it "should import array of values successfully" do
-          columns = [:tag_id, :publisher_id, :tag]
-          Tag.import columns, [[1, 1, 'Mystery']], validate: false
-
-          assert_difference "Tag.count", +0 do
-            Tag.import columns, [[1, 1, 'Science']], on_duplicate_key_update: [:tag], validate: false
+      unless ENV["SKIP_COMPOSITE_PK"]
+        context "with composite primary keys" do
+          it "should import array of values successfully" do
+            columns = [:tag_id, :publisher_id, :tag]
+            Tag.import columns, [[1, 1, 'Mystery']], validate: false
+
+            assert_difference "Tag.count", +0 do
+              Tag.import columns, [[1, 1, 'Science']], on_duplicate_key_update: [:tag], validate: false
+            end
+            assert_equal 'Science', Tag.first.tag
           end
-          assert_equal 'Science', Tag.first.tag
         end
       end
     end

data/test/support/shared_examples/recursive_import.rb

@@ -102,18 +102,20 @@ def should_support_recursive_import
       end
     end
 
-    describe "with composite primary keys" do
-      it "should import models and set id" do
-        tags = []
-        tags << Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')
-        tags << Tag.new(tag_id: 2, publisher_id: 1, tag: 'Science')
+    unless ENV["SKIP_COMPOSITE_PK"]
+      describe "with composite primary keys" do
+        it "should import models and set id" do
+          tags = []
+          tags << Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')
+          tags << Tag.new(tag_id: 2, publisher_id: 1, tag: 'Science')
 
-        assert_difference "Tag.count", +2 do
-          Tag.import tags
-        end
+          assert_difference "Tag.count", +2 do
+            Tag.import tags
+          end
 
-        assert_equal 1, tags[0].tag_id
-        assert_equal 2, tags[1].tag_id
+          assert_equal 1, tags[0].tag_id
+          assert_equal 2, tags[1].tag_id
+        end
       end
     end
 

data/test/test_helper.rb

@@ -26,7 +26,12 @@ end
 
 require 'timecop'
 require 'chronic'
-require 'composite_primary_keys'
+
+begin
+  require 'composite_primary_keys'
+rescue LoadError
+  ENV["SKIP_COMPOSITE_PK"] = "true"
+end
 
 require "ruby-debug" if RUBY_VERSION.to_f < 1.9
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 0.17.2
+  version: 0.18.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-14 00:00:00.000000000 Z
+date: 2017-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -73,6 +73,7 @@ files:
 - gemfiles/4.1.gemfile
 - gemfiles/4.2.gemfile
 - gemfiles/5.0.gemfile
+- gemfiles/5.1.gemfile
 - lib/activerecord-import.rb
 - lib/activerecord-import/active_record/adapters/abstract_adapter.rb
 - lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
@@ -175,7 +176,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord