activerecord-import 0.24.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c157e7eabb443b03aa347b06bb39d8e50a685a2
-  data.tar.gz: ec11c5ca04b49a03aa98a19eac8c768ade01e7c0
+  metadata.gz: 26224d6c0d4c23f3edaf1faea68466f18c03fbbc
+  data.tar.gz: 55107f77211b86e3be77b287d224e488dbb280ac
 SHA512:
-  metadata.gz: 3ac6e43987fe5b0309ca4a8346326d4f1c32cf5f30d8b342b260552d26caeb3bdc8bc0f8a89c9ac4671756f94797dc960cb70642eb70da1d8fec35b1a0eced95
-  data.tar.gz: ca600fdd1ec92ec39f34ab89216dee9137abfbf710dec6bd4678a6a7d130afab06a1a90631ab2aec1f3017842291e6e79177b8300d02a7b5498dd6ea42fb2dbf
+  metadata.gz: c0815961332178cf2858b39663f1d9f6ef3d61985cd4cb528cef410714e4bf07c354515b5e06124ad4bb5c7e57e7a179c48d7fb5c5e28886bc9e30c4399e82c9
+  data.tar.gz: df5e8ccd8a658d4981cd26167e09fa8c5e0425c58e4397c48e6f7721221d1de3cc4c6cdf903e3330f07cf6a74b93214cd11cca977f6d75cf6e8b43409807c23b

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.3.5
+  - 2.3.7
 
 env:
   global:
@@ -20,8 +20,6 @@ matrix:
   include:
     - rvm: jruby-9.1.14.0
       env: AR_VERSION=4.2
-      before_install:
-        - gem update --system
 
       script:
         - bundle exec rake test:jdbcsqlite3
@@ -30,23 +28,32 @@ matrix:
 
   fast_finish: true
 
-before_script:
-  - mysql -e 'create database activerecord_import_test;'
-  - psql -c 'create database activerecord_import_test;' -U postgres
-  - psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
-  - psql -c 'create extension if not exists postgis;' -U postgres
-  - psql -c 'create extension if not exists "uuid-ossp";' -U postgres
-  - cp test/travis/database.yml test/database.yml
-
 addons:
   postgresql: "9.5"
   apt:
     sources:
       - travis-ci/sqlite3
+      - mysql-5.7-trusty
     packages:
       - sqlite3
+      - mysql-server
+      - mysql-client
       - postgresql-9.5-postgis-2.3
 
+before_install:
+  - gem update --system
+  - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
+  - sudo mysql_upgrade
+  - sudo service mysql restart
+
+before_script:
+  - mysql -e 'create database activerecord_import_test;'
+  - psql -c 'create database activerecord_import_test;' -U postgres
+  - psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
+  - psql -c 'create extension if not exists postgis;' -U postgres
+  - psql -c 'create extension if not exists "uuid-ossp";' -U postgres
+  - cp test/travis/database.yml test/database.yml
+
 script:
   - bundle exec rake test:mysql2
   - bundle exec rake test:mysql2_makara
@@ -60,3 +67,5 @@ script:
   - bundle exec rubocop
 
 dist: trusty
+
+sudo: required

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## Changes in 0.25.0
+
+### New Features
+
+* Add support for makara_postgis adapter. Thanks to @chadwilken via \#527.
+* Skip validating presence of belongs_to associations. Thanks to @Sohair63, @naiyt, @jkowens via \#528.
+
+### Fixes
+
+* Add missing require for ActiveSupport.on_load. Thanks to @onk via \#529.
+* Support setting attribute values in before_validation callbacks.
+  Thanks to @SirRawlins, @jkowens via \#531.
+* Ignore virtual columns. Thanks to @dbourguignon, @jkowens via \#530.
+
 ## Changes in 0.24.0
 
 ### Fixes

data/Rakefile

@@ -23,6 +23,7 @@ ADAPTERS = %w(
   postgresql
   postgresql_makara
   postgis
+  makara_postgis
   sqlite3
   spatialite
   seamless_database_pool

data/gemfiles/5.0.gemfile

@@ -1 +1,2 @@
 gem 'activerecord', '~> 5.0.0'
+gem 'composite_primary_keys', '~> 9.0'

data/gemfiles/5.1.gemfile

@@ -1 +1,2 @@
 gem 'activerecord', '~> 5.1.0'
+gem 'composite_primary_keys', '~> 10.0'

data/gemfiles/5.2.gemfile

@@ -1,2 +1,2 @@
-gem 'activerecord', '~> 5.2.0.rc1'
-gem 'composite_primary_keys', '~> 11.0.0.rc2'
+gem 'activerecord', '~> 5.2.0'
+gem 'composite_primary_keys', '~> 11.0'

data/lib/activerecord-import.rb

@@ -1,4 +1,5 @@
 # rubocop:disable Style/FileName
+require "active_support/lazy_load_hooks"
 
 ActiveSupport.on_load(:active_record) do
   require "activerecord-import/base"

data/lib/activerecord-import/base.rb

@@ -11,6 +11,7 @@ module ActiveRecord::Import
     when 'mysql2spatial' then 'mysql2'
     when 'spatialite' then 'sqlite3'
     when 'postgresql_makara' then 'postgresql'
+    when 'makara_postgis' then 'postgresql'
     when 'postgis' then 'postgresql'
     else adapter
     end

data/lib/activerecord-import/import.rb

@@ -38,8 +38,14 @@ module ActiveRecord::Import #:nodoc:
         model.errors.clear
 
         validate_callbacks = model._validate_callbacks.dup
+        associations = model.class.reflect_on_all_associations(:belongs_to).map(&:name)
+
         model._validate_callbacks.each do |callback|
-          validate_callbacks.delete(callback) if callback.raw_filter.is_a? ActiveRecord::Validations::UniquenessValidator
+          filter = callback.raw_filter
+          if filter.is_a?(ActiveRecord::Validations::UniquenessValidator) ||
+             (defined?(ActiveRecord::Validations::PresenceValidator) && filter.is_a?(ActiveRecord::Validations::PresenceValidator) && associations.include?(filter.attributes.first))
+            validate_callbacks.delete(callback)
+          end
         end
 
         model.run_callbacks(:validation) do
@@ -95,7 +101,12 @@ class ActiveRecord::Associations::CollectionAssociation
 
     model_klass = reflection.klass
     symbolized_foreign_key = reflection.foreign_key.to_sym
-    symbolized_column_names = model_klass.column_names.map(&:to_sym)
+
+    symbolized_column_names = if model_klass.connection.respond_to?(:supports_virtual_columns?) && model_klass.connection.supports_virtual_columns?
+      model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
+    else
+      model_klass.column_names.map(&:to_sym)
+    end
 
     owner_primary_key = reflection.active_record_primary_key.to_sym
     owner_primary_key_value = owner.send(owner_primary_key)
@@ -494,8 +505,8 @@ class ActiveRecord::Base
         end
       end
 
-      is_validating = options[:validate]
-      is_validating = true unless options[:validate_with_context].nil?
+      is_validating = options[:validate_with_context].present? ? true : options[:validate]
+      validator = ActiveRecord::Import::Validator.new(options)
 
       # assume array of model objects
       if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
@@ -504,7 +515,11 @@ class ActiveRecord::Base
           column_names = args.first.dup
         else
           models = args.first
-          column_names = self.column_names.dup
+          column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
+            columns.reject(&:virtual?).map(&:name)
+          else
+            self.column_names.dup
+          end
         end
 
         if models.first.id.nil? && column_names.include?(primary_key) && columns_hash[primary_key].type == :uuid
@@ -520,12 +535,19 @@ class ActiveRecord::Base
           serialized_attributes
         end
 
-        array_of_attributes = models.map do |model|
+        array_of_attributes = []
+
+        models.each do |model|
           if supports_setting_primary_key_of_imported_objects?
             load_association_ids(model)
           end
 
-          column_names.map do |name|
+          if is_validating && !validator.valid_model?(model)
+            raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+            next
+          end
+
+          array_of_attributes << column_names.map do |name|
             if stored_attrs.key?(name.to_sym) ||
                serialized_attrs.key?(name) ||
                default_values.key?(name.to_s)
@@ -597,17 +619,27 @@ class ActiveRecord::Base
       end
 
       return_obj = if is_validating
-        if models
-          import_with_validations( column_names, array_of_attributes, options ) do |validator, failed|
-            models.each_with_index do |model, i|
-              next if validator.valid_model? model
+        import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
+          if models
+            models.each { |m| failed_instances << m if m.errors.any? }
+          else
+            # create instances for each of our column/value sets
+            arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
+
+            # keep track of the instance and the position it is currently at. if this fails
+            # validation we'll use the index to remove it from the array_of_attributes
+            arr.each_with_index do |hsh, i|
+              model = new
+              hsh.each_pair { |k, v| model[k] = v }
+              next if validator.valid_model?(model)
               raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
               array_of_attributes[i] = nil
-              failed << model
+              failure = model.dup
+              failure.errors.send(:initialize_dup, model.errors)
+              failed_instances << failure
             end
+            array_of_attributes.compact!
           end
-        else
-          import_with_validations( column_names, array_of_attributes, options )
         end
       else
         import_without_validations_or_callbacks( column_names, array_of_attributes, options )
@@ -640,29 +672,7 @@ class ActiveRecord::Base
     def import_with_validations( column_names, array_of_attributes, options = {} )
       failed_instances = []
 
-      validator = ActiveRecord::Import::Validator.new(options)
-
-      if block_given?
-        yield validator, failed_instances
-      else
-        # create instances for each of our column/value sets
-        arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
-
-        # keep track of the instance and the position it is currently at. if this fails
-        # validation we'll use the index to remove it from the array_of_attributes
-        arr.each_with_index do |hsh, i|
-          model = new
-          hsh.each_pair { |k, v| model[k] = v }
-          next if validator.valid_model? model
-          raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
-          array_of_attributes[i] = nil
-          failure = model.dup
-          failure.errors.send(:initialize_dup, model.errors)
-          failed_instances << failure
-        end
-      end
-
-      array_of_attributes.compact!
+      yield failed_instances if block_given?
 
       result = if options[:all_or_none] && failed_instances.any?
         ActiveRecord::Import::Result.new([], 0, [], [])
@@ -945,7 +955,7 @@ Hash key mismatch.
 
 When importing an array of hashes with provided columns_names, each hash must contain keys for all column_names.
 
-Required keys: #{column_names}
+Required keys: #{required_keys}
 Missing keys: #{missing_keys}
 
 Hash: #{hash}
@@ -958,7 +968,7 @@ When importing an array of hashes, all hashes must have the same keys.
 If you have records that are missing some values, we recommend you either set default values
 for the missing keys or group these records into batches by key set before importing.
 
-Required keys: #{column_names}
+Required keys: #{required_keys}
 Extra keys: #{extra_keys}
 Missing keys: #{missing_keys}
 

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "0.24.0".freeze
+    VERSION = "0.25.0".freeze
   end
 end

data/test/adapters/makara_postgis.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "postgis"

data/test/makara_postgis/import_test.rb

@@ -0,0 +1,8 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
+
+should_support_postgresql_import_functionality
+
+if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+  should_support_postgresql_upsert_functionality
+end

data/test/schema/mysql2_schema.rb

@@ -0,0 +1,19 @@
+ActiveRecord::Schema.define do
+  create_table :books, force: :cascade do |t|
+    t.string :title, null: false
+    t.virtual :upper_title, type: :string, as: "upper(`title`)" if t.respond_to?(:virtual)
+    t.string :publisher, null: false, default: 'Default Publisher'
+    t.string :author_name, null: false
+    t.datetime :created_at
+    t.datetime :created_on
+    t.datetime :updated_at
+    t.datetime :updated_on
+    t.date :publish_date
+    t.integer :topic_id
+    t.integer :tag_id
+    t.integer :publisher_id
+    t.boolean :for_sale, default: true
+    t.integer :status, default: 0
+    t.string :type
+  end
+end

data/test/support/mysql/import_examples.rb

@@ -82,5 +82,17 @@ def should_support_mysql_import_functionality
         assert_equal "Chad Fowler", topics.last.author_name, "wrong author!"
       end
     end
+
+    if ENV['AR_VERSION'].to_f >= 5.1
+      context "with virtual columns" do
+        let(:books) { [Book.new(author_name: "foo", title: "bar")] }
+
+        it "ignores virtual columns and creates record" do
+          assert_difference "Book.count", +1 do
+            Book.import books
+          end
+        end
+      end
+    end
   end
 end

data/test/test_helper.rb

@@ -33,6 +33,14 @@ rescue LoadError
   ENV["SKIP_COMPOSITE_PK"] = "true"
 end
 
+# Support MySQL 5.7
+if ActiveSupport::VERSION::STRING < "4.1"
+  require "active_record/connection_adapters/mysql2_adapter"
+  class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
+  end
+end
+
 require "ruby-debug" if RUBY_VERSION.to_f < 1.9
 
 adapter = ENV["ARE_DB"] || "sqlite3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.25.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-17 00:00:00.000000000 Z
+date: 2018-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -101,6 +101,7 @@ files:
 - test/adapters/jdbcmysql.rb
 - test/adapters/jdbcpostgresql.rb
 - test/adapters/jdbcsqlite3.rb
+- test/adapters/makara_postgis.rb
 - test/adapters/mysql2.rb
 - test/adapters/mysql2_makara.rb
 - test/adapters/mysql2spatial.rb
@@ -115,6 +116,7 @@ files:
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
 - test/jdbcsqlite3/import_test.rb
+- test/makara_postgis/import_test.rb
 - test/models/account.rb
 - test/models/alarm.rb
 - test/models/bike_maker.rb
@@ -141,7 +143,7 @@ files:
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
 - test/schema/jdbcpostgresql_schema.rb
-- test/schema/mysql_schema.rb
+- test/schema/mysql2_schema.rb
 - test/schema/postgis_schema.rb
 - test/schema/postgresql_schema.rb
 - test/schema/version.rb
@@ -189,6 +191,7 @@ test_files:
 - test/adapters/jdbcmysql.rb
 - test/adapters/jdbcpostgresql.rb
 - test/adapters/jdbcsqlite3.rb
+- test/adapters/makara_postgis.rb
 - test/adapters/mysql2.rb
 - test/adapters/mysql2_makara.rb
 - test/adapters/mysql2spatial.rb
@@ -203,6 +206,7 @@ test_files:
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
 - test/jdbcsqlite3/import_test.rb
+- test/makara_postgis/import_test.rb
 - test/models/account.rb
 - test/models/alarm.rb
 - test/models/bike_maker.rb
@@ -229,7 +233,7 @@ test_files:
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
 - test/schema/jdbcpostgresql_schema.rb
-- test/schema/mysql_schema.rb
+- test/schema/mysql2_schema.rb
 - test/schema/postgis_schema.rb
 - test/schema/postgresql_schema.rb
 - test/schema/version.rb

data/test/schema/mysql_schema.rb

@@ -1,16 +0,0 @@
-ActiveRecord::Schema.define do
-  create_table :books, options: 'ENGINE=MyISAM', force: true do |t|
-    t.column :title, :string, null: false
-    t.column :publisher, :string, null: false, default: 'Default Publisher'
-    t.column :author_name, :string, null: false
-    t.column :created_at, :datetime
-    t.column :created_on, :datetime
-    t.column :updated_at, :datetime
-    t.column :updated_on, :datetime
-    t.column :publish_date, :date
-    t.column :topic_id, :integer
-    t.column :for_sale, :boolean, default: true
-    t.column :status, :integer
-  end
-  execute "ALTER TABLE books ADD FULLTEXT( `title`, `publisher`, `author_name` )"
-end