activerecord-import 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c1b3ac9675567f96311618bfcf008c344559183
-  data.tar.gz: 796d923d2db2f0317c832081e2ea336b9f933a7c
+  metadata.gz: 64717eaa9cb0c49761ad29e6aa358c87e96b7166
+  data.tar.gz: 879ac9c11ee4a9a62988c97608ad195a1052893d
 SHA512:
-  metadata.gz: 17a0387cd93c752b91f8cee9bf0fa311463e927debf83141a759e277d5e632aa7f468fa0b36a082d54adad0596cae40f31002f699b6189c0b00ce1b072e666c9
-  data.tar.gz: 587d1a167feb65558bf10208404a23bc409c6a4f48e8dd64cc32e0e2af76e1f9b81247a17658c47de102d29a0b8b4619a021d1b78bde61aa00fa5403b38c2aaa
+  metadata.gz: 7e06cfe24bc207291c4a4c8737f8c3d6309218ccde2ca3246089f45d7f36c85f6fa7a63abe5758934dd6c9e939422c40395880b7ba950e10db7b96fa38b071af
+  data.tar.gz: 6ce0fc279753ab340269eefa2d64be18c40d2bed279f3cda27ba4057cb428582a207c5637da5a7c5222cffa77d2a43a0030578c87ebe05f76c81ad098da3fa21

data/.travis.yml

@@ -27,10 +27,13 @@ matrix:
 before_script:
   - mysql -e 'create database activerecord_import_test;'
   - psql -c 'create database activerecord_import_test;' -U postgres
-  - psql -U postgres -c "create extension postgis"
+  - psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
+  - psql -c 'create extension if not exists postgis;' -U postgres
+  - psql -c 'create extension if not exists "uuid-ossp";' -U postgres
   - cp test/travis/database.yml test/database.yml
 
 addons:
+  postgresql: "9.4"
   apt:
     sources:
       - travis-ci/sqlite3

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+## Changes in 0.16.0
+
+### New Features
+
+* Add partial index upsert support for PostgreSQL. Thanks to @luislew via \#305
+* Add UUID primary key support for PostgreSQL. Thanks to @jkowens via
+  \#312
+* Add store accessor support for JSON, JSON, and HSTORE data types.
+  Thanks to @jkowens via \#322
+* Log warning if database does not support :on_duplicate_key_update.
+  Thanks to @jkowens vi \#324
+* Add option :on_duplicate_key_ignore for MySQL and SQLite. Thanks to
+  @jkowens via \#326
+
+### Fixes
+
+* Fixes issue with recursive import using same primary key for all models.
+  Thanks to @chopraanmol1 via \#309
+* Fixes issue importing from STI subclass with polymorphic associations.
+  Thanks to @JNajera via \#314
+* Fixes issue setting returned IDs to wrong models when some fail validation. Also fixes issue with SQLite returning wrong IDs. Thanks to @mizukami234 via \#315
+
+
 ## Changes in 0.15.0
 
 ### New Features

data/Gemfile

@@ -25,6 +25,7 @@ end
 gem "factory_girl", "~> 4.2.0"
 gem "timecop"
 gem "chronic"
+gem "mocha"
 
 # Debugging
 platforms :jruby do

data/benchmarks/README

@@ -15,10 +15,10 @@ See "ruby benchmark.rb -h" for the complete listing of options.
 EXAMPLES
 --------
 To output to html format:
-  ruby benchmark.rb --adapter=mysql --to-html=results.html
+  ruby benchmark.rb --adapter=mysql2 --to-html=results.html
 
 To output to csv format:
-  ruby benchmark.rb --adapter=mysql --to-csv=results.csv
+  ruby benchmark.rb --adapter=mysql2 --to-csv=results.csv
 
 LIMITATIONS
 -----------

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -23,7 +23,6 @@ module ActiveRecord::Import::AbstractAdapter
       sql = []
       sql << options[:pre_sql] if options[:pre_sql]
       sql << options[:command] if options[:command]
-      sql << "IGNORE" if options[:ignore]
 
       # add keywords like IGNORE or DELAYED
       if options[:keywords].is_a?(Array)
@@ -45,15 +44,10 @@ module ActiveRecord::Import::AbstractAdapter
     def post_sql_statements( table_name, options ) # :nodoc:
       post_sql_statements = []
 
-      if supports_on_duplicate_key_update?
-        if options[:on_duplicate_key_ignore] && respond_to?(:sql_for_on_duplicate_key_ignore)
-          # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
-          unless options[:recursive]
-            post_sql_statements << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
-          end
-        elsif options[:on_duplicate_key_update]
-          post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] )
-        end
+      if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] )
+      elsif options[:on_duplicate_key_update]
+        logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
       end
 
       # custom user post_sql

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -39,10 +39,13 @@ module ActiveRecord::Import::MysqlAdapter
       value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
         reserved_bytes: sql_size,
         max_bytes: max)
-      value_sets.each do |value_set|
-        number_of_inserts += 1
-        sql2insert = base_sql + value_set.join( ',' ) + post_sql
-        insert( sql2insert, *args )
+
+      transaction(requires_new: true) do
+        value_sets.each do |value_set|
+          number_of_inserts += 1
+          sql2insert = base_sql + value_set.join( ',' ) + post_sql
+          insert( sql2insert, *args )
+        end
       end
     end
 
@@ -60,6 +63,12 @@ module ActiveRecord::Import::MysqlAdapter
     end
   end
 
+  def pre_sql_statements( options)
+    sql = []
+    sql << "IGNORE" if options[:ignore] || options[:on_duplicate_key_ignore]
+    sql + super
+  end
+
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
     if options.include?(:on_duplicate_key_update)
@@ -68,7 +77,7 @@ module ActiveRecord::Import::MysqlAdapter
       when Array then columns << column.to_sym unless columns.include?(column.to_sym)
       when Hash then columns[column.to_sym] = column.to_sym
       end
-    else
+    elsif !options[:ignore] && !options[:on_duplicate_key_ignore]
       options[:on_duplicate_key_update] = [column.to_sym]
     end
   end

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -31,11 +31,24 @@ module ActiveRecord::Import::PostgreSQLAdapter
   end
 
   def post_sql_statements( table_name, options ) # :nodoc:
-    if options[:no_returning] || options[:primary_key].blank?
-      super(table_name, options)
-    else
-      super(table_name, options) << "RETURNING #{options[:primary_key]}"
+    sql = []
+
+    if supports_on_duplicate_key_update?
+      # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
+      if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update] && !options[:recursive]
+        sql << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
+      end
+    elsif options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
+      logger.warn "Ignoring on_duplicate_key_ignore because it is not supported by the database."
+    end
+
+    sql += super(table_name, options)
+
+    unless options[:no_returning] || options[:primary_key].blank?
+      sql << "RETURNING #{options[:primary_key]}"
     end
+
+    sql
   end
 
   # Add a column to be updated on duplicate key update
@@ -113,10 +126,13 @@ module ActiveRecord::Import::PostgreSQLAdapter
   def sql_for_conflict_target( args = {} )
     constraint_name = args[:constraint_name]
     conflict_target = args[:conflict_target]
+    index_predicate = args[:index_predicate]
     if constraint_name.present?
       "ON CONSTRAINT #{constraint_name} "
     elsif conflict_target.present?
-      '(' << Array( conflict_target ).reject( &:empty? ).join( ', ' ) << ') '
+      '(' << Array( conflict_target ).reject( &:empty? ).join( ', ' ) << ') '.tap do |sql|
+        sql << "WHERE #{index_predicate} " if index_predicate
+      end
     end
   end
 
@@ -134,10 +150,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
     current_version >= MIN_VERSION_FOR_UPSERT
   end
 
-  def supports_on_duplicate_key_ignore?(current_version = postgresql_version)
-    supports_on_duplicate_key_update?(current_version)
-  end
-
   def support_setting_primary_key_of_imported_objects?
     true
   end

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -30,21 +30,37 @@ module ActiveRecord::Import::SQLite3Adapter
     value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
       max_records: SQLITE_LIMIT_COMPOUND_SELECT)
 
-    value_sets.each do |value_set|
-      number_of_inserts += 1
-      sql2insert = base_sql + value_set.join( ',' ) + post_sql
-      first_insert_id = insert( sql2insert, *args )
-      last_insert_id = first_insert_id + value_set.size - 1
-      ids.concat((first_insert_id..last_insert_id).to_a)
+    transaction(requires_new: true) do
+      value_sets.each do |value_set|
+        number_of_inserts += 1
+        sql2insert = base_sql + value_set.join( ',' ) + post_sql
+        last_insert_id = insert( sql2insert, *args )
+        first_insert_id = last_insert_id - affected_rows + 1
+        ids.concat((first_insert_id..last_insert_id).to_a)
+      end
     end
 
     [number_of_inserts, ids]
   end
 
+  def pre_sql_statements( options)
+    sql = []
+    # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
+    if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:recursive]
+      sql << "OR IGNORE"
+    end
+    sql + super
+  end
+
   def next_value_for_sequence(sequence_name)
     %{nextval('#{sequence_name}')}
   end
 
+  def affected_rows
+    result = execute('SELECT changes();')
+    result.first[0]
+  end
+
   def support_setting_primary_key_of_imported_objects?
     true
   end

data/lib/activerecord-import/import.rb

@@ -171,10 +171,11 @@ class ActiveRecord::Base
     # == Options
     # * +validate+ - true|false, tells import whether or not to use
     #    ActiveRecord validations. Validations are enforced by default.
-    # * +ignore+ - true|false, tells import to use MySQL's INSERT IGNORE
-    #    to discard records that contain duplicate keys.
-    # * +on_duplicate_key_ignore+ - true|false, tells import to use
-    #    Postgres 9.5+ ON CONFLICT DO NOTHING. Cannot be enabled on a
+    # * +ignore+ - true|false, an alias for on_duplicate_key_ignore.
+    # * +on_duplicate_key_ignore+ - true|false, tells import to discard
+    #    records that contain duplicate keys. For Postgres 9.5+ it adds
+    #    ON CONFLICT DO NOTHING, for MySQL it uses INSERT IGNORE, and for
+    #    SQLite it uses INSERT OR IGNORE. Cannot be enabled on a
     #    recursive import.
     # * +on_duplicate_key_update+ - an Array or Hash, tells import to
     #    use MySQL's ON DUPLICATE KEY UPDATE or Postgres 9.5+ ON CONFLICT
@@ -246,7 +247,8 @@ class ActiveRecord::Base
     # == On Duplicate Key Update (Postgres 9.5+)
     #
     # The :on_duplicate_key_update option can be an Array or a Hash with up to
-    # two attributes, :conflict_target or :constraint_name and :columns.
+    # three attributes, :conflict_target (and optionally :index_predicate) or
+    # :constraint_name, and :columns.
     #
     # ==== Using an Array
     #
@@ -260,10 +262,11 @@ class ActiveRecord::Base
     #
     # ====  Using a Hash
     #
-    # The :on_duplicate_update option can be a hash with up to two attributes,
-    # :conflict_target or constraint_name, and :columns. Unlike MySQL, Postgres
-    # requires the conflicting constraint to be explicitly specified. Using this
-    # option allows you to specify a constraint other than the primary key.
+    # The :on_duplicate_key_update option can be a hash with up to three
+    # attributes, :conflict_target (and optionally :index_predicate) or
+    # :constraint_name, and :columns. Unlike MySQL, Postgres requires the
+    # conflicting constraint to be explicitly specified. Using this option
+    # allows you to specify a constraint other than the primary key.
     #
     # ====== :conflict_target
     #
@@ -273,7 +276,16 @@ class ActiveRecord::Base
     # but it is the preferred method of identifying a constraint. It will
     # default to the primary key. Below is an example:
     #
-    #   BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [:author_id, :slug], columns: [ :date_modified ] }
+    #   BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id, :slug ], columns: [ :date_modified ] }
+    #
+    # ====== :index_predicate
+    #
+    # The :index_predicate attribute optionally specifies a WHERE condition
+    # on :conflict_target, which is required for matching against partial
+    # indexes. This attribute is ignored if :constraint_name is included.
+    # Below is an example:
+    #
+    #   BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id, :slug ], index_predicate: 'status <> 0', columns: [ :date_modified ] }
     #
     # ====== :constraint_name
     #
@@ -307,7 +319,7 @@ class ActiveRecord::Base
     # This returns an object which responds to +failed_instances+ and +num_inserts+.
     # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
     # * num_inserts - the number of insert statements it took to import the data
-    # * ids - the primary keys of the imported ids, if the adpater supports it, otherwise and empty array.
+    # * ids - the primary keys of the imported ids if the adapter supports it, otherwise an empty array.
     def import(*args)
       if args.first.is_a?( Array ) && args.first.first.is_a?(ActiveRecord::Base)
         options = {}
@@ -332,11 +344,13 @@ class ActiveRecord::Base
     end
 
     def import_helper( *args )
-      options = { validate: true, timestamps: true, primary_key: primary_key }
+      options = { validate: true, timestamps: true }
       options.merge!( args.pop ) if args.last.is_a? Hash
+      # making sure that current model's primary key is used
+      options[:primary_key] = primary_key
 
       # Don't modify incoming arguments
-      if options[:on_duplicate_key_update]
+      if options[:on_duplicate_key_update] && options[:on_duplicate_key_update].duplicable?
         options[:on_duplicate_key_update] = options[:on_duplicate_key_update].dup
       end
 
@@ -347,17 +361,27 @@ class ActiveRecord::Base
       if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
         if args.length == 2
           models = args.last
-          column_names = args.first
+          column_names = args.first.dup
         else
           models = args.first
           column_names = self.column_names.dup
         end
 
+        if column_names.include?(primary_key) && columns_hash[primary_key].type == :uuid
+          column_names.delete(primary_key)
+        end
+
+        stored_attrs = respond_to?(:stored_attributes) ? stored_attributes : {}
+
         array_of_attributes = models.map do |model|
           # this next line breaks sqlite.so with a segmentation fault
           # if model.new_record? || options[:on_duplicate_key_update]
           column_names.map do |name|
-            model.read_attribute_before_type_cast(name.to_s)
+            if stored_attrs.any? && stored_attrs.key?(name.to_sym)
+              model.read_attribute(name.to_s)
+            else
+              model.read_attribute_before_type_cast(name.to_s)
+            end
           end
           # end
         end
@@ -367,14 +391,14 @@ class ActiveRecord::Base
         # supports 2-element array and array
       elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
         column_names, array_of_attributes = args
+
+        # dup the passed args so we don't modify unintentionally
+        column_names = column_names.dup
         array_of_attributes = array_of_attributes.map(&:dup)
       else
         raise ArgumentError, "Invalid arguments!"
       end
 
-      # dup the passed in array so we don't modify it unintentionally
-      column_names = column_names.dup
-
       # Force the primary key col into the insert if it's not
       # on the list and we are using a sequence and stuff a nil
       # value for it into each row so the sequencer will fire later
@@ -394,11 +418,7 @@ class ActiveRecord::Base
             models.each_with_index do |model, i|
               model = model.dup if options[:recursive]
               next if model.valid?(options[:validate_with_context])
-              if options[:raise_error] && model.respond_to?(:raise_validation_error, true) # Rails 5.0 and higher
-                model.send(:raise_validation_error)
-              elsif options[:raise_error] # Rails 3.2, 4.0, 4.1 and 4.2
-                model.send(:raise_record_invalid)
-              end
+              raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
               array_of_attributes[i] = nil
               failed << model
             end
@@ -418,7 +438,7 @@ class ActiveRecord::Base
       return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
 
       # if we have ids, then set the id on the models and mark the models as clean.
-      if support_setting_primary_key_of_imported_objects?
+      if models && support_setting_primary_key_of_imported_objects?
         set_ids_and_mark_clean(models, return_obj)
 
         # if there are auto-save associations on the models we imported that are new, import them as well
@@ -502,7 +522,9 @@ class ActiveRecord::Base
       end
 
       columns_sql = "(#{column_names.map { |name| connection.quote_column_name(name) }.join(',')})"
-      insert_sql = "INSERT #{options[:ignore] ? 'IGNORE ' : ''}INTO #{quoted_table_name} #{columns_sql} VALUES "
+      pre_sql_statements = connection.pre_sql_statements( options )
+      insert_sql = ['INSERT', pre_sql_statements, "INTO #{quoted_table_name} #{columns_sql} VALUES "]
+      insert_sql = insert_sql.flatten.join(' ')
       values_sql = values_sql_for_columns_and_attributes(columns, array_of_attributes)
 
       number_inserted = 0
@@ -521,9 +543,11 @@ class ActiveRecord::Base
           ids += result[1]
         end
       else
-        values_sql.each do |values|
-          ids << connection.insert(insert_sql + values)
-          number_inserted += 1
+        transaction(requires_new: true) do
+          values_sql.each do |values|
+            ids << connection.insert(insert_sql + values)
+            number_inserted += 1
+          end
         end
       end
       [number_inserted, ids]
@@ -533,9 +557,10 @@ class ActiveRecord::Base
 
     def set_ids_and_mark_clean(models, import_result)
       return if models.nil?
+      models -= import_result.failed_instances
       import_result.ids.each_with_index do |id, index|
         model = models[index]
-        model.id = id.to_i
+        model.id = id
         if model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
           model.clear_changes_information
         else # Rails 3.2
@@ -550,6 +575,7 @@ class ActiveRecord::Base
       # notes:
       #    does not handle associations that reference themselves
       #    should probably take a hash to associations to follow.
+      return if models.nil?
       associated_objects_by_class = {}
       models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
 
@@ -585,7 +611,7 @@ class ActiveRecord::Base
           child.public_send("#{association_reflection.foreign_key}=", model.id)
           # For polymorphic associations
           association_reflection.type.try do |type|
-            child.public_send("#{type}=", model.class.name)
+            child.public_send("#{type}=", model.class.base_class.name)
           end
         end
         associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "0.15.0".freeze
+    VERSION = "0.16.0".freeze
   end
 end

data/test/import_test.rb

@@ -44,6 +44,25 @@ describe "#import" do
         end
       end
     end
+
+    context "that have no primary key" do
+      it "should import models successfully" do
+        assert_difference "Rule.count", +3 do
+          Rule.import Build(3, :rules)
+        end
+      end
+    end
+  end
+
+  describe "with STI models" do
+    it "should import models successfully" do
+      dictionaries = [Dictionary.new(author_name: "Noah Webster", title: "Webster's Dictionary")]
+
+      assert_difference "Dictionary.count", +1 do
+        Dictionary.import dictionaries
+      end
+      assert_equal "Dictionary", Dictionary.first.type
+    end
   end
 
   context "with :validation option" do
@@ -51,6 +70,8 @@ describe "#import" do
     let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
     let(:valid_values_with_context) { [[1111, "Jerry Carter"], [2222, "Chad Fowler"]] }
     let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] }
+    let(:valid_models) { valid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
+    let(:invalid_models) { invalid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
 
     context "with validation checks turned off" do
       it "should import valid data" do
@@ -103,6 +124,16 @@ describe "#import" do
         results.failed_instances.each { |e| assert_kind_of Topic, e }
       end
 
+      it "should set ids in valid models if adapter supports setting primary key of imported objects" do
+        if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
+          Topic.import (invalid_models + valid_models), validate: true
+          assert_nil invalid_models[0].id
+          assert_nil invalid_models[1].id
+          assert_equal valid_models[0].id, Topic.all[0].id
+          assert_equal valid_models[1].id, Topic.all[1].id
+        end
+      end
+
       it "should import valid data when mixed with invalid data" do
         assert_difference "Topic.count", +2 do
           Topic.import columns, valid_values + invalid_values, validate: true
@@ -112,6 +143,18 @@ describe "#import" do
     end
   end
 
+  context "without :validation option" do
+    let(:columns) { %w(title author_name) }
+    let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] }
+
+    it "should not import invalid data" do
+      assert_no_difference "Topic.count" do
+        result = Topic.import columns, invalid_values
+        assert_equal 2, result.failed_instances.size
+      end
+    end
+  end
+
   context "with :all_or_none option" do
     let(:columns) { %w(title author_name) }
     let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
@@ -281,6 +324,12 @@ describe "#import" do
       Topic.import [:id, :author_name, :title], [[99, "Bob Jones", "Topic 99"]]
       assert_equal 99, Topic.last.id
     end
+
+    it "ignores the recursive option" do
+      assert_difference "Topic.count", +1 do
+        Topic.import [:author_name, :title], [["David Chelimsky", "The RSpec Book"]], recursive: true
+      end
+    end
   end
 
   context "ActiveRecord timestamps" do
@@ -485,43 +534,90 @@ describe "#import" do
   end
 
   describe "importing serialized fields" do
-    it "imports values for serialized fields" do
+    it "imports values for serialized Hash fields" do
       assert_difference "Widget.unscoped.count", +1 do
         Widget.import [:w_id, :data], [[1, { a: :b }]]
       end
       assert_equal({ a: :b }, Widget.find_by_w_id(1).data)
     end
 
-    if ENV['AR_VERSION'].to_f >= 3.1
-      let(:data) { { a: :b } }
-      it "imports values for serialized JSON fields" do
-        assert_difference "Widget.unscoped.count", +1 do
-          Widget.import [:w_id, :json_data], [[9, data]]
+    it "imports values for serialized fields" do
+      assert_difference "Widget.unscoped.count", +1 do
+        Widget.import [:w_id, :unspecified_data], [[1, { a: :b }]]
+      end
+      assert_equal({ a: :b }, Widget.find_by_w_id(1).unspecified_data)
+    end
+
+    it "imports values for custom coder" do
+      assert_difference "Widget.unscoped.count", +1 do
+        Widget.import [:w_id, :custom_data], [[1, { a: :b }]]
+      end
+      assert_equal({ a: :b }, Widget.find_by_w_id(1).custom_data)
+    end
+
+    let(:data) { { a: :b } }
+    it "imports values for serialized JSON fields" do
+      assert_difference "Widget.unscoped.count", +1 do
+        Widget.import [:w_id, :json_data], [[9, data]]
+      end
+      assert_equal(data.as_json, Widget.find_by_w_id(9).json_data)
+    end
+
+    context "with a store" do
+      it "imports serialized attributes set using accessors" do
+        vendors = [Vendor.new(name: 'Vendor 1', color: 'blue')]
+        assert_difference "Vendor.count", +1 do
+          Vendor.import vendors
         end
-        assert_equal(data.as_json, Widget.find_by_w_id(9).json_data)
+        assert_equal('blue', Vendor.first.color)
       end
     end
   end
 
   describe "#import!" do
-    let(:columns) { %w(title author_name) }
-    let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
-    let(:invalid_values) { [["Rails Recipes", "Chad Fowler"], ["The RSpec Book", ""], ["Agile+UX", ""]] }
+    context "with an array of unsaved model instances" do
+      let(:topics) { Build(2, :topics) }
+      let(:invalid_topics) { Build(2, :invalid_topics) }
+
+      context "with invalid data" do
+        it "should raise ActiveRecord::RecordInvalid" do
+          assert_no_difference "Topic.count" do
+            assert_raise ActiveRecord::RecordInvalid do
+              Topic.import! invalid_topics
+            end
+          end
+        end
+      end
 
-    context "with invalid data" do
-      it "should raise ActiveRecord::RecordInvalid" do
-        assert_no_difference "Topic.count" do
-          assert_raise ActiveRecord::RecordInvalid do
-            Topic.import! columns, invalid_values
+      context "with valid data" do
+        it "should import data" do
+          assert_difference "Topic.count", +2 do
+            Topic.import! topics
           end
         end
       end
     end
 
-    context "with valid data" do
-      it "should import data" do
-        assert_difference "Topic.count", +2 do
-          Topic.import! columns, valid_values
+    context "with array of columns and array of values" do
+      let(:columns) { %w(title author_name) }
+      let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
+      let(:invalid_values) { [["Rails Recipes", "Chad Fowler"], ["The RSpec Book", ""], ["Agile+UX", ""]] }
+
+      context "with invalid data" do
+        it "should raise ActiveRecord::RecordInvalid" do
+          assert_no_difference "Topic.count" do
+            assert_raise ActiveRecord::RecordInvalid do
+              Topic.import! columns, invalid_values
+            end
+          end
+        end
+      end
+
+      context "with valid data" do
+        it "should import data" do
+          assert_difference "Topic.count", +2 do
+            Topic.import! columns, valid_values
+          end
         end
       end
     end

data/test/models/alarm.rb

@@ -0,0 +1,2 @@
+class Alarm < ActiveRecord::Base
+end

data/test/models/dictionary.rb

@@ -0,0 +1,2 @@
+class Dictionary < Book
+end

data/test/models/vendor.rb

@@ -0,0 +1,7 @@
+class Vendor < ActiveRecord::Base
+  store :preferences, accessors: [:color], coder: JSON
+
+  store_accessor :data, :size
+  store_accessor :config, :contact
+  store_accessor :settings, :charge_code
+end

data/test/models/widget.rb

@@ -1,3 +1,17 @@
+class CustomCoder
+  def load(value)
+    if value.nil?
+      {}
+    else
+      YAML.load(value)
+    end
+  end
+
+  def dump(value)
+    YAML.dump(value)
+  end
+end
+
 class Widget < ActiveRecord::Base
   self.primary_key = :w_id
 
@@ -5,4 +19,6 @@ class Widget < ActiveRecord::Base
 
   serialize :data, Hash
   serialize :json_data, JSON
+  serialize :unspecified_data
+  serialize :custom_data, CustomCoder.new
 end

data/test/postgis/import_test.rb

@@ -2,3 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
 require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
 
 should_support_postgresql_import_functionality
+
+if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+  should_support_postgresql_upsert_functionality
+end

data/test/schema/generic_schema.rb

@@ -64,6 +64,7 @@ ActiveRecord::Schema.define do
     t.integer :topic_id
     t.boolean :for_sale, default: true
     t.integer :status, default: 0
+    t.string :type
   end
 
   create_table :chapters, force: :cascade do |t|
@@ -73,7 +74,7 @@ ActiveRecord::Schema.define do
     t.datetime :updated_at
   end
 
-  create_table :end_notes, force: :cascade do |t|
+  create_table :end_notes, primary_key: :end_note_id, force: :cascade do |t|
     t.string :note
     t.integer :book_id, null: false
     t.datetime :created_at
@@ -115,6 +116,8 @@ ActiveRecord::Schema.define do
     t.boolean :active, default: false
     t.text :data
     t.text :json_data
+    t.text :unspecified_data
+    t.text :custom_data
   end
 
   create_table :promotions, primary_key: :promotion_id, force: :cascade do |t|
@@ -131,7 +134,8 @@ ActiveRecord::Schema.define do
     t.string :discountable_type
   end
 
-  create_table :rules, force: :cascade do |t|
+  create_table :rules, id: false, force: :cascade do |t|
+    t.integer :id
     t.string :condition_text
     t.integer :question_id
   end
@@ -139,4 +143,12 @@ ActiveRecord::Schema.define do
   create_table :questions, force: :cascade do |t|
     t.string :body
   end
+
+  create_table :vendors, force: :cascade do |t|
+    t.string :name, null: true
+    t.text :preferences
+    t.text :data
+    t.text :config
+    t.text :settings
+  end
 end

data/test/schema/jdbcpostgresql_schema.rb

@@ -0,0 +1 @@
+require File.expand_path(File.dirname(__FILE__) + '/postgresql_schema')

data/test/schema/postgis_schema.rb

@@ -0,0 +1 @@
+require File.expand_path(File.dirname(__FILE__) + '/postgresql_schema')

data/test/schema/postgresql_schema.rb

@@ -0,0 +1,41 @@
+ActiveRecord::Schema.define do
+  execute('CREATE extension IF NOT EXISTS "hstore";')
+  execute('CREATE extension IF NOT EXISTS "uuid-ossp";')
+
+  create_table :vendors, id: :uuid, force: :cascade do |t|
+    t.string :name, null: true
+    t.text :preferences
+
+    if t.respond_to?(:json)
+      t.json :data
+    else
+      t.text :data
+    end
+
+    if t.respond_to?(:hstore)
+      t.hstore :config
+    else
+      t.text :config
+    end
+
+    if t.respond_to?(:jsonb)
+      t.jsonb :settings
+    else
+      t.text :settings
+    end
+
+    t.datetime :created_at
+    t.datetime :updated_at
+  end
+
+  create_table :alarms, force: true do |t|
+    t.column :device_id, :integer, null: false
+    t.column :alarm_type, :integer, null: false
+    t.column :status, :integer, null: false
+    t.column :metadata, :text
+    t.datetime :created_at
+    t.datetime :updated_at
+  end
+
+  add_index :alarms, [:device_id, :alarm_type], unique: true, where: 'status <> 0'
+end

data/test/sqlite3/import_test.rb

@@ -1,6 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
 
 should_support_recursive_import
+should_support_on_duplicate_key_ignore
 
 describe "#supports_imports?" do
   context "and SQLite is 3.7.11 or higher" do
@@ -49,4 +50,18 @@ describe "#import" do
       assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
     end
   end
+
+  context "with :on_duplicate_key_update" do
+    let(:topics) { Build(1, :topics) }
+
+    it "should log a warning message" do
+      log = StringIO.new
+      logger = Logger.new(log)
+      logger.level = Logger::WARN
+      ActiveRecord::Base.connection.stubs(:logger).returns(logger)
+
+      Topic.import topics, on_duplicate_key_update: true
+      assert_match(/Ignoring on_duplicate_key_update/, log.string)
+    end
+  end
 end

data/test/support/factories.rb

@@ -32,6 +32,7 @@ FactoryGirl.define do
   end
 
   factory :rule do
+    sequence(:id) { |n| n }
     sequence(:condition_text) { |n| "q_#{n}_#{n}" }
   end
 

data/test/support/mysql/import_examples.rb

@@ -4,6 +4,7 @@ def should_support_mysql_import_functionality
   ActiveRecord::Base.connection.execute "set sql_mode='STRICT_ALL_TABLES'"
 
   should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
 
   describe "#import" do
     context "with :on_duplicate_key_update and validation checks turned off" do

data/test/support/postgresql/import_examples.rb

@@ -58,10 +58,57 @@ def should_support_postgresql_import_functionality
       end
     end
   end
+
+  describe "with a uuid primary key" do
+    let(:vendor) { Vendor.new(name: "foo") }
+    let(:vendors) { [vendor] }
+
+    it "creates records" do
+      assert_difference "Vendor.count", +1 do
+        Vendor.import vendors
+      end
+    end
+
+    it "assigns an id to the model objects" do
+      Vendor.import vendors
+      assert_not_nil vendor.id
+    end
+  end
+
+  describe "with store accessor fields" do
+    if ENV['AR_VERSION'].to_f >= 4.0
+      it "imports values for json fields" do
+        vendors = [Vendor.new(name: 'Vendor 1', size: 100)]
+        assert_difference "Vendor.count", +1 do
+          Vendor.import vendors
+        end
+        assert_equal(100, Vendor.first.size)
+      end
+
+      it "imports values for hstore fields" do
+        vendors = [Vendor.new(name: 'Vendor 1', contact: 'John Smith')]
+        assert_difference "Vendor.count", +1 do
+          Vendor.import vendors
+        end
+        assert_equal('John Smith', Vendor.first.contact)
+      end
+    end
+
+    if ENV['AR_VERSION'].to_f >= 4.2
+      it "imports values for jsonb fields" do
+        vendors = [Vendor.new(name: 'Vendor 1', charge_code: '12345')]
+        assert_difference "Vendor.count", +1 do
+          Vendor.import vendors
+        end
+        assert_equal('12345', Vendor.first.charge_code)
+      end
+    end
+  end
 end
 
 def should_support_postgresql_upsert_functionality
   should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
 
   describe "#import" do
     extend ActiveSupport::TestCase::ImportAssertions
@@ -151,6 +198,50 @@ def should_support_postgresql_upsert_functionality
           end
         end
 
+        context 'with :index_predicate' do
+          let(:columns) { %w( id device_id alarm_type status metadata ) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          context 'supports on duplicate key update for partial indexes' do
+            it 'should not update created_at timestamp columns' do
+              Timecop.freeze Chronic.parse("5 minutes from now") do
+                perform_import
+                assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
+              end
+            end
+
+            it 'should update updated_at timestamp columns' do
+              time = Chronic.parse("5 minutes from now")
+              Timecop.freeze time do
+                perform_import
+                assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
+              end
+            end
+
+            it 'should not update fields not mentioned' do
+              perform_import
+              assert_equal 'foo', updated_alarm.metadata
+            end
+
+            it 'should update fields mentioned with hash mappings' do
+              perform_import
+              assert_equal 2, updated_alarm.status
+            end
+          end
+        end
+
         context "with :constraint_name" do
           let(:columns) { %w( id title author_name author_email_address parent_id ) }
           let(:values) { [[100, "Book", "John Doe", "john@doe.com", 17]] }

data/test/support/shared_examples/on_duplicate_key_ignore.rb

@@ -0,0 +1,25 @@
+def should_support_on_duplicate_key_ignore
+  describe "#import" do
+    extend ActiveSupport::TestCase::ImportAssertions
+    let(:topic) { Topic.create!(title: "Book", author_name: "John Doe") }
+    let(:topics) { [topic] }
+
+    context "with :on_duplicate_key_ignore" do
+      it "should skip duplicates and continue import" do
+        topics << Topic.new(title: "Book 2", author_name: "Jane Doe")
+        assert_difference "Topic.count", +1 do
+          Topic.import topics, on_duplicate_key_ignore: true, validate: false
+        end
+      end
+    end
+
+    context "with :ignore" do
+      it "should skip duplicates and continue import" do
+        topics << Topic.new(title: "Book 2", author_name: "Jane Doe")
+        assert_difference "Topic.count", +1 do
+          Topic.import topics, ignore: true, validate: false
+        end
+      end
+    end
+  end
+end

data/test/support/shared_examples/recursive_import.rb

@@ -48,6 +48,22 @@ def should_support_recursive_import
       end
     end
 
+    it 'imports polymorphic associations from subclass' do
+      discounts = Array.new(1) { |i| Discount.new(amount: i) }
+      dictionaries = Array.new(1) { |i| Dictionary.new(author_name: "Author ##{i}", title: "Book ##{i}") }
+      dictionaries.each do |dictionary|
+        dictionary.discounts << discounts
+      end
+      Dictionary.import dictionaries, recursive: true
+      assert_equal 1, Dictionary.last.discounts.count
+      dictionaries.each do |dictionary|
+        dictionary.discounts.each do |discount|
+          assert_not_nil discount.discountable_id
+          assert_equal 'Book', discount.discountable_type
+        end
+      end
+    end
+
     [{ recursive: false }, {}].each do |import_options|
       it "skips recursion for #{import_options}" do
         assert_difference "Book.count", 0 do
@@ -106,14 +122,16 @@ def should_support_recursive_import
 
     # If adapter supports on_duplicate_key_update, it is only applied to top level models so that SQL with invalid
     # columns, keys, etc isn't generated for child associations when doing recursive import
-    describe "on_duplicate_key_update" do
-      let(:new_topics) { Build(1, :topic_with_book) }
+    if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+      describe "on_duplicate_key_update" do
+        let(:new_topics) { Build(1, :topic_with_book) }
 
-      it "imports objects with associations" do
-        assert_difference "Topic.count", +1 do
-          Topic.import new_topics, recursive: true, on_duplicate_key_update: [:updated_at], validate: false
-          new_topics.each do |topic|
-            assert_not_nil topic.id
+        it "imports objects with associations" do
+          assert_difference "Topic.count", +1 do
+            Topic.import new_topics, recursive: true, on_duplicate_key_update: [:updated_at], validate: false
+            new_topics.each do |topic|
+              assert_not_nil topic.id
+            end
           end
         end
       end

data/test/test_helper.rb

@@ -18,8 +18,10 @@ require "active_support/test_case"
 
 if ActiveSupport::VERSION::STRING < "4.0"
   require 'test/unit'
+  require 'mocha/test_unit'
 else
   require 'active_support/testing/autorun'
+  require "mocha/mini_test"
 end
 
 require 'timecop'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-04 00:00:00.000000000 Z
+date: 2016-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -111,8 +111,10 @@ files:
 - test/import_test.rb
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
+- test/models/alarm.rb
 - test/models/book.rb
 - test/models/chapter.rb
+- test/models/dictionary.rb
 - test/models/discount.rb
 - test/models/end_note.rb
 - test/models/group.rb
@@ -120,6 +122,7 @@ files:
 - test/models/question.rb
 - test/models/rule.rb
 - test/models/topic.rb
+- test/models/vendor.rb
 - test/models/widget.rb
 - test/mysql2/import_test.rb
 - test/mysql2_makara/import_test.rb
@@ -127,7 +130,10 @@ files:
 - test/postgis/import_test.rb
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
+- test/schema/jdbcpostgresql_schema.rb
 - test/schema/mysql_schema.rb
+- test/schema/postgis_schema.rb
+- test/schema/postgresql_schema.rb
 - test/schema/version.rb
 - test/sqlite3/import_test.rb
 - test/support/active_support/test_case_extensions.rb
@@ -136,6 +142,7 @@ files:
 - test/support/generate.rb
 - test/support/mysql/import_examples.rb
 - test/support/postgresql/import_examples.rb
+- test/support/shared_examples/on_duplicate_key_ignore.rb
 - test/support/shared_examples/on_duplicate_key_update.rb
 - test/support/shared_examples/recursive_import.rb
 - test/synchronize_test.rb
@@ -163,7 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Bulk-loading extension for ActiveRecord
@@ -183,8 +190,10 @@ test_files:
 - test/import_test.rb
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
+- test/models/alarm.rb
 - test/models/book.rb
 - test/models/chapter.rb
+- test/models/dictionary.rb
 - test/models/discount.rb
 - test/models/end_note.rb
 - test/models/group.rb
@@ -192,6 +201,7 @@ test_files:
 - test/models/question.rb
 - test/models/rule.rb
 - test/models/topic.rb
+- test/models/vendor.rb
 - test/models/widget.rb
 - test/mysql2/import_test.rb
 - test/mysql2_makara/import_test.rb
@@ -199,7 +209,10 @@ test_files:
 - test/postgis/import_test.rb
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
+- test/schema/jdbcpostgresql_schema.rb
 - test/schema/mysql_schema.rb
+- test/schema/postgis_schema.rb
+- test/schema/postgresql_schema.rb
 - test/schema/version.rb
 - test/sqlite3/import_test.rb
 - test/support/active_support/test_case_extensions.rb
@@ -208,6 +221,7 @@ test_files:
 - test/support/generate.rb
 - test/support/mysql/import_examples.rb
 - test/support/postgresql/import_examples.rb
+- test/support/shared_examples/on_duplicate_key_ignore.rb
 - test/support/shared_examples/on_duplicate_key_update.rb
 - test/support/shared_examples/recursive_import.rb
 - test/synchronize_test.rb
@@ -215,4 +229,3 @@ test_files:
 - test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
-has_rdoc: