activerecord-import 1.0.7 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fdf26ba513a4d3365d24cf9b2bf1fc814db9918d06ccdb91c5f32dec026fb7b7
-  data.tar.gz: 35ba31f95ef4b585cbe8ca4b0819d8ddc1ba50b12db6a70d3be2352669fd9289
+  metadata.gz: 71623b61f085adce5254fc92ae524f595594cb5a960bb4bf0cb46d961ca9d203
+  data.tar.gz: ce2200678685afd85dc0a887d967bae7e2e4220942ab2f8fc34bb949b8f5a31b
 SHA512:
-  metadata.gz: 3ef741e92b76c9705c63145b58351e82813eb71b003993a73cff5db2d0779d6dc14b48d4dc43a354ad8ba6a24a6acd2aa54bdf167775dee8e0404db4905c03f2
-  data.tar.gz: f8634e6ae1e9345ca9b1d68bca8fe4e67c04b1ae51101fdf97608a609d1fc349737d424ce23f809548d4e6a42931eb38d246d6be354acf49282fcaf20d827fa0
+  metadata.gz: 71bdd79b977164223d77c77ec796ac3914124318bb377c8aa044bc8085d0f7ba9536e35b99b6a670b0a4c825e4dfe49aef5a10626b92109b2ce7dd8cd529dbb1
+  data.tar.gz: 89db7ca7f321a473391d2083a2dc6cb4fe8614069441c2a16601a0a3e4ab848e889eafba67b53fc2042d60b7095aab50926f4795d07930c0c90820527f07c28c

data/.github/workflows/test.yaml

@@ -0,0 +1,107 @@
+name: Test
+on: [push, pull_request]
+jobs:
+  test:
+    services:
+      postgres:
+        image: postgis/postgis:10-2.5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - 3.1
+        env:
+          - AR_VERSION: '7.0'
+            RUBYOPT: --enable-frozen-string-literal
+          - AR_VERSION: 6.1
+            RUBYOPT: --enable-frozen-string-literal
+        include:
+          - ruby: '3.0'
+            env:
+              AR_VERSION: '7.0'
+          - ruby: '3.0'
+            env:
+              AR_VERSION: 6.1
+          - ruby: 2.7
+            env:
+              AR_VERSION: '7.0'
+          - ruby: 2.7
+            env:
+              AR_VERSION: 6.1
+          - ruby: 2.7
+            env:
+              AR_VERSION: '6.0'
+          - ruby: 2.6
+            env:
+              AR_VERSION: 5.2
+          - ruby: 2.6
+            env:
+              AR_VERSION: 5.1
+          - ruby: 2.4
+            env:
+              AR_VERSION: '5.0'
+          - ruby: 2.4
+            env:
+              AR_VERSION: 4.2
+    runs-on: ubuntu-latest
+    env:
+      AR_VERSION: ${{ matrix.env.AR_VERSION }}
+      DB_DATABASE: activerecord_import_test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Set up databases
+        run: |
+          sudo /etc/init.d/mysql start
+          mysql -e 'CREATE DATABASE ${{ env.DB_DATABASE }} CHARACTER SET utf8 COLLATE utf8_general_ci;' -u root -proot
+          psql -h localhost -U postgres -c 'create database ${{ env.DB_DATABASE }};'
+          psql -h localhost -U postgres -d ${{ env.DB_DATABASE }} -c 'create extension if not exists hstore;'
+          psql -h localhost -U postgres -c 'create extension if not exists postgis;'
+          psql -h localhost -U postgres -c 'create extension if not exists "uuid-ossp";'
+          cp test/github/database.yml test/database.yml
+        env:
+          PGPASSWORD: postgres
+      - name: Run tests with mysql2
+        run: |
+          bundle exec rake test:mysql2
+          bundle exec rake test:mysql2_makara
+          bundle exec rake test:mysql2spatial
+      - name: Run tests with postgresql
+        run: |
+          bundle exec rake test:postgis
+          bundle exec rake test:postgresql
+          bundle exec rake test:postgresql_makara
+      - name: Run tests with seamless_database_pool
+        run: |
+          bundle exec rake test:seamless_database_pool
+        if: ${{ matrix.ruby < '3.0' }}
+      - name: Run tests with sqlite
+        run: |
+          bundle exec rake test:spatialite
+          bundle exec rake test:sqlite3
+  lint:
+    runs-on: ubuntu-latest
+    env:
+      AR_VERSION: '7.0'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+          bundler-cache: true
+      - name: Run Rubocop
+        run: bundle exec rubocop

data/CHANGELOG.md

@@ -1,5 +1,48 @@
+## Changes in 1.4.0
+
+### New Features
+
+  * Enable compatibility with frozen string literals. Thanks to @desheikh via \##760.
+
+## Changes in 1.3.0
+
+### Fixes
+
+* Ensure correct timestamp values are returned for models after insert. Thanks to @kos1kov via \##756.
+* Restore database_version method to public scope. Thanks to @beauraF via \##753.
+
+### New Features
+
+* Add support for ActiveRecord 7.0. Thanks to @nickhammond, @ryanwood, @jkowens via \##749 and \##752.
+* Add support for compound foreign keys. Thanks to @Uladzimiro via \##750.
+* Add support for :recursive combined with on_duplicate_key_update: :all. Thanks to @deathwish via \##746.
+
+## Changes in 1.2.0
+
+### Fixes
+
+* Update JDBC MySQL adapter to use mysql2 connection adapter. Thanks to @terencechow via \##744.
+* Fix importing STI models with ActiveRecord 6. Thanks to @clemens1483 via \##743.
+* Use polymorphic_name instead of base_class.name for imports. Thanks to @kmhajjar via \##741.
+* Fix compatibility issue with composite primary keys. Thanks to @dlanileonardo via \##737.
+* Prevent double validation of associations on recursive import.
+
+## Changes in 1.1.0
+
+### New Features
+
+*  Add batch progress reporting. Thanks to @gee-forr via \##729.
+
+## Changes in 1.0.8
+
+### Fixes
+
+* Use correct method for clearing query cache. Thanks to @EtienneDepaulis via \##719.
+
 ## Changes in 1.0.7
 
+### New Features
+
 * Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
 * Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley  via \#684.
 
@@ -13,7 +56,7 @@
 
 * Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
   \#692.
-* Fix regression introduced in 1.0.4. Explicity allow adapters to
+* Fix regression introduced in 1.0.4. Explicitly allow adapters to
   support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
 
 ## Changes in 1.0.5
@@ -22,7 +65,7 @@
 
 * Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
 * Return ActiveRecord::Connection from
-  ActiveREcord::Base#establish_connection. Thanks to @reverentF via
+  ActiveRecord::Base#establish_connection. Thanks to @reverentF via
 \#663.
 * Support PostgreSQL array. Thanks to @ujihisa via \#669.
 * Skip loading association ids when column changed. Thanks to @Aristat
@@ -77,7 +120,7 @@
 * Fix import issue for models with Postgresql json/jsonb fields. Thanks
   to @stokarenko via \#594.
 * Fix issue importing models with timestamps that contain timezone
-  information. Thaks to @dekaikiwi, @jkowens via \#598.
+  information. Thanks to @dekaikiwi, @jkowens via \#598.
 * Ignore :no_returning when using :recursive option. Thanks to @dgollahon, @jkowens
   via \#599.
 

data/Gemfile

@@ -6,8 +6,11 @@ version = ENV['AR_VERSION'].to_f
 
 mysql2_version = '0.3.0'
 mysql2_version = '0.4.0' if version >= 4.2
+mysql2_version = '0.5.0' if version >= 6.1
 sqlite3_version = '1.3.0'
 sqlite3_version = '1.4.0' if version >= 6.0
+pg_version = '0.9'
+pg_version = '1.1' if version >= 6.1
 
 group :development, :test do
   gem 'rubocop', '~> 0.40.0'
@@ -17,9 +20,10 @@ end
 # Database Adapters
 platforms :ruby do
   gem "mysql2",                 "~> #{mysql2_version}"
-  gem "pg",                     "~> 0.9"
+  gem "pg",                     "~> #{pg_version}"
   gem "sqlite3",                "~> #{sqlite3_version}"
-  gem "seamless_database_pool", "~> 1.0.20"
+  # seamless_database_pool requires Ruby ~> 2.0
+  gem "seamless_database_pool", "~> 1.0.20" if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.0.0')
 end
 
 platforms :jruby do
@@ -41,14 +45,9 @@ platforms :jruby do
   gem "ruby-debug", "= 0.10.4"
 end
 
-platforms :mri_19 do
-  gem "debugger"
-end
-
 platforms :ruby do
   gem "pry-byebug"
   gem "pry", "~> 0.12.0"
-  gem "rb-readline"
 end
 
 if version >= 4.0

data/README.markdown

@@ -1,4 +1,4 @@
-# Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
+# Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
 
 Activerecord-Import is a library for bulk inserting data using ActiveRecord.
 
@@ -231,6 +231,18 @@ columns = [ :title ]
 Book.import columns, books, batch_size: 2
 ```
 
+If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
+
+```ruby
+my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
+  # Using the arguments provided to the callable, you can
+  # send an email, post to a websocket,
+  # update slack, alert if import is taking too long, etc.
+}
+
+Book.import columns, books, batch_size: 2, batch_progress: my_proc
+```
+
 #### Recursive
 
 NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
@@ -254,8 +266,8 @@ Key                       | Options               | Default            | Descrip
 ------------------------- | --------------------- | ------------------ | -----------
 :validate                 | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
 :validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
-:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
-:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]` 
+:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
+:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
 :on_duplicate_key_ignore  | `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
 :ignore                   | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
 :on_duplicate_key_update  | :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
@@ -380,7 +392,7 @@ Book.import books, validate_uniqueness: true
 
 ### Return Info
 
-The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed`.
+The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
 
 ```ruby
 articles = [
@@ -421,7 +433,8 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
 
 However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`,  `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
 
-If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
+If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
+NOTE: If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
 
 ### Callbacks
 
@@ -560,7 +573,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
 
 Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
 
-If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explcitly setting this value within the model. For example:
+If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
 
 ```ruby
 class Post < ActiveRecord::Base
@@ -603,13 +616,13 @@ After that, you can run the tests. They run against multiple tests and ActiveRec
 
 This is one example of how to run the tests:
 
-```ruby
+```bash
 rm Gemfile.lock
-AR_VERSION=4.2 bundle install
-AR_VERSION=4.2 bundle exec rake test:postgresql test:sqlite3 test:mysql2
+AR_VERSION=7.0 bundle install
+AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
 ```
 
-Once you have pushed up your changes, you can find your CI results [here](https://travis-ci.org/zdennis/activerecord-import/).
+Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
 
 ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
 

data/activerecord-import.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = ActiveRecord::Import::VERSION
 
-  gem.required_ruby_version = ">= 2.0.0"
+  gem.required_ruby_version = ">= 2.4.0"
 
-  gem.add_runtime_dependency "activerecord", ">= 3.2"
+  gem.add_runtime_dependency "activerecord", ">= 4.2"
   gem.add_development_dependency "rake"
 end

data/benchmarks/benchmark.rb

@@ -20,7 +20,11 @@ FileUtils.mkdir_p 'log'
 ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.default_timezone = :utc
+if ActiveRecord.respond_to?(:default_timezone)
+  ActiveRecord.default_timezone = :utc
+else
+  ActiveRecord::Base.default_timezone = :utc
+end
 
 require "activerecord-import"
 ActiveRecord::Base.establish_connection(:test)

data/gemfiles/6.0.gemfile

@@ -1 +1,2 @@
 gem 'activerecord', '~> 6.0.0'
+gem 'composite_primary_keys', '~> 12.0'

data/gemfiles/6.1.gemfile

@@ -1 +1,2 @@
-gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"
+gem 'activerecord', '~> 6.1.0'
+gem 'composite_primary_keys', '~> 13.0'

data/gemfiles/7.0.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 7.0.0'

data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb

@@ -1,6 +1,6 @@
-require "active_record/connection_adapters/mysql_adapter"
-require "activerecord-import/adapters/mysql_adapter"
+require "active_record/connection_adapters/mysql2_adapter"
+require "activerecord-import/adapters/mysql2_adapter"
 
-class ActiveRecord::ConnectionAdapters::MysqlAdapter
-  include ActiveRecord::Import::MysqlAdapter
+class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+  include ActiveRecord::Import::Mysql2Adapter
 end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -82,7 +82,7 @@ module ActiveRecord::Import::MysqlAdapter
   # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
-    sql = ' ON DUPLICATE KEY UPDATE '
+    sql = ' ON DUPLICATE KEY UPDATE '.dup
     arg = args.first
     locking_column = args.last
     if arg.is_a?( Array )

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -28,7 +28,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       else
         select_values( sql2insert, *args )
       end
-      query_cache.clear if query_cache_enabled
+      clear_query_cache if query_cache_enabled
     end
 
     if options[:returning].blank?
@@ -123,7 +123,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
 
-    sql = ' ON CONFLICT '
+    sql = ' ON CONFLICT '.dup
     conflict_target = sql_for_conflict_target( arg )
 
     columns = arg.fetch( :columns, [] )
@@ -179,9 +179,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
     if constraint_name.present?
       "ON CONSTRAINT #{constraint_name} "
     elsif conflict_target.present?
-      '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
-        sql << "WHERE #{index_predicate} " if index_predicate
-      end
+      sql = '(' + Array( conflict_target ).reject( &:blank? ).join( ', ' ) + ') '
+      sql += "WHERE #{index_predicate} " if index_predicate
+      sql
     end
   end
 
@@ -203,8 +203,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
     true
   end
 
-  private
-
   def database_version
     defined?(postgresql_version) ? postgresql_version : super
   end

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -97,7 +97,7 @@ module ActiveRecord::Import::SQLite3Adapter
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
 
-    sql = ' ON CONFLICT '
+    sql = ' ON CONFLICT '.dup
     conflict_target = sql_for_conflict_target( arg )
 
     columns = arg.fetch( :columns, [] )
@@ -150,9 +150,9 @@ module ActiveRecord::Import::SQLite3Adapter
     conflict_target = args[:conflict_target]
     index_predicate = args[:index_predicate]
     if conflict_target.present?
-      '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
-        sql << "WHERE #{index_predicate} " if index_predicate
-      end
+      sql = '(' + Array( conflict_target ).reject( &:blank? ).join( ', ' ) + ') '
+      sql += "WHERE #{index_predicate} " if index_predicate
+      sql
     end
   end
 
@@ -166,8 +166,6 @@ module ActiveRecord::Import::SQLite3Adapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
   end
 
-  private
-
   def database_version
     defined?(sqlite_version) ? sqlite_version : super
   end

data/lib/activerecord-import/import.rb

@@ -34,7 +34,7 @@ module ActiveRecord::Import #:nodoc:
       @validate_callbacks = klass._validate_callbacks.dup
 
       @validate_callbacks.each_with_index do |callback, i|
-        filter = callback.raw_filter
+        filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
         next unless filter.class.name =~ /Validations::PresenceValidator/ ||
                     (!@options[:validate_uniqueness] &&
                      filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
@@ -49,7 +49,7 @@ module ActiveRecord::Import #:nodoc:
           associations = klass.reflect_on_all_associations(:belongs_to)
           associations.each do |assoc|
             if (index = attrs.index(assoc.name))
-              key = assoc.foreign_key.to_sym
+              key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
               attrs[index] = key unless attrs.include?(key)
             end
           end
@@ -734,7 +734,10 @@ class ActiveRecord::Base
         set_attributes_and_mark_clean(models, return_obj, timestamps, options)
 
         # if there are auto-save associations on the models we imported that are new, import them as well
-        import_associations(models, options.dup) if options[:recursive]
+        if options[:recursive]
+          options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
+          import_associations(models, options.dup.merge(validate: false))
+        end
       end
 
       return_obj
@@ -775,21 +778,22 @@ class ActiveRecord::Base
       unless scope_columns.blank?
         scope_columns.zip(scope_values).each do |name, value|
           name_as_sym = name.to_sym
-          next if column_names.include?(name_as_sym)
-
-          is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
-          value = Array(value).first if is_sti
-
+          next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
           column_names << name_as_sym
           array_of_attributes.each { |attrs| attrs << value }
         end
       end
 
+      if finder_needs_type_condition?
+        unless column_names.include?(inheritance_column.to_sym)
+          column_names << inheritance_column.to_sym
+          array_of_attributes.each { |attrs| attrs << sti_name }
+        end
+      end
+
       columns = column_names.each_with_index.map do |name, i|
         column = columns_hash[name.to_s]
-
         raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
-
         column
       end
 
@@ -805,17 +809,29 @@ class ActiveRecord::Base
       if supports_import?
         # generate the sql
         post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
+        import_size = values_sql.size
+
+        batch_size = options[:batch_size] || import_size
+        run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
+        progress_proc = options[:batch_progress]
+        current_batch = 0
+        batches = (import_size / batch_size.to_f).ceil
 
-        batch_size = options[:batch_size] || values_sql.size
         values_sql.each_slice(batch_size) do |batch_values|
+          batch_started_at = Time.now.to_i
+
           # perform the inserts
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
             "#{model_name} Create Many" )
+
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results
+          current_batch += 1
+
+          progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
         end
       else
         transaction(requires_new: true) do
@@ -841,7 +857,7 @@ class ActiveRecord::Base
           model.id = id
 
           timestamps.each do |attr, value|
-            model.send(attr + "=", value)
+            model.send(attr + "=", value) if model.send(attr).nil?
           end
         end
       end
@@ -895,15 +911,19 @@ class ActiveRecord::Base
       changed_columns = model.changed
       association_reflections = model.class.reflect_on_all_associations(:belongs_to)
       association_reflections.each do |association_reflection|
-        column_name = association_reflection.foreign_key
         next if association_reflection.options[:polymorphic]
-        next if changed_columns.include?(column_name)
-        association = model.association(association_reflection.name)
-        association = association.target
-        next if association.blank? || model.public_send(column_name).present?
 
-        association_primary_key = association_reflection.association_primary_key
-        model.public_send("#{column_name}=", association.send(association_primary_key))
+        column_names = Array(association_reflection.foreign_key).map(&:to_s)
+        column_names.each_with_index do |column_name, column_index|
+          next if changed_columns.include?(column_name)
+
+          association = model.association(association_reflection.name)
+          association = association.target
+          next if association.blank? || model.public_send(column_name).present?
+
+          association_primary_key = Array(association_reflection.association_primary_key)[column_index]
+          model.public_send("#{column_name}=", association.send(association_primary_key))
+        end
       end
     end
 
@@ -916,8 +936,9 @@ class ActiveRecord::Base
       associated_objects_by_class = {}
       models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
 
-      # :on_duplicate_key_update and :returning not supported for associations
-      options.delete(:on_duplicate_key_update)
+      # :on_duplicate_key_update only supported for all fields
+      options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
+      # :returning not supported for associations
       options.delete(:returning)
 
       associated_objects_by_class.each_value do |associations|
@@ -949,8 +970,13 @@ class ActiveRecord::Base
         changed_objects.each do |child|
           child.public_send("#{association_reflection.foreign_key}=", model.id)
           # For polymorphic associations
+          association_name = if model.class.respond_to?(:polymorphic_name)
+            model.class.polymorphic_name
+          else
+            model.class.base_class
+          end
           association_reflection.type.try do |type|
-            child.public_send("#{type}=", model.class.base_class.name)
+            child.public_send("#{type}=", association_name)
           end
         end
         associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
@@ -1011,7 +1037,12 @@ class ActiveRecord::Base
       end
 
       # use tz as set in ActiveRecord::Base
-      timestamp = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
+      default_timezone = if ActiveRecord.respond_to?(:default_timezone)
+        ActiveRecord.default_timezone
+      else
+        ActiveRecord::Base.default_timezone
+      end
+      timestamp = default_timezone == :utc ? Time.now.utc : Time.now
 
       [:create, :update].each do |action|
         timestamp_columns[action].each do |column|

data/lib/activerecord-import/synchronize.rb

@@ -39,7 +39,7 @@ module ActiveRecord # :nodoc:
 
         next unless matched_instance
 
-        instance.send :clear_association_cache
+        instance.instance_variable_set :@association_cache, {}
         instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
         instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
 

data/lib/activerecord-import/value_sets_parser.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/array'
+
 module ActiveRecord::Import
   class ValueSetTooLargeError < StandardError
     attr_reader :size

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.7".freeze
+    VERSION = "1.4.0".freeze
   end
 end

data/test/{travis → github}/database.yml

@@ -1,7 +1,8 @@
 common: &common
   username: root
-  password:
+  password: root
   encoding: utf8
+  collation: utf8_general_ci
   host:     localhost
   database: activerecord_import_test
 
@@ -37,6 +38,7 @@ oracle:
 postgresql: &postgresql
   <<: *common
   username: postgres
+  password: postgres
   adapter: postgresql
   min_messages: warning
 

data/test/import_test.rb

@@ -169,7 +169,17 @@ describe "#import" do
       assert_difference "Dictionary.count", +1 do
         Dictionary.import dictionaries
       end
-      assert_equal "Dictionary", Dictionary.first.type
+      assert_equal "Dictionary", Dictionary.last.type
+    end
+
+    it "should import arrays successfully" do
+      columns = [:author_name, :title]
+      values =  [["Noah Webster", "Webster's Dictionary"]]
+
+      assert_difference "Dictionary.count", +1 do
+        Dictionary.import columns, values
+      end
+      assert_equal "Dictionary", Dictionary.last.type
     end
   end
 
@@ -405,6 +415,15 @@ describe "#import" do
         assert_equal 3, result.num_inserts if Topic.supports_import?
       end
     end
+
+    it "should accept and call an optional callable to run after each batch" do
+      lambda_called = 0
+
+      my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
+      Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
+
+      assert_equal 3, lambda_called
+    end
   end
 
   context "with :synchronize option" do
@@ -536,7 +555,11 @@ describe "#import" do
     context "when the timestamps columns are present" do
       setup do
         @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc, updated_at: 2.years.ago.utc, updated_on: 2.years.ago.utc)
-        ActiveRecord::Base.default_timezone = :utc
+        if ActiveRecord.respond_to?(:default_timezone)
+          ActiveRecord.default_timezone = :utc
+        else
+          ActiveRecord::Base.default_timezone = :utc
+        end
         Timecop.freeze(time) do
           assert_difference "Book.count", +2 do
             Book.import %w(title author_name publisher created_at created_on updated_at updated_on), [["LDAP", "Big Bird", "Del Rey", nil, nil, nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on, @existing_book.updated_at, @existing_book.updated_on]]
@@ -652,6 +675,14 @@ describe "#import" do
           assert_equal [val1, val2], scope.map(&column).sort
         end
 
+        context "for cards and decks" do
+          it "works when the polymorphic name is different than base class name" do
+            deck = Deck.create(id: 1, name: 'test')
+            deck.cards.import [:id, :deck_type], [[1, 'PlayingCard']]
+            assert_equal deck.cards.first.deck_type, "PlayingCard"
+          end
+        end
+
         it "works importing array of hashes" do
           scope.import [{ column => val1 }, { column => val2 }]
 

data/test/models/card.rb

@@ -0,0 +1,3 @@
+class Card < ActiveRecord::Base
+  belongs_to :deck, polymorphic: true
+end

data/test/models/customer.rb

@@ -0,0 +1,6 @@
+class Customer < ActiveRecord::Base
+  has_many :orders,
+    inverse_of: :customer,
+    primary_key: %i(account_id id),
+    foreign_key: %i(account_id customer_id)
+end

data/test/models/deck.rb

@@ -0,0 +1,6 @@
+class Deck < ActiveRecord::Base
+  has_many :cards
+  def self.polymorphic_name
+    "PlayingCard"
+  end
+end

data/test/models/order.rb

@@ -0,0 +1,6 @@
+class Order < ActiveRecord::Base
+  belongs_to :customer,
+    inverse_of: :orders,
+    primary_key: %i(account_id id),
+    foreign_key: %i(account_id customer_id)
+end

data/test/models/playing_card.rb

@@ -0,0 +1,2 @@
+class PlayingCard < ActiveRecord::Base
+end

data/test/schema/generic_schema.rb

@@ -52,6 +52,20 @@ ActiveRecord::Schema.define do
     t.string :name
   end
 
+  create_table :cards, force: :cascade do |t|
+    t.string :name
+    t.string :deck_type
+    t.integer :deck_id
+  end
+
+  create_table :decks, force: :cascade do |t|
+    t.string :name
+  end
+
+  create_table :playing_cards, force: :cascade do |t|
+    t.string :name
+  end
+
   create_table :books, force: :cascade do |t|
     t.string :title, null: false
     t.string :publisher, null: false, default: 'Default Publisher'
@@ -191,4 +205,15 @@ ActiveRecord::Schema.define do
       );
     ).split.join(' ').strip
   end
+
+  create_table :customers, force: :cascade do |t|
+    t.integer :account_id
+    t.string :name
+  end
+
+  create_table :orders, force: :cascade do |t|
+    t.integer :account_id
+    t.integer :customer_id
+    t.integer :amount
+  end
 end

data/test/support/postgresql/import_examples.rb

@@ -281,7 +281,7 @@ def should_support_postgresql_import_functionality
   end
 
   describe "with binary field" do
-    let(:binary_value) { "\xE0'c\xB2\xB0\xB3Bh\\\xC2M\xB1m\\I\xC4r".force_encoding('ASCII-8BIT') }
+    let(:binary_value) { "\xE0'c\xB2\xB0\xB3Bh\\\xC2M\xB1m\\I\xC4r".dup.force_encoding('ASCII-8BIT') }
     it "imports the correct values for binary fields" do
       alarms = [Alarm.new(device_id: 1, alarm_type: 1, status: 1, secret_key: binary_value)]
       assert_difference "Alarm.count", +1 do
@@ -290,6 +290,30 @@ def should_support_postgresql_import_functionality
       assert_equal(binary_value, Alarm.first.secret_key)
     end
   end
+
+  unless ENV["SKIP_COMPOSITE_PK"]
+    describe "with composite foreign keys" do
+      let(:account_id) { 555 }
+      let(:customer) { Customer.new(account_id: account_id, name: "foo") }
+      let(:order) { Order.new(account_id: account_id, amount: 100, customer: customer) }
+
+      it "imports and correctly maps foreign keys" do
+        assert_difference "Customer.count", +1 do
+          Customer.import [customer]
+        end
+
+        assert_difference "Order.count", +1 do
+          Order.import [order]
+        end
+
+        db_customer = Customer.last
+        db_order = Order.last
+
+        assert_equal db_customer.orders.last, db_order
+        assert_not_equal db_order.customer_id, nil
+      end
+    end
+  end
 end
 
 def should_support_postgresql_upsert_functionality

data/test/support/shared_examples/recursive_import.rb

@@ -176,7 +176,7 @@ def should_support_recursive_import
       end
     end
 
-    # If adapter supports on_duplicate_key_update, it is only applied to top level models so that SQL with invalid
+    # If adapter supports on_duplicate_key_update and specific columns are specified, it is only applied to top level models so that SQL with invalid
     # columns, keys, etc isn't generated for child associations when doing recursive import
     if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
       describe "on_duplicate_key_update" do
@@ -190,6 +190,26 @@ def should_support_recursive_import
             end
           end
         end
+
+        context "when :all fields are updated" do
+          setup do
+            Topic.import new_topics, recursive: true
+          end
+
+          it "updates associated objects" do
+            new_author_name = 'Richard Bachman'
+            topic = new_topics.first
+            topic.books.each do |book|
+              book.author_name = new_author_name
+            end
+            assert_nothing_raised do
+              Topic.import new_topics, recursive: true, on_duplicate_key_update: :all
+            end
+            Topic.find(topic.id).books.each do |book|
+              assert_equal new_author_name, book.author_name
+            end
+          end
+        end
       end
     end
 

data/test/test_helper.rb

@@ -1,4 +1,5 @@
 require 'pathname'
+require 'rake'
 test_dir = Pathname.new File.dirname(__FILE__)
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
@@ -48,8 +49,24 @@ adapter = ENV["ARE_DB"] || "sqlite3"
 FileUtils.mkdir_p 'log'
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
-ActiveRecord::Base.default_timezone = :utc
+
+if ENV['AR_VERSION'].to_f >= 6.0
+  yaml_config = if Gem::Version.new(Psych::VERSION) >= Gem::Version.new('3.2.1')
+    YAML.safe_load_file(test_dir.join("database.yml"), aliases: true)[adapter]
+  else
+    YAML.load_file(test_dir.join("database.yml"))[adapter]
+  end
+  config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
+  ActiveRecord::Base.configurations.configurations << config
+else
+  ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+end
+
+if ActiveRecord.respond_to?(:default_timezone)
+  ActiveRecord.default_timezone = :utc
+else
+  ActiveRecord::Base.default_timezone = :utc
+end
 
 require "activerecord-import"
 ActiveRecord::Base.establish_connection :test

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.4.0
 platform: ruby
 authors:
 - Zach Dennis
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2022-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -45,10 +45,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yaml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
-- ".travis.yml"
 - Brewfile
 - CHANGELOG.md
 - Gemfile
@@ -68,15 +68,13 @@ files:
 - benchmarks/models/test_memory.rb
 - benchmarks/models/test_myisam.rb
 - benchmarks/schema/mysql2_schema.rb
-- gemfiles/3.2.gemfile
-- gemfiles/4.0.gemfile
-- gemfiles/4.1.gemfile
 - gemfiles/4.2.gemfile
 - gemfiles/5.0.gemfile
 - gemfiles/5.1.gemfile
 - gemfiles/5.2.gemfile
 - gemfiles/6.0.gemfile
 - gemfiles/6.1.gemfile
+- gemfiles/7.0.gemfile
 - lib/activerecord-import.rb
 - lib/activerecord-import/active_record/adapters/abstract_adapter.rb
 - lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
@@ -114,6 +112,7 @@ files:
 - test/adapters/spatialite.rb
 - test/adapters/sqlite3.rb
 - test/database.yml.sample
+- test/github/database.yml
 - test/import_test.rb
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
@@ -125,11 +124,16 @@ files:
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
+- test/models/card.rb
 - test/models/chapter.rb
+- test/models/customer.rb
+- test/models/deck.rb
 - test/models/dictionary.rb
 - test/models/discount.rb
 - test/models/end_note.rb
 - test/models/group.rb
+- test/models/order.rb
+- test/models/playing_card.rb
 - test/models/promotion.rb
 - test/models/question.rb
 - test/models/rule.rb
@@ -164,14 +168,13 @@ files:
 - test/support/sqlite3/import_examples.rb
 - test/synchronize_test.rb
 - test/test_helper.rb
-- test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
 homepage: https://github.com/zdennis/activerecord-import
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -179,15 +182,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord
 test_files:
@@ -205,6 +208,7 @@ test_files:
 - test/adapters/spatialite.rb
 - test/adapters/sqlite3.rb
 - test/database.yml.sample
+- test/github/database.yml
 - test/import_test.rb
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
@@ -216,11 +220,16 @@ test_files:
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
+- test/models/card.rb
 - test/models/chapter.rb
+- test/models/customer.rb
+- test/models/deck.rb
 - test/models/dictionary.rb
 - test/models/discount.rb
 - test/models/end_note.rb
 - test/models/group.rb
+- test/models/order.rb
+- test/models/playing_card.rb
 - test/models/promotion.rb
 - test/models/question.rb
 - test/models/rule.rb
@@ -255,6 +264,5 @@ test_files:
 - test/support/sqlite3/import_examples.rb
 - test/synchronize_test.rb
 - test/test_helper.rb
-- test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb

data/.travis.yml

@@ -1,74 +0,0 @@
-language: ruby
-cache: bundler
-rvm:
-  - 2.5.5
-
-env:
-  global:
-    # https://github.com/discourse/discourse/blob/master/.travis.yml
-    - RUBY_GC_MALLOC_LIMIT=50000000
-  matrix:
-    - AR_VERSION=5.1
-    - AR_VERSION=5.2
-    - AR_VERSION=6.0
-
-matrix:
-  include:
-    - rvm: 2.3.8
-      env: AR_VERSION=3.2
-    - rvm: 2.3.8
-      env: AR_VERSION=4.0
-    - rvm: 2.3.8
-      env: AR_VERSION=4.1
-    - rvm: 2.3.8
-      env: AR_VERSION=4.2
-    - rvm: 2.3.8
-      env: AR_VERSION=5.0
-
-  fast_finish: true
-
-addons:
-  postgresql: "9.5"
-  apt:
-    sources:
-      - travis-ci/sqlite3
-      - mysql-5.7-trusty
-    packages:
-      - sqlite3
-      - mysql-server
-      - mysql-client
-      - postgresql-9.5-postgis-2.4
-
-before_install:
-  - gem update --system
-  - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
-  - sudo mysql_upgrade
-  - sudo service mysql restart
-
-before_script:
-  - mysql -e 'create database activerecord_import_test;'
-  - psql -c 'create database activerecord_import_test;' -U postgres
-  - psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
-  - psql -c 'create extension if not exists postgis;' -U postgres
-  - psql -c 'create extension if not exists "uuid-ossp";' -U postgres
-  - cp test/travis/database.yml test/database.yml
-
-script:
-  - bundle exec rake test:mysql2
-  - bundle exec rake test:mysql2_makara
-  - bundle exec rake test:mysql2spatial
-  - bundle exec rake test:postgis
-  - bundle exec rake test:postgresql
-  - bundle exec rake test:postgresql_makara
-  - bundle exec rake test:seamless_database_pool
-  - bundle exec rake test:spatialite
-  - bundle exec rake test:sqlite3
-  - bundle exec rubocop
-
-dist: xenial
-
-services:
-  - mysql
-  - postgresql
-
-sudo: required

data/gemfiles/3.2.gemfile

@@ -1,2 +0,0 @@
-gem 'activerecord', '~> 3.2.0'
-gem 'composite_primary_keys', '~> 5.0'

data/gemfiles/4.0.gemfile

@@ -1,2 +0,0 @@
-gem 'activerecord', '~> 4.0.0'
-gem 'composite_primary_keys', '~> 6.0'

data/gemfiles/4.1.gemfile

@@ -1,2 +0,0 @@
-gem 'activerecord', '~> 4.1.0'
-gem 'composite_primary_keys', '~> 7.0'