activerecord-import 1.4.1 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b9c717d31cf5a4012568c9d8497947466408be7fef5610fba9143f7a911ed7c
-  data.tar.gz: 925bc5f645152eeb09f9583e9d50b9be67c4bfa8049c303571f82649262e4d5b
+  metadata.gz: f8b122b955b4a926db451a6a3ef72405ade6d69b54876bccb9ea51680486b328
+  data.tar.gz: 3af5c65f25ffb54d338d012aab4f3608513071287c8a759f3ba678850a81f6e0
 SHA512:
-  metadata.gz: 409d66e4d4e6e9ac31940cea097837d76c93cbeb4b95461323afc6a5f2ace736f945705e5ef33f2a851bb92c13c27e19ffbc14c7e776a0366f1d76091cafd51f
-  data.tar.gz: 6a0eaa48b85f7b1863dfb2b04c3c193a5e7e1108084556ba1b25fa303bccb40f62dd706ec8d4b5c3394239f167eb1807dfff4fe292edc1e4ad550719a9a63446
+  metadata.gz: 638f6a8062713b07b1c2ebe866fa647c254d42e594c0fa056b5a710bcd9a2560b2217db73a89f5804c9fc1a71d5de05fc98625771e6225e9e148f58048618079
+  data.tar.gz: 806d66d95034276b0daec9c6c207b8a6187ec0950a44b312267aeef4d491b1c9f5e036641df35950296180f124315fc220e9af5d5b4d4057278176dcf667b1c3

data/.github/workflows/test.yaml

@@ -20,19 +20,33 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - 3.1
+          - 3.2
         env:
+          - AR_VERSION: '7.1'
+            RUBYOPT: --enable-frozen-string-literal
           - AR_VERSION: '7.0'
             RUBYOPT: --enable-frozen-string-literal
           - AR_VERSION: 6.1
             RUBYOPT: --enable-frozen-string-literal
         include:
+          - ruby: 3.1
+            env:
+              AR_VERSION: '7.1'
+          - ruby: 3.1
+            env:
+              AR_VERSION: '7.0'
+          - ruby: 3.1
+            env:
+              AR_VERSION: 6.1
           - ruby: '3.0'
             env:
               AR_VERSION: '7.0'
           - ruby: '3.0'
             env:
               AR_VERSION: 6.1
+          - ruby: jruby
+            env:
+              AR_VERSION: '7.0'
           - ruby: 2.7
             env:
               AR_VERSION: '7.0'
@@ -42,6 +56,9 @@ jobs:
           - ruby: 2.7
             env:
               AR_VERSION: '6.0'
+          - ruby: jruby-9.3.10.0
+            env:
+              AR_VERSION: '6.1'
           - ruby: 2.6
             env:
               AR_VERSION: 5.2
@@ -59,7 +76,7 @@ jobs:
       AR_VERSION: ${{ matrix.env.AR_VERSION }}
       DB_DATABASE: activerecord_import_test
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
@@ -98,7 +115,7 @@ jobs:
     env:
       AR_VERSION: '7.0'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7

data/.rubocop.yml

@@ -66,10 +66,10 @@ Style/SymbolArray:
 Style/TrailingCommaInArrayLiteral:
   Enabled: false
 
-Layout/AlignArguments:
+Layout/ArgumentAlignment:
   Enabled: false
 
-Layout/AlignParameters:
+Layout/ParameterAlignment:
   EnforcedStyle: with_fixed_indentation
 
 Layout/EndAlignment:
@@ -81,7 +81,7 @@ Layout/ElseAlignment:
 Layout/EmptyLineAfterGuardClause:
   Enabled: false
 
-Layout/IndentHeredoc:
+Layout/HeredocIndentation:
   Enabled: false
 
 Layout/SpaceInsideParens:
@@ -102,7 +102,7 @@ Lint/PercentStringArray:
 Naming/HeredocDelimiterNaming:
   Enabled: false
 
-Naming/UncommunicativeMethodParamName:
+Naming/MethodParameterName:
   Enabled: false
 
 Security/YAMLLoad:

data/.rubocop_todo.yml

@@ -1,36 +1,26 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2016-03-17 18:14:55 -0700 using RuboCop version 0.38.0.
+# on 2023-02-15 00:58:14 UTC using RuboCop version 1.45.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 2
-Lint/HandleExceptions:
-  Exclude:
-    - 'lib/activerecord-import/base.rb'
-    - 'test/import_test.rb'
-
 # Offense count: 2
 Lint/RescueException:
   Exclude:
     - 'benchmarks/lib/cli_parser.rb'
     - 'test/import_test.rb'
 
-# Offense count: 4
-# Cop supports --auto-correct.
-# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods.
+# Offense count: 3
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
 Lint/UnusedMethodArgument:
   Exclude:
     - 'lib/activerecord-import/adapters/postgresql_adapter.rb'
     - 'lib/activerecord-import/import.rb'
 
 # Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: Keywords.
-# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW
-Style/CommentAnnotation:
+Style/CombinableLoops:
   Exclude:
-    - 'benchmarks/lib/cli_parser.rb'
-    - 'lib/activerecord-import/import.rb'
+    - 'test/support/shared_examples/recursive_import.rb'

data/CHANGELOG.md

@@ -1,8 +1,26 @@
+## Changes in 1.5.1
+
+### Fixes
+
+* Stop memoizing schema_columns_hash so dynamic schema changes are picked up. Thanks to @koshigoe via \##812.
+
+## Changes in 1.5.0
+
+### New Features
+
+* Add Rails 7.1 support. Thanks to @gucki via \##807.
+* Add support for alias attributes. Thanks to @leonidkroka via \##799.
+
+### Fixes
+
+* Support for multi-byte column names when splitting queries. Thanks to @TakuyaKurimoto via \##801.
+* Fix issue with track_validation_failures when import models. Thanks to @OtaYohihiro via \##798.
+
 ## Changes in 1.4.1
 
 ### Fixes
 
-* Fix importing models that have required belongs_to associations and use composite primary keys. Thanks to @thoughtbot-summer vi \##783.
+* Fix importing models that have required belongs_to associations and use composite primary keys. Thanks to @thoughtbot-summer via \##783.
 
 ## Changes in 1.4.0
 

data/Dockerfile

@@ -0,0 +1,23 @@
+# Use the official Ruby 3.2 image as a base image
+ARG RUBY_VERSION=3.2
+FROM ruby:${RUBY_VERSION}-bullseye
+
+# Set the working directory
+WORKDIR /usr/src/app
+
+# Install system packages
+RUN apt-get update -qq && \
+    apt-get install -y default-mysql-client postgresql postgresql-contrib vim && \
+    apt-get clean
+
+# Set environment variables
+ENV AR_VERSION=7.0
+
+# Copy all files
+COPY . .
+
+# Move sample database.yml and install gems
+RUN mv test/database.yml.sample test/database.yml && \
+    bundle install
+
+CMD ["irb"]

data/Gemfile

@@ -15,7 +15,7 @@ pg_version = '0.9'
 pg_version = '1.1' if version >= 6.1
 
 group :development, :test do
-  gem 'rubocop', '~> 0.71.0'
+  gem 'rubocop'
   gem 'rake'
 end
 
@@ -31,16 +31,16 @@ end
 platforms :jruby do
   gem "jdbc-mysql"
   gem "jdbc-postgres"
-  gem "activerecord-jdbcsqlite3-adapter",    "~> 1.3"
-  gem "activerecord-jdbcmysql-adapter",      "~> 1.3"
-  gem "activerecord-jdbcpostgresql-adapter", "~> 1.3"
+  gem "activerecord-jdbcsqlite3-adapter"
+  gem "activerecord-jdbcmysql-adapter"
+  gem "activerecord-jdbcpostgresql-adapter"
 end
 
 # Support libs
 gem "factory_bot"
 gem "timecop"
 gem "chronic"
-gem "mocha", "~> 1.3.0"
+gem "mocha", "~> 2.1.0"
 
 # Debugging
 platforms :jruby do
@@ -49,7 +49,7 @@ end
 
 platforms :ruby do
   gem "pry-byebug"
-  gem "pry", "~> 0.12.0"
+  gem "pry", "~> 0.14.0"
 end
 
 if version >= 4.0

data/README.markdown

@@ -245,8 +245,8 @@ Book.import columns, books, batch_size: 2, batch_progress: my_proc
 
 #### Recursive
 
-NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
-hashes or arrays as recursive inputs.
+> **Note**
+> This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
 
 Assume that Books <code>has_many</code> Reviews.
 
@@ -434,7 +434,8 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
 However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`,  `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
 
 If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
-NOTE: If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
+> **Note**
+> If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
 
 ### Callbacks
 
@@ -506,7 +507,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
 
 ### Requiring
 
-Note: These instructions will only work if you are using version 0.2.0 or higher.
+> **Note**
+> These instructions will only work if you are using version 0.2.0 or higher.
 
 #### Autoloading via Bundler
 
@@ -624,6 +626,19 @@ AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
 
 Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
 
+#### Docker Setup
+
+Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
+
+```bash
+brew install docker && brew install docker-compose
+```
+##### Steps
+
+1. In your terminal run `docker-compose up --build`
+1. In another tab/window run `docker-compose exec app bash`
+1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
+
 ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
 
 You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).

data/benchmarks/benchmark.rb

@@ -41,7 +41,7 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
 adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
 require adapter_schema if File.exist?(adapter_schema)
 
-Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
+Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
 
 require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
 
@@ -53,8 +53,8 @@ else
 end
 
 letter = options.adapter[0].chr
-clazz_str = letter.upcase + options.adapter[1..-1].downcase
-clazz = Object.const_get( clazz_str + "Benchmark" )
+clazz_str = letter.upcase + options.adapter[1..].downcase
+clazz = Object.const_get( "#{clazz_str}Benchmark" )
 
 benchmarks = []
 options.number_of_objects.each do |num|

data/benchmarks/lib/base.rb

@@ -27,11 +27,11 @@ class BenchmarkBase
   # An OpenStruct object with the following attributes:
   #   * description - the description of the benchmark ran
   #   * tms - a Benchmark::Tms containing the results of the benchmark
-  def bm( description )
+  def bm( description, &block )
     tms = nil
     puts "Benchmarking #{description}"
 
-    Benchmark.bm { |x| tms = x.report { yield } }
+    Benchmark.bm { |x| tms = x.report(&block) }
     delete_all
     failed = false
 

data/benchmarks/lib/cli_parser.rb

@@ -29,7 +29,7 @@ module BenchmarkOptionParser
     print_valid_table_types( options, prefix: "    " )
   end
 
-  # TODO IMPLEMENT THIS
+  # TODO: IMPLEMENT THIS
   def self.print_valid_table_types( options, hsh = { prefix: '' } )
     if !options.table_types.keys.empty?
       options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }

data/docker-compose.yml

@@ -0,0 +1,34 @@
+version: "3.5"
+
+services:
+  mysql:
+    platform: linux/x86_64
+    image: mysql:5.7
+    volumes:
+      - mysql-data:/var/lib/mysql
+    ports:
+      - "3306:3306"
+
+  postgresql:
+    image: postgres:latest
+    volumes:
+      - postgresql-data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  app:
+    build:
+      context: .
+    environment:
+      DB_HOST: mysql
+      AR_VERSION: 7.0
+    volumes:
+      - .:/usr/src/app
+    depends_on:
+      - mysql
+      - postgresql
+    command: tail -f /dev/null
+
+volumes:
+  mysql-data:
+  postgresql-data:

data/gemfiles/7.1.gemfile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 7.1.0'

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -9,10 +9,11 @@ module ActiveRecord::Import::AbstractAdapter
     def insert_many( sql, values, _options = {}, *args ) # :nodoc:
       number_of_inserts = 1
 
-      base_sql, post_sql = if sql.is_a?( String )
-        [sql, '']
-      elsif sql.is_a?( Array )
-        [sql.shift, sql.join( ' ' )]
+      base_sql, post_sql = case sql
+                           when String
+                             [sql, '']
+                           when Array
+                             [sql.shift, sql.join( ' ' )]
       end
 
       sql2insert = base_sql + values.join( ',' ) + post_sql
@@ -47,7 +48,7 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements = []
 
       if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
-        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:model], options[:primary_key], options[:locking_column] )
       elsif logger && options[:on_duplicate_key_update]
         logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
       end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -13,13 +13,14 @@ module ActiveRecord::Import::MysqlAdapter
     # the number of inserts default
     number_of_inserts = 0
 
-    base_sql, post_sql = if sql.is_a?( String )
-      [sql, '']
-    elsif sql.is_a?( Array )
-      [sql.shift, sql.join( ' ' )]
+    base_sql, post_sql = case sql
+                         when String
+                           [sql, '']
+                         when Array
+                           [sql.shift, sql.join( ' ' )]
     end
 
-    sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
+    sql_size = QUERY_OVERHEAD + base_sql.bytesize + post_sql.bytesize
 
     # the number of bytes the requested insert statement values will take up
     values_in_bytes = values.sum(&:bytesize)
@@ -33,7 +34,7 @@ module ActiveRecord::Import::MysqlAdapter
     max = max_allowed_packet
 
     # if we can insert it all as one statement
-    if NO_MAX_PACKET == max || total_bytes <= max || options[:force_single_insert]
+    if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
       number_of_inserts += 1
       sql2insert = base_sql + values.join( ',' ) + post_sql
       insert( sql2insert, *args )
@@ -85,13 +86,13 @@ module ActiveRecord::Import::MysqlAdapter
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
     sql = ' ON DUPLICATE KEY UPDATE '.dup
-    arg = args.first
-    locking_column = args.last
-    if arg.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
-    elsif arg.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
-    elsif arg.is_a?( String )
+    arg, model, _primary_key, locking_column = args
+    case arg
+    when Array
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arg )
+    when Hash
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, arg )
+    when String
       sql << arg
     else
       raise ArgumentError, "Expected Array or Hash"
@@ -99,19 +100,24 @@ module ActiveRecord::Import::MysqlAdapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
-      qc = quote_column_name( column )
+      original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
+      qc = quote_column_name( original_column_name )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
     increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
-      qc1 = quote_column_name( column1 )
-      qc2 = quote_column_name( column2 )
+      original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
+      qc1 = quote_column_name( original_column1_name )
+
+      original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
+      qc2 = quote_column_name( original_column2_name )
+
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"
     end
     increment_locking_column!(table_name, results, locking_column)

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -12,10 +12,11 @@ module ActiveRecord::Import::PostgreSQLAdapter
     ids = []
     results = []
 
-    base_sql, post_sql = if sql.is_a?( String )
-      [sql, '']
-    elsif sql.is_a?( Array )
-      [sql.shift, sql.join( ' ' )]
+    base_sql, post_sql = case sql
+                         when String
+                           [sql, '']
+                         when Array
+                           [sql.shift, sql.join( ' ' )]
     end
 
     sql2insert = base_sql + values.join( ',' ) + post_sql
@@ -110,13 +111,14 @@ module ActiveRecord::Import::PostgreSQLAdapter
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
     arg = options[:on_duplicate_key_update]
-    if arg.is_a?( Hash )
+    case arg
+    when Hash
       columns = arg.fetch( :columns ) { arg[:columns] = [] }
       case columns
       when Array then columns << column.to_sym unless columns.include?( column.to_sym )
       when Hash then columns[column.to_sym] = column.to_sym
       end
-    elsif arg.is_a?( Array )
+    when Array
       arg << column.to_sym unless arg.include?( column.to_sym )
     end
   end
@@ -132,7 +134,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
   # Returns a generated ON CONFLICT DO UPDATE statement given the passed
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
-    arg, primary_key, locking_column = args
+    arg, model, primary_key, locking_column = args
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
 
@@ -151,11 +153,12 @@ module ActiveRecord::Import::PostgreSQLAdapter
     end
 
     sql << "#{conflict_target}DO UPDATE SET "
-    if columns.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
-    elsif columns.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
-    elsif columns.is_a?( String )
+    case columns
+    when Array
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
+    when Hash
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
+    when String
       sql << columns
     else
       raise ArgumentError, 'Expected :columns to be an Array or Hash'
@@ -166,19 +169,24 @@ module ActiveRecord::Import::PostgreSQLAdapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
-      qc = quote_column_name( column )
+      original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
+      qc = quote_column_name( original_column_name )
       "#{qc}=EXCLUDED.#{qc}"
     end
     increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
-      qc1 = quote_column_name( column1 )
-      qc2 = quote_column_name( column2 )
+      original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
+      qc1 = quote_column_name( original_column1_name )
+
+      original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
+      qc2 = quote_column_name( original_column2_name )
+
       "#{qc1}=EXCLUDED.#{qc2}"
     end
     increment_locking_column!(table_name, results, locking_column)
@@ -192,7 +200,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
     if constraint_name.present?
       "ON CONSTRAINT #{constraint_name} "
     elsif conflict_target.present?
-      sql = '(' + Array( conflict_target ).reject( &:blank? ).join( ', ' ) + ') '
+      sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
       sql += "WHERE #{index_predicate} " if index_predicate
       sql
     end