activerecord-import 0.22.0 → 1.4.1

data/benchmarks/benchmark.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require "fileutils"
 require "active_record"
@@ -20,7 +22,11 @@ FileUtils.mkdir_p 'log'
 ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.default_timezone = :utc
+if ActiveRecord.respond_to?(:default_timezone)
+  ActiveRecord.default_timezone = :utc
+else
+  ActiveRecord::Base.default_timezone = :utc
+end
 
 require "activerecord-import"
 ActiveRecord::Base.establish_connection(:test)

data/benchmarks/lib/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BenchmarkBase
   attr_reader :results
 

data/benchmarks/lib/cli_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'optparse'
 require 'ostruct'
 
@@ -8,7 +10,7 @@ require 'ostruct'
 # * t - the table types to test. ie: myisam, innodb, memory, temporary, etc.
 #
 module BenchmarkOptionParser
-  BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options.".freeze
+  BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options."
 
   def self.print_banner
     puts BANNER

data/benchmarks/lib/float.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Taken from http://www.programmingishard.com/posts/show/128
 # Posted by rbates
 class Float

data/benchmarks/lib/mysql2_benchmark.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Mysql2Benchmark < BenchmarkBase
   def benchmark_all( array_of_cols_and_vals )
     methods = self.methods.find_all { |m| m =~ /benchmark_/ }

data/benchmarks/lib/output_to_csv.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'csv'
 
 module OutputToCSV

data/benchmarks/lib/output_to_html.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 module OutputToHTML
-  TEMPLATE_HEADER = <<"EOT".freeze
+  TEMPLATE_HEADER = <<"EOT"
   <div>
     All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
     before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
@@ -9,7 +11,7 @@ module OutputToHTML
    </div>
 EOT
 
-  TEMPLATE = <<"EOT".freeze
+  TEMPLATE = <<"EOT"
  <style>
  td#benchmarkTitle {
    border: 1px solid black;

data/benchmarks/models/test_innodb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class TestInnoDb < ActiveRecord::Base
   self.table_name = 'test_innodb'
 end

data/benchmarks/models/test_memory.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class TestMemory < ActiveRecord::Base
   self.table_name = 'test_memory'
 end

data/benchmarks/models/test_myisam.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class TestMyISAM < ActiveRecord::Base
   self.table_name = 'test_myisam'
 end

data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ActiveRecord::Schema.define do
   create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t|
     t.column :my_name, :string, null: false

data/gemfiles/4.2.gemfile

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 gem 'activerecord', '~> 4.2.0'
 gem 'composite_primary_keys', '~> 8.0'

data/gemfiles/5.0.gemfile

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 gem 'activerecord', '~> 5.0.0'
 gem 'composite_primary_keys', '~> 9.0'

data/gemfiles/5.1.gemfile

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 gem 'activerecord', '~> 5.1.0'
 gem 'composite_primary_keys', '~> 10.0'

data/gemfiles/5.2.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 5.2.0'
+gem 'composite_primary_keys', '~> 11.0'

data/gemfiles/6.0.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 6.0.0'
+gem 'composite_primary_keys', '~> 12.0'

data/gemfiles/6.1.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 6.1.0'
+gem 'composite_primary_keys', '~> 13.0'

data/gemfiles/7.0.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 7.0.0'
+gem 'composite_primary_keys', '~> 14.0'

data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "activerecord-import/adapters/abstract_adapter"
 
 module ActiveRecord # :nodoc:

data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb

@@ -1,6 +1,8 @@
-require "active_record/connection_adapters/mysql_adapter"
-require "activerecord-import/adapters/mysql_adapter"
+# frozen_string_literal: true
 
-class ActiveRecord::ConnectionAdapters::MysqlAdapter
-  include ActiveRecord::Import::MysqlAdapter
+require "active_record/connection_adapters/mysql2_adapter"
+require "activerecord-import/adapters/mysql2_adapter"
+
+class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+  include ActiveRecord::Import::Mysql2Adapter
 end

data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/postgresql_adapter"
 require "activerecord-import/adapters/postgresql_adapter"
 

data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/sqlite3_adapter"
 require "activerecord-import/adapters/sqlite3_adapter"
 

data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/mysql2_adapter"
 require "activerecord-import/adapters/mysql2_adapter"
 

data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/postgresql_adapter"
 require "activerecord-import/adapters/postgresql_adapter"
 

data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "seamless_database_pool"
 require "active_record/connection_adapters/seamless_database_pool_adapter"
 require "activerecord-import/adapters/mysql_adapter"

data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/sqlite3_adapter"
 require "activerecord-import/adapters/sqlite3_adapter"
 

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord::Import::AbstractAdapter
   module InstanceMethods
     def next_value_for_sequence(sequence_name)
@@ -45,8 +47,8 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements = []
 
       if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
-        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key] )
-      elsif options[:on_duplicate_key_update]
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
+      elsif logger && options[:on_duplicate_key_update]
         logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
       end
 
@@ -59,6 +61,12 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements
     end
 
+    def increment_locking_column!(table_name, results, locking_column)
+      if locking_column.present?
+        results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
+      end
+    end
+
     def supports_on_duplicate_key_update?
       false
     end

data/lib/activerecord-import/adapters/em_mysql2_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "activerecord-import/adapters/mysql_adapter"
 
 module ActiveRecord::Import::EMMysql2Adapter

data/lib/activerecord-import/adapters/mysql2_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "activerecord-import/adapters/mysql_adapter"
 
 module ActiveRecord::Import::Mysql2Adapter

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord::Import::MysqlAdapter
   include ActiveRecord::Import::ImportSupport
   include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
@@ -56,9 +58,9 @@ module ActiveRecord::Import::MysqlAdapter
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end
@@ -71,26 +73,24 @@ module ActiveRecord::Import::MysqlAdapter
 
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
-    if options.include?(:on_duplicate_key_update)
-      columns = options[:on_duplicate_key_update]
+    if (columns = options[:on_duplicate_key_update])
       case columns
       when Array then columns << column.to_sym unless columns.include?(column.to_sym)
       when Hash then columns[column.to_sym] = column.to_sym
       end
-    elsif !options[:ignore] && !options[:on_duplicate_key_ignore]
-      options[:on_duplicate_key_update] = [column.to_sym]
     end
   end
 
   # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
-    sql = ' ON DUPLICATE KEY UPDATE '
+    sql = ' ON DUPLICATE KEY UPDATE '.dup
     arg = args.first
+    locking_column = args.last
     if arg.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
     elsif arg.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
     elsif arg.is_a?( String )
       sql << arg
     else
@@ -99,20 +99,22 @@ module ActiveRecord::Import::MysqlAdapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',')
   end
 
@@ -120,4 +122,10 @@ module ActiveRecord::Import::MysqlAdapter
   def duplicate_key_update_error?(exception) # :nodoc:
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
   end
+
+  def increment_locking_column!(table_name, results, locking_column)
+    if locking_column.present?
+      results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
+    end
+  end
 end

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord::Import::PostgreSQLAdapter
   include ActiveRecord::Import::ImportSupport
   include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
@@ -6,7 +8,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
   def insert_many( sql, values, options = {}, *args ) # :nodoc:
     number_of_inserts = 1
-    returned_values = []
+    returned_values = {}
     ids = []
     results = []
 
@@ -18,47 +20,53 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
     sql2insert = base_sql + values.join( ',' ) + post_sql
 
-    columns = returning_columns(options)
-    if columns.blank? || options[:no_returning]
+    selections = returning_selections(options)
+    if selections.blank? || (options[:no_returning] && !options[:recursive])
       insert( sql2insert, *args )
     else
-      returned_values = if columns.size > 1
+      returned_values = if selections.size > 1
         # Select composite columns
-        select_rows( sql2insert, *args )
+        db_result = select_all( sql2insert, *args )
+        { values: db_result.rows, columns: db_result.columns }
       else
-        select_values( sql2insert, *args )
+        { values: select_values( sql2insert, *args ) }
       end
-      query_cache.clear if query_cache_enabled
+      clear_query_cache if query_cache_enabled
     end
 
     if options[:returning].blank?
-      ids = returned_values
+      ids = Array(returned_values[:values])
     elsif options[:primary_key].blank?
-      results = returned_values
+      options[:returning_columns] ||= returned_values[:columns]
+      results = Array(returned_values[:values])
     else
       # split primary key and returning columns
-      ids, results = split_ids_and_results(returned_values, columns, options)
+      ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
     end
 
     ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
   end
 
-  def split_ids_and_results(values, columns, options)
+  def split_ids_and_results( selections, options )
     ids = []
-    results = []
+    returning_values = []
+
+    columns = Array(selections[:columns])
+    values = Array(selections[:values])
     id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
-    returning_indexes = Array(options[:returning]).map { |key| columns.index(key) }
+    returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
+    returning_indexes = returning_columns.map { |column| columns.index(column) }
 
     values.each do |value|
       value_array = Array(value)
-      ids << id_indexes.map { |i| value_array[i] }
-      results << returning_indexes.map { |i| value_array[i] }
+      ids << id_indexes.map { |index| value_array[index] }
+      returning_values << returning_indexes.map { |index| value_array[index] }
     end
 
     ids.map!(&:first) if id_indexes.size == 1
-    results.map!(&:first) if returning_indexes.size == 1
+    returning_values.map!(&:first) if returning_columns.size == 1
 
-    [ids, results]
+    [ids, returning_values, returning_columns]
   end
 
   def next_value_for_sequence(sequence_name)
@@ -73,25 +81,30 @@ module ActiveRecord::Import::PostgreSQLAdapter
       if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update] && !options[:recursive]
         sql << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
       end
-    elsif options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
+    elsif logger && options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
       logger.warn "Ignoring on_duplicate_key_ignore because it is not supported by the database."
     end
 
     sql += super(table_name, options)
 
-    columns = returning_columns(options)
-    unless columns.blank? || options[:no_returning]
-      sql << " RETURNING \"#{columns.join('", "')}\""
+    selections = returning_selections(options)
+    unless selections.blank? || (options[:no_returning] && !options[:recursive])
+      sql << " RETURNING #{selections.join(', ')}"
     end
 
     sql
   end
 
-  def returning_columns(options)
-    columns = []
-    columns += Array(options[:primary_key]) if options[:primary_key].present?
-    columns |= Array(options[:returning]) if options[:returning].present?
-    columns
+  def returning_selections(options)
+    selections = []
+    column_names = Array(options[:model].column_names)
+
+    selections += Array(options[:primary_key]) if options[:primary_key].present?
+    selections += Array(options[:returning]) if options[:returning].present?
+
+    selections.map do |selection|
+      column_names.include?(selection.to_s) ? "\"#{selection}\"" : selection
+    end
   end
 
   # Add a column to be updated on duplicate key update
@@ -119,11 +132,11 @@ module ActiveRecord::Import::PostgreSQLAdapter
   # Returns a generated ON CONFLICT DO UPDATE statement given the passed
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
-    arg, primary_key = args
+    arg, primary_key, locking_column = args
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
 
-    sql = ' ON CONFLICT '
+    sql = ' ON CONFLICT '.dup
     conflict_target = sql_for_conflict_target( arg )
 
     columns = arg.fetch( :columns, [] )
@@ -139,9 +152,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
     sql << "#{conflict_target}DO UPDATE SET "
     if columns.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, columns )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
     elsif columns.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, columns )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
     elsif columns.is_a?( String )
       sql << columns
     else
@@ -153,20 +166,22 @@ module ActiveRecord::Import::PostgreSQLAdapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{qc}=EXCLUDED.#{qc}"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{qc1}=EXCLUDED.#{qc2}"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -177,9 +192,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
     if constraint_name.present?
       "ON CONSTRAINT #{constraint_name} "
     elsif conflict_target.present?
-      '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
-        sql << "WHERE #{index_predicate} " if index_predicate
-      end
+      sql = '(' + Array( conflict_target ).reject( &:blank? ).join( ', ' ) + ') '
+      sql += "WHERE #{index_predicate} " if index_predicate
+      sql
     end
   end
 
@@ -193,11 +208,15 @@ module ActiveRecord::Import::PostgreSQLAdapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
   end
 
-  def supports_on_duplicate_key_update?(current_version = postgresql_version)
-    current_version >= MIN_VERSION_FOR_UPSERT
+  def supports_on_duplicate_key_update?
+    database_version >= MIN_VERSION_FOR_UPSERT
   end
 
   def supports_setting_primary_key_of_imported_objects?
     true
   end
+
+  def database_version
+    defined?(postgresql_version) ? postgresql_version : super
+  end
 end