activerecord-import 0.17.2 → 1.1.0

data/Rakefile

@@ -23,6 +23,7 @@ ADAPTERS = %w(
   postgresql
   postgresql_makara
   postgis
+  makara_postgis
   sqlite3
   spatialite
   seamless_database_pool
@@ -31,7 +32,7 @@ ADAPTERS.each do |adapter|
   namespace :test do
     desc "Runs #{adapter} database tests."
     Rake::TestTask.new(adapter) do |t|
-      # FactoryGirl has an issue with warnings, so turn off, so noisy
+      # FactoryBot has an issue with warnings, so turn off, so noisy
       # t.warning = true
       t.test_files = FileList["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/active_record/*_test.rb", "test/#{adapter}/**/*_test.rb"]
     end

data/activerecord-import.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |gem|
   gem.email         = ["zach.dennis@gmail.com"]
   gem.summary       = "Bulk insert extension for ActiveRecord"
   gem.description   = "A library for bulk inserting data using ActiveRecord."
-  gem.homepage      = "http://github.com/zdennis/activerecord-import"
-  gem.license       = "Ruby"
+  gem.homepage      = "https://github.com/zdennis/activerecord-import"
+  gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = ActiveRecord::Import::VERSION
 
-  gem.required_ruby_version = ">= 1.9.2"
+  gem.required_ruby_version = ">= 2.0.0"
 
   gem.add_runtime_dependency "activerecord", ">= 3.2"
   gem.add_development_dependency "rake"

data/benchmarks/lib/cli_parser.rb

@@ -42,7 +42,8 @@ module BenchmarkOptionParser
       table_types: {},
       delete_on_finish: true,
       number_of_objects: [],
-      outputs: [] )
+      outputs: []
+    )
 
     opt_parser = OptionParser.new do |opts|
       opts.banner = BANNER

data/gemfiles/5.1.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 5.1.0'
+gem 'composite_primary_keys', '~> 10.0'

data/gemfiles/5.2.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 5.2.0'
+gem 'composite_primary_keys', '~> 11.0'

data/gemfiles/6.0.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 6.0.0'
+gem 'composite_primary_keys', '~> 12.0'

data/gemfiles/6.1.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 6.1.0'

data/lib/activerecord-import.rb

@@ -1,19 +1,6 @@
 # rubocop:disable Style/FileName
+require "active_support/lazy_load_hooks"
 
 ActiveSupport.on_load(:active_record) do
-  class ActiveRecord::Base
-    class << self
-      def establish_connection_with_activerecord_import(*args)
-        conn = establish_connection_without_activerecord_import(*args)
-        if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool)
-          require "activerecord-import/base"
-        end
-
-        ActiveRecord::Import.load_from_connection_pool connection_pool
-        conn
-      end
-      alias establish_connection_without_activerecord_import establish_connection
-      alias establish_connection establish_connection_with_activerecord_import
-    end
-  end
+  require "activerecord-import/base"
 end

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -16,7 +16,7 @@ module ActiveRecord::Import::AbstractAdapter
       sql2insert = base_sql + values.join( ',' ) + post_sql
       insert( sql2insert, *args )
 
-      [number_of_inserts, []]
+      ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
     end
 
     def pre_sql_statements(options)
@@ -45,8 +45,8 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements = []
 
       if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
-        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key] )
-      elsif options[:on_duplicate_key_update]
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
+      elsif logger && options[:on_duplicate_key_update]
         logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
       end
 
@@ -59,6 +59,12 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements
     end
 
+    def increment_locking_column!(table_name, results, locking_column)
+      if locking_column.present?
+        results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
+      end
+    end
+
     def supports_on_duplicate_key_update?
       false
     end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -49,16 +49,16 @@ module ActiveRecord::Import::MysqlAdapter
       end
     end
 
-    [number_of_inserts, []]
+    ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
   end
 
   # Returns the maximum number of bytes that the server will allow
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end
@@ -71,14 +71,11 @@ module ActiveRecord::Import::MysqlAdapter
 
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
-    if options.include?(:on_duplicate_key_update)
-      columns = options[:on_duplicate_key_update]
+    if (columns = options[:on_duplicate_key_update])
       case columns
       when Array then columns << column.to_sym unless columns.include?(column.to_sym)
       when Hash then columns[column.to_sym] = column.to_sym
       end
-    elsif !options[:ignore] && !options[:on_duplicate_key_ignore]
-      options[:on_duplicate_key_update] = [column.to_sym]
     end
   end
 
@@ -87,10 +84,11 @@ module ActiveRecord::Import::MysqlAdapter
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
     sql = ' ON DUPLICATE KEY UPDATE '
     arg = args.first
+    locking_column = args.last
     if arg.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
     elsif arg.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
     elsif arg.is_a?( String )
       sql << arg
     else
@@ -99,20 +97,22 @@ module ActiveRecord::Import::MysqlAdapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',')
   end
 
@@ -120,4 +120,10 @@ module ActiveRecord::Import::MysqlAdapter
   def duplicate_key_update_error?(exception) # :nodoc:
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
   end
+
+  def increment_locking_column!(table_name, results, locking_column)
+    if locking_column.present?
+      results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
+    end
+  end
 end

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -5,9 +5,10 @@ module ActiveRecord::Import::PostgreSQLAdapter
   MIN_VERSION_FOR_UPSERT = 90_500
 
   def insert_many( sql, values, options = {}, *args ) # :nodoc:
-    primary_key = options[:primary_key]
     number_of_inserts = 1
+    returned_values = []
     ids = []
+    results = []
 
     base_sql, post_sql = if sql.is_a?( String )
       [sql, '']
@@ -17,19 +18,47 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
     sql2insert = base_sql + values.join( ',' ) + post_sql
 
-    if primary_key.blank? || options[:no_returning]
+    columns = returning_columns(options)
+    if columns.blank? || (options[:no_returning] && !options[:recursive])
       insert( sql2insert, *args )
     else
-      ids = if primary_key.is_a?( Array )
-        # Select composite primary keys
+      returned_values = if columns.size > 1
+        # Select composite columns
         select_rows( sql2insert, *args )
       else
         select_values( sql2insert, *args )
       end
-      query_cache.clear if query_cache_enabled
+      clear_query_cache if query_cache_enabled
     end
 
-    [number_of_inserts, ids]
+    if options[:returning].blank?
+      ids = returned_values
+    elsif options[:primary_key].blank?
+      results = returned_values
+    else
+      # split primary key and returning columns
+      ids, results = split_ids_and_results(returned_values, columns, options)
+    end
+
+    ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
+  end
+
+  def split_ids_and_results(values, columns, options)
+    ids = []
+    results = []
+    id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
+    returning_indexes = Array(options[:returning]).map { |key| columns.index(key) }
+
+    values.each do |value|
+      value_array = Array(value)
+      ids << id_indexes.map { |i| value_array[i] }
+      results << returning_indexes.map { |i| value_array[i] }
+    end
+
+    ids.map!(&:first) if id_indexes.size == 1
+    results.map!(&:first) if returning_indexes.size == 1
+
+    [ids, results]
   end
 
   def next_value_for_sequence(sequence_name)
@@ -44,20 +73,27 @@ module ActiveRecord::Import::PostgreSQLAdapter
       if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update] && !options[:recursive]
         sql << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
       end
-    elsif options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
+    elsif logger && options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
       logger.warn "Ignoring on_duplicate_key_ignore because it is not supported by the database."
     end
 
     sql += super(table_name, options)
 
-    unless options[:no_returning] || options[:primary_key].blank?
-      primary_key = Array(options[:primary_key])
-      sql << " RETURNING \"#{primary_key.join('", "')}\""
+    columns = returning_columns(options)
+    unless columns.blank? || (options[:no_returning] && !options[:recursive])
+      sql << " RETURNING \"#{columns.join('", "')}\""
     end
 
     sql
   end
 
+  def returning_columns(options)
+    columns = []
+    columns += Array(options[:primary_key]) if options[:primary_key].present?
+    columns |= Array(options[:returning]) if options[:returning].present?
+    columns
+  end
+
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
     arg = options[:on_duplicate_key_update]
@@ -83,14 +119,15 @@ module ActiveRecord::Import::PostgreSQLAdapter
   # Returns a generated ON CONFLICT DO UPDATE statement given the passed
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
-    arg, primary_key = args
+    arg, primary_key, locking_column = args
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
 
-    sql = " ON CONFLICT "
+    sql = ' ON CONFLICT '
     conflict_target = sql_for_conflict_target( arg )
 
     columns = arg.fetch( :columns, [] )
+    condition = arg[:condition]
     if columns.respond_to?( :empty? ) && columns.empty?
       return sql << "#{conflict_target}DO NOTHING"
     end
@@ -102,31 +139,36 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
     sql << "#{conflict_target}DO UPDATE SET "
     if columns.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, columns )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
     elsif columns.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, columns )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
     elsif columns.is_a?( String )
       sql << columns
     else
       raise ArgumentError, 'Expected :columns to be an Array or Hash'
     end
+
+    sql << " WHERE #{condition}" if condition.present?
+
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{qc}=EXCLUDED.#{qc}"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{qc1}=EXCLUDED.#{qc2}"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -137,7 +179,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
     if constraint_name.present?
       "ON CONSTRAINT #{constraint_name} "
     elsif conflict_target.present?
-      '(' << Array( conflict_target ).reject( &:empty? ).join( ', ' ) << ') '.tap do |sql|
+      '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
         sql << "WHERE #{index_predicate} " if index_predicate
       end
     end
@@ -153,11 +195,17 @@ module ActiveRecord::Import::PostgreSQLAdapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
   end
 
-  def supports_on_duplicate_key_update?(current_version = postgresql_version)
-    current_version >= MIN_VERSION_FOR_UPSERT
+  def supports_on_duplicate_key_update?
+    database_version >= MIN_VERSION_FOR_UPSERT
   end
 
-  def support_setting_primary_key_of_imported_objects?
+  def supports_setting_primary_key_of_imported_objects?
     true
   end
+
+  private
+
+  def database_version
+    defined?(postgresql_version) ? postgresql_version : super
+  end
 end

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -1,18 +1,20 @@
 module ActiveRecord::Import::SQLite3Adapter
   include ActiveRecord::Import::ImportSupport
+  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
 
   MIN_VERSION_FOR_IMPORT = "3.7.11".freeze
+  MIN_VERSION_FOR_UPSERT = "3.24.0".freeze
   SQLITE_LIMIT_COMPOUND_SELECT = 500
 
   # Override our conformance to ActiveRecord::Import::ImportSupport interface
   # to ensure that we only support import in supported version of SQLite.
   # Which INSERT statements with multiple value sets was introduced in 3.7.11.
-  def supports_import?(current_version = sqlite_version)
-    if current_version >= MIN_VERSION_FOR_IMPORT
-      true
-    else
-      false
-    end
+  def supports_import?
+    database_version >= MIN_VERSION_FOR_IMPORT
+  end
+
+  def supports_on_duplicate_key_update?
+    database_version >= MIN_VERSION_FOR_UPSERT
   end
 
   # +sql+ can be a single string or an array. If it is an array all
@@ -37,19 +39,136 @@ module ActiveRecord::Import::SQLite3Adapter
       end
     end
 
-    [number_of_inserts, []]
+    ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
   end
 
-  def pre_sql_statements( options)
+  def pre_sql_statements( options )
     sql = []
     # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
-    if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:recursive]
+    if !supports_on_duplicate_key_update? && (options[:ignore] || options[:on_duplicate_key_ignore])
       sql << "OR IGNORE"
     end
     sql + super
   end
 
+  def post_sql_statements( table_name, options ) # :nodoc:
+    sql = []
+
+    if supports_on_duplicate_key_update?
+      # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
+      if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update]
+        sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
+      end
+    end
+
+    sql + super
+  end
+
   def next_value_for_sequence(sequence_name)
     %{nextval('#{sequence_name}')}
   end
+
+  # Add a column to be updated on duplicate key update
+  def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
+    arg = options[:on_duplicate_key_update]
+    if arg.is_a?( Hash )
+      columns = arg.fetch( :columns ) { arg[:columns] = [] }
+      case columns
+      when Array then columns << column.to_sym unless columns.include?( column.to_sym )
+      when Hash then columns[column.to_sym] = column.to_sym
+      end
+    elsif arg.is_a?( Array )
+      arg << column.to_sym unless arg.include?( column.to_sym )
+    end
+  end
+
+  # Returns a generated ON CONFLICT DO NOTHING statement given the passed
+  # in +args+.
+  def sql_for_on_duplicate_key_ignore( *args ) # :nodoc:
+    arg = args.first
+    conflict_target = sql_for_conflict_target( arg ) if arg.is_a?( Hash )
+    " ON CONFLICT #{conflict_target}DO NOTHING"
+  end
+
+  # Returns a generated ON CONFLICT DO UPDATE statement given the passed
+  # in +args+.
+  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
+    arg, primary_key, locking_column = args
+    arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
+    return unless arg.is_a?( Hash )
+
+    sql = ' ON CONFLICT '
+    conflict_target = sql_for_conflict_target( arg )
+
+    columns = arg.fetch( :columns, [] )
+    condition = arg[:condition]
+    if columns.respond_to?( :empty? ) && columns.empty?
+      return sql << "#{conflict_target}DO NOTHING"
+    end
+
+    conflict_target ||= sql_for_default_conflict_target( primary_key )
+    unless conflict_target
+      raise ArgumentError, 'Expected :conflict_target to be specified'
+    end
+
+    sql << "#{conflict_target}DO UPDATE SET "
+    if columns.is_a?( Array )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
+    elsif columns.is_a?( Hash )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
+    elsif columns.is_a?( String )
+      sql << columns
+    else
+      raise ArgumentError, 'Expected :columns to be an Array or Hash'
+    end
+
+    sql << " WHERE #{condition}" if condition.present?
+
+    sql
+  end
+
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
+    results = arr.map do |column|
+      qc = quote_column_name( column )
+      "#{qc}=EXCLUDED.#{qc}"
+    end
+    increment_locking_column!(table_name, results, locking_column)
+    results.join( ',' )
+  end
+
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
+    results = hsh.map do |column1, column2|
+      qc1 = quote_column_name( column1 )
+      qc2 = quote_column_name( column2 )
+      "#{qc1}=EXCLUDED.#{qc2}"
+    end
+    increment_locking_column!(table_name, results, locking_column)
+    results.join( ',' )
+  end
+
+  def sql_for_conflict_target( args = {} )
+    conflict_target = args[:conflict_target]
+    index_predicate = args[:index_predicate]
+    if conflict_target.present?
+      '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
+        sql << "WHERE #{index_predicate} " if index_predicate
+      end
+    end
+  end
+
+  def sql_for_default_conflict_target( primary_key )
+    conflict_target = Array(primary_key).join(', ')
+    "(#{conflict_target}) " if conflict_target.present?
+  end
+
+  # Return true if the statement is a duplicate key record error
+  def duplicate_key_update_error?(exception) # :nodoc:
+    exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
+  end
+
+  private
+
+  def database_version
+    defined?(sqlite_version) ? sqlite_version : super
+  end
 end