activerecord-import 2.2.0 → 3.0.0

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -6,12 +6,17 @@ module ActiveRecord::Import::MysqlAdapter
 
   NO_MAX_PACKET = 0
   QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
+  MIN_MARIADB_VERSION_FOR_RETURNING = "10.5.0"
 
   # +sql+ can be a single string or an array. If it is an array all
   # elements that are in position >= 1 will be appended to the final SQL.
   def insert_many( sql, values, options = {}, *args ) # :nodoc:
     # the number of inserts default
     number_of_inserts = 0
+    affected_rows = nil
+    returning = import_returning?( options )
+    returned_columns = []
+    returned_rows = []
 
     base_sql, post_sql = case sql
                          when String
@@ -33,26 +38,55 @@ module ActiveRecord::Import::MysqlAdapter
 
     max = max_allowed_packet
 
+    insert_value_set = lambda do |value_set|
+      number_of_inserts += 1
+      sql2insert = base_sql + value_set.join( ',' ) + post_sql
+
+      if returning
+        result = exec_query( sql2insert, *args )
+        returned_columns = result.columns
+        returned_rows.concat(result.rows)
+
+        if result.respond_to?(:affected_rows) && result.affected_rows
+          affected_rows = (affected_rows || 0) + result.affected_rows
+        end
+      else
+        # Unlike #execute, whose result only has the affected rows with some drivers
+        # (e.g. trilogy, but not mysql2), #exec_update returns them with every driver
+        count = exec_update( sql2insert, *args )
+        affected_rows = (affected_rows || 0) + count if count.is_a?(Integer)
+      end
+    end
+
     # if we can insert it all as one statement
     if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
-      number_of_inserts += 1
-      sql2insert = base_sql + values.join( ',' ) + post_sql
-      insert( sql2insert, *args )
+      insert_value_set.call(values)
     else
       value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
         reserved_bytes: sql_size,
         max_bytes: max)
 
       transaction(requires_new: true) do
-        value_sets.each do |value_set|
-          number_of_inserts += 1
-          sql2insert = base_sql + value_set.join( ',' ) + post_sql
-          insert( sql2insert, *args )
-        end
+        value_sets.each { |value_set| insert_value_set.call(value_set) }
+      end
+    end
+
+    # ActiveRecord < 7.1 doesn't clear the query cache for #execute or #exec_query
+    clear_query_cache if query_cache_enabled
+
+    ids = []
+    results = []
+
+    if returning
+      returned_values = if returned_columns.size > 1
+        { values: returned_rows, columns: returned_columns }
+      else
+        { values: returned_rows.map(&:first) }
       end
+      ids, results = ids_and_results_from_returned_values(returned_values, options)
     end
 
-    ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
+    ActiveRecord::Import::Result.new([], number_of_inserts, ids, results, affected_rows)
   end
 
   # Returns the maximum number of bytes that the server will allow
@@ -68,10 +102,33 @@ module ActiveRecord::Import::MysqlAdapter
 
   def pre_sql_statements( options)
     sql = []
-    sql << "IGNORE" if options[:ignore] || options[:on_duplicate_key_ignore]
+    # Recursive imports need the id of every model, which INSERT IGNORE doesn't return
+    # for ignored rows, so :ignore and :on_duplicate_key_ignore don't apply to them.
+    if (options[:ignore] || options[:on_duplicate_key_ignore]) && !(options[:recursive] && supports_setting_primary_key_of_imported_objects?)
+      sql << "IGNORE"
+    end
     sql + super
   end
 
+  def post_sql_statements( table_name, options ) # :nodoc:
+    sql = super
+    sql << "RETURNING #{returning_selections(options).join(', ')}" if import_returning?( options )
+    sql
+  end
+
+  # MariaDB 10.5+ supports INSERT ... RETURNING, so the ids of imported models
+  # can be set. MySQL doesn't support it.
+  def supports_setting_primary_key_of_imported_objects?
+    respond_to?(:mariadb?) && mariadb? && database_version >= MIN_MARIADB_VERSION_FOR_RETURNING
+  end
+
+  # Returns true if the import should use INSERT ... RETURNING
+  def import_returning?( options ) # :nodoc:
+    supports_setting_primary_key_of_imported_objects? &&
+      returning_selections(options).present? &&
+      !(options[:no_returning] && !options[:recursive])
+  end
+
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
     if (columns = options[:on_duplicate_key_update])
@@ -102,7 +159,7 @@ module ActiveRecord::Import::MysqlAdapter
 
   def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
-      original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
+      original_column_name = model.column_name_for_import( column )
       qc = quote_column_name( original_column_name )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
@@ -112,10 +169,10 @@ module ActiveRecord::Import::MysqlAdapter
 
   def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
-      original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
+      original_column1_name = model.column_name_for_import( column1 )
       qc1 = quote_column_name( original_column1_name )
 
-      original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
+      original_column2_name = model.column_name_for_import( column2 )
       qc2 = quote_column_name( original_column2_name )
 
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -9,8 +9,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
   def insert_many( sql, values, options = {}, *args ) # :nodoc:
     number_of_inserts = 1
     returned_values = {}
-    ids = []
-    results = []
 
     base_sql, post_sql = case sql
                          when String
@@ -23,7 +21,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
     selections = returning_selections(options)
     if selections.blank? || (options[:no_returning] && !options[:recursive])
-      insert( sql2insert, *args )
+      # Don't use #insert here: ActiveRecord appends its own
+      # RETURNING <primary key> clause to statements run through it.
+      exec_update( sql2insert, *args )
     else
       returned_values = if selections.size > 1
         # Select composite columns
@@ -32,44 +32,14 @@ module ActiveRecord::Import::PostgreSQLAdapter
       else
         { values: select_values( sql2insert, *args ) }
       end
-      clear_query_cache if query_cache_enabled
     end
+    clear_query_cache if query_cache_enabled
 
-    if options[:returning].blank?
-      ids = Array(returned_values[:values])
-    elsif options[:primary_key].blank?
-      options[:returning_columns] ||= returned_values[:columns]
-      results = Array(returned_values[:values])
-    else
-      # split primary key and returning columns
-      ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
-    end
+    ids, results = ids_and_results_from_returned_values(returned_values, options)
 
     ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
   end
 
-  def split_ids_and_results( selections, options )
-    ids = []
-    returning_values = []
-
-    columns = Array(selections[:columns])
-    values = Array(selections[:values])
-    id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
-    returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
-    returning_indexes = returning_columns.map { |column| columns.index(column) }
-
-    values.each do |value|
-      value_array = Array(value)
-      ids << id_indexes.map { |index| value_array[index] }
-      returning_values << returning_indexes.map { |index| value_array[index] }
-    end
-
-    ids.map!(&:first) if id_indexes.size == 1
-    returning_values.map!(&:first) if returning_columns.size == 1
-
-    [ids, returning_values, returning_columns]
-  end
-
   def next_value_for_sequence(sequence_name)
     %{nextval('#{sequence_name}')}
   end
@@ -93,19 +63,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       sql << " RETURNING #{selections.join(', ')}"
     end
 
-    sql
-  end
-
-  def returning_selections(options)
-    selections = []
-    column_names = Array(options[:model].column_names)
-
-    selections += Array(options[:primary_key]) if options[:primary_key].present?
-    selections += Array(options[:returning]) if options[:returning].present?
-
-    selections.map do |selection|
-      column_names.include?(selection.to_s) ? "\"#{selection}\"" : selection
-    end
+    sql.uniq
   end
 
   # Add a column to be updated on duplicate key update
@@ -171,7 +129,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
   def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
-      original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
+      original_column_name = model.column_name_for_import( column )
       qc = quote_column_name( original_column_name )
       "#{qc}=EXCLUDED.#{qc}"
     end
@@ -181,10 +139,10 @@ module ActiveRecord::Import::PostgreSQLAdapter
 
   def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
-      original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
+      original_column1_name = model.column_name_for_import( column1 )
       qc1 = quote_column_name( original_column1_name )
 
-      original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
+      original_column2_name = model.column_name_for_import( column2 )
       qc2 = quote_column_name( original_column2_name )
 
       "#{qc1}=EXCLUDED.#{qc2}"
@@ -207,7 +165,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
   end
 
   def sql_for_default_conflict_target( table_name, primary_key )
-    conflict_target = Array(primary_key).join(', ')
+    conflict_target = Array(primary_key).map { |key| quote_column_name(key) }.join(', ')
     "(#{conflict_target}) " if conflict_target.present?
   end
 

data/lib/activerecord-import/adapters/postgresql_proxy_adapter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "activerecord-import/adapters/postgresql_adapter"
+require "activerecord-import/adapters/active_record_proxy_adapter"
+
+module ActiveRecord::Import::PostgreSQLProxyAdapter
+  include ActiveRecord::Import::PostgreSQLAdapter
+  include ActiveRecord::Import::ActiveRecordProxyAdapter
+end

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -19,6 +19,10 @@ module ActiveRecord::Import::SQLite3Adapter
     database_version >= MIN_VERSION_FOR_UPSERT
   end
 
+  def supports_default_keyword_in_values?
+    false
+  end
+
   # +sql+ can be a single string or an array. If it is an array all
   # elements that are in position >= 1 will be appended to the final SQL.
   def insert_many( sql, values, _options = {}, *args ) # :nodoc:
@@ -132,7 +136,7 @@ module ActiveRecord::Import::SQLite3Adapter
 
   def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
-      original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
+      original_column_name = model.column_name_for_import( column )
       qc = quote_column_name( original_column_name )
       "#{qc}=EXCLUDED.#{qc}"
     end
@@ -142,10 +146,10 @@ module ActiveRecord::Import::SQLite3Adapter
 
   def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
-      original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
+      original_column1_name = model.column_name_for_import( column1 )
       qc1 = quote_column_name( original_column1_name )
 
-      original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
+      original_column2_name = model.column_name_for_import( column2 )
       qc2 = quote_column_name( original_column2_name )
 
       "#{qc1}=EXCLUDED.#{qc2}"

data/lib/activerecord-import/adapters/sqlite3_proxy_adapter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "activerecord-import/adapters/sqlite3_adapter"
+require "activerecord-import/adapters/active_record_proxy_adapter"
+
+module ActiveRecord::Import::SQLite3ProxyAdapter
+  include ActiveRecord::Import::SQLite3Adapter
+  include ActiveRecord::Import::ActiveRecordProxyAdapter
+end

data/lib/activerecord-import/adapters/trilogy_proxy_adapter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "activerecord-import/adapters/trilogy_adapter"
+require "activerecord-import/adapters/active_record_proxy_adapter"
+
+module ActiveRecord::Import::TrilogyProxyAdapter
+  include ActiveRecord::Import::TrilogyAdapter
+  include ActiveRecord::Import::ActiveRecordProxyAdapter
+end

data/lib/activerecord-import/aliases.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module ActiveRecord::Import
+  # Methods aliased to +bulk_import+ and +bulk_import!+, keyed by the class
+  # they're defined on.
+  ALIASES = {
+    ActiveRecord::Associations::CollectionProxy => { import: :bulk_import },
+    ActiveRecord::Associations::CollectionAssociation => { import: :bulk_import },
+    ActiveRecord::Base.singleton_class => { import: :bulk_import, import!: :bulk_import! }
+  }.freeze
+
+  # Aliases +import+ and +import!+ to +bulk_import+ and +bulk_import!+, unless
+  # they are already defined or aliases have been skipped.
+  def self.define_aliases
+    return if skip_aliases?
+
+    ALIASES.each do |klass, aliases|
+      aliases.each do |name, target|
+        klass.send(:alias_method, name, target) unless klass.method_defined?(name)
+      end
+    end
+  end
+
+  # Prevents the +import+ and +import!+ aliases from being defined, and removes
+  # them if they already have been. Used by "activerecord-import/without_aliases".
+  def self.skip_aliases!
+    @skip_aliases = true
+
+    ALIASES.each do |klass, aliases|
+      aliases.each do |name, target|
+        next unless klass.method_defined?(name)
+
+        method = klass.instance_method(name)
+        klass.send(:remove_method, name) if method.owner == klass && method.original_name == target
+      end
+    end
+  end
+
+  def self.skip_aliases?
+    @skip_aliases == true
+  end
+end

data/lib/activerecord-import/base.rb

@@ -1,49 +1,5 @@
 # frozen_string_literal: true
 
-require "pathname"
-require "active_record"
-require "active_record/version"
+require "activerecord-import/core"
 
-module ActiveRecord::Import
-  ADAPTER_PATH = "activerecord-import/active_record/adapters"
-
-  def self.base_adapter(adapter)
-    case adapter
-    when 'mysql2_makara' then 'mysql2'
-    when 'mysql2spatial' then 'mysql2'
-    when 'mysql2_proxy' then 'mysql2'
-    when 'janus_mysql2' then 'mysql2'
-    when 'spatialite' then 'sqlite3'
-    when 'postgresql_makara' then 'postgresql'
-    when 'postgresql_proxy' then 'postgresql'
-    when 'makara_postgis' then 'postgresql'
-    when 'postgis' then 'postgresql'
-    when 'cockroachdb' then 'postgresql'
-    when 'janus_trilogy' then 'trilogy'
-    else adapter
-    end
-  end
-
-  # Loads the import functionality for a specific database adapter
-  def self.require_adapter(adapter)
-    require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
-  rescue LoadError
-    # fallback
-  end
-
-  # Loads the import functionality for the passed in ActiveRecord connection
-  def self.load_from_connection_pool(connection_pool)
-    adapter =
-      if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
-        connection_pool.db_config.adapter
-      else
-        connection_pool.spec.config[:adapter]
-      end
-    require_adapter adapter
-  end
-end
-
-require 'activerecord-import/import'
-require 'activerecord-import/active_record/adapters/abstract_adapter'
-require 'activerecord-import/synchronize'
-require 'activerecord-import/value_sets_parser'
+ActiveRecord::Import.define_aliases

data/lib/activerecord-import/core.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "concurrent/map"
+require "active_record"
+require "active_record/version"
+
+module ActiveRecord::Import
+  ADAPTER_PATH = "activerecord-import/active_record/adapters"
+  ATTEMPTED_ADAPTERS = Concurrent::Map.new # :nodoc:
+
+  def self.base_adapter(adapter)
+    case adapter
+    when 'mysql2_makara' then 'mysql2'
+    when 'mysql2spatial' then 'mysql2'
+    when 'janus_mysql2' then 'mysql2'
+    when 'spatialite' then 'sqlite3'
+    when 'postgresql_makara' then 'postgresql'
+    when 'makara_postgis' then 'postgresql'
+    when 'postgis' then 'postgresql'
+    when 'cockroachdb' then 'postgresql'
+    when 'janus_trilogy' then 'trilogy'
+    else adapter
+    end
+  end
+
+  # Loads the import functionality for a specific database adapter
+  def self.require_adapter(adapter)
+    require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
+  rescue LoadError
+    # fallback
+  end
+
+  # Loads the import functionality for the passed in ActiveRecord connection
+  def self.load_from_connection_pool(connection_pool)
+    require_adapter adapter_for(connection_pool)
+  end
+
+  # Loads the import functionality for the passed in ActiveRecord connection,
+  # unless it's been tried for the connection's adapter already. Import uses
+  # this when activerecord-import was required after the connection was
+  # established. Each adapter is only tried once, so an unsupported adapter
+  # doesn't search the load path on every import.
+  def self.load_from_connection_pool_once(connection_pool)
+    adapter = adapter_for(connection_pool)
+    ATTEMPTED_ADAPTERS.compute_if_absent(adapter) do
+      require_adapter adapter
+      true
+    end
+  end
+
+  def self.adapter_for(connection_pool) # :nodoc:
+    connection_pool.db_config.adapter
+  end
+  private_class_method :adapter_for
+end
+
+require 'activerecord-import/import'
+require 'activerecord-import/active_record/adapters/abstract_adapter'
+require 'activerecord-import/synchronize'
+require 'activerecord-import/value_sets_parser'
+require 'activerecord-import/aliases'