activerecord-import 0.14.1 → 1.4.1

data/activerecord-import.gemspec

@@ -1,13 +1,14 @@
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
+
 require File.expand_path('../lib/activerecord-import/version', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.authors       = ["Zach Dennis"]
   gem.email         = ["zach.dennis@gmail.com"]
-  gem.summary       = "Bulk-loading extension for ActiveRecord"
-  gem.description   = "Extraction of the ActiveRecord::Base#import functionality from ar-extensions for Rails 3 and beyond"
-  gem.homepage      = "http://github.com/zdennis/activerecord-import"
-  gem.license       = "Ruby"
+  gem.summary       = "Bulk insert extension for ActiveRecord"
+  gem.description   = "A library for bulk inserting data using ActiveRecord."
+  gem.homepage      = "https://github.com/zdennis/activerecord-import"
+  gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,8 +17,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = ActiveRecord::Import::VERSION
 
-  gem.required_ruby_version = ">= 1.9.2"
+  gem.required_ruby_version = ">= 2.4.0"
 
-  gem.add_runtime_dependency "activerecord", ">= 3.2"
+  gem.add_runtime_dependency "activerecord", ">= 4.2"
   gem.add_development_dependency "rake"
 end

data/benchmarks/README

@@ -15,10 +15,10 @@ See "ruby benchmark.rb -h" for the complete listing of options.
 EXAMPLES
 --------
 To output to html format:
-  ruby benchmark.rb --adapter=mysql --to-html=results.html
+  ruby benchmark.rb --adapter=mysql2 --to-html=results.html
 
 To output to csv format:
-  ruby benchmark.rb --adapter=mysql --to-csv=results.csv
+  ruby benchmark.rb --adapter=mysql2 --to-csv=results.csv
 
 LIMITATIONS
 -----------

data/benchmarks/benchmark.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require "fileutils"
 require "active_record"
+require "active_record/base"
 
 benchmark_dir = File.dirname(__FILE__)
 
@@ -19,7 +22,11 @@ FileUtils.mkdir_p 'log'
 ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.default_timezone = :utc
+if ActiveRecord.respond_to?(:default_timezone)
+  ActiveRecord.default_timezone = :utc
+else
+  ActiveRecord::Base.default_timezone = :utc
+end
 
 require "activerecord-import"
 ActiveRecord::Base.establish_connection(:test)

data/benchmarks/lib/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BenchmarkBase
   attr_reader :results
 

data/benchmarks/lib/cli_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'optparse'
 require 'ostruct'
 
@@ -8,7 +10,7 @@ require 'ostruct'
 # * t - the table types to test. ie: myisam, innodb, memory, temporary, etc.
 #
 module BenchmarkOptionParser
-  BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options.".freeze
+  BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options."
 
   def self.print_banner
     puts BANNER
@@ -42,7 +44,8 @@ module BenchmarkOptionParser
       table_types: {},
       delete_on_finish: true,
       number_of_objects: [],
-      outputs: [] )
+      outputs: []
+    )
 
     opt_parser = OptionParser.new do |opts|
       opts.banner = BANNER

data/benchmarks/lib/float.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Taken from http://www.programmingishard.com/posts/show/128
 # Posted by rbates
 class Float

data/benchmarks/lib/mysql2_benchmark.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Mysql2Benchmark < BenchmarkBase
   def benchmark_all( array_of_cols_and_vals )
     methods = self.methods.find_all { |m| m =~ /benchmark_/ }

data/benchmarks/lib/output_to_csv.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'csv'
 
 module OutputToCSV

data/benchmarks/lib/output_to_html.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 module OutputToHTML
-  TEMPLATE_HEADER = <<"EOT".freeze
+  TEMPLATE_HEADER = <<"EOT"
   <div>
     All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
     before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
@@ -9,7 +11,7 @@ module OutputToHTML
    </div>
 EOT
 
-  TEMPLATE = <<"EOT".freeze
+  TEMPLATE = <<"EOT"
  <style>
  td#benchmarkTitle {
    border: 1px solid black;

data/benchmarks/models/test_innodb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class TestInnoDb < ActiveRecord::Base
   self.table_name = 'test_innodb'
 end

data/benchmarks/models/test_memory.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class TestMemory < ActiveRecord::Base
   self.table_name = 'test_memory'
 end

data/benchmarks/models/test_myisam.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class TestMyISAM < ActiveRecord::Base
   self.table_name = 'test_myisam'
 end

data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ActiveRecord::Schema.define do
   create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t|
     t.column :my_name, :string, null: false

data/gemfiles/4.2.gemfile

@@ -1,3 +1,4 @@
-platforms :ruby do
-  gem 'activerecord', '~> 4.2.0'
-end
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 4.2.0'
+gem 'composite_primary_keys', '~> 8.0'

data/gemfiles/5.0.gemfile

@@ -1,3 +1,4 @@
-platforms :ruby do
-  gem 'activerecord', '~> 5.0.0.beta3'
-end
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 5.0.0'
+gem 'composite_primary_keys', '~> 9.0'

data/gemfiles/5.1.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 5.1.0'
+gem 'composite_primary_keys', '~> 10.0'

data/gemfiles/5.2.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 5.2.0'
+gem 'composite_primary_keys', '~> 11.0'

data/gemfiles/6.0.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 6.0.0'
+gem 'composite_primary_keys', '~> 12.0'

data/gemfiles/6.1.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 6.1.0'
+gem 'composite_primary_keys', '~> 13.0'

data/gemfiles/7.0.gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem 'activerecord', '~> 7.0.0'
+gem 'composite_primary_keys', '~> 14.0'

data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "activerecord-import/adapters/abstract_adapter"
 
 module ActiveRecord # :nodoc:

data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb

@@ -1,6 +1,8 @@
-require "active_record/connection_adapters/mysql_adapter"
-require "activerecord-import/adapters/mysql_adapter"
+# frozen_string_literal: true
 
-class ActiveRecord::ConnectionAdapters::MysqlAdapter
-  include ActiveRecord::Import::MysqlAdapter
+require "active_record/connection_adapters/mysql2_adapter"
+require "activerecord-import/adapters/mysql2_adapter"
+
+class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+  include ActiveRecord::Import::Mysql2Adapter
 end

data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/postgresql_adapter"
 require "activerecord-import/adapters/postgresql_adapter"
 

data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/sqlite3_adapter"
+require "activerecord-import/adapters/sqlite3_adapter"
+
+class ActiveRecord::ConnectionAdapters::SQLite3Adapter
+  include ActiveRecord::Import::SQLite3Adapter
+end

data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/mysql2_adapter"
 require "activerecord-import/adapters/mysql2_adapter"
 

data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/postgresql_adapter"
 require "activerecord-import/adapters/postgresql_adapter"
 

data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "seamless_database_pool"
 require "active_record/connection_adapters/seamless_database_pool_adapter"
 require "activerecord-import/adapters/mysql_adapter"

data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_record/connection_adapters/sqlite3_adapter"
 require "activerecord-import/adapters/sqlite3_adapter"
 

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module ActiveRecord::Import::AbstractAdapter
   module InstanceMethods
     def next_value_for_sequence(sequence_name)
       %(#{sequence_name}.nextval)
     end
 
-    def insert_many( sql, values, *args ) # :nodoc:
+    def insert_many( sql, values, _options = {}, *args ) # :nodoc:
       number_of_inserts = 1
 
       base_sql, post_sql = if sql.is_a?( String )
@@ -16,14 +18,13 @@ module ActiveRecord::Import::AbstractAdapter
       sql2insert = base_sql + values.join( ',' ) + post_sql
       insert( sql2insert, *args )
 
-      [number_of_inserts, []]
+      ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
     end
 
     def pre_sql_statements(options)
       sql = []
       sql << options[:pre_sql] if options[:pre_sql]
       sql << options[:command] if options[:command]
-      sql << "IGNORE" if options[:ignore]
 
       # add keywords like IGNORE or DELAYED
       if options[:keywords].is_a?(Array)
@@ -45,15 +46,10 @@ module ActiveRecord::Import::AbstractAdapter
     def post_sql_statements( table_name, options ) # :nodoc:
       post_sql_statements = []
 
-      if supports_on_duplicate_key_update?
-        if options[:on_duplicate_key_ignore] && respond_to?(:sql_for_on_duplicate_key_ignore)
-          # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
-          unless options[:recursive]
-            post_sql_statements << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
-          end
-        elsif options[:on_duplicate_key_update]
-          post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] )
-        end
+      if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
+      elsif logger && options[:on_duplicate_key_update]
+        logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
       end
 
       # custom user post_sql
@@ -65,10 +61,10 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements
     end
 
-    # Returns the maximum number of bytes that the server will allow
-    # in a single packet
-    def max_allowed_packet
-      NO_MAX_PACKET
+    def increment_locking_column!(table_name, results, locking_column)
+      if locking_column.present?
+        results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
+      end
     end
 
     def supports_on_duplicate_key_update?

data/lib/activerecord-import/adapters/em_mysql2_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "activerecord-import/adapters/mysql_adapter"
 
 module ActiveRecord::Import::EMMysql2Adapter

data/lib/activerecord-import/adapters/mysql2_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "activerecord-import/adapters/mysql_adapter"
 
 module ActiveRecord::Import::Mysql2Adapter

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord::Import::MysqlAdapter
   include ActiveRecord::Import::ImportSupport
   include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
@@ -7,7 +9,7 @@ module ActiveRecord::Import::MysqlAdapter
 
   # +sql+ can be a single string or an array. If it is an array all
   # elements that are in position >= 1 will be appended to the final SQL.
-  def insert_many( sql, values, *args ) # :nodoc:
+  def insert_many( sql, values, options = {}, *args ) # :nodoc:
     # the number of inserts default
     number_of_inserts = 0
 
@@ -31,7 +33,7 @@ module ActiveRecord::Import::MysqlAdapter
     max = max_allowed_packet
 
     # if we can insert it all as one statement
-    if NO_MAX_PACKET == max || total_bytes < max
+    if NO_MAX_PACKET == max || total_bytes <= max || options[:force_single_insert]
       number_of_inserts += 1
       sql2insert = base_sql + values.join( ',' ) + post_sql
       insert( sql2insert, *args )
@@ -39,49 +41,56 @@ module ActiveRecord::Import::MysqlAdapter
       value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
         reserved_bytes: sql_size,
         max_bytes: max)
-      value_sets.each do |value_set|
-        number_of_inserts += 1
-        sql2insert = base_sql + value_set.join( ',' ) + post_sql
-        insert( sql2insert, *args )
+
+      transaction(requires_new: true) do
+        value_sets.each do |value_set|
+          number_of_inserts += 1
+          sql2insert = base_sql + value_set.join( ',' ) + post_sql
+          insert( sql2insert, *args )
+        end
       end
     end
 
-    [number_of_inserts, []]
+    ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
   end
 
   # Returns the maximum number of bytes that the server will allow
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end
 
+  def pre_sql_statements( options)
+    sql = []
+    sql << "IGNORE" if options[:ignore] || options[:on_duplicate_key_ignore]
+    sql + super
+  end
+
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
-    if options.include?(:on_duplicate_key_update)
-      columns = options[:on_duplicate_key_update]
+    if (columns = options[:on_duplicate_key_update])
       case columns
       when Array then columns << column.to_sym unless columns.include?(column.to_sym)
       when Hash then columns[column.to_sym] = column.to_sym
       end
-    else
-      options[:on_duplicate_key_update] = [column.to_sym]
     end
   end
 
   # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
   # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
-    sql = ' ON DUPLICATE KEY UPDATE '
+    sql = ' ON DUPLICATE KEY UPDATE '.dup
     arg = args.first
+    locking_column = args.last
     if arg.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
     elsif arg.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
     elsif arg.is_a?( String )
       sql << arg
     else
@@ -90,20 +99,22 @@ module ActiveRecord::Import::MysqlAdapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"
     end
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',')
   end
 
@@ -111,4 +122,10 @@ module ActiveRecord::Import::MysqlAdapter
   def duplicate_key_update_error?(exception) # :nodoc:
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
   end
+
+  def increment_locking_column!(table_name, results, locking_column)
+    if locking_column.present?
+      results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
+    end
+  end
 end