activerecord-import 1.0.3

data/benchmarks/lib/mysql2_benchmark.rb

@@ -0,0 +1,19 @@
+class Mysql2Benchmark < BenchmarkBase
+  def benchmark_all( array_of_cols_and_vals )
+    methods = self.methods.find_all { |m| m =~ /benchmark_/ }
+    methods.delete_if { |m| m =~ /benchmark_(all|model)/ }
+    methods.each { |method| send( method, array_of_cols_and_vals ) }
+  end
+
+  def benchmark_myisam( array_of_cols_and_vals )
+    bm_model( TestMyISAM, array_of_cols_and_vals )
+  end
+
+  def benchmark_innodb( array_of_cols_and_vals )
+    bm_model( TestInnoDb, array_of_cols_and_vals )
+  end
+
+  def benchmark_memory( array_of_cols_and_vals )
+    bm_model( TestMemory, array_of_cols_and_vals )
+  end
+end

data/benchmarks/lib/output_to_csv.rb

@@ -0,0 +1,19 @@
+require 'csv'
+
+module OutputToCSV
+  def self.output_results( filename, results )
+    CSV.open( filename, 'w' ) do |csv|
+      # Iterate over each result set, which contains many results
+      results.each do |result_set|
+        columns = []
+        times = []
+        result_set.each do |result|
+          columns << result.description
+          times << result.tms.real
+        end
+        csv << columns
+        csv << times
+      end
+    end
+  end
+end

data/benchmarks/lib/output_to_html.rb

@@ -0,0 +1,64 @@
+require 'erb'
+
+module OutputToHTML
+  TEMPLATE_HEADER = <<"EOT".freeze
+  <div>
+    All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
+    before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
+    the very first column (which is always the default ActiveRecord::Base.create method.
+   </div>
+EOT
+
+  TEMPLATE = <<"EOT".freeze
+ <style>
+ td#benchmarkTitle {
+   border: 1px solid black;
+   padding: 2px;
+   font-size: 0.8em;
+   background-color: black;
+   color: white;
+ }
+ td#benchmarkCell {
+   border: 1px solid black;
+   padding: 2px;
+   font-size: 0.8em;
+ }
+ </style>
+   <table>
+     <tr>
+       <% columns.each do |col| %>
+         <td id="benchmarkTitle"><%= col %></td>
+       <% end %>
+     </tr>
+     <tr>
+       <% times.each do |time| %>
+         <td id="benchmarkCell"><%= time %></td>
+       <% end %>
+     </tr>
+     <tr><td>&nbsp;</td></tr>
+   </table>
+EOT
+
+  def self.output_results( filename, results )
+    html = ''
+    results.each do |result_set|
+      columns = []
+      times = []
+      result_set.each do |result|
+        columns << result.description
+        if result.failed
+          times << "failed"
+        else
+          time = result.tms.real.round_to( 3 )
+          speedup = ( result_set.first.tms.real / result.tms.real ).round
+          times << (result == result_set.first ? time.to_s : "#{time} (#{speedup}x speedup)")
+        end
+      end
+
+      template = ERB.new( TEMPLATE, 0, "%<>")
+      html << template.result( binding )
+    end
+
+    File.open( filename, 'w' ) { |file| file.write( TEMPLATE_HEADER + html ) }
+  end
+end

data/benchmarks/models/test_innodb.rb

@@ -0,0 +1,3 @@
+class TestInnoDb < ActiveRecord::Base
+  self.table_name = 'test_innodb'
+end

data/benchmarks/models/test_memory.rb

@@ -0,0 +1,3 @@
+class TestMemory < ActiveRecord::Base
+  self.table_name = 'test_memory'
+end

data/benchmarks/models/test_myisam.rb

@@ -0,0 +1,3 @@
+class TestMyISAM < ActiveRecord::Base
+  self.table_name = 'test_myisam'
+end

data/benchmarks/schema/mysql_schema.rb

@@ -0,0 +1,16 @@
+ActiveRecord::Schema.define do
+  create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t|
+    t.column :my_name, :string, null: false
+    t.column :description, :string
+  end
+
+  create_table :test_innodb, options: 'ENGINE=InnoDb', force: true do |t|
+    t.column :my_name, :string, null: false
+    t.column :description, :string
+  end
+
+  create_table :test_memory, options: 'ENGINE=Memory', force: true do |t|
+    t.column :my_name, :string, null: false
+    t.column :description, :string
+  end
+end

data/gemfiles/3.2.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 3.2.0'
+gem 'composite_primary_keys', '~> 5.0'

data/gemfiles/4.0.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 4.0.0'
+gem 'composite_primary_keys', '~> 6.0'

data/gemfiles/4.1.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 4.1.0'
+gem 'composite_primary_keys', '~> 7.0'

data/gemfiles/4.2.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 4.2.0'
+gem 'composite_primary_keys', '~> 8.0'

data/gemfiles/5.0.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 5.0.0'
+gem 'composite_primary_keys', '~> 9.0'

data/gemfiles/5.1.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 5.1.0'
+gem 'composite_primary_keys', '~> 10.0'

data/gemfiles/5.2.gemfile

@@ -0,0 +1,2 @@
+gem 'activerecord', '~> 5.2.0'
+gem 'composite_primary_keys', '~> 11.0'

data/gemfiles/6.0.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 6.0.0'

data/gemfiles/6.1.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"

data/lib/activerecord-import.rb

@@ -0,0 +1,6 @@
+# rubocop:disable Style/FileName
+require "active_support/lazy_load_hooks"
+
+ActiveSupport.on_load(:active_record) do
+  require "activerecord-import/base"
+end

data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb

@@ -0,0 +1,9 @@
+require "activerecord-import/adapters/abstract_adapter"
+
+module ActiveRecord # :nodoc:
+  module ConnectionAdapters # :nodoc:
+    class AbstractAdapter # :nodoc:
+      include ActiveRecord::Import::AbstractAdapter::InstanceMethods
+    end
+  end
+end

data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/mysql_adapter"
+require "activerecord-import/adapters/mysql_adapter"
+
+class ActiveRecord::ConnectionAdapters::MysqlAdapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/postgresql_adapter"
+require "activerecord-import/adapters/postgresql_adapter"
+
+class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+  include ActiveRecord::Import::PostgreSQLAdapter
+end

data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/sqlite3_adapter"
+require "activerecord-import/adapters/sqlite3_adapter"
+
+class ActiveRecord::ConnectionAdapters::SQLite3Adapter
+  include ActiveRecord::Import::SQLite3Adapter
+end

data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/mysql2_adapter"
+require "activerecord-import/adapters/mysql2_adapter"
+
+class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+  include ActiveRecord::Import::Mysql2Adapter
+end

data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/postgresql_adapter"
+require "activerecord-import/adapters/postgresql_adapter"
+
+class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+  include ActiveRecord::Import::PostgreSQLAdapter
+end

data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb

@@ -0,0 +1,7 @@
+require "seamless_database_pool"
+require "active_record/connection_adapters/seamless_database_pool_adapter"
+require "activerecord-import/adapters/mysql_adapter"
+
+class ActiveRecord::ConnectionAdapters::SeamlessDatabasePoolAdapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/sqlite3_adapter"
+require "activerecord-import/adapters/sqlite3_adapter"
+
+class ActiveRecord::ConnectionAdapters::SQLite3Adapter
+  include ActiveRecord::Import::SQLite3Adapter
+end

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -0,0 +1,66 @@
+module ActiveRecord::Import::AbstractAdapter
+  module InstanceMethods
+    def next_value_for_sequence(sequence_name)
+      %(#{sequence_name}.nextval)
+    end
+
+    def insert_many( sql, values, _options = {}, *args ) # :nodoc:
+      number_of_inserts = 1
+
+      base_sql, post_sql = if sql.is_a?( String )
+        [sql, '']
+      elsif sql.is_a?( Array )
+        [sql.shift, sql.join( ' ' )]
+      end
+
+      sql2insert = base_sql + values.join( ',' ) + post_sql
+      insert( sql2insert, *args )
+
+      ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
+    end
+
+    def pre_sql_statements(options)
+      sql = []
+      sql << options[:pre_sql] if options[:pre_sql]
+      sql << options[:command] if options[:command]
+
+      # add keywords like IGNORE or DELAYED
+      if options[:keywords].is_a?(Array)
+        sql.concat(options[:keywords])
+      elsif options[:keywords]
+        sql << options[:keywords].to_s
+      end
+
+      sql
+    end
+
+    # Synchronizes the passed in ActiveRecord instances with the records in
+    # the database by calling +reload+ on each instance.
+    def after_import_synchronize( instances )
+      instances.each(&:reload)
+    end
+
+    # Returns an array of post SQL statements given the passed in options.
+    def post_sql_statements( table_name, options ) # :nodoc:
+      post_sql_statements = []
+
+      if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
+      elsif logger && options[:on_duplicate_key_update]
+        logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
+      end
+
+      # custom user post_sql
+      post_sql_statements << options[:post_sql] if options[:post_sql]
+
+      # with rollup
+      post_sql_statements << rollup_sql if options[:rollup]
+
+      post_sql_statements
+    end
+
+    def supports_on_duplicate_key_update?
+      false
+    end
+  end
+end

data/lib/activerecord-import/adapters/em_mysql2_adapter.rb

@@ -0,0 +1,5 @@
+require "activerecord-import/adapters/mysql_adapter"
+
+module ActiveRecord::Import::EMMysql2Adapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import/adapters/mysql2_adapter.rb

@@ -0,0 +1,5 @@
+require "activerecord-import/adapters/mysql_adapter"
+
+module ActiveRecord::Import::Mysql2Adapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -0,0 +1,129 @@
+module ActiveRecord::Import::MysqlAdapter
+  include ActiveRecord::Import::ImportSupport
+  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
+
+  NO_MAX_PACKET = 0
+  QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
+
+  # +sql+ can be a single string or an array. If it is an array all
+  # elements that are in position >= 1 will be appended to the final SQL.
+  def insert_many( sql, values, options = {}, *args ) # :nodoc:
+    # the number of inserts default
+    number_of_inserts = 0
+
+    base_sql, post_sql = if sql.is_a?( String )
+      [sql, '']
+    elsif sql.is_a?( Array )
+      [sql.shift, sql.join( ' ' )]
+    end
+
+    sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
+
+    # the number of bytes the requested insert statement values will take up
+    values_in_bytes = values.sum(&:bytesize)
+
+    # the number of bytes (commas) it will take to comma separate our values
+    comma_separated_bytes = values.size - 1
+
+    # the total number of bytes required if this statement is one statement
+    total_bytes = sql_size + values_in_bytes + comma_separated_bytes
+
+    max = max_allowed_packet
+
+    # if we can insert it all as one statement
+    if NO_MAX_PACKET == max || total_bytes <= max || options[:force_single_insert]
+      number_of_inserts += 1
+      sql2insert = base_sql + values.join( ',' ) + post_sql
+      insert( sql2insert, *args )
+    else
+      value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
+        reserved_bytes: sql_size,
+        max_bytes: max)
+
+      transaction(requires_new: true) do
+        value_sets.each do |value_set|
+          number_of_inserts += 1
+          sql2insert = base_sql + value_set.join( ',' ) + post_sql
+          insert( sql2insert, *args )
+        end
+      end
+    end
+
+    ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
+  end
+
+  # Returns the maximum number of bytes that the server will allow
+  # in a single packet
+  def max_allowed_packet # :nodoc:
+    @max_allowed_packet ||= begin
+      result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
+      # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val.to_i
+    end
+  end
+
+  def pre_sql_statements( options)
+    sql = []
+    sql << "IGNORE" if options[:ignore] || options[:on_duplicate_key_ignore]
+    sql + super
+  end
+
+  # Add a column to be updated on duplicate key update
+  def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
+    if (columns = options[:on_duplicate_key_update])
+      case columns
+      when Array then columns << column.to_sym unless columns.include?(column.to_sym)
+      when Hash then columns[column.to_sym] = column.to_sym
+      end
+    end
+  end
+
+  # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
+  # in +args+.
+  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
+    sql = ' ON DUPLICATE KEY UPDATE '
+    arg = args.first
+    locking_column = args.last
+    if arg.is_a?( Array )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
+    elsif arg.is_a?( Hash )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
+    elsif arg.is_a?( String )
+      sql << arg
+    else
+      raise ArgumentError, "Expected Array or Hash"
+    end
+    sql
+  end
+
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
+    results = arr.map do |column|
+      qc = quote_column_name( column )
+      "#{table_name}.#{qc}=VALUES(#{qc})"
+    end
+    increment_locking_column!(results, table_name, locking_column)
+    results.join( ',' )
+  end
+
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
+    results = hsh.map do |column1, column2|
+      qc1 = quote_column_name( column1 )
+      qc2 = quote_column_name( column2 )
+      "#{table_name}.#{qc1}=VALUES( #{qc2} )"
+    end
+    increment_locking_column!(results, table_name, locking_column)
+    results.join( ',')
+  end
+
+  # Return true if the statement is a duplicate key record error
+  def duplicate_key_update_error?(exception) # :nodoc:
+    exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
+  end
+
+  def increment_locking_column!(results, table_name, locking_column)
+    if locking_column.present?
+      results << "#{table_name}.`#{locking_column}`=`#{locking_column}`+1"
+    end
+  end
+end