activerecord-import-rails4 0.5.0

data/benchmarks/lib/float.rb

@@ -0,0 +1,15 @@
+# Taken from http://www.programmingishard.com/posts/show/128
+# Posted by rbates
+class Float
+  def round_to(x)
+    (self * 10**x).round.to_f / 10**x
+  end
+
+  def ceil_to(x)
+    (self * 10**x).ceil.to_f / 10**x
+  end
+
+  def floor_to(x)
+    (self * 10**x).floor.to_f / 10**x
+  end
+end

data/benchmarks/lib/mysql_benchmark.rb

@@ -0,0 +1,22 @@
+class MysqlBenchmark < BenchmarkBase
+    
+  def benchmark_all( array_of_cols_and_vals )
+    methods = self.methods.find_all { |m| m =~ /benchmark_/ }
+    methods.delete_if{ |m| m =~ /benchmark_(all|model)/ }
+    methods.each { |method| self.send( method, array_of_cols_and_vals ) }
+  end
+  
+  def benchmark_myisam( array_of_cols_and_vals )
+    bm_model( TestMyISAM, array_of_cols_and_vals )
+  end
+    
+  def benchmark_innodb( array_of_cols_and_vals )
+    bm_model( TestInnoDb, array_of_cols_and_vals )
+  end
+  
+  def benchmark_memory( array_of_cols_and_vals )
+    bm_model( TestMemory, array_of_cols_and_vals )
+  end
+    
+end
+

data/benchmarks/lib/output_to_csv.rb

@@ -0,0 +1,18 @@
+require 'fastercsv'
+
+module OutputToCSV
+  def self.output_results( filename, results )
+    FasterCSV.open( filename, 'w' ) do |csv|
+      # Iterate over each result set, which contains many results
+      results.each do |result_set|
+        columns, times = [], []
+        result_set.each do |result| 
+          columns << result.description
+          times << result.tms.real
+        end
+        csv << columns
+        csv << times
+      end
+    end
+  end
+end

data/benchmarks/lib/output_to_html.rb

@@ -0,0 +1,69 @@
+require 'erb'
+
+module OutputToHTML
+
+TEMPLATE_HEADER =<<"EOT"
+  <div>
+    All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
+    before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
+    the very first column (which is always the default ActiveRecord::Base.create method.
+   </div>
+EOT
+
+TEMPLATE =<<"EOT"
+ <style>
+ td#benchmarkTitle {
+   border: 1px solid black;
+   padding: 2px;
+   font-size: 0.8em;
+   background-color: black;
+   color: white;
+ }
+ td#benchmarkCell {
+   border: 1px solid black;
+   padding: 2px;
+   font-size: 0.8em;
+ }
+ </style>
+   <table>
+     <tr>
+       <% columns.each do |col| %>
+         <td id="benchmarkTitle"><%= col %></td>
+       <% end %>
+     </tr>
+     <tr>
+       <% times.each do |time| %>
+         <td id="benchmarkCell"><%= time %></td>
+       <% end %>
+     </tr>
+     <tr><td>&nbsp;</td></tr>
+   </table>
+EOT
+
+  def self.output_results( filename, results )
+    html = ''
+    results.each do |result_set|
+      columns, times = [], []
+      result_set.each do |result|
+        columns << result.description
+        if result.failed
+          times << "failed"
+        else
+          time = result.tms.real.round_to( 3 )
+          speedup = ( result_set.first.tms.real / result.tms.real ).round 
+
+          if result == result_set.first
+            times << "#{time}"
+          else
+            times << "#{time} (#{speedup}x speedup)"
+          end
+        end
+      end
+
+      template = ERB.new( TEMPLATE, 0, "%<>")
+      html << template.result( binding )
+    end
+    
+    File.open( filename, 'w' ){ |file| file.write( TEMPLATE_HEADER + html ) }
+  end
+end

data/benchmarks/models/test_innodb.rb

@@ -0,0 +1,3 @@
+class TestInnoDb < ActiveRecord::Base
+  set_table_name 'test_innodb'
+end

data/benchmarks/models/test_memory.rb

@@ -0,0 +1,3 @@
+class TestMemory < ActiveRecord::Base
+  set_table_name 'test_memory'
+end

data/benchmarks/models/test_myisam.rb

@@ -0,0 +1,3 @@
+class TestMyISAM < ActiveRecord::Base
+  set_table_name 'test_myisam'
+end

data/benchmarks/schema/mysql_schema.rb

@@ -0,0 +1,16 @@
+ActiveRecord::Schema.define do
+  create_table :test_myisam, :options=>'ENGINE=MyISAM', :force=>true do |t|
+    t.column :my_name, :string, :null=>false
+    t.column :description, :string
+  end
+  
+  create_table :test_innodb, :options=>'ENGINE=InnoDb', :force=>true do |t|
+    t.column :my_name, :string, :null=>false
+    t.column :description, :string
+  end
+
+  create_table :test_memory, :options=>'ENGINE=Memory', :force=>true do |t|
+    t.column :my_name, :string, :null=>false
+    t.column :description, :string
+  end
+end

data/gemfiles/rails3.gemfile

@@ -0,0 +1,18 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "pg", "~> 0.9", :platforms=>:ruby
+gem "sqlite3-ruby", "~> 1.3.1", :platforms=>:ruby
+gem "seamless_database_pool", "~> 1.0.11", :platforms=>:ruby
+gem "jdbc-mysql", :platforms=>:jruby
+gem "activerecord-jdbcmysql-adapter", :platforms=>:jruby
+gem "factory_girl", "~> 4.2.0"
+gem "delorean", "~> 0.2.0"
+gem "ruby-debug-base", "= 0.10.4", :platforms=>:jruby
+gem "ruby-debug", "= 0.10.4", :platforms=>:jruby
+gem "debugger", :platforms=>:mri_19
+gem "mysql", "~> 2.8.1", :platforms=>:ruby
+gem "activerecord", "~> 3.0"
+
+gemspec :path=>"../"

data/gemfiles/rails4.gemfile

@@ -0,0 +1,18 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "pg", "~> 0.9", :platforms=>:ruby
+gem "sqlite3-ruby", "~> 1.3.1", :platforms=>:ruby
+gem "seamless_database_pool", "~> 1.0.11", :platforms=>:ruby
+gem "jdbc-mysql", :platforms=>:jruby
+gem "activerecord-jdbcmysql-adapter", :platforms=>:jruby
+gem "factory_girl", "~> 4.2.0"
+gem "delorean", "~> 0.2.0"
+gem "ruby-debug-base", "= 0.10.4", :platforms=>:jruby
+gem "ruby-debug", "= 0.10.4", :platforms=>:jruby
+gem "debugger", :platforms=>:mri_19
+gem "mysql", "~> 2.9", :platforms=>:ruby
+gem "activerecord", "~> 4.0.0.beta1"
+
+gemspec :path=>"../"

data/lib/activerecord-import-rails4.rb

@@ -0,0 +1,16 @@
+class ActiveRecord::Base
+  class << self
+    def establish_connection_with_activerecord_import(*args)
+      establish_connection_without_activerecord_import(*args)
+      ActiveSupport.run_load_hooks(:active_record_connection_established, connection_pool)
+    end
+    alias_method_chain :establish_connection, :activerecord_import
+  end
+end
+
+ActiveSupport.on_load(:active_record_connection_established) do |connection_pool|
+  if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool)
+    require File.join File.dirname(__FILE__),  "activerecord-import-rails4/base"
+  end
+  ActiveRecord::Import.load_from_connection_pool connection_pool
+end

data/lib/activerecord-import-rails4/active_record/adapters/abstract_adapter.rb

@@ -0,0 +1,10 @@
+require "activerecord-import-rails4/adapters/abstract_adapter"
+
+module ActiveRecord # :nodoc:
+  module ConnectionAdapters # :nodoc:
+    class AbstractAdapter # :nodoc:
+      extend ActiveRecord::Import::AbstractAdapter::ClassMethods
+      include ActiveRecord::Import::AbstractAdapter::InstanceMethods
+    end
+  end
+end

data/lib/activerecord-import-rails4/active_record/adapters/jdbcmysql_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/mysql_adapter"
+require "activerecord-import-rails4/adapters/mysql_adapter"
+
+class ActiveRecord::ConnectionAdapters::MysqlAdapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import-rails4/active_record/adapters/mysql2_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/mysql2_adapter"
+require "activerecord-import-rails4/adapters/mysql2_adapter"
+
+class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+  include ActiveRecord::Import::Mysql2Adapter
+end

data/lib/activerecord-import-rails4/active_record/adapters/mysql_adapter.rb

@@ -0,0 +1,6 @@
+require "active_record/connection_adapters/mysql_adapter"
+require "activerecord-import-rails4/adapters/mysql_adapter"
+
+class ActiveRecord::ConnectionAdapters::MysqlAdapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import-rails4/active_record/adapters/postgresql_adapter.rb

@@ -0,0 +1,7 @@
+require "active_record/connection_adapters/postgresql_adapter"
+require "activerecord-import-rails4/adapters/postgresql_adapter"
+
+class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+  include ActiveRecord::Import::PostgreSQLAdapter
+end
+

data/lib/activerecord-import-rails4/active_record/adapters/seamless_database_pool_adapter.rb

@@ -0,0 +1,7 @@
+require "seamless_database_pool"
+require "active_record/connection_adapters/seamless_database_pool_adapter"
+require "activerecord-import-rails4/adapters/mysql_adapter"
+
+class ActiveRecord::ConnectionAdapters::SeamlessDatabasePoolAdapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import-rails4/active_record/adapters/sqlite3_adapter.rb

@@ -0,0 +1,7 @@
+require "active_record/connection_adapters/sqlite3_adapter"
+require "activerecord-import-rails4/adapters/sqlite3_adapter"
+
+class ActiveRecord::ConnectionAdapters::SQLite3Adapter
+  include ActiveRecord::Import::SQLite3Adapter
+end
+

data/lib/activerecord-import-rails4/adapters/abstract_adapter.rb

@@ -0,0 +1,119 @@
+module ActiveRecord::Import::AbstractAdapter
+  NO_MAX_PACKET = 0
+  QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from.
+
+  module ClassMethods
+    def get_insert_value_sets( values, sql_size, max_bytes ) # :nodoc:
+      value_sets = []
+      arr, current_arr_values_size, current_size = [], 0, 0
+      values.each_with_index do |val,i|
+        comma_bytes = arr.size
+        sql_size_thus_far = sql_size + current_size + val.bytesize + comma_bytes
+        if NO_MAX_PACKET == max_bytes or sql_size_thus_far <= max_bytes
+          current_size += val.bytesize
+          arr << val
+        else
+          value_sets << arr
+          arr = [ val ]
+          current_size = val.bytesize
+        end
+
+        # if we're on the last iteration push whatever we have in arr to value_sets
+        value_sets << arr if i == (values.size-1)
+      end
+      [ *value_sets ]
+    end
+  end
+
+  module InstanceMethods
+    def next_value_for_sequence(sequence_name)
+      %{#{sequence_name}.nextval}
+    end
+
+    # +sql+ can be a single string or an array. If it is an array all 
+    # elements that are in position >= 1 will be appended to the final SQL.
+    def insert_many( sql, values, *args ) # :nodoc:
+      # the number of inserts default
+      number_of_inserts = 0
+
+      base_sql,post_sql = if sql.is_a?( String )
+        [ sql, '' ]
+      elsif sql.is_a?( Array )
+        [ sql.shift, sql.join( ' ' ) ]
+      end
+
+      sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
+
+      # the number of bytes the requested insert statement values will take up
+      values_in_bytes = values.sum {|value| value.bytesize }
+
+      # the number of bytes (commas) it will take to comma separate our values
+      comma_separated_bytes = values.size-1
+
+      # the total number of bytes required if this statement is one statement
+      total_bytes = sql_size + values_in_bytes + comma_separated_bytes
+
+      max = max_allowed_packet
+
+      # if we can insert it all as one statement
+      if NO_MAX_PACKET == max or total_bytes < max
+        number_of_inserts += 1
+        sql2insert = base_sql + values.join( ',' ) + post_sql
+        insert( sql2insert, *args )
+      else
+        value_sets = self.class.get_insert_value_sets( values, sql_size, max )
+        value_sets.each do |values|
+          number_of_inserts += 1
+          sql2insert = base_sql + values.join( ',' ) + post_sql
+          insert( sql2insert, *args )
+        end
+      end
+
+      number_of_inserts
+    end
+
+    def pre_sql_statements(options)
+      sql = []
+      sql << options[:pre_sql] if options[:pre_sql]
+      sql << options[:command] if options[:command]
+      sql << "IGNORE" if options[:ignore]
+
+      #add keywords like IGNORE or DELAYED
+      if options[:keywords].is_a?(Array)
+        sql.concat(options[:keywords])
+      elsif options[:keywords]
+        sql << options[:keywords].to_s
+      end
+
+      sql
+    end
+
+    # Synchronizes the passed in ActiveRecord instances with the records in
+    # the database by calling +reload+ on each instance.
+    def after_import_synchronize( instances )
+      instances.each { |e| e.reload }
+    end
+
+    # Returns an array of post SQL statements given the passed in options.
+    def post_sql_statements( table_name, options ) # :nodoc:
+      post_sql_statements = []
+      if options[:on_duplicate_key_update]
+        post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] )
+      end
+
+      #custom user post_sql
+      post_sql_statements << options[:post_sql] if options[:post_sql]
+
+      #with rollup
+      post_sql_statements << rollup_sql if options[:rollup]
+
+      post_sql_statements
+    end
+
+    # Returns the maximum number of bytes that the server will allow
+    # in a single packet
+    def max_allowed_packet
+      NO_MAX_PACKET
+    end
+  end
+end

data/lib/activerecord-import-rails4/adapters/mysql2_adapter.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + "/mysql_adapter"
+
+module ActiveRecord::Import::Mysql2Adapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import-rails4/adapters/mysql_adapter.rb

@@ -0,0 +1,55 @@
+module ActiveRecord::Import::MysqlAdapter
+  include ActiveRecord::Import::ImportSupport  
+  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
+
+  # Returns the maximum number of bytes that the server will allow
+  # in a single packet
+  def max_allowed_packet # :nodoc:
+    @max_allowed_packet ||= begin
+      result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
+      # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val.to_i
+    end
+  end
+
+  # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
+  # in +args+.
+  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
+    sql = ' ON DUPLICATE KEY UPDATE '
+    arg = args.first
+    if arg.is_a?( Array )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
+    elsif arg.is_a?( Hash )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
+    elsif arg.is_a?( String )
+      sql << arg
+    else
+      raise ArgumentError.new( "Expected Array or Hash" )
+    end
+    sql
+  end
+
+  def sql_for_on_duplicate_key_update_as_array( table_name, arr )  # :nodoc:
+    results = arr.map do |column|
+      qc = quote_column_name( column )
+      "#{table_name}.#{qc}=VALUES(#{qc})"
+    end
+    results.join( ',' )
+  end
+
+  def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
+    sql = ' ON DUPLICATE KEY UPDATE '
+    results = hsh.map do |column1, column2|
+      qc1 = quote_column_name( column1 )
+      qc2 = quote_column_name( column2 )
+      "#{table_name}.#{qc1}=VALUES( #{qc2} )"
+    end
+    results.join( ',')
+  end
+
+  #return true if the statement is a duplicate key record error
+  def duplicate_key_update_error?(exception)# :nodoc:
+    exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
+  end
+end