activerecord-import 0.3.1 → 0.4.0

data/benchmarks/lib/float.rb

@@ -0,0 +1,15 @@
+# Taken from http://www.programmingishard.com/posts/show/128
+# Posted by rbates
+class Float
+  def round_to(x)
+    (self * 10**x).round.to_f / 10**x
+  end
+
+  def ceil_to(x)
+    (self * 10**x).ceil.to_f / 10**x
+  end
+
+  def floor_to(x)
+    (self * 10**x).floor.to_f / 10**x
+  end
+end

data/benchmarks/lib/mysql_benchmark.rb

@@ -0,0 +1,22 @@
+class MysqlBenchmark < BenchmarkBase
+    
+  def benchmark_all( array_of_cols_and_vals )
+    methods = self.methods.find_all { |m| m =~ /benchmark_/ }
+    methods.delete_if{ |m| m =~ /benchmark_(all|model)/ }
+    methods.each { |method| self.send( method, array_of_cols_and_vals ) }
+  end
+  
+  def benchmark_myisam( array_of_cols_and_vals )
+    bm_model( TestMyISAM, array_of_cols_and_vals )
+  end
+    
+  def benchmark_innodb( array_of_cols_and_vals )
+    bm_model( TestInnoDb, array_of_cols_and_vals )
+  end
+  
+  def benchmark_memory( array_of_cols_and_vals )
+    bm_model( TestMemory, array_of_cols_and_vals )
+  end
+    
+end
+

data/benchmarks/lib/output_to_csv.rb

@@ -0,0 +1,18 @@
+require 'fastercsv'
+
+module OutputToCSV
+  def self.output_results( filename, results )
+    FasterCSV.open( filename, 'w' ) do |csv|
+      # Iterate over each result set, which contains many results
+      results.each do |result_set|
+        columns, times = [], []
+        result_set.each do |result| 
+          columns << result.description
+          times << result.tms.real
+        end
+        csv << columns
+        csv << times
+      end
+    end
+  end
+end

data/benchmarks/lib/output_to_html.rb

@@ -0,0 +1,69 @@
+require 'erb'
+
+module OutputToHTML
+
+TEMPLATE_HEADER =<<"EOT"
+  <div>
+    All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
+    before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
+    the very first column (which is always the default ActiveRecord::Base.create method.
+   </div>
+EOT
+
+TEMPLATE =<<"EOT"
+ <style>
+ td#benchmarkTitle {
+   border: 1px solid black;
+   padding: 2px;
+   font-size: 0.8em;
+   background-color: black;
+   color: white;
+ }
+ td#benchmarkCell {
+   border: 1px solid black;
+   padding: 2px;
+   font-size: 0.8em;
+ }
+ </style>
+   <table>
+     <tr>
+       <% columns.each do |col| %>
+         <td id="benchmarkTitle"><%= col %></td>
+       <% end %>
+     </tr>
+     <tr>
+       <% times.each do |time| %>
+         <td id="benchmarkCell"><%= time %></td>
+       <% end %>
+     </tr>
+     <tr><td>&nbsp;</td></tr>
+   </table>
+EOT
+
+  def self.output_results( filename, results )
+    html = ''
+    results.each do |result_set|
+      columns, times = [], []
+      result_set.each do |result|
+        columns << result.description
+        if result.failed
+          times << "failed"
+        else
+          time = result.tms.real.round_to( 3 )
+          speedup = ( result_set.first.tms.real / result.tms.real ).round 
+
+          if result == result_set.first
+            times << "#{time}"
+          else
+            times << "#{time} (#{speedup}x speedup)"
+          end
+        end
+      end
+
+      template = ERB.new( TEMPLATE, 0, "%<>")
+      html << template.result( binding )
+    end
+    
+    File.open( filename, 'w' ){ |file| file.write( TEMPLATE_HEADER + html ) }
+  end
+end

data/benchmarks/models/test_innodb.rb

@@ -0,0 +1,3 @@
+class TestInnoDb < ActiveRecord::Base
+  set_table_name 'test_innodb'
+end

data/benchmarks/models/test_memory.rb

@@ -0,0 +1,3 @@
+class TestMemory < ActiveRecord::Base
+  set_table_name 'test_memory'
+end

data/benchmarks/models/test_myisam.rb

@@ -0,0 +1,3 @@
+class TestMyISAM < ActiveRecord::Base
+  set_table_name 'test_myisam'
+end

data/benchmarks/schema/mysql_schema.rb

@@ -0,0 +1,16 @@
+ActiveRecord::Schema.define do
+  create_table :test_myisam, :options=>'ENGINE=MyISAM', :force=>true do |t|
+    t.column :my_name, :string, :null=>false
+    t.column :description, :string
+  end
+  
+  create_table :test_innodb, :options=>'ENGINE=InnoDb', :force=>true do |t|
+    t.column :my_name, :string, :null=>false
+    t.column :description, :string
+  end
+
+  create_table :test_memory, :options=>'ENGINE=Memory', :force=>true do |t|
+    t.column :my_name, :string, :null=>false
+    t.column :description, :string
+  end
+end

data/gemfiles/3.1.gemfile

@@ -0,0 +1,4 @@
+platforms :ruby do
+  gem 'mysql',        '~> 2.8.1'
+  gem 'activerecord', '~> 3.1.0'
+end

data/gemfiles/3.2.gemfile

@@ -0,0 +1,4 @@
+platforms :ruby do
+  gem 'mysql',        '~> 2.8.1'
+  gem 'activerecord', '~> 3.2.0'
+end

data/gemfiles/4.0.gemfile

@@ -0,0 +1,4 @@
+platforms :ruby do
+  gem 'mysql',        '~> 2.9'
+  gem 'activerecord', '~> 4.0.0.rc2'
+end

data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb

@@ -3,7 +3,6 @@ require "activerecord-import/adapters/abstract_adapter"
 module ActiveRecord # :nodoc:
   module ConnectionAdapters # :nodoc:
     class AbstractAdapter # :nodoc:
-      extend ActiveRecord::Import::AbstractAdapter::ClassMethods
       include ActiveRecord::Import::AbstractAdapter::InstanceMethods
     end
   end

data/lib/activerecord-import/active_record/adapters/em_mysql2_adapter.rb

@@ -0,0 +1,8 @@
+require "em-synchrony"
+require "em-synchrony/mysql2"
+require "em-synchrony/activerecord"
+require "activerecord-import/adapters/em_mysql2_adapter"
+
+class ActiveRecord::ConnectionAdapters::EMMysql2Adapter
+  include ActiveRecord::Import::EMMysql2Adapter
+end

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -1,73 +1,20 @@
 module ActiveRecord::Import::AbstractAdapter
-  NO_MAX_PACKET = 0
-  QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from.
-    
-  module ClassMethods
-    def get_insert_value_sets( values, sql_size, max_bytes ) # :nodoc:
-      value_sets = []          
-      arr, current_arr_values_size, current_size = [], 0, 0
-      values.each_with_index do |val,i|
-        comma_bytes = arr.size
-        sql_size_thus_far = sql_size + current_size + val.bytesize + comma_bytes
-        if NO_MAX_PACKET == max_bytes or sql_size_thus_far <= max_bytes
-          current_size += val.bytesize            
-          arr << val
-        else
-          value_sets << arr
-          arr = [ val ]
-          current_size = val.bytesize
-        end
-      
-        # if we're on the last iteration push whatever we have in arr to value_sets
-        value_sets << arr if i == (values.size-1)
-      end
-      [ *value_sets ]
-    end
-  end
-  
   module InstanceMethods
     def next_value_for_sequence(sequence_name)
       %{#{sequence_name}.nextval}
     end
-  
-    # +sql+ can be a single string or an array. If it is an array all 
-    # elements that are in position >= 1 will be appended to the final SQL.
+
     def insert_many( sql, values, *args ) # :nodoc:
-      # the number of inserts default
-      number_of_inserts = 0
-    
+      number_of_inserts = 1
+
       base_sql,post_sql = if sql.is_a?( String )
         [ sql, '' ]
       elsif sql.is_a?( Array )
         [ sql.shift, sql.join( ' ' ) ]
       end
-    
-      sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size 
 
-      # the number of bytes the requested insert statement values will take up
-      values_in_bytes = values.sum {|value| value.bytesize }
-    
-      # the number of bytes (commas) it will take to comma separate our values
-      comma_separated_bytes = values.size-1
-    
-      # the total number of bytes required if this statement is one statement
-      total_bytes = sql_size + values_in_bytes + comma_separated_bytes
-    
-      max = max_allowed_packet
-    
-      # if we can insert it all as one statement
-      if NO_MAX_PACKET == max or total_bytes < max
-        number_of_inserts += 1
-        sql2insert = base_sql + values.join( ',' ) + post_sql
-        insert( sql2insert, *args )
-      else
-        value_sets = self.class.get_insert_value_sets( values, sql_size, max )
-        value_sets.each do |values|
-          number_of_inserts += 1
-          sql2insert = base_sql + values.join( ',' ) + post_sql
-          insert( sql2insert, *args )
-        end
-      end
+      sql2insert = base_sql + values.join( ',' ) + post_sql
+      insert( sql2insert, *args )
 
       number_of_inserts
     end

data/lib/activerecord-import/adapters/em_mysql2_adapter.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + "/mysql_adapter"
+
+module ActiveRecord::Import::EMMysql2Adapter
+  include ActiveRecord::Import::MysqlAdapter
+end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -1,7 +1,54 @@
 module ActiveRecord::Import::MysqlAdapter
-  include ActiveRecord::Import::ImportSupport  
+  include ActiveRecord::Import::ImportSupport
   include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
 
+  NO_MAX_PACKET = 0
+  QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from.
+
+  # +sql+ can be a single string or an array. If it is an array all 
+  # elements that are in position >= 1 will be appended to the final SQL.
+  def insert_many( sql, values, *args ) # :nodoc:
+    # the number of inserts default
+    number_of_inserts = 0
+
+    base_sql,post_sql = if sql.is_a?( String )
+      [ sql, '' ]
+    elsif sql.is_a?( Array )
+      [ sql.shift, sql.join( ' ' ) ]
+    end
+
+    sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
+
+    # the number of bytes the requested insert statement values will take up
+    values_in_bytes = values.sum {|value| value.bytesize }
+
+    # the number of bytes (commas) it will take to comma separate our values
+    comma_separated_bytes = values.size-1
+
+    # the total number of bytes required if this statement is one statement
+    total_bytes = sql_size + values_in_bytes + comma_separated_bytes
+
+    max = max_allowed_packet
+
+    # if we can insert it all as one statement
+    if NO_MAX_PACKET == max or total_bytes < max
+      number_of_inserts += 1
+      sql2insert = base_sql + values.join( ',' ) + post_sql
+      insert( sql2insert, *args )
+    else
+      value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
+        :reserved_bytes => sql_size,
+        :max_bytes => max)
+      value_sets.each do |values|
+        number_of_inserts += 1
+        sql2insert = base_sql + values.join( ',' ) + post_sql
+        insert( sql2insert, *args )
+      end
+    end
+
+    number_of_inserts
+  end
+
   # Returns the maximum number of bytes that the server will allow
   # in a single packet
   def max_allowed_packet # :nodoc:
@@ -12,9 +59,9 @@ module ActiveRecord::Import::MysqlAdapter
       val.to_i
     end
   end
-  
+
   # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
-  # in +args+. 
+  # in +args+.
   def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
     sql = ' ON DUPLICATE KEY UPDATE '
     arg = args.first

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -1,18 +1,43 @@
+require 'debugger'
 module ActiveRecord::Import::SQLite3Adapter
   include ActiveRecord::Import::ImportSupport
 
+  MIN_VERSION_FOR_IMPORT = "3.7.11"
+  SQLITE_LIMIT_COMPOUND_SELECT = 500
+
   # Override our conformance to ActiveRecord::Import::ImportSupport interface
   # to ensure that we only support import in supported version of SQLite.
-  # Which INSERT statements with multiple value sets was introduced in 3.2.11.
+  # Which INSERT statements with multiple value sets was introduced in 3.7.11.
   def supports_import?(current_version=self.sqlite_version)
-    minimum_supported_version = "3.2.11"
-    if current_version >= minimum_supported_version
+    if current_version >= MIN_VERSION_FOR_IMPORT
       true
     else
       false
     end
   end
 
+  # +sql+ can be a single string or an array. If it is an array all
+  # elements that are in position >= 1 will be appended to the final SQL.
+  def insert_many(sql, values, *args) # :nodoc:
+    number_of_inserts = 0
+    base_sql,post_sql = if sql.is_a?( String )
+      [ sql, '' ]
+    elsif sql.is_a?( Array )
+      [ sql.shift, sql.join( ' ' ) ]
+    end
+
+    value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
+      :max_records => SQLITE_LIMIT_COMPOUND_SELECT)
+
+    value_sets.each do |values|
+      number_of_inserts += 1
+      sql2insert = base_sql + values.join( ',' ) + post_sql
+      insert( sql2insert, *args )
+    end
+
+    number_of_inserts
+  end
+
   def next_value_for_sequence(sequence_name)
     %{nextval('#{sequence_name}')}
   end

data/lib/activerecord-import/base.rb

@@ -14,7 +14,7 @@ module ActiveRecord::Import
     else adapter
     end
   end
-  
+
   # Loads the import functionality for a specific database adapter
   def self.require_adapter(adapter)
     require File.join(AdapterPath,"/abstract_adapter")
@@ -31,4 +31,5 @@ end
 this_dir = Pathname.new File.dirname(__FILE__)
 require this_dir.join("import").to_s
 require this_dir.join("active_record/adapters/abstract_adapter").to_s
-require this_dir.join("synchronize").to_s
+require this_dir.join("synchronize").to_s
+require this_dir.join("value_sets_parser").to_s

data/lib/activerecord-import/em_mysql2.rb

@@ -0,0 +1,7 @@
+warn <<-MSG
+[DEPRECATION] loading activerecord-import via 'require "activerecord-import/<adapter-name>"'
+  is deprecated. Update to autorequire using 'require "activerecord-import"'. See
+  http://github.com/zdennis/activerecord-import/wiki/Requiring for more information
+MSG
+
+require File.expand_path(File.join(File.dirname(__FILE__),  "/../activerecord-import"))

data/lib/activerecord-import/import.rb

@@ -11,7 +11,7 @@ module ActiveRecord::Import #:nodoc:
       true
     end
   end
-  
+
   module OnDuplicateKeyUpdateSupport #:nodoc:
     def supports_on_duplicate_key_update? #:nodoc:
       true
@@ -32,7 +32,7 @@ class ActiveRecord::Base
     tproc = lambda do
       ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
     end
-    
+
     AREXT_RAILS_COLUMNS = {
       :create => { "created_on" => tproc ,
                    "created_at" => tproc },
@@ -40,7 +40,7 @@ class ActiveRecord::Base
                    "updated_at" => tproc }
     }
     AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys
-  
+
     # Returns true if the current database connection adapter
     # supports import functionality, otherwise returns false.
     def supports_import?(*args)
@@ -57,14 +57,14 @@ class ActiveRecord::Base
     rescue NoMethodError
       false
     end
-    
-    # Imports a collection of values to the database.  
+
+    # Imports a collection of values to the database.
     #
     # This is more efficient than using ActiveRecord::Base#create or
     # ActiveRecord::Base#save multiple times. This method works well if
     # you want to create more than one record at a time and do not care
     # about having ActiveRecord objects returned for each record
-    # inserted. 
+    # inserted.
     #
     # This can be used with or without validations. It does not utilize
     # the ActiveRecord::Callbacks during creation/modification while
@@ -74,9 +74,9 @@ class ActiveRecord::Base
     #  Model.import array_of_models
     #  Model.import column_names, array_of_values
     #  Model.import column_names, array_of_values, options
-    # 
+    #
     # ==== Model.import array_of_models
-    # 
+    #
     # With this form you can call _import_ passing in an array of model
     # objects that you want updated.
     #
@@ -108,9 +108,9 @@ class ActiveRecord::Base
     # * +timestamps+ - true|false, tells import to not add timestamps \
     #   (if false) even if record timestamps is disabled in ActiveRecord::Base
     #
-    # == Examples  
+    # == Examples
     #  class BlogPost < ActiveRecord::Base ; end
-    #  
+    #
     #  # Example using array of model objects
     #  posts = [ BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT',
     #            BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT2',
@@ -128,7 +128,7 @@ class ActiveRecord::Base
     #  BlogPost.import( columns, values, :validate => false  )
     #
     #  # Example synchronizing existing instances in memory
-    #  post = BlogPost.find_by_author_name( 'zdennis' )
+    #  post = BlogPost.where(author_name: 'zdennis').first
     #  puts post.author_name # => 'zdennis'
     #  columns = [ :author_name, :title ]
     #  values = [ [ 'yoda', 'test post' ] ]
@@ -143,7 +143,7 @@ class ActiveRecord::Base
     # == On Duplicate Key Update (MySQL only)
     #
     # The :on_duplicate_key_update option can be either an Array or a Hash. 
-    # 
+    #
     # ==== Using an Array
     #
     # The :on_duplicate_key_update option can be an array of column
@@ -158,9 +158,9 @@ class ActiveRecord::Base
     # to model attribute name mappings. This gives you finer grained
     # control over what fields are updated with what attributes on your
     # model. Below is an example:
-    #   
-    #   BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title } 
-    #  
+    #
+    #   BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title }
+    #
     # = Returns
     # This returns an object which responds to +failed_instances+ and +num_inserts+.
     # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
@@ -180,7 +180,7 @@ class ActiveRecord::Base
           models = args.first
           column_names = self.column_names.dup
         end
-        
+
         array_of_attributes = models.map do |model|
           # this next line breaks sqlite.so with a segmentation fault
           # if model.new_record? || options[:on_duplicate_key_update]
@@ -230,23 +230,23 @@ class ActiveRecord::Base
       return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
       return_obj
     end
-    
-    # TODO import_from_table needs to be implemented. 
+
+    # TODO import_from_table needs to be implemented.
     def import_from_table( options ) # :nodoc:
     end
-    
+
     # Imports the passed in +column_names+ and +array_of_attributes+
     # given the passed in +options+ Hash with validations. Returns an
-    # object with the methods +failed_instances+ and +num_inserts+. 
-    # +failed_instances+ is an array of instances that failed validations. 
+    # object with the methods +failed_instances+ and +num_inserts+.
+    # +failed_instances+ is an array of instances that failed validations.
     # +num_inserts+ is the number of inserts it took to import the data. See
     # ActiveRecord::Base.import for more information on
     # +column_names+, +array_of_attributes+ and +options+.
     def import_with_validations( column_names, array_of_attributes, options={} )
       failed_instances = []
-    
+
       # create instances for each of our column/value sets
-      arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )    
+      arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
 
       # keep track of the instance and the position it is currently at. if this fails
       # validation we'll use the index to remove it from the array_of_attributes
@@ -257,7 +257,7 @@ class ActiveRecord::Base
         if not instance.valid?
           array_of_attributes[ i ] = nil
           failed_instances << instance
-        end    
+        end
       end
       array_of_attributes.compact!
 
@@ -268,7 +268,7 @@ class ActiveRecord::Base
                     end
       ActiveRecord::Import::Result.new(failed_instances, num_inserts)
     end
-    
+
     # Imports the passed in +column_names+ and +array_of_attributes+
     # given the passed in +options+ Hash. This will return the number
     # of insert operations it took to create these records without
@@ -309,7 +309,7 @@ class ActiveRecord::Base
         post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
 
         # perform the inserts
-        number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten, 
+        number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten,
                                                   values_sql,
                                                   "#{self.class.name} Create Many Without Validations Or Callbacks" )
       end
@@ -367,7 +367,7 @@ class ActiveRecord::Base
             column_names << key
             array_of_attributes.each { |arr| arr << value }
           end
-          
+
           if supports_on_duplicate_key_update?
             if options[:on_duplicate_key_update]
               options[:on_duplicate_key_update] << key.to_sym if options[:on_duplicate_key_update].is_a?(Array)
@@ -379,13 +379,13 @@ class ActiveRecord::Base
         end
       end
     end
-    
+
     # Returns an Array of Hashes for the passed in +column_names+ and +array_of_attributes+.
     def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc:
       array_of_attributes.map do |attributes|
         Hash[attributes.each_with_index.map {|attr, c| [column_names[c], attr] }]
       end
     end
-    
+
   end
 end