activerecord-import 1.0.3

data/Rakefile

@@ -0,0 +1,68 @@
+require "bundler"
+Bundler.setup
+
+require 'rake'
+require 'rake/testtask'
+
+namespace :display do
+  task :notice do
+    puts
+    puts "To run tests you must supply the adapter, see rake -T for more information."
+    puts
+  end
+end
+task default: ["display:notice"]
+
+ADAPTERS = %w(
+  mysql2
+  mysql2_makara
+  mysql2spatial
+  jdbcmysql
+  jdbcsqlite3
+  jdbcpostgresql
+  postgresql
+  postgresql_makara
+  postgis
+  makara_postgis
+  sqlite3
+  spatialite
+  seamless_database_pool
+).freeze
+ADAPTERS.each do |adapter|
+  namespace :test do
+    desc "Runs #{adapter} database tests."
+    Rake::TestTask.new(adapter) do |t|
+      # FactoryBot has an issue with warnings, so turn off, so noisy
+      # t.warning = true
+      t.test_files = FileList["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/active_record/*_test.rb", "test/#{adapter}/**/*_test.rb"]
+    end
+    task adapter
+  end
+end
+
+begin
+  require 'rcov/rcovtask'
+  adapter = ENV['ARE_DB']
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = ["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/#{adapter}/**/*_test.rb"]
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
+  end
+end
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "activerecord-import #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new

data/activerecord-import.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/activerecord-import/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Zach Dennis"]
+  gem.email         = ["zach.dennis@gmail.com"]
+  gem.summary       = "Bulk insert extension for ActiveRecord"
+  gem.description   = "A library for bulk inserting data using ActiveRecord."
+  gem.homepage      = "http://github.com/zdennis/activerecord-import"
+  gem.license       = "Ruby"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "activerecord-import"
+  gem.require_paths = ["lib"]
+  gem.version       = ActiveRecord::Import::VERSION
+
+  gem.required_ruby_version = ">= 1.9.2"
+
+  gem.add_runtime_dependency "activerecord", ">= 3.2"
+  gem.add_development_dependency "rake"
+end

data/benchmarks/README

@@ -0,0 +1,32 @@
+To run the benchmarks, from within the benchmarks run:
+   ruby benchmark.rb [options]
+
+The following options are supported:
+  --adapter [String]   The database adapter to use. IE: mysql, postgresql, oracle
+             
+  --do-not-delete       By default all records in the benchmark tables will be deleted at the end of the benchmark. This flag indicates not to delete the benchmark data.
+  --num [Integer]      The number of objects to benchmark. (Required!)
+  --table-type [String]  The table type to test. This can be used multiple times. By default it is all table types.
+  --to-csv [String]      Print results in a CSV file format
+  --to-html [String]     Print results in HTML format (String filename must be supplied)
+
+See "ruby benchmark.rb -h" for the complete listing of options.
+
+EXAMPLES
+--------
+To output to html format:
+  ruby benchmark.rb --adapter=mysql2 --to-html=results.html
+
+To output to csv format:
+  ruby benchmark.rb --adapter=mysql2 --to-csv=results.csv
+
+LIMITATIONS
+-----------
+Currently MySQL is the only supported adapter to benchmark.
+
+AUTHOR
+------
+Zach Dennis
+zach.dennis@gmail.com
+http://www.continuousthinking.com
+

data/benchmarks/benchmark.rb

@@ -0,0 +1,68 @@
+require 'pathname'
+require "fileutils"
+require "active_record"
+require "active_record/base"
+
+benchmark_dir = File.dirname(__FILE__)
+
+$LOAD_PATH.unshift('.')
+
+# Get the gem into the load path
+$LOAD_PATH.unshift(File.join(benchmark_dir, '..', 'lib'))
+
+# Load the benchmark files
+Dir[File.join( benchmark_dir, 'lib', '*.rb' )].sort.each { |f| require f }
+
+# Parse the options passed in via the command line
+options = BenchmarkOptionParser.parse( ARGV )
+
+FileUtils.mkdir_p 'log'
+ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
+ActiveRecord::Base.logger = Logger.new("log/test.log")
+ActiveRecord::Base.logger.level = Logger::DEBUG
+ActiveRecord::Base.default_timezone = :utc
+
+require "activerecord-import"
+ActiveRecord::Base.establish_connection(:test)
+
+ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
+  ActiveRecord::Base.logger.info hsh[:sql]
+end
+
+# Load base/generic schema
+require File.join(benchmark_dir, "../test/schema/version")
+require File.join(benchmark_dir, "../test/schema/generic_schema")
+adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
+require adapter_schema if File.exist?(adapter_schema)
+
+Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
+
+require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
+
+table_types = nil
+table_types = if options.benchmark_all_types
+  ["all"]
+else
+  options.table_types.keys
+end
+
+letter = options.adapter[0].chr
+clazz_str = letter.upcase + options.adapter[1..-1].downcase
+clazz = Object.const_get( clazz_str + "Benchmark" )
+
+benchmarks = []
+options.number_of_objects.each do |num|
+  benchmarks << (benchmark = clazz.new)
+  benchmark.send( "benchmark", table_types, num )
+end
+
+options.outputs.each do |output|
+  format = output.format.downcase
+  output_module = Object.const_get( "OutputTo#{format.upcase}" )
+  benchmarks.each do |benchmark|
+    output_module.output_results( output.filename, benchmark.results )
+  end
+end
+
+puts
+puts "Done with benchmark!"

data/benchmarks/lib/base.rb

@@ -0,0 +1,138 @@
+class BenchmarkBase
+  attr_reader :results
+
+  # The main benchmark method dispatcher. This dispatches the benchmarks
+  # to actual benchmark_xxxx methods.
+  #
+  # == PARAMETERS
+  #  * table_types - an array of table types to benchmark
+  #  * num - the number of record insertions to test
+  def benchmark( table_types, num )
+    array_of_cols_and_vals = build_array_of_cols_and_vals( num )
+    table_types.each do |table_type|
+      send( "benchmark_#{table_type}", array_of_cols_and_vals )
+    end
+  end
+
+  # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an
+  # actual benchmark.
+  #
+  # == PARAMETERS
+  #  * description - the description of the block that is getting benchmarked
+  #  * blk - the block of code to benchmark
+  #
+  # == RETURNS
+  # An OpenStruct object with the following attributes:
+  #   * description - the description of the benchmark ran
+  #   * tms - a Benchmark::Tms containing the results of the benchmark
+  def bm( description )
+    tms = nil
+    puts "Benchmarking #{description}"
+
+    Benchmark.bm { |x| tms = x.report { yield } }
+    delete_all
+    failed = false
+
+    OpenStruct.new description: description, tms: tms, failed: failed
+  end
+
+  # Given a model class (ie: Topic), and an array of columns and value sets
+  # this will perform all of the benchmarks necessary for this library.
+  #
+  # == PARAMETERS
+  #  * model_clazz - the model class to benchmark (ie: Topic)
+  #  * array_of_cols_and_vals - an array of column identifiers and value sets
+  #
+  # == RETURNS
+  #  returns true
+  def bm_model( model_clazz, array_of_cols_and_vals )
+    puts
+    puts "------ Benchmarking #{model_clazz.name} -------"
+
+    cols, vals = array_of_cols_and_vals
+    num_inserts = vals.size
+
+    # add a new result group for this particular benchmark
+    group = []
+    @results << group
+
+    description = "#{model_clazz.name}.create (#{num_inserts} records)"
+    group << bm( description ) do
+      vals.each do |values|
+        model_clazz.create create_hash_for_cols_and_vals( cols, values )
+      end
+    end
+
+    description = "#{model_clazz.name}.import(column, values) for #{num_inserts} records with validations"
+    group << bm( description ) { model_clazz.import cols, vals, validate: true }
+
+    description = "#{model_clazz.name}.import(columns, values) for #{num_inserts} records without validations"
+    group << bm( description ) { model_clazz.import cols, vals, validate: false }
+
+    models = []
+    array_of_attrs = []
+
+    vals.each do |arr|
+      array_of_attrs << (attrs = {})
+      arr.each_with_index { |value, i| attrs[cols[i]] = value }
+    end
+    array_of_attrs.each { |attrs| models << model_clazz.new(attrs) }
+
+    description = "#{model_clazz.name}.import(models) for #{num_inserts} records with validations"
+    group << bm( description ) { model_clazz.import models, validate: true }
+
+    description = "#{model_clazz.name}.import(models) for #{num_inserts} records without validations"
+    group << bm( description ) { model_clazz.import models, validate: false }
+
+    true
+  end
+
+  # Returns a two element array composing of an array of columns and an array of
+  # value sets given the passed +num+.
+  #
+  # === What is a value set?
+  # A value set is an array of arrays. Each child array represents an array of value sets
+  # for a given row of data.
+  #
+  # For example, say we wanted to represent an insertion of two records:
+  #   column_names = [ 'id', 'name', 'description' ]
+  #   record1 = [ 1, 'John Doe', 'A plumber' ]
+  #   record2 = [ 2, 'John Smith', 'A painter' ]
+  #   value_set [ record1, record2 ]
+  #
+  # == PARAMETER
+  #  * num - the number of records to create
+  def build_array_of_cols_and_vals( num )
+    cols = [:my_name, :description]
+    value_sets = []
+    num.times { |i| value_sets << ["My Name #{i}", "My Description #{i}"] }
+    [cols, value_sets]
+  end
+
+  # Returns a hash of column identifier to value mappings giving the passed in
+  # value array.
+  #
+  # Example:
+  #   cols = [ 'id', 'name', 'description' ]
+  #   values = [ 1, 'John Doe', 'A plumber' ]
+  #   hsh = create_hash_for_cols_and_vals( cols, values )
+  #   # hsh => { 'id'=>1, 'name'=>'John Doe', 'description'=>'A plumber' }
+  def create_hash_for_cols_and_vals( cols, vals )
+    h = {}
+    cols.zip( vals ) { |col, val| h[col] = val }
+    h
+  end
+
+  # Deletes all records from all ActiveRecord subclasses
+  def delete_all
+    ActiveRecord::Base.send( :subclasses ).each do |subclass|
+      if subclass.table_exists? && subclass.respond_to?(:delete_all)
+        subclass.delete_all
+      end
+    end
+  end
+
+  def initialize # :nodoc:
+    @results = []
+  end
+end

data/benchmarks/lib/cli_parser.rb

@@ -0,0 +1,107 @@
+require 'optparse'
+require 'ostruct'
+
+#
+# == PARAMETERS
+# * a - database adapter. ie: mysql, postgresql, oracle, etc.
+# * n - number of objects to test with. ie: 1, 100, 1000, etc.
+# * t - the table types to test. ie: myisam, innodb, memory, temporary, etc.
+#
+module BenchmarkOptionParser
+  BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options.".freeze
+
+  def self.print_banner
+    puts BANNER
+  end
+
+  def self.print_banner!
+    print_banner
+    exit
+  end
+
+  def self.print_options( options )
+    puts "Benchmarking the following options:"
+    puts "  Database adapter: #{options.adapter}"
+    puts "  Number of objects: #{options.number_of_objects}"
+    puts "  Table types:"
+    print_valid_table_types( options, prefix: "    " )
+  end
+
+  # TODO IMPLEMENT THIS
+  def self.print_valid_table_types( options, hsh = { prefix: '' } )
+    if !options.table_types.keys.empty?
+      options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }
+    else
+      puts 'No table types defined.'
+    end
+  end
+
+  def self.parse( args )
+    options = OpenStruct.new(
+      adapter: 'mysql2',
+      table_types: {},
+      delete_on_finish: true,
+      number_of_objects: [],
+      outputs: []
+    )
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = BANNER
+
+      # parse the database adapter
+      opts.on( "a", "--adapter [String]",
+        "The database adapter to use. IE: mysql, postgresql, oracle" ) do |arg|
+        options.adapter = arg
+      end
+
+      # parse do_not_delete flag
+      opts.on( "d", "--do-not-delete",
+        "By default all records in the benchmark tables will be deleted at the end of the benchmark. " \
+        "This flag indicates not to delete the benchmark data." ) do |_|
+        options.delete_on_finish = false
+      end
+
+      # parse the number of row objects to test
+      opts.on( "n", "--num [Integer]",
+        "The number of objects to benchmark." ) do |arg|
+        options.number_of_objects << arg.to_i
+      end
+
+      # parse the table types to test
+      opts.on( "t", "--table-type [String]",
+        "The table type to test. This can be used multiple times." ) do |arg|
+        if arg =~ /^all$/
+          options.table_types['all'] = options.benchmark_all_types = true
+        else
+          options.table_types[arg] = true
+        end
+      end
+
+      # print results in CSV format
+      opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename|
+        options.outputs << OpenStruct.new( format: 'csv', filename: filename)
+      end
+
+      # print results in HTML format
+      opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename|
+        options.outputs << OpenStruct.new( format: 'html', filename: filename )
+      end
+    end # end opt.parse!
+
+    begin
+      opt_parser.parse!( args )
+      if options.table_types.empty?
+        options.table_types['all'] = options.benchmark_all_types = true
+      end
+    rescue Exception
+      print_banner!
+    end
+
+    options.number_of_objects = [1000] if options.number_of_objects.empty?
+    options.outputs = [OpenStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
+
+    print_options( options )
+
+    options
+  end
+end

data/benchmarks/lib/float.rb

@@ -0,0 +1,15 @@
+# Taken from http://www.programmingishard.com/posts/show/128
+# Posted by rbates
+class Float
+  def round_to(x)
+    (self * 10**x).round.to_f / 10**x
+  end
+
+  def ceil_to(x)
+    (self * 10**x).ceil.to_f / 10**x
+  end
+
+  def floor_to(x)
+    (self * 10**x).floor.to_f / 10**x
+  end
+end