activerecord-import 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f8e8fa0974d52a903ef20a78f196030a515687d
-  data.tar.gz: 0ccc98cac5b73a78e9e9ffe57c2b74504c2b9e81
+  metadata.gz: 1f525ceb164067e5f2c49042b1a8f3312eab016a
+  data.tar.gz: c9817adb82be93259525dc52a85b42bf7829d585
 SHA512:
-  metadata.gz: 1562542547afcea6d646e3a5fcf2aabd806471f0000b2434d6fd979a60c2bc93b90e115e09e2b67df00b35a2244c67d0f2554e9480f0c5c6bb6f3dbdebd2e8be
-  data.tar.gz: 45f2552a368c3b9c70bf7abec385462f34a66e423a65db1e99536b8f3b383c02d21af4cb5b5a313c3769aa59a919bd4db34ddb6927ced52fae88272afaeb28de
+  metadata.gz: 974e9a0491e43cd9a9a22e167f1b9dede5bcdececbb3d480c2b4edf2e293e50c0cf3e85f39b4d10f624c541a0651db5de2f85bad0d5ca7142c4de28017e1809f
+  data.tar.gz: 4e31a0c10174ba285b08d47abbac06c2089c62488c5f3775d24c6b993b4c7bd465d051e5238e6df511da6ad8b152f47f5827b41cf316d5e8713dc9bd2c64e9bc

data/.rubocop.yml

@@ -0,0 +1,49 @@
+inherit_from: .rubocop_todo.yml
+
+Lint/EndAlignment:
+  AlignWith: variable
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Style/AlignParameters:
+  EnforcedStyle: with_fixed_indentation
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/ElseAlignment:
+  Enabled: false
+
+Style/SpaceInsideParens:
+  Enabled: false
+
+Style/SpecialGlobalVars:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/TrailingCommaInLiteral:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,36 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2016-03-17 18:14:55 -0700 using RuboCop version 0.38.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 2
+Lint/HandleExceptions:
+  Exclude:
+    - 'lib/activerecord-import/base.rb'
+    - 'test/import_test.rb'
+
+# Offense count: 2
+Lint/RescueException:
+  Exclude:
+    - 'benchmarks/lib/cli_parser.rb'
+    - 'test/import_test.rb'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods.
+Lint/UnusedMethodArgument:
+  Exclude:
+    - 'lib/activerecord-import/adapters/postgresql_adapter.rb'
+    - 'lib/activerecord-import/import.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: Keywords.
+# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW
+Style/CommentAnnotation:
+  Exclude:
+    - 'benchmarks/lib/cli_parser.rb'
+    - 'lib/activerecord-import/import.rb'

data/.travis.yml

@@ -1,20 +1,44 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.2.3
+  - 2.2.4
 
-gemfile:
-  - Gemfile
+env:
+  global:
+    # https://github.com/discourse/discourse/blob/master/.travis.yml
+    - RUBY_GC_MALLOC_LIMIT=50000000
+  matrix:
+    - AR_VERSION=3.1
+    - AR_VERSION=3.2
+    - AR_VERSION=4.0
+    - AR_VERSION=4.1
+    - AR_VERSION=4.2
+    - AR_VERSION=5.0
 
-bundler_args: -j2
+matrix:
+  fast_finish: true
 
 before_script:
   - mysql -e 'create database activerecord_import_test;'
   - psql -c 'create database activerecord_import_test;' -U postgres
   - psql -U postgres -c "create extension postgis"
   - cp test/travis/database.yml test/database.yml
-# https://github.com/discourse/discourse/blob/master/.travis.yml
-  - export RUBY_GC_MALLOC_LIMIT=50000000
+
+addons:
+  apt:
+    sources:
+      - travis-ci/sqlite3
+    packages:
+      - sqlite3
 
 script:
-  - test/travis/build.sh
+  - bundle exec rake test:mysql2
+  - bundle exec rake test:mysql2spatial
+  - bundle exec rake test:postgis
+  - bundle exec rake test:postgresql
+  - bundle exec rake test:seamless_database_pool
+  - bundle exec rake test:spatialite
+  - bundle exec rake test:sqlite3
+  - bundle exec rubocop
+
+sudo: false

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## Changes in 0.13.0
+
+### New Features
+
+* Addition of :batch_size option to control the number of rows to insert per INSERT statement. The default is the total number of records being inserted so there is a single INSERT statement. Thanks to @jkowens via \#245
+
+* Addition `import!` which will raise an exception if a validation occurs. It will fail fast. Thanks to @jkowens via \#246
+
+### Fixes
+
+* Fixing issue with recursive import when utilizing the `:on_duplicate_key_update` option. The `on_duplicate_key_update` only applies to parent models at this time. Thanks to @yuri-karpovich for reporting and  @jkowens for fixing via \#249
+
+### Misc
+
+* Refactoring of fetching and assigning attributes. Thanks to @jkownes via \#259
+* Lots of code cleanup and addition of Rubocop linter. Thanks to @sferik via \#256 and \#250
+* Resolving errors with the test suite when running against ActiveRecord 4.0 and 4.1. Thanks to @jkowens via \#262
+* Cleaning up the TravisCI settings and packages. Thanks to @sferik via \#258 and \#251
+
 ## Changes in 0.12.0
 
 ### New Features

data/Gemfile

@@ -2,6 +2,10 @@ source 'https://rubygems.org'
 
 gemspec
 
+group :development, :test do
+  gem 'rubocop', '~> 0.38.0'
+end
+
 # Database Adapters
 platforms :ruby do
   gem "mysql2",                 "~> 0.3.0"
@@ -22,7 +26,6 @@ gem "factory_girl", "~> 4.2.0"
 gem "timecop"
 gem "chronic"
 
-
 # Debugging
 platforms :jruby do
   gem "ruby-debug-base", "= 0.10.4"
@@ -42,7 +45,7 @@ end
 
 version = ENV['AR_VERSION'] || "4.2"
 
-if version > "4.0"
+if version >= "4.0"
   gem "minitest"
 else
   gem "test-unit"

data/README.markdown

@@ -21,6 +21,10 @@ and then the reviews:
 That would be about 4M SQL insert statements vs 3, which results in vastly improved performance. In our case, it converted
 an 18 hour batch process to <2 hrs.
 
+### Rails 5.0
+
+Use activerecord-import 0.11.0 or higher.
+
 ### Rails 4.0
 
 Use activerecord-import 0.4.0 or higher.
@@ -85,7 +89,8 @@ Zach Dennis (zach.dennis@gmail.com)
 
 # Contributors
 
-* Jordan Owens
+* Jordan Owens (@jkowens)
+* Erik Michaels-Ober (@sferik)
 * Blythe Dunham
 * Gabe da Silveira
 * Henry Work

data/Rakefile

@@ -11,9 +11,9 @@ namespace :display do
     puts
   end
 end
-task :default => ["display:notice"]
+task default: ["display:notice"]
 
-ADAPTERS = %w(mysql2 jdbcmysql jdbcpostgresql postgresql sqlite3 seamless_database_pool mysql2spatial spatialite postgis)
+ADAPTERS = %w(mysql2 jdbcmysql jdbcpostgresql postgresql sqlite3 seamless_database_pool mysql2spatial spatialite postgis).freeze
 ADAPTERS.each do |adapter|
   namespace :test do
     desc "Runs #{adapter} database tests."
@@ -49,3 +49,6 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new

data/activerecord-import.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |gem|
   gem.license       = "Ruby"
 
   gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "activerecord-import"
   gem.require_paths = ["lib"]

data/benchmarks/benchmark.rb

@@ -1,68 +1,67 @@
-require 'pathname'
-require "fileutils"
-require "active_record"
-
-benchmark_dir = File.dirname(__FILE__)
-
-$LOAD_PATH.unshift('.')
-
-# Get the gem into the load path
-$LOAD_PATH.unshift(File.join(benchmark_dir, '..', 'lib'))
-
-# Load the benchmark files
-Dir[File.join( benchmark_dir, 'lib', '*.rb' ) ].sort.each{ |f| require f }
-
-# Parse the options passed in via the command line
-options = BenchmarkOptionParser.parse( ARGV )
-
-FileUtils.mkdir_p 'log'
-ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
-ActiveRecord::Base.logger = Logger.new("log/test.log")
-ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.default_timezone = :utc
-
-require "activerecord-import"
-ActiveRecord::Base.establish_connection(:test)
-
-ActiveSupport::Notifications.subscribe(/active_record.sql/) do |event, _, _, _, hsh|
-  ActiveRecord::Base.logger.info hsh[:sql]
-end
-
-# Load base/generic schema
-require File.join(benchmark_dir, "../test/schema/version")
-require File.join(benchmark_dir, "../test/schema/generic_schema")
-adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
-require adapter_schema if File.exists?(adapter_schema)
-
-Dir[File.dirname(__FILE__) + "/models/*.rb"].each{ |file| require file }
-
-
-require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
-
-table_types = nil
-if options.benchmark_all_types
-  table_types = [ "all" ]
-else
-  table_types = options.table_types.keys
-end
-
-letter = options.adapter[0].chr
-clazz_str = letter.upcase + options.adapter[1..-1].downcase
-clazz = Object.const_get( clazz_str + "Benchmark" )
-
-benchmarks = []
-options.number_of_objects.each do |num|
-  benchmarks << (benchmark = clazz.new)
-  benchmark.send( "benchmark", table_types, num )
-end
-
-options.outputs.each do |output|
-  format = output.format.downcase
-  output_module = Object.const_get( "OutputTo#{format.upcase}" )
-  benchmarks.each do |benchmark|
-    output_module.output_results( output.filename, benchmark.results )
-  end
-end
-
-puts
-puts "Done with benchmark!"
+require 'pathname'
+require "fileutils"
+require "active_record"
+
+benchmark_dir = File.dirname(__FILE__)
+
+$LOAD_PATH.unshift('.')
+
+# Get the gem into the load path
+$LOAD_PATH.unshift(File.join(benchmark_dir, '..', 'lib'))
+
+# Load the benchmark files
+Dir[File.join( benchmark_dir, 'lib', '*.rb' )].sort.each { |f| require f }
+
+# Parse the options passed in via the command line
+options = BenchmarkOptionParser.parse( ARGV )
+
+FileUtils.mkdir_p 'log'
+ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
+ActiveRecord::Base.logger = Logger.new("log/test.log")
+ActiveRecord::Base.logger.level = Logger::DEBUG
+ActiveRecord::Base.default_timezone = :utc
+
+require "activerecord-import"
+ActiveRecord::Base.establish_connection(:test)
+
+ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
+  ActiveRecord::Base.logger.info hsh[:sql]
+end
+
+# Load base/generic schema
+require File.join(benchmark_dir, "../test/schema/version")
+require File.join(benchmark_dir, "../test/schema/generic_schema")
+adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
+require adapter_schema if File.exist?(adapter_schema)
+
+Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
+
+require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
+
+table_types = nil
+table_types = if options.benchmark_all_types
+  ["all"]
+else
+  options.table_types.keys
+end
+
+letter = options.adapter[0].chr
+clazz_str = letter.upcase + options.adapter[1..-1].downcase
+clazz = Object.const_get( clazz_str + "Benchmark" )
+
+benchmarks = []
+options.number_of_objects.each do |num|
+  benchmarks << (benchmark = clazz.new)
+  benchmark.send( "benchmark", table_types, num )
+end
+
+options.outputs.each do |output|
+  format = output.format.downcase
+  output_module = Object.const_get( "OutputTo#{format.upcase}" )
+  benchmarks.each do |benchmark|
+    output_module.output_results( output.filename, benchmark.results )
+  end
+end
+
+puts
+puts "Done with benchmark!"

data/benchmarks/lib/base.rb

@@ -1,137 +1,136 @@
-class BenchmarkBase
-
-  attr_reader :results
-
-  # The main benchmark method dispatcher. This dispatches the benchmarks
-  # to actual benchmark_xxxx methods.
-  # 
-  # == PARAMETERS
-  #  * table_types - an array of table types to benchmark
-  #  * num - the number of record insertions to test
-  def benchmark( table_types, num )
-    array_of_cols_and_vals = build_array_of_cols_and_vals( num )
-    table_types.each do |table_type|
-      self.send( "benchmark_#{table_type}", array_of_cols_and_vals )
-    end
-  end
-
-  # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an
-  # actual benchmark.
-  #
-  # == PARAMETERS
-  #  * description - the description of the block that is getting benchmarked
-  #  * blk - the block of code to benchmark
-  #
-  # == RETURNS
-  # An OpenStruct object with the following attributes:
-  #   * description - the description of the benchmark ran
-  #   * tms - a Benchmark::Tms containing the results of the benchmark
-  def bm( description, &blk )
-    tms  = nil
-    puts "Benchmarking #{description}"
-    
-    Benchmark.bm { |x| tms = x.report { blk.call } }
-    delete_all
-    failed = false
-
-    OpenStruct.new :description=>description, :tms=>tms, :failed=>failed
-  end
-  
-  # Given a model class (ie: Topic), and an array of columns and value sets
-  # this will perform all of the benchmarks necessary for this library.
-  # 
-  # == PARAMETERS
-  #  * model_clazz - the model class to benchmark (ie: Topic)
-  #  * array_of_cols_and_vals - an array of column identifiers and value sets
-  #
-  # == RETURNS
-  #  returns true
-  def bm_model( model_clazz, array_of_cols_and_vals )
-    puts
-    puts "------ Benchmarking #{model_clazz.name} -------"
-
-    cols,vals = array_of_cols_and_vals
-    num_inserts = vals.size
-
-    # add a new result group for this particular benchmark
-    group = []
-    @results << group
-
-    description = "#{model_clazz.name}.create (#{num_inserts} records)"
-    group << bm( description ) { 
-      vals.each do |values|
-        model_clazz.create create_hash_for_cols_and_vals( cols, values )
-      end  }
-
-    description = "#{model_clazz.name}.import(column, values) for #{num_inserts} records with validations"
-    group << bm( description ) { model_clazz.import cols, vals, :validate=>true }
-
-    description = "#{model_clazz.name}.import(columns, values) for #{num_inserts} records without validations"
-    group << bm( description ) { model_clazz.import cols, vals, :validate=>false }
-
-    models = []
-    array_of_attrs = []
-    
-    vals.each do |arr|
-      array_of_attrs << (attrs={})
-      arr.each_with_index { |value, i| attrs[cols[i]] = value }
-    end
-    array_of_attrs.each{ |attrs| models << model_clazz.new(attrs) }
-
-    description = "#{model_clazz.name}.import(models) for #{num_inserts} records with validations"
-    group << bm( description ) { model_clazz.import models, :validate=>true }
-
-    description = "#{model_clazz.name}.import(models) for #{num_inserts} records without validations"
-    group << bm( description ) { model_clazz.import models, :validate=>false }
-
-    true
-  end
-
-  # Returns a two element array composing of an array of columns and an array of
-  # value sets given the passed +num+.
-  #
-  # === What is a value set?
-  # A value set is an array of arrays. Each child array represents an array of value sets
-  # for a given row of data. 
-  #
-  # For example, say we wanted to represent an insertion of two records:
-  #   column_names = [ 'id', 'name', 'description' ]
-  #   record1 = [ 1, 'John Doe', 'A plumber' ]
-  #   record2 = [ 2, 'John Smith', 'A painter' ]
-  #   value_set [ record1, record2 ]
-  #
-  # == PARAMETER
-  #  * num - the number of records to create
-  def build_array_of_cols_and_vals( num )
-    cols = [ :my_name, :description ]
-    value_sets = []
-    num.times { |i| value_sets << [ "My Name #{i}", "My Description #{i}" ] }
-    [ cols, value_sets ]
-  end
-
-  # Returns a hash of column identifier to value mappings giving the passed in
-  # value array. 
-  #
-  # Example:
-  #   cols = [ 'id', 'name', 'description' ]
-  #   values = [ 1, 'John Doe', 'A plumber' ]
-  #   hsh = create_hash_for_cols_and_vals( cols, values )   
-  #   # hsh => { 'id'=>1, 'name'=>'John Doe', 'description'=>'A plumber' }
-  def create_hash_for_cols_and_vals( cols, vals )
-    h = {}
-    cols.zip( vals ){ |col,val| h[col] = val }
-    h
-  end
-
-  # Deletes all records from all ActiveRecord subclasses
-  def delete_all
-    ActiveRecord::Base.send( :subclasses ).each do |subclass|
-      subclass.delete_all if subclass.respond_to? :delete_all
-    end
-  end
-
-  def initialize   # :nodoc:
-    @results = []
-  end
-	
-end
+class BenchmarkBase
+  attr_reader :results
+
+  # The main benchmark method dispatcher. This dispatches the benchmarks
+  # to actual benchmark_xxxx methods.
+  #
+  # == PARAMETERS
+  #  * table_types - an array of table types to benchmark
+  #  * num - the number of record insertions to test
+  def benchmark( table_types, num )
+    array_of_cols_and_vals = build_array_of_cols_and_vals( num )
+    table_types.each do |table_type|
+      send( "benchmark_#{table_type}", array_of_cols_and_vals )
+    end
+  end
+
+  # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an
+  # actual benchmark.
+  #
+  # == PARAMETERS
+  #  * description - the description of the block that is getting benchmarked
+  #  * blk - the block of code to benchmark
+  #
+  # == RETURNS
+  # An OpenStruct object with the following attributes:
+  #   * description - the description of the benchmark ran
+  #   * tms - a Benchmark::Tms containing the results of the benchmark
+  def bm( description )
+    tms = nil
+    puts "Benchmarking #{description}"
+
+    Benchmark.bm { |x| tms = x.report { yield } }
+    delete_all
+    failed = false
+
+    OpenStruct.new description: description, tms: tms, failed: failed
+  end
+
+  # Given a model class (ie: Topic), and an array of columns and value sets
+  # this will perform all of the benchmarks necessary for this library.
+  #
+  # == PARAMETERS
+  #  * model_clazz - the model class to benchmark (ie: Topic)
+  #  * array_of_cols_and_vals - an array of column identifiers and value sets
+  #
+  # == RETURNS
+  #  returns true
+  def bm_model( model_clazz, array_of_cols_and_vals )
+    puts
+    puts "------ Benchmarking #{model_clazz.name} -------"
+
+    cols, vals = array_of_cols_and_vals
+    num_inserts = vals.size
+
+    # add a new result group for this particular benchmark
+    group = []
+    @results << group
+
+    description = "#{model_clazz.name}.create (#{num_inserts} records)"
+    group << bm( description ) do
+      vals.each do |values|
+        model_clazz.create create_hash_for_cols_and_vals( cols, values )
+      end
+    end
+
+    description = "#{model_clazz.name}.import(column, values) for #{num_inserts} records with validations"
+    group << bm( description ) { model_clazz.import cols, vals, validate: true }
+
+    description = "#{model_clazz.name}.import(columns, values) for #{num_inserts} records without validations"
+    group << bm( description ) { model_clazz.import cols, vals, validate: false }
+
+    models = []
+    array_of_attrs = []
+
+    vals.each do |arr|
+      array_of_attrs << (attrs = {})
+      arr.each_with_index { |value, i| attrs[cols[i]] = value }
+    end
+    array_of_attrs.each { |attrs| models << model_clazz.new(attrs) }
+
+    description = "#{model_clazz.name}.import(models) for #{num_inserts} records with validations"
+    group << bm( description ) { model_clazz.import models, validate: true }
+
+    description = "#{model_clazz.name}.import(models) for #{num_inserts} records without validations"
+    group << bm( description ) { model_clazz.import models, validate: false }
+
+    true
+  end
+
+  # Returns a two element array composing of an array of columns and an array of
+  # value sets given the passed +num+.
+  #
+  # === What is a value set?
+  # A value set is an array of arrays. Each child array represents an array of value sets
+  # for a given row of data.
+  #
+  # For example, say we wanted to represent an insertion of two records:
+  #   column_names = [ 'id', 'name', 'description' ]
+  #   record1 = [ 1, 'John Doe', 'A plumber' ]
+  #   record2 = [ 2, 'John Smith', 'A painter' ]
+  #   value_set [ record1, record2 ]
+  #
+  # == PARAMETER
+  #  * num - the number of records to create
+  def build_array_of_cols_and_vals( num )
+    cols = [:my_name, :description]
+    value_sets = []
+    num.times { |i| value_sets << ["My Name #{i}", "My Description #{i}"] }
+    [cols, value_sets]
+  end
+
+  # Returns a hash of column identifier to value mappings giving the passed in
+  # value array.
+  #
+  # Example:
+  #   cols = [ 'id', 'name', 'description' ]
+  #   values = [ 1, 'John Doe', 'A plumber' ]
+  #   hsh = create_hash_for_cols_and_vals( cols, values )
+  #   # hsh => { 'id'=>1, 'name'=>'John Doe', 'description'=>'A plumber' }
+  def create_hash_for_cols_and_vals( cols, vals )
+    h = {}
+    cols.zip( vals ) { |col, val| h[col] = val }
+    h
+  end
+
+  # Deletes all records from all ActiveRecord subclasses
+  def delete_all
+    ActiveRecord::Base.send( :subclasses ).each do |subclass|
+      subclass.delete_all if subclass.respond_to? :delete_all
+    end
+  end
+
+  def initialize # :nodoc:
+    @results = []
+  end
+end