csv_step_importer 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7e47c79367ebc96d77f98f8a222efed6058b41e297fd330a5819a3e7449214a
-  data.tar.gz: c60c54fbca29a869584a8ef08bd0149a6d90c4f4f6bfafc6addf89847922c740
+  metadata.gz: ace2a2fdb3fbbeca2a99be11e3d953917fae4db29e6a59d0799429f8f0732748
+  data.tar.gz: 1c3f7e4e4a79fd0e28549f23da924c24519d677baf8fc5ec7359929f09fc36a8
 SHA512:
-  metadata.gz: d03d12132eeccfa84468e1f861f25bc462b1d00957f3ccb0719d40437d6a23a8058a35997ada9eadeaaf087fb7051f5950c4a35780404426fcb4df99381ca19d
-  data.tar.gz: aefe551280cb52ca746f57d4d5c5bcd13ade807b3bececd1561fe81bfeea953a93a5ee3b7184b97a4d39d6f1ff91000ee5f12db83d704607f065395e78959e6a
+  metadata.gz: 6d88d7e05982dd7a7932a96f69e18b86a868ed392ac806eeef395971b8fb3847bd6cd87981515457b112368a5007994a615359a08320c9ad8f4f862c9df28b4c
+  data.tar.gz: b2022a3221bb8e6e210139c3eab6db1dab454e71db7551d84a67f6913a6b1fd5c684ba42b9ca5906619aa25054f18c0d9353442fd8a3bbd02d2541963354b6b3

data/CHANGELOG.md

@@ -157,3 +157,19 @@ See lib/csv_step_importer/file.rb for more options
 - dao_for's interface changed
   Before: `dao_for(model: some_model_class_or_instance, pluralize: optional_boolean)`
   After: `dao_for(some_model_class_or_instance, pluralize: optional_boolean)`
+
+## 2018-09-14 Version 0.13.0
+### Added
+- Added option `ignore_invalid_rows` in Row. Ignores all rows with failing validations.
+  Usage:
+    Row.set :ignore_invalid_rows, true # or you set it in your row class
+    loader = Loader.new *options
+    loader.valid? # => true # Bad rows are ignored, however all other validations still need to pass
+    loader.errors # => []
+    loader.save! # true
+  This also adds the `include_row?` method to the Row class, which will return false, if the row should not be included.
+- Add use_transaction to all nodes. Default is `false`, except in Chunk.
+  This causes all succeeding chunks to be committed, but will rollback failing chunks.
+  If you require data wide transactions, use `Loader.set :use_transaction, true`
+### Changed
+- Fix and optimize filtering of daos, when composite_key_columns are applied.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    csv_step_importer (0.12.0)
+    csv_step_importer (0.13.0)
       activemodel
       activerecord-import
       activesupport
@@ -87,7 +87,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubocop (0.59.0)
+    rubocop (0.59.1)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.5, != 2.5.1.1)
@@ -104,7 +104,7 @@ GEM
     rubocop-rspec (1.29.1)
       rubocop (>= 0.58.0)
     ruby-progressbar (1.10.0)
-    smarter_csv (1.2.4)
+    smarter_csv (1.2.5)
     thor (0.20.0)
     thread_safe (0.3.6)
     tzinfo (1.2.5)

data/lib/csv_step_importer/base.rb

@@ -13,11 +13,16 @@ module CSVStepImporter
     class CSVFileImportError < CSVImportError; end
 
     # defines a method with the specified proc or a proc returning the value of the second attribute
-    def self.set name, value_or_proc
+    def self.set(name, value_or_proc)
       procedure = value_or_proc.is_a?(Proc) ? value_or_proc : -> { value_or_proc }
       define_method name, procedure
     end
 
+    # do not use transactions by default, override if necessary in subclasses
+    # NOTE: Chunk's default is true
+    # NOTE: To turn on file wide transactions, use `Loader.set :use_transaction, true`
+    set :use_transaction, false
+
     def assign_attributes(attributes)
       attributes.each do |key, value|
         send("#{key}=", value)
@@ -36,9 +41,13 @@ module CSVStepImporter
       run_callbacks :save do
         return false unless valid?
 
-        status = !!::ActiveRecord::Base.transaction do
-          raise ::ActiveRecord::Rollback unless create_or_update
-          true
+        status = if use_transaction
+          !!::ActiveRecord::Base.transaction do
+            raise ::ActiveRecord::Rollback unless create_or_update
+            true
+          end
+        else
+          create_or_update
         end
 
         status

data/lib/csv_step_importer/chunk.rb

@@ -2,6 +2,8 @@
 
 module CSVStepImporter
   class Chunk < CSVStepImporter::Node
+    set :use_transaction, true
+
     attr_accessor :cache, :rows, :first_row
 
     def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 0, **attributes)
@@ -21,7 +23,7 @@ module CSVStepImporter
         row_number = self.first_row - 1
         rows = rows.collect do |row|
           row_class.new(parent: row_parent_node, row_number: row_number += 1, **row)
-        end
+        end.find_all(&:include_row?)
       end
 
       @rows = rows

data/lib/csv_step_importer/model/dao.rb

@@ -34,9 +34,7 @@ module CSVStepImporter
 
       # returns an array of all column values, used for batch importing
       def value
-        @value ||= columns.each_with_object({}) do |key, values|
-          values[key] = value_for_key key
-        end
+        @value ||= values_for columns
       end
 
       # retrieve a value for a key from the dao or row
@@ -54,6 +52,12 @@ module CSVStepImporter
         end
       end
 
+      def values_for(included_keys)
+        included_keys.each_with_object({}) do |key, values|
+          values[key] = value_for_key key
+        end
+      end
+
       def create_or_update
         # DAOs are usually processed in batches by the model and not saved one by one
         true

data/lib/csv_step_importer/model/model.rb

@@ -78,7 +78,7 @@ module CSVStepImporter
         unique_daos = {}
 
         daos.keep_if do |dao|
-          hash = dao.value.slice(*composite_key_columns).hash
+          hash = dao.values_for(composite_key_columns).hash
           keep = (unique_daos[hash] ||= dao) == dao
 
           # unlink to be deleted dao and add a link to

data/lib/csv_step_importer/row.rb

@@ -4,6 +4,16 @@ module CSVStepImporter
   class Row < CSVStepImporter::Node
     attr_accessor :attributes, :cache, :row_number
 
+    #########################################################
+    # Configuration
+    #########################################################
+
+    set :ignore_invalid_rows, false
+
+    #########################################################
+    # Logic
+    #########################################################
+
     def initialize(parent:, row_number:, **attributes)
       super parent: parent
 
@@ -21,5 +31,9 @@ module CSVStepImporter
     def dao_for(model, pluralize: false)
       cache[model.cache_key(pluralize: pluralize)]
     end
+
+    def include_row?
+      ignore_invalid_rows ? valid? : true
+    end
   end
 end

data/lib/csv_step_importer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CSVStepImporter
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv_step_importer
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Christian-Manuel Butzke
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-14 00:00:00.000000000 Z
+date: 2018-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler