csv_step_importer 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c7e47c79367ebc96d77f98f8a222efed6058b41e297fd330a5819a3e7449214a
4
- data.tar.gz: c60c54fbca29a869584a8ef08bd0149a6d90c4f4f6bfafc6addf89847922c740
3
+ metadata.gz: ace2a2fdb3fbbeca2a99be11e3d953917fae4db29e6a59d0799429f8f0732748
4
+ data.tar.gz: 1c3f7e4e4a79fd0e28549f23da924c24519d677baf8fc5ec7359929f09fc36a8
5
5
  SHA512:
6
- metadata.gz: d03d12132eeccfa84468e1f861f25bc462b1d00957f3ccb0719d40437d6a23a8058a35997ada9eadeaaf087fb7051f5950c4a35780404426fcb4df99381ca19d
7
- data.tar.gz: aefe551280cb52ca746f57d4d5c5bcd13ade807b3bececd1561fe81bfeea953a93a5ee3b7184b97a4d39d6f1ff91000ee5f12db83d704607f065395e78959e6a
6
+ metadata.gz: 6d88d7e05982dd7a7932a96f69e18b86a868ed392ac806eeef395971b8fb3847bd6cd87981515457b112368a5007994a615359a08320c9ad8f4f862c9df28b4c
7
+ data.tar.gz: b2022a3221bb8e6e210139c3eab6db1dab454e71db7551d84a67f6913a6b1fd5c684ba42b9ca5906619aa25054f18c0d9353442fd8a3bbd02d2541963354b6b3
@@ -157,3 +157,19 @@ See lib/csv_step_importer/file.rb for more options
157
157
  - dao_for's interface changed
158
158
  Before: `dao_for(model: some_model_class_or_instance, pluralize: optional_boolean)`
159
159
  After: `dao_for(some_model_class_or_instance, pluralize: optional_boolean)`
160
+
161
+ ## 2018-09-14 Version 0.13.0
162
+ ### Added
163
+ - Added option `ignore_invalid_rows` in Row. Ignores all rows with failing validations.
164
+ Usage:
165
+ Row.set :ignore_invalid_rows, true # or you set it in your row class
166
+ loader = Loader.new *options
167
+ loader.valid? # => true # Bad rows are ignored, however all other validations still need to pass
168
+ loader.errors # => []
169
+ loader.save! # true
170
+ This also adds the `include_row?` method to the Row class, which will return false, if the row should not be included.
171
+ - Add use_transaction to all nodes. Default is `false`, except in Chunk.
172
+ This causes all succeeding chunks to be committed, but will rollback failing chunks.
173
+ If you require data wide transactions, use `Loader.set :use_transaction, true`
174
+ ### Changed
175
+ - Fix and optimize filtering of daos, when composite_key_columns are applied.
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- csv_step_importer (0.12.0)
4
+ csv_step_importer (0.13.0)
5
5
  activemodel
6
6
  activerecord-import
7
7
  activesupport
@@ -87,7 +87,7 @@ GEM
87
87
  diff-lcs (>= 1.2.0, < 2.0)
88
88
  rspec-support (~> 3.8.0)
89
89
  rspec-support (3.8.0)
90
- rubocop (0.59.0)
90
+ rubocop (0.59.1)
91
91
  jaro_winkler (~> 1.5.1)
92
92
  parallel (~> 1.10)
93
93
  parser (>= 2.5, != 2.5.1.1)
@@ -104,7 +104,7 @@ GEM
104
104
  rubocop-rspec (1.29.1)
105
105
  rubocop (>= 0.58.0)
106
106
  ruby-progressbar (1.10.0)
107
- smarter_csv (1.2.4)
107
+ smarter_csv (1.2.5)
108
108
  thor (0.20.0)
109
109
  thread_safe (0.3.6)
110
110
  tzinfo (1.2.5)
@@ -13,11 +13,16 @@ module CSVStepImporter
13
13
  class CSVFileImportError < CSVImportError; end
14
14
 
15
15
  # defines a method with the specified proc or a proc returning the value of the second attribute
16
- def self.set name, value_or_proc
16
+ def self.set(name, value_or_proc)
17
17
  procedure = value_or_proc.is_a?(Proc) ? value_or_proc : -> { value_or_proc }
18
18
  define_method name, procedure
19
19
  end
20
20
 
21
+ # do not use transactions by default, override if necessary in subclasses
22
+ # NOTE: Chunk's default is true
23
+ # NOTE: To turn on file wide transactions, use `Loader.set :use_transaction, true`
24
+ set :use_transaction, false
25
+
21
26
  def assign_attributes(attributes)
22
27
  attributes.each do |key, value|
23
28
  send("#{key}=", value)
@@ -36,9 +41,13 @@ module CSVStepImporter
36
41
  run_callbacks :save do
37
42
  return false unless valid?
38
43
 
39
- status = !!::ActiveRecord::Base.transaction do
40
- raise ::ActiveRecord::Rollback unless create_or_update
41
- true
44
+ status = if use_transaction
45
+ !!::ActiveRecord::Base.transaction do
46
+ raise ::ActiveRecord::Rollback unless create_or_update
47
+ true
48
+ end
49
+ else
50
+ create_or_update
42
51
  end
43
52
 
44
53
  status
@@ -2,6 +2,8 @@
2
2
 
3
3
  module CSVStepImporter
4
4
  class Chunk < CSVStepImporter::Node
5
+ set :use_transaction, true
6
+
5
7
  attr_accessor :cache, :rows, :first_row
6
8
 
7
9
  def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 0, **attributes)
@@ -21,7 +23,7 @@ module CSVStepImporter
21
23
  row_number = self.first_row - 1
22
24
  rows = rows.collect do |row|
23
25
  row_class.new(parent: row_parent_node, row_number: row_number += 1, **row)
24
- end
26
+ end.find_all(&:include_row?)
25
27
  end
26
28
 
27
29
  @rows = rows
@@ -34,9 +34,7 @@ module CSVStepImporter
34
34
 
35
35
  # returns an array of all column values, used for batch importing
36
36
  def value
37
- @value ||= columns.each_with_object({}) do |key, values|
38
- values[key] = value_for_key key
39
- end
37
+ @value ||= values_for columns
40
38
  end
41
39
 
42
40
  # retrieve a value for a key from the dao or row
@@ -54,6 +52,12 @@ module CSVStepImporter
54
52
  end
55
53
  end
56
54
 
55
+ def values_for(included_keys)
56
+ included_keys.each_with_object({}) do |key, values|
57
+ values[key] = value_for_key key
58
+ end
59
+ end
60
+
57
61
  def create_or_update
58
62
  # DAOs are usually processed in batches by the model and not saved one by one
59
63
  true
@@ -78,7 +78,7 @@ module CSVStepImporter
78
78
  unique_daos = {}
79
79
 
80
80
  daos.keep_if do |dao|
81
- hash = dao.value.slice(*composite_key_columns).hash
81
+ hash = dao.values_for(composite_key_columns).hash
82
82
  keep = (unique_daos[hash] ||= dao) == dao
83
83
 
84
84
  # unlink to be deleted dao and add a link to
@@ -4,6 +4,16 @@ module CSVStepImporter
4
4
  class Row < CSVStepImporter::Node
5
5
  attr_accessor :attributes, :cache, :row_number
6
6
 
7
+ #########################################################
8
+ # Configuration
9
+ #########################################################
10
+
11
+ set :ignore_invalid_rows, false
12
+
13
+ #########################################################
14
+ # Logic
15
+ #########################################################
16
+
7
17
  def initialize(parent:, row_number:, **attributes)
8
18
  super parent: parent
9
19
 
@@ -21,5 +31,9 @@ module CSVStepImporter
21
31
  def dao_for(model, pluralize: false)
22
32
  cache[model.cache_key(pluralize: pluralize)]
23
33
  end
34
+
35
+ def include_row?
36
+ ignore_invalid_rows ? valid? : true
37
+ end
24
38
  end
25
39
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module CSVStepImporter
4
- VERSION = "0.12.0"
4
+ VERSION = "0.13.0"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: csv_step_importer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.0
4
+ version: 0.13.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christian-Manuel Butzke
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-09-14 00:00:00.000000000 Z
11
+ date: 2018-09-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler