csv_step_importer 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/Gemfile.lock +3 -3
- data/lib/csv_step_importer/base.rb +13 -4
- data/lib/csv_step_importer/chunk.rb +3 -1
- data/lib/csv_step_importer/model/dao.rb +7 -3
- data/lib/csv_step_importer/model/model.rb +1 -1
- data/lib/csv_step_importer/row.rb +14 -0
- data/lib/csv_step_importer/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ace2a2fdb3fbbeca2a99be11e3d953917fae4db29e6a59d0799429f8f0732748
|
4
|
+
data.tar.gz: 1c3f7e4e4a79fd0e28549f23da924c24519d677baf8fc5ec7359929f09fc36a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d88d7e05982dd7a7932a96f69e18b86a868ed392ac806eeef395971b8fb3847bd6cd87981515457b112368a5007994a615359a08320c9ad8f4f862c9df28b4c
|
7
|
+
data.tar.gz: b2022a3221bb8e6e210139c3eab6db1dab454e71db7551d84a67f6913a6b1fd5c684ba42b9ca5906619aa25054f18c0d9353442fd8a3bbd02d2541963354b6b3
|
data/CHANGELOG.md
CHANGED
@@ -157,3 +157,19 @@ See lib/csv_step_importer/file.rb for more options
|
|
157
157
|
- dao_for's interface changed
|
158
158
|
Before: `dao_for(model: some_model_class_or_instance, pluralize: optional_boolean)`
|
159
159
|
After: `dao_for(some_model_class_or_instance, pluralize: optional_boolean)`
|
160
|
+
|
161
|
+
## 2018-09-14 Version 0.13.0
|
162
|
+
### Added
|
163
|
+
- Added option `ignore_invalid_rows` in Row. Ignores all rows with failing validations.
|
164
|
+
Usage:
|
165
|
+
Row.set :ignore_invalid_rows, true # or you set it in your row class
|
166
|
+
loader = Loader.new *options
|
167
|
+
loader.valid? # => true # Bad rows are ignored, however all other validations still need to pass
|
168
|
+
loader.errors # => []
|
169
|
+
loader.save! # true
|
170
|
+
This also adds the `include_row?` method to the Row class, which will return false, if the row should not be included.
|
171
|
+
- Add use_transaction to all nodes. Default is `false`, except in Chunk.
|
172
|
+
This causes all succeeding chunks to be committed, but will rollback failing chunks.
|
173
|
+
If you require data wide transactions, use `Loader.set :use_transaction, true`
|
174
|
+
### Changed
|
175
|
+
- Fix and optimize filtering of daos, when composite_key_columns are applied.
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
csv_step_importer (0.
|
4
|
+
csv_step_importer (0.13.0)
|
5
5
|
activemodel
|
6
6
|
activerecord-import
|
7
7
|
activesupport
|
@@ -87,7 +87,7 @@ GEM
|
|
87
87
|
diff-lcs (>= 1.2.0, < 2.0)
|
88
88
|
rspec-support (~> 3.8.0)
|
89
89
|
rspec-support (3.8.0)
|
90
|
-
rubocop (0.59.
|
90
|
+
rubocop (0.59.1)
|
91
91
|
jaro_winkler (~> 1.5.1)
|
92
92
|
parallel (~> 1.10)
|
93
93
|
parser (>= 2.5, != 2.5.1.1)
|
@@ -104,7 +104,7 @@ GEM
|
|
104
104
|
rubocop-rspec (1.29.1)
|
105
105
|
rubocop (>= 0.58.0)
|
106
106
|
ruby-progressbar (1.10.0)
|
107
|
-
smarter_csv (1.2.
|
107
|
+
smarter_csv (1.2.5)
|
108
108
|
thor (0.20.0)
|
109
109
|
thread_safe (0.3.6)
|
110
110
|
tzinfo (1.2.5)
|
@@ -13,11 +13,16 @@ module CSVStepImporter
|
|
13
13
|
class CSVFileImportError < CSVImportError; end
|
14
14
|
|
15
15
|
# defines a method with the specified proc or a proc returning the value of the second attribute
|
16
|
-
def self.set
|
16
|
+
def self.set(name, value_or_proc)
|
17
17
|
procedure = value_or_proc.is_a?(Proc) ? value_or_proc : -> { value_or_proc }
|
18
18
|
define_method name, procedure
|
19
19
|
end
|
20
20
|
|
21
|
+
# do not use transactions by default, override if necessary in subclasses
|
22
|
+
# NOTE: Chunk's default is true
|
23
|
+
# NOTE: To turn on file wide transactions, use `Loader.set :use_transaction, true`
|
24
|
+
set :use_transaction, false
|
25
|
+
|
21
26
|
def assign_attributes(attributes)
|
22
27
|
attributes.each do |key, value|
|
23
28
|
send("#{key}=", value)
|
@@ -36,9 +41,13 @@ module CSVStepImporter
|
|
36
41
|
run_callbacks :save do
|
37
42
|
return false unless valid?
|
38
43
|
|
39
|
-
status =
|
40
|
-
|
41
|
-
|
44
|
+
status = if use_transaction
|
45
|
+
!!::ActiveRecord::Base.transaction do
|
46
|
+
raise ::ActiveRecord::Rollback unless create_or_update
|
47
|
+
true
|
48
|
+
end
|
49
|
+
else
|
50
|
+
create_or_update
|
42
51
|
end
|
43
52
|
|
44
53
|
status
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module CSVStepImporter
|
4
4
|
class Chunk < CSVStepImporter::Node
|
5
|
+
set :use_transaction, true
|
6
|
+
|
5
7
|
attr_accessor :cache, :rows, :first_row
|
6
8
|
|
7
9
|
def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 0, **attributes)
|
@@ -21,7 +23,7 @@ module CSVStepImporter
|
|
21
23
|
row_number = self.first_row - 1
|
22
24
|
rows = rows.collect do |row|
|
23
25
|
row_class.new(parent: row_parent_node, row_number: row_number += 1, **row)
|
24
|
-
end
|
26
|
+
end.find_all(&:include_row?)
|
25
27
|
end
|
26
28
|
|
27
29
|
@rows = rows
|
@@ -34,9 +34,7 @@ module CSVStepImporter
|
|
34
34
|
|
35
35
|
# returns an array of all column values, used for batch importing
|
36
36
|
def value
|
37
|
-
@value ||= columns
|
38
|
-
values[key] = value_for_key key
|
39
|
-
end
|
37
|
+
@value ||= values_for columns
|
40
38
|
end
|
41
39
|
|
42
40
|
# retrieve a value for a key from the dao or row
|
@@ -54,6 +52,12 @@ module CSVStepImporter
|
|
54
52
|
end
|
55
53
|
end
|
56
54
|
|
55
|
+
def values_for(included_keys)
|
56
|
+
included_keys.each_with_object({}) do |key, values|
|
57
|
+
values[key] = value_for_key key
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
57
61
|
def create_or_update
|
58
62
|
# DAOs are usually processed in batches by the model and not saved one by one
|
59
63
|
true
|
@@ -78,7 +78,7 @@ module CSVStepImporter
|
|
78
78
|
unique_daos = {}
|
79
79
|
|
80
80
|
daos.keep_if do |dao|
|
81
|
-
hash = dao.
|
81
|
+
hash = dao.values_for(composite_key_columns).hash
|
82
82
|
keep = (unique_daos[hash] ||= dao) == dao
|
83
83
|
|
84
84
|
# unlink to be deleted dao and add a link to
|
@@ -4,6 +4,16 @@ module CSVStepImporter
|
|
4
4
|
class Row < CSVStepImporter::Node
|
5
5
|
attr_accessor :attributes, :cache, :row_number
|
6
6
|
|
7
|
+
#########################################################
|
8
|
+
# Configuration
|
9
|
+
#########################################################
|
10
|
+
|
11
|
+
set :ignore_invalid_rows, false
|
12
|
+
|
13
|
+
#########################################################
|
14
|
+
# Logic
|
15
|
+
#########################################################
|
16
|
+
|
7
17
|
def initialize(parent:, row_number:, **attributes)
|
8
18
|
super parent: parent
|
9
19
|
|
@@ -21,5 +31,9 @@ module CSVStepImporter
|
|
21
31
|
def dao_for(model, pluralize: false)
|
22
32
|
cache[model.cache_key(pluralize: pluralize)]
|
23
33
|
end
|
34
|
+
|
35
|
+
def include_row?
|
36
|
+
ignore_invalid_rows ? valid? : true
|
37
|
+
end
|
24
38
|
end
|
25
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csv_step_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian-Manuel Butzke
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-09-
|
11
|
+
date: 2018-09-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|