csvbuilder-importer 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/Gemfile.lock +89 -0
- data/README.md +170 -5
- data/gemfiles/rails_edge.gemfile +17 -0
- data/lib/csvbuilder/importer/internal/import/csv.rb +1 -1
- data/lib/csvbuilder/importer/version.rb +1 -1
- metadata +5 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 52cdbeae4314f705e797a216730c276ccc68f57140f3c06ae86109fa9a9b6902
|
|
4
|
+
data.tar.gz: 45c9d16608cc0e86e920333bf9e08db34f98f9ca2fa17354c95323132e8afb43
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ae94d692fbfa627cdd37952a47b7db183414095e206d1bbd908d01a2e3e4ff0e719673a7f0fc8a1545bc2316bcce7d17d542ddcbe9b565f0f0d68137f5c25f23
|
|
7
|
+
data.tar.gz: a7c914fff4abb75256af5f3f996a13116db6408ff64290bf22430c75ab45503c85561bd1b959657dba6b95a6fb5975b76988a2f749d2a2ddb51ab52012de187e
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
csvbuilder-importer (0.1.4)
|
|
5
|
+
activemodel (>= 5.2)
|
|
6
|
+
activesupport (>= 5.2)
|
|
7
|
+
csvbuilder-core
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
activemodel (7.0.4.2)
|
|
13
|
+
activesupport (= 7.0.4.2)
|
|
14
|
+
activesupport (7.0.4.2)
|
|
15
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
16
|
+
i18n (>= 1.6, < 2)
|
|
17
|
+
minitest (>= 5.1)
|
|
18
|
+
tzinfo (~> 2.0)
|
|
19
|
+
ast (2.4.2)
|
|
20
|
+
concurrent-ruby (1.2.2)
|
|
21
|
+
csvbuilder-core (0.1.3)
|
|
22
|
+
activesupport (>= 5.2, < 8)
|
|
23
|
+
diff-lcs (1.5.0)
|
|
24
|
+
i18n (1.12.0)
|
|
25
|
+
concurrent-ruby (~> 1.0)
|
|
26
|
+
json (2.6.3)
|
|
27
|
+
minitest (5.17.0)
|
|
28
|
+
parallel (1.23.0)
|
|
29
|
+
parser (3.2.2.0)
|
|
30
|
+
ast (~> 2.4.1)
|
|
31
|
+
rainbow (3.1.1)
|
|
32
|
+
rake (13.0.6)
|
|
33
|
+
regexp_parser (2.8.0)
|
|
34
|
+
rexml (3.2.5)
|
|
35
|
+
rspec (3.12.0)
|
|
36
|
+
rspec-core (~> 3.12.0)
|
|
37
|
+
rspec-expectations (~> 3.12.0)
|
|
38
|
+
rspec-mocks (~> 3.12.0)
|
|
39
|
+
rspec-core (3.12.1)
|
|
40
|
+
rspec-support (~> 3.12.0)
|
|
41
|
+
rspec-expectations (3.12.2)
|
|
42
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
43
|
+
rspec-support (~> 3.12.0)
|
|
44
|
+
rspec-mocks (3.12.3)
|
|
45
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
46
|
+
rspec-support (~> 3.12.0)
|
|
47
|
+
rspec-support (3.12.0)
|
|
48
|
+
rubocop (1.50.2)
|
|
49
|
+
json (~> 2.3)
|
|
50
|
+
parallel (~> 1.10)
|
|
51
|
+
parser (>= 3.2.0.0)
|
|
52
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
53
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
54
|
+
rexml (>= 3.2.5, < 4.0)
|
|
55
|
+
rubocop-ast (>= 1.28.0, < 2.0)
|
|
56
|
+
ruby-progressbar (~> 1.7)
|
|
57
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
|
58
|
+
rubocop-ast (1.28.0)
|
|
59
|
+
parser (>= 3.2.1.0)
|
|
60
|
+
rubocop-capybara (2.18.0)
|
|
61
|
+
rubocop (~> 1.41)
|
|
62
|
+
rubocop-performance (1.17.1)
|
|
63
|
+
rubocop (>= 1.7.0, < 2.0)
|
|
64
|
+
rubocop-ast (>= 0.4.0)
|
|
65
|
+
rubocop-rake (0.6.0)
|
|
66
|
+
rubocop (~> 1.0)
|
|
67
|
+
rubocop-rspec (2.20.0)
|
|
68
|
+
rubocop (~> 1.33)
|
|
69
|
+
rubocop-capybara (~> 2.17)
|
|
70
|
+
ruby-progressbar (1.13.0)
|
|
71
|
+
tzinfo (2.0.6)
|
|
72
|
+
concurrent-ruby (~> 1.0)
|
|
73
|
+
unicode-display_width (2.4.2)
|
|
74
|
+
|
|
75
|
+
PLATFORMS
|
|
76
|
+
arm64-darwin-22
|
|
77
|
+
x86_64-linux
|
|
78
|
+
|
|
79
|
+
DEPENDENCIES
|
|
80
|
+
csvbuilder-importer!
|
|
81
|
+
rake
|
|
82
|
+
rspec
|
|
83
|
+
rubocop
|
|
84
|
+
rubocop-performance
|
|
85
|
+
rubocop-rake
|
|
86
|
+
rubocop-rspec
|
|
87
|
+
|
|
88
|
+
BUNDLED WITH
|
|
89
|
+
2.4.7
|
data/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Csvbuilder::Importer
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[Csvbuilder::Importer](https://github.com/joel/csvbuilder-importer) is part of the [csvbuilder-collection](https://github.com/joel/csvbuilder)
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
The importer contains the implementation for importing data from a CSV file.
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -14,9 +14,174 @@ If bundler is not being used to manage dependencies, install the gem by executin
|
|
|
14
14
|
|
|
15
15
|
$ gem install csvbuilder-importer
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
# Usage
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
Importing data from a CSV is critical and requires two validation layers. First, you need to ensure data from the CSV are correct, and Second, you need to check that data respects the business logic before inserting it into the system.
|
|
20
|
+
|
|
21
|
+
To do that, `Csvbuilder::Import` use `ActiveModel::Validations` so you can write your validations in the `CsvImportModel`.
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
class UserCsvImportModel
|
|
25
|
+
include Csvbuilder::Model
|
|
26
|
+
include Csvbuilder::Import
|
|
27
|
+
|
|
28
|
+
column :first_name
|
|
29
|
+
column :last_name
|
|
30
|
+
|
|
31
|
+
validates :first_name, presence: true, length: { minimum: 2 }
|
|
32
|
+
|
|
33
|
+
def abort?
|
|
34
|
+
"#{first_name} #{last_name}" == "Bill Gates"
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The import takes the CSV file and the Import class.
|
|
40
|
+
|
|
41
|
+
```ruby
|
|
42
|
+
rows = Csvbuilder::Import::File.new(file.path, UserCsvImportModel).each
|
|
43
|
+
row = rows.next
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`Csvbuilder::Import` implement two essential methods:
|
|
47
|
+
|
|
48
|
+
1. skip?
|
|
49
|
+
2. abort?
|
|
50
|
+
|
|
51
|
+
You have to provide your implementation of the method `abort?`. If the method `abort?` returns true, the iteration will stop.
|
|
52
|
+
|
|
53
|
+
By default, `skip?` return true if the `CsvImportClass` is invalid, but it is safe to override. This means the previous iteration will not return any invalid row.
|
|
54
|
+
|
|
55
|
+
# Integration
|
|
56
|
+
|
|
57
|
+
Let's say we want to do something useful and add users if those users are valid.
|
|
58
|
+
|
|
59
|
+
Let's extract the `CsvRowModel` for more clarity:
|
|
60
|
+
|
|
61
|
+
```ruby
|
|
62
|
+
class UserCsvRowModel
|
|
63
|
+
include Csvbuilder::Model
|
|
64
|
+
|
|
65
|
+
column :first_name
|
|
66
|
+
column :last_name
|
|
67
|
+
end
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
```ruby
|
|
71
|
+
class UserCsvImportModel < UserCsvRowModel
|
|
72
|
+
include Csvbuilder::Import
|
|
73
|
+
|
|
74
|
+
validates :first_name, presence: true, length: { minimum: 2 }
|
|
75
|
+
validates :last_name, presence: true, length: { minimum: 2 }
|
|
76
|
+
|
|
77
|
+
def user
|
|
78
|
+
User.new(first_name: first_name, last_name: last_name)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Skip if the row is not valid,
|
|
82
|
+
# the user is not valid or
|
|
83
|
+
# the user already exists
|
|
84
|
+
def skip?
|
|
85
|
+
super || !user.valid? || user.exists?
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Now, we can safely import our users.
|
|
91
|
+
|
|
92
|
+
```ruby
|
|
93
|
+
[
|
|
94
|
+
["First name", "Last name"],
|
|
95
|
+
["John" , "Doe" ],
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
Csvbuilder::Import::File.new(file.path, UserCsvImportModel).each do |row_model|
|
|
99
|
+
row_model.user.save
|
|
100
|
+
end
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
# Advance Integration
|
|
104
|
+
|
|
105
|
+
`Csvbuilder::Import::File` implement callbacks. It provides the following:
|
|
106
|
+
|
|
107
|
+
1. before_each_iteration
|
|
108
|
+
2. around_each_iteration
|
|
109
|
+
3. after_each_iteration
|
|
110
|
+
4. before_next
|
|
111
|
+
5. around_next
|
|
112
|
+
6. after_next
|
|
113
|
+
7. before_abort
|
|
114
|
+
8. before_skip
|
|
115
|
+
|
|
116
|
+
Let's extend `Csvbuilder::Import::File`
|
|
117
|
+
|
|
118
|
+
```ruby
|
|
119
|
+
class Importer < Csvbuilder::Import::File
|
|
120
|
+
attr_reader :row_in_errors
|
|
121
|
+
|
|
122
|
+
def initialize(*args)
|
|
123
|
+
super
|
|
124
|
+
@row_in_errors = RowErrors.new
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
after_next do
|
|
128
|
+
next true unless current_row_model # End of File
|
|
129
|
+
next true if current_row_model.valid? # No Errors To Collect
|
|
130
|
+
|
|
131
|
+
row_in_errors.append_errors(current_row_model)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Now the importer can report the errors encountered instead of ignoring them.
|
|
137
|
+
|
|
138
|
+
For documentation purposes, here is a possible implementation of the errors collector:
|
|
139
|
+
|
|
140
|
+
```ruby
|
|
141
|
+
class RowErrors
|
|
142
|
+
attr_reader :headers, :errors
|
|
143
|
+
|
|
144
|
+
def initialize
|
|
145
|
+
@errors = []
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def append_errors(row_model)
|
|
149
|
+
@headers ||= begin
|
|
150
|
+
errors << row_model.class.headers
|
|
151
|
+
row_model.class.headers
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
row_in_error = []
|
|
155
|
+
row_model.source_attributes.map do |key, value|
|
|
156
|
+
row_in_error << if row_model.errors.messages[key].present?
|
|
157
|
+
"Initial Value: [#{value}] - Errors: #{row_model.errors.messages[key].join(", ")}"
|
|
158
|
+
else
|
|
159
|
+
value
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
errors << row_in_error
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Now we can nicely show the errors which occur over the import.
|
|
168
|
+
|
|
169
|
+
```ruby
|
|
170
|
+
[
|
|
171
|
+
["First name", "Last name"],
|
|
172
|
+
["J", "Doe"]
|
|
173
|
+
]
|
|
174
|
+
|
|
175
|
+
importer.row_in_errors.errors
|
|
176
|
+
# => [
|
|
177
|
+
# => ["First Name", "Last Name"],
|
|
178
|
+
# => ["Initial Value: [J] - Errors: is too short (minimum is 2 characters)", "Doe"]
|
|
179
|
+
# => ]
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Thanks to the callback mechanism, the opportunities to interact with the import are immense. For instance, you can show the errors on a Web Form and offer the chance to the user to change the data and re-attempt.
|
|
183
|
+
|
|
184
|
+
For long imports, you can show a progress bar to help customers cope with the import time; as you know, if errors have occurred, you can change the colour of the progress bar accordingly and offer the possibility to stop the import earlier.
|
|
20
185
|
|
|
21
186
|
## Development
|
|
22
187
|
|
|
@@ -26,7 +191,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
|
26
191
|
|
|
27
192
|
## Contributing
|
|
28
193
|
|
|
29
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
|
194
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/joel/csvbuilder-importer. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/csvbuilder-importer/blob/main/CODE_OF_CONDUCT.md).
|
|
30
195
|
|
|
31
196
|
## License
|
|
32
197
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
source "https://rubygems.org"
|
|
4
|
+
|
|
5
|
+
# Runtime dependencies
|
|
6
|
+
gem "activemodel", github: "rails/rails", branch: "main"
|
|
7
|
+
gem "activesupport", github: "rails/rails", branch: "main"
|
|
8
|
+
|
|
9
|
+
gem "csvbuilder-core"
|
|
10
|
+
|
|
11
|
+
# Development dependencies
|
|
12
|
+
gem "rake"
|
|
13
|
+
gem "rspec"
|
|
14
|
+
gem "rubocop"
|
|
15
|
+
gem "rubocop-performance"
|
|
16
|
+
gem "rubocop-rake"
|
|
17
|
+
gem "rubocop-rspec"
|
|
@@ -23,7 +23,7 @@ module Csvbuilder
|
|
|
23
23
|
# http://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
|
|
24
24
|
# @return [Integer] the number of rows in the file, including empty new lines
|
|
25
25
|
def size
|
|
26
|
-
@size ||=
|
|
26
|
+
@size ||= ::File.readlines(file_path).length
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
# If the current position is at the headers, skip it and return it. Otherwise, only return false.
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: csvbuilder-importer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Joel Azemar
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-04-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activemodel
|
|
@@ -17,9 +17,6 @@ dependencies:
|
|
|
17
17
|
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
19
|
version: '5.2'
|
|
20
|
-
- - "<"
|
|
21
|
-
- !ruby/object:Gem::Version
|
|
22
|
-
version: '8'
|
|
23
20
|
type: :runtime
|
|
24
21
|
prerelease: false
|
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -27,9 +24,6 @@ dependencies:
|
|
|
27
24
|
- - ">="
|
|
28
25
|
- !ruby/object:Gem::Version
|
|
29
26
|
version: '5.2'
|
|
30
|
-
- - "<"
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: '8'
|
|
33
27
|
- !ruby/object:Gem::Dependency
|
|
34
28
|
name: activesupport
|
|
35
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -37,9 +31,6 @@ dependencies:
|
|
|
37
31
|
- - ">="
|
|
38
32
|
- !ruby/object:Gem::Version
|
|
39
33
|
version: '5.2'
|
|
40
|
-
- - "<"
|
|
41
|
-
- !ruby/object:Gem::Version
|
|
42
|
-
version: '8'
|
|
43
34
|
type: :runtime
|
|
44
35
|
prerelease: false
|
|
45
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -47,9 +38,6 @@ dependencies:
|
|
|
47
38
|
- - ">="
|
|
48
39
|
- !ruby/object:Gem::Version
|
|
49
40
|
version: '5.2'
|
|
50
|
-
- - "<"
|
|
51
|
-
- !ruby/object:Gem::Version
|
|
52
|
-
version: '8'
|
|
53
41
|
- !ruby/object:Gem::Dependency
|
|
54
42
|
name: csvbuilder-core
|
|
55
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -77,12 +65,14 @@ files:
|
|
|
77
65
|
- CHANGELOG.md
|
|
78
66
|
- CODE_OF_CONDUCT.md
|
|
79
67
|
- Gemfile
|
|
68
|
+
- Gemfile.lock
|
|
80
69
|
- LICENSE.txt
|
|
81
70
|
- README.md
|
|
82
71
|
- Rakefile
|
|
83
72
|
- gemfiles/Gemfile.5.2._gemfile
|
|
84
73
|
- gemfiles/Gemfile.6.1._gemfile
|
|
85
74
|
- gemfiles/Gemfile.7.0._gemfile
|
|
75
|
+
- gemfiles/rails_edge.gemfile
|
|
86
76
|
- lib/csvbuilder/importer.rb
|
|
87
77
|
- lib/csvbuilder/importer/concerns/import/attributes.rb
|
|
88
78
|
- lib/csvbuilder/importer/concerns/import/base.rb
|
|
@@ -115,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
115
105
|
- !ruby/object:Gem::Version
|
|
116
106
|
version: '0'
|
|
117
107
|
requirements: []
|
|
118
|
-
rubygems_version: 3.
|
|
108
|
+
rubygems_version: 3.4.12
|
|
119
109
|
signing_key:
|
|
120
110
|
specification_version: 4
|
|
121
111
|
summary: Csvbuilder Importer contain the components to handle CSV importing
|