csv_step_importer 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/CHANGELOG.md +29 -0
- data/Gemfile.lock +4 -4
- data/README.md +87 -22
- data/lib/csv_step_importer/chunk.rb +1 -1
- data/lib/csv_step_importer/file.rb +3 -3
- data/lib/csv_step_importer/loader.rb +12 -0
- data/lib/csv_step_importer/version.rb +1 -1
- data/lib/csv_step_importer.rb +1 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 930cc2407877056e544604dab4c86d885349db9206e217e6518ccddda4969a06
|
4
|
+
data.tar.gz: 2e2a217f8832430d9365018bfb0c9b99fa6f28787a25230748c74890d0efa5b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 596bd1bce8110e37d1ab496cb75a2469d2e0bc438ecf2389764b28f82e940003cf10a4e296c28ef6e4a36d30dbf835023d23793b70fdea59a64b1d2d4e774a0a
|
7
|
+
data.tar.gz: 1e296ae4f9c321f90f129eedaa4a26a55e8b5d472239d2d9061a639bec184a39162e5bf9277be66f8f754fb0e8a6e5e0d6853efe7ce555f43744b58436ae6135
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.5.1
|
data/CHANGELOG.md
CHANGED
@@ -57,3 +57,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
57
57
|
" A z " becomes :"A z"
|
58
58
|
"A\"z" becomes :"A\"z"
|
59
59
|
```
|
60
|
+
|
61
|
+
## 2018-09-11 Version 0.10.0
|
62
|
+
### Added
|
63
|
+
- Added a simple wrapper class for File and Chunk called Loader to use the same interface for importing a plain array of hashes as well as CSV files.
|
64
|
+
|
65
|
+
Usage:
|
66
|
+
|
67
|
+
```ruby
|
68
|
+
CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel]).save!
|
69
|
+
```
|
70
|
+
|
71
|
+
See lib/csv_step_importer/chunk.rb for more options
|
72
|
+
|
73
|
+
or
|
74
|
+
|
75
|
+
```ruby
|
76
|
+
CSVStepImporter::Loader.new(
|
77
|
+
path: 'authors.csv',
|
78
|
+
processor_classes: [Author::ImportableModel],
|
79
|
+
csv_options: {file_encoding: "UTF-8"}
|
80
|
+
).save!
|
81
|
+
```
|
82
|
+
|
83
|
+
See lib/csv_step_importer/file.rb for more options
|
84
|
+
|
85
|
+
### Changed
|
86
|
+
|
87
|
+
- Changed README, especially a note to include smarter_csv 2.0.0.pre1 into your project
|
88
|
+
- Chunks default for the first row index is now 0
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
csv_step_importer (0.
|
4
|
+
csv_step_importer (0.10.0)
|
5
5
|
activemodel
|
6
6
|
activerecord-import
|
7
7
|
activesupport
|
@@ -87,7 +87,7 @@ GEM
|
|
87
87
|
diff-lcs (>= 1.2.0, < 2.0)
|
88
88
|
rspec-support (~> 3.8.0)
|
89
89
|
rspec-support (3.8.0)
|
90
|
-
rubocop (0.
|
90
|
+
rubocop (0.59.0)
|
91
91
|
jaro_winkler (~> 1.5.1)
|
92
92
|
parallel (~> 1.10)
|
93
93
|
parser (>= 2.5, != 2.5.1.1)
|
@@ -101,7 +101,7 @@ GEM
|
|
101
101
|
rubocop-rails_config (0.2.3)
|
102
102
|
railties (>= 3.0)
|
103
103
|
rubocop (~> 0.56)
|
104
|
-
rubocop-rspec (1.
|
104
|
+
rubocop-rspec (1.29.1)
|
105
105
|
rubocop (>= 0.58.0)
|
106
106
|
ruby-progressbar (1.10.0)
|
107
107
|
smarter_csv (1.2.4)
|
@@ -124,4 +124,4 @@ DEPENDENCIES
|
|
124
124
|
rubocop-rspec
|
125
125
|
|
126
126
|
BUNDLED WITH
|
127
|
-
1.16.
|
127
|
+
1.16.4
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# csv_step_importer
|
2
2
|
|
3
|
-
A library to validate, speed up and organize bulk insertion of complex CSV data
|
3
|
+
A library to validate, speed up and organize bulk insertion of complex CSV data into multiple tables.
|
4
4
|
|
5
5
|
It depends on
|
6
6
|
|
@@ -12,7 +12,9 @@ It depends on
|
|
12
12
|
Add this line to your application's Gemfile:
|
13
13
|
|
14
14
|
```ruby
|
15
|
+
# Quicker CSV processing
|
15
16
|
gem 'csv_step_importer'
|
17
|
+
gem 'smarter_csv', github: 'tilo/smarter_csv'
|
16
18
|
```
|
17
19
|
|
18
20
|
NOTE: you might need to add `gem 'smarter_csv', github: 'tilo/smarter_csv'` if you encounter problems building rows
|
@@ -27,62 +29,125 @@ Or install it yourself as:
|
|
27
29
|
|
28
30
|
## Usage
|
29
31
|
|
30
|
-
### Hello world
|
32
|
+
### Hello world setup (super simple sample, single table, no user-defined row, no user-defined dao)
|
31
33
|
|
32
|
-
|
34
|
+
First let's create a basic rails application to play around with
|
33
35
|
|
34
36
|
```shell
|
35
|
-
rails new
|
36
|
-
cd
|
37
|
-
echo "gem 'csv_step_importer'
|
37
|
+
rails new bookshop --database=mysql
|
38
|
+
cd bookshop
|
39
|
+
echo "gem 'csv_step_importer'
|
40
|
+
gem 'smarter_csv', github: 'tilo/smarter_csv'" >> Gemfile
|
38
41
|
bundle install
|
39
|
-
rails g model
|
42
|
+
rails g model author name:string:uniq email:string
|
43
|
+
rails g model book author:references name:string:uniq
|
40
44
|
rails db:create db:migrate
|
41
45
|
```
|
42
46
|
|
43
47
|
Then edit the model like this:
|
44
48
|
|
45
|
-
|
49
|
+
app/models/author.rb
|
46
50
|
|
47
51
|
```ruby
|
48
|
-
class
|
52
|
+
class Author < ApplicationRecord
|
49
53
|
class ImportableModel < CSVStepImporter::Model::ImportableModel
|
50
54
|
# The model to be updated
|
51
55
|
def model_class
|
52
|
-
|
56
|
+
puts Module.nesting.inspect
|
57
|
+
Module.nesting[1]
|
53
58
|
end
|
54
59
|
|
55
60
|
def columns
|
56
|
-
[:
|
61
|
+
[:name, :email, :created_at, :updated_at]
|
57
62
|
end
|
58
63
|
|
59
64
|
def on_duplicate_key_update
|
60
|
-
[:
|
65
|
+
[:email, :updated_at]
|
61
66
|
end
|
62
67
|
end
|
63
68
|
end
|
64
69
|
```
|
65
70
|
|
71
|
+
### Simple upload of a single row
|
66
72
|
|
67
|
-
|
73
|
+
```shell
|
74
|
+
rails c
|
75
|
+
```
|
76
|
+
|
77
|
+
```ruby
|
78
|
+
irb(main)> data = [{ name: 'Milan Kundera', email: 'milan.kundera@example.com' }]
|
79
|
+
irb(main)> importer = CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel])
|
80
|
+
irb(main)> importer.valid?
|
81
|
+
=> true
|
82
|
+
irb(main)> importer.save!
|
83
|
+
(1.1ms) SET NAMES utf8, @@SESSION.sql_mode = CONCAT(CONCAT(@@sql_mode, ',STRICT_ALL_TABLES'), ',NO_AUTO_VALUE_ON_ZERO'), @@SESSION.sql_auto_is_null = 0, @@SESSION.wait_timeout = 2147483
|
84
|
+
(0.4ms) BEGIN
|
85
|
+
[Author::ImportableModel, Author(id: integer, name: string, email: string, created_at: datetime, updated_at: datetime)]
|
86
|
+
(13.3ms) SHOW VARIABLES like 'max_allowed_packet';
|
87
|
+
Author Create Many Without Validations Or Callbacks (9.2ms) INSERT INTO `authors` (`name`,`email`,`created_at`,`updated_at`) VALUES ('Milan Kundera','milan.kundera@example.com','2018-09-11 11:33:07','2018-09-11 11:33:07') ON DUPLICATE KEY UPDATE `authors`.`email`=VALUES(`email`),`authors`.`updated_at`=VALUES(`updated_at`)
|
88
|
+
(154.6ms) COMMIT
|
89
|
+
=> true
|
90
|
+
|
91
|
+
irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way to get clean yaml...
|
92
|
+
|
93
|
+
---
|
94
|
+
- id: 7
|
95
|
+
name: Milan Kundera
|
96
|
+
email: milan.kundera@example.com
|
97
|
+
created_at: '2018-09-11T11:42:15.000Z'
|
98
|
+
updated_at: '2018-09-11T11:42:15.000Z'
|
99
|
+
|
100
|
+
irb(main)> data = [{ name: 'Milan Kundera', email: 'milan.kundera2@example.com' }, { name: 'Immanuel Kant', email: 'immanuel.kant@example.com' }]
|
101
|
+
irb(main)> CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel]).save!
|
102
|
+
=> true
|
103
|
+
irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way to get clean yaml...
|
104
|
+
|
105
|
+
# NOTE: updated_at changed, but the id did not
|
106
|
+
---
|
107
|
+
- id: 7
|
108
|
+
name: Milan Kundera
|
109
|
+
email: milan.kundera2@example.com
|
110
|
+
created_at: '2018-09-11T11:42:15.000Z'
|
111
|
+
updated_at: '2018-09-11T12:19:17.000Z'
|
112
|
+
- id: 9
|
113
|
+
name: Immanuel Kant
|
114
|
+
email: immanuel.kant@example.com
|
115
|
+
created_at: '2018-09-11T12:19:17.000Z'
|
116
|
+
updated_at: '2018-09-11T12:19:17.000Z'
|
117
|
+
```
|
118
|
+
|
119
|
+
### File import using [tilo/smarter_csv](https://github.com/tilo/smarter_csv)
|
68
120
|
|
69
121
|
```shell
|
70
122
|
rails c
|
71
123
|
```
|
72
124
|
|
73
125
|
```ruby
|
74
|
-
File.open("
|
126
|
+
irb(main)> File.open("authors.csv", "w") do |file|
|
75
127
|
file.write(<<~CSV)
|
76
|
-
Name,
|
77
|
-
|
78
|
-
|
79
|
-
Japanese Yen,JPY
|
128
|
+
Name,Email
|
129
|
+
Milan Kundera,milan.kundera@example.com
|
130
|
+
Immanuel Kant,immanuel.kant@example.com
|
80
131
|
CSV
|
81
132
|
end
|
82
133
|
|
83
|
-
CSVStepImporter::
|
84
|
-
|
85
|
-
|
134
|
+
irb(main)> CSVStepImporter::Loader.new(path: 'authors.csv', processor_classes: [Author::ImportableModel], csv_options: {file_encoding: "UTF-8"}).save
|
135
|
+
=> true
|
136
|
+
|
137
|
+
irb(main)> puts JSON.parse(Author.all.to_json).to_yaml
|
138
|
+
|
139
|
+
# NOTE: The email and updated_at is updated as specified in on_duplicate_key_update
|
140
|
+
---
|
141
|
+
- id: 7
|
142
|
+
name: Milan Kundera
|
143
|
+
email: milan.kundera@example.com
|
144
|
+
created_at: '2018-09-11T11:42:15.000Z'
|
145
|
+
updated_at: '2018-09-11T12:24:13.000Z'
|
146
|
+
- id: 9
|
147
|
+
name: Immanuel Kant
|
148
|
+
email: immanuel.kant@example.com
|
149
|
+
created_at: '2018-09-11T12:19:17.000Z'
|
150
|
+
updated_at: '2018-09-11T12:24:13.000Z'
|
86
151
|
```
|
87
152
|
|
88
153
|
### Simple model
|
@@ -110,7 +175,7 @@ class SimpleModel < CSVStepImporter::Model::Model
|
|
110
175
|
end
|
111
176
|
end
|
112
177
|
|
113
|
-
CSVStepImporter::
|
178
|
+
CSVStepImporter::Loader.new(path: 'currencies.csv', processor_classes: [SimpleModel]).save
|
114
179
|
```
|
115
180
|
|
116
181
|
## Development
|
@@ -4,7 +4,7 @@ module CSVStepImporter
|
|
4
4
|
class Chunk < CSVStepImporter::Node
|
5
5
|
attr_accessor :cache, :rows, :first_row
|
6
6
|
|
7
|
-
def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row:
|
7
|
+
def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 0, **attributes)
|
8
8
|
super **attributes
|
9
9
|
|
10
10
|
self.cache = {}
|
@@ -14,12 +14,12 @@ module CSVStepImporter
|
|
14
14
|
validates :row_class, presence: true
|
15
15
|
validate :validate_csv_load_error
|
16
16
|
|
17
|
-
def initialize(path:, chunk_class:
|
17
|
+
def initialize(path:, chunk_class: CSVStepImporter::Chunk, row_class: CSVStepImporter::Row, csv_options: {}, processor_classes: nil, **attributes)
|
18
18
|
super **attributes
|
19
19
|
|
20
|
-
self.chunk_class = chunk_class
|
20
|
+
self.chunk_class = chunk_class
|
21
21
|
self.path = path
|
22
|
-
self.row_class = row_class
|
22
|
+
self.row_class = row_class
|
23
23
|
self.processor_classes = processor_classes
|
24
24
|
|
25
25
|
self.csv_options = csv_options
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "smarter_csv"
|
4
|
+
|
5
|
+
module CSVStepImporter
|
6
|
+
class Loader < CSVStepImporter::Node
|
7
|
+
def initialize(file_class: CSVStepImporter::File, chunk_class: CSVStepImporter::Chunk, **attributes)
|
8
|
+
super **attributes.slice(:parent, :children, :env)
|
9
|
+
add_children attributes[:path] ? file_class.new( **attributes.merge!(chunk_class: chunk_class) ) : chunk_class.new( **attributes )
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
data/lib/csv_step_importer.rb
CHANGED
@@ -7,6 +7,7 @@ require "csv_step_importer/node"
|
|
7
7
|
require "csv_step_importer/row"
|
8
8
|
require "csv_step_importer/file"
|
9
9
|
require "csv_step_importer/chunk"
|
10
|
+
require "csv_step_importer/loader"
|
10
11
|
require "csv_step_importer/model/dao"
|
11
12
|
require "csv_step_importer/model/model"
|
12
13
|
require "csv_step_importer/model/reflector"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csv_step_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian-Manuel Butzke
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -160,6 +160,7 @@ files:
|
|
160
160
|
- ".gitignore"
|
161
161
|
- ".rspec"
|
162
162
|
- ".rubocop.yml"
|
163
|
+
- ".ruby-version"
|
163
164
|
- ".travis.yml"
|
164
165
|
- CHANGELOG.md
|
165
166
|
- CODE_OF_CONDUCT.md
|
@@ -175,6 +176,7 @@ files:
|
|
175
176
|
- lib/csv_step_importer/base.rb
|
176
177
|
- lib/csv_step_importer/chunk.rb
|
177
178
|
- lib/csv_step_importer/file.rb
|
179
|
+
- lib/csv_step_importer/loader.rb
|
178
180
|
- lib/csv_step_importer/model/dao.rb
|
179
181
|
- lib/csv_step_importer/model/importable_model.rb
|
180
182
|
- lib/csv_step_importer/model/importer.rb
|
@@ -203,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
203
205
|
version: '0'
|
204
206
|
requirements: []
|
205
207
|
rubyforge_project:
|
206
|
-
rubygems_version: 2.7.
|
208
|
+
rubygems_version: 2.7.6
|
207
209
|
signing_key:
|
208
210
|
specification_version: 4
|
209
211
|
summary: Import your CSV files in multiple steps
|