csv_step_importer 0.9.2 → 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/CHANGELOG.md +29 -0
- data/Gemfile.lock +4 -4
- data/README.md +87 -22
- data/lib/csv_step_importer/chunk.rb +1 -1
- data/lib/csv_step_importer/file.rb +3 -3
- data/lib/csv_step_importer/loader.rb +12 -0
- data/lib/csv_step_importer/version.rb +1 -1
- data/lib/csv_step_importer.rb +1 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 930cc2407877056e544604dab4c86d885349db9206e217e6518ccddda4969a06
|
4
|
+
data.tar.gz: 2e2a217f8832430d9365018bfb0c9b99fa6f28787a25230748c74890d0efa5b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 596bd1bce8110e37d1ab496cb75a2469d2e0bc438ecf2389764b28f82e940003cf10a4e296c28ef6e4a36d30dbf835023d23793b70fdea59a64b1d2d4e774a0a
|
7
|
+
data.tar.gz: 1e296ae4f9c321f90f129eedaa4a26a55e8b5d472239d2d9061a639bec184a39162e5bf9277be66f8f754fb0e8a6e5e0d6853efe7ce555f43744b58436ae6135
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.5.1
|
data/CHANGELOG.md
CHANGED
@@ -57,3 +57,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
57
57
|
" A z " becomes :"A z"
|
58
58
|
"A\"z" becomes :"A\"z"
|
59
59
|
```
|
60
|
+
|
61
|
+
## 2018-09-11 Version 0.10.0
|
62
|
+
### Added
|
63
|
+
- Added a simple wrapper class for File and Chunk called Loader to use the same interface for importing a plain array of hashes as well as CSV files.
|
64
|
+
|
65
|
+
Usage:
|
66
|
+
|
67
|
+
```ruby
|
68
|
+
CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel]).save!
|
69
|
+
```
|
70
|
+
|
71
|
+
See lib/csv_step_importer/chunk.rb for more options
|
72
|
+
|
73
|
+
or
|
74
|
+
|
75
|
+
```ruby
|
76
|
+
CSVStepImporter::Loader.new(
|
77
|
+
path: 'authors.csv',
|
78
|
+
processor_classes: [Author::ImportableModel],
|
79
|
+
csv_options: {file_encoding: "UTF-8"}
|
80
|
+
).save!
|
81
|
+
```
|
82
|
+
|
83
|
+
See lib/csv_step_importer/file.rb for more options
|
84
|
+
|
85
|
+
### Changed
|
86
|
+
|
87
|
+
- Changed README, especially a note to include smarter_csv 2.0.0.pre1 into your project
|
88
|
+
- Chunks default for the first row index is now 0
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
csv_step_importer (0.
|
4
|
+
csv_step_importer (0.10.0)
|
5
5
|
activemodel
|
6
6
|
activerecord-import
|
7
7
|
activesupport
|
@@ -87,7 +87,7 @@ GEM
|
|
87
87
|
diff-lcs (>= 1.2.0, < 2.0)
|
88
88
|
rspec-support (~> 3.8.0)
|
89
89
|
rspec-support (3.8.0)
|
90
|
-
rubocop (0.
|
90
|
+
rubocop (0.59.0)
|
91
91
|
jaro_winkler (~> 1.5.1)
|
92
92
|
parallel (~> 1.10)
|
93
93
|
parser (>= 2.5, != 2.5.1.1)
|
@@ -101,7 +101,7 @@ GEM
|
|
101
101
|
rubocop-rails_config (0.2.3)
|
102
102
|
railties (>= 3.0)
|
103
103
|
rubocop (~> 0.56)
|
104
|
-
rubocop-rspec (1.
|
104
|
+
rubocop-rspec (1.29.1)
|
105
105
|
rubocop (>= 0.58.0)
|
106
106
|
ruby-progressbar (1.10.0)
|
107
107
|
smarter_csv (1.2.4)
|
@@ -124,4 +124,4 @@ DEPENDENCIES
|
|
124
124
|
rubocop-rspec
|
125
125
|
|
126
126
|
BUNDLED WITH
|
127
|
-
1.16.
|
127
|
+
1.16.4
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# csv_step_importer
|
2
2
|
|
3
|
-
A library to validate, speed up and organize bulk insertion of complex CSV data
|
3
|
+
A library to validate, speed up and organize bulk insertion of complex CSV data into multiple tables.
|
4
4
|
|
5
5
|
It depends on
|
6
6
|
|
@@ -12,7 +12,9 @@ It depends on
|
|
12
12
|
Add this line to your application's Gemfile:
|
13
13
|
|
14
14
|
```ruby
|
15
|
+
# Quicker CSV processing
|
15
16
|
gem 'csv_step_importer'
|
17
|
+
gem 'smarter_csv', github: 'tilo/smarter_csv'
|
16
18
|
```
|
17
19
|
|
18
20
|
NOTE: you might need to add `gem 'smarter_csv', github: 'tilo/smarter_csv'` if you encounter problems building rows
|
@@ -27,62 +29,125 @@ Or install it yourself as:
|
|
27
29
|
|
28
30
|
## Usage
|
29
31
|
|
30
|
-
### Hello world
|
32
|
+
### Hello world setup (super simple sample, single table, no user-defined row, no user-defined dao)
|
31
33
|
|
32
|
-
|
34
|
+
First let's create a basic rails application to play around with
|
33
35
|
|
34
36
|
```shell
|
35
|
-
rails new
|
36
|
-
cd
|
37
|
-
echo "gem 'csv_step_importer'
|
37
|
+
rails new bookshop --database=mysql
|
38
|
+
cd bookshop
|
39
|
+
echo "gem 'csv_step_importer'
|
40
|
+
gem 'smarter_csv', github: 'tilo/smarter_csv'" >> Gemfile
|
38
41
|
bundle install
|
39
|
-
rails g model
|
42
|
+
rails g model author name:string:uniq email:string
|
43
|
+
rails g model book author:references name:string:uniq
|
40
44
|
rails db:create db:migrate
|
41
45
|
```
|
42
46
|
|
43
47
|
Then edit the model like this:
|
44
48
|
|
45
|
-
|
49
|
+
app/models/author.rb
|
46
50
|
|
47
51
|
```ruby
|
48
|
-
class
|
52
|
+
class Author < ApplicationRecord
|
49
53
|
class ImportableModel < CSVStepImporter::Model::ImportableModel
|
50
54
|
# The model to be updated
|
51
55
|
def model_class
|
52
|
-
|
56
|
+
puts Module.nesting.inspect
|
57
|
+
Module.nesting[1]
|
53
58
|
end
|
54
59
|
|
55
60
|
def columns
|
56
|
-
[:
|
61
|
+
[:name, :email, :created_at, :updated_at]
|
57
62
|
end
|
58
63
|
|
59
64
|
def on_duplicate_key_update
|
60
|
-
[:
|
65
|
+
[:email, :updated_at]
|
61
66
|
end
|
62
67
|
end
|
63
68
|
end
|
64
69
|
```
|
65
70
|
|
71
|
+
### Simple upload of a single row
|
66
72
|
|
67
|
-
|
73
|
+
```shell
|
74
|
+
rails c
|
75
|
+
```
|
76
|
+
|
77
|
+
```ruby
|
78
|
+
irb(main)> data = [{ name: 'Milan Kundera', email: 'milan.kundera@example.com' }]
|
79
|
+
irb(main)> importer = CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel])
|
80
|
+
irb(main)> importer.valid?
|
81
|
+
=> true
|
82
|
+
irb(main)> importer.save!
|
83
|
+
(1.1ms) SET NAMES utf8, @@SESSION.sql_mode = CONCAT(CONCAT(@@sql_mode, ',STRICT_ALL_TABLES'), ',NO_AUTO_VALUE_ON_ZERO'), @@SESSION.sql_auto_is_null = 0, @@SESSION.wait_timeout = 2147483
|
84
|
+
(0.4ms) BEGIN
|
85
|
+
[Author::ImportableModel, Author(id: integer, name: string, email: string, created_at: datetime, updated_at: datetime)]
|
86
|
+
(13.3ms) SHOW VARIABLES like 'max_allowed_packet';
|
87
|
+
Author Create Many Without Validations Or Callbacks (9.2ms) INSERT INTO `authors` (`name`,`email`,`created_at`,`updated_at`) VALUES ('Milan Kundera','milan.kundera@example.com','2018-09-11 11:33:07','2018-09-11 11:33:07') ON DUPLICATE KEY UPDATE `authors`.`email`=VALUES(`email`),`authors`.`updated_at`=VALUES(`updated_at`)
|
88
|
+
(154.6ms) COMMIT
|
89
|
+
=> true
|
90
|
+
|
91
|
+
irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way to get clean yaml...
|
92
|
+
|
93
|
+
---
|
94
|
+
- id: 7
|
95
|
+
name: Milan Kundera
|
96
|
+
email: milan.kundera@example.com
|
97
|
+
created_at: '2018-09-11T11:42:15.000Z'
|
98
|
+
updated_at: '2018-09-11T11:42:15.000Z'
|
99
|
+
|
100
|
+
irb(main)> data = [{ name: 'Milan Kundera', email: 'milan.kundera2@example.com' }, { name: 'Immanuel Kant', email: 'immanuel.kant@example.com' }]
|
101
|
+
irb(main)> CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel]).save!
|
102
|
+
=> true
|
103
|
+
irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way to get clean yaml...
|
104
|
+
|
105
|
+
# NOTE: updated_at changed, but the id did not
|
106
|
+
---
|
107
|
+
- id: 7
|
108
|
+
name: Milan Kundera
|
109
|
+
email: milan.kundera2@example.com
|
110
|
+
created_at: '2018-09-11T11:42:15.000Z'
|
111
|
+
updated_at: '2018-09-11T12:19:17.000Z'
|
112
|
+
- id: 9
|
113
|
+
name: Immanuel Kant
|
114
|
+
email: immanuel.kant@example.com
|
115
|
+
created_at: '2018-09-11T12:19:17.000Z'
|
116
|
+
updated_at: '2018-09-11T12:19:17.000Z'
|
117
|
+
```
|
118
|
+
|
119
|
+
### File import using [tilo/smarter_csv](https://github.com/tilo/smarter_csv)
|
68
120
|
|
69
121
|
```shell
|
70
122
|
rails c
|
71
123
|
```
|
72
124
|
|
73
125
|
```ruby
|
74
|
-
File.open("
|
126
|
+
irb(main)> File.open("authors.csv", "w") do |file|
|
75
127
|
file.write(<<~CSV)
|
76
|
-
Name,
|
77
|
-
|
78
|
-
|
79
|
-
Japanese Yen,JPY
|
128
|
+
Name,Email
|
129
|
+
Milan Kundera,milan.kundera@example.com
|
130
|
+
Immanuel Kant,immanuel.kant@example.com
|
80
131
|
CSV
|
81
132
|
end
|
82
133
|
|
83
|
-
CSVStepImporter::
|
84
|
-
|
85
|
-
|
134
|
+
irb(main)> CSVStepImporter::Loader.new(path: 'authors.csv', processor_classes: [Author::ImportableModel], csv_options: {file_encoding: "UTF-8"}).save
|
135
|
+
=> true
|
136
|
+
|
137
|
+
irb(main)> puts JSON.parse(Author.all.to_json).to_yaml
|
138
|
+
|
139
|
+
# NOTE: The email and updated_at is updated as specified in on_duplicate_key_update
|
140
|
+
---
|
141
|
+
- id: 7
|
142
|
+
name: Milan Kundera
|
143
|
+
email: milan.kundera@example.com
|
144
|
+
created_at: '2018-09-11T11:42:15.000Z'
|
145
|
+
updated_at: '2018-09-11T12:24:13.000Z'
|
146
|
+
- id: 9
|
147
|
+
name: Immanuel Kant
|
148
|
+
email: immanuel.kant@example.com
|
149
|
+
created_at: '2018-09-11T12:19:17.000Z'
|
150
|
+
updated_at: '2018-09-11T12:24:13.000Z'
|
86
151
|
```
|
87
152
|
|
88
153
|
### Simple model
|
@@ -110,7 +175,7 @@ class SimpleModel < CSVStepImporter::Model::Model
|
|
110
175
|
end
|
111
176
|
end
|
112
177
|
|
113
|
-
CSVStepImporter::
|
178
|
+
CSVStepImporter::Loader.new(path: 'currencies.csv', processor_classes: [SimpleModel]).save
|
114
179
|
```
|
115
180
|
|
116
181
|
## Development
|
@@ -4,7 +4,7 @@ module CSVStepImporter
|
|
4
4
|
class Chunk < CSVStepImporter::Node
|
5
5
|
attr_accessor :cache, :rows, :first_row
|
6
6
|
|
7
|
-
def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row:
|
7
|
+
def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 0, **attributes)
|
8
8
|
super **attributes
|
9
9
|
|
10
10
|
self.cache = {}
|
@@ -14,12 +14,12 @@ module CSVStepImporter
|
|
14
14
|
validates :row_class, presence: true
|
15
15
|
validate :validate_csv_load_error
|
16
16
|
|
17
|
-
def initialize(path:, chunk_class:
|
17
|
+
def initialize(path:, chunk_class: CSVStepImporter::Chunk, row_class: CSVStepImporter::Row, csv_options: {}, processor_classes: nil, **attributes)
|
18
18
|
super **attributes
|
19
19
|
|
20
|
-
self.chunk_class = chunk_class
|
20
|
+
self.chunk_class = chunk_class
|
21
21
|
self.path = path
|
22
|
-
self.row_class = row_class
|
22
|
+
self.row_class = row_class
|
23
23
|
self.processor_classes = processor_classes
|
24
24
|
|
25
25
|
self.csv_options = csv_options
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "smarter_csv"
|
4
|
+
|
5
|
+
module CSVStepImporter
|
6
|
+
class Loader < CSVStepImporter::Node
|
7
|
+
def initialize(file_class: CSVStepImporter::File, chunk_class: CSVStepImporter::Chunk, **attributes)
|
8
|
+
super **attributes.slice(:parent, :children, :env)
|
9
|
+
add_children attributes[:path] ? file_class.new( **attributes.merge!(chunk_class: chunk_class) ) : chunk_class.new( **attributes )
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
data/lib/csv_step_importer.rb
CHANGED
@@ -7,6 +7,7 @@ require "csv_step_importer/node"
|
|
7
7
|
require "csv_step_importer/row"
|
8
8
|
require "csv_step_importer/file"
|
9
9
|
require "csv_step_importer/chunk"
|
10
|
+
require "csv_step_importer/loader"
|
10
11
|
require "csv_step_importer/model/dao"
|
11
12
|
require "csv_step_importer/model/model"
|
12
13
|
require "csv_step_importer/model/reflector"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csv_step_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian-Manuel Butzke
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -160,6 +160,7 @@ files:
|
|
160
160
|
- ".gitignore"
|
161
161
|
- ".rspec"
|
162
162
|
- ".rubocop.yml"
|
163
|
+
- ".ruby-version"
|
163
164
|
- ".travis.yml"
|
164
165
|
- CHANGELOG.md
|
165
166
|
- CODE_OF_CONDUCT.md
|
@@ -175,6 +176,7 @@ files:
|
|
175
176
|
- lib/csv_step_importer/base.rb
|
176
177
|
- lib/csv_step_importer/chunk.rb
|
177
178
|
- lib/csv_step_importer/file.rb
|
179
|
+
- lib/csv_step_importer/loader.rb
|
178
180
|
- lib/csv_step_importer/model/dao.rb
|
179
181
|
- lib/csv_step_importer/model/importable_model.rb
|
180
182
|
- lib/csv_step_importer/model/importer.rb
|
@@ -203,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
203
205
|
version: '0'
|
204
206
|
requirements: []
|
205
207
|
rubyforge_project:
|
206
|
-
rubygems_version: 2.7.
|
208
|
+
rubygems_version: 2.7.6
|
207
209
|
signing_key:
|
208
210
|
specification_version: 4
|
209
211
|
summary: Import your CSV files in multiple steps
|