mysql_import 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +21 -3
- data/README.md +153 -13
- data/lib/mysql_import/logger.rb +13 -15
- data/lib/mysql_import/version.rb +1 -1
- data/lib/mysql_import.rb +18 -21
- data/mysql_import.gemspec +1 -1
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa4a531d2bdd0606ab8ef9d54301a33a066cda2c
|
4
|
+
data.tar.gz: 8c10dbdb843d9be5edf3b0c49e7153318fa3cf51
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 741420989677a2c37e21bdbbfaf8184c272f10fc68f0ad4590d799a14b12dc8304db2d79a9266e0f81a9525b27b67db8706e40795cb6fa1effe4e951bd2d4023
|
7
|
+
data.tar.gz: 7fbb7a69ece5555bbbd64eb93931e339500001a855a0f2bdfd78585084dd01a220d8f4a0bceb886bdfaa7ac96288d174d323b9754aa1b16f0733a4b26c2221ad
|
data/.travis.yml
CHANGED
@@ -1,5 +1,23 @@
|
|
1
|
-
|
1
|
+
dist: trusty
|
2
|
+
sudo: required
|
2
3
|
language: ruby
|
3
4
|
rvm:
|
4
|
-
|
5
|
-
|
5
|
+
- 2.2.5
|
6
|
+
- 2.3.1
|
7
|
+
before_install: gem install bundler
|
8
|
+
cache:
|
9
|
+
- bundler
|
10
|
+
- apt
|
11
|
+
before_script:
|
12
|
+
- bin/db_setup
|
13
|
+
addons:
|
14
|
+
apt:
|
15
|
+
packages:
|
16
|
+
- mysql-server-5.6
|
17
|
+
- mysql-client-core-5.6
|
18
|
+
- mysql-client-5.6
|
19
|
+
code_climate:
|
20
|
+
repo_token: 7ccd2178b58732b43c4968b9472ee62fada44441a474183b0571a74d9c0db877
|
21
|
+
notifications:
|
22
|
+
slack:
|
23
|
+
secure: qLm5VTc7heQ+Ch73rKjrcX7VpP6T/RakKfzvK75c79z8ItpZNr8zCY+l3benZh8IvXde0+uXPIBHq3eaTzz8IEpeqK28TLlMHNCdpxGPaDkgdOtTqVYF9lQQ0MnJmgc8IUj0ESiIPCLxp258zY9zGg6LDmdf2OlwND5/Bvcqs2HJQWO8f9QcHzWonYGmsFTifHOSfz7LqA7/pPQ0yjQ3b2JUieMFy32x/h97/DackSNwn0nWawX5Z3850PFKK0ZBPi47J2GlF0xlZh20TEY0a8mTeS7NikNeBn5MKDtYqE2MDq9CllahaSLna86AP2+VL8QGvBX6seBIg3Y/bL6c+R6t1rT1iy4etd9IVFdjQw87NyMLef4QgdyVxVKtjHoj9KaRrz+7ghshHqlaNFmyo6IFwS0lQJ8aL8m5UGfcQ5yx3Kap1VbUZqRb5nQ1+uQJ3T0Sef5cLpUnvwhJ3ZophXhc8NT+juM5Ho/oaRQu57gKt3jAYeDO0kydbVcnChfEsValFxNTs8iB5hFbAZzkvCB2lUv+j+HhDt1E+g/sgpntQrGjs3T2Vb1XPvK+vjU+UQdNTJ+MHj5tkC6CmUZApES3+rTK5NXeFrJoMBK8yO2h3p7uYpKUrn5bQ/MnvluFowzpCDevP1jj54bVgzOGGUEQJg1LlxkfG9CM71r2T1k=
|
data/README.md
CHANGED
@@ -1,38 +1,178 @@
|
|
1
1
|
# MysqlImport
|
2
2
|
|
3
|
-
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/mysql_import.svg)](https://badge.fury.io/rb/mysql_import)
|
4
|
+
[![Build Status](https://travis-ci.org/nalabjp/mysql_import.svg?branch=master)](https://travis-ci.org/nalabjp/mysql_import)
|
5
|
+
[![Code Climate](https://codeclimate.com/github/nalabjp/mysql_import/badges/gpa.svg)](https://codeclimate.com/github/nalabjp/mysql_import)
|
6
|
+
[![Test Coverage](https://codeclimate.com/github/nalabjp/mysql_import/badges/coverage.svg)](https://codeclimate.com/github/nalabjp/mysql_import/coverage)
|
7
|
+
[![Dependency Status](https://gemnasium.com/badges/github.com/nalabjp/mysql_import.svg)](https://gemnasium.com/github.com/nalabjp/mysql_import)
|
4
8
|
|
5
|
-
|
9
|
+
Simple concurrent importer for MySQL using [load_data_infile2](https://github.com/nalabjp/load_data_infile2).
|
6
10
|
|
7
11
|
## Installation
|
8
12
|
|
9
|
-
Add
|
13
|
+
Add to your application's Gemfile:
|
10
14
|
|
11
15
|
```ruby
|
12
16
|
gem 'mysql_import'
|
13
17
|
```
|
14
18
|
|
15
|
-
And
|
19
|
+
And bundle.
|
16
20
|
|
17
|
-
|
21
|
+
## Examples
|
22
|
+
### Basic Usage
|
18
23
|
|
19
|
-
|
24
|
+
For exampole, if you want to import to `users` table from `/path/to/users.csv`:
|
25
|
+
```ruby
|
26
|
+
db_config = {
|
27
|
+
host: 'localhost'
|
28
|
+
database: 'mysql_import_test'
|
29
|
+
username: 'root'
|
30
|
+
}
|
31
|
+
importer = MysqlImport.new(db_config)
|
32
|
+
importer.add('/path/to/users.csv')
|
33
|
+
importer.import
|
34
|
+
# => Import to `users` tables
|
35
|
+
```
|
36
|
+
|
37
|
+
Multiple import:
|
38
|
+
```ruby
|
39
|
+
importer = MysqlImport.new(db_config)
|
40
|
+
importer.add('/path/to/users.csv')
|
41
|
+
importer.add('/path/to/groups.csv')
|
42
|
+
importer.add('/path/to/departments.csv')
|
43
|
+
importer.import
|
44
|
+
# => Import to three tables from three csv files
|
45
|
+
```
|
46
|
+
|
47
|
+
MysqlImport has the concurrency because it uses the [parallel](https://github.com/grosser/parallel) gem.
|
48
|
+
|
49
|
+
With import options:
|
50
|
+
|
51
|
+
```ruby
|
52
|
+
importer = MysqlImport.new(db_config)
|
53
|
+
importer.add('/path/to/users1.csv', table: 'users')
|
54
|
+
importer.add('/path/to/users2.csv', table: 'users')
|
55
|
+
importer.add('/path/to/users3.csv', table: 'users')
|
56
|
+
importer.import
|
57
|
+
# => Import to `users` table from three csv files
|
58
|
+
```
|
59
|
+
|
60
|
+
See more details for import options.
|
61
|
+
|
62
|
+
https://github.com/nalabjp/load_data_infile2#sql-options
|
63
|
+
|
64
|
+
### Filter
|
65
|
+
|
66
|
+
If you want to import only a specific file, you can specify the file.
|
67
|
+
|
68
|
+
The specification of the file will be used regular expression
|
69
|
+
|
70
|
+
```ruby
|
71
|
+
importer = MysqlImport.new(db_config)
|
72
|
+
importer.add('/path/to/users.csv')
|
73
|
+
importer.add('/path/to/groups.csv')
|
74
|
+
importer.add('/path/to/departments.csv')
|
75
|
+
importer.import('users')
|
76
|
+
# => Only import to `users` table
|
77
|
+
|
78
|
+
importer.import('users', 'groups')
|
79
|
+
# => Import to `users` and `groups` table
|
80
|
+
```
|
20
81
|
|
21
|
-
|
82
|
+
### Hook
|
22
83
|
|
23
|
-
|
84
|
+
You are able to set the hook immediately before and after import.
|
24
85
|
|
25
|
-
|
86
|
+
The hook will accept either String or Proc or Array.
|
26
87
|
|
27
|
-
|
88
|
+
#### String
|
28
89
|
|
29
|
-
|
90
|
+
String is evaluated directly as SQL.
|
30
91
|
|
31
|
-
|
92
|
+
```ruby
|
93
|
+
importer = MysqlImport.new(db_config)
|
94
|
+
importer.add(
|
95
|
+
'/path/to/users.csv',
|
96
|
+
{
|
97
|
+
before: 'TRUNCATE TABLE users;'
|
98
|
+
}
|
99
|
+
)
|
100
|
+
importer.import
|
101
|
+
# => Truncate query is always executed before import.
|
102
|
+
```
|
103
|
+
|
104
|
+
#### Proc
|
105
|
+
|
106
|
+
If you want to make the subsequent processing based on the execution result of SQL, you should use Proc.
|
107
|
+
|
108
|
+
Arguments that are passed to Proc is an instance of `LoadDataInfile2::Client`, which is a subclass of `Mysql2::Client`.
|
109
|
+
|
110
|
+
```ruby
|
111
|
+
importer = MysqlImport.new(db_config)
|
112
|
+
importer.add(
|
113
|
+
'/path/to/users.csv',
|
114
|
+
{
|
115
|
+
before: ->(cli) {
|
116
|
+
res = cli.query('SELECT COUNT(*) AS c FROM users;')
|
117
|
+
cli.query('TRUNCATE TABLE users;') if res.first['c'] > 0
|
118
|
+
}
|
119
|
+
}
|
120
|
+
)
|
121
|
+
importer.import
|
122
|
+
# => If there is one or more records in `users` table, truncate query is executed.
|
123
|
+
```
|
124
|
+
|
125
|
+
#### Array
|
126
|
+
|
127
|
+
Array of elements you need to use String or Proc.
|
128
|
+
|
129
|
+
```ruby
|
130
|
+
importer = MysqlImport.new(db_config)
|
131
|
+
importer.add(
|
132
|
+
'/path/to/users.csv',
|
133
|
+
{
|
134
|
+
before: [
|
135
|
+
"SET sql_mode = 'STRICT_TRANS_TABLES';",
|
136
|
+
->(cli) {
|
137
|
+
res = cli.query('SELECT COUNT(*) AS c FROM users;')
|
138
|
+
cli.query('TRUNCATE TABLE users;') if res.first['c'] > 0
|
139
|
+
}
|
140
|
+
],
|
141
|
+
after: [
|
142
|
+
'SET @i = 0;',
|
143
|
+
'UPDATE users SET order = (@i := @i + 1) ORDER BY name, email ASC;',
|
144
|
+
]
|
145
|
+
}
|
146
|
+
)
|
147
|
+
importer.import
|
148
|
+
```
|
149
|
+
|
150
|
+
#### Skip all subsequent processing
|
151
|
+
|
152
|
+
If you want to skip all subsequent processing, you will need to raise `MysqlImport::Break` in Proc.
|
153
|
+
|
154
|
+
```ruby
|
155
|
+
importer = MysqlImport.new(db_config)
|
156
|
+
importer.add(
|
157
|
+
'/path/to/users.csv',
|
158
|
+
{
|
159
|
+
before: ->(cli) {
|
160
|
+
res = cli.query('SELECT COUNT(*) AS c FROM users;')
|
161
|
+
raise MysqlImport::Break if res.first['c'] > 0
|
162
|
+
},
|
163
|
+
after: [
|
164
|
+
'SET @i = 0;',
|
165
|
+
'UPDATE users SET order = (@i := @i + 1) ORDER BY name, email ASC;',
|
166
|
+
]
|
167
|
+
}
|
168
|
+
)
|
169
|
+
importer.import
|
170
|
+
# => If there is one or more records in `users` table, import and after hook will be skipped.
|
171
|
+
```
|
32
172
|
|
33
173
|
## Contributing
|
34
174
|
|
35
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
175
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/nalabjp/mysql_import.
|
36
176
|
|
37
177
|
|
38
178
|
## License
|
data/lib/mysql_import/logger.rb
CHANGED
@@ -10,7 +10,7 @@ class MysqlImport
|
|
10
10
|
obj = ::Logger.new(nil)
|
11
11
|
when STDOUT, STDERR
|
12
12
|
obj = ::Logger.new(out)
|
13
|
-
obj.formatter = ->(
|
13
|
+
obj.formatter = ->(_, _, _, message) { "#{String === message ? message : message.inspect}\n" }
|
14
14
|
else
|
15
15
|
obj = out
|
16
16
|
end
|
@@ -21,7 +21,7 @@ class MysqlImport
|
|
21
21
|
end
|
22
22
|
|
23
23
|
module Logging
|
24
|
-
def initialize(config, opts =
|
24
|
+
def initialize(config, opts = {})
|
25
25
|
@logger = Logger.new(opts[:log], opts.fetch(:debug, false))
|
26
26
|
embed_logger
|
27
27
|
super
|
@@ -30,36 +30,34 @@ class MysqlImport
|
|
30
30
|
def import(*filters)
|
31
31
|
super
|
32
32
|
ensure
|
33
|
-
logger.info('Imported tables:')
|
34
|
-
if result.imported.size > 0
|
35
|
-
result.imported.sort.each {|t| logger.info(" #{t[0]} (#{t[1]} sec)") }
|
33
|
+
@logger.info('Imported tables:')
|
34
|
+
if @result.imported.size > 0
|
35
|
+
@result.imported.sort.each {|t| @logger.info(" #{t[0]} (#{t[1]} sec)") }
|
36
36
|
else
|
37
|
-
logger.info(' nothing...')
|
37
|
+
@logger.info(' nothing...')
|
38
38
|
end
|
39
|
-
if result.skipped.size > 0
|
40
|
-
logger.info('Skipped tables:')
|
41
|
-
result.skipped.sort.each {|t| logger.info(" #{t}") }
|
39
|
+
if @result.skipped.size > 0
|
40
|
+
@logger.info('Skipped tables:')
|
41
|
+
@result.skipped.sort.each {|t| @logger.info(" #{t}") }
|
42
42
|
end
|
43
43
|
|
44
|
-
result.clear
|
44
|
+
@result.clear
|
45
45
|
end
|
46
46
|
|
47
47
|
private
|
48
48
|
|
49
|
-
attr_reader :logger
|
50
|
-
|
51
49
|
def parallel_opts
|
52
50
|
@parallel_opts ||= super.merge(
|
53
51
|
finish: proc do |item, index, _result|
|
54
|
-
logger.debug("parallel_item: #{item.inspect}")
|
55
|
-
logger.debug("parallel_index: #{index}")
|
52
|
+
@logger.debug("parallel_item: #{item.inspect}")
|
53
|
+
@logger.debug("parallel_index: #{index}")
|
56
54
|
end
|
57
55
|
)
|
58
56
|
end
|
59
57
|
|
60
58
|
def embed_logger
|
61
59
|
unless LoadDataInfile2::Client.instance_methods.include?(:build_sql_with_logging)
|
62
|
-
LoadDataInfile2::Client.class_exec(logger) do |logger|
|
60
|
+
LoadDataInfile2::Client.class_exec(@logger) do |logger|
|
63
61
|
define_method :build_sql_with_logging do |file, options = {}|
|
64
62
|
build_sql_without_logging(file, options).tap {|sql| logger.debug("sql: #{sql}") }
|
65
63
|
end
|
data/lib/mysql_import/version.rb
CHANGED
data/lib/mysql_import.rb
CHANGED
@@ -5,23 +5,23 @@ require 'connection_pool'
|
|
5
5
|
require 'parallel'
|
6
6
|
|
7
7
|
class MysqlImport
|
8
|
-
def initialize(config, opts =
|
8
|
+
def initialize(config, opts = {})
|
9
9
|
@stash = []
|
10
|
-
@fileters = []
|
11
10
|
@concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
|
12
|
-
pool = concurrency.zero? ? 1 : concurrency
|
11
|
+
pool = @concurrency.zero? ? 1 : @concurrency
|
12
|
+
sql_opts = opts.fetch(:sql_opts, {})
|
13
13
|
|
14
14
|
@client = ConnectionPool.new(size: pool) { LoadDataInfile2::Client.new(config, sql_opts) }
|
15
15
|
@result = Result.new
|
16
16
|
end
|
17
17
|
|
18
|
-
def add(file_path,
|
19
|
-
stash.push([file_path,
|
18
|
+
def add(file_path, opts = {})
|
19
|
+
@stash.push([file_path, opts])
|
20
20
|
end
|
21
21
|
|
22
22
|
def import(*filters)
|
23
23
|
Parallel.each(filtered_list(filters), parallel_opts) do |args|
|
24
|
-
client.with do |cli|
|
24
|
+
@client.with do |cli|
|
25
25
|
run_import(cli, *args)
|
26
26
|
end
|
27
27
|
end
|
@@ -29,45 +29,42 @@ class MysqlImport
|
|
29
29
|
|
30
30
|
private
|
31
31
|
|
32
|
-
attr_reader :stash, :filters, :concurrency, :client, :result
|
33
|
-
|
34
32
|
def filtered_list(filters)
|
35
|
-
return stash if filters.empty?
|
33
|
+
return @stash if filters.empty?
|
36
34
|
|
37
35
|
regexps = filters.map{|f| Regexp.new(f) }
|
38
|
-
stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
|
36
|
+
@stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
|
39
37
|
end
|
40
38
|
|
41
39
|
def parallel_opts
|
42
|
-
{ in_threads: concurrency }
|
40
|
+
{ in_threads: @concurrency }
|
43
41
|
end
|
44
42
|
|
45
43
|
def run_import(cli, fpath, opts)
|
46
44
|
t = Time.now
|
47
45
|
|
48
|
-
|
49
|
-
|
50
|
-
table = opts[:table] || File.basename(fpath, '.*')
|
46
|
+
sql_opts = Marshal.load(Marshal.dump(opts.reject {|k, _| %i(before after).include?(k) }))
|
47
|
+
table = sql_opts[:table] || File.basename(fpath, '.*')
|
51
48
|
|
52
|
-
if before
|
49
|
+
if opts[:before]
|
53
50
|
begin
|
54
|
-
run_action(before, cli)
|
51
|
+
run_action(opts[:before], cli)
|
55
52
|
rescue Break
|
56
|
-
result.add(:skipped, table)
|
53
|
+
@result.add(:skipped, table)
|
57
54
|
return
|
58
55
|
end
|
59
56
|
end
|
60
57
|
|
61
|
-
cli.import(fpath,
|
58
|
+
cli.import(fpath, sql_opts)
|
62
59
|
|
63
|
-
if after
|
60
|
+
if opts[:after]
|
64
61
|
begin
|
65
|
-
run_action(after, cli)
|
62
|
+
run_action(opts[:after], cli)
|
66
63
|
rescue Break
|
67
64
|
end
|
68
65
|
end
|
69
66
|
|
70
|
-
result.add(:imported, [table, (Time.now - t)])
|
67
|
+
@result.add(:imported, [table, (Time.now - t)])
|
71
68
|
end
|
72
69
|
|
73
70
|
def run_action(action, cli)
|
data/mysql_import.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
-
spec.add_dependency 'load_data_infile2'
|
21
|
+
spec.add_dependency 'load_data_infile2', '~> 0.2'
|
22
22
|
spec.add_dependency 'connection_pool'
|
23
23
|
spec.add_dependency 'parallel'
|
24
24
|
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mysql_import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- nalabjp
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-07-
|
11
|
+
date: 2016-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: load_data_infile2
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
19
|
+
version: '0.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
26
|
+
version: '0.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: connection_pool
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|