mysql_import 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -2
- data/lib/mysql_import/logger.rb +22 -22
- data/lib/mysql_import/version.rb +1 -1
- data/lib/mysql_import.rb +26 -25
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 609ba688e11734a17c9b421fd8cc6ff576a400bb
|
4
|
+
data.tar.gz: 6911371e8f1d12a06adfdb590911739f55cb2d62
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 198bd007c859a481726fe34302242e79cbca316278094e033e1ac709fa9831a9a7ec66b7a0a1b4ed2b036acdf0a622f47c317dc3a9ba741698d20723c01b24cd
|
7
|
+
data.tar.gz: d87ca57f0955dfe6d1028441a8f88d3720d211ed927237fcbea13eaa28aeee364415cb8416e7976a0dc471b3c520007ba551f62fec04dc18a014aa03775a5142
|
data/.travis.yml
CHANGED
data/lib/mysql_import/logger.rb
CHANGED
@@ -1,30 +1,26 @@
|
|
1
1
|
require 'logger'
|
2
2
|
|
3
3
|
class MysqlImport
|
4
|
-
class Logger <
|
4
|
+
class Logger < ::Logger
|
5
5
|
def initialize(out, debug)
|
6
|
+
super(out)
|
7
|
+
|
8
|
+
@level = INFO unless debug
|
6
9
|
case out
|
7
|
-
when String
|
8
|
-
obj = ::Logger.new(out)
|
9
|
-
obj.level = ::Logger::INFO
|
10
|
-
when NilClass
|
11
|
-
obj = ::Logger.new(nil)
|
12
10
|
when STDOUT, STDERR
|
13
|
-
|
14
|
-
obj.formatter = ->(_, _, _, message) { "#{String === message ? message : message.inspect}\n" }
|
15
|
-
obj.level = ::Logger::INFO
|
16
|
-
else
|
17
|
-
obj = out
|
11
|
+
@formatter = ->(_, _, _, message) { "#{String === message ? message : message.inspect}\n" }
|
18
12
|
end
|
19
|
-
|
20
|
-
obj.level = ::Logger::DEBUG if debug
|
21
|
-
super(obj)
|
22
13
|
end
|
23
14
|
end
|
24
15
|
|
25
16
|
module Logging
|
26
17
|
def initialize(config, opts = {})
|
27
|
-
@
|
18
|
+
@debug = opts.fetch(:debug, false)
|
19
|
+
@logger = if opts[:log].is_a?(::Logger)
|
20
|
+
opts[:log]
|
21
|
+
else
|
22
|
+
Logger.new(opts[:log], @debug)
|
23
|
+
end
|
28
24
|
embed_logger
|
29
25
|
super
|
30
26
|
end
|
@@ -49,16 +45,20 @@ class MysqlImport
|
|
49
45
|
private
|
50
46
|
|
51
47
|
def parallel_opts
|
52
|
-
@parallel_opts ||=
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
48
|
+
@parallel_opts ||= if @debug
|
49
|
+
super.merge(
|
50
|
+
finish: proc do |item, index, _result|
|
51
|
+
@logger.debug("parallel_item: #{item.inspect}")
|
52
|
+
@logger.debug("parallel_index: #{index}")
|
53
|
+
end
|
54
|
+
)
|
55
|
+
else
|
56
|
+
super
|
57
|
+
end
|
58
58
|
end
|
59
59
|
|
60
60
|
def embed_logger
|
61
|
-
|
61
|
+
if @debug && !LoadDataInfile2::Client.instance_methods.include?(:build_sql_with_logging)
|
62
62
|
LoadDataInfile2::Client.class_exec(@logger) do |logger|
|
63
63
|
define_method :build_sql_with_logging do |file, options = {}|
|
64
64
|
build_sql_without_logging(file, options).tap {|sql| logger.debug("sql: #{sql}") }
|
data/lib/mysql_import/version.rb
CHANGED
data/lib/mysql_import.rb
CHANGED
@@ -7,6 +7,7 @@ require 'parallel'
|
|
7
7
|
class MysqlImport
|
8
8
|
def initialize(config, opts = {})
|
9
9
|
@stash = []
|
10
|
+
@lock = opts.fetch(:lock, false)
|
10
11
|
@concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
|
11
12
|
pool = @concurrency.zero? ? 1 : @concurrency
|
12
13
|
sql_opts = opts.fetch(:sql_opts, {})
|
@@ -44,32 +45,32 @@ class MysqlImport
|
|
44
45
|
|
45
46
|
def run_import(cli, fpath, opts)
|
46
47
|
t = Time.now
|
48
|
+
imported = false
|
47
49
|
|
48
|
-
sql_opts =
|
50
|
+
sql_opts = opts.reject {|k, _| %i(before after).include?(k) }
|
49
51
|
table = sql_opts[:table] || File.basename(fpath, '.*')
|
52
|
+
lock = opts.fetch(:lock, @lock)
|
50
53
|
|
51
|
-
|
52
|
-
|
53
|
-
run_action(opts[:before], cli)
|
54
|
-
rescue Break
|
55
|
-
@result.add(:skipped, table)
|
56
|
-
return
|
57
|
-
end
|
58
|
-
end
|
54
|
+
begin
|
55
|
+
write_lock(cli, table) if lock
|
59
56
|
|
60
|
-
|
57
|
+
run_action(opts[:before], cli)
|
61
58
|
|
62
|
-
|
63
|
-
|
64
|
-
run_action(opts[:after], cli)
|
65
|
-
rescue Break
|
66
|
-
end
|
67
|
-
end
|
59
|
+
cli.import(fpath, sql_opts)
|
60
|
+
imported = true
|
68
61
|
|
69
|
-
|
62
|
+
run_action(opts[:after], cli)
|
63
|
+
rescue Break
|
64
|
+
@result.skipped.push(table) unless imported
|
65
|
+
ensure
|
66
|
+
unlock(cli) if lock
|
67
|
+
@result.imported.push([table, (Time.now - t)]) if imported
|
68
|
+
end
|
70
69
|
end
|
71
70
|
|
72
71
|
def run_action(action, cli)
|
72
|
+
return unless action
|
73
|
+
|
73
74
|
case action
|
74
75
|
when Array
|
75
76
|
action.each { |act| run_action(act, cli) }
|
@@ -80,6 +81,14 @@ class MysqlImport
|
|
80
81
|
end
|
81
82
|
end
|
82
83
|
|
84
|
+
def write_lock(cli, table)
|
85
|
+
cli.query("LOCK TABLE `#{table}` WRITE;")
|
86
|
+
end
|
87
|
+
|
88
|
+
def unlock(cli)
|
89
|
+
cli.query("UNLOCK TABLES;")
|
90
|
+
end
|
91
|
+
|
83
92
|
class Result
|
84
93
|
def imported
|
85
94
|
@imported ||= []
|
@@ -89,14 +98,6 @@ class MysqlImport
|
|
89
98
|
@skipped ||= []
|
90
99
|
end
|
91
100
|
|
92
|
-
def mutex
|
93
|
-
@mutext ||= Mutex.new
|
94
|
-
end
|
95
|
-
|
96
|
-
def add(meth, res)
|
97
|
-
mutex.synchronize { __send__(meth).push(res) }
|
98
|
-
end
|
99
|
-
|
100
101
|
def clear
|
101
102
|
imported.clear
|
102
103
|
skipped.clear
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mysql_import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- nalabjp
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: load_data_infile2
|