mysql_import 0.4.1 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7019a15ca8ae5846ef8014d300a886464a8ab02e
4
- data.tar.gz: 2c8cf6cf654037eb44e047a7cf44874032a4f42d
3
+ metadata.gz: 609ba688e11734a17c9b421fd8cc6ff576a400bb
4
+ data.tar.gz: 6911371e8f1d12a06adfdb590911739f55cb2d62
5
5
  SHA512:
6
- metadata.gz: 758364cd6d87430daff77c55734aa09e1e9273352138410f91c40352920eb56e5b1a7d3a3d0140269c2ae7cf3f218b481059abfe1ad784716680b33a4845c178
7
- data.tar.gz: b251b30800e0a56517d8073faa8dae7b73a247e45f2909e91c2cac7c8cf96681fa9c39d1f1d3ebb615526863525d258482e3183c3eacc795329ecb7ab5a29dce
6
+ metadata.gz: 198bd007c859a481726fe34302242e79cbca316278094e033e1ac709fa9831a9a7ec66b7a0a1b4ed2b036acdf0a622f47c317dc3a9ba741698d20723c01b24cd
7
+ data.tar.gz: d87ca57f0955dfe6d1028441a8f88d3720d211ed927237fcbea13eaa28aeee364415cb8416e7976a0dc471b3c520007ba551f62fec04dc18a014aa03775a5142
data/.travis.yml CHANGED
@@ -2,8 +2,7 @@ dist: trusty
2
2
  sudo: required
3
3
  language: ruby
4
4
  rvm:
5
- - 2.2.5
6
- - 2.3.1
5
+ - 2.3.3
7
6
  before_install: gem install bundler
8
7
  cache:
9
8
  - bundler
@@ -1,30 +1,26 @@
1
1
  require 'logger'
2
2
 
3
3
  class MysqlImport
4
- class Logger < SimpleDelegator
4
+ class Logger < ::Logger
5
5
  def initialize(out, debug)
6
+ super(out)
7
+
8
+ @level = INFO unless debug
6
9
  case out
7
- when String
8
- obj = ::Logger.new(out)
9
- obj.level = ::Logger::INFO
10
- when NilClass
11
- obj = ::Logger.new(nil)
12
10
  when STDOUT, STDERR
13
- obj = ::Logger.new(out)
14
- obj.formatter = ->(_, _, _, message) { "#{String === message ? message : message.inspect}\n" }
15
- obj.level = ::Logger::INFO
16
- else
17
- obj = out
11
+ @formatter = ->(_, _, _, message) { "#{String === message ? message : message.inspect}\n" }
18
12
  end
19
-
20
- obj.level = ::Logger::DEBUG if debug
21
- super(obj)
22
13
  end
23
14
  end
24
15
 
25
16
  module Logging
26
17
  def initialize(config, opts = {})
27
- @logger = Logger.new(opts[:log], opts.fetch(:debug, false))
18
+ @debug = opts.fetch(:debug, false)
19
+ @logger = if opts[:log].is_a?(::Logger)
20
+ opts[:log]
21
+ else
22
+ Logger.new(opts[:log], @debug)
23
+ end
28
24
  embed_logger
29
25
  super
30
26
  end
@@ -49,16 +45,20 @@ class MysqlImport
49
45
  private
50
46
 
51
47
  def parallel_opts
52
- @parallel_opts ||= super.merge(
53
- finish: proc do |item, index, _result|
54
- @logger.debug("parallel_item: #{item.inspect}")
55
- @logger.debug("parallel_index: #{index}")
56
- end
57
- )
48
+ @parallel_opts ||= if @debug
49
+ super.merge(
50
+ finish: proc do |item, index, _result|
51
+ @logger.debug("parallel_item: #{item.inspect}")
52
+ @logger.debug("parallel_index: #{index}")
53
+ end
54
+ )
55
+ else
56
+ super
57
+ end
58
58
  end
59
59
 
60
60
  def embed_logger
61
- unless LoadDataInfile2::Client.instance_methods.include?(:build_sql_with_logging)
61
+ if @debug && !LoadDataInfile2::Client.instance_methods.include?(:build_sql_with_logging)
62
62
  LoadDataInfile2::Client.class_exec(@logger) do |logger|
63
63
  define_method :build_sql_with_logging do |file, options = {}|
64
64
  build_sql_without_logging(file, options).tap {|sql| logger.debug("sql: #{sql}") }
@@ -1,3 +1,3 @@
1
1
  class MysqlImport
2
- VERSION = '0.4.1'
2
+ VERSION = '0.5.0'
3
3
  end
data/lib/mysql_import.rb CHANGED
@@ -7,6 +7,7 @@ require 'parallel'
7
7
  class MysqlImport
8
8
  def initialize(config, opts = {})
9
9
  @stash = []
10
+ @lock = opts.fetch(:lock, false)
10
11
  @concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
11
12
  pool = @concurrency.zero? ? 1 : @concurrency
12
13
  sql_opts = opts.fetch(:sql_opts, {})
@@ -44,32 +45,32 @@ class MysqlImport
44
45
 
45
46
  def run_import(cli, fpath, opts)
46
47
  t = Time.now
48
+ imported = false
47
49
 
48
- sql_opts = Marshal.load(Marshal.dump(opts.reject {|k, _| %i(before after).include?(k) }))
50
+ sql_opts = opts.reject {|k, _| %i(before after).include?(k) }
49
51
  table = sql_opts[:table] || File.basename(fpath, '.*')
52
+ lock = opts.fetch(:lock, @lock)
50
53
 
51
- if opts[:before]
52
- begin
53
- run_action(opts[:before], cli)
54
- rescue Break
55
- @result.add(:skipped, table)
56
- return
57
- end
58
- end
54
+ begin
55
+ write_lock(cli, table) if lock
59
56
 
60
- cli.import(fpath, sql_opts)
57
+ run_action(opts[:before], cli)
61
58
 
62
- if opts[:after]
63
- begin
64
- run_action(opts[:after], cli)
65
- rescue Break
66
- end
67
- end
59
+ cli.import(fpath, sql_opts)
60
+ imported = true
68
61
 
69
- @result.add(:imported, [table, (Time.now - t)])
62
+ run_action(opts[:after], cli)
63
+ rescue Break
64
+ @result.skipped.push(table) unless imported
65
+ ensure
66
+ unlock(cli) if lock
67
+ @result.imported.push([table, (Time.now - t)]) if imported
68
+ end
70
69
  end
71
70
 
72
71
  def run_action(action, cli)
72
+ return unless action
73
+
73
74
  case action
74
75
  when Array
75
76
  action.each { |act| run_action(act, cli) }
@@ -80,6 +81,14 @@ class MysqlImport
80
81
  end
81
82
  end
82
83
 
84
+ def write_lock(cli, table)
85
+ cli.query("LOCK TABLE `#{table}` WRITE;")
86
+ end
87
+
88
+ def unlock(cli)
89
+ cli.query("UNLOCK TABLES;")
90
+ end
91
+
83
92
  class Result
84
93
  def imported
85
94
  @imported ||= []
@@ -89,14 +98,6 @@ class MysqlImport
89
98
  @skipped ||= []
90
99
  end
91
100
 
92
- def mutex
93
- @mutext ||= Mutex.new
94
- end
95
-
96
- def add(meth, res)
97
- mutex.synchronize { __send__(meth).push(res) }
98
- end
99
-
100
101
  def clear
101
102
  imported.clear
102
103
  skipped.clear
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mysql_import
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - nalabjp
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-08-16 00:00:00.000000000 Z
11
+ date: 2016-11-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: load_data_infile2