mysql_import 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 87f855e324bbd56af3b9ebd3652a752c8c5ddad9
4
- data.tar.gz: 64166a6f3265a9e33e47f4a102c2e9c967a8e3b3
3
+ metadata.gz: 6db72f5a5672da5ee918069f82585950e325cb47
4
+ data.tar.gz: a5db440198b9b79f3c2fb497abf2dc89c0ebbb6d
5
5
  SHA512:
6
- metadata.gz: f01b3bac43469acc2b0a5d14eae8c981e71085f38b9721dab09fcde3eacb0ad65889564ab6ec71870636d6ed1984078eb849f8f74e6d3adbff5ae18fe4c98e99
7
- data.tar.gz: 70287c5d1b20f527ad67317c026ee323db26fc028f6a089d97447967f9233b74327388115e0037e2f54e36a7156782cc233977907a82da68769a3eef2bd840cd
6
+ metadata.gz: f7d95ea5b111c74d41bf02551dd76d7b522a9dc611093452ef6dd64621cb8960a7902578c29738396d35f996a26cdd1a7bdd8cb6929615106e738cb62f25a462
7
+ data.tar.gz: fea69a70e010fc80136200dd814903892b367a415a6c5298afebe8b2d0eafb4370462782df042b4b59fba4c4dbb9c9fd40bf89d7d12128463bce32fb5baf914a
@@ -1,23 +1,71 @@
1
1
  require 'logger'
2
2
 
3
- class MysqlImport::Logger < SimpleDelegator
4
- def initialize(logger, debug = false)
5
- case logger
6
- when String
7
- obj = ::Logger.new(arg)
8
- when NilClass
9
- obj = ::Logger.new(STDOUT)
10
- obj.formatter = ->(seveity, datetime, progname, message) { "#{String === message ? message : message.inspect}\n" }
11
- when FalseClass
12
- obj = ::Logger.new('/dev/null')
13
- else
14
- obj = logger
3
+ class MysqlImport
4
+ class Logger < SimpleDelegator
5
+ def initialize(logger, debug)
6
+ case logger
7
+ when String
8
+ obj = ::Logger.new(arg)
9
+ when NilClass
10
+ obj = ::Logger.new(STDOUT)
11
+ obj.formatter = ->(seveity, datetime, progname, message) { "#{String === message ? message : message.inspect}\n" }
12
+ when FalseClass
13
+ obj = ::Logger.new('/dev/null')
14
+ else
15
+ obj = logger
16
+ end
17
+
18
+ obj.level = debug ? ::Logger::DEBUG : ::Logger::INFO
19
+ __setobj__(obj)
15
20
  end
21
+ end
22
+
23
+ module Logging
24
+ def initialize(config, opts ={}, sql_opts = {})
25
+ @logger = Logger.new(opts[:log], opts.fetch(:debug, false))
16
26
 
17
- if obj.respond_to?(:level=)
18
- obj.level = debug ? Logger::DEBUG : Logger::INFO
27
+ LoadDataInfile2::Client.class_exec(@logger) do |logger|
28
+ define_method :build_sql_with_logging do |file, options|
29
+ options = {} unless options
30
+ build_sql_without_logging(file, options).tap {|sql| logger.debug("sql: #{sql}") }
31
+ end
32
+ alias_method :build_sql_without_logging, :build_sql
33
+ alias_method :build_sql, :build_sql_with_logging
34
+ end
35
+
36
+ super
19
37
  end
20
38
 
21
- __setobj__(obj)
39
+ def import(*filters)
40
+ super
41
+ ensure
42
+ logger.info('Imported tables:')
43
+ if result.imported.size > 0
44
+ result.imported.sort.each {|t| logger.info(" #{t[0]} (#{t[1]} sec)") }
45
+ else
46
+ result.logger.info(' nothing...')
47
+ end
48
+ if result.skipped.size > 0
49
+ logger.info('Skipped tables:')
50
+ result.skipped.sort.each {|t| logger.info(" #{t}") }
51
+ end
52
+
53
+ result.clear
54
+ end
55
+
56
+ private
57
+
58
+ attr_reader :logger
59
+
60
+ def parallel_opts
61
+ @_parallel_opts ||= super.merge(
62
+ finish: proc do |item, index, _result|
63
+ logger.debug("parallel_item: #{item.inspect}")
64
+ logger.debug("parallel_index: #{index}")
65
+ end
66
+ )
67
+ end
22
68
  end
69
+
70
+ prepend Logging
23
71
  end
@@ -1,3 +1,3 @@
1
1
  class MysqlImport
2
- VERSION = '0.1.0'
2
+ VERSION = '0.2.0'
3
3
  end
data/lib/mysql_import.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'mysql_import/version'
2
+ require 'mysql_import/logger'
2
3
  require 'load_data_infile2'
3
4
  require 'connection_pool'
4
5
  require 'parallel'
@@ -8,7 +9,7 @@ class MysqlImport
8
9
  @stash = []
9
10
  @fileters = []
10
11
  @concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
11
- pool = concurrency == 0 ? 1 : concurrency
12
+ pool = concurrency.zero? ? 1 : concurrency
12
13
 
13
14
  @client = ConnectionPool.new(size: pool) { LoadDataInfile2::Client.new(config, sql_opts) }
14
15
  @result = Result.new
@@ -19,7 +20,7 @@ class MysqlImport
19
20
  end
20
21
 
21
22
  def import(*filters)
22
- Parallel.each(filtered_list(filters), in_threads: concurrency) do |args|
23
+ Parallel.each(filtered_list(filters), parallel_opts) do |args|
23
24
  client.with do |cli|
24
25
  run_import(cli, *args)
25
26
  end
@@ -37,7 +38,13 @@ class MysqlImport
37
38
  stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
38
39
  end
39
40
 
41
+ def parallel_opts
42
+ { in_threads: concurrency }
43
+ end
44
+
40
45
  def run_import(cli, fpath, opts)
46
+ t = Time.now
47
+
41
48
  before = opts.delete(:before)
42
49
  after = opts.delete(:after)
43
50
  table = opts[:table] || File.basename(fpath, '.*')
@@ -51,29 +58,26 @@ class MysqlImport
51
58
  end
52
59
  end
53
60
 
54
- res = cli.import(fpath, opts)
55
- result.add(:imported, table)
61
+ cli.import(fpath, opts)
56
62
 
57
63
  if after
58
64
  begin
59
- run_action(after, cli, res)
65
+ run_action(after, cli)
60
66
  rescue Break
61
67
  end
62
68
  end
69
+
70
+ result.add(:imported, [table, (Time.now - t)])
63
71
  end
64
72
 
65
- def run_action(action, cli, res = nil)
73
+ def run_action(action, cli)
66
74
  case action
67
75
  when Array
68
- action.map { |act| run_action(act, cli, res) }
76
+ action.each { |act| run_action(act, cli) }
69
77
  when String
70
78
  cli.query(action)
71
79
  else
72
- if res
73
- action.call(cli, res)
74
- else
75
- action.call(cli)
76
- end
80
+ action.call(cli)
77
81
  end
78
82
  end
79
83
 
@@ -93,6 +97,11 @@ class MysqlImport
93
97
  def add(meth, res)
94
98
  mutex.synchronize { __send__(meth).push(res) }
95
99
  end
100
+
101
+ def clear
102
+ imported.clear
103
+ skipped.clear
104
+ end
96
105
  end
97
106
 
98
107
  class Break < StandardError; end
data/mysql_import.gemspec CHANGED
@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
8
8
  spec.authors = ['nalabjp']
9
9
  spec.email = ['nalabjp@gmail.com']
10
10
 
11
- spec.summary = 'Concurrent importer for MySQL'
12
- spec.description = 'Concurrent importer for MySQL'
11
+ spec.summary = 'Simple concurrent importer for MySQL'
12
+ spec.description = 'Simple concurrent importer for MySQL'
13
13
  spec.homepage = 'https://github.com/nalabjp/mysql_import'
14
14
  spec.license = 'MIT'
15
15
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mysql_import
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - nalabjp
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-07-18 00:00:00.000000000 Z
11
+ date: 2016-07-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: load_data_infile2
@@ -94,7 +94,7 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
- description: Concurrent importer for MySQL
97
+ description: Simple concurrent importer for MySQL
98
98
  email:
99
99
  - nalabjp@gmail.com
100
100
  executables: []
@@ -137,6 +137,6 @@ rubyforge_project:
137
137
  rubygems_version: 2.5.1
138
138
  signing_key:
139
139
  specification_version: 4
140
- summary: Concurrent importer for MySQL
140
+ summary: Simple concurrent importer for MySQL
141
141
  test_files: []
142
142
  has_rdoc: