mysql_import 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87f855e324bbd56af3b9ebd3652a752c8c5ddad9
-  data.tar.gz: 64166a6f3265a9e33e47f4a102c2e9c967a8e3b3
+  metadata.gz: 6db72f5a5672da5ee918069f82585950e325cb47
+  data.tar.gz: a5db440198b9b79f3c2fb497abf2dc89c0ebbb6d
 SHA512:
-  metadata.gz: f01b3bac43469acc2b0a5d14eae8c981e71085f38b9721dab09fcde3eacb0ad65889564ab6ec71870636d6ed1984078eb849f8f74e6d3adbff5ae18fe4c98e99
-  data.tar.gz: 70287c5d1b20f527ad67317c026ee323db26fc028f6a089d97447967f9233b74327388115e0037e2f54e36a7156782cc233977907a82da68769a3eef2bd840cd
+  metadata.gz: f7d95ea5b111c74d41bf02551dd76d7b522a9dc611093452ef6dd64621cb8960a7902578c29738396d35f996a26cdd1a7bdd8cb6929615106e738cb62f25a462
+  data.tar.gz: fea69a70e010fc80136200dd814903892b367a415a6c5298afebe8b2d0eafb4370462782df042b4b59fba4c4dbb9c9fd40bf89d7d12128463bce32fb5baf914a

data/lib/mysql_import/logger.rb

@@ -1,23 +1,71 @@
 require 'logger'
 
-class MysqlImport::Logger < SimpleDelegator
-  def initialize(logger, debug = false)
-    case logger
-    when String
-      obj = ::Logger.new(arg)
-    when NilClass
-      obj = ::Logger.new(STDOUT)
-      obj.formatter = ->(seveity, datetime, progname, message) { "#{String === message ? message : message.inspect}\n" }
-    when FalseClass
-      obj = ::Logger.new('/dev/null')
-    else
-      obj = logger
+class MysqlImport
+  class Logger < SimpleDelegator
+    def initialize(logger, debug)
+      case logger
+      when String
+        obj = ::Logger.new(arg)
+      when NilClass
+        obj = ::Logger.new(STDOUT)
+        obj.formatter = ->(seveity, datetime, progname, message) { "#{String === message ? message : message.inspect}\n" }
+      when FalseClass
+        obj = ::Logger.new('/dev/null')
+      else
+        obj = logger
+      end
+
+      obj.level = debug ? ::Logger::DEBUG : ::Logger::INFO
+      __setobj__(obj)
     end
+  end
+
+  module Logging
+    def initialize(config, opts ={}, sql_opts = {})
+      @logger = Logger.new(opts[:log], opts.fetch(:debug, false))
 
-    if obj.respond_to?(:level=)
-      obj.level = debug ? Logger::DEBUG : Logger::INFO
+      LoadDataInfile2::Client.class_exec(@logger) do |logger|
+        define_method :build_sql_with_logging do |file, options|
+          options = {} unless options
+          build_sql_without_logging(file, options).tap {|sql| logger.debug("sql: #{sql}") }
+        end
+        alias_method :build_sql_without_logging, :build_sql
+        alias_method :build_sql, :build_sql_with_logging
+      end
+
+      super
     end
 
-    __setobj__(obj)
+    def import(*filters)
+      super
+    ensure
+      logger.info('Imported tables:')
+      if result.imported.size > 0
+        result.imported.sort.each {|t| logger.info("  #{t[0]} (#{t[1]} sec)") }
+      else
+        result.logger.info('  nothing...')
+      end
+      if result.skipped.size > 0
+        logger.info('Skipped tables:')
+        result.skipped.sort.each {|t| logger.info("  #{t}") }
+      end
+
+      result.clear
+    end
+
+    private
+
+    attr_reader :logger
+
+    def parallel_opts
+      @_parallel_opts ||= super.merge(
+        finish: proc do |item, index, _result|
+          logger.debug("parallel_item: #{item.inspect}")
+          logger.debug("parallel_index: #{index}")
+        end
+      )
+    end
   end
+
+  prepend Logging
 end

data/lib/mysql_import/version.rb

@@ -1,3 +1,3 @@
 class MysqlImport
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/mysql_import.rb

@@ -1,4 +1,5 @@
 require 'mysql_import/version'
+require 'mysql_import/logger'
 require 'load_data_infile2'
 require 'connection_pool'
 require 'parallel'
@@ -8,7 +9,7 @@ class MysqlImport
     @stash = []
     @fileters = []
     @concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
-    pool = concurrency == 0 ? 1 : concurrency
+    pool = concurrency.zero? ? 1 : concurrency
 
     @client = ConnectionPool.new(size: pool) { LoadDataInfile2::Client.new(config, sql_opts) }
     @result = Result.new
@@ -19,7 +20,7 @@ class MysqlImport
   end
 
   def import(*filters)
-    Parallel.each(filtered_list(filters), in_threads: concurrency) do |args|
+    Parallel.each(filtered_list(filters), parallel_opts) do |args|
       client.with do |cli|
         run_import(cli, *args)
       end
@@ -37,7 +38,13 @@ class MysqlImport
     stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
   end
 
+  def parallel_opts
+    { in_threads: concurrency }
+  end
+
   def run_import(cli, fpath, opts)
+    t = Time.now
+
     before = opts.delete(:before)
     after = opts.delete(:after)
     table = opts[:table] || File.basename(fpath, '.*')
@@ -51,29 +58,26 @@ class MysqlImport
       end
     end
 
-    res = cli.import(fpath, opts)
-    result.add(:imported, table)
+    cli.import(fpath, opts)
 
     if after
       begin
-        run_action(after, cli, res)
+        run_action(after, cli)
       rescue Break
       end
     end
+
+    result.add(:imported, [table, (Time.now - t)])
   end
 
-  def run_action(action, cli, res = nil)
+  def run_action(action, cli)
     case action
     when Array
-      action.map { |act| run_action(act, cli, res) }
+      action.each { |act| run_action(act, cli) }
     when String
       cli.query(action)
     else
-      if res
-        action.call(cli, res)
-      else
-        action.call(cli)
-      end
+      action.call(cli)
     end
   end
 
@@ -93,6 +97,11 @@ class MysqlImport
     def add(meth, res)
       mutex.synchronize { __send__(meth).push(res) }
     end
+
+    def clear
+      imported.clear
+      skipped.clear
+    end
   end
 
   class Break < StandardError; end

data/mysql_import.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['nalabjp']
   spec.email         = ['nalabjp@gmail.com']
 
-  spec.summary       = 'Concurrent importer for MySQL'
-  spec.description   = 'Concurrent importer for MySQL'
+  spec.summary       = 'Simple concurrent importer for MySQL'
+  spec.description   = 'Simple concurrent importer for MySQL'
   spec.homepage      = 'https://github.com/nalabjp/mysql_import'
   spec.license       = 'MIT'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mysql_import
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - nalabjp
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-18 00:00:00.000000000 Z
+date: 2016-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: load_data_infile2
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Concurrent importer for MySQL
+description: Simple concurrent importer for MySQL
 email:
 - nalabjp@gmail.com
 executables: []
@@ -137,6 +137,6 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Concurrent importer for MySQL
+summary: Simple concurrent importer for MySQL
 test_files: []
 has_rdoc: