mysql_import 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29c2cc452ed64d1ef3c41c20c1198437fa9e6d2c
-  data.tar.gz: 399d19ca719fc480d9031b2dbc9a55ce7c9d7549
+  metadata.gz: 06c89712eae3231ff8008e95ae2f918641392538
+  data.tar.gz: 8f070851b68245e1fe809997d91e7aaef3b6a079
 SHA512:
-  metadata.gz: 7e97ae3ea5baf48ab3507fe0bd72affaf2cb4de093a30cf152d9332de2fb81654e38e6a1b2a830672f169717bfe525982b0c6e6b190d55466134b9d1fec2965f
-  data.tar.gz: db0283a0db2d2d9995bde1f923be20c516725598c627cbe617322d563f8702be29918446f3ff6c6542b452ed2f1b7d355ff7bdcc0f3690783b3f77508b4354f2
+  metadata.gz: 6753df47ee3e60af5db61f00aab235a484fcf8a735f8c9b3f352683632514b4e70bd2cc402de9046952083a8d62041dc4ce850efacc2dd92281c8a1e354b5d91
+  data.tar.gz: 561b7c0146843c0ec9940b62a3c1b56bb3099448ef964435348fc2f18e996d5bc945621586b575366fd9da0900c10f7a8237fefa8d9333861db280e57e7e3759

data/lib/mysql_import/logger.rb

@@ -30,10 +30,12 @@ class MysqlImport
     ensure
       @logger.info('Imported tables:')
       if @result.imported.size > 0
+        max_len = @result.imported.map(&:first).max_by{|w| w.length}.length
         @result.imported.sort.each do |t|
-          msg = "  #{t[0]} (exec:#{format('%.3fs', t[1])}"
+          space = ' ' * ((max_len - t[0].length) + 1)
+          msg = "  #{t[0]}#{space}[exec:#{format('%.3fs', t[1])}"
           msg << " lock:#{format('%.3fs', t[2])}" if t[2]
-          msg << ')'
+          msg << ']'
           @logger.info(msg)
         end
       else

data/lib/mysql_import/version.rb

@@ -1,3 +1,3 @@
 class MysqlImport
-  VERSION = '0.5.2'
+  VERSION = '0.5.3'
 end

data/lib/mysql_import.rb

@@ -3,6 +3,7 @@ require 'mysql_import/logger'
 require 'load_data_infile2'
 require 'connection_pool'
 require 'parallel'
+require 'benchmark'
 
 class MysqlImport
   def initialize(config, opts = {})
@@ -23,7 +24,12 @@ class MysqlImport
   def import(filters = nil)
     Parallel.each(filtered_list(filters), parallel_opts) do |args|
       @client.with do |cli|
-        run_import(cli, *args)
+        begin
+          store[:client] = cli
+          import_internal(*args)
+        ensure
+          clear_store
+        end
       end
     end
   end
@@ -43,57 +49,107 @@ class MysqlImport
     { in_threads: @concurrency }
   end
 
-  def run_import(cli, fpath, opts)
-    t = Time.now
-    imported = false
-
+  def import_internal(fpath, opts)
     sql_opts = opts.reject {|k, _| %i(before after).include?(k) }
-    table = sql_opts[:table] || File.basename(fpath, '.*')
-    lock = opts.fetch(:lock, @lock)
+    store[:table] = sql_opts[:table] || File.basename(fpath, '.*')
 
-    begin
-      if lock
-        write_lock(cli, table)
-        lt = Time.now
-      end
+    with_recording do
+      with_lock_if_needed(opts.fetch(:lock, @lock)) do
+        with_skip_handling do
+          run_before_action(opts[:before])
 
-      run_action(opts[:before], cli)
+          store[:client].import(fpath, sql_opts)
 
-      cli.import(fpath, sql_opts)
-      imported = true
+          run_after_action(opts[:after])
+        end
+      end
+    end
+  end
 
-      run_action(opts[:after], cli)
-    rescue Break
-      @result.skipped.push(table) unless imported
-    ensure
-      res = [table, (Time.now - t)]
-      if lock
-        res.push(Time.now - lt)
-        unlock(cli)
+  def with_recording
+    store[:exec_time] = realtime { yield }
+  ensure
+    if store[:before_break]
+      @result.skipped.push(store[:table])
+    else
+      res = [store[:table], store[:exec_time]]
+      res.push(store[:lock_time]) if store[:lock_time]
+      @result.imported.push(res)
+    end
+  end
+
+  def with_lock_if_needed(need)
+    if need
+      begin
+        write_lock
+        store[:lock_time] = realtime { yield }
+      ensure
+        unlock
       end
-      @result.imported.push(res) if imported
+    else
+      yield
     end
   end
 
-  def run_action(action, cli)
+  def with_skip_handling
+    yield
+  rescue BeforeBreak
+    store[:before_break] = true
+  rescue AfterBreak
+    store[:after_break] = true
+  end
+
+  def realtime
+    Benchmark.realtime { yield }
+  end
+
+  def run_action(action)
     return unless action
 
     case action
     when Array
-      action.each { |act| run_action(act, cli) }
+      action.each { |act| run_action(act) }
     when String
-      cli.query(action)
+      store[:client].query(action)
     else
-      action.call(cli)
+      action.call(store[:client])
     end
   end
 
-  def write_lock(cli, table)
-    cli.query("LOCK TABLE `#{table}` WRITE;")
+  def run_before_action(action)
+    run_action(action)
+  rescue Break
+    raise BeforeBreak
+  end
+
+  def run_after_action(action)
+    run_action(action)
+  rescue Break
+    raise AfterBreak
+  end
+
+  def write_lock
+    [
+      'SET @old_autocommit=@@autocommit;',
+      'SET autocommit=0;',
+      "LOCK TABLE `#{store[:table]}` WRITE;"
+    ].each {|sql| store[:client].query(sql)}
+  end
+
+  def unlock
+    [
+      'COMMIT;',
+      'UNLOCK TABLES;',
+      'SET autocommit=@old_autocommit;'
+    ].each {|sql| store[:client].query(sql)}
+  end
+
+  def store
+    Thread.current[:store] ||= {}
   end
 
-  def unlock(cli)
-    cli.query("UNLOCK TABLES;")
+  def clear_store
+    Thread.current[:store] = nil
   end
 
   class Result
@@ -112,4 +168,6 @@ class MysqlImport
   end
 
   class Break < StandardError; end
+  class BeforeBreak < Break; end
+  class AfterBreak < Break; end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mysql_import
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - nalabjp
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-29 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: load_data_infile2