db_sucker 3.0.0

data/lib/db_sucker/application/sklaven_treiber/worker/helpers.rb

@@ -0,0 +1,144 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module Helpers
+          def runtime
+            if @started
+              human_seconds ((@ended || Time.current) - @started).to_i
+            end
+          end
+
+          def sftp_download *args, &block
+            IO::SftpDownload.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
+          def file_copy *args, &block
+            IO::FileCopy.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
+          def file_gunzip *args, &block
+            IO::FileGunzip.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
+          def file_shasum *args, &block
+            IO::Shasum.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
+          def pv_wrap *args, &block
+            IO::PvWrapper.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
+          def file_import_sql *args, &block
+            IO::FileImportSql.new(self, *args).tap do |op|
+              block.try(:call, op)
+            end
+          end
+
+          def aquire_slots *which, &block
+            target_thread = Thread.current
+            aquired = []
+            which.each_with_index do |wh, i|
+              if pool = sklaventreiber.slot_pools[wh]
+                waitlock = Queue.new
+                channel = app.channelfy_thread app.spawn_thread(:sklaventreiber_worker_slot_progress) {|thr|
+                  thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
+                  thr[:slot_pool_qindex] = Proc.new { pool.qindex(target_thread) }
+                  waitlock.pop
+                  pool.aquire(target_thread)
+                }
+                waitlock << true
+                target_thread.wait
+
+                label = "aquiring slot #{i+1}/#{which.length} `#{pool.name}' :slot_pool_qindex(– #%s in queue )(:seconds)..."
+                second_progress(channel, label, :blue).tap{ pool.wait_aquired(target_thread) }.join
+                if pool.aquired?(target_thread)
+                  aquired << wh
+                else
+                  break
+                end
+              else
+                raise SlotPoolNotInitializedError, "slot pool `#{wh}' was never initialized, can't aquire slot"
+              end
+            end
+            block.call if block && (which - aquired).empty?
+          ensure
+            release_slots(*which)
+          end
+
+          def release_slots *which
+            which.each_with_index do |wh, i|
+              if pool = sklaventreiber.slot_pools[wh]
+                pool.release(Thread.current)
+              else
+                raise SlotPoolNotInitializedError, "slot pool `#{wh}' was never initialized, can't release slot (was most likely never aquired)"
+              end
+            end
+          end
+
+          def wait_defer_ready label = nil
+            channel = app.fake_channel {|c| c[:slot_pool_qindex].call.zero? || @should_cancel }
+            channel[:slot_pool_qindex] = Proc.new { sklaventreiber.sync { sklaventreiber.workers.count{|w| !w.done? && !w.deferred? } } }
+
+            label = "deferred import: #{human_bytes(File.size(@local_file_raw))} raw SQL :slot_pool_qindex(– waiting for %s workers )(:seconds)"
+            second_progress(channel, label, :blue).join
+          end
+
+          def second_progress channel, status, color = :yellow
+            target_thread = Thread.current
+            app.spawn_thread(:sklaventreiber_worker_second_progress) do |thr|
+              thr[:iteration] = 0
+              thr[:started_at] = Time.current
+              thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
+              channel[:handler] = thr if channel.respond_to?(:[]=)
+              loop do
+                if @should_cancel && !thr[:canceled]
+                  if channel.is_a?(Net::SSH::Connection::Channel)
+                    if channel[:pty]
+                      channel.send_data("\C-c") rescue false
+                    elsif channel[:pid]
+                      @ctn.kill_remote_process(channel[:pid])
+                    end
+                  end
+                  channel.try(:close) rescue false
+                  Process.kill(:SIGINT, channel[:ipc_thread].pid) if channel[:ipc_thread]
+                  thr[:canceled] = true
+                end
+                stat = status.gsub(":seconds", human_seconds(Time.current - thr[:started_at]))
+                if channel[:slot_pool_qindex].respond_to?(:call)
+                  qi = channel[:slot_pool_qindex].call
+                  re = /:slot_pool_qindex\(([^\)]+)\)/
+                  if stat[re]
+                    stat[re] = qi ? stat[re].match(/\(([^\)]+)\)/)[1].gsub("%s", qi.to_s) : ""
+                  end
+                end
+                if channel[:error_message]
+                  @status = ["[ERROR] #{channel[:error_message]}", :red]
+                elsif !channel.active?
+                  @status = ["[CLOSED] #{stat}", :red]
+                elsif channel.closing?
+                  @status = ["[CLOSING] #{stat}", :red]
+                else
+                  @status = [stat, color]
+                end
+                break unless channel.active?
+                thr.wait(0.1)
+                thr[:iteration] += 1
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/base.rb

@@ -0,0 +1,240 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class Base
+            UnknownFormatterError = Class.new(::ArgumentError)
+            DataIntegrityError = Class.new(::RuntimeError)
+            STATUS_FORMATTERS = [:none, :minimal, :full]
+            attr_reader :status_format, :state, :operror, :offset, :closing, :local, :remote, :ctn, :verify_handle
+            attr_accessor :read_size, :label, :entity, :filesize, :throughput, :mode
+            OutputHelper.hook(self)
+
+            def initialize worker, ctn, fd
+              # references
+              @worker = worker
+              @ctn = ctn
+              @thread = Thread.current
+
+              # remote & local
+              @remote = fd.is_a?(Hash) ? fd.keys[0] : fd
+              @local = fd.values[0] if fd.is_a?(Hash)
+
+              # defaults
+              @label ||= "working"
+              @entity ||= "task"
+              @status_format = :off
+              @mode = :fs
+              @throughput = worker.sklaventreiber.throughput.register(self)
+              @read_size = 128 * 1024 # 128kb
+              @filesize = 0
+
+              # callbacks
+              @abort_if = Proc.new { false }
+              @on_error = Proc.new {}
+              @on_complete = Proc.new {}
+              @on_success = Proc.new {}
+              init
+              reset_state
+            end
+
+            def init
+            end
+
+            def reset_state
+              @operror = nil
+              @closing = false
+              @state = :idle
+              @offset = 0
+              @throughput.reset_stats
+            end
+
+            def abort_if &block
+              @abort_if = block
+            end
+
+            def on_error &block
+              @on_error = block
+            end
+
+            def on_complete &block
+              @on_complete = block
+            end
+
+            def on_success &block
+              @on_success = block
+            end
+
+            def status_format= which
+              which = which.to_sym
+              raise UnknownFormatterError, "unknown status format `#{which}', available options: #{STATUS_FORMATTERS * ", "}" unless STATUS_FORMATTERS.include?(which)
+              @status_format = which
+            end
+
+            def prepare_local_destination
+              FileUtils.mkdir_p(File.dirname(@local))
+            end
+
+            def handle_pause
+              @thread[:paused] ? throughput.pause! : throughput.unpause!
+            end
+
+            def execute opts = {}, &block
+              opts = opts.reverse_merge(tries: 1, sleep_error: 0)
+              try = 1
+              begin
+                reset_state
+                throughput.measure(&block)
+                if !@closing && !@worker.should_cancel
+                  @state = :done
+                  @on_success.call(self)
+                end
+              rescue StandardError => ex
+                @operror = "##{try} #{ex.class}: #{ex.message}"
+                @on_error.call(self, ex, @operror)
+                try += 1
+                if try > opts[:tries]
+                  raise ex
+                else
+                  Thread.current.wait(opts[:sleep_error])
+                  retry
+                end
+              ensure
+                @on_complete.call(self)
+              end
+            ensure
+              @throughput.unregister
+            end
+
+            def to_s
+              handle_pause
+              return @operror if @operror
+              tp = @throughput
+
+              [].tap do |r|
+                r << "[CLOSING]" if @closing
+                if @status_format == :none
+                  r << "#{@label}"
+                  break
+                end
+                case @state
+                when :idle, :init
+                  r << "#{@label}:"
+                  r << " initiating..."
+                when :finishing
+                  r << "#{@label}:"
+                  r << " finishing..."
+                when :verifying
+                  r << "#{@label}:"
+                  r << " verifying..."
+                  r << " #{@verify_handle.throughput.f_done_percentage}" if @verify_handle.respond_to?(:throughput)
+                when :done
+                  if @mode == :nofs
+                    r << "#{@entity || @label} complete(?): #{tp.f_offset}"
+                  else
+                    r << "#{@entity || @label} #{@offset == @filesize ? "complete" : "INCOMPLETE"}: #{tp.f_done_percentage} – #{tp.f_byte_progress}"
+                  end
+                when :downloading, :copying, :decompressing, :working
+                  r << "#{@label}:"
+                  if @mode == :nofs
+                    r << "[#{tp.f_offset} – #{tp.f_bps}/s]"
+                  else
+                    r << tp.f_done_percentage.rjust(7, " ")
+                    if @status_format == :minimal
+                      r << "[#{tp.f_eta}]"
+                    elsif @status_format == :full
+                      r << "[#{tp.f_eta} – #{tp.f_bps.rjust(9, " ")}/s]"
+                    end
+
+                    if @status_format == :full
+                      f_has, f_tot = tp.f_offset, tp.f_filesize
+                      r << "[#{f_has.rjust(f_tot.length, "0")}/#{f_tot}]"
+                    end
+                  end
+                end
+              end * " "
+            end
+
+            def to_curses target
+              handle_pause
+              _this = self
+              tp = @throughput
+              target.instance_eval do
+                if _this.operror
+                  red "#{_this.operror}"
+                  return
+                end
+
+                if _this.closing
+                  red "[CLOSING] "
+                end
+
+                if _this.status_format == :none
+                  blue "#{_this.label}"
+                  break
+                end
+
+                case _this.state
+                when :idle, :init
+                  yellow "#{_this.label}: "
+                  gray " initiating..."
+                when :finishing
+                  yellow "#{_this.label}: "
+                  gray " finishing..."
+                when :verifying
+                  yellow "#{_this.label}: "
+                  gray " verifying..."
+                  gray " #{_this.verify_handle.throughput.f_done_percentage}" if _this.verify_handle.respond_to?(:throughput)
+                when :done
+                  if _this.mode == :nofs
+                    green "#{_this.entity || _this.label} complete(?): "
+                    cyan "#{human_bytes _this.offset}"
+                  else
+                    if _this.offset == _this.filesize
+                      green "#{_this.entity || _this.label} complete: #{tp.f_done_percentage}"
+                      yellow " – "
+                      cyan "#{human_bytes _this.offset}"
+                    else
+                      red "#{_this.entity || _this.label} INCOMPLETE: #{tp.f_done_percentage}"
+                      yellow " – "
+                      cyan "#{tp.f_byte_progress}"
+                    end
+                  end
+                when :downloading, :copying, :decompressing, :working
+                  yellow "#{_this.label}: "
+                  if _this.mode == :nofs
+                    blue "#{tp.f_offset}"
+                    if _this.status_format == :minimal
+                      gray " [#{tp.f_runtime}]"
+                    elsif _this.status_format == :full
+                      gray " [#{tp.f_bps}/s – #{tp.f_runtime}]"
+                    end
+                    #progress_bar(-1)
+                  else
+                    diffp = tp.done_percentage
+                    color = diffp > 90 ? :green : diffp > 75 ? :blue : diffp > 50 ? :cyan : diffp > 25 ? :yellow : :red
+                    send(color, tp.f_done_percentage.rjust(7, " ") << " ")
+
+                    if _this.status_format == :minimal
+                      yellow "[#{tp.f_eta}]"
+                    elsif _this.status_format == :full
+                      yellow "[#{tp.f_eta} – #{tp.f_bps.rjust(9, " ")}/s]"
+                    end
+
+                    if _this.status_format == :full
+                      f_has, f_tot = tp.f_offset, tp.f_filesize
+                      gray " [#{f_has.rjust(f_tot.length, "0")}/#{f_tot}]"
+                    end
+
+                    progress_bar(diffp, prog_done_color: color, prog_current_color: color)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/file_copy.rb

@@ -0,0 +1,81 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class FileCopy < Base
+            attr_accessor :use_tmp, :integrity
+
+            def init
+              @label = "copying"
+              @entity = "copy"
+              @use_tmp = true
+              @integrity = true
+              @throughput.categories << :io << :io_file_copy
+            end
+
+            def copy! opts = {}
+              opts = opts.reverse_merge(tries: 1, read_size: @read_size)
+              prepare_local_destination
+
+              execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                @tmploc   = @use_tmp ? "#{@local}.tmp" : @local
+                @in_file  = File.new(@remote, "rb")
+                @out_file = File.new(@tmploc, "wb")
+                @filesize = @in_file.size
+
+                buf = ""
+                @state = :copying
+                begin
+                  while @in_file.sysread(opts[:read_size], buf)
+                    if !@closing && @abort_if.call(self)
+                      @closing = true
+                      break
+                    end
+
+                    @offset += buf.bytesize
+                    @out_file.syswrite(buf)
+                    GC.start if @offset % GC_FORCE_RATE == 0
+                  end
+                rescue EOFError
+                ensure
+                  @state = :finishing
+                  @in_file.close
+                  @out_file.close
+                end
+
+                FileUtils.mv(@tmploc, @local) if @use_tmp
+
+                @state = :verifying
+                src_hash = verify_file(@remote, 0)
+                dst_hash = verify_file(@local, 1)
+                if src_hash != dst_hash
+                  raise DataIntegrityError, "Integrity check failed! [SRC](#{src_hash}) != [DST](#{dst_hash})"
+                end
+              end
+            end
+
+            def verify_file file, index = 0
+              result = false
+              @worker.file_shasum(@ctn, file) do |fc|
+                fc.sha = @ctn.integrity_sha
+                fc.status_format = :none
+                fc.throughput.sopts[:perc_modifier] = 0.5
+                fc.throughput.sopts[:perc_base] = index * 50
+                @verify_handle = fc
+
+                fc.abort_if { @should_cancel }
+                fc.on_success do
+                  result = fc.result
+                end
+                fc.verify!
+              end
+              @verify_handle = false
+              return result
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/file_gunzip.rb

@@ -0,0 +1,58 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class FileGunzip < Base
+            attr_accessor :use_tmp
+            attr_accessor :preserve_original
+
+            def init
+              @label = "decompressing"
+              @entity = "decompress"
+              @use_tmp = true
+              @preserve_original = false
+              @local ||= "#{File.dirname(@remote)}/#{File.basename(@remote, ".gz")}"
+              @throughput.categories << :io << :io_gunzip
+            end
+
+            def gunzip! opts = {}
+              opts = opts.reverse_merge(tries: 1, read_size: @read_size)
+              prepare_local_destination
+
+              execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                @tmploc   = @use_tmp ? "#{@local}.tmp" : @local
+                @in_file  = File.new(@remote, "rb")
+                @out_file = File.new(@tmploc, "wb")
+                @filesize = @in_file.size if @filesize.zero?
+
+                @state = :decompressing
+                gz = Zlib::GzipReader.new(@in_file)
+                begin
+                  while buf = gz.read(opts[:read_size])
+                    if !@closing && @abort_if.call(self)
+                      @closing = true
+                      break
+                    end
+
+                    @offset += [opts[:read_size], @filesize - @offset].min
+                    @out_file.syswrite(buf)
+                    GC.start if @offset % GC_FORCE_RATE == 0
+                  end
+                ensure
+                  @state = :finishing
+                  gz.close
+                  @in_file.close
+                  @out_file.close
+                end
+
+                FileUtils.mv(@tmploc, @local) if @use_tmp
+                File.unlink(@remote) unless @preserve_original
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/file_import_sql.rb

@@ -0,0 +1,80 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class FileImportSql < Base
+            MissingInstructionError = Class.new(::ArgumentError)
+            InvalidInstructionError = Class.new(::ArgumentError)
+            ImportError = Class.new(::RuntimeError)
+            attr_accessor :instruction
+
+            def init
+              @label = "importing table"
+              @entity = "table"
+              @instruction = false
+              @throughput.categories << :io << :io_import
+            end
+
+            def import! opts = {}
+              raise MissingInstructionError, "no instruction given for import" unless @instruction.is_a?(Hash)
+              raise InvalidInstructionError, "import instruction must at least contain :bin and :file keys" if [:bin, :file].any?{|w| !@instruction.key?(w) }
+              opts = opts.reverse_merge(tries: 1, read_size: @read_size)
+
+              execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                @in_file  = File.new(@instruction[:file], "rb")
+                @filesize = @in_file.size if @filesize.zero?
+
+                buf = ""
+                @state = :working
+
+                begin
+                  debug "Opening process `#{@instruction[:bin]}'"
+                  Open3.popen2e(@instruction[:bin], pgroup: true) do |_stdin, _stdouterr, _thread|
+                    begin
+                      # file prepend
+                      _stdin.puts @instruction[:file_prepend] if @instruction[:file_prepend]
+
+                      # file contents
+                      begin
+                        while @in_file.sysread(opts[:read_size], buf)
+                          if !@closing && @abort_if.call(self)
+                            @closing = true
+                            break
+                          end
+
+                          @offset += buf.bytesize
+                          _stdin.write(buf)
+                          GC.start if @offset % GC_FORCE_RATE == 0
+                        end
+                      rescue EOFError
+                      ensure
+                        @in_file.close
+                      end
+
+                      # file append
+                      _stdin.puts @instruction[:file_append] if @instruction[:file_append]
+                    rescue Errno::EPIPE => ex
+                      raise ImportError, "#{ex.message} (#{_stdouterr.read.chomp})"
+                    end
+
+                    # close & exit status
+                    _stdin.close_write
+                    exit_status = _thread.value
+                    if exit_status == 0
+                      debug "Process exited (#{exit_status}) `#{@instruction[:bin]}'"
+                    else
+                      warning "Process exited (#{exit_status}) `#{@instruction[:bin]}'"
+                    end
+                  end
+                ensure
+                  @state = :finishing
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/file_shasum.rb

@@ -0,0 +1,49 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class Shasum < Base
+            attr_accessor :sha, :result
+
+            def init
+              @label = "verifying"
+              @entity = "verification"
+              @sha ||= 1
+              @throughput.categories.clear # IO read
+            end
+
+            def verify! opts = {}
+              opts = opts.reverse_merge(tries: 1, read_size: @read_size)
+
+              execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                @in_file  = File.new(@remote, "rb")
+                @filesize = @in_file.size
+
+                @state = :working
+                sha = "Digest::SHA#{@sha}".constantize.new
+                buf = ""
+                begin
+                  while @in_file.read(opts[:read_size], buf)
+                    if !@closing && @abort_if.call(self)
+                      @closing = true
+                      break
+                    end
+
+                    @offset += buf.bytesize
+                    sha << buf
+                    GC.start if @offset % GC_FORCE_RATE == 0
+                  end
+                ensure
+                  @state = :finishing
+                  @in_file.close
+                  @result = sha.hexdigest
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end