db_sucker 3.0.0

data/lib/db_sucker/application/sklaven_treiber/worker/io/pv_wrapper.rb

@@ -0,0 +1,73 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class PvWrapper < Base
+            NoCommandError = Class.new(::ArgumentError)
+            attr_accessor :cmd, :result
+
+            def init
+              @enabled ||= Proc.new {}
+              @fallback ||= Proc.new {}
+              @cmd ||= @local
+            end
+
+            def enabled &block
+              @enabled = block
+            end
+
+            def fallback &block
+              @fallback = block
+            end
+
+            def perform! opts = {}
+              if @ctn.pv_utility
+                @enabled.call(@ctn.pv_utility)
+                raise(NoCommandError, "no command was provided, set `pv.cmd = mycmd' in the enabled callback") if @cmd.blank?
+                execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                  _perform_with_wrapper
+                end
+              else
+                execute(opts.slice(:tries), &@fallback)
+              end
+            end
+
+            def _perform_with_wrapper
+              @state = :working
+              target_thread = Thread.current
+              channel, @result = @ctn.nonblocking_channel_result(cmd, channel: true, use_sh: true)
+
+              killer = @worker.app.spawn_thread(:sklaventreiber_worker_io_pv_killer) do |thr|
+                thr[:current_task] = target_thread[:current_task] if target_thread[:current_task]
+                loop do
+                  if @worker.should_cancel && !thr[:canceled]
+                    if channel.is_a?(Net::SSH::Connection::Channel)
+                      if channel[:pty]
+                        channel.send_data("\C-c") rescue false
+                      elsif channel[:pid]
+                        @ctn.kill_remote_process(channel[:pid])
+                      end
+                    end
+                    channel.close rescue false
+                    thr[:canceled] = true
+                  end
+                  break unless channel.active?
+                  thr.wait(0.1)
+                end
+              end
+
+              @result.each_linex do |grp, line|
+                next unless grp == :stderr
+                @offset = line.to_i
+              end
+              killer.signal.join
+            ensure
+              killer.kill
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/sftp_download.rb

@@ -0,0 +1,57 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class SftpDownload < Base
+            UnknownEventError = Class.new(::RuntimeError)
+            attr_reader :downloader
+
+            def init
+              @label = "downloading"
+              @entity = "download"
+              @throughput.categories << :inet << :inet_down
+            end
+
+            def reset_state
+              super
+              @downloader = nil
+            end
+
+            def download! opts = {}
+              opts = opts.reverse_merge(tries: 3, read_size: @read_size, force_new_connection: true)
+              prepare_local_destination
+              execute(opts.slice(:tries).merge(sleep_error: 3)) do
+                @ctn.sftp_start(opts[:force_new_connection]) do |sftp|
+                  @filesize = sftp.lstat!(@remote).size
+                  sftp.download!(@remote, @local, read_size: opts[:read_size], requests: 1) do |event, downloader, *args|
+                    if !@closing && @abort_if.call(self, event, downloader, *args)
+                      downloader.abort!
+                      @closing = true
+                    end
+
+                    case event
+                    when :open
+                      @downloader = downloader
+                      @state = :init
+                    when :get
+                      @state = :downloading
+                      @offset = args[1] + args[2].length
+                      GC.start if @offset % GC_FORCE_RATE == 0
+                    when :close
+                      @state = :finishing
+                    when :finish
+                      @state = :done
+                    else
+                      raise UnknownEventError, "unknown event `#{event}'"
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/io/throughput.rb

@@ -0,0 +1,219 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module IO
+          class Throughput
+            InstanceAlreadyRegisteredError = Class.new(::ArgumentError)
+
+            attr_reader :sklaventreiber, :stats
+
+            def initialize sklaventreiber
+              @sklaventreiber = sklaventreiber
+              @instances = {}
+              @stats = {}
+              @monitor = Monitor.new
+              @polling = []
+            end
+
+            def app
+              sklaventreiber.app
+            end
+
+            def sync &block
+              @monitor.synchronize(&block)
+            end
+
+            def poll! instance
+              sync { @polling.push(instance) }
+            end
+
+            def nopoll! instance
+              sync { @polling.delete(instance) }
+            end
+
+            def start_loop
+              @poll = app.spawn_thread(:sklaventreiber_throughput) do |thr|
+                thr[:polling] = 0
+                loop do
+                  sync {
+                    thr[:polling] = @polling.length
+                    @polling.each(&:ping)
+                  }
+                  break if thr[:stop]
+                  thr.wait(0.1)
+                end
+              end
+            end
+
+            def stop_loop
+              sync do
+                return unless @poll
+                @poll[:stop] = true
+              end
+              @poll.signal.join
+            end
+
+            def commit! bytes, *categories
+              sync do
+                return unless bytes
+                categories.flatten.each do |cat|
+                  @stats[cat] ||= 0
+                  @stats[cat] += bytes
+                end
+              end
+            end
+
+            def register target
+              sync do
+                if @instances[target]
+                  raise InstanceAlreadyRegisteredError, "throughput manager cannot register more than once on the same target: `#{target}'"
+                else
+                  raise NotImplementedError, "throughput manager requires the target to respond_to?(:filesize)" unless target.respond_to?(:filesize)
+                  raise NotImplementedError, "throughput manager requires the target to respond_to?(:offset)" unless target.respond_to?(:offset)
+                  @instances[target] = Instance.new(self, target)
+                end
+              end
+            end
+
+            def unregister instance
+              sync do
+                @instances.clone.each do |k, v|
+                  if v == instance
+                    @instances.delete(k)
+                    break
+                  end
+                end
+              end
+            end
+
+            class Instance
+              attr_reader :ioop, :categories, :sopts
+
+              def initialize manager, ioop
+                @manager = manager
+                @ioop = ioop
+                @monitor = Monitor.new
+                @pauses = 0
+                @categories = [:total]
+                @tracking = []
+                @tracking_offset = 0
+                @sopts = { perc_modifier: 1, perc_base: 0, tracking: 5.seconds }
+                reset_stats
+              end
+
+              def self.expose what, &how
+                define_method(what) do |*args, &block|
+                  sync { instance_exec(*args, &how) }
+                end
+              end
+
+              [:filesize, :offset].each do |m|
+                define_method(m) {|*a| @ioop.send(m, *a) }
+              end
+
+              [:human_bytes, :human_percentage, :human_seconds, :human_seconds2].each do |m|
+                define_method(m) {|*a| @manager.app.send(m, *a) }
+              end
+
+              def sync &block
+                @monitor.synchronize(&block)
+              end
+
+              def commit!
+                sync do
+                  ping
+                  return unless offset
+                  @manager.commit!(offset, @categories)
+                end
+              end
+
+              def pause!
+                sync do
+                  return if @pause_started
+                  @pause_started = Time.current
+                end
+              end
+
+              def unpause!
+                sync do
+                  return unless @pause_started
+                  @pauses += Time.current - @pause_started
+                  @pause_started = false
+                end
+              end
+
+              def ping
+                sync do
+                  return unless @started_at
+                  @stats[:bps_avg] = runtime.zero? ? 0 : (offset.to_d / runtime.to_d).to_i
+                  @stats[:eta2] = @stats[:bps_avg].zero? ? -1 : (bytes_remain.to_d / @stats[:bps_avg].to_d).to_i
+
+                  # eta tracking
+                  od = @tracking.last ? offset - @tracking_offset - @tracking.sum{|t,o| o }.to_d : 0
+                  @tracking << [Time.current, od] if !od.zero? || @tracking.empty?
+                  while @tracking.any? && @tracking.first[0] < @sopts[:tracking].ago
+                    @tracking_offset += @tracking.shift[1]
+                  end
+
+                  range = @tracking.any? ? @tracking.last[0] - @tracking.first[0] : 0
+                  @stats[:bps] = range.zero? ? 0 : @tracking.sum{|t,o| o }.to_d / range.to_d
+                  @stats[:eta] = @stats[:bps].zero? ? -1 : (bytes_remain.to_d / @stats[:bps].to_d).to_i
+                end
+              end
+
+              def unregister
+                @manager.unregister(self)
+              end
+
+              def reset_stats
+                sync do
+                  @tracking_offset = 0
+                  @tracking.clear
+                  @stats = { eta: -1, eta2: -1, bps: 0, bps_avg: 0 }
+                end
+              end
+
+              # =======
+              # = API =
+              # =======
+              expose(:eta) { ping; @stats[:eta] }
+              expose(:eta2) { ping; @stats[:eta2] }
+              expose(:bps) { ping; @stats[:bps] }
+              expose(:bps_avg) { ping; @stats[:bps_avg] }
+              expose(:done_percentage) { @sopts[:perc_base].to_d + @sopts[:perc_modifier].to_d * (filesize == 0 ? 100 : offset == 0 ? 0 : (offset.to_d / filesize.to_d * 100.to_d)) }
+              expose(:remain_percentage) { 100.to_d - done_percentage }
+              expose(:bytes_remain) { filesize - offset }
+              expose(:pauses) { @pauses + (@pause_started ? Time.current - @pause_started : 0) }
+              expose(:runtime) { @started_at ? (@ended_at || Time.current) - @started_at - pauses : 0 }
+              expose(:f_byte_progress) { "#{f_offset}/#{f_filesize}" }
+
+              [:bps, :bps_avg, :done_percentage, :remain_percentage, :bytes_remain, :offset, :filesize].each do |m|
+                expose(:"f_#{m}") { human_bytes(send(m)) }
+              end
+              [:done_percentage, :remain_percentage].each do |m|
+                expose(:"f_#{m}") { human_percentage(send(m)) }
+              end
+              [:runtime].each do |m|
+                expose(:"f_#{m}") { human_seconds(send(m)) }
+              end
+              [:eta, :eta2].each do |m|
+                expose(:"f_#{m}") { r = send(m); r == -1 ? "?:¿?:¿?" : human_seconds2(send(m)) }
+              end
+
+              def measure &block
+                @manager.poll!(self)
+                @started_at = Time.current
+                block.call(self)
+              ensure
+                @ended_at = Time.current
+                @manager.nopoll!(self)
+                commit!
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/routines.rb

@@ -0,0 +1,313 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module Routines
+          def _r_dump_file
+            @status = ["dumping table to remote file...", "yellow"]
+
+            pv_wrap(@ctn, nil) do |pv|
+              pv.enabled do |pvbinary|
+                pv.filesize = -1
+                pv.label = "dumping table"
+                pv.entity = "table dump"
+                pv.status_format = app.opts[:status_format]
+                pv.mode = :nofs
+                @status = [pv, "yellow"]
+                pv.abort_if { @should_cancel }
+
+                @remote_file_raw_tmp, pv.cmd = var.dump_to_remote_command(self, pvbinary)
+              end
+
+              pv.fallback do
+                @remote_file_raw_tmp, (channel, result) = var.dump_to_remote(self, false)
+                second_progress(channel, "dumping table to remote file (:seconds)...").join
+              end
+
+              pv.on_complete do
+                @remote_files_to_remove << @remote_file_raw_tmp
+              end
+
+              pv.perform!
+            end
+            _cancelpoint
+
+            # check if response has any sort of errors and abort
+            # if result.any?
+            #   r = result.join
+            #   if m = r.match(/(Unknown column '(.+)') in .+ \(([0-9]+)\)/i)
+            #     @status = ["[DUMP] Failed: #{m[1]} (#{m[3]})", :red]
+            #     throw :abort_execution, true
+            #   end
+            # end
+
+            @remote_file_raw = @remote_file_raw_tmp[0..-5]
+            ctn.sftp_start do |sftp|
+              # rename tmp file
+              sftp.rename!(@remote_file_raw_tmp, @remote_file_raw)
+
+              # save size for gzip progress
+              @remote_file_raw_filesize = sftp.lstat!(@remote_file_raw).size
+            end
+
+            @remote_files_to_remove.delete(@remote_file_raw_tmp)
+            @remote_files_to_remove << @remote_file_raw
+          end
+
+          def _r_calculate_raw_hash
+            @status = ["calculating integrity hash for raw file...", "yellow"]
+
+            pv_wrap(@ctn, nil) do |pv|
+              pv.enabled do |pvbinary|
+                pv.filesize = @remote_file_raw_filesize
+                pv.label = "hashing raw file"
+                pv.entity = "hashing raw file"
+                pv.status_format = app.opts[:status_format]
+                @status = [pv, "yellow"]
+                pv.abort_if { @should_cancel }
+                pv.cmd = ctn.calculate_remote_integrity_hash_command(@remote_file_raw, pvbinary)
+              end
+
+              pv.fallback do
+                cmd, (channel, pv.result) = ctn.calculate_remote_integrity_hash(@remote_file_raw, false)
+                second_progress(channel, "calculating integrity hash for raw file (:seconds)...").join
+              end
+
+              pv.on_success do
+                @integrity = { raw: pv.result.for_group(:stdout).join.split(" ").first.try(:strip).presence }
+              end
+
+              pv.perform!
+            end
+          end
+
+          def _r_compress_file
+            @status = ["compressing file for transfer...", "yellow"]
+
+            pv_wrap(@ctn, nil) do |pv|
+              pv.enabled do |pvbinary|
+                pv.filesize = @remote_file_raw_filesize
+                pv.label = "compressing"
+                pv.entity = "compress"
+                pv.status_format = app.opts[:status_format]
+                @status = [pv, "yellow"]
+                pv.abort_if { @should_cancel }
+                @remote_file_compressed, pv.cmd = var.compress_file_command(@remote_file_raw, pvbinary)
+                @remote_files_to_remove << @remote_file_compressed
+              end
+
+              pv.fallback do
+                @remote_file_compressed, (channel, result) = var.compress_file(@remote_file_raw, false)
+                @remote_files_to_remove << @remote_file_compressed
+                second_progress(channel, "compressing file for transfer (:seconds)...").join
+              end
+
+              pv.on_success do
+                @remote_files_to_remove.delete(@remote_file_raw)
+              end
+
+              pv.perform!
+            end
+          end
+
+          def _r_calculate_compressed_hash
+            @status = ["calculating integrity hash for compressed file...", "yellow"]
+
+            pv_wrap(@ctn, nil) do |pv|
+              pv.enabled do |pvbinary|
+                pv.filesize = @remote_file_raw_filesize
+                pv.label = "hashing compressed file"
+                pv.entity = "hashing compressed file"
+                pv.status_format = app.opts[:status_format]
+                @status = [pv, "yellow"]
+                pv.abort_if { @should_cancel }
+                pv.cmd = ctn.calculate_remote_integrity_hash_command(@remote_file_compressed, pvbinary)
+              end
+
+              pv.fallback do
+                cmd, (channel, pv.result) = ctn.calculate_remote_integrity_hash(@remote_file_compressed, false)
+                second_progress(channel, "calculating integrity hash for compressed file (:seconds)...").join
+              end
+
+              pv.on_success do
+                @integrity[:compressed] = pv.result.for_group(:stdout).join.split(" ").first.try(:strip).presence
+              end
+
+              pv.perform!
+            end
+          end
+
+          def _l_download_file
+            @status = ["initiating download...", "yellow"]
+            @local_file_compressed = local_tmp_file(File.basename(@remote_file_compressed))
+            @local_files_to_remove << @local_file_compressed
+
+            sftp_download(@ctn, @remote_file_compressed => @local_file_compressed) do |dl|
+              dl.status_format = app.opts[:status_format]
+              @status = [dl, "yellow"]
+              dl.abort_if { @should_cancel }
+              dl.download!
+            end
+          end
+
+          def _l_verify_compressed_hash
+            return unless @integrity[:compressed]
+            label = "verifying compressed file"
+            @status = ["#{label}...", :yellow]
+
+            file_shasum(@ctn, @local_file_compressed) do |fc|
+              fc.label = label
+              fc.sha = ctn.integrity_sha
+              fc.status_format = app.opts[:status_format]
+              @status = [fc, "yellow"]
+
+              fc.abort_if { @should_cancel }
+              fc.on_success do
+                @integrity[:compressed_local] = fc.result
+              end
+              fc.verify!
+            end
+
+
+            if !@should_cancel && @integrity[:compressed] != @integrity[:compressed_local]
+              @status = ["[INTEGRITY] downloaded compressed file corrupted! (remote: #{@integrity[:compressed]}, local: #{@integrity[:compressed_local]})", :red]
+              throw :abort_execution, true
+            end
+          end
+
+          def _l_copy_file file = nil
+            label = "copying #{var.copies_file_compressed? ? "gzipped" : "raw"} file"
+            @status = ["#{label}...", :yellow]
+
+            @copy_file_source = var.copies_file_compressed? ? @local_file_compressed : @local_file_raw
+            @copy_file_target = copy_file_destination(var.data["file"])
+
+            file_copy(@ctn, @copy_file_source => @copy_file_target) do |fc|
+              fc.label = label
+              fc.status_format = app.opts[:status_format]
+              @status = [fc, "yellow"]
+              fc.abort_if { @should_cancel }
+              fc.copy!
+            end
+          end
+
+          def _l_decompress_file
+            label = "decompressing file"
+            @status = ["#{label}...", :yellow]
+
+            file_gunzip(@ctn, @local_file_compressed) do |fc|
+              fc.filesize = @remote_file_raw_filesize
+              fc.label = label
+              fc.status_format = app.opts[:status_format]
+              @status = [fc, "yellow"]
+
+              fc.abort_if { @should_cancel }
+              fc.on_success do
+                @local_files_to_remove.delete(@local_file_compressed)
+                @local_file_raw = fc.local
+                @local_files_to_remove << @local_file_raw
+              end
+              fc.gunzip!
+            end
+          end
+
+          def _l_verify_raw_hash
+            return unless @integrity[:raw]
+            label = "verifying raw file"
+            @status = ["#{label}...", :yellow]
+
+            file_shasum(@ctn, @local_file_raw) do |fc|
+              fc.label = label
+              fc.sha = ctn.integrity_sha
+              fc.status_format = app.opts[:status_format]
+              @status = [fc, "yellow"]
+
+              fc.abort_if { @should_cancel }
+              fc.on_success do
+                @integrity[:raw_local] = fc.result
+              end
+              fc.verify!
+            end
+
+            if !@should_cancel && @integrity[:raw] != @integrity[:raw_local]
+              @status = ["[INTEGRITY] extracted raw file corrupted! (remote: #{@integrity[:raw]}, local: #{@integrity[:raw_local]})", :red]
+              throw :abort_execution, true
+            end
+          end
+
+          def _l_import_file
+            if File.size(@local_file_raw) > app.opts[:deferred_threshold] && app.opts[:deferred_import]
+              @deferred = true
+              @perform << "l_wait_for_workers"
+            else
+              # cancel!("importing not yet implemented", true)
+              _do_import_file
+            end
+          end
+
+          def _l_wait_for_workers
+            @perform << "l_import_file_deferred"
+            wait_defer_ready
+          end
+
+          def _l_import_file_deferred
+            @status = ["importing #{human_bytes(File.size(@local_file_raw))} SQL data into local server...", :yellow]
+            _do_import_file(@local_file_raw)
+          end
+
+          def _do_import_file
+            @status = ["importing #{human_bytes(File.size(@local_file_raw))} SQL data into local server...", :yellow]
+
+            imp = @var.data["importer"]
+            impf = @var.parse_flags(var.data["importer_flags"]).merge(deferred: @deferred)
+
+            if imp == "void10"
+              t = app.channelfy_thread app.spawn_thread(:sklaventreiber_worker_io_import_sql) {|thr| thr.wait(10) }
+              second_progress(t, "importing with void10, sleeping 10 seconds (:seconds)...").join
+            elsif imp == "sequel" || @var.constraint(table)
+              raise NotImplementedError, "SequelImporter is not yet implemented/ported to new db_sucker version!"
+              #     # imp_was_sequel = imp == "sequel"
+              #     # imp = "sequel"
+              #     # t = app.channelfy_thread Thread.new {
+              #     #   Thread.current[:importer] = imp = SequelImporter.new(worker, file, ignore_errors: !imp_was_sequel)
+              #     #   imp.start
+              #     # }
+              #   var.load_local_file(self, file) do |importer, channel|
+              #     case importer
+              #       when "sequel"
+              #         sequel_progress(channel).join
+              #         if channel[:importer].error
+              #           @status = ["importing with Sequel", :yellow]
+              #           raise channel[:importer].error
+              #         end
+              #       else second_progress(channel, "#{"(deferred) " if deferred}loading file (#{human_filesize(File.size(file))}) into local SQL server (:seconds)...").join
+              #     end
+              #     throw :abort_execution, channel[:error_message] if channel[:error_message]
+              #     @return_message = channel[:return_message] if channel[:return_message]
+              #   end
+            elsif imp == "binary"
+              t = app.channelfy_thread app.spawn_thread(:sklaventreiber_worker_io_import_sql) {|thr|
+                begin
+                  file_import_sql(@ctn, :instruction) do |fi|
+                    @status = [fi, "yellow"]
+                    fi.instruction = @var.import_instruction_for(@local_file_raw, impf)
+                    fi.filesize = File.size(@local_file_raw)
+                    fi.status_format = app.opts[:status_format]
+                    fi.abort_if { @should_cancel }
+                    fi.import!
+                  end
+                rescue Worker::IO::FileImportSql::ImportError => ex
+                  fail! "ImportError: #{ex.message}"
+                  sleep 3
+                end
+              }
+            else
+              raise ImporterNotFoundError, "variation `#{cfg.name}/#{name}' defines unknown importer `#{imp}' (in `#{cfg.src}')"
+            end
+            t.join
+          end
+        end
+      end
+    end
+  end
+end