db_sucker 3.0.0

data/lib/db_sucker/application/sklaven_treiber/worker.rb

@@ -0,0 +1,48 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        SlotPoolNotInitializedError = Class.new(::RuntimeError)
+        ChannelFailRetryError = Class.new(::RuntimeError)
+
+        include Core
+        include Accessors
+        include Helpers
+        include Routines
+
+        attr_reader :exception, :ctn, :var, :table, :thread, :monitor, :step, :perform, :should_cancel, :sklaventreiber, :timings, :sshing
+        OutputHelper.hook(self)
+
+        def initialize sklaventreiber, ctn, var, table
+          @sklaventreiber = sklaventreiber
+          @ctn = ctn
+          @var = var
+          @table = table
+          @monitor = Monitor.new
+          @timings = {}
+          @deferred = false
+          @spinner_frames = sklaventreiber.window.try(:spinner_frames).try(:dup) || []
+          @current_perform = :unknown
+          @perform = %w[].tap do |perform|
+            perform << "r_dump_file"
+            perform << "r_calculate_raw_hash" if ctn.integrity?
+            perform << "r_compress_file"
+            perform << "r_calculate_compressed_hash" if ctn.integrity?
+            perform << "l_download_file"
+            perform << "l_verify_compressed_hash" if ctn.integrity?
+            perform << "l_copy_file" if var.copies_file? && var.copies_file_compressed?
+            if var.requires_uncompression?
+              perform << "l_decompress_file"
+              perform << "l_verify_raw_hash" if ctn.integrity?
+              perform << "l_copy_file" if var.copies_file? && !var.copies_file_compressed?
+              perform << "l_import_file" if var.data["database"]
+            end
+          end
+
+          @state = :pending
+          @status = ["waiting...", "gray"]
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber.rb

@@ -0,0 +1,281 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      attr_reader :app, :trxid, :window, :data, :status, :monitor, :workers, :poll, :throughput, :slot_pools
+
+      def initialize app, trxid
+        @app = app
+        @trxid = trxid
+        @status = ["initializing", "gray"]
+        @monitor = Monitor.new
+        @workers = []
+        @threads = []
+        @slot_pools = {}
+        @sleep_before_exit = 0
+        @throughput = Worker::IO::Throughput.new(self)
+
+        @data = {
+          database: nil,
+          tables_transfer: nil,
+          tables_transfer_list: [],
+          tables_total: nil,
+          tables_done: 0,
+        }
+      end
+
+      def sync
+        @monitor.synchronize { yield }
+      end
+
+      def pause_worker worker
+        sync { worker.pause }
+      end
+
+      def unpause_worker worker
+        sync { worker.unpause }
+      end
+
+      def pause_all_workers
+        sync { @workers.each {|wrk| pause_worker(wrk) } }
+      end
+
+      def unpause_all_workers
+        sync { @workers.each {|wrk| unpause_worker(wrk) } }
+      end
+
+      def spooled
+        stdout_was = app.opts[:stdout]
+        app.opts[:stdout] = SklavenTreiber::LogSpool.new(stdout_was) if app.opts[:window_enabled]
+        yield if block_given?
+      ensure
+        app.opts[:stdout].spooldown do |meth, args, time|
+          stdout_was.send(meth, *args)
+        end if app.opts[:stdout].respond_to?(:spooldown)
+        app.opts[:stdout] = stdout_was
+      end
+
+      def whip_it! ctn, var
+        @ctn, @var = ctn, var
+
+        _start_ssh_poll
+        _init_window
+        _check_remote_tmp_directory
+        _select_tables
+        _initialize_slot_pools
+        _initialize_workers
+        @ctn.pv_utility # lazy load
+        @poll[:force] = false
+        @throughput.start_loop
+
+        @sleep_before_exit = 3 if @window
+        _run_consumers
+      ensure
+        app.sandboxed do
+          @status = ["terminating (canceling workers)", "red"]
+          @workers.each {|w| catch(:abort_execution) { w.cancel! } }
+        end
+        app.sandboxed do
+          @status = ["terminating (SSH poll)", "red"]
+          if @poll
+            @poll[:force] = false
+            @poll.join
+          end
+        end
+        @status = ["terminated", "red"]
+        sleep @sleep_before_exit
+        app.sandboxed { @window.try(:stop) }
+        app.sandboxed { @ctn.try(:sftp_end) }
+        app.sandboxed { @throughput.try(:stop_loop) }
+        app.sandboxed { @slot_pools.each{|n, p| p.close! } }
+        app.sandboxed do
+          app.puts @window.try(:_render_final_results)
+        end
+        @ctn, @var = nil, nil
+      end
+
+      def _init_window
+        return unless app.opts[:window_enabled]
+        @window = Window.new(app, self)
+        @window.init!
+        @window.start
+      end
+
+      def _check_remote_tmp_directory
+        @status = ["checking remote temp directory", "blue"]
+        @ctn.sftp_begin
+        @ctn.sftp_start do |sftp|
+          # check tmp directory
+          app.debug "Checking remote temp directory #{app.c @ctn.tmp_path, :magenta}"
+          begin
+            sftp.dir.glob("#{@ctn.tmp_path}", "**/*")
+          rescue Net::SFTP::StatusException => ex
+            if ex.message["no such file"]
+              app.abort "Temp directory `#{@ctn.tmp_path}' does not exist on the remote side!", 2
+            else
+              raise
+            end
+          end
+        end
+      end
+
+      def _select_tables
+        @status = ["selecting tables for transfer", "blue"]
+        ttt, at = @var.tables_to_transfer
+
+        # apply only/except filters provided via command line
+        if @app.opts[:suck_only].any? && @app.opts[:suck_except].any?
+          raise OptionParser::InvalidArgument, "only one of `--only' or `--except' option can be provided at the same time"
+        elsif @app.opts[:suck_only].any?
+          unless (r = @app.opts[:suck_only] - at).empty?
+            raise Container::TableNotFoundError, "table(s) `#{r * ", "}' for the database `#{@ctn.source["database"]}' could not be found (provided via --only, variation `#{@ctn.name}/#{@var.name}' in `#{@ctn.src}')"
+          end
+          ttt = @app.opts[:suck_only]
+        elsif @app.opts[:suck_except].any?
+          unless (r = @app.opts[:suck_except] - at).empty?
+            raise Container::TableNotFoundError, "table(s) `#{r * ", "}' for the database `#{@ctn.source["database"]}' could not be found (provided via --except, variation `#{@ctn.name}/#{@var.name}' in `#{@ctn.src}')"
+          end
+          ttt = ttt - @app.opts[:suck_except]
+        end
+
+        @data[:database] = @ctn.source["database"]
+        @data[:tables_transfer] = ttt.length
+        @data[:tables_transfer_list] = ttt
+        @data[:window_col1] = ttt.map(&:length).max
+        @data[:tables_total] = at.length
+      end
+
+      def _initialize_slot_pools
+        app.opts[:slot_pools].each do |name, slots|
+          @slot_pools[name] = SlotPool.new(slots, name)
+        end
+      end
+
+      def _initialize_workers
+        @status = ["initializing workers 0/#{@data[:tables_transfer]}", "blue"]
+
+        @data[:tables_transfer_list].each_with_index do |table, index|
+          @status = ["initializing workers #{index+1}/#{@data[:tables_transfer]}", "blue"]
+          @workers << Worker.new(self, @ctn, @var, table)
+        end
+      end
+
+      def _start_ssh_poll
+        wait_lock = Queue.new
+        @poll = app.spawn_thread(:sklaventreiber_ssh_poll) do |thr|
+          thr[:force] = true
+          thr[:iteration] = 0
+          thr[:errors] = 0
+          wait_lock << true
+          begin
+            @ctn.loop_ssh(0.1) {
+              thr[:iteration] += 1
+              thr[:last_iteration] = Time.current
+              thr[:force] || @workers.select{|w| !w.done? || w.sshing }.any?
+            }
+          rescue Container::SSH::ChannelOpenFailedError
+            thr[:errors] += 1
+            sleep 0.5
+            retry
+          end
+
+          if thr[:errors].zero?
+            app.debug "SSH error count (#{thr[:errors]})"
+          elsif thr[:errors] > 25
+            app.warning "SSH error count (#{thr[:errors]}) is high! Verify remote MaxSessions setting or lower concurrent worker count."
+          else
+            app.warning "SSH errors occured (#{thr[:errors]})! Verify remote MaxSessions setting or lower concurrent worker count."
+          end
+        end
+        wait_lock.pop
+        sleep 0.01 until @poll[:iteration] && @poll[:iteration] > 0
+      end
+
+      def _run_consumers
+        cnum = [app.opts[:consumers], @data[:tables_transfer]].min
+        @data[:window_col2] = cnum.to_s.length
+        if cnum <= 1
+          _run_in_main_thread
+        else
+          _run_in_threads(cnum)
+        end
+      end
+
+      def _run_in_main_thread
+        @status = ["running in main thread...", "green"]
+
+        # control thread
+        ctrlthr = app.spawn_thread(:sklaventreiber_worker_ctrl) do |thr|
+          loop do
+            _control_thread
+            break if thr[:stop]
+            thr.wait(0.1)
+          end
+        end
+
+        begin
+          Thread.current[:managed_worker] = :main
+          _queueoff
+        ensure
+          ctrlthr[:stop] = true
+          ctrlthr.signal.join
+        end
+      end
+
+      def _run_in_threads(cnum)
+        @status = ["starting consumer 0/#{cnum}", "blue"]
+
+        # initializing consumer threads
+        cnum.times do |wi|
+          @status = ["starting consumer #{wi+1}/#{cnum}", "blue"]
+          @threads << app.spawn_thread(:sklaventreiber_worker) {|thr|
+            begin
+              thr[:managed_worker] = wi
+              thr.wait(0.1) until thr[:start] || $core_runtime_exiting
+              _queueoff
+            rescue Interrupt
+            end
+          }
+        end
+
+        # start consumer threads
+        @status = ["running", "green"]
+        @threads.each{|t| t[:start] = true; t.signal }
+
+        # master thread (control)
+        while @threads.any?(&:alive?)
+          _control_thread
+          Thread.current.wait(0.1)
+        end
+        @threads.each(&:join)
+      end
+
+      def _queueoff
+        loop do
+          return if $core_runtime_exiting
+          worker = false
+          sync do
+            pending = @workers.select(&:pending?)
+            return unless pending.any?
+            worker = pending.first.aquire(Thread.current)
+          end
+          if worker
+            begin
+              worker.run
+            ensure
+              sync { @data[:tables_done] += 1 }
+            end
+          end
+        end
+      end
+
+      def _control_thread
+        if $core_runtime_exiting && $core_runtime_exiting < 100
+          $core_runtime_exiting += 100
+          app.sandboxed { @workers.each {|w| catch(:abort_execution) { w.cancel! } } }
+          app.sandboxed { @slot_pools.each{|n, p| p.softclose! } }
+          app.wakeup_handlers
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/slot_pool.rb

@@ -0,0 +1,137 @@
+module DbSucker
+  class Application
+    class SlotPool
+      SlotAllocationError = Class.new(::RuntimeError)
+      PoolAlreadyClosedError = Class.new(SlotAllocationError)
+
+      def initialize slots = 1, name = nil
+        @name = name
+        @slots = slots
+        @monitor = Monitor.new
+        @closed = false
+        @softclosed = false
+        @waiting = []
+        @active = []
+        @signal = @monitor.new_cond
+        @closed_signal = @monitor.new_cond
+      end
+
+      def self.expose what, &how
+        define_method(what) do |*args, &block|
+          sync { instance_exec(*args, &how) }
+        end
+      end
+
+      def sync &block
+        @monitor.synchronize(&block)
+      end
+
+      expose(:name) { @name }
+      expose(:active) { @active }
+      expose(:active?) { @active.any? }
+      expose(:waiting) { @waiting }
+      expose(:waiting?) { @waiting.any? }
+      expose(:closed?) { @closed }
+      expose(:softclosed?) { @softclosed }
+      expose(:slots) { @slots }
+      expose(:available_slots) { @slots ? @slots - @active.length : 1.0/0 }
+      expose(:slots?) { @slots ? available_slots > 0 : true }
+
+      def close
+        sync do
+          @closed = true
+          @signal.broadcast
+        end
+        true
+      end
+
+      def close!
+        sync do
+          close
+          @closed_signal.wait if active?
+        end
+      end
+
+      def softclose!
+        @softclosed = true
+        dequeue_waiting!
+      end
+
+      def dequeue_waiting!
+        sync do
+          while @waiting.any?
+            _wthr, _tthr = @waiting.shift
+            _wthr.signal
+          end
+          @signal.broadcast
+        end
+      end
+
+      def qindex thr = nil
+        thr ||= Thread.current
+        sync do
+          index = @waiting.find_index {|wthr, tthr| tthr == thr }
+          index ? index + 1 : false
+        end
+      end
+
+      def puts *a
+        Thread.main[:app].puts(*a)
+      end
+
+      def aquired? tthr = nil
+        @active.include?(tthr || Thread.current)
+      end
+
+      def wait_aquired tthr = nil
+        tthr ||= Thread.current
+        #tthr.wait(0.1) until qindex(tthr)
+        loop do
+          sync do
+            #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> wait for index"
+            #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> has #{available_slots} slots"
+            while slots? && @waiting.any?
+              _wthr, _tthr = @waiting.shift
+              #puts "<#{Time.current.to_f}-#{_tthr[:current_task]}> running now"
+              @active.push(_tthr) unless @softclosed
+              _tthr.signal
+              _wthr.signal
+            end
+            unless qindex(tthr)
+              #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> return"
+              return
+            end
+            #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> wait"
+            @signal.wait #(1)
+            #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> wait DONE"
+          end
+        end
+      end
+
+      def aquire tthr = nil
+        wthr = Thread.current
+        tthr ||= Thread.current
+        sync do
+          raise PoolAlreadyClosedError, "slot pool has already been closed, cannot aquire slot" if closed?
+          @waiting << [wthr, tthr]
+          #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> broadcasting signal after adding new waiter"
+          @signal.broadcast # signal polling threads
+        end
+        tthr.signal # signal target thread to continue and poll
+        wthr.wait # suspend thread until we aquired it
+        true
+      end
+
+      def release tthr = nil
+        sync do
+          tthr ||= Thread.current
+          ai = @active.delete(tthr)
+          return unless ai
+          #puts "<#{Time.current.to_f}-#{tthr[:current_task]}> broadcasting signal"
+          @signal.broadcast
+          @closed_signal.broadcast if @active.empty? && closed?
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/tie.rb

@@ -0,0 +1,25 @@
+module DbSucker
+  class Application
+    class Tie
+      def self.descendants
+        @descendants ||= []
+      end
+
+      # Descendant tracking for inherited classes.
+      def self.inherited(descendant)
+        descendants << descendant
+      end
+
+      def self.hook_all! app
+        descendants.uniq.each do |klass|
+          app.debug "[AppTie] Loading apptie `#{klass.name}'"
+          klass.hook!(app)
+        end
+      end
+
+      def self.hook! app
+        raise NotImplementedError, "AppTies must implement class method `.hook!(app)'!"
+      end
+    end
+  end
+end

data/lib/db_sucker/application/window/core.rb

@@ -0,0 +1,185 @@
+module DbSucker
+  class Application
+    class Window
+      module Core
+        UnknownSpinnerError = Class.new(::RuntimeError)
+        SPINNERS = {
+          arrows: "←↖↑↗→↘↓↙",
+          blocks: "▁▃▄▅▆▇█▇▆▅▄▃",
+          blocks2: "▉▊▋▌▍▎▏▎▍▌▋▊▉",
+          blocks3: "▖▘▝▗",
+          blocks4: "▌▀▐▄",
+          forks: "┤┘┴└├┌┬┐",
+          triangles: "◢◣◤◥",
+          trbl_square: "◰◳◲◱",
+          trbl_circle: "◴◷◶◵",
+          circle_half: "◐◓◑◒",
+          circle_quarter: "◜◝◞◟",
+          circle_quarter2: "╮╯╰╭",
+          unix: "|/-\\",
+          bomb: ".oO@*",
+          eye: "◡⊙◠",
+          diamond: "◇◈◆",
+        }
+
+        def start
+          @keypad.start_loop
+          start_window_loop
+        end
+
+        def stop
+          stop_window_loop
+          @keypad.stop_loop
+          close_screen
+          app.debug "Leaving curses screen mode"
+        end
+
+        def start_window_loop
+          @loop = app.spawn_thread(:window_draw_loop) do |thr|
+            loop do
+              break if thr[:stop] && (@view == :status || @force_kill)
+              refresh_screen if app.opts[:window_draw]
+              thr.wait(app.opts[:window_refresh_delay])
+            end
+          end
+        end
+
+        def stop_window_loop
+          return unless @loop
+          @loop[:stop] = true
+          @loop.signal.join
+        end
+
+        def choose_spinner
+          spinner = app.opts[:window_spinner]
+          spinner = SPINNERS.keys.sample if spinner == :random
+          if s = SPINNERS[spinner]
+            @spinner_frames = s.split("").reverse.freeze
+          else
+            raise UnknownSpinnerError, "The spinner `#{spinner}' does not exist, use :random or one of: #{SPINNERS.keys * ", "}"
+          end
+        end
+
+        def init!
+          app.debug "Entering curses screen mode"
+          init_screen
+          nl
+          if @app.opts[:window_keypad]
+            raw
+            nonl
+            noecho
+            cbreak
+            stdscr.keypad = true
+            set_cursor 0
+          end
+
+          # colors
+          start_color
+          use_default_colors
+          [:COLOR_BLACK, :COLOR_RED, :COLOR_GREEN, :COLOR_YELLOW, :COLOR_BLUE, :COLOR_MAGENTA, :COLOR_CYAN, :COLOR_WHITE].each do |cl|
+            c = Window.const_get(cl)
+            init_pair(c, c, -1)
+          end
+          init_pair(Window::COLOR_GRAY, 0, -1)
+        end
+
+        def set_cursor visibility
+          curs_set(visibility)
+        end
+
+        def force_cursor line, col = 0
+          if line.nil?
+            @force_cursor = nil
+          else
+            @force_cursor = [line, col]
+          end
+        end
+
+        def update
+          clear
+          @x_offset = 0
+          @line = -1
+          yield if block_given?
+          next_line
+          setpos(*@force_cursor) if @force_cursor
+          refresh
+        end
+
+        def line l = 1
+          setpos(l - 1, @x_offset)
+        end
+
+        def next_line
+          @line += 1
+          setpos(@line, @x_offset)
+        end
+
+        def change_view new_view
+          view_was = @view
+          if block_given?
+            begin
+              @view = new_view
+              yield
+            ensure
+              @view = view_was
+            end
+          else
+            @view = new_view
+            view_was
+          end
+        end
+
+        def progress_bar perc, opts = {}
+          opts = opts.reverse_merge({
+            width: cols - stdscr.curx - 3,
+            prog_open: "[",
+            prog_open_color: "yellow",
+            prog_done: "=",
+            prog_done_color: "green",
+            prog_current: ">",
+            prog_current_color: "yellow",
+            prog_remain: ".",
+            prog_remain_color: "gray",
+            prog_close: "]",
+            prog_close_color: "yellow",
+          })
+          pdone = (opts[:width].to_d * (perc.to_d / 100.to_d)).ceil.to_i
+          prem  = opts[:width] - pdone
+          pcur  = 0
+          if perc < 100
+            pdone.zero? ? (prem -= 1) : (pdone -= 1)
+            pcur  += 1
+          end
+
+          send(opts[:prog_open_color], " #{opts[:prog_open]}")
+          send(opts[:prog_done_color], "".ljust(pdone, opts[:prog_done])) unless pdone.zero?
+          send(opts[:prog_current_color], "".ljust(pcur, opts[:prog_current])) unless pcur.zero?
+          send(opts[:prog_remain_color], "".ljust(prem, opts[:prog_remain])) unless prem.zero?
+          send(opts[:prog_close_color], "#{opts[:prog_close]}")
+        end
+
+        def dialog &block
+          Dialog.new(self, &block)
+        end
+
+        def dialog! &block
+          dialog(&block).render!
+        end
+
+        # colors
+        [:red, :blue, :yellow, :cyan, :magenta, :gray, :green, :white].each do |c|
+          define_method(c) do |*args, &block|
+            color = Window.const_get "COLOR_#{c.to_s.upcase}"
+            attron(color_pair(color)|Window::A_NORMAL) do
+              if block
+                block.call
+              else
+                args.each {|a| addstr(a) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end