db_sucker 3.0.0

data/lib/db_sucker/application/evented_resultset.rb

@@ -0,0 +1,149 @@
+module DbSucker
+  class Application
+    class EventedResultset
+      SetAlreadyClosedError = Class.new(::RuntimeError)
+      include Enumerable
+
+      def initialize
+        @store = []
+        @monitor = Monitor.new
+        @closed = false
+        @close_signal = @monitor.new_cond
+        @value_signal = @monitor.new_cond
+      end
+
+      def sync &block
+        @monitor.synchronize(&block)
+      end
+
+      def enq data, group = nil
+        sync do
+          raise SetAlreadyClosedError, "failed to enqueue data: resultset is already closed" if closed?
+          @store << [group.try(:to_sym), data]
+          @value_signal.broadcast
+        end
+      end
+
+      def push *args
+        sync do
+          args.each {|a| enq(a) }
+          @store
+        end
+      end
+      alias_method :<<, :push
+
+      def close!
+        sync do
+          @closed = true
+          @value_signal.broadcast
+          @close_signal.broadcast
+        end
+        true
+      end
+
+      def closed?
+        sync { @closed }
+      end
+
+      def empty?
+        sync { @store.empty? }
+      end
+
+      def wait
+        sync do
+          return if closed?
+          @close_signal.wait
+        end
+        true
+      end
+
+      def for_group group
+        @store.map{|grp, data| grp == group.try(:to_sym) ? data : nil }.compact
+      end
+
+      def each &block
+        wait
+        if block
+          @store.each do |group, data|
+            block.call(data)
+          end
+        else
+          @store.map(&:second)
+        end
+      end
+
+      def join *a
+        wait
+        each.join(*a)
+      end
+
+      def [] which
+        wait
+        @store[which].try(:last)
+      end
+
+      def eachx &block
+        wait
+        @store.each(&block)
+      end
+
+      def each_line &block
+        Thread.current[self.object_id.to_s] = nil
+        loop do
+          data = gets
+          if !data
+            break if closed?
+            next
+          end
+          block.call(data)
+        end
+      ensure
+        Thread.current[self.object_id.to_s] = nil
+      end
+
+      def each_linex &block
+        Thread.current[self.object_id.to_s] = nil
+        loop do
+          group, entry = getx
+          unless entry
+            break if closed?
+            next
+          end
+          block.call(group, entry)
+        end
+      ensure
+        Thread.current[self.object_id.to_s] = nil
+      end
+
+      def gets
+        sync do
+          Thread.current[self.object_id.to_s] ||= -1
+          if !closed? && !@store[Thread.current[self.object_id.to_s]+1]
+            @value_signal.wait
+          end
+          if @store[Thread.current[self.object_id.to_s]+1]
+            Thread.current[self.object_id.to_s] += 1
+            @store[Thread.current[self.object_id.to_s]].try(:last)
+          else
+            false
+          end
+        end
+      end
+
+      def getx
+        sync do
+          Thread.current[self.object_id.to_s] ||= -1
+          if !closed? && !@store[Thread.current[self.object_id.to_s]+1]
+            @value_signal.wait
+          end
+          if @store[Thread.current[self.object_id.to_s]+1]
+            Thread.current[self.object_id.to_s] += 1
+            @store[Thread.current[self.object_id.to_s]]
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/fake_channel.rb

@@ -0,0 +1,22 @@
+module DbSucker
+  class Application
+    class FakeChannel
+      def initialize &block
+        @storage = {}
+        @termination = block
+      end
+
+      def [] k
+        @storage[k]
+      end
+
+      def []= k, v
+        @storage[k] = v
+      end
+
+      def alive?
+        !@termination.try(:call, self)
+      end
+    end
+  end
+end

data/lib/db_sucker/application/output_helper.rb

@@ -0,0 +1,197 @@
+module DbSucker
+  class Application
+    module OutputHelper
+      def self.hook ctx
+        [:puts, :print, :warn, :debug, :log, :warning, :error, :abort, :decolorize, :db_table_listing, :print_db_table_list, :decolorize, :render_table, :f_percentage, :human_bytes, :human_number, :human_percentage, :human_seconds, :human_seconds2, :rll, :c, :strbool].each do |meth|
+          ctx.__send__(:define_method, meth) do |*a|
+            Thread.main[:app].__send__(meth, *a)
+          end
+        end
+      end
+
+      def puts *a
+        outsync { @opts[:stdout].send(:puts, *a) }
+      end
+
+      def print *a
+        outsync { @opts[:stdout].send(:print, *a) }
+      end
+
+      def warn *a
+        outsync { @opts[:stdout].send(:warn, *a) }
+      end
+
+      def debug msg, lvl = 1
+        puts c("[DEBUG] #{msg}", :black) if @opts[:debug] && @opts[:debug] >= lvl
+      end
+
+      def log msg
+        puts c("#{msg}")
+      end
+
+      def warning msg
+        warn c("[WARN] #{msg}", :red)
+      end
+
+      def error msg
+        warn c("[ERROR] #{msg}", :red)
+      end
+
+      def abort msg, exit_code = 1
+        warn c("[ABORT] #{msg}", :red)
+        exit(exit_code) if exit_code
+      end
+
+      def rll
+        print "\033[A"
+        print "\033[2K\r"
+      end
+
+      def render_table table, headers = []
+        [].tap do |r|
+          col_sizes = table.map{|col| col.map{|i| decolorize(i.to_s) }.map(&:length).max }
+          headers.map{|i| decolorize(i.to_s) }.map(&:length).each_with_index do |length, header|
+            col_sizes[header] = [col_sizes[header] || 0, length || 0].max
+          end
+
+          # header
+          if headers.any?
+            r << [].tap do |line|
+              col_sizes.count.times do |col|
+                line << headers[col].ljust(col_sizes[col] + (headers[col].length - decolorize(headers[col]).length))
+              end
+            end.join(" | ")
+            r << "".ljust(col_sizes.inject(&:+) + ((col_sizes.count - 1) * 3), "-")
+          end
+
+          # records
+          table[0].count.times do |row|
+            r << [].tap do |line|
+              col_sizes.count.times do |col|
+                line << "#{table[col][row]}".ljust(col_sizes[col] + (table[col][row].to_s.length - decolorize(table[col][row]).length))
+              end
+            end.join(" | ")
+          end
+        end
+      end
+
+      def db_table_listing col
+        col.each do |id, ccfg|
+          log ""
+          log "====================="
+          log "=== #{c id, :magenta}"
+          log "====================="
+          a, b = [], []
+          ccfg.variations.map do |name, vd|
+            a << c(name, :blue)
+            b << (vd.label.present? ? c(vd.label) : c("no label", :black))
+          end
+          render_table([a, b], [c("variation"), c("label")]).each{|l| log("#{l}") }
+        end
+      end
+
+      def print_db_table_list host, dbs
+        log ""
+        log c(host, :red)
+
+        dbs.each_with_index do |db, i|
+          if db.is_a?(Array)
+            d = c(db[0], :magenta) << c(" (#{db[1].length})", :black)
+            if i == dbs.count - 1
+              log("#{db[1].any? ? "├──" : "└──"} #{d}")
+            else
+              log("├── #{d}")
+            end
+
+            table = render_table([db[1].map{|r| c(r[1], :cyan) }, db[1].map{|r| c(r[0], :green) }])
+            table.each_with_index do |l, i2|
+              if i2 == table.count - 1
+                log("│   └── #{l}")
+              else
+                log("│   ├── #{l}")
+              end
+            end
+          else
+            log "  #{c db, :blue}"
+          end
+        end
+      end
+
+      def strbool v
+        v = true if ["true", "t", "1", "y", "yes", "on"].include?(v)
+        v = false if ["false", "f", "0", "n", "no", "off"].include?(v)
+        v
+      end
+
+      def f_percentage have, total, nn = 2
+        human_percentage(total == 0 ? 100 : have == 0 ? 0 : (have.to_d / total.to_d * 100.to_d), nn)
+      end
+
+      def human_bytes bytes
+        return false unless bytes
+        {
+          'B'  => 1024,
+          'KB' => 1024 * 1024,
+          'MB' => 1024 * 1024 * 1024,
+          'GB' => 1024 * 1024 * 1024 * 1024,
+          'TB' => 1024 * 1024 * 1024 * 1024 * 1024
+        }.each_pair { |e, s| return "#{"%.2f" % (bytes.to_d / (s / 1024)).round(2)} #{e}" if bytes < s }
+      end
+
+      def human_number(n)
+        n.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
+      end
+
+      def human_percentage(n, nn = 2)
+        "%.#{nn}f%%" % n
+      end
+
+      def human_seconds secs, sprec = 0
+        t_minute = 60
+        t_hour = t_minute * 60
+        t_day = t_hour * 24
+        t_week = t_day * 7
+        t_month = t_day * 30
+        t_year = t_month * 12
+        "".tap do |r|
+          if secs >= t_year
+            r << "#{"%i" % (secs / t_year)}y "
+            secs = secs % t_year
+          end
+
+          if secs >= t_month
+            r << "#{"%i" % (secs / t_month)}m "
+            secs = secs % t_month
+          end
+
+          if secs >= t_week
+            r << "#{"%i" % (secs / t_week)}w "
+            secs = secs % t_week
+          end
+
+          if secs >= t_day || !r.blank?
+            r << "#{"%i" % (secs / t_day)}d "
+            secs = secs % t_day
+          end
+
+          if secs >= t_hour || !r.blank?
+            r << "#{"%i" % (secs / t_hour)}h "
+            secs = secs % t_hour
+          end
+
+          if secs >= t_minute || !r.blank?
+            r << "#{"%i" % (secs / t_minute)}m "
+            secs = secs % t_minute
+          end
+
+          r << "#{"%.#{sprec}f" % secs.round(sprec)}s" unless r.include?("d")
+        end.strip
+      end
+
+      def human_seconds2 secs
+        return "?:¿?:¿?" if secs.try(:infinite?)
+        Time.at(secs.round).utc.strftime("%k:%M:%S").strip
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/log_spool.rb

@@ -0,0 +1,57 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class LogSpool
+        attr_reader :original, :spool
+
+        def initialize original
+          @original = original
+          @enabled = true
+          @spool = []
+          @monitor = Monitor.new
+        end
+
+        def sync
+          @monitor.synchronize { yield }
+        end
+
+        def enable
+          sync do
+            @enabled = true
+          end
+        end
+
+        def clear
+          sync { @spool.clear }
+        end
+
+        def disable void_spool = false, &block
+          sync do
+            @enabled = false
+            (void_spool && clear) || (block && spooldown(&block))
+          end
+        end
+
+        def spooldown
+          sync do
+            while e = @spool.shift
+              yield(e + [original])
+            end
+          end
+        end
+
+        def puts *args
+          sync { @enabled ? (@spool << [:puts, args, Time.current]) : @original.send(:puts, *args) }
+        end
+
+        def print *args
+          sync { @enabled ? (@spool << [:print, args, Time.current]) : @original.send(:print, *args) }
+        end
+
+        def warn *args
+          sync { @enabled ? (@spool << [:warn, args, Time.current]) : @original.send(:warn, *args) }
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/accessors.rb

@@ -0,0 +1,105 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module Accessors
+          def pending?
+            @state == :pending
+          end
+
+          def done?
+            succeeded? || failed? || canceled?
+          end
+
+          def sshing?
+            @sshing
+          end
+
+          def failed?
+            @state == :failed
+          end
+
+          def succeeded?
+            @state == :done
+          end
+
+          def canceled?
+            @state == :canceled
+          end
+
+          def running?
+            @state == :running
+          end
+
+          def paused?
+            @state == :paused
+          end
+
+          def pausing?
+            @state == :pausing
+          end
+
+          def deferred?
+            @deferred
+          end
+
+          def status
+            @status
+          end
+
+          def state
+            @state
+          end
+
+          def trxid
+            sklaventreiber.trxid
+          end
+
+          def app
+            sklaventreiber.app
+          end
+
+          def descriptive
+            "#{ctn.source["database"]}-#{table}"
+          end
+
+          def identifier
+            "#{trxid}_table"
+          end
+
+          def to_s
+            "#<#{self.class}:#{self.object_id}-#{descriptive}(#{@state})>"
+          end
+
+          def tmp_filename tmp_suffix = false
+            "#{ctn.tmp_path}/#{trxid}_#{ctn.source["database"]}_#{table}.dbsc#{".tmp" if tmp_suffix}"
+          end
+
+          def local_tmp_path
+            sklaventreiber.app.core_tmp_path
+          end
+
+          def local_tmp_file file
+            "#{local_tmp_path}/#{file}"
+          end
+
+          def spinner_frame
+            @spinner_frames.unshift(@spinner_frames.pop)[0]
+          end
+
+          def copy_file_destination dstfile
+            d, dt = Time.current.strftime("%Y-%m-%d"), Time.current.strftime("%H-%M-%S")
+
+            File.expand_path dstfile.dup
+              .gsub!(":combined", ":datetime_-_:table")
+              .gsub!(":datetime", "#{d}_#{dt}")
+              .gsub!(":date", d)
+              .gsub!(":time", dt)
+              .gsub!(":table", table)
+              .gsub!(":id", sklaventreiber.trxid)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/sklaven_treiber/worker/core.rb

@@ -0,0 +1,168 @@
+module DbSucker
+  class Application
+    class SklavenTreiber
+      class Worker
+        module Core
+          def sync
+            @monitor.synchronize { yield }
+          end
+
+          def aquire thread
+            @thread = thread
+            thread[:current_task] = descriptive
+            thread[:current_worker] = self
+            if m = thread[:managed_worker]
+              debug "Consumer thread ##{m} aquired worker #{descriptive}"
+            else
+              debug "Main thread aquired worker #{descriptive}"
+            end
+            @status = ["initializing...", "gray"]
+            @state = :aquired
+            self
+          end
+
+          def pause wait = false
+            sync do
+              return if done?
+              return if @state == :pausing || @state == :paused
+              @pause_data = { state_was: @state, signal: @monitor.new_cond }
+              if @state == :pending
+                @state = :paused
+              else
+                @state = :pausing
+                @pause_data[:signal].wait if wait
+              end
+            end
+          end
+
+          def _pausepoint
+            sync do
+              return if !(@state == :pausing || @state == :paused)
+              return unless @pause_data
+              return unless @thread == Thread.current
+              @state = :paused
+              @pause_data[:signal].broadcast
+            end
+            @thread[:paused] = true
+            Thread.stop
+            @thread[:paused] = false
+            _cancelpoint
+          end
+
+          def unpause
+            sync do
+              return if !(@state == :pausing || @state == :paused)
+              return unless @pause_data
+              @state = @pause_data[:state_was]
+              @pause_data = false
+              @thread.wakeup if @thread
+            end
+          end
+
+          def cancel! reason = nil, now = false
+            return if done?
+            @should_cancel = reason || true
+            unpause
+            sync { _cancelpoint(reason) if pending? || now }
+          end
+
+          def fail! reason, now = false
+            @status = ["FAILED(#{@current_perform}) #{reason}", "red"]
+            @state = :failed
+            throw :abort_execution, true if now
+          end
+
+          def _cancelpoint reason = nil
+            if @should_cancel
+              reason ||= @should_cancel if @should_cancel.is_a?(String)
+              reason ||= @status[0]
+              @should_cancel = false
+              @state = :canceled
+              @status = ["CANCELED#{" (was #{reason.to_s.gsub("[CLOSING] ", "")})" if reason}", "red"]
+              throw :abort_execution, true
+              true
+            elsif @state == :failed
+              throw :abort_execution, true
+              true
+            end
+            _pausepoint
+          end
+
+          def priority
+            100 - ({
+              running: 50,
+              aquired: 50,
+              pausing: 50,
+              paused: 30,
+              canceled: 35,
+              pending: 30,
+              failed: 20,
+              done: 10,
+            }[@state] || 0)
+          end
+
+          def run
+            @state = :running
+            @sshing = true
+            @started = Time.current
+            @download_state = { state: :idle, offset: 0 }
+            @remote_files_to_remove = []
+            @local_files_to_remove = []
+            @current_perform = nil
+
+            app.fire(:worker_routine_before_all, self)
+            catch :abort_execution do
+              perform.each_with_index do |m, i|
+                @current_perform = m
+                _cancelpoint
+                @step = i + 1
+                r = catch(:abort_execution) {
+                  aquire_slots(*app.opts[:routine_pools][m.to_sym]) do
+                    begin
+                      r0 = Time.current
+                      app.fire(:worker_routine_before, self, @current_perform)
+                      send(:"_#{m}")
+                    ensure
+                      app.fire(:worker_routine_after, self, @current_perform)
+                      @timings[m] = Time.current - r0
+                    end
+                  end
+                  nil
+                }
+                throw :abort_execution if r
+                _cancelpoint
+              end
+              @status = ["DONE (#{runtime})", "green"]
+            end
+          rescue StandardError => ex
+            @exception = ex
+            fail! "#{ex.class}: #{ex.message}"
+            Thread.main[:app].notify_exception("SklavenTreiber::Worker encountered an error in `#{@current_perform}' (ctn: #{ctn.name}, var: #{var.name}, db: #{ctn.source["database"]}, table: #{table})", ex)
+          rescue Interrupt => ex
+            @state = :failed
+          ensure
+            # cleanup temp files
+            ctn.sftp_start do |sftp|
+              @remote_files_to_remove.each do |file|
+                sftp.remove!(file) rescue false
+              end
+            end if @remote_files_to_remove.any?
+
+            # cleanup local temp files
+            @local_files_to_remove.each do |file|
+              File.unlink(file) rescue false
+            end
+
+            app.fire(:worker_routine_after_all, self)
+            @state = :done if !canceled? && !failed?
+            @ended = Time.current
+            @sshing = false
+
+            # debug timings
+            debug "[Timings(#{table})] all: #{human_seconds(@timings.values.sum, 3)}, #{@timings.map{|a,t| "#{a}: #{human_seconds(t, 3)}" } * ", "}", 50
+          end
+        end
+      end
+    end
+  end
+end