db_sucker 3.0.0

data/lib/db_sucker/application/window.rb

@@ -0,0 +1,329 @@
+module DbSucker
+  class Application
+    class Window
+      include Core
+      include Curses
+      COLOR_GRAY = 8
+      COL1 = 20
+      COL2 = 25
+      COL3 = 20
+      OutputHelper.hook(self)
+
+      attr_reader :app, :sklaventreiber, :keypad, :tick, :spinner_frames
+      attr_accessor :view, :x_offset, :force_kill
+
+      def initialize app, sklaventreiber
+        @app = app
+        @keypad = Keypad.new(self)
+        @sklaventreiber = sklaventreiber
+        @monitor = Monitor.new
+        @line = 0
+        @tick = 0
+        @view = :status
+        @force_kill = false
+        choose_spinner
+      end
+
+      def refresh_screen
+        Thread.current[:last_render_duration] = rt = Benchmark.realtime do
+          @monitor.synchronize do
+            @tick += 1
+            update { __send__(:"_view_#{@view}") }
+          end
+        end
+        if rt > 0.020
+          @app.warning "window render took: #{"%.6f" % rt}"
+        else
+          @app.debug "window render took: #{"%.6f" % rt}", 125
+        end
+      rescue StandardError => ex
+        @app.notify_exception("DbSucker::Window encountered an render error on tick ##{@tick}", ex)
+
+        update do
+          next_line
+          red "RenderError occured!"
+          next_line
+          red "#{ex.class}: #{ex.message}"
+          ex.backtrace.each do |l|
+            next_line
+            red("    #{l}")
+          end
+        end
+        Thread.current.wait(1)
+      end
+
+      def _view_help
+        _render_status(threads: false, started: false, trxid: false, database: false)
+
+        #next_line; next_line
+        #magenta "db_sucker"
+        #blue " #{VERSION}"
+        #yellow " – "
+        #cyan "(C) 2016-#{Time.current.year} Sven Pachnit (bmonkeys.net)"
+
+        #next_line
+        #gray "Released under the MIT license."
+
+        next_line; next_line
+        blue "Key Bindings (case sensitive):"
+        next_line
+
+        Keypad::HELP_INFO[:key_bindings].each do |key, desc|
+          next_line
+          magenta "    #{key}"
+          yellow "  #{desc}"
+        end
+
+        next_line; next_line
+        blue "Main prompt commands:"
+        next_line
+
+        # only build output once and save it in memory
+        @_view_help_memory ||= begin
+          mchp = Keypad::HELP_INFO[:main_commands].map do |aliases, options, desc|
+            [].tap do |result|
+              result << [].tap{|r|
+                aliases.each do |al|
+                  r << (al.is_a?(Array) ? al.join("").length + 4 : al.length + 2)
+                end
+              }.sum
+
+              result << [].tap{|r|
+                options.each do |type, name|
+                  r << (type == :mandatory ? "<#{[*name] * "|"}> " : "[#{[*name] * "|"}] ").length
+                end
+              }.sum
+            end
+          end
+          columns = { aliases: mchp.map(&:first).max, options: mchp.map(&:second).max }
+
+          # render commands
+          [].tap do |instruct|
+            Keypad::HELP_INFO[:main_commands].each do |aliases, options, desc|
+              # aliases
+              instruct << [:next_line]
+              instruct << [:addstr, "    "]
+              cl = 0
+              aliases.each do |al|
+                instruct << [:magenta, ":"]
+                if al.is_a?(Array)
+                  instruct << [:blue, "#{al[0]}"]
+                  instruct << [:gray, "("]
+                  instruct << [:cyan, "#{al[1]}"]
+                  cl += al.join("").length + 4
+                  instruct << [:gray, ") "]
+                else
+                  cl += al.length + 2
+                  instruct << [:blue, "#{al} "]
+                end
+              end
+              instruct << [:addstr, "".ljust(columns[:aliases] - cl, " ")]
+
+              # options
+              cl = 0
+              instruct << [:addstr, " "]
+              options.each do |type, name|
+                if type == :mandatory
+                  instruct << [:red, "<"]
+                  [*name].each_with_index do |n, i|
+                    instruct << [:gray, "|"] if i > 0
+                    instruct << [:blue, n]
+                  end
+                  instruct << [:red, "> "]
+                else
+                  instruct << [:yellow, "["]
+                  [*name].each_with_index do |n, i|
+                    instruct << [:gray, "|"] if i > 0
+                    instruct << [:cyan, n]
+                  end
+                  instruct << [:yellow, "] "]
+                end
+                cl += [*name].join("").length + 3 + [*name].length - 1
+              end
+              instruct << [:addstr, "".ljust(columns[:options] - cl, " ")]
+
+              instruct << [:yellow, " #{desc}"]
+            end
+          end
+        end
+
+        @_view_help_memory.each do |a|
+          send(*a)
+        end
+
+        @keypad.prompt.render(self, lines-1)
+      end
+
+      def _view_log
+        _render_status(threads: false, started: false, trxid: false, database: false)
+        next_line
+        if app.opts[:stdout].is_a?(SklavenTreiber::LogSpool)
+          limit = [lines - @line - 1, app.opts[:stdout].spool.length].min
+          app.opts[:stdout].spool[-limit..-1].each do |m, l, t|
+            ts = "[#{t}] "
+            gray(ts)
+            white decolorize(l.join(" "))[0..(cols - ts.length)]
+            next_line
+          end
+        else
+          red "Log spooling is not enabled, can't show log entries!"
+        end
+        @keypad.prompt.render(self, lines-1)
+      end
+
+      def _view_status
+        _render_status
+        _render_workers
+        @keypad.prompt.render(self, lines-1)
+      end
+
+      def _render_status opts = {}
+        opts = opts.reverse_merge(status: true, threads: true, started: true, trxid: true, database: true, progress: true)
+
+        if opts[:status]
+          next_line
+          yellow "        Status: "
+          send(sklaventreiber.status[1].presence || :blue, sklaventreiber.status[0])
+          # shutdown message
+          if $core_runtime_exiting && sklaventreiber.status[0] != "terminated"
+            red " (HALTING … please wait)"
+          end
+        end
+
+        if opts[:threads]
+          next_line
+          yellow "       Threads: "
+          blue "#{Thread.list.length} ".ljust(COL1, " ")
+        end
+
+        if opts[:started]
+          next_line
+          yellow "       Started: "
+          blue "#{@app.boot}"
+          yellow " ("
+          blue human_seconds(Time.current - app.boot)
+          yellow ")"
+        end
+
+        if opts[:trxid]
+          next_line
+          yellow "Transaction ID: "
+          cyan sklaventreiber.trxid
+        end
+
+        if opts[:database]
+          next_line
+          yellow "      Database: "
+          magenta sklaventreiber.data[:database] || "?"
+          yellow " (transfering "
+          blue "#{sklaventreiber.data[:tables_transfer] || "?"}"
+          yellow " of "
+          blue "#{sklaventreiber.data[:tables_total] || "?"}"
+          yellow " tables)"
+        end
+
+        if opts[:progress]
+          next_line
+          total, done = sklaventreiber.data[:tables_transfer], sklaventreiber.data[:tables_done]
+          perc = total && done ? f_percentage(done, total) : "?"
+          yellow "      Progress: "
+          green perc
+          yellow " – "
+          blue "#{done}/#{total || "?"} workers done"
+        end
+      end
+
+      def _render_workers
+        if sklaventreiber.workers.any?
+          next_line
+          limit = lines - @line - 3 - (@keypad.prompt.active? ? 1 : 0) # @l starting at 0, 1 for blank line to come, placeholder
+          enum = sklaventreiber.workers.sort_by{|w| [w.priority, w.table] }
+          enum.each_with_index do |w, i|
+            # limit reached and more than one entry to come?
+            if i > limit && (enum.length - i - 1) > 0
+              next_line
+              rest = enum[i..-1]
+              part = rest.group_by(&:state).map do |k, v|
+                "#{v.length} #{k}"
+              end
+              yellow "… #{rest.length} more [#{part.join(", ")}]"
+              break
+            end
+            _render_worker_line(w)
+          end
+        end
+      end
+
+      def _render_worker_line worker
+        next_line
+        col1 = sklaventreiber.data[:window_col1]
+        col2 = sklaventreiber.data[:window_col2]
+
+        # status icon
+        case worker.state
+          when :pending then gray("⊙")
+          when :pausing, :paused then gray("♨")
+          when :aquired then white("⊙")
+          when :done then green("✔")
+          when :failed then red("✘")
+          when :canceled then red("⊘")
+          when :running then yellow("#{worker.spinner_frame}")
+        end
+
+        # table_name
+        send(worker.should_cancel ? :red : worker.paused? ? :gray : :magenta, " #{worker.table}".ljust(col1 + 1, " "))
+        gray " | "
+
+        # status
+        if worker.step
+          cyan "[#{worker.step}/#{worker.perform.length}] "
+        end
+        if worker.status[0].respond_to?(:to_curses)
+          worker.status[0].to_curses(self)
+        else
+          send(worker.status[1].presence || :blue, decolorize("#{worker.status[0]}"))
+        end
+      end
+
+      # used to render everything before exiting, can't fucking dump the pads I tried to implement -.-"
+      def _render_final_results
+        t_db, t_total, t_done = sklaventreiber.data[:database], sklaventreiber.data[:tables_transfer], sklaventreiber.data[:tables_done]
+        perc = t_total && t_done ? f_percentage(t_done, t_total) : "?"
+
+        puts
+        puts c("        Status: ") << c(sklaventreiber.status[0], sklaventreiber.status[1].presence || "red")
+        puts c("       Threads: ") << c("#{Thread.list.length} ".ljust(COL1, " "), :blue)
+        puts c("       Started: ") << c("#{@app.boot}", :blue) << c(" (") << c(human_seconds(Time.current - app.boot), :blue) << c(")")
+        puts c("Transaction ID: ") << c("#{sklaventreiber.trxid}", :cyan)
+        puts c("      Database: ") << c(t_db || "?", :magenta) << c(" (transferred ") << c(t_total || "?", :blue) << c(" of ") << c(t_done || "?", :blue) << c(" tables)")
+        puts c("      Progress: ") << c(perc, :green) << c(" – ") << c(t_total || "?", :blue) << c("#{t_done}/#{t_total || "?"} workers done", :blue)
+
+        if sklaventreiber.workers.any?
+          puts
+          enum = sklaventreiber.workers.sort_by{|w| [w.priority, w.table] }
+          enum.each do |worker|
+            col1 = sklaventreiber.data[:window_col1]
+            col2 = sklaventreiber.data[:window_col2]
+
+            puts "".tap{|res|
+              # status icon
+              res << case worker.state
+                when :done then c("✔", :green)
+                when :failed then c("✘", :red)
+                when :canceled then c("⊘", :red)
+                else "#{worker.state.inspect}"
+              end
+              # table
+              res << c(" #{worker.table}".ljust(col1 + 1, " "), :magenta) << c(" | ", :black)
+              # steps
+              res << c("[#{worker.step}/#{worker.perform.length}] ", :cyan) if worker.step
+              # status
+              res << c(worker.status[0], worker.status[1].presence || :blue)
+            }
+          end
+          puts
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application.rb

@@ -0,0 +1,168 @@
+module DbSucker
+  class Application
+    attr_reader :opts, :cfg, :sklaventreiber, :boot
+    include Core
+    include Colorize
+    include OutputHelper
+    include Dispatch
+    GC_FORCE_RATE = 25*1024*1024
+
+    # main dispatch routine for application
+    def self.dispatch *a
+      new(*a) do |app|
+        begin
+          app.signalify_thread(Thread.main)
+          Thread.main[:app] = app
+          app.load_appconfig
+          app.parse_params
+          app.debug "Running with PID #{Process.pid}"
+          app.debug "Runtime: #{RUBY_DESCRIPTION}"
+          app.dispatch
+          app.haltpoint
+        rescue Interrupt
+          app.abort("Interrupted", 1)
+        rescue OptionParser::ParseError => ex
+          app.fire(:core_exception, ex)
+          app.abort("#{ex.message}", false)
+          app.log app.c("Run `#{$0} --help' for more info", :blue)
+          exit 1
+        rescue StandardError => ex
+          app.fire(:core_exception, ex)
+          app.warn app.c("[FATAL] #{ex.class}: #{ex.message}", :red)
+          ex.backtrace.each do |l|
+            app.warn app.c("\t#{l}", :red)
+          end
+          app.abort case ex
+            when Container::TableNotFoundError then ex.message
+            else "Unhandled exception terminated application!"
+          end
+        ensure
+          app.fire(:core_shutdown)
+          remain = Thread.list.length
+          if remain > 1
+            app.warning "#{remain} threads remain (should be 1)..."
+          else
+            app.debug "1 thread remains..."
+          end
+          Thread.main[:app] = nil unless app.opts[:debug]
+        end
+      end
+    end
+
+    def initialize env, argv
+      @boot = Time.current
+      @env, @argv = env, argv
+      @hooks = {}
+      @monitor = Monitor.new
+      @output_monitor = Monitor.new
+      @cfg = ContainerCollection.new(self)
+      @opts = {
+        dispatch: :index,                # (internal) action to dispatch
+        mode: :default,                  # (internal) mode for action
+        check_for_updates: true,         # -z flag
+        colorize: true,                  # --monochrome flag
+        debug: false,                    # -d flag
+        stdout: STDOUT,                  # (internal) STDOUT redirect
+        pipein: ARGF,                    # (internal) INPUT redirect
+
+        list_databases: false,           # --list-databases flag
+        list_tables: false,              # --list-tables flag
+        suck_only: [],                   # --only flag
+        suck_except: [],                 # --except flag
+        simulate: false,                 # --simulate flag
+        deferred_import: true,           # -n flag
+        deferred_threshold: 50_000_000,  # 50 MB
+
+        # features
+        status_format: :full,            # used for IO operations, can be one of: none, minimal, full
+        pv_enabled: true,                # disable pv utility autodiscovery (force non-usage)
+
+        # sklaven treiber
+        window_enabled: true,            # if disabled effectively disables any status progress or window drawing
+        window_draw: true,               # wether to refresh screen or not
+        window_refresh_delay: 0.25,      # refresh screen every so many seconds
+        window_keypad: true,             # allow keyboard controls
+        window_spinner: :circle_quarter, # change spinner... why is this configurable?
+        consumers: 10,                   # amount of workers to run at the same time
+
+        # thread priorities (-3..+3)
+        tp_window_draw_loop: -3,
+        tp_window_keypad_loop: +2,
+        tp_sklaventreiber_ssh_poll: +3,
+        tp_sklaventreiber_throughput: +2,
+        tp_sklaventreiber_worker: -1,
+        tp_sklaventreiber_worker_ctrl: -1,
+        tp_sklaventreiber_worker_slot_progress: -2,
+        tp_sklaventreiber_worker_second_progress: -2,
+        tp_sklaventreiber_worker_io_pv_killer: -2,
+        tp_sklaventreiber_worker_io_import_sql: +3,
+
+        # used to open core dumps (should be a blocking call, e.g. `subl -w' or `mate -w')
+        # MUST be windowed! vim, nano, etc. will not work!
+        core_dump_editor: "subl -w",
+
+        # amount of workers that can use a slot (false = infinite)
+        # you can create as many pools as you want and use them in `routine_pools' setting
+        slot_pools: {
+          all: false,
+          remote: false,
+          download: false,
+          local: false,
+          import: 3,
+          deferred: 1,
+        },
+
+        # assign tasks to certain slot pools
+        routine_pools: {
+          r_dump_file: [:all, :remote],
+          r_calculate_raw_hash: [:all, :remote],
+          r_compress_file: [:all, :remote],
+          r_calculate_compressed_hash: [:all, :remote],
+          l_download_file: [:all, :download],
+          l_verify_compressed_hash: [:all, :local],
+          l_copy_file: [:all, :local],
+          l_decompress_file: [:all, :local],
+          l_verify_raw_hash: [:all, :local],
+          l_import_file: [:all, :local, :import],
+          l_wait_for_workers: [], # does nothing, no need for any slots
+          l_import_file_deferred: [:all, :local, :deferred],
+        }
+      }
+      init_params
+      Tie.hook_all!(self)
+      yield(self)
+    end
+
+    def init_params
+      @optparse = OptionParser.new do |opts|
+        opts.banner = "Usage: db_sucker [options] [identifier [variation]]"
+
+        opts.separator("\n" << "# Application options")
+        opts.on(      "--new NAME", String, "Generates new container config in #{core_cfg_path}") {|v| @opts[:dispatch] = :generate_config; @opts[:config_name] = v }
+        opts.on("-a", "--action ACTION", String, "Dispatch given action") {|v| @opts[:dispatch] = v }
+        opts.on("-m", "--mode MODE", String, "Dispatch action with given mode") {|v| @opts[:mode] = v.to_sym }
+        opts.on("-n", "--no-deffer", "Don't use deferred import for files > 50 MB SQL data size.") { @opts[:deferred_import] = false }
+        opts.on("-l", "--list-databases", "List databases for given identifier.") { @opts[:list_databases] = true }
+        opts.on("-t", "--list-tables [DATABASE]", String, "List tables for given identifier and database.", "If used with --list-databases the DATABASE parameter is optional.") {|s| @opts[:list_tables] = s || :all }
+        opts.on("-o", "--only table,table2", Array, "Only suck given tables. Identifier is required, variation is optional (defaults to default).", "WARNING: ignores ignore_always option") {|s| @opts[:suck_only] = s }
+        opts.on("-e", "--except table,table2", Array, "Don't suck given tables. Identifier is required, variation is optional (defaults to default).") {|s| @opts[:suck_except] = s }
+        opts.on("-c", "--consumers NUM=10", Integer, "Maximal amount of tasks to run simultaneously") {|n| @opts[:consumers] = n }
+        opts.on(      "--stat-tmp", "Show information about the remote temporary directory.", "If no identifier is given check local temp directory instead.") { @opts[:dispatch] = :stat_tmp }
+        opts.on(      "--cleanup-tmp", "Remove all temporary files from db_sucker in target directory.") { @opts[:dispatch] = :cleanup_tmp }
+        opts.on(      "--simulate", "To use with --cleanup-tmp to not actually remove anything.") { @opts[:simulate] = true }
+
+        opts.separator("\n" << "# General options")
+        opts.on("-d", "--debug [lvl=1]", Integer, "Enable debug output") {|l| @opts[:debug] = l || 1 }
+        opts.on("--monochrome", "Don't colorize output (does not apply to curses)") { @opts[:colorize] = false }
+        opts.on("--no-window", "Disables curses window alltogether (no progress)") { @opts[:window_enabled] = false }
+        opts.on("-h", "--help", "Shows this help") { @opts[:dispatch] = :help }
+        opts.on("-v", "--version", "Shows version and other info") { @opts[:dispatch] = :info }
+        opts.on("-z", "Do not check for updates on GitHub (with -v/--version)") { @opts[:check_for_updates] = false }
+      end
+    end
+
+    def parse_params
+      @optparse.parse!(@argv)
+    end
+  end
+end