db_sucker 3.0.0

data/lib/db_sucker/application/colorize.rb

@@ -0,0 +1,28 @@
+module DbSucker
+  class Application
+    module Colorize
+      UnknownColorError = Class.new(::ArgumentError)
+      COLORMAP = {
+        black: 30,
+        gray: 30,
+        red: 31,
+        green: 32,
+        yellow: 33,
+        blue: 34,
+        magenta: 35,
+        cyan: 36,
+        white: 37,
+      }
+
+      def colorize str, color = :yellow
+        ccode = COLORMAP[color.to_sym] || raise(UnknownColorError, "unknown color `#{color}'")
+        @opts[:colorize] ? "\e[#{ccode}m#{str}\e[0m" : "#{str}"
+      end
+      alias_method :c, :colorize
+
+      def decolorize str
+        str.to_s.gsub(/\e\[.*?(\d)+m/ , '')
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/accessors.rb

@@ -0,0 +1,60 @@
+module DbSucker
+  class Application
+    class Container
+      module Accessors
+        def ctn
+          self
+        end
+
+        def source
+          ctn.data["source"]
+        end
+
+        def tmp_path
+          source["ssh"]["tmp_location"].presence || "/tmp/db_sucker_tmp"
+        end
+
+        def variations
+          Hash[data["variations"].keys.map{|id| [id, variation(id)] }]
+        end
+
+        def variation v
+          return unless vd = data["variations"][v]
+          Variation.new(self, v, vd)
+        end
+
+        def integrity_binary
+          (source["integrity_binary"].nil? ? "shasum -ba" : source["integrity_binary"]).presence
+        end
+
+        def integrity_sha
+          source["integrity_sha"].nil? ? 512 : source["integrity_sha"]
+        end
+
+        def integrity?
+          !!integrity_sha
+        end
+
+        def ssh_key_files
+          @ssh_key_files ||= begin
+            files = [*data["source"]["ssh"]["keyfile"]].reject(&:blank?).map do |f|
+              if f.start_with?("~")
+                Pathname.new(File.expand_path(f))
+              else
+                Pathname.new(File.dirname(src)).join(f)
+              end
+            end
+            files.each do |f|
+              begin
+                File.open(f)
+              rescue Errno::ENOENT
+                warning("SSH identity file `#{f}' for identifier `#{name}' does not exist! (in `#{src}')")
+              end
+            end
+            files
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/ssh.rb

@@ -0,0 +1,225 @@
+module DbSucker
+  class Application
+    class Container
+      module SSH
+        CommandExecutionError = Class.new(::RuntimeError)
+        ChannelOpenFailedError = Class.new(::RuntimeError)
+
+        begin # SSH
+          def ssh_begin
+            debug "Opening SSH connection for identifier `#{name}'"
+            @ssh = ssh_start
+            begin
+              yield(@ssh)
+            ensure
+              ssh_end
+            end if block_given?
+          end
+
+          def ssh_end
+            return unless @ssh
+            debug "Closing SSH connection for identifier `#{name}'"
+            @ssh.try(:close) rescue false
+            debug "CLOSED SSH connection for identifier `#{name}'"
+            @ssh = nil
+          end
+
+          def ssh_sync &block
+            @ssh_mutex.synchronize(&block)
+          end
+
+          def ssh_start new_connection = false, &block
+            if @ssh && !new_connection
+              ssh_sync do
+                debug "Reusing SSH connection in start for identifier `#{name}'"
+                return block ? block.call(@ssh) : @ssh
+              end
+            end
+            debug "Opening new SSH connection in start for identifier `#{name}'"
+
+            opt = {}
+            opt[:user] = source["ssh"]["username"] if source["ssh"]["username"].present?
+            opt[:password] = source["ssh"]["password"] if source["ssh"]["password"].present?
+            opt[:keys] = ssh_key_files if ssh_key_files.any?
+            opt[:port] = source["ssh"]["port"] if source["ssh"]["port"].present?
+            if block
+              Net::SSH.start(source["ssh"]["hostname"], nil, opt) do |ssh|
+                block.call(ssh)
+              end
+            else
+              Net::SSH.start(source["ssh"]["hostname"], nil, opt)
+            end
+          end
+        end
+
+        begin # SFTP
+          def sftp_begin
+            debug "Opening SFTP connection for identifier `#{name}'"
+            @sftp = sftp_start
+            begin
+              yield(@sftp)
+            ensure
+              sftp_end
+            end if block_given?
+          end
+
+          def sftp_end
+            return unless @sftp
+            debug "Closing SFTP connection for identifier `#{name}'"
+            @sftp.try(:close) rescue false
+            debug "CLOSED SFTP connection for identifier `#{name}'"
+            @sftp = nil
+          end
+
+          def sftp_sync &block
+            @sftp_mutex.synchronize(&block)
+          end
+
+          def sftp_start new_connection = false, &block
+            if @sftp && !new_connection
+              sftp_sync do
+                debug "Reusing SFTP connection in start for identifier `#{name}'"
+                return block ? block.call(@sftp) : @sftp
+              end
+            end
+            debug "Opening new SFTP connection in start for identifier `#{name}'"
+
+            opt = {}
+            opt[:user] = source["ssh"]["username"] if source["ssh"]["username"].present?
+            opt[:password] = source["ssh"]["password"] if source["ssh"]["password"].present?
+            opt[:keys] = ssh_key_files if ssh_key_files.any?
+            opt[:port] = source["ssh"]["port"] if source["ssh"]["port"].present?
+            if block
+              Net::SFTP.start(source["ssh"]["hostname"], nil, opt) do |sftp|
+                block.call(sftp)
+              end
+            else
+              Net::SFTP.start(source["ssh"]["hostname"], nil, opt)
+            end
+          end
+        end
+
+        begin # SSH helpers
+          def loop_ssh *args, &block
+            return false unless @ssh
+            @ssh.loop(*args, &block)
+          end
+
+          def blocking_channel ssh = nil, result = nil, &block
+            waitlock = Queue.new
+            (ssh || ssh_start).open_channel do |ch|
+              waitlock.pop
+              block.call(ch)
+            end.tap do |ch|
+              # catch open_fail errors
+              ch.on_open_failed do |_ch, code, desc|
+                result.try(:close!)
+                _ch[:open_failed] = true
+                raise ChannelOpenFailedError, "#{code}: #{desc}"
+              end
+
+              ch[:wait_monitor] = Monitor.new
+              ch[:wait_condition] = ch[:wait_monitor].new_cond
+              st = app.sklaventreiber
+              waitlock << true
+              if !ssh && st && st.sync{ st.try(:poll) }
+                ch[:wait_monitor].synchronize do
+                  ch[:wait_condition].wait(0.1) while ch.active?
+                end
+              else
+                ch.wait
+              end
+            end
+          end
+
+          def nonblocking_channel ssh = nil, result = nil, &block
+            (ssh || ssh_start).open_channel do |ch|
+              ch[:wait_monitor] = Monitor.new
+              ch[:wait_condition] = ch[:wait_monitor].new_cond
+              block.call(ch)
+            end.tap do |ch|
+              # catch open_fail errors
+              ch.on_open_failed do |_ch, code, desc|
+                result.try(:close!)
+                _ch[:open_failed] = true
+                raise ChannelOpenFailedError, "#{code}: #{desc}"
+              end
+            end
+          end
+
+          def kill_remote_process pid, sig = :INT
+            ssh_start(true) do |ssh|
+              blocking_channel_result("kill -#{sig} -#{pid}", ssh: ssh)
+            end
+          end
+
+          def blocking_channel_result cmd, opts = {}
+            opts = opts.reverse_merge(ssh: nil, blocking: true, channel: false, request_pty: false, use_sh: false)
+            if opts[:use_sh]
+              cmd = %{/bin/sh -c 'echo $(ps -o pgid= $$ | grep -o [0-9]*) && #{cmd}'}
+              pid_monitor = Monitor.new
+              pid_signal = pid_monitor.new_cond
+            end
+            result = EventedResultset.new
+            chan = send(opts[:blocking] ? :blocking_channel : :nonblocking_channel, opts[:ssh], result) do |ch|
+              chproc = ->(ch, cmd, result) {
+                ch.exec(cmd) do |ch, success|
+                  Thread.main[:app].debug "START: #{Thread.current == Thread.main ? :main : Thread.current[:itype]}-#{cmd} #{success}"
+                  raise CommandExecutionError, "could not execute command" unless success
+
+                  # "on_data" is called when the process writes something to stdout
+                  ch.on_data do |c, data|
+                    Thread.main[:app].debug "#{Thread.current == Thread.main ? :main : Thread.current[:itype]}-#{Time.current.to_f}: STDOUT: #{data}".chomp
+                    if opts[:use_sh] && result.empty?
+                      ch[:pid] = data.to_i
+                      ch[:pid] = false if ch[:pid].zero?
+                      pid_monitor.synchronize { pid_signal.broadcast } if opts[:use_sh]
+                      next
+                    end
+                    result.enq(data, :stdout)
+                  end
+
+                  # "on_extended_data" is called when the process writes something to stderr
+                  ch.on_extended_data do |c, type, data|
+                    Thread.main[:app].debug "#{Thread.current == Thread.main ? :main : Thread.current[:itype]}-#{Time.current.to_f}: STDERR: #{data}".chomp
+                    result.enq(data, :stderr)
+                  end
+
+                  ch.on_request "exit-status" do |ch, data|
+                    Thread.main[:app].debug "#{Thread.current == Thread.main ? :main : Thread.current[:itype]}-#{Time.current.to_f}: EXIT: #{data.read_long} #{cmd}".chomp
+                  end
+
+                  ch.on_close do |ch|
+                    ch[:wait_monitor].synchronize { ch[:wait_condition].broadcast }
+                    Thread.main[:app].debug "#{Thread.current == Thread.main ? :main : Thread.current[:itype]}-#{Time.current.to_f}: CLOSED: #{cmd}".chomp
+                  end
+
+                  ch.on_eof do
+                    Thread.main[:app].debug "#{Thread.current == Thread.main ? :main : Thread.current[:itype]}-#{Time.current.to_f}: EOF: #{cmd}".chomp
+                    result.close!
+                    ch[:handler].try(:signal)
+                  end
+                end
+              }
+              if opts[:request_pty]
+                ch.request_pty do |ch, success|
+                  raise CommandExecutionError, "could not obtain pty" unless success
+                  ch[:pty] = true
+                  chproc.call(ch, cmd, result)
+                end
+              else
+                chproc.call(ch, cmd, result)
+              end
+            end
+            pid_monitor.synchronize { pid_signal.wait(1) if !chan[:pid] } if opts[:use_sh]
+            opts[:channel] ? [chan, result] : result
+          end
+
+          def nonblocking_channel_result cmd, opts = {}
+            blocking_channel_result(cmd, opts.merge(blocking: false))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/validations.rb

@@ -0,0 +1,53 @@
+module DbSucker
+  class Application
+    class Container
+      module Validations
+        def verify!
+          _verify("/", data, __keys_for(:root))
+
+          # validate source
+          if sd = data["source"]
+            _verify("/source", sd, __keys_for(:source))
+            if sd["ssh"]
+              _verify("/source/ssh", sd["ssh"], __keys_for(:source_ssh))
+              ssh_key_files
+            end
+          end
+
+          # validate variations
+          if sd = data["variations"]
+            sd.each do |name, vd|
+              begin
+                _verify("/variations/#{name}", vd, __keys_for(:variation))
+                base = sd[vd["base"]] if vd["base"]
+                raise(ConfigurationError, "variation `#{name}' cannot base from `#{vd["base"]}' since it doesn't exist (in `#{src}')") if vd["base"] && !base
+                raise ConfigurationError, "variation `#{name}' must define an adapter (mysql2, postgres, ...)" if vd["adapter"].blank? && vd["database"] != false && (!base || base["adapter"].blank?)
+              rescue ConfigurationError => ex
+                abort "#{ex.message} (in `#{src}' [/variations/#{name}])"
+              end
+            end
+          end
+        end
+
+        def _verify token, hash, keys
+          begin
+            hash.assert_valid_keys(keys)
+            raise ConfigurationError, "A source must define an adapter (mysql2, postgres, ...)" if token == "/source" && hash["adapter"].blank?
+            raise ConfigurationError, "A variation `#{name}' can only define either a `only' or `except' option" if hash["only"] && hash["except"]
+          rescue ConfigurationError, ArgumentError => ex
+            abort "#{ex.message} (in `#{src}' [#{token}])"
+          end
+        end
+
+        def __keys_for which
+          {
+            root: %w[source variations],
+            source: %w[adapter ssh database hostname username password args client_binary dump_binary gzip_binary integrity_sha integrity_binary],
+            source_ssh: %w[hostname username keyfile password port tmp_location],
+            variation: %w[adapter label base database hostname username password args client_binary incremental file only except importer importer_flags ignore_always constraints],
+          }[which] || []
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/variation/accessors.rb

@@ -0,0 +1,45 @@
+module DbSucker
+  class Application
+    class Container
+      class Variation
+        module Accessors
+          def ctn
+            cfg
+          end
+
+          def source
+            ctn.source
+          end
+
+          def label
+            data["label"]
+          end
+
+          def incrementals
+            data["incremental"] || {}
+          end
+
+          def gzip_binary
+            source["gzip_binary"] || "gzip"
+          end
+
+          def copies_file?
+            data["file"]
+          end
+
+          def copies_file_compressed?
+            copies_file? && data["file"].end_with?(".gz")
+          end
+
+          def requires_uncompression?
+            !copies_file_compressed? || data["database"]
+          end
+
+          def constraint table
+            data["constraints"] && (data["constraints"][table] || data["constraints"]["__default"])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/variation/helpers.rb

@@ -0,0 +1,21 @@
+module DbSucker
+  class Application
+    class Container
+      class Variation
+        module Helpers
+          def parse_flags flags
+            flags.to_s.split(" ").map(&:strip).reject(&:blank?).each_with_object({}) do |fstr, res|
+              if m = fstr.match(/\+(?<key>[^=]+)(?:=(?<value>))?/)
+                res[m[:key].strip] = m[:value].nil? ? true : m[:value]
+              elsif m = fstr.match(/\-(?<key>[^=]+)/)
+                res[m[:key]] = false
+              else
+                raise InvalidImporterFlagError, "invalid flag `#{fstr}' for variation `#{cfg.name}/#{name}' (in `#{cfg.src}')"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/variation/worker_api.rb

@@ -0,0 +1,65 @@
+module DbSucker
+  class Application
+    class Container
+      class Variation
+        module WorkerApi
+          def tables_to_transfer
+            all = cfg.table_list(cfg.data["source"]["database"]).map(&:first)
+            keep = []
+            if data["only"]
+              [*data["only"]].each do |t|
+                unless all.include?(t)
+                  raise TableNotFoundError, "table `#{t}' for the database `#{cfg.source["database"]}' could not be found (provided by variation `#{cfg.name}/#{name}' in `#{cfg.src}')"
+                end
+                keep << t
+              end
+            elsif data["except"]
+              keep = all.dup
+              [*data["except"]].each do |t|
+                unless all.include?(t)
+                  raise TableNotFoundError, "table `#{t}' for the database `#{cfg.source["database"]}' could not be found (provided by variation `#{cfg.name}/#{name}' in `#{cfg.src}')"
+                end
+                keep.delete(t)
+              end
+            else
+              keep = all.dup
+            end
+            keep -= data["ignore_always"] if data["ignore_always"].is_a?(Array)
+
+            [keep, all]
+          end
+
+          def dump_to_remote_command worker, pv_binary = false
+            tmpfile = worker.tmp_filename(true)
+            cmd = dump_command_for(worker.table)
+            if pv_binary.presence
+              cmd << %{ | #{pv_binary} -n -b > #{tmpfile}}
+            else
+              cmd << %{ > #{tmpfile}}
+            end
+            [tmpfile, cmd]
+          end
+
+          def dump_to_remote worker, blocking = true
+            tfile, cmd = dump_to_remote_command(worker)
+            [tfile, cfg.blocking_channel_result(cmd, channel: true, use_sh: true, blocking: blocking)]
+          end
+
+          def compress_file_command file, pv_binary = false
+            if pv_binary.presence
+              cmd = %{#{pv_binary} -n -b #{file} | #{gzip_binary} > #{file}.gz && rm #{file} }
+            else
+              cmd = %{#{gzip_binary} #{file}}
+            end
+            ["#{file}.gz", cmd]
+          end
+
+          def compress_file file, blocking = true
+            nfile, cmd = compress_file_command(file)
+            [nfile, cfg.blocking_channel_result(cmd, channel: true, use_sh: true, blocking: blocking)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container/variation.rb

@@ -0,0 +1,60 @@
+module DbSucker
+  class Application
+    class Container
+      class Variation
+        ImporterNotFoundError = Class.new(::RuntimeError)
+        InvalidImporterFlagError = Class.new(::RuntimeError)
+
+        include Accessors
+        include Helpers
+        include WorkerApi
+
+        attr_reader :cfg, :name, :data
+
+        def initialize cfg, name, data
+          @cfg, @name, @data = cfg, name, data
+
+          if data["base"]
+            bdata = cfg.variation(data["base"]) || raise(ConfigurationError, "variation `#{cfg.name}/#{name}' cannot base from `#{data["base"]}' since it doesn't exist (in `#{cfg.src}')")
+            @data = data.reverse_merge(bdata.data)
+          end
+
+          if @data["adapter"]
+            begin
+              adapter = "DbSucker::Adapters::#{@data["adapter"].camelize}::Api".constantize
+              @cfg.app.sync { adapter.require_dependencies }
+              extend adapter
+            rescue NameError => ex
+              raise(AdapterNotFoundError, "variation `#{cfg.name}/#{name}' defines invalid adapter `#{@data["adapter"]}' (in `#{cfg.src}'): #{ex.message}", ex.backtrace)
+            end
+          elsif @data["database"]
+            raise(ConfigurationError, "variation `#{cfg.name}/#{name}' must define an adapter (mysql2, postgres, ...) if database is provided (in `#{cfg.src}')")
+          end
+        end
+
+        # ===============
+        # = Adapter API =
+        # ===============
+        [
+          :client_binary,
+          :local_client_binary,
+          :dump_binary,
+          :client_call,
+          :local_client_call,
+          :dump_call,
+          :database_list,
+          :table_list,
+          :hostname,
+        ].each do |meth|
+          define_method meth do
+            raise NotImplementedError, "your selected adapter `#{@data["adapter"]}' must implement `##{meth}' for variation `#{cfg.name}/#{name}' (in `#{cfg.src}')"
+          end
+        end
+
+        def dump_command_for table
+          raise NotImplementedError, "your selected adapter `#{@data["adapter"]}' must implement `#dump_command_for(table)' for variation `#{cfg.name}/#{name}' (in `#{cfg.src}')"
+        end
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container.rb

@@ -0,0 +1,70 @@
+module DbSucker
+  class Application
+    class Container
+      AdapterNotFoundError = Class.new(::ArgumentError)
+      TableNotFoundError = Class.new(::RuntimeError)
+      ConfigurationError = Class.new(::ArgumentError)
+
+      include Accessors
+      include Validations
+      include SSH
+      OutputHelper.hook(self)
+
+      attr_reader :app, :name, :src, :data
+
+      def initialize app, name, src, data
+        @app = app
+        @name = name
+        @src = src
+        @data = data
+        @ssh_mutex = Monitor.new
+        @sftp_mutex = Monitor.new
+
+        verify!
+
+        begin
+          adapter = "DbSucker::Adapters::#{source["adapter"].camelize}::Api".constantize
+          app.sync { adapter.require_dependencies }
+          extend adapter
+        rescue NameError => ex
+          raise(AdapterNotFoundError, "identifier `#{name}' defines invalid source adapter `#{source["adapter"]}' (in `#{@src}'): #{ex.message}", ex.backtrace)
+        end
+      end
+
+      def pv_utility
+        app.sync do
+          if @_pv_utility.nil?
+            if app.opts[:pv_enabled]
+              res = blocking_channel_result("which pv && pv --version")
+              app.debug "#{Time.current.to_f}: #{res.to_a.inspect}"
+              if m = res[1].to_s.match(/pv\s([0-9\.]+)\s/i)
+                if Gem::Version.new(m[1]) >= Gem::Version.new("1.3.8")
+                  @_pv_utility = res[0].strip.presence
+                end
+              end
+            end
+            @_pv_utility = false unless @_pv_utility
+          end
+          @_pv_utility
+        end
+      rescue Errno::ECONNRESET
+      end
+
+      def calculate_remote_integrity_hash_command file, pv_binary = false
+        return unless integrity?
+        icmd = "#{integrity_binary}#{integrity_sha}"
+        if pv_binary.presence
+          %{#{pv_binary} -n -b #{file} | #{icmd}}
+        else
+          %{#{icmd} #{file}}
+        end
+      end
+
+      def calculate_remote_integrity_hash file, blocking = true
+        cmd = calculate_remote_integrity_hash_command(file)
+        return unless cmd
+        [cmd, blocking_channel_result(cmd, channel: true, use_sh: true, blocking: blocking)]
+      end
+    end
+  end
+end

data/lib/db_sucker/application/container_collection.rb

@@ -0,0 +1,47 @@
+module DbSucker
+  class Application
+    class ContainerCollection
+      DuplicateIdentifierError = Class.new(::ArgumentError)
+      YAMLParseError = Class.new(::RuntimeError)
+
+      attr_reader :app, :data
+
+      def initialize app
+        @app = app
+        @data = {}
+      end
+
+      def yml_configs disabled = false
+        files = Dir["#{app.core_cfg_path}/**/*.yml"].select{|f| File.file?(f) }
+        return files if disabled
+        files.reject do |f|
+          f.gsub("#{app.core_cfg_path}/", "").split("/").any?{|fp| fp.start_with?("__") }
+        end
+      end
+
+      def load_all_configs
+        yml_configs.each{|f| load_yml_config(f) }
+      end
+
+      def load_yml_config file
+        YAML.load_file(file).each do |id, cfg|
+          if @data.key?(id)
+            raise DuplicateIdentifierError, "double use of identifier `#{id}' in `#{file}'"
+          else
+            @data[id] = Container.new(app, id, file, cfg)
+          end
+        end
+      rescue Psych::SyntaxError => ex
+        app.abort ex.message
+      end
+
+      def get id
+        @data[id]
+      end
+
+      def each &block
+        @data.each(&block)
+      end
+    end
+  end
+end