pxcbackup 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 048fb039a8728b1281f0e6525e67570b4c218f51
-  data.tar.gz: d93238bcc021b538bf75c8551fc79e9631b6d976
+  metadata.gz: 95db3e154a55db8fe7bd7ea736d2f197a51a7f4f
+  data.tar.gz: 0c8da2018cecf4ee57bb5eb54ed28972a84d713f
 SHA512:
-  metadata.gz: 60ee86b267cfa743ab4ecf8bc7b792272a6fe3fbed583697cffdcc55be51087bceada6d92462342103c49fa3c11769cde53c4e72c3361215ed958b1681765f24
-  data.tar.gz: c88134fb4dc0ad0345bb860d1ccdb10bd17cfbd65e2bbe7556fc6ac68e0d4bbfdc80189cf96b857821fba02468d84faacadedbfc79380f2ecefb83088fecb1ec
+  metadata.gz: 2430921b6608a2d59aad06d6ffae4e538a6ba5bceabbe125ae8d8880c929ca4fee29f008de84a4b272cbc935c4f99e5ba0d8e4dc6d004acb99de886e3d7f60bd
+  data.tar.gz: efd83314d29a5c9cb5c21f86fdc0ff33158cc8b8f068f173e31412fc099e0e584fe7deb1f51f563656f901d3392436ecd71a8a7e0ad17c19e63f2368d5b2a3ce

data/README.md

@@ -1,5 +1,7 @@
 # PXCBackup
 
+[![Gem Version](https://badge.fury.io/rb/pxcbackup.svg)](http://badge.fury.io/rb/pxcbackup)
+
 PXCBackup is a database backup tool meant for [Percona XtraDB Cluster](http://www.percona.com/software/percona-xtradb-cluster) (PXC), although it could also be used on other related systems, like a [MariaDB](https://mariadb.org) [Galera](http://galeracluster.com/products/) cluster using [XtraBackup](http://www.percona.com/software/percona-xtrabackup), for example.
 
 The `innobackupex` script provided by Percona makes it very easy to create backups, however restoring backups can become quite complicated, since backups might need to be extracted, uncompressed, decrypted, before restoring they need to be prepared, incremental backups need to be applied on top of full backups, indexes might need to be rebuilt for compact backups, etc. Usually, backups need to be restored in stressful emergency situations, where all of these steps can slow you down quite a bit.

data/lib/pxcbackup/application.rb

@@ -2,6 +2,10 @@ require 'optparse'
 require 'time'
 require 'yaml'
 
+require 'pxcbackup/backupper'
+require 'pxcbackup/logger'
+require 'pxcbackup/version'
+
 module PXCBackup
   class Application
     def initialize(argv)
@@ -20,6 +24,7 @@ module PXCBackup
     end
 
     def run
+      Logger.color_output = ENV['TERM'] && !@options[:no_color]
       backupper = Backupper.new(@options)
 
       case @command
@@ -46,6 +51,10 @@ module PXCBackup
         opt.separator ''
         opt.separator 'Options'
 
+        opt.on('--no-color', 'disable color output') do |color_output|
+          @options[:no_color] = true
+        end
+
         opt.on('-c', '--config', '=CONFIG_FILE', 'config file to use instead of ~/.pxcbackup') do |config_file|
           @options[:config] = config_file
         end
@@ -71,7 +80,12 @@ module PXCBackup
         end
 
         opt.on('-v', '--verbose', 'verbose output') do
-          @options[:verbose] = true
+          Logger.raise_verbosity
+        end
+
+        opt.on('--version', 'print version and exit') do
+          puts "pxcbackup #{VERSION}"
+          exit
         end
 
         opt.on('-y', '--yes', 'skip confirmation on backup restore') do

data/lib/pxcbackup/backup.rb

@@ -23,7 +23,7 @@ module PXCBackup
     end
 
     def to_s
-      time.to_s
+      "#{time} - #{type.to_s[0..3]} (#{remote? ? 'remote' : 'local'})"
     end
 
     def time

data/lib/pxcbackup/backupper.rb

@@ -1,9 +1,10 @@
 require 'fileutils'
-require 'open3'
 require 'tmpdir'
 
 require 'pxcbackup/array'
 require 'pxcbackup/backup'
+require 'pxcbackup/command'
+require 'pxcbackup/logger'
 require 'pxcbackup/mysql'
 require 'pxcbackup/path_resolver'
 require 'pxcbackup/remote_repo'
@@ -12,9 +13,10 @@ require 'pxcbackup/repo'
 module PXCBackup
   class Backupper
     def initialize(options)
-      @verbose = options[:verbose] || false
       @threads = options[:threads] || 1
       @memory = options[:memory] || '100M'
+
+      @defaults_file = options[:defaults_file] || nil
       @throttle = options[:throttle] || nil
       @encrypt = options[:encrypt] || nil
       @encrypt_key = options[:encrypt_key] || nil
@@ -78,7 +80,7 @@ module PXCBackup
 
       Dir.mktmpdir('pxcbackup-') do |dir|
         arguments << dir.shellescape
-        log_action "Creating backup #{filename}" do
+        Logger.action "Creating backup #{filename}" do
           innobackupex(arguments, File.join(@local_repo.path, filename))
         end
       end
@@ -86,7 +88,11 @@ module PXCBackup
       desync_disable
       rotate(retention)
 
-      @remote_repo.sync(@local_repo) if @remote_repo
+      if @remote_repo
+        Logger.action 'Syncing backups to remote repository' do
+          @remote_repo.sync(@local_repo)
+        end
+      end
     end
 
     def restore_backup(time, skip_confirmation = false)
@@ -101,7 +107,8 @@ module PXCBackup
 
       full_backup = incremental_backups.shift
 
-      log "[1/#{incremental_backups.size + 1}] Processing #{full_backup.type.to_s} backup from #{full_backup}"
+      Logger.info "[1/#{incremental_backups.size + 1}] Processing #{full_backup.type.to_s} backup from #{full_backup.time}"
+      Logger.increase_indentation
       with_extracted_backup(full_backup) do |full_backup_path, full_backup_info|
         raise 'unexpected backup type' unless full_backup_info[:backup_type] == full_backup.type
         raise 'unexpected start LSN' unless full_backup_info[:from_lsn] == 0
@@ -109,22 +116,23 @@ module PXCBackup
         compact = full_backup_info[:compact]
 
         if full_backup_info[:compress]
-          log_action '  Decompressing' do
+          Logger.action 'Decompressing' do
             innobackupex(['--decompress', full_backup_path.shellescape])
           end
         end
 
         if incremental_backups.any?
-          log_action "  Preparing base backup (LSN #{full_backup_info[:to_lsn]})" do
+          Logger.action "Preparing base backup (LSN #{full_backup_info[:to_lsn]})" do
             innobackupex(['--apply-log', '--redo-only', full_backup_path.shellescape])
           end
 
           current_lsn = full_backup_info[:to_lsn]
+          Logger.decrease_indentation
 
           index = 2
           incremental_backups.each do |incremental_backup|
-            log "[#{index}/#{incremental_backups.size + 1}] Processing #{incremental_backup.type.to_s} backup from #{incremental_backup}"
-            index += 1
+            Logger.info "[#{index}/#{incremental_backups.size + 1}] Processing #{incremental_backup.type.to_s} backup from #{incremental_backup.time}"
+            Logger.increase_indentation
             with_extracted_backup(incremental_backup) do |incremental_backup_path, incremental_backup_info|
               raise 'unexpected backup type' unless incremental_backup_info[:backup_type] == incremental_backup.type
               raise 'unexpected start LSN' unless incremental_backup_info[:from_lsn] == current_lsn
@@ -132,17 +140,19 @@ module PXCBackup
               compact ||= incremental_backup_info[:compact]
 
               if incremental_backup_info[:compress]
-                log_action '  Decompressing' do
+                Logger.action 'Decompressing' do
                   innobackupex(['--decompress', incremental_backup_path.shellescape])
                 end
               end
 
-              log_action "  Applying increment (LSN #{incremental_backup_info[:from_lsn]} -> #{incremental_backup_info[:to_lsn]})" do
+              Logger.action "Applying increment (LSN #{incremental_backup_info[:from_lsn]} -> #{incremental_backup_info[:to_lsn]})" do
                 innobackupex(['--apply-log', '--redo-only', full_backup_path.shellescape, "--incremental-dir=#{incremental_backup_path.shellescape}"])
               end
 
               current_lsn = incremental_backup_info[:to_lsn]
+              Logger.decrease_indentation
             end
+            index += 1
           end
         end
 
@@ -156,12 +166,12 @@ module PXCBackup
           arguments << '--rebuild-indexes'
         end
 
-        log_action "#{action}" do
+        Logger.action "#{action}" do
           arguments << full_backup_path.shellescape
           innobackupex(arguments)
         end
 
-        log_action 'Attempting to restore Galera info' do
+        Logger.action 'Attempting to restore Galera info' do
           restore_galera_info(full_backup_path)
         end
 
@@ -194,8 +204,8 @@ module PXCBackup
           raise 'did not confirm restore' unless confirmation == 'yes'
         end
 
-        log_action 'Stopping MySQL server' do
-          system("#{@which.service.shellescape} mysql stop")
+        Logger.action 'Stopping MySQL server' do
+          Command.run("#{@which.service.shellescape} mysql stop")
         end
 
         stat = File.stat(mysql_datadir)
@@ -203,39 +213,33 @@ module PXCBackup
         gid = stat.gid
 
         mysql_datadir_old = mysql_datadir + '_' + Time.now.strftime('%Y%m%d%H%M%S')
-        log_action "Moving current datadir to #{mysql_datadir_old}" do
+        Logger.action "Moving current datadir to #{mysql_datadir_old}" do
           File.rename(mysql_datadir, mysql_datadir_old)
         end
 
-        log_action "Restoring backup to #{mysql_datadir}" do
+        Logger.action "Restoring backup to #{mysql_datadir}" do
           Dir.mkdir(mysql_datadir)
           innobackupex(['--move-back', full_backup_path.shellescape])
         end
 
-        log_action "Chowning #{mysql_datadir}" do
+        Logger.action "Chowning #{mysql_datadir}" do
           FileUtils.chown_R(uid, gid, mysql_datadir)
         end
 
         if @local_repo
-          log_action "Removing last backup info" do
+          Logger.action "Removing last backup info" do
             File.delete(File.join(@local_repo.path, 'xtrabackup_checkpoints'))
           end
         end
 
-        log_action 'Starting MySQL server' do
-          system("#{@which.service.shellescape} mysql start")
+        Logger.action 'Starting MySQL server' do
+          Command.run("#{@which.service.shellescape} mysql start")
         end
       end
     end
 
     def list_backups
-      all_backups.each do |backup|
-        if @verbose
-          puts "#{backup} - #{backup.type.to_s[0..3]} (#{backup.remote? ? 'remote' : 'local'})"
-        else
-          puts backup
-        end
-      end
+      all_backups.each { |backup| puts backup }
     end
 
     private
@@ -248,61 +252,38 @@ module PXCBackup
       backups.sort
     end
 
-    def log(text)
-      return unless @verbose
-      previous_stdout = $stdout
-      $stdout = STDOUT
-      puts text if @verbose
-      $stdout = previous_stdout
-    end
-
-    def log_action(text)
-      return yield unless @verbose
-
-      begin
-        print "#{text}... "
-        previous_stdout, previous_stderr = $stdout, $stderr
-        begin
-          $stdout = $stderr = File.new('/dev/null', 'w')
-          t1 = Time.now
-          yield
-          t2 = Time.now
-        ensure
-          $stdout, $stderr = previous_stdout, previous_stderr
-        end
-      rescue => e
-        puts "fail"
-        raise e
-      else
-        puts "done (%.1fs)" % (t2 - t1)
-      end
-    end
-
     def desync_enable(wait = 60)
-      log "Setting wsrep_desync=ON and waiting for #{wait} seconds"
+      Logger.info 'Setting wsrep_desync=ON'
       @mysql.set_variable('wsrep_desync', 'ON')
-      sleep(wait)
+      Logger.action "Waiting for #{wait} seconds" do
+        sleep(wait)
+      end
     end
 
     def desync_disable
-      log 'Waiting until wsrep_local_recv_queue is empty'
-      sleep(2) until @mysql.get_status('wsrep_local_recv_queue') == '0'
-      log 'Setting wsrep_desync=OFF'
+      Logger.action 'Waiting until wsrep_local_recv_queue is empty' do
+        sleep(2) until @mysql.get_status('wsrep_local_recv_queue') == '0'
+      end
+      Logger.info 'Setting wsrep_desync=OFF'
       @mysql.set_variable('wsrep_desync', 'OFF')
     end
 
     def rotate(retention)
-      log 'Checking if we have old backups to remove'
-      @local_repo.backups.each do |backup|
-        days = (Time.now - backup.time) / 86400
-        break if days < retention && backup.full?
-        log "Deleting backup #{backup}"
-        backup.delete
+      Logger.action 'Checking if we have old backups to remove' do
+        @local_repo.backups.each do |backup|
+          days = (Time.now - backup.time) / 86400
+          break if days < retention && backup.full?
+          Logger.info "Deleting backup from #{backup.time}"
+          backup.delete
+        end
       end
     end
 
     def innobackupex(arguments, output_file = nil)
       command = @which.innobackupex.shellescape
+      # --defaults-file has to be the first option passed!
+      command << " --defaults-file=#{@defaults_file.shellescape}" if @defaults_file
+
       arguments += [
         "--ibbackup=#{@which.xtrabackup.shellescape}",
         "--parallel=#{@threads}",
@@ -315,11 +296,8 @@ module PXCBackup
 
       command << ' ' + arguments.join(' ')
       command << " > #{output_file.shellescape}" if output_file
-      log = Open3.popen3(command) do |stdin, stdout, stderr|
-        stderr.read
-      end
-      exit_status = $?
-      raise 'something went wrong with innobackupex' unless exit_status.success? && log.lines.to_a.last.match(/: completed OK!$/)
+      result = Command.run(command)
+      raise 'unexpected output from innobackupex' unless result[:stderr].lines.to_a.last.match(/: completed OK!$/)
     end
 
     def read_backup_info(file)
@@ -359,8 +337,8 @@ module PXCBackup
           when :tar
             " | #{@which.tar.shellescape} -ixf - -C #{dir.shellescape}"
           end
-        log_action "  #{action}" do
-          system(command)
+        Logger.action action do
+          Command.run(command)
         end
 
         info = read_backup_info(File.join(dir, 'xtrabackup_checkpoints'))

data/lib/pxcbackup/command.rb

@@ -0,0 +1,32 @@
+require 'open3'
+
+require 'pxcbackup/logger'
+
+module PXCBackup
+  module Command
+    def self.run(command, ignore_exit_status = false)
+      Logger.debug "# #{command}"
+      captured_stdout = ''
+      captured_stderr = ''
+      Open3.popen3(command) do |stdin, stdout, stderr|
+        stdin.close
+        until stdout.closed? && stderr.closed?
+          sockets = []
+          sockets << stdout unless stdout.closed?
+          sockets << stderr unless stderr.closed?
+          IO.select(sockets).flatten.compact.each do |socket|
+            begin
+              data = socket.readpartial(1024)
+              captured_stdout << data if socket == stdout
+              captured_stderr << data if socket == stderr
+            rescue EOFError
+              socket.close
+            end
+          end
+        end
+      end
+      raise 'command "#{command.split.first}" exited with a non-zero status' unless $?.success? || ignore_exception
+      { :stdout => captured_stdout, :stderr => captured_stderr, :exit_status => $?.exitstatus }
+    end
+  end
+end

data/lib/pxcbackup/logger.rb

@@ -0,0 +1,96 @@
+module PXCBackup
+  module Logger
+    @verbosity_level = 0
+    @indentation = 0
+    @color_output = false
+    @partial = false
+
+    def self.raise_verbosity
+      @verbosity_level += 1
+    end
+
+    def self.increase_indentation
+      @indentation += 1
+    end
+
+    def self.decrease_indentation
+      @indentation -= 1
+    end
+
+    def self.color_output=(value)
+      @color_output = value
+    end
+
+    def self.output(message, skip_newline = false)
+      if @partial
+        puts
+        increase_indentation
+        @partial = false
+      end
+      print '  ' * @indentation + message
+      puts unless skip_newline
+      $stdout.flush
+    end
+
+    def self.action_start(message)
+      return unless @verbosity_level >= 1
+      output "#{message}: ", true
+      @partial = true
+    end
+
+    def self.action_end(message)
+      return unless @verbosity_level >= 1
+      if @partial
+        puts message
+        @partial = false
+      else
+        output message
+        decrease_indentation
+      end
+    end
+
+    def self.info(message)
+      output message if @verbosity_level >= 1
+    end
+
+    def self.debug(message)
+      output blue(message) if @verbosity_level >= 2
+    end
+
+    def self.action(message)
+      return yield unless @verbosity_level >= 1
+
+      action_start(message)
+      t1 = Time.now
+      begin
+        result = yield
+      rescue => e
+        action_end(red('fail'))
+        raise e
+      end
+      t2 = Time.now
+      action_end(green('done') + ' (%.1fs)' % (t2 - t1))
+      result
+    end
+
+    def self.colorize(text, color_code)
+      @color_output ? "\e[#{color_code}m#{text}\e[0m" : text
+    end
+
+    def self.red(text)
+      colorize(text, 31);
+    end
+
+    def self.green(text)
+      colorize(text, 32)
+    end
+
+    def self.yellow(text)
+      colorize(text, 33);
+    end
+
+    def self.blue(text)
+      colorize(text, 34);
+    end
+  end
+end

data/lib/pxcbackup/mysql.rb

@@ -1,5 +1,7 @@
 require 'shellwords'
 
+require 'pxcbackup/command'
+
 module PXCBackup
   class MySQL
     attr_reader :datadir
@@ -17,7 +19,8 @@ module PXCBackup
     end
 
     def exec(query)
-      lines = `echo #{query.shellescape} | #{@which.mysql.shellescape} #{auth} 2> /dev/null`.lines.to_a
+      output = Command.run("echo #{query.shellescape} | #{@which.mysql.shellescape} #{auth}", true)
+      lines = output[:stdout].lines.to_a
       return nil if lines.empty?
 
       keys = lines.shift.chomp.split("\t")

data/lib/pxcbackup/remote_repo.rb

@@ -1,7 +1,8 @@
 require 'shellwords'
 
-require	'pxcbackup/backup'
-require	'pxcbackup/repo'
+require 'pxcbackup/backup'
+require 'pxcbackup/command'
+require 'pxcbackup/repo'
 
 module PXCBackup
   class RemoteRepo < Repo
@@ -12,7 +13,8 @@ module PXCBackup
 
     def backups
       backups = []
-      `#{@which.s3cmd.shellescape} ls #{@path.shellescape}`.lines.to_a.each do |line|
+      output = Command.run("#{@which.s3cmd.shellescape} ls #{@path.shellescape}")
+      output[:stdout].lines.to_a.each do |line|
         path = line.chomp.split[3]
         next unless Backup.regexp.match(path)
         backups << Backup.new(self, path)
@@ -23,12 +25,12 @@ module PXCBackup
     def sync(local_repo)
       source = File.join(local_repo.path, '/')
       target = File.join(path, '/')
-      system("#{@which.s3cmd.shellescape} sync --no-progress --delete-removed #{source.shellescape} #{target.shellescape} > /dev/null")
+      Command.run("#{@which.s3cmd.shellescape} sync --no-progress --delete-removed #{source.shellescape} #{target.shellescape}")
     end
 
     def delete(backup)
       verify(backup)
-      system("#{@which.s3cmd.shellescape} del #{backup.path.shellescape} > /dev/null")
+      Command.run("#{@which.s3cmd.shellescape} del #{backup.path.shellescape}")
     end
 
     def stream_command(backup)

data/lib/pxcbackup/repo.rb

@@ -24,7 +24,7 @@ module PXCBackup
 
     def delete(backup)
       verify(backup)
-      system("#{@which.rm.shellescape} #{backup.path.shellescape}")
+      File.delete(backup.path)
     end
 
     def stream_command(backup)

data/lib/pxcbackup/version.rb

@@ -1,3 +1,3 @@
 module PXCBackup
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pxcbackup
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Robbert Klarenbeek
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-08 00:00:00.000000000 Z
+date: 2014-05-10 00:00:00.000000000 Z
 dependencies: []
 description: Backup tool for Percona XtraDB Cluster
 email: robbertkl@renbeek.nl
@@ -27,6 +27,8 @@ files:
 - lib/pxcbackup/array.rb
 - lib/pxcbackup/backup.rb
 - lib/pxcbackup/backupper.rb
+- lib/pxcbackup/command.rb
+- lib/pxcbackup/logger.rb
 - lib/pxcbackup/mysql.rb
 - lib/pxcbackup/path_resolver.rb
 - lib/pxcbackup/remote_repo.rb