foreman_maintain 0.2.1 → 0.2.2

data/lib/foreman_maintain/cli.rb

@@ -7,6 +7,7 @@ require 'foreman_maintain/cli/upgrade_command'
 require 'foreman_maintain/cli/backup_command'
 require 'foreman_maintain/cli/advanced_command'
 require 'foreman_maintain/cli/service_command'
+require 'foreman_maintain/cli/restore_command'
 
 module ForemanMaintain
   module Cli
@@ -15,9 +16,10 @@ module ForemanMaintain
 
       subcommand 'health', 'Health related commands', HealthCommand
       subcommand 'upgrade', 'Upgrade related commands', UpgradeCommand
+      subcommand 'service', 'Control applicable services', ServiceCommand
       subcommand 'backup', 'Backup server', BackupCommand
+      subcommand 'restore', 'Restore a backup', RestoreCommand
       subcommand 'advanced', 'Advanced tools for server maintenance', AdvancedCommand
-      subcommand 'service', 'Control applicable services', ServiceCommand
 
       def run(*arguments)
         logger.info("Running foreman-maintain command with arguments #{arguments.inspect}")

data/lib/foreman_maintain/cli/restore_command.rb

@@ -0,0 +1,26 @@
+module ForemanMaintain
+  module Cli
+    class RestoreCommand < Base
+      interactive_option
+      parameter 'BACKUP_DIR', 'Path to backup directory to restore',
+                :attribute_name => :backup_dir
+
+      option ['-i', '--incremental'], :flag, 'Restore an incremental backup',
+             :attribute_name => :incremental
+
+      def execute
+        scenario = Scenarios::Restore.new(
+          :backup_dir => @backup_dir,
+          :incremental_backup => @incremental || incremental_backup?
+        )
+        run_scenario(scenario)
+        exit runner.exit_code
+      end
+
+      def incremental_backup?
+        backup = ForemanMaintain::Utils::Backup.new(@backup_dir)
+        backup.metadata.fetch(:incremental, false)
+      end
+    end
+  end
+end

data/lib/foreman_maintain/concerns/base_database.rb

@@ -52,13 +52,34 @@ module ForemanMaintain
         execute!(dump_command(config) + " > #{file}", :hidden_patterns => [config['password']])
       end
 
+      def restore_dump(file, localdb, config = configuration)
+        if localdb
+          dump_cmd = "runuser - postgres -c 'pg_restore -C -d postgres #{file}'"
+          execute!(dump_cmd)
+        else
+          # TODO: figure out how to completely ignore errors. Currently this
+          # sometimes exits with 1 even though errors are ignored by pg_restore
+          dump_cmd = base_command(config, 'pg_restore') +
+                     ' --no-privileges --clean --disable-triggers -n public ' \
+                     "-d #{config['database']} #{file}"
+          execute!(dump_cmd, :hidden_patterns => [config['password']],
+                             :valid_exit_statuses => [0, 1])
+        end
+      end
+
+      def restore_pg_globals(pg_globals, config = configuration)
+        execute!(base_command(config, 'psql') + " -f #{pg_globals} postgres 2>/dev/null",
+                 :hidden_patterns => [config['password']])
+      end
+
       def backup_local(backup_file, extra_tar_options = {})
         dir = extra_tar_options.fetch(:data_dir, data_dir)
         FileUtils.cd(dir) do
           tar_options = {
             :archive => backup_file,
             :command => 'create',
-            :transform => 's,^,var/lib/pgsql/data/,S'
+            :transform => 's,^,var/lib/pgsql/data/,S',
+            :files => '*'
           }.merge(extra_tar_options)
           feature(:tar).run(tar_options)
         end
@@ -112,6 +133,19 @@ module ForemanMaintain
         find_dir_containing_file(directory, 'postgresql.conf')
       end
 
+      def dropdb(config = configuration)
+        if local?
+          execute!("runuser - postgres -c 'dropdb #{config['database']}'")
+        else
+          delete_statement = psql(<<-SQL)
+            select string_agg('drop table if exists \"' || tablename || '\" cascade;', '')
+            from pg_tables
+            where schemaname = 'public';
+          SQL
+          psql(delete_statement)
+        end
+      end
+
       private
 
       def base_command(config, command = 'psql')

data/lib/foreman_maintain/concerns/system_helpers.rb

@@ -7,6 +7,7 @@ module ForemanMaintain
   module Concerns
     module SystemHelpers
       include Logger
+      include Concerns::Finders
 
       def self.included(klass)
         klass.extend(self)
@@ -184,11 +185,6 @@ module ForemanMaintain
         execute("lvs --noheadings -o lv_path -S lv_name=#{lv_name}").strip
       end
 
-      def local_psql_database?
-        (feature(:foreman_database) && feature(:foreman_database).local?) ||
-          (feature(:candlepin_database) && feature(:candlepin_database).local?)
-      end
-
       def find_dir_containing_file(directory, target)
         result = nil
         Find.find(directory) do |path|

data/lib/foreman_maintain/reporter/cli_reporter.rb

@@ -88,7 +88,8 @@ module ForemanMaintain
       end
 
       def before_execution_starts(execution)
-        logger.info("--- Execution step '#{execution.name}' started ---")
+        label = execution.step.label.to_s.tr('_', '-')
+        logger.info("--- Execution step '#{execution.name}' [#{label}] started ---")
         puts(execution_info(execution, ''))
       end
 

data/lib/foreman_maintain/utils/backup.rb

@@ -0,0 +1,225 @@
+require 'zlib'
+require 'rubygems/package'
+require 'yaml'
+
+module ForemanMaintain
+  module Utils
+    class Backup
+      include Concerns::SystemHelpers
+
+      attr_accessor :standard_files, :katello_online_files, :katello_offline_files,
+                    :foreman_online_files, :foreman_offline_files, :fpc_offline_files,
+                    :fpc_online_files
+
+      def initialize(backup_dir)
+        # fpc stands for foreman proxy w/ content
+        @backup_dir = backup_dir
+        @standard_files = ['config_files.tar.gz']
+        @katello_online_files = ['mongo_dump', 'candlepin.dump', 'foreman.dump']
+        @katello_offline_files = ['mongo_data.tar.gz', 'pgsql_data.tar.gz']
+        @foreman_online_files = ['foreman.dump']
+        @foreman_offline_files = ['pgsql_data.tar.gz']
+        @fpc_online_files = ['mongo_dump']
+        @fpc_offline_files = ['mongo_data.tar.gz']
+      end
+
+      def file_map
+        @file_map ||= {
+          :mongo_data => map_file(@backup_dir, 'mongo_data.tar.gz'),
+          :pgsql_data => map_file(@backup_dir, 'pgsql_data.tar.gz'),
+          :pulp_data => map_file(@backup_dir, 'pulp_data.tar'),
+          :foreman_dump => map_file(@backup_dir, 'foreman.dump'),
+          :candlepin_dump => map_file(@backup_dir, 'candlepin.dump'),
+          :mongo_dump => map_file(@backup_dir, 'mongo_dump'),
+          :config_files => map_file(@backup_dir, 'config_files.tar.gz'),
+          :pg_globals => map_file(@backup_dir, 'pg_globals.dump'),
+          :metadata => map_file(@backup_dir, 'metadata.yml')
+        }
+      end
+
+      def map_file(backup_dir, filename)
+        file_path = File.join(backup_dir, filename)
+        present = File.exist?(file_path)
+        {
+          :present => present,
+          :path => file_path
+        }
+      end
+
+      def present_files
+        present_files = file_map.select { |_k, v| v[:present] }
+        present_files.values.map { |f| File.basename(f[:path]) }
+      end
+
+      def valid_backup?
+        file_map[:config_files][:present] && check_backup
+      end
+
+      def check_backup
+        if feature(:instance).foreman_proxy_with_content?
+          valid_fpc_backup?
+        elsif feature(:katello)
+          valid_katello_backup?
+        else
+          valid_foreman_backup?
+        end
+      end
+
+      def valid_fpc_backup?
+        fpc_online_backup? || fpc_standard_backup? || fpc_logical_backup?
+      end
+
+      def valid_katello_backup?
+        katello_online_backup? || katello_standard_backup? || katello_logical_backup?
+      end
+
+      def valid_foreman_backup?
+        foreman_standard_backup? || foreman_online_backup? || foreman_logical_backup?
+      end
+
+      def katello_standard_backup?
+        file_map[:mongo_data][:present] &&
+          file_map[:pgsql_data][:present] &&
+          !(
+            file_map[:candlepin_dump][:present] ||
+            file_map[:foreman_dump][:present] ||
+            file_map[:mongo_dump][:present]
+          )
+      end
+
+      def katello_online_backup?
+        file_map[:candlepin_dump][:present] &&
+          file_map[:foreman_dump][:present] &&
+          file_map[:mongo_dump][:present] &&
+          !(
+            file_map[:mongo_data][:present] ||
+            file_map[:pgsql_data][:present]
+          )
+      end
+
+      def katello_logical_backup?
+        file_map[:mongo_dump][:present] &&
+          file_map[:mongo_data][:present] &&
+          file_map[:pgsql_data][:present] &&
+          file_map[:candlepin_dump][:present] &&
+          file_map[:foreman_dump][:present]
+      end
+
+      def fpc_standard_backup?
+        file_map[:mongo_data][:present] &&
+          !(
+            file_map[:pgsql_data][:present] ||
+            file_map[:candlepin_dump][:present] ||
+            file_map[:foreman_dump][:present] ||
+            file_map[:mongo_dump][:present]
+          )
+      end
+
+      def fpc_online_backup?
+        file_map[:mongo_dump][:present] &&
+          !(
+            file_map[:mongo_data][:present] ||
+            file_map[:pgsql_data][:present] ||
+            file_map[:candlepin_dump][:present] ||
+            file_map[:foreman_dump][:present]
+          )
+      end
+
+      def fpc_logical_backup?
+        file_map[:mongo_dump][:present] &&
+          file_map[:mongo_data][:present] &&
+          !(
+            file_map[:pgsql_data][:present] ||
+            file_map[:candlepin_dump][:present] ||
+            file_map[:foreman_dump][:present]
+          )
+      end
+
+      def foreman_standard_backup?
+        file_map[:pgsql_data][:present] &&
+          !(
+            file_map[:candlepin_dump][:present] ||
+            file_map[:foreman_dump][:present] ||
+            file_map[:mongo_data][:present] ||
+            file_map[:mongo_dump][:present]
+          )
+      end
+
+      def foreman_online_backup?
+        file_map[:foreman_dump][:present] &&
+          !(
+            file_map[:candlepin_dump][:present] ||
+            file_map[:pgsql_data][:present] ||
+            file_map[:mongo_data][:present] ||
+            file_map[:mongo_dump][:present]
+          )
+      end
+
+      def foreman_logical_backup?
+        file_map[:pgsql_data][:present] &&
+          file_map[:foreman_dump][:present] &&
+          !(
+            file_map[:candlepin_dump][:present] ||
+            file_map[:mongo_data][:present] ||
+            file_map[:mongo_dump][:present]
+          )
+      end
+
+      def validate_hostname?
+        # make sure that the system hostname is the same as the backup
+        config_tarball = file_map[:config_files][:path]
+        config_files = tarball_file_list(config_tarball)
+
+        # Incremental backups sometimes don't include httpd.conf. Since a "base" backup
+        # is restored before an incremental, we can assume that the hostname is checked
+        # during the base backup restore
+        if config_files.include?('etc/httpd/conf/httpd.conf')
+          tar_cmd = "tar zxf #{config_tarball} etc/httpd/conf/httpd.conf " \
+                    "--to-stdout | grep ServerName | awk {'print $2'} | tr -d '\"'"
+          backup_hostname = execute(tar_cmd).chomp
+          backup_hostname == hostname
+        else
+          true
+        end
+      end
+
+      def metadata
+        if file_map[:metadata][:present]
+          YAML.load_file(file_map[:metadata][:path])
+        else
+          {}
+        end
+      end
+
+      def pulp_tar_split?
+        File.exist?(File.join(@backup_dir, 'pulp_data.part0002'))
+      end
+
+      def tar_backups_exist?
+        file_map[:mongo_data][:present] ||
+          file_map[:pulp_data][:present] ||
+          file_map[:pgsql_data][:present]
+      end
+
+      def sql_dump_files_exist?
+        file_map[:foreman_dump][:present] ||
+          file_map[:candlepin_dump][:present]
+      end
+
+      private
+
+      def tarball_file_list(tarball)
+        # accepts tar.gz files only
+        file_list = []
+        File.open(tarball, 'rb') do |file|
+          ::Zlib::GzipReader.wrap(file) do |gz|
+            ::Gem::Package::TarReader.new(gz) do |tar|
+              tar.each { |entry| file_list << entry.full_name }
+            end
+          end
+        end
+        file_list
+      end
+    end
+  end
+end

data/lib/foreman_maintain/utils/command_runner.rb

@@ -8,13 +8,15 @@ module ForemanMaintain
       attr_reader :logger, :command
 
       def initialize(logger, command, options)
-        options.validate_options!(:stdin, :hidden_patterns, :interactive)
+        options.validate_options!(:stdin, :hidden_patterns, :interactive, :valid_exit_statuses)
+        options[:valid_exit_statuses] ||= [0]
         @logger = logger
         @command = command
         @stdin = options[:stdin]
         @hidden_patterns = Array(options[:hidden_patterns])
         @interactive = options[:interactive]
         @options = options
+        @valid_exit_statuses = options[:valid_exit_statuses]
         raise ArgumentError, 'Can not pass stdin for interactive command' if @interactive && @stdin
       end
 
@@ -43,7 +45,7 @@ module ForemanMaintain
       end
 
       def success?
-        exit_status == 0
+        @valid_exit_statuses.include? exit_status
       end
 
       def execution_error

data/lib/foreman_maintain/utils/mongo_core.rb

@@ -15,6 +15,10 @@ module ForemanMaintain::Utils
     def dump_command
       'mongodump'
     end
+
+    def restore_command
+      'mongorestore'
+    end
   end
 
   class MongoCore34 < MongoCore
@@ -33,5 +37,50 @@ module ForemanMaintain::Utils
     def dump_command
       'scl enable rh-mongodb34 -- mongodump'
     end
+
+    def restore_command
+      'scl enable rh-mongodb34 -- mongorestore'
+    end
+  end
+
+  class MongoCoreInstalled < MongoCore
+    include ForemanMaintain::Concerns::SystemHelpers
+
+    attr_reader :services, :server_config_files, :client_command, :dump_command
+
+    def initialize
+      @services = {}
+      @server_config_files = []
+
+      detect_mongo_default
+      detect_mongo_34
+      raise ForemanMaintain::Error::Fail, 'Mongo client was not found' unless @client_command
+    end
+
+    private
+
+    def detect_mongo_34
+      if find_package('rh-mongodb34-mongodb-server')
+        @services['rh-mongodb34-mongod'] = 5
+        @server_config_files << '/etc/opt/rh/rh-mongodb34/mongod.conf'
+      end
+
+      if find_package('rh-mongodb34-mongodb')
+        @client_command = 'scl enable rh-mongodb34 -- mongo'
+        @dump_command = 'scl enable rh-mongodb34 -- mongodump'
+      end
+    end
+
+    def detect_mongo_default
+      if find_package('mongodb-server')
+        @services['mongod'] = 5
+        @server_config_files << '/etc/mongod.conf'
+      end
+
+      if find_package('mongodb')
+        @client_command = 'mongo'
+        @dump_command = 'mongodump'
+      end
+    end
   end
 end