foreman_maintain 0.5.4 → 0.6.4

data/definitions/features/pulpcore.rb

@@ -0,0 +1,28 @@
+require 'foreman_maintain/utils/service/systemd'
+
+class Features::Pulpcore < ForemanMaintain::Feature
+  metadata do
+    label :pulpcore
+
+    confine do
+      ForemanMaintain::Utils::Service::Systemd.new('pulpcore-api', 0).exist?
+    end
+  end
+
+  def services
+    [
+      system_service('rh-redis5-redis', 5),
+      system_service('pulpcore-api', 10),
+      system_service('pulpcore-content', 10),
+      system_service('pulpcore-resource-manager', 10),
+      system_service('pulpcore-worker@*', 20, :all => true, :skip_enablement => true),
+      system_service('httpd', 30)
+    ]
+  end
+
+  def config_files
+    [
+      '/etc/pulp/settings.py'
+    ]
+  end
+end

data/definitions/features/pulpcore_database.rb

@@ -0,0 +1,39 @@
+class Features::PulpcoreDatabase < ForemanMaintain::Feature
+  PULPCORE_DB_CONFIG = '/etc/pulp/settings.py'.freeze
+
+  include ForemanMaintain::Concerns::BaseDatabase
+
+  metadata do
+    label :pulpcore_database
+
+    confine do
+      file_nonzero?(PULPCORE_DB_CONFIG)
+    end
+  end
+
+  def configuration
+    @configuration || load_configuration
+  end
+
+  def services
+    [
+      system_service('postgresql', 10, :component => 'pulpcore',
+                                       :db_feature => feature(:pulpcore_database))
+    ]
+  end
+
+  private
+
+  def load_configuration
+    full_config = File.read(PULPCORE_DB_CONFIG).split(/[\s,'":]/).reject(&:empty?)
+
+    @configuration = {}
+    @configuration['adapter'] = 'postgresql'
+    @configuration['host'] = full_config[full_config.index('HOST') + 1]
+    @configuration['port'] = full_config[full_config.index('PORT') + 1]
+    @configuration['database'] = full_config[full_config.index('NAME') + 1]
+    @configuration['username'] = full_config[full_config.index('USER') + 1]
+    @configuration['password'] = full_config[full_config.index('PASSWORD') + 1]
+    @configuration
+  end
+end

data/definitions/features/redis.rb

@@ -0,0 +1,34 @@
+class Features::Redis < ForemanMaintain::Feature
+  metadata do
+    label :redis
+
+    confine do
+      # Luckily, the service name is the same as the package providing it
+      server? && find_package(service_name)
+    end
+  end
+
+  def services
+    [system_service(self.class.service_name, 10)]
+  end
+
+  def config_files
+    %w[redis redis.conf].map { |config| File.join(self.class.etc_prefix, config) }
+  end
+
+  class << self
+    SCL_NAME = 'rh-redis5'.freeze
+
+    def etc_prefix
+      "/etc/opt/rh/#{SCL_NAME}"
+    end
+
+    def scl_prefix
+      "#{SCL_NAME}-"
+    end
+
+    def service_name
+      "#{scl_prefix}redis"
+    end
+  end
+end

data/definitions/features/service.rb

@@ -27,7 +27,7 @@ class Features::Service < ForemanMaintain::Feature
   end
 
   def filtered_services(options)
-    service_list = existing_services
+    service_list = include_unregistered_services(existing_services, options[:include])
     service_list = filter_services(service_list, options)
     raise 'No services found matching your parameters' unless service_list.any?
 
@@ -99,9 +99,14 @@ class Features::Service < ForemanMaintain::Feature
     %w[start stop restart status enable disable].include?(action)
   end
 
-  def filter_services(service_list, options)
-    service_list = include_unregistered_services(service_list, options[:include])
+  def extend_service_list_with_sockets(service_list, options)
+    return service_list unless options[:include_sockets]
+
+    socket_list = service_list.map(&:socket).compact.select(&:exist?)
+    service_list + socket_list
+  end
 
+  def filter_services(service_list, options)
     if options[:only] && options[:only].any?
       service_list = service_list.select do |service|
         options[:only].any? { |opt| service.matches?(opt) }
@@ -112,6 +117,8 @@ class Features::Service < ForemanMaintain::Feature
     if options[:exclude] && options[:exclude].any?
       service_list = service_list.reject { |service| options[:exclude].include?(service.name) }
     end
+
+    service_list = extend_service_list_with_sockets(service_list, options)
     service_list.sort
   end
 

data/definitions/features/tar.rb

@@ -20,9 +20,23 @@ class Features::Tar < ForemanMaintain::Feature
   # @option options [Boolean] :gzip filter the archive through gzip
   # @option options [Boolean] :ignore_failed_read do not fail on missing files
   # @option options [Boolean] :allow_changing_files do not fail on changing files
+  def run(options = {})
+    logger.debug("Invoking tar from #{options[:directory] || FileUtils.pwd}")
+    statuses = options[:allow_changing_files] ? [0, 1] : [0]
+    execute!(tar_command(options), :valid_exit_statuses => statuses)
+  end
+
+  def validate_volume_size(size)
+    if size.nil? || size !~ /^\d+[bBcGKkMPTw]?$/
+      raise ForemanMaintain::Error::Validation,
+            "Please specify size according to 'tar --tape-length' format."
+    end
+    true
+  end
+
   # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
   # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-  def run(options = {})
+  def tar_command(options)
     volume_size = options.fetch(:volume_size, nil)
     absolute_names = options.fetch(:absolute_names, nil)
     validate_volume_size(volume_size) unless volume_size.nil?
@@ -66,29 +80,20 @@ class Features::Tar < ForemanMaintain::Feature
       tar_command << options.fetch(:files, '*')
     end
 
-    logger.debug("Invoking tar from #{options[:directory] || FileUtils.pwd}")
-    statuses = options[:allow_changing_files] ? [0, 1] : [0]
-    execute!(tar_command.join(' '), :valid_exit_statuses => statuses)
+    tar_command.join(' ')
   end
   # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
   # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
-  def validate_volume_size(size)
-    if size.nil? || size !~ /^\d+[bBcGKkMPTw]?$/
-      raise ForemanMaintain::Error::Validation,
-            "Please specify size according to 'tar --tape-length' format."
-    end
-    true
-  end
-
   private
 
   def default_split_tar_script
-    utils_path = File.expand_path('../../../bin', __FILE__)
+    utils_path = File.expand_path('../../bin', __dir__)
     split_tar_script = File.join(utils_path, 'foreman-maintain-rotate-tar')
     unless File.executable?(split_tar_script)
       raise ForemanMaintain::Error::Fail, "Script #{split_tar_script} is not executable"
     end
+
     split_tar_script
   end
 end

data/definitions/procedures/backup/config_files.rb

@@ -1,5 +1,8 @@
 module Procedures::Backup
   class ConfigFiles < ForemanMaintain::Procedure
+    MAX_RETRIES = 3
+    RETRY_DELAY = 10
+
     metadata do
       description 'Backup config files'
       tags :backup
@@ -8,6 +11,7 @@ module Procedures::Backup
           Checks::Backup::CertsTarExist.new
         end
       end
+
       param :backup_dir, 'Directory where to backup to', :required => true
       param :proxy_features, 'List of proxy features to backup (default: all)',
             :array => true, :default => ['all']
@@ -17,19 +21,31 @@ module Procedures::Backup
             :flag => true, :default => false
     end
 
+    # rubocop:disable Metrics/MethodLength
     def run
-      tarball = File.join(@backup_dir, 'config_files.tar.gz')
-      increments = File.join(@backup_dir, '.config.snar')
-      with_spinner('Collecting config files to backup') do
-        configs, to_exclude = config_files
-        feature(:tar).run(
-          :command => 'create', :gzip => true, :archive => tarball,
-          :listed_incremental => increments, :ignore_failed_read => true,
-          :exclude => to_exclude, :allow_changing_files => @ignore_changed_files,
-          :files => configs.join(' ')
-        )
+      logger.debug("Invoking tar from #{FileUtils.pwd}")
+      tar_cmd = tar_command
+      attempt_no = 1
+      loop do
+        runner = nil
+        with_spinner('Collecting config files to backup') do
+          runner = execute_runner(tar_cmd, :valid_exit_statuses => [0, 1])
+        end
+        break if runner.exit_status == 0 || @ignore_changed_files
+
+        puts "WARNING: Attempt #{attempt_no}/#{MAX_RETRIES} to collect all config files failed!"
+        puts 'Some files were modified during creation of the archive.'
+        if attempt_no == MAX_RETRIES
+          raise runner.execution_error
+        else
+          attempt_no += 1
+          FileUtils.rm_rf(tarball_path)
+          puts "Waiting #{RETRY_DELAY} seconds before re-try"
+          sleep(RETRY_DELAY)
+        end
       end
     end
+    # rubocop:enable Metrics/MethodLength
 
     # rubocop:disable Metrics/AbcSize
     def config_files
@@ -54,5 +70,22 @@ module Procedures::Backup
       [configs, exclude_configs]
     end
     # rubocop:enable Metrics/AbcSize
+
+    private
+
+    def tar_command
+      increments_path = File.join(@backup_dir, '.config.snar')
+      configs, to_exclude = config_files
+
+      feature(:tar).tar_command(
+        :command => 'create', :gzip => true, :archive => tarball_path,
+        :listed_incremental => increments_path, :ignore_failed_read => true,
+        :exclude => to_exclude, :files => configs.join(' ')
+      )
+    end
+
+    def tarball_path
+      @tarball_path ||= File.join(@backup_dir, 'config_files.tar.gz')
+    end
   end
 end

data/definitions/procedures/backup/offline/pulpcore_db.rb

@@ -0,0 +1,57 @@
+module Procedures::Backup
+  module Offline
+    class PulpcoreDB < ForemanMaintain::Procedure
+      metadata do
+        description 'Backup Pulpcore DB offline'
+        tags :backup
+        label :backup_offline_pulpcore_db
+        for_feature :pulpcore_database
+        preparation_steps { Checks::Pulpcore::DBUp.new unless feature(:pulpcore_database).local? }
+        param :backup_dir, 'Directory where to backup to', :required => true
+        param :tar_volume_size, 'Size of tar volume (indicates splitting)'
+        param :mount_dir, 'Snapshot mount directory'
+      end
+
+      def run
+        if feature(:pulpcore_database).local?
+          if File.exist?(pg_backup_file)
+            puts 'Already done'
+          else
+            local_backup
+          end
+        else
+          puts "Backup of #{pg_data_dir} is not supported for remote databases." \
+            ' Doing postgres dump instead...'
+          with_spinner('Getting Pulpcore DB dump') do
+            feature(:pulpcore_database).dump_db(File.join(@backup_dir, 'pulpcore.dump'))
+          end
+        end
+      end
+
+      private
+
+      def local_backup
+        with_spinner("Collecting data from #{pg_data_dir}") do
+          feature(:pulpcore_database).backup_local(
+            pg_backup_file,
+            :listed_incremental => File.join(@backup_dir, '.postgres.snar'),
+            :volume_size => @tar_volume_size,
+            :data_dir => pg_data_dir
+          )
+        end
+      end
+
+      def pg_backup_file
+        File.join(@backup_dir, 'pgsql_data.tar')
+      end
+
+      def pg_data_dir
+        return feature(:pulpcore_database).data_dir if @mount_dir.nil?
+        mount_point = File.join(@mount_dir, 'pgsql')
+        dir = feature(:pulpcore_database).find_base_directory(mount_point)
+        fail!("Snapshot of Pulpcore DB was not found mounted in #{mount_point}") if dir.nil?
+        dir
+      end
+    end
+  end
+end

data/definitions/procedures/backup/online/pulpcore_db.rb

@@ -0,0 +1,20 @@
+module Procedures::Backup
+  module Online
+    class PulpcoreDB < ForemanMaintain::Procedure
+      metadata do
+        description 'Backup Pulpcore database online'
+        tags :backup
+        label :backup_online_pulpcore_db
+        for_feature :pulpcore_database
+        preparation_steps { Checks::Pulpcore::DBUp.new }
+        param :backup_dir, 'Directory where to backup to', :required => true
+      end
+
+      def run
+        with_spinner('Getting Pulpcore DB dump') do
+          feature(:pulpcore_database).dump_db(File.join(@backup_dir, 'pulpcore.dump'))
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/pulp.rb

@@ -3,7 +3,7 @@ module Procedures::Backup
     metadata do
       description 'Backup Pulp data'
       tags :backup
-      for_feature :pulp
+      for_feature :pulp2
       param :backup_dir, 'Directory where to backup to', :required => true
       param :tar_volume_size, 'Size of tar volume (indicates splitting)'
       param :ensure_unchanged, 'Ensure the data did not change during backup'
@@ -39,9 +39,9 @@ module Procedures::Backup
     end
 
     def pulp_dir
-      return feature(:pulp).data_dir if @mount_dir.nil?
+      return feature(:pulp2).data_dir if @mount_dir.nil?
       mount_point = File.join(@mount_dir, 'pulp')
-      dir = feature(:pulp).find_marked_directory(mount_point)
+      dir = feature(:pulp2).find_marked_directory(mount_point)
       unless dir
         raise ForemanMaintain::Error::Fail,
               "Pulp base directory not found in the mount point (#{mount_point})"

data/definitions/procedures/backup/snapshot/logical_volume_confirmation.rb

@@ -12,10 +12,11 @@ module Procedures::Backup
         backup_lv = get_lv_info(@backup_dir)
 
         dbs = {}
-        dbs[:pulp] = 'Pulp' if feature(:pulp) && !@skip_pulp
+        dbs[:pulp] = 'Pulp' if feature(:pulp2) && !@skip_pulp
         dbs[:mongo] = 'Mongo' if db_local?(:mongo)
         dbs[:candlepin_database] = 'Candlepin' if db_local?(:candlepin_database)
         dbs[:foreman_database] = 'Foreman' if db_local?(:foreman_database)
+        dbs[:pulpcore_database] = 'Pulpcore' if db_local?(:pulpcore_database)
 
         shared_lv = dbs.inject([]) do |list, (db_label, db_name)|
           db_lv = get_lv_info(feature(db_label).data_dir)

data/definitions/procedures/backup/snapshot/mount_pulp.rb

@@ -5,7 +5,7 @@ module Procedures::Backup
       metadata do
         description 'Create and mount snapshot of Pulp data'
         tags :backup
-        for_feature :pulp
+        for_feature :pulp2
         MountBase.common_params(self)
         param :skip, 'Skip Pulp content during backup'
       end
@@ -13,8 +13,8 @@ module Procedures::Backup
       def run
         skip if @skip
         with_spinner('Creating snapshot of Pulp') do |spinner|
-          feature(:pulp).with_marked_directory(feature(:pulp).data_dir) do
-            lv_info = get_lv_info(feature(:pulp).data_dir)
+          feature(:pulp2).with_marked_directory(feature(:pulp2).data_dir) do
+            lv_info = get_lv_info(feature(:pulp2).data_dir)
             create_lv_snapshot('pulp-snap', @block_size, lv_info[0])
             spinner.update("Mounting snapshot of Pulp on #{mount_location('pulp')}")
             mount_snapshot('pulp', lv_info[1])

data/definitions/procedures/backup/snapshot/mount_pulpcore_db.rb

@@ -0,0 +1,48 @@
+require 'procedures/backup/snapshot/mount_base'
+module Procedures::Backup
+  module Snapshot
+    class MountPulpcoreDB < MountBase
+      metadata do
+        description 'Create and mount snapshot of Pulpcore DB'
+        tags :backup
+        label :backup_snapshot_mount_pulpcore_db
+        for_feature :pulpcore_database
+        preparation_steps { Checks::Pulpcore::DBUp.new unless feature(:pulpcore_database).local? }
+        MountBase.common_params(self)
+        param :backup_dir, 'Directory where to backup to'
+      end
+
+      def run
+        if feature(:pulpcore_database).local?
+          snapshot
+        else
+          dump_pulpcore
+        end
+      end
+
+      private
+
+      def dump_pulpcore
+        puts 'LV snapshots are not supported for remote databases. Doing postgres dump instead... '
+        with_spinner('Getting Pulpcore DB dump') do
+          feature(:pulpcore_database).dump_db(File.join(@backup_dir, 'pulpcore.dump'))
+        end
+      end
+
+      def snapshot
+        mount_point = mount_location('pgsql')
+        FileUtils.mkdir_p(mount_point) unless File.directory?(mount_point)
+        if directory_empty?(mount_point)
+          with_spinner('Creating snapshot of Postgres') do |spinner|
+            lv_info = get_lv_info(feature(:pulpcore_database).data_dir)
+            create_lv_snapshot('pgsql-snap', @block_size, lv_info[0])
+            spinner.update("Mounting snapshot of Postgres on #{mount_point}")
+            mount_snapshot('pgsql', lv_info[1])
+          end
+        else
+          puts 'Snapshot of Postgres is already mounted'
+        end
+      end
+    end
+  end
+end