foreman_maintain 0.1.6 → 0.2.0

data/definitions/procedures/backup/online/mongo.rb

@@ -0,0 +1,21 @@
+module Procedures::Backup
+  module Online
+    class Mongo < ForemanMaintain::Procedure
+      metadata do
+        description 'Backup Mongo online'
+        tags :backup
+        for_feature :mongo
+        preparation_steps do
+          [Checks::Mongo::DBUp.new, Checks::Mongo::ToolsInstalled.new]
+        end
+        param :backup_dir, 'Directory where to backup to', :required => true
+      end
+
+      def run
+        with_spinner('Getting dump of Mongo DB') do
+          feature(:mongo).dump(File.join(@backup_dir, 'mongo_dump'))
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/online/pg_global_objects.rb

@@ -0,0 +1,23 @@
+module Procedures::Backup
+  module Online
+    class PgGlobalObjects < ForemanMaintain::Procedure
+      metadata do
+        description 'Backup Postgres global objects online'
+        tags :backup
+        param :backup_dir, 'Directory where to backup to', :required => true
+        confine do
+          feature(:foreman_database) || feature(:candlepin_database)
+        end
+      end
+
+      def run
+        if feature(:foreman_database).local? || feature(:candlepin_database).local?
+          local_db = feature(:foreman_database).local? ? :foreman_database : :candlepin_database
+          feature(local_db).backup_global_objects(File.join(@backup_dir, 'pg_globals.dump'))
+        else
+          skip 'Backup of global objects is not supported for remote databases.'
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/online/safety_confirmation.rb

@@ -0,0 +1,25 @@
+module Procedures::Backup
+  module Online
+    class SafetyConfirmation < ForemanMaintain::Procedure
+      metadata do
+        description 'Data consistency warning'
+        tags :backup
+      end
+
+      def run
+        answer = ask_decision(
+          "*** WARNING: The online backup is intended for making a copy of the data\n" \
+          '*** for debugging purposes only.' \
+          " The backup routine can not ensure 100% consistency while the\n" \
+          "*** backup is taking place as there is a chance there may be data mismatch between\n" \
+          '*** Mongo and Postgres databases while the services are live.' \
+          " If you wish to utilize the online backup\n" \
+          '*** for production use you need to ensure that there are' \
+          " no modifications occurring during\n" \
+          "*** your backup run.\n\nDo you want to proceed?", 'y(yes), q(quit)'
+        )
+        abort! unless answer == :yes
+      end
+    end
+  end
+end

data/definitions/procedures/backup/prepare_directory.rb

@@ -0,0 +1,29 @@
+module Procedures::Backup
+  class PrepareDirectory < ForemanMaintain::Procedure
+    metadata do
+      description 'Prepare backup Directory'
+      tags :backup
+      param :backup_dir, 'Directory where to backup to', :required => true
+      param :preserve_dir, 'Directory where to backup to', :flag => true
+      param :incremental_dir, 'Changes since specified backup only'
+    end
+
+    def run
+      puts "Creating backup folder #{@backup_dir}"
+
+      unless @preserve_dir
+        FileUtils.mkdir_p @backup_dir
+        FileUtils.chmod_R 0o770, @backup_dir
+      end
+
+      if local_psql_database? && !@preserve_dir
+        FileUtils.chown_R(nil, 'postgres', @backup_dir)
+      end
+
+      FileUtils.rm(Dir.glob(File.join(@backup_dir, '.*.snar'))) if @preserve_dir
+      if @incremental_dir
+        FileUtils.cp(Dir.glob(File.join(@incremental_dir, '.*.snar')), @backup_dir)
+      end
+    end
+  end
+end

data/definitions/procedures/backup/pulp.rb

@@ -0,0 +1,68 @@
+module Procedures::Backup
+  class Pulp < ForemanMaintain::Procedure
+    metadata do
+      description 'Backup Pulp data'
+      tags :backup
+      for_feature :pulp
+      param :backup_dir, 'Directory where to backup to', :required => true
+      param :tar_volume_size, 'Size of tar volume (indicates splitting)'
+      param :ensure_unchanged, 'Ensure the data did not change during backup'
+      param :skip, 'Skip Pulp content during backup'
+      param :mount_dir, 'Snapshot mount directory'
+    end
+
+    def run
+      skip if @skip
+      with_spinner('Collecting Pulp data') do
+        FileUtils.cd(pulp_dir) do
+          if @ensure_unchanged
+            ensure_dir_unchanged { pulp_backup }
+          else
+            pulp_backup
+          end
+        end
+      end
+    end
+
+    private
+
+    def pulp_backup
+      feature(:tar).run(
+        :archive => File.join(@backup_dir, 'pulp_data.tar'),
+        :command => 'create',
+        :exclude => ['var/lib/pulp/katello-export'],
+        :listed_incremental => File.join(@backup_dir, '.pulp.snar'),
+        :transform => 's,^,var/lib/pulp/,S',
+        :volume_size => @tar_volume_size
+      )
+    end
+
+    def pulp_dir
+      return feature(:pulp).data_dir if @mount_dir.nil?
+      mount_point = File.join(@mount_dir, 'pulp')
+      feature(:pulp).find_base_directory(mount_point)
+    end
+
+    def ensure_dir_unchanged
+      matching = false
+      backup_file = File.join(@backup_dir, '.pulp.snar')
+      alternate_backup = File.join(@backup_dir, '.pulp.snar.backup')
+      until matching
+        FileUtils.cp(backup_file, alternate_backup) if File.exist? backup_file
+        checksum1 = compute_checksum
+        yield
+        checksum2 = compute_checksum
+        matching = (checksum1 == checksum2)
+        FileUtils.rm backup_file unless matching
+        if File.exist? alternate_backup
+          matching ? FileUtils.rm(alternate_backup) : FileUtils.mv(alternate_backup, backup_file)
+        end
+        logger.info("Data in #{pulp_dir} changed during backup. Retrying...") unless matching
+      end
+    end
+
+    def compute_checksum
+      execute!("find . -printf '%T@\n' | sha1sum")
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/clean_mount.rb

@@ -0,0 +1,24 @@
+module Procedures::Backup
+  module Snapshot
+    class CleanMount < ForemanMaintain::Procedure
+      metadata do
+        description 'Remove the snapshot mount points'
+        tags :backup
+        param :mount_dir, 'Snapshot mount directory', :required => true
+      end
+
+      def run
+        %w[pulp mongodb pgsql].each do |database|
+          mount_point = File.join(@mount_dir, database)
+
+          if File.exist?(mount_point) && !execute("mount|grep #{mount_point}").empty?
+            execute("umount #{mount_point}")
+          end
+
+          snapshot_location = get_lv_path("#{database}-snap")
+          execute("lvremove #{snapshot_location} -f") unless snapshot_location.empty?
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/logical_volume_confirmation.rb

@@ -0,0 +1,47 @@
+module Procedures::Backup
+  module Snapshot
+    class LogicalVolumeConfirmation < ForemanMaintain::Procedure
+      metadata do
+        description 'Check if backup is on different logical volume then the source'
+        tags :backup
+        param :backup_dir, 'Directory where to backup to', :required => true
+        param :skip_pulp, 'Skip Pulp content during backup'
+      end
+
+      def run
+        backup_lv = get_lv_info(@backup_dir)
+
+        dbs = {}
+        dbs[:pulp] = 'Pulp' if feature(:pulp) && !@skip_pulp
+        dbs[:mongo] = 'Mongo' if db_local?(:mongo)
+        dbs[:candlepin_database] = 'Candlepin' if db_local?(:candlepin_database)
+        dbs[:foreman_database] = 'Foreman' if db_local?(:foreman_database)
+
+        shared_lv = dbs.inject([]) do |list, (db_label, db_name)|
+          db_lv = get_lv_info(feature(db_label).data_dir)
+          list << db_name if db_lv == backup_lv
+          list
+        end
+        confirm(shared_lv) if shared_lv.any?
+      end
+
+      private
+
+      def db_local?(db)
+        feature(:instance).database_local?(db)
+      end
+
+      def confirm(shared_lv)
+        answer = ask_decision('*** WARNING: The chosen backup location is mounted on the same' \
+          " logical volume as the location of #{shared_lv.join(', ')}.\n" \
+          '*** It is highly suggested to backup to a different logical volume than' \
+          " the #{shared_lv.join(', ')} database.\n" \
+          '*** If you would like to continue, the snapshot size will be required to be at least' \
+          "the size of the actual #{shared_lv.join(', ')} database.\n" \
+          "*** You can skip this confirmation with the '-y' flag.\n\n" \
+          'Do you want to proceed(y/n)?', 'y(yes), q(quit)')
+        abort! unless answer == :yes
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/mount_base.rb

@@ -0,0 +1,27 @@
+module Procedures::Backup
+  module Snapshot
+    class MountBase < ForemanMaintain::Procedure
+      metadata do
+        advanced_run false
+      end
+
+      def self.common_params(context)
+        context.instance_eval do
+          param :mount_dir, 'Snapshot mount directory', :required => true
+          param :block_size, 'Snapshot block size', :default => '2G'
+        end
+      end
+
+      def mount_snapshot(database, lv_type)
+        FileUtils.mkdir_p(mount_location(database))
+        options = lv_type == 'xfs' ? '-onouuid,ro' : '-oro'
+        lv_name = "#{database}-snap"
+        execute!("mount #{get_lv_path(lv_name)} #{mount_location(database)} #{options}")
+      end
+
+      def mount_location(database)
+        File.join(@mount_dir, database)
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/mount_candlepin_db.rb

@@ -0,0 +1,48 @@
+require 'procedures/backup/snapshot/mount_base'
+module Procedures::Backup
+  module Snapshot
+    class MountCandlepinDB < MountBase
+      metadata do
+        description 'Create and mount snapshot of Candlepin DB'
+        tags :backup
+        label :backup_snapshot_mount_candlepin_db
+        for_feature :candlepin_database
+        preparation_steps { Checks::Candlepin::DBUp.new unless feature(:candlepin_database).local? }
+        MountBase.common_params(self)
+        param :backup_dir, 'Directory where to backup to'
+      end
+
+      def run
+        if feature(:candlepin_database).local?
+          snapshot
+        else
+          dump_candlepin
+        end
+      end
+
+      private
+
+      def dump_candlepin
+        puts 'LV snapshots are not supported for remote databases. Doing postgres dump instead... '
+        with_spinner('Getting Candlepin DB dump') do
+          feature(:candlepin_database).dump_db(File.join(@backup_dir, 'candlepin.dump'))
+        end
+      end
+
+      def snapshot
+        mount_point = mount_location('pgsql')
+        FileUtils.mkdir_p(mount_point) unless File.directory?(mount_point)
+        if directory_empty?(mount_point)
+          with_spinner('Creating snapshot of Postgres') do |spinner|
+            lv_info = get_lv_info(feature(:candlepin_database).data_dir)
+            create_lv_snapshot('pgsql-snap', @block_size, lv_info[0])
+            spinner.update("Mounting snapshot of Postgres on #{mount_point}")
+            mount_snapshot('pgsql', lv_info[1])
+          end
+        else
+          puts 'Snapshot of Postgres is already mounted'
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/mount_foreman_db.rb

@@ -0,0 +1,48 @@
+require 'procedures/backup/snapshot/mount_base'
+module Procedures::Backup
+  module Snapshot
+    class MountForemanDB < MountBase
+      metadata do
+        description 'Create and mount snapshot of Foreman DB'
+        tags :backup
+        label :backup_snapshot_mount_foreman_db
+        for_feature :foreman_database
+        preparation_steps { Checks::Foreman::DBUp.new unless feature(:foreman_database).local? }
+        MountBase.common_params(self)
+        param :backup_dir, 'Directory where to backup to'
+      end
+
+      def run
+        if feature(:foreman_database).local?
+          snapshot
+        else
+          dump_foreman
+        end
+      end
+
+      private
+
+      def dump_foreman
+        puts 'LV snapshots are not supported for remote databases. Doing postgres dump instead... '
+        with_spinner('Getting Foreman DB dump') do
+          feature(:foreman_database).dump_db(File.join(@backup_dir, 'foreman.dump'))
+        end
+      end
+
+      def snapshot
+        mount_point = mount_location('pgsql')
+        FileUtils.mkdir_p(mount_point) unless File.directory?(mount_point)
+        if directory_empty?(mount_point)
+          with_spinner('Creating snapshot of Postgres') do |spinner|
+            lv_info = get_lv_info(feature(:foreman_database).data_dir)
+            create_lv_snapshot('pgsql-snap', @block_size, lv_info[0])
+            spinner.update("Mounting snapshot of Postgres on #{mount_point}")
+            mount_snapshot('pgsql', lv_info[1])
+          end
+        else
+          puts 'Snapshot of Postgres is already mounted'
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/mount_mongo.rb

@@ -0,0 +1,35 @@
+require 'procedures/backup/snapshot/mount_base'
+module Procedures::Backup
+  module Snapshot
+    class MountMongo < MountBase
+      metadata do
+        description 'Create and mount snapshot of Mongo DB'
+        tags :backup
+        for_feature :mongo
+        preparation_steps do
+          unless feature(:mongo).local?
+            [Checks::Mongo::DBUp.new, Checks::Mongo::ToolsInstalled.new]
+          end
+        end
+        MountBase.common_params(self)
+        param :backup_dir, 'Directory where to backup to'
+      end
+
+      def run
+        if feature(:mongo).local?
+          with_spinner('Creating snapshot of Mongo DB') do |spinner|
+            lv_info = get_lv_info(feature(:mongo).data_dir)
+            create_lv_snapshot('mongodb-snap', @block_size, lv_info[0])
+            spinner.update("Mounting snapshot of Mongo DB on #{mount_location('mongodb')}")
+            mount_snapshot('mongodb', lv_info[1])
+          end
+        else
+          puts 'LV snapshots are not supported for remote databases. Doing dump instead...'
+          with_spinner('Getting dump of Mongo DB') do
+            feature(:mongo).dump(File.join(@backup_dir, 'mongo_dump'))
+          end
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/mount_pulp.rb

@@ -0,0 +1,24 @@
+require 'procedures/backup/snapshot/mount_base'
+module Procedures::Backup
+  module Snapshot
+    class MountPulp < MountBase
+      metadata do
+        description 'Create and mount snapshot of Pulp data'
+        tags :backup
+        for_feature :pulp
+        MountBase.common_params(self)
+        param :skip, 'Skip Pulp content during backup'
+      end
+
+      def run
+        skip if @skip
+        with_spinner('Creating snapshot of Pulp') do |spinner|
+          lv_info = get_lv_info(feature(:pulp).data_dir)
+          create_lv_snapshot('pulp-snap', @block_size, lv_info[0])
+          spinner.update("Mounting snapshot of Pulp on #{mount_location('pulp')}")
+          mount_snapshot('pulp', lv_info[1])
+        end
+      end
+    end
+  end
+end

data/definitions/procedures/backup/snapshot/prepare_mount.rb

@@ -0,0 +1,16 @@
+module Procedures::Backup
+  module Snapshot
+    class PrepareMount < ForemanMaintain::Procedure
+      metadata do
+        description 'Prepare mount point for the snapshot'
+        tags :backup
+        param :mount_dir, 'Snapshot mount directory', :required => true
+      end
+
+      def run
+        logger.debug("Creating snap dir: #{@mount_dir}")
+        FileUtils.mkdir_p @mount_dir
+      end
+    end
+  end
+end