digitalbits-core-backup 0.0.7

data/lib/digitalbits-core-backup/job.rb-e

@@ -0,0 +1,223 @@
+module DigitalbitsCoreBackup
+  class Job
+
+    class NoConfig < StandardError ; end
+
+    def initialize(**args)
+      if args.has_key?(:config) then
+        @config       = DigitalbitsCoreBackup::Config.new(args[:config])
+      else
+        puts "info: no config provided"
+        raise NoConfig
+      end
+
+      # Set run time options
+      @verify   = args[:verify] if args.has_key?(:verify)
+      @clean    = args[:clean] if args.has_key?(:clean)
+      @listlen  = args[:listlen]
+
+      # Set common run time parameters
+      @job_type        = args[:type]
+      @gpg_key         = @config.get('gpg_key')
+      @working_dir     = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @cmd             = DigitalbitsCoreBackup::Cmd.new(@working_dir)
+      @select          = args[:select] if args.has_key?(:select)
+      @s3              = DigitalbitsCoreBackup::S3.new(@config)
+      @pushgateway_url = @config.get('pushgateway_url')
+
+      # Set per operation type run time parameters
+      if args.has_key?(:type) then
+        case args[:type]
+          when 'backup'
+            puts 'info: backing up digitalbits-core'
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_start_time')
+            @backup_dir   = DigitalbitsCoreBackup::Utils.create_backup_dir(@config.get('backup_dir'))
+            @db           = DigitalbitsCoreBackup::Database.new(@config)
+            @fs           = DigitalbitsCoreBackup::Filesystem.new(@config)
+          when 'restore'
+            puts 'info: restoring digitalbits-core'
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_start_time')
+            @db_restore   = DigitalbitsCoreBackup::Restore::Database.new(@config)
+            @fs_restore   = DigitalbitsCoreBackup::Restore::Filesystem.new(@config)
+            @utils        = DigitalbitsCoreBackup::Utils.new(@config)
+          when 'getkey'
+            puts 'info: confirming public gpg key with key server'
+          when 'list'
+            puts "info: listing last #{@listlen} digitalbits-core backups"
+        end
+      end
+    end
+
+    def run()
+      case @job_type
+        when 'getkey'
+          begin
+            getkey = @cmd.run_and_capture('gpg', ['--keyserver', 'hkp://ipv4.pool.sks-keyservers.net', '--recv-key', @gpg_key, '2>&1'])
+            puts 'info: public gpg key installed'
+            if ! getkey.success then
+              puts "error: failed to get gpg key"
+              # dump the gpg output here for user level trouble shooting
+              puts "#{getkey.out}"
+              raise StandardError
+            end
+          rescue => e
+            puts e
+          end
+        when 'list'
+          begin
+            list=@s3.latest(@listlen)
+            puts "info: only #{list.length} backup files in bucket" if list.length < @listlen
+            puts list
+          rescue => e
+            puts e
+          end
+        when 'backup'
+          begin
+            if !DigitalbitsCoreBackup::Utils.core_healthy?(@config) then
+                puts "error: Can't back up unhealthy digitalbits-core"
+                raise StandardError
+            end
+            puts 'info: stopping digitalbits-core'
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'digitalbits-core'])
+            # only proceed if core is stopped
+            if stop_core.success then
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_db_dump_start_time')
+              @db.backup
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_db_dump_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_fs_backup_start_time')
+              @fs.backup
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_fs_backup_finish_time')
+              if @verify then
+                DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_verify_start_time')
+                create_hash_file = @cmd.run_and_capture('find', ['.', '-type', 'f', '!', '-name', 'SHA256SUMS', '|', 'xargs', 'sha256sum', '>', 'SHA256SUMS'])
+                if create_hash_file.success then
+                  puts "info: sha sums file created"
+                else
+                  puts 'error: error creating sha sums file'
+                  raise StandardError
+                end
+                sign_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--detach-sign', 'SHA256SUMS'])
+                if sign_hash_file.success then
+                  puts "info: gpg signature created ok"
+                  DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_verify_finish_time')
+                else
+                  puts 'error: error signing sha256sum file'
+                  raise StandardError
+                end
+              end
+              # create tar archive with fs, db backup files and if requested the file of shas256sums and corresponding gpg signature.
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_tar_start_time')
+              @backup = DigitalbitsCoreBackup::Utils.create_backup_tar(@working_dir, @backup_dir)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_tar_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_s3_push_start_time')
+              @s3.push(@backup)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_s3_push_finish_time')
+            else
+              puts 'error: can not stop digitalbits-core'
+              raise StandardError
+            end
+
+            # restart digitalbits-core post backup
+            puts 'info: starting digitalbits-core'
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'digitalbits-core'])
+            if start_core.success then
+              puts "info: digitalbits-core started"
+            else
+              puts 'error: can not start digitalbits-core'
+              raise StandardError
+            end
+
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+          rescue => e
+            puts e
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_fail_time')
+          else
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_success_time')
+          end
+        when 'restore'
+          begin
+            # confirm the bucket directory is set to be cleaned or is empty
+            # the fs_restore.core_data_dir_empty? throws an exception if it's not empty
+            if ! @clean then
+              @fs_restore.core_data_dir_empty?()
+            end
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'digitalbits-core'])
+            # only proceed if core is stopped
+            if stop_core.success then
+              # if no manual selection has been made, use the latest as derived from the s3.latest method
+              # this method returns an array so set @select to the first and only element
+              @select=@s3.latest(1)[0] if ! @select
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_s3_get_start_time')
+              @backup_archive = @s3.get(@select)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_s3_get_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_untar_start_time')
+              @utils.extract_backup(@backup_archive)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_untar_finish_time')
+              if @verify then
+                DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_verify_start_time')
+                verify_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--verify', 'SHA256SUMS.sig', 'SHA256SUMS', '2>&1'])
+                if verify_hash_file.success then
+                  puts "info: gpg signature processed ok"
+                else
+                  puts 'error: error verifying gpg signature'
+                  raise StandardError
+                end
+                verify_sha_file_content = @cmd.run_and_capture('sha256sum', ['--status', '--strict', '-c', 'SHA256SUMS'])
+                if verify_sha_file_content.success then
+                  puts "info: sha file sums match"
+                else
+                  puts 'error: error processing sha256sum file'
+                  raise StandardError
+                end
+                if DigitalbitsCoreBackup::Utils.confirm_shasums_definitive(@working_dir, @backup_archive) then
+                  puts 'info: SHA256SUMS file list matches delivered archive'
+                  DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_verify_finish_time')
+                else
+                  puts 'error: unknown additional file(s) detected in archive'
+                  raise StandardError
+                end
+              end
+              DigitalbitsCoreBackup::Utils.cleanbucket(@fs_restore.core_data_dir) if @clean
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_fs_restore_start_time')
+              @fs_restore.restore(@backup_archive)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_fs_restore_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_db_restore_start_time')
+              @db_restore.restore()
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_db_restore_finish_time')
+
+              # restart digitalbits-core post restore
+              puts 'info: starting digitalbits-core'
+              # using sudo, if running as non root uid then you will need to configure sudoers
+              start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'digitalbits-core'])
+              if start_core.success then
+                puts "info: digitalbits-core started"
+              else
+                puts 'error: can not start digitalbits-core'
+                raise StandardError
+              end
+            else
+              puts 'error: can not stop digitalbits-core'
+              raise StandardError
+            end
+
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+          rescue => e
+            puts e
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_fail_time')
+          else
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_success_time')
+          end
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/restore/database.rb

@@ -0,0 +1,36 @@
+require 'pg'
+
+module DigitalbitsCoreBackup::Restore
+  class Database < DigitalbitsCoreBackup::Database
+    include Contracts
+
+    attr_reader :dbname
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @dbname       = check_db_connection
+      @cmd          = DigitalbitsCoreBackup::Cmd.new(@working_dir)
+    end
+
+    public
+    Contract nil => nil
+    def restore()
+      puts "info: database restored" if pg_restore()
+    end
+
+    private
+    Contract nil => Bool
+    def pg_restore()
+      # we are restoring to public schema
+      pg_restore = @cmd.run('pg_restore', ['-n', 'public', '--jobs', "#{DigitalbitsCoreBackup::Utils.num_cores?()}", '-c', '-d', @dbname, 'core-db/'])
+      if pg_restore.success then
+        return true
+      else
+        return false
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/restore/database.rb-e

@@ -0,0 +1,36 @@
+require 'pg'
+
+module DigitalbitsCoreBackup::Restore
+  class Database < DigitalbitsCoreBackup::Database
+    include Contracts
+
+    attr_reader :dbname
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @dbname       = check_db_connection
+      @cmd          = DigitalbitsCoreBackup::Cmd.new(@working_dir)
+    end
+
+    public
+    Contract nil => nil
+    def restore()
+      puts "info: database restored" if pg_restore()
+    end
+
+    private
+    Contract nil => Bool
+    def pg_restore()
+      # we are restoring to public schema
+      pg_restore = @cmd.run('pg_restore', ['-n', 'public', '--jobs', "#{DigitalbitsCoreBackup::Utils.num_cores?()}", '-c', '-d', @dbname, 'core-db/'])
+      if pg_restore.success then
+        return true
+      else
+        return false
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/restore/filesystem.rb

@@ -0,0 +1,36 @@
+module DigitalbitsCoreBackup::Restore
+  class Filesystem < ::DigitalbitsCoreBackup::Filesystem
+    include Contracts
+
+    class DataDirNotEmpty < StandardError ; end
+
+    attr_reader :core_data_dir
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config         = config
+      @working_dir    = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @core_data_dir  = get_core_data_dir(@config.get('core_config'))
+    end
+
+    public
+
+    Contract String => nil
+    def restore(backup_archive)
+      # unpack the filesystem backup
+      puts "info: digitalbits-core buckets restored" if DigitalbitsCoreBackup::Tar.unpack("#{@working_dir}/core-fs.tar", @core_data_dir)
+    end
+
+    Contract nil => Bool
+    def core_data_dir_empty?()
+      # checks fs and asks user to remove fs manually if fs is already in place.
+      if (Dir.entries(@core_data_dir) - %w{ . .. }).empty? then
+        return true
+      else
+        puts "error: #{@core_data_dir} is not empty, you can only restore to an empty data directory"
+        raise DataDirNotEmpty
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/restore/filesystem.rb-e

@@ -0,0 +1,36 @@
+module DigitalbitsCoreBackup::Restore
+  class Filesystem < ::DigitalbitsCoreBackup::Filesystem
+    include Contracts
+
+    class DataDirNotEmpty < StandardError ; end
+
+    attr_reader :core_data_dir
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config         = config
+      @working_dir    = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @core_data_dir  = get_core_data_dir(@config.get('core_config'))
+    end
+
+    public
+
+    Contract String => nil
+    def restore(backup_archive)
+      # unpack the filesystem backup
+      puts "info: digitalbits-core buckets restored" if DigitalbitsCoreBackup::Tar.unpack("#{@working_dir}/core-fs.tar", @core_data_dir)
+    end
+
+    Contract nil => Bool
+    def core_data_dir_empty?()
+      # checks fs and asks user to remove fs manually if fs is already in place.
+      if (Dir.entries(@core_data_dir) - %w{ . .. }).empty? then
+        return true
+      else
+        puts "error: #{@core_data_dir} is not empty, you can only restore to an empty data directory"
+        raise DataDirNotEmpty
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/s3.rb

@@ -0,0 +1,64 @@
+require 'aws-sdk-s3'
+
+module DigitalbitsCoreBackup
+  class S3
+    include Contracts
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @s3_region    = @config.get('s3_region')
+      @s3_bucket    = @config.get('s3_bucket')
+      @s3_path      = @config.get('s3_path')
+      begin
+        @s3_client    = Aws::S3::Client.new(region: @s3_region)
+        @s3_resource  = Aws::S3::Resource.new(client: @s3_client)
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error connecting to s3"
+        puts e
+      end
+    end
+
+    def push(file)
+      begin
+        upload = @s3_resource.bucket(@s3_bucket).object("#{@s3_path}/#{File.basename(file)}")
+        if upload.upload_file(file) then
+          puts "info: pushed #{file} to s3 (#{@s3_bucket})"
+        else
+          puts "error: upload to s3 failed"
+        end
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error pushing #{file} to s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+    # fetches a backup tar file from s3, places in working dir
+    def get(file)
+      local_copy = "#{@working_dir}/#{File.basename(file)}"
+      begin
+        download = @s3_resource.bucket(@s3_bucket).object(file)
+        if download.download_file(local_copy) then
+          puts "info: fetched #{file} from s3 (#{@s3_bucket})"
+          return local_copy
+        else
+          puts "error: download from s3 failed"
+        end
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error downloading #{file} from s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+    def latest(listlen)
+      begin
+        @s3_client.list_objects_v2({bucket: @s3_bucket, prefix: @s3_path+'/core-backup-'}).contents.map{|o| o.key}.sort{|a,b| a.gsub(/(\d+)/,'\1') <=> b.gsub(/(\d+)/,'\1')}.last(listlen)
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error listing s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/s3.rb-e

@@ -0,0 +1,64 @@
+require 'aws-sdk-s3'
+
+module DigitalbitsCoreBackup
+  class S3
+    include Contracts
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @s3_region    = @config.get('s3_region')
+      @s3_bucket    = @config.get('s3_bucket')
+      @s3_path      = @config.get('s3_path')
+      begin
+        @s3_client    = Aws::S3::Client.new(region: @s3_region)
+        @s3_resource  = Aws::S3::Resource.new(client: @s3_client)
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error connecting to s3"
+        puts e
+      end
+    end
+
+    def push(file)
+      begin
+        upload = @s3_resource.bucket(@s3_bucket).object("#{@s3_path}/#{File.basename(file)}")
+        if upload.upload_file(file) then
+          puts "info: pushed #{file} to s3 (#{@s3_bucket})"
+        else
+          puts "error: upload to s3 failed"
+        end
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error pushing #{file} to s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+    # fetches a backup tar file from s3, places in working dir
+    def get(file)
+      local_copy = "#{@working_dir}/#{File.basename(file)}"
+      begin
+        download = @s3_resource.bucket(@s3_bucket).object(file)
+        if download.download_file(local_copy) then
+          puts "info: fetched #{file} from s3 (#{@s3_bucket})"
+          return local_copy
+        else
+          puts "error: download from s3 failed"
+        end
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error downloading #{file} from s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+    def latest(listlen)
+      begin
+        @s3_client.list_objects_v2({bucket: @s3_bucket, prefix: @s3_path+'/core-backup-'}).contents.map{|o| o.key}.sort{|a,b| a.gsub(/(\d+)/,'\1') <=> b.gsub(/(\d+)/,'\1')}.last(listlen)
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error listing s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/tar.rb

@@ -0,0 +1,37 @@
+require 'fileutils'
+
+module DigitalbitsCoreBackup
+  class Tar
+    include Contracts
+
+    Contract String, String => String
+    def self.pack(archive, directory)
+      if DigitalbitsCoreBackup::Utils.readable?(directory) then
+        # archive directory
+        puts "info: packing #{directory} in #{archive}"
+        %x{/bin/tar --create --file=#{archive} #{directory}}
+        if $?.exitstatus == 0 then
+          puts "info: #{archive} created"
+          return archive
+        else
+          raise StandardError
+        end
+      end
+    end
+
+    Contract String, String => String
+    def self.unpack(archive, destination)
+      if DigitalbitsCoreBackup::Utils.writable?(destination) then
+        # extract archive in destination directory
+        puts "info: unpacking #{archive} in #{destination}"
+        %x{/bin/tar --extract --file=#{archive} --directory=#{destination}}
+        if $?.exitstatus == 0 then
+          puts "info: #{archive} unpacked in #{destination}"
+          return destination
+        else
+          raise StandardError
+        end
+      end
+    end
+  end
+end

data/lib/digitalbits-core-backup/tar.rb-e

@@ -0,0 +1,37 @@
+require 'fileutils'
+
+module DigitalbitsCoreBackup
+  class Tar
+    include Contracts
+
+    Contract String, String => String
+    def self.pack(archive, directory)
+      if DigitalbitsCoreBackup::Utils.readable?(directory) then
+        # archive directory
+        puts "info: packing #{directory} in #{archive}"
+        %x{/bin/tar --create --file=#{archive} #{directory}}
+        if $?.exitstatus == 0 then
+          puts "info: #{archive} created"
+          return archive
+        else
+          raise StandardError
+        end
+      end
+    end
+
+    Contract String, String => String
+    def self.unpack(archive, destination)
+      if DigitalbitsCoreBackup::Utils.writable?(destination) then
+        # extract archive in destination directory
+        puts "info: unpacking #{archive} in #{destination}"
+        %x{/bin/tar --extract --file=#{archive} --directory=#{destination}}
+        if $?.exitstatus == 0 then
+          puts "info: #{archive} unpacked in #{destination}"
+          return destination
+        else
+          raise StandardError
+        end
+      end
+    end
+  end
+end