digitalbits-core-backup 0.0.7

data/lib/digitalbits-core-backup/database.rb

@@ -0,0 +1,67 @@
+require 'pg'
+
+module DigitalbitsCoreBackup
+  class Database
+    include Contracts
+
+    attr_reader :dbname
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @dbname       = check_db_connection
+    end
+
+    public
+    def backup()
+      pg_dump()
+    end
+
+    private
+    #Contract nil, DigitalbitsCoreBackup::CmdResult
+    def pg_dump()
+      cmd = DigitalbitsCoreBackup::Cmd.new(@working_dir)
+      puts "info: connecting (#{get_db_details(@config.get('core_config')).gsub(/password=(.*)/, 'password=********')})"
+      pg_dump = cmd.run('pg_dump', ['--dbname', @dbname, '--format', 'd', '--jobs', "#{DigitalbitsCoreBackup::Utils.num_cores?()}", '--file', "#{@working_dir}/core-db/"])
+      if pg_dump.success then
+        puts "info: database backup complete"
+      end
+    end
+
+    private
+    # check we have passwordless access to postgres
+    Contract nil => Contracts::Any
+    def check_db_connection()
+      # TODO move to config
+      dbname = get_db_details(@config.get('core_config'))
+      begin
+        conn = PG.connect(dbname)
+        conn.exec("SELECT * FROM pg_stat_activity") do |result|
+          # 2 => PGRES_TUPLES_OK
+          if result.result_status == 2 then
+            return dbname
+          end
+        end
+      rescue PG::Error
+        puts "error: failed to connect to db"
+        exit
+      end
+    end
+
+    private
+    Contract String => String
+    def get_db_details(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^DATABASE=/]) then
+            # "postgresql://dbname=digitalbits user=digitalbits"
+            connection_str = /^DATABASE=(.*)/.match(line).captures[0].gsub!(/"postgresql:\/\/(.*)"$/,'\1')
+            return connection_str
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/database.rb-e

@@ -0,0 +1,67 @@
+require 'pg'
+
+module DigitalbitsCoreBackup
+  class Database
+    include Contracts
+
+    attr_reader :dbname
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @dbname       = check_db_connection
+    end
+
+    public
+    def backup()
+      pg_dump()
+    end
+
+    private
+    #Contract nil, DigitalbitsCoreBackup::CmdResult
+    def pg_dump()
+      cmd = DigitalbitsCoreBackup::Cmd.new(@working_dir)
+      puts "info: connecting (#{get_db_details(@config.get('core_config')).gsub(/password=(.*)/, 'password=********')})"
+      pg_dump = cmd.run('pg_dump', ['--dbname', @dbname, '--format', 'd', '--jobs', "#{DigitalbitsCoreBackup::Utils.num_cores?()}", '--file', "#{@working_dir}/core-db/"])
+      if pg_dump.success then
+        puts "info: database backup complete"
+      end
+    end
+
+    private
+    # check we have passwordless access to postgres
+    Contract nil => Contracts::Any
+    def check_db_connection()
+      # TODO move to config
+      dbname = get_db_details(@config.get('core_config'))
+      begin
+        conn = PG.connect(dbname)
+        conn.exec("SELECT * FROM pg_stat_activity") do |result|
+          # 2 => PGRES_TUPLES_OK
+          if result.result_status == 2 then
+            return dbname
+          end
+        end
+      rescue PG::Error
+        puts "error: failed to connect to db"
+        exit
+      end
+    end
+
+    private
+    Contract String => String
+    def get_db_details(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^DATABASE=/]) then
+            # "postgresql://dbname=digitalbits user=digitalbits"
+            connection_str = /^DATABASE=(.*)/.match(line).captures[0].gsub!(/"postgresql:\/\/(.*)"$/,'\1')
+            return connection_str
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/filesystem.rb

@@ -0,0 +1,62 @@
+module DigitalbitsCoreBackup
+  class Filesystem
+    include Contracts
+
+    class ReadError < StandardError ; end
+
+    attr_reader :core_data_dir
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config         = config
+      @working_dir    = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @core_data_dir  = get_core_data_dir(@config.get('core_config'))
+    end
+
+    public
+    def backup()
+      create_core_data_tar()
+    end
+
+    private
+    #Contract nil, DigitalbitsCoreBackup::CmdResult
+    def create_core_data_tar()
+      if core_data_readable?() then
+        # create the tar balls
+        puts "info: creating filesystem backup"
+        Dir.chdir(@core_data_dir)
+        DigitalbitsCoreBackup::Tar.pack("#{@working_dir}/core-fs.tar", '.')
+        Dir.chdir(@working_dir)
+      end
+    end
+
+    private
+    # TODO: replace with Utils.readable ?
+    # check we have read access to the digitalbits-core data
+    Contract nil => Contracts::Any
+    def core_data_readable?()
+      if File.readable?("#{@core_data_dir}/digitalbits-core.lock") then
+        puts "info: processing #{@core_data_dir}"
+        return true
+      else
+        puts "error: can not access #{@core_data_dir}"
+        raise ReadError
+      end
+    end
+
+    private
+    Contract String => String
+    def get_core_data_dir(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^BUCKET_DIR_PATH=/]) then
+            # "postgresql://dbname=digitalbits user=digitalbits"
+            core_data_dir = /^BUCKET_DIR_PATH="(.*)"$/.match(line).captures[0]
+            return core_data_dir
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/filesystem.rb-e

@@ -0,0 +1,62 @@
+module DigitalbitsCoreBackup
+  class Filesystem
+    include Contracts
+
+    class ReadError < StandardError ; end
+
+    attr_reader :core_data_dir
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config         = config
+      @working_dir    = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @core_data_dir  = get_core_data_dir(@config.get('core_config'))
+    end
+
+    public
+    def backup()
+      create_core_data_tar()
+    end
+
+    private
+    #Contract nil, DigitalbitsCoreBackup::CmdResult
+    def create_core_data_tar()
+      if core_data_readable?() then
+        # create the tar balls
+        puts "info: creating filesystem backup"
+        Dir.chdir(@core_data_dir)
+        DigitalbitsCoreBackup::Tar.pack("#{@working_dir}/core-fs.tar", '.')
+        Dir.chdir(@working_dir)
+      end
+    end
+
+    private
+    # TODO: replace with Utils.readable ?
+    # check we have read access to the digitalbits-core data
+    Contract nil => Contracts::Any
+    def core_data_readable?()
+      if File.readable?("#{@core_data_dir}/digitalbits-core.lock") then
+        puts "info: processing #{@core_data_dir}"
+        return true
+      else
+        puts "error: can not access #{@core_data_dir}"
+        raise ReadError
+      end
+    end
+
+    private
+    Contract String => String
+    def get_core_data_dir(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^BUCKET_DIR_PATH=/]) then
+            # "postgresql://dbname=digitalbits user=digitalbits"
+            core_data_dir = /^BUCKET_DIR_PATH="(.*)"$/.match(line).captures[0]
+            return core_data_dir
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/job.rb

@@ -0,0 +1,223 @@
+module DigitalbitsCoreBackup
+  class Job
+
+    class NoConfig < StandardError ; end
+
+    def initialize(**args)
+      if args.has_key?(:config) then
+        @config       = DigitalbitsCoreBackup::Config.new(args[:config])
+      else
+        puts "info: no config provided"
+        raise NoConfig
+      end
+
+      # Set run time options
+      @verify   = args[:verify] if args.has_key?(:verify)
+      @clean    = args[:clean] if args.has_key?(:clean)
+      @listlen  = args[:listlen]
+
+      # Set common run time parameters
+      @job_type        = args[:type]
+      @gpg_key         = @config.get('gpg_key')
+      @working_dir     = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @cmd             = DigitalbitsCoreBackup::Cmd.new(@working_dir)
+      @select          = args[:select] if args.has_key?(:select)
+      @s3              = DigitalbitsCoreBackup::S3.new(@config)
+      @pushgateway_url = @config.get('pushgateway_url')
+
+      # Set per operation type run time parameters
+      if args.has_key?(:type) then
+        case args[:type]
+          when 'backup'
+            puts 'info: backing up digitalbits-core'
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_start_time')
+            @backup_dir   = DigitalbitsCoreBackup::Utils.create_backup_dir(@config.get('backup_dir'))
+            @db           = DigitalbitsCoreBackup::Database.new(@config)
+            @fs           = DigitalbitsCoreBackup::Filesystem.new(@config)
+          when 'restore'
+            puts 'info: restoring digitalbits-core'
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_start_time')
+            @db_restore   = DigitalbitsCoreBackup::Restore::Database.new(@config)
+            @fs_restore   = DigitalbitsCoreBackup::Restore::Filesystem.new(@config)
+            @utils        = DigitalbitsCoreBackup::Utils.new(@config)
+          when 'getkey'
+            puts 'info: confirming public gpg key with key server'
+          when 'list'
+            puts "info: listing last #{@listlen} digitalbits-core backups"
+        end
+      end
+    end
+
+    def run()
+      case @job_type
+        when 'getkey'
+          begin
+            getkey = @cmd.run_and_capture('gpg', ['--keyserver', 'hkp://ipv4.pool.sks-keyservers.net', '--recv-key', @gpg_key, '2>&1'])
+            puts 'info: public gpg key installed'
+            if ! getkey.success then
+              puts "error: failed to get gpg key"
+              # dump the gpg output here for user level trouble shooting
+              puts "#{getkey.out}"
+              raise StandardError
+            end
+          rescue => e
+            puts e
+          end
+        when 'list'
+          begin
+            list=@s3.latest(@listlen)
+            puts "info: only #{list.length} backup files in bucket" if list.length < @listlen
+            puts list
+          rescue => e
+            puts e
+          end
+        when 'backup'
+          begin
+            if !DigitalbitsCoreBackup::Utils.core_healthy?(@config) then
+                puts "error: Can't back up unhealthy digitalbits-core"
+                raise StandardError
+            end
+            puts 'info: stopping digitalbits-core'
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'digitalbits-core'])
+            # only proceed if core is stopped
+            if stop_core.success then
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_db_dump_start_time')
+              @db.backup
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_db_dump_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_fs_backup_start_time')
+              @fs.backup
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_fs_backup_finish_time')
+              if @verify then
+                DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_verify_start_time')
+                create_hash_file = @cmd.run_and_capture('find', ['.', '-type', 'f', '!', '-name', 'SHA256SUMS', '|', 'xargs', 'sha256sum', '>', 'SHA256SUMS'])
+                if create_hash_file.success then
+                  puts "info: sha sums file created"
+                else
+                  puts 'error: error creating sha sums file'
+                  raise StandardError
+                end
+                sign_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--detach-sign', 'SHA256SUMS'])
+                if sign_hash_file.success then
+                  puts "info: gpg signature created ok"
+                  DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_verify_finish_time')
+                else
+                  puts 'error: error signing sha256sum file'
+                  raise StandardError
+                end
+              end
+              # create tar archive with fs, db backup files and if requested the file of shas256sums and corresponding gpg signature.
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_tar_start_time')
+              @backup = DigitalbitsCoreBackup::Utils.create_backup_tar(@working_dir, @backup_dir)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_tar_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_s3_push_start_time')
+              @s3.push(@backup)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_s3_push_finish_time')
+            else
+              puts 'error: can not stop digitalbits-core'
+              raise StandardError
+            end
+
+            # restart digitalbits-core post backup
+            puts 'info: starting digitalbits-core'
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'digitalbits-core'])
+            if start_core.success then
+              puts "info: digitalbits-core started"
+            else
+              puts 'error: can not start digitalbits-core'
+              raise StandardError
+            end
+
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+          rescue => e
+            puts e
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_fail_time')
+          else
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_backup_success_time')
+          end
+        when 'restore'
+          begin
+            # confirm the bucket directory is set to be cleaned or is empty
+            # the fs_restore.core_data_dir_empty? throws an exception if it's not empty
+            if ! @clean then
+              @fs_restore.core_data_dir_empty?()
+            end
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'digitalbits-core'])
+            # only proceed if core is stopped
+            if stop_core.success then
+              # if no manual selection has been made, use the latest as derived from the s3.latest method
+              # this method returns an array so set @select to the first and only element
+              @select=@s3.latest(1)[0] if ! @select
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_s3_get_start_time')
+              @backup_archive = @s3.get(@select)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_s3_get_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_untar_start_time')
+              @utils.extract_backup(@backup_archive)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_untar_finish_time')
+              if @verify then
+                DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_verify_start_time')
+                verify_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--verify', 'SHA256SUMS.sig', 'SHA256SUMS', '2>&1'])
+                if verify_hash_file.success then
+                  puts "info: gpg signature processed ok"
+                else
+                  puts 'error: error verifying gpg signature'
+                  raise StandardError
+                end
+                verify_sha_file_content = @cmd.run_and_capture('sha256sum', ['--status', '--strict', '-c', 'SHA256SUMS'])
+                if verify_sha_file_content.success then
+                  puts "info: sha file sums match"
+                else
+                  puts 'error: error processing sha256sum file'
+                  raise StandardError
+                end
+                if DigitalbitsCoreBackup::Utils.confirm_shasums_definitive(@working_dir, @backup_archive) then
+                  puts 'info: SHA256SUMS file list matches delivered archive'
+                  DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_verify_finish_time')
+                else
+                  puts 'error: unknown additional file(s) detected in archive'
+                  raise StandardError
+                end
+              end
+              DigitalbitsCoreBackup::Utils.cleanbucket(@fs_restore.core_data_dir) if @clean
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_fs_restore_start_time')
+              @fs_restore.restore(@backup_archive)
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_fs_restore_finish_time')
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_db_restore_start_time')
+              @db_restore.restore()
+              DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_db_restore_finish_time')
+
+              # restart digitalbits-core post restore
+              puts 'info: starting digitalbits-core'
+              # using sudo, if running as non root uid then you will need to configure sudoers
+              start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'digitalbits-core'])
+              if start_core.success then
+                puts "info: digitalbits-core started"
+              else
+                puts 'error: can not start digitalbits-core'
+                raise StandardError
+              end
+            else
+              puts 'error: can not stop digitalbits-core'
+              raise StandardError
+            end
+
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+          rescue => e
+            puts e
+            # clean up working_dir
+            DigitalbitsCoreBackup::Utils.cleanup(@working_dir)
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_fail_time')
+          else
+            DigitalbitsCoreBackup::Utils.push_metric(@pushgateway_url, 'digitalbits_core_restore_success_time')
+          end
+      end
+    end
+
+  end
+end