stellar-core-backup 0.0.4

data/lib/stellar-core-backup.rb

@@ -0,0 +1,21 @@
+require "stellar-core-backup/version"
+require "contracts"
+require "fileutils"
+require "pg"
+
+module StellarCoreBackup
+  autoload :Cmd, "stellar-core-backup/cmd"
+  autoload :CmdResult, "stellar-core-backup/cmd_result"
+  autoload :Config, "stellar-core-backup/config"
+  autoload :Database, "stellar-core-backup/database"
+  autoload :Filesystem, "stellar-core-backup/filesystem"
+  autoload :Job, "stellar-core-backup/job"
+  autoload :S3, "stellar-core-backup/s3"
+  autoload :Tar, "stellar-core-backup/tar"
+  autoload :Utils, "stellar-core-backup/utils"
+
+  module Restore
+    autoload :Database, "stellar-core-backup/restore/database"
+    autoload :Filesystem, "stellar-core-backup/restore/filesystem"
+  end
+end

data/lib/stellar-core-backup/cmd.rb

@@ -0,0 +1,28 @@
+module StellarCoreBackup
+  class Cmd
+    include Contracts
+
+    Contract String => Any
+    def initialize(working_dir)
+      @working_dir = working_dir
+    end
+
+    Contract String, ArrayOf[String] => CmdResult
+    def run_and_capture(cmd, args)
+      Dir.chdir @working_dir do
+        stringArgs = args.map{|x| "#{x}"}.join(" ")
+        out = `#{cmd} #{stringArgs}`
+        CmdResult.new($?.exitstatus == 0, out)
+      end
+    end
+
+    Contract String, ArrayOf[String] => CmdResult
+    def run(cmd, args)
+      Dir.chdir @working_dir do
+        system(cmd, *args)
+      end
+      CmdResult.new($?.exitstatus == 0, nil)
+    end
+
+  end
+end

data/lib/stellar-core-backup/cmd_result.rb

@@ -0,0 +1,14 @@
+module StellarCoreBackup
+  class CmdResult
+    include Contracts
+
+    attr_reader :success
+    attr_reader :out
+
+    Contract Bool, Maybe[String] => Any
+    def initialize(success, out = None)
+      @success = success
+      @out = out
+    end
+  end
+end

data/lib/stellar-core-backup/config.rb

@@ -0,0 +1,37 @@
+require 'yaml'
+
+module StellarCoreBackup
+  class Config
+    include Contracts
+
+    class ReadError < StandardError ; end
+
+    Contract String => Any
+    def initialize(config)
+      if (File.exists?(config)) then
+        @config = YAML.load_file(config)
+      else
+        raise ReadError
+      end
+    end
+
+    Contract None => Bool
+    def configured?()
+      unless @config.nil? || @config.empty?
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String => Any
+    def get(item)
+      if self.configured?() then
+        @config[item]
+      else
+        raise ReadError
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/database.rb

@@ -0,0 +1,67 @@
+require 'pg'
+
+module StellarCoreBackup
+  class Database
+    include Contracts
+
+    attr_reader :dbname
+
+    Contract StellarCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @dbname       = check_db_connection
+    end
+
+    public
+    def backup()
+      pg_dump()
+    end
+
+    private
+    #Contract nil, StellarCoreBackup::CmdResult
+    def pg_dump()
+      cmd = StellarCoreBackup::Cmd.new(@working_dir)
+      puts "info: connecting (#{get_db_details(@config.get('core_config')).gsub(/password=(.*)/, 'password=********')})"
+      pg_dump = cmd.run('pg_dump', ['--dbname', @dbname, '--format', 'd', '--file', "#{@working_dir}/core-db/"])
+      if pg_dump.success then
+        puts "info: database backup complete"
+      end
+    end
+
+    private
+    # check we have passwordless access to postgres
+    Contract nil => Contracts::Any
+    def check_db_connection()
+      # TODO move to config
+      dbname = get_db_details(@config.get('core_config'))
+      begin
+        conn = PG.connect(dbname)
+        conn.exec("SELECT * FROM pg_stat_activity") do |result|
+          # 2 => PGRES_TUPLES_OK
+          if result.result_status == 2 then
+            return dbname
+          end
+        end
+      rescue PG::Error
+        puts "error: failed to connect to db"
+        exit
+      end
+    end
+
+    private
+    Contract String => String
+    def get_db_details(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^DATABASE=/]) then
+            # "postgresql://dbname=stellar user=stellar"
+            connection_str = /^DATABASE=(.*)/.match(line).captures[0].gsub!(/"postgresql:\/\/(.*)"$/,'\1')
+            return connection_str
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/filesystem.rb

@@ -0,0 +1,62 @@
+module StellarCoreBackup
+  class Filesystem
+    include Contracts
+
+    class ReadError < StandardError ; end
+
+    attr_reader :core_data_dir
+
+    Contract StellarCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config         = config
+      @working_dir    = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @core_data_dir  = get_core_data_dir(@config.get('core_config'))
+    end
+
+    public
+    def backup()
+      create_core_data_tar()
+    end
+
+    private
+    #Contract nil, StellarCoreBackup::CmdResult
+    def create_core_data_tar()
+      if core_data_readable?() then
+        # create the tar balls
+        puts "info: creating filesystem backup"
+        Dir.chdir(@core_data_dir)
+        StellarCoreBackup::Tar.pack("#{@working_dir}/core-fs.tar", '.')
+        Dir.chdir(@working_dir)
+      end
+    end
+
+    private
+    # TODO: replace with Utils.readable ?
+    # check we have read access to the stellar-core data
+    Contract nil => Contracts::Any
+    def core_data_readable?()
+      if File.readable?("#{@core_data_dir}/stellar-core.lock") then
+        puts "info: processing #{@core_data_dir}"
+        return true
+      else
+        puts "error: can not access #{@core_data_dir}"
+        raise ReadError
+      end
+    end
+
+    private
+    Contract String => String
+    def get_core_data_dir(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^BUCKET_DIR_PATH=/]) then
+            # "postgresql://dbname=stellar user=stellar"
+            core_data_dir = /^BUCKET_DIR_PATH="(.*)"$/.match(line).captures[0]
+            return core_data_dir
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/job.rb

@@ -0,0 +1,190 @@
+module StellarCoreBackup
+  class Job
+
+    class NoConfig < StandardError ; end
+
+    def initialize(**args)
+      if args.has_key?(:config) then
+        @config       = StellarCoreBackup::Config.new(args[:config])
+      else
+        puts "info: no config provided"
+        raise NoConfig
+      end
+
+      # Set run time options
+      @verify   = args[:verify] if args.has_key?(:verify)
+      @clean    = args[:clean] if args.has_key?(:clean)
+      @listlen  = args[:listlen]
+
+      # Set common run time parameters
+      @job_type     = args[:type]
+      @gpg_key      = @config.get('gpg_key')
+      @working_dir  = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @cmd          = StellarCoreBackup::Cmd.new(@working_dir)
+      @select       = args[:select] if args.has_key?(:select)
+      @s3           = StellarCoreBackup::S3.new(@config)
+
+      # Set per operation type run time parameters
+      if args.has_key?(:type) then
+        case args[:type]
+          when 'backup'
+            puts 'info: backing up stellar-core'
+            @backup_dir   = StellarCoreBackup::Utils.create_backup_dir(@config.get('backup_dir'))
+            @db           = StellarCoreBackup::Database.new(@config)
+            @fs           = StellarCoreBackup::Filesystem.new(@config)
+          when 'restore'
+            puts 'info: restoring stellar-core'
+            @db_restore   = StellarCoreBackup::Restore::Database.new(@config)
+            @fs_restore   = StellarCoreBackup::Restore::Filesystem.new(@config)
+            @utils        = StellarCoreBackup::Utils.new(@config)
+          when 'getkey'
+            puts 'info: confirming public gpg key with key server'
+          when 'list'
+            puts "info: listing last #{@listlen} stellar-core backups"
+        end
+      end
+    end
+
+    def run()
+      case @job_type
+        when 'getkey'
+          begin
+            getkey = @cmd.run_and_capture('gpg', ['--keyserver', 'hkp://ipv4.pool.sks-keyservers.net', '--recv-key', @gpg_key, '2>&1'])
+            puts 'info: public gpg key installed'
+            if ! getkey.success then
+              puts "error: failed to get gpg key"
+              # dump the gpg output here for user level trouble shooting
+              puts "#{getkey.out}"
+              raise StandardError
+            end
+          rescue => e
+            puts e
+          end
+        when 'list'
+          begin
+            list=@s3.latest(@listlen)
+            puts "info: only #{list.length} backup files in bucket" if list.length < @listlen
+            puts list
+          rescue => e
+            puts e
+          end
+        when 'backup'
+          begin
+            puts 'info: stopping stellar-core'
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
+            # only proceed if core is stopped
+            if stop_core.success then
+              @db.backup
+              @fs.backup
+              if @verify then
+                create_hash_file = @cmd.run_and_capture('find', ['.', '-type', 'f', '!', '-name', 'SHA256SUMS', '|', 'xargs', 'sha256sum', '>', 'SHA256SUMS'])
+                if create_hash_file.success then
+                  puts "info: sha sums file created"
+                else
+                  puts 'error: error creating sha sums file'
+                  raise StandardError
+                end
+                sign_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--detach-sign', 'SHA256SUMS'])
+                if sign_hash_file.success then
+                  puts "info: gpg signature created ok"
+                else
+                  puts 'error: error signing sha256sum file'
+                  raise StandardError
+                end
+              end
+              # create tar archive with fs, db backup files and if requested the file of shas256sums and corresponding gpg signature.
+              @backup = StellarCoreBackup::Utils.create_backup_tar(@working_dir, @backup_dir)
+              @s3.push(@backup)
+            else
+              puts 'error: can not stop stellar-core'
+              raise StandardError
+            end
+
+            # restart stellar-core post backup
+            puts 'info: starting stellar-core'
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'stellar-core'])
+            if start_core.success then
+              puts "info: stellar-core started"
+            else
+              puts 'error: can not start stellar-core'
+              raise StandardError
+            end
+
+            # clean up working_dir
+            StellarCoreBackup::Utils.cleanup(@working_dir)
+          rescue => e
+            puts e
+            # clean up working_dir
+            StellarCoreBackup::Utils.cleanup(@working_dir)
+          end
+        when 'restore'
+          begin
+            # confirm the bucket directory is set to be cleaned or is empty
+            # the fs_restore.core_data_dir_empty? throws an exception if it's not empty
+            if ! @clean then
+              @fs_restore.core_data_dir_empty?()
+            end
+            # using sudo, if running as non root uid then you will need to configure sudoers
+            stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
+            # only proceed if core is stopped
+            if stop_core.success then
+              # if no manual selection has been made, use the latest as derived from the s3.latest method
+              # this method returns an array so set @select to the first and only element
+              @select=@s3.latest(1)[0] if ! @select
+              @backup_archive = @s3.get(@select)
+              @utils.extract_backup(@backup_archive)
+              if @verify then
+                verify_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--verify', 'SHA256SUMS.sig', 'SHA256SUMS', '2>&1'])
+                if verify_hash_file.success then
+                  puts "info: gpg signature processed ok"
+                else
+                  puts 'error: error verifying gpg signature'
+                  raise StandardError
+                end
+                verify_sha_file_content = @cmd.run_and_capture('sha256sum', ['--status', '--strict', '-c', 'SHA256SUMS'])
+                if verify_sha_file_content.success then
+                  puts "info: sha file sums match"
+                else
+                  puts 'error: error processing sha256sum file'
+                  raise StandardError
+                end
+                if StellarCoreBackup::Utils.confirm_shasums_definitive(@working_dir, @backup_archive) then
+                  puts 'info: SHA256SUMS file list matches delivered archive'
+                else
+                  puts 'error: unknown additional file(s) detected in archive'
+                  raise StandardError
+                end
+              end
+              StellarCoreBackup::Utils.cleanbucket(@fs_restore.core_data_dir) if @clean
+              @fs_restore.restore(@backup_archive)
+              @db_restore.restore()
+
+              # restart stellar-core post restore
+              puts 'info: starting stellar-core'
+              # using sudo, if running as non root uid then you will need to configure sudoers
+              start_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'start', 'stellar-core'])
+              if start_core.success then
+                puts "info: stellar-core started"
+              else
+                puts 'error: can not start stellar-core'
+                raise StandardError
+              end
+            else
+              puts 'error: can not stop stellar-core'
+              raise StandardError
+            end
+
+            # clean up working_dir
+            StellarCoreBackup::Utils.cleanup(@working_dir)
+          rescue => e
+            puts e
+            # clean up working_dir
+            StellarCoreBackup::Utils.cleanup(@working_dir)
+          end
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/restore/database.rb

@@ -0,0 +1,36 @@
+require 'pg'
+
+module StellarCoreBackup::Restore
+  class Database < StellarCoreBackup::Database
+    include Contracts
+
+    attr_reader :dbname
+
+    Contract StellarCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @dbname       = check_db_connection
+      @cmd          = StellarCoreBackup::Cmd.new(@working_dir)
+    end
+
+    public
+    Contract nil => nil
+    def restore()
+      puts "info: database restored" if pg_restore()
+    end
+
+    private
+    Contract nil => Bool
+    def pg_restore()
+      # we are restoring to public schema
+      pg_restore = @cmd.run('pg_restore', ['-n', 'public', '-c', '-d', @dbname, 'core-db/'])
+      if pg_restore.success then
+        return true
+      else
+        return false
+      end
+    end
+
+  end
+end