stellar-core-backup 0.0.4

data/lib/stellar-core-backup/restore/filesystem.rb

@@ -0,0 +1,36 @@
+module StellarCoreBackup::Restore
+  class Filesystem < ::StellarCoreBackup::Filesystem
+    include Contracts
+
+    class DataDirNotEmpty < StandardError ; end
+
+    attr_reader :core_data_dir
+
+    Contract StellarCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config         = config
+      @working_dir    = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @core_data_dir  = get_core_data_dir(@config.get('core_config'))
+    end
+
+    public
+
+    Contract String => nil
+    def restore(backup_archive)
+      # unpack the filesystem backup
+      puts "info: stellar-core buckets restored" if StellarCoreBackup::Tar.unpack("#{@working_dir}/core-fs.tar", @core_data_dir)
+    end
+
+    Contract nil => Bool
+    def core_data_dir_empty?()
+      # checks fs and asks user to remove fs manually if fs is already in place.
+      if (Dir.entries(@core_data_dir) - %w{ . .. }).empty? then
+        return true
+      else
+        puts "error: #{@core_data_dir} is not empty, you can only restore to an empty data directory"
+        raise DataDirNotEmpty
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/s3.rb

@@ -0,0 +1,64 @@
+require 'aws-sdk-s3'
+
+module StellarCoreBackup
+  class S3
+    include Contracts
+
+    Contract StellarCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @s3_region    = @config.get('s3_region')
+      @s3_bucket    = @config.get('s3_bucket')
+      @s3_path      = @config.get('s3_path')
+      begin
+        @s3_client    = Aws::S3::Client.new(region: @s3_region)
+        @s3_resource  = Aws::S3::Resource.new(client: @s3_client)
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error connecting to s3"
+        puts e
+      end
+    end
+
+    def push(file)
+      begin
+        upload = @s3_resource.bucket(@s3_bucket).object("#{@s3_path}/#{File.basename(file)}")
+        if upload.upload_file(file) then
+          puts "info: pushed #{file} to s3 (#{@s3_bucket})"
+        else
+          puts "error: upload to s3 failed"
+        end
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error pushing #{file} to s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+    # fetches a backup tar file from s3, places in working dir
+    def get(file)
+      local_copy = "#{@working_dir}/#{File.basename(file)}"
+      begin
+        download = @s3_resource.bucket(@s3_bucket).object(file)
+        if download.download_file(local_copy) then
+          puts "info: fetched #{file} from s3 (#{@s3_bucket})"
+          return local_copy
+        else
+          puts "error: download from s3 failed"
+        end
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error downloading #{file} from s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+    def latest(listlen)
+      begin
+        @s3_client.list_objects_v2({bucket: @s3_bucket, prefix: @s3_path+'/core-backup-'}).contents.map{|o| o.key}.sort{|a,b| a.gsub(/(\d+)/,'\1') <=> b.gsub(/(\d+)/,'\1')}.last(listlen)
+      rescue Aws::S3::Errors::ServiceError => e
+        puts "info: error listing s3 (#{@s3_bucket})"
+        puts e
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/tar.rb

@@ -0,0 +1,37 @@
+require 'fileutils'
+
+module StellarCoreBackup
+  class Tar
+    include Contracts
+
+    Contract String, String => String
+    def self.pack(archive, directory)
+      if StellarCoreBackup::Utils.readable?(directory) then
+        # archive directory
+        puts "info: packing #{directory} in #{archive}"
+        %x{/bin/tar --create --file=#{archive} #{directory}}
+        if $?.exitstatus == 0 then
+          puts "info: #{archive} created"
+          return archive
+        else
+          raise StandardError
+        end
+      end
+    end
+
+    Contract String, String => String
+    def self.unpack(archive, destination)
+      if StellarCoreBackup::Utils.writable?(destination) then
+        # extract archive in destination directory
+        puts "info: unpacking #{archive} in #{destination}"
+        %x{/bin/tar --extract --file=#{archive} --directory=#{destination}}
+        if $?.exitstatus == 0 then
+          puts "info: #{archive} unpacked in #{destination}"
+          return destination
+        else
+          raise StandardError
+        end
+      end
+    end
+  end
+end

data/lib/stellar-core-backup/utils.rb

@@ -0,0 +1,130 @@
+require 'fileutils'
+
+module StellarCoreBackup
+  class Utils
+    include Contracts
+
+    Contract StellarCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @db_restore   = StellarCoreBackup::Restore::Database.new(@config)
+      @fs_restore   = StellarCoreBackup::Restore::Filesystem.new(@config)
+    end
+
+    Contract String => String
+    def self.create_working_dir(dir)
+      working_dir = dir + "/#{Process.pid}"
+      unless Dir.exists?(working_dir) then
+        Dir.mkdir working_dir
+      end
+      return working_dir
+    end
+
+    Contract String => nil
+    def self.remove_working_dir(working_dir)
+      if Dir.exists?(working_dir) then
+        Dir.rmdir working_dir + "/#{Process.pid}"
+      end
+    end
+
+    Contract String => String
+    def self.create_backup_dir(dir)
+      unless Dir.exists?(dir) then
+        Dir.mkdir dir
+      end
+      return dir
+    end
+
+    Contract String, String => String
+    def self.create_backup_tar(working_dir, backup_dir)
+      puts 'info: creating backup tarball'
+      tar_file = "#{backup_dir}/core-backup-#{Time.now.to_i}.tar"
+      Dir.chdir(working_dir)
+      # archive the working directory
+      StellarCoreBackup::Tar.pack(tar_file, '.')
+      return tar_file
+    end
+
+    Contract String => nil
+    def extract_backup(backup_archive)
+      # extract the backup archive into the working directory
+      StellarCoreBackup::Tar.unpack(backup_archive, @working_dir)
+      return
+    end
+
+#    Contract String => nil
+#    def restore(backup_archive)
+#      @fs_restore.restore(backup_archive)
+#      @db_restore.restore()
+#    end
+
+    Contract String => Bool
+    def self.cleanbucket(bucket_dir)
+      if FileUtils.remove(Dir.glob(bucket_dir+'/*')) then
+        puts 'info: cleaning up workspace'
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String => Bool
+    def self.cleanup(working_dir)
+      if FileUtils.remove_dir(working_dir) then
+        puts 'info: cleaning up workspace'
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String, String => Bool
+    def self.confirm_shasums_definitive(working_dir, backup_archive)
+
+      # create an array of filesunpacked into the working_dir
+      Dir.chdir(working_dir)
+      files_present=Dir.glob('./**/*')
+
+      # remove directories and shasum details from file array
+      files_present.delete('./'+File.basename(backup_archive))
+      files_present.delete('./core-db')
+      files_present.delete('./SHA256SUMS')
+      files_present.delete('./SHA256SUMS.sig')
+
+      # now delete the file names in the shasums file from the array
+      # we are expecting an array of zero length after this process
+      File.open("SHA256SUMS").each { |sha_file| files_present.delete(sha_file.split(' ')[1].chomp) }
+      if files_present.none? then
+        return true
+      else
+        return false
+      end
+    end
+
+    # check we have read permissions
+    Contract String => Bool
+    def self.readable?(file)
+      if File.readable?(file) then
+        puts "info: #{file} readable"
+        return true
+      else
+        puts "error: cannot read #{file}"
+        raise Errno::EACCES
+      end
+    end
+
+    # check we have write permissions
+    Contract String => Bool
+    def self.writable?(file)
+      if File.writable?(file) then
+        puts "info: #{file} writeable"
+        return true
+      else
+        puts "error: cannot write to #{file}"
+        raise Errno::EACCES
+      end
+    end
+
+  end
+end

data/lib/stellar-core-backup/version.rb

@@ -0,0 +1,3 @@
+module StellarCoreBackup
+  VERSION = "0.0.4"
+end

data/stellar-core-backup.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'stellar-core-backup/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "stellar-core-backup"
+  spec.version       = StellarCoreBackup::VERSION
+  spec.authors       = ["Tom Llewellyn-Smith"]
+  spec.email         = ["tom@stellar.org"]
+  spec.summary       = %q{A helper script to backup a stellar-core node}
+  spec.homepage      = "https://github.com/stellar/stellar-core-backup"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "contracts", "~> 0.9"
+  spec.add_dependency "pg", "~> 0.18.1"
+  spec.add_dependency "aws-sdk-s3", "~> 1"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: stellar-core-backup
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Tom Llewellyn-Smith
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: contracts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- tom@stellar.org
+executables:
+- stellar-core-backup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CONTRIBUTING.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/stellar-core-backup
+- config/sample.yaml
+- lib/stellar-core-backup.rb
+- lib/stellar-core-backup/cmd.rb
+- lib/stellar-core-backup/cmd_result.rb
+- lib/stellar-core-backup/config.rb
+- lib/stellar-core-backup/database.rb
+- lib/stellar-core-backup/filesystem.rb
+- lib/stellar-core-backup/job.rb
+- lib/stellar-core-backup/restore/database.rb
+- lib/stellar-core-backup/restore/filesystem.rb
+- lib/stellar-core-backup/s3.rb
+- lib/stellar-core-backup/tar.rb
+- lib/stellar-core-backup/utils.rb
+- lib/stellar-core-backup/version.rb
+- stellar-core-backup.gemspec
+homepage: https://github.com/stellar/stellar-core-backup
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: A helper script to backup a stellar-core node
+test_files: []